PostgreSQL Source Code  git master
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "access/rmgr.h"
17 #include "access/xact.h"
18 #include "access/xlog_internal.h"
19 #include "access/xlogreader.h"
20 #include "catalog/pg_control.h"
21 #include "catalog/storage_xlog.h"
23 #include "fe_utils/archive.h"
24 #include "filemap.h"
25 #include "pg_rewind.h"
26 
27 /*
28  * RmgrNames is an array of the built-in resource manager names, to make error
29  * messages a bit nicer.
30  */
31 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \
32  name,
33 
34 static const char *const RmgrNames[RM_MAX_ID + 1] = {
35 #include "access/rmgrlist.h"
36 };
37 
38 #define RmgrName(rmid) (((rmid) <= RM_MAX_BUILTIN_ID) ? \
39  RmgrNames[rmid] : "custom")
40 
41 static void extractPageInfo(XLogReaderState *record);
42 
43 static int xlogreadfd = -1;
45 static char xlogfpath[MAXPGPATH];
46 
47 typedef struct XLogPageReadPrivate
48 {
49  const char *restoreCommand;
50  int tliIndex;
52 
54  XLogRecPtr targetPagePtr,
55  int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
56 
57 /*
58  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
59  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
60  * the data blocks touched by the WAL records, and return them in a page map.
61  *
62  * 'endpoint' is the end of the last record to read. The record starting at
63  * 'endpoint' is the first one that is not read.
64  */
65 void
66 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
67  XLogRecPtr endpoint, const char *restoreCommand)
68 {
69  XLogRecord *record;
71  char *errormsg;
72  XLogPageReadPrivate private;
73 
74  private.tliIndex = tliIndex;
75  private.restoreCommand = restoreCommand;
77  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
78  &private);
79  if (xlogreader == NULL)
80  pg_fatal("out of memory while allocating a WAL reading processor");
81 
82  XLogBeginRead(xlogreader, startpoint);
83  do
84  {
85  record = XLogReadRecord(xlogreader, &errormsg);
86 
87  if (record == NULL)
88  {
90 
91  if (errormsg)
92  pg_fatal("could not read WAL record at %X/%X: %s",
93  LSN_FORMAT_ARGS(errptr),
94  errormsg);
95  else
96  pg_fatal("could not read WAL record at %X/%X",
97  LSN_FORMAT_ARGS(errptr));
98  }
99 
101  } while (xlogreader->EndRecPtr < endpoint);
102 
103  /*
104  * If 'endpoint' didn't point exactly at a record boundary, the caller
105  * messed up.
106  */
107  if (xlogreader->EndRecPtr != endpoint)
108  pg_fatal("end pointer %X/%X is not a valid end point; expected %X/%X",
110 
112  if (xlogreadfd != -1)
113  {
114  close(xlogreadfd);
115  xlogreadfd = -1;
116  }
117 }
118 
119 /*
120  * Reads one WAL record. Returns the end position of the record, without
121  * doing anything with the record itself.
122  */
124 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
125  const char *restoreCommand)
126 {
127  XLogRecord *record;
129  char *errormsg;
130  XLogPageReadPrivate private;
131  XLogRecPtr endptr;
132 
133  private.tliIndex = tliIndex;
134  private.restoreCommand = restoreCommand;
136  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
137  &private);
138  if (xlogreader == NULL)
139  pg_fatal("out of memory while allocating a WAL reading processor");
140 
142  record = XLogReadRecord(xlogreader, &errormsg);
143  if (record == NULL)
144  {
145  if (errormsg)
146  pg_fatal("could not read WAL record at %X/%X: %s",
147  LSN_FORMAT_ARGS(ptr), errormsg);
148  else
149  pg_fatal("could not read WAL record at %X/%X",
150  LSN_FORMAT_ARGS(ptr));
151  }
152  endptr = xlogreader->EndRecPtr;
153 
155  if (xlogreadfd != -1)
156  {
157  close(xlogreadfd);
158  xlogreadfd = -1;
159  }
160 
161  return endptr;
162 }
163 
164 /*
165  * Find the previous checkpoint preceding given WAL location.
166  */
167 void
168 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
169  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
170  XLogRecPtr *lastchkptredo, const char *restoreCommand)
171 {
172  /* Walk backwards, starting from the given record */
173  XLogRecord *record;
174  XLogRecPtr searchptr;
176  char *errormsg;
177  XLogPageReadPrivate private;
178  XLogSegNo current_segno = 0;
179  TimeLineID current_tli = 0;
180 
181  /*
182  * The given fork pointer points to the end of the last common record,
183  * which is not necessarily the beginning of the next record, if the
184  * previous record happens to end at a page boundary. Skip over the page
185  * header in that case to find the next record.
186  */
187  if (forkptr % XLOG_BLCKSZ == 0)
188  {
189  if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
190  forkptr += SizeOfXLogLongPHD;
191  else
192  forkptr += SizeOfXLogShortPHD;
193  }
194 
195  private.tliIndex = tliIndex;
196  private.restoreCommand = restoreCommand;
198  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
199  &private);
200  if (xlogreader == NULL)
201  pg_fatal("out of memory while allocating a WAL reading processor");
202 
203  searchptr = forkptr;
204  for (;;)
205  {
206  uint8 info;
207 
208  XLogBeginRead(xlogreader, searchptr);
209  record = XLogReadRecord(xlogreader, &errormsg);
210 
211  if (record == NULL)
212  {
213  if (errormsg)
214  pg_fatal("could not find previous WAL record at %X/%X: %s",
215  LSN_FORMAT_ARGS(searchptr),
216  errormsg);
217  else
218  pg_fatal("could not find previous WAL record at %X/%X",
219  LSN_FORMAT_ARGS(searchptr));
220  }
221 
222  /* Detect if a new WAL file has been opened */
223  if (xlogreader->seg.ws_tli != current_tli ||
224  xlogreader->seg.ws_segno != current_segno)
225  {
226  char xlogfname[MAXFNAMELEN];
227 
228  snprintf(xlogfname, MAXFNAMELEN, XLOGDIR "/");
229 
230  /* update curent values */
231  current_tli = xlogreader->seg.ws_tli;
232  current_segno = xlogreader->seg.ws_segno;
233 
234  XLogFileName(xlogfname + sizeof(XLOGDIR),
235  current_tli, current_segno, WalSegSz);
236 
237  /* Track this filename as one to not remove */
238  keepwal_add_entry(xlogfname);
239  }
240 
241  /*
242  * Check if it is a checkpoint record. This checkpoint record needs to
243  * be the latest checkpoint before WAL forked and not the checkpoint
244  * where the primary has been stopped to be rewound.
245  */
247  if (searchptr < forkptr &&
248  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
249  (info == XLOG_CHECKPOINT_SHUTDOWN ||
250  info == XLOG_CHECKPOINT_ONLINE))
251  {
252  CheckPoint checkPoint;
253 
254  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
255  *lastchkptrec = searchptr;
256  *lastchkpttli = checkPoint.ThisTimeLineID;
257  *lastchkptredo = checkPoint.redo;
258  break;
259  }
260 
261  /* Walk backwards to previous record. */
262  searchptr = record->xl_prev;
263  }
264 
266  if (xlogreadfd != -1)
267  {
268  close(xlogreadfd);
269  xlogreadfd = -1;
270  }
271 }
272 
273 /* XLogReader callback function, to read a WAL page */
274 static int
276  int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
277 {
279  uint32 targetPageOff;
280  XLogRecPtr targetSegEnd;
281  XLogSegNo targetSegNo;
282  int r;
283 
284  XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
285  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
286  targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
287 
288  /*
289  * See if we need to switch to a new segment because the requested record
290  * is not in the currently open one.
291  */
292  if (xlogreadfd >= 0 &&
293  !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
294  {
295  close(xlogreadfd);
296  xlogreadfd = -1;
297  }
298 
299  XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
300 
301  if (xlogreadfd < 0)
302  {
303  char xlogfname[MAXFNAMELEN];
304 
305  /*
306  * Since incomplete segments are copied into next timelines, switch to
307  * the timeline holding the required segment. Assuming this scan can
308  * be done both forward and backward, consider also switching timeline
309  * accordingly.
310  */
311  while (private->tliIndex < targetNentries - 1 &&
312  targetHistory[private->tliIndex].end < targetSegEnd)
313  private->tliIndex++;
314  while (private->tliIndex > 0 &&
315  targetHistory[private->tliIndex].begin >= targetSegEnd)
316  private->tliIndex--;
317 
318  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
320 
321  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
322  xlogreader->segcxt.ws_dir, xlogfname);
323 
324  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
325 
326  if (xlogreadfd < 0)
327  {
328  /*
329  * If we have no restore_command to execute, then exit.
330  */
331  if (private->restoreCommand == NULL)
332  {
333  pg_log_error("could not open file \"%s\": %m", xlogfpath);
334  return -1;
335  }
336 
337  /*
338  * Since we have restore_command, then try to retrieve missing WAL
339  * file from the archive.
340  */
342  xlogfname,
343  WalSegSz,
344  private->restoreCommand);
345 
346  if (xlogreadfd < 0)
347  return -1;
348  else
349  pg_log_debug("using file \"%s\" restored from archive",
350  xlogfpath);
351  }
352  }
353 
354  /*
355  * At this point, we have the right segment open.
356  */
357  Assert(xlogreadfd != -1);
358 
359  /* Read the requested page */
360  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
361  {
362  pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
363  return -1;
364  }
365 
366 
367  r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
368  if (r != XLOG_BLCKSZ)
369  {
370  if (r < 0)
371  pg_log_error("could not read file \"%s\": %m", xlogfpath);
372  else
373  pg_log_error("could not read file \"%s\": read %d of %zu",
374  xlogfpath, r, (Size) XLOG_BLCKSZ);
375 
376  return -1;
377  }
378 
379  Assert(targetSegNo == xlogreadsegno);
380 
381  xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
382  return XLOG_BLCKSZ;
383 }
384 
385 /*
386  * Extract information on which blocks the current record modifies.
387  */
388 static void
390 {
391  int block_id;
392  RmgrId rmid = XLogRecGetRmid(record);
393  uint8 info = XLogRecGetInfo(record);
394  uint8 rminfo = info & ~XLR_INFO_MASK;
395 
396  /* Is this a special record type that I recognize? */
397 
398  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_FILE_COPY)
399  {
400  /*
401  * New databases can be safely ignored. It won't be present in the
402  * source system, so it will be deleted. There's one corner-case,
403  * though: if a new, different, database is also created in the source
404  * system, we'll see that the files already exist and not copy them.
405  * That's OK, though; WAL replay of creating the new database, from
406  * the source systems's WAL, will re-copy the new database,
407  * overwriting the database created in the target system.
408  */
409  }
410  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_WAL_LOG)
411  {
412  /*
413  * New databases can be safely ignored. It won't be present in the
414  * source system, so it will be deleted.
415  */
416  }
417  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
418  {
419  /*
420  * An existing database was dropped. We'll see that the files don't
421  * exist in the target data dir, and copy them in toto from the source
422  * system. No need to do anything special here.
423  */
424  }
425  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
426  {
427  /*
428  * We can safely ignore these. The file will be removed from the
429  * target, if it doesn't exist in source system. If a file with same
430  * name is created in source system, too, there will be WAL records
431  * for all the blocks in it.
432  */
433  }
434  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
435  {
436  /*
437  * We can safely ignore these. When we compare the sizes later on,
438  * we'll notice that they differ, and copy the missing tail from
439  * source system.
440  */
441  }
442  else if (rmid == RM_XACT_ID &&
443  ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
445  (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT ||
447  {
448  /*
449  * These records can include "dropped rels". We can safely ignore
450  * them, we will see that they are missing and copy them from the
451  * source.
452  */
453  }
454  else if (info & XLR_SPECIAL_REL_UPDATE)
455  {
456  /*
457  * This record type modifies a relation file in some special way, but
458  * we don't recognize the type. That's bad - we don't know how to
459  * track that change.
460  */
461  pg_fatal("WAL record modifies a relation, but record type is not recognized: "
462  "lsn: %X/%X, rmid: %d, rmgr: %s, info: %02X",
463  LSN_FORMAT_ARGS(record->ReadRecPtr),
464  rmid, RmgrName(rmid), info);
465  }
466 
467  for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
468  {
469  RelFileLocator rlocator;
470  ForkNumber forknum;
471  BlockNumber blkno;
472 
473  if (!XLogRecGetBlockTagExtended(record, block_id,
474  &rlocator, &forknum, &blkno, NULL))
475  continue;
476 
477  /* We only care about the main fork; others are copied in toto */
478  if (forknum != MAIN_FORKNUM)
479  continue;
480 
481  process_target_wal_block_change(forknum, rlocator, blkno);
482  }
483 }
uint32 BlockNumber
Definition: block.h:31
uint8_t uint8
Definition: c.h:483
#define Assert(condition)
Definition: c.h:812
#define PG_BINARY
Definition: c.h:1227
uint32_t uint32
Definition: c.h:485
size_t Size
Definition: c.h:559
#define XLOG_DBASE_CREATE_WAL_LOG
#define XLOG_DBASE_DROP
#define XLOG_DBASE_CREATE_FILE_COPY
void process_target_wal_block_change(ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)
Definition: filemap.c:352
void keepwal_add_entry(const char *path)
Definition: filemap.c:250
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
#define pg_log_error(...)
Definition: logging.h:106
#define pg_log_debug(...)
Definition: logging.h:133
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
Definition: parsexlog.c:275
struct XLogPageReadPrivate XLogPageReadPrivate
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:44
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint, const char *restoreCommand)
Definition: parsexlog.c:66
static const char *const RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:34
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:389
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo, const char *restoreCommand)
Definition: parsexlog.c:168
static int xlogreadfd
Definition: parsexlog.c:43
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, const char *restoreCommand)
Definition: parsexlog.c:124
#define RmgrName(rmid)
Definition: parsexlog.c:38
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:45
#define pg_fatal(...)
#define MAXPGPATH
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:68
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:69
char * datadir
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:81
int targetNentries
Definition: pg_rewind.c:82
#define snprintf
Definition: port.h:238
ForkNumber
Definition: relpath.h:56
@ MAIN_FORKNUM
Definition: relpath.h:58
#define RM_MAX_ID
Definition: rmgr.h:33
uint8 RmgrId
Definition: rmgr.h:11
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
int WalSegSz
Definition: streamutil.c:32
TimeLineID ThisTimeLineID
Definition: pg_control.h:39
XLogRecPtr redo
Definition: pg_control.h:37
XLogRecPtr begin
Definition: timeline.h:28
TimeLineID tli
Definition: timeline.h:27
XLogRecPtr end
Definition: timeline.h:29
XLogSegNo ws_segno
Definition: xlogreader.h:48
TimeLineID ws_tli
Definition: xlogreader.h:49
char ws_dir[MAXPGPATH]
Definition: xlogreader.h:55
const char * restoreCommand
Definition: parsexlog.c:49
WALSegmentContext segcxt
Definition: xlogreader.h:271
XLogRecPtr EndRecPtr
Definition: xlogreader.h:207
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:206
WALOpenSegment seg
Definition: xlogreader.h:272
void * private_data
Definition: xlogreader.h:196
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
#define XLOG_XACT_COMMIT_PREPARED
Definition: xact.h:172
#define XLOG_XACT_COMMIT
Definition: xact.h:169
#define XLOG_XACT_OPMASK
Definition: xact.h:179
#define XLOG_XACT_ABORT
Definition: xact.h:171
#define XLOG_XACT_ABORT_PREPARED
Definition: xact.h:173
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define MAXFNAMELEN
#define XLOGDIR
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:52
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:69
static void XLogFileName(char *fname, TimeLineID tli, XLogSegNo logSegNo, int wal_segsz_bytes)
#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
bool RestoreArchivedFile(char *path, const char *xlogfname, const char *recovername, off_t expectedSize, bool cleanupEnabled)
Definition: xlogarchive.c:54
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
bool XLogRecGetBlockTagExtended(XLogReaderState *record, uint8 block_id, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *blknum, Buffer *prefetch_buffer)
Definition: xlogreader.c:1997
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:389
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:161
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogReaderRoutine *routine, void *private_data)
Definition: xlogreader.c:106
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:231
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:410
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:411
#define XLogRecGetData(decoder)
Definition: xlogreader.h:415
#define XL_ROUTINE(...)
Definition: xlogreader.h:117
#define XLogRecMaxBlockId(decoder)
Definition: xlogreader.h:418
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:82
static XLogReaderState * xlogreader
Definition: xlogrecovery.c:188