PostgreSQL Source Code  git master
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "access/rmgr.h"
17 #include "access/xact.h"
18 #include "access/xlog_internal.h"
19 #include "access/xlogreader.h"
20 #include "catalog/pg_control.h"
21 #include "catalog/storage_xlog.h"
23 #include "fe_utils/archive.h"
24 #include "filemap.h"
25 #include "pg_rewind.h"
26 
27 /*
28  * RmgrNames is an array of the built-in resource manager names, to make error
29  * messages a bit nicer.
30  */
31 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \
32  name,
33 
34 static const char *RmgrNames[RM_MAX_ID + 1] = {
35 #include "access/rmgrlist.h"
36 };
37 
38 #define RmgrName(rmid) (((rmid) <= RM_MAX_BUILTIN_ID) ? \
39  RmgrNames[rmid] : "custom")
40 
41 static void extractPageInfo(XLogReaderState *record);
42 
43 static int xlogreadfd = -1;
45 static char xlogfpath[MAXPGPATH];
46 
47 typedef struct XLogPageReadPrivate
48 {
49  const char *restoreCommand;
50  int tliIndex;
52 
54  XLogRecPtr targetPagePtr,
55  int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
56 
57 /*
58  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
59  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
60  * the data blocks touched by the WAL records, and return them in a page map.
61  *
62  * 'endpoint' is the end of the last record to read. The record starting at
63  * 'endpoint' is the first one that is not read.
64  */
65 void
66 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
67  XLogRecPtr endpoint, const char *restoreCommand)
68 {
69  XLogRecord *record;
71  char *errormsg;
72  XLogPageReadPrivate private;
73 
74  private.tliIndex = tliIndex;
75  private.restoreCommand = restoreCommand;
77  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
78  &private);
79  if (xlogreader == NULL)
80  pg_fatal("out of memory while allocating a WAL reading processor");
81 
82  XLogBeginRead(xlogreader, startpoint);
83  do
84  {
85  record = XLogReadRecord(xlogreader, &errormsg);
86 
87  if (record == NULL)
88  {
90 
91  if (errormsg)
92  pg_fatal("could not read WAL record at %X/%X: %s",
93  LSN_FORMAT_ARGS(errptr),
94  errormsg);
95  else
96  pg_fatal("could not read WAL record at %X/%X",
97  LSN_FORMAT_ARGS(errptr));
98  }
99 
101  } while (xlogreader->EndRecPtr < endpoint);
102 
103  /*
104  * If 'endpoint' didn't point exactly at a record boundary, the caller
105  * messed up.
106  */
107  if (xlogreader->EndRecPtr != endpoint)
108  pg_fatal("end pointer %X/%X is not a valid end point; expected %X/%X",
110 
112  if (xlogreadfd != -1)
113  {
114  close(xlogreadfd);
115  xlogreadfd = -1;
116  }
117 }
118 
119 /*
120  * Reads one WAL record. Returns the end position of the record, without
121  * doing anything with the record itself.
122  */
124 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
125  const char *restoreCommand)
126 {
127  XLogRecord *record;
129  char *errormsg;
130  XLogPageReadPrivate private;
131  XLogRecPtr endptr;
132 
133  private.tliIndex = tliIndex;
134  private.restoreCommand = restoreCommand;
136  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
137  &private);
138  if (xlogreader == NULL)
139  pg_fatal("out of memory while allocating a WAL reading processor");
140 
142  record = XLogReadRecord(xlogreader, &errormsg);
143  if (record == NULL)
144  {
145  if (errormsg)
146  pg_fatal("could not read WAL record at %X/%X: %s",
147  LSN_FORMAT_ARGS(ptr), errormsg);
148  else
149  pg_fatal("could not read WAL record at %X/%X",
150  LSN_FORMAT_ARGS(ptr));
151  }
152  endptr = xlogreader->EndRecPtr;
153 
155  if (xlogreadfd != -1)
156  {
157  close(xlogreadfd);
158  xlogreadfd = -1;
159  }
160 
161  return endptr;
162 }
163 
164 /*
165  * Find the previous checkpoint preceding given WAL location.
166  */
167 void
168 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
169  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
170  XLogRecPtr *lastchkptredo, const char *restoreCommand)
171 {
172  /* Walk backwards, starting from the given record */
173  XLogRecord *record;
174  XLogRecPtr searchptr;
176  char *errormsg;
177  XLogPageReadPrivate private;
178 
179  /*
180  * The given fork pointer points to the end of the last common record,
181  * which is not necessarily the beginning of the next record, if the
182  * previous record happens to end at a page boundary. Skip over the page
183  * header in that case to find the next record.
184  */
185  if (forkptr % XLOG_BLCKSZ == 0)
186  {
187  if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
188  forkptr += SizeOfXLogLongPHD;
189  else
190  forkptr += SizeOfXLogShortPHD;
191  }
192 
193  private.tliIndex = tliIndex;
194  private.restoreCommand = restoreCommand;
196  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
197  &private);
198  if (xlogreader == NULL)
199  pg_fatal("out of memory while allocating a WAL reading processor");
200 
201  searchptr = forkptr;
202  for (;;)
203  {
204  uint8 info;
205 
206  XLogBeginRead(xlogreader, searchptr);
207  record = XLogReadRecord(xlogreader, &errormsg);
208 
209  if (record == NULL)
210  {
211  if (errormsg)
212  pg_fatal("could not find previous WAL record at %X/%X: %s",
213  LSN_FORMAT_ARGS(searchptr),
214  errormsg);
215  else
216  pg_fatal("could not find previous WAL record at %X/%X",
217  LSN_FORMAT_ARGS(searchptr));
218  }
219 
220  /*
221  * Check if it is a checkpoint record. This checkpoint record needs to
222  * be the latest checkpoint before WAL forked and not the checkpoint
223  * where the primary has been stopped to be rewound.
224  */
226  if (searchptr < forkptr &&
227  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
228  (info == XLOG_CHECKPOINT_SHUTDOWN ||
229  info == XLOG_CHECKPOINT_ONLINE))
230  {
231  CheckPoint checkPoint;
232 
233  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
234  *lastchkptrec = searchptr;
235  *lastchkpttli = checkPoint.ThisTimeLineID;
236  *lastchkptredo = checkPoint.redo;
237  break;
238  }
239 
240  /* Walk backwards to previous record. */
241  searchptr = record->xl_prev;
242  }
243 
245  if (xlogreadfd != -1)
246  {
247  close(xlogreadfd);
248  xlogreadfd = -1;
249  }
250 }
251 
252 /* XLogReader callback function, to read a WAL page */
253 static int
255  int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
256 {
258  uint32 targetPageOff;
259  XLogRecPtr targetSegEnd;
260  XLogSegNo targetSegNo;
261  int r;
262 
263  XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
264  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
265  targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
266 
267  /*
268  * See if we need to switch to a new segment because the requested record
269  * is not in the currently open one.
270  */
271  if (xlogreadfd >= 0 &&
272  !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
273  {
274  close(xlogreadfd);
275  xlogreadfd = -1;
276  }
277 
278  XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
279 
280  if (xlogreadfd < 0)
281  {
282  char xlogfname[MAXFNAMELEN];
283 
284  /*
285  * Since incomplete segments are copied into next timelines, switch to
286  * the timeline holding the required segment. Assuming this scan can
287  * be done both forward and backward, consider also switching timeline
288  * accordingly.
289  */
290  while (private->tliIndex < targetNentries - 1 &&
291  targetHistory[private->tliIndex].end < targetSegEnd)
292  private->tliIndex++;
293  while (private->tliIndex > 0 &&
294  targetHistory[private->tliIndex].begin >= targetSegEnd)
295  private->tliIndex--;
296 
297  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
299 
300  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
301  xlogreader->segcxt.ws_dir, xlogfname);
302 
303  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
304 
305  if (xlogreadfd < 0)
306  {
307  /*
308  * If we have no restore_command to execute, then exit.
309  */
310  if (private->restoreCommand == NULL)
311  {
312  pg_log_error("could not open file \"%s\": %m", xlogfpath);
313  return -1;
314  }
315 
316  /*
317  * Since we have restore_command, then try to retrieve missing WAL
318  * file from the archive.
319  */
321  xlogfname,
322  WalSegSz,
323  private->restoreCommand);
324 
325  if (xlogreadfd < 0)
326  return -1;
327  else
328  pg_log_debug("using file \"%s\" restored from archive",
329  xlogfpath);
330  }
331  }
332 
333  /*
334  * At this point, we have the right segment open.
335  */
336  Assert(xlogreadfd != -1);
337 
338  /* Read the requested page */
339  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
340  {
341  pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
342  return -1;
343  }
344 
345 
346  r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
347  if (r != XLOG_BLCKSZ)
348  {
349  if (r < 0)
350  pg_log_error("could not read file \"%s\": %m", xlogfpath);
351  else
352  pg_log_error("could not read file \"%s\": read %d of %zu",
353  xlogfpath, r, (Size) XLOG_BLCKSZ);
354 
355  return -1;
356  }
357 
358  Assert(targetSegNo == xlogreadsegno);
359 
360  xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
361  return XLOG_BLCKSZ;
362 }
363 
364 /*
365  * Extract information on which blocks the current record modifies.
366  */
367 static void
369 {
370  int block_id;
371  RmgrId rmid = XLogRecGetRmid(record);
372  uint8 info = XLogRecGetInfo(record);
373  uint8 rminfo = info & ~XLR_INFO_MASK;
374 
375  /* Is this a special record type that I recognize? */
376 
377  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_FILE_COPY)
378  {
379  /*
380  * New databases can be safely ignored. It won't be present in the
381  * source system, so it will be deleted. There's one corner-case,
382  * though: if a new, different, database is also created in the source
383  * system, we'll see that the files already exist and not copy them.
384  * That's OK, though; WAL replay of creating the new database, from
385  * the source systems's WAL, will re-copy the new database,
386  * overwriting the database created in the target system.
387  */
388  }
389  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_WAL_LOG)
390  {
391  /*
392  * New databases can be safely ignored. It won't be present in the
393  * source system, so it will be deleted.
394  */
395  }
396  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
397  {
398  /*
399  * An existing database was dropped. We'll see that the files don't
400  * exist in the target data dir, and copy them in toto from the source
401  * system. No need to do anything special here.
402  */
403  }
404  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
405  {
406  /*
407  * We can safely ignore these. The file will be removed from the
408  * target, if it doesn't exist in source system. If a file with same
409  * name is created in source system, too, there will be WAL records
410  * for all the blocks in it.
411  */
412  }
413  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
414  {
415  /*
416  * We can safely ignore these. When we compare the sizes later on,
417  * we'll notice that they differ, and copy the missing tail from
418  * source system.
419  */
420  }
421  else if (rmid == RM_XACT_ID &&
422  ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
424  (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT ||
426  {
427  /*
428  * These records can include "dropped rels". We can safely ignore
429  * them, we will see that they are missing and copy them from the
430  * source.
431  */
432  }
433  else if (info & XLR_SPECIAL_REL_UPDATE)
434  {
435  /*
436  * This record type modifies a relation file in some special way, but
437  * we don't recognize the type. That's bad - we don't know how to
438  * track that change.
439  */
440  pg_fatal("WAL record modifies a relation, but record type is not recognized: "
441  "lsn: %X/%X, rmid: %d, rmgr: %s, info: %02X",
442  LSN_FORMAT_ARGS(record->ReadRecPtr),
443  rmid, RmgrName(rmid), info);
444  }
445 
446  for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
447  {
448  RelFileLocator rlocator;
449  ForkNumber forknum;
450  BlockNumber blkno;
451 
452  if (!XLogRecGetBlockTagExtended(record, block_id,
453  &rlocator, &forknum, &blkno, NULL))
454  continue;
455 
456  /* We only care about the main fork; others are copied in toto */
457  if (forknum != MAIN_FORKNUM)
458  continue;
459 
460  process_target_wal_block_change(forknum, rlocator, blkno);
461  }
462 }
uint32 BlockNumber
Definition: block.h:31
unsigned int uint32
Definition: c.h:452
#define PG_BINARY
Definition: c.h:1279
unsigned char uint8
Definition: c.h:450
size_t Size
Definition: c.h:551
#define XLOG_DBASE_CREATE_WAL_LOG
#define XLOG_DBASE_DROP
#define XLOG_DBASE_CREATE_FILE_COPY
void process_target_wal_block_change(ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)
Definition: filemap.c:291
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
Assert(fmt[strlen(fmt) - 1] !='\n')
#define pg_log_error(...)
Definition: logging.h:106
#define pg_log_debug(...)
Definition: logging.h:133
static const char * RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:34
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
Definition: parsexlog.c:254
struct XLogPageReadPrivate XLogPageReadPrivate
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:44
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint, const char *restoreCommand)
Definition: parsexlog.c:66
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:368
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo, const char *restoreCommand)
Definition: parsexlog.c:168
static int xlogreadfd
Definition: parsexlog.c:43
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, const char *restoreCommand)
Definition: parsexlog.c:124
#define RmgrName(rmid)
Definition: parsexlog.c:38
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:45
#define pg_fatal(...)
#define MAXPGPATH
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:68
char * datadir
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:73
int targetNentries
Definition: pg_rewind.c:74
#define snprintf
Definition: port.h:225
ForkNumber
Definition: relpath.h:41
@ MAIN_FORKNUM
Definition: relpath.h:43
#define RM_MAX_ID
Definition: rmgr.h:33
uint8 RmgrId
Definition: rmgr.h:11
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
uint32 WalSegSz
Definition: streamutil.c:34
TimeLineID ThisTimeLineID
Definition: pg_control.h:39
XLogRecPtr redo
Definition: pg_control.h:37
XLogRecPtr begin
Definition: timeline.h:28
TimeLineID tli
Definition: timeline.h:27
XLogRecPtr end
Definition: timeline.h:29
TimeLineID ws_tli
Definition: xlogreader.h:49
char ws_dir[MAXPGPATH]
Definition: xlogreader.h:55
const char * restoreCommand
Definition: parsexlog.c:49
WALSegmentContext segcxt
Definition: xlogreader.h:271
XLogRecPtr EndRecPtr
Definition: xlogreader.h:207
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:206
WALOpenSegment seg
Definition: xlogreader.h:272
void * private_data
Definition: xlogreader.h:196
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
#define XLOG_XACT_COMMIT_PREPARED
Definition: xact.h:159
#define XLOG_XACT_COMMIT
Definition: xact.h:156
#define XLOG_XACT_OPMASK
Definition: xact.h:166
#define XLOG_XACT_ABORT
Definition: xact.h:158
#define XLOG_XACT_ABORT_PREPARED
Definition: xact.h:160
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define MAXFNAMELEN
#define XLOGDIR
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:52
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:69
#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
bool RestoreArchivedFile(char *path, const char *xlogfname, const char *recovername, off_t expectedSize, bool cleanupEnabled)
Definition: xlogarchive.c:54
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
bool XLogRecGetBlockTagExtended(XLogReaderState *record, uint8 block_id, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *blknum, Buffer *prefetch_buffer)
Definition: xlogreader.c:1954
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:418
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:170
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogReaderRoutine *routine, void *private_data)
Definition: xlogreader.c:108
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:264
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:408
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:409
#define XLogRecGetData(decoder)
Definition: xlogreader.h:413
#define XL_ROUTINE(...)
Definition: xlogreader.h:117
#define XLogRecMaxBlockId(decoder)
Definition: xlogreader.h:416
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
static XLogReaderState * xlogreader
Definition: xlogrecovery.c:185