PostgreSQL Source Code  git master
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "access/rmgr.h"
17 #include "access/xlog_internal.h"
18 #include "access/xlogreader.h"
19 #include "catalog/pg_control.h"
20 #include "catalog/storage_xlog.h"
22 #include "fe_utils/archive.h"
23 #include "filemap.h"
24 #include "pg_rewind.h"
25 
26 /*
27  * RmgrNames is an array of resource manager names, to make error messages
28  * a bit nicer.
29  */
30 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
31  name,
32 
33 static const char *RmgrNames[RM_MAX_ID + 1] = {
34 #include "access/rmgrlist.h"
35 };
36 
37 static void extractPageInfo(XLogReaderState *record);
38 
39 static int xlogreadfd = -1;
41 static char xlogfpath[MAXPGPATH];
42 
43 typedef struct XLogPageReadPrivate
44 {
45  const char *restoreCommand;
46  int tliIndex;
48 
50  XLogRecPtr targetPagePtr,
51  int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
52 
53 /*
54  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
55  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
56  * the data blocks touched by the WAL records, and return them in a page map.
57  */
58 void
59 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
60  XLogRecPtr endpoint, const char *restoreCommand)
61 {
62  XLogRecord *record;
64  char *errormsg;
65  XLogPageReadPrivate private;
66 
67  private.tliIndex = tliIndex;
68  private.restoreCommand = restoreCommand;
69  xlogreader = XLogReaderAllocate(WalSegSz, datadir,
70  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
71  &private);
72  if (xlogreader == NULL)
73  pg_fatal("out of memory");
74 
75  XLogBeginRead(xlogreader, startpoint);
76  do
77  {
78  record = XLogReadRecord(xlogreader, &errormsg);
79 
80  if (record == NULL)
81  {
82  XLogRecPtr errptr = xlogreader->EndRecPtr;
83 
84  if (errormsg)
85  pg_fatal("could not read WAL record at %X/%X: %s",
86  (uint32) (errptr >> 32), (uint32) (errptr),
87  errormsg);
88  else
89  pg_fatal("could not read WAL record at %X/%X",
90  (uint32) (errptr >> 32), (uint32) (errptr));
91  }
92 
93  extractPageInfo(xlogreader);
94 
95  } while (xlogreader->ReadRecPtr != endpoint);
96 
97  XLogReaderFree(xlogreader);
98  if (xlogreadfd != -1)
99  {
100  close(xlogreadfd);
101  xlogreadfd = -1;
102  }
103 }
104 
105 /*
106  * Reads one WAL record. Returns the end position of the record, without
107  * doing anything with the record itself.
108  */
110 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
111  const char *restoreCommand)
112 {
113  XLogRecord *record;
115  char *errormsg;
116  XLogPageReadPrivate private;
117  XLogRecPtr endptr;
118 
119  private.tliIndex = tliIndex;
120  private.restoreCommand = restoreCommand;
121  xlogreader = XLogReaderAllocate(WalSegSz, datadir,
122  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
123  &private);
124  if (xlogreader == NULL)
125  pg_fatal("out of memory");
126 
127  XLogBeginRead(xlogreader, ptr);
128  record = XLogReadRecord(xlogreader, &errormsg);
129  if (record == NULL)
130  {
131  if (errormsg)
132  pg_fatal("could not read WAL record at %X/%X: %s",
133  (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
134  else
135  pg_fatal("could not read WAL record at %X/%X",
136  (uint32) (ptr >> 32), (uint32) (ptr));
137  }
138  endptr = xlogreader->EndRecPtr;
139 
140  XLogReaderFree(xlogreader);
141  if (xlogreadfd != -1)
142  {
143  close(xlogreadfd);
144  xlogreadfd = -1;
145  }
146 
147  return endptr;
148 }
149 
150 /*
151  * Find the previous checkpoint preceding given WAL location.
152  */
153 void
154 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
155  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
156  XLogRecPtr *lastchkptredo, const char *restoreCommand)
157 {
158  /* Walk backwards, starting from the given record */
159  XLogRecord *record;
160  XLogRecPtr searchptr;
162  char *errormsg;
163  XLogPageReadPrivate private;
164 
165  /*
166  * The given fork pointer points to the end of the last common record,
167  * which is not necessarily the beginning of the next record, if the
168  * previous record happens to end at a page boundary. Skip over the page
169  * header in that case to find the next record.
170  */
171  if (forkptr % XLOG_BLCKSZ == 0)
172  {
173  if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
174  forkptr += SizeOfXLogLongPHD;
175  else
176  forkptr += SizeOfXLogShortPHD;
177  }
178 
179  private.tliIndex = tliIndex;
180  private.restoreCommand = restoreCommand;
181  xlogreader = XLogReaderAllocate(WalSegSz, datadir,
182  XL_ROUTINE(.page_read = &SimpleXLogPageRead),
183  &private);
184  if (xlogreader == NULL)
185  pg_fatal("out of memory");
186 
187  searchptr = forkptr;
188  for (;;)
189  {
190  uint8 info;
191 
192  XLogBeginRead(xlogreader, searchptr);
193  record = XLogReadRecord(xlogreader, &errormsg);
194 
195  if (record == NULL)
196  {
197  if (errormsg)
198  pg_fatal("could not find previous WAL record at %X/%X: %s",
199  (uint32) (searchptr >> 32), (uint32) (searchptr),
200  errormsg);
201  else
202  pg_fatal("could not find previous WAL record at %X/%X",
203  (uint32) (searchptr >> 32), (uint32) (searchptr));
204  }
205 
206  /*
207  * Check if it is a checkpoint record. This checkpoint record needs to
208  * be the latest checkpoint before WAL forked and not the checkpoint
209  * where the primary has been stopped to be rewinded.
210  */
211  info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
212  if (searchptr < forkptr &&
213  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
214  (info == XLOG_CHECKPOINT_SHUTDOWN ||
215  info == XLOG_CHECKPOINT_ONLINE))
216  {
217  CheckPoint checkPoint;
218 
219  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
220  *lastchkptrec = searchptr;
221  *lastchkpttli = checkPoint.ThisTimeLineID;
222  *lastchkptredo = checkPoint.redo;
223  break;
224  }
225 
226  /* Walk backwards to previous record. */
227  searchptr = record->xl_prev;
228  }
229 
230  XLogReaderFree(xlogreader);
231  if (xlogreadfd != -1)
232  {
233  close(xlogreadfd);
234  xlogreadfd = -1;
235  }
236 }
237 
238 /* XLogReader callback function, to read a WAL page */
239 static int
241  int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
242 {
243  XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
244  uint32 targetPageOff;
245  XLogRecPtr targetSegEnd;
246  XLogSegNo targetSegNo;
247  int r;
248 
249  XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
250  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
251  targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
252 
253  /*
254  * See if we need to switch to a new segment because the requested record
255  * is not in the currently open one.
256  */
257  if (xlogreadfd >= 0 &&
258  !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
259  {
260  close(xlogreadfd);
261  xlogreadfd = -1;
262  }
263 
264  XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
265 
266  if (xlogreadfd < 0)
267  {
268  char xlogfname[MAXFNAMELEN];
269 
270  /*
271  * Since incomplete segments are copied into next timelines, switch to
272  * the timeline holding the required segment. Assuming this scan can
273  * be done both forward and backward, consider also switching timeline
274  * accordingly.
275  */
276  while (private->tliIndex < targetNentries - 1 &&
277  targetHistory[private->tliIndex].end < targetSegEnd)
278  private->tliIndex++;
279  while (private->tliIndex > 0 &&
280  targetHistory[private->tliIndex].begin >= targetSegEnd)
281  private->tliIndex--;
282 
283  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
285 
286  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
287  xlogreader->segcxt.ws_dir, xlogfname);
288 
289  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
290 
291  if (xlogreadfd < 0)
292  {
293  /*
294  * If we have no restore_command to execute, then exit.
295  */
296  if (private->restoreCommand == NULL)
297  {
298  pg_log_error("could not open file \"%s\": %m", xlogfpath);
299  return -1;
300  }
301 
302  /*
303  * Since we have restore_command, then try to retrieve missing WAL
304  * file from the archive.
305  */
307  xlogfname,
308  WalSegSz,
309  private->restoreCommand);
310 
311  if (xlogreadfd < 0)
312  return -1;
313  else
314  pg_log_debug("using file \"%s\" restored from archive",
315  xlogfpath);
316  }
317  }
318 
319  /*
320  * At this point, we have the right segment open.
321  */
322  Assert(xlogreadfd != -1);
323 
324  /* Read the requested page */
325  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
326  {
327  pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
328  return -1;
329  }
330 
331 
332  r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
333  if (r != XLOG_BLCKSZ)
334  {
335  if (r < 0)
336  pg_log_error("could not read file \"%s\": %m", xlogfpath);
337  else
338  pg_log_error("could not read file \"%s\": read %d of %zu",
339  xlogfpath, r, (Size) XLOG_BLCKSZ);
340 
341  return -1;
342  }
343 
344  Assert(targetSegNo == xlogreadsegno);
345 
346  xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
347  return XLOG_BLCKSZ;
348 }
349 
350 /*
351  * Extract information on which blocks the current record modifies.
352  */
353 static void
355 {
356  int block_id;
357  RmgrId rmid = XLogRecGetRmid(record);
358  uint8 info = XLogRecGetInfo(record);
359  uint8 rminfo = info & ~XLR_INFO_MASK;
360 
361  /* Is this a special record type that I recognize? */
362 
363  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
364  {
365  /*
366  * New databases can be safely ignored. It won't be present in the
367  * source system, so it will be deleted. There's one corner-case,
368  * though: if a new, different, database is also created in the source
369  * system, we'll see that the files already exist and not copy them.
370  * That's OK, though; WAL replay of creating the new database, from
371  * the source systems's WAL, will re-copy the new database,
372  * overwriting the database created in the target system.
373  */
374  }
375  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
376  {
377  /*
378  * An existing database was dropped. We'll see that the files don't
379  * exist in the target data dir, and copy them in toto from the source
380  * system. No need to do anything special here.
381  */
382  }
383  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
384  {
385  /*
386  * We can safely ignore these. The file will be removed from the
387  * target, if it doesn't exist in source system. If a file with same
388  * name is created in source system, too, there will be WAL records
389  * for all the blocks in it.
390  */
391  }
392  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
393  {
394  /*
395  * We can safely ignore these. When we compare the sizes later on,
396  * we'll notice that they differ, and copy the missing tail from
397  * source system.
398  */
399  }
400  else if (info & XLR_SPECIAL_REL_UPDATE)
401  {
402  /*
403  * This record type modifies a relation file in some special way, but
404  * we don't recognize the type. That's bad - we don't know how to
405  * track that change.
406  */
407  pg_fatal("WAL record modifies a relation, but record type is not recognized: "
408  "lsn: %X/%X, rmgr: %s, info: %02X",
409  (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
410  RmgrNames[rmid], info);
411  }
412 
413  for (block_id = 0; block_id <= record->max_block_id; block_id++)
414  {
415  RelFileNode rnode;
416  ForkNumber forknum;
417  BlockNumber blkno;
418 
419  if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
420  continue;
421 
422  /* We only care about the main fork; others are copied in toto */
423  if (forknum != MAIN_FORKNUM)
424  continue;
425 
426  process_block_change(forknum, rnode, blkno);
427  }
428 }
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint, const char *restoreCommand)
Definition: parsexlog.c:59
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
char ws_dir[MAXPGPATH]
Definition: xlogreader.h:54
uint32 TimeLineID
Definition: xlogdefs.h:52
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:68
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:67
#define pg_log_error(...)
Definition: logging.h:79
unsigned char uint8
Definition: c.h:372
bool RestoreArchivedFile(char *path, const char *xlogfname, const char *recovername, off_t expectedSize, bool cleanupEnabled)
Definition: xlogarchive.c:53
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo, const char *restoreCommand)
Definition: parsexlog.c:154
#define pg_fatal(...)
Definition: pg_rewind.h:41
static const char * RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:33
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
Definition: parsexlog.c:240
uint32 BlockNumber
Definition: block.h:31
TimeLineID tli
Definition: timeline.h:27
void * private_data
Definition: xlogreader.h:168
#define PG_BINARY
Definition: c.h:1211
void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
Definition: filemap.c:439
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
XLogRecPtr EndRecPtr
Definition: xlogreader.h:176
#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
WALOpenSegment seg
Definition: xlogreader.h:215
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:268
#define XLogRecGetData(decoder)
Definition: xlogreader.h:310
#define XLOG_DBASE_DROP
#define pg_log_debug(...)
Definition: logging.h:91
#define MAXPGPATH
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:175
char * datadir
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:240
unsigned int uint32
Definition: c.h:374
struct XLogPageReadPrivate XLogPageReadPrivate
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogReaderRoutine *routine, void *private_data)
Definition: xlogreader.c:75
static XLogReaderState * xlogreader
Definition: walsender.c:137
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:305
ForkNumber
Definition: relpath.h:40
#define XL_ROUTINE(...)
Definition: xlogreader.h:116
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:139
#define MAXFNAMELEN
#define RM_MAX_ID
Definition: rmgr.h:33
bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum)
Definition: xlogreader.c:1497
#define XLOGDIR
int targetNentries
Definition: pg_rewind.c:68
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
static int xlogreadfd
Definition: parsexlog.c:39
const char * restoreCommand
Definition: parsexlog.c:45
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:40
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:41
uint8 RmgrId
Definition: rmgr.h:11
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:354
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
XLogRecPtr end
Definition: timeline.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:745
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:55
size_t Size
Definition: c.h:473
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, const char *restoreCommand)
Definition: parsexlog.c:110
#define XLOG_DBASE_CREATE
TimeLineID ws_tli
Definition: xlogreader.h:48
int WalSegSz
Definition: pg_standby.c:38
TimeLineID ThisTimeLineID
Definition: pg_control.h:39
XLogRecPtr begin
Definition: timeline.h:28
WALSegmentContext segcxt
Definition: xlogreader.h:214
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:193
#define read(a, b, c)
Definition: win32.h:13
XLogRecPtr redo
Definition: pg_control.h:37
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:306
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:72
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)