PostgreSQL Source Code  git master
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "pg_rewind.h"
17 #include "filemap.h"
18 
19 #include "access/rmgr.h"
20 #include "access/xlog_internal.h"
21 #include "access/xlogreader.h"
22 #include "catalog/pg_control.h"
23 #include "catalog/storage_xlog.h"
25 
26 
27 /*
28  * RmgrNames is an array of resource manager names, to make error messages
29  * a bit nicer.
30  */
31 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
32  name,
33 
34 static const char *RmgrNames[RM_MAX_ID + 1] = {
35 #include "access/rmgrlist.h"
36 };
37 
38 static void extractPageInfo(XLogReaderState *record);
39 
40 static int xlogreadfd = -1;
42 static char xlogfpath[MAXPGPATH];
43 
44 typedef struct XLogPageReadPrivate
45 {
46  int tliIndex;
48 
49 static int SimpleXLogPageRead(XLogReaderState *xlogreader,
50  XLogRecPtr targetPagePtr,
51  int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
52 
53 /*
54  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
55  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
56  * the data blocks touched by the WAL records, and return them in a page map.
57  */
58 void
59 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
60  XLogRecPtr endpoint)
61 {
62  XLogRecord *record;
63  XLogReaderState *xlogreader;
64  char *errormsg;
65  XLogPageReadPrivate private;
66 
67  private.tliIndex = tliIndex;
68  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
69  &private);
70  if (xlogreader == NULL)
71  pg_fatal("out of memory");
72 
73  do
74  {
75  record = XLogReadRecord(xlogreader, startpoint, &errormsg);
76 
77  if (record == NULL)
78  {
79  XLogRecPtr errptr;
80 
81  errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
82 
83  if (errormsg)
84  pg_fatal("could not read WAL record at %X/%X: %s",
85  (uint32) (errptr >> 32), (uint32) (errptr),
86  errormsg);
87  else
88  pg_fatal("could not read WAL record at %X/%X",
89  (uint32) (errptr >> 32), (uint32) (errptr));
90  }
91 
92  extractPageInfo(xlogreader);
93 
94  startpoint = InvalidXLogRecPtr; /* continue reading at next record */
95 
96  } while (xlogreader->ReadRecPtr != endpoint);
97 
98  XLogReaderFree(xlogreader);
99  if (xlogreadfd != -1)
100  {
101  close(xlogreadfd);
102  xlogreadfd = -1;
103  }
104 }
105 
106 /*
107  * Reads one WAL record. Returns the end position of the record, without
108  * doing anything with the record itself.
109  */
111 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
112 {
113  XLogRecord *record;
114  XLogReaderState *xlogreader;
115  char *errormsg;
116  XLogPageReadPrivate private;
117  XLogRecPtr endptr;
118 
119  private.tliIndex = tliIndex;
120  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
121  &private);
122  if (xlogreader == NULL)
123  pg_fatal("out of memory");
124 
125  record = XLogReadRecord(xlogreader, ptr, &errormsg);
126  if (record == NULL)
127  {
128  if (errormsg)
129  pg_fatal("could not read WAL record at %X/%X: %s",
130  (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
131  else
132  pg_fatal("could not read WAL record at %X/%X",
133  (uint32) (ptr >> 32), (uint32) (ptr));
134  }
135  endptr = xlogreader->EndRecPtr;
136 
137  XLogReaderFree(xlogreader);
138  if (xlogreadfd != -1)
139  {
140  close(xlogreadfd);
141  xlogreadfd = -1;
142  }
143 
144  return endptr;
145 }
146 
147 /*
148  * Find the previous checkpoint preceding given WAL location.
149  */
150 void
151 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
152  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
153  XLogRecPtr *lastchkptredo)
154 {
155  /* Walk backwards, starting from the given record */
156  XLogRecord *record;
157  XLogRecPtr searchptr;
158  XLogReaderState *xlogreader;
159  char *errormsg;
160  XLogPageReadPrivate private;
161 
162  /*
163  * The given fork pointer points to the end of the last common record,
164  * which is not necessarily the beginning of the next record, if the
165  * previous record happens to end at a page boundary. Skip over the page
166  * header in that case to find the next record.
167  */
168  if (forkptr % XLOG_BLCKSZ == 0)
169  {
170  if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
171  forkptr += SizeOfXLogLongPHD;
172  else
173  forkptr += SizeOfXLogShortPHD;
174  }
175 
176  private.tliIndex = tliIndex;
177  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
178  &private);
179  if (xlogreader == NULL)
180  pg_fatal("out of memory");
181 
182  searchptr = forkptr;
183  for (;;)
184  {
185  uint8 info;
186 
187  record = XLogReadRecord(xlogreader, searchptr, &errormsg);
188 
189  if (record == NULL)
190  {
191  if (errormsg)
192  pg_fatal("could not find previous WAL record at %X/%X: %s",
193  (uint32) (searchptr >> 32), (uint32) (searchptr),
194  errormsg);
195  else
196  pg_fatal("could not find previous WAL record at %X/%X",
197  (uint32) (searchptr >> 32), (uint32) (searchptr));
198  }
199 
200  /*
201  * Check if it is a checkpoint record. This checkpoint record needs to
202  * be the latest checkpoint before WAL forked and not the checkpoint
203  * where the master has been stopped to be rewinded.
204  */
205  info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
206  if (searchptr < forkptr &&
207  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
208  (info == XLOG_CHECKPOINT_SHUTDOWN ||
209  info == XLOG_CHECKPOINT_ONLINE))
210  {
211  CheckPoint checkPoint;
212 
213  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
214  *lastchkptrec = searchptr;
215  *lastchkpttli = checkPoint.ThisTimeLineID;
216  *lastchkptredo = checkPoint.redo;
217  break;
218  }
219 
220  /* Walk backwards to previous record. */
221  searchptr = record->xl_prev;
222  }
223 
224  XLogReaderFree(xlogreader);
225  if (xlogreadfd != -1)
226  {
227  close(xlogreadfd);
228  xlogreadfd = -1;
229  }
230 }
231 
232 /* XLogReader callback function, to read a WAL page */
233 static int
234 SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
235  int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
236 {
237  XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
238  uint32 targetPageOff;
239  XLogRecPtr targetSegEnd;
240  XLogSegNo targetSegNo;
241  int r;
242 
243  XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
244  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
245  targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
246 
247  /*
248  * See if we need to switch to a new segment because the requested record
249  * is not in the currently open one.
250  */
251  if (xlogreadfd >= 0 &&
252  !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
253  {
254  close(xlogreadfd);
255  xlogreadfd = -1;
256  }
257 
258  XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
259 
260  if (xlogreadfd < 0)
261  {
262  char xlogfname[MAXFNAMELEN];
263 
264  /*
265  * Since incomplete segments are copied into next timelines, switch to
266  * the timeline holding the required segment. Assuming this scan can
267  * be done both forward and backward, consider also switching timeline
268  * accordingly.
269  */
270  while (private->tliIndex < targetNentries - 1 &&
271  targetHistory[private->tliIndex].end < targetSegEnd)
272  private->tliIndex++;
273  while (private->tliIndex > 0 &&
274  targetHistory[private->tliIndex].begin >= targetSegEnd)
275  private->tliIndex--;
276 
277  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
279 
280  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
281  xlogreader->segcxt.ws_dir, xlogfname);
282 
283  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
284 
285  if (xlogreadfd < 0)
286  {
287  pg_log_error("could not open file \"%s\": %m", xlogfpath);
288  return -1;
289  }
290  }
291 
292  /*
293  * At this point, we have the right segment open.
294  */
295  Assert(xlogreadfd != -1);
296 
297  /* Read the requested page */
298  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
299  {
300  pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
301  return -1;
302  }
303 
304 
305  r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
306  if (r != XLOG_BLCKSZ)
307  {
308  if (r < 0)
309  pg_log_error("could not read file \"%s\": %m", xlogfpath);
310  else
311  pg_log_error("could not read file \"%s\": read %d of %zu",
312  xlogfpath, r, (Size) XLOG_BLCKSZ);
313 
314  return -1;
315  }
316 
317  Assert(targetSegNo == xlogreadsegno);
318 
319  xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
320  return XLOG_BLCKSZ;
321 }
322 
323 /*
324  * Extract information on which blocks the current record modifies.
325  */
326 static void
328 {
329  int block_id;
330  RmgrId rmid = XLogRecGetRmid(record);
331  uint8 info = XLogRecGetInfo(record);
332  uint8 rminfo = info & ~XLR_INFO_MASK;
333 
334  /* Is this a special record type that I recognize? */
335 
336  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
337  {
338  /*
339  * New databases can be safely ignored. It won't be present in the
340  * source system, so it will be deleted. There's one corner-case,
341  * though: if a new, different, database is also created in the source
342  * system, we'll see that the files already exist and not copy them.
343  * That's OK, though; WAL replay of creating the new database, from
344  * the source systems's WAL, will re-copy the new database,
345  * overwriting the database created in the target system.
346  */
347  }
348  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
349  {
350  /*
351  * An existing database was dropped. We'll see that the files don't
352  * exist in the target data dir, and copy them in toto from the source
353  * system. No need to do anything special here.
354  */
355  }
356  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
357  {
358  /*
359  * We can safely ignore these. The file will be removed from the
360  * target, if it doesn't exist in source system. If a file with same
361  * name is created in source system, too, there will be WAL records
362  * for all the blocks in it.
363  */
364  }
365  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
366  {
367  /*
368  * We can safely ignore these. When we compare the sizes later on,
369  * we'll notice that they differ, and copy the missing tail from
370  * source system.
371  */
372  }
373  else if (info & XLR_SPECIAL_REL_UPDATE)
374  {
375  /*
376  * This record type modifies a relation file in some special way, but
377  * we don't recognize the type. That's bad - we don't know how to
378  * track that change.
379  */
380  pg_fatal("WAL record modifies a relation, but record type is not recognized: "
381  "lsn: %X/%X, rmgr: %s, info: %02X",
382  (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
383  RmgrNames[rmid], info);
384  }
385 
386  for (block_id = 0; block_id <= record->max_block_id; block_id++)
387  {
388  RelFileNode rnode;
389  ForkNumber forknum;
390  BlockNumber blkno;
391 
392  if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
393  continue;
394 
395  /* We only care about the main fork; others are copied in toto */
396  if (forknum != MAIN_FORKNUM)
397  continue;
398 
399  process_block_change(forknum, rnode, blkno);
400  }
401 }
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
char ws_dir[MAXPGPATH]
Definition: xlogreader.h:46
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
uint32 TimeLineID
Definition: xlogdefs.h:52
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:68
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:64
#define pg_log_error(...)
Definition: logging.h:79
unsigned char uint8
Definition: c.h:356
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint)
Definition: parsexlog.c:59
#define pg_fatal(...)
Definition: pg_rewind.h:43
static const char * RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:34
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
Definition: parsexlog.c:234
uint32 BlockNumber
Definition: block.h:31
TimeLineID tli
Definition: timeline.h:27
void * private_data
Definition: xlogreader.h:126
#define PG_BINARY
Definition: c.h:1191
XLogRecord * XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
Definition: xlogreader.c:237
void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
Definition: filemap.c:418
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
XLogRecPtr EndRecPtr
Definition: xlogreader.h:133
#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
WALOpenSegment seg
Definition: xlogreader.h:170
#define XLogRecGetData(decoder)
Definition: xlogreader.h:246
#define XLOG_DBASE_DROP
#define MAXPGPATH
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
Definition: parsexlog.c:111
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:132
char * datadir
unsigned int uint32
Definition: c.h:358
struct XLogPageReadPrivate XLogPageReadPrivate
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:242
ForkNumber
Definition: relpath.h:40
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:133
#define MAXFNAMELEN
#define RM_MAX_ID
Definition: rmgr.h:33
bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum)
Definition: xlogreader.c:1367
#define XLOGDIR
int targetNentries
Definition: pg_rewind.c:65
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
static int xlogreadfd
Definition: parsexlog.c:40
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:41
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:42
uint8 RmgrId
Definition: rmgr.h:11
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:327
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
XLogRecPtr end
Definition: timeline.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:732
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:55
size_t Size
Definition: c.h:466
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define XLOG_DBASE_CREATE
TimeLineID ws_tli
Definition: xlogreader.h:40
int WalSegSz
Definition: pg_standby.c:39
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogPageReadCB pagereadfunc, void *private_data)
Definition: xlogreader.c:71
TimeLineID ThisTimeLineID
Definition: pg_control.h:39
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo)
Definition: parsexlog.c:151
XLogRecPtr begin
Definition: timeline.h:28
WALSegmentContext segcxt
Definition: xlogreader.h:169
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:192
#define read(a, b, c)
Definition: win32.h:13
XLogRecPtr redo
Definition: pg_control.h:37
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:243
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:72
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)