PostgreSQL Source Code  git master
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "access/rmgr.h"
17 #include "access/xlog_internal.h"
18 #include "access/xlogreader.h"
19 #include "catalog/pg_control.h"
20 #include "catalog/storage_xlog.h"
22 #include "filemap.h"
23 #include "pg_rewind.h"
24 
25 /*
26  * RmgrNames is an array of resource manager names, to make error messages
27  * a bit nicer.
28  */
29 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
30  name,
31 
32 static const char *RmgrNames[RM_MAX_ID + 1] = {
33 #include "access/rmgrlist.h"
34 };
35 
36 static void extractPageInfo(XLogReaderState *record);
37 
38 static int xlogreadfd = -1;
40 static char xlogfpath[MAXPGPATH];
41 
42 typedef struct XLogPageReadPrivate
43 {
44  int tliIndex;
46 
47 static int SimpleXLogPageRead(XLogReaderState *xlogreader,
48  XLogRecPtr targetPagePtr,
49  int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
50 
51 /*
52  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
53  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
54  * the data blocks touched by the WAL records, and return them in a page map.
55  */
56 void
57 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
58  XLogRecPtr endpoint)
59 {
60  XLogRecord *record;
61  XLogReaderState *xlogreader;
62  char *errormsg;
63  XLogPageReadPrivate private;
64 
65  private.tliIndex = tliIndex;
66  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
67  &private);
68  if (xlogreader == NULL)
69  pg_fatal("out of memory");
70 
71  XLogBeginRead(xlogreader, startpoint);
72  do
73  {
74  record = XLogReadRecord(xlogreader, &errormsg);
75 
76  if (record == NULL)
77  {
78  XLogRecPtr errptr = xlogreader->EndRecPtr;
79 
80  if (errormsg)
81  pg_fatal("could not read WAL record at %X/%X: %s",
82  (uint32) (errptr >> 32), (uint32) (errptr),
83  errormsg);
84  else
85  pg_fatal("could not read WAL record at %X/%X",
86  (uint32) (errptr >> 32), (uint32) (errptr));
87  }
88 
89  extractPageInfo(xlogreader);
90 
91  } while (xlogreader->ReadRecPtr != endpoint);
92 
93  XLogReaderFree(xlogreader);
94  if (xlogreadfd != -1)
95  {
97  xlogreadfd = -1;
98  }
99 }
100 
101 /*
102  * Reads one WAL record. Returns the end position of the record, without
103  * doing anything with the record itself.
104  */
106 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
107 {
108  XLogRecord *record;
109  XLogReaderState *xlogreader;
110  char *errormsg;
111  XLogPageReadPrivate private;
112  XLogRecPtr endptr;
113 
114  private.tliIndex = tliIndex;
115  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
116  &private);
117  if (xlogreader == NULL)
118  pg_fatal("out of memory");
119 
120  XLogBeginRead(xlogreader, ptr);
121  record = XLogReadRecord(xlogreader, &errormsg);
122  if (record == NULL)
123  {
124  if (errormsg)
125  pg_fatal("could not read WAL record at %X/%X: %s",
126  (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
127  else
128  pg_fatal("could not read WAL record at %X/%X",
129  (uint32) (ptr >> 32), (uint32) (ptr));
130  }
131  endptr = xlogreader->EndRecPtr;
132 
133  XLogReaderFree(xlogreader);
134  if (xlogreadfd != -1)
135  {
136  close(xlogreadfd);
137  xlogreadfd = -1;
138  }
139 
140  return endptr;
141 }
142 
143 /*
144  * Find the previous checkpoint preceding given WAL location.
145  */
146 void
147 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
148  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
149  XLogRecPtr *lastchkptredo)
150 {
151  /* Walk backwards, starting from the given record */
152  XLogRecord *record;
153  XLogRecPtr searchptr;
154  XLogReaderState *xlogreader;
155  char *errormsg;
156  XLogPageReadPrivate private;
157 
158  /*
159  * The given fork pointer points to the end of the last common record,
160  * which is not necessarily the beginning of the next record, if the
161  * previous record happens to end at a page boundary. Skip over the page
162  * header in that case to find the next record.
163  */
164  if (forkptr % XLOG_BLCKSZ == 0)
165  {
166  if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
167  forkptr += SizeOfXLogLongPHD;
168  else
169  forkptr += SizeOfXLogShortPHD;
170  }
171 
172  private.tliIndex = tliIndex;
173  xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
174  &private);
175  if (xlogreader == NULL)
176  pg_fatal("out of memory");
177 
178  searchptr = forkptr;
179  for (;;)
180  {
181  uint8 info;
182 
183  XLogBeginRead(xlogreader, searchptr);
184  record = XLogReadRecord(xlogreader, &errormsg);
185 
186  if (record == NULL)
187  {
188  if (errormsg)
189  pg_fatal("could not find previous WAL record at %X/%X: %s",
190  (uint32) (searchptr >> 32), (uint32) (searchptr),
191  errormsg);
192  else
193  pg_fatal("could not find previous WAL record at %X/%X",
194  (uint32) (searchptr >> 32), (uint32) (searchptr));
195  }
196 
197  /*
198  * Check if it is a checkpoint record. This checkpoint record needs to
199  * be the latest checkpoint before WAL forked and not the checkpoint
200  * where the master has been stopped to be rewinded.
201  */
202  info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
203  if (searchptr < forkptr &&
204  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
205  (info == XLOG_CHECKPOINT_SHUTDOWN ||
206  info == XLOG_CHECKPOINT_ONLINE))
207  {
208  CheckPoint checkPoint;
209 
210  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
211  *lastchkptrec = searchptr;
212  *lastchkpttli = checkPoint.ThisTimeLineID;
213  *lastchkptredo = checkPoint.redo;
214  break;
215  }
216 
217  /* Walk backwards to previous record. */
218  searchptr = record->xl_prev;
219  }
220 
221  XLogReaderFree(xlogreader);
222  if (xlogreadfd != -1)
223  {
224  close(xlogreadfd);
225  xlogreadfd = -1;
226  }
227 }
228 
229 /* XLogReader callback function, to read a WAL page */
230 static int
231 SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
232  int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
233 {
234  XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
235  uint32 targetPageOff;
236  XLogRecPtr targetSegEnd;
237  XLogSegNo targetSegNo;
238  int r;
239 
240  XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
241  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
242  targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
243 
244  /*
245  * See if we need to switch to a new segment because the requested record
246  * is not in the currently open one.
247  */
248  if (xlogreadfd >= 0 &&
249  !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
250  {
251  close(xlogreadfd);
252  xlogreadfd = -1;
253  }
254 
255  XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
256 
257  if (xlogreadfd < 0)
258  {
259  char xlogfname[MAXFNAMELEN];
260 
261  /*
262  * Since incomplete segments are copied into next timelines, switch to
263  * the timeline holding the required segment. Assuming this scan can
264  * be done both forward and backward, consider also switching timeline
265  * accordingly.
266  */
267  while (private->tliIndex < targetNentries - 1 &&
268  targetHistory[private->tliIndex].end < targetSegEnd)
269  private->tliIndex++;
270  while (private->tliIndex > 0 &&
271  targetHistory[private->tliIndex].begin >= targetSegEnd)
272  private->tliIndex--;
273 
274  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
276 
277  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
278  xlogreader->segcxt.ws_dir, xlogfname);
279 
280  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
281 
282  if (xlogreadfd < 0)
283  {
284  pg_log_error("could not open file \"%s\": %m", xlogfpath);
285  return -1;
286  }
287  }
288 
289  /*
290  * At this point, we have the right segment open.
291  */
292  Assert(xlogreadfd != -1);
293 
294  /* Read the requested page */
295  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
296  {
297  pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
298  return -1;
299  }
300 
301 
302  r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
303  if (r != XLOG_BLCKSZ)
304  {
305  if (r < 0)
306  pg_log_error("could not read file \"%s\": %m", xlogfpath);
307  else
308  pg_log_error("could not read file \"%s\": read %d of %zu",
309  xlogfpath, r, (Size) XLOG_BLCKSZ);
310 
311  return -1;
312  }
313 
314  Assert(targetSegNo == xlogreadsegno);
315 
316  xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
317  return XLOG_BLCKSZ;
318 }
319 
320 /*
321  * Extract information on which blocks the current record modifies.
322  */
323 static void
325 {
326  int block_id;
327  RmgrId rmid = XLogRecGetRmid(record);
328  uint8 info = XLogRecGetInfo(record);
329  uint8 rminfo = info & ~XLR_INFO_MASK;
330 
331  /* Is this a special record type that I recognize? */
332 
333  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
334  {
335  /*
336  * New databases can be safely ignored. It won't be present in the
337  * source system, so it will be deleted. There's one corner-case,
338  * though: if a new, different, database is also created in the source
339  * system, we'll see that the files already exist and not copy them.
340  * That's OK, though; WAL replay of creating the new database, from
341  * the source systems's WAL, will re-copy the new database,
342  * overwriting the database created in the target system.
343  */
344  }
345  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
346  {
347  /*
348  * An existing database was dropped. We'll see that the files don't
349  * exist in the target data dir, and copy them in toto from the source
350  * system. No need to do anything special here.
351  */
352  }
353  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
354  {
355  /*
356  * We can safely ignore these. The file will be removed from the
357  * target, if it doesn't exist in source system. If a file with same
358  * name is created in source system, too, there will be WAL records
359  * for all the blocks in it.
360  */
361  }
362  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
363  {
364  /*
365  * We can safely ignore these. When we compare the sizes later on,
366  * we'll notice that they differ, and copy the missing tail from
367  * source system.
368  */
369  }
370  else if (info & XLR_SPECIAL_REL_UPDATE)
371  {
372  /*
373  * This record type modifies a relation file in some special way, but
374  * we don't recognize the type. That's bad - we don't know how to
375  * track that change.
376  */
377  pg_fatal("WAL record modifies a relation, but record type is not recognized: "
378  "lsn: %X/%X, rmgr: %s, info: %02X",
379  (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
380  RmgrNames[rmid], info);
381  }
382 
383  for (block_id = 0; block_id <= record->max_block_id; block_id++)
384  {
385  RelFileNode rnode;
386  ForkNumber forknum;
387  BlockNumber blkno;
388 
389  if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
390  continue;
391 
392  /* We only care about the main fork; others are copied in toto */
393  if (forknum != MAIN_FORKNUM)
394  continue;
395 
396  process_block_change(forknum, rnode, blkno);
397  }
398 }
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
char ws_dir[MAXPGPATH]
Definition: xlogreader.h:47
uint32 TimeLineID
Definition: xlogdefs.h:52
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:68
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:63
#define pg_log_error(...)
Definition: logging.h:79
unsigned char uint8
Definition: c.h:357
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint)
Definition: parsexlog.c:57
#define pg_fatal(...)
Definition: pg_rewind.h:41
static const char * RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:32
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
Definition: parsexlog.c:231
uint32 BlockNumber
Definition: block.h:31
TimeLineID tli
Definition: timeline.h:27
void * private_data
Definition: xlogreader.h:127
#define PG_BINARY
Definition: c.h:1221
void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
Definition: filemap.c:417
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
XLogRecPtr EndRecPtr
Definition: xlogreader.h:135
#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
WALOpenSegment seg
Definition: xlogreader.h:172
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:261
#define XLogRecGetData(decoder)
Definition: xlogreader.h:288
#define XLOG_DBASE_DROP
#define MAXPGPATH
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
Definition: parsexlog.c:106
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:134
char * datadir
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:233
unsigned int uint32
Definition: c.h:359
struct XLogPageReadPrivate XLogPageReadPrivate
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:284
ForkNumber
Definition: relpath.h:40
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:135
#define MAXFNAMELEN
#define RM_MAX_ID
Definition: rmgr.h:33
bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum)
Definition: xlogreader.c:1481
#define XLOGDIR
int targetNentries
Definition: pg_rewind.c:64
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
static int xlogreadfd
Definition: parsexlog.c:38
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:39
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:40
uint8 RmgrId
Definition: rmgr.h:11
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:324
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
XLogRecPtr end
Definition: timeline.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:739
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:55
size_t Size
Definition: c.h:467
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define XLOG_DBASE_CREATE
TimeLineID ws_tli
Definition: xlogreader.h:41
int WalSegSz
Definition: pg_standby.c:38
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogPageReadCB pagereadfunc, void *private_data)
Definition: xlogreader.c:73
TimeLineID ThisTimeLineID
Definition: pg_control.h:39
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo)
Definition: parsexlog.c:147
XLogRecPtr begin
Definition: timeline.h:28
WALSegmentContext segcxt
Definition: xlogreader.h:171
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:192
#define read(a, b, c)
Definition: win32.h:13
XLogRecPtr redo
Definition: pg_control.h:37
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:285
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:72
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)