PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
parsexlog.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * parsexlog.c
4  * Functions for reading Write-Ahead-Log
5  *
6  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *-------------------------------------------------------------------------
10  */
11 
12 #include "postgres_fe.h"
13 
14 #include <unistd.h>
15 
16 #include "pg_rewind.h"
17 #include "filemap.h"
18 #include "logging.h"
19 
20 #include "access/rmgr.h"
21 #include "access/xlog_internal.h"
22 #include "access/xlogreader.h"
23 #include "catalog/pg_control.h"
24 #include "catalog/storage_xlog.h"
26 
27 
28 /*
29  * RmgrNames is an array of resource manager names, to make error messages
30  * a bit nicer.
31  */
32 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
33  name,
34 
35 static const char *RmgrNames[RM_MAX_ID + 1] = {
36 #include "access/rmgrlist.h"
37 };
38 
39 static void extractPageInfo(XLogReaderState *record);
40 
41 static int xlogreadfd = -1;
43 static char xlogfpath[MAXPGPATH];
44 
45 typedef struct XLogPageReadPrivate
46 {
47  const char *datadir;
48  int tliIndex;
50 
51 static int SimpleXLogPageRead(XLogReaderState *xlogreader,
52  XLogRecPtr targetPagePtr,
53  int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
54  TimeLineID *pageTLI);
55 
56 /*
57  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
58  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
59  * the data blocks touched by the WAL records, and return them in a page map.
60  */
61 void
62 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
63  XLogRecPtr endpoint)
64 {
65  XLogRecord *record;
66  XLogReaderState *xlogreader;
67  char *errormsg;
68  XLogPageReadPrivate private;
69 
70  private.datadir = datadir;
71  private.tliIndex = tliIndex;
72  xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
73  if (xlogreader == NULL)
74  pg_fatal("out of memory\n");
75 
76  do
77  {
78  record = XLogReadRecord(xlogreader, startpoint, &errormsg);
79 
80  if (record == NULL)
81  {
82  XLogRecPtr errptr;
83 
84  errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
85 
86  if (errormsg)
87  pg_fatal("could not read WAL record at %X/%X: %s\n",
88  (uint32) (errptr >> 32), (uint32) (errptr),
89  errormsg);
90  else
91  pg_fatal("could not read WAL record at %X/%X\n",
92  (uint32) (startpoint >> 32),
93  (uint32) (startpoint));
94  }
95 
96  extractPageInfo(xlogreader);
97 
98  startpoint = InvalidXLogRecPtr; /* continue reading at next record */
99 
100  } while (xlogreader->ReadRecPtr != endpoint);
101 
102  XLogReaderFree(xlogreader);
103  if (xlogreadfd != -1)
104  {
105  close(xlogreadfd);
106  xlogreadfd = -1;
107  }
108 }
109 
110 /*
111  * Reads one WAL record. Returns the end position of the record, without
112  * doing anything with the record itself.
113  */
115 readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
116 {
117  XLogRecord *record;
118  XLogReaderState *xlogreader;
119  char *errormsg;
120  XLogPageReadPrivate private;
121  XLogRecPtr endptr;
122 
123  private.datadir = datadir;
124  private.tliIndex = tliIndex;
125  xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
126  if (xlogreader == NULL)
127  pg_fatal("out of memory\n");
128 
129  record = XLogReadRecord(xlogreader, ptr, &errormsg);
130  if (record == NULL)
131  {
132  if (errormsg)
133  pg_fatal("could not read WAL record at %X/%X: %s\n",
134  (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
135  else
136  pg_fatal("could not read WAL record at %X/%X\n",
137  (uint32) (ptr >> 32), (uint32) (ptr));
138  }
139  endptr = xlogreader->EndRecPtr;
140 
141  XLogReaderFree(xlogreader);
142  if (xlogreadfd != -1)
143  {
144  close(xlogreadfd);
145  xlogreadfd = -1;
146  }
147 
148  return endptr;
149 }
150 
151 /*
152  * Find the previous checkpoint preceding given WAL location.
153  */
154 void
155 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
156  XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
157  XLogRecPtr *lastchkptredo)
158 {
159  /* Walk backwards, starting from the given record */
160  XLogRecord *record;
161  XLogRecPtr searchptr;
162  XLogReaderState *xlogreader;
163  char *errormsg;
164  XLogPageReadPrivate private;
165 
166  /*
167  * The given fork pointer points to the end of the last common record,
168  * which is not necessarily the beginning of the next record, if the
169  * previous record happens to end at a page boundary. Skip over the page
170  * header in that case to find the next record.
171  */
172  if (forkptr % XLOG_BLCKSZ == 0)
173  forkptr += (forkptr % XLogSegSize == 0) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD;
174 
175  private.datadir = datadir;
176  private.tliIndex = tliIndex;
177  xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
178  if (xlogreader == NULL)
179  pg_fatal("out of memory\n");
180 
181  searchptr = forkptr;
182  for (;;)
183  {
184  uint8 info;
185 
186  record = XLogReadRecord(xlogreader, searchptr, &errormsg);
187 
188  if (record == NULL)
189  {
190  if (errormsg)
191  pg_fatal("could not find previous WAL record at %X/%X: %s\n",
192  (uint32) (searchptr >> 32), (uint32) (searchptr),
193  errormsg);
194  else
195  pg_fatal("could not find previous WAL record at %X/%X\n",
196  (uint32) (searchptr >> 32), (uint32) (searchptr));
197  }
198 
199  /*
200  * Check if it is a checkpoint record. This checkpoint record needs to
201  * be the latest checkpoint before WAL forked and not the checkpoint
202  * where the master has been stopped to be rewinded.
203  */
204  info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
205  if (searchptr < forkptr &&
206  XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
207  (info == XLOG_CHECKPOINT_SHUTDOWN ||
208  info == XLOG_CHECKPOINT_ONLINE))
209  {
210  CheckPoint checkPoint;
211 
212  memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
213  *lastchkptrec = searchptr;
214  *lastchkpttli = checkPoint.ThisTimeLineID;
215  *lastchkptredo = checkPoint.redo;
216  break;
217  }
218 
219  /* Walk backwards to previous record. */
220  searchptr = record->xl_prev;
221  }
222 
223  XLogReaderFree(xlogreader);
224  if (xlogreadfd != -1)
225  {
226  close(xlogreadfd);
227  xlogreadfd = -1;
228  }
229 }
230 
231 /* XLogreader callback function, to read a WAL page */
232 static int
233 SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
234  int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
235  TimeLineID *pageTLI)
236 {
237  XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
238  uint32 targetPageOff;
239  XLogRecPtr targetSegEnd;
240  XLogSegNo targetSegNo;
241 
242  XLByteToSeg(targetPagePtr, targetSegNo);
243  XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, targetSegEnd);
244  targetPageOff = targetPagePtr % XLogSegSize;
245 
246  /*
247  * See if we need to switch to a new segment because the requested record
248  * is not in the currently open one.
249  */
250  if (xlogreadfd >= 0 && !XLByteInSeg(targetPagePtr, xlogreadsegno))
251  {
252  close(xlogreadfd);
253  xlogreadfd = -1;
254  }
255 
256  XLByteToSeg(targetPagePtr, xlogreadsegno);
257 
258  if (xlogreadfd < 0)
259  {
260  char xlogfname[MAXFNAMELEN];
261 
262  /*
263  * Since incomplete segments are copied into next timelines, switch to
264  * the timeline holding the required segment. Assuming this scan can
265  * be done both forward and backward, consider also switching timeline
266  * accordingly.
267  */
268  while (private->tliIndex < targetNentries - 1 &&
269  targetHistory[private->tliIndex].end < targetSegEnd)
270  private->tliIndex++;
271  while (private->tliIndex > 0 &&
272  targetHistory[private->tliIndex].begin >= targetSegEnd)
273  private->tliIndex--;
274 
275  XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, xlogreadsegno);
276 
277  snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
278 
279  xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
280 
281  if (xlogreadfd < 0)
282  {
283  printf(_("could not open file \"%s\": %s\n"), xlogfpath,
284  strerror(errno));
285  return -1;
286  }
287  }
288 
289  /*
290  * At this point, we have the right segment open.
291  */
292  Assert(xlogreadfd != -1);
293 
294  /* Read the requested page */
295  if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
296  {
297  printf(_("could not seek in file \"%s\": %s\n"), xlogfpath,
298  strerror(errno));
299  return -1;
300  }
301 
302  if (read(xlogreadfd, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ)
303  {
304  printf(_("could not read from file \"%s\": %s\n"), xlogfpath,
305  strerror(errno));
306  return -1;
307  }
308 
309  Assert(targetSegNo == xlogreadsegno);
310 
311  *pageTLI = targetHistory[private->tliIndex].tli;
312  return XLOG_BLCKSZ;
313 }
314 
315 /*
316  * Extract information on which blocks the current record modifies.
317  */
318 static void
320 {
321  int block_id;
322  RmgrId rmid = XLogRecGetRmid(record);
323  uint8 info = XLogRecGetInfo(record);
324  uint8 rminfo = info & ~XLR_INFO_MASK;
325 
326  /* Is this a special record type that I recognize? */
327 
328  if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
329  {
330  /*
331  * New databases can be safely ignored. It won't be present in the
332  * source system, so it will be deleted. There's one corner-case,
333  * though: if a new, different, database is also created in the source
334  * system, we'll see that the files already exist and not copy them.
335  * That's OK, though; WAL replay of creating the new database, from
336  * the source systems's WAL, will re-copy the new database,
337  * overwriting the database created in the target system.
338  */
339  }
340  else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
341  {
342  /*
343  * An existing database was dropped. We'll see that the files don't
344  * exist in the target data dir, and copy them in toto from the source
345  * system. No need to do anything special here.
346  */
347  }
348  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
349  {
350  /*
351  * We can safely ignore these. The file will be removed from the
352  * target, if it doesn't exist in source system. If a file with same
353  * name is created in source system, too, there will be WAL records
354  * for all the blocks in it.
355  */
356  }
357  else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
358  {
359  /*
360  * We can safely ignore these. When we compare the sizes later on,
361  * we'll notice that they differ, and copy the missing tail from
362  * source system.
363  */
364  }
365  else if (info & XLR_SPECIAL_REL_UPDATE)
366  {
367  /*
368  * This record type modifies a relation file in some special way, but
369  * we don't recognize the type. That's bad - we don't know how to
370  * track that change.
371  */
372  pg_fatal("WAL record modifies a relation, but record type is not recognized\n"
373  "lsn: %X/%X, rmgr: %s, info: %02X\n",
374  (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
375  RmgrNames[rmid], info);
376  }
377 
378  for (block_id = 0; block_id <= record->max_block_id; block_id++)
379  {
380  RelFileNode rnode;
381  ForkNumber forknum;
382  BlockNumber blkno;
383 
384  if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
385  continue;
386 
387  /* We only care about the main fork; others are copied in toto */
388  if (forknum != MAIN_FORKNUM)
389  continue;
390 
391  process_block_change(forknum, rnode, blkno);
392  }
393 }
XLogReaderState * XLogReaderAllocate(XLogPageReadCB pagereadfunc, void *private_data)
Definition: xlogreader.c:67
#define XLogSegSize
Definition: xlog_internal.h:92
XLogRecPtr xl_prev
Definition: xlogrecord.h:45
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
uint32 TimeLineID
Definition: xlogdefs.h:45
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
#define XLOG_CHECKPOINT_ONLINE
Definition: pg_control.h:68
TimeLineHistoryEntry * targetHistory
Definition: pg_rewind.c:58
#define XLogFileName(fname, tli, logSegNo)
unsigned char uint8
Definition: c.h:256
void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint)
Definition: parsexlog.c:62
static const char * RmgrNames[RM_MAX_ID+1]
Definition: parsexlog.c:35
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
uint32 BlockNumber
Definition: block.h:31
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
TimeLineID tli
Definition: timeline.h:27
void * private_data
Definition: xlogreader.h:108
#define PG_BINARY
Definition: c.h:1027
XLogRecord * XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
Definition: xlogreader.c:193
void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
Definition: filemap.c:343
#define XLogSegNoOffsetToRecPtr(segno, offset, dest)
Definition: xlog_internal.h:95
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
XLogRecPtr EndRecPtr
Definition: xlogreader.h:115
#define XLogRecGetData(decoder)
Definition: xlogreader.h:220
#define XLOG_DBASE_DROP
#define MAXPGPATH
XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
Definition: parsexlog.c:115
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
uint64 XLogSegNo
Definition: xlogdefs.h:34
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:114
char * datadir
unsigned int uint32
Definition: c.h:258
struct XLogPageReadPrivate XLogPageReadPrivate
const char * datadir
Definition: parsexlog.c:47
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:216
ForkNumber
Definition: relpath.h:24
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:125
#define MAXFNAMELEN
#define RM_MAX_ID
Definition: rmgr.h:33
bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum)
Definition: xlogreader.c:1307
#define XLOGDIR
int targetNentries
Definition: pg_rewind.c:59
static int xlogreadfd
Definition: parsexlog.c:41
static XLogSegNo xlogreadsegno
Definition: parsexlog.c:42
static char xlogfpath[MAXPGPATH]
Definition: parsexlog.c:43
uint8 RmgrId
Definition: rmgr.h:11
#define XLByteToSeg(xlrp, logSegNo)
static void extractPageInfo(XLogReaderState *record)
Definition: parsexlog.c:319
XLogRecPtr end
Definition: timeline.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:664
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
#define SizeOfXLogShortPHD
Definition: xlog_internal.h:55
#define XLOG_DBASE_CREATE
TimeLineID ThisTimeLineID
Definition: pg_control.h:38
void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo)
Definition: parsexlog.c:155
XLogRecPtr begin
Definition: timeline.h:28
const char * strerror(int errnum)
Definition: strerror.c:19
static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI)
Definition: parsexlog.c:233
#define close(a)
Definition: win32.h:12
#define XLByteInSeg(xlrp, logSegNo)
#define _(x)
Definition: elog.c:84
#define read(a, b, c)
Definition: win32.h:13
XLogRecPtr redo
Definition: pg_control.h:36
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:217
#define SizeOfXLogLongPHD
Definition: xlog_internal.h:72