PostgreSQL Source Code  git master
xlogreader.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * xlogreader.h
4  * Definitions for the generic XLog reading facility
5  *
6  * Portions Copyright (c) 2013-2021, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/include/access/xlogreader.h
10  *
11  * NOTES
12  * See the definition of the XLogReaderState struct for instructions on
13  * how to use the XLogReader infrastructure.
14  *
15  * The basic idea is to allocate an XLogReaderState via
16  * XLogReaderAllocate(), position the reader to the first record with
17  * XLogBeginRead() or XLogFindNextRecord(), and call XLogReadRecord()
18  * until it returns NULL.
19  *
20  * Callers supply a page_read callback if they want to call
21  * XLogReadRecord or XLogFindNextRecord; it can be passed in as NULL
22  * otherwise. The WALRead function can be used as a helper to write
23  * page_read callbacks, but it is not mandatory; callers that use it,
24  * must supply segment_open callbacks. The segment_close callback
25  * must always be supplied.
26  *
27  * After reading a record with XLogReadRecord(), it's decomposed into
28  * the per-block and main data parts, and the parts can be accessed
29  * with the XLogRec* macros and functions. You can also decode a
30  * record that's already constructed in memory, without reading from
31  * disk, by calling the DecodeXLogRecord() function.
32  *-------------------------------------------------------------------------
33  */
34 #ifndef XLOGREADER_H
35 #define XLOGREADER_H
36 
37 #ifndef FRONTEND
38 #include "access/transam.h"
39 #endif
40 
41 #include "access/xlogrecord.h"
42 #include "storage/buf.h"
43 
44 /* WALOpenSegment represents a WAL segment being read. */
45 typedef struct WALOpenSegment
46 {
47  int ws_file; /* segment file descriptor */
48  XLogSegNo ws_segno; /* segment number */
49  TimeLineID ws_tli; /* timeline ID of the currently open file */
51 
52 /* WALSegmentContext carries context information about WAL segments to read */
53 typedef struct WALSegmentContext
54 {
55  char ws_dir[MAXPGPATH];
58 
61 
62 /* Function type definition for the segment cleanup callback */
64 
65 /* Function type definition for the open/close callbacks for WALRead() */
67  XLogSegNo nextSegNo,
68  TimeLineID *tli_p);
70 
71 typedef struct
72 {
73  /* Is this block ref in use? */
74  bool in_use;
75 
76  /* Identify the block this refers to */
80 
81  /* Workspace for remembering last known buffer holding this block. */
83 
84  /* copy of the fork_flags field from the XLogRecordBlockHeader */
86 
87  /* Information on full-page image, if any */
88  bool has_image; /* has image, even for consistency checking */
89  bool apply_image; /* has image that should be restored */
90  char *bkp_image;
95 
96  /* Buffer holding the rmgr-specific data associated with this block */
97  bool has_data;
98  char *data;
102 
103 /* Return code from XLogReadRecord */
105 {
106  XLREAD_SUCCESS, /* record is successfully read */
107  XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
108  XLREAD_FULL, /* cannot hold more data while reading ahead */
109  XLREAD_FAIL /* failed during reading a record */
111 
112 /*
113  * internal state of XLogReadRecord
114  *
115  * XLogReadState runs a state machine while reading a record. Theses states
116  * are not seen outside the function. Each state may repeat several times
117  * exiting requesting caller for new data. See the comment of XLogReadRecrod
118  * for details.
119  */
121 {
127 
128 /*
129  * The decoded contents of a record. This occupies a contiguous region of
130  * memory, with main_data and blocks[n].data pointing to memory after the
131  * members declared here.
132  */
133 typedef struct DecodedXLogRecord
134 {
135  /* Private member used for resource management. */
136  size_t size; /* total size of decoded record */
137  bool oversized; /* outside the regular decode buffer? */
138  struct DecodedXLogRecord *next; /* decoded record queue link */
139 
140  /* Public members. */
141  XLogRecPtr lsn; /* location */
142  XLogRecPtr next_lsn; /* location of next record */
143  XLogRecord header; /* header */
145  TransactionId toplevel_xid; /* XID of top-level transaction */
146  char *main_data; /* record's main data portion */
147  uint32 main_data_len; /* main data portion's length */
148  int max_block_id; /* highest block_id in use (-1 if none) */
151 
153 {
154  /*
155  * Operational callbacks
156  */
158 
159  /* ----------------------------------------
160  * Public parameters
161  * ----------------------------------------
162  */
163 
164  /*
165  * System identifier of the xlog files we're about to read. Set to zero
166  * (the default value) if unknown or unimportant.
167  */
169 
170  /*
171  * Start and end point of last record read. EndRecPtr is also used as the
172  * position to read next. Calling XLogBeginRead() sets EndRecPtr to the
173  * starting position and ReadRecPtr to invalid.
174  *
175  * Start and end point of last record returned by XLogReadRecord(). These
176  * are also available as record->lsn and record->next_lsn.
177  */
178  XLogRecPtr ReadRecPtr; /* start of last record read or being read */
179  XLogRecPtr EndRecPtr; /* end+1 of last record read */
180 
181  /* ----------------------------------------
182  * Communication with page reader
183  * readBuf is XLOG_BLCKSZ bytes, valid up to at least reqLen bytes.
184  * ----------------------------------------
185  */
186  /* variables the clients of xlogreader can examine */
187  XLogRecPtr readPagePtr; /* page pointer to read */
188  int32 reqLen; /* bytes requested to the caller */
189  char *readBuf; /* buffer to store data */
190  bool page_verified; /* is the page header on the buffer verified? */
191  bool record_verified;/* is the current record header verified? */
192 
193  /* variables set by the client of xlogreader */
194  int32 readLen; /* actual bytes copied into readBuf by client,
195  * which should be >= reqLen. Client should
196  * use XLogReaderSetInputData() to set. */
197 
198  /* ----------------------------------------
199  * Decoded representation of current record
200  *
201  * Use XLogRecGet* functions to investigate the record; these fields
202  * should not be accessed directly.
203  * ----------------------------------------
204  * Start and end point of the last record read and decoded by
205  * XLogReadRecordInternal(). NextRecPtr is also used as the position to
206  * decode next. Calling XLogBeginRead() sets NextRecPtr and EndRecPtr to
207  * the requested starting position.
208  */
209  XLogRecPtr DecodeRecPtr; /* start of last record decoded */
210  XLogRecPtr NextRecPtr; /* end+1 of last record decoded */
211  XLogRecPtr PrevRecPtr; /* start of previous record decoded */
212 
213  /* Last record returned by XLogReadRecord(). */
215 
216  /* ----------------------------------------
217  * private/internal state
218  * ----------------------------------------
219  */
220 
221  /*
222  * Buffer for decoded records. This is a circular buffer, though
223  * individual records can't be split in the middle, so some space is often
224  * wasted at the end. Oversized records that don't fit in this space are
225  * allocated separately.
226  */
229  bool free_decode_buffer; /* need to free? */
230  char *decode_buffer_head; /* write head */
231  char *decode_buffer_tail; /* read head */
232 
233  /*
234  * Queue of records that have been decoded. This is a linked list that
235  * usually consists of consecutive records in decode_buffer, but may also
236  * contain oversized records allocated with palloc().
237  */
238  DecodedXLogRecord *decode_queue_head; /* newest decoded record */
239  DecodedXLogRecord *decode_queue_tail; /* oldest decoded record */
240 
241  /* last read XLOG position for data currently in readBuf */
245 
246  /*
247  * beginning of prior page read, and its TLI. Doesn't necessarily
248  * correspond to what's in readBuf; used for timeline sanity checks.
249  */
252 
253  /* timeline to read it from, 0 if a lookup is required */
255 
256  /*
257  * Safe point to read to in currTLI if current TLI is historical
258  * (tliSwitchPoint) or InvalidXLogRecPtr if on current timeline.
259  *
260  * Actually set to the start of the segment containing the timeline switch
261  * that ends currTLI's validity, not the LSN of the switch its self, since
262  * we can't assume the old segment will be present.
263  */
265 
266  /*
267  * If currTLI is not the most recent known timeline, the next timeline to
268  * read from when currTLIValidUntil is reached.
269  */
271 
272  /*
273  * Buffer for current ReadRecord result (expandable), used when a record
274  * crosses a page boundary.
275  */
278 
279  /*
280  * XLogReadRecordInternal() state
281  */
282  XLogReadRecordState readRecordState; /* state machine state */
283  int recordGotLen; /* amount of current record that has already
284  * been read */
285  int recordRemainLen; /* length of current record that remains */
286  XLogRecPtr recordContRecPtr; /* where the current record continues */
287 
288  DecodedXLogRecord *decoding; /* record currently being decoded */
289 
290  /* Buffer to hold error message */
293 };
294 
296 {
300 };
301 
302 /* Report that data is available for decoding. */
303 static inline void
305 {
306  state->readLen = len;
307 }
308 
309 /* Get a new XLogReader */
311  const char *waldir,
312  WALSegmentCleanupCB cleanup_cb);
313 
314 /* Free an XLogReader */
315 extern void XLogReaderFree(XLogReaderState *state);
316 
317 /* Optionally provide a circular decoding buffer to allow readahead. */
318 extern void XLogReaderSetDecodeBuffer(XLogReaderState *state,
319  void *buffer,
320  size_t size);
321 
322 /* Position the XLogReader to given record */
323 extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
324 #ifdef FRONTEND
325 extern XLogFindNextRecordState *InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr);
326 extern bool XLogFindNextRecord(XLogFindNextRecordState *state);
327 #endif /* FRONTEND */
328 
329 /* Read the next record's header. Returns NULL on end-of-WAL or failure. */
331  XLogRecord **record,
332  char **errormsg);
333 
334 /* Read the next decoded record. Returns NULL on end-of-WAL or failure. */
336  DecodedXLogRecord **record,
337  char **errormsg);
338 
339 /* Try to read ahead, if there is space in the decoding buffer. */
341  DecodedXLogRecord **record,
342  char **errormsg);
343 
344 /* Validate a page */
346  XLogRecPtr recptr, char *phdr);
347 
348 /*
349  * Error information from WALRead that both backend and frontend caller can
350  * process. Currently only errors from pg_pread can be reported.
351  */
352 typedef struct WALReadError
353 {
354  int wre_errno; /* errno set by the last pg_pread() */
355  int wre_off; /* Offset we tried to read from. */
356  int wre_req; /* Bytes requested to be read. */
357  int wre_read; /* Bytes read by the last read(). */
358  WALOpenSegment wre_seg; /* Segment we tried to read from. */
359 } WALReadError;
360 
361 extern bool WALRead(XLogReaderState *state,
362  WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
363  char *buf, XLogRecPtr startptr, Size count,
364  TimeLineID tli, WALReadError *errinfo);
365 
366 /* Functions for decoding an XLogRecord */
367 
368 extern size_t DecodeXLogRecordRequiredSpace(size_t xl_tot_len);
369 extern bool DecodeXLogRecord(XLogReaderState *state,
370  DecodedXLogRecord *decoded,
371  XLogRecord *record,
372  XLogRecPtr lsn,
373  char **errmsg);
374 
375 #define XLogRecGetTotalLen(decoder) ((decoder)->record->header.xl_tot_len)
376 #define XLogRecGetPrev(decoder) ((decoder)->record->header.xl_prev)
377 #define XLogRecGetInfo(decoder) ((decoder)->record->header.xl_info)
378 #define XLogRecGetRmid(decoder) ((decoder)->record->header.xl_rmid)
379 #define XLogRecGetXid(decoder) ((decoder)->record->header.xl_xid)
380 #define XLogRecGetOrigin(decoder) ((decoder)->record->record_origin)
381 #define XLogRecGetTopXid(decoder) ((decoder)->record->toplevel_xid)
382 #define XLogRecGetData(decoder) ((decoder)->record->main_data)
383 #define XLogRecGetDataLen(decoder) ((decoder)->record->main_data_len)
384 #define XLogRecHasAnyBlockRefs(decoder) ((decoder)->record->max_block_id >= 0)
385 #define XLogRecMaxBlockId(decoder) ((decoder)->record->max_block_id)
386 #define XLogRecGetBlock(decoder, i) (&(decoder)->record->blocks[(i)])
387 #define XLogRecHasBlockRef(decoder, block_id) \
388  ((decoder)->record->max_block_id >= (block_id) && \
389  (decoder)->record->blocks[block_id].in_use)
390 #define XLogRecHasBlockImage(decoder, block_id) \
391  ((decoder)->record->blocks[block_id].has_image)
392 #define XLogRecBlockImageApply(decoder, block_id) \
393  ((decoder)->record->blocks[block_id].apply_image)
394 
395 #ifndef FRONTEND
397 #endif
398 
399 extern bool RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page);
400 extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len);
401 extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id,
402  RelFileNode *rnode, ForkNumber *forknum,
403  BlockNumber *blknum);
404 extern bool XLogRecGetRecentBuffer(XLogReaderState *record, uint8 block_id,
405  RelFileNode *rnode, ForkNumber *forknum,
406  BlockNumber *blknum, Buffer *recent_buffer);
407 
408 #endif /* XLOGREADER_H */
WALOpenSegment wre_seg
Definition: xlogreader.h:358
BlockNumber blkno
Definition: xlogreader.h:79
bool RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
Definition: xlogreader.c:2203
XLogReadRecordState
Definition: xlogreader.h:120
DecodedXLogRecord * decode_queue_tail
Definition: xlogreader.h:239
struct WALSegmentContext WALSegmentContext
size_t DecodeXLogRecordRequiredSpace(size_t xl_tot_len)
Definition: xlogreader.c:1793
char * decode_buffer
Definition: xlogreader.h:227
char * readRecordBuf
Definition: xlogreader.h:276
struct DecodedXLogRecord * next
Definition: xlogreader.h:138
uint32 TimeLineID
Definition: xlogdefs.h:59
XLogRecPtr recordContRecPtr
Definition: xlogreader.h:286
int wal_segment_size
Definition: xlog.c:121
bool record_verified
Definition: xlogreader.h:191
uint32 TransactionId
Definition: c.h:587
XLogRecPtr NextRecPtr
Definition: xlogreader.h:210
TransactionId toplevel_xid
Definition: xlogreader.h:145
XLogReadRecordResult XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Definition: xlogreader.c:346
char * decode_buffer_tail
Definition: xlogreader.h:231
uint16 hole_offset
Definition: xlogreader.h:91
bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum)
Definition: xlogreader.c:2137
unsigned char uint8
Definition: c.h:439
uint16 RepOriginId
Definition: xlogdefs.h:65
XLogRecPtr next_lsn
Definition: xlogreader.h:142
#define FLEXIBLE_ARRAY_MEMBER
Definition: c.h:350
size_t decode_buffer_size
Definition: xlogreader.h:228
uint32 BlockNumber
Definition: block.h:31
bool DecodeXLogRecord(XLogReaderState *state, DecodedXLogRecord *decoded, XLogRecord *record, XLogRecPtr lsn, char **errmsg)
Definition: xlogreader.c:1826
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:252
uint16 bimg_len
Definition: xlogreader.h:93
char * decode_buffer_head
Definition: xlogreader.h:230
XLogReadRecordResult
Definition: xlogreader.h:104
uint32 main_data_len
Definition: xlogreader.h:147
signed int int32
Definition: c.h:429
XLogRecPtr EndRecPtr
Definition: xlogreader.h:179
WALOpenSegment seg
Definition: xlogreader.h:243
void(* WALSegmentCleanupCB)(XLogReaderState *xlogreader)
Definition: xlogreader.h:63
unsigned short uint16
Definition: c.h:440
XLogRecPtr latestPagePtr
Definition: xlogreader.h:250
uint16 hole_length
Definition: xlogreader.h:92
struct DecodedXLogRecord DecodedXLogRecord
struct WALReadError WALReadError
Buffer recent_buffer
Definition: xlogreader.h:82
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:68
XLogRecPtr readPagePtr
Definition: xlogreader.h:187
uint64 XLogSegNo
Definition: xlogdefs.h:48
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:178
struct WALOpenSegment WALOpenSegment
XLogSegNo ws_segno
Definition: xlogreader.h:48
XLogRecPtr PrevRecPtr
Definition: xlogreader.h:211
bool free_decode_buffer
Definition: xlogreader.h:229
XLogRecPtr currTLIValidUntil
Definition: xlogreader.h:264
XLogRecPtr DecodeRecPtr
Definition: xlogreader.h:209
unsigned int uint32
Definition: c.h:441
XLogRecPtr lsn
Definition: xlogreader.h:141
RepOriginId record_origin
Definition: xlogreader.h:144
void(* WALSegmentCloseCB)(XLogReaderState *xlogreader)
Definition: xlogreader.h:69
static XLogReaderState * xlogreader
Definition: walsender.c:137
void(* WALSegmentOpenCB)(XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p)
Definition: xlogreader.h:66
ForkNumber
Definition: relpath.h:40
uint32 readRecordBufSize
Definition: xlogreader.h:277
TimeLineID nextTLI
Definition: xlogreader.h:270
DecodedXLogRecord * decoding
Definition: xlogreader.h:288
FullTransactionId XLogRecGetFullXid(XLogReaderState *record)
Definition: xlogreader.c:2256
XLogReaderState * reader_state
Definition: xlogreader.h:297
ForkNumber forknum
Definition: xlogreader.h:78
XLogReadRecordResult XLogNextRecord(XLogReaderState *state, DecodedXLogRecord **record, char **errormsg)
Definition: xlogreader.c:397
bool errormsg_deferred
Definition: xlogreader.h:292
DecodedXLogRecord * record
Definition: xlogreader.h:214
uint16 data_len
Definition: xlogreader.h:99
TimeLineID currTLI
Definition: xlogreader.h:254
XLogReadRecordResult XLogReadAhead(XLogReaderState *state, DecodedXLogRecord **record, char **errormsg)
Definition: xlogreader.c:511
uint64 XLogRecPtr
Definition: xlogdefs.h:21
Definition: regguts.h:317
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:142
bool XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, char *phdr)
Definition: xlogreader.c:1370
uint16 data_bufsz
Definition: xlogreader.h:100
size_t Size
Definition: c.h:540
bool XLogRecGetRecentBuffer(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum, Buffer *recent_buffer)
Definition: xlogreader.c:2145
static int64 blocks
Definition: pg_checksums.c:35
TimeLineID ws_tli
Definition: xlogreader.h:49
char * bkp_image
Definition: xlogreader.h:90
char * XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
Definition: xlogreader.c:2173
int errmsg(const char *fmt,...)
Definition: elog.c:909
XLogRecord header
Definition: xlogreader.h:143
XLogReadRecordState readRecordState
Definition: xlogreader.h:282
bool WALRead(XLogReaderState *state, WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo)
Definition: xlogreader.c:1665
uint64 system_identifier
Definition: xlogreader.h:168
WALSegmentContext segcxt
Definition: xlogreader.h:242
char * errormsg_buf
Definition: xlogreader.h:291
TimeLineID latestPageTLI
Definition: xlogreader.h:251
RelFileNode rnode
Definition: xlogreader.h:77
int Buffer
Definition: buf.h:23
DecodedXLogRecord * decode_queue_head
Definition: xlogreader.h:238
static void XLogReaderSetInputData(XLogReaderState *state, int32 len)
Definition: xlogreader.h:304
void XLogReaderSetDecodeBuffer(XLogReaderState *state, void *buffer, size_t size)
Definition: xlogreader.c:163
WALSegmentCleanupCB cleanup_cb
Definition: xlogreader.h:157
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, WALSegmentCleanupCB cleanup_cb)
Definition: xlogreader.c:82
static XLogRecPtr startptr
Definition: basebackup.c:116