PostgreSQL Source Code  git master
basebackup_sink.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * basebackup_sink.h
4  * API for filtering or sending to a final destination the archives
5  * produced by the base backup process
6  *
7  * Taking a base backup produces one archive per tablespace directory,
8  * plus a backup manifest unless that feature has been disabled. The
9  * goal of the backup process is to put those archives and that manifest
10  * someplace, possibly after postprocessing them in some way. A 'bbsink'
11  * is an object to which those archives, and the manifest if present,
12  * can be sent.
13  *
14  * In practice, there will be a chain of 'bbsink' objects rather than
15  * just one, with callbacks being forwarded from one to the next,
16  * possibly with modification. Each object is responsible for a
17  * single task e.g. command progress reporting, throttling, or
18  * communication with the client.
19  *
20  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
21  *
22  * src/include/backup/basebackup_sink.h
23  *
24  *-------------------------------------------------------------------------
25  */
26 #ifndef BASEBACKUP_SINK_H
27 #define BASEBACKUP_SINK_H
28 
29 #include "access/xlogdefs.h"
30 #include "common/compression.h"
31 #include "nodes/pg_list.h"
32 
33 /* Forward declarations. */
34 struct bbsink;
35 struct bbsink_ops;
36 typedef struct bbsink bbsink;
37 typedef struct bbsink_ops bbsink_ops;
38 
39 /*
40  * Overall backup state shared by all bbsink objects for a backup.
41  *
42  * Before calling bbstate_begin_backup, caller must initiate a bbsink_state
43  * object which will last for the lifetime of the backup, and must thereafter
44  * update it as required before each new call to a bbsink method. The bbsink
45  * will retain a pointer to the state object and will consult it to understand
46  * the progress of the backup.
47  *
48  * 'tablespaces' is a list of tablespaceinfo objects. It must be set before
49  * calling bbstate_begin_backup() and must not be modified thereafter.
50  *
51  * 'tablespace_num' is the index of the current tablespace within the list
52  * stored in 'tablespaces'.
53  *
54  * 'bytes_done' is the number of bytes read so far from $PGDATA.
55  *
56  * 'bytes_total' is the total number of bytes estimated to be present in
57  * $PGDATA, if we have estimated this.
58  *
59  * 'bytes_total_is_valid' is true if and only if a proper estimate has been
60  * stored into 'bytes_total'.
61  *
62  * 'startptr' and 'starttli' identify the point in the WAL stream at which
63  * the backup began. They must be set before calling bbstate_begin_backup()
64  * and must not be modified thereafter.
65  */
66 typedef struct bbsink_state
67 {
70  uint64 bytes_done;
71  uint64 bytes_total;
76 
77 /*
78  * Common data for any type of basebackup sink.
79  *
80  * 'bbs_ops' is the relevant callback table.
81  *
82  * 'bbs_buffer' is the buffer into which data destined for the bbsink
83  * should be stored. It must be a multiple of BLCKSZ.
84  *
85  * 'bbs_buffer_length' is the allocated length of the buffer.
86  *
87  * 'bbs_next' is a pointer to another bbsink to which this bbsink is
88  * forwarding some or all operations.
89  *
90  * 'bbs_state' is a pointer to the bbsink_state object for this backup.
91  * Every bbsink associated with this backup should point to the same
92  * underlying state object.
93  *
94  * In general it is expected that the values of these fields are set when
95  * a bbsink is created and that they do not change thereafter. It's OK
96  * to modify the data to which bbs_buffer or bbs_state point, but no changes
97  * should be made to the contents of this struct.
98  */
99 struct bbsink
100 {
102  char *bbs_buffer;
106 };
107 
108 /*
109  * Callbacks for a base backup sink.
110  *
111  * All of these callbacks are required. If a particular callback just needs to
112  * forward the call to sink->bbs_next, use bbsink_forward_<callback_name> as
113  * the callback.
114  *
115  * Callers should always invoke these callbacks via the bbsink_* inline
116  * functions rather than calling them directly.
117  */
119 {
120  /*
121  * This callback is invoked just once, at the very start of the backup. It
122  * must set bbs_buffer to point to a chunk of storage where at least
123  * bbs_buffer_length bytes of data can be written.
124  */
125  void (*begin_backup) (bbsink *sink);
126 
127  /*
128  * For each archive transmitted to a bbsink, there will be one call to the
129  * begin_archive() callback, some number of calls to the
130  * archive_contents() callback, and then one call to the end_archive()
131  * callback.
132  *
133  * Before invoking the archive_contents() callback, the caller should copy
134  * a number of bytes equal to what will be passed as len into bbs_buffer,
135  * but not more than bbs_buffer_length.
136  *
137  * It's generally good if the buffer is as full as possible before the
138  * archive_contents() callback is invoked, but it's not worth expending
139  * extra cycles to make sure it's absolutely 100% full.
140  */
141  void (*begin_archive) (bbsink *sink, const char *archive_name);
142  void (*archive_contents) (bbsink *sink, size_t len);
143  void (*end_archive) (bbsink *sink);
144 
145  /*
146  * If a backup manifest is to be transmitted to a bbsink, there will be
147  * one call to the begin_manifest() callback, some number of calls to the
148  * manifest_contents() callback, and then one call to the end_manifest()
149  * callback. These calls will occur after all archives are transmitted.
150  *
151  * The rules for invoking the manifest_contents() callback are the same as
152  * for the archive_contents() callback above.
153  */
154  void (*begin_manifest) (bbsink *sink);
155  void (*manifest_contents) (bbsink *sink, size_t len);
156  void (*end_manifest) (bbsink *sink);
157 
158  /*
159  * This callback is invoked just once, after all archives and the manifest
160  * have been sent.
161  */
162  void (*end_backup) (bbsink *sink, XLogRecPtr endptr, TimeLineID endtli);
163 
164  /*
165  * If a backup is aborted by an error, this callback is invoked before the
166  * bbsink object is destroyed, so that it can release any resources that
167  * would not be released automatically. If no error occurs, this callback
168  * is invoked after the end_backup callback.
169  */
170  void (*cleanup) (bbsink *sink);
171 };
172 
173 /* Begin a backup. */
174 static inline void
175 bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
176 {
177  Assert(sink != NULL);
178 
179  Assert(buffer_length > 0);
180 
181  sink->bbs_state = state;
182  sink->bbs_buffer_length = buffer_length;
183  sink->bbs_ops->begin_backup(sink);
184 
185  Assert(sink->bbs_buffer != NULL);
186  Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
187 }
188 
189 /* Begin an archive. */
190 static inline void
191 bbsink_begin_archive(bbsink *sink, const char *archive_name)
192 {
193  Assert(sink != NULL);
194 
195  sink->bbs_ops->begin_archive(sink, archive_name);
196 }
197 
198 /* Process some of the contents of an archive. */
199 static inline void
201 {
202  Assert(sink != NULL);
203 
204  /*
205  * The caller should make a reasonable attempt to fill the buffer before
206  * calling this function, so it shouldn't be completely empty. Nor should
207  * it be filled beyond capacity.
208  */
209  Assert(len > 0 && len <= sink->bbs_buffer_length);
210 
211  sink->bbs_ops->archive_contents(sink, len);
212 }
213 
214 /* Finish an archive. */
215 static inline void
217 {
218  Assert(sink != NULL);
219 
220  sink->bbs_ops->end_archive(sink);
221 }
222 
223 /* Begin the backup manifest. */
224 static inline void
226 {
227  Assert(sink != NULL);
228 
229  sink->bbs_ops->begin_manifest(sink);
230 }
231 
232 /* Process some of the manifest contents. */
233 static inline void
235 {
236  Assert(sink != NULL);
237 
238  /* See comments in bbsink_archive_contents. */
239  Assert(len > 0 && len <= sink->bbs_buffer_length);
240 
241  sink->bbs_ops->manifest_contents(sink, len);
242 }
243 
244 /* Finish the backup manifest. */
245 static inline void
247 {
248  Assert(sink != NULL);
249 
250  sink->bbs_ops->end_manifest(sink);
251 }
252 
253 /* Finish a backup. */
254 static inline void
256 {
257  Assert(sink != NULL);
259 
260  sink->bbs_ops->end_backup(sink, endptr, endtli);
261 }
262 
263 /* Release resources before destruction. */
264 static inline void
266 {
267  Assert(sink != NULL);
268 
269  sink->bbs_ops->cleanup(sink);
270 }
271 
272 /* Forwarding callbacks. Use these to pass operations through to next sink. */
273 extern void bbsink_forward_begin_backup(bbsink *sink);
274 extern void bbsink_forward_begin_archive(bbsink *sink,
275  const char *archive_name);
276 extern void bbsink_forward_archive_contents(bbsink *sink, size_t len);
277 extern void bbsink_forward_end_archive(bbsink *sink);
278 extern void bbsink_forward_begin_manifest(bbsink *sink);
279 extern void bbsink_forward_manifest_contents(bbsink *sink, size_t len);
280 extern void bbsink_forward_end_manifest(bbsink *sink);
281 extern void bbsink_forward_end_backup(bbsink *sink, XLogRecPtr endptr,
282  TimeLineID endtli);
283 extern void bbsink_forward_cleanup(bbsink *sink);
284 
285 /* Constructors for various types of sinks. */
286 extern bbsink *bbsink_copystream_new(bool send_to_client);
290 extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size);
291 extern bbsink *bbsink_server_new(bbsink *next, char *pathname);
293 
294 /* Extra interface functions for progress reporting. */
295 extern void basebackup_progress_wait_checkpoint(void);
298 extern void basebackup_progress_transfer_wal(void);
299 extern void basebackup_progress_done(void);
300 
301 #endif
bbsink * bbsink_copystream_new(bool send_to_client)
static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
void basebackup_progress_wait_checkpoint(void)
void bbsink_forward_begin_backup(bbsink *sink)
void bbsink_forward_begin_manifest(bbsink *sink)
bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)
void bbsink_forward_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
static void bbsink_begin_archive(bbsink *sink, const char *archive_name)
void bbsink_forward_cleanup(bbsink *sink)
void bbsink_forward_manifest_contents(bbsink *sink, size_t len)
static void bbsink_end_archive(bbsink *sink)
static void bbsink_begin_manifest(bbsink *sink)
struct bbsink_state bbsink_state
static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
void bbsink_forward_end_archive(bbsink *sink)
void bbsink_forward_archive_contents(bbsink *sink, size_t len)
static void bbsink_cleanup(bbsink *sink)
void basebackup_progress_wait_wal_archive(bbsink_state *)
bbsink * bbsink_server_new(bbsink *next, char *pathname)
void bbsink_forward_begin_archive(bbsink *sink, const char *archive_name)
bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *)
static void bbsink_end_manifest(bbsink *sink)
void bbsink_forward_end_manifest(bbsink *sink)
bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *)
static void bbsink_archive_contents(bbsink *sink, size_t len)
static void bbsink_manifest_contents(bbsink *sink, size_t len)
bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *)
bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)
void basebackup_progress_done(void)
void basebackup_progress_transfer_wal(void)
void basebackup_progress_estimate_backup_size(void)
static int32 next
Definition: blutils.c:221
unsigned int uint32
Definition: c.h:493
Assert(fmt[strlen(fmt) - 1] !='\n')
static int32 maxrate
const void size_t len
static int list_length(const List *l)
Definition: pg_list.h:152
Definition: pg_list.h:54
void(* begin_manifest)(bbsink *sink)
void(* cleanup)(bbsink *sink)
void(* end_manifest)(bbsink *sink)
void(* end_archive)(bbsink *sink)
void(* manifest_contents)(bbsink *sink, size_t len)
void(* begin_backup)(bbsink *sink)
void(* end_backup)(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
void(* archive_contents)(bbsink *sink, size_t len)
void(* begin_archive)(bbsink *sink, const char *archive_name)
bool bytes_total_is_valid
XLogRecPtr startptr
uint64 bytes_total
List * tablespaces
TimeLineID starttli
bbsink * bbs_next
bbsink_state * bbs_state
char * bbs_buffer
const bbsink_ops * bbs_ops
size_t bbs_buffer_length
Definition: regguts.h:323
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59