PostgreSQL Source Code  git master
bbstreamer.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * bbstreamer.h
4  *
5  * Each tar archive returned by the server is passed to one or more
6  * bbstreamer objects for further processing. The bbstreamer may do
7  * something simple, like write the archive to a file, perhaps after
8  * compressing it, but it can also do more complicated things, like
9  * annotating the byte stream to indicate which parts of the data
10  * correspond to tar headers or trailing padding, vs. which parts are
11  * payload data. A subsequent bbstreamer may use this information to
12  * make further decisions about how to process the data; for example,
13  * it might choose to modify the archive contents.
14  *
15  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
16  *
17  * IDENTIFICATION
18  * src/bin/pg_basebackup/bbstreamer.h
19  *-------------------------------------------------------------------------
20  */
21 
22 #ifndef BBSTREAMER_H
23 #define BBSTREAMER_H
24 
25 #include "common/compression.h"
26 #include "lib/stringinfo.h"
27 #include "pqexpbuffer.h"
28 
29 struct bbstreamer;
30 struct bbstreamer_ops;
31 typedef struct bbstreamer bbstreamer;
32 typedef struct bbstreamer_ops bbstreamer_ops;
33 
34 /*
35  * Each chunk of archive data passed to a bbstreamer is classified into one
36  * of these categories. When data is first received from the remote server,
37  * each chunk will be categorized as BBSTREAMER_UNKNOWN, and the chunks will
38  * be of whatever size the remote server chose to send.
39  *
40  * If the archive is parsed (e.g. see bbstreamer_tar_parser_new()), then all
41  * chunks should be labelled as one of the other types listed here. In
42  * addition, there should be exactly one BBSTREAMER_MEMBER_HEADER chunk and
43  * exactly one BBSTREAMER_MEMBER_TRAILER chunk per archive member, even if
44  * that means a zero-length call. There can be any number of
45  * BBSTREAMER_MEMBER_CONTENTS chunks in between those calls. There
46  * should exactly BBSTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
47  * last BBSTREAMER_MEMBER_TRAILER chunk.
48  *
49  * In theory, we could need other classifications here, such as a way of
50  * indicating an archive header, but the "tar" format doesn't need anything
51  * else, so for the time being there's no point.
52  */
53 typedef enum
54 {
61 
62 /*
63  * Each chunk of data that is classified as BBSTREAMER_MEMBER_HEADER,
64  * BBSTREAMER_MEMBER_CONTENTS, or BBSTREAMER_MEMBER_TRAILER should also
65  * pass a pointer to an instance of this struct. The details are expected
66  * to be present in the archive header and used to fill the struct, after
67  * which all subsequent calls for the same archive member are expected to
68  * pass the same details.
69  */
70 typedef struct
71 {
72  char pathname[MAXPGPATH];
74  mode_t mode;
78  bool is_link;
79  char linktarget[MAXPGPATH];
81 
82 /*
83  * Generally, each type of bbstreamer will define its own struct, but the
84  * first element should be 'bbstreamer base'. A bbstreamer that does not
85  * require any additional private data could use this structure directly.
86  *
87  * bbs_ops is a pointer to the bbstreamer_ops object which contains the
88  * function pointers appropriate to this type of bbstreamer.
89  *
90  * bbs_next is a pointer to the successor bbstreamer, for those types of
91  * bbstreamer which forward data to a successor. It need not be used and
92  * should be set to NULL when not relevant.
93  *
94  * bbs_buffer is a buffer for accumulating data for temporary storage. Each
95  * type of bbstreamer makes its own decisions about whether and how to use
96  * this buffer.
97  */
98 struct bbstreamer
99 {
103 };
104 
105 /*
106  * There are three callbacks for a bbstreamer. The 'content' callback is
107  * called repeatedly, as described in the bbstreamer_archive_context comments.
108  * Then, the 'finalize' callback is called once at the end, to give the
109  * bbstreamer a chance to perform cleanup such as closing files. Finally,
110  * because this code is running in a frontend environment where, as of this
111  * writing, there are no memory contexts, the 'free' callback is called to
112  * release memory. These callbacks should always be invoked using the static
113  * inline functions defined below.
114  */
116 {
117  void (*content) (bbstreamer *streamer, bbstreamer_member *member,
118  const char *data, int len,
120  void (*finalize) (bbstreamer *streamer);
121  void (*free) (bbstreamer *streamer);
122 };
123 
124 /* Send some content to a bbstreamer. */
125 static inline void
127  const char *data, int len,
129 {
130  Assert(streamer != NULL);
131  streamer->bbs_ops->content(streamer, member, data, len, context);
132 }
133 
134 /* Finalize a bbstreamer. */
135 static inline void
137 {
138  Assert(streamer != NULL);
139  streamer->bbs_ops->finalize(streamer);
140 }
141 
142 /* Free a bbstreamer. */
143 static inline void
145 {
146  Assert(streamer != NULL);
147  streamer->bbs_ops->free(streamer);
148 }
149 
150 /*
151  * This is a convenience method for use when implementing a bbstreamer; it is
152  * not for use by outside callers. It adds the amount of data specified by
153  * 'nbytes' to the bbstreamer's buffer and adjusts '*len' and '*data'
154  * accordingly.
155  */
156 static inline void
157 bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len,
158  int nbytes)
159 {
160  Assert(nbytes <= *len);
161 
162  appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
163  *len -= nbytes;
164  *data += nbytes;
165 }
166 
167 /*
168  * This is a convenence method for use when implementing a bbstreamer; it is
169  * not for use by outsider callers. It attempts to add enough data to the
170  * bbstreamer's buffer to reach a length of target_bytes and adjusts '*len'
171  * and '*data' accordingly. It returns true if the target length has been
172  * reached and false otherwise.
173  */
174 static inline bool
175 bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len,
176  int target_bytes)
177 {
178  int buflen = streamer->bbs_buffer.len;
179 
180  if (buflen >= target_bytes)
181  {
182  /* Target length already reached; nothing to do. */
183  return true;
184  }
185 
186  if (buflen + *len < target_bytes)
187  {
188  /* Not enough data to reach target length; buffer all of it. */
189  bbstreamer_buffer_bytes(streamer, data, len, *len);
190  return false;
191  }
192 
193  /* Buffer just enough to reach the target length. */
194  bbstreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
195  return true;
196 }
197 
198 /*
199  * Functions for creating bbstreamer objects of various types. See the header
200  * comments for each of these functions for details.
201  */
202 extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file);
203 extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
204  pg_compress_specification *compress);
205 extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
206  const char *(*link_map) (const char *),
207  void (*report_output_file) (const char *));
208 
211  pg_compress_specification *compress);
214  pg_compress_specification *compress);
219 
221  bool is_recovery_guc_supported,
223 extern void bbstreamer_inject_file(bbstreamer *streamer, char *pathname,
224  char *data, int len);
225 
226 #endif
bbstreamer * bbstreamer_gzip_writer_new(char *pathname, FILE *file, pg_compress_specification *compress)
bbstreamer * bbstreamer_zstd_compressor_new(bbstreamer *next, pg_compress_specification *compress)
static void bbstreamer_content(bbstreamer *streamer, bbstreamer_member *member, const char *data, int len, bbstreamer_archive_context context)
Definition: bbstreamer.h:126
static void bbstreamer_finalize(bbstreamer *streamer)
Definition: bbstreamer.h:136
static void bbstreamer_buffer_bytes(bbstreamer *streamer, const char **data, int *len, int nbytes)
Definition: bbstreamer.h:157
static bool bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len, int target_bytes)
Definition: bbstreamer.h:175
bbstreamer * bbstreamer_tar_terminator_new(bbstreamer *next)
bbstreamer * bbstreamer_lz4_compressor_new(bbstreamer *next, pg_compress_specification *compress)
bbstreamer * bbstreamer_tar_parser_new(bbstreamer *next)
bbstreamer * bbstreamer_lz4_decompressor_new(bbstreamer *next)
bbstreamer_archive_context
Definition: bbstreamer.h:54
@ BBSTREAMER_ARCHIVE_TRAILER
Definition: bbstreamer.h:59
@ BBSTREAMER_MEMBER_HEADER
Definition: bbstreamer.h:56
@ BBSTREAMER_MEMBER_TRAILER
Definition: bbstreamer.h:58
@ BBSTREAMER_UNKNOWN
Definition: bbstreamer.h:55
@ BBSTREAMER_MEMBER_CONTENTS
Definition: bbstreamer.h:57
static void bbstreamer_free(bbstreamer *streamer)
Definition: bbstreamer.h:144
bbstreamer * bbstreamer_recovery_injector_new(bbstreamer *next, bool is_recovery_guc_supported, PQExpBuffer recoveryconfcontents)
bbstreamer * bbstreamer_tar_archiver_new(bbstreamer *next)
bbstreamer * bbstreamer_zstd_decompressor_new(bbstreamer *next)
bbstreamer * bbstreamer_plain_writer_new(char *pathname, FILE *file)
bbstreamer * bbstreamer_extractor_new(const char *basepath, const char *(*link_map)(const char *), void(*report_output_file)(const char *))
bbstreamer * bbstreamer_gzip_decompressor_new(bbstreamer *next)
void bbstreamer_inject_file(bbstreamer *streamer, char *pathname, char *data, int len)
static int32 next
Definition: blutils.c:219
Assert(fmt[strlen(fmt) - 1] !='\n')
static PQExpBuffer recoveryconfcontents
#define MAXPGPATH
const void size_t len
const void * data
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:227
void(* content)(bbstreamer *streamer, bbstreamer_member *member, const char *data, int len, bbstreamer_archive_context context)
Definition: bbstreamer.h:117
void(* finalize)(bbstreamer *streamer)
Definition: bbstreamer.h:120
void(* free)(bbstreamer *streamer)
Definition: bbstreamer.h:121
const bbstreamer_ops * bbs_ops
Definition: bbstreamer.h:100
StringInfoData bbs_buffer
Definition: bbstreamer.h:102
bbstreamer * bbs_next
Definition: bbstreamer.h:101
int gid_t
Definition: win32_port.h:245
#define pgoff_t
Definition: win32_port.h:208
int uid_t
Definition: win32_port.h:244