PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
load_manifest.c File Reference
#include "postgres_fe.h"
#include <sys/stat.h>
#include <unistd.h>
#include "common/hashfn_unstable.h"
#include "common/logging.h"
#include "common/parse_manifest.h"
#include "load_manifest.h"
#include "lib/simplehash.h"
Include dependency graph for load_manifest.c:

Go to the source code of this file.

Macros

#define ESTIMATED_BYTES_PER_MANIFEST_LINE   100
 
#define READ_CHUNK_SIZE   (128 * 1024)
 
#define SH_PREFIX   manifest_files
 
#define SH_ELEMENT_TYPE   manifest_file
 
#define SH_KEY_TYPE   const char *
 
#define SH_KEY   pathname
 
#define SH_HASH_KEY(tb, key)   hash_string(key)
 
#define SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
 
#define SH_SCOPE   extern
 
#define SH_RAW_ALLOCATOR   pg_malloc0
 
#define SH_DEFINE
 

Functions

static void combinebackup_version_cb (JsonManifestParseContext *context, int manifest_version)
 
static void combinebackup_system_identifier_cb (JsonManifestParseContext *context, uint64 manifest_system_identifier)
 
static void combinebackup_per_file_cb (JsonManifestParseContext *context, const char *pathname, uint64 size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
 
static void combinebackup_per_wal_range_cb (JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
 
static void report_manifest_error (JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
 
static void pg_attribute_noreturn ()
 
manifest_data ** load_backup_manifests (int n_backups, char **backup_directories)
 
manifest_dataload_backup_manifest (char *backup_directory)
 

Macro Definition Documentation

◆ ESTIMATED_BYTES_PER_MANIFEST_LINE

#define ESTIMATED_BYTES_PER_MANIFEST_LINE   100

Definition at line 35 of file load_manifest.c.

◆ READ_CHUNK_SIZE

#define READ_CHUNK_SIZE   (128 * 1024)

Definition at line 41 of file load_manifest.c.

◆ SH_DEFINE

#define SH_DEFINE

Definition at line 55 of file load_manifest.c.

◆ SH_ELEMENT_TYPE

#define SH_ELEMENT_TYPE   manifest_file

Definition at line 48 of file load_manifest.c.

◆ SH_EQUAL

#define SH_EQUAL (   tb,
  a,
  b 
)    (strcmp(a, b) == 0)

Definition at line 52 of file load_manifest.c.

◆ SH_HASH_KEY

#define SH_HASH_KEY (   tb,
  key 
)    hash_string(key)

Definition at line 51 of file load_manifest.c.

◆ SH_KEY

#define SH_KEY   pathname

Definition at line 50 of file load_manifest.c.

◆ SH_KEY_TYPE

#define SH_KEY_TYPE   const char *

Definition at line 49 of file load_manifest.c.

◆ SH_PREFIX

#define SH_PREFIX   manifest_files

Definition at line 47 of file load_manifest.c.

◆ SH_RAW_ALLOCATOR

#define SH_RAW_ALLOCATOR   pg_malloc0

Definition at line 54 of file load_manifest.c.

◆ SH_SCOPE

#define SH_SCOPE   extern

Definition at line 53 of file load_manifest.c.

Function Documentation

◆ combinebackup_per_file_cb()

static void combinebackup_per_file_cb ( JsonManifestParseContext context,
const char *  pathname,
uint64  size,
pg_checksum_type  checksum_type,
int  checksum_length,
uint8 checksum_payload 
)
static

Definition at line 268 of file load_manifest.c.

272{
274 manifest_file *m;
275 bool found;
276
277 /* Make a new entry in the hash table for this file. */
278 m = manifest_files_insert(manifest->files, pathname, &found);
279 if (found)
280 pg_fatal("duplicate path name in backup manifest: \"%s\"", pathname);
281
282 /* Initialize the entry. */
283 m->size = size;
284 m->checksum_type = checksum_type;
285 m->checksum_length = checksum_length;
286 m->checksum_payload = checksum_payload;
287}
#define pg_fatal(...)
static bool manifest
static pg_noinline void Size size
Definition: slab.c:607
uint8 * checksum_payload
Definition: load_manifest.h:29
pg_checksum_type checksum_type
Definition: load_manifest.h:27

References manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, manifest, pg_fatal, JsonManifestParseContext::private_data, size, and manifest_file::size.

Referenced by load_backup_manifest().

◆ combinebackup_per_wal_range_cb()

static void combinebackup_per_wal_range_cb ( JsonManifestParseContext context,
TimeLineID  tli,
XLogRecPtr  start_lsn,
XLogRecPtr  end_lsn 
)
static

Definition at line 293 of file load_manifest.c.

296{
299
300 /* Allocate and initialize a struct describing this WAL range. */
302 range->tli = tli;
303 range->start_lsn = start_lsn;
304 range->end_lsn = end_lsn;
305 range->prev = manifest->last_wal_range;
306 range->next = NULL;
307
308 /* Add it to the end of the list. */
309 if (manifest->first_wal_range == NULL)
310 manifest->first_wal_range = range;
311 else
312 manifest->last_wal_range->next = range;
313 manifest->last_wal_range = range;
314}
void * palloc(Size size)
Definition: mcxt.c:1317
static struct cvec * range(struct vars *v, chr a, chr b, int cases)
Definition: regc_locale.c:412

References manifest, palloc(), JsonManifestParseContext::private_data, and range().

Referenced by load_backup_manifest().

◆ combinebackup_system_identifier_cb()

static void combinebackup_system_identifier_cb ( JsonManifestParseContext context,
uint64  manifest_system_identifier 
)
static

Definition at line 255 of file load_manifest.c.

257{
259
260 /* Validation will be at the later stage */
261 manifest->system_identifier = manifest_system_identifier;
262}

References manifest, and JsonManifestParseContext::private_data.

Referenced by load_backup_manifest().

◆ combinebackup_version_cb()

static void combinebackup_version_cb ( JsonManifestParseContext context,
int  manifest_version 
)
static

Definition at line 243 of file load_manifest.c.

245{
246 /* Incremental backups supported on manifest version 2 or later */
247 if (manifest_version == 1)
248 pg_fatal("backup manifest version 1 does not support incremental backup");
249}

References pg_fatal.

Referenced by load_backup_manifest().

◆ load_backup_manifest()

manifest_data * load_backup_manifest ( char *  backup_directory)

Definition at line 105 of file load_manifest.c.

106{
107 char pathname[MAXPGPATH];
108 int fd;
109 struct stat statbuf;
110 off_t estimate;
111 uint32 initial_size;
112 manifest_files_hash *ht;
113 char *buffer;
114 int rc;
116 manifest_data *result;
117 int chunk_size = READ_CHUNK_SIZE;
118
119 /* Open the manifest file. */
120 snprintf(pathname, MAXPGPATH, "%s/backup_manifest", backup_directory);
121 if ((fd = open(pathname, O_RDONLY | PG_BINARY, 0)) < 0)
122 {
123 if (errno == ENOENT)
124 {
125 pg_log_warning("file \"%s\" does not exist", pathname);
126 return NULL;
127 }
128 pg_fatal("could not open file \"%s\": %m", pathname);
129 }
130
131 /* Figure out how big the manifest is. */
132 if (fstat(fd, &statbuf) != 0)
133 pg_fatal("could not stat file \"%s\": %m", pathname);
134
135 /* Guess how large to make the hash table based on the manifest size. */
136 estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
137 initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
138
139 /* Create the hash table. */
140 ht = manifest_files_create(initial_size, NULL);
141
142 result = pg_malloc0(sizeof(manifest_data));
143 result->files = ht;
144 context.private_data = result;
150
151 /*
152 * Parse the file, in chunks if necessary.
153 */
154 if (statbuf.st_size <= chunk_size)
155 {
156 buffer = pg_malloc(statbuf.st_size);
157 rc = read(fd, buffer, statbuf.st_size);
158 if (rc != statbuf.st_size)
159 {
160 if (rc < 0)
161 pg_fatal("could not read file \"%s\": %m", pathname);
162 else
163 pg_fatal("could not read file \"%s\": read %d of %lld",
164 pathname, rc, (long long int) statbuf.st_size);
165 }
166
167 /* Close the manifest file. */
168 close(fd);
169
170 /* Parse the manifest. */
171 json_parse_manifest(&context, buffer, statbuf.st_size);
172 }
173 else
174 {
175 int bytes_left = statbuf.st_size;
177
178 inc_state = json_parse_manifest_incremental_init(&context);
179
180 buffer = pg_malloc(chunk_size + 1);
181
182 while (bytes_left > 0)
183 {
184 int bytes_to_read = chunk_size;
185
186 /*
187 * Make sure that the last chunk is sufficiently large. (i.e. at
188 * least half the chunk size) so that it will contain fully the
189 * piece at the end with the checksum.
190 */
191 if (bytes_left < chunk_size)
192 bytes_to_read = bytes_left;
193 else if (bytes_left < 2 * chunk_size)
194 bytes_to_read = bytes_left / 2;
195 rc = read(fd, buffer, bytes_to_read);
196 if (rc != bytes_to_read)
197 {
198 if (rc < 0)
199 pg_fatal("could not read file \"%s\": %m", pathname);
200 else
201 pg_fatal("could not read file \"%s\": read %lld of %lld",
202 pathname,
203 (long long int) (statbuf.st_size + rc - bytes_left),
204 (long long int) statbuf.st_size);
205 }
206 bytes_left -= rc;
207 json_parse_manifest_incremental_chunk(inc_state, buffer, rc, bytes_left == 0);
208 }
209
210 /* Release the incremental state memory */
212
213 close(fd);
214 }
215
216 /* All done. */
217 pfree(buffer);
218 return result;
219}
#define Min(x, y)
Definition: c.h:958
#define PG_UINT32_MAX
Definition: c.h:544
#define Max(x, y)
Definition: c.h:952
#define PG_BINARY
Definition: c.h:1227
uint32_t uint32
Definition: c.h:485
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
#define ESTIMATED_BYTES_PER_MANIFEST_LINE
Definition: load_manifest.c:35
#define READ_CHUNK_SIZE
Definition: load_manifest.c:41
static void combinebackup_per_file_cb(JsonManifestParseContext *context, const char *pathname, uint64 size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
static void report_manifest_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void combinebackup_version_cb(JsonManifestParseContext *context, int manifest_version)
static void combinebackup_per_wal_range_cb(JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
static void combinebackup_system_identifier_cb(JsonManifestParseContext *context, uint64 manifest_system_identifier)
void pfree(void *pointer)
Definition: mcxt.c:1521
void json_parse_manifest(JsonManifestParseContext *context, const char *buffer, size_t size)
JsonManifestParseIncrementalState * json_parse_manifest_incremental_init(JsonManifestParseContext *context)
void json_parse_manifest_incremental_shutdown(JsonManifestParseIncrementalState *incstate)
void json_parse_manifest_incremental_chunk(JsonManifestParseIncrementalState *incstate, const char *chunk, size_t size, bool is_last)
#define MAXPGPATH
#define pg_log_warning(...)
Definition: pgfnames.c:24
#define snprintf
Definition: port.h:238
static int fd(const char *x, int i)
Definition: preproc-init.c:105
json_manifest_per_wal_range_callback per_wal_range_cb
json_manifest_system_identifier_callback system_identifier_cb
json_manifest_error_callback error_cb
json_manifest_per_file_callback per_file_cb
json_manifest_version_callback version_cb
manifest_files_hash * files
Definition: load_manifest.h:59
#define fstat
Definition: win32_port.h:283

References close, combinebackup_per_file_cb(), combinebackup_per_wal_range_cb(), combinebackup_system_identifier_cb(), combinebackup_version_cb(), JsonManifestParseContext::error_cb, ESTIMATED_BYTES_PER_MANIFEST_LINE, fd(), manifest_data::files, fstat, json_parse_manifest(), json_parse_manifest_incremental_chunk(), json_parse_manifest_incremental_init(), json_parse_manifest_incremental_shutdown(), Max, MAXPGPATH, Min, JsonManifestParseContext::per_file_cb, JsonManifestParseContext::per_wal_range_cb, pfree(), PG_BINARY, pg_fatal, pg_log_warning, pg_malloc(), pg_malloc0(), PG_UINT32_MAX, JsonManifestParseContext::private_data, read, READ_CHUNK_SIZE, report_manifest_error(), snprintf, stat::st_size, JsonManifestParseContext::system_identifier_cb, and JsonManifestParseContext::version_cb.

Referenced by load_backup_manifests().

◆ load_backup_manifests()

manifest_data ** load_backup_manifests ( int  n_backups,
char **  backup_directories 
)

Definition at line 83 of file load_manifest.c.

84{
85 manifest_data **result;
86 int i;
87
88 result = pg_malloc(sizeof(manifest_data *) * n_backups);
89 for (i = 0; i < n_backups; ++i)
90 result[i] = load_backup_manifest(backup_directories[i]);
91
92 return result;
93}
int i
Definition: isn.c:72
manifest_data * load_backup_manifest(char *backup_directory)

References i, load_backup_manifest(), and pg_malloc().

Referenced by main().

◆ pg_attribute_noreturn()

static void pg_attribute_noreturn ( )

◆ report_manifest_error()

static void report_manifest_error ( JsonManifestParseContext context,
const char *  fmt,
  ... 
)
static

Definition at line 228 of file load_manifest.c.

229{
230 va_list ap;
231
232 va_start(ap, fmt);
234 va_end(ap);
235
236 exit(1);
237}
#define gettext(x)
Definition: c.h:1133
static void const char * fmt
va_end(args)
exit(1)
va_start(args, fmt)
void pg_log_generic_v(enum pg_log_level level, enum pg_log_part part, const char *pg_restrict fmt, va_list ap)
Definition: logging.c:219
@ PG_LOG_PRIMARY
Definition: logging.h:67
@ PG_LOG_ERROR
Definition: logging.h:43

References exit(), fmt, gettext, PG_LOG_ERROR, pg_log_generic_v(), PG_LOG_PRIMARY, va_end(), and va_start().

Referenced by load_backup_manifest().