PostgreSQL Source Code  git master
load_manifest.c File Reference
#include "postgres_fe.h"
#include <sys/stat.h>
#include <unistd.h>
#include "common/hashfn_unstable.h"
#include "common/logging.h"
#include "common/parse_manifest.h"
#include "load_manifest.h"
#include "lib/simplehash.h"
Include dependency graph for load_manifest.c:

Go to the source code of this file.

Macros

#define ESTIMATED_BYTES_PER_MANIFEST_LINE   100
 
#define READ_CHUNK_SIZE   (128 * 1024)
 
#define SH_PREFIX   manifest_files
 
#define SH_ELEMENT_TYPE   manifest_file
 
#define SH_KEY_TYPE   char *
 
#define SH_KEY   pathname
 
#define SH_HASH_KEY(tb, key)   hash_string(key)
 
#define SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
 
#define SH_SCOPE   extern
 
#define SH_RAW_ALLOCATOR   pg_malloc0
 
#define SH_DEFINE
 

Functions

static void combinebackup_version_cb (JsonManifestParseContext *context, int manifest_version)
 
static void combinebackup_system_identifier_cb (JsonManifestParseContext *context, uint64 manifest_system_identifier)
 
static void combinebackup_per_file_cb (JsonManifestParseContext *context, char *pathname, size_t size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
 
static void combinebackup_per_wal_range_cb (JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
 
static void report_manifest_error (JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
 
static void pg_attribute_noreturn ()
 
manifest_data ** load_backup_manifests (int n_backups, char **backup_directories)
 
manifest_dataload_backup_manifest (char *backup_directory)
 

Macro Definition Documentation

◆ ESTIMATED_BYTES_PER_MANIFEST_LINE

#define ESTIMATED_BYTES_PER_MANIFEST_LINE   100

Definition at line 35 of file load_manifest.c.

◆ READ_CHUNK_SIZE

#define READ_CHUNK_SIZE   (128 * 1024)

Definition at line 41 of file load_manifest.c.

◆ SH_DEFINE

#define SH_DEFINE

Definition at line 55 of file load_manifest.c.

◆ SH_ELEMENT_TYPE

#define SH_ELEMENT_TYPE   manifest_file

Definition at line 48 of file load_manifest.c.

◆ SH_EQUAL

#define SH_EQUAL (   tb,
  a,
  b 
)    (strcmp(a, b) == 0)

Definition at line 52 of file load_manifest.c.

◆ SH_HASH_KEY

#define SH_HASH_KEY (   tb,
  key 
)    hash_string(key)

Definition at line 51 of file load_manifest.c.

◆ SH_KEY

#define SH_KEY   pathname

Definition at line 50 of file load_manifest.c.

◆ SH_KEY_TYPE

#define SH_KEY_TYPE   char *

Definition at line 49 of file load_manifest.c.

◆ SH_PREFIX

#define SH_PREFIX   manifest_files

Definition at line 47 of file load_manifest.c.

◆ SH_RAW_ALLOCATOR

#define SH_RAW_ALLOCATOR   pg_malloc0

Definition at line 54 of file load_manifest.c.

◆ SH_SCOPE

#define SH_SCOPE   extern

Definition at line 53 of file load_manifest.c.

Function Documentation

◆ combinebackup_per_file_cb()

static void combinebackup_per_file_cb ( JsonManifestParseContext context,
char *  pathname,
size_t  size,
pg_checksum_type  checksum_type,
int  checksum_length,
uint8 checksum_payload 
)
static

Definition at line 269 of file load_manifest.c.

273 {
274  manifest_data *manifest = context->private_data;
275  manifest_file *m;
276  bool found;
277 
278  /* Make a new entry in the hash table for this file. */
279  m = manifest_files_insert(manifest->files, pathname, &found);
280  if (found)
281  pg_fatal("duplicate path name in backup manifest: \"%s\"", pathname);
282 
283  /* Initialize the entry. */
284  m->size = size;
285  m->checksum_type = checksum_type;
286  m->checksum_length = checksum_length;
287  m->checksum_payload = checksum_payload;
288 }
#define pg_fatal(...)
static bool manifest
tree context
Definition: radixtree.h:1829
static pg_noinline void Size size
Definition: slab.c:607
uint8 * checksum_payload
Definition: load_manifest.h:29
pg_checksum_type checksum_type
Definition: load_manifest.h:27

References manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, context, manifest, pg_fatal, size, and manifest_file::size.

Referenced by load_backup_manifest().

◆ combinebackup_per_wal_range_cb()

static void combinebackup_per_wal_range_cb ( JsonManifestParseContext context,
TimeLineID  tli,
XLogRecPtr  start_lsn,
XLogRecPtr  end_lsn 
)
static

Definition at line 294 of file load_manifest.c.

297 {
298  manifest_data *manifest = context->private_data;
300 
301  /* Allocate and initialize a struct describing this WAL range. */
302  range = palloc(sizeof(manifest_wal_range));
303  range->tli = tli;
304  range->start_lsn = start_lsn;
305  range->end_lsn = end_lsn;
306  range->prev = manifest->last_wal_range;
307  range->next = NULL;
308 
309  /* Add it to the end of the list. */
310  if (manifest->first_wal_range == NULL)
311  manifest->first_wal_range = range;
312  else
313  manifest->last_wal_range->next = range;
314  manifest->last_wal_range = range;
315 }
void * palloc(Size size)
Definition: mcxt.c:1316
static struct cvec * range(struct vars *v, chr a, chr b, int cases)
Definition: regc_locale.c:412

References context, manifest, palloc(), and range().

Referenced by load_backup_manifest().

◆ combinebackup_system_identifier_cb()

static void combinebackup_system_identifier_cb ( JsonManifestParseContext context,
uint64  manifest_system_identifier 
)
static

Definition at line 256 of file load_manifest.c.

258 {
259  manifest_data *manifest = context->private_data;
260 
261  /* Validation will be at the later stage */
262  manifest->system_identifier = manifest_system_identifier;
263 }

References context, and manifest.

Referenced by load_backup_manifest().

◆ combinebackup_version_cb()

static void combinebackup_version_cb ( JsonManifestParseContext context,
int  manifest_version 
)
static

Definition at line 244 of file load_manifest.c.

246 {
247  /* Incremental backups supported on manifest version 2 or later */
248  if (manifest_version == 1)
249  pg_fatal("backup manifest version 1 does not support incremental backup");
250 }

References pg_fatal.

Referenced by load_backup_manifest().

◆ load_backup_manifest()

manifest_data* load_backup_manifest ( char *  backup_directory)

Definition at line 105 of file load_manifest.c.

106 {
107  char pathname[MAXPGPATH];
108  int fd;
109  struct stat statbuf;
110  off_t estimate;
111  uint32 initial_size;
112  manifest_files_hash *ht;
113  char *buffer;
114  int rc;
116  manifest_data *result;
117  int chunk_size = READ_CHUNK_SIZE;
118 
119  /* Open the manifest file. */
120  snprintf(pathname, MAXPGPATH, "%s/backup_manifest", backup_directory);
121  if ((fd = open(pathname, O_RDONLY | PG_BINARY, 0)) < 0)
122  {
123  if (errno == ENOENT)
124  {
125  pg_log_warning("\"%s\" does not exist", pathname);
126  return NULL;
127  }
128  pg_fatal("could not open file \"%s\": %m", pathname);
129  }
130 
131  /* Figure out how big the manifest is. */
132  if (fstat(fd, &statbuf) != 0)
133  pg_fatal("could not stat file \"%s\": %m", pathname);
134 
135  /* Guess how large to make the hash table based on the manifest size. */
136  estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
137  initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
138 
139  /* Create the hash table. */
140  ht = manifest_files_create(initial_size, NULL);
141 
142  result = pg_malloc0(sizeof(manifest_data));
143  result->files = ht;
144  context.private_data = result;
145  context.version_cb = combinebackup_version_cb;
146  context.system_identifier_cb = combinebackup_system_identifier_cb;
147  context.per_file_cb = combinebackup_per_file_cb;
148  context.per_wal_range_cb = combinebackup_per_wal_range_cb;
149  context.error_cb = report_manifest_error;
150 
151  /*
152  * Parse the file, in chunks if necessary.
153  */
154  if (statbuf.st_size <= chunk_size)
155  {
156  buffer = pg_malloc(statbuf.st_size);
157  rc = read(fd, buffer, statbuf.st_size);
158  if (rc != statbuf.st_size)
159  {
160  if (rc < 0)
161  pg_fatal("could not read file \"%s\": %m", pathname);
162  else
163  pg_fatal("could not read file \"%s\": read %d of %lld",
164  pathname, rc, (long long int) statbuf.st_size);
165  }
166 
167  /* Close the manifest file. */
168  close(fd);
169 
170  /* Parse the manifest. */
171  json_parse_manifest(&context, buffer, statbuf.st_size);
172  }
173  else
174  {
175  int bytes_left = statbuf.st_size;
177 
179 
180  buffer = pg_malloc(chunk_size + 1);
181 
182  while (bytes_left > 0)
183  {
184  int bytes_to_read = chunk_size;
185 
186  /*
187  * Make sure that the last chunk is sufficiently large. (i.e. at
188  * least half the chunk size) so that it will contain fully the
189  * piece at the end with the checksum.
190  */
191  if (bytes_left < chunk_size)
192  bytes_to_read = bytes_left;
193  else if (bytes_left < 2 * chunk_size)
194  bytes_to_read = bytes_left / 2;
195  rc = read(fd, buffer, bytes_to_read);
196  if (rc != bytes_to_read)
197  {
198  if (rc < 0)
199  pg_fatal("could not read file \"%s\": %m", pathname);
200  else
201  pg_fatal("could not read file \"%s\": read %lld of %lld",
202  pathname,
203  (long long int) (statbuf.st_size + rc - bytes_left),
204  (long long int) statbuf.st_size);
205  }
206  bytes_left -= rc;
208  inc_state, buffer, rc, bytes_left == 0);
209  }
210 
211  /* Release the incremental state memory */
213 
214  close(fd);
215  }
216 
217  /* All done. */
218  pfree(buffer);
219  return result;
220 }
unsigned int uint32
Definition: c.h:506
#define Min(x, y)
Definition: c.h:1004
#define PG_UINT32_MAX
Definition: c.h:590
#define Max(x, y)
Definition: c.h:998
#define PG_BINARY
Definition: c.h:1273
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
static void combinebackup_per_file_cb(JsonManifestParseContext *context, char *pathname, size_t size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
#define ESTIMATED_BYTES_PER_MANIFEST_LINE
Definition: load_manifest.c:35
#define READ_CHUNK_SIZE
Definition: load_manifest.c:41
static void report_manifest_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void combinebackup_version_cb(JsonManifestParseContext *context, int manifest_version)
static void combinebackup_per_wal_range_cb(JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
static void combinebackup_system_identifier_cb(JsonManifestParseContext *context, uint64 manifest_system_identifier)
void pfree(void *pointer)
Definition: mcxt.c:1520
void json_parse_manifest_incremental_chunk(JsonManifestParseIncrementalState *incstate, char *chunk, int size, bool is_last)
JsonManifestParseIncrementalState * json_parse_manifest_incremental_init(JsonManifestParseContext *context)
void json_parse_manifest(JsonManifestParseContext *context, char *buffer, size_t size)
void json_parse_manifest_incremental_shutdown(JsonManifestParseIncrementalState *incstate)
#define MAXPGPATH
#define pg_log_warning(...)
Definition: pgfnames.c:24
#define snprintf
Definition: port.h:238
static int fd(const char *x, int i)
Definition: preproc-init.c:105
manifest_files_hash * files
Definition: load_manifest.h:59
#define fstat
Definition: win32_port.h:283

References close, combinebackup_per_file_cb(), combinebackup_per_wal_range_cb(), combinebackup_system_identifier_cb(), combinebackup_version_cb(), context, ESTIMATED_BYTES_PER_MANIFEST_LINE, fd(), manifest_data::files, fstat, json_parse_manifest(), json_parse_manifest_incremental_chunk(), json_parse_manifest_incremental_init(), json_parse_manifest_incremental_shutdown(), Max, MAXPGPATH, Min, pfree(), PG_BINARY, pg_fatal, pg_log_warning, pg_malloc(), pg_malloc0(), PG_UINT32_MAX, read, READ_CHUNK_SIZE, report_manifest_error(), snprintf, and stat::st_size.

Referenced by load_backup_manifests().

◆ load_backup_manifests()

manifest_data** load_backup_manifests ( int  n_backups,
char **  backup_directories 
)

Definition at line 83 of file load_manifest.c.

84 {
85  manifest_data **result;
86  int i;
87 
88  result = pg_malloc(sizeof(manifest_data *) * n_backups);
89  for (i = 0; i < n_backups; ++i)
90  result[i] = load_backup_manifest(backup_directories[i]);
91 
92  return result;
93 }
int i
Definition: isn.c:73
manifest_data * load_backup_manifest(char *backup_directory)

References i, load_backup_manifest(), and pg_malloc().

Referenced by main().

◆ pg_attribute_noreturn()

static void pg_attribute_noreturn ( )

◆ report_manifest_error()

static void report_manifest_error ( JsonManifestParseContext context,
const char *  fmt,
  ... 
)
static

Definition at line 229 of file load_manifest.c.

230 {
231  va_list ap;
232 
233  va_start(ap, fmt);
235  va_end(ap);
236 
237  exit(1);
238 }
#define gettext(x)
Definition: c.h:1179
static void const char * fmt
va_end(args)
exit(1)
va_start(args, fmt)
void pg_log_generic_v(enum pg_log_level level, enum pg_log_part part, const char *pg_restrict fmt, va_list ap)
Definition: logging.c:216
@ PG_LOG_PRIMARY
Definition: logging.h:67
@ PG_LOG_ERROR
Definition: logging.h:43

References exit(), fmt, gettext, PG_LOG_ERROR, pg_log_generic_v(), PG_LOG_PRIMARY, va_end(), and va_start().

Referenced by load_backup_manifest().