PostgreSQL Source Code  git master
backup_manifest.c File Reference
#include "postgres.h"
#include "access/timeline.h"
#include "access/xlog.h"
#include "backup/backup_manifest.h"
#include "backup/basebackup_sink.h"
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "utils/json.h"
Include dependency graph for backup_manifest.c:

Go to the source code of this file.

Macros

#define AppendToManifest(manifest, ...)
 

Functions

static void AppendStringToManifest (backup_manifest_info *manifest, const char *s)
 
static bool IsManifestEnabled (backup_manifest_info *manifest)
 
void InitializeBackupManifest (backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
 
void FreeBackupManifest (backup_manifest_info *manifest)
 
void AddFileToBackupManifest (backup_manifest_info *manifest, Oid spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
 
void AddWALInfoToBackupManifest (backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
 
void SendBackupManifest (backup_manifest_info *manifest, bbsink *sink)
 

Macro Definition Documentation

◆ AppendToManifest

#define AppendToManifest (   manifest,
  ... 
)
Value:
{ \
char *_manifest_s = psprintf(__VA_ARGS__); \
AppendStringToManifest(manifest, _manifest_s); \
pfree(_manifest_s); \
}
static bool manifest
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46

Definition at line 41 of file backup_manifest.c.

Function Documentation

◆ AddFileToBackupManifest()

void AddFileToBackupManifest ( backup_manifest_info manifest,
Oid  spcoid,
const char *  pathname,
size_t  size,
pg_time_t  mtime,
pg_checksum_context checksum_ctx 
)

Definition at line 101 of file backup_manifest.c.

104 {
105  char pathbuf[MAXPGPATH];
106  int pathlen;
108 
110  return;
111 
112  /*
113  * If this file is part of a tablespace, the pathname passed to this
114  * function will be relative to the tar file that contains it. We want the
115  * pathname relative to the data directory (ignoring the intermediate
116  * symlink traversal).
117  */
118  if (OidIsValid(spcoid))
119  {
120  snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%u/%s", spcoid,
121  pathname);
122  pathname = pathbuf;
123  }
124 
125  /*
126  * Each file's entry needs to be separated from any entry that follows by
127  * a comma, but there's no comma before the first one or after the last
128  * one. To make that work, adding a file to the manifest starts by
129  * terminating the most recently added line, with a comma if appropriate,
130  * but does not terminate the line inserted for this file.
131  */
133  if (manifest->first_file)
134  {
135  appendStringInfoChar(&buf, '\n');
136  manifest->first_file = false;
137  }
138  else
139  appendStringInfoString(&buf, ",\n");
140 
141  /*
142  * Write the relative pathname to this file out to the manifest. The
143  * manifest is always stored in UTF-8, so we have to encode paths that are
144  * not valid in that encoding.
145  */
146  pathlen = strlen(pathname);
147  if (!manifest->force_encode &&
148  pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
149  {
150  appendStringInfoString(&buf, "{ \"Path\": ");
151  escape_json(&buf, pathname);
152  appendStringInfoString(&buf, ", ");
153  }
154  else
155  {
156  appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
157  enlargeStringInfo(&buf, 2 * pathlen);
158  buf.len += hex_encode(pathname, pathlen,
159  &buf.data[buf.len]);
160  appendStringInfoString(&buf, "\", ");
161  }
162 
163  appendStringInfo(&buf, "\"Size\": %zu, ", size);
164 
165  /*
166  * Convert last modification time to a string and append it to the
167  * manifest. Since it's not clear what time zone to use and since time
168  * zone definitions can change, possibly causing confusion, use GMT
169  * always.
170  */
171  appendStringInfoString(&buf, "\"Last-Modified\": \"");
172  enlargeStringInfo(&buf, 128);
173  buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
174  pg_gmtime(&mtime));
175  appendStringInfoChar(&buf, '"');
176 
177  /* Add checksum information. */
178  if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
179  {
180  uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
181  int checksumlen;
182 
183  checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
184  if (checksumlen < 0)
185  elog(ERROR, "could not finalize checksum of file \"%s\"",
186  pathname);
187 
189  ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
190  pg_checksum_type_name(checksum_ctx->type));
191  enlargeStringInfo(&buf, 2 * checksumlen);
192  buf.len += hex_encode((char *) checksumbuf, checksumlen,
193  &buf.data[buf.len]);
194  appendStringInfoChar(&buf, '"');
195  }
196 
197  /* Close out the object. */
198  appendStringInfoString(&buf, " }");
199 
200  /* OK, add it to the manifest. */
202 
203  /* Avoid leaking memory. */
204  pfree(buf.data);
205 }
static void AppendStringToManifest(backup_manifest_info *manifest, const char *s)
static bool IsManifestEnabled(backup_manifest_info *manifest)
unsigned char uint8
Definition: c.h:491
#define OidIsValid(objectId)
Definition: c.h:762
char * pg_checksum_type_name(pg_checksum_type type)
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
#define PG_CHECKSUM_MAX_LENGTH
@ CHECKSUM_TYPE_NONE
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
uint64 hex_encode(const char *src, size_t len, char *dst)
Definition: encode.c:162
void escape_json(StringInfo buf, const char *str)
Definition: json.c:1549
bool pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
Definition: mbutils.c:1566
void pfree(void *pointer)
Definition: mcxt.c:1508
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:73
@ PG_UTF8
Definition: pg_wchar.h:232
struct pg_tm * pg_gmtime(const pg_time_t *timep)
Definition: localtime.c:1389
size_t pg_strftime(char *s, size_t maxsize, const char *format, const struct pg_tm *t)
Definition: strftime.c:128
#define snprintf
Definition: port.h:238
static pg_noinline void Size size
Definition: slab.c:607
void appendStringInfo(StringInfo str, const char *fmt,...)
Definition: stringinfo.c:97
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:289
void appendStringInfoString(StringInfo str, const char *s)
Definition: stringinfo.c:182
void appendStringInfoChar(StringInfo str, char ch)
Definition: stringinfo.c:194
void initStringInfo(StringInfo str)
Definition: stringinfo.c:59
pg_checksum_type type

References appendStringInfo(), appendStringInfoChar(), appendStringInfoString(), AppendStringToManifest(), buf, CHECKSUM_TYPE_NONE, elog, enlargeStringInfo(), ERROR, escape_json(), hex_encode(), initStringInfo(), IsManifestEnabled(), manifest, MAXPGPATH, OidIsValid, pfree(), pg_checksum_final(), PG_CHECKSUM_MAX_LENGTH, pg_checksum_type_name(), pg_gmtime(), pg_strftime(), PG_UTF8, pg_verify_mbstr(), size, snprintf, and pg_checksum_context::type.

Referenced by sendFile(), and sendFileWithContent().

◆ AddWALInfoToBackupManifest()

void AddWALInfoToBackupManifest ( backup_manifest_info manifest,
XLogRecPtr  startptr,
TimeLineID  starttli,
XLogRecPtr  endptr,
TimeLineID  endtli 
)

Definition at line 212 of file backup_manifest.c.

215 {
216  List *timelines;
217  ListCell *lc;
218  bool first_wal_range = true;
219  bool found_start_timeline = false;
220 
222  return;
223 
224  /* Terminate the list of files. */
225  AppendStringToManifest(manifest, "\n],\n");
226 
227  /* Read the timeline history for the ending timeline. */
228  timelines = readTimeLineHistory(endtli);
229 
230  /* Start a list of LSN ranges. */
231  AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
232 
233  foreach(lc, timelines)
234  {
235  TimeLineHistoryEntry *entry = lfirst(lc);
236  XLogRecPtr tl_beginptr;
237 
238  /*
239  * We only care about timelines that were active during the backup.
240  * Skip any that ended before the backup started. (Note that if
241  * entry->end is InvalidXLogRecPtr, it means that the timeline has not
242  * yet ended.)
243  */
244  if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
245  continue;
246 
247  /*
248  * Because the timeline history file lists newer timelines before
249  * older ones, the first timeline we encounter that is new enough to
250  * matter ought to match the ending timeline of the backup.
251  */
252  if (first_wal_range && endtli != entry->tli)
253  ereport(ERROR,
254  errmsg("expected end timeline %u but found timeline %u",
255  starttli, entry->tli));
256 
257  /*
258  * If this timeline entry matches with the timeline on which the
259  * backup started, WAL needs to be checked from the start LSN of the
260  * backup. If this entry refers to a newer timeline, WAL needs to be
261  * checked since the beginning of this timeline, so use the LSN where
262  * the timeline began.
263  */
264  if (starttli == entry->tli)
265  tl_beginptr = startptr;
266  else
267  {
268  tl_beginptr = entry->begin;
269 
270  /*
271  * If we reach a TLI that has no valid beginning LSN, there can't
272  * be any more timelines in the history after this point, so we'd
273  * better have arrived at the expected starting TLI. If not,
274  * something's gone horribly wrong.
275  */
276  if (XLogRecPtrIsInvalid(entry->begin))
277  ereport(ERROR,
278  errmsg("expected start timeline %u but found timeline %u",
279  starttli, entry->tli));
280  }
281 
283  "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
284  first_wal_range ? "" : ",\n",
285  entry->tli,
286  LSN_FORMAT_ARGS(tl_beginptr),
287  LSN_FORMAT_ARGS(endptr));
288 
289  if (starttli == entry->tli)
290  {
291  found_start_timeline = true;
292  break;
293  }
294 
295  endptr = entry->begin;
296  first_wal_range = false;
297  }
298 
299  /*
300  * The last entry in the timeline history for the ending timeline should
301  * be the ending timeline itself. Verify that this is what we observed.
302  */
303  if (!found_start_timeline)
304  ereport(ERROR,
305  errmsg("start timeline %u not found in history of timeline %u",
306  starttli, endtli));
307 
308  /* Terminate the list of WAL ranges. */
309  AppendStringToManifest(manifest, "\n],\n");
310 }
List * readTimeLineHistory(TimeLineID targetTLI)
Definition: timeline.c:76
#define AppendToManifest(manifest,...)
int errmsg(const char *fmt,...)
Definition: elog.c:1072
#define ereport(elevel,...)
Definition: elog.h:149
#define lfirst(lc)
Definition: pg_list.h:172
Definition: pg_list.h:54
XLogRecPtr begin
Definition: timeline.h:28
TimeLineID tli
Definition: timeline.h:27
XLogRecPtr end
Definition: timeline.h:29
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21

References AppendStringToManifest(), AppendToManifest, TimeLineHistoryEntry::begin, TimeLineHistoryEntry::end, ereport, errmsg(), ERROR, IsManifestEnabled(), lfirst, LSN_FORMAT_ARGS, manifest, readTimeLineHistory(), TimeLineHistoryEntry::tli, and XLogRecPtrIsInvalid.

Referenced by perform_base_backup().

◆ AppendStringToManifest()

static void AppendStringToManifest ( backup_manifest_info manifest,
const char *  s 
)
static

Definition at line 383 of file backup_manifest.c.

384 {
385  int len = strlen(s);
386 
387  Assert(manifest != NULL);
388  if (manifest->still_checksumming)
389  {
390  if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
391  elog(ERROR, "failed to update checksum of backup manifest: %s",
392  pg_cryptohash_error(manifest->manifest_ctx));
393  }
394  BufFileWrite(manifest->buffile, s, len);
395  manifest->manifest_size += len;
396 }
void BufFileWrite(BufFile *file, const void *ptr, size_t size)
Definition: buffile.c:676
int pg_cryptohash_update(pg_cryptohash_ctx *ctx, const uint8 *data, size_t len)
Definition: cryptohash.c:136
const char * pg_cryptohash_error(pg_cryptohash_ctx *ctx)
Definition: cryptohash.c:254
Assert(fmt[strlen(fmt) - 1] !='\n')
const void size_t len

References Assert(), BufFileWrite(), elog, ERROR, len, manifest, pg_cryptohash_error(), and pg_cryptohash_update().

Referenced by AddFileToBackupManifest(), AddWALInfoToBackupManifest(), and SendBackupManifest().

◆ FreeBackupManifest()

void FreeBackupManifest ( backup_manifest_info manifest)

Definition at line 91 of file backup_manifest.c.

92 {
93  pg_cryptohash_free(manifest->manifest_ctx);
94  manifest->manifest_ctx = NULL;
95 }
void pg_cryptohash_free(pg_cryptohash_ctx *ctx)
Definition: cryptohash.c:238

References manifest, and pg_cryptohash_free().

Referenced by perform_base_backup().

◆ InitializeBackupManifest()

void InitializeBackupManifest ( backup_manifest_info manifest,
backup_manifest_option  want_manifest,
pg_checksum_type  manifest_checksum_type 
)

Definition at line 56 of file backup_manifest.c.

59 {
60  memset(manifest, 0, sizeof(backup_manifest_info));
61  manifest->checksum_type = manifest_checksum_type;
62 
63  if (want_manifest == MANIFEST_OPTION_NO)
64  manifest->buffile = NULL;
65  else
66  {
67  manifest->buffile = BufFileCreateTemp(false);
68  manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
69  if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
70  elog(ERROR, "failed to initialize checksum of backup manifest: %s",
71  pg_cryptohash_error(manifest->manifest_ctx));
72  }
73 
74  manifest->manifest_size = UINT64CONST(0);
75  manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
76  manifest->first_file = true;
77  manifest->still_checksumming = true;
78 
79  if (want_manifest != MANIFEST_OPTION_NO)
81  "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
82  "\"System-Identifier\": " UINT64_FORMAT ",\n"
83  "\"Files\": [",
85 }
@ MANIFEST_OPTION_NO
@ MANIFEST_OPTION_FORCE_ENCODE
BufFile * BufFileCreateTemp(bool interXact)
Definition: buffile.c:193
#define UINT64_FORMAT
Definition: c.h:536
int pg_cryptohash_init(pg_cryptohash_ctx *ctx)
Definition: cryptohash.c:100
pg_cryptohash_ctx * pg_cryptohash_create(pg_cryptohash_type type)
Definition: cryptohash.c:74
@ PG_SHA256
Definition: cryptohash.h:24
uint64 GetSystemIdentifier(void)
Definition: xlog.c:4458

References AppendToManifest, BufFileCreateTemp(), elog, ERROR, GetSystemIdentifier(), manifest, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, pg_cryptohash_create(), pg_cryptohash_error(), pg_cryptohash_init(), PG_SHA256, and UINT64_FORMAT.

Referenced by perform_base_backup().

◆ IsManifestEnabled()

static bool IsManifestEnabled ( backup_manifest_info manifest)
inlinestatic

Definition at line 33 of file backup_manifest.c.

34 {
35  return (manifest->buffile != NULL);
36 }

References manifest.

Referenced by AddFileToBackupManifest(), AddWALInfoToBackupManifest(), and SendBackupManifest().

◆ SendBackupManifest()

void SendBackupManifest ( backup_manifest_info manifest,
bbsink sink 
)

Definition at line 316 of file backup_manifest.c.

317 {
318  uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
319  char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
320  size_t manifest_bytes_done = 0;
321 
323  return;
324 
325  /*
326  * Append manifest checksum, so that the problems with the manifest itself
327  * can be detected.
328  *
329  * We always use SHA-256 for this, regardless of what algorithm is chosen
330  * for checksumming the files. If we ever want to make the checksum
331  * algorithm used for the manifest file variable, the client will need a
332  * way to figure out which algorithm to use as close to the beginning of
333  * the manifest file as possible, to avoid having to read the whole thing
334  * twice.
335  */
336  manifest->still_checksumming = false;
337  if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
338  sizeof(checksumbuf)) < 0)
339  elog(ERROR, "failed to finalize checksum of backup manifest: %s",
340  pg_cryptohash_error(manifest->manifest_ctx));
341  AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
342 
343  hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
344  checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
345 
346  AppendStringToManifest(manifest, checksumstringbuf);
348 
349  /*
350  * We've written all the data to the manifest file. Rewind the file so
351  * that we can read it all back.
352  */
353  if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
354  ereport(ERROR,
356  errmsg("could not rewind temporary file")));
357 
358 
359  /*
360  * Send the backup manifest.
361  */
362  bbsink_begin_manifest(sink);
363  while (manifest_bytes_done < manifest->manifest_size)
364  {
365  size_t bytes_to_read;
366 
367  bytes_to_read = Min(sink->bbs_buffer_length,
368  manifest->manifest_size - manifest_bytes_done);
369  BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
370  bbsink_manifest_contents(sink, bytes_to_read);
371  manifest_bytes_done += bytes_to_read;
372  }
373  bbsink_end_manifest(sink);
374 
375  /* Release resources */
376  BufFileClose(manifest->buffile);
377 }
static void bbsink_begin_manifest(bbsink *sink)
static void bbsink_end_manifest(bbsink *sink)
static void bbsink_manifest_contents(bbsink *sink, size_t len)
void BufFileReadExact(BufFile *file, void *ptr, size_t size)
Definition: buffile.c:654
int BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
Definition: buffile.c:740
void BufFileClose(BufFile *file)
Definition: buffile.c:412
#define Min(x, y)
Definition: c.h:991
int pg_cryptohash_final(pg_cryptohash_ctx *ctx, uint8 *dest, size_t len)
Definition: cryptohash.c:172
int errcode_for_file_access(void)
Definition: elog.c:882
#define PG_SHA256_DIGEST_LENGTH
Definition: sha2.h:23
#define PG_SHA256_DIGEST_STRING_LENGTH
Definition: sha2.h:24
char * bbs_buffer
size_t bbs_buffer_length

References AppendStringToManifest(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_begin_manifest(), bbsink_end_manifest(), bbsink_manifest_contents(), BufFileClose(), BufFileReadExact(), BufFileSeek(), elog, ereport, errcode_for_file_access(), errmsg(), ERROR, hex_encode(), IsManifestEnabled(), manifest, Min, pg_cryptohash_error(), pg_cryptohash_final(), PG_SHA256_DIGEST_LENGTH, and PG_SHA256_DIGEST_STRING_LENGTH.

Referenced by perform_base_backup().