PostgreSQL Source Code  git master
write_manifest.c File Reference
#include "postgres_fe.h"
#include <fcntl.h>
#include <time.h>
#include <unistd.h>
#include "common/checksum_helper.h"
#include "common/file_perm.h"
#include "common/logging.h"
#include "lib/stringinfo.h"
#include "load_manifest.h"
#include "mb/pg_wchar.h"
#include "write_manifest.h"
Include dependency graph for write_manifest.c:

Go to the source code of this file.

Data Structures

struct  manifest_writer
 

Functions

static void escape_json (StringInfo buf, const char *str)
 
static void flush_manifest (manifest_writer *mwriter)
 
static size_t hex_encode (const uint8 *src, size_t len, char *dst)
 
manifest_writercreate_manifest_writer (char *directory, uint64 system_identifier)
 
void add_file_to_manifest (manifest_writer *mwriter, const char *manifest_path, uint64 size, time_t mtime, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
 
void finalize_manifest (manifest_writer *mwriter, manifest_wal_range *first_wal_range)
 

Function Documentation

◆ add_file_to_manifest()

void add_file_to_manifest ( manifest_writer mwriter,
const char *  manifest_path,
uint64  size,
time_t  mtime,
pg_checksum_type  checksum_type,
int  checksum_length,
uint8 checksum_payload 
)

Definition at line 76 of file write_manifest.c.

81 {
82  int pathlen = strlen(manifest_path);
83 
84  if (mwriter->first_file)
85  {
86  appendStringInfoChar(&mwriter->buf, '\n');
87  mwriter->first_file = false;
88  }
89  else
90  appendStringInfoString(&mwriter->buf, ",\n");
91 
92  if (pg_encoding_verifymbstr(PG_UTF8, manifest_path, pathlen) == pathlen)
93  {
94  appendStringInfoString(&mwriter->buf, "{ \"Path\": ");
95  escape_json(&mwriter->buf, manifest_path);
96  appendStringInfoString(&mwriter->buf, ", ");
97  }
98  else
99  {
100  appendStringInfoString(&mwriter->buf, "{ \"Encoded-Path\": \"");
101  enlargeStringInfo(&mwriter->buf, 2 * pathlen);
102  mwriter->buf.len += hex_encode((const uint8 *) manifest_path, pathlen,
103  &mwriter->buf.data[mwriter->buf.len]);
104  appendStringInfoString(&mwriter->buf, "\", ");
105  }
106 
107  appendStringInfo(&mwriter->buf, "\"Size\": %llu, ",
108  (unsigned long long) size);
109 
110  appendStringInfoString(&mwriter->buf, "\"Last-Modified\": \"");
111  enlargeStringInfo(&mwriter->buf, 128);
112  mwriter->buf.len += strftime(&mwriter->buf.data[mwriter->buf.len], 128,
113  "%Y-%m-%d %H:%M:%S %Z",
114  gmtime(&mtime));
115  appendStringInfoChar(&mwriter->buf, '"');
116 
117  if (mwriter->buf.len > 128 * 1024)
118  flush_manifest(mwriter);
119 
120  if (checksum_length > 0)
121  {
122  appendStringInfo(&mwriter->buf,
123  ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
124  pg_checksum_type_name(checksum_type));
125 
126  enlargeStringInfo(&mwriter->buf, 2 * checksum_length);
127  mwriter->buf.len += hex_encode(checksum_payload, checksum_length,
128  &mwriter->buf.data[mwriter->buf.len]);
129 
130  appendStringInfoChar(&mwriter->buf, '"');
131  }
132 
133  appendStringInfoString(&mwriter->buf, " }");
134 
135  if (mwriter->buf.len > 128 * 1024)
136  flush_manifest(mwriter);
137 }
unsigned char uint8
Definition: c.h:516
char * pg_checksum_type_name(pg_checksum_type type)
@ PG_UTF8
Definition: pg_wchar.h:232
static pg_noinline void Size size
Definition: slab.c:607
void appendStringInfo(StringInfo str, const char *fmt,...)
Definition: stringinfo.c:94
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:286
void appendStringInfoString(StringInfo str, const char *s)
Definition: stringinfo.c:179
void appendStringInfoChar(StringInfo str, char ch)
Definition: stringinfo.c:191
StringInfoData buf
int pg_encoding_verifymbstr(int encoding, const char *mbstr, int len)
Definition: wchar.c:2116
static void flush_manifest(manifest_writer *mwriter)
static size_t hex_encode(const uint8 *src, size_t len, char *dst)
static void escape_json(StringInfo buf, const char *str)

References appendStringInfo(), appendStringInfoChar(), appendStringInfoString(), manifest_writer::buf, StringInfoData::data, enlargeStringInfo(), escape_json(), manifest_writer::first_file, flush_manifest(), hex_encode(), StringInfoData::len, pg_checksum_type_name(), pg_encoding_verifymbstr(), PG_UTF8, and size.

Referenced by process_directory_recursively(), and write_backup_label().

◆ create_manifest_writer()

manifest_writer* create_manifest_writer ( char *  directory,
uint64  system_identifier 
)

Definition at line 48 of file write_manifest.c.

49 {
50  manifest_writer *mwriter = pg_malloc(sizeof(manifest_writer));
51 
52  snprintf(mwriter->pathname, MAXPGPATH, "%s/backup_manifest", directory);
53  mwriter->fd = -1;
54  initStringInfo(&mwriter->buf);
55  mwriter->first_file = true;
56  mwriter->still_checksumming = true;
58 
59  appendStringInfo(&mwriter->buf,
60  "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
61  "\"System-Identifier\": " UINT64_FORMAT ",\n"
62  "\"Files\": [",
63  system_identifier);
64 
65  return mwriter;
66 }
#define UINT64_FORMAT
Definition: c.h:552
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
@ CHECKSUM_TYPE_SHA256
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define MAXPGPATH
#define snprintf
Definition: port.h:238
void initStringInfo(StringInfo str)
Definition: stringinfo.c:56
char pathname[MAXPGPATH]
pg_checksum_context manifest_ctx
static const char * directory
Definition: zic.c:634

References appendStringInfo(), manifest_writer::buf, CHECKSUM_TYPE_SHA256, directory, manifest_writer::fd, manifest_writer::first_file, initStringInfo(), manifest_writer::manifest_ctx, MAXPGPATH, manifest_writer::pathname, pg_checksum_init(), pg_malloc(), snprintf, manifest_writer::still_checksumming, and UINT64_FORMAT.

Referenced by main().

◆ escape_json()

static void escape_json ( StringInfo  buf,
const char *  str 
)
static

Definition at line 196 of file write_manifest.c.

197 {
198  const char *p;
199 
201  for (p = str; *p; p++)
202  {
203  switch (*p)
204  {
205  case '\b':
206  appendStringInfoString(buf, "\\b");
207  break;
208  case '\f':
209  appendStringInfoString(buf, "\\f");
210  break;
211  case '\n':
212  appendStringInfoString(buf, "\\n");
213  break;
214  case '\r':
215  appendStringInfoString(buf, "\\r");
216  break;
217  case '\t':
218  appendStringInfoString(buf, "\\t");
219  break;
220  case '"':
221  appendStringInfoString(buf, "\\\"");
222  break;
223  case '\\':
224  appendStringInfoString(buf, "\\\\");
225  break;
226  default:
227  if ((unsigned char) *p < ' ')
228  appendStringInfo(buf, "\\u%04x", (int) *p);
229  else
231  break;
232  }
233  }
235 }
const char * str
static char * buf
Definition: pg_test_fsync.c:72
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:204

References appendStringInfo(), appendStringInfoCharMacro, appendStringInfoString(), buf, and str.

Referenced by add_file_to_manifest().

◆ finalize_manifest()

void finalize_manifest ( manifest_writer mwriter,
manifest_wal_range first_wal_range 
)

Definition at line 143 of file write_manifest.c.

145 {
146  uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
147  int len;
148  manifest_wal_range *wal_range;
149 
150  /* Terminate the list of files. */
151  appendStringInfoString(&mwriter->buf, "\n],\n");
152 
153  /* Start a list of LSN ranges. */
154  appendStringInfoString(&mwriter->buf, "\"WAL-Ranges\": [\n");
155 
156  for (wal_range = first_wal_range; wal_range != NULL;
157  wal_range = wal_range->next)
158  appendStringInfo(&mwriter->buf,
159  "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
160  wal_range == first_wal_range ? "" : ",\n",
161  wal_range->tli,
162  LSN_FORMAT_ARGS(wal_range->start_lsn),
163  LSN_FORMAT_ARGS(wal_range->end_lsn));
164 
165  /* Terminate the list of WAL ranges. */
166  appendStringInfoString(&mwriter->buf, "\n],\n");
167 
168  /* Flush accumulated data and update checksum calculation. */
169  flush_manifest(mwriter);
170 
171  /* Checksum only includes data up to this point. */
172  mwriter->still_checksumming = false;
173 
174  /* Compute and insert manifest checksum. */
175  appendStringInfoString(&mwriter->buf, "\"Manifest-Checksum\": \"");
177  len = pg_checksum_final(&mwriter->manifest_ctx, checksumbuf);
179  mwriter->buf.len +=
180  hex_encode(checksumbuf, len, &mwriter->buf.data[mwriter->buf.len]);
181  appendStringInfoString(&mwriter->buf, "\"}\n");
182 
183  /* Flush the last manifest checksum itself. */
184  flush_manifest(mwriter);
185 
186  /* Close the file. */
187  if (close(mwriter->fd) != 0)
188  pg_fatal("could not close file \"%s\": %m", mwriter->pathname);
189  mwriter->fd = -1;
190 }
#define Assert(condition)
Definition: c.h:861
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
#define close(a)
Definition: win32.h:12
#define pg_fatal(...)
const void size_t len
#define PG_SHA256_DIGEST_LENGTH
Definition: sha2.h:23
#define PG_SHA256_DIGEST_STRING_LENGTH
Definition: sha2.h:24
XLogRecPtr end_lsn
Definition: load_manifest.h:48
struct manifest_wal_range * next
Definition: load_manifest.h:49
XLogRecPtr start_lsn
Definition: load_manifest.h:47
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43

References appendStringInfo(), appendStringInfoString(), Assert, manifest_writer::buf, close, StringInfoData::data, manifest_wal_range::end_lsn, enlargeStringInfo(), manifest_writer::fd, flush_manifest(), hex_encode(), StringInfoData::len, len, LSN_FORMAT_ARGS, manifest_writer::manifest_ctx, manifest_wal_range::next, manifest_writer::pathname, pg_checksum_final(), pg_fatal, PG_SHA256_DIGEST_LENGTH, PG_SHA256_DIGEST_STRING_LENGTH, manifest_wal_range::start_lsn, manifest_writer::still_checksumming, and manifest_wal_range::tli.

Referenced by main().

◆ flush_manifest()

static void flush_manifest ( manifest_writer mwriter)
static

Definition at line 245 of file write_manifest.c.

246 {
247  if (mwriter->fd == -1 &&
248  (mwriter->fd = open(mwriter->pathname,
249  O_WRONLY | O_CREAT | O_EXCL | PG_BINARY,
250  pg_file_create_mode)) < 0)
251  pg_fatal("could not open file \"%s\": %m", mwriter->pathname);
252 
253  if (mwriter->buf.len > 0)
254  {
255  ssize_t wb;
256 
257  wb = write(mwriter->fd, mwriter->buf.data, mwriter->buf.len);
258  if (wb != mwriter->buf.len)
259  {
260  if (wb < 0)
261  pg_fatal("could not write file \"%s\": %m", mwriter->pathname);
262  else
263  pg_fatal("could not write file \"%s\": wrote %d of %d",
264  mwriter->pathname, (int) wb, mwriter->buf.len);
265  }
266 
267  if (mwriter->still_checksumming &&
269  (uint8 *) mwriter->buf.data,
270  mwriter->buf.len) < 0)
271  pg_fatal("could not update checksum of file \"%s\"",
272  mwriter->pathname);
273  resetStringInfo(&mwriter->buf);
274  }
275 }
#define PG_BINARY
Definition: c.h:1276
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_file_create_mode
Definition: file_perm.c:19
#define write(a, b, c)
Definition: win32.h:14
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:75

References manifest_writer::buf, StringInfoData::data, manifest_writer::fd, StringInfoData::len, manifest_writer::manifest_ctx, manifest_writer::pathname, PG_BINARY, pg_checksum_update(), pg_fatal, pg_file_create_mode, resetStringInfo(), manifest_writer::still_checksumming, and write.

Referenced by add_file_to_manifest(), and finalize_manifest().

◆ hex_encode()

static size_t hex_encode ( const uint8 src,
size_t  len,
char *  dst 
)
static

Definition at line 281 of file write_manifest.c.

282 {
283  const uint8 *end = src + len;
284 
285  while (src < end)
286  {
287  unsigned n1 = (*src >> 4) & 0xF;
288  unsigned n2 = *src & 0xF;
289 
290  *dst++ = n1 < 10 ? '0' + n1 : 'a' + n1 - 10;
291  *dst++ = n2 < 10 ? '0' + n2 : 'a' + n2 - 10;
292  ++src;
293  }
294 
295  return len * 2;
296 }

References len.

Referenced by add_file_to_manifest(), and finalize_manifest().