PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
basebackup.c File Reference
#include "postgres.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/catalog.h"
#include "catalog/pg_type.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgtar.h"
#include "pgstat.h"
#include "replication/basebackup.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "utils/builtins.h"
#include "utils/elog.h"
#include "utils/ps_status.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 

Functions

static int64 sendDir (char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
 
static bool sendFile (char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
 
static void sendFileWithContent (const char *filename, const char *content)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void base_backup_cleanup (int code, Datum arg)
 
static void perform_base_backup (basebackup_options *opt, DIR *tblspcdir)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const void *a, const void *b)
 
static void throttle (size_t increment)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 
int64 sendTablespace (char *path, bool sizeonly)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static const char * excludeDirContents []
 
static const char * excludeFiles []
 

Macro Definition Documentation

#define TAR_SEND_SIZE   32768

Definition at line 81 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

#define THROTTLING_FREQUENCY   8

Definition at line 86 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1314 of file basebackup.c.

References _tarWriteHeader(), and NULL.

Referenced by sendDir().

1316 {
1317  /* If symlink, write it as a directory anyway */
1318 #ifndef WIN32
1319  if (S_ISLNK(statbuf->st_mode))
1320 #else
1321  if (pgwin32_is_junction(pathbuf))
1322 #endif
1323  statbuf->st_mode = S_IFDIR | S_IRWXU;
1324 
1325  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1326 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
#define NULL
Definition: c.h:226
static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1272 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1274 {
1275  char h[512];
1276  enum tarError rc;
1277 
1278  if (!sizeonly)
1279  {
1280  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1281  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1282  statbuf->st_mtime);
1283 
1284  switch (rc)
1285  {
1286  case TAR_OK:
1287  break;
1288  case TAR_NAME_TOO_LONG:
1289  ereport(ERROR,
1290  (errmsg("file name too long for tar format: \"%s\"",
1291  filename)));
1292  break;
1293  case TAR_SYMLINK_TOO_LONG:
1294  ereport(ERROR,
1295  (errmsg("symbolic link target too long for tar format: "
1296  "file name \"%s\", target \"%s\"",
1297  filename, linktarget)));
1298  break;
1299  default:
1300  elog(ERROR, "unrecognized tar error: %d", rc);
1301  }
1302 
1303  pq_putmessage('d', h, sizeof(h));
1304  }
1305 
1306  return sizeof(h);
1307 }
Definition: pgtar.h:17
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define ereport(elevel, rest)
Definition: elog.h:122
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
static char * filename
Definition: pg_dumpall.c:84
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define elog
Definition: elog.h:219
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:43
static void base_backup_cleanup ( int  code,
Datum  arg 
)
static

Definition at line 172 of file basebackup.c.

References do_pg_abort_backup().

Referenced by perform_base_backup().

173 {
175 }
void do_pg_abort_backup(void)
Definition: xlog.c:10964
static int compareWalFileNames ( const void *  a,
const void *  b 
)
static

Definition at line 565 of file basebackup.c.

Referenced by perform_base_backup().

566 {
567  char *fna = *((char **) a);
568  char *fnb = *((char **) b);
569 
570  return strcmp(fna + 8, fnb + 8);
571 }
static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 577 of file basebackup.c.

References DefElem::arg, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, basebackup_options::nowait, NULL, basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

578 {
579  ListCell *lopt;
580  bool o_label = false;
581  bool o_progress = false;
582  bool o_fast = false;
583  bool o_nowait = false;
584  bool o_wal = false;
585  bool o_maxrate = false;
586  bool o_tablespace_map = false;
587 
588  MemSet(opt, 0, sizeof(*opt));
589  foreach(lopt, options)
590  {
591  DefElem *defel = (DefElem *) lfirst(lopt);
592 
593  if (strcmp(defel->defname, "label") == 0)
594  {
595  if (o_label)
596  ereport(ERROR,
597  (errcode(ERRCODE_SYNTAX_ERROR),
598  errmsg("duplicate option \"%s\"", defel->defname)));
599  opt->label = strVal(defel->arg);
600  o_label = true;
601  }
602  else if (strcmp(defel->defname, "progress") == 0)
603  {
604  if (o_progress)
605  ereport(ERROR,
606  (errcode(ERRCODE_SYNTAX_ERROR),
607  errmsg("duplicate option \"%s\"", defel->defname)));
608  opt->progress = true;
609  o_progress = true;
610  }
611  else if (strcmp(defel->defname, "fast") == 0)
612  {
613  if (o_fast)
614  ereport(ERROR,
615  (errcode(ERRCODE_SYNTAX_ERROR),
616  errmsg("duplicate option \"%s\"", defel->defname)));
617  opt->fastcheckpoint = true;
618  o_fast = true;
619  }
620  else if (strcmp(defel->defname, "nowait") == 0)
621  {
622  if (o_nowait)
623  ereport(ERROR,
624  (errcode(ERRCODE_SYNTAX_ERROR),
625  errmsg("duplicate option \"%s\"", defel->defname)));
626  opt->nowait = true;
627  o_nowait = true;
628  }
629  else if (strcmp(defel->defname, "wal") == 0)
630  {
631  if (o_wal)
632  ereport(ERROR,
633  (errcode(ERRCODE_SYNTAX_ERROR),
634  errmsg("duplicate option \"%s\"", defel->defname)));
635  opt->includewal = true;
636  o_wal = true;
637  }
638  else if (strcmp(defel->defname, "max_rate") == 0)
639  {
640  long maxrate;
641 
642  if (o_maxrate)
643  ereport(ERROR,
644  (errcode(ERRCODE_SYNTAX_ERROR),
645  errmsg("duplicate option \"%s\"", defel->defname)));
646 
647  maxrate = intVal(defel->arg);
648  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
649  ereport(ERROR,
650  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
651  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
652  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
653 
654  opt->maxrate = (uint32) maxrate;
655  o_maxrate = true;
656  }
657  else if (strcmp(defel->defname, "tablespace_map") == 0)
658  {
659  if (o_tablespace_map)
660  ereport(ERROR,
661  (errcode(ERRCODE_SYNTAX_ERROR),
662  errmsg("duplicate option \"%s\"", defel->defname)));
663  opt->sendtblspcmapfile = true;
664  o_tablespace_map = true;
665  }
666  else
667  elog(ERROR, "option \"%s\" not recognized",
668  defel->defname);
669  }
670  if (opt->label == NULL)
671  opt->label = "base backup";
672 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
#define MemSet(start, val, len)
Definition: c.h:853
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:43
static int32 maxrate
Definition: pg_basebackup.c:95
const char * label
Definition: basebackup.c:44
unsigned int uint32
Definition: c.h:265
#define ereport(elevel, rest)
Definition: elog.h:122
Node * arg
Definition: parsenodes.h:676
#define NULL
Definition: c.h:226
#define lfirst(lc)
Definition: pg_list.h:106
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:797
char * defname
Definition: parsenodes.h:675
#define elog
Definition: elog.h:219
static void perform_base_backup ( basebackup_options opt,
DIR tblspcdir 
)
static

Definition at line 184 of file basebackup.c.

References _tarWriteHeader(), AllocateDir(), AllocateFile(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, base_backup_cleanup(), buf, CheckXLogRemoved(), compareWalFileNames(), dirent::d_name, StringInfoData::data, DataDir, do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, FreeDir(), FreeFile(), GetCurrentTimestamp(), i, basebackup_options::includewal, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, list_length(), lnext, lstat, makeStringInfo(), MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, NULL, palloc(), palloc0(), tablespaceinfo::path, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint(), basebackup_options::progress, psprintf(), pstrdup(), qsort, ReadDir(), RecoveryInProgress(), SendBackupHeader(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf(), statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, USECS_PER_SEC, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, XLogFromFileName, and XLogSegSize.

Referenced by SendBaseBackup().

185 {
186  XLogRecPtr startptr;
187  TimeLineID starttli;
188  XLogRecPtr endptr;
189  TimeLineID endtli;
190  StringInfo labelfile;
192  int datadirpathlen;
193  List *tablespaces = NIL;
194 
195  datadirpathlen = strlen(DataDir);
196 
198 
199  labelfile = makeStringInfo();
200  tblspc_map_file = makeStringInfo();
201 
202  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
203  labelfile, tblspcdir, &tablespaces,
204  tblspc_map_file,
205  opt->progress, opt->sendtblspcmapfile);
206 
207  /*
208  * Once do_pg_start_backup has been called, ensure that any failure causes
209  * us to abort the backup so we don't "leak" a backup counter. For this
210  * reason, *all* functionality between do_pg_start_backup() and
211  * do_pg_stop_backup() should be inside the error cleanup block!
212  */
213 
215  {
216  ListCell *lc;
217  tablespaceinfo *ti;
218 
219  SendXlogRecPtrResult(startptr, starttli);
220 
221  /*
222  * Calculate the relative path of temporary statistics directory in
223  * order to skip the files which are located in that directory later.
224  */
226  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
227  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
228  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
230  else
232 
233  /* Add a node for the base directory at the end */
234  ti = palloc0(sizeof(tablespaceinfo));
235  ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
236  tablespaces = lappend(tablespaces, ti);
237 
238  /* Send tablespace header */
239  SendBackupHeader(tablespaces);
240 
241  /* Setup and activate network throttling, if client requested it */
242  if (opt->maxrate > 0)
243  {
245  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
246 
247  /*
248  * The minimum amount of time for throttling_sample bytes to be
249  * transferred.
250  */
252 
253  /* Enable throttling. */
254  throttling_counter = 0;
255 
256  /* The 'real data' starts now (header was ignored). */
258  }
259  else
260  {
261  /* Disable throttling. */
262  throttling_counter = -1;
263  }
264 
265  /* Send off our tablespaces one by one */
266  foreach(lc, tablespaces)
267  {
268  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
270 
271  /* Send CopyOutResponse message */
272  pq_beginmessage(&buf, 'H');
273  pq_sendbyte(&buf, 0); /* overall format */
274  pq_sendint(&buf, 0, 2); /* natts */
275  pq_endmessage(&buf);
276 
277  if (ti->path == NULL)
278  {
279  struct stat statbuf;
280 
281  /* In the main tar, include the backup_label first... */
283 
284  /*
285  * Send tablespace_map file if required and then the bulk of
286  * the files.
287  */
288  if (tblspc_map_file && opt->sendtblspcmapfile)
289  {
290  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
291  sendDir(".", 1, false, tablespaces, false);
292  }
293  else
294  sendDir(".", 1, false, tablespaces, true);
295 
296  /* ... and pg_control after everything else. */
297  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
298  ereport(ERROR,
300  errmsg("could not stat control file \"%s\": %m",
302  sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false);
303  }
304  else
305  sendTablespace(ti->path, false);
306 
307  /*
308  * If we're including WAL, and this is the main data directory we
309  * don't terminate the tar stream here. Instead, we will append
310  * the xlog files below and terminate it then. This is safe since
311  * the main data directory is always sent *last*.
312  */
313  if (opt->includewal && ti->path == NULL)
314  {
315  Assert(lnext(lc) == NULL);
316  }
317  else
318  pq_putemptymessage('c'); /* CopyDone */
319  }
320  }
322 
323  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
324 
325  if (opt->includewal)
326  {
327  /*
328  * We've left the last tar file "open", so we can now append the
329  * required WAL files to it.
330  */
331  char pathbuf[MAXPGPATH];
332  XLogSegNo segno;
333  XLogSegNo startsegno;
334  XLogSegNo endsegno;
335  struct stat statbuf;
336  List *historyFileList = NIL;
337  List *walFileList = NIL;
338  char **walFiles;
339  int nWalFiles;
340  char firstoff[MAXFNAMELEN];
341  char lastoff[MAXFNAMELEN];
342  DIR *dir;
343  struct dirent *de;
344  int i;
345  ListCell *lc;
346  TimeLineID tli;
347 
348  /*
349  * I'd rather not worry about timelines here, so scan pg_wal and
350  * include all WAL files in the range between 'startptr' and 'endptr',
351  * regardless of the timeline the file is stamped with. If there are
352  * some spurious WAL files belonging to timelines that don't belong in
353  * this server's history, they will be included too. Normally there
354  * shouldn't be such files, but if there are, there's little harm in
355  * including them.
356  */
357  XLByteToSeg(startptr, startsegno);
358  XLogFileName(firstoff, ThisTimeLineID, startsegno);
359  XLByteToPrevSeg(endptr, endsegno);
360  XLogFileName(lastoff, ThisTimeLineID, endsegno);
361 
362  dir = AllocateDir("pg_wal");
363  if (!dir)
364  ereport(ERROR,
365  (errmsg("could not open directory \"%s\": %m", "pg_wal")));
366  while ((de = ReadDir(dir, "pg_wal")) != NULL)
367  {
368  /* Does it look like a WAL segment, and is it in the range? */
369  if (IsXLogFileName(de->d_name) &&
370  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
371  strcmp(de->d_name + 8, lastoff + 8) <= 0)
372  {
373  walFileList = lappend(walFileList, pstrdup(de->d_name));
374  }
375  /* Does it look like a timeline history file? */
376  else if (IsTLHistoryFileName(de->d_name))
377  {
378  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
379  }
380  }
381  FreeDir(dir);
382 
383  /*
384  * Before we go any further, check that none of the WAL segments we
385  * need were removed.
386  */
387  CheckXLogRemoved(startsegno, ThisTimeLineID);
388 
389  /*
390  * Put the WAL filenames into an array, and sort. We send the files in
391  * order from oldest to newest, to reduce the chance that a file is
392  * recycled before we get a chance to send it over.
393  */
394  nWalFiles = list_length(walFileList);
395  walFiles = palloc(nWalFiles * sizeof(char *));
396  i = 0;
397  foreach(lc, walFileList)
398  {
399  walFiles[i++] = lfirst(lc);
400  }
401  qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
402 
403  /*
404  * There must be at least one xlog file in the pg_wal directory,
405  * since we are doing backup-including-xlog.
406  */
407  if (nWalFiles < 1)
408  ereport(ERROR,
409  (errmsg("could not find any WAL files")));
410 
411  /*
412  * Sanity check: the first and last segment should cover startptr and
413  * endptr, with no gaps in between.
414  */
415  XLogFromFileName(walFiles[0], &tli, &segno);
416  if (segno != startsegno)
417  {
418  char startfname[MAXFNAMELEN];
419 
420  XLogFileName(startfname, ThisTimeLineID, startsegno);
421  ereport(ERROR,
422  (errmsg("could not find WAL file \"%s\"", startfname)));
423  }
424  for (i = 0; i < nWalFiles; i++)
425  {
426  XLogSegNo currsegno = segno;
427  XLogSegNo nextsegno = segno + 1;
428 
429  XLogFromFileName(walFiles[i], &tli, &segno);
430  if (!(nextsegno == segno || currsegno == segno))
431  {
432  char nextfname[MAXFNAMELEN];
433 
434  XLogFileName(nextfname, ThisTimeLineID, nextsegno);
435  ereport(ERROR,
436  (errmsg("could not find WAL file \"%s\"", nextfname)));
437  }
438  }
439  if (segno != endsegno)
440  {
441  char endfname[MAXFNAMELEN];
442 
443  XLogFileName(endfname, ThisTimeLineID, endsegno);
444  ereport(ERROR,
445  (errmsg("could not find WAL file \"%s\"", endfname)));
446  }
447 
448  /* Ok, we have everything we need. Send the WAL files. */
449  for (i = 0; i < nWalFiles; i++)
450  {
451  FILE *fp;
452  char buf[TAR_SEND_SIZE];
453  size_t cnt;
454  pgoff_t len = 0;
455 
456  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
457  XLogFromFileName(walFiles[i], &tli, &segno);
458 
459  fp = AllocateFile(pathbuf, "rb");
460  if (fp == NULL)
461  {
462  /*
463  * Most likely reason for this is that the file was already
464  * removed by a checkpoint, so check for that to get a better
465  * error message.
466  */
467  CheckXLogRemoved(segno, tli);
468 
469  ereport(ERROR,
471  errmsg("could not open file \"%s\": %m", pathbuf)));
472  }
473 
474  if (fstat(fileno(fp), &statbuf) != 0)
475  ereport(ERROR,
477  errmsg("could not stat file \"%s\": %m",
478  pathbuf)));
479  if (statbuf.st_size != XLogSegSize)
480  {
481  CheckXLogRemoved(segno, tli);
482  ereport(ERROR,
484  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
485  }
486 
487  /* send the WAL file itself */
488  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
489 
490  while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
491  {
492  CheckXLogRemoved(segno, tli);
493  /* Send the chunk as a CopyData message */
494  if (pq_putmessage('d', buf, cnt))
495  ereport(ERROR,
496  (errmsg("base backup could not send data, aborting backup")));
497 
498  len += cnt;
499  throttle(cnt);
500 
501  if (len == XLogSegSize)
502  break;
503  }
504 
505  if (len != XLogSegSize)
506  {
507  CheckXLogRemoved(segno, tli);
508  ereport(ERROR,
510  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
511  }
512 
513  /* XLogSegSize is a multiple of 512, so no need for padding */
514 
515  FreeFile(fp);
516 
517  /*
518  * Mark file as archived, otherwise files can get archived again
519  * after promotion of a new node. This is in line with
520  * walreceiver.c always doing an XLogArchiveForceDone() after a
521  * complete segment.
522  */
523  StatusFilePath(pathbuf, walFiles[i], ".done");
524  sendFileWithContent(pathbuf, "");
525  }
526 
527  /*
528  * Send timeline history files too. Only the latest timeline history
529  * file is required for recovery, and even that only if there happens
530  * to be a timeline switch in the first WAL segment that contains the
531  * checkpoint record, or if we're taking a base backup from a standby
532  * server and the target timeline changes while the backup is taken.
533  * But they are small and highly useful for debugging purposes, so
534  * better include them all, always.
535  */
536  foreach(lc, historyFileList)
537  {
538  char *fname = lfirst(lc);
539 
540  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
541 
542  if (lstat(pathbuf, &statbuf) != 0)
543  ereport(ERROR,
545  errmsg("could not stat file \"%s\": %m", pathbuf)));
546 
547  sendFile(pathbuf, pathbuf, &statbuf, false);
548 
549  /* unconditionally mark file as archived */
550  StatusFilePath(pathbuf, fname, ".done");
551  sendFileWithContent(pathbuf, "");
552  }
553 
554  /* Send CopyDone message for the last tar file */
555  pq_putemptymessage('c');
556  }
557  SendXlogRecPtrResult(endptr, endtli);
558 }
#define StatusFilePath(path, xlog, suffix)
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:954
#define NIL
Definition: pg_list.h:69
#define XLogSegSize
Definition: xlog_internal.h:92
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1199
static void throttle(size_t increment)
Definition: basebackup.c:1334
uint32 TimeLineID
Definition: xlogdefs.h:45
void pq_sendbyte(StringInfo buf, int byt)
Definition: pqformat.c:105
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1569
char * pstrdup(const char *in)
Definition: mcxt.c:1165
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:29
#define Min(x, y)
Definition: c.h:802
#define XLogFileName(fname, tli, logSegNo)
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:723
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:421
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
bool RecoveryInProgress(void)
Definition: xlog.c:7805
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
#define TABLESPACE_MAP
Definition: xlog.h:304
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:10581
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3743
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:65
uint64 XLogSegNo
Definition: xlogdefs.h:34
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
const char * label
Definition: basebackup.c:44
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2043
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:801
int64 sendTablespace(char *path, bool sizeonly)
Definition: basebackup.c:903
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2284
char * pgstat_stat_directory
Definition: pgstat.c:118
#define XLByteToPrevSeg(xlrp, logSegNo)
#define lnext(lc)
Definition: pg_list.h:105
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
List * lappend(List *list, void *datum)
Definition: list.c:128
#define MAXFNAMELEN
#define pgoff_t
Definition: win32.h:241
void * palloc0(Size size)
Definition: mcxt.c:920
#define XLOGDIR
uintptr_t Datum
Definition: postgres.h:374
#define XLOG_CONTROL_FILE
TimeLineID ThisTimeLineID
Definition: xlog.c:178
static TimestampTz throttled_last
Definition: basebackup.c:98
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:44
#define XLByteToSeg(xlrp, logSegNo)
#define NULL
Definition: c.h:226
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:671
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2350
#define THROTTLING_FREQUENCY
Definition: basebackup.c:86
static int list_length(const List *l)
Definition: pg_list.h:89
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
int FreeFile(FILE *file)
Definition: fd.c:2226
#define IsTLHistoryFileName(fname)
void * palloc(Size size)
Definition: mcxt.c:891
int errmsg(const char *fmt,...)
Definition: elog.c:797
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
#define TAR_SEND_SIZE
Definition: basebackup.c:81
int i
char * DataDir
Definition: globals.c:59
#define BACKUP_LABEL_FILE
Definition: xlog.h:301
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:43
static int64 throttling_counter
Definition: basebackup.c:92
#define qsort(a, b, c, d)
Definition: port.h:440
#define XLogFromFileName(fname, tli, logSegNo)
static void base_backup_cleanup(int code, Datum arg)
Definition: basebackup.c:172
#define lstat(path, sb)
Definition: win32.h:272
Definition: pg_list.h:45
static uint64 throttling_sample
Definition: basebackup.c:89
static void sendFileWithContent(const char *filename, const char *content)
Definition: basebackup.c:856
int FreeDir(DIR *dir)
Definition: fd.c:2393
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, DIR *tblspcdir, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10087
static int compareWalFileNames(const void *a, const void *b)
Definition: basebackup.c:565
static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 713 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), and pq_sendint().

Referenced by SendBackupHeader().

714 {
715  char is[32];
716 
717  sprintf(is, INT64_FORMAT, intval);
718  pq_sendint(buf, strlen(is), 4);
719  pq_sendbytes(buf, is, strlen(is));
720 }
#define INT64_FORMAT
Definition: c.h:312
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
static void SendBackupHeader ( List tablespaces)
static

Definition at line 723 of file basebackup.c.

References buf, INT8OID, lfirst, NULL, tablespaceinfo::oid, OIDOID, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint(), pq_sendstring(), send_int8_string(), tablespaceinfo::size, and TEXTOID.

Referenced by perform_base_backup().

724 {
726  ListCell *lc;
727 
728  /* Construct and send the directory information */
729  pq_beginmessage(&buf, 'T'); /* RowDescription */
730  pq_sendint(&buf, 3, 2); /* 3 fields */
731 
732  /* First field - spcoid */
733  pq_sendstring(&buf, "spcoid");
734  pq_sendint(&buf, 0, 4); /* table oid */
735  pq_sendint(&buf, 0, 2); /* attnum */
736  pq_sendint(&buf, OIDOID, 4); /* type oid */
737  pq_sendint(&buf, 4, 2); /* typlen */
738  pq_sendint(&buf, 0, 4); /* typmod */
739  pq_sendint(&buf, 0, 2); /* format code */
740 
741  /* Second field - spcpath */
742  pq_sendstring(&buf, "spclocation");
743  pq_sendint(&buf, 0, 4);
744  pq_sendint(&buf, 0, 2);
745  pq_sendint(&buf, TEXTOID, 4);
746  pq_sendint(&buf, -1, 2);
747  pq_sendint(&buf, 0, 4);
748  pq_sendint(&buf, 0, 2);
749 
750  /* Third field - size */
751  pq_sendstring(&buf, "size");
752  pq_sendint(&buf, 0, 4);
753  pq_sendint(&buf, 0, 2);
754  pq_sendint(&buf, INT8OID, 4);
755  pq_sendint(&buf, 8, 2);
756  pq_sendint(&buf, 0, 4);
757  pq_sendint(&buf, 0, 2);
758  pq_endmessage(&buf);
759 
760  foreach(lc, tablespaces)
761  {
762  tablespaceinfo *ti = lfirst(lc);
763 
764  /* Send one datarow message */
765  pq_beginmessage(&buf, 'D');
766  pq_sendint(&buf, 3, 2); /* number of columns */
767  if (ti->path == NULL)
768  {
769  pq_sendint(&buf, -1, 4); /* Length = -1 ==> NULL */
770  pq_sendint(&buf, -1, 4);
771  }
772  else
773  {
774  Size len;
775 
776  len = strlen(ti->oid);
777  pq_sendint(&buf, len, 4);
778  pq_sendbytes(&buf, ti->oid, len);
779 
780  len = strlen(ti->path);
781  pq_sendint(&buf, len, 4);
782  pq_sendbytes(&buf, ti->path, len);
783  }
784  if (ti->size >= 0)
785  send_int8_string(&buf, ti->size / 1024);
786  else
787  pq_sendint(&buf, -1, 4); /* NULL */
788 
789  pq_endmessage(&buf);
790  }
791 
792  /* Send a CommandComplete message */
793  pq_puttextmessage('C', "SELECT");
794 }
#define OIDOID
Definition: pg_type.h:328
#define TEXTOID
Definition: pg_type.h:324
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:186
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:713
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
static char * buf
Definition: pg_test_fsync.c:65
#define INT8OID
Definition: pg_type.h:304
#define NULL
Definition: c.h:226
#define lfirst(lc)
Definition: pg_list.h:106
size_t Size
Definition: c.h:353
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:400
void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 683 of file basebackup.c.

References AllocateDir(), ereport, errmsg(), ERROR, FreeDir(), basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf(), update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

684 {
685  DIR *dir;
686  basebackup_options opt;
687 
688  parse_basebackup_options(cmd->options, &opt);
689 
691 
693  {
694  char activitymsg[50];
695 
696  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
697  opt.label);
698  set_ps_display(activitymsg, false);
699  }
700 
701  /* Make sure we can open the directory with tablespaces in it */
702  dir = AllocateDir("pg_tblspc");
703  if (!dir)
704  ereport(ERROR,
705  (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
706 
707  perform_base_backup(&opt, dir);
708 
709  FreeDir(dir);
710 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:35
void set_ps_display(const char *activity, bool force)
Definition: ps_status.c:326
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:577
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
Definition: basebackup.c:184
const char * label
Definition: basebackup.c:44
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2284
#define ereport(elevel, rest)
Definition: elog.h:122
void WalSndSetState(WalSndState state)
Definition: walsender.c:2655
int errmsg(const char *fmt,...)
Definition: elog.c:797
int FreeDir(DIR *dir)
Definition: fd.c:2393
static int64 sendDir ( char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks 
)
static

Definition at line 954 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), lfirst, lstat, MAXPGPATH, NULL, PG_TEMP_FILE_PREFIX, ReadDir(), RecoveryInProgress(), tablespaceinfo::rpath, sendFile(), snprintf(), statrelpath, and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

956 {
957  DIR *dir;
958  struct dirent *de;
959  char pathbuf[MAXPGPATH];
960  struct stat statbuf;
961  int64 size = 0;
962 
963  dir = AllocateDir(path);
964  while ((de = ReadDir(dir, path)) != NULL)
965  {
966  int excludeIdx;
967  bool excludeFound;
968 
969  /* Skip special stuff */
970  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
971  continue;
972 
973  /* Skip temporary files */
974  if (strncmp(de->d_name,
976  strlen(PG_TEMP_FILE_PREFIX)) == 0)
977  continue;
978 
979  /*
980  * Check if the postmaster has signaled us to exit, and abort with an
981  * error in that case. The error handler further up will call
982  * do_pg_abort_backup() for us. Also check that if the backup was
983  * started while still in recovery, the server wasn't promoted.
984  * dp_pg_stop_backup() will check that too, but it's better to stop
985  * the backup early than continue to the end and fail there.
986  */
989  ereport(ERROR,
990  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
991  errmsg("the standby was promoted during online backup"),
992  errhint("This means that the backup being taken is corrupt "
993  "and should not be used. "
994  "Try taking another online backup.")));
995 
996  /* Scan for files that should be excluded */
997  excludeFound = false;
998  for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
999  {
1000  if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1001  {
1002  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1003  excludeFound = true;
1004  break;
1005  }
1006  }
1007 
1008  if (excludeFound)
1009  continue;
1010 
1011  snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name);
1012 
1013  /* Skip pg_control here to back up it last */
1014  if (strcmp(pathbuf, "./global/pg_control") == 0)
1015  continue;
1016 
1017  if (lstat(pathbuf, &statbuf) != 0)
1018  {
1019  if (errno != ENOENT)
1020  ereport(ERROR,
1022  errmsg("could not stat file or directory \"%s\": %m",
1023  pathbuf)));
1024 
1025  /* If the file went away while scanning, it's not an error. */
1026  continue;
1027  }
1028 
1029  /* Scan for directories whose contents should be excluded */
1030  excludeFound = false;
1031  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1032  {
1033  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1034  {
1035  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1036  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1037  excludeFound = true;
1038  break;
1039  }
1040  }
1041 
1042  if (excludeFound)
1043  continue;
1044 
1045  /*
1046  * Exclude contents of directory specified by statrelpath if not set
1047  * to the default (pg_stat_tmp) which is caught in the loop above.
1048  */
1049  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1050  {
1051  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1052  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1053  continue;
1054  }
1055 
1056  /*
1057  * We can skip pg_wal, the WAL segments need to be fetched from the
1058  * WAL archive anyway. But include it as an empty directory anyway, so
1059  * we get permissions right.
1060  */
1061  if (strcmp(pathbuf, "./pg_wal") == 0)
1062  {
1063  /* If pg_wal is a symlink, write it as a directory anyway */
1064  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1065 
1066  /*
1067  * Also send archive_status directory (by hackishly reusing
1068  * statbuf from above ...).
1069  */
1070  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1071  sizeonly);
1072 
1073  continue; /* don't recurse into pg_wal */
1074  }
1075 
1076  /* Allow symbolic links in pg_tblspc only */
1077  if (strcmp(path, "./pg_tblspc") == 0 &&
1078 #ifndef WIN32
1079  S_ISLNK(statbuf.st_mode)
1080 #else
1081  pgwin32_is_junction(pathbuf)
1082 #endif
1083  )
1084  {
1085 #if defined(HAVE_READLINK) || defined(WIN32)
1086  char linkpath[MAXPGPATH];
1087  int rllen;
1088 
1089  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1090  if (rllen < 0)
1091  ereport(ERROR,
1093  errmsg("could not read symbolic link \"%s\": %m",
1094  pathbuf)));
1095  if (rllen >= sizeof(linkpath))
1096  ereport(ERROR,
1097  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1098  errmsg("symbolic link \"%s\" target is too long",
1099  pathbuf)));
1100  linkpath[rllen] = '\0';
1101 
1102  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1103  &statbuf, sizeonly);
1104 #else
1105 
1106  /*
1107  * If the platform does not have symbolic links, it should not be
1108  * possible to have tablespaces - clearly somebody else created
1109  * them. Warn about it and ignore.
1110  */
1111  ereport(WARNING,
1112  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1113  errmsg("tablespaces are not supported on this platform")));
1114  continue;
1115 #endif /* HAVE_READLINK */
1116  }
1117  else if (S_ISDIR(statbuf.st_mode))
1118  {
1119  bool skip_this_dir = false;
1120  ListCell *lc;
1121 
1122  /*
1123  * Store a directory entry in the tar file so we can get the
1124  * permissions right.
1125  */
1126  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1127  sizeonly);
1128 
1129  /*
1130  * Call ourselves recursively for a directory, unless it happens
1131  * to be a separate tablespace located within PGDATA.
1132  */
1133  foreach(lc, tablespaces)
1134  {
1135  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1136 
1137  /*
1138  * ti->rpath is the tablespace relative path within PGDATA, or
1139  * NULL if the tablespace has been properly located somewhere
1140  * else.
1141  *
1142  * Skip past the leading "./" in pathbuf when comparing.
1143  */
1144  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1145  {
1146  skip_this_dir = true;
1147  break;
1148  }
1149  }
1150 
1151  /*
1152  * skip sending directories inside pg_tblspc, if not required.
1153  */
1154  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1155  skip_this_dir = true;
1156 
1157  if (!skip_this_dir)
1158  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1159  }
1160  else if (S_ISREG(statbuf.st_mode))
1161  {
1162  bool sent = false;
1163 
1164  if (!sizeonly)
1165  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1166  true);
1167 
1168  if (sent || sizeonly)
1169  {
1170  /* Add size, rounded up to 512byte block */
1171  size += ((statbuf.st_size + 511) & ~511);
1172  size += 512; /* Size of the header of the file */
1173  }
1174  }
1175  else
1176  ereport(WARNING,
1177  (errmsg("skipping special file \"%s\"", pathbuf)));
1178  }
1179  FreeDir(dir);
1180  return size;
1181 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:954
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1199
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:987
char * rpath
Definition: basebackup.h:28
int errcode(int sqlerrcode)
Definition: elog.c:575
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define PG_TEMP_FILE_PREFIX
Definition: fd.h:127
bool RecoveryInProgress(void)
Definition: xlog.c:7805
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2284
static const char * excludeFiles[]
Definition: basebackup.c:146
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
#define WARNING
Definition: elog.h:40
#define NULL
Definition: c.h:226
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2350
int errmsg(const char *fmt,...)
Definition: elog.c:797
static const char * excludeDirContents[]
Definition: basebackup.c:105
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
char d_name[MAX_PATH]
Definition: dirent.h:14
#define elog
Definition: elog.h:219
#define lstat(path, sb)
Definition: win32.h:272
int FreeDir(DIR *dir)
Definition: fd.c:2393
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1314
static bool sendFile ( char *  readfilename,
char *  tarfilename,
struct stat *  statbuf,
bool  missing_ok 
)
static

Definition at line 1199 of file basebackup.c.

References _tarWriteHeader(), AllocateFile(), buf, ereport, errcode_for_file_access(), errmsg(), ERROR, FreeFile(), MemSet, Min, NULL, pgoff_t, pq_putmessage, TAR_SEND_SIZE, and throttle().

Referenced by perform_base_backup(), and sendDir().

1201 {
1202  FILE *fp;
1203  char buf[TAR_SEND_SIZE];
1204  size_t cnt;
1205  pgoff_t len = 0;
1206  size_t pad;
1207 
1208  fp = AllocateFile(readfilename, "rb");
1209  if (fp == NULL)
1210  {
1211  if (errno == ENOENT && missing_ok)
1212  return false;
1213  ereport(ERROR,
1215  errmsg("could not open file \"%s\": %m", readfilename)));
1216  }
1217 
1218  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1219 
1220  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1221  {
1222  /* Send the chunk as a CopyData message */
1223  if (pq_putmessage('d', buf, cnt))
1224  ereport(ERROR,
1225  (errmsg("base backup could not send data, aborting backup")));
1226 
1227  len += cnt;
1228  throttle(cnt);
1229 
1230  if (len >= statbuf->st_size)
1231  {
1232  /*
1233  * Reached end of file. The file could be longer, if it was
1234  * extended while we were sending it, but for a base backup we can
1235  * ignore such extended data. It will be restored from WAL.
1236  */
1237  break;
1238  }
1239  }
1240 
1241  /* If the file was truncated while we were sending it, pad it with zeros */
1242  if (len < statbuf->st_size)
1243  {
1244  MemSet(buf, 0, sizeof(buf));
1245  while (len < statbuf->st_size)
1246  {
1247  cnt = Min(sizeof(buf), statbuf->st_size - len);
1248  pq_putmessage('d', buf, cnt);
1249  len += cnt;
1250  throttle(cnt);
1251  }
1252  }
1253 
1254  /*
1255  * Pad to 512 byte boundary, per tar format requirements. (This small
1256  * piece of data is probably not worth throttling.)
1257  */
1258  pad = ((len + 511) & ~511) - len;
1259  if (pad > 0)
1260  {
1261  MemSet(buf, 0, pad);
1262  pq_putmessage('d', buf, pad);
1263  }
1264 
1265  FreeFile(fp);
1266 
1267  return true;
1268 }
static void throttle(size_t increment)
Definition: basebackup.c:1334
#define Min(x, y)
Definition: c.h:802
#define MemSet(start, val, len)
Definition: c.h:853
#define ERROR
Definition: elog.h:43
static char * buf
Definition: pg_test_fsync.c:65
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2043
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
#define ereport(elevel, rest)
Definition: elog.h:122
#define pgoff_t
Definition: win32.h:241
#define NULL
Definition: c.h:226
int FreeFile(FILE *file)
Definition: fd.c:2226
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define TAR_SEND_SIZE
Definition: basebackup.c:81
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:43
static void sendFileWithContent ( const char *  filename,
const char *  content 
)
static

Definition at line 856 of file basebackup.c.

References _tarWriteHeader(), buf, MemSet, NULL, and pq_putmessage.

Referenced by perform_base_backup().

857 {
858  struct stat statbuf;
859  int pad,
860  len;
861 
862  len = strlen(content);
863 
864  /*
865  * Construct a stat struct for the backup_label file we're injecting in
866  * the tar.
867  */
868  /* Windows doesn't have the concept of uid and gid */
869 #ifdef WIN32
870  statbuf.st_uid = 0;
871  statbuf.st_gid = 0;
872 #else
873  statbuf.st_uid = geteuid();
874  statbuf.st_gid = getegid();
875 #endif
876  statbuf.st_mtime = time(NULL);
877  statbuf.st_mode = S_IRUSR | S_IWUSR;
878  statbuf.st_size = len;
879 
880  _tarWriteHeader(filename, NULL, &statbuf, false);
881  /* Send the contents as a CopyData message */
882  pq_putmessage('d', content, len);
883 
884  /* Pad to 512 byte boundary, per tar format requirements */
885  pad = ((len + 511) & ~511) - len;
886  if (pad > 0)
887  {
888  char buf[512];
889 
890  MemSet(buf, 0, pad);
891  pq_putmessage('d', buf, pad);
892  }
893 }
#define MemSet(start, val, len)
Definition: c.h:853
static char * buf
Definition: pg_test_fsync.c:65
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
#define NULL
Definition: c.h:226
static char * filename
Definition: pg_dumpall.c:84
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:43
int64 sendTablespace ( char *  path,
bool  sizeonly 
)

Definition at line 903 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, NULL, sendDir(), snprintf(), and TABLESPACE_VERSION_DIRECTORY.

Referenced by do_pg_start_backup(), and perform_base_backup().

904 {
905  int64 size;
906  char pathbuf[MAXPGPATH];
907  struct stat statbuf;
908 
909  /*
910  * 'path' points to the tablespace location, but we only want to include
911  * the version directory in it that belongs to us.
912  */
913  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
915 
916  /*
917  * Store a directory entry in the tar file so we get the permissions
918  * right.
919  */
920  if (lstat(pathbuf, &statbuf) != 0)
921  {
922  if (errno != ENOENT)
923  ereport(ERROR,
925  errmsg("could not stat file or directory \"%s\": %m",
926  pathbuf)));
927 
928  /* If the tablespace went away while scanning, it's no error. */
929  return 0;
930  }
931 
933  sizeonly);
934 
935  /* Send all the files in the tablespace version directory */
936  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
937 
938  return size;
939 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:954
#define NIL
Definition: pg_list.h:69
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
#define ereport(elevel, rest)
Definition: elog.h:122
#define NULL
Definition: c.h:226
#define TABLESPACE_VERSION_DIRECTORY
Definition: catalog.h:26
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define lstat(path, sb)
Definition: win32.h:272
static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 801 of file basebackup.c.

References buf, INT8OID, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint(), pq_sendstring(), snprintf(), and TEXTOID.

Referenced by perform_base_backup().

802 {
804  char str[MAXFNAMELEN];
805  Size len;
806 
807  pq_beginmessage(&buf, 'T'); /* RowDescription */
808  pq_sendint(&buf, 2, 2); /* 2 fields */
809 
810  /* Field headers */
811  pq_sendstring(&buf, "recptr");
812  pq_sendint(&buf, 0, 4); /* table oid */
813  pq_sendint(&buf, 0, 2); /* attnum */
814  pq_sendint(&buf, TEXTOID, 4); /* type oid */
815  pq_sendint(&buf, -1, 2);
816  pq_sendint(&buf, 0, 4);
817  pq_sendint(&buf, 0, 2);
818 
819  pq_sendstring(&buf, "tli");
820  pq_sendint(&buf, 0, 4); /* table oid */
821  pq_sendint(&buf, 0, 2); /* attnum */
822 
823  /*
824  * int8 may seem like a surprising data type for this, but in theory int4
825  * would not be wide enough for this, as TimeLineID is unsigned.
826  */
827  pq_sendint(&buf, INT8OID, 4); /* type oid */
828  pq_sendint(&buf, -1, 2);
829  pq_sendint(&buf, 0, 4);
830  pq_sendint(&buf, 0, 2);
831  pq_endmessage(&buf);
832 
833  /* Data row */
834  pq_beginmessage(&buf, 'D');
835  pq_sendint(&buf, 2, 2); /* number of columns */
836 
837  len = snprintf(str, sizeof(str),
838  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
839  pq_sendint(&buf, len, 4);
840  pq_sendbytes(&buf, str, len);
841 
842  len = snprintf(str, sizeof(str), "%u", tli);
843  pq_sendint(&buf, len, 4);
844  pq_sendbytes(&buf, str, len);
845 
846  pq_endmessage(&buf);
847 
848  /* Send a CommandComplete message */
849  pq_puttextmessage('C', "SELECT");
850 }
#define TEXTOID
Definition: pg_type.h:324
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:186
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
static char * buf
Definition: pg_test_fsync.c:65
unsigned int uint32
Definition: c.h:265
#define MAXFNAMELEN
#define INT8OID
Definition: pg_type.h:304
size_t Size
Definition: c.h:353
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:400
static void throttle ( size_t  increment)
static

Definition at line 1334 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_LATCH_SET, WL_POSTMASTER_DEATH, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1335 {
1336  TimeOffset elapsed,
1337  elapsed_min,
1338  sleep;
1339  int wait_result;
1340 
1341  if (throttling_counter < 0)
1342  return;
1343 
1344  throttling_counter += increment;
1346  return;
1347 
1348  /* Time elapsed since the last measurement (and possible wake up). */
1349  elapsed = GetCurrentTimestamp() - throttled_last;
1350  /* How much should have elapsed at minimum? */
1352  sleep = elapsed_min - elapsed;
1353  /* Only sleep if the transfer is faster than it should be. */
1354  if (sleep > 0)
1355  {
1357 
1358  /* We're eating a potentially set latch, so check for interrupts */
1360 
1361  /*
1362  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1363  * the maximum time to sleep. Thus the cast to long is safe.
1364  */
1365  wait_result = WaitLatch(MyLatch,
1367  (long) (sleep / 1000),
1369 
1370  if (wait_result & WL_LATCH_SET)
1372  }
1373 
1374  /*
1375  * As we work with integers, only whole multiple of throttling_sample was
1376  * processed. The rest will be done during the next call of this function.
1377  */
1379 
1380  /*
1381  * Time interval for the remaining amount and possible next increments
1382  * starts now.
1383  */
1385 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1569
void ResetLatch(volatile Latch *latch)
Definition: latch.c:462
int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:301
int64 TimeOffset
Definition: timestamp.h:40
#define WL_POSTMASTER_DEATH
Definition: latch.h:128
static TimestampTz throttled_last
Definition: basebackup.c:98
struct Latch * MyLatch
Definition: globals.c:51
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
static int64 throttling_counter
Definition: basebackup.c:92
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:89
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95

Variable Documentation

bool backup_started_in_recovery = false
static

Definition at line 73 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

TimeOffset elapsed_min_unit
static

Definition at line 95 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

const char* excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define NULL
Definition: c.h:226
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 105 of file basebackup.c.

Referenced by sendDir().

const char* excludeFiles[]
static
Initial value:
=
{
"postmaster.pid",
"postmaster.opts",
}
#define TABLESPACE_MAP
Definition: xlog.h:304
#define NULL
Definition: c.h:226
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define BACKUP_LABEL_FILE
Definition: xlog.h:301

Definition at line 146 of file basebackup.c.

Referenced by sendDir().

char* statrelpath = NULL
static

Definition at line 76 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

TimestampTz throttled_last
static

Definition at line 98 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

int64 throttling_counter
static

Definition at line 92 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

uint64 throttling_sample
static

Definition at line 89 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().