PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/pg_type.h"
#include "common/file_perm.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 
#define CHECK_FREAD_ERROR(fp, filename)
 

Functions

static int64 sendDir (const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
 
static bool sendFile (const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid)
 
static void sendFileWithContent (const char *filename, const char *content)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void perform_base_backup (basebackup_options *opt)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static void throttle (size_t increment)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 
int64 sendTablespace (char *path, bool sizeonly)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static XLogRecPtr startptr
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static const char *const excludeDirContents []
 
static const char *const excludeFiles []
 
static const char *const noChecksumFiles []
 

Macro Definition Documentation

◆ CHECK_FREAD_ERROR

#define CHECK_FREAD_ERROR (   fp,
  filename 
)
Value:
do { \
if (ferror(fp)) \
ereport(ERROR, \
(errmsg("could not read from file \"%s\"", filename))); \
} while (0)
#define ERROR
Definition: elog.h:43
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:822

Definition at line 96 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ TAR_SEND_SIZE

#define TAR_SEND_SIZE   32768

Definition at line 84 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ THROTTLING_FREQUENCY

#define THROTTLING_FREQUENCY   8

Definition at line 89 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

◆ _tarWriteDir()

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1646 of file basebackup.c.

References _tarWriteHeader(), pg_dir_create_mode, and pgwin32_is_junction().

Referenced by sendDir().

1648 {
1649  /* If symlink, write it as a directory anyway */
1650 #ifndef WIN32
1651  if (S_ISLNK(statbuf->st_mode))
1652 #else
1653  if (pgwin32_is_junction(pathbuf))
1654 #endif
1655  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1656 
1657  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1658 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
int pg_dir_create_mode
Definition: file_perm.c:18
bool pgwin32_is_junction(const char *path)

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1604 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1606 {
1607  char h[512];
1608  enum tarError rc;
1609 
1610  if (!sizeonly)
1611  {
1612  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1613  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1614  statbuf->st_mtime);
1615 
1616  switch (rc)
1617  {
1618  case TAR_OK:
1619  break;
1620  case TAR_NAME_TOO_LONG:
1621  ereport(ERROR,
1622  (errmsg("file name too long for tar format: \"%s\"",
1623  filename)));
1624  break;
1625  case TAR_SYMLINK_TOO_LONG:
1626  ereport(ERROR,
1627  (errmsg("symbolic link target too long for tar format: "
1628  "file name \"%s\", target \"%s\"",
1629  filename, linktarget)));
1630  break;
1631  default:
1632  elog(ERROR, "unrecognized tar error: %d", rc);
1633  }
1634 
1635  pq_putmessage('d', h, sizeof(h));
1636  }
1637 
1638  return sizeof(h);
1639 }
Definition: pgtar.h:17
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define ereport(elevel, rest)
Definition: elog.h:141
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define elog(elevel,...)
Definition: elog.h:228
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 619 of file basebackup.c.

References lfirst.

Referenced by perform_base_backup().

620 {
621  char *fna = (char *) lfirst(a);
622  char *fnb = (char *) lfirst(b);
623 
624  return strcmp(fna + 8, fnb + 8);
625 }
#define lfirst(lc)
Definition: pg_list.h:190

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1318 of file basebackup.c.

References noChecksumFiles.

Referenced by sendFile().

1319 {
1320  const char *const *f;
1321 
1322  /* Check that the file is in a tablespace */
1323  if (strncmp(fullpath, "./global/", 9) == 0 ||
1324  strncmp(fullpath, "./base/", 7) == 0 ||
1325  strncmp(fullpath, "/", 1) == 0)
1326  {
1327  /* Compare file against noChecksumFiles skiplist */
1328  for (f = noChecksumFiles; *f; f++)
1329  if (strcmp(*f, filename) == 0)
1330  return false;
1331 
1332  return true;
1333  }
1334  else
1335  return false;
1336 }
static char * filename
Definition: pg_dumpall.c:90
static const char *const noChecksumFiles[]
Definition: basebackup.c:206

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 631 of file basebackup.c.

References DefElem::arg, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

632 {
633  ListCell *lopt;
634  bool o_label = false;
635  bool o_progress = false;
636  bool o_fast = false;
637  bool o_nowait = false;
638  bool o_wal = false;
639  bool o_maxrate = false;
640  bool o_tablespace_map = false;
641  bool o_noverify_checksums = false;
642 
643  MemSet(opt, 0, sizeof(*opt));
644  foreach(lopt, options)
645  {
646  DefElem *defel = (DefElem *) lfirst(lopt);
647 
648  if (strcmp(defel->defname, "label") == 0)
649  {
650  if (o_label)
651  ereport(ERROR,
652  (errcode(ERRCODE_SYNTAX_ERROR),
653  errmsg("duplicate option \"%s\"", defel->defname)));
654  opt->label = strVal(defel->arg);
655  o_label = true;
656  }
657  else if (strcmp(defel->defname, "progress") == 0)
658  {
659  if (o_progress)
660  ereport(ERROR,
661  (errcode(ERRCODE_SYNTAX_ERROR),
662  errmsg("duplicate option \"%s\"", defel->defname)));
663  opt->progress = true;
664  o_progress = true;
665  }
666  else if (strcmp(defel->defname, "fast") == 0)
667  {
668  if (o_fast)
669  ereport(ERROR,
670  (errcode(ERRCODE_SYNTAX_ERROR),
671  errmsg("duplicate option \"%s\"", defel->defname)));
672  opt->fastcheckpoint = true;
673  o_fast = true;
674  }
675  else if (strcmp(defel->defname, "nowait") == 0)
676  {
677  if (o_nowait)
678  ereport(ERROR,
679  (errcode(ERRCODE_SYNTAX_ERROR),
680  errmsg("duplicate option \"%s\"", defel->defname)));
681  opt->nowait = true;
682  o_nowait = true;
683  }
684  else if (strcmp(defel->defname, "wal") == 0)
685  {
686  if (o_wal)
687  ereport(ERROR,
688  (errcode(ERRCODE_SYNTAX_ERROR),
689  errmsg("duplicate option \"%s\"", defel->defname)));
690  opt->includewal = true;
691  o_wal = true;
692  }
693  else if (strcmp(defel->defname, "max_rate") == 0)
694  {
695  long maxrate;
696 
697  if (o_maxrate)
698  ereport(ERROR,
699  (errcode(ERRCODE_SYNTAX_ERROR),
700  errmsg("duplicate option \"%s\"", defel->defname)));
701 
702  maxrate = intVal(defel->arg);
703  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
704  ereport(ERROR,
705  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
706  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
707  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
708 
709  opt->maxrate = (uint32) maxrate;
710  o_maxrate = true;
711  }
712  else if (strcmp(defel->defname, "tablespace_map") == 0)
713  {
714  if (o_tablespace_map)
715  ereport(ERROR,
716  (errcode(ERRCODE_SYNTAX_ERROR),
717  errmsg("duplicate option \"%s\"", defel->defname)));
718  opt->sendtblspcmapfile = true;
719  o_tablespace_map = true;
720  }
721  else if (strcmp(defel->defname, "noverify_checksums") == 0)
722  {
723  if (o_noverify_checksums)
724  ereport(ERROR,
725  (errcode(ERRCODE_SYNTAX_ERROR),
726  errmsg("duplicate option \"%s\"", defel->defname)));
727  noverify_checksums = true;
728  o_noverify_checksums = true;
729  }
730  else
731  elog(ERROR, "option \"%s\" not recognized",
732  defel->defname);
733  }
734  if (opt->label == NULL)
735  opt->label = "base backup";
736 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:608
#define MemSet(start, val, len)
Definition: c.h:962
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:43
static bool noverify_checksums
Definition: basebackup.c:122
static int32 maxrate
const char * label
Definition: basebackup.c:47
unsigned int uint32
Definition: c.h:359
#define ereport(elevel, rest)
Definition: elog.h:141
Node * arg
Definition: parsenodes.h:731
#define lfirst(lc)
Definition: pg_list.h:190
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define elog(elevel,...)
Definition: elog.h:228
char * defname
Definition: parsenodes.h:730

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt)
static

Definition at line 225 of file basebackup.c.

References _tarWriteHeader(), AllocateDir(), AllocateFile(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, BoolGetDatum, buf, CHECK_FREAD_ERROR, CheckXLogRemoved(), compareWalFileNames(), dirent::d_name, StringInfoData::data, DataDir, do_pg_abort_backup(), do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, FreeDir(), FreeFile(), GetCurrentTimestamp(), basebackup_options::includewal, InvalidOid, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, linitial, list_sort(), lnext(), lstat, makeStringInfo(), MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, palloc0(), tablespaceinfo::path, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint16(), basebackup_options::progress, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), SendBackupHeader(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf, startptr, stat, statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, total_checksum_failures, USECS_PER_SEC, wal_segment_size, WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

226 {
227  TimeLineID starttli;
228  XLogRecPtr endptr;
229  TimeLineID endtli;
230  StringInfo labelfile;
232  int datadirpathlen;
233  List *tablespaces = NIL;
234 
235  datadirpathlen = strlen(DataDir);
236 
238 
239  labelfile = makeStringInfo();
240  tblspc_map_file = makeStringInfo();
241 
243 
244  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
245  labelfile, &tablespaces,
246  tblspc_map_file,
247  opt->progress, opt->sendtblspcmapfile);
248 
249  /*
250  * Once do_pg_start_backup has been called, ensure that any failure causes
251  * us to abort the backup so we don't "leak" a backup counter. For this
252  * reason, *all* functionality between do_pg_start_backup() and the end of
253  * do_pg_stop_backup() should be inside the error cleanup block!
254  */
255 
257  {
258  ListCell *lc;
259  tablespaceinfo *ti;
260 
261  SendXlogRecPtrResult(startptr, starttli);
262 
263  /*
264  * Calculate the relative path of temporary statistics directory in
265  * order to skip the files which are located in that directory later.
266  */
268  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
269  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
270  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
272  else
274 
275  /* Add a node for the base directory at the end */
276  ti = palloc0(sizeof(tablespaceinfo));
277  ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
278  tablespaces = lappend(tablespaces, ti);
279 
280  /* Send tablespace header */
281  SendBackupHeader(tablespaces);
282 
283  /* Setup and activate network throttling, if client requested it */
284  if (opt->maxrate > 0)
285  {
287  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
288 
289  /*
290  * The minimum amount of time for throttling_sample bytes to be
291  * transferred.
292  */
294 
295  /* Enable throttling. */
296  throttling_counter = 0;
297 
298  /* The 'real data' starts now (header was ignored). */
300  }
301  else
302  {
303  /* Disable throttling. */
304  throttling_counter = -1;
305  }
306 
307  /* Send off our tablespaces one by one */
308  foreach(lc, tablespaces)
309  {
310  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
312 
313  /* Send CopyOutResponse message */
314  pq_beginmessage(&buf, 'H');
315  pq_sendbyte(&buf, 0); /* overall format */
316  pq_sendint16(&buf, 0); /* natts */
317  pq_endmessage(&buf);
318 
319  if (ti->path == NULL)
320  {
321  struct stat statbuf;
322 
323  /* In the main tar, include the backup_label first... */
325 
326  /*
327  * Send tablespace_map file if required and then the bulk of
328  * the files.
329  */
330  if (tblspc_map_file && opt->sendtblspcmapfile)
331  {
332  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
333  sendDir(".", 1, false, tablespaces, false);
334  }
335  else
336  sendDir(".", 1, false, tablespaces, true);
337 
338  /* ... and pg_control after everything else. */
339  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
340  ereport(ERROR,
342  errmsg("could not stat file \"%s\": %m",
345  }
346  else
347  sendTablespace(ti->path, false);
348 
349  /*
350  * If we're including WAL, and this is the main data directory we
351  * don't terminate the tar stream here. Instead, we will append
352  * the xlog files below and terminate it then. This is safe since
353  * the main data directory is always sent *last*.
354  */
355  if (opt->includewal && ti->path == NULL)
356  {
357  Assert(lnext(tablespaces, lc) == NULL);
358  }
359  else
360  pq_putemptymessage('c'); /* CopyDone */
361  }
362 
363  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
364  }
366 
367 
368  if (opt->includewal)
369  {
370  /*
371  * We've left the last tar file "open", so we can now append the
372  * required WAL files to it.
373  */
374  char pathbuf[MAXPGPATH];
375  XLogSegNo segno;
376  XLogSegNo startsegno;
377  XLogSegNo endsegno;
378  struct stat statbuf;
379  List *historyFileList = NIL;
380  List *walFileList = NIL;
381  char firstoff[MAXFNAMELEN];
382  char lastoff[MAXFNAMELEN];
383  DIR *dir;
384  struct dirent *de;
385  ListCell *lc;
386  TimeLineID tli;
387 
388  /*
389  * I'd rather not worry about timelines here, so scan pg_wal and
390  * include all WAL files in the range between 'startptr' and 'endptr',
391  * regardless of the timeline the file is stamped with. If there are
392  * some spurious WAL files belonging to timelines that don't belong in
393  * this server's history, they will be included too. Normally there
394  * shouldn't be such files, but if there are, there's little harm in
395  * including them.
396  */
397  XLByteToSeg(startptr, startsegno, wal_segment_size);
398  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
399  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
400  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
401 
402  dir = AllocateDir("pg_wal");
403  while ((de = ReadDir(dir, "pg_wal")) != NULL)
404  {
405  /* Does it look like a WAL segment, and is it in the range? */
406  if (IsXLogFileName(de->d_name) &&
407  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
408  strcmp(de->d_name + 8, lastoff + 8) <= 0)
409  {
410  walFileList = lappend(walFileList, pstrdup(de->d_name));
411  }
412  /* Does it look like a timeline history file? */
413  else if (IsTLHistoryFileName(de->d_name))
414  {
415  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
416  }
417  }
418  FreeDir(dir);
419 
420  /*
421  * Before we go any further, check that none of the WAL segments we
422  * need were removed.
423  */
424  CheckXLogRemoved(startsegno, ThisTimeLineID);
425 
426  /*
427  * Sort the WAL filenames. We want to send the files in order from
428  * oldest to newest, to reduce the chance that a file is recycled
429  * before we get a chance to send it over.
430  */
431  list_sort(walFileList, compareWalFileNames);
432 
433  /*
434  * There must be at least one xlog file in the pg_wal directory, since
435  * we are doing backup-including-xlog.
436  */
437  if (walFileList == NIL)
438  ereport(ERROR,
439  (errmsg("could not find any WAL files")));
440 
441  /*
442  * Sanity check: the first and last segment should cover startptr and
443  * endptr, with no gaps in between.
444  */
445  XLogFromFileName((char *) linitial(walFileList),
446  &tli, &segno, wal_segment_size);
447  if (segno != startsegno)
448  {
449  char startfname[MAXFNAMELEN];
450 
451  XLogFileName(startfname, ThisTimeLineID, startsegno,
453  ereport(ERROR,
454  (errmsg("could not find WAL file \"%s\"", startfname)));
455  }
456  foreach(lc, walFileList)
457  {
458  char *walFileName = (char *) lfirst(lc);
459  XLogSegNo currsegno = segno;
460  XLogSegNo nextsegno = segno + 1;
461 
462  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
463  if (!(nextsegno == segno || currsegno == segno))
464  {
465  char nextfname[MAXFNAMELEN];
466 
467  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
469  ereport(ERROR,
470  (errmsg("could not find WAL file \"%s\"", nextfname)));
471  }
472  }
473  if (segno != endsegno)
474  {
475  char endfname[MAXFNAMELEN];
476 
477  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
478  ereport(ERROR,
479  (errmsg("could not find WAL file \"%s\"", endfname)));
480  }
481 
482  /* Ok, we have everything we need. Send the WAL files. */
483  foreach(lc, walFileList)
484  {
485  char *walFileName = (char *) lfirst(lc);
486  FILE *fp;
487  char buf[TAR_SEND_SIZE];
488  size_t cnt;
489  pgoff_t len = 0;
490 
491  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
492  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
493 
494  fp = AllocateFile(pathbuf, "rb");
495  if (fp == NULL)
496  {
497  int save_errno = errno;
498 
499  /*
500  * Most likely reason for this is that the file was already
501  * removed by a checkpoint, so check for that to get a better
502  * error message.
503  */
504  CheckXLogRemoved(segno, tli);
505 
506  errno = save_errno;
507  ereport(ERROR,
509  errmsg("could not open file \"%s\": %m", pathbuf)));
510  }
511 
512  if (fstat(fileno(fp), &statbuf) != 0)
513  ereport(ERROR,
515  errmsg("could not stat file \"%s\": %m",
516  pathbuf)));
517  if (statbuf.st_size != wal_segment_size)
518  {
519  CheckXLogRemoved(segno, tli);
520  ereport(ERROR,
522  errmsg("unexpected WAL file size \"%s\"", walFileName)));
523  }
524 
525  /* send the WAL file itself */
526  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
527 
528  while ((cnt = fread(buf, 1,
529  Min(sizeof(buf), wal_segment_size - len),
530  fp)) > 0)
531  {
532  CheckXLogRemoved(segno, tli);
533  /* Send the chunk as a CopyData message */
534  if (pq_putmessage('d', buf, cnt))
535  ereport(ERROR,
536  (errmsg("base backup could not send data, aborting backup")));
537 
538  len += cnt;
539  throttle(cnt);
540 
541  if (len == wal_segment_size)
542  break;
543  }
544 
545  CHECK_FREAD_ERROR(fp, pathbuf);
546 
547  if (len != wal_segment_size)
548  {
549  CheckXLogRemoved(segno, tli);
550  ereport(ERROR,
552  errmsg("unexpected WAL file size \"%s\"", walFileName)));
553  }
554 
555  /* wal_segment_size is a multiple of 512, so no need for padding */
556 
557  FreeFile(fp);
558 
559  /*
560  * Mark file as archived, otherwise files can get archived again
561  * after promotion of a new node. This is in line with
562  * walreceiver.c always doing an XLogArchiveForceDone() after a
563  * complete segment.
564  */
565  StatusFilePath(pathbuf, walFileName, ".done");
566  sendFileWithContent(pathbuf, "");
567  }
568 
569  /*
570  * Send timeline history files too. Only the latest timeline history
571  * file is required for recovery, and even that only if there happens
572  * to be a timeline switch in the first WAL segment that contains the
573  * checkpoint record, or if we're taking a base backup from a standby
574  * server and the target timeline changes while the backup is taken.
575  * But they are small and highly useful for debugging purposes, so
576  * better include them all, always.
577  */
578  foreach(lc, historyFileList)
579  {
580  char *fname = lfirst(lc);
581 
582  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
583 
584  if (lstat(pathbuf, &statbuf) != 0)
585  ereport(ERROR,
587  errmsg("could not stat file \"%s\": %m", pathbuf)));
588 
589  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid);
590 
591  /* unconditionally mark file as archived */
592  StatusFilePath(pathbuf, fname, ".done");
593  sendFileWithContent(pathbuf, "");
594  }
595 
596  /* Send CopyDone message for the last tar file */
597  pq_putemptymessage('c');
598  }
599  SendXlogRecPtrResult(endptr, endtli);
600 
602  {
603  if (total_checksum_failures > 1)
605  (errmsg("%lld total checksum verification failures", total_checksum_failures)));
606 
607  ereport(ERROR,
609  errmsg("checksum verification failure during base backup")));
610  }
611 
612 }
#define StatusFilePath(path, xlog, suffix)
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
static void throttle(size_t increment)
Definition: basebackup.c:1666
uint32 TimeLineID
Definition: xlogdefs.h:52
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11145
int wal_segment_size
Definition: xlog.c:112
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:321
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1583
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:911
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:778
int errcode(int sqlerrcode)
Definition: elog.c:608
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool RecoveryInProgress(void)
Definition: xlog.c:7930
static bool backup_started_in_recovery
Definition: basebackup.c:76
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10215
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
#define pgoff_t
Definition: win32_port.h:195
#define TABLESPACE_MAP
Definition: xlog.h:362
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define linitial(l)
Definition: pg_list.h:195
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:10734
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3859
#define MAXPGPATH
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
int errcode_for_file_access(void)
Definition: elog.c:631
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:47
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2242
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:856
int64 sendTablespace(char *path, bool sizeonly)
Definition: basebackup.c:958
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2503
char * pgstat_stat_directory
Definition: pgstat.c:133
#define ereport(elevel, rest)
Definition: elog.h:141
#define CHECK_FREAD_ERROR(fp, filename)
Definition: basebackup.c:96
static char * statrelpath
Definition: basebackup.c:79
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:322
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
#define MAXFNAMELEN
void * palloc0(Size size)
Definition: mcxt.c:980
#define XLOGDIR
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:187
static TimestampTz throttled_last
Definition: basebackup.c:113
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:739
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2569
#define THROTTLING_FREQUENCY
Definition: basebackup.c:89
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
#define lstat(path, sb)
Definition: win32_port.h:244
int FreeFile(FILE *file)
Definition: fd.c:2441
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:619
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:1009
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1482
int errmsg(const char *fmt,...)
Definition: elog.c:822
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:84
char * DataDir
Definition: globals.c:62
#define BACKUP_LABEL_FILE
Definition: xlog.h:359
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:107
Definition: pg_list.h:50
#define snprintf
Definition: port.h:192
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid)
Definition: basebackup.c:1357
static uint64 throttling_sample
Definition: basebackup.c:104
static void sendFileWithContent(const char *filename, const char *content)
Definition: basebackup.c:911
int FreeDir(DIR *dir)
Definition: fd.c:2621
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
static XLogRecPtr startptr
Definition: basebackup.c:116
static long long int total_checksum_failures
Definition: basebackup.c:119
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)

◆ send_int8_string()

static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 768 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), pq_sendint32(), and sprintf.

Referenced by SendBackupHeader().

769 {
770  char is[32];
771 
772  sprintf(is, INT64_FORMAT, intval);
773  pq_sendint32(buf, strlen(is));
774  pq_sendbytes(buf, is, strlen(is));
775 }
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:194
#define INT64_FORMAT
Definition: c.h:401
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125

◆ SendBackupHeader()

static void SendBackupHeader ( List tablespaces)
static

Definition at line 778 of file basebackup.c.

References buf, lfirst, tablespaceinfo::oid, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), send_int8_string(), and tablespaceinfo::size.

Referenced by perform_base_backup().

779 {
781  ListCell *lc;
782 
783  /* Construct and send the directory information */
784  pq_beginmessage(&buf, 'T'); /* RowDescription */
785  pq_sendint16(&buf, 3); /* 3 fields */
786 
787  /* First field - spcoid */
788  pq_sendstring(&buf, "spcoid");
789  pq_sendint32(&buf, 0); /* table oid */
790  pq_sendint16(&buf, 0); /* attnum */
791  pq_sendint32(&buf, OIDOID); /* type oid */
792  pq_sendint16(&buf, 4); /* typlen */
793  pq_sendint32(&buf, 0); /* typmod */
794  pq_sendint16(&buf, 0); /* format code */
795 
796  /* Second field - spclocation */
797  pq_sendstring(&buf, "spclocation");
798  pq_sendint32(&buf, 0);
799  pq_sendint16(&buf, 0);
800  pq_sendint32(&buf, TEXTOID);
801  pq_sendint16(&buf, -1);
802  pq_sendint32(&buf, 0);
803  pq_sendint16(&buf, 0);
804 
805  /* Third field - size */
806  pq_sendstring(&buf, "size");
807  pq_sendint32(&buf, 0);
808  pq_sendint16(&buf, 0);
809  pq_sendint32(&buf, INT8OID);
810  pq_sendint16(&buf, 8);
811  pq_sendint32(&buf, 0);
812  pq_sendint16(&buf, 0);
813  pq_endmessage(&buf);
814 
815  foreach(lc, tablespaces)
816  {
817  tablespaceinfo *ti = lfirst(lc);
818 
819  /* Send one datarow message */
820  pq_beginmessage(&buf, 'D');
821  pq_sendint16(&buf, 3); /* number of columns */
822  if (ti->path == NULL)
823  {
824  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
825  pq_sendint32(&buf, -1);
826  }
827  else
828  {
829  Size len;
830 
831  len = strlen(ti->oid);
832  pq_sendint32(&buf, len);
833  pq_sendbytes(&buf, ti->oid, len);
834 
835  len = strlen(ti->path);
836  pq_sendint32(&buf, len);
837  pq_sendbytes(&buf, ti->path, len);
838  }
839  if (ti->size >= 0)
840  send_int8_string(&buf, ti->size / 1024);
841  else
842  pq_sendint32(&buf, -1); /* NULL */
843 
844  pq_endmessage(&buf);
845  }
846 
847  /* Send a CommandComplete message */
848  pq_puttextmessage('C', "SELECT");
849 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:768
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
#define lfirst(lc)
Definition: pg_list.h:190
size_t Size
Definition: c.h:467
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 747 of file basebackup.c.

References basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

748 {
749  basebackup_options opt;
750 
751  parse_basebackup_options(cmd->options, &opt);
752 
754 
756  {
757  char activitymsg[50];
758 
759  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
760  opt.label);
761  set_ps_display(activitymsg, false);
762  }
763 
764  perform_base_backup(&opt);
765 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:35
void set_ps_display(const char *activity, bool force)
Definition: ps_status.c:335
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:631
const char * label
Definition: basebackup.c:47
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:225
void WalSndSetState(WalSndState state)
Definition: walsender.c:3137
#define snprintf
Definition: port.h:192

◆ sendDir()

static int64 sendDir ( const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks 
)
static

Definition at line 1009 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, DEBUG2, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, MAXPGPATH, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, pgwin32_is_junction(), ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISREG, sendFile(), snprintf, stat, statrelpath, TABLESPACE_VERSION_DIRECTORY, and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

1011 {
1012  DIR *dir;
1013  struct dirent *de;
1014  char pathbuf[MAXPGPATH * 2];
1015  struct stat statbuf;
1016  int64 size = 0;
1017  const char *lastDir; /* Split last dir from parent path. */
1018  bool isDbDir = false; /* Does this directory contain relations? */
1019 
1020  /*
1021  * Determine if the current path is a database directory that can contain
1022  * relations.
1023  *
1024  * Start by finding the location of the delimiter between the parent path
1025  * and the current path.
1026  */
1027  lastDir = last_dir_separator(path);
1028 
1029  /* Does this path look like a database path (i.e. all digits)? */
1030  if (lastDir != NULL &&
1031  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1032  {
1033  /* Part of path that contains the parent directory. */
1034  int parentPathLen = lastDir - path;
1035 
1036  /*
1037  * Mark path as a database directory if the parent path is either
1038  * $PGDATA/base or a tablespace version path.
1039  */
1040  if (strncmp(path, "./base", parentPathLen) == 0 ||
1041  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1042  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1044  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1045  isDbDir = true;
1046  }
1047 
1048  dir = AllocateDir(path);
1049  while ((de = ReadDir(dir, path)) != NULL)
1050  {
1051  int excludeIdx;
1052  bool excludeFound;
1053  ForkNumber relForkNum; /* Type of fork if file is a relation */
1054  int relOidChars; /* Chars in filename that are the rel oid */
1055 
1056  /* Skip special stuff */
1057  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1058  continue;
1059 
1060  /* Skip temporary files */
1061  if (strncmp(de->d_name,
1063  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1064  continue;
1065 
1066  /*
1067  * Check if the postmaster has signaled us to exit, and abort with an
1068  * error in that case. The error handler further up will call
1069  * do_pg_abort_backup() for us. Also check that if the backup was
1070  * started while still in recovery, the server wasn't promoted.
1071  * do_pg_stop_backup() will check that too, but it's better to stop
1072  * the backup early than continue to the end and fail there.
1073  */
1076  ereport(ERROR,
1077  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1078  errmsg("the standby was promoted during online backup"),
1079  errhint("This means that the backup being taken is corrupt "
1080  "and should not be used. "
1081  "Try taking another online backup.")));
1082 
1083  /* Scan for files that should be excluded */
1084  excludeFound = false;
1085  for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
1086  {
1087  if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1088  {
1089  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1090  excludeFound = true;
1091  break;
1092  }
1093  }
1094 
1095  if (excludeFound)
1096  continue;
1097 
1098  /* Exclude all forks for unlogged tables except the init fork */
1099  if (isDbDir &&
1100  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1101  &relForkNum))
1102  {
1103  /* Never exclude init forks */
1104  if (relForkNum != INIT_FORKNUM)
1105  {
1106  char initForkFile[MAXPGPATH];
1107  char relOid[OIDCHARS + 1];
1108 
1109  /*
1110  * If any other type of fork, check if there is an init fork
1111  * with the same OID. If so, the file can be excluded.
1112  */
1113  memcpy(relOid, de->d_name, relOidChars);
1114  relOid[relOidChars] = '\0';
1115  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1116  path, relOid);
1117 
1118  if (lstat(initForkFile, &statbuf) == 0)
1119  {
1120  elog(DEBUG2,
1121  "unlogged relation file \"%s\" excluded from backup",
1122  de->d_name);
1123 
1124  continue;
1125  }
1126  }
1127  }
1128 
1129  /* Exclude temporary relations */
1130  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1131  {
1132  elog(DEBUG2,
1133  "temporary relation file \"%s\" excluded from backup",
1134  de->d_name);
1135 
1136  continue;
1137  }
1138 
1139  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1140 
1141  /* Skip pg_control here to back up it last */
1142  if (strcmp(pathbuf, "./global/pg_control") == 0)
1143  continue;
1144 
1145  if (lstat(pathbuf, &statbuf) != 0)
1146  {
1147  if (errno != ENOENT)
1148  ereport(ERROR,
1150  errmsg("could not stat file or directory \"%s\": %m",
1151  pathbuf)));
1152 
1153  /* If the file went away while scanning, it's not an error. */
1154  continue;
1155  }
1156 
1157  /* Scan for directories whose contents should be excluded */
1158  excludeFound = false;
1159  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1160  {
1161  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1162  {
1163  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1164  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1165  excludeFound = true;
1166  break;
1167  }
1168  }
1169 
1170  if (excludeFound)
1171  continue;
1172 
1173  /*
1174  * Exclude contents of directory specified by statrelpath if not set
1175  * to the default (pg_stat_tmp) which is caught in the loop above.
1176  */
1177  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1178  {
1179  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1180  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1181  continue;
1182  }
1183 
1184  /*
1185  * We can skip pg_wal, the WAL segments need to be fetched from the
1186  * WAL archive anyway. But include it as an empty directory anyway, so
1187  * we get permissions right.
1188  */
1189  if (strcmp(pathbuf, "./pg_wal") == 0)
1190  {
1191  /* If pg_wal is a symlink, write it as a directory anyway */
1192  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1193 
1194  /*
1195  * Also send archive_status directory (by hackishly reusing
1196  * statbuf from above ...).
1197  */
1198  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1199  sizeonly);
1200 
1201  continue; /* don't recurse into pg_wal */
1202  }
1203 
1204  /* Allow symbolic links in pg_tblspc only */
1205  if (strcmp(path, "./pg_tblspc") == 0 &&
1206 #ifndef WIN32
1207  S_ISLNK(statbuf.st_mode)
1208 #else
1209  pgwin32_is_junction(pathbuf)
1210 #endif
1211  )
1212  {
1213 #if defined(HAVE_READLINK) || defined(WIN32)
1214  char linkpath[MAXPGPATH];
1215  int rllen;
1216 
1217  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1218  if (rllen < 0)
1219  ereport(ERROR,
1221  errmsg("could not read symbolic link \"%s\": %m",
1222  pathbuf)));
1223  if (rllen >= sizeof(linkpath))
1224  ereport(ERROR,
1225  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1226  errmsg("symbolic link \"%s\" target is too long",
1227  pathbuf)));
1228  linkpath[rllen] = '\0';
1229 
1230  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1231  &statbuf, sizeonly);
1232 #else
1233 
1234  /*
1235  * If the platform does not have symbolic links, it should not be
1236  * possible to have tablespaces - clearly somebody else created
1237  * them. Warn about it and ignore.
1238  */
1239  ereport(WARNING,
1240  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1241  errmsg("tablespaces are not supported on this platform")));
1242  continue;
1243 #endif /* HAVE_READLINK */
1244  }
1245  else if (S_ISDIR(statbuf.st_mode))
1246  {
1247  bool skip_this_dir = false;
1248  ListCell *lc;
1249 
1250  /*
1251  * Store a directory entry in the tar file so we can get the
1252  * permissions right.
1253  */
1254  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1255  sizeonly);
1256 
1257  /*
1258  * Call ourselves recursively for a directory, unless it happens
1259  * to be a separate tablespace located within PGDATA.
1260  */
1261  foreach(lc, tablespaces)
1262  {
1263  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1264 
1265  /*
1266  * ti->rpath is the tablespace relative path within PGDATA, or
1267  * NULL if the tablespace has been properly located somewhere
1268  * else.
1269  *
1270  * Skip past the leading "./" in pathbuf when comparing.
1271  */
1272  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1273  {
1274  skip_this_dir = true;
1275  break;
1276  }
1277  }
1278 
1279  /*
1280  * skip sending directories inside pg_tblspc, if not required.
1281  */
1282  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1283  skip_this_dir = true;
1284 
1285  if (!skip_this_dir)
1286  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1287  }
1288  else if (S_ISREG(statbuf.st_mode))
1289  {
1290  bool sent = false;
1291 
1292  if (!sizeonly)
1293  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1294  true, isDbDir ? atooid(lastDir + 1) : InvalidOid);
1295 
1296  if (sent || sizeonly)
1297  {
1298  /* Add size, rounded up to 512byte block */
1299  size += ((statbuf.st_size + 511) & ~511);
1300  size += 512; /* Size of the header of the file */
1301  }
1302  }
1303  else
1304  ereport(WARNING,
1305  (errmsg("skipping special file \"%s\"", pathbuf)));
1306  }
1307  FreeDir(dir);
1308  return size;
1309 }
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1069
char * rpath
Definition: basebackup.h:28
int errcode(int sqlerrcode)
Definition: elog.c:608
bool RecoveryInProgress(void)
Definition: xlog.c:7930
static bool backup_started_in_recovery
Definition: basebackup.c:76
Definition: dirent.h:9
static const char *const excludeDirContents[]
Definition: basebackup.c:132
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
#define readlink(path, buf, size)
Definition: win32_port.h:222
int errcode_for_file_access(void)
Definition: elog.c:631
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2503
#define atooid(x)
Definition: postgres_ext.h:42
#define ereport(elevel, rest)
Definition: elog.h:141
static char * statrelpath
Definition: basebackup.c:79
ForkNumber
Definition: relpath.h:40
static const char *const excludeFiles[]
Definition: basebackup.c:173
#define S_ISREG(m)
Definition: win32_port.h:299
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define InvalidOid
Definition: postgres_ext.h:36
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2569
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3101
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:374
#define S_ISDIR(m)
Definition: win32_port.h:296
#define lstat(path, sb)
Definition: win32_port.h:244
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:1009
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define elog(elevel,...)
Definition: elog.h:228
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
char d_name[MAX_PATH]
Definition: dirent.h:14
#define snprintf
Definition: port.h:192
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid)
Definition: basebackup.c:1357
#define OIDCHARS
Definition: relpath.h:30
int FreeDir(DIR *dir)
Definition: fd.c:2621
bool pgwin32_is_junction(const char *path)
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1646

◆ sendFile()

static bool sendFile ( const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid 
)
static

Definition at line 1357 of file basebackup.c.

References _tarWriteHeader(), AllocateFile(), Assert, buf, CHECK_FREAD_ERROR, DataChecksumsEnabled(), ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, filename, FreeFile(), i, is_checksummed_file(), last_dir_separator(), MemSet, Min, noverify_checksums, PageGetLSN, PageIsNew, PageHeaderData::pd_checksum, pg_checksum_page(), pgoff_t, pgstat_report_checksum_failures_in_db(), pq_putmessage, startptr, TAR_SEND_SIZE, throttle(), total_checksum_failures, and WARNING.

Referenced by perform_base_backup(), and sendDir().

1359 {
1360  FILE *fp;
1361  BlockNumber blkno = 0;
1362  bool block_retry = false;
1363  char buf[TAR_SEND_SIZE];
1364  uint16 checksum;
1365  int checksum_failures = 0;
1366  off_t cnt;
1367  int i;
1368  pgoff_t len = 0;
1369  char *page;
1370  size_t pad;
1371  PageHeader phdr;
1372  int segmentno = 0;
1373  char *segmentpath;
1374  bool verify_checksum = false;
1375 
1376  fp = AllocateFile(readfilename, "rb");
1377  if (fp == NULL)
1378  {
1379  if (errno == ENOENT && missing_ok)
1380  return false;
1381  ereport(ERROR,
1383  errmsg("could not open file \"%s\": %m", readfilename)));
1384  }
1385 
1386  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1387 
1389  {
1390  char *filename;
1391 
1392  /*
1393  * Get the filename (excluding path). As last_dir_separator()
1394  * includes the last directory separator, we chop that off by
1395  * incrementing the pointer.
1396  */
1397  filename = last_dir_separator(readfilename) + 1;
1398 
1399  if (is_checksummed_file(readfilename, filename))
1400  {
1401  verify_checksum = true;
1402 
1403  /*
1404  * Cut off at the segment boundary (".") to get the segment number
1405  * in order to mix it into the checksum.
1406  */
1407  segmentpath = strstr(filename, ".");
1408  if (segmentpath != NULL)
1409  {
1410  segmentno = atoi(segmentpath + 1);
1411  if (segmentno == 0)
1412  ereport(ERROR,
1413  (errmsg("invalid segment number %d in file \"%s\"",
1414  segmentno, filename)));
1415  }
1416  }
1417  }
1418 
1419  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1420  {
1421  /*
1422  * The checksums are verified at block level, so we iterate over the
1423  * buffer in chunks of BLCKSZ, after making sure that
1424  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1425  * BLCKSZ bytes.
1426  */
1427  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1428 
1429  if (verify_checksum && (cnt % BLCKSZ != 0))
1430  {
1431  ereport(WARNING,
1432  (errmsg("could not verify checksum in file \"%s\", block "
1433  "%d: read buffer size %d and page size %d "
1434  "differ",
1435  readfilename, blkno, (int) cnt, BLCKSZ)));
1436  verify_checksum = false;
1437  }
1438 
1439  if (verify_checksum)
1440  {
1441  for (i = 0; i < cnt / BLCKSZ; i++)
1442  {
1443  page = buf + BLCKSZ * i;
1444 
1445  /*
1446  * Only check pages which have not been modified since the
1447  * start of the base backup. Otherwise, they might have been
1448  * written only halfway and the checksum would not be valid.
1449  * However, replaying WAL would reinstate the correct page in
1450  * this case. We also skip completely new pages, since they
1451  * don't have a checksum yet.
1452  */
1453  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1454  {
1455  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1456  phdr = (PageHeader) page;
1457  if (phdr->pd_checksum != checksum)
1458  {
1459  /*
1460  * Retry the block on the first failure. It's
1461  * possible that we read the first 4K page of the
1462  * block just before postgres updated the entire block
1463  * so it ends up looking torn to us. We only need to
1464  * retry once because the LSN should be updated to
1465  * something we can ignore on the next pass. If the
1466  * error happens again then it is a true validation
1467  * failure.
1468  */
1469  if (block_retry == false)
1470  {
1471  /* Reread the failed block */
1472  if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
1473  {
1474  ereport(ERROR,
1476  errmsg("could not fseek in file \"%s\": %m",
1477  readfilename)));
1478  }
1479 
1480  if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
1481  {
1482  /*
1483  * If we hit end-of-file, a concurrent
1484  * truncation must have occurred, so break out
1485  * of this loop just as if the initial fread()
1486  * returned 0. We'll drop through to the same
1487  * code that handles that case. (We must fix
1488  * up cnt first, though.)
1489  */
1490  if (feof(fp))
1491  {
1492  cnt = BLCKSZ * i;
1493  break;
1494  }
1495 
1496  ereport(ERROR,
1498  errmsg("could not reread block %d of file \"%s\": %m",
1499  blkno, readfilename)));
1500  }
1501 
1502  if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
1503  {
1504  ereport(ERROR,
1506  errmsg("could not fseek in file \"%s\": %m",
1507  readfilename)));
1508  }
1509 
1510  /* Set flag so we know a retry was attempted */
1511  block_retry = true;
1512 
1513  /* Reset loop to validate the block again */
1514  i--;
1515  continue;
1516  }
1517 
1518  checksum_failures++;
1519 
1520  if (checksum_failures <= 5)
1521  ereport(WARNING,
1522  (errmsg("checksum verification failed in "
1523  "file \"%s\", block %d: calculated "
1524  "%X but expected %X",
1525  readfilename, blkno, checksum,
1526  phdr->pd_checksum)));
1527  if (checksum_failures == 5)
1528  ereport(WARNING,
1529  (errmsg("further checksum verification "
1530  "failures in file \"%s\" will not "
1531  "be reported", readfilename)));
1532  }
1533  }
1534  block_retry = false;
1535  blkno++;
1536  }
1537  }
1538 
1539  /* Send the chunk as a CopyData message */
1540  if (pq_putmessage('d', buf, cnt))
1541  ereport(ERROR,
1542  (errmsg("base backup could not send data, aborting backup")));
1543 
1544  len += cnt;
1545  throttle(cnt);
1546 
1547  if (feof(fp) || len >= statbuf->st_size)
1548  {
1549  /*
1550  * Reached end of file. The file could be longer, if it was
1551  * extended while we were sending it, but for a base backup we can
1552  * ignore such extended data. It will be restored from WAL.
1553  */
1554  break;
1555  }
1556  }
1557 
1558  CHECK_FREAD_ERROR(fp, readfilename);
1559 
1560  /* If the file was truncated while we were sending it, pad it with zeros */
1561  if (len < statbuf->st_size)
1562  {
1563  MemSet(buf, 0, sizeof(buf));
1564  while (len < statbuf->st_size)
1565  {
1566  cnt = Min(sizeof(buf), statbuf->st_size - len);
1567  pq_putmessage('d', buf, cnt);
1568  len += cnt;
1569  throttle(cnt);
1570  }
1571  }
1572 
1573  /*
1574  * Pad to 512 byte boundary, per tar format requirements. (This small
1575  * piece of data is probably not worth throttling.)
1576  */
1577  pad = ((len + 511) & ~511) - len;
1578  if (pad > 0)
1579  {
1580  MemSet(buf, 0, pad);
1581  pq_putmessage('d', buf, pad);
1582  }
1583 
1584  FreeFile(fp);
1585 
1586  if (checksum_failures > 1)
1587  {
1588  ereport(WARNING,
1589  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1590  "file \"%s\" has a total of %d checksum verification failures",
1591  checksum_failures,
1592  readfilename, checksum_failures)));
1593 
1594  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1595  }
1596 
1597  total_checksum_failures += checksum_failures;
1598 
1599  return true;
1600 }
static void throttle(size_t increment)
Definition: basebackup.c:1666
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:932
bool DataChecksumsEnabled(void)
Definition: xlog.c:4818
#define Min(x, y)
Definition: c.h:911
#define MemSet(start, val, len)
Definition: c.h:962
uint32 BlockNumber
Definition: block.h:31
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1525
uint16 pd_checksum
Definition: bufpage.h:156
#define pgoff_t
Definition: win32_port.h:195
unsigned short uint16
Definition: c.h:358
#define ERROR
Definition: elog.h:43
static bool noverify_checksums
Definition: basebackup.c:122
static char * buf
Definition: pg_test_fsync.c:67
int errcode_for_file_access(void)
Definition: elog.c:631
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2242
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
#define ereport(elevel, rest)
Definition: elog.h:141
#define CHECK_FREAD_ERROR(fp, filename)
Definition: basebackup.c:96
#define WARNING
Definition: elog.h:40
char * last_dir_separator(const char *filename)
Definition: path.c:138
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1318
PageHeaderData * PageHeader
Definition: bufpage.h:166
#define Assert(condition)
Definition: c.h:739
#define PageGetLSN(page)
Definition: bufpage.h:366
int FreeFile(FILE *file)
Definition: fd.c:2441
static char * filename
Definition: pg_dumpall.c:90
#define PageIsNew(page)
Definition: bufpage.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define TAR_SEND_SIZE
Definition: basebackup.c:84
int i
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
uint16 pg_checksum_page(char *page, BlockNumber blkno)
static XLogRecPtr startptr
Definition: basebackup.c:116
static long long int total_checksum_failures
Definition: basebackup.c:119

◆ sendFileWithContent()

static void sendFileWithContent ( const char *  filename,
const char *  content 
)
static

Definition at line 911 of file basebackup.c.

References _tarWriteHeader(), buf, MemSet, pg_file_create_mode, pq_putmessage, and stat.

Referenced by perform_base_backup().

912 {
913  struct stat statbuf;
914  int pad,
915  len;
916 
917  len = strlen(content);
918 
919  /*
920  * Construct a stat struct for the backup_label file we're injecting in
921  * the tar.
922  */
923  /* Windows doesn't have the concept of uid and gid */
924 #ifdef WIN32
925  statbuf.st_uid = 0;
926  statbuf.st_gid = 0;
927 #else
928  statbuf.st_uid = geteuid();
929  statbuf.st_gid = getegid();
930 #endif
931  statbuf.st_mtime = time(NULL);
932  statbuf.st_mode = pg_file_create_mode;
933  statbuf.st_size = len;
934 
935  _tarWriteHeader(filename, NULL, &statbuf, false);
936  /* Send the contents as a CopyData message */
937  pq_putmessage('d', content, len);
938 
939  /* Pad to 512 byte boundary, per tar format requirements */
940  pad = ((len + 511) & ~511) - len;
941  if (pad > 0)
942  {
943  char buf[512];
944 
945  MemSet(buf, 0, pad);
946  pq_putmessage('d', buf, pad);
947  }
948 }
int pg_file_create_mode
Definition: file_perm.c:19
#define MemSet(start, val, len)
Definition: c.h:962
static char * buf
Definition: pg_test_fsync.c:67
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
#define stat(a, b)
Definition: win32_port.h:255
static char * filename
Definition: pg_dumpall.c:90
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ sendTablespace()

int64 sendTablespace ( char *  path,
bool  sizeonly 
)

Definition at line 958 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, sendDir(), snprintf, stat, and TABLESPACE_VERSION_DIRECTORY.

Referenced by do_pg_start_backup(), and perform_base_backup().

959 {
960  int64 size;
961  char pathbuf[MAXPGPATH];
962  struct stat statbuf;
963 
964  /*
965  * 'path' points to the tablespace location, but we only want to include
966  * the version directory in it that belongs to us.
967  */
968  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
970 
971  /*
972  * Store a directory entry in the tar file so we get the permissions
973  * right.
974  */
975  if (lstat(pathbuf, &statbuf) != 0)
976  {
977  if (errno != ENOENT)
978  ereport(ERROR,
980  errmsg("could not stat file or directory \"%s\": %m",
981  pathbuf)));
982 
983  /* If the tablespace went away while scanning, it's no error. */
984  return 0;
985  }
986 
987  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
988  sizeonly);
989 
990  /* Send all the files in the tablespace version directory */
991  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
992 
993  return size;
994 }
#define NIL
Definition: pg_list.h:65
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
int errcode_for_file_access(void)
Definition: elog.c:631
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1604
#define ereport(elevel, rest)
Definition: elog.h:141
#define stat(a, b)
Definition: win32_port.h:255
#define lstat(path, sb)
Definition: win32_port.h:244
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:1009
int errmsg(const char *fmt,...)
Definition: elog.c:822
#define snprintf
Definition: port.h:192

◆ SendXlogRecPtrResult()

static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 856 of file basebackup.c.

References buf, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), snprintf, and generate_unaccent_rules::str.

Referenced by perform_base_backup().

857 {
859  char str[MAXFNAMELEN];
860  Size len;
861 
862  pq_beginmessage(&buf, 'T'); /* RowDescription */
863  pq_sendint16(&buf, 2); /* 2 fields */
864 
865  /* Field headers */
866  pq_sendstring(&buf, "recptr");
867  pq_sendint32(&buf, 0); /* table oid */
868  pq_sendint16(&buf, 0); /* attnum */
869  pq_sendint32(&buf, TEXTOID); /* type oid */
870  pq_sendint16(&buf, -1);
871  pq_sendint32(&buf, 0);
872  pq_sendint16(&buf, 0);
873 
874  pq_sendstring(&buf, "tli");
875  pq_sendint32(&buf, 0); /* table oid */
876  pq_sendint16(&buf, 0); /* attnum */
877 
878  /*
879  * int8 may seem like a surprising data type for this, but in theory int4
880  * would not be wide enough for this, as TimeLineID is unsigned.
881  */
882  pq_sendint32(&buf, INT8OID); /* type oid */
883  pq_sendint16(&buf, -1);
884  pq_sendint32(&buf, 0);
885  pq_sendint16(&buf, 0);
886  pq_endmessage(&buf);
887 
888  /* Data row */
889  pq_beginmessage(&buf, 'D');
890  pq_sendint16(&buf, 2); /* number of columns */
891 
892  len = snprintf(str, sizeof(str),
893  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
894  pq_sendint32(&buf, len);
895  pq_sendbytes(&buf, str, len);
896 
897  len = snprintf(str, sizeof(str), "%u", tli);
898  pq_sendint32(&buf, len);
899  pq_sendbytes(&buf, str, len);
900 
901  pq_endmessage(&buf);
902 
903  /* Send a CommandComplete message */
904  pq_puttextmessage('C', "SELECT");
905 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:359
#define MAXFNAMELEN
size_t Size
Definition: c.h:467
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
#define snprintf
Definition: port.h:192

◆ throttle()

static void throttle ( size_t  increment)
static

Definition at line 1666 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_EXIT_ON_PM_DEATH, WL_LATCH_SET, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1667 {
1668  TimeOffset elapsed_min;
1669 
1670  if (throttling_counter < 0)
1671  return;
1672 
1673  throttling_counter += increment;
1675  return;
1676 
1677  /* How much time should have elapsed at minimum? */
1678  elapsed_min = elapsed_min_unit *
1680 
1681  /*
1682  * Since the latch could be set repeatedly because of concurrently WAL
1683  * activity, sleep in a loop to ensure enough time has passed.
1684  */
1685  for (;;)
1686  {
1687  TimeOffset elapsed,
1688  sleep;
1689  int wait_result;
1690 
1691  /* Time elapsed since the last measurement (and possible wake up). */
1692  elapsed = GetCurrentTimestamp() - throttled_last;
1693 
1694  /* sleep if the transfer is faster than it should be */
1695  sleep = elapsed_min - elapsed;
1696  if (sleep <= 0)
1697  break;
1698 
1700 
1701  /* We're eating a potentially set latch, so check for interrupts */
1703 
1704  /*
1705  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1706  * the maximum time to sleep. Thus the cast to long is safe.
1707  */
1708  wait_result = WaitLatch(MyLatch,
1710  (long) (sleep / 1000),
1712 
1713  if (wait_result & WL_LATCH_SET)
1715 
1716  /* Done waiting? */
1717  if (wait_result & WL_TIMEOUT)
1718  break;
1719  }
1720 
1721  /*
1722  * As we work with integers, only whole multiple of throttling_sample was
1723  * processed. The rest will be done during the next call of this function.
1724  */
1726 
1727  /*
1728  * Time interval for the remaining amount and possible next increments
1729  * starts now.
1730  */
1732 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1583
void ResetLatch(Latch *latch)
Definition: latch.c:519
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:344
int64 TimeOffset
Definition: timestamp.h:40
static TimestampTz throttled_last
Definition: basebackup.c:113
struct Latch * MyLatch
Definition: globals.c:54
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
static int64 throttling_counter
Definition: basebackup.c:107
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:104
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 76 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

◆ elapsed_min_unit

TimeOffset elapsed_min_unit
static

Definition at line 110 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:50
#define PG_STAT_TMP_DIR
Definition: pgstat.h:33

Definition at line 132 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const char* const excludeFiles[]
static
Initial value:
=
{
"postmaster.pid",
"postmaster.opts",
NULL
}
#define TABLESPACE_MAP
Definition: xlog.h:362
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:24
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:98
#define BACKUP_LABEL_FILE
Definition: xlog.h:359

Definition at line 173 of file basebackup.c.

Referenced by sendDir().

◆ noChecksumFiles

const char* const noChecksumFiles[]
static
Initial value:
= {
"pg_control",
"pg_filenode.map",
"pg_internal.init",
"PG_VERSION",
NULL,
}

Definition at line 206 of file basebackup.c.

Referenced by is_checksummed_file().

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 122 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ startptr

◆ statrelpath

char* statrelpath = NULL
static

Definition at line 79 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

◆ throttled_last

TimestampTz throttled_last
static

Definition at line 113 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_counter

int64 throttling_counter
static

Definition at line 107 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_sample

uint64 throttling_sample
static

Definition at line 104 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 119 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().