PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/pg_type.h"
#include "common/file_perm.h"
#include "commands/progress.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/backup_manifest.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 
struct  exclude_list_item
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 

Functions

static int64 sendTablespace (char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
 
static int64 sendDir (const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
 
static bool sendFile (const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 
static void sendFileWithContent (const char *filename, const char *content, backup_manifest_info *manifest)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void perform_base_backup (basebackup_options *opt)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static void throttle (size_t increment)
 
static void update_basebackup_progress (int64 delta)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
static int basebackup_read_file (int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static XLogRecPtr startptr
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static int64 backup_total = 0
 
static int64 backup_streamed = 0
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 
static const struct exclude_list_item noChecksumFiles []
 

Macro Definition Documentation

◆ TAR_SEND_SIZE

#define TAR_SEND_SIZE   32768

Definition at line 96 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ THROTTLING_FREQUENCY

#define THROTTLING_FREQUENCY   8

Definition at line 101 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

◆ _tarWriteDir()

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1858 of file basebackup.c.

References _tarWriteHeader(), pg_dir_create_mode, and pgwin32_is_junction().

Referenced by sendDir().

1860 {
1861  /* If symlink, write it as a directory anyway */
1862 #ifndef WIN32
1863  if (S_ISLNK(statbuf->st_mode))
1864 #else
1865  if (pgwin32_is_junction(pathbuf))
1866 #endif
1867  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1868 
1869  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1870 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
int pg_dir_create_mode
Definition: file_perm.c:18
bool pgwin32_is_junction(const char *path)

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1815 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_BLOCK_SIZE, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, tarCreateHeader(), and update_basebackup_progress().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1817 {
1818  char h[TAR_BLOCK_SIZE];
1819  enum tarError rc;
1820 
1821  if (!sizeonly)
1822  {
1823  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1824  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1825  statbuf->st_mtime);
1826 
1827  switch (rc)
1828  {
1829  case TAR_OK:
1830  break;
1831  case TAR_NAME_TOO_LONG:
1832  ereport(ERROR,
1833  (errmsg("file name too long for tar format: \"%s\"",
1834  filename)));
1835  break;
1836  case TAR_SYMLINK_TOO_LONG:
1837  ereport(ERROR,
1838  (errmsg("symbolic link target too long for tar format: "
1839  "file name \"%s\", target \"%s\"",
1840  filename, linktarget)));
1841  break;
1842  default:
1843  elog(ERROR, "unrecognized tar error: %d", rc);
1844  }
1845 
1846  pq_putmessage('d', h, sizeof(h));
1847  update_basebackup_progress(sizeof(h));
1848  }
1849 
1850  return sizeof(h);
1851 }
Definition: pgtar.h:21
tarError
Definition: pgtar.h:19
#define ERROR
Definition: elog.h:43
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
#define ereport(elevel,...)
Definition: elog.h:144
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1952
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ basebackup_read_file()

static int basebackup_read_file ( int  fd,
char *  buf,
size_t  nbytes,
off_t  offset,
const char *  filename,
bool  partial_read_ok 
)
static

Definition at line 1989 of file basebackup.c.

References ereport, errcode_for_file_access(), errmsg(), ERROR, pg_pread(), pgstat_report_wait_end(), pgstat_report_wait_start(), and WAIT_EVENT_BASEBACKUP_READ.

Referenced by perform_base_backup(), and sendFile().

1991 {
1992  int rc;
1993 
1995  rc = pg_pread(fd, buf, nbytes, offset);
1997 
1998  if (rc < 0)
1999  ereport(ERROR,
2001  errmsg("could not read file \"%s\": %m", filename)));
2002  if (!partial_read_ok && rc > 0 && rc != nbytes)
2003  ereport(ERROR,
2005  errmsg("could not read file \"%s\": read %d of %zu",
2006  filename, rc, nbytes)));
2007 
2008  return rc;
2009 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset)
Definition: pread.c:27
#define ERROR
Definition: elog.h:43
static char * buf
Definition: pg_test_fsync.c:67
int errcode_for_file_access(void)
Definition: elog.c:633
static void pgstat_report_wait_end(void)
Definition: pgstat.h:1381
#define ereport(elevel,...)
Definition: elog.h:144
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: pgstat.h:1357
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:824

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 740 of file basebackup.c.

References lfirst.

Referenced by perform_base_backup().

741 {
742  char *fna = (char *) lfirst(a);
743  char *fnb = (char *) lfirst(b);
744 
745  return strcmp(fna + 8, fnb + 8);
746 }
#define lfirst(lc)
Definition: pg_list.h:190

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1517 of file basebackup.c.

References exclude_list_item::match_prefix, and exclude_list_item::name.

Referenced by sendFile().

1518 {
1519  /* Check that the file is in a tablespace */
1520  if (strncmp(fullpath, "./global/", 9) == 0 ||
1521  strncmp(fullpath, "./base/", 7) == 0 ||
1522  strncmp(fullpath, "/", 1) == 0)
1523  {
1524  int excludeIdx;
1525 
1526  /* Compare file against noChecksumFiles skip list */
1527  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1528  {
1529  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1530 
1531  if (!noChecksumFiles[excludeIdx].match_prefix)
1532  cmplen++;
1533  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1534  cmplen) == 0)
1535  return false;
1536  }
1537 
1538  return true;
1539  }
1540  else
1541  return false;
1542 }
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:238
const char * name
Definition: encode.c:561
const char * name
Definition: basebackup.c:141
static char * filename
Definition: pg_dumpall.c:90

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 752 of file basebackup.c.

References DefElem::arg, CHECKSUM_TYPE_CRC32C, CHECKSUM_TYPE_NONE, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, basebackup_options::manifest, basebackup_options::manifest_checksum_type, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, MANIFEST_OPTION_YES, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, parse_bool(), pg_checksum_parse_type(), pg_strcasecmp(), basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

753 {
754  ListCell *lopt;
755  bool o_label = false;
756  bool o_progress = false;
757  bool o_fast = false;
758  bool o_nowait = false;
759  bool o_wal = false;
760  bool o_maxrate = false;
761  bool o_tablespace_map = false;
762  bool o_noverify_checksums = false;
763  bool o_manifest = false;
764  bool o_manifest_checksums = false;
765 
766  MemSet(opt, 0, sizeof(*opt));
769 
770  foreach(lopt, options)
771  {
772  DefElem *defel = (DefElem *) lfirst(lopt);
773 
774  if (strcmp(defel->defname, "label") == 0)
775  {
776  if (o_label)
777  ereport(ERROR,
778  (errcode(ERRCODE_SYNTAX_ERROR),
779  errmsg("duplicate option \"%s\"", defel->defname)));
780  opt->label = strVal(defel->arg);
781  o_label = true;
782  }
783  else if (strcmp(defel->defname, "progress") == 0)
784  {
785  if (o_progress)
786  ereport(ERROR,
787  (errcode(ERRCODE_SYNTAX_ERROR),
788  errmsg("duplicate option \"%s\"", defel->defname)));
789  opt->progress = true;
790  o_progress = true;
791  }
792  else if (strcmp(defel->defname, "fast") == 0)
793  {
794  if (o_fast)
795  ereport(ERROR,
796  (errcode(ERRCODE_SYNTAX_ERROR),
797  errmsg("duplicate option \"%s\"", defel->defname)));
798  opt->fastcheckpoint = true;
799  o_fast = true;
800  }
801  else if (strcmp(defel->defname, "nowait") == 0)
802  {
803  if (o_nowait)
804  ereport(ERROR,
805  (errcode(ERRCODE_SYNTAX_ERROR),
806  errmsg("duplicate option \"%s\"", defel->defname)));
807  opt->nowait = true;
808  o_nowait = true;
809  }
810  else if (strcmp(defel->defname, "wal") == 0)
811  {
812  if (o_wal)
813  ereport(ERROR,
814  (errcode(ERRCODE_SYNTAX_ERROR),
815  errmsg("duplicate option \"%s\"", defel->defname)));
816  opt->includewal = true;
817  o_wal = true;
818  }
819  else if (strcmp(defel->defname, "max_rate") == 0)
820  {
821  long maxrate;
822 
823  if (o_maxrate)
824  ereport(ERROR,
825  (errcode(ERRCODE_SYNTAX_ERROR),
826  errmsg("duplicate option \"%s\"", defel->defname)));
827 
828  maxrate = intVal(defel->arg);
829  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
830  ereport(ERROR,
831  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
832  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
833  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
834 
835  opt->maxrate = (uint32) maxrate;
836  o_maxrate = true;
837  }
838  else if (strcmp(defel->defname, "tablespace_map") == 0)
839  {
840  if (o_tablespace_map)
841  ereport(ERROR,
842  (errcode(ERRCODE_SYNTAX_ERROR),
843  errmsg("duplicate option \"%s\"", defel->defname)));
844  opt->sendtblspcmapfile = true;
845  o_tablespace_map = true;
846  }
847  else if (strcmp(defel->defname, "noverify_checksums") == 0)
848  {
849  if (o_noverify_checksums)
850  ereport(ERROR,
851  (errcode(ERRCODE_SYNTAX_ERROR),
852  errmsg("duplicate option \"%s\"", defel->defname)));
853  noverify_checksums = true;
854  o_noverify_checksums = true;
855  }
856  else if (strcmp(defel->defname, "manifest") == 0)
857  {
858  char *optval = strVal(defel->arg);
859  bool manifest_bool;
860 
861  if (o_manifest)
862  ereport(ERROR,
863  (errcode(ERRCODE_SYNTAX_ERROR),
864  errmsg("duplicate option \"%s\"", defel->defname)));
865  if (parse_bool(optval, &manifest_bool))
866  {
867  if (manifest_bool)
869  else
871  }
872  else if (pg_strcasecmp(optval, "force-encode") == 0)
874  else
875  ereport(ERROR,
876  (errcode(ERRCODE_SYNTAX_ERROR),
877  errmsg("unrecognized manifest option: \"%s\"",
878  optval)));
879  o_manifest = true;
880  }
881  else if (strcmp(defel->defname, "manifest_checksums") == 0)
882  {
883  char *optval = strVal(defel->arg);
884 
885  if (o_manifest_checksums)
886  ereport(ERROR,
887  (errcode(ERRCODE_SYNTAX_ERROR),
888  errmsg("duplicate option \"%s\"", defel->defname)));
889  if (!pg_checksum_parse_type(optval,
890  &opt->manifest_checksum_type))
891  ereport(ERROR,
892  (errcode(ERRCODE_SYNTAX_ERROR),
893  errmsg("unrecognized checksum algorithm: \"%s\"",
894  optval)));
895  o_manifest_checksums = true;
896  }
897  else
898  elog(ERROR, "option \"%s\" not recognized",
899  defel->defname);
900  }
901  if (opt->label == NULL)
902  opt->label = "base backup";
903  if (opt->manifest == MANIFEST_OPTION_NO)
904  {
905  if (o_manifest_checksums)
906  ereport(ERROR,
907  (errcode(ERRCODE_SYNTAX_ERROR),
908  errmsg("manifest checksums require a backup manifest")));
910  }
911 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:610
#define MemSet(start, val, len)
Definition: c.h:978
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:43
backup_manifest_option manifest
Definition: basebackup.c:57
static bool noverify_checksums
Definition: basebackup.c:122
static int32 maxrate
const char * label
Definition: basebackup.c:50
unsigned int uint32
Definition: c.h:374
Node * arg
Definition: parsenodes.h:733
#define ereport(elevel,...)
Definition: elog.h:144
#define lfirst(lc)
Definition: pg_list.h:190
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
char * defname
Definition: parsenodes.h:732
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt)
static

Definition at line 256 of file basebackup.c.

References _tarWriteHeader(), AddWALInfoToBackupManifest(), AllocateDir(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, backup_streamed, backup_total, basebackup_read_file(), BoolGetDatum, buf, CheckXLogRemoved(), CloseTransientFile(), compareWalFileNames(), CurrentResourceOwner, dirent::d_name, StringInfoData::data, DataDir, do_pg_abort_backup(), do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, fd(), FreeDir(), GetCurrentTimestamp(), basebackup_options::includewal, InitializeBackupManifest(), InvalidOid, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, linitial, list_length(), list_sort(), lnext(), lstat, makeStringInfo(), basebackup_options::manifest, manifest, basebackup_options::manifest_checksum_type, MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, tablespaceinfo::oid, OpenTransientFile(), palloc0(), tablespaceinfo::path, PG_BINARY, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_progress_end_command(), pgstat_progress_start_command(), pgstat_progress_update_multi_param(), pgstat_progress_update_param(), pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint16(), basebackup_options::progress, PROGRESS_BASEBACKUP_BACKUP_TOTAL, PROGRESS_BASEBACKUP_PHASE, PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE, PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP, PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL, PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT, PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE, PROGRESS_BASEBACKUP_TBLSPC_STREAMED, PROGRESS_BASEBACKUP_TBLSPC_TOTAL, PROGRESS_COMMAND_BASEBACKUP, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), ResourceOwnerCreate(), SendBackupHeader(), SendBackupManifest(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf, startptr, stat, statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_BLOCK_SIZE, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, total_checksum_failures, update_basebackup_progress(), USECS_PER_SEC, val, wal_segment_size, WalSndResourceCleanup(), WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

257 {
258  TimeLineID starttli;
259  XLogRecPtr endptr;
260  TimeLineID endtli;
261  StringInfo labelfile;
264  int datadirpathlen;
265  List *tablespaces = NIL;
266 
267  backup_total = 0;
268  backup_streamed = 0;
270 
271  /*
272  * If the estimation of the total backup size is disabled, make the
273  * backup_total column in the view return NULL by setting the parameter to
274  * -1.
275  */
276  if (!opt->progress)
277  {
278  backup_total = -1;
280  backup_total);
281  }
282 
283  /* we're going to use a BufFile, so we need a ResourceOwner */
284  Assert(CurrentResourceOwner == NULL);
285  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
286 
287  datadirpathlen = strlen(DataDir);
288 
290 
291  labelfile = makeStringInfo();
292  tblspc_map_file = makeStringInfo();
293  InitializeBackupManifest(&manifest, opt->manifest,
295 
297 
300  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
301  labelfile, &tablespaces,
302  tblspc_map_file, opt->sendtblspcmapfile);
303 
304  /*
305  * Once do_pg_start_backup has been called, ensure that any failure causes
306  * us to abort the backup so we don't "leak" a backup counter. For this
307  * reason, *all* functionality between do_pg_start_backup() and the end of
308  * do_pg_stop_backup() should be inside the error cleanup block!
309  */
310 
312  {
313  ListCell *lc;
314  tablespaceinfo *ti;
315  int tblspc_streamed = 0;
316 
317  /*
318  * Calculate the relative path of temporary statistics directory in
319  * order to skip the files which are located in that directory later.
320  */
322  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
323  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
324  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
326  else
328 
329  /* Add a node for the base directory at the end */
330  ti = palloc0(sizeof(tablespaceinfo));
331  ti->size = -1;
332  tablespaces = lappend(tablespaces, ti);
333 
334  /*
335  * Calculate the total backup size by summing up the size of each
336  * tablespace
337  */
338  if (opt->progress)
339  {
342 
343  foreach(lc, tablespaces)
344  {
345  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
346 
347  if (tmp->path == NULL)
348  tmp->size = sendDir(".", 1, true, tablespaces, true, NULL,
349  NULL);
350  else
351  tmp->size = sendTablespace(tmp->path, tmp->oid, true,
352  NULL);
353  backup_total += tmp->size;
354  }
355  }
356 
357  /* Report that we are now streaming database files as a base backup */
358  {
359  const int index[] = {
363  };
364  const int64 val[] = {
366  backup_total, list_length(tablespaces)
367  };
368 
369  pgstat_progress_update_multi_param(3, index, val);
370  }
371 
372  /* Send the starting position of the backup */
373  SendXlogRecPtrResult(startptr, starttli);
374 
375  /* Send tablespace header */
376  SendBackupHeader(tablespaces);
377 
378  /* Setup and activate network throttling, if client requested it */
379  if (opt->maxrate > 0)
380  {
382  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
383 
384  /*
385  * The minimum amount of time for throttling_sample bytes to be
386  * transferred.
387  */
389 
390  /* Enable throttling. */
391  throttling_counter = 0;
392 
393  /* The 'real data' starts now (header was ignored). */
395  }
396  else
397  {
398  /* Disable throttling. */
399  throttling_counter = -1;
400  }
401 
402  /* Send off our tablespaces one by one */
403  foreach(lc, tablespaces)
404  {
405  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
407 
408  /* Send CopyOutResponse message */
409  pq_beginmessage(&buf, 'H');
410  pq_sendbyte(&buf, 0); /* overall format */
411  pq_sendint16(&buf, 0); /* natts */
412  pq_endmessage(&buf);
413 
414  if (ti->path == NULL)
415  {
416  struct stat statbuf;
417  bool sendtblspclinks = true;
418 
419  /* In the main tar, include the backup_label first... */
421  &manifest);
422 
423  /* Then the tablespace_map file, if required... */
424  if (opt->sendtblspcmapfile)
425  {
426  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
427  &manifest);
428  sendtblspclinks = false;
429  }
430 
431  /* Then the bulk of the files... */
432  sendDir(".", 1, false, tablespaces, sendtblspclinks,
433  &manifest, NULL);
434 
435  /* ... and pg_control after everything else. */
436  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
437  ereport(ERROR,
439  errmsg("could not stat file \"%s\": %m",
442  false, InvalidOid, &manifest, NULL);
443  }
444  else
445  sendTablespace(ti->path, ti->oid, false, &manifest);
446 
447  /*
448  * If we're including WAL, and this is the main data directory we
449  * don't terminate the tar stream here. Instead, we will append
450  * the xlog files below and terminate it then. This is safe since
451  * the main data directory is always sent *last*.
452  */
453  if (opt->includewal && ti->path == NULL)
454  {
455  Assert(lnext(tablespaces, lc) == NULL);
456  }
457  else
458  pq_putemptymessage('c'); /* CopyDone */
459 
460  tblspc_streamed++;
462  tblspc_streamed);
463  }
464 
467  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
468  }
470 
471 
472  if (opt->includewal)
473  {
474  /*
475  * We've left the last tar file "open", so we can now append the
476  * required WAL files to it.
477  */
478  char pathbuf[MAXPGPATH];
479  XLogSegNo segno;
480  XLogSegNo startsegno;
481  XLogSegNo endsegno;
482  struct stat statbuf;
483  List *historyFileList = NIL;
484  List *walFileList = NIL;
485  char firstoff[MAXFNAMELEN];
486  char lastoff[MAXFNAMELEN];
487  DIR *dir;
488  struct dirent *de;
489  ListCell *lc;
490  TimeLineID tli;
491 
494 
495  /*
496  * I'd rather not worry about timelines here, so scan pg_wal and
497  * include all WAL files in the range between 'startptr' and 'endptr',
498  * regardless of the timeline the file is stamped with. If there are
499  * some spurious WAL files belonging to timelines that don't belong in
500  * this server's history, they will be included too. Normally there
501  * shouldn't be such files, but if there are, there's little harm in
502  * including them.
503  */
504  XLByteToSeg(startptr, startsegno, wal_segment_size);
505  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
506  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
507  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
508 
509  dir = AllocateDir("pg_wal");
510  while ((de = ReadDir(dir, "pg_wal")) != NULL)
511  {
512  /* Does it look like a WAL segment, and is it in the range? */
513  if (IsXLogFileName(de->d_name) &&
514  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
515  strcmp(de->d_name + 8, lastoff + 8) <= 0)
516  {
517  walFileList = lappend(walFileList, pstrdup(de->d_name));
518  }
519  /* Does it look like a timeline history file? */
520  else if (IsTLHistoryFileName(de->d_name))
521  {
522  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
523  }
524  }
525  FreeDir(dir);
526 
527  /*
528  * Before we go any further, check that none of the WAL segments we
529  * need were removed.
530  */
531  CheckXLogRemoved(startsegno, ThisTimeLineID);
532 
533  /*
534  * Sort the WAL filenames. We want to send the files in order from
535  * oldest to newest, to reduce the chance that a file is recycled
536  * before we get a chance to send it over.
537  */
538  list_sort(walFileList, compareWalFileNames);
539 
540  /*
541  * There must be at least one xlog file in the pg_wal directory, since
542  * we are doing backup-including-xlog.
543  */
544  if (walFileList == NIL)
545  ereport(ERROR,
546  (errmsg("could not find any WAL files")));
547 
548  /*
549  * Sanity check: the first and last segment should cover startptr and
550  * endptr, with no gaps in between.
551  */
552  XLogFromFileName((char *) linitial(walFileList),
553  &tli, &segno, wal_segment_size);
554  if (segno != startsegno)
555  {
556  char startfname[MAXFNAMELEN];
557 
558  XLogFileName(startfname, ThisTimeLineID, startsegno,
560  ereport(ERROR,
561  (errmsg("could not find WAL file \"%s\"", startfname)));
562  }
563  foreach(lc, walFileList)
564  {
565  char *walFileName = (char *) lfirst(lc);
566  XLogSegNo currsegno = segno;
567  XLogSegNo nextsegno = segno + 1;
568 
569  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
570  if (!(nextsegno == segno || currsegno == segno))
571  {
572  char nextfname[MAXFNAMELEN];
573 
574  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
576  ereport(ERROR,
577  (errmsg("could not find WAL file \"%s\"", nextfname)));
578  }
579  }
580  if (segno != endsegno)
581  {
582  char endfname[MAXFNAMELEN];
583 
584  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
585  ereport(ERROR,
586  (errmsg("could not find WAL file \"%s\"", endfname)));
587  }
588 
589  /* Ok, we have everything we need. Send the WAL files. */
590  foreach(lc, walFileList)
591  {
592  char *walFileName = (char *) lfirst(lc);
593  int fd;
594  char buf[TAR_SEND_SIZE];
595  size_t cnt;
596  pgoff_t len = 0;
597 
598  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
599  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
600 
601  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
602  if (fd < 0)
603  {
604  int save_errno = errno;
605 
606  /*
607  * Most likely reason for this is that the file was already
608  * removed by a checkpoint, so check for that to get a better
609  * error message.
610  */
611  CheckXLogRemoved(segno, tli);
612 
613  errno = save_errno;
614  ereport(ERROR,
616  errmsg("could not open file \"%s\": %m", pathbuf)));
617  }
618 
619  if (fstat(fd, &statbuf) != 0)
620  ereport(ERROR,
622  errmsg("could not stat file \"%s\": %m",
623  pathbuf)));
624  if (statbuf.st_size != wal_segment_size)
625  {
626  CheckXLogRemoved(segno, tli);
627  ereport(ERROR,
629  errmsg("unexpected WAL file size \"%s\"", walFileName)));
630  }
631 
632  /* send the WAL file itself */
633  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
634 
635  while ((cnt = basebackup_read_file(fd, buf,
636  Min(sizeof(buf),
637  wal_segment_size - len),
638  len, pathbuf, true)) > 0)
639  {
640  CheckXLogRemoved(segno, tli);
641  /* Send the chunk as a CopyData message */
642  if (pq_putmessage('d', buf, cnt))
643  ereport(ERROR,
644  (errmsg("base backup could not send data, aborting backup")));
646 
647  len += cnt;
648  throttle(cnt);
649 
650  if (len == wal_segment_size)
651  break;
652  }
653 
654  if (len != wal_segment_size)
655  {
656  CheckXLogRemoved(segno, tli);
657  ereport(ERROR,
659  errmsg("unexpected WAL file size \"%s\"", walFileName)));
660  }
661 
662  /*
663  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
664  * for padding.
665  */
667 
668  CloseTransientFile(fd);
669 
670  /*
671  * Mark file as archived, otherwise files can get archived again
672  * after promotion of a new node. This is in line with
673  * walreceiver.c always doing an XLogArchiveForceDone() after a
674  * complete segment.
675  */
676  StatusFilePath(pathbuf, walFileName, ".done");
677  sendFileWithContent(pathbuf, "", &manifest);
678  }
679 
680  /*
681  * Send timeline history files too. Only the latest timeline history
682  * file is required for recovery, and even that only if there happens
683  * to be a timeline switch in the first WAL segment that contains the
684  * checkpoint record, or if we're taking a base backup from a standby
685  * server and the target timeline changes while the backup is taken.
686  * But they are small and highly useful for debugging purposes, so
687  * better include them all, always.
688  */
689  foreach(lc, historyFileList)
690  {
691  char *fname = lfirst(lc);
692 
693  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
694 
695  if (lstat(pathbuf, &statbuf) != 0)
696  ereport(ERROR,
698  errmsg("could not stat file \"%s\": %m", pathbuf)));
699 
700  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
701  &manifest, NULL);
702 
703  /* unconditionally mark file as archived */
704  StatusFilePath(pathbuf, fname, ".done");
705  sendFileWithContent(pathbuf, "", &manifest);
706  }
707 
708  /* Send CopyDone message for the last tar file */
709  pq_putemptymessage('c');
710  }
711 
712  AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
713 
714  SendBackupManifest(&manifest);
715 
716  SendXlogRecPtrResult(endptr, endtli);
717 
719  {
720  if (total_checksum_failures > 1)
722  (errmsg("%lld total checksum verification failures", total_checksum_failures)));
723 
724  ereport(ERROR,
726  errmsg("checksum verification failure during base backup")));
727  }
728 
729  /* clean up the resource owner we created */
730  WalSndResourceCleanup(true);
731 
733 }
#define StatusFilePath(path, xlog, suffix)
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:1989
static void throttle(size_t increment)
Definition: basebackup.c:1878
static int64 backup_total
Definition: basebackup.c:128
uint32 TimeLineID
Definition: xlogdefs.h:52
#define PROGRESS_BASEBACKUP_PHASE
Definition: progress.h:123
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11427
int wal_segment_size
Definition: xlog.c:116
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:321
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
Definition: pgstat.c:3210
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1574
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3231
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:927
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:953
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED
Definition: progress.h:127
int errcode(int sqlerrcode)
Definition: elog.c:610
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE
Definition: progress.h:133
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:88
Definition: dirent.h:9
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1240
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool needtblspcmapfile)
Definition: xlog.c:10496
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
Definition: type.h:89
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT
Definition: progress.h:130
#define pgoff_t
Definition: win32_port.h:194
#define TABLESPACE_MAP
Definition: xlog.h:392
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
static void sendFileWithContent(const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1086
#define linitial(l)
Definition: pg_list.h:195
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
backup_manifest_option manifest
Definition: basebackup.c:57
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:11014
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2372
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3925
#define MAXPGPATH
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
void SendBackupManifest(backup_manifest_info *manifest)
static int64 backup_streamed
Definition: basebackup.c:131
int errcode_for_file_access(void)
Definition: elog.c:633
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:50
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:1031
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2583
static int64 sendTablespace(char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
Definition: basebackup.c:1143
char * pgstat_stat_directory
Definition: pgstat.c:133
static char * statrelpath
Definition: basebackup.c:91
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:321
int CloseTransientFile(int fd)
Definition: fd.c:2549
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
#define MAXFNAMELEN
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void * palloc0(Size size)
Definition: mcxt.c:980
void pgstat_progress_end_command(void)
Definition: pgstat.c:3282
#define XLOGDIR
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL
Definition: progress.h:126
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL
Definition: progress.h:134
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:330
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:191
static TimestampTz throttled_last
Definition: basebackup.c:113
#define ereport(elevel,...)
Definition: elog.h:144
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1952
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:745
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2649
#define THROTTLING_FREQUENCY
Definition: basebackup.c:101
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3253
static bool manifest
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static int list_length(const List *l)
Definition: pg_list.h:169
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
#define PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE
Definition: progress.h:131
#define lstat(path, sb)
Definition: win32_port.h:244
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP
Definition: progress.h:132
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:740
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1196
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1481
int errmsg(const char *fmt,...)
Definition: elog.c:824
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:96
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1563
char * DataDir
Definition: globals.c:62
#define BACKUP_LABEL_FILE
Definition: xlog.h:389
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:107
Definition: pg_list.h:50
#define snprintf
Definition: port.h:193
static uint64 throttling_sample
Definition: basebackup.c:104
long val
Definition: informix.c:664
int FreeDir(DIR *dir)
Definition: fd.c:2701
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
static XLogRecPtr startptr
Definition: basebackup.c:116
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:422
static long long int total_checksum_failures
Definition: basebackup.c:119
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)

◆ send_int8_string()

static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 943 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), pq_sendint32(), and sprintf.

Referenced by SendBackupHeader().

944 {
945  char is[32];
946 
947  sprintf(is, INT64_FORMAT, intval);
948  pq_sendint32(buf, strlen(is));
949  pq_sendbytes(buf, is, strlen(is));
950 }
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:195
#define INT64_FORMAT
Definition: c.h:416
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125

◆ SendBackupHeader()

static void SendBackupHeader ( List tablespaces)
static

Definition at line 953 of file basebackup.c.

References buf, lfirst, tablespaceinfo::oid, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), send_int8_string(), and tablespaceinfo::size.

Referenced by perform_base_backup().

954 {
956  ListCell *lc;
957 
958  /* Construct and send the directory information */
959  pq_beginmessage(&buf, 'T'); /* RowDescription */
960  pq_sendint16(&buf, 3); /* 3 fields */
961 
962  /* First field - spcoid */
963  pq_sendstring(&buf, "spcoid");
964  pq_sendint32(&buf, 0); /* table oid */
965  pq_sendint16(&buf, 0); /* attnum */
966  pq_sendint32(&buf, OIDOID); /* type oid */
967  pq_sendint16(&buf, 4); /* typlen */
968  pq_sendint32(&buf, 0); /* typmod */
969  pq_sendint16(&buf, 0); /* format code */
970 
971  /* Second field - spclocation */
972  pq_sendstring(&buf, "spclocation");
973  pq_sendint32(&buf, 0);
974  pq_sendint16(&buf, 0);
975  pq_sendint32(&buf, TEXTOID);
976  pq_sendint16(&buf, -1);
977  pq_sendint32(&buf, 0);
978  pq_sendint16(&buf, 0);
979 
980  /* Third field - size */
981  pq_sendstring(&buf, "size");
982  pq_sendint32(&buf, 0);
983  pq_sendint16(&buf, 0);
984  pq_sendint32(&buf, INT8OID);
985  pq_sendint16(&buf, 8);
986  pq_sendint32(&buf, 0);
987  pq_sendint16(&buf, 0);
988  pq_endmessage(&buf);
989 
990  foreach(lc, tablespaces)
991  {
992  tablespaceinfo *ti = lfirst(lc);
993 
994  /* Send one datarow message */
995  pq_beginmessage(&buf, 'D');
996  pq_sendint16(&buf, 3); /* number of columns */
997  if (ti->path == NULL)
998  {
999  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
1000  pq_sendint32(&buf, -1);
1001  }
1002  else
1003  {
1004  Size len;
1005 
1006  len = strlen(ti->oid);
1007  pq_sendint32(&buf, len);
1008  pq_sendbytes(&buf, ti->oid, len);
1009 
1010  len = strlen(ti->path);
1011  pq_sendint32(&buf, len);
1012  pq_sendbytes(&buf, ti->path, len);
1013  }
1014  if (ti->size >= 0)
1015  send_int8_string(&buf, ti->size / 1024);
1016  else
1017  pq_sendint32(&buf, -1); /* NULL */
1018 
1019  pq_endmessage(&buf);
1020  }
1021 
1022  /* Send a CommandComplete message */
1023  pq_puttextmessage('C', "SELECT");
1024 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:943
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
#define lfirst(lc)
Definition: pg_list.h:190
size_t Size
Definition: c.h:473
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 922 of file basebackup.c.

References basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

923 {
924  basebackup_options opt;
925 
926  parse_basebackup_options(cmd->options, &opt);
927 
929 
931  {
932  char activitymsg[50];
933 
934  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
935  opt.label);
936  set_ps_display(activitymsg);
937  }
938 
939  perform_base_backup(&opt);
940 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:36
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:752
void set_ps_display(const char *activity)
Definition: ps_status.c:349
const char * label
Definition: basebackup.c:50
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:256
void WalSndSetState(WalSndState state)
Definition: walsender.c:3187
#define snprintf
Definition: port.h:193

◆ sendDir()

static int64 sendDir ( const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1196 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, DEBUG2, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, exclude_list_item::match_prefix, MAXPGPATH, exclude_list_item::name, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, pgwin32_is_junction(), ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISREG, sendFile(), snprintf, stat, statrelpath, TABLESPACE_VERSION_DIRECTORY, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

1199 {
1200  DIR *dir;
1201  struct dirent *de;
1202  char pathbuf[MAXPGPATH * 2];
1203  struct stat statbuf;
1204  int64 size = 0;
1205  const char *lastDir; /* Split last dir from parent path. */
1206  bool isDbDir = false; /* Does this directory contain relations? */
1207 
1208  /*
1209  * Determine if the current path is a database directory that can contain
1210  * relations.
1211  *
1212  * Start by finding the location of the delimiter between the parent path
1213  * and the current path.
1214  */
1215  lastDir = last_dir_separator(path);
1216 
1217  /* Does this path look like a database path (i.e. all digits)? */
1218  if (lastDir != NULL &&
1219  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1220  {
1221  /* Part of path that contains the parent directory. */
1222  int parentPathLen = lastDir - path;
1223 
1224  /*
1225  * Mark path as a database directory if the parent path is either
1226  * $PGDATA/base or a tablespace version path.
1227  */
1228  if (strncmp(path, "./base", parentPathLen) == 0 ||
1229  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1230  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1232  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1233  isDbDir = true;
1234  }
1235 
1236  dir = AllocateDir(path);
1237  while ((de = ReadDir(dir, path)) != NULL)
1238  {
1239  int excludeIdx;
1240  bool excludeFound;
1241  ForkNumber relForkNum; /* Type of fork if file is a relation */
1242  int relOidChars; /* Chars in filename that are the rel oid */
1243 
1244  /* Skip special stuff */
1245  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1246  continue;
1247 
1248  /* Skip temporary files */
1249  if (strncmp(de->d_name,
1251  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1252  continue;
1253 
1254  /*
1255  * Check if the postmaster has signaled us to exit, and abort with an
1256  * error in that case. The error handler further up will call
1257  * do_pg_abort_backup() for us. Also check that if the backup was
1258  * started while still in recovery, the server wasn't promoted.
1259  * do_pg_stop_backup() will check that too, but it's better to stop
1260  * the backup early than continue to the end and fail there.
1261  */
1264  ereport(ERROR,
1265  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1266  errmsg("the standby was promoted during online backup"),
1267  errhint("This means that the backup being taken is corrupt "
1268  "and should not be used. "
1269  "Try taking another online backup.")));
1270 
1271  /* Scan for files that should be excluded */
1272  excludeFound = false;
1273  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1274  {
1275  int cmplen = strlen(excludeFiles[excludeIdx].name);
1276 
1277  if (!excludeFiles[excludeIdx].match_prefix)
1278  cmplen++;
1279  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1280  {
1281  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1282  excludeFound = true;
1283  break;
1284  }
1285  }
1286 
1287  if (excludeFound)
1288  continue;
1289 
1290  /* Exclude all forks for unlogged tables except the init fork */
1291  if (isDbDir &&
1292  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1293  &relForkNum))
1294  {
1295  /* Never exclude init forks */
1296  if (relForkNum != INIT_FORKNUM)
1297  {
1298  char initForkFile[MAXPGPATH];
1299  char relOid[OIDCHARS + 1];
1300 
1301  /*
1302  * If any other type of fork, check if there is an init fork
1303  * with the same OID. If so, the file can be excluded.
1304  */
1305  memcpy(relOid, de->d_name, relOidChars);
1306  relOid[relOidChars] = '\0';
1307  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1308  path, relOid);
1309 
1310  if (lstat(initForkFile, &statbuf) == 0)
1311  {
1312  elog(DEBUG2,
1313  "unlogged relation file \"%s\" excluded from backup",
1314  de->d_name);
1315 
1316  continue;
1317  }
1318  }
1319  }
1320 
1321  /* Exclude temporary relations */
1322  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1323  {
1324  elog(DEBUG2,
1325  "temporary relation file \"%s\" excluded from backup",
1326  de->d_name);
1327 
1328  continue;
1329  }
1330 
1331  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1332 
1333  /* Skip pg_control here to back up it last */
1334  if (strcmp(pathbuf, "./global/pg_control") == 0)
1335  continue;
1336 
1337  if (lstat(pathbuf, &statbuf) != 0)
1338  {
1339  if (errno != ENOENT)
1340  ereport(ERROR,
1342  errmsg("could not stat file or directory \"%s\": %m",
1343  pathbuf)));
1344 
1345  /* If the file went away while scanning, it's not an error. */
1346  continue;
1347  }
1348 
1349  /* Scan for directories whose contents should be excluded */
1350  excludeFound = false;
1351  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1352  {
1353  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1354  {
1355  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1356  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1357  excludeFound = true;
1358  break;
1359  }
1360  }
1361 
1362  if (excludeFound)
1363  continue;
1364 
1365  /*
1366  * Exclude contents of directory specified by statrelpath if not set
1367  * to the default (pg_stat_tmp) which is caught in the loop above.
1368  */
1369  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1370  {
1371  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1372  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1373  continue;
1374  }
1375 
1376  /*
1377  * We can skip pg_wal, the WAL segments need to be fetched from the
1378  * WAL archive anyway. But include it as an empty directory anyway, so
1379  * we get permissions right.
1380  */
1381  if (strcmp(pathbuf, "./pg_wal") == 0)
1382  {
1383  /* If pg_wal is a symlink, write it as a directory anyway */
1384  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1385 
1386  /*
1387  * Also send archive_status directory (by hackishly reusing
1388  * statbuf from above ...).
1389  */
1390  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1391  sizeonly);
1392 
1393  continue; /* don't recurse into pg_wal */
1394  }
1395 
1396  /* Allow symbolic links in pg_tblspc only */
1397  if (strcmp(path, "./pg_tblspc") == 0 &&
1398 #ifndef WIN32
1399  S_ISLNK(statbuf.st_mode)
1400 #else
1401  pgwin32_is_junction(pathbuf)
1402 #endif
1403  )
1404  {
1405 #if defined(HAVE_READLINK) || defined(WIN32)
1406  char linkpath[MAXPGPATH];
1407  int rllen;
1408 
1409  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1410  if (rllen < 0)
1411  ereport(ERROR,
1413  errmsg("could not read symbolic link \"%s\": %m",
1414  pathbuf)));
1415  if (rllen >= sizeof(linkpath))
1416  ereport(ERROR,
1417  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1418  errmsg("symbolic link \"%s\" target is too long",
1419  pathbuf)));
1420  linkpath[rllen] = '\0';
1421 
1422  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1423  &statbuf, sizeonly);
1424 #else
1425 
1426  /*
1427  * If the platform does not have symbolic links, it should not be
1428  * possible to have tablespaces - clearly somebody else created
1429  * them. Warn about it and ignore.
1430  */
1431  ereport(WARNING,
1432  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1433  errmsg("tablespaces are not supported on this platform")));
1434  continue;
1435 #endif /* HAVE_READLINK */
1436  }
1437  else if (S_ISDIR(statbuf.st_mode))
1438  {
1439  bool skip_this_dir = false;
1440  ListCell *lc;
1441 
1442  /*
1443  * Store a directory entry in the tar file so we can get the
1444  * permissions right.
1445  */
1446  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1447  sizeonly);
1448 
1449  /*
1450  * Call ourselves recursively for a directory, unless it happens
1451  * to be a separate tablespace located within PGDATA.
1452  */
1453  foreach(lc, tablespaces)
1454  {
1455  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1456 
1457  /*
1458  * ti->rpath is the tablespace relative path within PGDATA, or
1459  * NULL if the tablespace has been properly located somewhere
1460  * else.
1461  *
1462  * Skip past the leading "./" in pathbuf when comparing.
1463  */
1464  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1465  {
1466  skip_this_dir = true;
1467  break;
1468  }
1469  }
1470 
1471  /*
1472  * skip sending directories inside pg_tblspc, if not required.
1473  */
1474  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1475  skip_this_dir = true;
1476 
1477  if (!skip_this_dir)
1478  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
1479  sendtblspclinks, manifest, spcoid);
1480  }
1481  else if (S_ISREG(statbuf.st_mode))
1482  {
1483  bool sent = false;
1484 
1485  if (!sizeonly)
1486  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1487  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1488  manifest, spcoid);
1489 
1490  if (sent || sizeonly)
1491  {
1492  /* Add size. */
1493  size += statbuf.st_size;
1494 
1495  /* Pad to a multiple of the tar block size. */
1496  size += tarPaddingBytesRequired(statbuf.st_size);
1497 
1498  /* Size of the header for the file. */
1499  size += TAR_BLOCK_SIZE;
1500  }
1501  }
1502  else
1503  ereport(WARNING,
1504  (errmsg("skipping special file \"%s\"", pathbuf)));
1505  }
1506  FreeDir(dir);
1507  return size;
1508 }
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1071
char * rpath
Definition: basebackup.h:27
int errcode(int sqlerrcode)
Definition: elog.c:610
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:88
Definition: dirent.h:9
static const char *const excludeDirContents[]
Definition: basebackup.c:153
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
#define readlink(path, buf, size)
Definition: win32_port.h:222
int errcode_for_file_access(void)
Definition: elog.c:633
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2583
#define atooid(x)
Definition: postgres_ext.h:42
static char * statrelpath
Definition: basebackup.c:91
ForkNumber
Definition: relpath.h:40
#define S_ISREG(m)
Definition: win32_port.h:299
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define InvalidOid
Definition: postgres_ext.h:36
#define ereport(elevel,...)
Definition: elog.h:144
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2649
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3187
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:374
const char * name
Definition: encode.c:561
#define S_ISDIR(m)
Definition: win32_port.h:296
#define lstat(path, sb)
Definition: win32_port.h:244
const char * name
Definition: basebackup.c:141
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1196
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1563
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
char d_name[MAX_PATH]
Definition: dirent.h:14
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:194
#define snprintf
Definition: port.h:193
#define OIDCHARS
Definition: relpath.h:30
int FreeDir(DIR *dir)
Definition: fd.c:2701
bool pgwin32_is_junction(const char *path)
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1858

◆ sendFile()

static bool sendFile ( const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1563 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), Assert, basebackup_read_file(), buf, backup_manifest_info::checksum_type, CloseTransientFile(), DataChecksumsEnabled(), ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, fd(), filename, i, is_checksummed_file(), last_dir_separator(), MemSet, Min, noverify_checksums, OpenTransientFile(), PageGetLSN, PageIsNew, PageHeaderData::pd_checksum, PG_BINARY, pg_checksum_init(), pg_checksum_page(), pg_checksum_update(), pgoff_t, pgstat_report_checksum_failures_in_db(), pq_putmessage, startptr, TAR_SEND_SIZE, tarPaddingBytesRequired(), throttle(), total_checksum_failures, update_basebackup_progress(), and WARNING.

Referenced by perform_base_backup(), and sendDir().

1566 {
1567  int fd;
1568  BlockNumber blkno = 0;
1569  bool block_retry = false;
1570  char buf[TAR_SEND_SIZE];
1571  uint16 checksum;
1572  int checksum_failures = 0;
1573  off_t cnt;
1574  int i;
1575  pgoff_t len = 0;
1576  char *page;
1577  size_t pad;
1578  PageHeader phdr;
1579  int segmentno = 0;
1580  char *segmentpath;
1581  bool verify_checksum = false;
1582  pg_checksum_context checksum_ctx;
1583 
1584  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1585 
1586  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1587  if (fd < 0)
1588  {
1589  if (errno == ENOENT && missing_ok)
1590  return false;
1591  ereport(ERROR,
1593  errmsg("could not open file \"%s\": %m", readfilename)));
1594  }
1595 
1596  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1597 
1599  {
1600  char *filename;
1601 
1602  /*
1603  * Get the filename (excluding path). As last_dir_separator()
1604  * includes the last directory separator, we chop that off by
1605  * incrementing the pointer.
1606  */
1607  filename = last_dir_separator(readfilename) + 1;
1608 
1609  if (is_checksummed_file(readfilename, filename))
1610  {
1611  verify_checksum = true;
1612 
1613  /*
1614  * Cut off at the segment boundary (".") to get the segment number
1615  * in order to mix it into the checksum.
1616  */
1617  segmentpath = strstr(filename, ".");
1618  if (segmentpath != NULL)
1619  {
1620  segmentno = atoi(segmentpath + 1);
1621  if (segmentno == 0)
1622  ereport(ERROR,
1623  (errmsg("invalid segment number %d in file \"%s\"",
1624  segmentno, filename)));
1625  }
1626  }
1627  }
1628 
1629  /*
1630  * Loop until we read the amount of data the caller told us to expect. The
1631  * file could be longer, if it was extended while we were sending it, but
1632  * for a base backup we can ignore such extended data. It will be restored
1633  * from WAL.
1634  */
1635  while (len < statbuf->st_size)
1636  {
1637  /* Try to read some more data. */
1638  cnt = basebackup_read_file(fd, buf,
1639  Min(sizeof(buf), statbuf->st_size - len),
1640  len, readfilename, true);
1641 
1642  /*
1643  * If we hit end-of-file, a concurrent truncation must have occurred.
1644  * That's not an error condition, because WAL replay will fix things
1645  * up.
1646  */
1647  if (cnt == 0)
1648  break;
1649 
1650  /*
1651  * The checksums are verified at block level, so we iterate over the
1652  * buffer in chunks of BLCKSZ, after making sure that
1653  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1654  * BLCKSZ bytes.
1655  */
1656  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1657 
1658  if (verify_checksum && (cnt % BLCKSZ != 0))
1659  {
1660  ereport(WARNING,
1661  (errmsg("could not verify checksum in file \"%s\", block "
1662  "%d: read buffer size %d and page size %d "
1663  "differ",
1664  readfilename, blkno, (int) cnt, BLCKSZ)));
1665  verify_checksum = false;
1666  }
1667 
1668  if (verify_checksum)
1669  {
1670  for (i = 0; i < cnt / BLCKSZ; i++)
1671  {
1672  page = buf + BLCKSZ * i;
1673 
1674  /*
1675  * Only check pages which have not been modified since the
1676  * start of the base backup. Otherwise, they might have been
1677  * written only halfway and the checksum would not be valid.
1678  * However, replaying WAL would reinstate the correct page in
1679  * this case. We also skip completely new pages, since they
1680  * don't have a checksum yet.
1681  */
1682  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1683  {
1684  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1685  phdr = (PageHeader) page;
1686  if (phdr->pd_checksum != checksum)
1687  {
1688  /*
1689  * Retry the block on the first failure. It's
1690  * possible that we read the first 4K page of the
1691  * block just before postgres updated the entire block
1692  * so it ends up looking torn to us. We only need to
1693  * retry once because the LSN should be updated to
1694  * something we can ignore on the next pass. If the
1695  * error happens again then it is a true validation
1696  * failure.
1697  */
1698  if (block_retry == false)
1699  {
1700  int reread_cnt;
1701 
1702  /* Reread the failed block */
1703  reread_cnt =
1704  basebackup_read_file(fd, buf + BLCKSZ * i,
1705  BLCKSZ, len + BLCKSZ * i,
1706  readfilename,
1707  false);
1708  if (reread_cnt == 0)
1709  {
1710  /*
1711  * If we hit end-of-file, a concurrent
1712  * truncation must have occurred, so break out
1713  * of this loop just as if the initial fread()
1714  * returned 0. We'll drop through to the same
1715  * code that handles that case. (We must fix
1716  * up cnt first, though.)
1717  */
1718  cnt = BLCKSZ * i;
1719  break;
1720  }
1721 
1722  /* Set flag so we know a retry was attempted */
1723  block_retry = true;
1724 
1725  /* Reset loop to validate the block again */
1726  i--;
1727  continue;
1728  }
1729 
1730  checksum_failures++;
1731 
1732  if (checksum_failures <= 5)
1733  ereport(WARNING,
1734  (errmsg("checksum verification failed in "
1735  "file \"%s\", block %d: calculated "
1736  "%X but expected %X",
1737  readfilename, blkno, checksum,
1738  phdr->pd_checksum)));
1739  if (checksum_failures == 5)
1740  ereport(WARNING,
1741  (errmsg("further checksum verification "
1742  "failures in file \"%s\" will not "
1743  "be reported", readfilename)));
1744  }
1745  }
1746  block_retry = false;
1747  blkno++;
1748  }
1749  }
1750 
1751  /* Send the chunk as a CopyData message */
1752  if (pq_putmessage('d', buf, cnt))
1753  ereport(ERROR,
1754  (errmsg("base backup could not send data, aborting backup")));
1756 
1757  /* Also feed it to the checksum machinery. */
1758  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1759 
1760  len += cnt;
1761  throttle(cnt);
1762  }
1763 
1764  /* If the file was truncated while we were sending it, pad it with zeros */
1765  if (len < statbuf->st_size)
1766  {
1767  MemSet(buf, 0, sizeof(buf));
1768  while (len < statbuf->st_size)
1769  {
1770  cnt = Min(sizeof(buf), statbuf->st_size - len);
1771  pq_putmessage('d', buf, cnt);
1772  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1774  len += cnt;
1775  throttle(cnt);
1776  }
1777  }
1778 
1779  /*
1780  * Pad to a block boundary, per tar format requirements. (This small
1781  * piece of data is probably not worth throttling, and is not checksummed
1782  * because it's not actually part of the file.)
1783  */
1784  pad = tarPaddingBytesRequired(len);
1785  if (pad > 0)
1786  {
1787  MemSet(buf, 0, pad);
1788  pq_putmessage('d', buf, pad);
1790  }
1791 
1792  CloseTransientFile(fd);
1793 
1794  if (checksum_failures > 1)
1795  {
1796  ereport(WARNING,
1797  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1798  "file \"%s\" has a total of %d checksum verification failures",
1799  checksum_failures,
1800  readfilename, checksum_failures)));
1801 
1802  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1803  }
1804 
1805  total_checksum_failures += checksum_failures;
1806 
1807  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1808  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1809 
1810  return true;
1811 }
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:1989
static void throttle(size_t increment)
Definition: basebackup.c:1878
int64 pg_time_t
Definition: pgtime.h:23
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:934
bool DataChecksumsEnabled(void)
Definition: xlog.c:4930
#define Min(x, y)
Definition: c.h:927
unsigned char uint8
Definition: c.h:372
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define MemSet(start, val, len)
Definition: c.h:978
uint32 BlockNumber
Definition: block.h:31
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1240
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1585
uint16 pd_checksum
Definition: bufpage.h:156
#define pgoff_t
Definition: win32_port.h:194
unsigned short uint16
Definition: c.h:373
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define ERROR
Definition: elog.h:43
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2372
static bool noverify_checksums
Definition: basebackup.c:122
static char * buf
Definition: pg_test_fsync.c:67
int errcode_for_file_access(void)
Definition: elog.c:633
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
int CloseTransientFile(int fd)
Definition: fd.c:2549
#define WARNING
Definition: elog.h:40
pg_checksum_type checksum_type
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define ereport(elevel,...)
Definition: elog.h:144
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1517
PageHeaderData * PageHeader
Definition: bufpage.h:166
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1952
#define Assert(condition)
Definition: c.h:745
#define PageGetLSN(page)
Definition: bufpage.h:366
static char * filename
Definition: pg_dumpall.c:90
#define PageIsNew(page)
Definition: bufpage.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define TAR_SEND_SIZE
Definition: basebackup.c:96
int i
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
uint16 pg_checksum_page(char *page, BlockNumber blkno)
static XLogRecPtr startptr
Definition: basebackup.c:116
static long long int total_checksum_failures
Definition: basebackup.c:119

◆ sendFileWithContent()

static void sendFileWithContent ( const char *  filename,
const char *  content,
backup_manifest_info manifest 
)
static

Definition at line 1086 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), buf, backup_manifest_info::checksum_type, MemSet, pg_checksum_init(), pg_checksum_update(), pg_file_create_mode, pq_putmessage, stat, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and update_basebackup_progress().

Referenced by perform_base_backup().

1088 {
1089  struct stat statbuf;
1090  int pad,
1091  len;
1092  pg_checksum_context checksum_ctx;
1093 
1094  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1095 
1096  len = strlen(content);
1097 
1098  /*
1099  * Construct a stat struct for the backup_label file we're injecting in
1100  * the tar.
1101  */
1102  /* Windows doesn't have the concept of uid and gid */
1103 #ifdef WIN32
1104  statbuf.st_uid = 0;
1105  statbuf.st_gid = 0;
1106 #else
1107  statbuf.st_uid = geteuid();
1108  statbuf.st_gid = getegid();
1109 #endif
1110  statbuf.st_mtime = time(NULL);
1111  statbuf.st_mode = pg_file_create_mode;
1112  statbuf.st_size = len;
1113 
1114  _tarWriteHeader(filename, NULL, &statbuf, false);
1115  /* Send the contents as a CopyData message */
1116  pq_putmessage('d', content, len);
1118 
1119  /* Pad to a multiple of the tar block size. */
1120  pad = tarPaddingBytesRequired(len);
1121  if (pad > 0)
1122  {
1123  char buf[TAR_BLOCK_SIZE];
1124 
1125  MemSet(buf, 0, pad);
1126  pq_putmessage('d', buf, pad);
1128  }
1129 
1130  pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
1131  AddFileToBackupManifest(manifest, NULL, filename, len,
1132  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1133 }
int pg_file_create_mode
Definition: file_perm.c:19
int64 pg_time_t
Definition: pgtime.h:23
unsigned char uint8
Definition: c.h:372
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define MemSet(start, val, len)
Definition: c.h:978
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
static char * buf
Definition: pg_test_fsync.c:67
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
#define stat(a, b)
Definition: win32_port.h:255
pg_checksum_type checksum_type
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1952
static char * filename
Definition: pg_dumpall.c:90
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ sendTablespace()

static int64 sendTablespace ( char *  path,
char *  oid,
bool  sizeonly,
struct backup_manifest_info manifest 
)
static

Definition at line 1143 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, sendDir(), snprintf, stat, and TABLESPACE_VERSION_DIRECTORY.

Referenced by perform_base_backup().

1145 {
1146  int64 size;
1147  char pathbuf[MAXPGPATH];
1148  struct stat statbuf;
1149 
1150  /*
1151  * 'path' points to the tablespace location, but we only want to include
1152  * the version directory in it that belongs to us.
1153  */
1154  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1156 
1157  /*
1158  * Store a directory entry in the tar file so we get the permissions
1159  * right.
1160  */
1161  if (lstat(pathbuf, &statbuf) != 0)
1162  {
1163  if (errno != ENOENT)
1164  ereport(ERROR,
1166  errmsg("could not stat file or directory \"%s\": %m",
1167  pathbuf)));
1168 
1169  /* If the tablespace went away while scanning, it's no error. */
1170  return 0;
1171  }
1172 
1173  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1174  sizeonly);
1175 
1176  /* Send all the files in the tablespace version directory */
1177  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1178  spcoid);
1179 
1180  return size;
1181 }
#define NIL
Definition: pg_list.h:65
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
int errcode_for_file_access(void)
Definition: elog.c:633
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
#define stat(a, b)
Definition: win32_port.h:255
#define ereport(elevel,...)
Definition: elog.h:144
#define lstat(path, sb)
Definition: win32_port.h:244
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1196
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define snprintf
Definition: port.h:193

◆ SendXlogRecPtrResult()

static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 1031 of file basebackup.c.

References buf, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), snprintf, and generate_unaccent_rules::str.

Referenced by perform_base_backup().

1032 {
1034  char str[MAXFNAMELEN];
1035  Size len;
1036 
1037  pq_beginmessage(&buf, 'T'); /* RowDescription */
1038  pq_sendint16(&buf, 2); /* 2 fields */
1039 
1040  /* Field headers */
1041  pq_sendstring(&buf, "recptr");
1042  pq_sendint32(&buf, 0); /* table oid */
1043  pq_sendint16(&buf, 0); /* attnum */
1044  pq_sendint32(&buf, TEXTOID); /* type oid */
1045  pq_sendint16(&buf, -1);
1046  pq_sendint32(&buf, 0);
1047  pq_sendint16(&buf, 0);
1048 
1049  pq_sendstring(&buf, "tli");
1050  pq_sendint32(&buf, 0); /* table oid */
1051  pq_sendint16(&buf, 0); /* attnum */
1052 
1053  /*
1054  * int8 may seem like a surprising data type for this, but in theory int4
1055  * would not be wide enough for this, as TimeLineID is unsigned.
1056  */
1057  pq_sendint32(&buf, INT8OID); /* type oid */
1058  pq_sendint16(&buf, -1);
1059  pq_sendint32(&buf, 0);
1060  pq_sendint16(&buf, 0);
1061  pq_endmessage(&buf);
1062 
1063  /* Data row */
1064  pq_beginmessage(&buf, 'D');
1065  pq_sendint16(&buf, 2); /* number of columns */
1066 
1067  len = snprintf(str, sizeof(str),
1068  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
1069  pq_sendint32(&buf, len);
1070  pq_sendbytes(&buf, str, len);
1071 
1072  len = snprintf(str, sizeof(str), "%u", tli);
1073  pq_sendint32(&buf, len);
1074  pq_sendbytes(&buf, str, len);
1075 
1076  pq_endmessage(&buf);
1077 
1078  /* Send a CommandComplete message */
1079  pq_puttextmessage('C', "SELECT");
1080 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:374
#define MAXFNAMELEN
size_t Size
Definition: c.h:473
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
#define snprintf
Definition: port.h:193

◆ throttle()

static void throttle ( size_t  increment)
static

Definition at line 1878 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_EXIT_ON_PM_DEATH, WL_LATCH_SET, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1879 {
1880  TimeOffset elapsed_min;
1881 
1882  if (throttling_counter < 0)
1883  return;
1884 
1885  throttling_counter += increment;
1887  return;
1888 
1889  /* How much time should have elapsed at minimum? */
1890  elapsed_min = elapsed_min_unit *
1892 
1893  /*
1894  * Since the latch could be set repeatedly because of concurrently WAL
1895  * activity, sleep in a loop to ensure enough time has passed.
1896  */
1897  for (;;)
1898  {
1899  TimeOffset elapsed,
1900  sleep;
1901  int wait_result;
1902 
1903  /* Time elapsed since the last measurement (and possible wake up). */
1904  elapsed = GetCurrentTimestamp() - throttled_last;
1905 
1906  /* sleep if the transfer is faster than it should be */
1907  sleep = elapsed_min - elapsed;
1908  if (sleep <= 0)
1909  break;
1910 
1912 
1913  /* We're eating a potentially set latch, so check for interrupts */
1915 
1916  /*
1917  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1918  * the maximum time to sleep. Thus the cast to long is safe.
1919  */
1920  wait_result = WaitLatch(MyLatch,
1922  (long) (sleep / 1000),
1924 
1925  if (wait_result & WL_LATCH_SET)
1927 
1928  /* Done waiting? */
1929  if (wait_result & WL_TIMEOUT)
1930  break;
1931  }
1932 
1933  /*
1934  * As we work with integers, only whole multiple of throttling_sample was
1935  * processed. The rest will be done during the next call of this function.
1936  */
1938 
1939  /*
1940  * Time interval for the remaining amount and possible next increments
1941  * starts now.
1942  */
1944 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1574
void ResetLatch(Latch *latch)
Definition: latch.c:588
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:390
int64 TimeOffset
Definition: timestamp.h:40
static TimestampTz throttled_last
Definition: basebackup.c:113
struct Latch * MyLatch
Definition: globals.c:54
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
static int64 throttling_counter
Definition: basebackup.c:107
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:104
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129

◆ update_basebackup_progress()

static void update_basebackup_progress ( int64  delta)
static

Definition at line 1952 of file basebackup.c.

References backup_streamed, backup_total, pgstat_progress_update_multi_param(), PROGRESS_BASEBACKUP_BACKUP_STREAMED, PROGRESS_BASEBACKUP_BACKUP_TOTAL, and val.

Referenced by _tarWriteHeader(), perform_base_backup(), sendFile(), and sendFileWithContent().

1953 {
1954  const int index[] = {
1957  };
1958  int64 val[2];
1959  int nparam = 0;
1960 
1961  backup_streamed += delta;
1962  val[nparam++] = backup_streamed;
1963 
1964  /*
1965  * Avoid overflowing past 100% or the full size. This may make the total
1966  * size number change as we approach the end of the backup (the estimate
1967  * will always be wrong if WAL is included), but that's better than having
1968  * the done column be bigger than the total.
1969  */
1971  {
1973  val[nparam++] = backup_total;
1974  }
1975 
1976  pgstat_progress_update_multi_param(nparam, index, val);
1977 }
static int64 backup_total
Definition: basebackup.c:128
Definition: type.h:89
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED
Definition: progress.h:125
static int64 backup_streamed
Definition: basebackup.c:131
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3253
long val
Definition: informix.c:664

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 88 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

◆ backup_streamed

int64 backup_streamed = 0
static

Definition at line 131 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ backup_total

int64 backup_total = 0
static

Definition at line 128 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ elapsed_min_unit

TimeOffset elapsed_min_unit
static

Definition at line 110 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 153 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{PG_AUTOCONF_FILENAME ".tmp", false},
{TABLESPACE_MAP, false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}
#define TABLESPACE_MAP
Definition: xlog.h:392
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:96
#define BACKUP_LABEL_FILE
Definition: xlog.h:389

Definition at line 194 of file basebackup.c.

◆ noChecksumFiles

const struct exclude_list_item noChecksumFiles[]
static
Initial value:
= {
{"pg_control", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
{NULL, false}
}

Definition at line 238 of file basebackup.c.

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 122 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ startptr

◆ statrelpath

char* statrelpath = NULL
static

Definition at line 91 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

◆ throttled_last

TimestampTz throttled_last
static

Definition at line 113 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_counter

int64 throttling_counter
static

Definition at line 107 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_sample

uint64 throttling_sample
static

Definition at line 104 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 119 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().