PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/pg_type.h"
#include "common/file_perm.h"
#include "commands/progress.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/backup_manifest.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 
struct  exclude_list_item
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 

Functions

static int64 sendTablespace (char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
 
static int64 sendDir (const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
 
static bool sendFile (const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 
static void sendFileWithContent (const char *filename, const char *content, backup_manifest_info *manifest)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void perform_base_backup (basebackup_options *opt)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static void throttle (size_t increment)
 
static void update_basebackup_progress (int64 delta)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
static int basebackup_read_file (int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static XLogRecPtr startptr
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static int64 backup_total = 0
 
static int64 backup_streamed = 0
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 
static const struct exclude_list_item noChecksumFiles []
 

Macro Definition Documentation

◆ TAR_SEND_SIZE

#define TAR_SEND_SIZE   32768

Definition at line 96 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ THROTTLING_FREQUENCY

#define THROTTLING_FREQUENCY   8

Definition at line 101 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

◆ _tarWriteDir()

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1877 of file basebackup.c.

References _tarWriteHeader(), pg_dir_create_mode, pgwin32_is_junction(), and stat::st_mode.

Referenced by sendDir().

1879 {
1880  /* If symlink, write it as a directory anyway */
1881 #ifndef WIN32
1882  if (S_ISLNK(statbuf->st_mode))
1883 #else
1884  if (pgwin32_is_junction(pathbuf))
1885 #endif
1886  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1887 
1888  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1889 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
int pg_dir_create_mode
Definition: file_perm.c:18
unsigned short st_mode
Definition: win32_port.h:260
bool pgwin32_is_junction(const char *path)

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1834 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, stat::st_uid, TAR_BLOCK_SIZE, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, tarCreateHeader(), and update_basebackup_progress().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1836 {
1837  char h[TAR_BLOCK_SIZE];
1838  enum tarError rc;
1839 
1840  if (!sizeonly)
1841  {
1842  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1843  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1844  statbuf->st_mtime);
1845 
1846  switch (rc)
1847  {
1848  case TAR_OK:
1849  break;
1850  case TAR_NAME_TOO_LONG:
1851  ereport(ERROR,
1852  (errmsg("file name too long for tar format: \"%s\"",
1853  filename)));
1854  break;
1855  case TAR_SYMLINK_TOO_LONG:
1856  ereport(ERROR,
1857  (errmsg("symbolic link target too long for tar format: "
1858  "file name \"%s\", target \"%s\"",
1859  filename, linktarget)));
1860  break;
1861  default:
1862  elog(ERROR, "unrecognized tar error: %d", rc);
1863  }
1864 
1865  pq_putmessage('d', h, sizeof(h));
1866  update_basebackup_progress(sizeof(h));
1867  }
1868 
1869  return sizeof(h);
1870 }
Definition: pgtar.h:21
short st_uid
Definition: win32_port.h:262
tarError
Definition: pgtar.h:19
#define ERROR
Definition: elog.h:45
short st_gid
Definition: win32_port.h:263
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
__int64 st_size
Definition: win32_port.h:265
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
unsigned short st_mode
Definition: win32_port.h:260
#define ereport(elevel,...)
Definition: elog.h:155
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1971
static char * filename
Definition: pg_dumpall.c:91
int errmsg(const char *fmt,...)
Definition: elog.c:915
#define elog(elevel,...)
Definition: elog.h:228
__time64_t st_mtime
Definition: win32_port.h:267
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ basebackup_read_file()

static int basebackup_read_file ( int  fd,
char *  buf,
size_t  nbytes,
off_t  offset,
const char *  filename,
bool  partial_read_ok 
)
static

Definition at line 2008 of file basebackup.c.

References ereport, errcode_for_file_access(), errmsg(), ERROR, pg_pread(), pgstat_report_wait_end(), pgstat_report_wait_start(), and WAIT_EVENT_BASEBACKUP_READ.

Referenced by perform_base_backup(), and sendFile().

2010 {
2011  int rc;
2012 
2014  rc = pg_pread(fd, buf, nbytes, offset);
2016 
2017  if (rc < 0)
2018  ereport(ERROR,
2020  errmsg("could not read file \"%s\": %m", filename)));
2021  if (!partial_read_ok && rc > 0 && rc != nbytes)
2022  ereport(ERROR,
2024  errmsg("could not read file \"%s\": read %d of %zu",
2025  filename, rc, nbytes)));
2026 
2027  return rc;
2028 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset)
Definition: pread.c:27
#define ERROR
Definition: elog.h:45
static char * buf
Definition: pg_test_fsync.c:68
int errcode_for_file_access(void)
Definition: elog.c:727
static void pgstat_report_wait_end(void)
Definition: pgstat.h:1512
#define ereport(elevel,...)
Definition: elog.h:155
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: pgstat.h:1488
static char * filename
Definition: pg_dumpall.c:91
int errmsg(const char *fmt,...)
Definition: elog.c:915

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 750 of file basebackup.c.

References lfirst.

Referenced by perform_base_backup().

751 {
752  char *fna = (char *) lfirst(a);
753  char *fnb = (char *) lfirst(b);
754 
755  return strcmp(fna + 8, fnb + 8);
756 }
#define lfirst(lc)
Definition: pg_list.h:169

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1532 of file basebackup.c.

References exclude_list_item::match_prefix, and exclude_list_item::name.

Referenced by sendFile().

1533 {
1534  /* Check that the file is in a tablespace */
1535  if (strncmp(fullpath, "./global/", 9) == 0 ||
1536  strncmp(fullpath, "./base/", 7) == 0 ||
1537  strncmp(fullpath, "/", 1) == 0)
1538  {
1539  int excludeIdx;
1540 
1541  /* Compare file against noChecksumFiles skip list */
1542  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1543  {
1544  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1545 
1546  if (!noChecksumFiles[excludeIdx].match_prefix)
1547  cmplen++;
1548  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1549  cmplen) == 0)
1550  return false;
1551  }
1552 
1553  return true;
1554  }
1555  else
1556  return false;
1557 }
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:238
const char * name
Definition: encode.c:515
const char * name
Definition: basebackup.c:141
static char * filename
Definition: pg_dumpall.c:91

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 762 of file basebackup.c.

References DefElem::arg, CHECKSUM_TYPE_CRC32C, CHECKSUM_TYPE_NONE, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, basebackup_options::manifest, basebackup_options::manifest_checksum_type, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, MANIFEST_OPTION_YES, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, parse_bool(), pg_checksum_parse_type(), pg_strcasecmp(), basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

763 {
764  ListCell *lopt;
765  bool o_label = false;
766  bool o_progress = false;
767  bool o_fast = false;
768  bool o_nowait = false;
769  bool o_wal = false;
770  bool o_maxrate = false;
771  bool o_tablespace_map = false;
772  bool o_noverify_checksums = false;
773  bool o_manifest = false;
774  bool o_manifest_checksums = false;
775 
776  MemSet(opt, 0, sizeof(*opt));
779 
780  foreach(lopt, options)
781  {
782  DefElem *defel = (DefElem *) lfirst(lopt);
783 
784  if (strcmp(defel->defname, "label") == 0)
785  {
786  if (o_label)
787  ereport(ERROR,
788  (errcode(ERRCODE_SYNTAX_ERROR),
789  errmsg("duplicate option \"%s\"", defel->defname)));
790  opt->label = strVal(defel->arg);
791  o_label = true;
792  }
793  else if (strcmp(defel->defname, "progress") == 0)
794  {
795  if (o_progress)
796  ereport(ERROR,
797  (errcode(ERRCODE_SYNTAX_ERROR),
798  errmsg("duplicate option \"%s\"", defel->defname)));
799  opt->progress = true;
800  o_progress = true;
801  }
802  else if (strcmp(defel->defname, "fast") == 0)
803  {
804  if (o_fast)
805  ereport(ERROR,
806  (errcode(ERRCODE_SYNTAX_ERROR),
807  errmsg("duplicate option \"%s\"", defel->defname)));
808  opt->fastcheckpoint = true;
809  o_fast = true;
810  }
811  else if (strcmp(defel->defname, "nowait") == 0)
812  {
813  if (o_nowait)
814  ereport(ERROR,
815  (errcode(ERRCODE_SYNTAX_ERROR),
816  errmsg("duplicate option \"%s\"", defel->defname)));
817  opt->nowait = true;
818  o_nowait = true;
819  }
820  else if (strcmp(defel->defname, "wal") == 0)
821  {
822  if (o_wal)
823  ereport(ERROR,
824  (errcode(ERRCODE_SYNTAX_ERROR),
825  errmsg("duplicate option \"%s\"", defel->defname)));
826  opt->includewal = true;
827  o_wal = true;
828  }
829  else if (strcmp(defel->defname, "max_rate") == 0)
830  {
831  long maxrate;
832 
833  if (o_maxrate)
834  ereport(ERROR,
835  (errcode(ERRCODE_SYNTAX_ERROR),
836  errmsg("duplicate option \"%s\"", defel->defname)));
837 
838  maxrate = intVal(defel->arg);
839  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
840  ereport(ERROR,
841  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
842  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
843  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
844 
845  opt->maxrate = (uint32) maxrate;
846  o_maxrate = true;
847  }
848  else if (strcmp(defel->defname, "tablespace_map") == 0)
849  {
850  if (o_tablespace_map)
851  ereport(ERROR,
852  (errcode(ERRCODE_SYNTAX_ERROR),
853  errmsg("duplicate option \"%s\"", defel->defname)));
854  opt->sendtblspcmapfile = true;
855  o_tablespace_map = true;
856  }
857  else if (strcmp(defel->defname, "noverify_checksums") == 0)
858  {
859  if (o_noverify_checksums)
860  ereport(ERROR,
861  (errcode(ERRCODE_SYNTAX_ERROR),
862  errmsg("duplicate option \"%s\"", defel->defname)));
863  noverify_checksums = true;
864  o_noverify_checksums = true;
865  }
866  else if (strcmp(defel->defname, "manifest") == 0)
867  {
868  char *optval = strVal(defel->arg);
869  bool manifest_bool;
870 
871  if (o_manifest)
872  ereport(ERROR,
873  (errcode(ERRCODE_SYNTAX_ERROR),
874  errmsg("duplicate option \"%s\"", defel->defname)));
875  if (parse_bool(optval, &manifest_bool))
876  {
877  if (manifest_bool)
879  else
881  }
882  else if (pg_strcasecmp(optval, "force-encode") == 0)
884  else
885  ereport(ERROR,
886  (errcode(ERRCODE_SYNTAX_ERROR),
887  errmsg("unrecognized manifest option: \"%s\"",
888  optval)));
889  o_manifest = true;
890  }
891  else if (strcmp(defel->defname, "manifest_checksums") == 0)
892  {
893  char *optval = strVal(defel->arg);
894 
895  if (o_manifest_checksums)
896  ereport(ERROR,
897  (errcode(ERRCODE_SYNTAX_ERROR),
898  errmsg("duplicate option \"%s\"", defel->defname)));
899  if (!pg_checksum_parse_type(optval,
900  &opt->manifest_checksum_type))
901  ereport(ERROR,
902  (errcode(ERRCODE_SYNTAX_ERROR),
903  errmsg("unrecognized checksum algorithm: \"%s\"",
904  optval)));
905  o_manifest_checksums = true;
906  }
907  else
908  elog(ERROR, "option \"%s\" not recognized",
909  defel->defname);
910  }
911  if (opt->label == NULL)
912  opt->label = "base backup";
913  if (opt->manifest == MANIFEST_OPTION_NO)
914  {
915  if (o_manifest_checksums)
916  ereport(ERROR,
917  (errcode(ERRCODE_SYNTAX_ERROR),
918  errmsg("manifest checksums require a backup manifest")));
920  }
921 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:704
#define MemSet(start, val, len)
Definition: c.h:996
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:45
backup_manifest_option manifest
Definition: basebackup.c:57
static bool noverify_checksums
Definition: basebackup.c:122
static int32 maxrate
const char * label
Definition: basebackup.c:50
unsigned int uint32
Definition: c.h:429
Node * arg
Definition: parsenodes.h:734
#define ereport(elevel,...)
Definition: elog.h:155
#define lfirst(lc)
Definition: pg_list.h:169
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:915
#define elog(elevel,...)
Definition: elog.h:228
char * defname
Definition: parsenodes.h:733
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt)
static

Definition at line 256 of file basebackup.c.

References _tarWriteHeader(), AddWALInfoToBackupManifest(), AllocateDir(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, backup_streamed, backup_total, basebackup_read_file(), BoolGetDatum, buf, CheckXLogRemoved(), CloseTransientFile(), compareWalFileNames(), CurrentResourceOwner, dirent::d_name, StringInfoData::data, DataDir, do_pg_abort_backup(), do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, basebackup_options::fastcheckpoint, fd(), FreeBackupManifest(), FreeDir(), fstat, GetCurrentTimestamp(), basebackup_options::includewal, InitializeBackupManifest(), InvalidOid, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, linitial, list_length(), list_sort(), lnext(), lstat, makeStringInfo(), basebackup_options::manifest, manifest, basebackup_options::manifest_checksum_type, MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, tablespaceinfo::oid, OpenTransientFile(), palloc0(), tablespaceinfo::path, PG_BINARY, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_progress_end_command(), pgstat_progress_start_command(), pgstat_progress_update_multi_param(), pgstat_progress_update_param(), pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint16(), basebackup_options::progress, PROGRESS_BASEBACKUP_BACKUP_TOTAL, PROGRESS_BASEBACKUP_PHASE, PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE, PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP, PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL, PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT, PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE, PROGRESS_BASEBACKUP_TBLSPC_STREAMED, PROGRESS_BASEBACKUP_TBLSPC_TOTAL, PROGRESS_COMMAND_BASEBACKUP, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), ResourceOwnerCreate(), SendBackupHeader(), SendBackupManifest(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf, stat::st_size, startptr, statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_BLOCK_SIZE, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, total_checksum_failures, update_basebackup_progress(), USECS_PER_SEC, val, wal_segment_size, WalSndResourceCleanup(), WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

257 {
258  TimeLineID starttli;
259  XLogRecPtr endptr;
260  TimeLineID endtli;
261  StringInfo labelfile;
264  int datadirpathlen;
265  List *tablespaces = NIL;
266 
267  backup_total = 0;
268  backup_streamed = 0;
270 
271  /*
272  * If the estimation of the total backup size is disabled, make the
273  * backup_total column in the view return NULL by setting the parameter to
274  * -1.
275  */
276  if (!opt->progress)
277  {
278  backup_total = -1;
280  backup_total);
281  }
282 
283  /* we're going to use a BufFile, so we need a ResourceOwner */
284  Assert(CurrentResourceOwner == NULL);
285  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
286 
287  datadirpathlen = strlen(DataDir);
288 
290 
291  labelfile = makeStringInfo();
292  tblspc_map_file = makeStringInfo();
293  InitializeBackupManifest(&manifest, opt->manifest,
295 
297 
300  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
301  labelfile, &tablespaces,
302  tblspc_map_file, opt->sendtblspcmapfile);
303 
304  /*
305  * Once do_pg_start_backup has been called, ensure that any failure causes
306  * us to abort the backup so we don't "leak" a backup counter. For this
307  * reason, *all* functionality between do_pg_start_backup() and the end of
308  * do_pg_stop_backup() should be inside the error cleanup block!
309  */
310 
312  {
313  ListCell *lc;
314  tablespaceinfo *ti;
315  int tblspc_streamed = 0;
316 
317  /*
318  * Calculate the relative path of temporary statistics directory in
319  * order to skip the files which are located in that directory later.
320  */
322  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
323  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
324  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
326  else
328 
329  /* Add a node for the base directory at the end */
330  ti = palloc0(sizeof(tablespaceinfo));
331  ti->size = -1;
332  tablespaces = lappend(tablespaces, ti);
333 
334  /*
335  * Calculate the total backup size by summing up the size of each
336  * tablespace
337  */
338  if (opt->progress)
339  {
342 
343  foreach(lc, tablespaces)
344  {
345  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
346 
347  if (tmp->path == NULL)
348  tmp->size = sendDir(".", 1, true, tablespaces, true, NULL,
349  NULL);
350  else
351  tmp->size = sendTablespace(tmp->path, tmp->oid, true,
352  NULL);
353  backup_total += tmp->size;
354  }
355  }
356 
357  /* Report that we are now streaming database files as a base backup */
358  {
359  const int index[] = {
363  };
364  const int64 val[] = {
366  backup_total, list_length(tablespaces)
367  };
368 
369  pgstat_progress_update_multi_param(3, index, val);
370  }
371 
372  /* Send the starting position of the backup */
373  SendXlogRecPtrResult(startptr, starttli);
374 
375  /* Send tablespace header */
376  SendBackupHeader(tablespaces);
377 
378  /* Setup and activate network throttling, if client requested it */
379  if (opt->maxrate > 0)
380  {
382  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
383 
384  /*
385  * The minimum amount of time for throttling_sample bytes to be
386  * transferred.
387  */
389 
390  /* Enable throttling. */
391  throttling_counter = 0;
392 
393  /* The 'real data' starts now (header was ignored). */
395  }
396  else
397  {
398  /* Disable throttling. */
399  throttling_counter = -1;
400  }
401 
402  /* Send off our tablespaces one by one */
403  foreach(lc, tablespaces)
404  {
405  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
407 
408  /* Send CopyOutResponse message */
409  pq_beginmessage(&buf, 'H');
410  pq_sendbyte(&buf, 0); /* overall format */
411  pq_sendint16(&buf, 0); /* natts */
412  pq_endmessage(&buf);
413 
414  if (ti->path == NULL)
415  {
416  struct stat statbuf;
417  bool sendtblspclinks = true;
418 
419  /* In the main tar, include the backup_label first... */
421  &manifest);
422 
423  /* Then the tablespace_map file, if required... */
424  if (opt->sendtblspcmapfile)
425  {
426  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
427  &manifest);
428  sendtblspclinks = false;
429  }
430 
431  /* Then the bulk of the files... */
432  sendDir(".", 1, false, tablespaces, sendtblspclinks,
433  &manifest, NULL);
434 
435  /* ... and pg_control after everything else. */
436  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
437  ereport(ERROR,
439  errmsg("could not stat file \"%s\": %m",
442  false, InvalidOid, &manifest, NULL);
443  }
444  else
445  sendTablespace(ti->path, ti->oid, false, &manifest);
446 
447  /*
448  * If we're including WAL, and this is the main data directory we
449  * don't terminate the tar stream here. Instead, we will append
450  * the xlog files below and terminate it then. This is safe since
451  * the main data directory is always sent *last*.
452  */
453  if (opt->includewal && ti->path == NULL)
454  {
455  Assert(lnext(tablespaces, lc) == NULL);
456  }
457  else
458  pq_putemptymessage('c'); /* CopyDone */
459 
460  tblspc_streamed++;
462  tblspc_streamed);
463  }
464 
467  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
468  }
470 
471 
472  if (opt->includewal)
473  {
474  /*
475  * We've left the last tar file "open", so we can now append the
476  * required WAL files to it.
477  */
478  char pathbuf[MAXPGPATH];
479  XLogSegNo segno;
480  XLogSegNo startsegno;
481  XLogSegNo endsegno;
482  struct stat statbuf;
483  List *historyFileList = NIL;
484  List *walFileList = NIL;
485  char firstoff[MAXFNAMELEN];
486  char lastoff[MAXFNAMELEN];
487  DIR *dir;
488  struct dirent *de;
489  ListCell *lc;
490  TimeLineID tli;
491 
494 
495  /*
496  * I'd rather not worry about timelines here, so scan pg_wal and
497  * include all WAL files in the range between 'startptr' and 'endptr',
498  * regardless of the timeline the file is stamped with. If there are
499  * some spurious WAL files belonging to timelines that don't belong in
500  * this server's history, they will be included too. Normally there
501  * shouldn't be such files, but if there are, there's little harm in
502  * including them.
503  */
504  XLByteToSeg(startptr, startsegno, wal_segment_size);
505  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
506  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
507  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
508 
509  dir = AllocateDir("pg_wal");
510  while ((de = ReadDir(dir, "pg_wal")) != NULL)
511  {
512  /* Does it look like a WAL segment, and is it in the range? */
513  if (IsXLogFileName(de->d_name) &&
514  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
515  strcmp(de->d_name + 8, lastoff + 8) <= 0)
516  {
517  walFileList = lappend(walFileList, pstrdup(de->d_name));
518  }
519  /* Does it look like a timeline history file? */
520  else if (IsTLHistoryFileName(de->d_name))
521  {
522  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
523  }
524  }
525  FreeDir(dir);
526 
527  /*
528  * Before we go any further, check that none of the WAL segments we
529  * need were removed.
530  */
531  CheckXLogRemoved(startsegno, ThisTimeLineID);
532 
533  /*
534  * Sort the WAL filenames. We want to send the files in order from
535  * oldest to newest, to reduce the chance that a file is recycled
536  * before we get a chance to send it over.
537  */
538  list_sort(walFileList, compareWalFileNames);
539 
540  /*
541  * There must be at least one xlog file in the pg_wal directory, since
542  * we are doing backup-including-xlog.
543  */
544  if (walFileList == NIL)
545  ereport(ERROR,
546  (errmsg("could not find any WAL files")));
547 
548  /*
549  * Sanity check: the first and last segment should cover startptr and
550  * endptr, with no gaps in between.
551  */
552  XLogFromFileName((char *) linitial(walFileList),
553  &tli, &segno, wal_segment_size);
554  if (segno != startsegno)
555  {
556  char startfname[MAXFNAMELEN];
557 
558  XLogFileName(startfname, ThisTimeLineID, startsegno,
560  ereport(ERROR,
561  (errmsg("could not find WAL file \"%s\"", startfname)));
562  }
563  foreach(lc, walFileList)
564  {
565  char *walFileName = (char *) lfirst(lc);
566  XLogSegNo currsegno = segno;
567  XLogSegNo nextsegno = segno + 1;
568 
569  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
570  if (!(nextsegno == segno || currsegno == segno))
571  {
572  char nextfname[MAXFNAMELEN];
573 
574  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
576  ereport(ERROR,
577  (errmsg("could not find WAL file \"%s\"", nextfname)));
578  }
579  }
580  if (segno != endsegno)
581  {
582  char endfname[MAXFNAMELEN];
583 
584  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
585  ereport(ERROR,
586  (errmsg("could not find WAL file \"%s\"", endfname)));
587  }
588 
589  /* Ok, we have everything we need. Send the WAL files. */
590  foreach(lc, walFileList)
591  {
592  char *walFileName = (char *) lfirst(lc);
593  int fd;
594  char buf[TAR_SEND_SIZE];
595  size_t cnt;
596  pgoff_t len = 0;
597 
598  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
599  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
600 
601  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
602  if (fd < 0)
603  {
604  int save_errno = errno;
605 
606  /*
607  * Most likely reason for this is that the file was already
608  * removed by a checkpoint, so check for that to get a better
609  * error message.
610  */
611  CheckXLogRemoved(segno, tli);
612 
613  errno = save_errno;
614  ereport(ERROR,
616  errmsg("could not open file \"%s\": %m", pathbuf)));
617  }
618 
619  if (fstat(fd, &statbuf) != 0)
620  ereport(ERROR,
622  errmsg("could not stat file \"%s\": %m",
623  pathbuf)));
624  if (statbuf.st_size != wal_segment_size)
625  {
626  CheckXLogRemoved(segno, tli);
627  ereport(ERROR,
629  errmsg("unexpected WAL file size \"%s\"", walFileName)));
630  }
631 
632  /* send the WAL file itself */
633  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
634 
635  while ((cnt = basebackup_read_file(fd, buf,
636  Min(sizeof(buf),
637  wal_segment_size - len),
638  len, pathbuf, true)) > 0)
639  {
640  CheckXLogRemoved(segno, tli);
641  /* Send the chunk as a CopyData message */
642  if (pq_putmessage('d', buf, cnt))
643  ereport(ERROR,
644  (errmsg("base backup could not send data, aborting backup")));
646 
647  len += cnt;
648  throttle(cnt);
649 
650  if (len == wal_segment_size)
651  break;
652  }
653 
654  if (len != wal_segment_size)
655  {
656  CheckXLogRemoved(segno, tli);
657  ereport(ERROR,
659  errmsg("unexpected WAL file size \"%s\"", walFileName)));
660  }
661 
662  /*
663  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
664  * for padding.
665  */
667 
668  CloseTransientFile(fd);
669 
670  /*
671  * Mark file as archived, otherwise files can get archived again
672  * after promotion of a new node. This is in line with
673  * walreceiver.c always doing an XLogArchiveForceDone() after a
674  * complete segment.
675  */
676  StatusFilePath(pathbuf, walFileName, ".done");
677  sendFileWithContent(pathbuf, "", &manifest);
678  }
679 
680  /*
681  * Send timeline history files too. Only the latest timeline history
682  * file is required for recovery, and even that only if there happens
683  * to be a timeline switch in the first WAL segment that contains the
684  * checkpoint record, or if we're taking a base backup from a standby
685  * server and the target timeline changes while the backup is taken.
686  * But they are small and highly useful for debugging purposes, so
687  * better include them all, always.
688  */
689  foreach(lc, historyFileList)
690  {
691  char *fname = lfirst(lc);
692 
693  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
694 
695  if (lstat(pathbuf, &statbuf) != 0)
696  ereport(ERROR,
698  errmsg("could not stat file \"%s\": %m", pathbuf)));
699 
700  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
701  &manifest, NULL);
702 
703  /* unconditionally mark file as archived */
704  StatusFilePath(pathbuf, fname, ".done");
705  sendFileWithContent(pathbuf, "", &manifest);
706  }
707 
708  /* Send CopyDone message for the last tar file */
709  pq_putemptymessage('c');
710  }
711 
712  AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
713 
714  SendBackupManifest(&manifest);
715 
716  SendXlogRecPtrResult(endptr, endtli);
717 
719  {
720  if (total_checksum_failures > 1)
722  (errmsg_plural("%lld total checksum verification failure",
723  "%lld total checksum verification failures",
726 
727  ereport(ERROR,
729  errmsg("checksum verification failure during base backup")));
730  }
731 
732  /*
733  * Make sure to free the manifest before the resource owners as manifests
734  * use cryptohash contexts that may depend on resource owners (like
735  * OpenSSL).
736  */
737  FreeBackupManifest(&manifest);
738 
739  /* clean up the resource owner we created */
740  WalSndResourceCleanup(true);
741 
743 }
#define StatusFilePath(path, xlog, suffix)
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:2008
static void throttle(size_t increment)
Definition: basebackup.c:1897
static int64 backup_total
Definition: basebackup.c:128
uint32 TimeLineID
Definition: xlogdefs.h:52
#define PROGRESS_BASEBACKUP_PHASE
Definition: progress.h:123
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11590
int wal_segment_size
Definition: xlog.c:118
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:322
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
Definition: pgstat.c:3457
#define USECS_PER_SEC
Definition: timestamp.h:94
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1025
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1578
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
ResourceOwner CurrentResourceOwner
Definition: resowner.c:144
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3478
char * pstrdup(const char *in)
Definition: mcxt.c:1187
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:974
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:963
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED
Definition: progress.h:127
int errcode(int sqlerrcode)
Definition: elog.c:704
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE
Definition: progress.h:133
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool RecoveryInProgress(void)
Definition: xlog.c:8148
static bool backup_started_in_recovery
Definition: basebackup.c:88
Definition: dirent.h:9
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1259
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool needtblspcmapfile)
Definition: xlog.c:10659
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
Definition: type.h:89
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT
Definition: progress.h:130
#define pgoff_t
Definition: win32_port.h:200
#define TABLESPACE_MAP
Definition: xlog.h:392
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define fstat
Definition: win32_port.h:274
static void sendFileWithContent(const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1096
#define linitial(l)
Definition: pg_list.h:174
Definition: dirent.c:25
#define ERROR
Definition: elog.h:45
#define IsXLogFileName(fname)
backup_manifest_option manifest
Definition: basebackup.c:57
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:11177
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2404
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3956
#define MAXPGPATH
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
static char * buf
Definition: pg_test_fsync.c:68
uint64 XLogSegNo
Definition: xlogdefs.h:41
void SendBackupManifest(backup_manifest_info *manifest)
static int64 backup_streamed
Definition: basebackup.c:131
int errcode_for_file_access(void)
Definition: elog.c:727
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:50
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:1041
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2615
void FreeBackupManifest(backup_manifest_info *manifest)
static int64 sendTablespace(char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
Definition: basebackup.c:1158
char * pgstat_stat_directory
Definition: pgstat.c:135
static char * statrelpath
Definition: basebackup.c:91
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:336
int CloseTransientFile(int fd)
Definition: fd.c:2581
#define WARNING
Definition: elog.h:40
#define MAXFNAMELEN
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void * palloc0(Size size)
Definition: mcxt.c:981
void pgstat_progress_end_command(void)
Definition: pgstat.c:3529
#define XLOGDIR
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL
Definition: progress.h:126
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL
Definition: progress.h:134
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:330
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:193
static TimestampTz throttled_last
Definition: basebackup.c:113
#define ereport(elevel,...)
Definition: elog.h:155
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1971
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:792
#define lfirst(lc)
Definition: pg_list.h:169
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2681
#define THROTTLING_FREQUENCY
Definition: basebackup.c:101
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3500
static bool manifest
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static int list_length(const List *l)
Definition: pg_list.h:149
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
#define PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE
Definition: progress.h:131
#define lstat(path, sb)
Definition: win32_port.h:276
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP
Definition: progress.h:132
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:750
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1211
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1496
int errmsg(const char *fmt,...)
Definition: elog.c:915
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:96
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1578
char * DataDir
Definition: globals.c:63
#define BACKUP_LABEL_FILE
Definition: xlog.h:389
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58
char d_name[MAX_PATH]
Definition: dirent.h:15
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:107
Definition: pg_list.h:50
#define snprintf
Definition: port.h:215
static uint64 throttling_sample
Definition: basebackup.c:104
long val
Definition: informix.c:664
int FreeDir(DIR *dir)
Definition: fd.c:2733
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
static XLogRecPtr startptr
Definition: basebackup.c:116
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:425
static long long int total_checksum_failures
Definition: basebackup.c:119
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)

◆ send_int8_string()

static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 953 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), pq_sendint32(), and sprintf.

Referenced by SendBackupHeader().

954 {
955  char is[32];
956 
957  sprintf(is, INT64_FORMAT, intval);
958  pq_sendint32(buf, strlen(is));
959  pq_sendbytes(buf, is, strlen(is));
960 }
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:217
#define INT64_FORMAT
Definition: c.h:471
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125

◆ SendBackupHeader()

static void SendBackupHeader ( List tablespaces)
static

Definition at line 963 of file basebackup.c.

References buf, lfirst, tablespaceinfo::oid, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), send_int8_string(), and tablespaceinfo::size.

Referenced by perform_base_backup().

964 {
966  ListCell *lc;
967 
968  /* Construct and send the directory information */
969  pq_beginmessage(&buf, 'T'); /* RowDescription */
970  pq_sendint16(&buf, 3); /* 3 fields */
971 
972  /* First field - spcoid */
973  pq_sendstring(&buf, "spcoid");
974  pq_sendint32(&buf, 0); /* table oid */
975  pq_sendint16(&buf, 0); /* attnum */
976  pq_sendint32(&buf, OIDOID); /* type oid */
977  pq_sendint16(&buf, 4); /* typlen */
978  pq_sendint32(&buf, 0); /* typmod */
979  pq_sendint16(&buf, 0); /* format code */
980 
981  /* Second field - spclocation */
982  pq_sendstring(&buf, "spclocation");
983  pq_sendint32(&buf, 0);
984  pq_sendint16(&buf, 0);
985  pq_sendint32(&buf, TEXTOID);
986  pq_sendint16(&buf, -1);
987  pq_sendint32(&buf, 0);
988  pq_sendint16(&buf, 0);
989 
990  /* Third field - size */
991  pq_sendstring(&buf, "size");
992  pq_sendint32(&buf, 0);
993  pq_sendint16(&buf, 0);
994  pq_sendint32(&buf, INT8OID);
995  pq_sendint16(&buf, 8);
996  pq_sendint32(&buf, 0);
997  pq_sendint16(&buf, 0);
998  pq_endmessage(&buf);
999 
1000  foreach(lc, tablespaces)
1001  {
1002  tablespaceinfo *ti = lfirst(lc);
1003 
1004  /* Send one datarow message */
1005  pq_beginmessage(&buf, 'D');
1006  pq_sendint16(&buf, 3); /* number of columns */
1007  if (ti->path == NULL)
1008  {
1009  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
1010  pq_sendint32(&buf, -1);
1011  }
1012  else
1013  {
1014  Size len;
1015 
1016  len = strlen(ti->oid);
1017  pq_sendint32(&buf, len);
1018  pq_sendbytes(&buf, ti->oid, len);
1019 
1020  len = strlen(ti->path);
1021  pq_sendint32(&buf, len);
1022  pq_sendbytes(&buf, ti->path, len);
1023  }
1024  if (ti->size >= 0)
1025  send_int8_string(&buf, ti->size / 1024);
1026  else
1027  pq_sendint32(&buf, -1); /* NULL */
1028 
1029  pq_endmessage(&buf);
1030  }
1031 
1032  /* Send a CommandComplete message */
1033  pq_puttextmessage('C', "SELECT");
1034 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:953
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:68
#define lfirst(lc)
Definition: pg_list.h:169
size_t Size
Definition: c.h:528
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 932 of file basebackup.c.

References basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

933 {
934  basebackup_options opt;
935 
936  parse_basebackup_options(cmd->options, &opt);
937 
939 
941  {
942  char activitymsg[50];
943 
944  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
945  opt.label);
946  set_ps_display(activitymsg);
947  }
948 
949  perform_base_backup(&opt);
950 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:36
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:762
void set_ps_display(const char *activity)
Definition: ps_status.c:349
const char * label
Definition: basebackup.c:50
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:256
void WalSndSetState(WalSndState state)
Definition: walsender.c:3201
#define snprintf
Definition: port.h:215

◆ sendDir()

static int64 sendDir ( const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1211 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, DEBUG2, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, exclude_list_item::match_prefix, MAXPGPATH, exclude_list_item::name, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, pgwin32_is_junction(), ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISREG, sendFile(), snprintf, stat::st_mode, stat::st_size, statrelpath, TABLESPACE_VERSION_DIRECTORY, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

1214 {
1215  DIR *dir;
1216  struct dirent *de;
1217  char pathbuf[MAXPGPATH * 2];
1218  struct stat statbuf;
1219  int64 size = 0;
1220  const char *lastDir; /* Split last dir from parent path. */
1221  bool isDbDir = false; /* Does this directory contain relations? */
1222 
1223  /*
1224  * Determine if the current path is a database directory that can contain
1225  * relations.
1226  *
1227  * Start by finding the location of the delimiter between the parent path
1228  * and the current path.
1229  */
1230  lastDir = last_dir_separator(path);
1231 
1232  /* Does this path look like a database path (i.e. all digits)? */
1233  if (lastDir != NULL &&
1234  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1235  {
1236  /* Part of path that contains the parent directory. */
1237  int parentPathLen = lastDir - path;
1238 
1239  /*
1240  * Mark path as a database directory if the parent path is either
1241  * $PGDATA/base or a tablespace version path.
1242  */
1243  if (strncmp(path, "./base", parentPathLen) == 0 ||
1244  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1245  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1247  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1248  isDbDir = true;
1249  }
1250 
1251  dir = AllocateDir(path);
1252  while ((de = ReadDir(dir, path)) != NULL)
1253  {
1254  int excludeIdx;
1255  bool excludeFound;
1256  ForkNumber relForkNum; /* Type of fork if file is a relation */
1257  int relOidChars; /* Chars in filename that are the rel oid */
1258 
1259  /* Skip special stuff */
1260  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1261  continue;
1262 
1263  /* Skip temporary files */
1264  if (strncmp(de->d_name,
1266  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1267  continue;
1268 
1269  /*
1270  * Check if the postmaster has signaled us to exit, and abort with an
1271  * error in that case. The error handler further up will call
1272  * do_pg_abort_backup() for us. Also check that if the backup was
1273  * started while still in recovery, the server wasn't promoted.
1274  * do_pg_stop_backup() will check that too, but it's better to stop
1275  * the backup early than continue to the end and fail there.
1276  */
1279  ereport(ERROR,
1280  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1281  errmsg("the standby was promoted during online backup"),
1282  errhint("This means that the backup being taken is corrupt "
1283  "and should not be used. "
1284  "Try taking another online backup.")));
1285 
1286  /* Scan for files that should be excluded */
1287  excludeFound = false;
1288  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1289  {
1290  int cmplen = strlen(excludeFiles[excludeIdx].name);
1291 
1292  if (!excludeFiles[excludeIdx].match_prefix)
1293  cmplen++;
1294  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1295  {
1296  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1297  excludeFound = true;
1298  break;
1299  }
1300  }
1301 
1302  if (excludeFound)
1303  continue;
1304 
1305  /* Exclude all forks for unlogged tables except the init fork */
1306  if (isDbDir &&
1307  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1308  &relForkNum))
1309  {
1310  /* Never exclude init forks */
1311  if (relForkNum != INIT_FORKNUM)
1312  {
1313  char initForkFile[MAXPGPATH];
1314  char relOid[OIDCHARS + 1];
1315 
1316  /*
1317  * If any other type of fork, check if there is an init fork
1318  * with the same OID. If so, the file can be excluded.
1319  */
1320  memcpy(relOid, de->d_name, relOidChars);
1321  relOid[relOidChars] = '\0';
1322  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1323  path, relOid);
1324 
1325  if (lstat(initForkFile, &statbuf) == 0)
1326  {
1327  elog(DEBUG2,
1328  "unlogged relation file \"%s\" excluded from backup",
1329  de->d_name);
1330 
1331  continue;
1332  }
1333  }
1334  }
1335 
1336  /* Exclude temporary relations */
1337  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1338  {
1339  elog(DEBUG2,
1340  "temporary relation file \"%s\" excluded from backup",
1341  de->d_name);
1342 
1343  continue;
1344  }
1345 
1346  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1347 
1348  /* Skip pg_control here to back up it last */
1349  if (strcmp(pathbuf, "./global/pg_control") == 0)
1350  continue;
1351 
1352  if (lstat(pathbuf, &statbuf) != 0)
1353  {
1354  if (errno != ENOENT)
1355  ereport(ERROR,
1357  errmsg("could not stat file or directory \"%s\": %m",
1358  pathbuf)));
1359 
1360  /* If the file went away while scanning, it's not an error. */
1361  continue;
1362  }
1363 
1364  /* Scan for directories whose contents should be excluded */
1365  excludeFound = false;
1366  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1367  {
1368  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1369  {
1370  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1371  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1372  excludeFound = true;
1373  break;
1374  }
1375  }
1376 
1377  if (excludeFound)
1378  continue;
1379 
1380  /*
1381  * Exclude contents of directory specified by statrelpath if not set
1382  * to the default (pg_stat_tmp) which is caught in the loop above.
1383  */
1384  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1385  {
1386  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1387  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1388  continue;
1389  }
1390 
1391  /*
1392  * We can skip pg_wal, the WAL segments need to be fetched from the
1393  * WAL archive anyway. But include it as an empty directory anyway, so
1394  * we get permissions right.
1395  */
1396  if (strcmp(pathbuf, "./pg_wal") == 0)
1397  {
1398  /* If pg_wal is a symlink, write it as a directory anyway */
1399  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1400 
1401  /*
1402  * Also send archive_status directory (by hackishly reusing
1403  * statbuf from above ...).
1404  */
1405  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1406  sizeonly);
1407 
1408  continue; /* don't recurse into pg_wal */
1409  }
1410 
1411  /* Allow symbolic links in pg_tblspc only */
1412  if (strcmp(path, "./pg_tblspc") == 0 &&
1413 #ifndef WIN32
1414  S_ISLNK(statbuf.st_mode)
1415 #else
1416  pgwin32_is_junction(pathbuf)
1417 #endif
1418  )
1419  {
1420 #if defined(HAVE_READLINK) || defined(WIN32)
1421  char linkpath[MAXPGPATH];
1422  int rllen;
1423 
1424  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1425  if (rllen < 0)
1426  ereport(ERROR,
1428  errmsg("could not read symbolic link \"%s\": %m",
1429  pathbuf)));
1430  if (rllen >= sizeof(linkpath))
1431  ereport(ERROR,
1432  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1433  errmsg("symbolic link \"%s\" target is too long",
1434  pathbuf)));
1435  linkpath[rllen] = '\0';
1436 
1437  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1438  &statbuf, sizeonly);
1439 #else
1440 
1441  /*
1442  * If the platform does not have symbolic links, it should not be
1443  * possible to have tablespaces - clearly somebody else created
1444  * them. Warn about it and ignore.
1445  */
1446  ereport(WARNING,
1447  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1448  errmsg("tablespaces are not supported on this platform")));
1449  continue;
1450 #endif /* HAVE_READLINK */
1451  }
1452  else if (S_ISDIR(statbuf.st_mode))
1453  {
1454  bool skip_this_dir = false;
1455  ListCell *lc;
1456 
1457  /*
1458  * Store a directory entry in the tar file so we can get the
1459  * permissions right.
1460  */
1461  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1462  sizeonly);
1463 
1464  /*
1465  * Call ourselves recursively for a directory, unless it happens
1466  * to be a separate tablespace located within PGDATA.
1467  */
1468  foreach(lc, tablespaces)
1469  {
1470  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1471 
1472  /*
1473  * ti->rpath is the tablespace relative path within PGDATA, or
1474  * NULL if the tablespace has been properly located somewhere
1475  * else.
1476  *
1477  * Skip past the leading "./" in pathbuf when comparing.
1478  */
1479  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1480  {
1481  skip_this_dir = true;
1482  break;
1483  }
1484  }
1485 
1486  /*
1487  * skip sending directories inside pg_tblspc, if not required.
1488  */
1489  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1490  skip_this_dir = true;
1491 
1492  if (!skip_this_dir)
1493  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
1494  sendtblspclinks, manifest, spcoid);
1495  }
1496  else if (S_ISREG(statbuf.st_mode))
1497  {
1498  bool sent = false;
1499 
1500  if (!sizeonly)
1501  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1502  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1503  manifest, spcoid);
1504 
1505  if (sent || sizeonly)
1506  {
1507  /* Add size. */
1508  size += statbuf.st_size;
1509 
1510  /* Pad to a multiple of the tar block size. */
1511  size += tarPaddingBytesRequired(statbuf.st_size);
1512 
1513  /* Size of the header for the file. */
1514  size += TAR_BLOCK_SIZE;
1515  }
1516  }
1517  else
1518  ereport(WARNING,
1519  (errmsg("skipping special file \"%s\"", pathbuf)));
1520  }
1521  FreeDir(dir);
1522  return size;
1523 }
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1162
char * rpath
Definition: basebackup.h:27
int errcode(int sqlerrcode)
Definition: elog.c:704
bool RecoveryInProgress(void)
Definition: xlog.c:8148
static bool backup_started_in_recovery
Definition: basebackup.c:88
Definition: dirent.h:9
static const char *const excludeDirContents[]
Definition: basebackup.c:153
Definition: dirent.c:25
#define ERROR
Definition: elog.h:45
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
#define readlink(path, buf, size)
Definition: win32_port.h:228
int errcode_for_file_access(void)
Definition: elog.c:727
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2615
#define atooid(x)
Definition: postgres_ext.h:42
static char * statrelpath
Definition: basebackup.c:91
ForkNumber
Definition: relpath.h:40
#define S_ISREG(m)
Definition: win32_port.h:319
#define WARNING
Definition: elog.h:40
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define InvalidOid
Definition: postgres_ext.h:36
#define ereport(elevel,...)
Definition: elog.h:155
#define lfirst(lc)
Definition: pg_list.h:169
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2681
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3219
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:369
const char * name
Definition: encode.c:515
#define S_ISDIR(m)
Definition: win32_port.h:316
#define lstat(path, sb)
Definition: win32_port.h:276
const char * name
Definition: basebackup.c:141
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1211
int errmsg(const char *fmt,...)
Definition: elog.c:915
#define elog(elevel,...)
Definition: elog.h:228
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1578
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:100
char d_name[MAX_PATH]
Definition: dirent.h:15
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:194
#define snprintf
Definition: port.h:215
#define OIDCHARS
Definition: relpath.h:30
int FreeDir(DIR *dir)
Definition: fd.c:2733
bool pgwin32_is_junction(const char *path)
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1877

◆ sendFile()

static bool sendFile ( const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1578 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), Assert, basebackup_read_file(), buf, backup_manifest_info::checksum_type, CloseTransientFile(), DataChecksumsEnabled(), elog, ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, fd(), filename, i, is_checksummed_file(), last_dir_separator(), MemSet, Min, noverify_checksums, OpenTransientFile(), PageGetLSN, PageIsNew, PageHeaderData::pd_checksum, PG_BINARY, pg_checksum_init(), pg_checksum_page(), pg_checksum_update(), pgoff_t, pgstat_report_checksum_failures_in_db(), pq_putmessage, stat::st_mtime, stat::st_size, startptr, TAR_SEND_SIZE, tarPaddingBytesRequired(), throttle(), total_checksum_failures, update_basebackup_progress(), and WARNING.

Referenced by perform_base_backup(), and sendDir().

1581 {
1582  int fd;
1583  BlockNumber blkno = 0;
1584  bool block_retry = false;
1585  char buf[TAR_SEND_SIZE];
1586  uint16 checksum;
1587  int checksum_failures = 0;
1588  off_t cnt;
1589  int i;
1590  pgoff_t len = 0;
1591  char *page;
1592  size_t pad;
1593  PageHeader phdr;
1594  int segmentno = 0;
1595  char *segmentpath;
1596  bool verify_checksum = false;
1597  pg_checksum_context checksum_ctx;
1598 
1599  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1600  elog(ERROR, "could not initialize checksum of file \"%s\"",
1601  readfilename);
1602 
1603  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1604  if (fd < 0)
1605  {
1606  if (errno == ENOENT && missing_ok)
1607  return false;
1608  ereport(ERROR,
1610  errmsg("could not open file \"%s\": %m", readfilename)));
1611  }
1612 
1613  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1614 
1616  {
1617  char *filename;
1618 
1619  /*
1620  * Get the filename (excluding path). As last_dir_separator()
1621  * includes the last directory separator, we chop that off by
1622  * incrementing the pointer.
1623  */
1624  filename = last_dir_separator(readfilename) + 1;
1625 
1626  if (is_checksummed_file(readfilename, filename))
1627  {
1628  verify_checksum = true;
1629 
1630  /*
1631  * Cut off at the segment boundary (".") to get the segment number
1632  * in order to mix it into the checksum.
1633  */
1634  segmentpath = strstr(filename, ".");
1635  if (segmentpath != NULL)
1636  {
1637  segmentno = atoi(segmentpath + 1);
1638  if (segmentno == 0)
1639  ereport(ERROR,
1640  (errmsg("invalid segment number %d in file \"%s\"",
1641  segmentno, filename)));
1642  }
1643  }
1644  }
1645 
1646  /*
1647  * Loop until we read the amount of data the caller told us to expect. The
1648  * file could be longer, if it was extended while we were sending it, but
1649  * for a base backup we can ignore such extended data. It will be restored
1650  * from WAL.
1651  */
1652  while (len < statbuf->st_size)
1653  {
1654  /* Try to read some more data. */
1655  cnt = basebackup_read_file(fd, buf,
1656  Min(sizeof(buf), statbuf->st_size - len),
1657  len, readfilename, true);
1658 
1659  /*
1660  * If we hit end-of-file, a concurrent truncation must have occurred.
1661  * That's not an error condition, because WAL replay will fix things
1662  * up.
1663  */
1664  if (cnt == 0)
1665  break;
1666 
1667  /*
1668  * The checksums are verified at block level, so we iterate over the
1669  * buffer in chunks of BLCKSZ, after making sure that
1670  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1671  * BLCKSZ bytes.
1672  */
1673  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1674 
1675  if (verify_checksum && (cnt % BLCKSZ != 0))
1676  {
1677  ereport(WARNING,
1678  (errmsg("could not verify checksum in file \"%s\", block "
1679  "%d: read buffer size %d and page size %d "
1680  "differ",
1681  readfilename, blkno, (int) cnt, BLCKSZ)));
1682  verify_checksum = false;
1683  }
1684 
1685  if (verify_checksum)
1686  {
1687  for (i = 0; i < cnt / BLCKSZ; i++)
1688  {
1689  page = buf + BLCKSZ * i;
1690 
1691  /*
1692  * Only check pages which have not been modified since the
1693  * start of the base backup. Otherwise, they might have been
1694  * written only halfway and the checksum would not be valid.
1695  * However, replaying WAL would reinstate the correct page in
1696  * this case. We also skip completely new pages, since they
1697  * don't have a checksum yet.
1698  */
1699  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1700  {
1701  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1702  phdr = (PageHeader) page;
1703  if (phdr->pd_checksum != checksum)
1704  {
1705  /*
1706  * Retry the block on the first failure. It's
1707  * possible that we read the first 4K page of the
1708  * block just before postgres updated the entire block
1709  * so it ends up looking torn to us. We only need to
1710  * retry once because the LSN should be updated to
1711  * something we can ignore on the next pass. If the
1712  * error happens again then it is a true validation
1713  * failure.
1714  */
1715  if (block_retry == false)
1716  {
1717  int reread_cnt;
1718 
1719  /* Reread the failed block */
1720  reread_cnt =
1721  basebackup_read_file(fd, buf + BLCKSZ * i,
1722  BLCKSZ, len + BLCKSZ * i,
1723  readfilename,
1724  false);
1725  if (reread_cnt == 0)
1726  {
1727  /*
1728  * If we hit end-of-file, a concurrent
1729  * truncation must have occurred, so break out
1730  * of this loop just as if the initial fread()
1731  * returned 0. We'll drop through to the same
1732  * code that handles that case. (We must fix
1733  * up cnt first, though.)
1734  */
1735  cnt = BLCKSZ * i;
1736  break;
1737  }
1738 
1739  /* Set flag so we know a retry was attempted */
1740  block_retry = true;
1741 
1742  /* Reset loop to validate the block again */
1743  i--;
1744  continue;
1745  }
1746 
1747  checksum_failures++;
1748 
1749  if (checksum_failures <= 5)
1750  ereport(WARNING,
1751  (errmsg("checksum verification failed in "
1752  "file \"%s\", block %d: calculated "
1753  "%X but expected %X",
1754  readfilename, blkno, checksum,
1755  phdr->pd_checksum)));
1756  if (checksum_failures == 5)
1757  ereport(WARNING,
1758  (errmsg("further checksum verification "
1759  "failures in file \"%s\" will not "
1760  "be reported", readfilename)));
1761  }
1762  }
1763  block_retry = false;
1764  blkno++;
1765  }
1766  }
1767 
1768  /* Send the chunk as a CopyData message */
1769  if (pq_putmessage('d', buf, cnt))
1770  ereport(ERROR,
1771  (errmsg("base backup could not send data, aborting backup")));
1773 
1774  /* Also feed it to the checksum machinery. */
1775  if (pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt) < 0)
1776  elog(ERROR, "could not update checksum of base backup");
1777 
1778  len += cnt;
1779  throttle(cnt);
1780  }
1781 
1782  /* If the file was truncated while we were sending it, pad it with zeros */
1783  if (len < statbuf->st_size)
1784  {
1785  MemSet(buf, 0, sizeof(buf));
1786  while (len < statbuf->st_size)
1787  {
1788  cnt = Min(sizeof(buf), statbuf->st_size - len);
1789  pq_putmessage('d', buf, cnt);
1790  if (pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt) < 0)
1791  elog(ERROR, "could not update checksum of base backup");
1793  len += cnt;
1794  throttle(cnt);
1795  }
1796  }
1797 
1798  /*
1799  * Pad to a block boundary, per tar format requirements. (This small piece
1800  * of data is probably not worth throttling, and is not checksummed
1801  * because it's not actually part of the file.)
1802  */
1803  pad = tarPaddingBytesRequired(len);
1804  if (pad > 0)
1805  {
1806  MemSet(buf, 0, pad);
1807  pq_putmessage('d', buf, pad);
1809  }
1810 
1811  CloseTransientFile(fd);
1812 
1813  if (checksum_failures > 1)
1814  {
1815  ereport(WARNING,
1816  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1817  "file \"%s\" has a total of %d checksum verification failures",
1818  checksum_failures,
1819  readfilename, checksum_failures)));
1820 
1821  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1822  }
1823 
1824  total_checksum_failures += checksum_failures;
1825 
1826  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1827  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1828 
1829  return true;
1830 }
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:2008
static void throttle(size_t increment)
Definition: basebackup.c:1897
int64 pg_time_t
Definition: pgtime.h:23
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1025
bool DataChecksumsEnabled(void)
Definition: xlog.c:4961
#define Min(x, y)
Definition: c.h:974
unsigned char uint8
Definition: c.h:427
#define MemSet(start, val, len)
Definition: c.h:996
uint32 BlockNumber
Definition: block.h:31
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1259
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1722
uint16 pd_checksum
Definition: bufpage.h:156
#define pgoff_t
Definition: win32_port.h:200
unsigned short uint16
Definition: c.h:428
#define ERROR
Definition: elog.h:45
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2404
static bool noverify_checksums
Definition: basebackup.c:122
static char * buf
Definition: pg_test_fsync.c:68
int errcode_for_file_access(void)
Definition: elog.c:727
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
__int64 st_size
Definition: win32_port.h:265
int CloseTransientFile(int fd)
Definition: fd.c:2581
#define WARNING
Definition: elog.h:40
pg_checksum_type checksum_type
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define ereport(elevel,...)
Definition: elog.h:155
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1532
PageHeaderData * PageHeader
Definition: bufpage.h:166
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1971
#define Assert(condition)
Definition: c.h:792
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define PageGetLSN(page)
Definition: bufpage.h:366
static char * filename
Definition: pg_dumpall.c:91
#define PageIsNew(page)
Definition: bufpage.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:915
#define TAR_SEND_SIZE
Definition: basebackup.c:96
#define elog(elevel,...)
Definition: elog.h:228
int i
__time64_t st_mtime
Definition: win32_port.h:267
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
uint16 pg_checksum_page(char *page, BlockNumber blkno)
static XLogRecPtr startptr
Definition: basebackup.c:116
static long long int total_checksum_failures
Definition: basebackup.c:119

◆ sendFileWithContent()

static void sendFileWithContent ( const char *  filename,
const char *  content,
backup_manifest_info manifest 
)
static

Definition at line 1096 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), buf, backup_manifest_info::checksum_type, elog, ERROR, MemSet, pg_checksum_init(), pg_checksum_update(), pg_file_create_mode, pq_putmessage, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, stat::st_uid, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and update_basebackup_progress().

Referenced by perform_base_backup().

1098 {
1099  struct stat statbuf;
1100  int pad,
1101  len;
1102  pg_checksum_context checksum_ctx;
1103 
1104  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1105  elog(ERROR, "could not initialize checksum of file \"%s\"",
1106  filename);
1107 
1108  len = strlen(content);
1109 
1110  /*
1111  * Construct a stat struct for the backup_label file we're injecting in
1112  * the tar.
1113  */
1114  /* Windows doesn't have the concept of uid and gid */
1115 #ifdef WIN32
1116  statbuf.st_uid = 0;
1117  statbuf.st_gid = 0;
1118 #else
1119  statbuf.st_uid = geteuid();
1120  statbuf.st_gid = getegid();
1121 #endif
1122  statbuf.st_mtime = time(NULL);
1123  statbuf.st_mode = pg_file_create_mode;
1124  statbuf.st_size = len;
1125 
1126  _tarWriteHeader(filename, NULL, &statbuf, false);
1127  /* Send the contents as a CopyData message */
1128  pq_putmessage('d', content, len);
1130 
1131  /* Pad to a multiple of the tar block size. */
1132  pad = tarPaddingBytesRequired(len);
1133  if (pad > 0)
1134  {
1135  char buf[TAR_BLOCK_SIZE];
1136 
1137  MemSet(buf, 0, pad);
1138  pq_putmessage('d', buf, pad);
1140  }
1141 
1142  if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1143  elog(ERROR, "could not update checksum of file \"%s\"",
1144  filename);
1145 
1146  AddFileToBackupManifest(manifest, NULL, filename, len,
1147  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1148 }
int pg_file_create_mode
Definition: file_perm.c:19
int64 pg_time_t
Definition: pgtime.h:23
unsigned char uint8
Definition: c.h:427
#define MemSet(start, val, len)
Definition: c.h:996
#define ERROR
Definition: elog.h:45
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
static char * buf
Definition: pg_test_fsync.c:68
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
pg_checksum_type checksum_type
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1971
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
static char * filename
Definition: pg_dumpall.c:91
#define elog(elevel,...)
Definition: elog.h:228
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ sendTablespace()

static int64 sendTablespace ( char *  path,
char *  oid,
bool  sizeonly,
struct backup_manifest_info manifest 
)
static

Definition at line 1158 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, sendDir(), snprintf, and TABLESPACE_VERSION_DIRECTORY.

Referenced by perform_base_backup().

1160 {
1161  int64 size;
1162  char pathbuf[MAXPGPATH];
1163  struct stat statbuf;
1164 
1165  /*
1166  * 'path' points to the tablespace location, but we only want to include
1167  * the version directory in it that belongs to us.
1168  */
1169  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1171 
1172  /*
1173  * Store a directory entry in the tar file so we get the permissions
1174  * right.
1175  */
1176  if (lstat(pathbuf, &statbuf) != 0)
1177  {
1178  if (errno != ENOENT)
1179  ereport(ERROR,
1181  errmsg("could not stat file or directory \"%s\": %m",
1182  pathbuf)));
1183 
1184  /* If the tablespace went away while scanning, it's no error. */
1185  return 0;
1186  }
1187 
1188  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1189  sizeonly);
1190 
1191  /* Send all the files in the tablespace version directory */
1192  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1193  spcoid);
1194 
1195  return size;
1196 }
#define NIL
Definition: pg_list.h:65
#define ERROR
Definition: elog.h:45
#define MAXPGPATH
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
int errcode_for_file_access(void)
Definition: elog.c:727
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1834
#define ereport(elevel,...)
Definition: elog.h:155
#define lstat(path, sb)
Definition: win32_port.h:276
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1211
int errmsg(const char *fmt,...)
Definition: elog.c:915
#define snprintf
Definition: port.h:215

◆ SendXlogRecPtrResult()

static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 1041 of file basebackup.c.

References buf, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), snprintf, and generate_unaccent_rules::str.

Referenced by perform_base_backup().

1042 {
1044  char str[MAXFNAMELEN];
1045  Size len;
1046 
1047  pq_beginmessage(&buf, 'T'); /* RowDescription */
1048  pq_sendint16(&buf, 2); /* 2 fields */
1049 
1050  /* Field headers */
1051  pq_sendstring(&buf, "recptr");
1052  pq_sendint32(&buf, 0); /* table oid */
1053  pq_sendint16(&buf, 0); /* attnum */
1054  pq_sendint32(&buf, TEXTOID); /* type oid */
1055  pq_sendint16(&buf, -1);
1056  pq_sendint32(&buf, 0);
1057  pq_sendint16(&buf, 0);
1058 
1059  pq_sendstring(&buf, "tli");
1060  pq_sendint32(&buf, 0); /* table oid */
1061  pq_sendint16(&buf, 0); /* attnum */
1062 
1063  /*
1064  * int8 may seem like a surprising data type for this, but in theory int4
1065  * would not be wide enough for this, as TimeLineID is unsigned.
1066  */
1067  pq_sendint32(&buf, INT8OID); /* type oid */
1068  pq_sendint16(&buf, -1);
1069  pq_sendint32(&buf, 0);
1070  pq_sendint16(&buf, 0);
1071  pq_endmessage(&buf);
1072 
1073  /* Data row */
1074  pq_beginmessage(&buf, 'D');
1075  pq_sendint16(&buf, 2); /* number of columns */
1076 
1077  len = snprintf(str, sizeof(str),
1078  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
1079  pq_sendint32(&buf, len);
1080  pq_sendbytes(&buf, str, len);
1081 
1082  len = snprintf(str, sizeof(str), "%u", tli);
1083  pq_sendint32(&buf, len);
1084  pq_sendbytes(&buf, str, len);
1085 
1086  pq_endmessage(&buf);
1087 
1088  /* Send a CommandComplete message */
1089  pq_puttextmessage('C', "SELECT");
1090 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:68
unsigned int uint32
Definition: c.h:429
#define MAXFNAMELEN
size_t Size
Definition: c.h:528
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
#define snprintf
Definition: port.h:215

◆ throttle()

static void throttle ( size_t  increment)
static

Definition at line 1897 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_EXIT_ON_PM_DEATH, WL_LATCH_SET, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1898 {
1899  TimeOffset elapsed_min;
1900 
1901  if (throttling_counter < 0)
1902  return;
1903 
1904  throttling_counter += increment;
1906  return;
1907 
1908  /* How much time should have elapsed at minimum? */
1909  elapsed_min = elapsed_min_unit *
1911 
1912  /*
1913  * Since the latch could be set repeatedly because of concurrently WAL
1914  * activity, sleep in a loop to ensure enough time has passed.
1915  */
1916  for (;;)
1917  {
1918  TimeOffset elapsed,
1919  sleep;
1920  int wait_result;
1921 
1922  /* Time elapsed since the last measurement (and possible wake up). */
1923  elapsed = GetCurrentTimestamp() - throttled_last;
1924 
1925  /* sleep if the transfer is faster than it should be */
1926  sleep = elapsed_min - elapsed;
1927  if (sleep <= 0)
1928  break;
1929 
1931 
1932  /* We're eating a potentially set latch, so check for interrupts */
1934 
1935  /*
1936  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1937  * the maximum time to sleep. Thus the cast to long is safe.
1938  */
1939  wait_result = WaitLatch(MyLatch,
1941  (long) (sleep / 1000),
1943 
1944  if (wait_result & WL_LATCH_SET)
1946 
1947  /* Done waiting? */
1948  if (wait_result & WL_TIMEOUT)
1949  break;
1950  }
1951 
1952  /*
1953  * As we work with integers, only whole multiple of throttling_sample was
1954  * processed. The rest will be done during the next call of this function.
1955  */
1957 
1958  /*
1959  * Time interval for the remaining amount and possible next increments
1960  * starts now.
1961  */
1963 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1578
void ResetLatch(Latch *latch)
Definition: latch.c:588
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:390
int64 TimeOffset
Definition: timestamp.h:40
static TimestampTz throttled_last
Definition: basebackup.c:113
struct Latch * MyLatch
Definition: globals.c:55
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:100
static int64 throttling_counter
Definition: basebackup.c:107
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:104
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129

◆ update_basebackup_progress()

static void update_basebackup_progress ( int64  delta)
static

Definition at line 1971 of file basebackup.c.

References backup_streamed, backup_total, pgstat_progress_update_multi_param(), PROGRESS_BASEBACKUP_BACKUP_STREAMED, PROGRESS_BASEBACKUP_BACKUP_TOTAL, and val.

Referenced by _tarWriteHeader(), perform_base_backup(), sendFile(), and sendFileWithContent().

1972 {
1973  const int index[] = {
1976  };
1977  int64 val[2];
1978  int nparam = 0;
1979 
1980  backup_streamed += delta;
1981  val[nparam++] = backup_streamed;
1982 
1983  /*
1984  * Avoid overflowing past 100% or the full size. This may make the total
1985  * size number change as we approach the end of the backup (the estimate
1986  * will always be wrong if WAL is included), but that's better than having
1987  * the done column be bigger than the total.
1988  */
1990  {
1992  val[nparam++] = backup_total;
1993  }
1994 
1995  pgstat_progress_update_multi_param(nparam, index, val);
1996 }
static int64 backup_total
Definition: basebackup.c:128
Definition: type.h:89
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED
Definition: progress.h:125
static int64 backup_streamed
Definition: basebackup.c:131
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3500
long val
Definition: informix.c:664

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 88 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

◆ backup_streamed

int64 backup_streamed = 0
static

Definition at line 131 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ backup_total

int64 backup_total = 0
static

Definition at line 128 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ elapsed_min_unit

TimeOffset elapsed_min_unit
static

Definition at line 110 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 153 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{PG_AUTOCONF_FILENAME ".tmp", false},
{TABLESPACE_MAP, false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}
#define TABLESPACE_MAP
Definition: xlog.h:392
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:96
#define BACKUP_LABEL_FILE
Definition: xlog.h:389

Definition at line 194 of file basebackup.c.

◆ noChecksumFiles

const struct exclude_list_item noChecksumFiles[]
static
Initial value:
= {
{"pg_control", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
{NULL, false}
}

Definition at line 238 of file basebackup.c.

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 122 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ startptr

◆ statrelpath

char* statrelpath = NULL
static

Definition at line 91 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

◆ throttled_last

TimestampTz throttled_last
static

Definition at line 113 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_counter

int64 throttling_counter
static

Definition at line 107 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_sample

uint64 throttling_sample
static

Definition at line 104 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 119 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().