PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/pg_type.h"
#include "common/file_perm.h"
#include "commands/progress.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/backup_manifest.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 
struct  exclude_list_item
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 
#define CHECK_FREAD_ERROR(fp, filename)
 

Functions

static int64 sendDir (const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
 
static bool sendFile (const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 
static void sendFileWithContent (const char *filename, const char *content, backup_manifest_info *manifest)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void perform_base_backup (basebackup_options *opt)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static void throttle (size_t increment)
 
static void update_basebackup_progress (int64 delta)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 
int64 sendTablespace (char *path, char *spcoid, bool sizeonly, backup_manifest_info *manifest)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static XLogRecPtr startptr
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static int64 backup_total = 0
 
static int64 backup_streamed = 0
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 
static const struct exclude_list_item noChecksumFiles []
 

Macro Definition Documentation

◆ CHECK_FREAD_ERROR

#define CHECK_FREAD_ERROR (   fp,
  filename 
)
Value:
do { \
if (ferror(fp)) \
ereport(ERROR, \
(errmsg("could not read from file \"%s\"", filename))); \
} while (0)
#define ERROR
Definition: elog.h:43
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:824

Definition at line 104 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ TAR_SEND_SIZE

#define TAR_SEND_SIZE   32768

Definition at line 92 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

◆ THROTTLING_FREQUENCY

#define THROTTLING_FREQUENCY   8

Definition at line 97 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

◆ _tarWriteDir()

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1865 of file basebackup.c.

References _tarWriteHeader(), pg_dir_create_mode, and pgwin32_is_junction().

Referenced by sendDir().

1867 {
1868  /* If symlink, write it as a directory anyway */
1869 #ifndef WIN32
1870  if (S_ISLNK(statbuf->st_mode))
1871 #else
1872  if (pgwin32_is_junction(pathbuf))
1873 #endif
1874  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1875 
1876  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1877 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
int pg_dir_create_mode
Definition: file_perm.c:18
bool pgwin32_is_junction(const char *path)

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1822 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, tarCreateHeader(), and update_basebackup_progress().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1824 {
1825  char h[512];
1826  enum tarError rc;
1827 
1828  if (!sizeonly)
1829  {
1830  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1831  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1832  statbuf->st_mtime);
1833 
1834  switch (rc)
1835  {
1836  case TAR_OK:
1837  break;
1838  case TAR_NAME_TOO_LONG:
1839  ereport(ERROR,
1840  (errmsg("file name too long for tar format: \"%s\"",
1841  filename)));
1842  break;
1843  case TAR_SYMLINK_TOO_LONG:
1844  ereport(ERROR,
1845  (errmsg("symbolic link target too long for tar format: "
1846  "file name \"%s\", target \"%s\"",
1847  filename, linktarget)));
1848  break;
1849  default:
1850  elog(ERROR, "unrecognized tar error: %d", rc);
1851  }
1852 
1853  pq_putmessage('d', h, sizeof(h));
1854  update_basebackup_progress(sizeof(h));
1855  }
1856 
1857  return sizeof(h);
1858 }
Definition: pgtar.h:17
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
#define ereport(elevel,...)
Definition: elog.h:144
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1959
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 742 of file basebackup.c.

References lfirst.

Referenced by perform_base_backup().

743 {
744  char *fna = (char *) lfirst(a);
745  char *fnb = (char *) lfirst(b);
746 
747  return strcmp(fna + 8, fnb + 8);
748 }
#define lfirst(lc)
Definition: pg_list.h:190

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1514 of file basebackup.c.

References exclude_list_item::match_prefix, and exclude_list_item::name.

Referenced by sendFile().

1515 {
1516  /* Check that the file is in a tablespace */
1517  if (strncmp(fullpath, "./global/", 9) == 0 ||
1518  strncmp(fullpath, "./base/", 7) == 0 ||
1519  strncmp(fullpath, "/", 1) == 0)
1520  {
1521  int excludeIdx;
1522 
1523  /* Compare file against noChecksumFiles skip list */
1524  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1525  {
1526  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1527 
1528  if (!noChecksumFiles[excludeIdx].match_prefix)
1529  cmplen++;
1530  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1531  cmplen) == 0)
1532  return false;
1533  }
1534 
1535  return true;
1536  }
1537  else
1538  return false;
1539 }
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:246
const char * name
Definition: encode.c:555
const char * name
Definition: basebackup.c:149
static char * filename
Definition: pg_dumpall.c:90

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 754 of file basebackup.c.

References DefElem::arg, CHECKSUM_TYPE_CRC32C, CHECKSUM_TYPE_NONE, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, basebackup_options::manifest, basebackup_options::manifest_checksum_type, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, MANIFEST_OPTION_YES, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, parse_bool(), pg_checksum_parse_type(), pg_strcasecmp(), basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

755 {
756  ListCell *lopt;
757  bool o_label = false;
758  bool o_progress = false;
759  bool o_fast = false;
760  bool o_nowait = false;
761  bool o_wal = false;
762  bool o_maxrate = false;
763  bool o_tablespace_map = false;
764  bool o_noverify_checksums = false;
765  bool o_manifest = false;
766  bool o_manifest_checksums = false;
767 
768  MemSet(opt, 0, sizeof(*opt));
771 
772  foreach(lopt, options)
773  {
774  DefElem *defel = (DefElem *) lfirst(lopt);
775 
776  if (strcmp(defel->defname, "label") == 0)
777  {
778  if (o_label)
779  ereport(ERROR,
780  (errcode(ERRCODE_SYNTAX_ERROR),
781  errmsg("duplicate option \"%s\"", defel->defname)));
782  opt->label = strVal(defel->arg);
783  o_label = true;
784  }
785  else if (strcmp(defel->defname, "progress") == 0)
786  {
787  if (o_progress)
788  ereport(ERROR,
789  (errcode(ERRCODE_SYNTAX_ERROR),
790  errmsg("duplicate option \"%s\"", defel->defname)));
791  opt->progress = true;
792  o_progress = true;
793  }
794  else if (strcmp(defel->defname, "fast") == 0)
795  {
796  if (o_fast)
797  ereport(ERROR,
798  (errcode(ERRCODE_SYNTAX_ERROR),
799  errmsg("duplicate option \"%s\"", defel->defname)));
800  opt->fastcheckpoint = true;
801  o_fast = true;
802  }
803  else if (strcmp(defel->defname, "nowait") == 0)
804  {
805  if (o_nowait)
806  ereport(ERROR,
807  (errcode(ERRCODE_SYNTAX_ERROR),
808  errmsg("duplicate option \"%s\"", defel->defname)));
809  opt->nowait = true;
810  o_nowait = true;
811  }
812  else if (strcmp(defel->defname, "wal") == 0)
813  {
814  if (o_wal)
815  ereport(ERROR,
816  (errcode(ERRCODE_SYNTAX_ERROR),
817  errmsg("duplicate option \"%s\"", defel->defname)));
818  opt->includewal = true;
819  o_wal = true;
820  }
821  else if (strcmp(defel->defname, "max_rate") == 0)
822  {
823  long maxrate;
824 
825  if (o_maxrate)
826  ereport(ERROR,
827  (errcode(ERRCODE_SYNTAX_ERROR),
828  errmsg("duplicate option \"%s\"", defel->defname)));
829 
830  maxrate = intVal(defel->arg);
831  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
832  ereport(ERROR,
833  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
834  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
835  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
836 
837  opt->maxrate = (uint32) maxrate;
838  o_maxrate = true;
839  }
840  else if (strcmp(defel->defname, "tablespace_map") == 0)
841  {
842  if (o_tablespace_map)
843  ereport(ERROR,
844  (errcode(ERRCODE_SYNTAX_ERROR),
845  errmsg("duplicate option \"%s\"", defel->defname)));
846  opt->sendtblspcmapfile = true;
847  o_tablespace_map = true;
848  }
849  else if (strcmp(defel->defname, "noverify_checksums") == 0)
850  {
851  if (o_noverify_checksums)
852  ereport(ERROR,
853  (errcode(ERRCODE_SYNTAX_ERROR),
854  errmsg("duplicate option \"%s\"", defel->defname)));
855  noverify_checksums = true;
856  o_noverify_checksums = true;
857  }
858  else if (strcmp(defel->defname, "manifest") == 0)
859  {
860  char *optval = strVal(defel->arg);
861  bool manifest_bool;
862 
863  if (o_manifest)
864  ereport(ERROR,
865  (errcode(ERRCODE_SYNTAX_ERROR),
866  errmsg("duplicate option \"%s\"", defel->defname)));
867  if (parse_bool(optval, &manifest_bool))
868  {
869  if (manifest_bool)
871  else
873  }
874  else if (pg_strcasecmp(optval, "force-encode") == 0)
876  else
877  ereport(ERROR,
878  (errcode(ERRCODE_SYNTAX_ERROR),
879  errmsg("unrecognized manifest option: \"%s\"",
880  optval)));
881  o_manifest = true;
882  }
883  else if (strcmp(defel->defname, "manifest_checksums") == 0)
884  {
885  char *optval = strVal(defel->arg);
886 
887  if (o_manifest_checksums)
888  ereport(ERROR,
889  (errcode(ERRCODE_SYNTAX_ERROR),
890  errmsg("duplicate option \"%s\"", defel->defname)));
891  if (!pg_checksum_parse_type(optval,
892  &opt->manifest_checksum_type))
893  ereport(ERROR,
894  (errcode(ERRCODE_SYNTAX_ERROR),
895  errmsg("unrecognized checksum algorithm: \"%s\"",
896  optval)));
897  o_manifest_checksums = true;
898  }
899  else
900  elog(ERROR, "option \"%s\" not recognized",
901  defel->defname);
902  }
903  if (opt->label == NULL)
904  opt->label = "base backup";
905  if (opt->manifest == MANIFEST_OPTION_NO)
906  {
907  if (o_manifest_checksums)
908  ereport(ERROR,
909  (errcode(ERRCODE_SYNTAX_ERROR),
910  errmsg("manifest checksums require a backup manifest")));
912  }
913 }
#define MAX_RATE_LOWER
Definition: basebackup.h:23
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:610
#define MemSet(start, val, len)
Definition: c.h:971
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define MAX_RATE_UPPER
Definition: basebackup.h:24
#define ERROR
Definition: elog.h:43
backup_manifest_option manifest
Definition: basebackup.c:57
static bool noverify_checksums
Definition: basebackup.c:130
static int32 maxrate
const char * label
Definition: basebackup.c:50
unsigned int uint32
Definition: c.h:367
Node * arg
Definition: parsenodes.h:733
#define ereport(elevel,...)
Definition: elog.h:144
#define lfirst(lc)
Definition: pg_list.h:190
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
char * defname
Definition: parsenodes.h:732
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt)
static

Definition at line 264 of file basebackup.c.

References _tarWriteHeader(), AddWALInfoToBackupManifest(), AllocateDir(), AllocateFile(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, backup_streamed, backup_total, BoolGetDatum, buf, CHECK_FREAD_ERROR, CheckXLogRemoved(), compareWalFileNames(), CurrentResourceOwner, dirent::d_name, StringInfoData::data, DataDir, do_pg_abort_backup(), do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, FreeDir(), FreeFile(), GetCurrentTimestamp(), basebackup_options::includewal, InitializeBackupManifest(), InvalidOid, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, linitial, list_length(), list_sort(), lnext(), lstat, makeStringInfo(), basebackup_options::manifest, manifest, basebackup_options::manifest_checksum_type, MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, tablespaceinfo::oid, palloc0(), tablespaceinfo::path, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_progress_end_command(), pgstat_progress_start_command(), pgstat_progress_update_multi_param(), pgstat_progress_update_param(), pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint16(), basebackup_options::progress, PROGRESS_BASEBACKUP_BACKUP_TOTAL, PROGRESS_BASEBACKUP_PHASE, PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP, PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL, PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT, PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE, PROGRESS_BASEBACKUP_TBLSPC_STREAMED, PROGRESS_BASEBACKUP_TBLSPC_TOTAL, PROGRESS_COMMAND_BASEBACKUP, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), ResourceOwnerCreate(), SendBackupHeader(), SendBackupManifest(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf, startptr, stat, statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, total_checksum_failures, update_basebackup_progress(), USECS_PER_SEC, val, wal_segment_size, WalSndResourceCleanup(), WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

265 {
266  TimeLineID starttli;
267  XLogRecPtr endptr;
268  TimeLineID endtli;
269  StringInfo labelfile;
272  int datadirpathlen;
273  List *tablespaces = NIL;
274 
275  backup_total = 0;
276  backup_streamed = 0;
278 
279  /*
280  * If the estimation of the total backup size is disabled, make the
281  * backup_total column in the view return NULL by setting the parameter to
282  * -1.
283  */
284  if (!opt->progress)
285  {
286  backup_total = -1;
288  backup_total);
289  }
290 
291  /* we're going to use a BufFile, so we need a ResourceOwner */
292  Assert(CurrentResourceOwner == NULL);
293  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
294 
295  datadirpathlen = strlen(DataDir);
296 
298 
299  labelfile = makeStringInfo();
300  tblspc_map_file = makeStringInfo();
301  InitializeBackupManifest(&manifest, opt->manifest,
303 
305 
308  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
309  labelfile, &tablespaces,
310  tblspc_map_file,
311  opt->progress, opt->sendtblspcmapfile);
312 
313  /*
314  * Once do_pg_start_backup has been called, ensure that any failure causes
315  * us to abort the backup so we don't "leak" a backup counter. For this
316  * reason, *all* functionality between do_pg_start_backup() and the end of
317  * do_pg_stop_backup() should be inside the error cleanup block!
318  */
319 
321  {
322  ListCell *lc;
323  tablespaceinfo *ti;
324  int tblspc_streamed = 0;
325 
326  /*
327  * Calculate the relative path of temporary statistics directory in
328  * order to skip the files which are located in that directory later.
329  */
331  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
332  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
333  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
335  else
337 
338  /* Add a node for the base directory at the end */
339  ti = palloc0(sizeof(tablespaceinfo));
340  if (opt->progress)
341  ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL);
342  else
343  ti->size = -1;
344  tablespaces = lappend(tablespaces, ti);
345 
346  /*
347  * Calculate the total backup size by summing up the size of each
348  * tablespace
349  */
350  if (opt->progress)
351  {
352  foreach(lc, tablespaces)
353  {
354  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
355 
356  backup_total += tmp->size;
357  }
358  }
359 
360  /* Report that we are now streaming database files as a base backup */
361  {
362  const int index[] = {
366  };
367  const int64 val[] = {
369  backup_total, list_length(tablespaces)
370  };
371 
372  pgstat_progress_update_multi_param(3, index, val);
373  }
374 
375  /* Send the starting position of the backup */
376  SendXlogRecPtrResult(startptr, starttli);
377 
378  /* Send tablespace header */
379  SendBackupHeader(tablespaces);
380 
381  /* Setup and activate network throttling, if client requested it */
382  if (opt->maxrate > 0)
383  {
385  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
386 
387  /*
388  * The minimum amount of time for throttling_sample bytes to be
389  * transferred.
390  */
392 
393  /* Enable throttling. */
394  throttling_counter = 0;
395 
396  /* The 'real data' starts now (header was ignored). */
398  }
399  else
400  {
401  /* Disable throttling. */
402  throttling_counter = -1;
403  }
404 
405  /* Send off our tablespaces one by one */
406  foreach(lc, tablespaces)
407  {
408  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
410 
411  /* Send CopyOutResponse message */
412  pq_beginmessage(&buf, 'H');
413  pq_sendbyte(&buf, 0); /* overall format */
414  pq_sendint16(&buf, 0); /* natts */
415  pq_endmessage(&buf);
416 
417  if (ti->path == NULL)
418  {
419  struct stat statbuf;
420 
421  /* In the main tar, include the backup_label first... */
423  &manifest);
424 
425  /*
426  * Send tablespace_map file if required and then the bulk of
427  * the files.
428  */
429  if (tblspc_map_file && opt->sendtblspcmapfile)
430  {
431  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
432  &manifest);
433  sendDir(".", 1, false, tablespaces, false,
434  &manifest, NULL);
435  }
436  else
437  sendDir(".", 1, false, tablespaces, true,
438  &manifest, NULL);
439 
440  /* ... and pg_control after everything else. */
441  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
442  ereport(ERROR,
444  errmsg("could not stat file \"%s\": %m",
447  false, InvalidOid, &manifest, NULL);
448  }
449  else
450  sendTablespace(ti->path, ti->oid, false, &manifest);
451 
452  /*
453  * If we're including WAL, and this is the main data directory we
454  * don't terminate the tar stream here. Instead, we will append
455  * the xlog files below and terminate it then. This is safe since
456  * the main data directory is always sent *last*.
457  */
458  if (opt->includewal && ti->path == NULL)
459  {
460  Assert(lnext(tablespaces, lc) == NULL);
461  }
462  else
463  pq_putemptymessage('c'); /* CopyDone */
464 
465  tblspc_streamed++;
467  tblspc_streamed);
468  }
469 
472  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
473  }
475 
476 
477  if (opt->includewal)
478  {
479  /*
480  * We've left the last tar file "open", so we can now append the
481  * required WAL files to it.
482  */
483  char pathbuf[MAXPGPATH];
484  XLogSegNo segno;
485  XLogSegNo startsegno;
486  XLogSegNo endsegno;
487  struct stat statbuf;
488  List *historyFileList = NIL;
489  List *walFileList = NIL;
490  char firstoff[MAXFNAMELEN];
491  char lastoff[MAXFNAMELEN];
492  DIR *dir;
493  struct dirent *de;
494  ListCell *lc;
495  TimeLineID tli;
496 
499 
500  /*
501  * I'd rather not worry about timelines here, so scan pg_wal and
502  * include all WAL files in the range between 'startptr' and 'endptr',
503  * regardless of the timeline the file is stamped with. If there are
504  * some spurious WAL files belonging to timelines that don't belong in
505  * this server's history, they will be included too. Normally there
506  * shouldn't be such files, but if there are, there's little harm in
507  * including them.
508  */
509  XLByteToSeg(startptr, startsegno, wal_segment_size);
510  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
511  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
512  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
513 
514  dir = AllocateDir("pg_wal");
515  while ((de = ReadDir(dir, "pg_wal")) != NULL)
516  {
517  /* Does it look like a WAL segment, and is it in the range? */
518  if (IsXLogFileName(de->d_name) &&
519  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
520  strcmp(de->d_name + 8, lastoff + 8) <= 0)
521  {
522  walFileList = lappend(walFileList, pstrdup(de->d_name));
523  }
524  /* Does it look like a timeline history file? */
525  else if (IsTLHistoryFileName(de->d_name))
526  {
527  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
528  }
529  }
530  FreeDir(dir);
531 
532  /*
533  * Before we go any further, check that none of the WAL segments we
534  * need were removed.
535  */
536  CheckXLogRemoved(startsegno, ThisTimeLineID);
537 
538  /*
539  * Sort the WAL filenames. We want to send the files in order from
540  * oldest to newest, to reduce the chance that a file is recycled
541  * before we get a chance to send it over.
542  */
543  list_sort(walFileList, compareWalFileNames);
544 
545  /*
546  * There must be at least one xlog file in the pg_wal directory, since
547  * we are doing backup-including-xlog.
548  */
549  if (walFileList == NIL)
550  ereport(ERROR,
551  (errmsg("could not find any WAL files")));
552 
553  /*
554  * Sanity check: the first and last segment should cover startptr and
555  * endptr, with no gaps in between.
556  */
557  XLogFromFileName((char *) linitial(walFileList),
558  &tli, &segno, wal_segment_size);
559  if (segno != startsegno)
560  {
561  char startfname[MAXFNAMELEN];
562 
563  XLogFileName(startfname, ThisTimeLineID, startsegno,
565  ereport(ERROR,
566  (errmsg("could not find WAL file \"%s\"", startfname)));
567  }
568  foreach(lc, walFileList)
569  {
570  char *walFileName = (char *) lfirst(lc);
571  XLogSegNo currsegno = segno;
572  XLogSegNo nextsegno = segno + 1;
573 
574  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
575  if (!(nextsegno == segno || currsegno == segno))
576  {
577  char nextfname[MAXFNAMELEN];
578 
579  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
581  ereport(ERROR,
582  (errmsg("could not find WAL file \"%s\"", nextfname)));
583  }
584  }
585  if (segno != endsegno)
586  {
587  char endfname[MAXFNAMELEN];
588 
589  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
590  ereport(ERROR,
591  (errmsg("could not find WAL file \"%s\"", endfname)));
592  }
593 
594  /* Ok, we have everything we need. Send the WAL files. */
595  foreach(lc, walFileList)
596  {
597  char *walFileName = (char *) lfirst(lc);
598  FILE *fp;
599  char buf[TAR_SEND_SIZE];
600  size_t cnt;
601  pgoff_t len = 0;
602 
603  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
604  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
605 
606  fp = AllocateFile(pathbuf, "rb");
607  if (fp == NULL)
608  {
609  int save_errno = errno;
610 
611  /*
612  * Most likely reason for this is that the file was already
613  * removed by a checkpoint, so check for that to get a better
614  * error message.
615  */
616  CheckXLogRemoved(segno, tli);
617 
618  errno = save_errno;
619  ereport(ERROR,
621  errmsg("could not open file \"%s\": %m", pathbuf)));
622  }
623 
624  if (fstat(fileno(fp), &statbuf) != 0)
625  ereport(ERROR,
627  errmsg("could not stat file \"%s\": %m",
628  pathbuf)));
629  if (statbuf.st_size != wal_segment_size)
630  {
631  CheckXLogRemoved(segno, tli);
632  ereport(ERROR,
634  errmsg("unexpected WAL file size \"%s\"", walFileName)));
635  }
636 
637  /* send the WAL file itself */
638  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
639 
640  while ((cnt = fread(buf, 1,
641  Min(sizeof(buf), wal_segment_size - len),
642  fp)) > 0)
643  {
644  CheckXLogRemoved(segno, tli);
645  /* Send the chunk as a CopyData message */
646  if (pq_putmessage('d', buf, cnt))
647  ereport(ERROR,
648  (errmsg("base backup could not send data, aborting backup")));
650 
651  len += cnt;
652  throttle(cnt);
653 
654  if (len == wal_segment_size)
655  break;
656  }
657 
658  CHECK_FREAD_ERROR(fp, pathbuf);
659 
660  if (len != wal_segment_size)
661  {
662  CheckXLogRemoved(segno, tli);
663  ereport(ERROR,
665  errmsg("unexpected WAL file size \"%s\"", walFileName)));
666  }
667 
668  /* wal_segment_size is a multiple of 512, so no need for padding */
669 
670  FreeFile(fp);
671 
672  /*
673  * Mark file as archived, otherwise files can get archived again
674  * after promotion of a new node. This is in line with
675  * walreceiver.c always doing an XLogArchiveForceDone() after a
676  * complete segment.
677  */
678  StatusFilePath(pathbuf, walFileName, ".done");
679  sendFileWithContent(pathbuf, "", &manifest);
680  }
681 
682  /*
683  * Send timeline history files too. Only the latest timeline history
684  * file is required for recovery, and even that only if there happens
685  * to be a timeline switch in the first WAL segment that contains the
686  * checkpoint record, or if we're taking a base backup from a standby
687  * server and the target timeline changes while the backup is taken.
688  * But they are small and highly useful for debugging purposes, so
689  * better include them all, always.
690  */
691  foreach(lc, historyFileList)
692  {
693  char *fname = lfirst(lc);
694 
695  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
696 
697  if (lstat(pathbuf, &statbuf) != 0)
698  ereport(ERROR,
700  errmsg("could not stat file \"%s\": %m", pathbuf)));
701 
702  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
703  &manifest, NULL);
704 
705  /* unconditionally mark file as archived */
706  StatusFilePath(pathbuf, fname, ".done");
707  sendFileWithContent(pathbuf, "", &manifest);
708  }
709 
710  /* Send CopyDone message for the last tar file */
711  pq_putemptymessage('c');
712  }
713 
714  AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
715 
716  SendBackupManifest(&manifest);
717 
718  SendXlogRecPtrResult(endptr, endtli);
719 
721  {
722  if (total_checksum_failures > 1)
724  (errmsg("%lld total checksum verification failures", total_checksum_failures)));
725 
726  ereport(ERROR,
728  errmsg("checksum verification failure during base backup")));
729  }
730 
731  /* clean up the resource owner we created */
732  WalSndResourceCleanup(true);
733 
735 }
#define StatusFilePath(path, xlog, suffix)
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
static void throttle(size_t increment)
Definition: basebackup.c:1885
static int64 backup_total
Definition: basebackup.c:136
uint32 TimeLineID
Definition: xlogdefs.h:52
#define PROGRESS_BASEBACKUP_PHASE
Definition: progress.h:123
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11415
int wal_segment_size
Definition: xlog.c:116
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:321
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
Definition: pgstat.c:3214
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1574
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3235
char * pstrdup(const char *in)
Definition: mcxt.c:1186
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:920
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:955
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED
Definition: progress.h:127
int errcode(int sqlerrcode)
Definition: elog.c:610
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE
Definition: progress.h:133
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:84
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
int64 sendTablespace(char *path, char *spcoid, bool sizeonly, backup_manifest_info *manifest)
Definition: basebackup.c:1145
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10474
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
Definition: type.h:89
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT
Definition: progress.h:130
#define pgoff_t
Definition: win32_port.h:194
#define TABLESPACE_MAP
Definition: xlog.h:389
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
static void sendFileWithContent(const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1088
#define linitial(l)
Definition: pg_list.h:195
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
backup_manifest_option manifest
Definition: basebackup.c:57
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:11002
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3925
#define MAXPGPATH
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
void SendBackupManifest(backup_manifest_info *manifest)
static int64 backup_streamed
Definition: basebackup.c:139
int errcode_for_file_access(void)
Definition: elog.c:633
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:50
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2320
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:1033
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2581
char * pgstat_stat_directory
Definition: pgstat.c:133
#define CHECK_FREAD_ERROR(fp, filename)
Definition: basebackup.c:104
static char * statrelpath
Definition: basebackup.c:87
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:321
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
#define MAXFNAMELEN
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void * palloc0(Size size)
Definition: mcxt.c:980
void pgstat_progress_end_command(void)
Definition: pgstat.c:3286
#define XLOGDIR
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL
Definition: progress.h:126
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL
Definition: progress.h:134
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:347
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:191
static TimestampTz throttled_last
Definition: basebackup.c:121
#define ereport(elevel,...)
Definition: elog.h:144
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1959
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:738
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2647
#define THROTTLING_FREQUENCY
Definition: basebackup.c:97
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3257
static bool manifest
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static int list_length(const List *l)
Definition: pg_list.h:169
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
#define lstat(path, sb)
Definition: win32_port.h:244
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP
Definition: progress.h:132
int FreeFile(FILE *file)
Definition: fd.c:2519
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:742
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1198
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1481
int errmsg(const char *fmt,...)
Definition: elog.c:824
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:92
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1560
char * DataDir
Definition: globals.c:62
#define BACKUP_LABEL_FILE
Definition: xlog.h:386
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:115
Definition: pg_list.h:50
#define snprintf
Definition: port.h:193
static uint64 throttling_sample
Definition: basebackup.c:112
long val
Definition: informix.c:664
int FreeDir(DIR *dir)
Definition: fd.c:2699
static TimeOffset elapsed_min_unit
Definition: basebackup.c:118
static XLogRecPtr startptr
Definition: basebackup.c:124
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:422
static long long int total_checksum_failures
Definition: basebackup.c:127
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)

◆ send_int8_string()

static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 945 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), pq_sendint32(), and sprintf.

Referenced by SendBackupHeader().

946 {
947  char is[32];
948 
949  sprintf(is, INT64_FORMAT, intval);
950  pq_sendint32(buf, strlen(is));
951  pq_sendbytes(buf, is, strlen(is));
952 }
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:195
#define INT64_FORMAT
Definition: c.h:409
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125

◆ SendBackupHeader()

static void SendBackupHeader ( List tablespaces)
static

Definition at line 955 of file basebackup.c.

References buf, lfirst, tablespaceinfo::oid, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), send_int8_string(), and tablespaceinfo::size.

Referenced by perform_base_backup().

956 {
958  ListCell *lc;
959 
960  /* Construct and send the directory information */
961  pq_beginmessage(&buf, 'T'); /* RowDescription */
962  pq_sendint16(&buf, 3); /* 3 fields */
963 
964  /* First field - spcoid */
965  pq_sendstring(&buf, "spcoid");
966  pq_sendint32(&buf, 0); /* table oid */
967  pq_sendint16(&buf, 0); /* attnum */
968  pq_sendint32(&buf, OIDOID); /* type oid */
969  pq_sendint16(&buf, 4); /* typlen */
970  pq_sendint32(&buf, 0); /* typmod */
971  pq_sendint16(&buf, 0); /* format code */
972 
973  /* Second field - spclocation */
974  pq_sendstring(&buf, "spclocation");
975  pq_sendint32(&buf, 0);
976  pq_sendint16(&buf, 0);
977  pq_sendint32(&buf, TEXTOID);
978  pq_sendint16(&buf, -1);
979  pq_sendint32(&buf, 0);
980  pq_sendint16(&buf, 0);
981 
982  /* Third field - size */
983  pq_sendstring(&buf, "size");
984  pq_sendint32(&buf, 0);
985  pq_sendint16(&buf, 0);
986  pq_sendint32(&buf, INT8OID);
987  pq_sendint16(&buf, 8);
988  pq_sendint32(&buf, 0);
989  pq_sendint16(&buf, 0);
990  pq_endmessage(&buf);
991 
992  foreach(lc, tablespaces)
993  {
994  tablespaceinfo *ti = lfirst(lc);
995 
996  /* Send one datarow message */
997  pq_beginmessage(&buf, 'D');
998  pq_sendint16(&buf, 3); /* number of columns */
999  if (ti->path == NULL)
1000  {
1001  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
1002  pq_sendint32(&buf, -1);
1003  }
1004  else
1005  {
1006  Size len;
1007 
1008  len = strlen(ti->oid);
1009  pq_sendint32(&buf, len);
1010  pq_sendbytes(&buf, ti->oid, len);
1011 
1012  len = strlen(ti->path);
1013  pq_sendint32(&buf, len);
1014  pq_sendbytes(&buf, ti->path, len);
1015  }
1016  if (ti->size >= 0)
1017  send_int8_string(&buf, ti->size / 1024);
1018  else
1019  pq_sendint32(&buf, -1); /* NULL */
1020 
1021  pq_endmessage(&buf);
1022  }
1023 
1024  /* Send a CommandComplete message */
1025  pq_puttextmessage('C', "SELECT");
1026 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:945
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
#define lfirst(lc)
Definition: pg_list.h:190
size_t Size
Definition: c.h:466
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 924 of file basebackup.c.

References basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

925 {
926  basebackup_options opt;
927 
928  parse_basebackup_options(cmd->options, &opt);
929 
931 
933  {
934  char activitymsg[50];
935 
936  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
937  opt.label);
938  set_ps_display(activitymsg);
939  }
940 
941  perform_base_backup(&opt);
942 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:36
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:754
void set_ps_display(const char *activity)
Definition: ps_status.c:349
const char * label
Definition: basebackup.c:50
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:264
void WalSndSetState(WalSndState state)
Definition: walsender.c:3205
#define snprintf
Definition: port.h:193

◆ sendDir()

static int64 sendDir ( const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1198 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, DEBUG2, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, exclude_list_item::match_prefix, MAXPGPATH, exclude_list_item::name, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, pgwin32_is_junction(), ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISREG, sendFile(), snprintf, stat, statrelpath, TABLESPACE_VERSION_DIRECTORY, and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

1201 {
1202  DIR *dir;
1203  struct dirent *de;
1204  char pathbuf[MAXPGPATH * 2];
1205  struct stat statbuf;
1206  int64 size = 0;
1207  const char *lastDir; /* Split last dir from parent path. */
1208  bool isDbDir = false; /* Does this directory contain relations? */
1209 
1210  /*
1211  * Determine if the current path is a database directory that can contain
1212  * relations.
1213  *
1214  * Start by finding the location of the delimiter between the parent path
1215  * and the current path.
1216  */
1217  lastDir = last_dir_separator(path);
1218 
1219  /* Does this path look like a database path (i.e. all digits)? */
1220  if (lastDir != NULL &&
1221  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1222  {
1223  /* Part of path that contains the parent directory. */
1224  int parentPathLen = lastDir - path;
1225 
1226  /*
1227  * Mark path as a database directory if the parent path is either
1228  * $PGDATA/base or a tablespace version path.
1229  */
1230  if (strncmp(path, "./base", parentPathLen) == 0 ||
1231  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1232  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1234  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1235  isDbDir = true;
1236  }
1237 
1238  dir = AllocateDir(path);
1239  while ((de = ReadDir(dir, path)) != NULL)
1240  {
1241  int excludeIdx;
1242  bool excludeFound;
1243  ForkNumber relForkNum; /* Type of fork if file is a relation */
1244  int relOidChars; /* Chars in filename that are the rel oid */
1245 
1246  /* Skip special stuff */
1247  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1248  continue;
1249 
1250  /* Skip temporary files */
1251  if (strncmp(de->d_name,
1253  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1254  continue;
1255 
1256  /*
1257  * Check if the postmaster has signaled us to exit, and abort with an
1258  * error in that case. The error handler further up will call
1259  * do_pg_abort_backup() for us. Also check that if the backup was
1260  * started while still in recovery, the server wasn't promoted.
1261  * do_pg_stop_backup() will check that too, but it's better to stop
1262  * the backup early than continue to the end and fail there.
1263  */
1266  ereport(ERROR,
1267  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1268  errmsg("the standby was promoted during online backup"),
1269  errhint("This means that the backup being taken is corrupt "
1270  "and should not be used. "
1271  "Try taking another online backup.")));
1272 
1273  /* Scan for files that should be excluded */
1274  excludeFound = false;
1275  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1276  {
1277  int cmplen = strlen(excludeFiles[excludeIdx].name);
1278 
1279  if (!excludeFiles[excludeIdx].match_prefix)
1280  cmplen++;
1281  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1282  {
1283  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1284  excludeFound = true;
1285  break;
1286  }
1287  }
1288 
1289  if (excludeFound)
1290  continue;
1291 
1292  /* Exclude all forks for unlogged tables except the init fork */
1293  if (isDbDir &&
1294  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1295  &relForkNum))
1296  {
1297  /* Never exclude init forks */
1298  if (relForkNum != INIT_FORKNUM)
1299  {
1300  char initForkFile[MAXPGPATH];
1301  char relOid[OIDCHARS + 1];
1302 
1303  /*
1304  * If any other type of fork, check if there is an init fork
1305  * with the same OID. If so, the file can be excluded.
1306  */
1307  memcpy(relOid, de->d_name, relOidChars);
1308  relOid[relOidChars] = '\0';
1309  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1310  path, relOid);
1311 
1312  if (lstat(initForkFile, &statbuf) == 0)
1313  {
1314  elog(DEBUG2,
1315  "unlogged relation file \"%s\" excluded from backup",
1316  de->d_name);
1317 
1318  continue;
1319  }
1320  }
1321  }
1322 
1323  /* Exclude temporary relations */
1324  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1325  {
1326  elog(DEBUG2,
1327  "temporary relation file \"%s\" excluded from backup",
1328  de->d_name);
1329 
1330  continue;
1331  }
1332 
1333  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1334 
1335  /* Skip pg_control here to back up it last */
1336  if (strcmp(pathbuf, "./global/pg_control") == 0)
1337  continue;
1338 
1339  if (lstat(pathbuf, &statbuf) != 0)
1340  {
1341  if (errno != ENOENT)
1342  ereport(ERROR,
1344  errmsg("could not stat file or directory \"%s\": %m",
1345  pathbuf)));
1346 
1347  /* If the file went away while scanning, it's not an error. */
1348  continue;
1349  }
1350 
1351  /* Scan for directories whose contents should be excluded */
1352  excludeFound = false;
1353  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1354  {
1355  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1356  {
1357  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1358  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1359  excludeFound = true;
1360  break;
1361  }
1362  }
1363 
1364  if (excludeFound)
1365  continue;
1366 
1367  /*
1368  * Exclude contents of directory specified by statrelpath if not set
1369  * to the default (pg_stat_tmp) which is caught in the loop above.
1370  */
1371  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1372  {
1373  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1374  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1375  continue;
1376  }
1377 
1378  /*
1379  * We can skip pg_wal, the WAL segments need to be fetched from the
1380  * WAL archive anyway. But include it as an empty directory anyway, so
1381  * we get permissions right.
1382  */
1383  if (strcmp(pathbuf, "./pg_wal") == 0)
1384  {
1385  /* If pg_wal is a symlink, write it as a directory anyway */
1386  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1387 
1388  /*
1389  * Also send archive_status directory (by hackishly reusing
1390  * statbuf from above ...).
1391  */
1392  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1393  sizeonly);
1394 
1395  continue; /* don't recurse into pg_wal */
1396  }
1397 
1398  /* Allow symbolic links in pg_tblspc only */
1399  if (strcmp(path, "./pg_tblspc") == 0 &&
1400 #ifndef WIN32
1401  S_ISLNK(statbuf.st_mode)
1402 #else
1403  pgwin32_is_junction(pathbuf)
1404 #endif
1405  )
1406  {
1407 #if defined(HAVE_READLINK) || defined(WIN32)
1408  char linkpath[MAXPGPATH];
1409  int rllen;
1410 
1411  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1412  if (rllen < 0)
1413  ereport(ERROR,
1415  errmsg("could not read symbolic link \"%s\": %m",
1416  pathbuf)));
1417  if (rllen >= sizeof(linkpath))
1418  ereport(ERROR,
1419  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1420  errmsg("symbolic link \"%s\" target is too long",
1421  pathbuf)));
1422  linkpath[rllen] = '\0';
1423 
1424  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1425  &statbuf, sizeonly);
1426 #else
1427 
1428  /*
1429  * If the platform does not have symbolic links, it should not be
1430  * possible to have tablespaces - clearly somebody else created
1431  * them. Warn about it and ignore.
1432  */
1433  ereport(WARNING,
1434  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1435  errmsg("tablespaces are not supported on this platform")));
1436  continue;
1437 #endif /* HAVE_READLINK */
1438  }
1439  else if (S_ISDIR(statbuf.st_mode))
1440  {
1441  bool skip_this_dir = false;
1442  ListCell *lc;
1443 
1444  /*
1445  * Store a directory entry in the tar file so we can get the
1446  * permissions right.
1447  */
1448  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1449  sizeonly);
1450 
1451  /*
1452  * Call ourselves recursively for a directory, unless it happens
1453  * to be a separate tablespace located within PGDATA.
1454  */
1455  foreach(lc, tablespaces)
1456  {
1457  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1458 
1459  /*
1460  * ti->rpath is the tablespace relative path within PGDATA, or
1461  * NULL if the tablespace has been properly located somewhere
1462  * else.
1463  *
1464  * Skip past the leading "./" in pathbuf when comparing.
1465  */
1466  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1467  {
1468  skip_this_dir = true;
1469  break;
1470  }
1471  }
1472 
1473  /*
1474  * skip sending directories inside pg_tblspc, if not required.
1475  */
1476  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1477  skip_this_dir = true;
1478 
1479  if (!skip_this_dir)
1480  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
1481  sendtblspclinks, manifest, spcoid);
1482  }
1483  else if (S_ISREG(statbuf.st_mode))
1484  {
1485  bool sent = false;
1486 
1487  if (!sizeonly)
1488  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1489  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1490  manifest, spcoid);
1491 
1492  if (sent || sizeonly)
1493  {
1494  /* Add size, rounded up to 512byte block */
1495  size += ((statbuf.st_size + 511) & ~511);
1496  size += 512; /* Size of the header of the file */
1497  }
1498  }
1499  else
1500  ereport(WARNING,
1501  (errmsg("skipping special file \"%s\"", pathbuf)));
1502  }
1503  FreeDir(dir);
1504  return size;
1505 }
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1071
char * rpath
Definition: basebackup.h:30
int errcode(int sqlerrcode)
Definition: elog.c:610
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:84
Definition: dirent.h:9
static const char *const excludeDirContents[]
Definition: basebackup.c:161
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
#define readlink(path, buf, size)
Definition: win32_port.h:222
int errcode_for_file_access(void)
Definition: elog.c:633
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2581
#define atooid(x)
Definition: postgres_ext.h:42
static char * statrelpath
Definition: basebackup.c:87
ForkNumber
Definition: relpath.h:40
#define S_ISREG(m)
Definition: win32_port.h:299
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define InvalidOid
Definition: postgres_ext.h:36
#define ereport(elevel,...)
Definition: elog.h:144
#define lfirst(lc)
Definition: pg_list.h:190
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2647
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3179
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:374
const char * name
Definition: encode.c:555
#define S_ISDIR(m)
Definition: win32_port.h:296
#define lstat(path, sb)
Definition: win32_port.h:244
const char * name
Definition: basebackup.c:149
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1198
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define elog(elevel,...)
Definition: elog.h:214
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1560
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
char d_name[MAX_PATH]
Definition: dirent.h:14
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:202
#define snprintf
Definition: port.h:193
#define OIDCHARS
Definition: relpath.h:30
int FreeDir(DIR *dir)
Definition: fd.c:2699
bool pgwin32_is_junction(const char *path)
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1865

◆ sendFile()

static bool sendFile ( const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1560 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), AllocateFile(), Assert, buf, CHECK_FREAD_ERROR, backup_manifest_info::checksum_type, DataChecksumsEnabled(), ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, filename, FreeFile(), i, is_checksummed_file(), last_dir_separator(), MemSet, Min, noverify_checksums, PageGetLSN, PageIsNew, PageHeaderData::pd_checksum, pg_checksum_init(), pg_checksum_page(), pg_checksum_update(), pgoff_t, pgstat_report_checksum_failures_in_db(), pq_putmessage, startptr, TAR_SEND_SIZE, throttle(), total_checksum_failures, update_basebackup_progress(), and WARNING.

Referenced by perform_base_backup(), and sendDir().

1563 {
1564  FILE *fp;
1565  BlockNumber blkno = 0;
1566  bool block_retry = false;
1567  char buf[TAR_SEND_SIZE];
1568  uint16 checksum;
1569  int checksum_failures = 0;
1570  off_t cnt;
1571  int i;
1572  pgoff_t len = 0;
1573  char *page;
1574  size_t pad;
1575  PageHeader phdr;
1576  int segmentno = 0;
1577  char *segmentpath;
1578  bool verify_checksum = false;
1579  pg_checksum_context checksum_ctx;
1580 
1581  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1582 
1583  fp = AllocateFile(readfilename, "rb");
1584  if (fp == NULL)
1585  {
1586  if (errno == ENOENT && missing_ok)
1587  return false;
1588  ereport(ERROR,
1590  errmsg("could not open file \"%s\": %m", readfilename)));
1591  }
1592 
1593  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1594 
1596  {
1597  char *filename;
1598 
1599  /*
1600  * Get the filename (excluding path). As last_dir_separator()
1601  * includes the last directory separator, we chop that off by
1602  * incrementing the pointer.
1603  */
1604  filename = last_dir_separator(readfilename) + 1;
1605 
1606  if (is_checksummed_file(readfilename, filename))
1607  {
1608  verify_checksum = true;
1609 
1610  /*
1611  * Cut off at the segment boundary (".") to get the segment number
1612  * in order to mix it into the checksum.
1613  */
1614  segmentpath = strstr(filename, ".");
1615  if (segmentpath != NULL)
1616  {
1617  segmentno = atoi(segmentpath + 1);
1618  if (segmentno == 0)
1619  ereport(ERROR,
1620  (errmsg("invalid segment number %d in file \"%s\"",
1621  segmentno, filename)));
1622  }
1623  }
1624  }
1625 
1626  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1627  {
1628  /*
1629  * The checksums are verified at block level, so we iterate over the
1630  * buffer in chunks of BLCKSZ, after making sure that
1631  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1632  * BLCKSZ bytes.
1633  */
1634  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1635 
1636  if (verify_checksum && (cnt % BLCKSZ != 0))
1637  {
1638  ereport(WARNING,
1639  (errmsg("could not verify checksum in file \"%s\", block "
1640  "%d: read buffer size %d and page size %d "
1641  "differ",
1642  readfilename, blkno, (int) cnt, BLCKSZ)));
1643  verify_checksum = false;
1644  }
1645 
1646  if (verify_checksum)
1647  {
1648  for (i = 0; i < cnt / BLCKSZ; i++)
1649  {
1650  page = buf + BLCKSZ * i;
1651 
1652  /*
1653  * Only check pages which have not been modified since the
1654  * start of the base backup. Otherwise, they might have been
1655  * written only halfway and the checksum would not be valid.
1656  * However, replaying WAL would reinstate the correct page in
1657  * this case. We also skip completely new pages, since they
1658  * don't have a checksum yet.
1659  */
1660  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1661  {
1662  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1663  phdr = (PageHeader) page;
1664  if (phdr->pd_checksum != checksum)
1665  {
1666  /*
1667  * Retry the block on the first failure. It's
1668  * possible that we read the first 4K page of the
1669  * block just before postgres updated the entire block
1670  * so it ends up looking torn to us. We only need to
1671  * retry once because the LSN should be updated to
1672  * something we can ignore on the next pass. If the
1673  * error happens again then it is a true validation
1674  * failure.
1675  */
1676  if (block_retry == false)
1677  {
1678  /* Reread the failed block */
1679  if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
1680  {
1681  ereport(ERROR,
1683  errmsg("could not fseek in file \"%s\": %m",
1684  readfilename)));
1685  }
1686 
1687  if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
1688  {
1689  /*
1690  * If we hit end-of-file, a concurrent
1691  * truncation must have occurred, so break out
1692  * of this loop just as if the initial fread()
1693  * returned 0. We'll drop through to the same
1694  * code that handles that case. (We must fix
1695  * up cnt first, though.)
1696  */
1697  if (feof(fp))
1698  {
1699  cnt = BLCKSZ * i;
1700  break;
1701  }
1702 
1703  ereport(ERROR,
1705  errmsg("could not reread block %d of file \"%s\": %m",
1706  blkno, readfilename)));
1707  }
1708 
1709  if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
1710  {
1711  ereport(ERROR,
1713  errmsg("could not fseek in file \"%s\": %m",
1714  readfilename)));
1715  }
1716 
1717  /* Set flag so we know a retry was attempted */
1718  block_retry = true;
1719 
1720  /* Reset loop to validate the block again */
1721  i--;
1722  continue;
1723  }
1724 
1725  checksum_failures++;
1726 
1727  if (checksum_failures <= 5)
1728  ereport(WARNING,
1729  (errmsg("checksum verification failed in "
1730  "file \"%s\", block %d: calculated "
1731  "%X but expected %X",
1732  readfilename, blkno, checksum,
1733  phdr->pd_checksum)));
1734  if (checksum_failures == 5)
1735  ereport(WARNING,
1736  (errmsg("further checksum verification "
1737  "failures in file \"%s\" will not "
1738  "be reported", readfilename)));
1739  }
1740  }
1741  block_retry = false;
1742  blkno++;
1743  }
1744  }
1745 
1746  /* Send the chunk as a CopyData message */
1747  if (pq_putmessage('d', buf, cnt))
1748  ereport(ERROR,
1749  (errmsg("base backup could not send data, aborting backup")));
1751 
1752  /* Also feed it to the checksum machinery. */
1753  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1754 
1755  len += cnt;
1756  throttle(cnt);
1757 
1758  if (feof(fp) || len >= statbuf->st_size)
1759  {
1760  /*
1761  * Reached end of file. The file could be longer, if it was
1762  * extended while we were sending it, but for a base backup we can
1763  * ignore such extended data. It will be restored from WAL.
1764  */
1765  break;
1766  }
1767  }
1768 
1769  CHECK_FREAD_ERROR(fp, readfilename);
1770 
1771  /* If the file was truncated while we were sending it, pad it with zeros */
1772  if (len < statbuf->st_size)
1773  {
1774  MemSet(buf, 0, sizeof(buf));
1775  while (len < statbuf->st_size)
1776  {
1777  cnt = Min(sizeof(buf), statbuf->st_size - len);
1778  pq_putmessage('d', buf, cnt);
1779  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1781  len += cnt;
1782  throttle(cnt);
1783  }
1784  }
1785 
1786  /*
1787  * Pad to 512 byte boundary, per tar format requirements. (This small
1788  * piece of data is probably not worth throttling, and is not checksummed
1789  * because it's not actually part of the file.)
1790  */
1791  pad = ((len + 511) & ~511) - len;
1792  if (pad > 0)
1793  {
1794  MemSet(buf, 0, pad);
1795  pq_putmessage('d', buf, pad);
1797  }
1798 
1799  FreeFile(fp);
1800 
1801  if (checksum_failures > 1)
1802  {
1803  ereport(WARNING,
1804  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1805  "file \"%s\" has a total of %d checksum verification failures",
1806  checksum_failures,
1807  readfilename, checksum_failures)));
1808 
1809  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1810  }
1811 
1812  total_checksum_failures += checksum_failures;
1813 
1814  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1815  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1816 
1817  return true;
1818 }
static void throttle(size_t increment)
Definition: basebackup.c:1885
int64 pg_time_t
Definition: pgtime.h:23
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:934
bool DataChecksumsEnabled(void)
Definition: xlog.c:4930
#define Min(x, y)
Definition: c.h:920
unsigned char uint8
Definition: c.h:365
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define MemSet(start, val, len)
Definition: c.h:971
uint32 BlockNumber
Definition: block.h:31
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1585
uint16 pd_checksum
Definition: bufpage.h:156
#define pgoff_t
Definition: win32_port.h:194
unsigned short uint16
Definition: c.h:366
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define ERROR
Definition: elog.h:43
static bool noverify_checksums
Definition: basebackup.c:130
static char * buf
Definition: pg_test_fsync.c:67
int errcode_for_file_access(void)
Definition: elog.c:633
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2320
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
#define CHECK_FREAD_ERROR(fp, filename)
Definition: basebackup.c:104
#define WARNING
Definition: elog.h:40
pg_checksum_type checksum_type
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define ereport(elevel,...)
Definition: elog.h:144
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1514
PageHeaderData * PageHeader
Definition: bufpage.h:166
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1959
#define Assert(condition)
Definition: c.h:738
#define PageGetLSN(page)
Definition: bufpage.h:366
int FreeFile(FILE *file)
Definition: fd.c:2519
static char * filename
Definition: pg_dumpall.c:90
#define PageIsNew(page)
Definition: bufpage.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define TAR_SEND_SIZE
Definition: basebackup.c:92
int i
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
uint16 pg_checksum_page(char *page, BlockNumber blkno)
static XLogRecPtr startptr
Definition: basebackup.c:124
static long long int total_checksum_failures
Definition: basebackup.c:127

◆ sendFileWithContent()

static void sendFileWithContent ( const char *  filename,
const char *  content,
backup_manifest_info manifest 
)
static

Definition at line 1088 of file basebackup.c.

References _tarWriteHeader(), AddFileToBackupManifest(), buf, backup_manifest_info::checksum_type, MemSet, pg_checksum_init(), pg_checksum_update(), pg_file_create_mode, pq_putmessage, stat, and update_basebackup_progress().

Referenced by perform_base_backup().

1090 {
1091  struct stat statbuf;
1092  int pad,
1093  len;
1094  pg_checksum_context checksum_ctx;
1095 
1096  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1097 
1098  len = strlen(content);
1099 
1100  /*
1101  * Construct a stat struct for the backup_label file we're injecting in
1102  * the tar.
1103  */
1104  /* Windows doesn't have the concept of uid and gid */
1105 #ifdef WIN32
1106  statbuf.st_uid = 0;
1107  statbuf.st_gid = 0;
1108 #else
1109  statbuf.st_uid = geteuid();
1110  statbuf.st_gid = getegid();
1111 #endif
1112  statbuf.st_mtime = time(NULL);
1113  statbuf.st_mode = pg_file_create_mode;
1114  statbuf.st_size = len;
1115 
1116  _tarWriteHeader(filename, NULL, &statbuf, false);
1117  /* Send the contents as a CopyData message */
1118  pq_putmessage('d', content, len);
1120 
1121  /* Pad to 512 byte boundary, per tar format requirements */
1122  pad = ((len + 511) & ~511) - len;
1123  if (pad > 0)
1124  {
1125  char buf[512];
1126 
1127  MemSet(buf, 0, pad);
1128  pq_putmessage('d', buf, pad);
1130  }
1131 
1132  pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
1133  AddFileToBackupManifest(manifest, NULL, filename, len,
1134  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1135 }
int pg_file_create_mode
Definition: file_perm.c:19
int64 pg_time_t
Definition: pgtime.h:23
unsigned char uint8
Definition: c.h:365
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define MemSet(start, val, len)
Definition: c.h:971
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
static char * buf
Definition: pg_test_fsync.c:67
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
#define stat(a, b)
Definition: win32_port.h:255
pg_checksum_type checksum_type
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1959
static char * filename
Definition: pg_dumpall.c:90
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42

◆ sendTablespace()

int64 sendTablespace ( char *  path,
char *  spcoid,
bool  sizeonly,
backup_manifest_info manifest 
)

Definition at line 1145 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, sendDir(), snprintf, stat, and TABLESPACE_VERSION_DIRECTORY.

Referenced by do_pg_start_backup(), and perform_base_backup().

1147 {
1148  int64 size;
1149  char pathbuf[MAXPGPATH];
1150  struct stat statbuf;
1151 
1152  /*
1153  * 'path' points to the tablespace location, but we only want to include
1154  * the version directory in it that belongs to us.
1155  */
1156  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1158 
1159  /*
1160  * Store a directory entry in the tar file so we get the permissions
1161  * right.
1162  */
1163  if (lstat(pathbuf, &statbuf) != 0)
1164  {
1165  if (errno != ENOENT)
1166  ereport(ERROR,
1168  errmsg("could not stat file or directory \"%s\": %m",
1169  pathbuf)));
1170 
1171  /* If the tablespace went away while scanning, it's no error. */
1172  return 0;
1173  }
1174 
1175  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1176  sizeonly);
1177 
1178  /* Send all the files in the tablespace version directory */
1179  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1180  spcoid);
1181 
1182  return size;
1183 }
#define NIL
Definition: pg_list.h:65
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
int errcode_for_file_access(void)
Definition: elog.c:633
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
#define stat(a, b)
Definition: win32_port.h:255
#define ereport(elevel,...)
Definition: elog.h:144
#define lstat(path, sb)
Definition: win32_port.h:244
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1198
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define snprintf
Definition: port.h:193

◆ SendXlogRecPtrResult()

static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 1033 of file basebackup.c.

References buf, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), snprintf, and generate_unaccent_rules::str.

Referenced by perform_base_backup().

1034 {
1036  char str[MAXFNAMELEN];
1037  Size len;
1038 
1039  pq_beginmessage(&buf, 'T'); /* RowDescription */
1040  pq_sendint16(&buf, 2); /* 2 fields */
1041 
1042  /* Field headers */
1043  pq_sendstring(&buf, "recptr");
1044  pq_sendint32(&buf, 0); /* table oid */
1045  pq_sendint16(&buf, 0); /* attnum */
1046  pq_sendint32(&buf, TEXTOID); /* type oid */
1047  pq_sendint16(&buf, -1);
1048  pq_sendint32(&buf, 0);
1049  pq_sendint16(&buf, 0);
1050 
1051  pq_sendstring(&buf, "tli");
1052  pq_sendint32(&buf, 0); /* table oid */
1053  pq_sendint16(&buf, 0); /* attnum */
1054 
1055  /*
1056  * int8 may seem like a surprising data type for this, but in theory int4
1057  * would not be wide enough for this, as TimeLineID is unsigned.
1058  */
1059  pq_sendint32(&buf, INT8OID); /* type oid */
1060  pq_sendint16(&buf, -1);
1061  pq_sendint32(&buf, 0);
1062  pq_sendint16(&buf, 0);
1063  pq_endmessage(&buf);
1064 
1065  /* Data row */
1066  pq_beginmessage(&buf, 'D');
1067  pq_sendint16(&buf, 2); /* number of columns */
1068 
1069  len = snprintf(str, sizeof(str),
1070  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
1071  pq_sendint32(&buf, len);
1072  pq_sendbytes(&buf, str, len);
1073 
1074  len = snprintf(str, sizeof(str), "%u", tli);
1075  pq_sendint32(&buf, len);
1076  pq_sendbytes(&buf, str, len);
1077 
1078  pq_endmessage(&buf);
1079 
1080  /* Send a CommandComplete message */
1081  pq_puttextmessage('C', "SELECT");
1082 }
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:367
#define MAXFNAMELEN
size_t Size
Definition: c.h:466
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
#define snprintf
Definition: port.h:193

◆ throttle()

static void throttle ( size_t  increment)
static

Definition at line 1885 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_EXIT_ON_PM_DEATH, WL_LATCH_SET, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1886 {
1887  TimeOffset elapsed_min;
1888 
1889  if (throttling_counter < 0)
1890  return;
1891 
1892  throttling_counter += increment;
1894  return;
1895 
1896  /* How much time should have elapsed at minimum? */
1897  elapsed_min = elapsed_min_unit *
1899 
1900  /*
1901  * Since the latch could be set repeatedly because of concurrently WAL
1902  * activity, sleep in a loop to ensure enough time has passed.
1903  */
1904  for (;;)
1905  {
1906  TimeOffset elapsed,
1907  sleep;
1908  int wait_result;
1909 
1910  /* Time elapsed since the last measurement (and possible wake up). */
1911  elapsed = GetCurrentTimestamp() - throttled_last;
1912 
1913  /* sleep if the transfer is faster than it should be */
1914  sleep = elapsed_min - elapsed;
1915  if (sleep <= 0)
1916  break;
1917 
1919 
1920  /* We're eating a potentially set latch, so check for interrupts */
1922 
1923  /*
1924  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1925  * the maximum time to sleep. Thus the cast to long is safe.
1926  */
1927  wait_result = WaitLatch(MyLatch,
1929  (long) (sleep / 1000),
1931 
1932  if (wait_result & WL_LATCH_SET)
1934 
1935  /* Done waiting? */
1936  if (wait_result & WL_TIMEOUT)
1937  break;
1938  }
1939 
1940  /*
1941  * As we work with integers, only whole multiple of throttling_sample was
1942  * processed. The rest will be done during the next call of this function.
1943  */
1945 
1946  /*
1947  * Time interval for the remaining amount and possible next increments
1948  * starts now.
1949  */
1951 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1574
void ResetLatch(Latch *latch)
Definition: latch.c:540
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:365
int64 TimeOffset
Definition: timestamp.h:40
static TimestampTz throttled_last
Definition: basebackup.c:121
struct Latch * MyLatch
Definition: globals.c:54
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
static int64 throttling_counter
Definition: basebackup.c:115
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:112
static TimeOffset elapsed_min_unit
Definition: basebackup.c:118
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129

◆ update_basebackup_progress()

static void update_basebackup_progress ( int64  delta)
static

Definition at line 1959 of file basebackup.c.

References backup_streamed, backup_total, pgstat_progress_update_multi_param(), PROGRESS_BASEBACKUP_BACKUP_STREAMED, PROGRESS_BASEBACKUP_BACKUP_TOTAL, and val.

Referenced by _tarWriteHeader(), perform_base_backup(), sendFile(), and sendFileWithContent().

1960 {
1961  const int index[] = {
1964  };
1965  int64 val[2];
1966  int nparam = 0;
1967 
1968  backup_streamed += delta;
1969  val[nparam++] = backup_streamed;
1970 
1971  /*
1972  * Avoid overflowing past 100% or the full size. This may make the total
1973  * size number change as we approach the end of the backup (the estimate
1974  * will always be wrong if WAL is included), but that's better than having
1975  * the done column be bigger than the total.
1976  */
1978  {
1980  val[nparam++] = backup_total;
1981  }
1982 
1983  pgstat_progress_update_multi_param(nparam, index, val);
1984 }
static int64 backup_total
Definition: basebackup.c:136
Definition: type.h:89
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED
Definition: progress.h:125
static int64 backup_streamed
Definition: basebackup.c:139
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3257
long val
Definition: informix.c:664

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 84 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

◆ backup_streamed

int64 backup_streamed = 0
static

Definition at line 139 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ backup_total

int64 backup_total = 0
static

Definition at line 136 of file basebackup.c.

Referenced by perform_base_backup(), and update_basebackup_progress().

◆ elapsed_min_unit

TimeOffset elapsed_min_unit
static

Definition at line 118 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:50
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 161 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{PG_AUTOCONF_FILENAME ".tmp", false},
{TABLESPACE_MAP, false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}
#define TABLESPACE_MAP
Definition: xlog.h:389
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:96
#define BACKUP_LABEL_FILE
Definition: xlog.h:386

Definition at line 202 of file basebackup.c.

◆ noChecksumFiles

const struct exclude_list_item noChecksumFiles[]
static
Initial value:
= {
{"pg_control", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
{NULL, false}
}

Definition at line 246 of file basebackup.c.

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 130 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ startptr

◆ statrelpath

char* statrelpath = NULL
static

Definition at line 87 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

◆ throttled_last

TimestampTz throttled_last
static

Definition at line 121 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_counter

int64 throttling_counter
static

Definition at line 115 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ throttling_sample

uint64 throttling_sample
static

Definition at line 112 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 127 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().