PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "access/xlogbackup.h"
#include "backup/backup_manifest.h"
#include "backup/basebackup.h"
#include "backup/basebackup_sink.h"
#include "backup/basebackup_target.h"
#include "commands/defrem.h"
#include "common/compression.h"
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/guc.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 
struct  exclude_list_item
 

Macros

#define SINK_BUFFER_LENGTH   Max(32768, BLCKSZ)
 

Functions

static int64 sendTablespace (bbsink *sink, char *path, char *spcoid, bool sizeonly, struct backup_manifest_info *manifest)
 
static int64 sendDir (bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
 
static bool sendFile (bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 
static void sendFileWithContent (bbsink *sink, const char *filename, const char *content, backup_manifest_info *manifest)
 
static int64 _tarWriteHeader (bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static void _tarWritePadding (bbsink *sink, int len)
 
static void convert_link_to_directory (const char *pathbuf, struct stat *statbuf)
 
static void perform_base_backup (basebackup_options *opt, bbsink *sink)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
static int basebackup_read_file (int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 

Variables

static bool backup_started_in_recovery = false
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 
static const struct exclude_list_item noChecksumFiles []
 

Macro Definition Documentation

◆ SINK_BUFFER_LENGTH

#define SINK_BUFFER_LENGTH   Max(32768, BLCKSZ)

Definition at line 58 of file basebackup.c.

Function Documentation

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( bbsink sink,
const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1738 of file basebackup.c.

1740 {
1741  enum tarError rc;
1742 
1743  if (!sizeonly)
1744  {
1745  /*
1746  * As of this writing, the smallest supported block size is 1kB, which
1747  * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
1748  * multiple of BLCKSZ, it should be safe to assume that the buffer is
1749  * large enough to fit an entire tar block. We double-check by means
1750  * of these assertions.
1751  */
1752  StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
1753  "BLCKSZ too small for tar block");
1755 
1756  rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
1757  statbuf->st_size, statbuf->st_mode,
1758  statbuf->st_uid, statbuf->st_gid,
1759  statbuf->st_mtime);
1760 
1761  switch (rc)
1762  {
1763  case TAR_OK:
1764  break;
1765  case TAR_NAME_TOO_LONG:
1766  ereport(ERROR,
1767  (errmsg("file name too long for tar format: \"%s\"",
1768  filename)));
1769  break;
1770  case TAR_SYMLINK_TOO_LONG:
1771  ereport(ERROR,
1772  (errmsg("symbolic link target too long for tar format: "
1773  "file name \"%s\", target \"%s\"",
1774  filename, linktarget)));
1775  break;
1776  default:
1777  elog(ERROR, "unrecognized tar error: %d", rc);
1778  }
1779 
1781  }
1782 
1783  return TAR_BLOCK_SIZE;
1784 }
static void bbsink_archive_contents(bbsink *sink, size_t len)
#define StaticAssertDecl(condition, errmessage)
Definition: c.h:925
int errmsg(const char *fmt,...)
Definition: elog.c:1069
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
Assert(fmt[strlen(fmt) - 1] !='\n')
static char * filename
Definition: pg_dumpall.c:119
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
tarError
Definition: pgtar.h:20
@ TAR_SYMLINK_TOO_LONG
Definition: pgtar.h:23
@ TAR_OK
Definition: pgtar.h:21
@ TAR_NAME_TOO_LONG
Definition: pgtar.h:22
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
char * bbs_buffer
size_t bbs_buffer_length
__time64_t st_mtime
Definition: win32_port.h:275
__int64 st_size
Definition: win32_port.h:273
short st_gid
Definition: win32_port.h:271
unsigned short st_mode
Definition: win32_port.h:268
short st_uid
Definition: win32_port.h:270

References Assert(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), elog(), ereport, errmsg(), ERROR, filename, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, stat::st_uid, StaticAssertDecl, TAR_BLOCK_SIZE, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

◆ _tarWritePadding()

static void _tarWritePadding ( bbsink sink,
int  len 
)
static

Definition at line 1790 of file basebackup.c.

1791 {
1792  int pad = tarPaddingBytesRequired(len);
1793 
1794  /*
1795  * As in _tarWriteHeader, it should be safe to assume that the buffer is
1796  * large enough that we don't need to do this in multiple chunks.
1797  */
1799  Assert(pad <= TAR_BLOCK_SIZE);
1800 
1801  if (pad > 0)
1802  {
1803  MemSet(sink->bbs_buffer, 0, pad);
1804  bbsink_archive_contents(sink, pad);
1805  }
1806 }
#define MemSet(start, val, len)
Definition: c.h:1009
const void size_t len
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:79

References Assert(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), len, MemSet, TAR_BLOCK_SIZE, and tarPaddingBytesRequired().

Referenced by sendFile(), and sendFileWithContent().

◆ basebackup_read_file()

static int basebackup_read_file ( int  fd,
char *  buf,
size_t  nbytes,
off_t  offset,
const char *  filename,
bool  partial_read_ok 
)
static

Definition at line 1830 of file basebackup.c.

1832 {
1833  int rc;
1834 
1835  pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1836  rc = pg_pread(fd, buf, nbytes, offset);
1838 
1839  if (rc < 0)
1840  ereport(ERROR,
1842  errmsg("could not read file \"%s\": %m", filename)));
1843  if (!partial_read_ok && rc > 0 && rc != nbytes)
1844  ereport(ERROR,
1846  errmsg("could not read file \"%s\": read %d of %zu",
1847  filename, rc, nbytes)));
1848 
1849  return rc;
1850 }
int errcode_for_file_access(void)
Definition: elog.c:881
static char * buf
Definition: pg_test_fsync.c:67
#define pg_pread
Definition: port.h:225
static int fd(const char *x, int i)
Definition: preproc-init.c:105
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:88
static void pgstat_report_wait_end(void)
Definition: wait_event.h:104

References buf, ereport, errcode_for_file_access(), errmsg(), ERROR, fd(), filename, pg_pread, pgstat_report_wait_end(), and pgstat_report_wait_start().

Referenced by perform_base_backup(), and sendFile().

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 674 of file basebackup.c.

675 {
676  char *fna = (char *) lfirst(a);
677  char *fnb = (char *) lfirst(b);
678 
679  return strcmp(fna + 8, fnb + 8);
680 }
int b
Definition: isn.c:70
int a
Definition: isn.c:69
#define lfirst(lc)
Definition: pg_list.h:172

References a, b, and lfirst.

Referenced by perform_base_backup().

◆ convert_link_to_directory()

static void convert_link_to_directory ( const char *  pathbuf,
struct stat statbuf 
)
static

Definition at line 1813 of file basebackup.c.

1814 {
1815  /* If symlink, write it as a directory anyway */
1816  if (S_ISLNK(statbuf->st_mode))
1817  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1818 }
int pg_dir_create_mode
Definition: file_perm.c:18
#define S_ISLNK(m)
Definition: win32_port.h:344

References pg_dir_create_mode, S_ISLNK, and stat::st_mode.

Referenced by sendDir().

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1443 of file basebackup.c.

1444 {
1445  /* Check that the file is in a tablespace */
1446  if (strncmp(fullpath, "./global/", 9) == 0 ||
1447  strncmp(fullpath, "./base/", 7) == 0 ||
1448  strncmp(fullpath, "/", 1) == 0)
1449  {
1450  int excludeIdx;
1451 
1452  /* Compare file against noChecksumFiles skip list */
1453  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1454  {
1455  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1456 
1457  if (!noChecksumFiles[excludeIdx].match_prefix)
1458  cmplen++;
1459  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1460  cmplen) == 0)
1461  return false;
1462  }
1463 
1464  return true;
1465  }
1466  else
1467  return false;
1468 }
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:212
const char * name
Definition: basebackup.c:118
const char * name

References filename, exclude_list_item::name, name, and noChecksumFiles.

Referenced by sendFile().

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 686 of file basebackup.c.

687 {
688  ListCell *lopt;
689  bool o_label = false;
690  bool o_progress = false;
691  bool o_checkpoint = false;
692  bool o_nowait = false;
693  bool o_wal = false;
694  bool o_maxrate = false;
695  bool o_tablespace_map = false;
696  bool o_noverify_checksums = false;
697  bool o_manifest = false;
698  bool o_manifest_checksums = false;
699  bool o_target = false;
700  bool o_target_detail = false;
701  char *target_str = NULL;
702  char *target_detail_str = NULL;
703  bool o_compression = false;
704  bool o_compression_detail = false;
705  char *compression_detail_str = NULL;
706 
707  MemSet(opt, 0, sizeof(*opt));
712 
713  foreach(lopt, options)
714  {
715  DefElem *defel = (DefElem *) lfirst(lopt);
716 
717  if (strcmp(defel->defname, "label") == 0)
718  {
719  if (o_label)
720  ereport(ERROR,
721  (errcode(ERRCODE_SYNTAX_ERROR),
722  errmsg("duplicate option \"%s\"", defel->defname)));
723  opt->label = defGetString(defel);
724  o_label = true;
725  }
726  else if (strcmp(defel->defname, "progress") == 0)
727  {
728  if (o_progress)
729  ereport(ERROR,
730  (errcode(ERRCODE_SYNTAX_ERROR),
731  errmsg("duplicate option \"%s\"", defel->defname)));
732  opt->progress = defGetBoolean(defel);
733  o_progress = true;
734  }
735  else if (strcmp(defel->defname, "checkpoint") == 0)
736  {
737  char *optval = defGetString(defel);
738 
739  if (o_checkpoint)
740  ereport(ERROR,
741  (errcode(ERRCODE_SYNTAX_ERROR),
742  errmsg("duplicate option \"%s\"", defel->defname)));
743  if (pg_strcasecmp(optval, "fast") == 0)
744  opt->fastcheckpoint = true;
745  else if (pg_strcasecmp(optval, "spread") == 0)
746  opt->fastcheckpoint = false;
747  else
748  ereport(ERROR,
749  (errcode(ERRCODE_SYNTAX_ERROR),
750  errmsg("unrecognized checkpoint type: \"%s\"",
751  optval)));
752  o_checkpoint = true;
753  }
754  else if (strcmp(defel->defname, "wait") == 0)
755  {
756  if (o_nowait)
757  ereport(ERROR,
758  (errcode(ERRCODE_SYNTAX_ERROR),
759  errmsg("duplicate option \"%s\"", defel->defname)));
760  opt->nowait = !defGetBoolean(defel);
761  o_nowait = true;
762  }
763  else if (strcmp(defel->defname, "wal") == 0)
764  {
765  if (o_wal)
766  ereport(ERROR,
767  (errcode(ERRCODE_SYNTAX_ERROR),
768  errmsg("duplicate option \"%s\"", defel->defname)));
769  opt->includewal = defGetBoolean(defel);
770  o_wal = true;
771  }
772  else if (strcmp(defel->defname, "max_rate") == 0)
773  {
774  int64 maxrate;
775 
776  if (o_maxrate)
777  ereport(ERROR,
778  (errcode(ERRCODE_SYNTAX_ERROR),
779  errmsg("duplicate option \"%s\"", defel->defname)));
780 
781  maxrate = defGetInt64(defel);
782  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
783  ereport(ERROR,
784  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
785  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
786  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
787 
788  opt->maxrate = (uint32) maxrate;
789  o_maxrate = true;
790  }
791  else if (strcmp(defel->defname, "tablespace_map") == 0)
792  {
793  if (o_tablespace_map)
794  ereport(ERROR,
795  (errcode(ERRCODE_SYNTAX_ERROR),
796  errmsg("duplicate option \"%s\"", defel->defname)));
797  opt->sendtblspcmapfile = defGetBoolean(defel);
798  o_tablespace_map = true;
799  }
800  else if (strcmp(defel->defname, "verify_checksums") == 0)
801  {
802  if (o_noverify_checksums)
803  ereport(ERROR,
804  (errcode(ERRCODE_SYNTAX_ERROR),
805  errmsg("duplicate option \"%s\"", defel->defname)));
807  o_noverify_checksums = true;
808  }
809  else if (strcmp(defel->defname, "manifest") == 0)
810  {
811  char *optval = defGetString(defel);
812  bool manifest_bool;
813 
814  if (o_manifest)
815  ereport(ERROR,
816  (errcode(ERRCODE_SYNTAX_ERROR),
817  errmsg("duplicate option \"%s\"", defel->defname)));
818  if (parse_bool(optval, &manifest_bool))
819  {
820  if (manifest_bool)
822  else
824  }
825  else if (pg_strcasecmp(optval, "force-encode") == 0)
827  else
828  ereport(ERROR,
829  (errcode(ERRCODE_SYNTAX_ERROR),
830  errmsg("unrecognized manifest option: \"%s\"",
831  optval)));
832  o_manifest = true;
833  }
834  else if (strcmp(defel->defname, "manifest_checksums") == 0)
835  {
836  char *optval = defGetString(defel);
837 
838  if (o_manifest_checksums)
839  ereport(ERROR,
840  (errcode(ERRCODE_SYNTAX_ERROR),
841  errmsg("duplicate option \"%s\"", defel->defname)));
842  if (!pg_checksum_parse_type(optval,
843  &opt->manifest_checksum_type))
844  ereport(ERROR,
845  (errcode(ERRCODE_SYNTAX_ERROR),
846  errmsg("unrecognized checksum algorithm: \"%s\"",
847  optval)));
848  o_manifest_checksums = true;
849  }
850  else if (strcmp(defel->defname, "target") == 0)
851  {
852  if (o_target)
853  ereport(ERROR,
854  (errcode(ERRCODE_SYNTAX_ERROR),
855  errmsg("duplicate option \"%s\"", defel->defname)));
856  target_str = defGetString(defel);
857  o_target = true;
858  }
859  else if (strcmp(defel->defname, "target_detail") == 0)
860  {
861  char *optval = defGetString(defel);
862 
863  if (o_target_detail)
864  ereport(ERROR,
865  (errcode(ERRCODE_SYNTAX_ERROR),
866  errmsg("duplicate option \"%s\"", defel->defname)));
867  target_detail_str = optval;
868  o_target_detail = true;
869  }
870  else if (strcmp(defel->defname, "compression") == 0)
871  {
872  char *optval = defGetString(defel);
873 
874  if (o_compression)
875  ereport(ERROR,
876  (errcode(ERRCODE_SYNTAX_ERROR),
877  errmsg("duplicate option \"%s\"", defel->defname)));
878  if (!parse_compress_algorithm(optval, &opt->compression))
879  ereport(ERROR,
880  (errcode(ERRCODE_SYNTAX_ERROR),
881  errmsg("unrecognized compression algorithm: \"%s\"",
882  optval)));
883  o_compression = true;
884  }
885  else if (strcmp(defel->defname, "compression_detail") == 0)
886  {
887  if (o_compression_detail)
888  ereport(ERROR,
889  (errcode(ERRCODE_SYNTAX_ERROR),
890  errmsg("duplicate option \"%s\"", defel->defname)));
891  compression_detail_str = defGetString(defel);
892  o_compression_detail = true;
893  }
894  else
895  ereport(ERROR,
896  (errcode(ERRCODE_SYNTAX_ERROR),
897  errmsg("unrecognized base backup option: \"%s\"",
898  defel->defname)));
899  }
900 
901  if (opt->label == NULL)
902  opt->label = "base backup";
903  if (opt->manifest == MANIFEST_OPTION_NO)
904  {
905  if (o_manifest_checksums)
906  ereport(ERROR,
907  (errcode(ERRCODE_SYNTAX_ERROR),
908  errmsg("manifest checksums require a backup manifest")));
910  }
911 
912  if (target_str == NULL)
913  {
914  if (target_detail_str != NULL)
915  ereport(ERROR,
916  (errcode(ERRCODE_SYNTAX_ERROR),
917  errmsg("target detail cannot be used without target")));
918  opt->use_copytblspc = true;
919  opt->send_to_client = true;
920  }
921  else if (strcmp(target_str, "client") == 0)
922  {
923  if (target_detail_str != NULL)
924  ereport(ERROR,
925  (errcode(ERRCODE_SYNTAX_ERROR),
926  errmsg("target \"%s\" does not accept a target detail",
927  target_str)));
928  opt->send_to_client = true;
929  }
930  else
931  opt->target_handle =
932  BaseBackupGetTargetHandle(target_str, target_detail_str);
933 
934  if (o_compression_detail && !o_compression)
935  ereport(ERROR,
936  (errcode(ERRCODE_SYNTAX_ERROR),
937  errmsg("compression detail cannot be specified unless compression is enabled")));
938 
939  if (o_compression)
940  {
941  char *error_detail;
942 
943  parse_compress_specification(opt->compression, compression_detail_str,
945  error_detail =
947  if (error_detail != NULL)
948  ereport(ERROR,
949  errcode(ERRCODE_SYNTAX_ERROR),
950  errmsg("invalid compression specification: %s",
951  error_detail));
952  }
953 }
@ MANIFEST_OPTION_NO
@ MANIFEST_OPTION_FORCE_ENCODE
@ MANIFEST_OPTION_YES
static bool noverify_checksums
Definition: basebackup.c:108
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define MAX_RATE_LOWER
Definition: basebackup.h:20
BaseBackupTargetHandle * BaseBackupGetTargetHandle(char *target, char *target_detail)
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
unsigned int uint32
Definition: c.h:495
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
@ CHECKSUM_TYPE_NONE
@ CHECKSUM_TYPE_CRC32C
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:49
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:107
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:344
@ PG_COMPRESSION_NONE
Definition: compression.h:23
bool defGetBoolean(DefElem *def)
Definition: define.c:108
char * defGetString(DefElem *def)
Definition: define.c:49
int64 defGetInt64(DefElem *def)
Definition: define.c:187
int errcode(int sqlerrcode)
Definition: elog.c:858
static int32 maxrate
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * defname
Definition: parsenodes.h:809
pg_compress_specification compression_specification
Definition: basebackup.c:74
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:75
const char * label
Definition: basebackup.c:62
pg_compress_algorithm compression
Definition: basebackup.c:73
backup_manifest_option manifest
Definition: basebackup.c:72
BaseBackupTargetHandle * target_handle
Definition: basebackup.c:71
pg_compress_algorithm algorithm
Definition: compression.h:34

References pg_compress_specification::algorithm, BaseBackupGetTargetHandle(), CHECKSUM_TYPE_CRC32C, CHECKSUM_TYPE_NONE, basebackup_options::compression, basebackup_options::compression_specification, defGetBoolean(), defGetInt64(), defGetString(), DefElem::defname, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, basebackup_options::label, lfirst, basebackup_options::manifest, basebackup_options::manifest_checksum_type, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, MANIFEST_OPTION_YES, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, parse_bool(), parse_compress_algorithm(), parse_compress_specification(), pg_checksum_parse_type(), PG_COMPRESSION_NONE, pg_strcasecmp(), basebackup_options::progress, basebackup_options::send_to_client, basebackup_options::sendtblspcmapfile, basebackup_options::target_handle, basebackup_options::use_copytblspc, and validate_compress_specification().

Referenced by SendBaseBackup().

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt,
bbsink sink 
)
static

Definition at line 230 of file basebackup.c.

231 {
233  XLogRecPtr endptr;
234  TimeLineID endtli;
238 
239  /* Initial backup state, insofar as we know it now. */
240  state.tablespaces = NIL;
241  state.tablespace_num = 0;
242  state.bytes_done = 0;
243  state.bytes_total = 0;
244  state.bytes_total_is_valid = false;
245 
246  /* we're going to use a BufFile, so we need a ResourceOwner */
247  Assert(CurrentResourceOwner == NULL);
248  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
249 
251 
254 
256 
257  /* Allocate backup related variables. */
260 
262  do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
264 
265  state.startptr = backup_state->startpoint;
266  state.starttli = backup_state->starttli;
267 
268  /*
269  * Once do_pg_backup_start has been called, ensure that any failure causes
270  * us to abort the backup so we don't "leak" a backup counter. For this
271  * reason, *all* functionality between do_pg_backup_start() and the end of
272  * do_pg_backup_stop() should be inside the error cleanup block!
273  */
274 
276  {
277  ListCell *lc;
278  tablespaceinfo *newti;
279 
280  /* Add a node for the base directory at the end */
281  newti = palloc0(sizeof(tablespaceinfo));
282  newti->size = -1;
283  state.tablespaces = lappend(state.tablespaces, newti);
284 
285  /*
286  * Calculate the total backup size by summing up the size of each
287  * tablespace
288  */
289  if (opt->progress)
290  {
292 
293  foreach(lc, state.tablespaces)
294  {
295  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
296 
297  if (tmp->path == NULL)
298  tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
299  true, NULL, NULL);
300  else
301  tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
302  NULL);
303  state.bytes_total += tmp->size;
304  }
305  state.bytes_total_is_valid = true;
306  }
307 
308  /* notify basebackup sink about start of backup */
310 
311  /* Send off our tablespaces one by one */
312  foreach(lc, state.tablespaces)
313  {
314  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
315 
316  if (ti->path == NULL)
317  {
318  struct stat statbuf;
319  bool sendtblspclinks = true;
320  char *backup_label;
321 
322  bbsink_begin_archive(sink, "base.tar");
323 
324  /* In the main tar, include the backup_label first... */
325  backup_label = build_backup_content(backup_state, false);
327  backup_label, &manifest);
328  pfree(backup_label);
329 
330  /* Then the tablespace_map file, if required... */
331  if (opt->sendtblspcmapfile)
332  {
335  sendtblspclinks = false;
336  }
337 
338  /* Then the bulk of the files... */
339  sendDir(sink, ".", 1, false, state.tablespaces,
340  sendtblspclinks, &manifest, NULL);
341 
342  /* ... and pg_control after everything else. */
343  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
344  ereport(ERROR,
346  errmsg("could not stat file \"%s\": %m",
349  false, InvalidOid, &manifest, NULL);
350  }
351  else
352  {
353  char *archive_name = psprintf("%s.tar", ti->oid);
354 
355  bbsink_begin_archive(sink, archive_name);
356 
357  sendTablespace(sink, ti->path, ti->oid, false, &manifest);
358  }
359 
360  /*
361  * If we're including WAL, and this is the main data directory we
362  * don't treat this as the end of the tablespace. Instead, we will
363  * include the xlog files below and stop afterwards. This is safe
364  * since the main data directory is always sent *last*.
365  */
366  if (opt->includewal && ti->path == NULL)
367  {
368  Assert(lnext(state.tablespaces, lc) == NULL);
369  }
370  else
371  {
372  /* Properly terminate the tarfile. */
373  StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
374  "BLCKSZ too small for 2 tar blocks");
375  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
377 
378  /* OK, that's the end of the archive. */
379  bbsink_end_archive(sink);
380  }
381  }
382 
385 
386  endptr = backup_state->stoppoint;
387  endtli = backup_state->stoptli;
388 
389  /* Deallocate backup-related variables. */
393  }
395 
396 
397  if (opt->includewal)
398  {
399  /*
400  * We've left the last tar file "open", so we can now append the
401  * required WAL files to it.
402  */
403  char pathbuf[MAXPGPATH];
404  XLogSegNo segno;
405  XLogSegNo startsegno;
406  XLogSegNo endsegno;
407  struct stat statbuf;
408  List *historyFileList = NIL;
409  List *walFileList = NIL;
410  char firstoff[MAXFNAMELEN];
411  char lastoff[MAXFNAMELEN];
412  DIR *dir;
413  struct dirent *de;
414  ListCell *lc;
415  TimeLineID tli;
416 
418 
419  /*
420  * I'd rather not worry about timelines here, so scan pg_wal and
421  * include all WAL files in the range between 'startptr' and 'endptr',
422  * regardless of the timeline the file is stamped with. If there are
423  * some spurious WAL files belonging to timelines that don't belong in
424  * this server's history, they will be included too. Normally there
425  * shouldn't be such files, but if there are, there's little harm in
426  * including them.
427  */
428  XLByteToSeg(state.startptr, startsegno, wal_segment_size);
429  XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
430  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
431  XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
432 
433  dir = AllocateDir("pg_wal");
434  while ((de = ReadDir(dir, "pg_wal")) != NULL)
435  {
436  /* Does it look like a WAL segment, and is it in the range? */
437  if (IsXLogFileName(de->d_name) &&
438  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
439  strcmp(de->d_name + 8, lastoff + 8) <= 0)
440  {
441  walFileList = lappend(walFileList, pstrdup(de->d_name));
442  }
443  /* Does it look like a timeline history file? */
444  else if (IsTLHistoryFileName(de->d_name))
445  {
446  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
447  }
448  }
449  FreeDir(dir);
450 
451  /*
452  * Before we go any further, check that none of the WAL segments we
453  * need were removed.
454  */
455  CheckXLogRemoved(startsegno, state.starttli);
456 
457  /*
458  * Sort the WAL filenames. We want to send the files in order from
459  * oldest to newest, to reduce the chance that a file is recycled
460  * before we get a chance to send it over.
461  */
462  list_sort(walFileList, compareWalFileNames);
463 
464  /*
465  * There must be at least one xlog file in the pg_wal directory, since
466  * we are doing backup-including-xlog.
467  */
468  if (walFileList == NIL)
469  ereport(ERROR,
470  (errmsg("could not find any WAL files")));
471 
472  /*
473  * Sanity check: the first and last segment should cover startptr and
474  * endptr, with no gaps in between.
475  */
476  XLogFromFileName((char *) linitial(walFileList),
477  &tli, &segno, wal_segment_size);
478  if (segno != startsegno)
479  {
480  char startfname[MAXFNAMELEN];
481 
482  XLogFileName(startfname, state.starttli, startsegno,
484  ereport(ERROR,
485  (errmsg("could not find WAL file \"%s\"", startfname)));
486  }
487  foreach(lc, walFileList)
488  {
489  char *walFileName = (char *) lfirst(lc);
490  XLogSegNo currsegno = segno;
491  XLogSegNo nextsegno = segno + 1;
492 
493  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
494  if (!(nextsegno == segno || currsegno == segno))
495  {
496  char nextfname[MAXFNAMELEN];
497 
498  XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
499  ereport(ERROR,
500  (errmsg("could not find WAL file \"%s\"", nextfname)));
501  }
502  }
503  if (segno != endsegno)
504  {
505  char endfname[MAXFNAMELEN];
506 
507  XLogFileName(endfname, endtli, endsegno, wal_segment_size);
508  ereport(ERROR,
509  (errmsg("could not find WAL file \"%s\"", endfname)));
510  }
511 
512  /* Ok, we have everything we need. Send the WAL files. */
513  foreach(lc, walFileList)
514  {
515  char *walFileName = (char *) lfirst(lc);
516  int fd;
517  size_t cnt;
518  pgoff_t len = 0;
519 
520  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
521  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
522 
523  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
524  if (fd < 0)
525  {
526  int save_errno = errno;
527 
528  /*
529  * Most likely reason for this is that the file was already
530  * removed by a checkpoint, so check for that to get a better
531  * error message.
532  */
533  CheckXLogRemoved(segno, tli);
534 
535  errno = save_errno;
536  ereport(ERROR,
538  errmsg("could not open file \"%s\": %m", pathbuf)));
539  }
540 
541  if (fstat(fd, &statbuf) != 0)
542  ereport(ERROR,
544  errmsg("could not stat file \"%s\": %m",
545  pathbuf)));
546  if (statbuf.st_size != wal_segment_size)
547  {
548  CheckXLogRemoved(segno, tli);
549  ereport(ERROR,
551  errmsg("unexpected WAL file size \"%s\"", walFileName)));
552  }
553 
554  /* send the WAL file itself */
555  _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
556 
557  while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
558  Min(sink->bbs_buffer_length,
560  len, pathbuf, true)) > 0)
561  {
562  CheckXLogRemoved(segno, tli);
563  bbsink_archive_contents(sink, cnt);
564 
565  len += cnt;
566 
567  if (len == wal_segment_size)
568  break;
569  }
570 
571  if (len != wal_segment_size)
572  {
573  CheckXLogRemoved(segno, tli);
574  ereport(ERROR,
576  errmsg("unexpected WAL file size \"%s\"", walFileName)));
577  }
578 
579  /*
580  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
581  * for padding.
582  */
584 
586 
587  /*
588  * Mark file as archived, otherwise files can get archived again
589  * after promotion of a new node. This is in line with
590  * walreceiver.c always doing an XLogArchiveForceDone() after a
591  * complete segment.
592  */
593  StatusFilePath(pathbuf, walFileName, ".done");
594  sendFileWithContent(sink, pathbuf, "", &manifest);
595  }
596 
597  /*
598  * Send timeline history files too. Only the latest timeline history
599  * file is required for recovery, and even that only if there happens
600  * to be a timeline switch in the first WAL segment that contains the
601  * checkpoint record, or if we're taking a base backup from a standby
602  * server and the target timeline changes while the backup is taken.
603  * But they are small and highly useful for debugging purposes, so
604  * better include them all, always.
605  */
606  foreach(lc, historyFileList)
607  {
608  char *fname = lfirst(lc);
609 
610  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
611 
612  if (lstat(pathbuf, &statbuf) != 0)
613  ereport(ERROR,
615  errmsg("could not stat file \"%s\": %m", pathbuf)));
616 
617  sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
618  &manifest, NULL);
619 
620  /* unconditionally mark file as archived */
621  StatusFilePath(pathbuf, fname, ".done");
622  sendFileWithContent(sink, pathbuf, "", &manifest);
623  }
624 
625  /* Properly terminate the tar file. */
626  StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
627  "BLCKSZ too small for 2 tar blocks");
628  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
630 
631  /* OK, that's the end of the archive. */
632  bbsink_end_archive(sink);
633  }
634 
635  AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
636  endptr, endtli);
637 
639 
640  bbsink_end_backup(sink, endptr, endtli);
641 
643  {
644  if (total_checksum_failures > 1)
646  (errmsg_plural("%lld total checksum verification failure",
647  "%lld total checksum verification failures",
650 
651  ereport(ERROR,
653  errmsg("checksum verification failure during base backup")));
654  }
655 
656  /*
657  * Make sure to free the manifest before the resource owners as manifests
658  * use cryptohash contexts that may depend on resource owners (like
659  * OpenSSL).
660  */
662 
663  /* clean up the resource owner we created */
664  WalSndResourceCleanup(true);
665 
667 }
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
void SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
void FreeBackupManifest(backup_manifest_info *manifest)
static void sendFileWithContent(bbsink *sink, const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1032
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:674
#define SINK_BUFFER_LENGTH
Definition: basebackup.c:58
static bool backup_started_in_recovery
Definition: basebackup.c:102
static int64 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1738
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1482
static int64 sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly, struct backup_manifest_info *manifest)
Definition: basebackup.c:1093
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:1830
static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1146
static long long int total_checksum_failures
Definition: basebackup.c:105
void basebackup_progress_wait_checkpoint(void)
void basebackup_progress_wait_wal_archive(bbsink_state *state)
void basebackup_progress_done(void)
void basebackup_progress_transfer_wal(void)
void basebackup_progress_estimate_backup_size(void)
static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
static void bbsink_begin_archive(bbsink *sink, const char *archive_name)
static void bbsink_end_archive(bbsink *sink)
static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
#define Min(x, y)
Definition: c.h:993
#define PG_BINARY
Definition: c.h:1283
#define StaticAssertStmt(condition, errmessage)
Definition: c.h:927
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1179
#define WARNING
Definition: elog.h:36
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2854
int FreeDir(DIR *dir)
Definition: fd.c:2906
int CloseTransientFile(int fd)
Definition: fd.c:2754
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2578
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2788
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1673
List * lappend(List *list, void *datum)
Definition: list.c:338
char * pstrdup(const char *in)
Definition: mcxt.c:1644
void pfree(void *pointer)
Definition: mcxt.c:1456
void * palloc0(Size size)
Definition: mcxt.c:1257
static bool manifest
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define MAXPGPATH
#define NIL
Definition: pg_list.h:68
#define linitial(l)
Definition: pg_list.h:178
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:343
#define snprintf
Definition: port.h:238
static Datum BoolGetDatum(bool X)
Definition: postgres.h:102
#define InvalidOid
Definition: postgres_ext.h:36
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:429
ResourceOwner CurrentResourceOwner
Definition: resowner.c:147
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
TimeLineID starttli
Definition: xlogbackup.h:27
XLogRecPtr startpoint
Definition: xlogbackup.h:26
TimeLineID stoptli
Definition: xlogbackup.h:34
XLogRecPtr stoppoint
Definition: xlogbackup.h:33
Definition: dirent.c:26
Definition: pg_list.h:54
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
Definition: regguts.h:323
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:350
#define lstat(path, sb)
Definition: win32_port.h:285
#define fstat
Definition: win32_port.h:283
#define pgoff_t
Definition: win32_port.h:207
bool RecoveryInProgress(void)
Definition: xlog.c:5948
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:8882
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3460
int wal_segment_size
Definition: xlog.c:146
void do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, BackupState *state, StringInfo tblspcmapfile)
Definition: xlog.c:8293
void do_pg_backup_stop(BackupState *state, bool waitforarchive)
Definition: xlog.c:8608
#define TABLESPACE_MAP
Definition: xlog.h:294
#define BACKUP_LABEL_FILE
Definition: xlog.h:291
#define XLOG_CONTROL_FILE
static bool IsXLogFileName(const char *fname)
static void XLogFromFileName(const char *fname, TimeLineID *tli, XLogSegNo *logSegNo, int wal_segsz_bytes)
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
static bool IsTLHistoryFileName(const char *fname)
#define MAXFNAMELEN
#define XLOGDIR
static void StatusFilePath(char *path, const char *xlog, const char *suffix)
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
static void XLogFileName(char *fname, TimeLineID tli, XLogSegNo logSegNo, int wal_segsz_bytes)
char * build_backup_content(BackupState *state, bool ishistoryfile)
Definition: xlogbackup.c:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
static BackupState * backup_state
Definition: xlogfuncs.c:46
static StringInfo tablespace_map
Definition: xlogfuncs.c:47

References _tarWriteHeader(), AddWALInfoToBackupManifest(), AllocateDir(), Assert(), BACKUP_LABEL_FILE, backup_started_in_recovery, backup_state, basebackup_progress_done(), basebackup_progress_estimate_backup_size(), basebackup_progress_transfer_wal(), basebackup_progress_wait_checkpoint(), basebackup_progress_wait_wal_archive(), basebackup_read_file(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), bbsink_begin_archive(), bbsink_begin_backup(), bbsink_end_archive(), bbsink_end_backup(), BoolGetDatum(), build_backup_content(), CheckXLogRemoved(), CloseTransientFile(), compareWalFileNames(), CurrentResourceOwner, dirent::d_name, StringInfoData::data, do_pg_abort_backup(), do_pg_backup_start(), do_pg_backup_stop(), ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, basebackup_options::fastcheckpoint, fd(), FreeBackupManifest(), FreeDir(), fstat, basebackup_options::includewal, InitializeBackupManifest(), InvalidOid, IsTLHistoryFileName(), IsXLogFileName(), basebackup_options::label, lappend(), len, lfirst, linitial, list_sort(), lnext(), lstat, makeStringInfo(), basebackup_options::manifest, manifest, basebackup_options::manifest_checksum_type, MAXFNAMELEN, MAXPGPATH, Min, NIL, basebackup_options::nowait, tablespaceinfo::oid, OpenTransientFile(), palloc0(), tablespaceinfo::path, pfree(), PG_BINARY, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, basebackup_options::progress, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), ResourceOwnerCreate(), SendBackupManifest(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SINK_BUFFER_LENGTH, tablespaceinfo::size, snprintf, stat::st_size, BackupState::startpoint, BackupState::starttli, StaticAssertDecl, StaticAssertStmt, StatusFilePath(), BackupState::stoppoint, BackupState::stoptli, tablespace_map, TABLESPACE_MAP, TAR_BLOCK_SIZE, total_checksum_failures, wal_segment_size, WalSndResourceCleanup(), WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName(), and XLogFromFileName().

Referenced by SendBaseBackup().

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 964 of file basebackup.c.

965 {
966  basebackup_options opt;
967  bbsink *sink;
969 
970  if (status == SESSION_BACKUP_RUNNING)
971  ereport(ERROR,
972  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
973  errmsg("a backup is already in progress in this session")));
974 
975  parse_basebackup_options(cmd->options, &opt);
976 
978 
980  {
981  char activitymsg[50];
982 
983  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
984  opt.label);
985  set_ps_display(activitymsg);
986  }
987 
988  /*
989  * If the target is specifically 'client' then set up to stream the backup
990  * to the client; otherwise, it's being sent someplace else and should not
991  * be sent to the client. BaseBackupGetSink has the job of setting up a
992  * sink to send the backup data wherever it needs to go.
993  */
995  if (opt.target_handle != NULL)
996  sink = BaseBackupGetSink(opt.target_handle, sink);
997 
998  /* Set up network throttling, if client requested it */
999  if (opt.maxrate > 0)
1000  sink = bbsink_throttle_new(sink, opt.maxrate);
1001 
1002  /* Set up server-side compression, if client requested it */
1003  if (opt.compression == PG_COMPRESSION_GZIP)
1004  sink = bbsink_gzip_new(sink, &opt.compression_specification);
1005  else if (opt.compression == PG_COMPRESSION_LZ4)
1006  sink = bbsink_lz4_new(sink, &opt.compression_specification);
1007  else if (opt.compression == PG_COMPRESSION_ZSTD)
1008  sink = bbsink_zstd_new(sink, &opt.compression_specification);
1009 
1010  /* Set up progress reporting. */
1011  sink = bbsink_progress_new(sink, opt.progress);
1012 
1013  /*
1014  * Perform the base backup, but make sure we clean up the bbsink even if
1015  * an error occurs.
1016  */
1017  PG_TRY();
1018  {
1019  perform_base_backup(&opt, sink);
1020  }
1021  PG_FINALLY();
1022  {
1023  bbsink_cleanup(sink);
1024  }
1025  PG_END_TRY();
1026 }
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:686
static void perform_base_backup(basebackup_options *opt, bbsink *sink)
Definition: basebackup.c:230
bbsink * bbsink_copystream_new(bool send_to_client)
bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *compress)
bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *compress)
bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)
static void bbsink_cleanup(bbsink *sink)
bbsink * BaseBackupGetSink(BaseBackupTargetHandle *handle, bbsink *next_sink)
bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)
bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *compress)
@ PG_COMPRESSION_GZIP
Definition: compression.h:24
@ PG_COMPRESSION_LZ4
Definition: compression.h:25
@ PG_COMPRESSION_ZSTD
Definition: compression.h:26
#define PG_TRY(...)
Definition: elog.h:370
#define PG_END_TRY(...)
Definition: elog.h:395
#define PG_FINALLY(...)
Definition: elog.h:387
bool update_process_title
Definition: ps_status.c:31
static void set_ps_display(const char *activity)
Definition: ps_status.h:40
List * options
Definition: replnodes.h:44
void WalSndSetState(WalSndState state)
Definition: walsender.c:3470
@ WALSNDSTATE_BACKUP
SessionBackupState get_backup_status(void)
Definition: xlog.c:8589
SessionBackupState
Definition: xlog.h:275
@ SESSION_BACKUP_RUNNING
Definition: xlog.h:277

References BaseBackupGetSink(), bbsink_cleanup(), bbsink_copystream_new(), bbsink_gzip_new(), bbsink_lz4_new(), bbsink_progress_new(), bbsink_throttle_new(), bbsink_zstd_new(), basebackup_options::compression, basebackup_options::compression_specification, ereport, errcode(), errmsg(), ERROR, get_backup_status(), basebackup_options::label, basebackup_options::maxrate, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), PG_COMPRESSION_GZIP, PG_COMPRESSION_LZ4, PG_COMPRESSION_ZSTD, PG_END_TRY, PG_FINALLY, PG_TRY, basebackup_options::progress, basebackup_options::send_to_client, SESSION_BACKUP_RUNNING, set_ps_display(), snprintf, basebackup_options::target_handle, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

◆ sendDir()

static int64 sendDir ( bbsink sink,
const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1146 of file basebackup.c.

1149 {
1150  DIR *dir;
1151  struct dirent *de;
1152  char pathbuf[MAXPGPATH * 2];
1153  struct stat statbuf;
1154  int64 size = 0;
1155  const char *lastDir; /* Split last dir from parent path. */
1156  bool isDbDir = false; /* Does this directory contain relations? */
1157 
1158  /*
1159  * Determine if the current path is a database directory that can contain
1160  * relations.
1161  *
1162  * Start by finding the location of the delimiter between the parent path
1163  * and the current path.
1164  */
1165  lastDir = last_dir_separator(path);
1166 
1167  /* Does this path look like a database path (i.e. all digits)? */
1168  if (lastDir != NULL &&
1169  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1170  {
1171  /* Part of path that contains the parent directory. */
1172  int parentPathLen = lastDir - path;
1173 
1174  /*
1175  * Mark path as a database directory if the parent path is either
1176  * $PGDATA/base or a tablespace version path.
1177  */
1178  if (strncmp(path, "./base", parentPathLen) == 0 ||
1179  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1180  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1182  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1183  isDbDir = true;
1184  }
1185 
1186  dir = AllocateDir(path);
1187  while ((de = ReadDir(dir, path)) != NULL)
1188  {
1189  int excludeIdx;
1190  bool excludeFound;
1191  ForkNumber relForkNum; /* Type of fork if file is a relation */
1192  int relnumchars; /* Chars in filename that are the
1193  * relnumber */
1194 
1195  /* Skip special stuff */
1196  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1197  continue;
1198 
1199  /* Skip temporary files */
1200  if (strncmp(de->d_name,
1202  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1203  continue;
1204 
1205  /*
1206  * Check if the postmaster has signaled us to exit, and abort with an
1207  * error in that case. The error handler further up will call
1208  * do_pg_abort_backup() for us. Also check that if the backup was
1209  * started while still in recovery, the server wasn't promoted.
1210  * do_pg_backup_stop() will check that too, but it's better to stop
1211  * the backup early than continue to the end and fail there.
1212  */
1215  ereport(ERROR,
1216  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1217  errmsg("the standby was promoted during online backup"),
1218  errhint("This means that the backup being taken is corrupt "
1219  "and should not be used. "
1220  "Try taking another online backup.")));
1221 
1222  /* Scan for files that should be excluded */
1223  excludeFound = false;
1224  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1225  {
1226  int cmplen = strlen(excludeFiles[excludeIdx].name);
1227 
1228  if (!excludeFiles[excludeIdx].match_prefix)
1229  cmplen++;
1230  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1231  {
1232  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1233  excludeFound = true;
1234  break;
1235  }
1236  }
1237 
1238  if (excludeFound)
1239  continue;
1240 
1241  /* Exclude all forks for unlogged tables except the init fork */
1242  if (isDbDir &&
1243  parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
1244  &relForkNum))
1245  {
1246  /* Never exclude init forks */
1247  if (relForkNum != INIT_FORKNUM)
1248  {
1249  char initForkFile[MAXPGPATH];
1250  char relNumber[OIDCHARS + 1];
1251 
1252  /*
1253  * If any other type of fork, check if there is an init fork
1254  * with the same RelFileNumber. If so, the file can be
1255  * excluded.
1256  */
1257  memcpy(relNumber, de->d_name, relnumchars);
1258  relNumber[relnumchars] = '\0';
1259  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1260  path, relNumber);
1261 
1262  if (lstat(initForkFile, &statbuf) == 0)
1263  {
1264  elog(DEBUG2,
1265  "unlogged relation file \"%s\" excluded from backup",
1266  de->d_name);
1267 
1268  continue;
1269  }
1270  }
1271  }
1272 
1273  /* Exclude temporary relations */
1274  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1275  {
1276  elog(DEBUG2,
1277  "temporary relation file \"%s\" excluded from backup",
1278  de->d_name);
1279 
1280  continue;
1281  }
1282 
1283  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1284 
1285  /* Skip pg_control here to back up it last */
1286  if (strcmp(pathbuf, "./global/pg_control") == 0)
1287  continue;
1288 
1289  if (lstat(pathbuf, &statbuf) != 0)
1290  {
1291  if (errno != ENOENT)
1292  ereport(ERROR,
1294  errmsg("could not stat file or directory \"%s\": %m",
1295  pathbuf)));
1296 
1297  /* If the file went away while scanning, it's not an error. */
1298  continue;
1299  }
1300 
1301  /* Scan for directories whose contents should be excluded */
1302  excludeFound = false;
1303  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1304  {
1305  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1306  {
1307  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1308  convert_link_to_directory(pathbuf, &statbuf);
1309  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1310  &statbuf, sizeonly);
1311  excludeFound = true;
1312  break;
1313  }
1314  }
1315 
1316  if (excludeFound)
1317  continue;
1318 
1319  /*
1320  * We can skip pg_wal, the WAL segments need to be fetched from the
1321  * WAL archive anyway. But include it as an empty directory anyway, so
1322  * we get permissions right.
1323  */
1324  if (strcmp(pathbuf, "./pg_wal") == 0)
1325  {
1326  /* If pg_wal is a symlink, write it as a directory anyway */
1327  convert_link_to_directory(pathbuf, &statbuf);
1328  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1329  &statbuf, sizeonly);
1330 
1331  /*
1332  * Also send archive_status directory (by hackishly reusing
1333  * statbuf from above ...).
1334  */
1335  size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1336  &statbuf, sizeonly);
1337 
1338  continue; /* don't recurse into pg_wal */
1339  }
1340 
1341  /* Allow symbolic links in pg_tblspc only */
1342  if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
1343  {
1344  char linkpath[MAXPGPATH];
1345  int rllen;
1346 
1347  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1348  if (rllen < 0)
1349  ereport(ERROR,
1351  errmsg("could not read symbolic link \"%s\": %m",
1352  pathbuf)));
1353  if (rllen >= sizeof(linkpath))
1354  ereport(ERROR,
1355  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1356  errmsg("symbolic link \"%s\" target is too long",
1357  pathbuf)));
1358  linkpath[rllen] = '\0';
1359 
1360  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1361  &statbuf, sizeonly);
1362  }
1363  else if (S_ISDIR(statbuf.st_mode))
1364  {
1365  bool skip_this_dir = false;
1366  ListCell *lc;
1367 
1368  /*
1369  * Store a directory entry in the tar file so we can get the
1370  * permissions right.
1371  */
1372  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1373  sizeonly);
1374 
1375  /*
1376  * Call ourselves recursively for a directory, unless it happens
1377  * to be a separate tablespace located within PGDATA.
1378  */
1379  foreach(lc, tablespaces)
1380  {
1381  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1382 
1383  /*
1384  * ti->rpath is the tablespace relative path within PGDATA, or
1385  * NULL if the tablespace has been properly located somewhere
1386  * else.
1387  *
1388  * Skip past the leading "./" in pathbuf when comparing.
1389  */
1390  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1391  {
1392  skip_this_dir = true;
1393  break;
1394  }
1395  }
1396 
1397  /*
1398  * skip sending directories inside pg_tblspc, if not required.
1399  */
1400  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1401  skip_this_dir = true;
1402 
1403  if (!skip_this_dir)
1404  size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1405  sendtblspclinks, manifest, spcoid);
1406  }
1407  else if (S_ISREG(statbuf.st_mode))
1408  {
1409  bool sent = false;
1410 
1411  if (!sizeonly)
1412  sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
1413  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1414  manifest, spcoid);
1415 
1416  if (sent || sizeonly)
1417  {
1418  /* Add size. */
1419  size += statbuf.st_size;
1420 
1421  /* Pad to a multiple of the tar block size. */
1422  size += tarPaddingBytesRequired(statbuf.st_size);
1423 
1424  /* Size of the header for the file. */
1425  size += TAR_BLOCK_SIZE;
1426  }
1427  }
1428  else
1429  ereport(WARNING,
1430  (errmsg("skipping special file \"%s\"", pathbuf)));
1431  }
1432  FreeDir(dir);
1433  return size;
1434 }
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:170
static const char *const excludeDirContents[]
Definition: basebackup.c:130
static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
Definition: basebackup.c:1813
int errhint(const char *fmt,...)
Definition: elog.c:1316
#define DEBUG2
Definition: elog.h:29
#define DEBUG1
Definition: elog.h:30
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3394
#define PG_TEMP_FILE_PREFIX
Definition: file_utils.h:58
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:121
char * last_dir_separator(const char *filename)
Definition: path.c:139
#define atooid(x)
Definition: postgres_ext.h:42
bool parse_filename_for_nontemp_relation(const char *name, int *relnumchars, ForkNumber *fork)
Definition: reinit.c:381
ForkNumber
Definition: relpath.h:48
@ INIT_FORKNUM
Definition: relpath.h:53
#define OIDCHARS
Definition: relpath.h:37
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:33
char * rpath
Definition: basebackup.h:32
#define S_ISDIR(m)
Definition: win32_port.h:325
#define readlink(path, buf, size)
Definition: win32_port.h:236
#define S_ISREG(m)
Definition: win32_port.h:328

References _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, convert_link_to_directory(), dirent::d_name, DEBUG1, DEBUG2, elog(), ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, manifest, MAXPGPATH, exclude_list_item::name, name, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISLNK, S_ISREG, sendFile(), snprintf, stat::st_mode, stat::st_size, TABLESPACE_VERSION_DIRECTORY, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

◆ sendFile()

static bool sendFile ( bbsink sink,
const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1482 of file basebackup.c.

1485 {
1486  int fd;
1487  BlockNumber blkno = 0;
1488  bool block_retry = false;
1489  uint16 checksum;
1490  int checksum_failures = 0;
1491  off_t cnt;
1492  int i;
1493  pgoff_t len = 0;
1494  char *page;
1495  PageHeader phdr;
1496  int segmentno = 0;
1497  char *segmentpath;
1498  bool verify_checksum = false;
1499  pg_checksum_context checksum_ctx;
1500 
1501  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1502  elog(ERROR, "could not initialize checksum of file \"%s\"",
1503  readfilename);
1504 
1505  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1506  if (fd < 0)
1507  {
1508  if (errno == ENOENT && missing_ok)
1509  return false;
1510  ereport(ERROR,
1512  errmsg("could not open file \"%s\": %m", readfilename)));
1513  }
1514 
1515  _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1516 
1518  {
1519  char *filename;
1520 
1521  /*
1522  * Get the filename (excluding path). As last_dir_separator()
1523  * includes the last directory separator, we chop that off by
1524  * incrementing the pointer.
1525  */
1526  filename = last_dir_separator(readfilename) + 1;
1527 
1528  if (is_checksummed_file(readfilename, filename))
1529  {
1530  verify_checksum = true;
1531 
1532  /*
1533  * Cut off at the segment boundary (".") to get the segment number
1534  * in order to mix it into the checksum.
1535  */
1536  segmentpath = strstr(filename, ".");
1537  if (segmentpath != NULL)
1538  {
1539  segmentno = atoi(segmentpath + 1);
1540  if (segmentno == 0)
1541  ereport(ERROR,
1542  (errmsg("invalid segment number %d in file \"%s\"",
1543  segmentno, filename)));
1544  }
1545  }
1546  }
1547 
1548  /*
1549  * Loop until we read the amount of data the caller told us to expect. The
1550  * file could be longer, if it was extended while we were sending it, but
1551  * for a base backup we can ignore such extended data. It will be restored
1552  * from WAL.
1553  */
1554  while (len < statbuf->st_size)
1555  {
1556  size_t remaining = statbuf->st_size - len;
1557 
1558  /* Try to read some more data. */
1559  cnt = basebackup_read_file(fd, sink->bbs_buffer,
1561  len, readfilename, true);
1562 
1563  /*
1564  * The checksums are verified at block level, so we iterate over the
1565  * buffer in chunks of BLCKSZ, after making sure that
1566  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1567  * BLCKSZ bytes.
1568  */
1569  Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1570 
1571  if (verify_checksum && (cnt % BLCKSZ != 0))
1572  {
1573  ereport(WARNING,
1574  (errmsg("could not verify checksum in file \"%s\", block "
1575  "%u: read buffer size %d and page size %d "
1576  "differ",
1577  readfilename, blkno, (int) cnt, BLCKSZ)));
1578  verify_checksum = false;
1579  }
1580 
1581  if (verify_checksum)
1582  {
1583  for (i = 0; i < cnt / BLCKSZ; i++)
1584  {
1585  page = sink->bbs_buffer + BLCKSZ * i;
1586 
1587  /*
1588  * Only check pages which have not been modified since the
1589  * start of the base backup. Otherwise, they might have been
1590  * written only halfway and the checksum would not be valid.
1591  * However, replaying WAL would reinstate the correct page in
1592  * this case. We also skip completely new pages, since they
1593  * don't have a checksum yet.
1594  */
1595  if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr)
1596  {
1597  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1598  phdr = (PageHeader) page;
1599  if (phdr->pd_checksum != checksum)
1600  {
1601  /*
1602  * Retry the block on the first failure. It's
1603  * possible that we read the first 4K page of the
1604  * block just before postgres updated the entire block
1605  * so it ends up looking torn to us. If, before we
1606  * retry the read, the concurrent write of the block
1607  * finishes, the page LSN will be updated and we'll
1608  * realize that we should ignore this block.
1609  *
1610  * There's no guarantee that this will actually
1611  * happen, though: the torn write could take an
1612  * arbitrarily long time to complete. Retrying
1613  * multiple times wouldn't fix this problem, either,
1614  * though it would reduce the chances of it happening
1615  * in practice. The only real fix here seems to be to
1616  * have some kind of interlock that allows us to wait
1617  * until we can be certain that no write to the block
1618  * is in progress. Since we don't have any such thing
1619  * right now, we just do this and hope for the best.
1620  */
1621  if (block_retry == false)
1622  {
1623  int reread_cnt;
1624 
1625  /* Reread the failed block */
1626  reread_cnt =
1628  sink->bbs_buffer + BLCKSZ * i,
1629  BLCKSZ, len + BLCKSZ * i,
1630  readfilename,
1631  false);
1632  if (reread_cnt == 0)
1633  {
1634  /*
1635  * If we hit end-of-file, a concurrent
1636  * truncation must have occurred, so break out
1637  * of this loop just as if the initial fread()
1638  * returned 0. We'll drop through to the same
1639  * code that handles that case. (We must fix
1640  * up cnt first, though.)
1641  */
1642  cnt = BLCKSZ * i;
1643  break;
1644  }
1645 
1646  /* Set flag so we know a retry was attempted */
1647  block_retry = true;
1648 
1649  /* Reset loop to validate the block again */
1650  i--;
1651  continue;
1652  }
1653 
1654  checksum_failures++;
1655 
1656  if (checksum_failures <= 5)
1657  ereport(WARNING,
1658  (errmsg("checksum verification failed in "
1659  "file \"%s\", block %u: calculated "
1660  "%X but expected %X",
1661  readfilename, blkno, checksum,
1662  phdr->pd_checksum)));
1663  if (checksum_failures == 5)
1664  ereport(WARNING,
1665  (errmsg("further checksum verification "
1666  "failures in file \"%s\" will not "
1667  "be reported", readfilename)));
1668  }
1669  }
1670  block_retry = false;
1671  blkno++;
1672  }
1673  }
1674 
1675  /*
1676  * If we hit end-of-file, a concurrent truncation must have occurred.
1677  * That's not an error condition, because WAL replay will fix things
1678  * up.
1679  */
1680  if (cnt == 0)
1681  break;
1682 
1683  /* Archive the data we just read. */
1684  bbsink_archive_contents(sink, cnt);
1685 
1686  /* Also feed it to the checksum machinery. */
1687  if (pg_checksum_update(&checksum_ctx,
1688  (uint8 *) sink->bbs_buffer, cnt) < 0)
1689  elog(ERROR, "could not update checksum of base backup");
1690 
1691  len += cnt;
1692  }
1693 
1694  /* If the file was truncated while we were sending it, pad it with zeros */
1695  while (len < statbuf->st_size)
1696  {
1697  size_t remaining = statbuf->st_size - len;
1698  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1699 
1700  MemSet(sink->bbs_buffer, 0, nbytes);
1701  if (pg_checksum_update(&checksum_ctx,
1702  (uint8 *) sink->bbs_buffer,
1703  nbytes) < 0)
1704  elog(ERROR, "could not update checksum of base backup");
1705  bbsink_archive_contents(sink, nbytes);
1706  len += nbytes;
1707  }
1708 
1709  /*
1710  * Pad to a block boundary, per tar format requirements. (This small piece
1711  * of data is probably not worth throttling, and is not checksummed
1712  * because it's not actually part of the file.)
1713  */
1714  _tarWritePadding(sink, len);
1715 
1717 
1718  if (checksum_failures > 1)
1719  {
1720  ereport(WARNING,
1721  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1722  "file \"%s\" has a total of %d checksum verification failures",
1723  checksum_failures,
1724  readfilename, checksum_failures)));
1725 
1726  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1727  }
1728 
1729  total_checksum_failures += checksum_failures;
1730 
1731  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1732  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1733 
1734  return true;
1735 }
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1443
static void _tarWritePadding(bbsink *sink, int len)
Definition: basebackup.c:1790
uint32 BlockNumber
Definition: block.h:31
PageHeaderData * PageHeader
Definition: bufpage.h:170
static bool PageIsNew(Page page)
Definition: bufpage.h:230
static XLogRecPtr PageGetLSN(Page page)
Definition: bufpage.h:383
unsigned short uint16
Definition: c.h:494
unsigned char uint8
Definition: c.h:493
uint16 pg_checksum_page(char *page, BlockNumber blkno)
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
int remaining
Definition: informix.c:667
int i
Definition: isn.c:73
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
int64 pg_time_t
Definition: pgtime.h:23
uint16 pd_checksum
Definition: bufpage.h:160
XLogRecPtr startptr
bbsink_state * bbs_state
bool DataChecksumsEnabled(void)
Definition: xlog.c:4223

References _tarWriteHeader(), _tarWritePadding(), AddFileToBackupManifest(), Assert(), basebackup_read_file(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink::bbs_state, bbsink_archive_contents(), CloseTransientFile(), DataChecksumsEnabled(), elog(), ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, fd(), filename, i, is_checksummed_file(), last_dir_separator(), len, manifest, MemSet, Min, noverify_checksums, OpenTransientFile(), PageGetLSN(), PageIsNew(), PageHeaderData::pd_checksum, PG_BINARY, pg_checksum_init(), pg_checksum_page(), pg_checksum_update(), pgoff_t, pgstat_report_checksum_failures_in_db(), remaining, stat::st_mtime, stat::st_size, bbsink_state::startptr, total_checksum_failures, and WARNING.

Referenced by perform_base_backup(), and sendDir().

◆ sendFileWithContent()

static void sendFileWithContent ( bbsink sink,
const char *  filename,
const char *  content,
backup_manifest_info manifest 
)
static

Definition at line 1032 of file basebackup.c.

1034 {
1035  struct stat statbuf;
1036  int bytes_done = 0,
1037  len;
1038  pg_checksum_context checksum_ctx;
1039 
1040  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1041  elog(ERROR, "could not initialize checksum of file \"%s\"",
1042  filename);
1043 
1044  len = strlen(content);
1045 
1046  /*
1047  * Construct a stat struct for the backup_label file we're injecting in
1048  * the tar.
1049  */
1050  /* Windows doesn't have the concept of uid and gid */
1051 #ifdef WIN32
1052  statbuf.st_uid = 0;
1053  statbuf.st_gid = 0;
1054 #else
1055  statbuf.st_uid = geteuid();
1056  statbuf.st_gid = getegid();
1057 #endif
1058  statbuf.st_mtime = time(NULL);
1059  statbuf.st_mode = pg_file_create_mode;
1060  statbuf.st_size = len;
1061 
1062  _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1063 
1064  if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1065  elog(ERROR, "could not update checksum of file \"%s\"",
1066  filename);
1067 
1068  while (bytes_done < len)
1069  {
1070  size_t remaining = len - bytes_done;
1071  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1072 
1073  memcpy(sink->bbs_buffer, content, nbytes);
1074  bbsink_archive_contents(sink, nbytes);
1075  bytes_done += nbytes;
1076  content += nbytes;
1077  }
1078 
1079  _tarWritePadding(sink, len);
1080 
1082  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1083 }
int pg_file_create_mode
Definition: file_perm.c:19

References _tarWriteHeader(), _tarWritePadding(), AddFileToBackupManifest(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), elog(), ERROR, filename, len, manifest, Min, pg_checksum_init(), pg_checksum_update(), pg_file_create_mode, remaining, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, and stat::st_uid.

Referenced by perform_base_backup().

◆ sendTablespace()

static int64 sendTablespace ( bbsink sink,
char *  path,
char *  spcoid,
bool  sizeonly,
struct backup_manifest_info manifest 
)
static

Definition at line 1093 of file basebackup.c.

1095 {
1096  int64 size;
1097  char pathbuf[MAXPGPATH];
1098  struct stat statbuf;
1099 
1100  /*
1101  * 'path' points to the tablespace location, but we only want to include
1102  * the version directory in it that belongs to us.
1103  */
1104  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1106 
1107  /*
1108  * Store a directory entry in the tar file so we get the permissions
1109  * right.
1110  */
1111  if (lstat(pathbuf, &statbuf) != 0)
1112  {
1113  if (errno != ENOENT)
1114  ereport(ERROR,
1116  errmsg("could not stat file or directory \"%s\": %m",
1117  pathbuf)));
1118 
1119  /* If the tablespace went away while scanning, it's no error. */
1120  return 0;
1121  }
1122 
1123  size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1124  sizeonly);
1125 
1126  /* Send all the files in the tablespace version directory */
1127  size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1128  spcoid);
1129 
1130  return size;
1131 }

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, manifest, MAXPGPATH, NIL, sendDir(), snprintf, and TABLESPACE_VERSION_DIRECTORY.

Referenced by perform_base_backup().

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 102 of file basebackup.c.

Referenced by do_pg_backup_start(), perform_base_backup(), and sendDir().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define PG_STAT_TMP_DIR
Definition: pgstat.h:32

Definition at line 130 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{PG_AUTOCONF_FILENAME ".tmp", false},
{TABLESPACE_MAP, false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}
#define PG_AUTOCONF_FILENAME
Definition: guc.h:33
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:101

Definition at line 130 of file basebackup.c.

Referenced by sendDir().

◆ noChecksumFiles

const struct exclude_list_item noChecksumFiles[]
static
Initial value:
= {
{"pg_control", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
{NULL, false}
}

Definition at line 130 of file basebackup.c.

Referenced by is_checksummed_file().

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 108 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 105 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().