PostgreSQL Source Code  git master
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "common/compression.h"
#include "common/file_perm.h"
#include "commands/defrem.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgstat.h"
#include "pgtar.h"
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/basebackup_sink.h"
#include "replication/basebackup_target.h"
#include "replication/backup_manifest.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 
struct  exclude_list_item
 

Macros

#define SINK_BUFFER_LENGTH   Max(32768, BLCKSZ)
 

Functions

static int64 sendTablespace (bbsink *sink, char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
 
static int64 sendDir (bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
 
static bool sendFile (bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 
static void sendFileWithContent (bbsink *sink, const char *filename, const char *content, backup_manifest_info *manifest)
 
static int64 _tarWriteHeader (bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static void _tarWritePadding (bbsink *sink, int len)
 
static void convert_link_to_directory (const char *pathbuf, struct stat *statbuf)
 
static void perform_base_backup (basebackup_options *opt, bbsink *sink)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static int compareWalFileNames (const ListCell *a, const ListCell *b)
 
static bool is_checksummed_file (const char *fullpath, const char *filename)
 
static int basebackup_read_file (int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 

Variables

static bool backup_started_in_recovery = false
 
static long long int total_checksum_failures
 
static bool noverify_checksums = false
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 
static const struct exclude_list_item noChecksumFiles []
 

Macro Definition Documentation

◆ SINK_BUFFER_LENGTH

#define SINK_BUFFER_LENGTH   Max(32768, BLCKSZ)

Definition at line 56 of file basebackup.c.

Function Documentation

◆ _tarWriteHeader()

static int64 _tarWriteHeader ( bbsink sink,
const char *  filename,
const char *  linktarget,
struct stat statbuf,
bool  sizeonly 
)
static

Definition at line 1728 of file basebackup.c.

1730 {
1731  enum tarError rc;
1732 
1733  if (!sizeonly)
1734  {
1735  /*
1736  * As of this writing, the smallest supported block size is 1kB, which
1737  * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
1738  * multiple of BLCKSZ, it should be safe to assume that the buffer is
1739  * large enough to fit an entire tar block. We double-check by means
1740  * of these assertions.
1741  */
1742  StaticAssertStmt(TAR_BLOCK_SIZE <= BLCKSZ,
1743  "BLCKSZ too small for tar block");
1745 
1746  rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
1747  statbuf->st_size, statbuf->st_mode,
1748  statbuf->st_uid, statbuf->st_gid,
1749  statbuf->st_mtime);
1750 
1751  switch (rc)
1752  {
1753  case TAR_OK:
1754  break;
1755  case TAR_NAME_TOO_LONG:
1756  ereport(ERROR,
1757  (errmsg("file name too long for tar format: \"%s\"",
1758  filename)));
1759  break;
1760  case TAR_SYMLINK_TOO_LONG:
1761  ereport(ERROR,
1762  (errmsg("symbolic link target too long for tar format: "
1763  "file name \"%s\", target \"%s\"",
1764  filename, linktarget)));
1765  break;
1766  default:
1767  elog(ERROR, "unrecognized tar error: %d", rc);
1768  }
1769 
1771  }
1772 
1773  return TAR_BLOCK_SIZE;
1774 }
static void bbsink_archive_contents(bbsink *sink, size_t len)
#define StaticAssertStmt(condition, errmessage)
Definition: c.h:918
int errmsg(const char *fmt,...)
Definition: elog.c:904
#define ERROR
Definition: elog.h:33
#define elog(elevel,...)
Definition: elog.h:218
#define ereport(elevel,...)
Definition: elog.h:143
Assert(fmt[strlen(fmt) - 1] !='\n')
static char * filename
Definition: pg_dumpall.c:94
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
tarError
Definition: pgtar.h:20
@ TAR_SYMLINK_TOO_LONG
Definition: pgtar.h:23
@ TAR_OK
Definition: pgtar.h:21
@ TAR_NAME_TOO_LONG
Definition: pgtar.h:22
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
char * bbs_buffer
size_t bbs_buffer_length
__time64_t st_mtime
Definition: win32_port.h:275
__int64 st_size
Definition: win32_port.h:273
short st_gid
Definition: win32_port.h:271
unsigned short st_mode
Definition: win32_port.h:268
short st_uid
Definition: win32_port.h:270

References Assert(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), elog, ereport, errmsg(), ERROR, filename, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, stat::st_uid, StaticAssertStmt, TAR_BLOCK_SIZE, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

◆ _tarWritePadding()

static void _tarWritePadding ( bbsink sink,
int  len 
)
static

Definition at line 1780 of file basebackup.c.

1781 {
1782  int pad = tarPaddingBytesRequired(len);
1783 
1784  /*
1785  * As in _tarWriteHeader, it should be safe to assume that the buffer is
1786  * large enough that we don't need to do this in multiple chunks.
1787  */
1789  Assert(pad <= TAR_BLOCK_SIZE);
1790 
1791  if (pad > 0)
1792  {
1793  MemSet(sink->bbs_buffer, 0, pad);
1794  bbsink_archive_contents(sink, pad);
1795  }
1796 }
#define MemSet(start, val, len)
Definition: c.h:1008
const void size_t len
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40

References Assert(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), len, MemSet, TAR_BLOCK_SIZE, and tarPaddingBytesRequired().

Referenced by sendFile(), and sendFileWithContent().

◆ basebackup_read_file()

static int basebackup_read_file ( int  fd,
char *  buf,
size_t  nbytes,
off_t  offset,
const char *  filename,
bool  partial_read_ok 
)
static

Definition at line 1824 of file basebackup.c.

1826 {
1827  int rc;
1828 
1830  rc = pg_pread(fd, buf, nbytes, offset);
1832 
1833  if (rc < 0)
1834  ereport(ERROR,
1836  errmsg("could not read file \"%s\": %m", filename)));
1837  if (!partial_read_ok && rc > 0 && rc != nbytes)
1838  ereport(ERROR,
1840  errmsg("could not read file \"%s\": read %d of %zu",
1841  filename, rc, nbytes)));
1842 
1843  return rc;
1844 }
int errcode_for_file_access(void)
Definition: elog.c:716
static char * buf
Definition: pg_test_fsync.c:67
ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset)
Definition: pread.c:27
static int fd(const char *x, int i)
Definition: preproc-init.c:105
@ WAIT_EVENT_BASEBACKUP_READ
Definition: wait_event.h:160
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:266
static void pgstat_report_wait_end(void)
Definition: wait_event.h:282

References buf, ereport, errcode_for_file_access(), errmsg(), ERROR, fd(), filename, pg_pread(), pgstat_report_wait_end(), pgstat_report_wait_start(), and WAIT_EVENT_BASEBACKUP_READ.

Referenced by perform_base_backup(), and sendFile().

◆ compareWalFileNames()

static int compareWalFileNames ( const ListCell a,
const ListCell b 
)
static

Definition at line 658 of file basebackup.c.

659 {
660  char *fna = (char *) lfirst(a);
661  char *fnb = (char *) lfirst(b);
662 
663  return strcmp(fna + 8, fnb + 8);
664 }
int b
Definition: isn.c:70
int a
Definition: isn.c:69
#define lfirst(lc)
Definition: pg_list.h:169

References a, b, and lfirst.

Referenced by perform_base_backup().

◆ convert_link_to_directory()

static void convert_link_to_directory ( const char *  pathbuf,
struct stat statbuf 
)
static

Definition at line 1803 of file basebackup.c.

1804 {
1805  /* If symlink, write it as a directory anyway */
1806 #ifndef WIN32
1807  if (S_ISLNK(statbuf->st_mode))
1808 #else
1809  if (pgwin32_is_junction(pathbuf))
1810 #endif
1811  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1812 }
int pg_dir_create_mode
Definition: file_perm.c:18
bool pgwin32_is_junction(const char *path)

References pg_dir_create_mode, pgwin32_is_junction(), and stat::st_mode.

Referenced by sendDir().

◆ is_checksummed_file()

static bool is_checksummed_file ( const char *  fullpath,
const char *  filename 
)
static

Definition at line 1437 of file basebackup.c.

1438 {
1439  /* Check that the file is in a tablespace */
1440  if (strncmp(fullpath, "./global/", 9) == 0 ||
1441  strncmp(fullpath, "./base/", 7) == 0 ||
1442  strncmp(fullpath, "/", 1) == 0)
1443  {
1444  int excludeIdx;
1445 
1446  /* Compare file against noChecksumFiles skip list */
1447  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1448  {
1449  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1450 
1451  if (!noChecksumFiles[excludeIdx].match_prefix)
1452  cmplen++;
1453  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1454  cmplen) == 0)
1455  return false;
1456  }
1457 
1458  return true;
1459  }
1460  else
1461  return false;
1462 }
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:210
const char * name
Definition: encode.c:561
const char * name
Definition: basebackup.c:116

References filename, exclude_list_item::name, name, and noChecksumFiles.

Referenced by sendFile().

◆ parse_basebackup_options()

static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 670 of file basebackup.c.

671 {
672  ListCell *lopt;
673  bool o_label = false;
674  bool o_progress = false;
675  bool o_checkpoint = false;
676  bool o_nowait = false;
677  bool o_wal = false;
678  bool o_maxrate = false;
679  bool o_tablespace_map = false;
680  bool o_noverify_checksums = false;
681  bool o_manifest = false;
682  bool o_manifest_checksums = false;
683  bool o_target = false;
684  bool o_target_detail = false;
685  char *target_str = NULL;
686  char *target_detail_str = NULL;
687  bool o_compression = false;
688  bool o_compression_detail = false;
689  char *compression_detail_str = NULL;
690 
691  MemSet(opt, 0, sizeof(*opt));
696 
697  foreach(lopt, options)
698  {
699  DefElem *defel = (DefElem *) lfirst(lopt);
700 
701  if (strcmp(defel->defname, "label") == 0)
702  {
703  if (o_label)
704  ereport(ERROR,
705  (errcode(ERRCODE_SYNTAX_ERROR),
706  errmsg("duplicate option \"%s\"", defel->defname)));
707  opt->label = defGetString(defel);
708  o_label = true;
709  }
710  else if (strcmp(defel->defname, "progress") == 0)
711  {
712  if (o_progress)
713  ereport(ERROR,
714  (errcode(ERRCODE_SYNTAX_ERROR),
715  errmsg("duplicate option \"%s\"", defel->defname)));
716  opt->progress = defGetBoolean(defel);
717  o_progress = true;
718  }
719  else if (strcmp(defel->defname, "checkpoint") == 0)
720  {
721  char *optval = defGetString(defel);
722 
723  if (o_checkpoint)
724  ereport(ERROR,
725  (errcode(ERRCODE_SYNTAX_ERROR),
726  errmsg("duplicate option \"%s\"", defel->defname)));
727  if (pg_strcasecmp(optval, "fast") == 0)
728  opt->fastcheckpoint = true;
729  else if (pg_strcasecmp(optval, "spread") == 0)
730  opt->fastcheckpoint = false;
731  else
732  ereport(ERROR,
733  (errcode(ERRCODE_SYNTAX_ERROR),
734  errmsg("unrecognized checkpoint type: \"%s\"",
735  optval)));
736  o_checkpoint = true;
737  }
738  else if (strcmp(defel->defname, "wait") == 0)
739  {
740  if (o_nowait)
741  ereport(ERROR,
742  (errcode(ERRCODE_SYNTAX_ERROR),
743  errmsg("duplicate option \"%s\"", defel->defname)));
744  opt->nowait = !defGetBoolean(defel);
745  o_nowait = true;
746  }
747  else if (strcmp(defel->defname, "wal") == 0)
748  {
749  if (o_wal)
750  ereport(ERROR,
751  (errcode(ERRCODE_SYNTAX_ERROR),
752  errmsg("duplicate option \"%s\"", defel->defname)));
753  opt->includewal = defGetBoolean(defel);
754  o_wal = true;
755  }
756  else if (strcmp(defel->defname, "max_rate") == 0)
757  {
758  int64 maxrate;
759 
760  if (o_maxrate)
761  ereport(ERROR,
762  (errcode(ERRCODE_SYNTAX_ERROR),
763  errmsg("duplicate option \"%s\"", defel->defname)));
764 
765  maxrate = defGetInt64(defel);
766  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
767  ereport(ERROR,
768  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
769  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
770  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
771 
772  opt->maxrate = (uint32) maxrate;
773  o_maxrate = true;
774  }
775  else if (strcmp(defel->defname, "tablespace_map") == 0)
776  {
777  if (o_tablespace_map)
778  ereport(ERROR,
779  (errcode(ERRCODE_SYNTAX_ERROR),
780  errmsg("duplicate option \"%s\"", defel->defname)));
781  opt->sendtblspcmapfile = defGetBoolean(defel);
782  o_tablespace_map = true;
783  }
784  else if (strcmp(defel->defname, "verify_checksums") == 0)
785  {
786  if (o_noverify_checksums)
787  ereport(ERROR,
788  (errcode(ERRCODE_SYNTAX_ERROR),
789  errmsg("duplicate option \"%s\"", defel->defname)));
791  o_noverify_checksums = true;
792  }
793  else if (strcmp(defel->defname, "manifest") == 0)
794  {
795  char *optval = defGetString(defel);
796  bool manifest_bool;
797 
798  if (o_manifest)
799  ereport(ERROR,
800  (errcode(ERRCODE_SYNTAX_ERROR),
801  errmsg("duplicate option \"%s\"", defel->defname)));
802  if (parse_bool(optval, &manifest_bool))
803  {
804  if (manifest_bool)
806  else
808  }
809  else if (pg_strcasecmp(optval, "force-encode") == 0)
811  else
812  ereport(ERROR,
813  (errcode(ERRCODE_SYNTAX_ERROR),
814  errmsg("unrecognized manifest option: \"%s\"",
815  optval)));
816  o_manifest = true;
817  }
818  else if (strcmp(defel->defname, "manifest_checksums") == 0)
819  {
820  char *optval = defGetString(defel);
821 
822  if (o_manifest_checksums)
823  ereport(ERROR,
824  (errcode(ERRCODE_SYNTAX_ERROR),
825  errmsg("duplicate option \"%s\"", defel->defname)));
826  if (!pg_checksum_parse_type(optval,
827  &opt->manifest_checksum_type))
828  ereport(ERROR,
829  (errcode(ERRCODE_SYNTAX_ERROR),
830  errmsg("unrecognized checksum algorithm: \"%s\"",
831  optval)));
832  o_manifest_checksums = true;
833  }
834  else if (strcmp(defel->defname, "target") == 0)
835  {
836  if (o_target)
837  ereport(ERROR,
838  (errcode(ERRCODE_SYNTAX_ERROR),
839  errmsg("duplicate option \"%s\"", defel->defname)));
840  target_str = defGetString(defel);
841  o_target = true;
842  }
843  else if (strcmp(defel->defname, "target_detail") == 0)
844  {
845  char *optval = defGetString(defel);
846 
847  if (o_target_detail)
848  ereport(ERROR,
849  (errcode(ERRCODE_SYNTAX_ERROR),
850  errmsg("duplicate option \"%s\"", defel->defname)));
851  target_detail_str = optval;
852  o_target_detail = true;
853  }
854  else if (strcmp(defel->defname, "compression") == 0)
855  {
856  char *optval = defGetString(defel);
857 
858  if (o_compression)
859  ereport(ERROR,
860  (errcode(ERRCODE_SYNTAX_ERROR),
861  errmsg("duplicate option \"%s\"", defel->defname)));
862  if (!parse_compress_algorithm(optval, &opt->compression))
863  ereport(ERROR,
864  (errcode(ERRCODE_SYNTAX_ERROR),
865  errmsg("unrecognized compression algorithm \"%s\"",
866  optval)));
867  o_compression = true;
868  }
869  else if (strcmp(defel->defname, "compression_detail") == 0)
870  {
871  if (o_compression_detail)
872  ereport(ERROR,
873  (errcode(ERRCODE_SYNTAX_ERROR),
874  errmsg("duplicate option \"%s\"", defel->defname)));
875  compression_detail_str = defGetString(defel);
876  o_compression_detail = true;
877  }
878  else
879  ereport(ERROR,
880  (errcode(ERRCODE_SYNTAX_ERROR),
881  errmsg("unrecognized base backup option: \"%s\"",
882  defel->defname)));
883  }
884 
885  if (opt->label == NULL)
886  opt->label = "base backup";
887  if (opt->manifest == MANIFEST_OPTION_NO)
888  {
889  if (o_manifest_checksums)
890  ereport(ERROR,
891  (errcode(ERRCODE_SYNTAX_ERROR),
892  errmsg("manifest checksums require a backup manifest")));
894  }
895 
896  if (target_str == NULL)
897  {
898  if (target_detail_str != NULL)
899  ereport(ERROR,
900  (errcode(ERRCODE_SYNTAX_ERROR),
901  errmsg("target detail cannot be used without target")));
902  opt->use_copytblspc = true;
903  opt->send_to_client = true;
904  }
905  else if (strcmp(target_str, "client") == 0)
906  {
907  if (target_detail_str != NULL)
908  ereport(ERROR,
909  (errcode(ERRCODE_SYNTAX_ERROR),
910  errmsg("target '%s' does not accept a target detail",
911  target_str)));
912  opt->send_to_client = true;
913  }
914  else
915  opt->target_handle =
916  BaseBackupGetTargetHandle(target_str, target_detail_str);
917 
918  if (o_compression_detail && !o_compression)
919  ereport(ERROR,
920  (errcode(ERRCODE_SYNTAX_ERROR),
921  errmsg("compression detail requires compression")));
922 
923  if (o_compression)
924  {
925  char *error_detail;
926 
927  parse_compress_specification(opt->compression, compression_detail_str,
929  error_detail =
931  if (error_detail != NULL)
932  ereport(ERROR,
933  errcode(ERRCODE_SYNTAX_ERROR),
934  errmsg("invalid compression specification: %s",
935  error_detail));
936  }
937 }
@ MANIFEST_OPTION_NO
@ MANIFEST_OPTION_FORCE_ENCODE
@ MANIFEST_OPTION_YES
static bool noverify_checksums
Definition: basebackup.c:106
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define MAX_RATE_LOWER
Definition: basebackup.h:20
BaseBackupTargetHandle * BaseBackupGetTargetHandle(char *target, char *target_detail)
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
unsigned int uint32
Definition: c.h:441
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
@ CHECKSUM_TYPE_NONE
@ CHECKSUM_TYPE_CRC32C
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:40
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:95
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:250
@ PG_COMPRESSION_NONE
Definition: compression.h:19
bool defGetBoolean(DefElem *def)
Definition: define.c:108
char * defGetString(DefElem *def)
Definition: define.c:49
int64 defGetInt64(DefElem *def)
Definition: define.c:187
int errcode(int sqlerrcode)
Definition: elog.c:693
static int32 maxrate
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * defname
Definition: parsenodes.h:765
pg_compress_specification compression_specification
Definition: basebackup.c:72
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:73
const char * label
Definition: basebackup.c:60
pg_compress_algorithm compression
Definition: basebackup.c:71
backup_manifest_option manifest
Definition: basebackup.c:70
BaseBackupTargetHandle * target_handle
Definition: basebackup.c:69
pg_compress_algorithm algorithm
Definition: compression.h:30

References pg_compress_specification::algorithm, BaseBackupGetTargetHandle(), CHECKSUM_TYPE_CRC32C, CHECKSUM_TYPE_NONE, basebackup_options::compression, basebackup_options::compression_specification, defGetBoolean(), defGetInt64(), defGetString(), DefElem::defname, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, basebackup_options::label, lfirst, basebackup_options::manifest, basebackup_options::manifest_checksum_type, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, MANIFEST_OPTION_YES, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, noverify_checksums, basebackup_options::nowait, parse_bool(), parse_compress_algorithm(), parse_compress_specification(), pg_checksum_parse_type(), PG_COMPRESSION_NONE, pg_strcasecmp(), basebackup_options::progress, basebackup_options::send_to_client, basebackup_options::sendtblspcmapfile, basebackup_options::target_handle, basebackup_options::use_copytblspc, and validate_compress_specification().

Referenced by SendBaseBackup().

◆ perform_base_backup()

static void perform_base_backup ( basebackup_options opt,
bbsink sink 
)
static

Definition at line 228 of file basebackup.c.

229 {
231  XLogRecPtr endptr;
232  TimeLineID endtli;
233  StringInfo labelfile;
236 
237  /* Initial backup state, insofar as we know it now. */
238  state.tablespaces = NIL;
239  state.tablespace_num = 0;
240  state.bytes_done = 0;
241  state.bytes_total = 0;
242  state.bytes_total_is_valid = false;
243 
244  /* we're going to use a BufFile, so we need a ResourceOwner */
245  Assert(CurrentResourceOwner == NULL);
246  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
247 
249 
250  labelfile = makeStringInfo();
254 
256 
258  state.startptr = do_pg_backup_start(opt->label, opt->fastcheckpoint,
259  &state.starttli,
260  labelfile, &state.tablespaces,
262 
263  /*
264  * Once do_pg_backup_start has been called, ensure that any failure causes
265  * us to abort the backup so we don't "leak" a backup counter. For this
266  * reason, *all* functionality between do_pg_backup_start() and the end of
267  * do_pg_backup_stop() should be inside the error cleanup block!
268  */
269 
271  {
272  ListCell *lc;
273  tablespaceinfo *ti;
274 
275  /* Add a node for the base directory at the end */
276  ti = palloc0(sizeof(tablespaceinfo));
277  ti->size = -1;
278  state.tablespaces = lappend(state.tablespaces, ti);
279 
280  /*
281  * Calculate the total backup size by summing up the size of each
282  * tablespace
283  */
284  if (opt->progress)
285  {
287 
288  foreach(lc, state.tablespaces)
289  {
290  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
291 
292  if (tmp->path == NULL)
293  tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
294  true, NULL, NULL);
295  else
296  tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
297  NULL);
298  state.bytes_total += tmp->size;
299  }
300  state.bytes_total_is_valid = true;
301  }
302 
303  /* notify basebackup sink about start of backup */
305 
306  /* Send off our tablespaces one by one */
307  foreach(lc, state.tablespaces)
308  {
309  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
310 
311  if (ti->path == NULL)
312  {
313  struct stat statbuf;
314  bool sendtblspclinks = true;
315 
316  bbsink_begin_archive(sink, "base.tar");
317 
318  /* In the main tar, include the backup_label first... */
319  sendFileWithContent(sink, BACKUP_LABEL_FILE, labelfile->data,
320  &manifest);
321 
322  /* Then the tablespace_map file, if required... */
323  if (opt->sendtblspcmapfile)
324  {
326  &manifest);
327  sendtblspclinks = false;
328  }
329 
330  /* Then the bulk of the files... */
331  sendDir(sink, ".", 1, false, state.tablespaces,
332  sendtblspclinks, &manifest, NULL);
333 
334  /* ... and pg_control after everything else. */
335  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
336  ereport(ERROR,
338  errmsg("could not stat file \"%s\": %m",
341  false, InvalidOid, &manifest, NULL);
342  }
343  else
344  {
345  char *archive_name = psprintf("%s.tar", ti->oid);
346 
347  bbsink_begin_archive(sink, archive_name);
348 
349  sendTablespace(sink, ti->path, ti->oid, false, &manifest);
350  }
351 
352  /*
353  * If we're including WAL, and this is the main data directory we
354  * don't treat this as the end of the tablespace. Instead, we will
355  * include the xlog files below and stop afterwards. This is safe
356  * since the main data directory is always sent *last*.
357  */
358  if (opt->includewal && ti->path == NULL)
359  {
360  Assert(lnext(state.tablespaces, lc) == NULL);
361  }
362  else
363  {
364  /* Properly terminate the tarfile. */
365  StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
366  "BLCKSZ too small for 2 tar blocks");
367  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
369 
370  /* OK, that's the end of the archive. */
371  bbsink_end_archive(sink);
372  }
373  }
374 
376  endptr = do_pg_backup_stop(labelfile->data, !opt->nowait, &endtli);
377  }
379 
380 
381  if (opt->includewal)
382  {
383  /*
384  * We've left the last tar file "open", so we can now append the
385  * required WAL files to it.
386  */
387  char pathbuf[MAXPGPATH];
388  XLogSegNo segno;
389  XLogSegNo startsegno;
390  XLogSegNo endsegno;
391  struct stat statbuf;
392  List *historyFileList = NIL;
393  List *walFileList = NIL;
394  char firstoff[MAXFNAMELEN];
395  char lastoff[MAXFNAMELEN];
396  DIR *dir;
397  struct dirent *de;
398  ListCell *lc;
399  TimeLineID tli;
400 
402 
403  /*
404  * I'd rather not worry about timelines here, so scan pg_wal and
405  * include all WAL files in the range between 'startptr' and 'endptr',
406  * regardless of the timeline the file is stamped with. If there are
407  * some spurious WAL files belonging to timelines that don't belong in
408  * this server's history, they will be included too. Normally there
409  * shouldn't be such files, but if there are, there's little harm in
410  * including them.
411  */
412  XLByteToSeg(state.startptr, startsegno, wal_segment_size);
413  XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
414  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
415  XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
416 
417  dir = AllocateDir("pg_wal");
418  while ((de = ReadDir(dir, "pg_wal")) != NULL)
419  {
420  /* Does it look like a WAL segment, and is it in the range? */
421  if (IsXLogFileName(de->d_name) &&
422  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
423  strcmp(de->d_name + 8, lastoff + 8) <= 0)
424  {
425  walFileList = lappend(walFileList, pstrdup(de->d_name));
426  }
427  /* Does it look like a timeline history file? */
428  else if (IsTLHistoryFileName(de->d_name))
429  {
430  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
431  }
432  }
433  FreeDir(dir);
434 
435  /*
436  * Before we go any further, check that none of the WAL segments we
437  * need were removed.
438  */
439  CheckXLogRemoved(startsegno, state.starttli);
440 
441  /*
442  * Sort the WAL filenames. We want to send the files in order from
443  * oldest to newest, to reduce the chance that a file is recycled
444  * before we get a chance to send it over.
445  */
446  list_sort(walFileList, compareWalFileNames);
447 
448  /*
449  * There must be at least one xlog file in the pg_wal directory, since
450  * we are doing backup-including-xlog.
451  */
452  if (walFileList == NIL)
453  ereport(ERROR,
454  (errmsg("could not find any WAL files")));
455 
456  /*
457  * Sanity check: the first and last segment should cover startptr and
458  * endptr, with no gaps in between.
459  */
460  XLogFromFileName((char *) linitial(walFileList),
461  &tli, &segno, wal_segment_size);
462  if (segno != startsegno)
463  {
464  char startfname[MAXFNAMELEN];
465 
466  XLogFileName(startfname, state.starttli, startsegno,
468  ereport(ERROR,
469  (errmsg("could not find WAL file \"%s\"", startfname)));
470  }
471  foreach(lc, walFileList)
472  {
473  char *walFileName = (char *) lfirst(lc);
474  XLogSegNo currsegno = segno;
475  XLogSegNo nextsegno = segno + 1;
476 
477  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
478  if (!(nextsegno == segno || currsegno == segno))
479  {
480  char nextfname[MAXFNAMELEN];
481 
482  XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
483  ereport(ERROR,
484  (errmsg("could not find WAL file \"%s\"", nextfname)));
485  }
486  }
487  if (segno != endsegno)
488  {
489  char endfname[MAXFNAMELEN];
490 
491  XLogFileName(endfname, endtli, endsegno, wal_segment_size);
492  ereport(ERROR,
493  (errmsg("could not find WAL file \"%s\"", endfname)));
494  }
495 
496  /* Ok, we have everything we need. Send the WAL files. */
497  foreach(lc, walFileList)
498  {
499  char *walFileName = (char *) lfirst(lc);
500  int fd;
501  size_t cnt;
502  pgoff_t len = 0;
503 
504  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
505  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
506 
507  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
508  if (fd < 0)
509  {
510  int save_errno = errno;
511 
512  /*
513  * Most likely reason for this is that the file was already
514  * removed by a checkpoint, so check for that to get a better
515  * error message.
516  */
517  CheckXLogRemoved(segno, tli);
518 
519  errno = save_errno;
520  ereport(ERROR,
522  errmsg("could not open file \"%s\": %m", pathbuf)));
523  }
524 
525  if (fstat(fd, &statbuf) != 0)
526  ereport(ERROR,
528  errmsg("could not stat file \"%s\": %m",
529  pathbuf)));
530  if (statbuf.st_size != wal_segment_size)
531  {
532  CheckXLogRemoved(segno, tli);
533  ereport(ERROR,
535  errmsg("unexpected WAL file size \"%s\"", walFileName)));
536  }
537 
538  /* send the WAL file itself */
539  _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
540 
541  while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
542  Min(sink->bbs_buffer_length,
544  len, pathbuf, true)) > 0)
545  {
546  CheckXLogRemoved(segno, tli);
547  bbsink_archive_contents(sink, cnt);
548 
549  len += cnt;
550 
551  if (len == wal_segment_size)
552  break;
553  }
554 
555  if (len != wal_segment_size)
556  {
557  CheckXLogRemoved(segno, tli);
558  ereport(ERROR,
560  errmsg("unexpected WAL file size \"%s\"", walFileName)));
561  }
562 
563  /*
564  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
565  * for padding.
566  */
568 
570 
571  /*
572  * Mark file as archived, otherwise files can get archived again
573  * after promotion of a new node. This is in line with
574  * walreceiver.c always doing an XLogArchiveForceDone() after a
575  * complete segment.
576  */
577  StatusFilePath(pathbuf, walFileName, ".done");
578  sendFileWithContent(sink, pathbuf, "", &manifest);
579  }
580 
581  /*
582  * Send timeline history files too. Only the latest timeline history
583  * file is required for recovery, and even that only if there happens
584  * to be a timeline switch in the first WAL segment that contains the
585  * checkpoint record, or if we're taking a base backup from a standby
586  * server and the target timeline changes while the backup is taken.
587  * But they are small and highly useful for debugging purposes, so
588  * better include them all, always.
589  */
590  foreach(lc, historyFileList)
591  {
592  char *fname = lfirst(lc);
593 
594  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
595 
596  if (lstat(pathbuf, &statbuf) != 0)
597  ereport(ERROR,
599  errmsg("could not stat file \"%s\": %m", pathbuf)));
600 
601  sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
602  &manifest, NULL);
603 
604  /* unconditionally mark file as archived */
605  StatusFilePath(pathbuf, fname, ".done");
606  sendFileWithContent(sink, pathbuf, "", &manifest);
607  }
608 
609  /* Properly terminate the tar file. */
610  StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
611  "BLCKSZ too small for 2 tar blocks");
612  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
614 
615  /* OK, that's the end of the archive. */
616  bbsink_end_archive(sink);
617  }
618 
619  AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
620  endptr, endtli);
621 
623 
624  bbsink_end_backup(sink, endptr, endtli);
625 
627  {
628  if (total_checksum_failures > 1)
630  (errmsg_plural("%lld total checksum verification failure",
631  "%lld total checksum verification failures",
634 
635  ereport(ERROR,
637  errmsg("checksum verification failure during base backup")));
638  }
639 
640  /*
641  * Make sure to free the manifest before the resource owners as manifests
642  * use cryptohash contexts that may depend on resource owners (like
643  * OpenSSL).
644  */
646 
647  /* clean up the resource owner we created */
648  WalSndResourceCleanup(true);
649 
651 }
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
void SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
void FreeBackupManifest(backup_manifest_info *manifest)
static void sendFileWithContent(bbsink *sink, const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1010
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:658
#define SINK_BUFFER_LENGTH
Definition: basebackup.c:56
static bool backup_started_in_recovery
Definition: basebackup.c:100
static int64 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1728
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1483
static int64 sendTablespace(bbsink *sink, char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
Definition: basebackup.c:1070
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:1824
static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1123
static long long int total_checksum_failures
Definition: basebackup.c:103
void basebackup_progress_wait_checkpoint(void)
void basebackup_progress_wait_wal_archive(bbsink_state *state)
void basebackup_progress_done(void)
void basebackup_progress_transfer_wal(void)
void basebackup_progress_estimate_backup_size(void)
static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
static void bbsink_begin_archive(bbsink *sink, const char *archive_name)
static void bbsink_end_archive(bbsink *sink)
static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
#define Min(x, y)
Definition: c.h:986
#define PG_BINARY
Definition: c.h:1268
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1014
#define WARNING
Definition: elog.h:30
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2788
int FreeDir(DIR *dir)
Definition: fd.c:2840
int CloseTransientFile(int fd)
Definition: fd.c:2688
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2511
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2722
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1612
List * lappend(List *list, void *datum)
Definition: list.c:336
char * pstrdup(const char *in)
Definition: mcxt.c:1305
void * palloc0(Size size)
Definition: mcxt.c:1099
static bool manifest
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:43
#define MAXPGPATH
#define NIL
Definition: pg_list.h:65
#define linitial(l)
Definition: pg_list.h:174
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:322
#define snprintf
Definition: port.h:225
#define BoolGetDatum(X)
Definition: postgres.h:446
#define InvalidOid
Definition: postgres_ext.h:36
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:428
ResourceOwner CurrentResourceOwner
Definition: resowner.c:146
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
Definition: dirent.c:26
Definition: pg_list.h:51
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
Definition: regguts.h:318
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:350
#define lstat(path, sb)
Definition: win32_port.h:284
#define fstat
Definition: win32_port.h:282
#define pgoff_t
Definition: win32_port.h:208
XLogRecPtr do_pg_backup_start(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile)
Definition: xlog.c:8077
bool RecoveryInProgress(void)
Definition: xlog.c:5753
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:8759
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3454
int wal_segment_size
Definition: xlog.c:144
XLogRecPtr do_pg_backup_stop(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:8437
#define TABLESPACE_MAP
Definition: xlog.h:295
#define BACKUP_LABEL_FILE
Definition: xlog.h:292
#define XLOG_CONTROL_FILE
#define IsTLHistoryFileName(fname)
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define IsXLogFileName(fname)
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
#define MAXFNAMELEN
#define XLOGDIR
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
#define StatusFilePath(path, xlog, suffix)
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45

References _tarWriteHeader(), AddWALInfoToBackupManifest(), AllocateDir(), Assert(), BACKUP_LABEL_FILE, backup_started_in_recovery, basebackup_progress_done(), basebackup_progress_estimate_backup_size(), basebackup_progress_transfer_wal(), basebackup_progress_wait_checkpoint(), basebackup_progress_wait_wal_archive(), basebackup_read_file(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), bbsink_begin_archive(), bbsink_begin_backup(), bbsink_end_archive(), bbsink_end_backup(), BoolGetDatum, CheckXLogRemoved(), CloseTransientFile(), compareWalFileNames(), CurrentResourceOwner, dirent::d_name, StringInfoData::data, do_pg_abort_backup(), do_pg_backup_start(), do_pg_backup_stop(), ereport, errcode(), ERRCODE_DATA_CORRUPTED, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, basebackup_options::fastcheckpoint, fd(), FreeBackupManifest(), FreeDir(), fstat, basebackup_options::includewal, InitializeBackupManifest(), InvalidOid, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), len, lfirst, linitial, list_sort(), lnext(), lstat, makeStringInfo(), basebackup_options::manifest, manifest, basebackup_options::manifest_checksum_type, MAXFNAMELEN, MAXPGPATH, Min, NIL, basebackup_options::nowait, tablespaceinfo::oid, OpenTransientFile(), palloc0(), tablespaceinfo::path, PG_BINARY, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, basebackup_options::progress, psprintf(), pstrdup(), ReadDir(), RecoveryInProgress(), ResourceOwnerCreate(), SendBackupManifest(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SINK_BUFFER_LENGTH, tablespaceinfo::size, snprintf, stat::st_size, StaticAssertStmt, StatusFilePath, TABLESPACE_MAP, TAR_BLOCK_SIZE, tblspc_map_file, total_checksum_failures, wal_segment_size, WalSndResourceCleanup(), WARNING, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

◆ SendBaseBackup()

void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 948 of file basebackup.c.

949 {
950  basebackup_options opt;
951  bbsink *sink;
952 
953  parse_basebackup_options(cmd->options, &opt);
954 
956 
958  {
959  char activitymsg[50];
960 
961  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
962  opt.label);
963  set_ps_display(activitymsg);
964  }
965 
966  /*
967  * If the target is specifically 'client' then set up to stream the backup
968  * to the client; otherwise, it's being sent someplace else and should not
969  * be sent to the client. BaseBackupGetSink has the job of setting up a
970  * sink to send the backup data wherever it needs to go.
971  */
973  if (opt.target_handle != NULL)
974  sink = BaseBackupGetSink(opt.target_handle, sink);
975 
976  /* Set up network throttling, if client requested it */
977  if (opt.maxrate > 0)
978  sink = bbsink_throttle_new(sink, opt.maxrate);
979 
980  /* Set up server-side compression, if client requested it */
981  if (opt.compression == PG_COMPRESSION_GZIP)
982  sink = bbsink_gzip_new(sink, &opt.compression_specification);
983  else if (opt.compression == PG_COMPRESSION_LZ4)
984  sink = bbsink_lz4_new(sink, &opt.compression_specification);
985  else if (opt.compression == PG_COMPRESSION_ZSTD)
986  sink = bbsink_zstd_new(sink, &opt.compression_specification);
987 
988  /* Set up progress reporting. */
989  sink = bbsink_progress_new(sink, opt.progress);
990 
991  /*
992  * Perform the base backup, but make sure we clean up the bbsink even if
993  * an error occurs.
994  */
995  PG_TRY();
996  {
997  perform_base_backup(&opt, sink);
998  }
999  PG_FINALLY();
1000  {
1001  bbsink_cleanup(sink);
1002  }
1003  PG_END_TRY();
1004 }
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:670
static void perform_base_backup(basebackup_options *opt, bbsink *sink)
Definition: basebackup.c:228
bbsink * bbsink_copystream_new(bool send_to_client)
bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *compress)
bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *compress)
bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)
static void bbsink_cleanup(bbsink *sink)
bbsink * BaseBackupGetSink(BaseBackupTargetHandle *handle, bbsink *next_sink)
bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)
bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *compress)
@ PG_COMPRESSION_GZIP
Definition: compression.h:20
@ PG_COMPRESSION_LZ4
Definition: compression.h:21
@ PG_COMPRESSION_ZSTD
Definition: compression.h:22
#define PG_END_TRY()
Definition: elog.h:324
#define PG_TRY()
Definition: elog.h:299
#define PG_FINALLY()
Definition: elog.h:316
bool update_process_title
Definition: ps_status.c:36
void set_ps_display(const char *activity)
Definition: ps_status.c:349
List * options
Definition: replnodes.h:44
void WalSndSetState(WalSndState state)
Definition: walsender.c:3418
@ WALSNDSTATE_BACKUP

References BaseBackupGetSink(), bbsink_cleanup(), bbsink_copystream_new(), bbsink_gzip_new(), bbsink_lz4_new(), bbsink_progress_new(), bbsink_throttle_new(), bbsink_zstd_new(), basebackup_options::compression, basebackup_options::compression_specification, basebackup_options::label, basebackup_options::maxrate, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), PG_COMPRESSION_GZIP, PG_COMPRESSION_LZ4, PG_COMPRESSION_ZSTD, PG_END_TRY, PG_FINALLY, PG_TRY, basebackup_options::progress, basebackup_options::send_to_client, set_ps_display(), snprintf, basebackup_options::target_handle, update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

◆ sendDir()

static int64 sendDir ( bbsink sink,
const char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1123 of file basebackup.c.

1126 {
1127  DIR *dir;
1128  struct dirent *de;
1129  char pathbuf[MAXPGPATH * 2];
1130  struct stat statbuf;
1131  int64 size = 0;
1132  const char *lastDir; /* Split last dir from parent path. */
1133  bool isDbDir = false; /* Does this directory contain relations? */
1134 
1135  /*
1136  * Determine if the current path is a database directory that can contain
1137  * relations.
1138  *
1139  * Start by finding the location of the delimiter between the parent path
1140  * and the current path.
1141  */
1142  lastDir = last_dir_separator(path);
1143 
1144  /* Does this path look like a database path (i.e. all digits)? */
1145  if (lastDir != NULL &&
1146  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1147  {
1148  /* Part of path that contains the parent directory. */
1149  int parentPathLen = lastDir - path;
1150 
1151  /*
1152  * Mark path as a database directory if the parent path is either
1153  * $PGDATA/base or a tablespace version path.
1154  */
1155  if (strncmp(path, "./base", parentPathLen) == 0 ||
1156  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1157  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1159  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1160  isDbDir = true;
1161  }
1162 
1163  dir = AllocateDir(path);
1164  while ((de = ReadDir(dir, path)) != NULL)
1165  {
1166  int excludeIdx;
1167  bool excludeFound;
1168  ForkNumber relForkNum; /* Type of fork if file is a relation */
1169  int relOidChars; /* Chars in filename that are the rel oid */
1170 
1171  /* Skip special stuff */
1172  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1173  continue;
1174 
1175  /* Skip temporary files */
1176  if (strncmp(de->d_name,
1178  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1179  continue;
1180 
1181  /*
1182  * Check if the postmaster has signaled us to exit, and abort with an
1183  * error in that case. The error handler further up will call
1184  * do_pg_abort_backup() for us. Also check that if the backup was
1185  * started while still in recovery, the server wasn't promoted.
1186  * do_pg_backup_stop() will check that too, but it's better to stop
1187  * the backup early than continue to the end and fail there.
1188  */
1191  ereport(ERROR,
1192  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1193  errmsg("the standby was promoted during online backup"),
1194  errhint("This means that the backup being taken is corrupt "
1195  "and should not be used. "
1196  "Try taking another online backup.")));
1197 
1198  /* Scan for files that should be excluded */
1199  excludeFound = false;
1200  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1201  {
1202  int cmplen = strlen(excludeFiles[excludeIdx].name);
1203 
1204  if (!excludeFiles[excludeIdx].match_prefix)
1205  cmplen++;
1206  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1207  {
1208  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1209  excludeFound = true;
1210  break;
1211  }
1212  }
1213 
1214  if (excludeFound)
1215  continue;
1216 
1217  /* Exclude all forks for unlogged tables except the init fork */
1218  if (isDbDir &&
1219  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1220  &relForkNum))
1221  {
1222  /* Never exclude init forks */
1223  if (relForkNum != INIT_FORKNUM)
1224  {
1225  char initForkFile[MAXPGPATH];
1226  char relOid[OIDCHARS + 1];
1227 
1228  /*
1229  * If any other type of fork, check if there is an init fork
1230  * with the same OID. If so, the file can be excluded.
1231  */
1232  memcpy(relOid, de->d_name, relOidChars);
1233  relOid[relOidChars] = '\0';
1234  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1235  path, relOid);
1236 
1237  if (lstat(initForkFile, &statbuf) == 0)
1238  {
1239  elog(DEBUG2,
1240  "unlogged relation file \"%s\" excluded from backup",
1241  de->d_name);
1242 
1243  continue;
1244  }
1245  }
1246  }
1247 
1248  /* Exclude temporary relations */
1249  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1250  {
1251  elog(DEBUG2,
1252  "temporary relation file \"%s\" excluded from backup",
1253  de->d_name);
1254 
1255  continue;
1256  }
1257 
1258  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1259 
1260  /* Skip pg_control here to back up it last */
1261  if (strcmp(pathbuf, "./global/pg_control") == 0)
1262  continue;
1263 
1264  if (lstat(pathbuf, &statbuf) != 0)
1265  {
1266  if (errno != ENOENT)
1267  ereport(ERROR,
1269  errmsg("could not stat file or directory \"%s\": %m",
1270  pathbuf)));
1271 
1272  /* If the file went away while scanning, it's not an error. */
1273  continue;
1274  }
1275 
1276  /* Scan for directories whose contents should be excluded */
1277  excludeFound = false;
1278  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1279  {
1280  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1281  {
1282  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1283  convert_link_to_directory(pathbuf, &statbuf);
1284  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1285  &statbuf, sizeonly);
1286  excludeFound = true;
1287  break;
1288  }
1289  }
1290 
1291  if (excludeFound)
1292  continue;
1293 
1294  /*
1295  * We can skip pg_wal, the WAL segments need to be fetched from the
1296  * WAL archive anyway. But include it as an empty directory anyway, so
1297  * we get permissions right.
1298  */
1299  if (strcmp(pathbuf, "./pg_wal") == 0)
1300  {
1301  /* If pg_wal is a symlink, write it as a directory anyway */
1302  convert_link_to_directory(pathbuf, &statbuf);
1303  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1304  &statbuf, sizeonly);
1305 
1306  /*
1307  * Also send archive_status directory (by hackishly reusing
1308  * statbuf from above ...).
1309  */
1310  size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1311  &statbuf, sizeonly);
1312 
1313  continue; /* don't recurse into pg_wal */
1314  }
1315 
1316  /* Allow symbolic links in pg_tblspc only */
1317  if (strcmp(path, "./pg_tblspc") == 0 &&
1318 #ifndef WIN32
1319  S_ISLNK(statbuf.st_mode)
1320 #else
1321  pgwin32_is_junction(pathbuf)
1322 #endif
1323  )
1324  {
1325 #if defined(HAVE_READLINK) || defined(WIN32)
1326  char linkpath[MAXPGPATH];
1327  int rllen;
1328 
1329  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1330  if (rllen < 0)
1331  ereport(ERROR,
1333  errmsg("could not read symbolic link \"%s\": %m",
1334  pathbuf)));
1335  if (rllen >= sizeof(linkpath))
1336  ereport(ERROR,
1337  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1338  errmsg("symbolic link \"%s\" target is too long",
1339  pathbuf)));
1340  linkpath[rllen] = '\0';
1341 
1342  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1343  &statbuf, sizeonly);
1344 #else
1345 
1346  /*
1347  * If the platform does not have symbolic links, it should not be
1348  * possible to have tablespaces - clearly somebody else created
1349  * them. Warn about it and ignore.
1350  */
1351  ereport(WARNING,
1352  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1353  errmsg("tablespaces are not supported on this platform")));
1354  continue;
1355 #endif /* HAVE_READLINK */
1356  }
1357  else if (S_ISDIR(statbuf.st_mode))
1358  {
1359  bool skip_this_dir = false;
1360  ListCell *lc;
1361 
1362  /*
1363  * Store a directory entry in the tar file so we can get the
1364  * permissions right.
1365  */
1366  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1367  sizeonly);
1368 
1369  /*
1370  * Call ourselves recursively for a directory, unless it happens
1371  * to be a separate tablespace located within PGDATA.
1372  */
1373  foreach(lc, tablespaces)
1374  {
1375  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1376 
1377  /*
1378  * ti->rpath is the tablespace relative path within PGDATA, or
1379  * NULL if the tablespace has been properly located somewhere
1380  * else.
1381  *
1382  * Skip past the leading "./" in pathbuf when comparing.
1383  */
1384  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1385  {
1386  skip_this_dir = true;
1387  break;
1388  }
1389  }
1390 
1391  /*
1392  * skip sending directories inside pg_tblspc, if not required.
1393  */
1394  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1395  skip_this_dir = true;
1396 
1397  if (!skip_this_dir)
1398  size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1399  sendtblspclinks, manifest, spcoid);
1400  }
1401  else if (S_ISREG(statbuf.st_mode))
1402  {
1403  bool sent = false;
1404 
1405  if (!sizeonly)
1406  sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
1407  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1408  manifest, spcoid);
1409 
1410  if (sent || sizeonly)
1411  {
1412  /* Add size. */
1413  size += statbuf.st_size;
1414 
1415  /* Pad to a multiple of the tar block size. */
1416  size += tarPaddingBytesRequired(statbuf.st_size);
1417 
1418  /* Size of the header for the file. */
1419  size += TAR_BLOCK_SIZE;
1420  }
1421  }
1422  else
1423  ereport(WARNING,
1424  (errmsg("skipping special file \"%s\"", pathbuf)));
1425  }
1426  FreeDir(dir);
1427  return size;
1428 }
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:168
static const char *const excludeDirContents[]
Definition: basebackup.c:128
static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
Definition: basebackup.c:1803
int errhint(const char *fmt,...)
Definition: elog.c:1151
#define DEBUG2
Definition: elog.h:23
#define DEBUG1
Definition: elog.h:24
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3334
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:121
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:63
char * last_dir_separator(const char *filename)
Definition: path.c:139
#define atooid(x)
Definition: postgres_ext.h:42
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:381
ForkNumber
Definition: relpath.h:41
@ INIT_FORKNUM
Definition: relpath.h:46
#define OIDCHARS
Definition: relpath.h:30
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
char * rpath
Definition: basebackup.h:32
#define S_ISDIR(m)
Definition: win32_port.h:324
#define readlink(path, buf, size)
Definition: win32_port.h:236
#define S_ISREG(m)
Definition: win32_port.h:327

References _tarWriteHeader(), AllocateDir(), atooid, backup_started_in_recovery, CHECK_FOR_INTERRUPTS, convert_link_to_directory(), dirent::d_name, DEBUG1, DEBUG2, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), INIT_FORKNUM, InvalidOid, last_dir_separator(), lfirst, looks_like_temp_rel_name(), lstat, manifest, MAXPGPATH, exclude_list_item::name, name, OIDCHARS, parse_filename_for_nontemp_relation(), PG_TEMP_FILE_PREFIX, pgwin32_is_junction(), ReadDir(), readlink, RecoveryInProgress(), tablespaceinfo::rpath, S_ISDIR, S_ISREG, sendFile(), snprintf, stat::st_mode, stat::st_size, TABLESPACE_VERSION_DIRECTORY, TAR_BLOCK_SIZE, tarPaddingBytesRequired(), and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

◆ sendFile()

static bool sendFile ( bbsink sink,
const char *  readfilename,
const char *  tarfilename,
struct stat statbuf,
bool  missing_ok,
Oid  dboid,
backup_manifest_info manifest,
const char *  spcoid 
)
static

Definition at line 1483 of file basebackup.c.

1486 {
1487  int fd;
1488  BlockNumber blkno = 0;
1489  bool block_retry = false;
1490  uint16 checksum;
1491  int checksum_failures = 0;
1492  off_t cnt;
1493  int i;
1494  pgoff_t len = 0;
1495  char *page;
1496  PageHeader phdr;
1497  int segmentno = 0;
1498  char *segmentpath;
1499  bool verify_checksum = false;
1500  pg_checksum_context checksum_ctx;
1501 
1502  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1503  elog(ERROR, "could not initialize checksum of file \"%s\"",
1504  readfilename);
1505 
1506  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1507  if (fd < 0)
1508  {
1509  if (errno == ENOENT && missing_ok)
1510  return false;
1511  ereport(ERROR,
1513  errmsg("could not open file \"%s\": %m", readfilename)));
1514  }
1515 
1516  _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1517 
1519  {
1520  char *filename;
1521 
1522  /*
1523  * Get the filename (excluding path). As last_dir_separator()
1524  * includes the last directory separator, we chop that off by
1525  * incrementing the pointer.
1526  */
1527  filename = last_dir_separator(readfilename) + 1;
1528 
1529  if (is_checksummed_file(readfilename, filename))
1530  {
1531  verify_checksum = true;
1532 
1533  /*
1534  * Cut off at the segment boundary (".") to get the segment number
1535  * in order to mix it into the checksum.
1536  */
1537  segmentpath = strstr(filename, ".");
1538  if (segmentpath != NULL)
1539  {
1540  segmentno = atoi(segmentpath + 1);
1541  if (segmentno == 0)
1542  ereport(ERROR,
1543  (errmsg("invalid segment number %d in file \"%s\"",
1544  segmentno, filename)));
1545  }
1546  }
1547  }
1548 
1549  /*
1550  * Loop until we read the amount of data the caller told us to expect. The
1551  * file could be longer, if it was extended while we were sending it, but
1552  * for a base backup we can ignore such extended data. It will be restored
1553  * from WAL.
1554  */
1555  while (len < statbuf->st_size)
1556  {
1557  size_t remaining = statbuf->st_size - len;
1558 
1559  /* Try to read some more data. */
1560  cnt = basebackup_read_file(fd, sink->bbs_buffer,
1562  len, readfilename, true);
1563 
1564  /*
1565  * If we hit end-of-file, a concurrent truncation must have occurred.
1566  * That's not an error condition, because WAL replay will fix things
1567  * up.
1568  */
1569  if (cnt == 0)
1570  break;
1571 
1572  /*
1573  * The checksums are verified at block level, so we iterate over the
1574  * buffer in chunks of BLCKSZ, after making sure that
1575  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1576  * BLCKSZ bytes.
1577  */
1578  Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1579 
1580  if (verify_checksum && (cnt % BLCKSZ != 0))
1581  {
1582  ereport(WARNING,
1583  (errmsg("could not verify checksum in file \"%s\", block "
1584  "%u: read buffer size %d and page size %d "
1585  "differ",
1586  readfilename, blkno, (int) cnt, BLCKSZ)));
1587  verify_checksum = false;
1588  }
1589 
1590  if (verify_checksum)
1591  {
1592  for (i = 0; i < cnt / BLCKSZ; i++)
1593  {
1594  page = sink->bbs_buffer + BLCKSZ * i;
1595 
1596  /*
1597  * Only check pages which have not been modified since the
1598  * start of the base backup. Otherwise, they might have been
1599  * written only halfway and the checksum would not be valid.
1600  * However, replaying WAL would reinstate the correct page in
1601  * this case. We also skip completely new pages, since they
1602  * don't have a checksum yet.
1603  */
1604  if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr)
1605  {
1606  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1607  phdr = (PageHeader) page;
1608  if (phdr->pd_checksum != checksum)
1609  {
1610  /*
1611  * Retry the block on the first failure. It's
1612  * possible that we read the first 4K page of the
1613  * block just before postgres updated the entire block
1614  * so it ends up looking torn to us. We only need to
1615  * retry once because the LSN should be updated to
1616  * something we can ignore on the next pass. If the
1617  * error happens again then it is a true validation
1618  * failure.
1619  */
1620  if (block_retry == false)
1621  {
1622  int reread_cnt;
1623 
1624  /* Reread the failed block */
1625  reread_cnt =
1627  sink->bbs_buffer + BLCKSZ * i,
1628  BLCKSZ, len + BLCKSZ * i,
1629  readfilename,
1630  false);
1631  if (reread_cnt == 0)
1632  {
1633  /*
1634  * If we hit end-of-file, a concurrent
1635  * truncation must have occurred, so break out
1636  * of this loop just as if the initial fread()
1637  * returned 0. We'll drop through to the same
1638  * code that handles that case. (We must fix
1639  * up cnt first, though.)
1640  */
1641  cnt = BLCKSZ * i;
1642  break;
1643  }
1644 
1645  /* Set flag so we know a retry was attempted */
1646  block_retry = true;
1647 
1648  /* Reset loop to validate the block again */
1649  i--;
1650  continue;
1651  }
1652 
1653  checksum_failures++;
1654 
1655  if (checksum_failures <= 5)
1656  ereport(WARNING,
1657  (errmsg("checksum verification failed in "
1658  "file \"%s\", block %u: calculated "
1659  "%X but expected %X",
1660  readfilename, blkno, checksum,
1661  phdr->pd_checksum)));
1662  if (checksum_failures == 5)
1663  ereport(WARNING,
1664  (errmsg("further checksum verification "
1665  "failures in file \"%s\" will not "
1666  "be reported", readfilename)));
1667  }
1668  }
1669  block_retry = false;
1670  blkno++;
1671  }
1672  }
1673 
1674  bbsink_archive_contents(sink, cnt);
1675 
1676  /* Also feed it to the checksum machinery. */
1677  if (pg_checksum_update(&checksum_ctx,
1678  (uint8 *) sink->bbs_buffer, cnt) < 0)
1679  elog(ERROR, "could not update checksum of base backup");
1680 
1681  len += cnt;
1682  }
1683 
1684  /* If the file was truncated while we were sending it, pad it with zeros */
1685  while (len < statbuf->st_size)
1686  {
1687  size_t remaining = statbuf->st_size - len;
1688  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1689 
1690  MemSet(sink->bbs_buffer, 0, nbytes);
1691  if (pg_checksum_update(&checksum_ctx,
1692  (uint8 *) sink->bbs_buffer,
1693  nbytes) < 0)
1694  elog(ERROR, "could not update checksum of base backup");
1695  bbsink_archive_contents(sink, nbytes);
1696  len += nbytes;
1697  }
1698 
1699  /*
1700  * Pad to a block boundary, per tar format requirements. (This small piece
1701  * of data is probably not worth throttling, and is not checksummed
1702  * because it's not actually part of the file.)
1703  */
1704  _tarWritePadding(sink, len);
1705 
1707 
1708  if (checksum_failures > 1)
1709  {
1710  ereport(WARNING,
1711  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1712  "file \"%s\" has a total of %d checksum verification failures",
1713  checksum_failures,
1714  readfilename, checksum_failures)));
1715 
1716  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1717  }
1718 
1719  total_checksum_failures += checksum_failures;
1720 
1721  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1722  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1723 
1724  return true;
1725 }
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1437
static void _tarWritePadding(bbsink *sink, int len)
Definition: basebackup.c:1780
uint32 BlockNumber
Definition: block.h:31
PageHeaderData * PageHeader
Definition: bufpage.h:166
#define PageIsNew(page)
Definition: bufpage.h:228
#define PageGetLSN(page)
Definition: bufpage.h:365
unsigned short uint16
Definition: c.h:440
unsigned char uint8
Definition: c.h:439
uint16 pg_checksum_page(char *page, BlockNumber blkno)
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
int remaining
Definition: informix.c:667
int i
Definition: isn.c:73
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
int64 pg_time_t
Definition: pgtime.h:23
uint16 pd_checksum
Definition: bufpage.h:156
XLogRecPtr startptr
bbsink_state * bbs_state
bool DataChecksumsEnabled(void)
Definition: xlog.c:4226

References _tarWriteHeader(), _tarWritePadding(), AddFileToBackupManifest(), Assert(), basebackup_read_file(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink::bbs_state, bbsink_archive_contents(), CloseTransientFile(), DataChecksumsEnabled(), elog, ereport, errcode_for_file_access(), errmsg(), errmsg_plural(), ERROR, fd(), filename, i, is_checksummed_file(), last_dir_separator(), len, manifest, MemSet, Min, noverify_checksums, OpenTransientFile(), PageGetLSN, PageIsNew, PageHeaderData::pd_checksum, PG_BINARY, pg_checksum_init(), pg_checksum_page(), pg_checksum_update(), pgoff_t, pgstat_report_checksum_failures_in_db(), remaining, stat::st_mtime, stat::st_size, bbsink_state::startptr, total_checksum_failures, and WARNING.

Referenced by perform_base_backup(), and sendDir().

◆ sendFileWithContent()

static void sendFileWithContent ( bbsink sink,
const char *  filename,
const char *  content,
backup_manifest_info manifest 
)
static

Definition at line 1010 of file basebackup.c.

1012 {
1013  struct stat statbuf;
1014  int bytes_done = 0,
1015  len;
1016  pg_checksum_context checksum_ctx;
1017 
1018  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1019  elog(ERROR, "could not initialize checksum of file \"%s\"",
1020  filename);
1021 
1022  len = strlen(content);
1023 
1024  /*
1025  * Construct a stat struct for the backup_label file we're injecting in
1026  * the tar.
1027  */
1028  /* Windows doesn't have the concept of uid and gid */
1029 #ifdef WIN32
1030  statbuf.st_uid = 0;
1031  statbuf.st_gid = 0;
1032 #else
1033  statbuf.st_uid = geteuid();
1034  statbuf.st_gid = getegid();
1035 #endif
1036  statbuf.st_mtime = time(NULL);
1037  statbuf.st_mode = pg_file_create_mode;
1038  statbuf.st_size = len;
1039 
1040  _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1041 
1042  if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1043  elog(ERROR, "could not update checksum of file \"%s\"",
1044  filename);
1045 
1046  while (bytes_done < len)
1047  {
1048  size_t remaining = len - bytes_done;
1049  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1050 
1051  memcpy(sink->bbs_buffer, content, nbytes);
1052  bbsink_archive_contents(sink, nbytes);
1053  bytes_done += nbytes;
1054  }
1055 
1056  _tarWritePadding(sink, len);
1057 
1059  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1060 }
int pg_file_create_mode
Definition: file_perm.c:19

References _tarWriteHeader(), _tarWritePadding(), AddFileToBackupManifest(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_archive_contents(), elog, ERROR, filename, len, manifest, Min, pg_checksum_init(), pg_checksum_update(), pg_file_create_mode, remaining, stat::st_gid, stat::st_mode, stat::st_mtime, stat::st_size, and stat::st_uid.

Referenced by perform_base_backup().

◆ sendTablespace()

static int64 sendTablespace ( bbsink sink,
char *  path,
char *  oid,
bool  sizeonly,
struct backup_manifest_info manifest 
)
static

Definition at line 1070 of file basebackup.c.

1072 {
1073  int64 size;
1074  char pathbuf[MAXPGPATH];
1075  struct stat statbuf;
1076 
1077  /*
1078  * 'path' points to the tablespace location, but we only want to include
1079  * the version directory in it that belongs to us.
1080  */
1081  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1083 
1084  /*
1085  * Store a directory entry in the tar file so we get the permissions
1086  * right.
1087  */
1088  if (lstat(pathbuf, &statbuf) != 0)
1089  {
1090  if (errno != ENOENT)
1091  ereport(ERROR,
1093  errmsg("could not stat file or directory \"%s\": %m",
1094  pathbuf)));
1095 
1096  /* If the tablespace went away while scanning, it's no error. */
1097  return 0;
1098  }
1099 
1100  size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1101  sizeonly);
1102 
1103  /* Send all the files in the tablespace version directory */
1104  size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1105  spcoid);
1106 
1107  return size;
1108 }

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, manifest, MAXPGPATH, NIL, sendDir(), snprintf, and TABLESPACE_VERSION_DIRECTORY.

Referenced by perform_base_backup().

Variable Documentation

◆ backup_started_in_recovery

bool backup_started_in_recovery = false
static

Definition at line 100 of file basebackup.c.

Referenced by do_pg_backup_start(), do_pg_backup_stop(), perform_base_backup(), and sendDir().

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define PG_STAT_TMP_DIR
Definition: pgstat.h:32

Definition at line 128 of file basebackup.c.

Referenced by sendDir().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{PG_AUTOCONF_FILENAME ".tmp", false},
{TABLESPACE_MAP, false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:24
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:101

Definition at line 128 of file basebackup.c.

Referenced by sendDir().

◆ noChecksumFiles

const struct exclude_list_item noChecksumFiles[]
static
Initial value:
= {
{"pg_control", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
{NULL, false}
}

Definition at line 128 of file basebackup.c.

Referenced by is_checksummed_file().

◆ noverify_checksums

bool noverify_checksums = false
static

Definition at line 106 of file basebackup.c.

Referenced by parse_basebackup_options(), and sendFile().

◆ total_checksum_failures

long long int total_checksum_failures
static

Definition at line 103 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().