57 static int64
sendDir(
const char *path,
int basepathlen,
bool sizeonly,
58 List *tablespaces,
bool sendtblspclinks);
59 static bool sendFile(
const char *readfilename,
const char *tarfilename,
60 struct stat *statbuf,
bool missing_ok,
Oid dboid);
63 struct stat *statbuf,
bool sizeonly);
64 static int64
_tarWriteDir(
const char *pathbuf,
int basepathlen,
struct stat *statbuf,
73 static void throttle(
size_t increment);
85 #define TAR_SEND_SIZE 32768 90 #define THROTTLING_FREQUENCY 8 97 #define CHECK_FREAD_ERROR(fp, filename) \ 101 (errmsg("could not read from file \"%s\"", filename))); \ 213 "config_exec_params",
214 "config_exec_params.new",
247 datadirpathlen = strlen(
DataDir);
257 labelfile, &tablespaces,
290 tablespaces =
lappend(tablespaces, ti);
320 foreach(lc, tablespaces)
331 if (ti->
path == NULL)
345 sendDir(
".", 1,
false, tablespaces,
false);
348 sendDir(
".", 1,
false, tablespaces,
true);
354 errmsg(
"could not stat file \"%s\": %m",
415 while ((de =
ReadDir(dir,
"pg_wal")) != NULL)
419 strcmp(de->
d_name + 8, firstoff + 8) >= 0 &&
420 strcmp(de->
d_name + 8, lastoff + 8) <= 0)
449 if (walFileList ==
NIL)
451 (
errmsg(
"could not find any WAL files")));
459 if (segno != startsegno)
466 (
errmsg(
"could not find WAL file \"%s\"", startfname)));
468 foreach(lc, walFileList)
470 char *walFileName = (
char *)
lfirst(lc);
475 if (!(nextsegno == segno || currsegno == segno))
482 (
errmsg(
"could not find WAL file \"%s\"", nextfname)));
485 if (segno != endsegno)
491 (
errmsg(
"could not find WAL file \"%s\"", endfname)));
495 foreach(lc, walFileList)
497 char *walFileName = (
char *)
lfirst(lc);
509 int save_errno = errno;
521 errmsg(
"could not open file \"%s\": %m", pathbuf)));
524 if (fstat(fileno(fp), &statbuf) != 0)
527 errmsg(
"could not stat file \"%s\": %m",
534 errmsg(
"unexpected WAL file size \"%s\"", walFileName)));
540 while ((cnt = fread(buf, 1,
548 (
errmsg(
"base backup could not send data, aborting backup")));
564 errmsg(
"unexpected WAL file size \"%s\"", walFileName)));
590 foreach(lc, historyFileList)
596 if (
lstat(pathbuf, &statbuf) != 0)
599 errmsg(
"could not stat file \"%s\": %m", pathbuf)));
621 errmsg(
"checksum verification failure during base backup")));
633 char *fna = (
char *)
lfirst(a);
634 char *fnb = (
char *)
lfirst(b);
636 return strcmp(fna + 8, fnb + 8);
646 bool o_label =
false;
647 bool o_progress =
false;
649 bool o_nowait =
false;
651 bool o_maxrate =
false;
652 bool o_tablespace_map =
false;
653 bool o_noverify_checksums =
false;
655 MemSet(opt, 0,
sizeof(*opt));
656 foreach(lopt, options)
660 if (strcmp(defel->
defname,
"label") == 0)
664 (
errcode(ERRCODE_SYNTAX_ERROR),
669 else if (strcmp(defel->
defname,
"progress") == 0)
673 (
errcode(ERRCODE_SYNTAX_ERROR),
678 else if (strcmp(defel->
defname,
"fast") == 0)
682 (
errcode(ERRCODE_SYNTAX_ERROR),
687 else if (strcmp(defel->
defname,
"nowait") == 0)
691 (
errcode(ERRCODE_SYNTAX_ERROR),
696 else if (strcmp(defel->
defname,
"wal") == 0)
700 (
errcode(ERRCODE_SYNTAX_ERROR),
705 else if (strcmp(defel->
defname,
"max_rate") == 0)
711 (
errcode(ERRCODE_SYNTAX_ERROR),
717 (
errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
718 errmsg(
"%d is outside the valid range for parameter \"%s\" (%d .. %d)",
724 else if (strcmp(defel->
defname,
"tablespace_map") == 0)
726 if (o_tablespace_map)
728 (
errcode(ERRCODE_SYNTAX_ERROR),
731 o_tablespace_map =
true;
733 else if (strcmp(defel->
defname,
"noverify_checksums") == 0)
735 if (o_noverify_checksums)
737 (
errcode(ERRCODE_SYNTAX_ERROR),
740 o_noverify_checksums =
true;
743 elog(
ERROR,
"option \"%s\" not recognized",
746 if (opt->
label == NULL)
747 opt->
label =
"base backup";
769 char activitymsg[50];
771 snprintf(activitymsg,
sizeof(activitymsg),
"sending backup \"%s\"",
827 foreach(lc, tablespaces)
834 if (ti->
path == NULL)
843 len = strlen(ti->
oid);
847 len = strlen(ti->
path);
909 len =
snprintf(str,
sizeof(str),
"%u", tli);
929 len = strlen(content);
940 statbuf.st_uid = geteuid();
941 statbuf.st_gid = getegid();
943 statbuf.st_mtime = time(NULL);
945 statbuf.st_size = len;
952 pad = ((len + 511) & ~511) - len;
980 snprintf(pathbuf,
sizeof(pathbuf),
"%s/%s", path,
987 if (
lstat(pathbuf, &statbuf) != 0)
992 errmsg(
"could not stat file or directory \"%s\": %m",
1003 size +=
sendDir(pathbuf, strlen(path), sizeonly,
NIL,
true);
1021 sendDir(
const char *path,
int basepathlen,
bool sizeonly,
List *tablespaces,
1022 bool sendtblspclinks)
1027 struct stat statbuf;
1029 const char *lastDir;
1030 bool isDbDir =
false;
1042 if (lastDir != NULL &&
1043 strspn(lastDir + 1,
"0123456789") == strlen(lastDir + 1))
1046 int parentPathLen = lastDir - path;
1052 if (strncmp(path,
"./base", parentPathLen) == 0 ||
1061 while ((de =
ReadDir(dir, path)) != NULL)
1069 if (strcmp(de->
d_name,
".") == 0 || strcmp(de->
d_name,
"..") == 0)
1089 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1090 errmsg(
"the standby was promoted during online backup"),
1091 errhint(
"This means that the backup being taken is corrupt " 1092 "and should not be used. " 1093 "Try taking another online backup.")));
1096 excludeFound =
false;
1097 for (excludeIdx = 0;
excludeFiles[excludeIdx] != NULL; excludeIdx++)
1102 excludeFound =
true;
1125 memcpy(relOid, de->
d_name, relOidChars);
1126 relOid[relOidChars] =
'\0';
1127 snprintf(initForkFile,
sizeof(initForkFile),
"%s/%s_init",
1130 if (
lstat(initForkFile, &statbuf) == 0)
1133 "unlogged relation file \"%s\" excluded from backup",
1145 "temporary relation file \"%s\" excluded from backup",
1154 if (strcmp(pathbuf,
"./global/pg_control") == 0)
1157 if (
lstat(pathbuf, &statbuf) != 0)
1159 if (errno != ENOENT)
1162 errmsg(
"could not stat file or directory \"%s\": %m",
1170 excludeFound =
false;
1175 elog(
DEBUG1,
"contents of directory \"%s\" excluded from backup", de->
d_name);
1176 size +=
_tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1177 excludeFound =
true;
1192 size +=
_tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1201 if (strcmp(pathbuf,
"./pg_wal") == 0)
1204 size +=
_tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1217 if (strcmp(path,
"./pg_tblspc") == 0 &&
1219 S_ISLNK(statbuf.st_mode)
1225 #if defined(HAVE_READLINK) || defined(WIN32) 1229 rllen =
readlink(pathbuf, linkpath,
sizeof(linkpath));
1233 errmsg(
"could not read symbolic link \"%s\": %m",
1235 if (rllen >=
sizeof(linkpath))
1237 (
errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1238 errmsg(
"symbolic link \"%s\" target is too long",
1240 linkpath[rllen] =
'\0';
1243 &statbuf, sizeonly);
1252 (
errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1253 errmsg(
"tablespaces are not supported on this platform")));
1257 else if (
S_ISDIR(statbuf.st_mode))
1259 bool skip_this_dir =
false;
1273 foreach(lc, tablespaces)
1284 if (ti->
rpath && strcmp(ti->
rpath, pathbuf + 2) == 0)
1286 skip_this_dir =
true;
1294 if (strcmp(pathbuf,
"./pg_tblspc") == 0 && !sendtblspclinks)
1295 skip_this_dir =
true;
1298 size +=
sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1300 else if (
S_ISREG(statbuf.st_mode))
1305 sent =
sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1308 if (sent || sizeonly)
1311 size += ((statbuf.st_size + 511) & ~511);
1317 (
errmsg(
"skipping special file \"%s\"", pathbuf)));
1332 const char *
const *f;
1335 if (strncmp(fullpath,
"./global/", 9) == 0 ||
1336 strncmp(fullpath,
"./base/", 7) == 0 ||
1337 strncmp(fullpath,
"/", 1) == 0)
1341 if (strcmp(*f, filename) == 0)
1369 sendFile(
const char *readfilename,
const char *tarfilename,
struct stat *statbuf,
1370 bool missing_ok,
Oid dboid)
1374 bool block_retry =
false;
1377 int checksum_failures = 0;
1386 bool verify_checksum =
false;
1391 if (errno == ENOENT && missing_ok)
1395 errmsg(
"could not open file \"%s\": %m", readfilename)));
1413 verify_checksum =
true;
1419 segmentpath = strstr(filename,
".");
1420 if (segmentpath != NULL)
1422 segmentno = atoi(segmentpath + 1);
1425 (
errmsg(
"invalid segment number %d in file \"%s\"",
1426 segmentno, filename)));
1431 while ((cnt = fread(buf, 1,
Min(
sizeof(buf), statbuf->st_size - len), fp)) > 0)
1441 if (verify_checksum && (cnt % BLCKSZ != 0))
1444 (
errmsg(
"could not verify checksum in file \"%s\", block " 1445 "%d: read buffer size %d and page size %d " 1447 readfilename, blkno, (
int) cnt, BLCKSZ)));
1448 verify_checksum =
false;
1451 if (verify_checksum)
1453 for (i = 0; i < cnt / BLCKSZ; i++)
1455 page = buf + BLCKSZ *
i;
1467 checksum =
pg_checksum_page((
char *) page, blkno + segmentno * RELSEG_SIZE);
1481 if (block_retry ==
false)
1484 if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
1488 errmsg(
"could not fseek in file \"%s\": %m",
1492 if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
1510 errmsg(
"could not reread block %d of file \"%s\": %m",
1511 blkno, readfilename)));
1514 if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
1518 errmsg(
"could not fseek in file \"%s\": %m",
1530 checksum_failures++;
1532 if (checksum_failures <= 5)
1534 (
errmsg(
"checksum verification failed in " 1535 "file \"%s\", block %d: calculated " 1536 "%X but expected %X",
1537 readfilename, blkno, checksum,
1539 if (checksum_failures == 5)
1541 (
errmsg(
"further checksum verification " 1542 "failures in file \"%s\" will not " 1543 "be reported", readfilename)));
1546 block_retry =
false;
1554 (
errmsg(
"base backup could not send data, aborting backup")));
1559 if (feof(fp) || len >= statbuf->st_size)
1573 if (len < statbuf->st_size)
1575 MemSet(buf, 0,
sizeof(buf));
1576 while (len < statbuf->st_size)
1578 cnt =
Min(
sizeof(buf), statbuf->st_size - len);
1589 pad = ((len + 511) & ~511) - len;
1598 if (checksum_failures > 1)
1601 (
errmsg_plural(
"file \"%s\" has a total of %d checksum verification failure",
1602 "file \"%s\" has a total of %d checksum verification failures",
1604 readfilename, checksum_failures)));
1617 struct stat *statbuf,
bool sizeonly)
1625 statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1634 (
errmsg(
"file name too long for tar format: \"%s\"",
1639 (
errmsg(
"symbolic link target too long for tar format: " 1640 "file name \"%s\", target \"%s\"",
1641 filename, linktarget)));
1644 elog(
ERROR,
"unrecognized tar error: %d", rc);
1663 if (S_ISLNK(statbuf->st_mode))
1669 return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1707 sleep = elapsed_min - elapsed;
1722 (
long) (sleep / 1000),
#define StatusFilePath(path, xlog, suffix)
int errhint(const char *fmt,...)
static void throttle(size_t increment)
static ListCell * lnext(const List *l, const ListCell *c)
bool update_process_title
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
TimestampTz GetCurrentTimestamp(void)
bool DataChecksumsEnabled(void)
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
char * pstrdup(const char *in)
char * psprintf(const char *fmt,...)
StringInfo makeStringInfo(void)
static void SendBackupHeader(List *tablespaces)
void set_ps_display(const char *activity, bool force)
static void parse_basebackup_options(List *options, basebackup_options *opt)
int errcode(int sqlerrcode)
#define MemSet(start, val, len)
bool RecoveryInProgress(void)
static bool backup_started_in_recovery
static void send_int8_string(StringInfoData *buf, int64 intval)
void ResetLatch(Latch *latch)
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
static const char *const excludeDirContents[]
#define PG_TEMP_FILE_PREFIX
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
static bool noverify_checksums
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
void SendBaseBackup(BaseBackupCmd *cmd)
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define TABLESPACE_VERSION_DIRECTORY
#define readlink(path, buf, size)
int errcode_for_file_access(void)
#define is_absolute_path(filename)
FILE * AllocateFile(const char *name, const char *mode)
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
int64 sendTablespace(char *path, bool sizeonly)
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
DIR * AllocateDir(const char *dirname)
char * pgstat_stat_directory
#define ereport(elevel, rest)
#define CHECK_FREAD_ERROR(fp, filename)
static char * statrelpath
static const char *const excludeFiles[]
#define ERRCODE_DATA_CORRUPTED
List * lappend(List *list, void *datum)
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
static void perform_base_backup(basebackup_options *opt)
void * palloc0(Size size)
char * last_dir_separator(const char *filename)
#define XLOG_CONTROL_FILE
TimeLineID ThisTimeLineID
static TimestampTz throttled_last
#define RELCACHE_INIT_FILENAME
static StringInfo tblspc_map_file
static bool is_checksummed_file(const char *fullpath, const char *filename)
PageHeaderData * PageHeader
#define Assert(condition)
void WalSndSetState(WalSndState state)
struct dirent * ReadDir(DIR *dir, const char *dirname)
#define THROTTLING_FREQUENCY
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define PG_AUTOCONF_FILENAME
#define LOG_METAINFO_DATAFILE_TMP
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
bool looks_like_temp_rel_name(const char *name)
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
void list_sort(List *list, list_sort_comparator cmp)
int errmsg(const char *fmt,...)
void do_pg_abort_backup(void)
#define BACKUP_LABEL_FILE
#define CHECK_FOR_INTERRUPTS()
#define pq_putmessage(msgtype, s, len)
static int64 throttling_counter
static void base_backup_cleanup(int code, Datum arg)
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid)
static uint64 throttling_sample
static void sendFileWithContent(const char *filename, const char *content)
uint16 pg_checksum_page(char *page, BlockNumber blkno)
static TimeOffset elapsed_min_unit
int32 pg_atoi(const char *s, int size, int c)
#define WL_EXIT_ON_PM_DEATH
static XLogRecPtr startptr
bool pgwin32_is_junction(const char *path)
static const char *const noChecksumFiles[]
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
static long long int total_checksum_failures
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)