35 #define INCREMENTAL_PREFIX "INCREMENTAL."
36 #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
103 char *input_directory,
104 char *output_directory,
107 char **prior_backup_dirs,
124 static struct option long_options[] = {
139 char *last_input_dir;
146 uint64 system_identifier;
147 char **prior_backup_dirs;
160 memset(&opt, 0,
sizeof(opt));
167 long_options, &optindex)) != -1)
190 pg_fatal(
"unrecognized checksum algorithm: \"%s\"",
221 pg_fatal(
"no output directory specified");
230 #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
231 (defined(__linux__) && defined(FICLONE))
239 pg_fatal(
"file cloning not supported on this platform");
244 #if defined(HAVE_COPY_FILE_RANGE)
247 pg_log_debug(
"would use copy_file_range to copy blocks");
249 pg_log_debug(
"will use copy_file_range to copy blocks");
252 pg_fatal(
"copy_file_range not supported on this platform");
260 n_backups = argc -
optind;
270 n_prior_backups = argc -
optind - 1;
271 prior_backup_dirs = argv +
optind;
280 for (
i = 0;
i < n_backups;
i++)
283 manifests[
i]->system_identifier != system_identifier)
287 controlpath =
psprintf(
"%s/%s", prior_backup_dirs[
i],
"global/pg_control");
289 pg_fatal(
"%s: manifest system identifier is %llu, but control file has %llu",
291 (
unsigned long long) manifests[
i]->system_identifier,
292 (
unsigned long long) system_identifier);
297 last_input_dir = argv[argc - 1];
314 for (ts = tablespaces; ts != NULL; ts = ts->
next)
327 if (manifests[n_prior_backups] == NULL)
328 pg_fatal(
"can't generate a manifest because no manifest is available for the final input backup");
339 last_backup_label->
cursor = 0;
345 pg_log_debug(
"processing backup directory \"%s\"", last_input_dir);
347 NULL, n_prior_backups, prior_backup_dirs,
348 manifests, mwriter, &opt);
351 for (ts = tablespaces; ts != NULL; ts = ts->
next)
368 pg_log_debug(
"would create symbolic link from \"%s\" to \"%s\"",
372 pg_log_debug(
"creating symbolic link from \"%s\" to \"%s\"",
375 pg_fatal(
"could not create symbolic link from \"%s\" to \"%s\": %m",
387 pg_fatal(
"could not create directory \"%s\": %m",
394 NULL, n_prior_backups, prior_backup_dirs,
395 manifests, mwriter, &opt);
401 manifests[n_prior_backups]->first_wal_range);
438 dst_ptr = dst = tsmap->
old_dir;
439 for (arg_ptr =
arg; *arg_ptr !=
'\0'; arg_ptr++)
442 pg_fatal(
"directory name too long");
444 if (*arg_ptr ==
'\\' && *(arg_ptr + 1) ==
'=')
446 else if (*arg_ptr ==
'=' && (arg_ptr ==
arg || *(arg_ptr - 1) !=
'\\'))
449 pg_fatal(
"multiple \"=\" signs in tablespace mapping");
451 dst = dst_ptr = tsmap->
new_dir;
454 *dst_ptr++ = *arg_ptr;
457 pg_fatal(
"invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"",
arg);
468 pg_fatal(
"old directory is not an absolute path in tablespace mapping: %s",
472 pg_fatal(
"old directory is not an absolute path in tablespace mapping: %s",
498 for (
i = n_backups - 1;
i >= 0; --
i)
510 if ((
fd = open(pathbuf, O_RDONLY, 0)) < 0)
511 pg_fatal(
"could not open file \"%s\": %m", pathbuf);
526 pg_fatal(
"could not close \"%s\": %m", pathbuf);
530 &previous_tli, &previous_lsn);
540 if (
i > 0 && previous_tli == 0)
541 pg_fatal(
"backup at \"%s\" is a full backup, but only the first backup should be a full backup",
543 if (
i == 0 && previous_tli != 0)
544 pg_fatal(
"backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
546 if (
i < n_backups - 1 && start_tli != check_tli)
547 pg_fatal(
"backup at \"%s\" starts on timeline %u, but expected %u",
548 backup_dirs[
i], start_tli, check_tli);
549 if (
i < n_backups - 1 && start_lsn != check_lsn)
550 pg_fatal(
"backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
554 check_tli = previous_tli;
555 check_lsn = previous_lsn;
585 uint64 system_identifier = 0;
586 uint32 data_checksum_version = 0;
587 bool data_checksum_mismatch =
false;
590 for (
i = n_backups - 1;
i >= 0; --
i)
596 controlpath =
psprintf(
"%s/%s", backup_dirs[
i],
"global/pg_control");
602 pg_fatal(
"%s: CRC is incorrect", controlpath);
606 pg_fatal(
"%s: unexpected control file version",
610 if (
i == n_backups - 1)
613 pg_fatal(
"%s: expected system identifier %llu, but found %llu",
614 controlpath, (
unsigned long long) system_identifier,
621 if (
i == n_backups - 1)
623 else if (data_checksum_version != 0 &&
625 data_checksum_mismatch =
true;
637 (
unsigned long long) system_identifier);
643 if (data_checksum_mismatch)
646 pg_log_warning_hint(
"disable, and optionally reenable, checksums on the output directory to avoid failures");
649 return system_identifier;
663 if (
stat(dir, &st) != 0)
664 pg_fatal(
"could not stat \"%s\": %m", dir);
687 pg_log_info(
"removing contents of output directory \"%s\"",
690 pg_log_error(
"failed to remove contents of output directory");
718 pg_fatal(
"could not create directory \"%s\": %m", dirname);
723 pg_log_debug(
"using existing directory \"%s\"", dirname);
730 pg_fatal(
"directory \"%s\" exists but is not empty", dirname);
733 pg_fatal(
"could not access directory \"%s\": %m", dirname);
747 printf(
_(
"%s reconstructs full backups from incrementals.\n\n"),
progname);
751 printf(
_(
" -d, --debug generate lots of debugging output\n"));
752 printf(
_(
" -n, --dry-run do not actually do anything\n"));
753 printf(
_(
" -N, --no-sync do not wait for changes to be written safely to disk\n"));
754 printf(
_(
" -o, --output output directory\n"));
755 printf(
_(
" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
756 " relocate tablespace in OLDDIR to NEWDIR\n"));
757 printf(
_(
" --clone clone (reflink) instead of copying files\n"));
758 printf(
_(
" --copy-file-range copy using copy_file_range() syscall\n"));
759 printf(
_(
" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
760 " use algorithm for manifest checksums\n"));
761 printf(
_(
" --no-manifest suppress generation of backup manifest\n"));
762 printf(
_(
" --sync-method=METHOD set method for syncing files to disk\n"));
763 printf(
_(
" -V, --version output version information, then exit\n"));
764 printf(
_(
" -?, --help show this help, then exit\n"));
766 printf(
_(
"\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
767 printf(
_(
"%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
782 oid = strtoul(s, &ep, 10);
783 if (errno != 0 || *ep !=
'\0' || oid < 1 || oid >
PG_UINT32_MAX)
811 char *input_directory,
812 char *output_directory,
815 char **prior_backup_dirs,
825 bool is_pg_tblspc =
false;
826 bool is_pg_wal =
false;
827 bool is_incremental_dir =
false;
854 is_incremental_dir =
true;
855 else if (relative_path != NULL)
857 is_pg_tblspc = strcmp(relative_path,
"pg_tblspc") == 0;
858 is_pg_wal = (strcmp(relative_path,
"pg_wal") == 0 ||
859 strncmp(relative_path,
"pg_wal/", 7) == 0);
860 is_incremental_dir = strncmp(relative_path,
"base/", 5) == 0 ||
861 strcmp(relative_path,
"global") == 0 ||
862 strncmp(relative_path,
"pg_tblspc/", 10) == 0;
880 if (relative_path == NULL)
887 manifest_prefix[0] =
'\0';
897 tsoid, relative_path);
906 if (relative_path != NULL)
909 pg_log_debug(
"would create directory \"%s\"", ofulldir);
914 pg_fatal(
"could not create directory \"%s\": %m", ofulldir);
919 if ((dir =
opendir(ifulldir)) == NULL)
920 pg_fatal(
"could not open directory \"%s\": %m", ifulldir);
921 while (errno = 0, (de =
readdir(dir)) != NULL)
928 int checksum_length = 0;
929 uint8 *checksum_payload = NULL;
933 if (strcmp(de->
d_name,
".") == 0 ||
934 strcmp(de->
d_name,
"..") == 0)
963 if (relative_path == NULL)
971 input_directory, output_directory,
973 n_prior_backups, prior_backup_dirs,
974 manifests, mwriter, opt);
992 if (relative_path == NULL &&
993 (strcmp(de->
d_name,
"backup_label") == 0 ||
994 strcmp(de->
d_name,
"backup_manifest") == 0))
1001 if (is_incremental_dir &&
1044 latest_manifest != NULL)
1048 mfile = manifest_files_lookup(latest_manifest->
files,
1058 bmpath =
psprintf(
"%s/%s", input_directory,
1061 bmpath, manifest_path);
1076 if (checksum_length != 0)
1083 copy_file(ifullpath, ofullpath, &checksum_ctx,
1100 if (mwriter != NULL)
1117 if (
stat(ofullpath, &sb) < 0)
1118 pg_fatal(
"could not stat file \"%s\": %m", ofullpath);
1123 checksum_type, checksum_length,
1128 if (checksum_payload != NULL)
1129 pfree(checksum_payload);
1166 version = strtoul(
buf.data, &ep, 10);
1167 if (errno != 0 || *ep !=
'\n')
1175 if (version < 10 && *ep ==
'.')
1185 return version * 10000;
1242 if ((dir =
opendir(pg_tblspc)) == NULL)
1243 pg_fatal(
"could not open directory \"%s\": %m", pg_tblspc);
1245 while (errno = 0, (de =
readdir(dir)) != NULL)
1256 if (strcmp(de->
d_name,
".") == 0 || strcmp(de->
d_name,
"..") == 0)
1265 pg_log_debug(
"skipping \"%s\" because the filename is not a legal tablespace OID",
1276 pg_log_debug(
"skipping \"%s\" because it is neither a symbolic link nor a directory",
1294 link_length =
readlink(tblspcdir, link_target,
sizeof(link_target));
1295 if (link_length < 0)
1296 pg_fatal(
"could not read symbolic link \"%s\": %m",
1298 if (link_length >=
sizeof(link_target))
1299 pg_fatal(
"symbolic link \"%s\" is too long", tblspcdir);
1300 link_target[link_length] =
'\0';
1302 pg_fatal(
"symbolic link \"%s\" is relative", tblspcdir);
1311 for (tsmap = opt->
tsmappings; tsmap != NULL; tsmap = tsmap->
next)
1313 if (strcmp(tsmap->
old_dir, link_target) == 0)
1324 pg_fatal(
"tablespace at \"%s\" has no tablespace mapping",
1340 for (otherts = tslist; otherts != NULL; otherts = otherts->
next)
1342 pg_fatal(
"tablespaces with OIDs %u and %u both point at \"%s\"",
1351 pg_fatal(
"could not close directory \"%s\": %m", pg_tblspc);
1389 pg_fatal(
"could not read file \"%s\": read only %zd of %lld bytes",
1395 buf->data[
buf->len] =
'\0';
void parse_backup_label(char *filename, StringInfo buf, TimeLineID *start_tli, XLogRecPtr *start_lsn, TimeLineID *previous_tli, XLogRecPtr *previous_lsn)
void write_backup_label(char *output_directory, StringInfo buf, pg_checksum_type checksum_type, manifest_writer *mwriter)
#define PG_TEXTDOMAIN(domain)
#define OidIsValid(objectId)
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define PG_CHECKSUM_MAX_LENGTH
void set_pglocale_pgservice(const char *argv0, const char *app)
ControlFileData * get_controlfile_by_exact_path(const char *ControlFilePath, bool *crc_ok_p)
@ COPY_METHOD_COPY_FILE_RANGE
void copy_file(const char *fromfile, const char *tofile)
struct dirent * readdir(DIR *)
DIR * opendir(const char *)
void * pg_malloc0(size_t size)
void * pg_malloc(size_t size)
void SetDataDirectoryCreatePerm(int dataDirMode)
PGFileType get_dirent_type(const char *path, const struct dirent *de, bool look_through_symlinks, int elevel)
@ DATA_DIR_SYNC_METHOD_FSYNC
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
#define required_argument
manifest_data ** load_backup_manifests(int n_backups, char **backup_directories)
void pg_logging_increase_verbosity(void)
void pg_logging_init(const char *argv0)
#define pg_log_error(...)
#define pg_log_error_hint(...)
#define pg_log_warning_hint(...)
#define pg_log_debug(...)
void pfree(void *pointer)
void handle_help_version_opts(int argc, char *argv[], const char *fixed_progname, help_handler hlp)
bool parse_sync_method(const char *optarg, DataDirSyncMethod *sync_method)
static cb_tablespace * scan_for_existing_tablespaces(char *pathname, cb_options *opt)
int main(int argc, char *argv[])
struct cb_cleanup_dir cb_cleanup_dir
static void remember_to_cleanup_directory(char *target_path, bool rmtopdir)
static void help(const char *progname)
static void process_directory_recursively(Oid tsoid, char *input_directory, char *output_directory, char *relative_path, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, manifest_writer *mwriter, cb_options *opt)
static void create_output_directory(char *dirname, cb_options *opt)
static void check_input_dir_permissions(char *dir)
static uint64 check_control_files(int n_backups, char **backup_dirs)
cb_cleanup_dir * cleanup_dir_list
struct cb_tablespace cb_tablespace
#define INCREMENTAL_PREFIX_LENGTH
static void cleanup_directories_atexit(void)
static StringInfo check_backup_label_files(int n_backups, char **backup_dirs)
static void add_tablespace_mapping(cb_options *opt, char *arg)
#define INCREMENTAL_PREFIX
struct cb_tablespace_mapping cb_tablespace_mapping
static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
struct cb_options cb_options
static bool parse_oid(char *s, Oid *result)
static int read_pg_version_file(char *directory)
static void reset_directory_cleanup_list(void)
#define PG_CONTROL_VERSION
PGDLLIMPORT char * optarg
#define pg_log_warning(...)
int pg_mkdir_p(char *path, int omode)
#define is_absolute_path(filename)
void canonicalize_path(char *path)
int pg_check_dir(const char *dir)
const char * get_progname(const char *argv0)
size_t strlcpy(char *dst, const char *src, size_t siz)
static int fd(const char *x, int i)
char * psprintf(const char *fmt,...)
void reconstruct_from_incremental_file(char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)
bool rmtree(const char *path, bool rmtopdir)
void destroyStringInfo(StringInfo str)
StringInfo makeStringInfo(void)
void resetStringInfo(StringInfo str)
void enlargeStringInfo(StringInfo str, int needed)
void initStringInfo(StringInfo str)
uint32 pg_control_version
uint32 data_checksum_version
struct cb_cleanup_dir * next
cb_tablespace_mapping * tsmappings
DataDirSyncMethod sync_method
pg_checksum_type manifest_checksums
struct cb_tablespace_mapping * next
struct cb_tablespace * next
manifest_files_hash * files
pg_checksum_type checksum_type
#define symlink(oldpath, newpath)
#define readlink(path, buf, size)
manifest_writer * create_manifest_writer(char *directory, uint64 system_identifier)
void add_file_to_manifest(manifest_writer *mwriter, const char *manifest_path, size_t size, time_t mtime, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
void finalize_manifest(manifest_writer *mwriter, manifest_wal_range *first_wal_range)
#define LSN_FORMAT_ARGS(lsn)
#define InvalidXLogRecPtr
static const char * directory