PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/catalog.h"
#include "catalog/pg_type.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgtar.h"
#include "pgstat.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "utils/builtins.h"
#include "utils/elog.h"
#include "utils/ps_status.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 

Functions

static int64 sendDir (char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
 
static bool sendFile (char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
 
static void sendFileWithContent (const char *filename, const char *content)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void base_backup_cleanup (int code, Datum arg)
 
static void perform_base_backup (basebackup_options *opt, DIR *tblspcdir)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const void *a, const void *b)
 
static void throttle (size_t increment)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 
int64 sendTablespace (char *path, bool sizeonly)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static const char * excludeDirContents []
 
static const char * excludeFiles []
 

Macro Definition Documentation

#define TAR_SEND_SIZE   32768

Definition at line 81 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

#define THROTTLING_FREQUENCY   8

Definition at line 86 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1317 of file basebackup.c.

References _tarWriteHeader(), and NULL.

Referenced by sendDir().

1319 {
1320  /* If symlink, write it as a directory anyway */
1321 #ifndef WIN32
1322  if (S_ISLNK(statbuf->st_mode))
1323 #else
1324  if (pgwin32_is_junction(pathbuf))
1325 #endif
1326  statbuf->st_mode = S_IFDIR | S_IRWXU;
1327 
1328  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1329 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
#define NULL
Definition: c.h:229
static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1275 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1277 {
1278  char h[512];
1279  enum tarError rc;
1280 
1281  if (!sizeonly)
1282  {
1283  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1284  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1285  statbuf->st_mtime);
1286 
1287  switch (rc)
1288  {
1289  case TAR_OK:
1290  break;
1291  case TAR_NAME_TOO_LONG:
1292  ereport(ERROR,
1293  (errmsg("file name too long for tar format: \"%s\"",
1294  filename)));
1295  break;
1296  case TAR_SYMLINK_TOO_LONG:
1297  ereport(ERROR,
1298  (errmsg("symbolic link target too long for tar format: "
1299  "file name \"%s\", target \"%s\"",
1300  filename, linktarget)));
1301  break;
1302  default:
1303  elog(ERROR, "unrecognized tar error: %d", rc);
1304  }
1305 
1306  pq_putmessage('d', h, sizeof(h));
1307  }
1308 
1309  return sizeof(h);
1310 }
Definition: pgtar.h:17
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define ereport(elevel, rest)
Definition: elog.h:122
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
static char * filename
Definition: pg_dumpall.c:87
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define elog
Definition: elog.h:219
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static void base_backup_cleanup ( int  code,
Datum  arg 
)
static

Definition at line 175 of file basebackup.c.

References do_pg_abort_backup().

Referenced by perform_base_backup().

176 {
178 }
void do_pg_abort_backup(void)
Definition: xlog.c:11057
static int compareWalFileNames ( const void *  a,
const void *  b 
)
static

Definition at line 568 of file basebackup.c.

Referenced by perform_base_backup().

569 {
570  char *fna = *((char **) a);
571  char *fnb = *((char **) b);
572 
573  return strcmp(fna + 8, fnb + 8);
574 }
static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 580 of file basebackup.c.

References DefElem::arg, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, basebackup_options::nowait, NULL, basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

581 {
582  ListCell *lopt;
583  bool o_label = false;
584  bool o_progress = false;
585  bool o_fast = false;
586  bool o_nowait = false;
587  bool o_wal = false;
588  bool o_maxrate = false;
589  bool o_tablespace_map = false;
590 
591  MemSet(opt, 0, sizeof(*opt));
592  foreach(lopt, options)
593  {
594  DefElem *defel = (DefElem *) lfirst(lopt);
595 
596  if (strcmp(defel->defname, "label") == 0)
597  {
598  if (o_label)
599  ereport(ERROR,
600  (errcode(ERRCODE_SYNTAX_ERROR),
601  errmsg("duplicate option \"%s\"", defel->defname)));
602  opt->label = strVal(defel->arg);
603  o_label = true;
604  }
605  else if (strcmp(defel->defname, "progress") == 0)
606  {
607  if (o_progress)
608  ereport(ERROR,
609  (errcode(ERRCODE_SYNTAX_ERROR),
610  errmsg("duplicate option \"%s\"", defel->defname)));
611  opt->progress = true;
612  o_progress = true;
613  }
614  else if (strcmp(defel->defname, "fast") == 0)
615  {
616  if (o_fast)
617  ereport(ERROR,
618  (errcode(ERRCODE_SYNTAX_ERROR),
619  errmsg("duplicate option \"%s\"", defel->defname)));
620  opt->fastcheckpoint = true;
621  o_fast = true;
622  }
623  else if (strcmp(defel->defname, "nowait") == 0)
624  {
625  if (o_nowait)
626  ereport(ERROR,
627  (errcode(ERRCODE_SYNTAX_ERROR),
628  errmsg("duplicate option \"%s\"", defel->defname)));
629  opt->nowait = true;
630  o_nowait = true;
631  }
632  else if (strcmp(defel->defname, "wal") == 0)
633  {
634  if (o_wal)
635  ereport(ERROR,
636  (errcode(ERRCODE_SYNTAX_ERROR),
637  errmsg("duplicate option \"%s\"", defel->defname)));
638  opt->includewal = true;
639  o_wal = true;
640  }
641  else if (strcmp(defel->defname, "max_rate") == 0)
642  {
643  long maxrate;
644 
645  if (o_maxrate)
646  ereport(ERROR,
647  (errcode(ERRCODE_SYNTAX_ERROR),
648  errmsg("duplicate option \"%s\"", defel->defname)));
649 
650  maxrate = intVal(defel->arg);
651  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
652  ereport(ERROR,
653  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
654  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
655  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
656 
657  opt->maxrate = (uint32) maxrate;
658  o_maxrate = true;
659  }
660  else if (strcmp(defel->defname, "tablespace_map") == 0)
661  {
662  if (o_tablespace_map)
663  ereport(ERROR,
664  (errcode(ERRCODE_SYNTAX_ERROR),
665  errmsg("duplicate option \"%s\"", defel->defname)));
666  opt->sendtblspcmapfile = true;
667  o_tablespace_map = true;
668  }
669  else
670  elog(ERROR, "option \"%s\" not recognized",
671  defel->defname);
672  }
673  if (opt->label == NULL)
674  opt->label = "base backup";
675 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
#define MemSet(start, val, len)
Definition: c.h:857
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:43
static int32 maxrate
Definition: pg_basebackup.c:92
const char * label
Definition: basebackup.c:44
unsigned int uint32
Definition: c.h:268
#define ereport(elevel, rest)
Definition: elog.h:122
Node * arg
Definition: parsenodes.h:721
#define NULL
Definition: c.h:229
#define lfirst(lc)
Definition: pg_list.h:106
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:797
char * defname
Definition: parsenodes.h:720
#define elog
Definition: elog.h:219
static void perform_base_backup ( basebackup_options opt,
DIR tblspcdir 
)
static

Definition at line 187 of file basebackup.c.

References _tarWriteHeader(), AllocateDir(), AllocateFile(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, base_backup_cleanup(), buf, CheckXLogRemoved(), compareWalFileNames(), dirent::d_name, StringInfoData::data, DataDir, do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, FreeDir(), FreeFile(), GetCurrentTimestamp(), i, basebackup_options::includewal, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, list_length(), lnext, lstat, makeStringInfo(), MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, NULL, palloc(), palloc0(), tablespaceinfo::path, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint(), basebackup_options::progress, psprintf(), pstrdup(), qsort, ReadDir(), RecoveryInProgress(), SendBackupHeader(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf(), statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, USECS_PER_SEC, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, XLogFromFileName, and XLogSegSize.

Referenced by SendBaseBackup().

188 {
189  XLogRecPtr startptr;
190  TimeLineID starttli;
191  XLogRecPtr endptr;
192  TimeLineID endtli;
193  StringInfo labelfile;
195  int datadirpathlen;
196  List *tablespaces = NIL;
197 
198  datadirpathlen = strlen(DataDir);
199 
201 
202  labelfile = makeStringInfo();
203  tblspc_map_file = makeStringInfo();
204 
205  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
206  labelfile, tblspcdir, &tablespaces,
207  tblspc_map_file,
208  opt->progress, opt->sendtblspcmapfile);
209 
210  /*
211  * Once do_pg_start_backup has been called, ensure that any failure causes
212  * us to abort the backup so we don't "leak" a backup counter. For this
213  * reason, *all* functionality between do_pg_start_backup() and
214  * do_pg_stop_backup() should be inside the error cleanup block!
215  */
216 
218  {
219  ListCell *lc;
220  tablespaceinfo *ti;
221 
222  SendXlogRecPtrResult(startptr, starttli);
223 
224  /*
225  * Calculate the relative path of temporary statistics directory in
226  * order to skip the files which are located in that directory later.
227  */
229  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
230  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
231  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
233  else
235 
236  /* Add a node for the base directory at the end */
237  ti = palloc0(sizeof(tablespaceinfo));
238  ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
239  tablespaces = lappend(tablespaces, ti);
240 
241  /* Send tablespace header */
242  SendBackupHeader(tablespaces);
243 
244  /* Setup and activate network throttling, if client requested it */
245  if (opt->maxrate > 0)
246  {
248  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
249 
250  /*
251  * The minimum amount of time for throttling_sample bytes to be
252  * transferred.
253  */
255 
256  /* Enable throttling. */
257  throttling_counter = 0;
258 
259  /* The 'real data' starts now (header was ignored). */
261  }
262  else
263  {
264  /* Disable throttling. */
265  throttling_counter = -1;
266  }
267 
268  /* Send off our tablespaces one by one */
269  foreach(lc, tablespaces)
270  {
271  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
273 
274  /* Send CopyOutResponse message */
275  pq_beginmessage(&buf, 'H');
276  pq_sendbyte(&buf, 0); /* overall format */
277  pq_sendint(&buf, 0, 2); /* natts */
278  pq_endmessage(&buf);
279 
280  if (ti->path == NULL)
281  {
282  struct stat statbuf;
283 
284  /* In the main tar, include the backup_label first... */
286 
287  /*
288  * Send tablespace_map file if required and then the bulk of
289  * the files.
290  */
291  if (tblspc_map_file && opt->sendtblspcmapfile)
292  {
293  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
294  sendDir(".", 1, false, tablespaces, false);
295  }
296  else
297  sendDir(".", 1, false, tablespaces, true);
298 
299  /* ... and pg_control after everything else. */
300  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
301  ereport(ERROR,
303  errmsg("could not stat control file \"%s\": %m",
305  sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false);
306  }
307  else
308  sendTablespace(ti->path, false);
309 
310  /*
311  * If we're including WAL, and this is the main data directory we
312  * don't terminate the tar stream here. Instead, we will append
313  * the xlog files below and terminate it then. This is safe since
314  * the main data directory is always sent *last*.
315  */
316  if (opt->includewal && ti->path == NULL)
317  {
318  Assert(lnext(lc) == NULL);
319  }
320  else
321  pq_putemptymessage('c'); /* CopyDone */
322  }
323  }
325 
326  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
327 
328  if (opt->includewal)
329  {
330  /*
331  * We've left the last tar file "open", so we can now append the
332  * required WAL files to it.
333  */
334  char pathbuf[MAXPGPATH];
335  XLogSegNo segno;
336  XLogSegNo startsegno;
337  XLogSegNo endsegno;
338  struct stat statbuf;
339  List *historyFileList = NIL;
340  List *walFileList = NIL;
341  char **walFiles;
342  int nWalFiles;
343  char firstoff[MAXFNAMELEN];
344  char lastoff[MAXFNAMELEN];
345  DIR *dir;
346  struct dirent *de;
347  int i;
348  ListCell *lc;
349  TimeLineID tli;
350 
351  /*
352  * I'd rather not worry about timelines here, so scan pg_wal and
353  * include all WAL files in the range between 'startptr' and 'endptr',
354  * regardless of the timeline the file is stamped with. If there are
355  * some spurious WAL files belonging to timelines that don't belong in
356  * this server's history, they will be included too. Normally there
357  * shouldn't be such files, but if there are, there's little harm in
358  * including them.
359  */
360  XLByteToSeg(startptr, startsegno);
361  XLogFileName(firstoff, ThisTimeLineID, startsegno);
362  XLByteToPrevSeg(endptr, endsegno);
363  XLogFileName(lastoff, ThisTimeLineID, endsegno);
364 
365  dir = AllocateDir("pg_wal");
366  if (!dir)
367  ereport(ERROR,
368  (errmsg("could not open directory \"%s\": %m", "pg_wal")));
369  while ((de = ReadDir(dir, "pg_wal")) != NULL)
370  {
371  /* Does it look like a WAL segment, and is it in the range? */
372  if (IsXLogFileName(de->d_name) &&
373  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
374  strcmp(de->d_name + 8, lastoff + 8) <= 0)
375  {
376  walFileList = lappend(walFileList, pstrdup(de->d_name));
377  }
378  /* Does it look like a timeline history file? */
379  else if (IsTLHistoryFileName(de->d_name))
380  {
381  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
382  }
383  }
384  FreeDir(dir);
385 
386  /*
387  * Before we go any further, check that none of the WAL segments we
388  * need were removed.
389  */
390  CheckXLogRemoved(startsegno, ThisTimeLineID);
391 
392  /*
393  * Put the WAL filenames into an array, and sort. We send the files in
394  * order from oldest to newest, to reduce the chance that a file is
395  * recycled before we get a chance to send it over.
396  */
397  nWalFiles = list_length(walFileList);
398  walFiles = palloc(nWalFiles * sizeof(char *));
399  i = 0;
400  foreach(lc, walFileList)
401  {
402  walFiles[i++] = lfirst(lc);
403  }
404  qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
405 
406  /*
407  * There must be at least one xlog file in the pg_wal directory,
408  * since we are doing backup-including-xlog.
409  */
410  if (nWalFiles < 1)
411  ereport(ERROR,
412  (errmsg("could not find any WAL files")));
413 
414  /*
415  * Sanity check: the first and last segment should cover startptr and
416  * endptr, with no gaps in between.
417  */
418  XLogFromFileName(walFiles[0], &tli, &segno);
419  if (segno != startsegno)
420  {
421  char startfname[MAXFNAMELEN];
422 
423  XLogFileName(startfname, ThisTimeLineID, startsegno);
424  ereport(ERROR,
425  (errmsg("could not find WAL file \"%s\"", startfname)));
426  }
427  for (i = 0; i < nWalFiles; i++)
428  {
429  XLogSegNo currsegno = segno;
430  XLogSegNo nextsegno = segno + 1;
431 
432  XLogFromFileName(walFiles[i], &tli, &segno);
433  if (!(nextsegno == segno || currsegno == segno))
434  {
435  char nextfname[MAXFNAMELEN];
436 
437  XLogFileName(nextfname, ThisTimeLineID, nextsegno);
438  ereport(ERROR,
439  (errmsg("could not find WAL file \"%s\"", nextfname)));
440  }
441  }
442  if (segno != endsegno)
443  {
444  char endfname[MAXFNAMELEN];
445 
446  XLogFileName(endfname, ThisTimeLineID, endsegno);
447  ereport(ERROR,
448  (errmsg("could not find WAL file \"%s\"", endfname)));
449  }
450 
451  /* Ok, we have everything we need. Send the WAL files. */
452  for (i = 0; i < nWalFiles; i++)
453  {
454  FILE *fp;
455  char buf[TAR_SEND_SIZE];
456  size_t cnt;
457  pgoff_t len = 0;
458 
459  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
460  XLogFromFileName(walFiles[i], &tli, &segno);
461 
462  fp = AllocateFile(pathbuf, "rb");
463  if (fp == NULL)
464  {
465  /*
466  * Most likely reason for this is that the file was already
467  * removed by a checkpoint, so check for that to get a better
468  * error message.
469  */
470  CheckXLogRemoved(segno, tli);
471 
472  ereport(ERROR,
474  errmsg("could not open file \"%s\": %m", pathbuf)));
475  }
476 
477  if (fstat(fileno(fp), &statbuf) != 0)
478  ereport(ERROR,
480  errmsg("could not stat file \"%s\": %m",
481  pathbuf)));
482  if (statbuf.st_size != XLogSegSize)
483  {
484  CheckXLogRemoved(segno, tli);
485  ereport(ERROR,
487  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
488  }
489 
490  /* send the WAL file itself */
491  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
492 
493  while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
494  {
495  CheckXLogRemoved(segno, tli);
496  /* Send the chunk as a CopyData message */
497  if (pq_putmessage('d', buf, cnt))
498  ereport(ERROR,
499  (errmsg("base backup could not send data, aborting backup")));
500 
501  len += cnt;
502  throttle(cnt);
503 
504  if (len == XLogSegSize)
505  break;
506  }
507 
508  if (len != XLogSegSize)
509  {
510  CheckXLogRemoved(segno, tli);
511  ereport(ERROR,
513  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
514  }
515 
516  /* XLogSegSize is a multiple of 512, so no need for padding */
517 
518  FreeFile(fp);
519 
520  /*
521  * Mark file as archived, otherwise files can get archived again
522  * after promotion of a new node. This is in line with
523  * walreceiver.c always doing an XLogArchiveForceDone() after a
524  * complete segment.
525  */
526  StatusFilePath(pathbuf, walFiles[i], ".done");
527  sendFileWithContent(pathbuf, "");
528  }
529 
530  /*
531  * Send timeline history files too. Only the latest timeline history
532  * file is required for recovery, and even that only if there happens
533  * to be a timeline switch in the first WAL segment that contains the
534  * checkpoint record, or if we're taking a base backup from a standby
535  * server and the target timeline changes while the backup is taken.
536  * But they are small and highly useful for debugging purposes, so
537  * better include them all, always.
538  */
539  foreach(lc, historyFileList)
540  {
541  char *fname = lfirst(lc);
542 
543  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
544 
545  if (lstat(pathbuf, &statbuf) != 0)
546  ereport(ERROR,
548  errmsg("could not stat file \"%s\": %m", pathbuf)));
549 
550  sendFile(pathbuf, pathbuf, &statbuf, false);
551 
552  /* unconditionally mark file as archived */
553  StatusFilePath(pathbuf, fname, ".done");
554  sendFileWithContent(pathbuf, "");
555  }
556 
557  /* Send CopyDone message for the last tar file */
558  pq_putemptymessage('c');
559  }
560  SendXlogRecPtrResult(endptr, endtli);
561 }
#define StatusFilePath(path, xlog, suffix)
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:957
#define NIL
Definition: pg_list.h:69
#define XLogSegSize
Definition: xlog_internal.h:92
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1202
static void throttle(size_t increment)
Definition: basebackup.c:1337
uint32 TimeLineID
Definition: xlogdefs.h:45
void pq_sendbyte(StringInfo buf, int byt)
Definition: pqformat.c:105
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1570
char * pstrdup(const char *in)
Definition: mcxt.c:1077
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:29
#define Min(x, y)
Definition: c.h:806
#define XLogFileName(fname, tli, logSegNo)
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:726
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:421
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
bool RecoveryInProgress(void)
Definition: xlog.c:7874
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
#define TABLESPACE_MAP
Definition: xlog.h:324
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:10674
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3766
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:66
uint64 XLogSegNo
Definition: xlogdefs.h:34
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
const char * label
Definition: basebackup.c:44
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:804
int64 sendTablespace(char *path, bool sizeonly)
Definition: basebackup.c:906
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2335
char * pgstat_stat_directory
Definition: pgstat.c:131
#define XLByteToPrevSeg(xlrp, logSegNo)
#define lnext(lc)
Definition: pg_list.h:105
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
List * lappend(List *list, void *datum)
Definition: list.c:128
#define MAXFNAMELEN
#define pgoff_t
Definition: win32.h:231
void * palloc0(Size size)
Definition: mcxt.c:878
#define XLOGDIR
uintptr_t Datum
Definition: postgres.h:372
#define XLOG_CONTROL_FILE
TimeLineID ThisTimeLineID
Definition: xlog.c:179
static TimestampTz throttled_last
Definition: basebackup.c:98
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:44
#define XLByteToSeg(xlrp, logSegNo)
#define NULL
Definition: c.h:229
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:675
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2401
#define THROTTLING_FREQUENCY
Definition: basebackup.c:86
static int list_length(const List *l)
Definition: pg_list.h:89
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
int FreeFile(FILE *file)
Definition: fd.c:2277
#define IsTLHistoryFileName(fname)
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
#define TAR_SEND_SIZE
Definition: basebackup.c:81
int i
char * DataDir
Definition: globals.c:59
#define BACKUP_LABEL_FILE
Definition: xlog.h:321
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:92
#define qsort(a, b, c, d)
Definition: port.h:440
#define XLogFromFileName(fname, tli, logSegNo)
static void base_backup_cleanup(int code, Datum arg)
Definition: basebackup.c:175
#define lstat(path, sb)
Definition: win32.h:262
Definition: pg_list.h:45
static uint64 throttling_sample
Definition: basebackup.c:89
static void sendFileWithContent(const char *filename, const char *content)
Definition: basebackup.c:859
int FreeDir(DIR *dir)
Definition: fd.c:2444
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, DIR *tblspcdir, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10167
static int compareWalFileNames(const void *a, const void *b)
Definition: basebackup.c:568
static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 716 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), and pq_sendint().

Referenced by SendBackupHeader().

717 {
718  char is[32];
719 
720  sprintf(is, INT64_FORMAT, intval);
721  pq_sendint(buf, strlen(is), 4);
722  pq_sendbytes(buf, is, strlen(is));
723 }
#define INT64_FORMAT
Definition: c.h:315
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
static void SendBackupHeader ( List tablespaces)
static

Definition at line 726 of file basebackup.c.

References buf, INT8OID, lfirst, NULL, tablespaceinfo::oid, OIDOID, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint(), pq_sendstring(), send_int8_string(), tablespaceinfo::size, and TEXTOID.

Referenced by perform_base_backup().

727 {
729  ListCell *lc;
730 
731  /* Construct and send the directory information */
732  pq_beginmessage(&buf, 'T'); /* RowDescription */
733  pq_sendint(&buf, 3, 2); /* 3 fields */
734 
735  /* First field - spcoid */
736  pq_sendstring(&buf, "spcoid");
737  pq_sendint(&buf, 0, 4); /* table oid */
738  pq_sendint(&buf, 0, 2); /* attnum */
739  pq_sendint(&buf, OIDOID, 4); /* type oid */
740  pq_sendint(&buf, 4, 2); /* typlen */
741  pq_sendint(&buf, 0, 4); /* typmod */
742  pq_sendint(&buf, 0, 2); /* format code */
743 
744  /* Second field - spcpath */
745  pq_sendstring(&buf, "spclocation");
746  pq_sendint(&buf, 0, 4);
747  pq_sendint(&buf, 0, 2);
748  pq_sendint(&buf, TEXTOID, 4);
749  pq_sendint(&buf, -1, 2);
750  pq_sendint(&buf, 0, 4);
751  pq_sendint(&buf, 0, 2);
752 
753  /* Third field - size */
754  pq_sendstring(&buf, "size");
755  pq_sendint(&buf, 0, 4);
756  pq_sendint(&buf, 0, 2);
757  pq_sendint(&buf, INT8OID, 4);
758  pq_sendint(&buf, 8, 2);
759  pq_sendint(&buf, 0, 4);
760  pq_sendint(&buf, 0, 2);
761  pq_endmessage(&buf);
762 
763  foreach(lc, tablespaces)
764  {
765  tablespaceinfo *ti = lfirst(lc);
766 
767  /* Send one datarow message */
768  pq_beginmessage(&buf, 'D');
769  pq_sendint(&buf, 3, 2); /* number of columns */
770  if (ti->path == NULL)
771  {
772  pq_sendint(&buf, -1, 4); /* Length = -1 ==> NULL */
773  pq_sendint(&buf, -1, 4);
774  }
775  else
776  {
777  Size len;
778 
779  len = strlen(ti->oid);
780  pq_sendint(&buf, len, 4);
781  pq_sendbytes(&buf, ti->oid, len);
782 
783  len = strlen(ti->path);
784  pq_sendint(&buf, len, 4);
785  pq_sendbytes(&buf, ti->path, len);
786  }
787  if (ti->size >= 0)
788  send_int8_string(&buf, ti->size / 1024);
789  else
790  pq_sendint(&buf, -1, 4); /* NULL */
791 
792  pq_endmessage(&buf);
793  }
794 
795  /* Send a CommandComplete message */
796  pq_puttextmessage('C', "SELECT");
797 }
#define OIDOID
Definition: pg_type.h:328
#define TEXTOID
Definition: pg_type.h:324
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:186
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:716
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
static char * buf
Definition: pg_test_fsync.c:66
#define INT8OID
Definition: pg_type.h:304
#define NULL
Definition: c.h:229
#define lfirst(lc)
Definition: pg_list.h:106
size_t Size
Definition: c.h:356
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:400
void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 686 of file basebackup.c.

References AllocateDir(), ereport, errmsg(), ERROR, FreeDir(), basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf(), update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

687 {
688  DIR *dir;
689  basebackup_options opt;
690 
691  parse_basebackup_options(cmd->options, &opt);
692 
694 
696  {
697  char activitymsg[50];
698 
699  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
700  opt.label);
701  set_ps_display(activitymsg, false);
702  }
703 
704  /* Make sure we can open the directory with tablespaces in it */
705  dir = AllocateDir("pg_tblspc");
706  if (!dir)
707  ereport(ERROR,
708  (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
709 
710  perform_base_backup(&opt, dir);
711 
712  FreeDir(dir);
713 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:35
void set_ps_display(const char *activity, bool force)
Definition: ps_status.c:326
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:580
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
Definition: basebackup.c:187
const char * label
Definition: basebackup.c:44
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2335
#define ereport(elevel, rest)
Definition: elog.h:122
void WalSndSetState(WalSndState state)
Definition: walsender.c:2959
int errmsg(const char *fmt,...)
Definition: elog.c:797
int FreeDir(DIR *dir)
Definition: fd.c:2444
static int64 sendDir ( char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks 
)
static

Definition at line 957 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), lfirst, lstat, MAXPGPATH, NULL, PG_TEMP_FILE_PREFIX, ReadDir(), RecoveryInProgress(), tablespaceinfo::rpath, sendFile(), snprintf(), statrelpath, and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

959 {
960  DIR *dir;
961  struct dirent *de;
962  char pathbuf[MAXPGPATH * 2];
963  struct stat statbuf;
964  int64 size = 0;
965 
966  dir = AllocateDir(path);
967  while ((de = ReadDir(dir, path)) != NULL)
968  {
969  int excludeIdx;
970  bool excludeFound;
971 
972  /* Skip special stuff */
973  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
974  continue;
975 
976  /* Skip temporary files */
977  if (strncmp(de->d_name,
979  strlen(PG_TEMP_FILE_PREFIX)) == 0)
980  continue;
981 
982  /*
983  * Check if the postmaster has signaled us to exit, and abort with an
984  * error in that case. The error handler further up will call
985  * do_pg_abort_backup() for us. Also check that if the backup was
986  * started while still in recovery, the server wasn't promoted.
987  * dp_pg_stop_backup() will check that too, but it's better to stop
988  * the backup early than continue to the end and fail there.
989  */
992  ereport(ERROR,
993  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
994  errmsg("the standby was promoted during online backup"),
995  errhint("This means that the backup being taken is corrupt "
996  "and should not be used. "
997  "Try taking another online backup.")));
998 
999  /* Scan for files that should be excluded */
1000  excludeFound = false;
1001  for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
1002  {
1003  if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1004  {
1005  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1006  excludeFound = true;
1007  break;
1008  }
1009  }
1010 
1011  if (excludeFound)
1012  continue;
1013 
1014  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1015 
1016  /* Skip pg_control here to back up it last */
1017  if (strcmp(pathbuf, "./global/pg_control") == 0)
1018  continue;
1019 
1020  if (lstat(pathbuf, &statbuf) != 0)
1021  {
1022  if (errno != ENOENT)
1023  ereport(ERROR,
1025  errmsg("could not stat file or directory \"%s\": %m",
1026  pathbuf)));
1027 
1028  /* If the file went away while scanning, it's not an error. */
1029  continue;
1030  }
1031 
1032  /* Scan for directories whose contents should be excluded */
1033  excludeFound = false;
1034  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1035  {
1036  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1037  {
1038  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1039  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1040  excludeFound = true;
1041  break;
1042  }
1043  }
1044 
1045  if (excludeFound)
1046  continue;
1047 
1048  /*
1049  * Exclude contents of directory specified by statrelpath if not set
1050  * to the default (pg_stat_tmp) which is caught in the loop above.
1051  */
1052  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1053  {
1054  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1055  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1056  continue;
1057  }
1058 
1059  /*
1060  * We can skip pg_wal, the WAL segments need to be fetched from the
1061  * WAL archive anyway. But include it as an empty directory anyway, so
1062  * we get permissions right.
1063  */
1064  if (strcmp(pathbuf, "./pg_wal") == 0)
1065  {
1066  /* If pg_wal is a symlink, write it as a directory anyway */
1067  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1068 
1069  /*
1070  * Also send archive_status directory (by hackishly reusing
1071  * statbuf from above ...).
1072  */
1073  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1074  sizeonly);
1075 
1076  continue; /* don't recurse into pg_wal */
1077  }
1078 
1079  /* Allow symbolic links in pg_tblspc only */
1080  if (strcmp(path, "./pg_tblspc") == 0 &&
1081 #ifndef WIN32
1082  S_ISLNK(statbuf.st_mode)
1083 #else
1084  pgwin32_is_junction(pathbuf)
1085 #endif
1086  )
1087  {
1088 #if defined(HAVE_READLINK) || defined(WIN32)
1089  char linkpath[MAXPGPATH];
1090  int rllen;
1091 
1092  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1093  if (rllen < 0)
1094  ereport(ERROR,
1096  errmsg("could not read symbolic link \"%s\": %m",
1097  pathbuf)));
1098  if (rllen >= sizeof(linkpath))
1099  ereport(ERROR,
1100  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1101  errmsg("symbolic link \"%s\" target is too long",
1102  pathbuf)));
1103  linkpath[rllen] = '\0';
1104 
1105  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1106  &statbuf, sizeonly);
1107 #else
1108 
1109  /*
1110  * If the platform does not have symbolic links, it should not be
1111  * possible to have tablespaces - clearly somebody else created
1112  * them. Warn about it and ignore.
1113  */
1114  ereport(WARNING,
1115  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1116  errmsg("tablespaces are not supported on this platform")));
1117  continue;
1118 #endif /* HAVE_READLINK */
1119  }
1120  else if (S_ISDIR(statbuf.st_mode))
1121  {
1122  bool skip_this_dir = false;
1123  ListCell *lc;
1124 
1125  /*
1126  * Store a directory entry in the tar file so we can get the
1127  * permissions right.
1128  */
1129  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1130  sizeonly);
1131 
1132  /*
1133  * Call ourselves recursively for a directory, unless it happens
1134  * to be a separate tablespace located within PGDATA.
1135  */
1136  foreach(lc, tablespaces)
1137  {
1138  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1139 
1140  /*
1141  * ti->rpath is the tablespace relative path within PGDATA, or
1142  * NULL if the tablespace has been properly located somewhere
1143  * else.
1144  *
1145  * Skip past the leading "./" in pathbuf when comparing.
1146  */
1147  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1148  {
1149  skip_this_dir = true;
1150  break;
1151  }
1152  }
1153 
1154  /*
1155  * skip sending directories inside pg_tblspc, if not required.
1156  */
1157  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1158  skip_this_dir = true;
1159 
1160  if (!skip_this_dir)
1161  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1162  }
1163  else if (S_ISREG(statbuf.st_mode))
1164  {
1165  bool sent = false;
1166 
1167  if (!sizeonly)
1168  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1169  true);
1170 
1171  if (sent || sizeonly)
1172  {
1173  /* Add size, rounded up to 512byte block */
1174  size += ((statbuf.st_size + 511) & ~511);
1175  size += 512; /* Size of the header of the file */
1176  }
1177  }
1178  else
1179  ereport(WARNING,
1180  (errmsg("skipping special file \"%s\"", pathbuf)));
1181  }
1182  FreeDir(dir);
1183  return size;
1184 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:957
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1202
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:987
char * rpath
Definition: basebackup.h:28
int errcode(int sqlerrcode)
Definition: elog.c:575
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define PG_TEMP_FILE_PREFIX
Definition: fd.h:128
bool RecoveryInProgress(void)
Definition: xlog.c:7874
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2335
static const char * excludeFiles[]
Definition: basebackup.c:146
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
#define WARNING
Definition: elog.h:40
#define NULL
Definition: c.h:229
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2401
int errmsg(const char *fmt,...)
Definition: elog.c:797
static const char * excludeDirContents[]
Definition: basebackup.c:105
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
char d_name[MAX_PATH]
Definition: dirent.h:14
#define elog
Definition: elog.h:219
#define lstat(path, sb)
Definition: win32.h:262
int FreeDir(DIR *dir)
Definition: fd.c:2444
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1317
static bool sendFile ( char *  readfilename,
char *  tarfilename,
struct stat *  statbuf,
bool  missing_ok 
)
static

Definition at line 1202 of file basebackup.c.

References _tarWriteHeader(), AllocateFile(), buf, ereport, errcode_for_file_access(), errmsg(), ERROR, FreeFile(), MemSet, Min, NULL, pgoff_t, pq_putmessage, TAR_SEND_SIZE, and throttle().

Referenced by perform_base_backup(), and sendDir().

1204 {
1205  FILE *fp;
1206  char buf[TAR_SEND_SIZE];
1207  size_t cnt;
1208  pgoff_t len = 0;
1209  size_t pad;
1210 
1211  fp = AllocateFile(readfilename, "rb");
1212  if (fp == NULL)
1213  {
1214  if (errno == ENOENT && missing_ok)
1215  return false;
1216  ereport(ERROR,
1218  errmsg("could not open file \"%s\": %m", readfilename)));
1219  }
1220 
1221  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1222 
1223  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1224  {
1225  /* Send the chunk as a CopyData message */
1226  if (pq_putmessage('d', buf, cnt))
1227  ereport(ERROR,
1228  (errmsg("base backup could not send data, aborting backup")));
1229 
1230  len += cnt;
1231  throttle(cnt);
1232 
1233  if (len >= statbuf->st_size)
1234  {
1235  /*
1236  * Reached end of file. The file could be longer, if it was
1237  * extended while we were sending it, but for a base backup we can
1238  * ignore such extended data. It will be restored from WAL.
1239  */
1240  break;
1241  }
1242  }
1243 
1244  /* If the file was truncated while we were sending it, pad it with zeros */
1245  if (len < statbuf->st_size)
1246  {
1247  MemSet(buf, 0, sizeof(buf));
1248  while (len < statbuf->st_size)
1249  {
1250  cnt = Min(sizeof(buf), statbuf->st_size - len);
1251  pq_putmessage('d', buf, cnt);
1252  len += cnt;
1253  throttle(cnt);
1254  }
1255  }
1256 
1257  /*
1258  * Pad to 512 byte boundary, per tar format requirements. (This small
1259  * piece of data is probably not worth throttling.)
1260  */
1261  pad = ((len + 511) & ~511) - len;
1262  if (pad > 0)
1263  {
1264  MemSet(buf, 0, pad);
1265  pq_putmessage('d', buf, pad);
1266  }
1267 
1268  FreeFile(fp);
1269 
1270  return true;
1271 }
static void throttle(size_t increment)
Definition: basebackup.c:1337
#define Min(x, y)
Definition: c.h:806
#define MemSet(start, val, len)
Definition: c.h:857
#define ERROR
Definition: elog.h:43
static char * buf
Definition: pg_test_fsync.c:66
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
#define ereport(elevel, rest)
Definition: elog.h:122
#define pgoff_t
Definition: win32.h:231
#define NULL
Definition: c.h:229
int FreeFile(FILE *file)
Definition: fd.c:2277
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define TAR_SEND_SIZE
Definition: basebackup.c:81
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static void sendFileWithContent ( const char *  filename,
const char *  content 
)
static

Definition at line 859 of file basebackup.c.

References _tarWriteHeader(), buf, MemSet, NULL, and pq_putmessage.

Referenced by perform_base_backup().

860 {
861  struct stat statbuf;
862  int pad,
863  len;
864 
865  len = strlen(content);
866 
867  /*
868  * Construct a stat struct for the backup_label file we're injecting in
869  * the tar.
870  */
871  /* Windows doesn't have the concept of uid and gid */
872 #ifdef WIN32
873  statbuf.st_uid = 0;
874  statbuf.st_gid = 0;
875 #else
876  statbuf.st_uid = geteuid();
877  statbuf.st_gid = getegid();
878 #endif
879  statbuf.st_mtime = time(NULL);
880  statbuf.st_mode = S_IRUSR | S_IWUSR;
881  statbuf.st_size = len;
882 
883  _tarWriteHeader(filename, NULL, &statbuf, false);
884  /* Send the contents as a CopyData message */
885  pq_putmessage('d', content, len);
886 
887  /* Pad to 512 byte boundary, per tar format requirements */
888  pad = ((len + 511) & ~511) - len;
889  if (pad > 0)
890  {
891  char buf[512];
892 
893  MemSet(buf, 0, pad);
894  pq_putmessage('d', buf, pad);
895  }
896 }
#define MemSet(start, val, len)
Definition: c.h:857
static char * buf
Definition: pg_test_fsync.c:66
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
#define NULL
Definition: c.h:229
static char * filename
Definition: pg_dumpall.c:87
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
int64 sendTablespace ( char *  path,
bool  sizeonly 
)

Definition at line 906 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, NULL, sendDir(), snprintf(), and TABLESPACE_VERSION_DIRECTORY.

Referenced by do_pg_start_backup(), and perform_base_backup().

907 {
908  int64 size;
909  char pathbuf[MAXPGPATH];
910  struct stat statbuf;
911 
912  /*
913  * 'path' points to the tablespace location, but we only want to include
914  * the version directory in it that belongs to us.
915  */
916  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
918 
919  /*
920  * Store a directory entry in the tar file so we get the permissions
921  * right.
922  */
923  if (lstat(pathbuf, &statbuf) != 0)
924  {
925  if (errno != ENOENT)
926  ereport(ERROR,
928  errmsg("could not stat file or directory \"%s\": %m",
929  pathbuf)));
930 
931  /* If the tablespace went away while scanning, it's no error. */
932  return 0;
933  }
934 
936  sizeonly);
937 
938  /* Send all the files in the tablespace version directory */
939  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
940 
941  return size;
942 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:957
#define NIL
Definition: pg_list.h:69
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1275
#define ereport(elevel, rest)
Definition: elog.h:122
#define NULL
Definition: c.h:229
#define TABLESPACE_VERSION_DIRECTORY
Definition: catalog.h:26
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define lstat(path, sb)
Definition: win32.h:262
static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 804 of file basebackup.c.

References buf, INT8OID, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint(), pq_sendstring(), snprintf(), and TEXTOID.

Referenced by perform_base_backup().

805 {
807  char str[MAXFNAMELEN];
808  Size len;
809 
810  pq_beginmessage(&buf, 'T'); /* RowDescription */
811  pq_sendint(&buf, 2, 2); /* 2 fields */
812 
813  /* Field headers */
814  pq_sendstring(&buf, "recptr");
815  pq_sendint(&buf, 0, 4); /* table oid */
816  pq_sendint(&buf, 0, 2); /* attnum */
817  pq_sendint(&buf, TEXTOID, 4); /* type oid */
818  pq_sendint(&buf, -1, 2);
819  pq_sendint(&buf, 0, 4);
820  pq_sendint(&buf, 0, 2);
821 
822  pq_sendstring(&buf, "tli");
823  pq_sendint(&buf, 0, 4); /* table oid */
824  pq_sendint(&buf, 0, 2); /* attnum */
825 
826  /*
827  * int8 may seem like a surprising data type for this, but in theory int4
828  * would not be wide enough for this, as TimeLineID is unsigned.
829  */
830  pq_sendint(&buf, INT8OID, 4); /* type oid */
831  pq_sendint(&buf, -1, 2);
832  pq_sendint(&buf, 0, 4);
833  pq_sendint(&buf, 0, 2);
834  pq_endmessage(&buf);
835 
836  /* Data row */
837  pq_beginmessage(&buf, 'D');
838  pq_sendint(&buf, 2, 2); /* number of columns */
839 
840  len = snprintf(str, sizeof(str),
841  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
842  pq_sendint(&buf, len, 4);
843  pq_sendbytes(&buf, str, len);
844 
845  len = snprintf(str, sizeof(str), "%u", tli);
846  pq_sendint(&buf, len, 4);
847  pq_sendbytes(&buf, str, len);
848 
849  pq_endmessage(&buf);
850 
851  /* Send a CommandComplete message */
852  pq_puttextmessage('C', "SELECT");
853 }
#define TEXTOID
Definition: pg_type.h:324
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:186
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
static char * buf
Definition: pg_test_fsync.c:66
unsigned int uint32
Definition: c.h:268
#define MAXFNAMELEN
#define INT8OID
Definition: pg_type.h:304
size_t Size
Definition: c.h:356
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:400
static void throttle ( size_t  increment)
static

Definition at line 1337 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_LATCH_SET, WL_POSTMASTER_DEATH, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1338 {
1339  TimeOffset elapsed,
1340  elapsed_min,
1341  sleep;
1342  int wait_result;
1343 
1344  if (throttling_counter < 0)
1345  return;
1346 
1347  throttling_counter += increment;
1349  return;
1350 
1351  /* Time elapsed since the last measurement (and possible wake up). */
1352  elapsed = GetCurrentTimestamp() - throttled_last;
1353  /* How much should have elapsed at minimum? */
1355  sleep = elapsed_min - elapsed;
1356  /* Only sleep if the transfer is faster than it should be. */
1357  if (sleep > 0)
1358  {
1360 
1361  /* We're eating a potentially set latch, so check for interrupts */
1363 
1364  /*
1365  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1366  * the maximum time to sleep. Thus the cast to long is safe.
1367  */
1368  wait_result = WaitLatch(MyLatch,
1370  (long) (sleep / 1000),
1372 
1373  if (wait_result & WL_LATCH_SET)
1375  }
1376 
1377  /*
1378  * As we work with integers, only whole multiple of throttling_sample was
1379  * processed. The rest will be done during the next call of this function.
1380  */
1382 
1383  /*
1384  * Time interval for the remaining amount and possible next increments
1385  * starts now.
1386  */
1388 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1570
void ResetLatch(volatile Latch *latch)
Definition: latch.c:498
int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:336
int64 TimeOffset
Definition: timestamp.h:40
#define WL_POSTMASTER_DEATH
Definition: latch.h:128
static TimestampTz throttled_last
Definition: basebackup.c:98
struct Latch * MyLatch
Definition: globals.c:51
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
static int64 throttling_counter
Definition: basebackup.c:92
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:89
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95

Variable Documentation

bool backup_started_in_recovery = false
static

Definition at line 73 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

TimeOffset elapsed_min_unit
static

Definition at line 95 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

const char* excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define NULL
Definition: c.h:229
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 105 of file basebackup.c.

Referenced by sendDir().

const char* excludeFiles[]
static
Initial value:
=
{
"postmaster.pid",
"postmaster.opts",
}
#define TABLESPACE_MAP
Definition: xlog.h:324
#define NULL
Definition: c.h:229
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:95
#define BACKUP_LABEL_FILE
Definition: xlog.h:321

Definition at line 146 of file basebackup.c.

Referenced by sendDir().

char* statrelpath = NULL
static

Definition at line 76 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

TimestampTz throttled_last
static

Definition at line 98 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

int64 throttling_counter
static

Definition at line 92 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

uint64 throttling_sample
static

Definition at line 89 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().