PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
basebackup.c File Reference
#include "postgres.h"
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "access/xlog_internal.h"
#include "catalog/catalog.h"
#include "catalog/pg_type.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "miscadmin.h"
#include "nodes/pg_list.h"
#include "pgtar.h"
#include "pgstat.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "utils/builtins.h"
#include "utils/elog.h"
#include "utils/ps_status.h"
#include "utils/timestamp.h"
Include dependency graph for basebackup.c:

Go to the source code of this file.

Data Structures

struct  basebackup_options
 

Macros

#define TAR_SEND_SIZE   32768
 
#define THROTTLING_FREQUENCY   8
 

Functions

static int64 sendDir (char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
 
static bool sendFile (char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
 
static void sendFileWithContent (const char *filename, const char *content)
 
static int64 _tarWriteHeader (const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
 
static int64 _tarWriteDir (const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
 
static void send_int8_string (StringInfoData *buf, int64 intval)
 
static void SendBackupHeader (List *tablespaces)
 
static void base_backup_cleanup (int code, Datum arg)
 
static void perform_base_backup (basebackup_options *opt, DIR *tblspcdir)
 
static void parse_basebackup_options (List *options, basebackup_options *opt)
 
static void SendXlogRecPtrResult (XLogRecPtr ptr, TimeLineID tli)
 
static int compareWalFileNames (const void *a, const void *b)
 
static void throttle (size_t increment)
 
void SendBaseBackup (BaseBackupCmd *cmd)
 
int64 sendTablespace (char *path, bool sizeonly)
 

Variables

static bool backup_started_in_recovery = false
 
static char * statrelpath = NULL
 
static uint64 throttling_sample
 
static int64 throttling_counter
 
static TimeOffset elapsed_min_unit
 
static TimestampTz throttled_last
 
static const char * excludeDirContents []
 
static const char * excludeFiles []
 

Macro Definition Documentation

#define TAR_SEND_SIZE   32768

Definition at line 81 of file basebackup.c.

Referenced by perform_base_backup(), and sendFile().

#define THROTTLING_FREQUENCY   8

Definition at line 86 of file basebackup.c.

Referenced by perform_base_backup().

Function Documentation

static int64 _tarWriteDir ( const char *  pathbuf,
int  basepathlen,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1321 of file basebackup.c.

References _tarWriteHeader().

Referenced by sendDir().

1323 {
1324  /* If symlink, write it as a directory anyway */
1325 #ifndef WIN32
1326  if (S_ISLNK(statbuf->st_mode))
1327 #else
1328  if (pgwin32_is_junction(pathbuf))
1329 #endif
1330  statbuf->st_mode = S_IFDIR | S_IRWXU;
1331 
1332  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1333 }
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
static int64 _tarWriteHeader ( const char *  filename,
const char *  linktarget,
struct stat *  statbuf,
bool  sizeonly 
)
static

Definition at line 1279 of file basebackup.c.

References elog, ereport, errmsg(), ERROR, pq_putmessage, TAR_NAME_TOO_LONG, TAR_OK, TAR_SYMLINK_TOO_LONG, and tarCreateHeader().

Referenced by _tarWriteDir(), perform_base_backup(), sendDir(), sendFile(), sendFileWithContent(), and sendTablespace().

1281 {
1282  char h[512];
1283  enum tarError rc;
1284 
1285  if (!sizeonly)
1286  {
1287  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1288  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1289  statbuf->st_mtime);
1290 
1291  switch (rc)
1292  {
1293  case TAR_OK:
1294  break;
1295  case TAR_NAME_TOO_LONG:
1296  ereport(ERROR,
1297  (errmsg("file name too long for tar format: \"%s\"",
1298  filename)));
1299  break;
1300  case TAR_SYMLINK_TOO_LONG:
1301  ereport(ERROR,
1302  (errmsg("symbolic link target too long for tar format: "
1303  "file name \"%s\", target \"%s\"",
1304  filename, linktarget)));
1305  break;
1306  default:
1307  elog(ERROR, "unrecognized tar error: %d", rc);
1308  }
1309 
1310  pq_putmessage('d', h, sizeof(h));
1311  }
1312 
1313  return sizeof(h);
1314 }
Definition: pgtar.h:17
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define ereport(elevel, rest)
Definition: elog.h:122
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define elog
Definition: elog.h:219
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static void base_backup_cleanup ( int  code,
Datum  arg 
)
static

Definition at line 175 of file basebackup.c.

References do_pg_abort_backup().

Referenced by perform_base_backup().

176 {
178 }
void do_pg_abort_backup(void)
Definition: xlog.c:11157
static int compareWalFileNames ( const void *  a,
const void *  b 
)
static

Definition at line 572 of file basebackup.c.

Referenced by perform_base_backup().

573 {
574  char *fna = *((char **) a);
575  char *fnb = *((char **) b);
576 
577  return strcmp(fna + 8, fnb + 8);
578 }
static void parse_basebackup_options ( List options,
basebackup_options opt 
)
static

Definition at line 584 of file basebackup.c.

References DefElem::arg, DefElem::defname, elog, ereport, errcode(), errmsg(), ERROR, basebackup_options::fastcheckpoint, basebackup_options::includewal, intVal, basebackup_options::label, lfirst, MAX_RATE_LOWER, MAX_RATE_UPPER, basebackup_options::maxrate, maxrate, MemSet, basebackup_options::nowait, basebackup_options::progress, basebackup_options::sendtblspcmapfile, and strVal.

Referenced by SendBaseBackup().

585 {
586  ListCell *lopt;
587  bool o_label = false;
588  bool o_progress = false;
589  bool o_fast = false;
590  bool o_nowait = false;
591  bool o_wal = false;
592  bool o_maxrate = false;
593  bool o_tablespace_map = false;
594 
595  MemSet(opt, 0, sizeof(*opt));
596  foreach(lopt, options)
597  {
598  DefElem *defel = (DefElem *) lfirst(lopt);
599 
600  if (strcmp(defel->defname, "label") == 0)
601  {
602  if (o_label)
603  ereport(ERROR,
604  (errcode(ERRCODE_SYNTAX_ERROR),
605  errmsg("duplicate option \"%s\"", defel->defname)));
606  opt->label = strVal(defel->arg);
607  o_label = true;
608  }
609  else if (strcmp(defel->defname, "progress") == 0)
610  {
611  if (o_progress)
612  ereport(ERROR,
613  (errcode(ERRCODE_SYNTAX_ERROR),
614  errmsg("duplicate option \"%s\"", defel->defname)));
615  opt->progress = true;
616  o_progress = true;
617  }
618  else if (strcmp(defel->defname, "fast") == 0)
619  {
620  if (o_fast)
621  ereport(ERROR,
622  (errcode(ERRCODE_SYNTAX_ERROR),
623  errmsg("duplicate option \"%s\"", defel->defname)));
624  opt->fastcheckpoint = true;
625  o_fast = true;
626  }
627  else if (strcmp(defel->defname, "nowait") == 0)
628  {
629  if (o_nowait)
630  ereport(ERROR,
631  (errcode(ERRCODE_SYNTAX_ERROR),
632  errmsg("duplicate option \"%s\"", defel->defname)));
633  opt->nowait = true;
634  o_nowait = true;
635  }
636  else if (strcmp(defel->defname, "wal") == 0)
637  {
638  if (o_wal)
639  ereport(ERROR,
640  (errcode(ERRCODE_SYNTAX_ERROR),
641  errmsg("duplicate option \"%s\"", defel->defname)));
642  opt->includewal = true;
643  o_wal = true;
644  }
645  else if (strcmp(defel->defname, "max_rate") == 0)
646  {
647  long maxrate;
648 
649  if (o_maxrate)
650  ereport(ERROR,
651  (errcode(ERRCODE_SYNTAX_ERROR),
652  errmsg("duplicate option \"%s\"", defel->defname)));
653 
654  maxrate = intVal(defel->arg);
655  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
656  ereport(ERROR,
657  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
658  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
659  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
660 
661  opt->maxrate = (uint32) maxrate;
662  o_maxrate = true;
663  }
664  else if (strcmp(defel->defname, "tablespace_map") == 0)
665  {
666  if (o_tablespace_map)
667  ereport(ERROR,
668  (errcode(ERRCODE_SYNTAX_ERROR),
669  errmsg("duplicate option \"%s\"", defel->defname)));
670  opt->sendtblspcmapfile = true;
671  o_tablespace_map = true;
672  }
673  else
674  elog(ERROR, "option \"%s\" not recognized",
675  defel->defname);
676  }
677  if (opt->label == NULL)
678  opt->label = "base backup";
679 }
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
#define MemSet(start, val, len)
Definition: c.h:863
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define ERROR
Definition: elog.h:43
static int32 maxrate
Definition: pg_basebackup.c:93
const char * label
Definition: basebackup.c:44
unsigned int uint32
Definition: c.h:258
#define ereport(elevel, rest)
Definition: elog.h:122
Node * arg
Definition: parsenodes.h:720
#define lfirst(lc)
Definition: pg_list.h:106
#define intVal(v)
Definition: value.h:52
int errmsg(const char *fmt,...)
Definition: elog.c:797
char * defname
Definition: parsenodes.h:719
#define elog
Definition: elog.h:219
static void perform_base_backup ( basebackup_options opt,
DIR tblspcdir 
)
static

Definition at line 187 of file basebackup.c.

References _tarWriteHeader(), AllocateDir(), AllocateFile(), Assert, BACKUP_LABEL_FILE, backup_started_in_recovery, base_backup_cleanup(), buf, CheckXLogRemoved(), compareWalFileNames(), dirent::d_name, StringInfoData::data, DataDir, do_pg_start_backup(), do_pg_stop_backup(), elapsed_min_unit, ereport, errcode_for_file_access(), errmsg(), ERROR, basebackup_options::fastcheckpoint, FreeDir(), FreeFile(), GetCurrentTimestamp(), i, basebackup_options::includewal, is_absolute_path, IsTLHistoryFileName, IsXLogFileName, basebackup_options::label, lappend(), lfirst, list_length(), lnext, lstat, makeStringInfo(), MAXFNAMELEN, MAXPGPATH, basebackup_options::maxrate, Min, NIL, basebackup_options::nowait, palloc(), palloc0(), tablespaceinfo::path, PG_END_ENSURE_ERROR_CLEANUP, PG_ENSURE_ERROR_CLEANUP, pgoff_t, pgstat_stat_directory, pq_beginmessage(), pq_endmessage(), pq_putemptymessage(), pq_putmessage, pq_sendbyte(), pq_sendint16(), basebackup_options::progress, psprintf(), pstrdup(), qsort, ReadDir(), RecoveryInProgress(), SendBackupHeader(), sendDir(), sendFile(), sendFileWithContent(), sendTablespace(), basebackup_options::sendtblspcmapfile, SendXlogRecPtrResult(), tablespaceinfo::size, snprintf(), statrelpath, StatusFilePath, TABLESPACE_MAP, TAR_SEND_SIZE, tblspc_map_file, ThisTimeLineID, throttle(), throttled_last, throttling_counter, THROTTLING_FREQUENCY, throttling_sample, USECS_PER_SEC, wal_segment_size, XLByteToPrevSeg, XLByteToSeg, XLOG_CONTROL_FILE, XLOGDIR, XLogFileName, and XLogFromFileName.

Referenced by SendBaseBackup().

188 {
189  XLogRecPtr startptr;
190  TimeLineID starttli;
191  XLogRecPtr endptr;
192  TimeLineID endtli;
193  StringInfo labelfile;
195  int datadirpathlen;
196  List *tablespaces = NIL;
197 
198  datadirpathlen = strlen(DataDir);
199 
201 
202  labelfile = makeStringInfo();
203  tblspc_map_file = makeStringInfo();
204 
205  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
206  labelfile, tblspcdir, &tablespaces,
207  tblspc_map_file,
208  opt->progress, opt->sendtblspcmapfile);
209 
210  /*
211  * Once do_pg_start_backup has been called, ensure that any failure causes
212  * us to abort the backup so we don't "leak" a backup counter. For this
213  * reason, *all* functionality between do_pg_start_backup() and
214  * do_pg_stop_backup() should be inside the error cleanup block!
215  */
216 
218  {
219  ListCell *lc;
220  tablespaceinfo *ti;
221 
222  SendXlogRecPtrResult(startptr, starttli);
223 
224  /*
225  * Calculate the relative path of temporary statistics directory in
226  * order to skip the files which are located in that directory later.
227  */
229  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
230  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
231  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
233  else
235 
236  /* Add a node for the base directory at the end */
237  ti = palloc0(sizeof(tablespaceinfo));
238  ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
239  tablespaces = lappend(tablespaces, ti);
240 
241  /* Send tablespace header */
242  SendBackupHeader(tablespaces);
243 
244  /* Setup and activate network throttling, if client requested it */
245  if (opt->maxrate > 0)
246  {
248  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
249 
250  /*
251  * The minimum amount of time for throttling_sample bytes to be
252  * transferred.
253  */
255 
256  /* Enable throttling. */
257  throttling_counter = 0;
258 
259  /* The 'real data' starts now (header was ignored). */
261  }
262  else
263  {
264  /* Disable throttling. */
265  throttling_counter = -1;
266  }
267 
268  /* Send off our tablespaces one by one */
269  foreach(lc, tablespaces)
270  {
271  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
273 
274  /* Send CopyOutResponse message */
275  pq_beginmessage(&buf, 'H');
276  pq_sendbyte(&buf, 0); /* overall format */
277  pq_sendint16(&buf, 0); /* natts */
278  pq_endmessage(&buf);
279 
280  if (ti->path == NULL)
281  {
282  struct stat statbuf;
283 
284  /* In the main tar, include the backup_label first... */
286 
287  /*
288  * Send tablespace_map file if required and then the bulk of
289  * the files.
290  */
291  if (tblspc_map_file && opt->sendtblspcmapfile)
292  {
293  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
294  sendDir(".", 1, false, tablespaces, false);
295  }
296  else
297  sendDir(".", 1, false, tablespaces, true);
298 
299  /* ... and pg_control after everything else. */
300  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
301  ereport(ERROR,
303  errmsg("could not stat control file \"%s\": %m",
305  sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false);
306  }
307  else
308  sendTablespace(ti->path, false);
309 
310  /*
311  * If we're including WAL, and this is the main data directory we
312  * don't terminate the tar stream here. Instead, we will append
313  * the xlog files below and terminate it then. This is safe since
314  * the main data directory is always sent *last*.
315  */
316  if (opt->includewal && ti->path == NULL)
317  {
318  Assert(lnext(lc) == NULL);
319  }
320  else
321  pq_putemptymessage('c'); /* CopyDone */
322  }
323  }
325 
326  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
327 
328  if (opt->includewal)
329  {
330  /*
331  * We've left the last tar file "open", so we can now append the
332  * required WAL files to it.
333  */
334  char pathbuf[MAXPGPATH];
335  XLogSegNo segno;
336  XLogSegNo startsegno;
337  XLogSegNo endsegno;
338  struct stat statbuf;
339  List *historyFileList = NIL;
340  List *walFileList = NIL;
341  char **walFiles;
342  int nWalFiles;
343  char firstoff[MAXFNAMELEN];
344  char lastoff[MAXFNAMELEN];
345  DIR *dir;
346  struct dirent *de;
347  int i;
348  ListCell *lc;
349  TimeLineID tli;
350 
351  /*
352  * I'd rather not worry about timelines here, so scan pg_wal and
353  * include all WAL files in the range between 'startptr' and 'endptr',
354  * regardless of the timeline the file is stamped with. If there are
355  * some spurious WAL files belonging to timelines that don't belong in
356  * this server's history, they will be included too. Normally there
357  * shouldn't be such files, but if there are, there's little harm in
358  * including them.
359  */
360  XLByteToSeg(startptr, startsegno, wal_segment_size);
361  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
362  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
363  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
364 
365  dir = AllocateDir("pg_wal");
366  if (!dir)
367  ereport(ERROR,
368  (errmsg("could not open directory \"%s\": %m", "pg_wal")));
369  while ((de = ReadDir(dir, "pg_wal")) != NULL)
370  {
371  /* Does it look like a WAL segment, and is it in the range? */
372  if (IsXLogFileName(de->d_name) &&
373  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
374  strcmp(de->d_name + 8, lastoff + 8) <= 0)
375  {
376  walFileList = lappend(walFileList, pstrdup(de->d_name));
377  }
378  /* Does it look like a timeline history file? */
379  else if (IsTLHistoryFileName(de->d_name))
380  {
381  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
382  }
383  }
384  FreeDir(dir);
385 
386  /*
387  * Before we go any further, check that none of the WAL segments we
388  * need were removed.
389  */
390  CheckXLogRemoved(startsegno, ThisTimeLineID);
391 
392  /*
393  * Put the WAL filenames into an array, and sort. We send the files in
394  * order from oldest to newest, to reduce the chance that a file is
395  * recycled before we get a chance to send it over.
396  */
397  nWalFiles = list_length(walFileList);
398  walFiles = palloc(nWalFiles * sizeof(char *));
399  i = 0;
400  foreach(lc, walFileList)
401  {
402  walFiles[i++] = lfirst(lc);
403  }
404  qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
405 
406  /*
407  * There must be at least one xlog file in the pg_wal directory, since
408  * we are doing backup-including-xlog.
409  */
410  if (nWalFiles < 1)
411  ereport(ERROR,
412  (errmsg("could not find any WAL files")));
413 
414  /*
415  * Sanity check: the first and last segment should cover startptr and
416  * endptr, with no gaps in between.
417  */
418  XLogFromFileName(walFiles[0], &tli, &segno, wal_segment_size);
419  if (segno != startsegno)
420  {
421  char startfname[MAXFNAMELEN];
422 
423  XLogFileName(startfname, ThisTimeLineID, startsegno,
425  ereport(ERROR,
426  (errmsg("could not find WAL file \"%s\"", startfname)));
427  }
428  for (i = 0; i < nWalFiles; i++)
429  {
430  XLogSegNo currsegno = segno;
431  XLogSegNo nextsegno = segno + 1;
432 
433  XLogFromFileName(walFiles[i], &tli, &segno, wal_segment_size);
434  if (!(nextsegno == segno || currsegno == segno))
435  {
436  char nextfname[MAXFNAMELEN];
437 
438  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
440  ereport(ERROR,
441  (errmsg("could not find WAL file \"%s\"", nextfname)));
442  }
443  }
444  if (segno != endsegno)
445  {
446  char endfname[MAXFNAMELEN];
447 
448  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
449  ereport(ERROR,
450  (errmsg("could not find WAL file \"%s\"", endfname)));
451  }
452 
453  /* Ok, we have everything we need. Send the WAL files. */
454  for (i = 0; i < nWalFiles; i++)
455  {
456  FILE *fp;
457  char buf[TAR_SEND_SIZE];
458  size_t cnt;
459  pgoff_t len = 0;
460 
461  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
462  XLogFromFileName(walFiles[i], &tli, &segno, wal_segment_size);
463 
464  fp = AllocateFile(pathbuf, "rb");
465  if (fp == NULL)
466  {
467  /*
468  * Most likely reason for this is that the file was already
469  * removed by a checkpoint, so check for that to get a better
470  * error message.
471  */
472  CheckXLogRemoved(segno, tli);
473 
474  ereport(ERROR,
476  errmsg("could not open file \"%s\": %m", pathbuf)));
477  }
478 
479  if (fstat(fileno(fp), &statbuf) != 0)
480  ereport(ERROR,
482  errmsg("could not stat file \"%s\": %m",
483  pathbuf)));
484  if (statbuf.st_size != wal_segment_size)
485  {
486  CheckXLogRemoved(segno, tli);
487  ereport(ERROR,
489  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
490  }
491 
492  /* send the WAL file itself */
493  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
494 
495  while ((cnt = fread(buf, 1,
496  Min(sizeof(buf), wal_segment_size - len),
497  fp)) > 0)
498  {
499  CheckXLogRemoved(segno, tli);
500  /* Send the chunk as a CopyData message */
501  if (pq_putmessage('d', buf, cnt))
502  ereport(ERROR,
503  (errmsg("base backup could not send data, aborting backup")));
504 
505  len += cnt;
506  throttle(cnt);
507 
508  if (len == wal_segment_size)
509  break;
510  }
511 
512  if (len != wal_segment_size)
513  {
514  CheckXLogRemoved(segno, tli);
515  ereport(ERROR,
517  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
518  }
519 
520  /* wal_segment_size is a multiple of 512, so no need for padding */
521 
522  FreeFile(fp);
523 
524  /*
525  * Mark file as archived, otherwise files can get archived again
526  * after promotion of a new node. This is in line with
527  * walreceiver.c always doing an XLogArchiveForceDone() after a
528  * complete segment.
529  */
530  StatusFilePath(pathbuf, walFiles[i], ".done");
531  sendFileWithContent(pathbuf, "");
532  }
533 
534  /*
535  * Send timeline history files too. Only the latest timeline history
536  * file is required for recovery, and even that only if there happens
537  * to be a timeline switch in the first WAL segment that contains the
538  * checkpoint record, or if we're taking a base backup from a standby
539  * server and the target timeline changes while the backup is taken.
540  * But they are small and highly useful for debugging purposes, so
541  * better include them all, always.
542  */
543  foreach(lc, historyFileList)
544  {
545  char *fname = lfirst(lc);
546 
547  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
548 
549  if (lstat(pathbuf, &statbuf) != 0)
550  ereport(ERROR,
552  errmsg("could not stat file \"%s\": %m", pathbuf)));
553 
554  sendFile(pathbuf, pathbuf, &statbuf, false);
555 
556  /* unconditionally mark file as archived */
557  StatusFilePath(pathbuf, fname, ".done");
558  sendFileWithContent(pathbuf, "");
559  }
560 
561  /* Send CopyDone message for the last tar file */
562  pq_putemptymessage('c');
563  }
564  SendXlogRecPtrResult(endptr, endtli);
565 }
#define StatusFilePath(path, xlog, suffix)
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:961
#define NIL
Definition: pg_list.h:69
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1206
static void throttle(size_t increment)
Definition: basebackup.c:1341
uint32 TimeLineID
Definition: xlogdefs.h:45
int wal_segment_size
Definition: xlog.c:113
#define USECS_PER_SEC
Definition: timestamp.h:94
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1570
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
char * pstrdup(const char *in)
Definition: mcxt.c:1076
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
#define Min(x, y)
Definition: c.h:812
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:730
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
static void pq_sendbyte(StringInfo buf, int8 byt)
Definition: pqformat.h:164
bool RecoveryInProgress(void)
Definition: xlog.c:7954
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
#define TABLESPACE_MAP
Definition: xlog.h:325
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:10770
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3779
#define MAXPGPATH
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:34
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
const char * label
Definition: basebackup.c:44
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2117
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:808
int64 sendTablespace(char *path, bool sizeonly)
Definition: basebackup.c:910
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2367
char * pgstat_stat_directory
Definition: pgstat.c:134
#define lnext(lc)
Definition: pg_list.h:105
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
List * lappend(List *list, void *datum)
Definition: list.c:128
#define MAXFNAMELEN
#define pgoff_t
Definition: win32.h:231
void * palloc0(Size size)
Definition: mcxt.c:877
#define XLOGDIR
uintptr_t Datum
Definition: postgres.h:372
#define XLOG_CONTROL_FILE
TimeLineID ThisTimeLineID
Definition: xlog.c:181
static TimestampTz throttled_last
Definition: basebackup.c:98
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:44
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:681
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2433
#define THROTTLING_FREQUENCY
Definition: basebackup.c:86
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static int list_length(const List *l)
Definition: pg_list.h:89
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
static void pq_sendint16(StringInfo buf, int16 i)
Definition: pqformat.h:140
int FreeFile(FILE *file)
Definition: fd.c:2309
#define IsTLHistoryFileName(fname)
void * palloc(Size size)
Definition: mcxt.c:848
int errmsg(const char *fmt,...)
Definition: elog.c:797
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:81
int i
char * DataDir
Definition: globals.c:60
#define BACKUP_LABEL_FILE
Definition: xlog.h:322
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:92
#define qsort(a, b, c, d)
Definition: port.h:447
static void base_backup_cleanup(int code, Datum arg)
Definition: basebackup.c:175
#define lstat(path, sb)
Definition: win32.h:262
Definition: pg_list.h:45
static uint64 throttling_sample
Definition: basebackup.c:89
static void sendFileWithContent(const char *filename, const char *content)
Definition: basebackup.c:863
int FreeDir(DIR *dir)
Definition: fd.c:2476
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, DIR *tblspcdir, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10262
static int compareWalFileNames(const void *a, const void *b)
Definition: basebackup.c:572
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
static void send_int8_string ( StringInfoData buf,
int64  intval 
)
static

Definition at line 720 of file basebackup.c.

References INT64_FORMAT, pq_sendbytes(), and pq_sendint32().

Referenced by SendBackupHeader().

721 {
722  char is[32];
723 
724  sprintf(is, INT64_FORMAT, intval);
725  pq_sendint32(buf, strlen(is));
726  pq_sendbytes(buf, is, strlen(is));
727 }
static void pq_sendint32(StringInfo buf, int32 i)
Definition: pqformat.h:148
#define INT64_FORMAT
Definition: c.h:300
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
static void SendBackupHeader ( List tablespaces)
static

Definition at line 730 of file basebackup.c.

References buf, INT8OID, lfirst, tablespaceinfo::oid, OIDOID, tablespaceinfo::path, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), send_int8_string(), tablespaceinfo::size, and TEXTOID.

Referenced by perform_base_backup().

731 {
733  ListCell *lc;
734 
735  /* Construct and send the directory information */
736  pq_beginmessage(&buf, 'T'); /* RowDescription */
737  pq_sendint16(&buf, 3); /* 3 fields */
738 
739  /* First field - spcoid */
740  pq_sendstring(&buf, "spcoid");
741  pq_sendint32(&buf, 0); /* table oid */
742  pq_sendint16(&buf, 0); /* attnum */
743  pq_sendint32(&buf, OIDOID); /* type oid */
744  pq_sendint16(&buf, 4); /* typlen */
745  pq_sendint32(&buf, 0); /* typmod */
746  pq_sendint16(&buf, 0); /* format code */
747 
748  /* Second field - spcpath */
749  pq_sendstring(&buf, "spclocation");
750  pq_sendint32(&buf, 0);
751  pq_sendint16(&buf, 0);
752  pq_sendint32(&buf, TEXTOID);
753  pq_sendint16(&buf, -1);
754  pq_sendint32(&buf, 0);
755  pq_sendint16(&buf, 0);
756 
757  /* Third field - size */
758  pq_sendstring(&buf, "size");
759  pq_sendint32(&buf, 0);
760  pq_sendint16(&buf, 0);
761  pq_sendint32(&buf, INT8OID);
762  pq_sendint16(&buf, 8);
763  pq_sendint32(&buf, 0);
764  pq_sendint16(&buf, 0);
765  pq_endmessage(&buf);
766 
767  foreach(lc, tablespaces)
768  {
769  tablespaceinfo *ti = lfirst(lc);
770 
771  /* Send one datarow message */
772  pq_beginmessage(&buf, 'D');
773  pq_sendint16(&buf, 3); /* number of columns */
774  if (ti->path == NULL)
775  {
776  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
777  pq_sendint32(&buf, -1);
778  }
779  else
780  {
781  Size len;
782 
783  len = strlen(ti->oid);
784  pq_sendint32(&buf, len);
785  pq_sendbytes(&buf, ti->oid, len);
786 
787  len = strlen(ti->path);
788  pq_sendint32(&buf, len);
789  pq_sendbytes(&buf, ti->path, len);
790  }
791  if (ti->size >= 0)
792  send_int8_string(&buf, ti->size / 1024);
793  else
794  pq_sendint32(&buf, -1); /* NULL */
795 
796  pq_endmessage(&buf);
797  }
798 
799  /* Send a CommandComplete message */
800  pq_puttextmessage('C', "SELECT");
801 }
#define OIDOID
Definition: pg_type.h:328
#define TEXTOID
Definition: pg_type.h:324
static void pq_sendint32(StringInfo buf, int32 i)
Definition: pqformat.h:148
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:720
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static char * buf
Definition: pg_test_fsync.c:67
#define INT8OID
Definition: pg_type.h:304
#define lfirst(lc)
Definition: pg_list.h:106
size_t Size
Definition: c.h:350
static void pq_sendint16(StringInfo buf, int16 i)
Definition: pqformat.h:140
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
void SendBaseBackup ( BaseBackupCmd cmd)

Definition at line 690 of file basebackup.c.

References AllocateDir(), ereport, errmsg(), ERROR, FreeDir(), basebackup_options::label, BaseBackupCmd::options, parse_basebackup_options(), perform_base_backup(), set_ps_display(), snprintf(), update_process_title, WalSndSetState(), and WALSNDSTATE_BACKUP.

Referenced by exec_replication_command().

691 {
692  DIR *dir;
693  basebackup_options opt;
694 
695  parse_basebackup_options(cmd->options, &opt);
696 
698 
700  {
701  char activitymsg[50];
702 
703  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
704  opt.label);
705  set_ps_display(activitymsg, false);
706  }
707 
708  /* Make sure we can open the directory with tablespaces in it */
709  dir = AllocateDir("pg_tblspc");
710  if (!dir)
711  ereport(ERROR,
712  (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
713 
714  perform_base_backup(&opt, dir);
715 
716  FreeDir(dir);
717 }
List * options
Definition: replnodes.h:44
bool update_process_title
Definition: ps_status.c:35
void set_ps_display(const char *activity, bool force)
Definition: ps_status.c:326
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:584
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
Definition: basebackup.c:187
const char * label
Definition: basebackup.c:44
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2367
#define ereport(elevel, rest)
Definition: elog.h:122
void WalSndSetState(WalSndState state)
Definition: walsender.c:3097
int errmsg(const char *fmt,...)
Definition: elog.c:797
int FreeDir(DIR *dir)
Definition: fd.c:2476
static int64 sendDir ( char *  path,
int  basepathlen,
bool  sizeonly,
List tablespaces,
bool  sendtblspclinks 
)
static

Definition at line 961 of file basebackup.c.

References _tarWriteDir(), _tarWriteHeader(), AllocateDir(), backup_started_in_recovery, CHECK_FOR_INTERRUPTS, dirent::d_name, DEBUG1, elog, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, excludeDirContents, excludeFiles, FreeDir(), lfirst, lstat, MAXPGPATH, PG_TEMP_FILE_PREFIX, ReadDir(), RecoveryInProgress(), tablespaceinfo::rpath, sendFile(), snprintf(), statrelpath, and WARNING.

Referenced by perform_base_backup(), and sendTablespace().

963 {
964  DIR *dir;
965  struct dirent *de;
966  char pathbuf[MAXPGPATH * 2];
967  struct stat statbuf;
968  int64 size = 0;
969 
970  dir = AllocateDir(path);
971  while ((de = ReadDir(dir, path)) != NULL)
972  {
973  int excludeIdx;
974  bool excludeFound;
975 
976  /* Skip special stuff */
977  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
978  continue;
979 
980  /* Skip temporary files */
981  if (strncmp(de->d_name,
983  strlen(PG_TEMP_FILE_PREFIX)) == 0)
984  continue;
985 
986  /*
987  * Check if the postmaster has signaled us to exit, and abort with an
988  * error in that case. The error handler further up will call
989  * do_pg_abort_backup() for us. Also check that if the backup was
990  * started while still in recovery, the server wasn't promoted.
991  * dp_pg_stop_backup() will check that too, but it's better to stop
992  * the backup early than continue to the end and fail there.
993  */
996  ereport(ERROR,
997  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
998  errmsg("the standby was promoted during online backup"),
999  errhint("This means that the backup being taken is corrupt "
1000  "and should not be used. "
1001  "Try taking another online backup.")));
1002 
1003  /* Scan for files that should be excluded */
1004  excludeFound = false;
1005  for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
1006  {
1007  if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1008  {
1009  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1010  excludeFound = true;
1011  break;
1012  }
1013  }
1014 
1015  if (excludeFound)
1016  continue;
1017 
1018  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1019 
1020  /* Skip pg_control here to back up it last */
1021  if (strcmp(pathbuf, "./global/pg_control") == 0)
1022  continue;
1023 
1024  if (lstat(pathbuf, &statbuf) != 0)
1025  {
1026  if (errno != ENOENT)
1027  ereport(ERROR,
1029  errmsg("could not stat file or directory \"%s\": %m",
1030  pathbuf)));
1031 
1032  /* If the file went away while scanning, it's not an error. */
1033  continue;
1034  }
1035 
1036  /* Scan for directories whose contents should be excluded */
1037  excludeFound = false;
1038  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1039  {
1040  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1041  {
1042  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1043  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1044  excludeFound = true;
1045  break;
1046  }
1047  }
1048 
1049  if (excludeFound)
1050  continue;
1051 
1052  /*
1053  * Exclude contents of directory specified by statrelpath if not set
1054  * to the default (pg_stat_tmp) which is caught in the loop above.
1055  */
1056  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1057  {
1058  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1059  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1060  continue;
1061  }
1062 
1063  /*
1064  * We can skip pg_wal, the WAL segments need to be fetched from the
1065  * WAL archive anyway. But include it as an empty directory anyway, so
1066  * we get permissions right.
1067  */
1068  if (strcmp(pathbuf, "./pg_wal") == 0)
1069  {
1070  /* If pg_wal is a symlink, write it as a directory anyway */
1071  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1072 
1073  /*
1074  * Also send archive_status directory (by hackishly reusing
1075  * statbuf from above ...).
1076  */
1077  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1078  sizeonly);
1079 
1080  continue; /* don't recurse into pg_wal */
1081  }
1082 
1083  /* Allow symbolic links in pg_tblspc only */
1084  if (strcmp(path, "./pg_tblspc") == 0 &&
1085 #ifndef WIN32
1086  S_ISLNK(statbuf.st_mode)
1087 #else
1088  pgwin32_is_junction(pathbuf)
1089 #endif
1090  )
1091  {
1092 #if defined(HAVE_READLINK) || defined(WIN32)
1093  char linkpath[MAXPGPATH];
1094  int rllen;
1095 
1096  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1097  if (rllen < 0)
1098  ereport(ERROR,
1100  errmsg("could not read symbolic link \"%s\": %m",
1101  pathbuf)));
1102  if (rllen >= sizeof(linkpath))
1103  ereport(ERROR,
1104  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1105  errmsg("symbolic link \"%s\" target is too long",
1106  pathbuf)));
1107  linkpath[rllen] = '\0';
1108 
1109  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1110  &statbuf, sizeonly);
1111 #else
1112 
1113  /*
1114  * If the platform does not have symbolic links, it should not be
1115  * possible to have tablespaces - clearly somebody else created
1116  * them. Warn about it and ignore.
1117  */
1118  ereport(WARNING,
1119  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1120  errmsg("tablespaces are not supported on this platform")));
1121  continue;
1122 #endif /* HAVE_READLINK */
1123  }
1124  else if (S_ISDIR(statbuf.st_mode))
1125  {
1126  bool skip_this_dir = false;
1127  ListCell *lc;
1128 
1129  /*
1130  * Store a directory entry in the tar file so we can get the
1131  * permissions right.
1132  */
1133  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1134  sizeonly);
1135 
1136  /*
1137  * Call ourselves recursively for a directory, unless it happens
1138  * to be a separate tablespace located within PGDATA.
1139  */
1140  foreach(lc, tablespaces)
1141  {
1142  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1143 
1144  /*
1145  * ti->rpath is the tablespace relative path within PGDATA, or
1146  * NULL if the tablespace has been properly located somewhere
1147  * else.
1148  *
1149  * Skip past the leading "./" in pathbuf when comparing.
1150  */
1151  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1152  {
1153  skip_this_dir = true;
1154  break;
1155  }
1156  }
1157 
1158  /*
1159  * skip sending directories inside pg_tblspc, if not required.
1160  */
1161  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1162  skip_this_dir = true;
1163 
1164  if (!skip_this_dir)
1165  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1166  }
1167  else if (S_ISREG(statbuf.st_mode))
1168  {
1169  bool sent = false;
1170 
1171  if (!sizeonly)
1172  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1173  true);
1174 
1175  if (sent || sizeonly)
1176  {
1177  /* Add size, rounded up to 512byte block */
1178  size += ((statbuf.st_size + 511) & ~511);
1179  size += 512; /* Size of the header of the file */
1180  }
1181  }
1182  else
1183  ereport(WARNING,
1184  (errmsg("skipping special file \"%s\"", pathbuf)));
1185  }
1186  FreeDir(dir);
1187  return size;
1188 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:961
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1206
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:987
char * rpath
Definition: basebackup.h:28
int errcode(int sqlerrcode)
Definition: elog.c:575
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define PG_TEMP_FILE_PREFIX
Definition: fd.h:129
bool RecoveryInProgress(void)
Definition: xlog.c:7954
static bool backup_started_in_recovery
Definition: basebackup.c:73
Definition: dirent.h:9
Definition: dirent.c:25
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2367
static const char * excludeFiles[]
Definition: basebackup.c:146
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
#define WARNING
Definition: elog.h:40
#define lfirst(lc)
Definition: pg_list.h:106
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2433
int errmsg(const char *fmt,...)
Definition: elog.c:797
static const char * excludeDirContents[]
Definition: basebackup.c:105
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
char d_name[MAX_PATH]
Definition: dirent.h:14
#define elog
Definition: elog.h:219
#define lstat(path, sb)
Definition: win32.h:262
int FreeDir(DIR *dir)
Definition: fd.c:2476
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1321
static bool sendFile ( char *  readfilename,
char *  tarfilename,
struct stat *  statbuf,
bool  missing_ok 
)
static

Definition at line 1206 of file basebackup.c.

References _tarWriteHeader(), AllocateFile(), buf, ereport, errcode_for_file_access(), errmsg(), ERROR, FreeFile(), MemSet, Min, pgoff_t, pq_putmessage, TAR_SEND_SIZE, and throttle().

Referenced by perform_base_backup(), and sendDir().

1208 {
1209  FILE *fp;
1210  char buf[TAR_SEND_SIZE];
1211  size_t cnt;
1212  pgoff_t len = 0;
1213  size_t pad;
1214 
1215  fp = AllocateFile(readfilename, "rb");
1216  if (fp == NULL)
1217  {
1218  if (errno == ENOENT && missing_ok)
1219  return false;
1220  ereport(ERROR,
1222  errmsg("could not open file \"%s\": %m", readfilename)));
1223  }
1224 
1225  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1226 
1227  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1228  {
1229  /* Send the chunk as a CopyData message */
1230  if (pq_putmessage('d', buf, cnt))
1231  ereport(ERROR,
1232  (errmsg("base backup could not send data, aborting backup")));
1233 
1234  len += cnt;
1235  throttle(cnt);
1236 
1237  if (len >= statbuf->st_size)
1238  {
1239  /*
1240  * Reached end of file. The file could be longer, if it was
1241  * extended while we were sending it, but for a base backup we can
1242  * ignore such extended data. It will be restored from WAL.
1243  */
1244  break;
1245  }
1246  }
1247 
1248  /* If the file was truncated while we were sending it, pad it with zeros */
1249  if (len < statbuf->st_size)
1250  {
1251  MemSet(buf, 0, sizeof(buf));
1252  while (len < statbuf->st_size)
1253  {
1254  cnt = Min(sizeof(buf), statbuf->st_size - len);
1255  pq_putmessage('d', buf, cnt);
1256  len += cnt;
1257  throttle(cnt);
1258  }
1259  }
1260 
1261  /*
1262  * Pad to 512 byte boundary, per tar format requirements. (This small
1263  * piece of data is probably not worth throttling.)
1264  */
1265  pad = ((len + 511) & ~511) - len;
1266  if (pad > 0)
1267  {
1268  MemSet(buf, 0, pad);
1269  pq_putmessage('d', buf, pad);
1270  }
1271 
1272  FreeFile(fp);
1273 
1274  return true;
1275 }
static void throttle(size_t increment)
Definition: basebackup.c:1341
#define Min(x, y)
Definition: c.h:812
#define MemSet(start, val, len)
Definition: c.h:863
#define ERROR
Definition: elog.h:43
static char * buf
Definition: pg_test_fsync.c:67
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2117
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
#define ereport(elevel, rest)
Definition: elog.h:122
#define pgoff_t
Definition: win32.h:231
int FreeFile(FILE *file)
Definition: fd.c:2309
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define TAR_SEND_SIZE
Definition: basebackup.c:81
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static void sendFileWithContent ( const char *  filename,
const char *  content 
)
static

Definition at line 863 of file basebackup.c.

References _tarWriteHeader(), buf, MemSet, and pq_putmessage.

Referenced by perform_base_backup().

864 {
865  struct stat statbuf;
866  int pad,
867  len;
868 
869  len = strlen(content);
870 
871  /*
872  * Construct a stat struct for the backup_label file we're injecting in
873  * the tar.
874  */
875  /* Windows doesn't have the concept of uid and gid */
876 #ifdef WIN32
877  statbuf.st_uid = 0;
878  statbuf.st_gid = 0;
879 #else
880  statbuf.st_uid = geteuid();
881  statbuf.st_gid = getegid();
882 #endif
883  statbuf.st_mtime = time(NULL);
884  statbuf.st_mode = S_IRUSR | S_IWUSR;
885  statbuf.st_size = len;
886 
887  _tarWriteHeader(filename, NULL, &statbuf, false);
888  /* Send the contents as a CopyData message */
889  pq_putmessage('d', content, len);
890 
891  /* Pad to 512 byte boundary, per tar format requirements */
892  pad = ((len + 511) & ~511) - len;
893  if (pad > 0)
894  {
895  char buf[512];
896 
897  MemSet(buf, 0, pad);
898  pq_putmessage('d', buf, pad);
899  }
900 }
#define MemSet(start, val, len)
Definition: c.h:863
static char * buf
Definition: pg_test_fsync.c:67
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
static char * filename
Definition: pg_dumpall.c:90
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
int64 sendTablespace ( char *  path,
bool  sizeonly 
)

Definition at line 910 of file basebackup.c.

References _tarWriteHeader(), ereport, errcode_for_file_access(), errmsg(), ERROR, lstat, MAXPGPATH, NIL, sendDir(), snprintf(), and TABLESPACE_VERSION_DIRECTORY.

Referenced by do_pg_start_backup(), and perform_base_backup().

911 {
912  int64 size;
913  char pathbuf[MAXPGPATH];
914  struct stat statbuf;
915 
916  /*
917  * 'path' points to the tablespace location, but we only want to include
918  * the version directory in it that belongs to us.
919  */
920  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
922 
923  /*
924  * Store a directory entry in the tar file so we get the permissions
925  * right.
926  */
927  if (lstat(pathbuf, &statbuf) != 0)
928  {
929  if (errno != ENOENT)
930  ereport(ERROR,
932  errmsg("could not stat file or directory \"%s\": %m",
933  pathbuf)));
934 
935  /* If the tablespace went away while scanning, it's no error. */
936  return 0;
937  }
938 
939  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
940  sizeonly);
941 
942  /* Send all the files in the tablespace version directory */
943  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
944 
945  return size;
946 }
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:961
#define NIL
Definition: pg_list.h:69
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define ERROR
Definition: elog.h:43
#define MAXPGPATH
int errcode_for_file_access(void)
Definition: elog.c:598
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1279
#define ereport(elevel, rest)
Definition: elog.h:122
#define TABLESPACE_VERSION_DIRECTORY
Definition: catalog.h:26
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define lstat(path, sb)
Definition: win32.h:262
static void SendXlogRecPtrResult ( XLogRecPtr  ptr,
TimeLineID  tli 
)
static

Definition at line 808 of file basebackup.c.

References buf, INT8OID, MAXFNAMELEN, pq_beginmessage(), pq_endmessage(), pq_puttextmessage(), pq_sendbytes(), pq_sendint16(), pq_sendint32(), pq_sendstring(), snprintf(), and TEXTOID.

Referenced by perform_base_backup().

809 {
811  char str[MAXFNAMELEN];
812  Size len;
813 
814  pq_beginmessage(&buf, 'T'); /* RowDescription */
815  pq_sendint16(&buf, 2); /* 2 fields */
816 
817  /* Field headers */
818  pq_sendstring(&buf, "recptr");
819  pq_sendint32(&buf, 0); /* table oid */
820  pq_sendint16(&buf, 0); /* attnum */
821  pq_sendint32(&buf, TEXTOID); /* type oid */
822  pq_sendint16(&buf, -1);
823  pq_sendint32(&buf, 0);
824  pq_sendint16(&buf, 0);
825 
826  pq_sendstring(&buf, "tli");
827  pq_sendint32(&buf, 0); /* table oid */
828  pq_sendint16(&buf, 0); /* attnum */
829 
830  /*
831  * int8 may seem like a surprising data type for this, but in theory int4
832  * would not be wide enough for this, as TimeLineID is unsigned.
833  */
834  pq_sendint32(&buf, INT8OID); /* type oid */
835  pq_sendint16(&buf, -1);
836  pq_sendint32(&buf, 0);
837  pq_sendint16(&buf, 0);
838  pq_endmessage(&buf);
839 
840  /* Data row */
841  pq_beginmessage(&buf, 'D');
842  pq_sendint16(&buf, 2); /* number of columns */
843 
844  len = snprintf(str, sizeof(str),
845  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
846  pq_sendint32(&buf, len);
847  pq_sendbytes(&buf, str, len);
848 
849  len = snprintf(str, sizeof(str), "%u", tli);
850  pq_sendint32(&buf, len);
851  pq_sendbytes(&buf, str, len);
852 
853  pq_endmessage(&buf);
854 
855  /* Send a CommandComplete message */
856  pq_puttextmessage('C', "SELECT");
857 }
#define TEXTOID
Definition: pg_type.h:324
static void pq_sendint32(StringInfo buf, int32 i)
Definition: pqformat.h:148
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:258
#define MAXFNAMELEN
#define INT8OID
Definition: pg_type.h:304
size_t Size
Definition: c.h:350
static void pq_sendint16(StringInfo buf, int16 i)
Definition: pqformat.h:140
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
static void throttle ( size_t  increment)
static

Definition at line 1341 of file basebackup.c.

References CHECK_FOR_INTERRUPTS, elapsed_min_unit, GetCurrentTimestamp(), MyLatch, ResetLatch(), throttled_last, throttling_counter, throttling_sample, WAIT_EVENT_BASE_BACKUP_THROTTLE, WaitLatch(), WL_LATCH_SET, WL_POSTMASTER_DEATH, and WL_TIMEOUT.

Referenced by perform_base_backup(), and sendFile().

1342 {
1343  TimeOffset elapsed_min;
1344 
1345  if (throttling_counter < 0)
1346  return;
1347 
1348  throttling_counter += increment;
1350  return;
1351 
1352  /* How much time should have elapsed at minimum? */
1353  elapsed_min = elapsed_min_unit *
1355 
1356  /*
1357  * Since the latch could be set repeatedly because of concurrently WAL
1358  * activity, sleep in a loop to ensure enough time has passed.
1359  */
1360  for (;;)
1361  {
1362  TimeOffset elapsed,
1363  sleep;
1364  int wait_result;
1365 
1366  /* Time elapsed since the last measurement (and possible wake up). */
1367  elapsed = GetCurrentTimestamp() - throttled_last;
1368 
1369  /* sleep if the transfer is faster than it should be */
1370  sleep = elapsed_min - elapsed;
1371  if (sleep <= 0)
1372  break;
1373 
1375 
1376  /* We're eating a potentially set latch, so check for interrupts */
1378 
1379  /*
1380  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1381  * the maximum time to sleep. Thus the cast to long is safe.
1382  */
1383  wait_result = WaitLatch(MyLatch,
1385  (long) (sleep / 1000),
1387 
1388  if (wait_result & WL_LATCH_SET)
1390 
1391  /* Done waiting? */
1392  if (wait_result & WL_TIMEOUT)
1393  break;
1394  }
1395 
1396  /*
1397  * As we work with integers, only whole multiple of throttling_sample was
1398  * processed. The rest will be done during the next call of this function.
1399  */
1401 
1402  /*
1403  * Time interval for the remaining amount and possible next increments
1404  * starts now.
1405  */
1407 }
#define WL_TIMEOUT
Definition: latch.h:127
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1570
void ResetLatch(volatile Latch *latch)
Definition: latch.c:497
int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:336
int64 TimeOffset
Definition: timestamp.h:40
#define WL_POSTMASTER_DEATH
Definition: latch.h:128
static TimestampTz throttled_last
Definition: basebackup.c:98
struct Latch * MyLatch
Definition: globals.c:52
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
static int64 throttling_counter
Definition: basebackup.c:92
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:89
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95

Variable Documentation

bool backup_started_in_recovery = false
static

Definition at line 73 of file basebackup.c.

Referenced by do_pg_start_backup(), do_pg_stop_backup(), perform_base_backup(), and sendDir().

TimeOffset elapsed_min_unit
static

Definition at line 95 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

const char* excludeDirContents[]
static
Initial value:
=
{
"pg_replslot",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34

Definition at line 105 of file basebackup.c.

Referenced by sendDir().

const char* excludeFiles[]
static
Initial value:
=
{
"postmaster.pid",
"postmaster.opts",
NULL
}
#define TABLESPACE_MAP
Definition: xlog.h:325
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:95
#define BACKUP_LABEL_FILE
Definition: xlog.h:322

Definition at line 146 of file basebackup.c.

Referenced by sendDir().

char* statrelpath = NULL
static

Definition at line 76 of file basebackup.c.

Referenced by perform_base_backup(), and sendDir().

TimestampTz throttled_last
static

Definition at line 98 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

int64 throttling_counter
static

Definition at line 92 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().

uint64 throttling_sample
static

Definition at line 89 of file basebackup.c.

Referenced by perform_base_backup(), and throttle().