PostgreSQL Source Code  git master
pg_archivecleanup.c
Go to the documentation of this file.
1 /*
2  * pg_archivecleanup.c
3  *
4  * To be used as archive_cleanup_command to clean an archive when using
5  * standby mode.
6  *
7  * src/bin/pg_archivecleanup/pg_archivecleanup.c
8  */
9 #include "postgres_fe.h"
10 
11 #include <ctype.h>
12 #include <dirent.h>
13 #include <sys/stat.h>
14 #include <fcntl.h>
15 #include <signal.h>
16 #include <sys/time.h>
17 
18 #include "access/xlog_internal.h"
19 #include "common/logging.h"
20 #include "getopt_long.h"
21 
22 const char *progname;
23 
24 /* Options and defaults */
25 bool dryrun = false; /* are we performing a dry-run operation? */
26 bool cleanBackupHistory = false; /* remove files including backup
27  * history files */
28 char *additional_ext = NULL; /* Extension to remove from filenames */
29 
30 char *archiveLocation; /* where to find the archive? */
31 char *restartWALFileName; /* the file from which we can restart restore */
32 char exclusiveCleanupFileName[MAXFNAMELEN]; /* the oldest file we want
33  * to remain in archive */
34 
35 
36 /* =====================================================================
37  *
38  * Customizable section
39  *
40  * =====================================================================
41  *
42  * Currently, this section assumes that the Archive is a locally
43  * accessible directory. If you want to make other assumptions,
44  * such as using a vendor-specific archive and access API, these
45  * routines are the ones you'll need to change. You're
46  * encouraged to submit any changes to pgsql-hackers@lists.postgresql.org
47  * or personally to the current maintainer. Those changes may be
48  * folded in to later versions of this program.
49  */
50 
51 /*
52  * Initialize allows customized commands into the archive cleanup program.
53  *
54  * You may wish to add code to check for tape libraries, etc..
55  */
56 static void
58 {
59  /*
60  * This code assumes that archiveLocation is a directory, so we use stat
61  * to test if it's accessible.
62  */
63  struct stat stat_buf;
64 
65  if (stat(archiveLocation, &stat_buf) != 0 ||
66  !S_ISDIR(stat_buf.st_mode))
67  {
68  pg_log_error("archive location \"%s\" does not exist",
70  exit(2);
71  }
72 }
73 
74 static void
75 TrimExtension(char *filename, char *extension)
76 {
77  int flen;
78  int elen;
79 
80  if (extension == NULL)
81  return;
82 
83  elen = strlen(extension);
84  flen = strlen(filename);
85 
86  if (flen > elen && strcmp(filename + flen - elen, extension) == 0)
87  filename[flen - elen] = '\0';
88 }
89 
90 static void
92 {
93  int rc;
94  DIR *xldir;
95  struct dirent *xlde;
96  char walfile[MAXPGPATH];
97 
98  xldir = opendir(archiveLocation);
99  if (xldir == NULL)
100  pg_fatal("could not open archive location \"%s\": %m",
102 
103  while (errno = 0, (xlde = readdir(xldir)) != NULL)
104  {
105  char WALFilePath[MAXPGPATH * 2]; /* the file path including
106  * archive */
107 
108  /*
109  * Truncation is essentially harmless, because we skip files whose
110  * format is different from WAL files and backup history files. (In
111  * principle, one could use a 1000-character additional_ext and get
112  * trouble.)
113  */
114  strlcpy(walfile, xlde->d_name, MAXPGPATH);
116 
117  /*
118  * Ignore anything does that not look like a WAL segment, a .partial
119  * WAL segment or a backup history file (if requested).
120  */
123  continue;
124 
125  /*
126  * We ignore the timeline part of the XLOG segment identifiers in
127  * deciding whether a segment is still needed. This ensures that we
128  * won't prematurely remove a segment from a parent timeline. We could
129  * probably be a little more proactive about removing segments of
130  * non-parent timelines, but that would be a whole lot more
131  * complicated.
132  *
133  * We use the alphanumeric sorting property of the filenames to decide
134  * which ones are earlier than the exclusiveCleanupFileName file. Note
135  * that this means files are not removed in the order they were
136  * originally written, in case this worries you.
137  */
138  if (strcmp(walfile + 8, exclusiveCleanupFileName + 8) >= 0)
139  continue;
140 
141  /*
142  * Use the original file name again now, including any extension that
143  * might have been chopped off before testing the sequence.
144  */
145  snprintf(WALFilePath, sizeof(WALFilePath), "%s/%s",
146  archiveLocation, xlde->d_name);
147 
148  if (dryrun)
149  {
150  /*
151  * Prints the name of the file to be removed and skips the actual
152  * removal. The regular printout is so that the user can pipe the
153  * output into some other program.
154  */
155  printf("%s\n", WALFilePath);
156  pg_log_debug("file \"%s\" would be removed", WALFilePath);
157  continue;
158  }
159 
160  pg_log_debug("removing file \"%s\"", WALFilePath);
161 
162  rc = unlink(WALFilePath);
163  if (rc != 0)
164  pg_fatal("could not remove file \"%s\": %m",
165  WALFilePath);
166  }
167 
168  if (errno)
169  pg_fatal("could not read archive location \"%s\": %m",
171  if (closedir(xldir))
172  pg_fatal("could not close archive location \"%s\": %m",
174 }
175 
176 /*
177  * SetWALFileNameForCleanup()
178  *
179  * Set the earliest WAL filename that we want to keep on the archive
180  * and decide whether we need cleanup
181  */
182 static void
184 {
185  bool fnameOK = false;
186 
188 
189  /*
190  * If restartWALFileName is a WAL file name then just use it directly. If
191  * restartWALFileName is a .partial or .backup filename, make sure we use
192  * the prefix of the filename, otherwise we will remove wrong files since
193  * 000000010000000000000010.partial and
194  * 000000010000000000000010.00000020.backup are after
195  * 000000010000000000000010.
196  */
198  {
200  fnameOK = true;
201  }
203  {
204  int args;
205  uint32 tli = 1,
206  log = 0,
207  seg = 0;
208 
209  args = sscanf(restartWALFileName, "%08X%08X%08X.partial",
210  &tli, &log, &seg);
211  if (args == 3)
212  {
213  fnameOK = true;
214 
215  /*
216  * Use just the prefix of the filename, ignore everything after
217  * first period
218  */
220  }
221  }
223  {
224  int args;
225  uint32 tli = 1,
226  log = 0,
227  seg = 0,
228  offset = 0;
229 
230  args = sscanf(restartWALFileName, "%08X%08X%08X.%08X.backup", &tli, &log, &seg, &offset);
231  if (args == 4)
232  {
233  fnameOK = true;
234 
235  /*
236  * Use just the prefix of the filename, ignore everything after
237  * first period
238  */
240  }
241  }
242 
243  if (!fnameOK)
244  {
245  pg_log_error("invalid file name argument");
246  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
247  exit(2);
248  }
249 }
250 
251 /* =====================================================================
252  * End of Customizable section
253  * =====================================================================
254  */
255 
256 static void
257 usage(void)
258 {
259  printf(_("%s removes older WAL files from PostgreSQL archives.\n\n"), progname);
260  printf(_("Usage:\n"));
261  printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
262  printf(_("\nOptions:\n"));
263  printf(_(" -b, --clean-backup-history clean up files including backup history files\n"));
264  printf(_(" -d, --debug generate debug output (verbose mode)\n"));
265  printf(_(" -n, --dry-run dry run, show the names of the files that would be\n"
266  " removed\n"));
267  printf(_(" -V, --version output version information, then exit\n"));
268  printf(_(" -x, --strip-extension=EXT strip this extension before identifying files for\n"
269  " clean up\n"));
270  printf(_(" -?, --help show this help, then exit\n"));
271  printf(_("\n"
272  "For use as archive_cleanup_command in postgresql.conf:\n"
273  " archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n"
274  "e.g.\n"
275  " archive_cleanup_command = 'pg_archivecleanup /mnt/server/archiverdir %%r'\n"));
276  printf(_("\n"
277  "Or for use as a standalone archive cleaner:\n"
278  "e.g.\n"
279  " pg_archivecleanup /mnt/server/archiverdir 000000010000000000000010.00000020.backup\n"));
280  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
281  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
282 }
283 
284 /*------------ MAIN ----------------------------------------*/
285 int
286 main(int argc, char **argv)
287 {
288  static struct option long_options[] = {
289  {"clean-backup-history", no_argument, NULL, 'b'},
290  {"debug", no_argument, NULL, 'd'},
291  {"dry-run", no_argument, NULL, 'n'},
292  {"strip-extension", required_argument, NULL, 'x'},
293  {NULL, 0, NULL, 0}
294  };
295  int c;
296 
297  pg_logging_init(argv[0]);
298  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_archivecleanup"));
299  progname = get_progname(argv[0]);
300 
301  if (argc > 1)
302  {
303  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
304  {
305  usage();
306  exit(0);
307  }
308  if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
309  {
310  puts("pg_archivecleanup (PostgreSQL) " PG_VERSION);
311  exit(0);
312  }
313  }
314 
315  while ((c = getopt_long(argc, argv, "bdnx:", long_options, NULL)) != -1)
316  {
317  switch (c)
318  {
319  case 'b': /* Remove backup history files as well */
320  cleanBackupHistory = true;
321  break;
322  case 'd': /* Debug mode */
324  break;
325  case 'n': /* Dry-Run mode */
326  dryrun = true;
327  break;
328  case 'x':
329  additional_ext = pg_strdup(optarg); /* Extension to remove
330  * from xlogfile names */
331  break;
332  default:
333  /* getopt already emitted a complaint */
334  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
335  exit(2);
336  }
337  }
338 
339  /*
340  * We will go to the archiveLocation to check restartWALFileName.
341  * restartWALFileName may not exist anymore, which would not be an error,
342  * so we separate the archiveLocation and restartWALFileName so we can
343  * check separately whether archiveLocation exists, if not that is an
344  * error
345  */
346  if (optind < argc)
347  {
348  archiveLocation = argv[optind];
349  optind++;
350  }
351  else
352  {
353  pg_log_error("must specify archive location");
354  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
355  exit(2);
356  }
357 
358  if (optind < argc)
359  {
360  restartWALFileName = argv[optind];
361  optind++;
362  }
363  else
364  {
365  pg_log_error("must specify oldest kept WAL file");
366  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
367  exit(2);
368  }
369 
370  if (optind < argc)
371  {
372  pg_log_error("too many command-line arguments");
373  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
374  exit(2);
375  }
376 
377  /*
378  * Check archive exists and other initialization if required.
379  */
380  Initialize();
381 
382  /*
383  * Check filename is a valid name, then process to find cut-off
384  */
386 
387  pg_log_debug("keeping WAL file \"%s/%s\" and later",
389 
390  /*
391  * Remove WAL files older than cut-off
392  */
394 
395  exit(0);
396 }
unsigned int uint32
Definition: c.h:493
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1201
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:448
int closedir(DIR *)
Definition: dirent.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
#define _(x)
Definition: elog.c:90
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:60
#define no_argument
Definition: getopt_long.h:24
#define required_argument
Definition: getopt_long.h:25
exit(1)
void pg_logging_increase_verbosity(void)
Definition: logging.c:182
void pg_logging_init(const char *argv0)
Definition: logging.c:83
#define pg_log_error(...)
Definition: logging.h:106
#define pg_log_error_hint(...)
Definition: logging.h:112
#define pg_log_debug(...)
Definition: logging.h:133
char * restartWALFileName
int main(int argc, char **argv)
char * archiveLocation
char * additional_ext
bool cleanBackupHistory
char exclusiveCleanupFileName[MAXFNAMELEN]
static void SetWALFileNameForCleanup(void)
static void CleanupPriorWALFiles(void)
static void Initialize(void)
const char * progname
static void usage(void)
static void TrimExtension(char *filename, char *extension)
bool dryrun
#define pg_fatal(...)
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:121
PGDLLIMPORT int optind
Definition: getopt.c:50
PGDLLIMPORT char * optarg
Definition: getopt.c:52
const char * get_progname(const char *argv0)
Definition: path.c:574
#define snprintf
Definition: port.h:238
#define printf(...)
Definition: port.h:244
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
char * c
static Walfile * walfile
Definition: receivelog.c:29
Definition: dirent.c:26
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
unsigned short st_mode
Definition: win32_port.h:268
#define stat
Definition: win32_port.h:284
#define S_ISDIR(m)
Definition: win32_port.h:325
static bool IsXLogFileName(const char *fname)
#define MAXFNAMELEN
static bool IsBackupHistoryFileName(const char *fname)
static void XLogFileNameById(char *fname, TimeLineID tli, uint32 log, uint32 seg)
static bool IsPartialXLogFileName(const char *fname)