PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
pg_archivecleanup.c
Go to the documentation of this file.
1 /*
2  * src/bin/pg_archivecleanup/pg_archivecleanup.c
3  *
4  * pg_archivecleanup.c
5  *
6  * Production-ready example of an archive_cleanup_command
7  * used to clean an archive when using standby_mode = on in 9.0
8  * or for standalone use for any version of PostgreSQL 8.0+.
9  *
10  * Original author: Simon Riggs simon@2ndquadrant.com
11  * Current maintainer: Simon Riggs
12  */
13 #include "postgres_fe.h"
14 
15 #include <ctype.h>
16 #include <dirent.h>
17 #include <sys/stat.h>
18 #include <fcntl.h>
19 #include <signal.h>
20 #include <sys/time.h>
21 
22 #include "pg_getopt.h"
23 
24 #include "access/xlog_internal.h"
25 
26 const char *progname;
27 
28 /* Options and defaults */
29 bool debug = false; /* are we debugging? */
30 bool dryrun = false; /* are we performing a dry-run operation? */
31 char *additional_ext = NULL; /* Extension to remove from filenames */
32 
33 char *archiveLocation; /* where to find the archive? */
34 char *restartWALFileName; /* the file from which we can restart restore */
35 char WALFilePath[MAXPGPATH]; /* the file path including archive */
36 char exclusiveCleanupFileName[MAXFNAMELEN]; /* the oldest file we
37  * want to remain in
38  * archive */
39 
40 
41 /* =====================================================================
42  *
43  * Customizable section
44  *
45  * =====================================================================
46  *
47  * Currently, this section assumes that the Archive is a locally
48  * accessible directory. If you want to make other assumptions,
49  * such as using a vendor-specific archive and access API, these
50  * routines are the ones you'll need to change. You're
51  * encouraged to submit any changes to pgsql-hackers@postgresql.org
52  * or personally to the current maintainer. Those changes may be
53  * folded in to later versions of this program.
54  */
55 
56 /*
57  * Initialize allows customized commands into the archive cleanup program.
58  *
59  * You may wish to add code to check for tape libraries, etc..
60  */
61 static void
63 {
64  /*
65  * This code assumes that archiveLocation is a directory, so we use stat
66  * to test if it's accessible.
67  */
68  struct stat stat_buf;
69 
70  if (stat(archiveLocation, &stat_buf) != 0 ||
71  !S_ISDIR(stat_buf.st_mode))
72  {
73  fprintf(stderr, _("%s: archive location \"%s\" does not exist\n"),
75  exit(2);
76  }
77 }
78 
79 static void
80 TrimExtension(char *filename, char *extension)
81 {
82  int flen;
83  int elen;
84 
85  if (extension == NULL)
86  return;
87 
88  elen = strlen(extension);
89  flen = strlen(filename);
90 
91  if (flen > elen && strcmp(filename + flen - elen, extension) == 0)
92  filename[flen - elen] = '\0';
93 }
94 
95 static void
97 {
98  int rc;
99  DIR *xldir;
100  struct dirent *xlde;
101  char walfile[MAXPGPATH];
102 
103  if ((xldir = opendir(archiveLocation)) != NULL)
104  {
105  while (errno = 0, (xlde = readdir(xldir)) != NULL)
106  {
107  /*
108  * Truncation is essentially harmless, because we skip names of
109  * length other than XLOG_FNAME_LEN. (In principle, one could use
110  * a 1000-character additional_ext and get trouble.)
111  */
112  strlcpy(walfile, xlde->d_name, MAXPGPATH);
113  TrimExtension(walfile, additional_ext);
114 
115  /*
116  * We ignore the timeline part of the XLOG segment identifiers in
117  * deciding whether a segment is still needed. This ensures that
118  * we won't prematurely remove a segment from a parent timeline.
119  * We could probably be a little more proactive about removing
120  * segments of non-parent timelines, but that would be a whole lot
121  * more complicated.
122  *
123  * We use the alphanumeric sorting property of the filenames to
124  * decide which ones are earlier than the exclusiveCleanupFileName
125  * file. Note that this means files are not removed in the order
126  * they were originally written, in case this worries you.
127  */
128  if ((IsXLogFileName(walfile) || IsPartialXLogFileName(walfile)) &&
129  strcmp(walfile + 8, exclusiveCleanupFileName + 8) < 0)
130  {
131  /*
132  * Use the original file name again now, including any
133  * extension that might have been chopped off before testing
134  * the sequence.
135  */
136  snprintf(WALFilePath, MAXPGPATH, "%s/%s",
137  archiveLocation, xlde->d_name);
138 
139  if (dryrun)
140  {
141  /*
142  * Prints the name of the file to be removed and skips the
143  * actual removal. The regular printout is so that the
144  * user can pipe the output into some other program.
145  */
146  printf("%s\n", WALFilePath);
147  if (debug)
148  fprintf(stderr,
149  _("%s: file \"%s\" would be removed\n"),
151  continue;
152  }
153 
154  if (debug)
155  fprintf(stderr, _("%s: removing file \"%s\"\n"),
157 
158  rc = unlink(WALFilePath);
159  if (rc != 0)
160  {
161  fprintf(stderr, _("%s: ERROR: could not remove file \"%s\": %s\n"),
162  progname, WALFilePath, strerror(errno));
163  break;
164  }
165  }
166  }
167 
168  if (errno)
169  fprintf(stderr, _("%s: could not read archive location \"%s\": %s\n"),
171  if (closedir(xldir))
172  fprintf(stderr, _("%s: could not close archive location \"%s\": %s\n"),
174  }
175  else
176  fprintf(stderr, _("%s: could not open archive location \"%s\": %s\n"),
178 }
179 
180 /*
181  * SetWALFileNameForCleanup()
182  *
183  * Set the earliest WAL filename that we want to keep on the archive
184  * and decide whether we need cleanup
185  */
186 static void
188 {
189  bool fnameOK = false;
190 
192 
193  /*
194  * If restartWALFileName is a WAL file name then just use it directly. If
195  * restartWALFileName is a .partial or .backup filename, make sure we use
196  * the prefix of the filename, otherwise we will remove wrong files since
197  * 000000010000000000000010.partial and
198  * 000000010000000000000010.00000020.backup are after
199  * 000000010000000000000010.
200  */
202  {
204  fnameOK = true;
205  }
207  {
208  int args;
209  uint32 tli = 1,
210  log = 0,
211  seg = 0;
212 
213  args = sscanf(restartWALFileName, "%08X%08X%08X.partial",
214  &tli, &log, &seg);
215  if (args == 3)
216  {
217  fnameOK = true;
218 
219  /*
220  * Use just the prefix of the filename, ignore everything after
221  * first period
222  */
224  }
225  }
227  {
228  int args;
229  uint32 tli = 1,
230  log = 0,
231  seg = 0,
232  offset = 0;
233 
234  args = sscanf(restartWALFileName, "%08X%08X%08X.%08X.backup", &tli, &log, &seg, &offset);
235  if (args == 4)
236  {
237  fnameOK = true;
238 
239  /*
240  * Use just the prefix of the filename, ignore everything after
241  * first period
242  */
244  }
245  }
246 
247  if (!fnameOK)
248  {
249  fprintf(stderr, _("%s: invalid filename input\n"), progname);
250  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
251  exit(2);
252  }
253 }
254 
255 /* =====================================================================
256  * End of Customizable section
257  * =====================================================================
258  */
259 
260 static void
261 usage(void)
262 {
263  printf(_("%s removes older WAL files from PostgreSQL archives.\n\n"), progname);
264  printf(_("Usage:\n"));
265  printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
266  printf(_("\nOptions:\n"));
267  printf(_(" -d generate debug output (verbose mode)\n"));
268  printf(_(" -n dry run, show the names of the files that would be removed\n"));
269  printf(_(" -V, --version output version information, then exit\n"));
270  printf(_(" -x EXT clean up files if they have this extension\n"));
271  printf(_(" -?, --help show this help, then exit\n"));
272  printf(_("\n"
273  "For use as archive_cleanup_command in recovery.conf when standby_mode = on:\n"
274  " archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n"
275  "e.g.\n"
276  " archive_cleanup_command = 'pg_archivecleanup /mnt/server/archiverdir %%r'\n"));
277  printf(_("\n"
278  "Or for use as a standalone archive cleaner:\n"
279  "e.g.\n"
280  " pg_archivecleanup /mnt/server/archiverdir 000000010000000000000010.00000020.backup\n"));
281  printf(_("\nReport bugs to <pgsql-bugs@postgresql.org>.\n"));
282 }
283 
284 /*------------ MAIN ----------------------------------------*/
285 int
286 main(int argc, char **argv)
287 {
288  int c;
289 
290  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_archivecleanup"));
291  progname = get_progname(argv[0]);
292 
293  if (argc > 1)
294  {
295  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
296  {
297  usage();
298  exit(0);
299  }
300  if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
301  {
302  puts("pg_archivecleanup (PostgreSQL) " PG_VERSION);
303  exit(0);
304  }
305  }
306 
307  while ((c = getopt(argc, argv, "x:dn")) != -1)
308  {
309  switch (c)
310  {
311  case 'd': /* Debug mode */
312  debug = true;
313  break;
314  case 'n': /* Dry-Run mode */
315  dryrun = true;
316  break;
317  case 'x':
318  additional_ext = pg_strdup(optarg); /* Extension to remove
319  * from xlogfile names */
320  break;
321  default:
322  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
323  exit(2);
324  break;
325  }
326  }
327 
328  /*
329  * We will go to the archiveLocation to check restartWALFileName.
330  * restartWALFileName may not exist anymore, which would not be an error,
331  * so we separate the archiveLocation and restartWALFileName so we can
332  * check separately whether archiveLocation exists, if not that is an
333  * error
334  */
335  if (optind < argc)
336  {
337  archiveLocation = argv[optind];
338  optind++;
339  }
340  else
341  {
342  fprintf(stderr, _("%s: must specify archive location\n"), progname);
343  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
344  exit(2);
345  }
346 
347  if (optind < argc)
348  {
349  restartWALFileName = argv[optind];
350  optind++;
351  }
352  else
353  {
354  fprintf(stderr, _("%s: must specify restartfilename\n"), progname);
355  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
356  exit(2);
357  }
358 
359  if (optind < argc)
360  {
361  fprintf(stderr, _("%s: too many parameters\n"), progname);
362  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
363  exit(2);
364  }
365 
366  /*
367  * Check archive exists and other initialization if required.
368  */
369  Initialize();
370 
371  /*
372  * Check filename is a valid name, then process to find cut-off
373  */
375 
376  if (debug)
377  {
378  snprintf(WALFilePath, MAXPGPATH, "%s/%s",
380  fprintf(stderr, _("%s: keep WAL file \"%s\" and later\n"),
382  }
383 
384  /*
385  * Remove WAL files older than cut-off
386  */
388 
389  exit(0);
390 }
static void CleanupPriorWALFiles(void)
static void SetWALFileNameForCleanup(void)
const char * get_progname(const char *argv0)
Definition: path.c:453
int closedir(DIR *)
Definition: dirent.c:113
#define XLogFileNameById(fname, tli, log, seg)
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
char * archiveLocation
Definition: dirent.h:9
int getopt(int nargc, char *const *nargv, const char *ostr)
Definition: getopt.c:72
static void usage(void)
bool debug
static void TrimExtension(char *filename, char *extension)
int optind
Definition: getopt.c:51
Definition: dirent.c:25
#define IsXLogFileName(fname)
#define MAXPGPATH
static Walfile * walfile
Definition: receivelog.c:33
DIR * opendir(const char *)
Definition: dirent.c:33
char * restartWALFileName
char * c
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
unsigned int uint32
Definition: c.h:268
int unlink(const char *filename)
#define IsBackupHistoryFileName(fname)
const char * progname
#define MAXFNAMELEN
bool dryrun
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1012
char exclusiveCleanupFileName[MAXFNAMELEN]
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
char WALFilePath[MAXPGPATH]
#define NULL
Definition: c.h:229
struct dirent * readdir(DIR *)
Definition: dirent.c:78
int main(int argc, char **argv)
static char * filename
Definition: pg_dumpall.c:87
#define IsPartialXLogFileName(fname)
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:550
char * optarg
Definition: getopt.c:53
const char * strerror(int errnum)
Definition: strerror.c:19
static void Initialize(void)
char d_name[MAX_PATH]
Definition: dirent.h:14
#define _(x)
Definition: elog.c:84
char * additional_ext