PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
pg_backup_tar.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_tar.c
4  *
5  * This file is copied from the 'files' format file, but dumps data into
6  * one temp file then sends it to the output TAR archive.
7  *
8  * The tar format also includes a 'restore.sql' script which is there for
9  * the benefit of humans. This script is never used by pg_restore.
10  *
11  * NOTE: If you untar the created 'tar' file, the resulting files are
12  * compatible with the 'directory' format. Please keep the two formats in
13  * sync.
14  *
15  * See the headers to pg_backup_directory & pg_restore for more details.
16  *
17  * Copyright (c) 2000, Philip Warner
18  * Rights are granted to use this software in any way so long
19  * as this notice is not removed.
20  *
21  * The author is not responsible for loss or damages that may
22  * result from it's use.
23  *
24  *
25  * IDENTIFICATION
26  * src/bin/pg_dump/pg_backup_tar.c
27  *
28  *-------------------------------------------------------------------------
29  */
30 #include "postgres_fe.h"
31 
32 #include "pg_backup_archiver.h"
33 #include "pg_backup_tar.h"
34 #include "pg_backup_utils.h"
35 #include "pgtar.h"
36 #include "common/file_utils.h"
37 #include "fe_utils/string_utils.h"
38 
39 #include <sys/stat.h>
40 #include <ctype.h>
41 #include <limits.h>
42 #include <unistd.h>
43 
44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 static int _WriteByte(ArchiveHandle *AH, const int i);
49 static int _ReadByte(ArchiveHandle *);
50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 static void _CloseArchive(ArchiveHandle *AH);
53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 
58 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
59 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
62 
63 #define K_STD_BUF_SIZE 1024
64 
65 
66 typedef struct
67 {
68 #ifdef HAVE_LIBZ
69  gzFile zFH;
70 #else
71  FILE *zFH;
72 #endif
73  FILE *nFH;
74  FILE *tarFH;
75  FILE *tmpFH;
76  char *targetFile;
77  char mode;
81 } TAR_MEMBER;
82 
83 typedef struct
84 {
85  int hasSeek;
86  pgoff_t filePos;
88  FILE *tarFH;
94 } lclContext;
95 
96 typedef struct
97 {
99  char *filename;
100 } lclTocEntry;
101 
102 /* translator: this is a module name */
103 static const char *modulename = gettext_noop("tar archiver");
104 
105 static void _LoadBlobs(ArchiveHandle *AH);
106 
107 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
108 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
109 
110 #ifdef __NOT_USED__
111 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
112 #endif
113 static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3, 4);
114 
115 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
116 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
117 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
118 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
119 static void _tarWriteHeader(TAR_MEMBER *th);
120 static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
121 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
122 
123 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
124 
125 /*
126  * Initializer
127  */
128 void
130 {
131  lclContext *ctx;
132 
133  /* Assuming static functions, this can be copied for each format. */
134  AH->ArchiveEntryPtr = _ArchiveEntry;
135  AH->StartDataPtr = _StartData;
136  AH->WriteDataPtr = _WriteData;
137  AH->EndDataPtr = _EndData;
138  AH->WriteBytePtr = _WriteByte;
139  AH->ReadBytePtr = _ReadByte;
140  AH->WriteBufPtr = _WriteBuf;
141  AH->ReadBufPtr = _ReadBuf;
142  AH->ClosePtr = _CloseArchive;
143  AH->ReopenPtr = NULL;
144  AH->PrintTocDataPtr = _PrintTocData;
145  AH->ReadExtraTocPtr = _ReadExtraToc;
146  AH->WriteExtraTocPtr = _WriteExtraToc;
147  AH->PrintExtraTocPtr = _PrintExtraToc;
148 
149  AH->StartBlobsPtr = _StartBlobs;
150  AH->StartBlobPtr = _StartBlob;
151  AH->EndBlobPtr = _EndBlob;
152  AH->EndBlobsPtr = _EndBlobs;
153  AH->ClonePtr = NULL;
154  AH->DeClonePtr = NULL;
155 
156  AH->WorkerJobDumpPtr = NULL;
157  AH->WorkerJobRestorePtr = NULL;
158 
159  /*
160  * Set up some special context used in compressing data.
161  */
162  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
163  AH->formatData = (void *) ctx;
164  ctx->filePos = 0;
165  ctx->isSpecialScript = 0;
166 
167  /* Initialize LO buffering */
168  AH->lo_buf_size = LOBBUFSIZE;
169  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
170 
171  /*
172  * Now open the tar file, and load the TOC if we're in read mode.
173  */
174  if (AH->mode == archModeWrite)
175  {
176  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
177  {
178  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
179  if (ctx->tarFH == NULL)
181  "could not open TOC file \"%s\" for output: %s\n",
182  AH->fSpec, strerror(errno));
183  }
184  else
185  {
186  ctx->tarFH = stdout;
187  if (ctx->tarFH == NULL)
189  "could not open TOC file for output: %s\n",
190  strerror(errno));
191  }
192 
193  ctx->tarFHpos = 0;
194 
195  /*
196  * Make unbuffered since we will dup() it, and the buffers screw each
197  * other
198  */
199  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
200 
201  ctx->hasSeek = checkSeek(ctx->tarFH);
202 
203  /*
204  * We don't support compression because reading the files back is not
205  * possible since gzdopen uses buffered IO which totally screws file
206  * positioning.
207  */
208  if (AH->compression != 0)
210  "compression is not supported by tar archive format\n");
211  }
212  else
213  { /* Read Mode */
214  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
215  {
216  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
217  if (ctx->tarFH == NULL)
218  exit_horribly(modulename, "could not open TOC file \"%s\" for input: %s\n",
219  AH->fSpec, strerror(errno));
220  }
221  else
222  {
223  ctx->tarFH = stdin;
224  if (ctx->tarFH == NULL)
225  exit_horribly(modulename, "could not open TOC file for input: %s\n",
226  strerror(errno));
227  }
228 
229  /*
230  * Make unbuffered since we will dup() it, and the buffers screw each
231  * other
232  */
233  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
234 
235  ctx->tarFHpos = 0;
236 
237  ctx->hasSeek = checkSeek(ctx->tarFH);
238 
239  /*
240  * Forcibly unmark the header as read since we use the lookahead
241  * buffer
242  */
243  AH->readHeader = 0;
244 
245  ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
246  ReadHead(AH);
247  ReadToc(AH);
248  tarClose(AH, ctx->FH); /* Nothing else in the file... */
249  }
250 }
251 
252 /*
253  * - Start a new TOC entry
254  * Setup the output file name.
255  */
256 static void
258 {
259  lclTocEntry *ctx;
260  char fn[K_STD_BUF_SIZE];
261 
262  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
263  if (te->dataDumper != NULL)
264  {
265 #ifdef HAVE_LIBZ
266  if (AH->compression == 0)
267  sprintf(fn, "%d.dat", te->dumpId);
268  else
269  sprintf(fn, "%d.dat.gz", te->dumpId);
270 #else
271  sprintf(fn, "%d.dat", te->dumpId);
272 #endif
273  ctx->filename = pg_strdup(fn);
274  }
275  else
276  {
277  ctx->filename = NULL;
278  ctx->TH = NULL;
279  }
280  te->formatData = (void *) ctx;
281 }
282 
283 static void
285 {
286  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
287 
288  if (ctx->filename)
289  WriteStr(AH, ctx->filename);
290  else
291  WriteStr(AH, "");
292 }
293 
294 static void
296 {
297  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
298 
299  if (ctx == NULL)
300  {
301  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
302  te->formatData = (void *) ctx;
303  }
304 
305  ctx->filename = ReadStr(AH);
306  if (strlen(ctx->filename) == 0)
307  {
308  free(ctx->filename);
309  ctx->filename = NULL;
310  }
311  ctx->TH = NULL;
312 }
313 
314 static void
316 {
317  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
318 
319  if (AH->public.verbose && ctx->filename != NULL)
320  ahprintf(AH, "-- File: %s\n", ctx->filename);
321 }
322 
323 static void
325 {
326  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
327 
328  tctx->TH = tarOpen(AH, tctx->filename, 'w');
329 }
330 
331 static TAR_MEMBER *
332 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
333 {
334  lclContext *ctx = (lclContext *) AH->formatData;
335  TAR_MEMBER *tm;
336 
337 #ifdef HAVE_LIBZ
338  char fmode[10];
339 #endif
340 
341  if (mode == 'r')
342  {
343  tm = _tarPositionTo(AH, filename);
344  if (!tm) /* Not found */
345  {
346  if (filename)
347  {
348  /*
349  * Couldn't find the requested file. Future: do SEEK(0) and
350  * retry.
351  */
352  exit_horribly(modulename, "could not find file \"%s\" in archive\n", filename);
353  }
354  else
355  {
356  /* Any file OK, none left, so return NULL */
357  return NULL;
358  }
359  }
360 
361 #ifdef HAVE_LIBZ
362 
363  if (AH->compression == 0)
364  tm->nFH = ctx->tarFH;
365  else
366  exit_horribly(modulename, "compression is not supported by tar archive format\n");
367  /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
368 #else
369  tm->nFH = ctx->tarFH;
370 #endif
371  }
372  else
373  {
374  int old_umask;
375 
376  tm = pg_malloc0(sizeof(TAR_MEMBER));
377 
378  /*
379  * POSIX does not require, but permits, tmpfile() to restrict file
380  * permissions. Given an OS crash after we write data, the filesystem
381  * might retain the data but forget tmpfile()'s unlink(). If so, the
382  * file mode protects confidentiality of the data written.
383  */
384  old_umask = umask(S_IRWXG | S_IRWXO);
385 
386 #ifndef WIN32
387  tm->tmpFH = tmpfile();
388 #else
389 
390  /*
391  * On WIN32, tmpfile() generates a filename in the root directory,
392  * which requires administrative permissions on certain systems. Loop
393  * until we find a unique file name we can create.
394  */
395  while (1)
396  {
397  char *name;
398  int fd;
399 
400  name = _tempnam(NULL, "pg_temp_");
401  if (name == NULL)
402  break;
403  fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
404  O_TEMPORARY, S_IRUSR | S_IWUSR);
405  free(name);
406 
407  if (fd != -1) /* created a file */
408  {
409  tm->tmpFH = fdopen(fd, "w+b");
410  break;
411  }
412  else if (errno != EEXIST) /* failure other than file exists */
413  break;
414  }
415 #endif
416 
417  if (tm->tmpFH == NULL)
418  exit_horribly(modulename, "could not generate temporary file name: %s\n", strerror(errno));
419 
420  umask(old_umask);
421 
422 #ifdef HAVE_LIBZ
423 
424  if (AH->compression != 0)
425  {
426  sprintf(fmode, "wb%d", AH->compression);
427  tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
428  if (tm->zFH == NULL)
429  exit_horribly(modulename, "could not open temporary file\n");
430  }
431  else
432  tm->nFH = tm->tmpFH;
433 #else
434 
435  tm->nFH = tm->tmpFH;
436 #endif
437 
438  tm->AH = AH;
439  tm->targetFile = pg_strdup(filename);
440  }
441 
442  tm->mode = mode;
443  tm->tarFH = ctx->tarFH;
444 
445  return tm;
446 }
447 
448 static void
450 {
451  /*
452  * Close the GZ file since we dup'd. This will flush the buffers.
453  */
454  if (AH->compression != 0)
455  if (GZCLOSE(th->zFH) != 0)
456  exit_horribly(modulename, "could not close tar member\n");
457 
458  if (th->mode == 'w')
459  _tarAddFile(AH, th); /* This will close the temp file */
460 
461  /*
462  * else Nothing to do for normal read since we don't dup() normal file
463  * handle, and we don't use temp files.
464  */
465 
466  if (th->targetFile)
467  free(th->targetFile);
468 
469  th->nFH = NULL;
470  th->zFH = NULL;
471 }
472 
473 #ifdef __NOT_USED__
474 static char *
475 tarGets(char *buf, size_t len, TAR_MEMBER *th)
476 {
477  char *s;
478  size_t cnt = 0;
479  char c = ' ';
480  int eof = 0;
481 
482  /* Can't read past logical EOF */
483  if (len > (th->fileLen - th->pos))
484  len = th->fileLen - th->pos;
485 
486  while (cnt < len && c != '\n')
487  {
488  if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
489  {
490  eof = 1;
491  break;
492  }
493  buf[cnt++] = c;
494  }
495 
496  if (eof && cnt == 0)
497  s = NULL;
498  else
499  {
500  buf[cnt++] = '\0';
501  s = buf;
502  }
503 
504  if (s)
505  {
506  len = strlen(s);
507  th->pos += len;
508  }
509 
510  return s;
511 }
512 #endif
513 
514 /*
515  * Just read bytes from the archive. This is the low level read routine
516  * that is used for ALL reads on a tar file.
517  */
518 static size_t
519 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
520 {
521  lclContext *ctx = (lclContext *) AH->formatData;
522  size_t avail;
523  size_t used = 0;
524  size_t res = 0;
525 
526  avail = AH->lookaheadLen - AH->lookaheadPos;
527  if (avail > 0)
528  {
529  /* We have some lookahead bytes to use */
530  if (avail >= len) /* Just use the lookahead buffer */
531  used = len;
532  else
533  used = avail;
534 
535  /* Copy, and adjust buffer pos */
536  memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
537  AH->lookaheadPos += used;
538 
539  /* Adjust required length */
540  len -= used;
541  }
542 
543  /* Read the file if len > 0 */
544  if (len > 0)
545  {
546  if (fh)
547  {
548  res = fread(&((char *) buf)[used], 1, len, fh);
549  if (res != len && !feof(fh))
550  READ_ERROR_EXIT(fh);
551  }
552  else if (th)
553  {
554  if (th->zFH)
555  {
556  res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
557  if (res != len && !GZEOF(th->zFH))
559  "could not read from input file: %s\n", strerror(errno));
560  }
561  else
562  {
563  res = fread(&((char *) buf)[used], 1, len, th->nFH);
564  if (res != len && !feof(th->nFH))
565  READ_ERROR_EXIT(th->nFH);
566  }
567  }
568  else
569  exit_horribly(modulename, "internal error -- neither th nor fh specified in tarReadRaw()\n");
570  }
571 
572  ctx->tarFHpos += res + used;
573 
574  return (res + used);
575 }
576 
577 static size_t
578 tarRead(void *buf, size_t len, TAR_MEMBER *th)
579 {
580  size_t res;
581 
582  if (th->pos + len > th->fileLen)
583  len = th->fileLen - th->pos;
584 
585  if (len <= 0)
586  return 0;
587 
588  res = _tarReadRaw(th->AH, buf, len, th, NULL);
589 
590  th->pos += res;
591 
592  return res;
593 }
594 
595 static size_t
596 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
597 {
598  size_t res;
599 
600  if (th->zFH != NULL)
601  res = GZWRITE(buf, 1, len, th->zFH);
602  else
603  res = fwrite(buf, 1, len, th->nFH);
604 
605  th->pos += res;
606  return res;
607 }
608 
609 static void
610 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
611 {
612  lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
613 
614  if (tarWrite(data, dLen, tctx->TH) != dLen)
616 
617  return;
618 }
619 
620 static void
622 {
623  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
624 
625  /* Close the file */
626  tarClose(AH, tctx->TH);
627  tctx->TH = NULL;
628 }
629 
630 /*
631  * Print data for a given file
632  */
633 static void
635 {
636  lclContext *ctx = (lclContext *) AH->formatData;
637  char buf[4096];
638  size_t cnt;
639  TAR_MEMBER *th;
640 
641  if (!filename)
642  return;
643 
644  th = tarOpen(AH, filename, 'r');
645  ctx->FH = th;
646 
647  while ((cnt = tarRead(buf, 4095, th)) > 0)
648  {
649  buf[cnt] = '\0';
650  ahwrite(buf, 1, cnt, AH);
651  }
652 
653  tarClose(AH, th);
654 }
655 
656 
657 /*
658  * Print data for a given TOC entry
659 */
660 static void
662 {
663  lclContext *ctx = (lclContext *) AH->formatData;
664  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
665  int pos1;
666 
667  if (!tctx->filename)
668  return;
669 
670  /*
671  * If we're writing the special restore.sql script, emit a suitable
672  * command to include each table's data from the corresponding file.
673  *
674  * In the COPY case this is a bit klugy because the regular COPY command
675  * was already printed before we get control.
676  */
677  if (ctx->isSpecialScript)
678  {
679  if (te->copyStmt)
680  {
681  /* Abort the COPY FROM stdin */
682  ahprintf(AH, "\\.\n");
683 
684  /*
685  * The COPY statement should look like "COPY ... FROM stdin;\n",
686  * see dumpTableData().
687  */
688  pos1 = (int) strlen(te->copyStmt) - 13;
689  if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
690  strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
692  "unexpected COPY statement syntax: \"%s\"\n",
693  te->copyStmt);
694 
695  /* Emit all but the FROM part ... */
696  ahwrite(te->copyStmt, 1, pos1, AH);
697  /* ... and insert modified FROM */
698  ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
699  }
700  else
701  {
702  /* --inserts mode, no worries, just include the data file */
703  ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
704  }
705 
706  return;
707  }
708 
709  if (strcmp(te->desc, "BLOBS") == 0)
710  _LoadBlobs(AH);
711  else
712  _PrintFileData(AH, tctx->filename);
713 }
714 
715 static void
717 {
718  Oid oid;
719  lclContext *ctx = (lclContext *) AH->formatData;
720  TAR_MEMBER *th;
721  size_t cnt;
722  bool foundBlob = false;
723  char buf[4096];
724 
725  StartRestoreBlobs(AH);
726 
727  th = tarOpen(AH, NULL, 'r'); /* Open next file */
728  while (th != NULL)
729  {
730  ctx->FH = th;
731 
732  if (strncmp(th->targetFile, "blob_", 5) == 0)
733  {
734  oid = atooid(&th->targetFile[5]);
735  if (oid != 0)
736  {
737  ahlog(AH, 1, "restoring large object with OID %u\n", oid);
738 
739  StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
740 
741  while ((cnt = tarRead(buf, 4095, th)) > 0)
742  {
743  buf[cnt] = '\0';
744  ahwrite(buf, 1, cnt, AH);
745  }
746  EndRestoreBlob(AH, oid);
747  foundBlob = true;
748  }
749  tarClose(AH, th);
750  }
751  else
752  {
753  tarClose(AH, th);
754 
755  /*
756  * Once we have found the first blob, stop at the first non-blob
757  * entry (which will be 'blobs.toc'). This coding would eat all
758  * the rest of the archive if there are no blobs ... but this
759  * function shouldn't be called at all in that case.
760  */
761  if (foundBlob)
762  break;
763  }
764 
765  th = tarOpen(AH, NULL, 'r');
766  }
767  EndRestoreBlobs(AH);
768 }
769 
770 
771 static int
772 _WriteByte(ArchiveHandle *AH, const int i)
773 {
774  lclContext *ctx = (lclContext *) AH->formatData;
775  char b = i; /* Avoid endian problems */
776 
777  if (tarWrite(&b, 1, ctx->FH) != 1)
779 
780  ctx->filePos += 1;
781  return 1;
782 }
783 
784 static int
786 {
787  lclContext *ctx = (lclContext *) AH->formatData;
788  size_t res;
789  unsigned char c;
790 
791  res = tarRead(&c, 1, ctx->FH);
792  if (res != 1)
793  /* We already would have exited for errors on reads, must be EOF */
795  "could not read from input file: end of file\n");
796  ctx->filePos += 1;
797  return c;
798 }
799 
800 static void
801 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
802 {
803  lclContext *ctx = (lclContext *) AH->formatData;
804 
805  if (tarWrite(buf, len, ctx->FH) != len)
807 
808  ctx->filePos += len;
809 }
810 
811 static void
812 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
813 {
814  lclContext *ctx = (lclContext *) AH->formatData;
815 
816  if (tarRead(buf, len, ctx->FH) != len)
817  /* We already would have exited for errors on reads, must be EOF */
819  "could not read from input file: end of file\n");
820 
821  ctx->filePos += len;
822  return;
823 }
824 
825 static void
827 {
828  lclContext *ctx = (lclContext *) AH->formatData;
829  TAR_MEMBER *th;
830  RestoreOptions *ropt;
831  RestoreOptions *savRopt;
832  DumpOptions *savDopt;
833  int savVerbose,
834  i;
835 
836  if (AH->mode == archModeWrite)
837  {
838  /*
839  * Write the Header & TOC to the archive FIRST
840  */
841  th = tarOpen(AH, "toc.dat", 'w');
842  ctx->FH = th;
843  WriteHead(AH);
844  WriteToc(AH);
845  tarClose(AH, th); /* Not needed any more */
846 
847  /*
848  * Now send the data (tables & blobs)
849  */
850  WriteDataChunks(AH, NULL);
851 
852  /*
853  * Now this format wants to append a script which does a full restore
854  * if the files have been extracted.
855  */
856  th = tarOpen(AH, "restore.sql", 'w');
857 
858  tarPrintf(AH, th, "--\n"
859  "-- NOTE:\n"
860  "--\n"
861  "-- File paths need to be edited. Search for $$PATH$$ and\n"
862  "-- replace it with the path to the directory containing\n"
863  "-- the extracted data files.\n"
864  "--\n");
865 
866  AH->CustomOutPtr = _scriptOut;
867 
868  ctx->isSpecialScript = 1;
869  ctx->scriptTH = th;
870 
871  ropt = NewRestoreOptions();
872  memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
873  ropt->filename = NULL;
874  ropt->dropSchema = 1;
875  ropt->compression = 0;
876  ropt->superuser = NULL;
877  ropt->suppressDumpWarnings = true;
878 
879  savDopt = AH->public.dopt;
880  savRopt = AH->public.ropt;
881 
882  SetArchiveOptions((Archive *) AH, NULL, ropt);
883 
884  savVerbose = AH->public.verbose;
885  AH->public.verbose = 0;
886 
887  RestoreArchive((Archive *) AH);
888 
889  SetArchiveOptions((Archive *) AH, savDopt, savRopt);
890 
891  AH->public.verbose = savVerbose;
892 
893  tarClose(AH, th);
894 
895  ctx->isSpecialScript = 0;
896 
897  /*
898  * EOF marker for tar files is two blocks of NULLs.
899  */
900  for (i = 0; i < 512 * 2; i++)
901  {
902  if (fputc(0, ctx->tarFH) == EOF)
904  }
905 
906  /* Sync the output file if one is defined */
907  if (AH->dosync && AH->fSpec)
908  (void) fsync_fname(AH->fSpec, false, progname);
909  }
910 
911  AH->FH = NULL;
912 }
913 
914 static size_t
915 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
916 {
917  lclContext *ctx = (lclContext *) AH->formatData;
918 
919  return tarWrite(buf, len, ctx->scriptTH);
920 }
921 
922 /*
923  * BLOB support
924  */
925 
926 /*
927  * Called by the archiver when starting to save all BLOB DATA (not schema).
928  * This routine should save whatever format-specific information is needed
929  * to read the BLOBs back into memory.
930  *
931  * It is called just prior to the dumper's DataDumper routine.
932  *
933  * Optional, but strongly recommended.
934  *
935  */
936 static void
938 {
939  lclContext *ctx = (lclContext *) AH->formatData;
940  char fname[K_STD_BUF_SIZE];
941 
942  sprintf(fname, "blobs.toc");
943  ctx->blobToc = tarOpen(AH, fname, 'w');
944 }
945 
946 /*
947  * Called by the archiver when the dumper calls StartBlob.
948  *
949  * Mandatory.
950  *
951  * Must save the passed OID for retrieval at restore-time.
952  */
953 static void
955 {
956  lclContext *ctx = (lclContext *) AH->formatData;
957  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
958  char fname[255];
959  char *sfx;
960 
961  if (oid == 0)
962  exit_horribly(modulename, "invalid OID for large object (%u)\n", oid);
963 
964  if (AH->compression != 0)
965  sfx = ".gz";
966  else
967  sfx = "";
968 
969  sprintf(fname, "blob_%u.dat%s", oid, sfx);
970 
971  tarPrintf(AH, ctx->blobToc, "%u %s\n", oid, fname);
972 
973  tctx->TH = tarOpen(AH, fname, 'w');
974 }
975 
976 /*
977  * Called by the archiver when the dumper calls EndBlob.
978  *
979  * Optional.
980  *
981  */
982 static void
984 {
985  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
986 
987  tarClose(AH, tctx->TH);
988 }
989 
990 /*
991  * Called by the archiver when finishing saving all BLOB DATA.
992  *
993  * Optional.
994  *
995  */
996 static void
998 {
999  lclContext *ctx = (lclContext *) AH->formatData;
1000 
1001  /* Write out a fake zero OID to mark end-of-blobs. */
1002  /* WriteInt(AH, 0); */
1003 
1004  tarClose(AH, ctx->blobToc);
1005 }
1006 
1007 
1008 
1009 /*------------
1010  * TAR Support
1011  *------------
1012  */
1013 
1014 static int
1015 tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
1016 {
1017  char *p;
1018  size_t len = 128; /* initial assumption about buffer size */
1019  size_t cnt;
1020 
1021  for (;;)
1022  {
1023  va_list args;
1024 
1025  /* Allocate work buffer. */
1026  p = (char *) pg_malloc(len);
1027 
1028  /* Try to format the data. */
1029  va_start(args, fmt);
1030  cnt = pvsnprintf(p, len, fmt, args);
1031  va_end(args);
1032 
1033  if (cnt < len)
1034  break; /* success */
1035 
1036  /* Release buffer and loop around to try again with larger len. */
1037  free(p);
1038  len = cnt;
1039  }
1040 
1041  cnt = tarWrite(p, cnt, th);
1042  free(p);
1043  return (int) cnt;
1044 }
1045 
1046 bool
1048 {
1049  int sum;
1050  int chk = tarChecksum(header);
1051 
1052  sum = read_tar_number(&header[148], 8);
1053 
1054  if (sum != chk)
1055  return false;
1056 
1057  /* POSIX tar format */
1058  if (memcmp(&header[257], "ustar\0", 6) == 0 &&
1059  memcmp(&header[263], "00", 2) == 0)
1060  return true;
1061  /* GNU tar format */
1062  if (memcmp(&header[257], "ustar \0", 8) == 0)
1063  return true;
1064  /* not-quite-POSIX format written by pre-9.3 pg_dump */
1065  if (memcmp(&header[257], "ustar00\0", 8) == 0)
1066  return true;
1067 
1068  return false;
1069 }
1070 
1071 /* Given the member, write the TAR header & copy the file */
1072 static void
1074 {
1075  lclContext *ctx = (lclContext *) AH->formatData;
1076  FILE *tmp = th->tmpFH; /* Grab it for convenience */
1077  char buf[32768];
1078  size_t cnt;
1079  pgoff_t len = 0;
1080  size_t res;
1081  size_t i,
1082  pad;
1083 
1084  /*
1085  * Find file len & go back to start.
1086  */
1087  fseeko(tmp, 0, SEEK_END);
1088  th->fileLen = ftello(tmp);
1089  if (th->fileLen < 0)
1090  exit_horribly(modulename, "could not determine seek position in archive file: %s\n",
1091  strerror(errno));
1092  fseeko(tmp, 0, SEEK_SET);
1093 
1094  _tarWriteHeader(th);
1095 
1096  while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1097  {
1098  if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1100  len += res;
1101  }
1102  if (!feof(tmp))
1103  READ_ERROR_EXIT(tmp);
1104 
1105  if (fclose(tmp) != 0) /* This *should* delete it... */
1106  exit_horribly(modulename, "could not close temporary file: %s\n",
1107  strerror(errno));
1108 
1109  if (len != th->fileLen)
1110  {
1111  char buf1[32],
1112  buf2[32];
1113 
1114  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) len);
1115  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) th->fileLen);
1116  exit_horribly(modulename, "actual file length (%s) does not match expected (%s)\n",
1117  buf1, buf2);
1118  }
1119 
1120  pad = ((len + 511) & ~511) - len;
1121  for (i = 0; i < pad; i++)
1122  {
1123  if (fputc('\0', th->tarFH) == EOF)
1125  }
1126 
1127  ctx->tarFHpos += len + pad;
1128 }
1129 
1130 /* Locate the file in the archive, read header and position to data */
1131 static TAR_MEMBER *
1133 {
1134  lclContext *ctx = (lclContext *) AH->formatData;
1135  TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
1136  char c;
1137  char header[512];
1138  size_t i,
1139  len,
1140  blks;
1141  int id;
1142 
1143  th->AH = AH;
1144 
1145  /* Go to end of current file, if any */
1146  if (ctx->tarFHpos != 0)
1147  {
1148  char buf1[100],
1149  buf2[100];
1150 
1151  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ctx->tarFHpos);
1152  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ctx->tarNextMember);
1153  ahlog(AH, 4, "moving from position %s to next member at file position %s\n",
1154  buf1, buf2);
1155 
1156  while (ctx->tarFHpos < ctx->tarNextMember)
1157  _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1158  }
1159 
1160  {
1161  char buf[100];
1162 
1163  snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ctx->tarFHpos);
1164  ahlog(AH, 4, "now at file position %s\n", buf);
1165  }
1166 
1167  /* We are at the start of the file, or at the next member */
1168 
1169  /* Get the header */
1170  if (!_tarGetHeader(AH, th))
1171  {
1172  if (filename)
1173  exit_horribly(modulename, "could not find header for file \"%s\" in tar archive\n", filename);
1174  else
1175  {
1176  /*
1177  * We're just scanning the archive for the next file, so return
1178  * null
1179  */
1180  free(th);
1181  return NULL;
1182  }
1183  }
1184 
1185  while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1186  {
1187  ahlog(AH, 4, "skipping tar member %s\n", th->targetFile);
1188 
1189  id = atoi(th->targetFile);
1190  if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1191  exit_horribly(modulename, "restoring data out of order is not supported in this archive format: "
1192  "\"%s\" is required, but comes before \"%s\" in the archive file.\n",
1193  th->targetFile, filename);
1194 
1195  /* Header doesn't match, so read to next header */
1196  len = ((th->fileLen + 511) & ~511); /* Padded length */
1197  blks = len >> 9; /* # of 512 byte blocks */
1198 
1199  for (i = 0; i < blks; i++)
1200  _tarReadRaw(AH, &header[0], 512, NULL, ctx->tarFH);
1201 
1202  if (!_tarGetHeader(AH, th))
1203  exit_horribly(modulename, "could not find header for file \"%s\" in tar archive\n", filename);
1204  }
1205 
1206  ctx->tarNextMember = ctx->tarFHpos + ((th->fileLen + 511) & ~511);
1207  th->pos = 0;
1208 
1209  return th;
1210 }
1211 
1212 /* Read & verify a header */
1213 static int
1215 {
1216  lclContext *ctx = (lclContext *) AH->formatData;
1217  char h[512];
1218  char tag[100 + 1];
1219  int sum,
1220  chk;
1221  pgoff_t len;
1222  pgoff_t hPos;
1223  bool gotBlock = false;
1224 
1225  while (!gotBlock)
1226  {
1227  /* Save the pos for reporting purposes */
1228  hPos = ctx->tarFHpos;
1229 
1230  /* Read a 512 byte block, return EOF, exit if short */
1231  len = _tarReadRaw(AH, h, 512, NULL, ctx->tarFH);
1232  if (len == 0) /* EOF */
1233  return 0;
1234 
1235  if (len != 512)
1237  ngettext("incomplete tar header found (%lu byte)\n",
1238  "incomplete tar header found (%lu bytes)\n",
1239  len),
1240  (unsigned long) len);
1241 
1242  /* Calc checksum */
1243  chk = tarChecksum(h);
1244  sum = read_tar_number(&h[148], 8);
1245 
1246  /*
1247  * If the checksum failed, see if it is a null block. If so, silently
1248  * continue to the next block.
1249  */
1250  if (chk == sum)
1251  gotBlock = true;
1252  else
1253  {
1254  int i;
1255 
1256  for (i = 0; i < 512; i++)
1257  {
1258  if (h[i] != 0)
1259  {
1260  gotBlock = true;
1261  break;
1262  }
1263  }
1264  }
1265  }
1266 
1267  /* Name field is 100 bytes, might not be null-terminated */
1268  strlcpy(tag, &h[0], 100 + 1);
1269 
1270  len = read_tar_number(&h[124], 12);
1271 
1272  {
1273  char posbuf[32];
1274  char lenbuf[32];
1275 
1276  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
1277  snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
1278  ahlog(AH, 3, "TOC Entry %s at %s (length %s, checksum %d)\n",
1279  tag, posbuf, lenbuf, sum);
1280  }
1281 
1282  if (chk != sum)
1283  {
1284  char posbuf[32];
1285 
1286  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
1287  (uint64) ftello(ctx->tarFH));
1289  "corrupt tar header found in %s "
1290  "(expected %d, computed %d) file position %s\n",
1291  tag, sum, chk, posbuf);
1292  }
1293 
1294  th->targetFile = pg_strdup(tag);
1295  th->fileLen = len;
1296 
1297  return 1;
1298 }
1299 
1300 
1301 static void
1303 {
1304  char h[512];
1305 
1306  tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1307  0600, 04000, 02000, time(NULL));
1308 
1309  /* Now write the completed header. */
1310  if (fwrite(h, 1, 512, th->tarFH) != 512)
1312 }
static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3
static void _LoadBlobs(ArchiveHandle *AH)
TAR_MEMBER * FH
Definition: pg_backup_tar.c:91
FILE * zFH
Definition: pg_backup_tar.c:71
void ReadToc(ArchiveHandle *AH)
#define TH
static void _StartData(ArchiveHandle *AH, TocEntry *te)
void ReadHead(ArchiveHandle *AH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
TAR_MEMBER * blobToc
Definition: pg_backup_tar.c:87
TAR_MEMBER * scriptTH
Definition: pg_backup_tar.c:93
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
RestoreOptions * ropt
Definition: pg_backup.h:181
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
static TAR_MEMBER * tarOpen(ArchiveHandle *AH, const char *filename, char mode)
pgoff_t tarFHpos
Definition: pg_backup_tar.c:89
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:567
#define gettext_noop(x)
Definition: c.h:139
#define PG_BINARY_W
Definition: c.h:1041
void WriteToc(ArchiveHandle *AH)
ArchiveHandle * AH
Definition: pg_backup_tar.c:80
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
DataDumperPtr dataDumper
unsigned int Oid
Definition: postgres_ext.h:31
#define fseeko(stream, offset, origin)
Definition: win32.h:237
const char * progname
Definition: pg_standby.c:37
#define PG_BINARY_R
Definition: c.h:1040
void InitArchiveFmt_Tar(ArchiveHandle *AH)
static int fd(const char *x, int i)
Definition: preproc-init.c:105
void StartRestoreBlobs(ArchiveHandle *AH)
struct _tocEntry * currToc
RestoreOptions * NewRestoreOptions(void)
#define pg_attribute_printf(f, a)
Definition: c.h:638
pgoff_t tarNextMember
Definition: pg_backup_tar.c:90
static struct pg_tm tm
Definition: localtime.c:111
size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args)
Definition: psprintf.c:104
int isSpecialScript
Definition: pg_backup_tar.c:92
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
DumpOptions * dopt
Definition: pg_backup.h:180
FILE * tarFH
Definition: pg_backup_tar.c:88
void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop)
char * c
static void _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static char * buf
Definition: pg_test_fsync.c:66
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
static TAR_MEMBER * _tarPositionTo(ArchiveHandle *AH, const char *filename)
#define GZEOF(fh)
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH)
#define atooid(x)
Definition: postgres_ext.h:42
bool isValidTarHeader(char *header)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _EndBlobs(ArchiveHandle *AH, TocEntry *te)
#define GZWRITE(p, s, n, fh)
void EndRestoreBlobs(ArchiveHandle *AH)
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
#define pgoff_t
Definition: win32.h:231
#define ngettext(s, p, n)
Definition: c.h:127
FILE * nFH
Definition: pg_backup_tar.c:73
static const char * modulename
int verbose
Definition: pg_backup.h:183
static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
#define S_IRWXO
Definition: win32.h:455
int tarChecksum(char *header)
Definition: tar.c:88
pgoff_t pos
Definition: pg_backup_tar.c:78
static void * fn(void *arg)
#define free(a)
Definition: header.h:65
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
#define NULL
Definition: c.h:229
CustomOutPtrType CustomOutPtr
static void _tarWriteHeader(TAR_MEMBER *th)
char * ReadStr(ArchiveHandle *AH)
teReqs TocIDRequired(ArchiveHandle *AH, DumpId id)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
FILE * tmpFH
Definition: pg_backup_tar.c:75
pgoff_t fileLen
Definition: pg_backup_tar.c:79
#define ftello(stream)
Definition: win32.h:240
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
#define GZCLOSE(fh)
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:207
#define INT64_FORMAT
Definition: c.h:315
const char * name
Definition: encode.c:521
static int static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
void WriteHead(ArchiveHandle *AH)
#define GZREAD(p, s, n, fh)
#define WRITE_ERROR_EXIT
void exit_horribly(const char *modulename, const char *fmt,...)
#define S_IRWXG
Definition: win32.h:451
static void _CloseArchive(ArchiveHandle *AH)
#define K_STD_BUF_SIZE
Definition: pg_backup_tar.c:63
static char * filename
Definition: pg_dumpall.c:89
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
void ahlog(ArchiveHandle *AH, int level, const char *fmt,...)
void EndRestoreBlob(ArchiveHandle *AH, Oid oid)
void RestoreArchive(Archive *AH)
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th)
int i
const char * strerror(int errnum)
Definition: strerror.c:19
size_t WriteStr(ArchiveHandle *AH, const char *c)
TAR_MEMBER * TH
Definition: pg_backup_tar.c:98
#define READ_ERROR_EXIT(fd)
static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bool checkSeek(FILE *fp)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
pgoff_t filePos
static int _ReadByte(ArchiveHandle *)
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
FILE * tarFH
Definition: pg_backup_tar.c:74
#define UINT64_FORMAT
Definition: c.h:316
#define LOBBUFSIZE
static int _WriteByte(ArchiveHandle *AH, const int i)
char * targetFile
Definition: pg_backup_tar.c:76
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ropt)
static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)