PostgreSQL Source Code  git master
pg_backup_tar.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_tar.c
4  *
5  * This file is copied from the 'files' format file, but dumps data into
6  * one temp file then sends it to the output TAR archive.
7  *
8  * The tar format also includes a 'restore.sql' script which is there for
9  * the benefit of humans. This script is never used by pg_restore.
10  *
11  * NOTE: If you untar the created 'tar' file, the resulting files are
12  * compatible with the 'directory' format. Please keep the two formats in
13  * sync.
14  *
15  * See the headers to pg_backup_directory & pg_restore for more details.
16  *
17  * Copyright (c) 2000, Philip Warner
18  * Rights are granted to use this software in any way so long
19  * as this notice is not removed.
20  *
21  * The author is not responsible for loss or damages that may
22  * result from its use.
23  *
24  *
25  * IDENTIFICATION
26  * src/bin/pg_dump/pg_backup_tar.c
27  *
28  *-------------------------------------------------------------------------
29  */
30 #include "postgres_fe.h"
31 
32 #include <sys/stat.h>
33 #include <ctype.h>
34 #include <limits.h>
35 #include <unistd.h>
36 
37 #include "common/file_utils.h"
38 #include "fe_utils/string_utils.h"
39 #include "pg_backup_archiver.h"
40 #include "pg_backup_tar.h"
41 #include "pg_backup_utils.h"
42 #include "pgtar.h"
43 
44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 static int _WriteByte(ArchiveHandle *AH, const int i);
49 static int _ReadByte(ArchiveHandle *);
50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 static void _CloseArchive(ArchiveHandle *AH);
53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 
58 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
59 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
62 
63 #define K_STD_BUF_SIZE 1024
64 
65 
66 typedef struct
67 {
68 #ifdef HAVE_LIBZ
69  gzFile zFH;
70 #else
71  FILE *zFH;
72 #endif
73  FILE *nFH;
74  FILE *tarFH;
75  FILE *tmpFH;
76  char *targetFile;
77  char mode;
81 } TAR_MEMBER;
82 
83 typedef struct
84 {
85  int hasSeek;
88  FILE *tarFH;
94 } lclContext;
95 
96 typedef struct
97 {
99  char *filename;
100 } lclTocEntry;
101 
102 static void _LoadBlobs(ArchiveHandle *AH);
103 
104 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
105 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
106 
107 #ifdef __NOT_USED__
108 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
109 #endif
110 static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
111 
112 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
113 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
114 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
115 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
116 static void _tarWriteHeader(TAR_MEMBER *th);
117 static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
118 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
119 
120 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
121 
122 /*
123  * Initializer
124  */
125 void
127 {
128  lclContext *ctx;
129 
130  /* Assuming static functions, this can be copied for each format. */
131  AH->ArchiveEntryPtr = _ArchiveEntry;
132  AH->StartDataPtr = _StartData;
133  AH->WriteDataPtr = _WriteData;
134  AH->EndDataPtr = _EndData;
135  AH->WriteBytePtr = _WriteByte;
136  AH->ReadBytePtr = _ReadByte;
137  AH->WriteBufPtr = _WriteBuf;
138  AH->ReadBufPtr = _ReadBuf;
139  AH->ClosePtr = _CloseArchive;
140  AH->ReopenPtr = NULL;
141  AH->PrintTocDataPtr = _PrintTocData;
142  AH->ReadExtraTocPtr = _ReadExtraToc;
143  AH->WriteExtraTocPtr = _WriteExtraToc;
144  AH->PrintExtraTocPtr = _PrintExtraToc;
145 
146  AH->StartBlobsPtr = _StartBlobs;
147  AH->StartBlobPtr = _StartBlob;
148  AH->EndBlobPtr = _EndBlob;
149  AH->EndBlobsPtr = _EndBlobs;
150  AH->ClonePtr = NULL;
151  AH->DeClonePtr = NULL;
152 
153  AH->WorkerJobDumpPtr = NULL;
154  AH->WorkerJobRestorePtr = NULL;
155 
156  /*
157  * Set up some special context used in compressing data.
158  */
159  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
160  AH->formatData = (void *) ctx;
161  ctx->filePos = 0;
162  ctx->isSpecialScript = 0;
163 
164  /* Initialize LO buffering */
165  AH->lo_buf_size = LOBBUFSIZE;
166  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
167 
168  /*
169  * Now open the tar file, and load the TOC if we're in read mode.
170  */
171  if (AH->mode == archModeWrite)
172  {
173  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
174  {
175  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
176  if (ctx->tarFH == NULL)
177  fatal("could not open TOC file \"%s\" for output: %m",
178  AH->fSpec);
179  }
180  else
181  {
182  ctx->tarFH = stdout;
183  if (ctx->tarFH == NULL)
184  fatal("could not open TOC file for output: %m");
185  }
186 
187  ctx->tarFHpos = 0;
188 
189  /*
190  * Make unbuffered since we will dup() it, and the buffers screw each
191  * other
192  */
193  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
194 
195  ctx->hasSeek = checkSeek(ctx->tarFH);
196 
197  /*
198  * We don't support compression because reading the files back is not
199  * possible since gzdopen uses buffered IO which totally screws file
200  * positioning.
201  */
202  if (AH->compression != 0)
203  fatal("compression is not supported by tar archive format");
204  }
205  else
206  { /* Read Mode */
207  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
208  {
209  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
210  if (ctx->tarFH == NULL)
211  fatal("could not open TOC file \"%s\" for input: %m",
212  AH->fSpec);
213  }
214  else
215  {
216  ctx->tarFH = stdin;
217  if (ctx->tarFH == NULL)
218  fatal("could not open TOC file for input: %m");
219  }
220 
221  /*
222  * Make unbuffered since we will dup() it, and the buffers screw each
223  * other
224  */
225  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
226 
227  ctx->tarFHpos = 0;
228 
229  ctx->hasSeek = checkSeek(ctx->tarFH);
230 
231  ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
232  ReadHead(AH);
233  ReadToc(AH);
234  tarClose(AH, ctx->FH); /* Nothing else in the file... */
235  }
236 }
237 
238 /*
239  * - Start a new TOC entry
240  * Setup the output file name.
241  */
242 static void
244 {
245  lclTocEntry *ctx;
246  char fn[K_STD_BUF_SIZE];
247 
248  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
249  if (te->dataDumper != NULL)
250  {
251 #ifdef HAVE_LIBZ
252  if (AH->compression == 0)
253  sprintf(fn, "%d.dat", te->dumpId);
254  else
255  sprintf(fn, "%d.dat.gz", te->dumpId);
256 #else
257  sprintf(fn, "%d.dat", te->dumpId);
258 #endif
259  ctx->filename = pg_strdup(fn);
260  }
261  else
262  {
263  ctx->filename = NULL;
264  ctx->TH = NULL;
265  }
266  te->formatData = (void *) ctx;
267 }
268 
269 static void
271 {
272  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
273 
274  if (ctx->filename)
275  WriteStr(AH, ctx->filename);
276  else
277  WriteStr(AH, "");
278 }
279 
280 static void
282 {
283  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
284 
285  if (ctx == NULL)
286  {
287  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
288  te->formatData = (void *) ctx;
289  }
290 
291  ctx->filename = ReadStr(AH);
292  if (strlen(ctx->filename) == 0)
293  {
294  free(ctx->filename);
295  ctx->filename = NULL;
296  }
297  ctx->TH = NULL;
298 }
299 
300 static void
302 {
303  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
304 
305  if (AH->public.verbose && ctx->filename != NULL)
306  ahprintf(AH, "-- File: %s\n", ctx->filename);
307 }
308 
309 static void
311 {
312  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
313 
314  tctx->TH = tarOpen(AH, tctx->filename, 'w');
315 }
316 
317 static TAR_MEMBER *
318 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
319 {
320  lclContext *ctx = (lclContext *) AH->formatData;
321  TAR_MEMBER *tm;
322 
323 #ifdef HAVE_LIBZ
324  char fmode[14];
325 #endif
326 
327  if (mode == 'r')
328  {
329  tm = _tarPositionTo(AH, filename);
330  if (!tm) /* Not found */
331  {
332  if (filename)
333  {
334  /*
335  * Couldn't find the requested file. Future: do SEEK(0) and
336  * retry.
337  */
338  fatal("could not find file \"%s\" in archive", filename);
339  }
340  else
341  {
342  /* Any file OK, none left, so return NULL */
343  return NULL;
344  }
345  }
346 
347 #ifdef HAVE_LIBZ
348 
349  if (AH->compression == 0)
350  tm->nFH = ctx->tarFH;
351  else
352  fatal("compression is not supported by tar archive format");
353  /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
354 #else
355  tm->nFH = ctx->tarFH;
356 #endif
357  }
358  else
359  {
360  int old_umask;
361 
362  tm = pg_malloc0(sizeof(TAR_MEMBER));
363 
364  /*
365  * POSIX does not require, but permits, tmpfile() to restrict file
366  * permissions. Given an OS crash after we write data, the filesystem
367  * might retain the data but forget tmpfile()'s unlink(). If so, the
368  * file mode protects confidentiality of the data written.
369  */
370  old_umask = umask(S_IRWXG | S_IRWXO);
371 
372 #ifndef WIN32
373  tm->tmpFH = tmpfile();
374 #else
375 
376  /*
377  * On WIN32, tmpfile() generates a filename in the root directory,
378  * which requires administrative permissions on certain systems. Loop
379  * until we find a unique file name we can create.
380  */
381  while (1)
382  {
383  char *name;
384  int fd;
385 
386  name = _tempnam(NULL, "pg_temp_");
387  if (name == NULL)
388  break;
389  fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
390  O_TEMPORARY, S_IRUSR | S_IWUSR);
391  free(name);
392 
393  if (fd != -1) /* created a file */
394  {
395  tm->tmpFH = fdopen(fd, "w+b");
396  break;
397  }
398  else if (errno != EEXIST) /* failure other than file exists */
399  break;
400  }
401 #endif
402 
403  if (tm->tmpFH == NULL)
404  fatal("could not generate temporary file name: %m");
405 
406  umask(old_umask);
407 
408 #ifdef HAVE_LIBZ
409 
410  if (AH->compression != 0)
411  {
412  sprintf(fmode, "wb%d", AH->compression);
413  tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
414  if (tm->zFH == NULL)
415  fatal("could not open temporary file");
416  }
417  else
418  tm->nFH = tm->tmpFH;
419 #else
420 
421  tm->nFH = tm->tmpFH;
422 #endif
423 
424  tm->AH = AH;
425  tm->targetFile = pg_strdup(filename);
426  }
427 
428  tm->mode = mode;
429  tm->tarFH = ctx->tarFH;
430 
431  return tm;
432 }
433 
434 static void
436 {
437  /*
438  * Close the GZ file since we dup'd. This will flush the buffers.
439  */
440  if (AH->compression != 0)
441  {
442  errno = 0; /* in case gzclose() doesn't set it */
443  if (GZCLOSE(th->zFH) != 0)
444  fatal("could not close tar member: %m");
445  }
446 
447  if (th->mode == 'w')
448  _tarAddFile(AH, th); /* This will close the temp file */
449 
450  /*
451  * else Nothing to do for normal read since we don't dup() normal file
452  * handle, and we don't use temp files.
453  */
454 
455  if (th->targetFile)
456  free(th->targetFile);
457 
458  th->nFH = NULL;
459  th->zFH = NULL;
460 }
461 
462 #ifdef __NOT_USED__
463 static char *
464 tarGets(char *buf, size_t len, TAR_MEMBER *th)
465 {
466  char *s;
467  size_t cnt = 0;
468  char c = ' ';
469  int eof = 0;
470 
471  /* Can't read past logical EOF */
472  if (len > (th->fileLen - th->pos))
473  len = th->fileLen - th->pos;
474 
475  while (cnt < len && c != '\n')
476  {
477  if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
478  {
479  eof = 1;
480  break;
481  }
482  buf[cnt++] = c;
483  }
484 
485  if (eof && cnt == 0)
486  s = NULL;
487  else
488  {
489  buf[cnt++] = '\0';
490  s = buf;
491  }
492 
493  if (s)
494  {
495  len = strlen(s);
496  th->pos += len;
497  }
498 
499  return s;
500 }
501 #endif
502 
503 /*
504  * Just read bytes from the archive. This is the low level read routine
505  * that is used for ALL reads on a tar file.
506  */
507 static size_t
508 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
509 {
510  lclContext *ctx = (lclContext *) AH->formatData;
511  size_t avail;
512  size_t used = 0;
513  size_t res = 0;
514 
515  Assert(th || fh);
516 
517  avail = AH->lookaheadLen - AH->lookaheadPos;
518  if (avail > 0)
519  {
520  /* We have some lookahead bytes to use */
521  if (avail >= len) /* Just use the lookahead buffer */
522  used = len;
523  else
524  used = avail;
525 
526  /* Copy, and adjust buffer pos */
527  memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
528  AH->lookaheadPos += used;
529 
530  /* Adjust required length */
531  len -= used;
532  }
533 
534  /* Read the file if len > 0 */
535  if (len > 0)
536  {
537  if (fh)
538  {
539  res = fread(&((char *) buf)[used], 1, len, fh);
540  if (res != len && !feof(fh))
541  READ_ERROR_EXIT(fh);
542  }
543  else if (th)
544  {
545  if (th->zFH)
546  {
547  res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
548  if (res != len && !GZEOF(th->zFH))
549  {
550 #ifdef HAVE_LIBZ
551  int errnum;
552  const char *errmsg = gzerror(th->zFH, &errnum);
553 
554  fatal("could not read from input file: %s",
555  errnum == Z_ERRNO ? strerror(errno) : errmsg);
556 #else
557  fatal("could not read from input file: %s",
558  strerror(errno));
559 #endif
560  }
561  }
562  else
563  {
564  res = fread(&((char *) buf)[used], 1, len, th->nFH);
565  if (res != len && !feof(th->nFH))
566  READ_ERROR_EXIT(th->nFH);
567  }
568  }
569  }
570 
571  ctx->tarFHpos += res + used;
572 
573  return (res + used);
574 }
575 
576 static size_t
577 tarRead(void *buf, size_t len, TAR_MEMBER *th)
578 {
579  size_t res;
580 
581  if (th->pos + len > th->fileLen)
582  len = th->fileLen - th->pos;
583 
584  if (len <= 0)
585  return 0;
586 
587  res = _tarReadRaw(th->AH, buf, len, th, NULL);
588 
589  th->pos += res;
590 
591  return res;
592 }
593 
594 static size_t
595 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
596 {
597  size_t res;
598 
599  if (th->zFH != NULL)
600  res = GZWRITE(buf, 1, len, th->zFH);
601  else
602  res = fwrite(buf, 1, len, th->nFH);
603 
604  th->pos += res;
605  return res;
606 }
607 
608 static void
609 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
610 {
611  lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
612 
613  if (tarWrite(data, dLen, tctx->TH) != dLen)
615 }
616 
617 static void
619 {
620  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
621 
622  /* Close the file */
623  tarClose(AH, tctx->TH);
624  tctx->TH = NULL;
625 }
626 
627 /*
628  * Print data for a given file
629  */
630 static void
632 {
633  lclContext *ctx = (lclContext *) AH->formatData;
634  char buf[4096];
635  size_t cnt;
636  TAR_MEMBER *th;
637 
638  if (!filename)
639  return;
640 
641  th = tarOpen(AH, filename, 'r');
642  ctx->FH = th;
643 
644  while ((cnt = tarRead(buf, 4095, th)) > 0)
645  {
646  buf[cnt] = '\0';
647  ahwrite(buf, 1, cnt, AH);
648  }
649 
650  tarClose(AH, th);
651 }
652 
653 
654 /*
655  * Print data for a given TOC entry
656 */
657 static void
659 {
660  lclContext *ctx = (lclContext *) AH->formatData;
661  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
662  int pos1;
663 
664  if (!tctx->filename)
665  return;
666 
667  /*
668  * If we're writing the special restore.sql script, emit a suitable
669  * command to include each table's data from the corresponding file.
670  *
671  * In the COPY case this is a bit klugy because the regular COPY command
672  * was already printed before we get control.
673  */
674  if (ctx->isSpecialScript)
675  {
676  if (te->copyStmt)
677  {
678  /* Abort the COPY FROM stdin */
679  ahprintf(AH, "\\.\n");
680 
681  /*
682  * The COPY statement should look like "COPY ... FROM stdin;\n",
683  * see dumpTableData().
684  */
685  pos1 = (int) strlen(te->copyStmt) - 13;
686  if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
687  strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
688  fatal("unexpected COPY statement syntax: \"%s\"",
689  te->copyStmt);
690 
691  /* Emit all but the FROM part ... */
692  ahwrite(te->copyStmt, 1, pos1, AH);
693  /* ... and insert modified FROM */
694  ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
695  }
696  else
697  {
698  /* --inserts mode, no worries, just include the data file */
699  ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
700  }
701 
702  return;
703  }
704 
705  if (strcmp(te->desc, "BLOBS") == 0)
706  _LoadBlobs(AH);
707  else
708  _PrintFileData(AH, tctx->filename);
709 }
710 
711 static void
713 {
714  Oid oid;
715  lclContext *ctx = (lclContext *) AH->formatData;
716  TAR_MEMBER *th;
717  size_t cnt;
718  bool foundBlob = false;
719  char buf[4096];
720 
721  StartRestoreBlobs(AH);
722 
723  th = tarOpen(AH, NULL, 'r'); /* Open next file */
724  while (th != NULL)
725  {
726  ctx->FH = th;
727 
728  if (strncmp(th->targetFile, "blob_", 5) == 0)
729  {
730  oid = atooid(&th->targetFile[5]);
731  if (oid != 0)
732  {
733  pg_log_info("restoring large object with OID %u", oid);
734 
735  StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
736 
737  while ((cnt = tarRead(buf, 4095, th)) > 0)
738  {
739  buf[cnt] = '\0';
740  ahwrite(buf, 1, cnt, AH);
741  }
742  EndRestoreBlob(AH, oid);
743  foundBlob = true;
744  }
745  tarClose(AH, th);
746  }
747  else
748  {
749  tarClose(AH, th);
750 
751  /*
752  * Once we have found the first blob, stop at the first non-blob
753  * entry (which will be 'blobs.toc'). This coding would eat all
754  * the rest of the archive if there are no blobs ... but this
755  * function shouldn't be called at all in that case.
756  */
757  if (foundBlob)
758  break;
759  }
760 
761  th = tarOpen(AH, NULL, 'r');
762  }
763  EndRestoreBlobs(AH);
764 }
765 
766 
767 static int
768 _WriteByte(ArchiveHandle *AH, const int i)
769 {
770  lclContext *ctx = (lclContext *) AH->formatData;
771  char b = i; /* Avoid endian problems */
772 
773  if (tarWrite(&b, 1, ctx->FH) != 1)
775 
776  ctx->filePos += 1;
777  return 1;
778 }
779 
780 static int
782 {
783  lclContext *ctx = (lclContext *) AH->formatData;
784  size_t res;
785  unsigned char c;
786 
787  res = tarRead(&c, 1, ctx->FH);
788  if (res != 1)
789  /* We already would have exited for errors on reads, must be EOF */
790  fatal("could not read from input file: end of file");
791  ctx->filePos += 1;
792  return c;
793 }
794 
795 static void
796 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
797 {
798  lclContext *ctx = (lclContext *) AH->formatData;
799 
800  if (tarWrite(buf, len, ctx->FH) != len)
802 
803  ctx->filePos += len;
804 }
805 
806 static void
807 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
808 {
809  lclContext *ctx = (lclContext *) AH->formatData;
810 
811  if (tarRead(buf, len, ctx->FH) != len)
812  /* We already would have exited for errors on reads, must be EOF */
813  fatal("could not read from input file: end of file");
814 
815  ctx->filePos += len;
816 }
817 
818 static void
820 {
821  lclContext *ctx = (lclContext *) AH->formatData;
822  TAR_MEMBER *th;
823  RestoreOptions *ropt;
824  RestoreOptions *savRopt;
825  DumpOptions *savDopt;
826  int savVerbose,
827  i;
828 
829  if (AH->mode == archModeWrite)
830  {
831  /*
832  * Write the Header & TOC to the archive FIRST
833  */
834  th = tarOpen(AH, "toc.dat", 'w');
835  ctx->FH = th;
836  WriteHead(AH);
837  WriteToc(AH);
838  tarClose(AH, th); /* Not needed any more */
839 
840  /*
841  * Now send the data (tables & blobs)
842  */
843  WriteDataChunks(AH, NULL);
844 
845  /*
846  * Now this format wants to append a script which does a full restore
847  * if the files have been extracted.
848  */
849  th = tarOpen(AH, "restore.sql", 'w');
850 
851  tarPrintf(th, "--\n"
852  "-- NOTE:\n"
853  "--\n"
854  "-- File paths need to be edited. Search for $$PATH$$ and\n"
855  "-- replace it with the path to the directory containing\n"
856  "-- the extracted data files.\n"
857  "--\n");
858 
859  AH->CustomOutPtr = _scriptOut;
860 
861  ctx->isSpecialScript = 1;
862  ctx->scriptTH = th;
863 
864  ropt = NewRestoreOptions();
865  memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
866  ropt->filename = NULL;
867  ropt->dropSchema = 1;
868  ropt->compression = 0;
869  ropt->superuser = NULL;
870  ropt->suppressDumpWarnings = true;
871 
872  savDopt = AH->public.dopt;
873  savRopt = AH->public.ropt;
874 
875  SetArchiveOptions((Archive *) AH, NULL, ropt);
876 
877  savVerbose = AH->public.verbose;
878  AH->public.verbose = 0;
879 
880  RestoreArchive((Archive *) AH);
881 
882  SetArchiveOptions((Archive *) AH, savDopt, savRopt);
883 
884  AH->public.verbose = savVerbose;
885 
886  tarClose(AH, th);
887 
888  ctx->isSpecialScript = 0;
889 
890  /*
891  * EOF marker for tar files is two blocks of NULLs.
892  */
893  for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
894  {
895  if (fputc(0, ctx->tarFH) == EOF)
897  }
898 
899  /* Sync the output file if one is defined */
900  if (AH->dosync && AH->fSpec)
901  (void) fsync_fname(AH->fSpec, false);
902  }
903 
904  AH->FH = NULL;
905 }
906 
907 static size_t
908 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
909 {
910  lclContext *ctx = (lclContext *) AH->formatData;
911 
912  return tarWrite(buf, len, ctx->scriptTH);
913 }
914 
915 /*
916  * BLOB support
917  */
918 
919 /*
920  * Called by the archiver when starting to save all BLOB DATA (not schema).
921  * This routine should save whatever format-specific information is needed
922  * to read the BLOBs back into memory.
923  *
924  * It is called just prior to the dumper's DataDumper routine.
925  *
926  * Optional, but strongly recommended.
927  *
928  */
929 static void
931 {
932  lclContext *ctx = (lclContext *) AH->formatData;
933  char fname[K_STD_BUF_SIZE];
934 
935  sprintf(fname, "blobs.toc");
936  ctx->blobToc = tarOpen(AH, fname, 'w');
937 }
938 
939 /*
940  * Called by the archiver when the dumper calls StartBlob.
941  *
942  * Mandatory.
943  *
944  * Must save the passed OID for retrieval at restore-time.
945  */
946 static void
948 {
949  lclContext *ctx = (lclContext *) AH->formatData;
950  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
951  char fname[255];
952  char *sfx;
953 
954  if (oid == 0)
955  fatal("invalid OID for large object (%u)", oid);
956 
957  if (AH->compression != 0)
958  sfx = ".gz";
959  else
960  sfx = "";
961 
962  sprintf(fname, "blob_%u.dat%s", oid, sfx);
963 
964  tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
965 
966  tctx->TH = tarOpen(AH, fname, 'w');
967 }
968 
969 /*
970  * Called by the archiver when the dumper calls EndBlob.
971  *
972  * Optional.
973  *
974  */
975 static void
977 {
978  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
979 
980  tarClose(AH, tctx->TH);
981 }
982 
983 /*
984  * Called by the archiver when finishing saving all BLOB DATA.
985  *
986  * Optional.
987  *
988  */
989 static void
991 {
992  lclContext *ctx = (lclContext *) AH->formatData;
993 
994  /* Write out a fake zero OID to mark end-of-blobs. */
995  /* WriteInt(AH, 0); */
996 
997  tarClose(AH, ctx->blobToc);
998 }
999 
1000 
1001 
1002 /*------------
1003  * TAR Support
1004  *------------
1005  */
1006 
1007 static int
1008 tarPrintf(TAR_MEMBER *th, const char *fmt,...)
1009 {
1010  int save_errno = errno;
1011  char *p;
1012  size_t len = 128; /* initial assumption about buffer size */
1013  size_t cnt;
1014 
1015  for (;;)
1016  {
1017  va_list args;
1018 
1019  /* Allocate work buffer. */
1020  p = (char *) pg_malloc(len);
1021 
1022  /* Try to format the data. */
1023  errno = save_errno;
1024  va_start(args, fmt);
1025  cnt = pvsnprintf(p, len, fmt, args);
1026  va_end(args);
1027 
1028  if (cnt < len)
1029  break; /* success */
1030 
1031  /* Release buffer and loop around to try again with larger len. */
1032  free(p);
1033  len = cnt;
1034  }
1035 
1036  cnt = tarWrite(p, cnt, th);
1037  free(p);
1038  return (int) cnt;
1039 }
1040 
1041 bool
1043 {
1044  int sum;
1045  int chk = tarChecksum(header);
1046 
1047  sum = read_tar_number(&header[148], 8);
1048 
1049  if (sum != chk)
1050  return false;
1051 
1052  /* POSIX tar format */
1053  if (memcmp(&header[257], "ustar\0", 6) == 0 &&
1054  memcmp(&header[263], "00", 2) == 0)
1055  return true;
1056  /* GNU tar format */
1057  if (memcmp(&header[257], "ustar \0", 8) == 0)
1058  return true;
1059  /* not-quite-POSIX format written by pre-9.3 pg_dump */
1060  if (memcmp(&header[257], "ustar00\0", 8) == 0)
1061  return true;
1062 
1063  return false;
1064 }
1065 
1066 /* Given the member, write the TAR header & copy the file */
1067 static void
1069 {
1070  lclContext *ctx = (lclContext *) AH->formatData;
1071  FILE *tmp = th->tmpFH; /* Grab it for convenience */
1072  char buf[32768];
1073  size_t cnt;
1074  pgoff_t len = 0;
1075  size_t res;
1076  size_t i,
1077  pad;
1078 
1079  /*
1080  * Find file len & go back to start.
1081  */
1082  if (fseeko(tmp, 0, SEEK_END) != 0)
1083  fatal("error during file seek: %m");
1084  th->fileLen = ftello(tmp);
1085  if (th->fileLen < 0)
1086  fatal("could not determine seek position in archive file: %m");
1087  if (fseeko(tmp, 0, SEEK_SET) != 0)
1088  fatal("error during file seek: %m");
1089 
1090  _tarWriteHeader(th);
1091 
1092  while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1093  {
1094  if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1096  len += res;
1097  }
1098  if (!feof(tmp))
1099  READ_ERROR_EXIT(tmp);
1100 
1101  if (fclose(tmp) != 0) /* This *should* delete it... */
1102  fatal("could not close temporary file: %m");
1103 
1104  if (len != th->fileLen)
1105  {
1106  char buf1[32],
1107  buf2[32];
1108 
1109  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) len);
1110  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) th->fileLen);
1111  fatal("actual file length (%s) does not match expected (%s)",
1112  buf1, buf2);
1113  }
1114 
1116  for (i = 0; i < pad; i++)
1117  {
1118  if (fputc('\0', th->tarFH) == EOF)
1120  }
1121 
1122  ctx->tarFHpos += len + pad;
1123 }
1124 
1125 /* Locate the file in the archive, read header and position to data */
1126 static TAR_MEMBER *
1128 {
1129  lclContext *ctx = (lclContext *) AH->formatData;
1130  TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
1131  char c;
1132  char header[TAR_BLOCK_SIZE];
1133  size_t i,
1134  len,
1135  blks;
1136  int id;
1137 
1138  th->AH = AH;
1139 
1140  /* Go to end of current file, if any */
1141  if (ctx->tarFHpos != 0)
1142  {
1143  char buf1[100],
1144  buf2[100];
1145 
1146  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ctx->tarFHpos);
1147  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ctx->tarNextMember);
1148  pg_log_debug("moving from position %s to next member at file position %s",
1149  buf1, buf2);
1150 
1151  while (ctx->tarFHpos < ctx->tarNextMember)
1152  _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1153  }
1154 
1155  {
1156  char buf[100];
1157 
1158  snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ctx->tarFHpos);
1159  pg_log_debug("now at file position %s", buf);
1160  }
1161 
1162  /* We are at the start of the file, or at the next member */
1163 
1164  /* Get the header */
1165  if (!_tarGetHeader(AH, th))
1166  {
1167  if (filename)
1168  fatal("could not find header for file \"%s\" in tar archive", filename);
1169  else
1170  {
1171  /*
1172  * We're just scanning the archive for the next file, so return
1173  * null
1174  */
1175  free(th);
1176  return NULL;
1177  }
1178  }
1179 
1180  while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1181  {
1182  pg_log_debug("skipping tar member %s", th->targetFile);
1183 
1184  id = atoi(th->targetFile);
1185  if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1186  fatal("restoring data out of order is not supported in this archive format: "
1187  "\"%s\" is required, but comes before \"%s\" in the archive file.",
1188  th->targetFile, filename);
1189 
1190  /* Header doesn't match, so read to next header */
1191  len = th->fileLen;
1193  blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1194 
1195  for (i = 0; i < blks; i++)
1196  _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1197 
1198  if (!_tarGetHeader(AH, th))
1199  fatal("could not find header for file \"%s\" in tar archive", filename);
1200  }
1201 
1202  ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1204  th->pos = 0;
1205 
1206  return th;
1207 }
1208 
1209 /* Read & verify a header */
1210 static int
1212 {
1213  lclContext *ctx = (lclContext *) AH->formatData;
1214  char h[TAR_BLOCK_SIZE];
1215  char tag[100 + 1];
1216  int sum,
1217  chk;
1218  pgoff_t len;
1219  pgoff_t hPos;
1220  bool gotBlock = false;
1221 
1222  while (!gotBlock)
1223  {
1224  /* Save the pos for reporting purposes */
1225  hPos = ctx->tarFHpos;
1226 
1227  /* Read the next tar block, return EOF, exit if short */
1228  len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1229  if (len == 0) /* EOF */
1230  return 0;
1231 
1232  if (len != TAR_BLOCK_SIZE)
1233  fatal(ngettext("incomplete tar header found (%lu byte)",
1234  "incomplete tar header found (%lu bytes)",
1235  len),
1236  (unsigned long) len);
1237 
1238  /* Calc checksum */
1239  chk = tarChecksum(h);
1240  sum = read_tar_number(&h[148], 8);
1241 
1242  /*
1243  * If the checksum failed, see if it is a null block. If so, silently
1244  * continue to the next block.
1245  */
1246  if (chk == sum)
1247  gotBlock = true;
1248  else
1249  {
1250  int i;
1251 
1252  for (i = 0; i < TAR_BLOCK_SIZE; i++)
1253  {
1254  if (h[i] != 0)
1255  {
1256  gotBlock = true;
1257  break;
1258  }
1259  }
1260  }
1261  }
1262 
1263  /* Name field is 100 bytes, might not be null-terminated */
1264  strlcpy(tag, &h[0], 100 + 1);
1265 
1266  len = read_tar_number(&h[124], 12);
1267 
1268  {
1269  char posbuf[32];
1270  char lenbuf[32];
1271 
1272  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
1273  snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
1274  pg_log_debug("TOC Entry %s at %s (length %s, checksum %d)",
1275  tag, posbuf, lenbuf, sum);
1276  }
1277 
1278  if (chk != sum)
1279  {
1280  char posbuf[32];
1281 
1282  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
1283  (uint64) ftello(ctx->tarFH));
1284  fatal("corrupt tar header found in %s (expected %d, computed %d) file position %s",
1285  tag, sum, chk, posbuf);
1286  }
1287 
1288  th->targetFile = pg_strdup(tag);
1289  th->fileLen = len;
1290 
1291  return 1;
1292 }
1293 
1294 
1295 static void
1297 {
1298  char h[TAR_BLOCK_SIZE];
1299 
1300  tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1301  0600, 04000, 02000, time(NULL));
1302 
1303  /* Now write the completed header. */
1304  if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1306 }
#define PG_BINARY_R
Definition: c.h:1270
#define ngettext(s, p, n)
Definition: c.h:1179
#define INT64_FORMAT
Definition: c.h:483
#define pg_attribute_printf(f, a)
Definition: c.h:164
#define UINT64_FORMAT
Definition: c.h:484
#define PG_BINARY_W
Definition: c.h:1271
int errmsg(const char *fmt,...)
Definition: elog.c:909
const char * name
Definition: encode.c:561
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:673
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define TH
#define free(a)
Definition: header.h:65
int b
Definition: isn.c:70
int i
Definition: isn.c:73
va_end(args)
Assert(fmt[strlen(fmt) - 1] !='\n')
static void const char * fmt
va_start(args, fmt)
static struct pg_tm tm
Definition: localtime.c:102
#define pg_log_info(...)
Definition: logging.h:88
#define pg_log_debug(...)
Definition: logging.h:92
void RestoreArchive(Archive *AH)
RestoreOptions * NewRestoreOptions(void)
@ archModeWrite
Definition: pg_backup.h:49
void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ropt)
int TocIDRequired(ArchiveHandle *AH, DumpId id)
bool checkSeek(FILE *fp)
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteHead(ArchiveHandle *AH)
void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void StartRestoreBlobs(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void WriteToc(ArchiveHandle *AH)
void EndRestoreBlob(ArchiveHandle *AH, Oid oid)
void EndRestoreBlobs(ArchiveHandle *AH)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define LOBBUFSIZE
#define GZEOF(fh)
#define GZWRITE(p, s, n, fh)
#define WRITE_ERROR_EXIT
#define GZREAD(p, s, n, fh)
#define REQ_DATA
#define READ_ERROR_EXIT(fd)
#define GZCLOSE(fh)
static void _LoadBlobs(ArchiveHandle *AH)
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2
static void _tarWriteHeader(TAR_MEMBER *th)
void InitArchiveFmt_Tar(ArchiveHandle *AH)
static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
#define K_STD_BUF_SIZE
Definition: pg_backup_tar.c:63
static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
bool isValidTarHeader(char *header)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static TAR_MEMBER * tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static void _EndBlobs(ArchiveHandle *AH, TocEntry *te)
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
static int _ReadByte(ArchiveHandle *)
static TAR_MEMBER * _tarPositionTo(ArchiveHandle *AH, const char *filename)
static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static int static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
#define fatal(...)
static PgChecksumMode mode
Definition: pg_checksums.c:65
const void size_t len
const void * data
while(p+4<=pend)
static char * filename
Definition: pg_dumpall.c:92
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:212
static char * buf
Definition: pg_test_fsync.c:70
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:58
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
int tarChecksum(char *header)
Definition: tar.c:90
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define sprintf
Definition: port.h:224
#define strerror
Definition: port.h:235
#define snprintf
Definition: port.h:222
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
char * c
static int fd(const char *x, int i)
Definition: preproc-init.c:105
size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args)
Definition: psprintf.c:106
DumpOptions * dopt
Definition: pg_backup.h:191
int verbose
Definition: pg_backup.h:194
RestoreOptions * ropt
Definition: pg_backup.h:192
pgoff_t pos
Definition: pg_backup_tar.c:78
char * targetFile
Definition: pg_backup_tar.c:76
pgoff_t fileLen
Definition: pg_backup_tar.c:79
FILE * tarFH
Definition: pg_backup_tar.c:74
ArchiveHandle * AH
Definition: pg_backup_tar.c:80
FILE * zFH
Definition: pg_backup_tar.c:71
FILE * nFH
Definition: pg_backup_tar.c:73
FILE * tmpFH
Definition: pg_backup_tar.c:75
CustomOutPtrType CustomOutPtr
struct _tocEntry * currToc
DataDumperPtr dataDumper
FILE * tarFH
Definition: pg_backup_tar.c:88
pgoff_t filePos
Definition: pg_backup_tar.c:86
pgoff_t tarFHpos
Definition: pg_backup_tar.c:89
TAR_MEMBER * blobToc
Definition: pg_backup_tar.c:87
TAR_MEMBER * FH
Definition: pg_backup_tar.c:91
pgoff_t tarNextMember
Definition: pg_backup_tar.c:90
TAR_MEMBER * scriptTH
Definition: pg_backup_tar.c:93
int isSpecialScript
Definition: pg_backup_tar.c:92
TAR_MEMBER * TH
Definition: pg_backup_tar.c:98
static void * fn(void *arg)
#define S_IRWXG
Definition: win32_port.h:309
#define S_IRWXO
Definition: win32_port.h:321
#define ftello(stream)
Definition: win32_port.h:218
#define S_IRUSR
Definition: win32_port.h:288
#define fseeko(stream, offset, origin)
Definition: win32_port.h:215
#define S_IWUSR
Definition: win32_port.h:291
#define pgoff_t
Definition: win32_port.h:208