PostgreSQL Source Code  git master
pg_backup_tar.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_tar.c
4  *
5  * This file is copied from the 'files' format file, but dumps data into
6  * one temp file then sends it to the output TAR archive.
7  *
8  * The tar format also includes a 'restore.sql' script which is there for
9  * the benefit of humans. This script is never used by pg_restore.
10  *
11  * NOTE: If you untar the created 'tar' file, the resulting files are
12  * compatible with the 'directory' format. Please keep the two formats in
13  * sync.
14  *
15  * See the headers to pg_backup_directory & pg_restore for more details.
16  *
17  * Copyright (c) 2000, Philip Warner
18  * Rights are granted to use this software in any way so long
19  * as this notice is not removed.
20  *
21  * The author is not responsible for loss or damages that may
22  * result from its use.
23  *
24  *
25  * IDENTIFICATION
26  * src/bin/pg_dump/pg_backup_tar.c
27  *
28  *-------------------------------------------------------------------------
29  */
30 #include "postgres_fe.h"
31 
32 #include "pg_backup_archiver.h"
33 #include "pg_backup_tar.h"
34 #include "pg_backup_utils.h"
35 #include "pgtar.h"
36 #include "common/file_utils.h"
37 #include "fe_utils/string_utils.h"
38 
39 #include <sys/stat.h>
40 #include <ctype.h>
41 #include <limits.h>
42 #include <unistd.h>
43 
44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 static int _WriteByte(ArchiveHandle *AH, const int i);
49 static int _ReadByte(ArchiveHandle *);
50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 static void _CloseArchive(ArchiveHandle *AH);
53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 
58 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
59 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
62 
63 #define K_STD_BUF_SIZE 1024
64 
65 
66 typedef struct
67 {
68 #ifdef HAVE_LIBZ
69  gzFile zFH;
70 #else
71  FILE *zFH;
72 #endif
73  FILE *nFH;
74  FILE *tarFH;
75  FILE *tmpFH;
76  char *targetFile;
77  char mode;
81 } TAR_MEMBER;
82 
83 typedef struct
84 {
85  int hasSeek;
86  pgoff_t filePos;
88  FILE *tarFH;
94 } lclContext;
95 
96 typedef struct
97 {
99  char *filename;
100 } lclTocEntry;
101 
102 static void _LoadBlobs(ArchiveHandle *AH);
103 
104 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
105 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
106 
107 #ifdef __NOT_USED__
108 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
109 #endif
110 static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3, 4);
111 
112 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
113 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
114 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
115 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
116 static void _tarWriteHeader(TAR_MEMBER *th);
117 static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
118 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
119 
120 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
121 
122 /*
123  * Initializer
124  */
125 void
127 {
128  lclContext *ctx;
129 
130  /* Assuming static functions, this can be copied for each format. */
131  AH->ArchiveEntryPtr = _ArchiveEntry;
132  AH->StartDataPtr = _StartData;
133  AH->WriteDataPtr = _WriteData;
134  AH->EndDataPtr = _EndData;
135  AH->WriteBytePtr = _WriteByte;
136  AH->ReadBytePtr = _ReadByte;
137  AH->WriteBufPtr = _WriteBuf;
138  AH->ReadBufPtr = _ReadBuf;
139  AH->ClosePtr = _CloseArchive;
140  AH->ReopenPtr = NULL;
141  AH->PrintTocDataPtr = _PrintTocData;
142  AH->ReadExtraTocPtr = _ReadExtraToc;
143  AH->WriteExtraTocPtr = _WriteExtraToc;
144  AH->PrintExtraTocPtr = _PrintExtraToc;
145 
146  AH->StartBlobsPtr = _StartBlobs;
147  AH->StartBlobPtr = _StartBlob;
148  AH->EndBlobPtr = _EndBlob;
149  AH->EndBlobsPtr = _EndBlobs;
150  AH->ClonePtr = NULL;
151  AH->DeClonePtr = NULL;
152 
153  AH->WorkerJobDumpPtr = NULL;
154  AH->WorkerJobRestorePtr = NULL;
155 
156  /*
157  * Set up some special context used in compressing data.
158  */
159  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
160  AH->formatData = (void *) ctx;
161  ctx->filePos = 0;
162  ctx->isSpecialScript = 0;
163 
164  /* Initialize LO buffering */
165  AH->lo_buf_size = LOBBUFSIZE;
166  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
167 
168  /*
169  * Now open the tar file, and load the TOC if we're in read mode.
170  */
171  if (AH->mode == archModeWrite)
172  {
173  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
174  {
175  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
176  if (ctx->tarFH == NULL)
177  fatal("could not open TOC file \"%s\" for output: %m",
178  AH->fSpec);
179  }
180  else
181  {
182  ctx->tarFH = stdout;
183  if (ctx->tarFH == NULL)
184  fatal("could not open TOC file for output: %m");
185  }
186 
187  ctx->tarFHpos = 0;
188 
189  /*
190  * Make unbuffered since we will dup() it, and the buffers screw each
191  * other
192  */
193  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
194 
195  ctx->hasSeek = checkSeek(ctx->tarFH);
196 
197  /*
198  * We don't support compression because reading the files back is not
199  * possible since gzdopen uses buffered IO which totally screws file
200  * positioning.
201  */
202  if (AH->compression != 0)
203  fatal("compression is not supported by tar archive format");
204  }
205  else
206  { /* Read Mode */
207  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
208  {
209  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
210  if (ctx->tarFH == NULL)
211  fatal("could not open TOC file \"%s\" for input: %m",
212  AH->fSpec);
213  }
214  else
215  {
216  ctx->tarFH = stdin;
217  if (ctx->tarFH == NULL)
218  fatal("could not open TOC file for input: %m");
219  }
220 
221  /*
222  * Make unbuffered since we will dup() it, and the buffers screw each
223  * other
224  */
225  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
226 
227  ctx->tarFHpos = 0;
228 
229  ctx->hasSeek = checkSeek(ctx->tarFH);
230 
231  /*
232  * Forcibly unmark the header as read since we use the lookahead
233  * buffer
234  */
235  AH->readHeader = 0;
236 
237  ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
238  ReadHead(AH);
239  ReadToc(AH);
240  tarClose(AH, ctx->FH); /* Nothing else in the file... */
241  }
242 }
243 
244 /*
245  * - Start a new TOC entry
246  * Setup the output file name.
247  */
248 static void
250 {
251  lclTocEntry *ctx;
252  char fn[K_STD_BUF_SIZE];
253 
254  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
255  if (te->dataDumper != NULL)
256  {
257 #ifdef HAVE_LIBZ
258  if (AH->compression == 0)
259  sprintf(fn, "%d.dat", te->dumpId);
260  else
261  sprintf(fn, "%d.dat.gz", te->dumpId);
262 #else
263  sprintf(fn, "%d.dat", te->dumpId);
264 #endif
265  ctx->filename = pg_strdup(fn);
266  }
267  else
268  {
269  ctx->filename = NULL;
270  ctx->TH = NULL;
271  }
272  te->formatData = (void *) ctx;
273 }
274 
275 static void
277 {
278  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
279 
280  if (ctx->filename)
281  WriteStr(AH, ctx->filename);
282  else
283  WriteStr(AH, "");
284 }
285 
286 static void
288 {
289  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
290 
291  if (ctx == NULL)
292  {
293  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
294  te->formatData = (void *) ctx;
295  }
296 
297  ctx->filename = ReadStr(AH);
298  if (strlen(ctx->filename) == 0)
299  {
300  free(ctx->filename);
301  ctx->filename = NULL;
302  }
303  ctx->TH = NULL;
304 }
305 
306 static void
308 {
309  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
310 
311  if (AH->public.verbose && ctx->filename != NULL)
312  ahprintf(AH, "-- File: %s\n", ctx->filename);
313 }
314 
315 static void
317 {
318  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
319 
320  tctx->TH = tarOpen(AH, tctx->filename, 'w');
321 }
322 
323 static TAR_MEMBER *
324 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
325 {
326  lclContext *ctx = (lclContext *) AH->formatData;
327  TAR_MEMBER *tm;
328 
329 #ifdef HAVE_LIBZ
330  char fmode[14];
331 #endif
332 
333  if (mode == 'r')
334  {
335  tm = _tarPositionTo(AH, filename);
336  if (!tm) /* Not found */
337  {
338  if (filename)
339  {
340  /*
341  * Couldn't find the requested file. Future: do SEEK(0) and
342  * retry.
343  */
344  fatal("could not find file \"%s\" in archive", filename);
345  }
346  else
347  {
348  /* Any file OK, none left, so return NULL */
349  return NULL;
350  }
351  }
352 
353 #ifdef HAVE_LIBZ
354 
355  if (AH->compression == 0)
356  tm->nFH = ctx->tarFH;
357  else
358  fatal("compression is not supported by tar archive format");
359  /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
360 #else
361  tm->nFH = ctx->tarFH;
362 #endif
363  }
364  else
365  {
366  int old_umask;
367 
368  tm = pg_malloc0(sizeof(TAR_MEMBER));
369 
370  /*
371  * POSIX does not require, but permits, tmpfile() to restrict file
372  * permissions. Given an OS crash after we write data, the filesystem
373  * might retain the data but forget tmpfile()'s unlink(). If so, the
374  * file mode protects confidentiality of the data written.
375  */
376  old_umask = umask(S_IRWXG | S_IRWXO);
377 
378 #ifndef WIN32
379  tm->tmpFH = tmpfile();
380 #else
381 
382  /*
383  * On WIN32, tmpfile() generates a filename in the root directory,
384  * which requires administrative permissions on certain systems. Loop
385  * until we find a unique file name we can create.
386  */
387  while (1)
388  {
389  char *name;
390  int fd;
391 
392  name = _tempnam(NULL, "pg_temp_");
393  if (name == NULL)
394  break;
395  fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
396  O_TEMPORARY, S_IRUSR | S_IWUSR);
397  free(name);
398 
399  if (fd != -1) /* created a file */
400  {
401  tm->tmpFH = fdopen(fd, "w+b");
402  break;
403  }
404  else if (errno != EEXIST) /* failure other than file exists */
405  break;
406  }
407 #endif
408 
409  if (tm->tmpFH == NULL)
410  fatal("could not generate temporary file name: %m");
411 
412  umask(old_umask);
413 
414 #ifdef HAVE_LIBZ
415 
416  if (AH->compression != 0)
417  {
418  sprintf(fmode, "wb%d", AH->compression);
419  tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
420  if (tm->zFH == NULL)
421  fatal("could not open temporary file");
422  }
423  else
424  tm->nFH = tm->tmpFH;
425 #else
426 
427  tm->nFH = tm->tmpFH;
428 #endif
429 
430  tm->AH = AH;
431  tm->targetFile = pg_strdup(filename);
432  }
433 
434  tm->mode = mode;
435  tm->tarFH = ctx->tarFH;
436 
437  return tm;
438 }
439 
440 static void
442 {
443  /*
444  * Close the GZ file since we dup'd. This will flush the buffers.
445  */
446  if (AH->compression != 0)
447  if (GZCLOSE(th->zFH) != 0)
448  fatal("could not close tar member");
449 
450  if (th->mode == 'w')
451  _tarAddFile(AH, th); /* This will close the temp file */
452 
453  /*
454  * else Nothing to do for normal read since we don't dup() normal file
455  * handle, and we don't use temp files.
456  */
457 
458  if (th->targetFile)
459  free(th->targetFile);
460 
461  th->nFH = NULL;
462  th->zFH = NULL;
463 }
464 
465 #ifdef __NOT_USED__
466 static char *
467 tarGets(char *buf, size_t len, TAR_MEMBER *th)
468 {
469  char *s;
470  size_t cnt = 0;
471  char c = ' ';
472  int eof = 0;
473 
474  /* Can't read past logical EOF */
475  if (len > (th->fileLen - th->pos))
476  len = th->fileLen - th->pos;
477 
478  while (cnt < len && c != '\n')
479  {
480  if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
481  {
482  eof = 1;
483  break;
484  }
485  buf[cnt++] = c;
486  }
487 
488  if (eof && cnt == 0)
489  s = NULL;
490  else
491  {
492  buf[cnt++] = '\0';
493  s = buf;
494  }
495 
496  if (s)
497  {
498  len = strlen(s);
499  th->pos += len;
500  }
501 
502  return s;
503 }
504 #endif
505 
506 /*
507  * Just read bytes from the archive. This is the low level read routine
508  * that is used for ALL reads on a tar file.
509  */
510 static size_t
511 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
512 {
513  lclContext *ctx = (lclContext *) AH->formatData;
514  size_t avail;
515  size_t used = 0;
516  size_t res = 0;
517 
518  avail = AH->lookaheadLen - AH->lookaheadPos;
519  if (avail > 0)
520  {
521  /* We have some lookahead bytes to use */
522  if (avail >= len) /* Just use the lookahead buffer */
523  used = len;
524  else
525  used = avail;
526 
527  /* Copy, and adjust buffer pos */
528  memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
529  AH->lookaheadPos += used;
530 
531  /* Adjust required length */
532  len -= used;
533  }
534 
535  /* Read the file if len > 0 */
536  if (len > 0)
537  {
538  if (fh)
539  {
540  res = fread(&((char *) buf)[used], 1, len, fh);
541  if (res != len && !feof(fh))
542  READ_ERROR_EXIT(fh);
543  }
544  else if (th)
545  {
546  if (th->zFH)
547  {
548  res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
549  if (res != len && !GZEOF(th->zFH))
550  {
551 #ifdef HAVE_LIBZ
552  int errnum;
553  const char *errmsg = gzerror(th->zFH, &errnum);
554 
555  fatal("could not read from input file: %s",
556  errnum == Z_ERRNO ? strerror(errno) : errmsg);
557 #else
558  fatal("could not read from input file: %s",
559  strerror(errno));
560 #endif
561  }
562  }
563  else
564  {
565  res = fread(&((char *) buf)[used], 1, len, th->nFH);
566  if (res != len && !feof(th->nFH))
567  READ_ERROR_EXIT(th->nFH);
568  }
569  }
570  else
571  fatal("internal error -- neither th nor fh specified in _tarReadRaw()");
572  }
573 
574  ctx->tarFHpos += res + used;
575 
576  return (res + used);
577 }
578 
579 static size_t
580 tarRead(void *buf, size_t len, TAR_MEMBER *th)
581 {
582  size_t res;
583 
584  if (th->pos + len > th->fileLen)
585  len = th->fileLen - th->pos;
586 
587  if (len <= 0)
588  return 0;
589 
590  res = _tarReadRaw(th->AH, buf, len, th, NULL);
591 
592  th->pos += res;
593 
594  return res;
595 }
596 
597 static size_t
598 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
599 {
600  size_t res;
601 
602  if (th->zFH != NULL)
603  res = GZWRITE(buf, 1, len, th->zFH);
604  else
605  res = fwrite(buf, 1, len, th->nFH);
606 
607  th->pos += res;
608  return res;
609 }
610 
611 static void
612 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
613 {
614  lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
615 
616  if (tarWrite(data, dLen, tctx->TH) != dLen)
618 
619  return;
620 }
621 
622 static void
624 {
625  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
626 
627  /* Close the file */
628  tarClose(AH, tctx->TH);
629  tctx->TH = NULL;
630 }
631 
632 /*
633  * Print data for a given file
634  */
635 static void
637 {
638  lclContext *ctx = (lclContext *) AH->formatData;
639  char buf[4096];
640  size_t cnt;
641  TAR_MEMBER *th;
642 
643  if (!filename)
644  return;
645 
646  th = tarOpen(AH, filename, 'r');
647  ctx->FH = th;
648 
649  while ((cnt = tarRead(buf, 4095, th)) > 0)
650  {
651  buf[cnt] = '\0';
652  ahwrite(buf, 1, cnt, AH);
653  }
654 
655  tarClose(AH, th);
656 }
657 
658 
659 /*
660  * Print data for a given TOC entry
661 */
662 static void
664 {
665  lclContext *ctx = (lclContext *) AH->formatData;
666  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
667  int pos1;
668 
669  if (!tctx->filename)
670  return;
671 
672  /*
673  * If we're writing the special restore.sql script, emit a suitable
674  * command to include each table's data from the corresponding file.
675  *
676  * In the COPY case this is a bit klugy because the regular COPY command
677  * was already printed before we get control.
678  */
679  if (ctx->isSpecialScript)
680  {
681  if (te->copyStmt)
682  {
683  /* Abort the COPY FROM stdin */
684  ahprintf(AH, "\\.\n");
685 
686  /*
687  * The COPY statement should look like "COPY ... FROM stdin;\n",
688  * see dumpTableData().
689  */
690  pos1 = (int) strlen(te->copyStmt) - 13;
691  if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
692  strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
693  fatal("unexpected COPY statement syntax: \"%s\"",
694  te->copyStmt);
695 
696  /* Emit all but the FROM part ... */
697  ahwrite(te->copyStmt, 1, pos1, AH);
698  /* ... and insert modified FROM */
699  ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
700  }
701  else
702  {
703  /* --inserts mode, no worries, just include the data file */
704  ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
705  }
706 
707  return;
708  }
709 
710  if (strcmp(te->desc, "BLOBS") == 0)
711  _LoadBlobs(AH);
712  else
713  _PrintFileData(AH, tctx->filename);
714 }
715 
716 static void
718 {
719  Oid oid;
720  lclContext *ctx = (lclContext *) AH->formatData;
721  TAR_MEMBER *th;
722  size_t cnt;
723  bool foundBlob = false;
724  char buf[4096];
725 
726  StartRestoreBlobs(AH);
727 
728  th = tarOpen(AH, NULL, 'r'); /* Open next file */
729  while (th != NULL)
730  {
731  ctx->FH = th;
732 
733  if (strncmp(th->targetFile, "blob_", 5) == 0)
734  {
735  oid = atooid(&th->targetFile[5]);
736  if (oid != 0)
737  {
738  pg_log_info("restoring large object with OID %u", oid);
739 
740  StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
741 
742  while ((cnt = tarRead(buf, 4095, th)) > 0)
743  {
744  buf[cnt] = '\0';
745  ahwrite(buf, 1, cnt, AH);
746  }
747  EndRestoreBlob(AH, oid);
748  foundBlob = true;
749  }
750  tarClose(AH, th);
751  }
752  else
753  {
754  tarClose(AH, th);
755 
756  /*
757  * Once we have found the first blob, stop at the first non-blob
758  * entry (which will be 'blobs.toc'). This coding would eat all
759  * the rest of the archive if there are no blobs ... but this
760  * function shouldn't be called at all in that case.
761  */
762  if (foundBlob)
763  break;
764  }
765 
766  th = tarOpen(AH, NULL, 'r');
767  }
768  EndRestoreBlobs(AH);
769 }
770 
771 
772 static int
773 _WriteByte(ArchiveHandle *AH, const int i)
774 {
775  lclContext *ctx = (lclContext *) AH->formatData;
776  char b = i; /* Avoid endian problems */
777 
778  if (tarWrite(&b, 1, ctx->FH) != 1)
780 
781  ctx->filePos += 1;
782  return 1;
783 }
784 
785 static int
787 {
788  lclContext *ctx = (lclContext *) AH->formatData;
789  size_t res;
790  unsigned char c;
791 
792  res = tarRead(&c, 1, ctx->FH);
793  if (res != 1)
794  /* We already would have exited for errors on reads, must be EOF */
795  fatal("could not read from input file: end of file");
796  ctx->filePos += 1;
797  return c;
798 }
799 
800 static void
801 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
802 {
803  lclContext *ctx = (lclContext *) AH->formatData;
804 
805  if (tarWrite(buf, len, ctx->FH) != len)
807 
808  ctx->filePos += len;
809 }
810 
811 static void
812 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
813 {
814  lclContext *ctx = (lclContext *) AH->formatData;
815 
816  if (tarRead(buf, len, ctx->FH) != len)
817  /* We already would have exited for errors on reads, must be EOF */
818  fatal("could not read from input file: end of file");
819 
820  ctx->filePos += len;
821  return;
822 }
823 
824 static void
826 {
827  lclContext *ctx = (lclContext *) AH->formatData;
828  TAR_MEMBER *th;
829  RestoreOptions *ropt;
830  RestoreOptions *savRopt;
831  DumpOptions *savDopt;
832  int savVerbose,
833  i;
834 
835  if (AH->mode == archModeWrite)
836  {
837  /*
838  * Write the Header & TOC to the archive FIRST
839  */
840  th = tarOpen(AH, "toc.dat", 'w');
841  ctx->FH = th;
842  WriteHead(AH);
843  WriteToc(AH);
844  tarClose(AH, th); /* Not needed any more */
845 
846  /*
847  * Now send the data (tables & blobs)
848  */
849  WriteDataChunks(AH, NULL);
850 
851  /*
852  * Now this format wants to append a script which does a full restore
853  * if the files have been extracted.
854  */
855  th = tarOpen(AH, "restore.sql", 'w');
856 
857  tarPrintf(AH, th, "--\n"
858  "-- NOTE:\n"
859  "--\n"
860  "-- File paths need to be edited. Search for $$PATH$$ and\n"
861  "-- replace it with the path to the directory containing\n"
862  "-- the extracted data files.\n"
863  "--\n");
864 
865  AH->CustomOutPtr = _scriptOut;
866 
867  ctx->isSpecialScript = 1;
868  ctx->scriptTH = th;
869 
870  ropt = NewRestoreOptions();
871  memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
872  ropt->filename = NULL;
873  ropt->dropSchema = 1;
874  ropt->compression = 0;
875  ropt->superuser = NULL;
876  ropt->suppressDumpWarnings = true;
877 
878  savDopt = AH->public.dopt;
879  savRopt = AH->public.ropt;
880 
881  SetArchiveOptions((Archive *) AH, NULL, ropt);
882 
883  savVerbose = AH->public.verbose;
884  AH->public.verbose = 0;
885 
886  RestoreArchive((Archive *) AH);
887 
888  SetArchiveOptions((Archive *) AH, savDopt, savRopt);
889 
890  AH->public.verbose = savVerbose;
891 
892  tarClose(AH, th);
893 
894  ctx->isSpecialScript = 0;
895 
896  /*
897  * EOF marker for tar files is two blocks of NULLs.
898  */
899  for (i = 0; i < 512 * 2; i++)
900  {
901  if (fputc(0, ctx->tarFH) == EOF)
903  }
904 
905  /* Sync the output file if one is defined */
906  if (AH->dosync && AH->fSpec)
907  (void) fsync_fname(AH->fSpec, false);
908  }
909 
910  AH->FH = NULL;
911 }
912 
913 static size_t
914 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
915 {
916  lclContext *ctx = (lclContext *) AH->formatData;
917 
918  return tarWrite(buf, len, ctx->scriptTH);
919 }
920 
921 /*
922  * BLOB support
923  */
924 
925 /*
926  * Called by the archiver when starting to save all BLOB DATA (not schema).
927  * This routine should save whatever format-specific information is needed
928  * to read the BLOBs back into memory.
929  *
930  * It is called just prior to the dumper's DataDumper routine.
931  *
932  * Optional, but strongly recommended.
933  *
934  */
935 static void
937 {
938  lclContext *ctx = (lclContext *) AH->formatData;
939  char fname[K_STD_BUF_SIZE];
940 
941  sprintf(fname, "blobs.toc");
942  ctx->blobToc = tarOpen(AH, fname, 'w');
943 }
944 
945 /*
946  * Called by the archiver when the dumper calls StartBlob.
947  *
948  * Mandatory.
949  *
950  * Must save the passed OID for retrieval at restore-time.
951  */
952 static void
954 {
955  lclContext *ctx = (lclContext *) AH->formatData;
956  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
957  char fname[255];
958  char *sfx;
959 
960  if (oid == 0)
961  fatal("invalid OID for large object (%u)", oid);
962 
963  if (AH->compression != 0)
964  sfx = ".gz";
965  else
966  sfx = "";
967 
968  sprintf(fname, "blob_%u.dat%s", oid, sfx);
969 
970  tarPrintf(AH, ctx->blobToc, "%u %s\n", oid, fname);
971 
972  tctx->TH = tarOpen(AH, fname, 'w');
973 }
974 
975 /*
976  * Called by the archiver when the dumper calls EndBlob.
977  *
978  * Optional.
979  *
980  */
981 static void
983 {
984  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
985 
986  tarClose(AH, tctx->TH);
987 }
988 
989 /*
990  * Called by the archiver when finishing saving all BLOB DATA.
991  *
992  * Optional.
993  *
994  */
995 static void
997 {
998  lclContext *ctx = (lclContext *) AH->formatData;
999 
1000  /* Write out a fake zero OID to mark end-of-blobs. */
1001  /* WriteInt(AH, 0); */
1002 
1003  tarClose(AH, ctx->blobToc);
1004 }
1005 
1006 
1007 
1008 /*------------
1009  * TAR Support
1010  *------------
1011  */
1012 
1013 static int
1014 tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
1015 {
1016  int save_errno = errno;
1017  char *p;
1018  size_t len = 128; /* initial assumption about buffer size */
1019  size_t cnt;
1020 
1021  for (;;)
1022  {
1023  va_list args;
1024 
1025  /* Allocate work buffer. */
1026  p = (char *) pg_malloc(len);
1027 
1028  /* Try to format the data. */
1029  errno = save_errno;
1030  va_start(args, fmt);
1031  cnt = pvsnprintf(p, len, fmt, args);
1032  va_end(args);
1033 
1034  if (cnt < len)
1035  break; /* success */
1036 
1037  /* Release buffer and loop around to try again with larger len. */
1038  free(p);
1039  len = cnt;
1040  }
1041 
1042  cnt = tarWrite(p, cnt, th);
1043  free(p);
1044  return (int) cnt;
1045 }
1046 
1047 bool
1049 {
1050  int sum;
1051  int chk = tarChecksum(header);
1052 
1053  sum = read_tar_number(&header[148], 8);
1054 
1055  if (sum != chk)
1056  return false;
1057 
1058  /* POSIX tar format */
1059  if (memcmp(&header[257], "ustar\0", 6) == 0 &&
1060  memcmp(&header[263], "00", 2) == 0)
1061  return true;
1062  /* GNU tar format */
1063  if (memcmp(&header[257], "ustar \0", 8) == 0)
1064  return true;
1065  /* not-quite-POSIX format written by pre-9.3 pg_dump */
1066  if (memcmp(&header[257], "ustar00\0", 8) == 0)
1067  return true;
1068 
1069  return false;
1070 }
1071 
1072 /* Given the member, write the TAR header & copy the file */
1073 static void
1075 {
1076  lclContext *ctx = (lclContext *) AH->formatData;
1077  FILE *tmp = th->tmpFH; /* Grab it for convenience */
1078  char buf[32768];
1079  size_t cnt;
1080  pgoff_t len = 0;
1081  size_t res;
1082  size_t i,
1083  pad;
1084 
1085  /*
1086  * Find file len & go back to start.
1087  */
1088  fseeko(tmp, 0, SEEK_END);
1089  th->fileLen = ftello(tmp);
1090  if (th->fileLen < 0)
1091  fatal("could not determine seek position in archive file: %m");
1092  fseeko(tmp, 0, SEEK_SET);
1093 
1094  _tarWriteHeader(th);
1095 
1096  while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1097  {
1098  if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1100  len += res;
1101  }
1102  if (!feof(tmp))
1103  READ_ERROR_EXIT(tmp);
1104 
1105  if (fclose(tmp) != 0) /* This *should* delete it... */
1106  fatal("could not close temporary file: %m");
1107 
1108  if (len != th->fileLen)
1109  {
1110  char buf1[32],
1111  buf2[32];
1112 
1113  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) len);
1114  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) th->fileLen);
1115  fatal("actual file length (%s) does not match expected (%s)",
1116  buf1, buf2);
1117  }
1118 
1119  pad = ((len + 511) & ~511) - len;
1120  for (i = 0; i < pad; i++)
1121  {
1122  if (fputc('\0', th->tarFH) == EOF)
1124  }
1125 
1126  ctx->tarFHpos += len + pad;
1127 }
1128 
1129 /* Locate the file in the archive, read header and position to data */
1130 static TAR_MEMBER *
1132 {
1133  lclContext *ctx = (lclContext *) AH->formatData;
1134  TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
1135  char c;
1136  char header[512];
1137  size_t i,
1138  len,
1139  blks;
1140  int id;
1141 
1142  th->AH = AH;
1143 
1144  /* Go to end of current file, if any */
1145  if (ctx->tarFHpos != 0)
1146  {
1147  char buf1[100],
1148  buf2[100];
1149 
1150  snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ctx->tarFHpos);
1151  snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ctx->tarNextMember);
1152  pg_log_debug("moving from position %s to next member at file position %s",
1153  buf1, buf2);
1154 
1155  while (ctx->tarFHpos < ctx->tarNextMember)
1156  _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1157  }
1158 
1159  {
1160  char buf[100];
1161 
1162  snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ctx->tarFHpos);
1163  pg_log_debug("now at file position %s", buf);
1164  }
1165 
1166  /* We are at the start of the file, or at the next member */
1167 
1168  /* Get the header */
1169  if (!_tarGetHeader(AH, th))
1170  {
1171  if (filename)
1172  fatal("could not find header for file \"%s\" in tar archive", filename);
1173  else
1174  {
1175  /*
1176  * We're just scanning the archive for the next file, so return
1177  * null
1178  */
1179  free(th);
1180  return NULL;
1181  }
1182  }
1183 
1184  while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1185  {
1186  pg_log_debug("skipping tar member %s", th->targetFile);
1187 
1188  id = atoi(th->targetFile);
1189  if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1190  fatal("restoring data out of order is not supported in this archive format: "
1191  "\"%s\" is required, but comes before \"%s\" in the archive file.",
1192  th->targetFile, filename);
1193 
1194  /* Header doesn't match, so read to next header */
1195  len = ((th->fileLen + 511) & ~511); /* Padded length */
1196  blks = len >> 9; /* # of 512 byte blocks */
1197 
1198  for (i = 0; i < blks; i++)
1199  _tarReadRaw(AH, &header[0], 512, NULL, ctx->tarFH);
1200 
1201  if (!_tarGetHeader(AH, th))
1202  fatal("could not find header for file \"%s\" in tar archive", filename);
1203  }
1204 
1205  ctx->tarNextMember = ctx->tarFHpos + ((th->fileLen + 511) & ~511);
1206  th->pos = 0;
1207 
1208  return th;
1209 }
1210 
1211 /* Read & verify a header */
1212 static int
1214 {
1215  lclContext *ctx = (lclContext *) AH->formatData;
1216  char h[512];
1217  char tag[100 + 1];
1218  int sum,
1219  chk;
1220  pgoff_t len;
1221  pgoff_t hPos;
1222  bool gotBlock = false;
1223 
1224  while (!gotBlock)
1225  {
1226  /* Save the pos for reporting purposes */
1227  hPos = ctx->tarFHpos;
1228 
1229  /* Read a 512 byte block, return EOF, exit if short */
1230  len = _tarReadRaw(AH, h, 512, NULL, ctx->tarFH);
1231  if (len == 0) /* EOF */
1232  return 0;
1233 
1234  if (len != 512)
1235  fatal(ngettext("incomplete tar header found (%lu byte)",
1236  "incomplete tar header found (%lu bytes)",
1237  len),
1238  (unsigned long) len);
1239 
1240  /* Calc checksum */
1241  chk = tarChecksum(h);
1242  sum = read_tar_number(&h[148], 8);
1243 
1244  /*
1245  * If the checksum failed, see if it is a null block. If so, silently
1246  * continue to the next block.
1247  */
1248  if (chk == sum)
1249  gotBlock = true;
1250  else
1251  {
1252  int i;
1253 
1254  for (i = 0; i < 512; i++)
1255  {
1256  if (h[i] != 0)
1257  {
1258  gotBlock = true;
1259  break;
1260  }
1261  }
1262  }
1263  }
1264 
1265  /* Name field is 100 bytes, might not be null-terminated */
1266  strlcpy(tag, &h[0], 100 + 1);
1267 
1268  len = read_tar_number(&h[124], 12);
1269 
1270  {
1271  char posbuf[32];
1272  char lenbuf[32];
1273 
1274  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
1275  snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
1276  pg_log_debug("TOC Entry %s at %s (length %s, checksum %d)",
1277  tag, posbuf, lenbuf, sum);
1278  }
1279 
1280  if (chk != sum)
1281  {
1282  char posbuf[32];
1283 
1284  snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
1285  (uint64) ftello(ctx->tarFH));
1286  fatal("corrupt tar header found in %s (expected %d, computed %d) file position %s",
1287  tag, sum, chk, posbuf);
1288  }
1289 
1290  th->targetFile = pg_strdup(tag);
1291  th->fileLen = len;
1292 
1293  return 1;
1294 }
1295 
1296 
1297 static void
1299 {
1300  char h[512];
1301 
1302  tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1303  0600, 04000, 02000, time(NULL));
1304 
1305  /* Now write the completed header. */
1306  if (fwrite(h, 1, 512, th->tarFH) != 512)
1308 }
static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3
static void _LoadBlobs(ArchiveHandle *AH)
TAR_MEMBER * FH
Definition: pg_backup_tar.c:91
FILE * zFH
Definition: pg_backup_tar.c:71
void ReadToc(ArchiveHandle *AH)
#define TH
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static PgChecksumMode mode
Definition: pg_checksums.c:61
void ReadHead(ArchiveHandle *AH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
TAR_MEMBER * blobToc
Definition: pg_backup_tar.c:87
TAR_MEMBER * scriptTH
Definition: pg_backup_tar.c:93
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
RestoreOptions * ropt
Definition: pg_backup.h:183
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
static TAR_MEMBER * tarOpen(ArchiveHandle *AH, const char *filename, char mode)
pgoff_t tarFHpos
Definition: pg_backup_tar.c:89
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:580
#define PG_BINARY_W
Definition: c.h:1194
void WriteToc(ArchiveHandle *AH)
ArchiveHandle * AH
Definition: pg_backup_tar.c:80
static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
DataDumperPtr dataDumper
unsigned int Oid
Definition: postgres_ext.h:31
#define PG_BINARY_R
Definition: c.h:1193
void InitArchiveFmt_Tar(ArchiveHandle *AH)
static int fd(const char *x, int i)
Definition: preproc-init.c:105
void StartRestoreBlobs(ArchiveHandle *AH)
struct _tocEntry * currToc
RestoreOptions * NewRestoreOptions(void)
#define pg_attribute_printf(f, a)
Definition: c.h:132
pgoff_t tarNextMember
Definition: pg_backup_tar.c:90
static struct pg_tm tm
Definition: localtime.c:108
#define pgoff_t
Definition: win32_port.h:195
#define ftello(stream)
Definition: win32_port.h:204
#define sprintf
Definition: port.h:194
size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args)
Definition: psprintf.c:106
int isSpecialScript
Definition: pg_backup_tar.c:92
#define S_IWUSR
Definition: win32_port.h:263
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
DumpOptions * dopt
Definition: pg_backup.h:182
#define pg_log_debug(...)
Definition: logging.h:91
FILE * tarFH
Definition: pg_backup_tar.c:88
void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop)
char * c
static void _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static char * buf
Definition: pg_test_fsync.c:68
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
static TAR_MEMBER * _tarPositionTo(ArchiveHandle *AH, const char *filename)
#define GZEOF(fh)
#define S_IRWXG
Definition: win32_port.h:281
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH)
#define atooid(x)
Definition: postgres_ext.h:42
bool isValidTarHeader(char *header)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _EndBlobs(ArchiveHandle *AH, TocEntry *te)
#define GZWRITE(p, s, n, fh)
void EndRestoreBlobs(ArchiveHandle *AH)
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
#define ngettext(s, p, n)
Definition: c.h:1103
FILE * nFH
Definition: pg_backup_tar.c:73
int verbose
Definition: pg_backup.h:185
static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
#define fseeko(stream, offset, origin)
Definition: win32_port.h:201
int tarChecksum(char *header)
Definition: tar.c:88
pgoff_t pos
Definition: pg_backup_tar.c:78
static void * fn(void *arg)
#define free(a)
Definition: header.h:65
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
CustomOutPtrType CustomOutPtr
static void _tarWriteHeader(TAR_MEMBER *th)
char * ReadStr(ArchiveHandle *AH)
teReqs TocIDRequired(ArchiveHandle *AH, DumpId id)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
FILE * tmpFH
Definition: pg_backup_tar.c:75
pgoff_t fileLen
Definition: pg_backup_tar.c:79
#define strerror
Definition: port.h:205
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
#define fatal(...)
static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
#define GZCLOSE(fh)
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:210
#define INT64_FORMAT
Definition: c.h:400
const char * name
Definition: encode.c:521
#define S_IRUSR
Definition: win32_port.h:260
static int static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
void WriteHead(ArchiveHandle *AH)
#define GZREAD(p, s, n, fh)
#define WRITE_ERROR_EXIT
static void _CloseArchive(ArchiveHandle *AH)
#define K_STD_BUF_SIZE
Definition: pg_backup_tar.c:63
static char * filename
Definition: pg_dumpall.c:91
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
int errmsg(const char *fmt,...)
Definition: elog.c:784
void EndRestoreBlob(ArchiveHandle *AH, Oid oid)
void RestoreArchive(Archive *AH)
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th)
int i
size_t WriteStr(ArchiveHandle *AH, const char *c)
TAR_MEMBER * TH
Definition: pg_backup_tar.c:98
#define READ_ERROR_EXIT(fd)
static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bool checkSeek(FILE *fp)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
pgoff_t filePos
static int _ReadByte(ArchiveHandle *)
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
FILE * tarFH
Definition: pg_backup_tar.c:74
#define snprintf
Definition: port.h:192
#define UINT64_FORMAT
Definition: c.h:401
#define LOBBUFSIZE
static int _WriteByte(ArchiveHandle *AH, const int i)
char * targetFile
Definition: pg_backup_tar.c:76
#define S_IRWXO
Definition: win32_port.h:293
#define pg_log_info(...)
Definition: logging.h:87
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ropt)
static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)