PostgreSQL Source Code  git master
pg_backup_tar.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_tar.c
4  *
5  * This file is copied from the 'files' format file, but dumps data into
6  * one temp file then sends it to the output TAR archive.
7  *
8  * The tar format also includes a 'restore.sql' script which is there for
9  * the benefit of humans. This script is never used by pg_restore.
10  *
11  * NOTE: If you untar the created 'tar' file, the resulting files are
12  * compatible with the 'directory' format. Please keep the two formats in
13  * sync.
14  *
15  * See the headers to pg_backup_directory & pg_restore for more details.
16  *
17  * Copyright (c) 2000, Philip Warner
18  * Rights are granted to use this software in any way so long
19  * as this notice is not removed.
20  *
21  * The author is not responsible for loss or damages that may
22  * result from its use.
23  *
24  *
25  * IDENTIFICATION
26  * src/bin/pg_dump/pg_backup_tar.c
27  *
28  *-------------------------------------------------------------------------
29  */
30 #include "postgres_fe.h"
31 
32 #include <sys/stat.h>
33 #include <ctype.h>
34 #include <limits.h>
35 #include <unistd.h>
36 
37 #include "common/file_utils.h"
38 #include "fe_utils/string_utils.h"
39 #include "pg_backup_archiver.h"
40 #include "pg_backup_tar.h"
41 #include "pg_backup_utils.h"
42 #include "pgtar.h"
43 
44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 static int _WriteByte(ArchiveHandle *AH, const int i);
49 static int _ReadByte(ArchiveHandle *);
50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 static void _CloseArchive(ArchiveHandle *AH);
53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 
58 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
59 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
62 
63 #define K_STD_BUF_SIZE 1024
64 
65 
66 typedef struct
67 {
68  FILE *nFH;
69  FILE *tarFH;
70  FILE *tmpFH;
71  char *targetFile;
72  char mode;
76 } TAR_MEMBER;
77 
78 typedef struct
79 {
80  int hasSeek;
83  FILE *tarFH;
89 } lclContext;
90 
91 typedef struct
92 {
94  char *filename;
95 } lclTocEntry;
96 
97 static void _LoadBlobs(ArchiveHandle *AH);
98 
99 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
100 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
101 
102 #ifdef __NOT_USED__
103 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
104 #endif
105 static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
106 
107 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
108 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
109 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
110 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
111 static void _tarWriteHeader(TAR_MEMBER *th);
112 static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
113 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
114 
115 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
116 
117 /*
118  * Initializer
119  */
120 void
122 {
123  lclContext *ctx;
124 
125  /* Assuming static functions, this can be copied for each format. */
126  AH->ArchiveEntryPtr = _ArchiveEntry;
127  AH->StartDataPtr = _StartData;
128  AH->WriteDataPtr = _WriteData;
129  AH->EndDataPtr = _EndData;
130  AH->WriteBytePtr = _WriteByte;
131  AH->ReadBytePtr = _ReadByte;
132  AH->WriteBufPtr = _WriteBuf;
133  AH->ReadBufPtr = _ReadBuf;
134  AH->ClosePtr = _CloseArchive;
135  AH->ReopenPtr = NULL;
136  AH->PrintTocDataPtr = _PrintTocData;
137  AH->ReadExtraTocPtr = _ReadExtraToc;
138  AH->WriteExtraTocPtr = _WriteExtraToc;
139  AH->PrintExtraTocPtr = _PrintExtraToc;
140 
141  AH->StartBlobsPtr = _StartBlobs;
142  AH->StartBlobPtr = _StartBlob;
143  AH->EndBlobPtr = _EndBlob;
144  AH->EndBlobsPtr = _EndBlobs;
145  AH->ClonePtr = NULL;
146  AH->DeClonePtr = NULL;
147 
148  AH->WorkerJobDumpPtr = NULL;
149  AH->WorkerJobRestorePtr = NULL;
150 
151  /*
152  * Set up some special context used in compressing data.
153  */
154  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
155  AH->formatData = (void *) ctx;
156  ctx->filePos = 0;
157  ctx->isSpecialScript = 0;
158 
159  /* Initialize LO buffering */
160  AH->lo_buf_size = LOBBUFSIZE;
161  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
162 
163  /*
164  * Now open the tar file, and load the TOC if we're in read mode.
165  */
166  if (AH->mode == archModeWrite)
167  {
168  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
169  {
170  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
171  if (ctx->tarFH == NULL)
172  pg_fatal("could not open TOC file \"%s\" for output: %m",
173  AH->fSpec);
174  }
175  else
176  {
177  ctx->tarFH = stdout;
178  if (ctx->tarFH == NULL)
179  pg_fatal("could not open TOC file for output: %m");
180  }
181 
182  ctx->tarFHpos = 0;
183 
184  /*
185  * Make unbuffered since we will dup() it, and the buffers screw each
186  * other
187  */
188  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
189 
190  ctx->hasSeek = checkSeek(ctx->tarFH);
191 
192  /*
193  * We don't support compression because reading the files back is not
194  * possible since gzdopen uses buffered IO which totally screws file
195  * positioning.
196  */
197  if (AH->compression != 0)
198  pg_fatal("compression is not supported by tar archive format");
199  }
200  else
201  { /* Read Mode */
202  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
203  {
204  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
205  if (ctx->tarFH == NULL)
206  pg_fatal("could not open TOC file \"%s\" for input: %m",
207  AH->fSpec);
208  }
209  else
210  {
211  ctx->tarFH = stdin;
212  if (ctx->tarFH == NULL)
213  pg_fatal("could not open TOC file for input: %m");
214  }
215 
216  /*
217  * Make unbuffered since we will dup() it, and the buffers screw each
218  * other
219  */
220  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
221 
222  ctx->tarFHpos = 0;
223 
224  ctx->hasSeek = checkSeek(ctx->tarFH);
225 
226  ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
227  ReadHead(AH);
228  ReadToc(AH);
229  tarClose(AH, ctx->FH); /* Nothing else in the file... */
230  }
231 }
232 
233 /*
234  * - Start a new TOC entry
235  * Setup the output file name.
236  */
237 static void
239 {
240  lclTocEntry *ctx;
241  char fn[K_STD_BUF_SIZE];
242 
243  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
244  if (te->dataDumper != NULL)
245  {
246  snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
247  ctx->filename = pg_strdup(fn);
248  }
249  else
250  {
251  ctx->filename = NULL;
252  ctx->TH = NULL;
253  }
254  te->formatData = (void *) ctx;
255 }
256 
257 static void
259 {
260  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
261 
262  if (ctx->filename)
263  WriteStr(AH, ctx->filename);
264  else
265  WriteStr(AH, "");
266 }
267 
268 static void
270 {
271  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
272 
273  if (ctx == NULL)
274  {
275  ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
276  te->formatData = (void *) ctx;
277  }
278 
279  ctx->filename = ReadStr(AH);
280  if (strlen(ctx->filename) == 0)
281  {
282  free(ctx->filename);
283  ctx->filename = NULL;
284  }
285  ctx->TH = NULL;
286 }
287 
288 static void
290 {
291  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
292 
293  if (AH->public.verbose && ctx->filename != NULL)
294  ahprintf(AH, "-- File: %s\n", ctx->filename);
295 }
296 
297 static void
299 {
300  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
301 
302  tctx->TH = tarOpen(AH, tctx->filename, 'w');
303 }
304 
305 static TAR_MEMBER *
306 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
307 {
308  lclContext *ctx = (lclContext *) AH->formatData;
309  TAR_MEMBER *tm;
310 
311  if (mode == 'r')
312  {
313  tm = _tarPositionTo(AH, filename);
314  if (!tm) /* Not found */
315  {
316  if (filename)
317  {
318  /*
319  * Couldn't find the requested file. Future: do SEEK(0) and
320  * retry.
321  */
322  pg_fatal("could not find file \"%s\" in archive", filename);
323  }
324  else
325  {
326  /* Any file OK, none left, so return NULL */
327  return NULL;
328  }
329  }
330 
331  if (AH->compression == 0)
332  tm->nFH = ctx->tarFH;
333  else
334  pg_fatal("compression is not supported by tar archive format");
335  }
336  else
337  {
338  int old_umask;
339 
340  tm = pg_malloc0(sizeof(TAR_MEMBER));
341 
342  /*
343  * POSIX does not require, but permits, tmpfile() to restrict file
344  * permissions. Given an OS crash after we write data, the filesystem
345  * might retain the data but forget tmpfile()'s unlink(). If so, the
346  * file mode protects confidentiality of the data written.
347  */
348  old_umask = umask(S_IRWXG | S_IRWXO);
349 
350 #ifndef WIN32
351  tm->tmpFH = tmpfile();
352 #else
353 
354  /*
355  * On WIN32, tmpfile() generates a filename in the root directory,
356  * which requires administrative permissions on certain systems. Loop
357  * until we find a unique file name we can create.
358  */
359  while (1)
360  {
361  char *name;
362  int fd;
363 
364  name = _tempnam(NULL, "pg_temp_");
365  if (name == NULL)
366  break;
367  fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
368  O_TEMPORARY, S_IRUSR | S_IWUSR);
369  free(name);
370 
371  if (fd != -1) /* created a file */
372  {
373  tm->tmpFH = fdopen(fd, "w+b");
374  break;
375  }
376  else if (errno != EEXIST) /* failure other than file exists */
377  break;
378  }
379 #endif
380 
381  if (tm->tmpFH == NULL)
382  pg_fatal("could not generate temporary file name: %m");
383 
384  umask(old_umask);
385 
386  if (AH->compression == 0)
387  tm->nFH = tm->tmpFH;
388  else
389  pg_fatal("compression is not supported by tar archive format");
390 
391  tm->AH = AH;
392  tm->targetFile = pg_strdup(filename);
393  }
394 
395  tm->mode = mode;
396  tm->tarFH = ctx->tarFH;
397 
398  return tm;
399 }
400 
401 static void
403 {
404  if (AH->compression != 0)
405  pg_fatal("compression is not supported by tar archive format");
406 
407  if (th->mode == 'w')
408  _tarAddFile(AH, th); /* This will close the temp file */
409 
410  /*
411  * else Nothing to do for normal read since we don't dup() normal file
412  * handle, and we don't use temp files.
413  */
414 
415  if (th->targetFile)
416  free(th->targetFile);
417 
418  th->nFH = NULL;
419 }
420 
421 #ifdef __NOT_USED__
422 static char *
423 tarGets(char *buf, size_t len, TAR_MEMBER *th)
424 {
425  char *s;
426  size_t cnt = 0;
427  char c = ' ';
428  int eof = 0;
429 
430  /* Can't read past logical EOF */
431  if (len > (th->fileLen - th->pos))
432  len = th->fileLen - th->pos;
433 
434  while (cnt < len && c != '\n')
435  {
436  if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
437  {
438  eof = 1;
439  break;
440  }
441  buf[cnt++] = c;
442  }
443 
444  if (eof && cnt == 0)
445  s = NULL;
446  else
447  {
448  buf[cnt++] = '\0';
449  s = buf;
450  }
451 
452  if (s)
453  {
454  len = strlen(s);
455  th->pos += len;
456  }
457 
458  return s;
459 }
460 #endif
461 
462 /*
463  * Just read bytes from the archive. This is the low level read routine
464  * that is used for ALL reads on a tar file.
465  */
466 static size_t
467 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
468 {
469  lclContext *ctx = (lclContext *) AH->formatData;
470  size_t avail;
471  size_t used = 0;
472  size_t res = 0;
473 
474  Assert(th || fh);
475 
476  avail = AH->lookaheadLen - AH->lookaheadPos;
477  if (avail > 0)
478  {
479  /* We have some lookahead bytes to use */
480  if (avail >= len) /* Just use the lookahead buffer */
481  used = len;
482  else
483  used = avail;
484 
485  /* Copy, and adjust buffer pos */
486  memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
487  AH->lookaheadPos += used;
488 
489  /* Adjust required length */
490  len -= used;
491  }
492 
493  /* Read the file if len > 0 */
494  if (len > 0)
495  {
496  if (fh)
497  {
498  res = fread(&((char *) buf)[used], 1, len, fh);
499  if (res != len && !feof(fh))
500  READ_ERROR_EXIT(fh);
501  }
502  else if (th)
503  {
504  res = fread(&((char *) buf)[used], 1, len, th->nFH);
505  if (res != len && !feof(th->nFH))
506  READ_ERROR_EXIT(th->nFH);
507  }
508  }
509 
510  ctx->tarFHpos += res + used;
511 
512  return (res + used);
513 }
514 
515 static size_t
516 tarRead(void *buf, size_t len, TAR_MEMBER *th)
517 {
518  size_t res;
519 
520  if (th->pos + len > th->fileLen)
521  len = th->fileLen - th->pos;
522 
523  if (len <= 0)
524  return 0;
525 
526  res = _tarReadRaw(th->AH, buf, len, th, NULL);
527 
528  th->pos += res;
529 
530  return res;
531 }
532 
533 static size_t
534 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
535 {
536  size_t res;
537 
538  res = fwrite(buf, 1, len, th->nFH);
539 
540  th->pos += res;
541  return res;
542 }
543 
544 static void
545 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
546 {
547  lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
548 
549  if (tarWrite(data, dLen, tctx->TH) != dLen)
551 }
552 
553 static void
555 {
556  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
557 
558  /* Close the file */
559  tarClose(AH, tctx->TH);
560  tctx->TH = NULL;
561 }
562 
563 /*
564  * Print data for a given file
565  */
566 static void
568 {
569  lclContext *ctx = (lclContext *) AH->formatData;
570  char buf[4096];
571  size_t cnt;
572  TAR_MEMBER *th;
573 
574  if (!filename)
575  return;
576 
577  th = tarOpen(AH, filename, 'r');
578  ctx->FH = th;
579 
580  while ((cnt = tarRead(buf, 4095, th)) > 0)
581  {
582  buf[cnt] = '\0';
583  ahwrite(buf, 1, cnt, AH);
584  }
585 
586  tarClose(AH, th);
587 }
588 
589 
590 /*
591  * Print data for a given TOC entry
592 */
593 static void
595 {
596  lclContext *ctx = (lclContext *) AH->formatData;
597  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
598  int pos1;
599 
600  if (!tctx->filename)
601  return;
602 
603  /*
604  * If we're writing the special restore.sql script, emit a suitable
605  * command to include each table's data from the corresponding file.
606  *
607  * In the COPY case this is a bit klugy because the regular COPY command
608  * was already printed before we get control.
609  */
610  if (ctx->isSpecialScript)
611  {
612  if (te->copyStmt)
613  {
614  /* Abort the COPY FROM stdin */
615  ahprintf(AH, "\\.\n");
616 
617  /*
618  * The COPY statement should look like "COPY ... FROM stdin;\n",
619  * see dumpTableData().
620  */
621  pos1 = (int) strlen(te->copyStmt) - 13;
622  if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
623  strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
624  pg_fatal("unexpected COPY statement syntax: \"%s\"",
625  te->copyStmt);
626 
627  /* Emit all but the FROM part ... */
628  ahwrite(te->copyStmt, 1, pos1, AH);
629  /* ... and insert modified FROM */
630  ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
631  }
632  else
633  {
634  /* --inserts mode, no worries, just include the data file */
635  ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
636  }
637 
638  return;
639  }
640 
641  if (strcmp(te->desc, "BLOBS") == 0)
642  _LoadBlobs(AH);
643  else
644  _PrintFileData(AH, tctx->filename);
645 }
646 
647 static void
649 {
650  Oid oid;
651  lclContext *ctx = (lclContext *) AH->formatData;
652  TAR_MEMBER *th;
653  size_t cnt;
654  bool foundBlob = false;
655  char buf[4096];
656 
657  StartRestoreBlobs(AH);
658 
659  th = tarOpen(AH, NULL, 'r'); /* Open next file */
660  while (th != NULL)
661  {
662  ctx->FH = th;
663 
664  if (strncmp(th->targetFile, "blob_", 5) == 0)
665  {
666  oid = atooid(&th->targetFile[5]);
667  if (oid != 0)
668  {
669  pg_log_info("restoring large object with OID %u", oid);
670 
671  StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
672 
673  while ((cnt = tarRead(buf, 4095, th)) > 0)
674  {
675  buf[cnt] = '\0';
676  ahwrite(buf, 1, cnt, AH);
677  }
678  EndRestoreBlob(AH, oid);
679  foundBlob = true;
680  }
681  tarClose(AH, th);
682  }
683  else
684  {
685  tarClose(AH, th);
686 
687  /*
688  * Once we have found the first blob, stop at the first non-blob
689  * entry (which will be 'blobs.toc'). This coding would eat all
690  * the rest of the archive if there are no blobs ... but this
691  * function shouldn't be called at all in that case.
692  */
693  if (foundBlob)
694  break;
695  }
696 
697  th = tarOpen(AH, NULL, 'r');
698  }
699  EndRestoreBlobs(AH);
700 }
701 
702 
703 static int
704 _WriteByte(ArchiveHandle *AH, const int i)
705 {
706  lclContext *ctx = (lclContext *) AH->formatData;
707  char b = i; /* Avoid endian problems */
708 
709  if (tarWrite(&b, 1, ctx->FH) != 1)
711 
712  ctx->filePos += 1;
713  return 1;
714 }
715 
716 static int
718 {
719  lclContext *ctx = (lclContext *) AH->formatData;
720  size_t res;
721  unsigned char c;
722 
723  res = tarRead(&c, 1, ctx->FH);
724  if (res != 1)
725  /* We already would have exited for errors on reads, must be EOF */
726  pg_fatal("could not read from input file: end of file");
727  ctx->filePos += 1;
728  return c;
729 }
730 
731 static void
732 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
733 {
734  lclContext *ctx = (lclContext *) AH->formatData;
735 
736  if (tarWrite(buf, len, ctx->FH) != len)
738 
739  ctx->filePos += len;
740 }
741 
742 static void
743 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
744 {
745  lclContext *ctx = (lclContext *) AH->formatData;
746 
747  if (tarRead(buf, len, ctx->FH) != len)
748  /* We already would have exited for errors on reads, must be EOF */
749  pg_fatal("could not read from input file: end of file");
750 
751  ctx->filePos += len;
752 }
753 
754 static void
756 {
757  lclContext *ctx = (lclContext *) AH->formatData;
758  TAR_MEMBER *th;
759  RestoreOptions *ropt;
760  RestoreOptions *savRopt;
761  DumpOptions *savDopt;
762  int savVerbose,
763  i;
764 
765  if (AH->mode == archModeWrite)
766  {
767  /*
768  * Write the Header & TOC to the archive FIRST
769  */
770  th = tarOpen(AH, "toc.dat", 'w');
771  ctx->FH = th;
772  WriteHead(AH);
773  WriteToc(AH);
774  tarClose(AH, th); /* Not needed any more */
775 
776  /*
777  * Now send the data (tables & blobs)
778  */
779  WriteDataChunks(AH, NULL);
780 
781  /*
782  * Now this format wants to append a script which does a full restore
783  * if the files have been extracted.
784  */
785  th = tarOpen(AH, "restore.sql", 'w');
786 
787  tarPrintf(th, "--\n"
788  "-- NOTE:\n"
789  "--\n"
790  "-- File paths need to be edited. Search for $$PATH$$ and\n"
791  "-- replace it with the path to the directory containing\n"
792  "-- the extracted data files.\n"
793  "--\n");
794 
795  AH->CustomOutPtr = _scriptOut;
796 
797  ctx->isSpecialScript = 1;
798  ctx->scriptTH = th;
799 
800  ropt = NewRestoreOptions();
801  memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
802  ropt->filename = NULL;
803  ropt->dropSchema = 1;
804  ropt->compression = 0;
805  ropt->superuser = NULL;
806  ropt->suppressDumpWarnings = true;
807 
808  savDopt = AH->public.dopt;
809  savRopt = AH->public.ropt;
810 
811  SetArchiveOptions((Archive *) AH, NULL, ropt);
812 
813  savVerbose = AH->public.verbose;
814  AH->public.verbose = 0;
815 
816  RestoreArchive((Archive *) AH);
817 
818  SetArchiveOptions((Archive *) AH, savDopt, savRopt);
819 
820  AH->public.verbose = savVerbose;
821 
822  tarClose(AH, th);
823 
824  ctx->isSpecialScript = 0;
825 
826  /*
827  * EOF marker for tar files is two blocks of NULLs.
828  */
829  for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
830  {
831  if (fputc(0, ctx->tarFH) == EOF)
833  }
834 
835  /* Sync the output file if one is defined */
836  if (AH->dosync && AH->fSpec)
837  (void) fsync_fname(AH->fSpec, false);
838  }
839 
840  AH->FH = NULL;
841 }
842 
843 static size_t
844 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
845 {
846  lclContext *ctx = (lclContext *) AH->formatData;
847 
848  return tarWrite(buf, len, ctx->scriptTH);
849 }
850 
851 /*
852  * BLOB support
853  */
854 
855 /*
856  * Called by the archiver when starting to save all BLOB DATA (not schema).
857  * This routine should save whatever format-specific information is needed
858  * to read the BLOBs back into memory.
859  *
860  * It is called just prior to the dumper's DataDumper routine.
861  *
862  * Optional, but strongly recommended.
863  *
864  */
865 static void
867 {
868  lclContext *ctx = (lclContext *) AH->formatData;
869  char fname[K_STD_BUF_SIZE];
870 
871  sprintf(fname, "blobs.toc");
872  ctx->blobToc = tarOpen(AH, fname, 'w');
873 }
874 
875 /*
876  * Called by the archiver when the dumper calls StartBlob.
877  *
878  * Mandatory.
879  *
880  * Must save the passed OID for retrieval at restore-time.
881  */
882 static void
884 {
885  lclContext *ctx = (lclContext *) AH->formatData;
886  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
887  char fname[255];
888 
889  if (oid == 0)
890  pg_fatal("invalid OID for large object (%u)", oid);
891 
892  if (AH->compression != 0)
893  pg_fatal("compression is not supported by tar archive format");
894 
895  sprintf(fname, "blob_%u.dat", oid);
896 
897  tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
898 
899  tctx->TH = tarOpen(AH, fname, 'w');
900 }
901 
902 /*
903  * Called by the archiver when the dumper calls EndBlob.
904  *
905  * Optional.
906  *
907  */
908 static void
910 {
911  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
912 
913  tarClose(AH, tctx->TH);
914 }
915 
916 /*
917  * Called by the archiver when finishing saving all BLOB DATA.
918  *
919  * Optional.
920  *
921  */
922 static void
924 {
925  lclContext *ctx = (lclContext *) AH->formatData;
926 
927  /* Write out a fake zero OID to mark end-of-blobs. */
928  /* WriteInt(AH, 0); */
929 
930  tarClose(AH, ctx->blobToc);
931 }
932 
933 
934 
935 /*------------
936  * TAR Support
937  *------------
938  */
939 
940 static int
941 tarPrintf(TAR_MEMBER *th, const char *fmt,...)
942 {
943  int save_errno = errno;
944  char *p;
945  size_t len = 128; /* initial assumption about buffer size */
946  size_t cnt;
947 
948  for (;;)
949  {
950  va_list args;
951 
952  /* Allocate work buffer. */
953  p = (char *) pg_malloc(len);
954 
955  /* Try to format the data. */
956  errno = save_errno;
957  va_start(args, fmt);
958  cnt = pvsnprintf(p, len, fmt, args);
959  va_end(args);
960 
961  if (cnt < len)
962  break; /* success */
963 
964  /* Release buffer and loop around to try again with larger len. */
965  free(p);
966  len = cnt;
967  }
968 
969  cnt = tarWrite(p, cnt, th);
970  free(p);
971  return (int) cnt;
972 }
973 
974 bool
976 {
977  int sum;
978  int chk = tarChecksum(header);
979 
980  sum = read_tar_number(&header[148], 8);
981 
982  if (sum != chk)
983  return false;
984 
985  /* POSIX tar format */
986  if (memcmp(&header[257], "ustar\0", 6) == 0 &&
987  memcmp(&header[263], "00", 2) == 0)
988  return true;
989  /* GNU tar format */
990  if (memcmp(&header[257], "ustar \0", 8) == 0)
991  return true;
992  /* not-quite-POSIX format written by pre-9.3 pg_dump */
993  if (memcmp(&header[257], "ustar00\0", 8) == 0)
994  return true;
995 
996  return false;
997 }
998 
999 /* Given the member, write the TAR header & copy the file */
1000 static void
1002 {
1003  lclContext *ctx = (lclContext *) AH->formatData;
1004  FILE *tmp = th->tmpFH; /* Grab it for convenience */
1005  char buf[32768];
1006  size_t cnt;
1007  pgoff_t len = 0;
1008  size_t res;
1009  size_t i,
1010  pad;
1011 
1012  /*
1013  * Find file len & go back to start.
1014  */
1015  if (fseeko(tmp, 0, SEEK_END) != 0)
1016  pg_fatal("error during file seek: %m");
1017  th->fileLen = ftello(tmp);
1018  if (th->fileLen < 0)
1019  pg_fatal("could not determine seek position in archive file: %m");
1020  if (fseeko(tmp, 0, SEEK_SET) != 0)
1021  pg_fatal("error during file seek: %m");
1022 
1023  _tarWriteHeader(th);
1024 
1025  while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1026  {
1027  if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1029  len += res;
1030  }
1031  if (!feof(tmp))
1032  READ_ERROR_EXIT(tmp);
1033 
1034  if (fclose(tmp) != 0) /* This *should* delete it... */
1035  pg_fatal("could not close temporary file: %m");
1036 
1037  if (len != th->fileLen)
1038  pg_fatal("actual file length (%lld) does not match expected (%lld)",
1039  (long long) len, (long long) th->fileLen);
1040 
1042  for (i = 0; i < pad; i++)
1043  {
1044  if (fputc('\0', th->tarFH) == EOF)
1046  }
1047 
1048  ctx->tarFHpos += len + pad;
1049 }
1050 
1051 /* Locate the file in the archive, read header and position to data */
1052 static TAR_MEMBER *
1054 {
1055  lclContext *ctx = (lclContext *) AH->formatData;
1056  TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
1057  char c;
1058  char header[TAR_BLOCK_SIZE];
1059  size_t i,
1060  len,
1061  blks;
1062  int id;
1063 
1064  th->AH = AH;
1065 
1066  /* Go to end of current file, if any */
1067  if (ctx->tarFHpos != 0)
1068  {
1069  pg_log_debug("moving from position %lld to next member at file position %lld",
1070  (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1071 
1072  while (ctx->tarFHpos < ctx->tarNextMember)
1073  _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1074  }
1075 
1076  pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1077 
1078  /* We are at the start of the file, or at the next member */
1079 
1080  /* Get the header */
1081  if (!_tarGetHeader(AH, th))
1082  {
1083  if (filename)
1084  pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1085  else
1086  {
1087  /*
1088  * We're just scanning the archive for the next file, so return
1089  * null
1090  */
1091  free(th);
1092  return NULL;
1093  }
1094  }
1095 
1096  while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1097  {
1098  pg_log_debug("skipping tar member %s", th->targetFile);
1099 
1100  id = atoi(th->targetFile);
1101  if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1102  pg_fatal("restoring data out of order is not supported in this archive format: "
1103  "\"%s\" is required, but comes before \"%s\" in the archive file.",
1104  th->targetFile, filename);
1105 
1106  /* Header doesn't match, so read to next header */
1107  len = th->fileLen;
1109  blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1110 
1111  for (i = 0; i < blks; i++)
1112  _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1113 
1114  if (!_tarGetHeader(AH, th))
1115  pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1116  }
1117 
1118  ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1120  th->pos = 0;
1121 
1122  return th;
1123 }
1124 
1125 /* Read & verify a header */
1126 static int
1128 {
1129  lclContext *ctx = (lclContext *) AH->formatData;
1130  char h[TAR_BLOCK_SIZE];
1131  char tag[100 + 1];
1132  int sum,
1133  chk;
1134  pgoff_t len;
1135  pgoff_t hPos;
1136  bool gotBlock = false;
1137 
1138  while (!gotBlock)
1139  {
1140  /* Save the pos for reporting purposes */
1141  hPos = ctx->tarFHpos;
1142 
1143  /* Read the next tar block, return EOF, exit if short */
1144  len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1145  if (len == 0) /* EOF */
1146  return 0;
1147 
1148  if (len != TAR_BLOCK_SIZE)
1149  pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1150  "incomplete tar header found (%lu bytes)",
1151  len),
1152  (unsigned long) len);
1153 
1154  /* Calc checksum */
1155  chk = tarChecksum(h);
1156  sum = read_tar_number(&h[148], 8);
1157 
1158  /*
1159  * If the checksum failed, see if it is a null block. If so, silently
1160  * continue to the next block.
1161  */
1162  if (chk == sum)
1163  gotBlock = true;
1164  else
1165  {
1166  int i;
1167 
1168  for (i = 0; i < TAR_BLOCK_SIZE; i++)
1169  {
1170  if (h[i] != 0)
1171  {
1172  gotBlock = true;
1173  break;
1174  }
1175  }
1176  }
1177  }
1178 
1179  /* Name field is 100 bytes, might not be null-terminated */
1180  strlcpy(tag, &h[0], 100 + 1);
1181 
1182  len = read_tar_number(&h[124], 12);
1183 
1184  pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1185  tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1186 
1187  if (chk != sum)
1188  pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1189  tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1190 
1191  th->targetFile = pg_strdup(tag);
1192  th->fileLen = len;
1193 
1194  return 1;
1195 }
1196 
1197 
1198 static void
1200 {
1201  char h[TAR_BLOCK_SIZE];
1202 
1203  tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1204  0600, 04000, 02000, time(NULL));
1205 
1206  /* Now write the completed header. */
1207  if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1209 }
#define PG_BINARY_R
Definition: c.h:1270
#define ngettext(s, p, n)
Definition: c.h:1179
#define pg_attribute_printf(f, a)
Definition: c.h:164
#define PG_BINARY_W
Definition: c.h:1271
const char * name
Definition: encode.c:561
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:673
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define TH
#define free(a)
Definition: header.h:65
int b
Definition: isn.c:70
int i
Definition: isn.c:73
va_end(args)
Assert(fmt[strlen(fmt) - 1] !='\n')
static void const char * fmt
va_start(args, fmt)
static struct pg_tm tm
Definition: localtime.c:102
#define pg_log_info(...)
Definition: logging.h:124
#define pg_log_debug(...)
Definition: logging.h:133
void RestoreArchive(Archive *AH)
RestoreOptions * NewRestoreOptions(void)
@ archModeWrite
Definition: pg_backup.h:49
void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ropt)
int TocIDRequired(ArchiveHandle *AH, DumpId id)
bool checkSeek(FILE *fp)
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteHead(ArchiveHandle *AH)
void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void StartRestoreBlobs(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void WriteToc(ArchiveHandle *AH)
void EndRestoreBlob(ArchiveHandle *AH, Oid oid)
void EndRestoreBlobs(ArchiveHandle *AH)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define LOBBUFSIZE
#define WRITE_ERROR_EXIT
#define REQ_DATA
#define READ_ERROR_EXIT(fd)
static void _LoadBlobs(ArchiveHandle *AH)
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2
static void _tarWriteHeader(TAR_MEMBER *th)
void InitArchiveFmt_Tar(ArchiveHandle *AH)
static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
#define K_STD_BUF_SIZE
Definition: pg_backup_tar.c:63
static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
bool isValidTarHeader(char *header)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static TAR_MEMBER * tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static void _EndBlobs(ArchiveHandle *AH, TocEntry *te)
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
static int _ReadByte(ArchiveHandle *)
static TAR_MEMBER * _tarPositionTo(ArchiveHandle *AH, const char *filename)
static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static int static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
#define pg_fatal(...)
static PgChecksumMode mode
Definition: pg_checksums.c:65
const void size_t len
const void * data
while(p+4<=pend)
static char * filename
Definition: pg_dumpall.c:94
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:212
static char * buf
Definition: pg_test_fsync.c:67
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:58
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
int tarChecksum(char *header)
Definition: tar.c:90
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define sprintf
Definition: port.h:227
#define snprintf
Definition: port.h:225
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
char * c
static int fd(const char *x, int i)
Definition: preproc-init.c:105
size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args)
Definition: psprintf.c:106
DumpOptions * dopt
Definition: pg_backup.h:209
int verbose
Definition: pg_backup.h:212
RestoreOptions * ropt
Definition: pg_backup.h:210
pgoff_t pos
Definition: pg_backup_tar.c:73
char * targetFile
Definition: pg_backup_tar.c:71
pgoff_t fileLen
Definition: pg_backup_tar.c:74
FILE * tarFH
Definition: pg_backup_tar.c:69
ArchiveHandle * AH
Definition: pg_backup_tar.c:75
FILE * nFH
Definition: pg_backup_tar.c:68
FILE * tmpFH
Definition: pg_backup_tar.c:70
CustomOutPtrType CustomOutPtr
struct _tocEntry * currToc
DataDumperPtr dataDumper
FILE * tarFH
Definition: pg_backup_tar.c:83
pgoff_t filePos
Definition: pg_backup_tar.c:81
pgoff_t tarFHpos
Definition: pg_backup_tar.c:84
TAR_MEMBER * blobToc
Definition: pg_backup_tar.c:82
TAR_MEMBER * FH
Definition: pg_backup_tar.c:86
pgoff_t tarNextMember
Definition: pg_backup_tar.c:85
TAR_MEMBER * scriptTH
Definition: pg_backup_tar.c:88
int isSpecialScript
Definition: pg_backup_tar.c:87
TAR_MEMBER * TH
Definition: pg_backup_tar.c:93
static void * fn(void *arg)
#define S_IRWXG
Definition: win32_port.h:309
#define S_IRWXO
Definition: win32_port.h:321
#define ftello(stream)
Definition: win32_port.h:218
#define S_IRUSR
Definition: win32_port.h:288
#define fseeko(stream, offset, origin)
Definition: win32_port.h:215
#define S_IWUSR
Definition: win32_port.h:291
#define pgoff_t
Definition: win32_port.h:208