PostgreSQL Source Code  git master
pg_backup_tar.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_tar.c
4  *
5  * This file is copied from the 'files' format file, but dumps data into
6  * one temp file then sends it to the output TAR archive.
7  *
8  * The tar format also includes a 'restore.sql' script which is there for
9  * the benefit of humans. This script is never used by pg_restore.
10  *
11  * NOTE: If you untar the created 'tar' file, the resulting files are
12  * compatible with the 'directory' format. Please keep the two formats in
13  * sync.
14  *
15  * See the headers to pg_backup_directory & pg_restore for more details.
16  *
17  * Copyright (c) 2000, Philip Warner
18  * Rights are granted to use this software in any way so long
19  * as this notice is not removed.
20  *
21  * The author is not responsible for loss or damages that may
22  * result from its use.
23  *
24  *
25  * IDENTIFICATION
26  * src/bin/pg_dump/pg_backup_tar.c
27  *
28  *-------------------------------------------------------------------------
29  */
30 #include "postgres_fe.h"
31 
32 #include <sys/stat.h>
33 #include <ctype.h>
34 #include <limits.h>
35 #include <unistd.h>
36 
37 #include "common/file_utils.h"
38 #include "fe_utils/string_utils.h"
39 #include "pg_backup_archiver.h"
40 #include "pg_backup_tar.h"
41 #include "pg_backup_utils.h"
42 #include "pgtar.h"
43 
44 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 static int _WriteByte(ArchiveHandle *AH, const int i);
49 static int _ReadByte(ArchiveHandle *AH);
50 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 static void _CloseArchive(ArchiveHandle *AH);
53 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 
58 static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
59 static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
62 
63 #define K_STD_BUF_SIZE 1024
64 
65 
66 typedef struct
67 {
68  FILE *nFH;
69  FILE *tarFH;
70  FILE *tmpFH;
71  char *targetFile;
72  char mode;
76 } TAR_MEMBER;
77 
78 typedef struct
79 {
80  int hasSeek;
83  FILE *tarFH;
89 } lclContext;
90 
91 typedef struct
92 {
94  char *filename;
95 } lclTocEntry;
96 
97 static void _LoadLOs(ArchiveHandle *AH);
98 
99 static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
100 static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th);
101 
102 #ifdef __NOT_USED__
103 static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
104 #endif
105 static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
106 
107 static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
108 static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
109 static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
110 static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
111 static void _tarWriteHeader(TAR_MEMBER *th);
112 static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
113 static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
114 
115 static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
116 
117 /*
118  * Initializer
119  */
120 void
122 {
123  lclContext *ctx;
124 
125  /* Assuming static functions, this can be copied for each format. */
126  AH->ArchiveEntryPtr = _ArchiveEntry;
127  AH->StartDataPtr = _StartData;
128  AH->WriteDataPtr = _WriteData;
129  AH->EndDataPtr = _EndData;
130  AH->WriteBytePtr = _WriteByte;
131  AH->ReadBytePtr = _ReadByte;
132  AH->WriteBufPtr = _WriteBuf;
133  AH->ReadBufPtr = _ReadBuf;
134  AH->ClosePtr = _CloseArchive;
135  AH->ReopenPtr = NULL;
136  AH->PrintTocDataPtr = _PrintTocData;
137  AH->ReadExtraTocPtr = _ReadExtraToc;
138  AH->WriteExtraTocPtr = _WriteExtraToc;
139  AH->PrintExtraTocPtr = _PrintExtraToc;
140 
141  AH->StartLOsPtr = _StartLOs;
142  AH->StartLOPtr = _StartLO;
143  AH->EndLOPtr = _EndLO;
144  AH->EndLOsPtr = _EndLOs;
145  AH->ClonePtr = NULL;
146  AH->DeClonePtr = NULL;
147 
148  AH->WorkerJobDumpPtr = NULL;
149  AH->WorkerJobRestorePtr = NULL;
150 
151  /*
152  * Set up some special context used in compressing data.
153  */
155  AH->formatData = (void *) ctx;
156  ctx->filePos = 0;
157  ctx->isSpecialScript = 0;
158 
159  /* Initialize LO buffering */
160  AH->lo_buf_size = LOBBUFSIZE;
161  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
162 
163  /*
164  * Now open the tar file, and load the TOC if we're in read mode.
165  */
166  if (AH->mode == archModeWrite)
167  {
168  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
169  {
170  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
171  if (ctx->tarFH == NULL)
172  pg_fatal("could not open TOC file \"%s\" for output: %m",
173  AH->fSpec);
174  }
175  else
176  {
177  ctx->tarFH = stdout;
178  if (ctx->tarFH == NULL)
179  pg_fatal("could not open TOC file for output: %m");
180  }
181 
182  ctx->tarFHpos = 0;
183 
184  /*
185  * Make unbuffered since we will dup() it, and the buffers screw each
186  * other
187  */
188  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
189 
190  ctx->hasSeek = checkSeek(ctx->tarFH);
191 
192  /*
193  * We don't support compression because reading the files back is not
194  * possible since gzdopen uses buffered IO which totally screws file
195  * positioning.
196  */
197  if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
198  pg_fatal("compression is not supported by tar archive format");
199  }
200  else
201  { /* Read Mode */
202  if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
203  {
204  ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
205  if (ctx->tarFH == NULL)
206  pg_fatal("could not open TOC file \"%s\" for input: %m",
207  AH->fSpec);
208  }
209  else
210  {
211  ctx->tarFH = stdin;
212  if (ctx->tarFH == NULL)
213  pg_fatal("could not open TOC file for input: %m");
214  }
215 
216  /*
217  * Make unbuffered since we will dup() it, and the buffers screw each
218  * other
219  */
220  /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
221 
222  ctx->tarFHpos = 0;
223 
224  ctx->hasSeek = checkSeek(ctx->tarFH);
225 
226  ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
227  ReadHead(AH);
228  ReadToc(AH);
229  tarClose(AH, ctx->FH); /* Nothing else in the file... */
230  }
231 }
232 
233 /*
234  * - Start a new TOC entry
235  * Setup the output file name.
236  */
237 static void
239 {
240  lclTocEntry *ctx;
241  char fn[K_STD_BUF_SIZE];
242 
244  if (te->dataDumper != NULL)
245  {
246  snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
247  ctx->filename = pg_strdup(fn);
248  }
249  else
250  {
251  ctx->filename = NULL;
252  ctx->TH = NULL;
253  }
254  te->formatData = (void *) ctx;
255 }
256 
257 static void
259 {
260  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
261 
262  if (ctx->filename)
263  WriteStr(AH, ctx->filename);
264  else
265  WriteStr(AH, "");
266 }
267 
268 static void
270 {
271  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
272 
273  if (ctx == NULL)
274  {
276  te->formatData = (void *) ctx;
277  }
278 
279  ctx->filename = ReadStr(AH);
280  if (strlen(ctx->filename) == 0)
281  {
282  free(ctx->filename);
283  ctx->filename = NULL;
284  }
285  ctx->TH = NULL;
286 }
287 
288 static void
290 {
291  lclTocEntry *ctx = (lclTocEntry *) te->formatData;
292 
293  if (AH->public.verbose && ctx->filename != NULL)
294  ahprintf(AH, "-- File: %s\n", ctx->filename);
295 }
296 
297 static void
299 {
300  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
301 
302  tctx->TH = tarOpen(AH, tctx->filename, 'w');
303 }
304 
305 static TAR_MEMBER *
306 tarOpen(ArchiveHandle *AH, const char *filename, char mode)
307 {
308  lclContext *ctx = (lclContext *) AH->formatData;
309  TAR_MEMBER *tm;
310 
311  if (mode == 'r')
312  {
313  tm = _tarPositionTo(AH, filename);
314  if (!tm) /* Not found */
315  {
316  if (filename)
317  {
318  /*
319  * Couldn't find the requested file. Future: do SEEK(0) and
320  * retry.
321  */
322  pg_fatal("could not find file \"%s\" in archive", filename);
323  }
324  else
325  {
326  /* Any file OK, none left, so return NULL */
327  return NULL;
328  }
329  }
330 
332  tm->nFH = ctx->tarFH;
333  else
334  pg_fatal("compression is not supported by tar archive format");
335  }
336  else
337  {
338  int old_umask;
339 
341 
342  /*
343  * POSIX does not require, but permits, tmpfile() to restrict file
344  * permissions. Given an OS crash after we write data, the filesystem
345  * might retain the data but forget tmpfile()'s unlink(). If so, the
346  * file mode protects confidentiality of the data written.
347  */
348  old_umask = umask(S_IRWXG | S_IRWXO);
349 
350 #ifndef WIN32
351  tm->tmpFH = tmpfile();
352 #else
353 
354  /*
355  * On WIN32, tmpfile() generates a filename in the root directory,
356  * which requires administrative permissions on certain systems. Loop
357  * until we find a unique file name we can create.
358  */
359  while (1)
360  {
361  char *name;
362  int fd;
363 
364  name = _tempnam(NULL, "pg_temp_");
365  if (name == NULL)
366  break;
367  fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
368  O_TEMPORARY, S_IRUSR | S_IWUSR);
369  free(name);
370 
371  if (fd != -1) /* created a file */
372  {
373  tm->tmpFH = fdopen(fd, "w+b");
374  break;
375  }
376  else if (errno != EEXIST) /* failure other than file exists */
377  break;
378  }
379 #endif
380 
381  if (tm->tmpFH == NULL)
382  pg_fatal("could not generate temporary file name: %m");
383 
384  umask(old_umask);
385 
387  tm->nFH = tm->tmpFH;
388  else
389  pg_fatal("compression is not supported by tar archive format");
390 
391  tm->AH = AH;
392  tm->targetFile = pg_strdup(filename);
393  }
394 
395  tm->mode = mode;
396  tm->tarFH = ctx->tarFH;
397 
398  return tm;
399 }
400 
401 static void
403 {
405  pg_fatal("compression is not supported by tar archive format");
406 
407  if (th->mode == 'w')
408  _tarAddFile(AH, th); /* This will close the temp file */
409 
410  /*
411  * else Nothing to do for normal read since we don't dup() normal file
412  * handle, and we don't use temp files.
413  */
414 
415  free(th->targetFile);
416 
417  th->nFH = NULL;
418 }
419 
420 #ifdef __NOT_USED__
421 static char *
422 tarGets(char *buf, size_t len, TAR_MEMBER *th)
423 {
424  char *s;
425  size_t cnt = 0;
426  char c = ' ';
427  int eof = 0;
428 
429  /* Can't read past logical EOF */
430  if (len > (th->fileLen - th->pos))
431  len = th->fileLen - th->pos;
432 
433  while (cnt < len && c != '\n')
434  {
435  if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
436  {
437  eof = 1;
438  break;
439  }
440  buf[cnt++] = c;
441  }
442 
443  if (eof && cnt == 0)
444  s = NULL;
445  else
446  {
447  buf[cnt++] = '\0';
448  s = buf;
449  }
450 
451  if (s)
452  {
453  len = strlen(s);
454  th->pos += len;
455  }
456 
457  return s;
458 }
459 #endif
460 
461 /*
462  * Just read bytes from the archive. This is the low level read routine
463  * that is used for ALL reads on a tar file.
464  */
465 static size_t
466 _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
467 {
468  lclContext *ctx = (lclContext *) AH->formatData;
469  size_t avail;
470  size_t used = 0;
471  size_t res = 0;
472 
473  Assert(th || fh);
474 
475  avail = AH->lookaheadLen - AH->lookaheadPos;
476  if (avail > 0)
477  {
478  /* We have some lookahead bytes to use */
479  if (avail >= len) /* Just use the lookahead buffer */
480  used = len;
481  else
482  used = avail;
483 
484  /* Copy, and adjust buffer pos */
485  memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
486  AH->lookaheadPos += used;
487 
488  /* Adjust required length */
489  len -= used;
490  }
491 
492  /* Read the file if len > 0 */
493  if (len > 0)
494  {
495  if (fh)
496  {
497  res = fread(&((char *) buf)[used], 1, len, fh);
498  if (res != len && !feof(fh))
499  READ_ERROR_EXIT(fh);
500  }
501  else if (th)
502  {
503  res = fread(&((char *) buf)[used], 1, len, th->nFH);
504  if (res != len && !feof(th->nFH))
505  READ_ERROR_EXIT(th->nFH);
506  }
507  }
508 
509  ctx->tarFHpos += res + used;
510 
511  return (res + used);
512 }
513 
514 static size_t
515 tarRead(void *buf, size_t len, TAR_MEMBER *th)
516 {
517  size_t res;
518 
519  if (th->pos + len > th->fileLen)
520  len = th->fileLen - th->pos;
521 
522  if (len <= 0)
523  return 0;
524 
525  res = _tarReadRaw(th->AH, buf, len, th, NULL);
526 
527  th->pos += res;
528 
529  return res;
530 }
531 
532 static size_t
533 tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
534 {
535  size_t res;
536 
537  res = fwrite(buf, 1, len, th->nFH);
538 
539  th->pos += res;
540  return res;
541 }
542 
543 static void
544 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
545 {
546  lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
547 
548  if (tarWrite(data, dLen, tctx->TH) != dLen)
550 }
551 
552 static void
554 {
555  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
556 
557  /* Close the file */
558  tarClose(AH, tctx->TH);
559  tctx->TH = NULL;
560 }
561 
562 /*
563  * Print data for a given file
564  */
565 static void
567 {
568  lclContext *ctx = (lclContext *) AH->formatData;
569  char buf[4096];
570  size_t cnt;
571  TAR_MEMBER *th;
572 
573  if (!filename)
574  return;
575 
576  th = tarOpen(AH, filename, 'r');
577  ctx->FH = th;
578 
579  while ((cnt = tarRead(buf, 4095, th)) > 0)
580  {
581  buf[cnt] = '\0';
582  ahwrite(buf, 1, cnt, AH);
583  }
584 
585  tarClose(AH, th);
586 }
587 
588 
589 /*
590  * Print data for a given TOC entry
591 */
592 static void
594 {
595  lclContext *ctx = (lclContext *) AH->formatData;
596  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
597  int pos1;
598 
599  if (!tctx->filename)
600  return;
601 
602  /*
603  * If we're writing the special restore.sql script, emit a suitable
604  * command to include each table's data from the corresponding file.
605  *
606  * In the COPY case this is a bit klugy because the regular COPY command
607  * was already printed before we get control.
608  */
609  if (ctx->isSpecialScript)
610  {
611  if (te->copyStmt)
612  {
613  /* Abort the COPY FROM stdin */
614  ahprintf(AH, "\\.\n");
615 
616  /*
617  * The COPY statement should look like "COPY ... FROM stdin;\n",
618  * see dumpTableData().
619  */
620  pos1 = (int) strlen(te->copyStmt) - 13;
621  if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
622  strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
623  pg_fatal("unexpected COPY statement syntax: \"%s\"",
624  te->copyStmt);
625 
626  /* Emit all but the FROM part ... */
627  ahwrite(te->copyStmt, 1, pos1, AH);
628  /* ... and insert modified FROM */
629  ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
630  }
631  else
632  {
633  /* --inserts mode, no worries, just include the data file */
634  ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
635  }
636 
637  return;
638  }
639 
640  if (strcmp(te->desc, "BLOBS") == 0)
641  _LoadLOs(AH);
642  else
643  _PrintFileData(AH, tctx->filename);
644 }
645 
646 static void
648 {
649  Oid oid;
650  lclContext *ctx = (lclContext *) AH->formatData;
651  TAR_MEMBER *th;
652  size_t cnt;
653  bool foundLO = false;
654  char buf[4096];
655 
656  StartRestoreLOs(AH);
657 
658  th = tarOpen(AH, NULL, 'r'); /* Open next file */
659  while (th != NULL)
660  {
661  ctx->FH = th;
662 
663  if (strncmp(th->targetFile, "blob_", 5) == 0)
664  {
665  oid = atooid(&th->targetFile[5]);
666  if (oid != 0)
667  {
668  pg_log_info("restoring large object with OID %u", oid);
669 
670  StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
671 
672  while ((cnt = tarRead(buf, 4095, th)) > 0)
673  {
674  buf[cnt] = '\0';
675  ahwrite(buf, 1, cnt, AH);
676  }
677  EndRestoreLO(AH, oid);
678  foundLO = true;
679  }
680  tarClose(AH, th);
681  }
682  else
683  {
684  tarClose(AH, th);
685 
686  /*
687  * Once we have found the first LO, stop at the first non-LO entry
688  * (which will be 'blobs.toc'). This coding would eat all the
689  * rest of the archive if there are no LOs ... but this function
690  * shouldn't be called at all in that case.
691  */
692  if (foundLO)
693  break;
694  }
695 
696  th = tarOpen(AH, NULL, 'r');
697  }
698  EndRestoreLOs(AH);
699 }
700 
701 
702 static int
703 _WriteByte(ArchiveHandle *AH, const int i)
704 {
705  lclContext *ctx = (lclContext *) AH->formatData;
706  char b = i; /* Avoid endian problems */
707 
708  if (tarWrite(&b, 1, ctx->FH) != 1)
710 
711  ctx->filePos += 1;
712  return 1;
713 }
714 
715 static int
717 {
718  lclContext *ctx = (lclContext *) AH->formatData;
719  size_t res;
720  unsigned char c;
721 
722  res = tarRead(&c, 1, ctx->FH);
723  if (res != 1)
724  /* We already would have exited for errors on reads, must be EOF */
725  pg_fatal("could not read from input file: end of file");
726  ctx->filePos += 1;
727  return c;
728 }
729 
730 static void
731 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
732 {
733  lclContext *ctx = (lclContext *) AH->formatData;
734 
735  if (tarWrite(buf, len, ctx->FH) != len)
737 
738  ctx->filePos += len;
739 }
740 
741 static void
742 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
743 {
744  lclContext *ctx = (lclContext *) AH->formatData;
745 
746  if (tarRead(buf, len, ctx->FH) != len)
747  /* We already would have exited for errors on reads, must be EOF */
748  pg_fatal("could not read from input file: end of file");
749 
750  ctx->filePos += len;
751 }
752 
753 static void
755 {
756  lclContext *ctx = (lclContext *) AH->formatData;
757  TAR_MEMBER *th;
758  RestoreOptions *ropt;
759  RestoreOptions *savRopt;
760  DumpOptions *savDopt;
761  int savVerbose,
762  i;
763 
764  if (AH->mode == archModeWrite)
765  {
766  /*
767  * Write the Header & TOC to the archive FIRST
768  */
769  th = tarOpen(AH, "toc.dat", 'w');
770  ctx->FH = th;
771  WriteHead(AH);
772  WriteToc(AH);
773  tarClose(AH, th); /* Not needed any more */
774 
775  /*
776  * Now send the data (tables & LOs)
777  */
778  WriteDataChunks(AH, NULL);
779 
780  /*
781  * Now this format wants to append a script which does a full restore
782  * if the files have been extracted.
783  */
784  th = tarOpen(AH, "restore.sql", 'w');
785 
786  tarPrintf(th, "--\n"
787  "-- NOTE:\n"
788  "--\n"
789  "-- File paths need to be edited. Search for $$PATH$$ and\n"
790  "-- replace it with the path to the directory containing\n"
791  "-- the extracted data files.\n"
792  "--\n");
793 
794  AH->CustomOutPtr = _scriptOut;
795 
796  ctx->isSpecialScript = 1;
797  ctx->scriptTH = th;
798 
799  ropt = NewRestoreOptions();
800  memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
801  ropt->filename = NULL;
802  ropt->dropSchema = 1;
803  ropt->superuser = NULL;
804  ropt->suppressDumpWarnings = true;
805 
806  savDopt = AH->public.dopt;
807  savRopt = AH->public.ropt;
808 
809  SetArchiveOptions((Archive *) AH, NULL, ropt);
810 
811  savVerbose = AH->public.verbose;
812  AH->public.verbose = 0;
813 
814  RestoreArchive((Archive *) AH);
815 
816  SetArchiveOptions((Archive *) AH, savDopt, savRopt);
817 
818  AH->public.verbose = savVerbose;
819 
820  tarClose(AH, th);
821 
822  ctx->isSpecialScript = 0;
823 
824  /*
825  * EOF marker for tar files is two blocks of NULLs.
826  */
827  for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
828  {
829  if (fputc(0, ctx->tarFH) == EOF)
831  }
832 
833  /* Sync the output file if one is defined */
834  if (AH->dosync && AH->fSpec)
835  (void) fsync_fname(AH->fSpec, false);
836  }
837 
838  AH->FH = NULL;
839 }
840 
841 static size_t
842 _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
843 {
844  lclContext *ctx = (lclContext *) AH->formatData;
845 
846  return tarWrite(buf, len, ctx->scriptTH);
847 }
848 
849 /*
850  * Large Object support
851  */
852 
853 /*
854  * Called by the archiver when starting to save all BLOB DATA (not schema).
855  * This routine should save whatever format-specific information is needed
856  * to read the LOs back into memory.
857  *
858  * It is called just prior to the dumper's DataDumper routine.
859  *
860  * Optional, but strongly recommended.
861  *
862  */
863 static void
865 {
866  lclContext *ctx = (lclContext *) AH->formatData;
867  char fname[K_STD_BUF_SIZE];
868 
869  sprintf(fname, "blobs.toc");
870  ctx->loToc = tarOpen(AH, fname, 'w');
871 }
872 
873 /*
874  * Called by the archiver when the dumper calls StartLO.
875  *
876  * Mandatory.
877  *
878  * Must save the passed OID for retrieval at restore-time.
879  */
880 static void
882 {
883  lclContext *ctx = (lclContext *) AH->formatData;
884  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
885  char fname[255];
886 
887  if (oid == 0)
888  pg_fatal("invalid OID for large object (%u)", oid);
889 
891  pg_fatal("compression is not supported by tar archive format");
892 
893  sprintf(fname, "blob_%u.dat", oid);
894 
895  tarPrintf(ctx->loToc, "%u %s\n", oid, fname);
896 
897  tctx->TH = tarOpen(AH, fname, 'w');
898 }
899 
900 /*
901  * Called by the archiver when the dumper calls EndLO.
902  *
903  * Optional.
904  *
905  */
906 static void
908 {
909  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
910 
911  tarClose(AH, tctx->TH);
912 }
913 
914 /*
915  * Called by the archiver when finishing saving all BLOB DATA.
916  *
917  * Optional.
918  *
919  */
920 static void
922 {
923  lclContext *ctx = (lclContext *) AH->formatData;
924 
925  /* Write out a fake zero OID to mark end-of-LOs. */
926  /* WriteInt(AH, 0); */
927 
928  tarClose(AH, ctx->loToc);
929 }
930 
931 
932 
933 /*------------
934  * TAR Support
935  *------------
936  */
937 
938 static int
939 tarPrintf(TAR_MEMBER *th, const char *fmt,...)
940 {
941  int save_errno = errno;
942  char *p;
943  size_t len = 128; /* initial assumption about buffer size */
944  size_t cnt;
945 
946  for (;;)
947  {
948  va_list args;
949 
950  /* Allocate work buffer. */
951  p = (char *) pg_malloc(len);
952 
953  /* Try to format the data. */
954  errno = save_errno;
955  va_start(args, fmt);
956  cnt = pvsnprintf(p, len, fmt, args);
957  va_end(args);
958 
959  if (cnt < len)
960  break; /* success */
961 
962  /* Release buffer and loop around to try again with larger len. */
963  free(p);
964  len = cnt;
965  }
966 
967  cnt = tarWrite(p, cnt, th);
968  free(p);
969  return (int) cnt;
970 }
971 
972 bool
973 isValidTarHeader(char *header)
974 {
975  int sum;
976  int chk = tarChecksum(header);
977 
978  sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8);
979 
980  if (sum != chk)
981  return false;
982 
983  /* POSIX tar format */
984  if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 &&
985  memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0)
986  return true;
987  /* GNU tar format */
988  if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar \0", 8) == 0)
989  return true;
990  /* not-quite-POSIX format written by pre-9.3 pg_dump */
991  if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0)
992  return true;
993 
994  return false;
995 }
996 
997 /* Given the member, write the TAR header & copy the file */
998 static void
1000 {
1001  lclContext *ctx = (lclContext *) AH->formatData;
1002  FILE *tmp = th->tmpFH; /* Grab it for convenience */
1003  char buf[32768];
1004  size_t cnt;
1005  pgoff_t len = 0;
1006  size_t res;
1007  size_t i,
1008  pad;
1009 
1010  /*
1011  * Find file len & go back to start.
1012  */
1013  if (fseeko(tmp, 0, SEEK_END) != 0)
1014  pg_fatal("error during file seek: %m");
1015  th->fileLen = ftello(tmp);
1016  if (th->fileLen < 0)
1017  pg_fatal("could not determine seek position in archive file: %m");
1018  if (fseeko(tmp, 0, SEEK_SET) != 0)
1019  pg_fatal("error during file seek: %m");
1020 
1021  _tarWriteHeader(th);
1022 
1023  while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1024  {
1025  if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1027  len += res;
1028  }
1029  if (!feof(tmp))
1030  READ_ERROR_EXIT(tmp);
1031 
1032  if (fclose(tmp) != 0) /* This *should* delete it... */
1033  pg_fatal("could not close temporary file: %m");
1034 
1035  if (len != th->fileLen)
1036  pg_fatal("actual file length (%lld) does not match expected (%lld)",
1037  (long long) len, (long long) th->fileLen);
1038 
1040  for (i = 0; i < pad; i++)
1041  {
1042  if (fputc('\0', th->tarFH) == EOF)
1044  }
1045 
1046  ctx->tarFHpos += len + pad;
1047 }
1048 
1049 /* Locate the file in the archive, read header and position to data */
1050 static TAR_MEMBER *
1052 {
1053  lclContext *ctx = (lclContext *) AH->formatData;
1055  char c;
1056  char header[TAR_BLOCK_SIZE];
1057  size_t i,
1058  len,
1059  blks;
1060  int id;
1061 
1062  th->AH = AH;
1063 
1064  /* Go to end of current file, if any */
1065  if (ctx->tarFHpos != 0)
1066  {
1067  pg_log_debug("moving from position %lld to next member at file position %lld",
1068  (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1069 
1070  while (ctx->tarFHpos < ctx->tarNextMember)
1071  _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1072  }
1073 
1074  pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1075 
1076  /* We are at the start of the file, or at the next member */
1077 
1078  /* Get the header */
1079  if (!_tarGetHeader(AH, th))
1080  {
1081  if (filename)
1082  pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1083  else
1084  {
1085  /*
1086  * We're just scanning the archive for the next file, so return
1087  * null
1088  */
1089  free(th);
1090  return NULL;
1091  }
1092  }
1093 
1094  while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1095  {
1096  pg_log_debug("skipping tar member %s", th->targetFile);
1097 
1098  id = atoi(th->targetFile);
1099  if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1100  pg_fatal("restoring data out of order is not supported in this archive format: "
1101  "\"%s\" is required, but comes before \"%s\" in the archive file.",
1102  th->targetFile, filename);
1103 
1104  /* Header doesn't match, so read to next header */
1105  len = th->fileLen;
1107  blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1108 
1109  for (i = 0; i < blks; i++)
1110  _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1111 
1112  if (!_tarGetHeader(AH, th))
1113  pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1114  }
1115 
1116  ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1118  th->pos = 0;
1119 
1120  return th;
1121 }
1122 
1123 /* Read & verify a header */
1124 static int
1126 {
1127  lclContext *ctx = (lclContext *) AH->formatData;
1128  char h[TAR_BLOCK_SIZE];
1129  char tag[100 + 1];
1130  int sum,
1131  chk;
1132  pgoff_t len;
1133  pgoff_t hPos;
1134  bool gotBlock = false;
1135 
1136  while (!gotBlock)
1137  {
1138  /* Save the pos for reporting purposes */
1139  hPos = ctx->tarFHpos;
1140 
1141  /* Read the next tar block, return EOF, exit if short */
1142  len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1143  if (len == 0) /* EOF */
1144  return 0;
1145 
1146  if (len != TAR_BLOCK_SIZE)
1147  pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1148  "incomplete tar header found (%lu bytes)",
1149  len),
1150  (unsigned long) len);
1151 
1152  /* Calc checksum */
1153  chk = tarChecksum(h);
1154  sum = read_tar_number(&h[TAR_OFFSET_CHECKSUM], 8);
1155 
1156  /*
1157  * If the checksum failed, see if it is a null block. If so, silently
1158  * continue to the next block.
1159  */
1160  if (chk == sum)
1161  gotBlock = true;
1162  else
1163  {
1164  int i;
1165 
1166  for (i = 0; i < TAR_BLOCK_SIZE; i++)
1167  {
1168  if (h[i] != 0)
1169  {
1170  gotBlock = true;
1171  break;
1172  }
1173  }
1174  }
1175  }
1176 
1177  /* Name field is 100 bytes, might not be null-terminated */
1178  strlcpy(tag, &h[TAR_OFFSET_NAME], 100 + 1);
1179 
1180  len = read_tar_number(&h[TAR_OFFSET_SIZE], 12);
1181 
1182  pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1183  tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1184 
1185  if (chk != sum)
1186  pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1187  tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1188 
1189  th->targetFile = pg_strdup(tag);
1190  th->fileLen = len;
1191 
1192  return 1;
1193 }
1194 
1195 
1196 static void
1198 {
1199  char h[TAR_BLOCK_SIZE];
1200 
1201  tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1202  0600, 04000, 02000, time(NULL));
1203 
1204  /* Now write the completed header. */
1205  if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1207 }
#define PG_BINARY_R
Definition: c.h:1262
#define ngettext(s, p, n)
Definition: c.h:1168
#define pg_attribute_printf(f, a)
Definition: c.h:178
#define PG_BINARY_W
Definition: c.h:1263
@ PG_COMPRESSION_NONE
Definition: compression.h:23
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:756
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define pg_malloc0_object(type)
Definition: fe_memutils.h:39
#define free(a)
Definition: header.h:65
int b
Definition: isn.c:70
int i
Definition: isn.c:73
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
static void const char * fmt
va_end(args)
Assert(fmt[strlen(fmt) - 1] !='\n')
va_start(args, fmt)
static struct pg_tm tm
Definition: localtime.c:104
#define pg_log_info(...)
Definition: logging.h:124
#define pg_log_debug(...)
Definition: logging.h:133
RestoreOptions * NewRestoreOptions(void)
@ archModeWrite
Definition: pg_backup.h:51
void SetArchiveOptions(Archive *AH, DumpOptions *dopt, RestoreOptions *ropt)
void RestoreArchive(Archive *AHX)
int TocIDRequired(ArchiveHandle *AH, DumpId id)
bool checkSeek(FILE *fp)
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteHead(ArchiveHandle *AH)
void StartRestoreLOs(ArchiveHandle *AH)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void EndRestoreLO(ArchiveHandle *AH, Oid oid)
void WriteToc(ArchiveHandle *AH)
void EndRestoreLOs(ArchiveHandle *AH)
void StartRestoreLO(ArchiveHandle *AH, Oid oid, bool drop)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define LOBBUFSIZE
#define WRITE_ERROR_EXIT
#define REQ_DATA
#define READ_ERROR_EXIT(fd)
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2
static void _tarWriteHeader(TAR_MEMBER *th)
static void _StartLOs(ArchiveHandle *AH, TocEntry *te)
void InitArchiveFmt_Tar(ArchiveHandle *AH)
static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
static void _LoadLOs(ArchiveHandle *AH)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndLOs(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
#define K_STD_BUF_SIZE
Definition: pg_backup_tar.c:63
static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static int _ReadByte(ArchiveHandle *AH)
bool isValidTarHeader(char *header)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static TAR_MEMBER * tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
static TAR_MEMBER * _tarPositionTo(ArchiveHandle *AH, const char *filename)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static int static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
#define pg_fatal(...)
static PgChecksumMode mode
Definition: pg_checksums.c:56
const void size_t len
const void * data
while(p+4<=pend)
static char * filename
Definition: pg_dumpall.c:121
static char * buf
Definition: pg_test_fsync.c:73
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:58
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:79
int tarChecksum(char *header)
Definition: tar.c:90
@ TAR_OFFSET_VERSION
Definition: pgtar.h:49
@ TAR_OFFSET_NAME
Definition: pgtar.h:39
@ TAR_OFFSET_SIZE
Definition: pgtar.h:43
@ TAR_OFFSET_CHECKSUM
Definition: pgtar.h:45
@ TAR_OFFSET_MAGIC
Definition: pgtar.h:48
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define sprintf
Definition: port.h:240
#define snprintf
Definition: port.h:238
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
char * c
static int fd(const char *x, int i)
Definition: preproc-init.c:105
size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args)
Definition: psprintf.c:106
DumpOptions * dopt
Definition: pg_backup.h:212
int verbose
Definition: pg_backup.h:215
RestoreOptions * ropt
Definition: pg_backup.h:213
pgoff_t pos
Definition: pg_backup_tar.c:73
char * targetFile
Definition: pg_backup_tar.c:71
pgoff_t fileLen
Definition: pg_backup_tar.c:74
FILE * tarFH
Definition: pg_backup_tar.c:69
ArchiveHandle * AH
Definition: pg_backup_tar.c:75
FILE * nFH
Definition: pg_backup_tar.c:68
FILE * tmpFH
Definition: pg_backup_tar.c:70
CustomOutPtrType CustomOutPtr
pg_compress_specification compression_spec
struct _tocEntry * currToc
DataDumperPtr dataDumper
TAR_MEMBER * loToc
Definition: pg_backup_tar.c:82
FILE * tarFH
Definition: pg_backup_tar.c:83
pgoff_t filePos
Definition: pg_backup_tar.c:81
pgoff_t tarFHpos
Definition: pg_backup_tar.c:84
TAR_MEMBER * FH
Definition: pg_backup_tar.c:86
pgoff_t tarNextMember
Definition: pg_backup_tar.c:85
TAR_MEMBER * scriptTH
Definition: pg_backup_tar.c:88
int isSpecialScript
Definition: pg_backup_tar.c:87
TAR_MEMBER * TH
Definition: pg_backup_tar.c:93
pg_compress_algorithm algorithm
Definition: compression.h:34
static void * fn(void *arg)
Definition: thread-alloc.c:119
const char * name
#define S_IRWXG
Definition: win32_port.h:310
#define S_IRWXO
Definition: win32_port.h:322
#define ftello(stream)
Definition: win32_port.h:219
#define S_IRUSR
Definition: win32_port.h:289
#define fseeko(stream, offset, origin)
Definition: win32_port.h:216
#define S_IWUSR
Definition: win32_port.h:292
#define pgoff_t
Definition: win32_port.h:207