PostgreSQL Source Code  git master
pg_backup_directory.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_directory.c
4  *
5  * A directory format dump is a directory, which contains a "toc.dat" file
6  * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7  * Large objects (BLOBs) are stored in separate files named "blob_<oid>.dat",
8  * and there's a plain-text TOC file for them called "blobs.toc". If
9  * compression is used, each data file is individually compressed and the
10  * ".gz" suffix is added to the filenames. The TOC files are never
11  * compressed by pg_dump, however they are accepted with the .gz suffix too,
12  * in case the user has manually compressed them with 'gzip'.
13  *
14  * NOTE: This format is identical to the files written in the tar file in
15  * the 'tar' format, except that we don't write the restore.sql file (TODO),
16  * and the tar format doesn't support compression. Please keep the formats in
17  * sync.
18  *
19  *
20  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
21  * Portions Copyright (c) 1994, Regents of the University of California
22  * Portions Copyright (c) 2000, Philip Warner
23  *
24  * Rights are granted to use this software in any way so long
25  * as this notice is not removed.
26  *
27  * The author is not responsible for loss or damages that may
28  * result from its use.
29  *
30  * IDENTIFICATION
31  * src/bin/pg_dump/pg_backup_directory.c
32  *
33  *-------------------------------------------------------------------------
34  */
35 #include "postgres_fe.h"
36 
37 #include <dirent.h>
38 #include <sys/stat.h>
39 
40 #include "common/file_utils.h"
41 #include "compress_io.h"
42 #include "parallel.h"
43 #include "pg_backup_utils.h"
44 
45 typedef struct
46 {
47  /*
48  * Our archive location. This is basically what the user specified as his
49  * backup file but of course here it is a directory.
50  */
51  char *directory;
52 
53  cfp *dataFH; /* currently open data file */
54 
55  cfp *blobsTocFH; /* file handle for blobs.toc */
56  ParallelState *pstate; /* for parallel backup / restore */
57 } lclContext;
58 
59 typedef struct
60 {
61  char *filename; /* filename excluding the directory (basename) */
62 } lclTocEntry;
63 
64 /* prototypes for private functions */
65 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
66 static void _StartData(ArchiveHandle *AH, TocEntry *te);
67 static void _EndData(ArchiveHandle *AH, TocEntry *te);
68 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
69 static int _WriteByte(ArchiveHandle *AH, const int i);
70 static int _ReadByte(ArchiveHandle *AH);
71 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
72 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
73 static void _CloseArchive(ArchiveHandle *AH);
74 static void _ReopenArchive(ArchiveHandle *AH);
75 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
76 
77 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
78 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
79 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
80 
81 static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
82 static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
83 static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
84 static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
85 static void _LoadBlobs(ArchiveHandle *AH);
86 
87 static void _PrepParallelRestore(ArchiveHandle *AH);
88 static void _Clone(ArchiveHandle *AH);
89 static void _DeClone(ArchiveHandle *AH);
90 
93 
94 static void setFilePath(ArchiveHandle *AH, char *buf,
95  const char *relativeFilename);
96 
97 /*
98  * Init routine required by ALL formats. This is a global routine
99  * and should be declared in pg_backup_archiver.h
100  *
101  * Its task is to create any extra archive context (using AH->formatData),
102  * and to initialize the supported function pointers.
103  *
104  * It should also prepare whatever its input source is for reading/writing,
105  * and in the case of a read mode connection, it should load the Header & TOC.
106  */
107 void
109 {
110  lclContext *ctx;
111 
112  /* Assuming static functions, this can be copied for each format. */
114  AH->StartDataPtr = _StartData;
115  AH->WriteDataPtr = _WriteData;
116  AH->EndDataPtr = _EndData;
117  AH->WriteBytePtr = _WriteByte;
118  AH->ReadBytePtr = _ReadByte;
119  AH->WriteBufPtr = _WriteBuf;
120  AH->ReadBufPtr = _ReadBuf;
121  AH->ClosePtr = _CloseArchive;
127 
129  AH->StartBlobPtr = _StartBlob;
130  AH->EndBlobPtr = _EndBlob;
131  AH->EndBlobsPtr = _EndBlobs;
132 
134  AH->ClonePtr = _Clone;
135  AH->DeClonePtr = _DeClone;
136 
139 
140  /* Set up our private context */
141  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
142  AH->formatData = (void *) ctx;
143 
144  ctx->dataFH = NULL;
145  ctx->blobsTocFH = NULL;
146 
147  /* Initialize LO buffering */
148  AH->lo_buf_size = LOBBUFSIZE;
149  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
150 
151  /*
152  * Now open the TOC file
153  */
154 
155  if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
156  pg_fatal("no output directory specified");
157 
158  ctx->directory = AH->fSpec;
159 
160  if (AH->mode == archModeWrite)
161  {
162  struct stat st;
163  bool is_empty = false;
164 
165  /* we accept an empty existing directory */
166  if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
167  {
168  DIR *dir = opendir(ctx->directory);
169 
170  if (dir)
171  {
172  struct dirent *d;
173 
174  is_empty = true;
175  while (errno = 0, (d = readdir(dir)))
176  {
177  if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
178  {
179  is_empty = false;
180  break;
181  }
182  }
183 
184  if (errno)
185  pg_fatal("could not read directory \"%s\": %m",
186  ctx->directory);
187 
188  if (closedir(dir))
189  pg_fatal("could not close directory \"%s\": %m",
190  ctx->directory);
191  }
192  }
193 
194  if (!is_empty && mkdir(ctx->directory, 0700) < 0)
195  pg_fatal("could not create directory \"%s\": %m",
196  ctx->directory);
197  }
198  else
199  { /* Read Mode */
200  char fname[MAXPGPATH];
201  cfp *tocFH;
202 
203  setFilePath(AH, fname, "toc.dat");
204 
205  tocFH = cfopen_read(fname, PG_BINARY_R);
206  if (tocFH == NULL)
207  pg_fatal("could not open input file \"%s\": %m", fname);
208 
209  ctx->dataFH = tocFH;
210 
211  /*
212  * The TOC of a directory format dump shares the format code of the
213  * tar format.
214  */
215  AH->format = archTar;
216  ReadHead(AH);
217  AH->format = archDirectory;
218  ReadToc(AH);
219 
220  /* Nothing else in the file, so close it again... */
221  if (cfclose(tocFH) != 0)
222  pg_fatal("could not close TOC file: %m");
223  ctx->dataFH = NULL;
224  }
225 }
226 
227 /*
228  * Called by the Archiver when the dumper creates a new TOC entry.
229  *
230  * We determine the filename for this entry.
231 */
232 static void
234 {
235  lclTocEntry *tctx;
236  char fn[MAXPGPATH];
237 
238  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
239  if (strcmp(te->desc, "BLOBS") == 0)
240  tctx->filename = pg_strdup("blobs.toc");
241  else if (te->dataDumper)
242  {
243  snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
244  tctx->filename = pg_strdup(fn);
245  }
246  else
247  tctx->filename = NULL;
248 
249  te->formatData = (void *) tctx;
250 }
251 
252 /*
253  * Called by the Archiver to save any extra format-related TOC entry
254  * data.
255  *
256  * Use the Archiver routines to write data - they are non-endian, and
257  * maintain other important file information.
258  */
259 static void
261 {
262  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
263 
264  /*
265  * A dumpable object has set tctx->filename, any other object has not.
266  * (see _ArchiveEntry).
267  */
268  if (tctx->filename)
269  WriteStr(AH, tctx->filename);
270  else
271  WriteStr(AH, "");
272 }
273 
274 /*
275  * Called by the Archiver to read any extra format-related TOC data.
276  *
277  * Needs to match the order defined in _WriteExtraToc, and should also
278  * use the Archiver input routines.
279  */
280 static void
282 {
283  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
284 
285  if (tctx == NULL)
286  {
287  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
288  te->formatData = (void *) tctx;
289  }
290 
291  tctx->filename = ReadStr(AH);
292  if (strlen(tctx->filename) == 0)
293  {
294  free(tctx->filename);
295  tctx->filename = NULL;
296  }
297 }
298 
299 /*
300  * Called by the Archiver when restoring an archive to output a comment
301  * that includes useful information about the TOC entry.
302  */
303 static void
305 {
306  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
307 
308  if (AH->public.verbose && tctx->filename)
309  ahprintf(AH, "-- File: %s\n", tctx->filename);
310 }
311 
312 /*
313  * Called by the archiver when saving TABLE DATA (not schema). This routine
314  * should save whatever format-specific information is needed to read
315  * the archive back.
316  *
317  * It is called just prior to the dumper's 'DataDumper' routine being called.
318  *
319  * We create the data file for writing.
320  */
321 static void
323 {
324  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
325  lclContext *ctx = (lclContext *) AH->formatData;
326  char fname[MAXPGPATH];
327 
328  setFilePath(AH, fname, tctx->filename);
329 
330  ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
331  AH->compression_spec);
332  if (ctx->dataFH == NULL)
333  pg_fatal("could not open output file \"%s\": %m", fname);
334 }
335 
336 /*
337  * Called by archiver when dumper calls WriteData. This routine is
338  * called for both BLOB and TABLE data; it is the responsibility of
339  * the format to manage each kind of data using StartBlob/StartData.
340  *
341  * It should only be called from within a DataDumper routine.
342  *
343  * We write the data to the open data file.
344  */
345 static void
346 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
347 {
348  lclContext *ctx = (lclContext *) AH->formatData;
349 
350  errno = 0;
351  if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
352  {
353  /* if write didn't set errno, assume problem is no disk space */
354  if (errno == 0)
355  errno = ENOSPC;
356  pg_fatal("could not write to output file: %s",
357  get_cfp_error(ctx->dataFH));
358  }
359 }
360 
361 /*
362  * Called by the archiver when a dumper's 'DataDumper' routine has
363  * finished.
364  *
365  * We close the data file.
366  */
367 static void
369 {
370  lclContext *ctx = (lclContext *) AH->formatData;
371 
372  /* Close the file */
373  if (cfclose(ctx->dataFH) != 0)
374  pg_fatal("could not close data file: %m");
375 
376  ctx->dataFH = NULL;
377 }
378 
379 /*
380  * Print data for a given file (can be a BLOB as well)
381  */
382 static void
384 {
385  size_t cnt;
386  char *buf;
387  size_t buflen;
388  cfp *cfp;
389 
390  if (!filename)
391  return;
392 
394 
395  if (!cfp)
396  pg_fatal("could not open input file \"%s\": %m", filename);
397 
399  buflen = ZLIB_OUT_SIZE;
400 
401  while ((cnt = cfread(buf, buflen, cfp)))
402  {
403  ahwrite(buf, 1, cnt, AH);
404  }
405 
406  free(buf);
407  if (cfclose(cfp) != 0)
408  pg_fatal("could not close data file \"%s\": %m", filename);
409 }
410 
411 /*
412  * Print data for a given TOC entry
413 */
414 static void
416 {
417  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
418 
419  if (!tctx->filename)
420  return;
421 
422  if (strcmp(te->desc, "BLOBS") == 0)
423  _LoadBlobs(AH);
424  else
425  {
426  char fname[MAXPGPATH];
427 
428  setFilePath(AH, fname, tctx->filename);
429  _PrintFileData(AH, fname);
430  }
431 }
432 
433 static void
435 {
436  Oid oid;
437  lclContext *ctx = (lclContext *) AH->formatData;
438  char tocfname[MAXPGPATH];
439  char line[MAXPGPATH];
440 
441  StartRestoreBlobs(AH);
442 
443  setFilePath(AH, tocfname, "blobs.toc");
444 
445  ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
446 
447  if (ctx->blobsTocFH == NULL)
448  pg_fatal("could not open large object TOC file \"%s\" for input: %m",
449  tocfname);
450 
451  /* Read the blobs TOC file line-by-line, and process each blob */
452  while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
453  {
454  char blobfname[MAXPGPATH + 1];
455  char path[MAXPGPATH];
456 
457  /* Can't overflow because line and blobfname are the same length */
458  if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, blobfname) != 2)
459  pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
460  tocfname, line);
461 
462  StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
463  snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, blobfname);
464  _PrintFileData(AH, path);
465  EndRestoreBlob(AH, oid);
466  }
467  if (!cfeof(ctx->blobsTocFH))
468  pg_fatal("error reading large object TOC file \"%s\"",
469  tocfname);
470 
471  if (cfclose(ctx->blobsTocFH) != 0)
472  pg_fatal("could not close large object TOC file \"%s\": %m",
473  tocfname);
474 
475  ctx->blobsTocFH = NULL;
476 
477  EndRestoreBlobs(AH);
478 }
479 
480 
481 /*
482  * Write a byte of data to the archive.
483  * Called by the archiver to do integer & byte output to the archive.
484  * These routines are only used to read & write the headers & TOC.
485  */
486 static int
487 _WriteByte(ArchiveHandle *AH, const int i)
488 {
489  unsigned char c = (unsigned char) i;
490  lclContext *ctx = (lclContext *) AH->formatData;
491 
492  errno = 0;
493  if (cfwrite(&c, 1, ctx->dataFH) != 1)
494  {
495  /* if write didn't set errno, assume problem is no disk space */
496  if (errno == 0)
497  errno = ENOSPC;
498  pg_fatal("could not write to output file: %s",
499  get_cfp_error(ctx->dataFH));
500  }
501 
502  return 1;
503 }
504 
505 /*
506  * Read a byte of data from the archive.
507  * Called by the archiver to read bytes & integers from the archive.
508  * These routines are only used to read & write headers & TOC.
509  * EOF should be treated as a fatal error.
510  */
511 static int
513 {
514  lclContext *ctx = (lclContext *) AH->formatData;
515 
516  return cfgetc(ctx->dataFH);
517 }
518 
519 /*
520  * Write a buffer of data to the archive.
521  * Called by the archiver to write a block of bytes to the TOC or a data file.
522  */
523 static void
524 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
525 {
526  lclContext *ctx = (lclContext *) AH->formatData;
527 
528  errno = 0;
529  if (cfwrite(buf, len, ctx->dataFH) != len)
530  {
531  /* if write didn't set errno, assume problem is no disk space */
532  if (errno == 0)
533  errno = ENOSPC;
534  pg_fatal("could not write to output file: %s",
535  get_cfp_error(ctx->dataFH));
536  }
537 }
538 
539 /*
540  * Read a block of bytes from the archive.
541  *
542  * Called by the archiver to read a block of bytes from the archive
543  */
544 static void
545 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
546 {
547  lclContext *ctx = (lclContext *) AH->formatData;
548 
549  /*
550  * If there was an I/O error, we already exited in cfread(), so here we
551  * exit on short reads.
552  */
553  if (cfread(buf, len, ctx->dataFH) != len)
554  pg_fatal("could not read from input file: end of file");
555 }
556 
557 /*
558  * Close the archive.
559  *
560  * When writing the archive, this is the routine that actually starts
561  * the process of saving it to files. No data should be written prior
562  * to this point, since the user could sort the TOC after creating it.
563  *
564  * If an archive is to be written, this routine must call:
565  * WriteHead to save the archive header
566  * WriteToc to save the TOC entries
567  * WriteDataChunks to save all DATA & BLOBs.
568  */
569 static void
571 {
572  lclContext *ctx = (lclContext *) AH->formatData;
573 
574  if (AH->mode == archModeWrite)
575  {
576  cfp *tocFH;
577  pg_compress_specification compression_spec = {0};
578  char fname[MAXPGPATH];
579 
580  setFilePath(AH, fname, "toc.dat");
581 
582  /* this will actually fork the processes for a parallel backup */
583  ctx->pstate = ParallelBackupStart(AH);
584 
585  /* The TOC is always created uncompressed */
586  compression_spec.algorithm = PG_COMPRESSION_NONE;
587  tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
588  if (tocFH == NULL)
589  pg_fatal("could not open output file \"%s\": %m", fname);
590  ctx->dataFH = tocFH;
591 
592  /*
593  * Write 'tar' in the format field of the toc.dat file. The directory
594  * is compatible with 'tar', so there's no point having a different
595  * format code for it.
596  */
597  AH->format = archTar;
598  WriteHead(AH);
599  AH->format = archDirectory;
600  WriteToc(AH);
601  if (cfclose(tocFH) != 0)
602  pg_fatal("could not close TOC file: %m");
603  WriteDataChunks(AH, ctx->pstate);
604 
605  ParallelBackupEnd(AH, ctx->pstate);
606 
607  /*
608  * In directory mode, there is no need to sync all the entries
609  * individually. Just recurse once through all the files generated.
610  */
611  if (AH->dosync)
612  fsync_dir_recurse(ctx->directory);
613  }
614  AH->FH = NULL;
615 }
616 
617 /*
618  * Reopen the archive's file handle.
619  */
620 static void
622 {
623  /*
624  * Our TOC is in memory, our data files are opened by each child anyway as
625  * they are separate. We support reopening the archive by just doing
626  * nothing.
627  */
628 }
629 
630 /*
631  * BLOB support
632  */
633 
634 /*
635  * Called by the archiver when starting to save all BLOB DATA (not schema).
636  * It is called just prior to the dumper's DataDumper routine.
637  *
638  * We open the large object TOC file here, so that we can append a line to
639  * it for each blob.
640  */
641 static void
643 {
644  lclContext *ctx = (lclContext *) AH->formatData;
645  pg_compress_specification compression_spec = {0};
646  char fname[MAXPGPATH];
647 
648  setFilePath(AH, fname, "blobs.toc");
649 
650  /* The blob TOC file is never compressed */
651  compression_spec.algorithm = PG_COMPRESSION_NONE;
652  ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec);
653  if (ctx->blobsTocFH == NULL)
654  pg_fatal("could not open output file \"%s\": %m", fname);
655 }
656 
657 /*
658  * Called by the archiver when we're about to start dumping a blob.
659  *
660  * We create a file to write the blob to.
661  */
662 static void
664 {
665  lclContext *ctx = (lclContext *) AH->formatData;
666  char fname[MAXPGPATH];
667 
668  snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
669 
670  ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
671 
672  if (ctx->dataFH == NULL)
673  pg_fatal("could not open output file \"%s\": %m", fname);
674 }
675 
676 /*
677  * Called by the archiver when the dumper is finished writing a blob.
678  *
679  * We close the blob file and write an entry to the blob TOC file for it.
680  */
681 static void
683 {
684  lclContext *ctx = (lclContext *) AH->formatData;
685  char buf[50];
686  int len;
687 
688  /* Close the BLOB data file itself */
689  if (cfclose(ctx->dataFH) != 0)
690  pg_fatal("could not close blob data file: %m");
691  ctx->dataFH = NULL;
692 
693  /* register the blob in blobs.toc */
694  len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
695  if (cfwrite(buf, len, ctx->blobsTocFH) != len)
696  pg_fatal("could not write to blobs TOC file");
697 }
698 
699 /*
700  * Called by the archiver when finishing saving all BLOB DATA.
701  *
702  * We close the blobs TOC file.
703  */
704 static void
706 {
707  lclContext *ctx = (lclContext *) AH->formatData;
708 
709  if (cfclose(ctx->blobsTocFH) != 0)
710  pg_fatal("could not close blobs TOC file: %m");
711  ctx->blobsTocFH = NULL;
712 }
713 
714 /*
715  * Gets a relative file name and prepends the output directory, writing the
716  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
717  * big. Can't use a static char[MAXPGPATH] inside the function because we run
718  * multithreaded on Windows.
719  */
720 static void
721 setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
722 {
723  lclContext *ctx = (lclContext *) AH->formatData;
724  char *dname;
725 
726  dname = ctx->directory;
727 
728  if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
729  pg_fatal("file name too long: \"%s\"", dname);
730 
731  strcpy(buf, dname);
732  strcat(buf, "/");
733  strcat(buf, relativeFilename);
734 }
735 
736 /*
737  * Prepare for parallel restore.
738  *
739  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
740  * TOC entries' dataLength fields with appropriate values to guide the
741  * ordering of restore jobs. The source of said data is format-dependent,
742  * as is the exact meaning of the values.
743  *
744  * A format module might also choose to do other setup here.
745  */
746 static void
748 {
749  TocEntry *te;
750 
751  for (te = AH->toc->next; te != AH->toc; te = te->next)
752  {
753  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
754  char fname[MAXPGPATH];
755  struct stat st;
756 
757  /*
758  * A dumpable object has set tctx->filename, any other object has not.
759  * (see _ArchiveEntry).
760  */
761  if (tctx->filename == NULL)
762  continue;
763 
764  /* We may ignore items not due to be restored */
765  if ((te->reqs & REQ_DATA) == 0)
766  continue;
767 
768  /*
769  * Stat the file and, if successful, put its size in dataLength. When
770  * using compression, the physical file size might not be a very good
771  * guide to the amount of work involved in restoring the file, but we
772  * only need an approximate indicator of that.
773  */
774  setFilePath(AH, fname, tctx->filename);
775 
776  if (stat(fname, &st) == 0)
777  te->dataLength = st.st_size;
778  else
779  {
780  /* It might be compressed */
781  strlcat(fname, ".gz", sizeof(fname));
782  if (stat(fname, &st) == 0)
783  te->dataLength = st.st_size;
784  }
785 
786  /*
787  * If this is the BLOBS entry, what we stat'd was blobs.toc, which
788  * most likely is a lot smaller than the actual blob data. We don't
789  * have a cheap way to estimate how much smaller, but fortunately it
790  * doesn't matter too much as long as we get the blobs processed
791  * reasonably early. Arbitrarily scale up by a factor of 1K.
792  */
793  if (strcmp(te->desc, "BLOBS") == 0)
794  te->dataLength *= 1024;
795  }
796 }
797 
798 /*
799  * Clone format-specific fields during parallel restoration.
800  */
801 static void
803 {
804  lclContext *ctx = (lclContext *) AH->formatData;
805 
806  AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
807  memcpy(AH->formatData, ctx, sizeof(lclContext));
808  ctx = (lclContext *) AH->formatData;
809 
810  /*
811  * Note: we do not make a local lo_buf because we expect at most one BLOBS
812  * entry per archive, so no parallelism is possible. Likewise,
813  * TOC-entry-local state isn't an issue because any one TOC entry is
814  * touched by just one worker child.
815  */
816 
817  /*
818  * We also don't copy the ParallelState pointer (pstate), only the leader
819  * process ever writes to it.
820  */
821 }
822 
823 static void
825 {
826  lclContext *ctx = (lclContext *) AH->formatData;
827 
828  free(ctx);
829 }
830 
831 /*
832  * This function is executed in the child of a parallel backup for a
833  * directory-format archive and dumps the actual data for one TOC entry.
834  */
835 static int
837 {
838  /*
839  * This function returns void. We either fail and die horribly or
840  * succeed... A failure will be detected by the parent when the child dies
841  * unexpectedly.
842  */
844 
845  return 0;
846 }
847 
848 /*
849  * This function is executed in the child of a parallel restore from a
850  * directory-format archive and restores the actual data for one TOC entry.
851  */
852 static int
854 {
855  return parallel_restore(AH, te);
856 }
void ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate)
Definition: parallel.c:1059
ParallelState * ParallelBackupStart(ArchiveHandle *AH)
Definition: parallel.c:897
#define PG_BINARY_R
Definition: c.h:1211
#define CppAsString2(x)
Definition: c.h:311
#define PG_BINARY_W
Definition: c.h:1212
int cfwrite(const void *ptr, int size, cfp *fp)
Definition: compress_io.c:581
char * cfgets(cfp *fp, char *buf, int len)
Definition: compress_io.c:620
int cfclose(cfp *fp)
Definition: compress_io.c:631
cfp * cfopen_write(const char *path, const char *mode, const pg_compress_specification compression_spec)
Definition: compress_io.c:469
int cfread(void *ptr, int size, cfp *fp)
Definition: compress_io.c:550
int cfgetc(cfp *fp)
Definition: compress_io.c:592
cfp * cfopen_read(const char *path, const char *mode)
Definition: compress_io.c:425
int cfeof(cfp *fp)
Definition: compress_io.c:658
const char * get_cfp_error(cfp *fp)
Definition: compress_io.c:669
struct cfp cfp
Definition: compress_io.h:53
@ PG_COMPRESSION_NONE
Definition: compression.h:19
int closedir(DIR *)
Definition: dirent.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define free(a)
Definition: header.h:65
int i
Definition: isn.c:73
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
@ archModeWrite
Definition: pg_backup.h:50
@ archTar
Definition: pg_backup.h:42
@ archDirectory
Definition: pg_backup.h:44
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te)
void WriteHead(ArchiveHandle *AH)
void StartRestoreBlob(ArchiveHandle *AH, Oid oid, bool drop)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void StartRestoreBlobs(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void WriteToc(ArchiveHandle *AH)
void EndRestoreBlob(ArchiveHandle *AH, Oid oid)
void EndRestoreBlobs(ArchiveHandle *AH)
int parallel_restore(ArchiveHandle *AH, TocEntry *te)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define LOBBUFSIZE
#define REQ_DATA
static void _LoadBlobs(ArchiveHandle *AH)
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static void _DeClone(ArchiveHandle *AH)
static void _ReopenArchive(ArchiveHandle *AH)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
static void _PrepParallelRestore(ArchiveHandle *AH)
static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
static int _ReadByte(ArchiveHandle *AH)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static void _EndBlobs(ArchiveHandle *AH, TocEntry *te)
static void _Clone(ArchiveHandle *AH)
static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
void InitArchiveFmt_Directory(ArchiveHandle *AH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static void setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
#define pg_fatal(...)
#define MAXPGPATH
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:119
static char * buf
Definition: pg_test_fsync.c:67
#define snprintf
Definition: port.h:238
size_t strlcat(char *dst, const char *src, size_t siz)
Definition: strlcat.c:33
unsigned int Oid
Definition: postgres_ext.h:31
char * c
int verbose
Definition: pg_backup.h:214
RestoreOptions * ropt
Definition: pg_backup.h:212
Definition: dirent.c:26
ArchiveFormat format
struct _tocEntry * toc
DeClonePtrType DeClonePtr
ReadExtraTocPtrType ReadExtraTocPtr
WorkerJobDumpPtrType WorkerJobDumpPtr
EndBlobPtrType EndBlobPtr
ArchiveEntryPtrType ArchiveEntryPtr
pg_compress_specification compression_spec
WriteDataPtrType WriteDataPtr
ClonePtrType ClonePtr
StartBlobsPtrType StartBlobsPtr
WriteBufPtrType WriteBufPtr
PrepParallelRestorePtrType PrepParallelRestorePtr
StartBlobPtrType StartBlobPtr
WriteExtraTocPtrType WriteExtraTocPtr
ReadBytePtrType ReadBytePtr
WorkerJobRestorePtrType WorkerJobRestorePtr
PrintTocDataPtrType PrintTocDataPtr
WriteBytePtrType WriteBytePtr
ReadBufPtrType ReadBufPtr
PrintExtraTocPtrType PrintExtraTocPtr
EndBlobsPtrType EndBlobsPtr
StartDataPtrType StartDataPtr
ReopenPtrType ReopenPtr
EndDataPtrType EndDataPtr
ClosePtrType ClosePtr
DataDumperPtr dataDumper
pgoff_t dataLength
struct _tocEntry * next
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
ParallelState * pstate
pg_compress_algorithm algorithm
Definition: compression.h:29
unsigned short st_mode
Definition: win32_port.h:270
static void * fn(void *arg)
#define ZLIB_OUT_SIZE
Definition: walmethods.c:33
#define stat
Definition: win32_port.h:286
#define S_ISDIR(m)
Definition: win32_port.h:327
#define mkdir(a, b)
Definition: win32_port.h:80