PostgreSQL Source Code  git master
pg_backup_directory.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_directory.c
4  *
5  * A directory format dump is a directory, which contains a "toc.dat" file
6  * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7  * Large objects are stored in separate files named "blob_<oid>.dat",
8  * and there's a plain-text TOC file for each BLOBS TOC entry named
9  * "blobs_<dumpID>.toc" (or just "blobs.toc" in archive versions before 16).
10  *
11  * If compression is used, each data file is individually compressed and the
12  * ".gz" suffix is added to the filenames. The TOC files are never
13  * compressed by pg_dump, however they are accepted with the .gz suffix too,
14  * in case the user has manually compressed them with 'gzip'.
15  *
16  * NOTE: This format is identical to the files written in the tar file in
17  * the 'tar' format, except that we don't write the restore.sql file (TODO),
18  * and the tar format doesn't support compression. Please keep the formats in
19  * sync.
20  *
21  *
22  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
23  * Portions Copyright (c) 1994, Regents of the University of California
24  * Portions Copyright (c) 2000, Philip Warner
25  *
26  * Rights are granted to use this software in any way so long
27  * as this notice is not removed.
28  *
29  * The author is not responsible for loss or damages that may
30  * result from its use.
31  *
32  * IDENTIFICATION
33  * src/bin/pg_dump/pg_backup_directory.c
34  *
35  *-------------------------------------------------------------------------
36  */
37 #include "postgres_fe.h"
38 
39 #include <dirent.h>
40 #include <sys/stat.h>
41 
42 #include "common/file_utils.h"
43 #include "compress_io.h"
44 #include "parallel.h"
45 #include "pg_backup_utils.h"
46 
47 typedef struct
48 {
49  /*
50  * Our archive location. This is basically what the user specified as his
51  * backup file but of course here it is a directory.
52  */
53  char *directory;
54 
55  CompressFileHandle *dataFH; /* currently open data file */
56  CompressFileHandle *LOsTocFH; /* file handle for blobs_NNN.toc */
57  ParallelState *pstate; /* for parallel backup / restore */
58 } lclContext;
59 
60 typedef struct
61 {
62  char *filename; /* filename excluding the directory (basename) */
63 } lclTocEntry;
64 
65 /* prototypes for private functions */
66 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
67 static void _StartData(ArchiveHandle *AH, TocEntry *te);
68 static void _EndData(ArchiveHandle *AH, TocEntry *te);
69 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
70 static int _WriteByte(ArchiveHandle *AH, const int i);
71 static int _ReadByte(ArchiveHandle *AH);
72 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
73 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
74 static void _CloseArchive(ArchiveHandle *AH);
75 static void _ReopenArchive(ArchiveHandle *AH);
76 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
77 
78 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
79 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
80 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
81 
82 static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
83 static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
84 static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
85 static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
86 static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
87 
88 static void _PrepParallelRestore(ArchiveHandle *AH);
89 static void _Clone(ArchiveHandle *AH);
90 static void _DeClone(ArchiveHandle *AH);
91 
94 
95 static void setFilePath(ArchiveHandle *AH, char *buf,
96  const char *relativeFilename);
97 
98 /*
99  * Init routine required by ALL formats. This is a global routine
100  * and should be declared in pg_backup_archiver.h
101  *
102  * Its task is to create any extra archive context (using AH->formatData),
103  * and to initialize the supported function pointers.
104  *
105  * It should also prepare whatever its input source is for reading/writing,
106  * and in the case of a read mode connection, it should load the Header & TOC.
107  */
108 void
110 {
111  lclContext *ctx;
112 
113  /* Assuming static functions, this can be copied for each format. */
115  AH->StartDataPtr = _StartData;
116  AH->WriteDataPtr = _WriteData;
117  AH->EndDataPtr = _EndData;
118  AH->WriteBytePtr = _WriteByte;
119  AH->ReadBytePtr = _ReadByte;
120  AH->WriteBufPtr = _WriteBuf;
121  AH->ReadBufPtr = _ReadBuf;
122  AH->ClosePtr = _CloseArchive;
128 
129  AH->StartLOsPtr = _StartLOs;
130  AH->StartLOPtr = _StartLO;
131  AH->EndLOPtr = _EndLO;
132  AH->EndLOsPtr = _EndLOs;
133 
135  AH->ClonePtr = _Clone;
136  AH->DeClonePtr = _DeClone;
137 
140 
141  /* Set up our private context */
142  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
143  AH->formatData = (void *) ctx;
144 
145  ctx->dataFH = NULL;
146  ctx->LOsTocFH = NULL;
147 
148  /*
149  * Now open the TOC file
150  */
151 
152  if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
153  pg_fatal("no output directory specified");
154 
155  ctx->directory = AH->fSpec;
156 
157  if (AH->mode == archModeWrite)
158  {
159  struct stat st;
160  bool is_empty = false;
161 
162  /* we accept an empty existing directory */
163  if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
164  {
165  DIR *dir = opendir(ctx->directory);
166 
167  if (dir)
168  {
169  struct dirent *d;
170 
171  is_empty = true;
172  while (errno = 0, (d = readdir(dir)))
173  {
174  if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
175  {
176  is_empty = false;
177  break;
178  }
179  }
180 
181  if (errno)
182  pg_fatal("could not read directory \"%s\": %m",
183  ctx->directory);
184 
185  if (closedir(dir))
186  pg_fatal("could not close directory \"%s\": %m",
187  ctx->directory);
188  }
189  }
190 
191  if (!is_empty && mkdir(ctx->directory, 0700) < 0)
192  pg_fatal("could not create directory \"%s\": %m",
193  ctx->directory);
194  }
195  else
196  { /* Read Mode */
197  char fname[MAXPGPATH];
198  CompressFileHandle *tocFH;
199 
200  setFilePath(AH, fname, "toc.dat");
201 
203  if (tocFH == NULL)
204  pg_fatal("could not open input file \"%s\": %m", fname);
205 
206  ctx->dataFH = tocFH;
207 
208  /*
209  * The TOC of a directory format dump shares the format code of the
210  * tar format.
211  */
212  AH->format = archTar;
213  ReadHead(AH);
214  AH->format = archDirectory;
215  ReadToc(AH);
216 
217  /* Nothing else in the file, so close it again... */
218  if (!EndCompressFileHandle(tocFH))
219  pg_fatal("could not close TOC file: %m");
220  ctx->dataFH = NULL;
221  }
222 }
223 
224 /*
225  * Called by the Archiver when the dumper creates a new TOC entry.
226  *
227  * We determine the filename for this entry.
228 */
229 static void
231 {
232  lclTocEntry *tctx;
233  char fn[MAXPGPATH];
234 
235  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
236  if (strcmp(te->desc, "BLOBS") == 0)
237  {
238  snprintf(fn, MAXPGPATH, "blobs_%d.toc", te->dumpId);
239  tctx->filename = pg_strdup(fn);
240  }
241  else if (te->dataDumper)
242  {
243  snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
244  tctx->filename = pg_strdup(fn);
245  }
246  else
247  tctx->filename = NULL;
248 
249  te->formatData = (void *) tctx;
250 }
251 
252 /*
253  * Called by the Archiver to save any extra format-related TOC entry
254  * data.
255  *
256  * Use the Archiver routines to write data - they are non-endian, and
257  * maintain other important file information.
258  */
259 static void
261 {
262  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
263 
264  /*
265  * A dumpable object has set tctx->filename, any other object has not.
266  * (see _ArchiveEntry).
267  */
268  if (tctx->filename)
269  WriteStr(AH, tctx->filename);
270  else
271  WriteStr(AH, "");
272 }
273 
274 /*
275  * Called by the Archiver to read any extra format-related TOC data.
276  *
277  * Needs to match the order defined in _WriteExtraToc, and should also
278  * use the Archiver input routines.
279  */
280 static void
282 {
283  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
284 
285  if (tctx == NULL)
286  {
287  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
288  te->formatData = (void *) tctx;
289  }
290 
291  tctx->filename = ReadStr(AH);
292  if (strlen(tctx->filename) == 0)
293  {
294  free(tctx->filename);
295  tctx->filename = NULL;
296  }
297 }
298 
299 /*
300  * Called by the Archiver when restoring an archive to output a comment
301  * that includes useful information about the TOC entry.
302  */
303 static void
305 {
306  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
307 
308  if (AH->public.verbose && tctx->filename)
309  ahprintf(AH, "-- File: %s\n", tctx->filename);
310 }
311 
312 /*
313  * Called by the archiver when saving TABLE DATA (not schema). This routine
314  * should save whatever format-specific information is needed to read
315  * the archive back.
316  *
317  * It is called just prior to the dumper's 'DataDumper' routine being called.
318  *
319  * We create the data file for writing.
320  */
321 static void
323 {
324  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
325  lclContext *ctx = (lclContext *) AH->formatData;
326  char fname[MAXPGPATH];
327 
328  setFilePath(AH, fname, tctx->filename);
329 
330  ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
331 
332  if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
333  pg_fatal("could not open output file \"%s\": %m", fname);
334 }
335 
336 /*
337  * Called by archiver when dumper calls WriteData. This routine is
338  * called for both LO and table data; it is the responsibility of
339  * the format to manage each kind of data using StartLO/StartData.
340  *
341  * It should only be called from within a DataDumper routine.
342  *
343  * We write the data to the open data file.
344  */
345 static void
346 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
347 {
348  lclContext *ctx = (lclContext *) AH->formatData;
349  CompressFileHandle *CFH = ctx->dataFH;
350 
351  errno = 0;
352  if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
353  {
354  /* if write didn't set errno, assume problem is no disk space */
355  if (errno == 0)
356  errno = ENOSPC;
357  pg_fatal("could not write to output file: %s",
358  CFH->get_error_func(CFH));
359  }
360 }
361 
362 /*
363  * Called by the archiver when a dumper's 'DataDumper' routine has
364  * finished.
365  *
366  * We close the data file.
367  */
368 static void
370 {
371  lclContext *ctx = (lclContext *) AH->formatData;
372 
373  /* Close the file */
375  pg_fatal("could not close data file: %m");
376 
377  ctx->dataFH = NULL;
378 }
379 
380 /*
381  * Print data for a given file (can be a LO as well)
382  */
383 static void
385 {
386  size_t cnt = 0;
387  char *buf;
388  size_t buflen;
389  CompressFileHandle *CFH;
390 
391  if (!filename)
392  return;
393 
395  if (!CFH)
396  pg_fatal("could not open input file \"%s\": %m", filename);
397 
398  buflen = DEFAULT_IO_BUFFER_SIZE;
399  buf = pg_malloc(buflen);
400 
401  while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
402  {
403  ahwrite(buf, 1, cnt, AH);
404  }
405 
406  free(buf);
407  if (!EndCompressFileHandle(CFH))
408  pg_fatal("could not close data file \"%s\": %m", filename);
409 }
410 
411 /*
412  * Print data for a given TOC entry
413 */
414 static void
416 {
417  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
418 
419  if (!tctx->filename)
420  return;
421 
422  if (strcmp(te->desc, "BLOBS") == 0)
423  _LoadLOs(AH, te);
424  else
425  {
426  char fname[MAXPGPATH];
427 
428  setFilePath(AH, fname, tctx->filename);
429  _PrintFileData(AH, fname);
430  }
431 }
432 
433 static void
435 {
436  Oid oid;
437  lclContext *ctx = (lclContext *) AH->formatData;
438  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
439  CompressFileHandle *CFH;
440  char tocfname[MAXPGPATH];
441  char line[MAXPGPATH];
442 
443  StartRestoreLOs(AH);
444 
445  /*
446  * Note: before archive v16, there was always only one BLOBS TOC entry,
447  * now there can be multiple. We don't need to worry what version we are
448  * reading though, because tctx->filename should be correct either way.
449  */
450  setFilePath(AH, tocfname, tctx->filename);
451 
452  CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
453 
454  if (ctx->LOsTocFH == NULL)
455  pg_fatal("could not open large object TOC file \"%s\" for input: %m",
456  tocfname);
457 
458  /* Read the LOs TOC file line-by-line, and process each LO */
459  while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
460  {
461  char lofname[MAXPGPATH + 1];
462  char path[MAXPGPATH];
463 
464  /* Can't overflow because line and lofname are the same length */
465  if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
466  pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
467  tocfname, line);
468 
469  StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
470  snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
471  _PrintFileData(AH, path);
472  EndRestoreLO(AH, oid);
473  }
474  if (!CFH->eof_func(CFH))
475  pg_fatal("error reading large object TOC file \"%s\"",
476  tocfname);
477 
478  if (!EndCompressFileHandle(ctx->LOsTocFH))
479  pg_fatal("could not close large object TOC file \"%s\": %m",
480  tocfname);
481 
482  ctx->LOsTocFH = NULL;
483 
484  EndRestoreLOs(AH);
485 }
486 
487 
488 /*
489  * Write a byte of data to the archive.
490  * Called by the archiver to do integer & byte output to the archive.
491  * These routines are only used to read & write the headers & TOC.
492  */
493 static int
494 _WriteByte(ArchiveHandle *AH, const int i)
495 {
496  unsigned char c = (unsigned char) i;
497  lclContext *ctx = (lclContext *) AH->formatData;
498  CompressFileHandle *CFH = ctx->dataFH;
499 
500  errno = 0;
501  if (!CFH->write_func(&c, 1, CFH))
502  {
503  /* if write didn't set errno, assume problem is no disk space */
504  if (errno == 0)
505  errno = ENOSPC;
506  pg_fatal("could not write to output file: %s",
507  CFH->get_error_func(CFH));
508  }
509 
510  return 1;
511 }
512 
513 /*
514  * Read a byte of data from the archive.
515  * Called by the archiver to read bytes & integers from the archive.
516  * These routines are only used to read & write headers & TOC.
517  * EOF should be treated as a fatal error.
518  */
519 static int
521 {
522  lclContext *ctx = (lclContext *) AH->formatData;
523  CompressFileHandle *CFH = ctx->dataFH;
524 
525  return CFH->getc_func(CFH);
526 }
527 
528 /*
529  * Write a buffer of data to the archive.
530  * Called by the archiver to write a block of bytes to the TOC or a data file.
531  */
532 static void
533 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
534 {
535  lclContext *ctx = (lclContext *) AH->formatData;
536  CompressFileHandle *CFH = ctx->dataFH;
537 
538  errno = 0;
539  if (!CFH->write_func(buf, len, CFH))
540  {
541  /* if write didn't set errno, assume problem is no disk space */
542  if (errno == 0)
543  errno = ENOSPC;
544  pg_fatal("could not write to output file: %s",
545  CFH->get_error_func(CFH));
546  }
547 }
548 
549 /*
550  * Read a block of bytes from the archive.
551  *
552  * Called by the archiver to read a block of bytes from the archive
553  */
554 static void
555 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
556 {
557  lclContext *ctx = (lclContext *) AH->formatData;
558  CompressFileHandle *CFH = ctx->dataFH;
559 
560  /*
561  * If there was an I/O error, we already exited in readF(), so here we
562  * exit on short reads.
563  */
564  if (!CFH->read_func(buf, len, NULL, CFH))
565  pg_fatal("could not read from input file: end of file");
566 }
567 
568 /*
569  * Close the archive.
570  *
571  * When writing the archive, this is the routine that actually starts
572  * the process of saving it to files. No data should be written prior
573  * to this point, since the user could sort the TOC after creating it.
574  *
575  * If an archive is to be written, this routine must call:
576  * WriteHead to save the archive header
577  * WriteToc to save the TOC entries
578  * WriteDataChunks to save all data & LOs.
579  */
580 static void
582 {
583  lclContext *ctx = (lclContext *) AH->formatData;
584 
585  if (AH->mode == archModeWrite)
586  {
587  CompressFileHandle *tocFH;
588  pg_compress_specification compression_spec = {0};
589  char fname[MAXPGPATH];
590 
591  setFilePath(AH, fname, "toc.dat");
592 
593  /* this will actually fork the processes for a parallel backup */
594  ctx->pstate = ParallelBackupStart(AH);
595 
596  /* The TOC is always created uncompressed */
597  compression_spec.algorithm = PG_COMPRESSION_NONE;
598  tocFH = InitCompressFileHandle(compression_spec);
599  if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
600  pg_fatal("could not open output file \"%s\": %m", fname);
601  ctx->dataFH = tocFH;
602 
603  /*
604  * Write 'tar' in the format field of the toc.dat file. The directory
605  * is compatible with 'tar', so there's no point having a different
606  * format code for it.
607  */
608  AH->format = archTar;
609  WriteHead(AH);
610  AH->format = archDirectory;
611  WriteToc(AH);
612  if (!EndCompressFileHandle(tocFH))
613  pg_fatal("could not close TOC file: %m");
614  WriteDataChunks(AH, ctx->pstate);
615 
616  ParallelBackupEnd(AH, ctx->pstate);
617 
618  /*
619  * In directory mode, there is no need to sync all the entries
620  * individually. Just recurse once through all the files generated.
621  */
622  if (AH->dosync)
623  sync_dir_recurse(ctx->directory, AH->sync_method);
624  }
625  AH->FH = NULL;
626 }
627 
628 /*
629  * Reopen the archive's file handle.
630  */
631 static void
633 {
634  /*
635  * Our TOC is in memory, our data files are opened by each child anyway as
636  * they are separate. We support reopening the archive by just doing
637  * nothing.
638  */
639 }
640 
641 /*
642  * LO support
643  */
644 
645 /*
646  * Called by the archiver when starting to save BLOB DATA (not schema).
647  * It is called just prior to the dumper's DataDumper routine.
648  *
649  * We open the large object TOC file here, so that we can append a line to
650  * it for each LO.
651  */
652 static void
654 {
655  lclContext *ctx = (lclContext *) AH->formatData;
656  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
657  pg_compress_specification compression_spec = {0};
658  char fname[MAXPGPATH];
659 
660  setFilePath(AH, fname, tctx->filename);
661 
662  /* The LO TOC file is never compressed */
663  compression_spec.algorithm = PG_COMPRESSION_NONE;
664  ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
665  if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
666  pg_fatal("could not open output file \"%s\": %m", fname);
667 }
668 
669 /*
670  * Called by the archiver when we're about to start dumping a LO.
671  *
672  * We create a file to write the LO to.
673  */
674 static void
676 {
677  lclContext *ctx = (lclContext *) AH->formatData;
678  char fname[MAXPGPATH];
679 
680  snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
681 
683  if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
684  pg_fatal("could not open output file \"%s\": %m", fname);
685 }
686 
687 /*
688  * Called by the archiver when the dumper is finished writing a LO.
689  *
690  * We close the LO file and write an entry to the LO TOC file for it.
691  */
692 static void
694 {
695  lclContext *ctx = (lclContext *) AH->formatData;
696  CompressFileHandle *CFH = ctx->LOsTocFH;
697  char buf[50];
698  int len;
699 
700  /* Close the BLOB data file itself */
702  pg_fatal("could not close LO data file: %m");
703  ctx->dataFH = NULL;
704 
705  /* register the LO in blobs_NNN.toc */
706  len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
707  if (!CFH->write_func(buf, len, CFH))
708  {
709  /* if write didn't set errno, assume problem is no disk space */
710  if (errno == 0)
711  errno = ENOSPC;
712  pg_fatal("could not write to LOs TOC file: %s",
713  CFH->get_error_func(CFH));
714  }
715 }
716 
717 /*
718  * Called by the archiver when finishing saving BLOB DATA.
719  *
720  * We close the LOs TOC file.
721  */
722 static void
724 {
725  lclContext *ctx = (lclContext *) AH->formatData;
726 
728  pg_fatal("could not close LOs TOC file: %m");
729  ctx->LOsTocFH = NULL;
730 }
731 
732 /*
733  * Gets a relative file name and prepends the output directory, writing the
734  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
735  * big. Can't use a static char[MAXPGPATH] inside the function because we run
736  * multithreaded on Windows.
737  */
738 static void
739 setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
740 {
741  lclContext *ctx = (lclContext *) AH->formatData;
742  char *dname;
743 
744  dname = ctx->directory;
745 
746  if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
747  pg_fatal("file name too long: \"%s\"", dname);
748 
749  strcpy(buf, dname);
750  strcat(buf, "/");
751  strcat(buf, relativeFilename);
752 }
753 
754 /*
755  * Prepare for parallel restore.
756  *
757  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
758  * TOC entries' dataLength fields with appropriate values to guide the
759  * ordering of restore jobs. The source of said data is format-dependent,
760  * as is the exact meaning of the values.
761  *
762  * A format module might also choose to do other setup here.
763  */
764 static void
766 {
767  TocEntry *te;
768 
769  for (te = AH->toc->next; te != AH->toc; te = te->next)
770  {
771  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
772  char fname[MAXPGPATH];
773  struct stat st;
774 
775  /*
776  * A dumpable object has set tctx->filename, any other object has not.
777  * (see _ArchiveEntry).
778  */
779  if (tctx->filename == NULL)
780  continue;
781 
782  /* We may ignore items not due to be restored */
783  if ((te->reqs & REQ_DATA) == 0)
784  continue;
785 
786  /*
787  * Stat the file and, if successful, put its size in dataLength. When
788  * using compression, the physical file size might not be a very good
789  * guide to the amount of work involved in restoring the file, but we
790  * only need an approximate indicator of that.
791  */
792  setFilePath(AH, fname, tctx->filename);
793 
794  if (stat(fname, &st) == 0)
795  te->dataLength = st.st_size;
797  {
799  strlcat(fname, ".gz", sizeof(fname));
801  strlcat(fname, ".lz4", sizeof(fname));
803  strlcat(fname, ".zst", sizeof(fname));
804 
805  if (stat(fname, &st) == 0)
806  te->dataLength = st.st_size;
807  }
808 
809  /*
810  * If this is a BLOBS entry, what we stat'd was blobs_NNN.toc, which
811  * most likely is a lot smaller than the actual blob data. We don't
812  * have a cheap way to estimate how much smaller, but fortunately it
813  * doesn't matter too much as long as we get the LOs processed
814  * reasonably early. Arbitrarily scale up by a factor of 1K.
815  */
816  if (strcmp(te->desc, "BLOBS") == 0)
817  te->dataLength *= 1024;
818  }
819 }
820 
821 /*
822  * Clone format-specific fields during parallel restoration.
823  */
824 static void
826 {
827  lclContext *ctx = (lclContext *) AH->formatData;
828 
829  AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
830  memcpy(AH->formatData, ctx, sizeof(lclContext));
831  ctx = (lclContext *) AH->formatData;
832 
833  /*
834  * TOC-entry-local state isn't an issue because any one TOC entry is
835  * touched by just one worker child.
836  */
837 
838  /*
839  * We also don't copy the ParallelState pointer (pstate), only the leader
840  * process ever writes to it.
841  */
842 }
843 
844 static void
846 {
847  lclContext *ctx = (lclContext *) AH->formatData;
848 
849  free(ctx);
850 }
851 
852 /*
853  * This function is executed in the child of a parallel backup for a
854  * directory-format archive and dumps the actual data for one TOC entry.
855  */
856 static int
858 {
859  /*
860  * This function returns void. We either fail and die horribly or
861  * succeed... A failure will be detected by the parent when the child dies
862  * unexpectedly.
863  */
865 
866  return 0;
867 }
868 
869 /*
870  * This function is executed in the child of a parallel restore from a
871  * directory-format archive and restores the actual data for one TOC entry.
872  */
873 static int
875 {
876  return parallel_restore(AH, te);
877 }
void ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate)
Definition: parallel.c:1059
ParallelState * ParallelBackupStart(ArchiveHandle *AH)
Definition: parallel.c:897
#define PG_BINARY_R
Definition: c.h:1266
#define CppAsString2(x)
Definition: c.h:330
#define PG_BINARY_W
Definition: c.h:1267
bool EndCompressFileHandle(CompressFileHandle *CFH)
Definition: compress_io.c:289
CompressFileHandle * InitDiscoverCompressFileHandle(const char *path, const char *mode)
Definition: compress_io.c:241
CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compression_spec)
Definition: compress_io.c:195
#define DEFAULT_IO_BUFFER_SIZE
Definition: compress_io.h:27
@ PG_COMPRESSION_GZIP
Definition: compression.h:24
@ PG_COMPRESSION_LZ4
Definition: compression.h:25
@ PG_COMPRESSION_NONE
Definition: compression.h:23
@ PG_COMPRESSION_ZSTD
Definition: compression.h:26
int closedir(DIR *)
Definition: dirent.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define free(a)
Definition: header.h:65
int i
Definition: isn.c:72
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:76
@ archModeWrite
Definition: pg_backup.h:51
@ archTar
Definition: pg_backup.h:43
@ archDirectory
Definition: pg_backup.h:45
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te)
void WriteHead(ArchiveHandle *AH)
void StartRestoreLOs(ArchiveHandle *AH)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void EndRestoreLO(ArchiveHandle *AH, Oid oid)
void WriteToc(ArchiveHandle *AH)
void EndRestoreLOs(ArchiveHandle *AH)
void StartRestoreLO(ArchiveHandle *AH, Oid oid, bool drop)
int parallel_restore(ArchiveHandle *AH, TocEntry *te)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define REQ_DATA
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static void _DeClone(ArchiveHandle *AH)
static void _ReopenArchive(ArchiveHandle *AH)
static void _StartLOs(ArchiveHandle *AH, TocEntry *te)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndLOs(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
static void _PrepParallelRestore(ArchiveHandle *AH)
static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static int _ReadByte(ArchiveHandle *AH)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static void _Clone(ArchiveHandle *AH)
static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
void InitArchiveFmt_Directory(ArchiveHandle *AH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static void setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
static void _LoadLOs(ArchiveHandle *AH, TocEntry *te)
static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
#define pg_fatal(...)
#define MAXPGPATH
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:119
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
size_t strlcat(char *dst, const char *src, size_t siz)
Definition: strlcat.c:33
unsigned int Oid
Definition: postgres_ext.h:31
char * c
int verbose
Definition: pg_backup.h:218
RestoreOptions * ropt
Definition: pg_backup.h:216
bool(* open_write_func)(const char *path, const char *mode, CompressFileHandle *CFH)
Definition: compress_io.h:122
int(* getc_func)(CompressFileHandle *CFH)
Definition: compress_io.h:161
char *(* gets_func)(char *s, int size, CompressFileHandle *CFH)
Definition: compress_io.h:152
bool(* eof_func)(CompressFileHandle *CFH)
Definition: compress_io.h:168
bool(* read_func)(void *ptr, size_t size, size_t *rsize, CompressFileHandle *CFH)
Definition: compress_io.h:131
Definition: dirent.c:26
ArchiveFormat format
struct _tocEntry * toc
DeClonePtrType DeClonePtr
EndLOsPtrType EndLOsPtr
DataDirSyncMethod sync_method
ReadExtraTocPtrType ReadExtraTocPtr
WorkerJobDumpPtrType WorkerJobDumpPtr
StartLOsPtrType StartLOsPtr
ArchiveEntryPtrType ArchiveEntryPtr
pg_compress_specification compression_spec
WriteDataPtrType WriteDataPtr
StartLOPtrType StartLOPtr
ClonePtrType ClonePtr
WriteBufPtrType WriteBufPtr
PrepParallelRestorePtrType PrepParallelRestorePtr
EndLOPtrType EndLOPtr
WriteExtraTocPtrType WriteExtraTocPtr
ReadBytePtrType ReadBytePtr
WorkerJobRestorePtrType WorkerJobRestorePtr
PrintTocDataPtrType PrintTocDataPtr
WriteBytePtrType WriteBytePtr
ReadBufPtrType ReadBufPtr
PrintExtraTocPtrType PrintExtraTocPtr
StartDataPtrType StartDataPtr
ReopenPtrType ReopenPtr
EndDataPtrType EndDataPtr
ClosePtrType ClosePtr
DataDumperPtr dataDumper
pgoff_t dataLength
struct _tocEntry * next
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
ParallelState * pstate
CompressFileHandle * LOsTocFH
CompressFileHandle * dataFH
pg_compress_algorithm algorithm
Definition: compression.h:34
unsigned short st_mode
Definition: win32_port.h:268
static void * fn(void *arg)
Definition: thread-alloc.c:119
#define stat
Definition: win32_port.h:284
#define S_ISDIR(m)
Definition: win32_port.h:325
#define mkdir(a, b)
Definition: win32_port.h:80