PostgreSQL Source Code  git master
pg_backup_directory.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_backup_directory.c
4  *
5  * A directory format dump is a directory, which contains a "toc.dat" file
6  * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7  * Large objects are stored in separate files named "blob_<oid>.dat",
8  * and there's a plain-text TOC file for them called "blobs.toc". If
9  * compression is used, each data file is individually compressed and the
10  * ".gz" suffix is added to the filenames. The TOC files are never
11  * compressed by pg_dump, however they are accepted with the .gz suffix too,
12  * in case the user has manually compressed them with 'gzip'.
13  *
14  * NOTE: This format is identical to the files written in the tar file in
15  * the 'tar' format, except that we don't write the restore.sql file (TODO),
16  * and the tar format doesn't support compression. Please keep the formats in
17  * sync.
18  *
19  *
20  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
21  * Portions Copyright (c) 1994, Regents of the University of California
22  * Portions Copyright (c) 2000, Philip Warner
23  *
24  * Rights are granted to use this software in any way so long
25  * as this notice is not removed.
26  *
27  * The author is not responsible for loss or damages that may
28  * result from its use.
29  *
30  * IDENTIFICATION
31  * src/bin/pg_dump/pg_backup_directory.c
32  *
33  *-------------------------------------------------------------------------
34  */
35 #include "postgres_fe.h"
36 
37 #include <dirent.h>
38 #include <sys/stat.h>
39 
40 #include "common/file_utils.h"
41 #include "compress_io.h"
42 #include "parallel.h"
43 #include "pg_backup_utils.h"
44 
45 typedef struct
46 {
47  /*
48  * Our archive location. This is basically what the user specified as his
49  * backup file but of course here it is a directory.
50  */
51  char *directory;
52 
53  CompressFileHandle *dataFH; /* currently open data file */
54  CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
55  ParallelState *pstate; /* for parallel backup / restore */
56 } lclContext;
57 
58 typedef struct
59 {
60  char *filename; /* filename excluding the directory (basename) */
61 } lclTocEntry;
62 
63 /* prototypes for private functions */
64 static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
65 static void _StartData(ArchiveHandle *AH, TocEntry *te);
66 static void _EndData(ArchiveHandle *AH, TocEntry *te);
67 static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
68 static int _WriteByte(ArchiveHandle *AH, const int i);
69 static int _ReadByte(ArchiveHandle *AH);
70 static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
71 static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
72 static void _CloseArchive(ArchiveHandle *AH);
73 static void _ReopenArchive(ArchiveHandle *AH);
74 static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
75 
76 static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
77 static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
78 static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
79 
80 static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
81 static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
82 static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
83 static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
84 static void _LoadLOs(ArchiveHandle *AH);
85 
86 static void _PrepParallelRestore(ArchiveHandle *AH);
87 static void _Clone(ArchiveHandle *AH);
88 static void _DeClone(ArchiveHandle *AH);
89 
92 
93 static void setFilePath(ArchiveHandle *AH, char *buf,
94  const char *relativeFilename);
95 
96 /*
97  * Init routine required by ALL formats. This is a global routine
98  * and should be declared in pg_backup_archiver.h
99  *
100  * Its task is to create any extra archive context (using AH->formatData),
101  * and to initialize the supported function pointers.
102  *
103  * It should also prepare whatever its input source is for reading/writing,
104  * and in the case of a read mode connection, it should load the Header & TOC.
105  */
106 void
108 {
109  lclContext *ctx;
110 
111  /* Assuming static functions, this can be copied for each format. */
113  AH->StartDataPtr = _StartData;
114  AH->WriteDataPtr = _WriteData;
115  AH->EndDataPtr = _EndData;
116  AH->WriteBytePtr = _WriteByte;
117  AH->ReadBytePtr = _ReadByte;
118  AH->WriteBufPtr = _WriteBuf;
119  AH->ReadBufPtr = _ReadBuf;
120  AH->ClosePtr = _CloseArchive;
126 
127  AH->StartLOsPtr = _StartLOs;
128  AH->StartLOPtr = _StartLO;
129  AH->EndLOPtr = _EndLO;
130  AH->EndLOsPtr = _EndLOs;
131 
133  AH->ClonePtr = _Clone;
134  AH->DeClonePtr = _DeClone;
135 
138 
139  /* Set up our private context */
140  ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
141  AH->formatData = (void *) ctx;
142 
143  ctx->dataFH = NULL;
144  ctx->LOsTocFH = NULL;
145 
146  /* Initialize LO buffering */
147  AH->lo_buf_size = LOBBUFSIZE;
148  AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
149 
150  /*
151  * Now open the TOC file
152  */
153 
154  if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
155  pg_fatal("no output directory specified");
156 
157  ctx->directory = AH->fSpec;
158 
159  if (AH->mode == archModeWrite)
160  {
161  struct stat st;
162  bool is_empty = false;
163 
164  /* we accept an empty existing directory */
165  if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
166  {
167  DIR *dir = opendir(ctx->directory);
168 
169  if (dir)
170  {
171  struct dirent *d;
172 
173  is_empty = true;
174  while (errno = 0, (d = readdir(dir)))
175  {
176  if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
177  {
178  is_empty = false;
179  break;
180  }
181  }
182 
183  if (errno)
184  pg_fatal("could not read directory \"%s\": %m",
185  ctx->directory);
186 
187  if (closedir(dir))
188  pg_fatal("could not close directory \"%s\": %m",
189  ctx->directory);
190  }
191  }
192 
193  if (!is_empty && mkdir(ctx->directory, 0700) < 0)
194  pg_fatal("could not create directory \"%s\": %m",
195  ctx->directory);
196  }
197  else
198  { /* Read Mode */
199  char fname[MAXPGPATH];
200  CompressFileHandle *tocFH;
201 
202  setFilePath(AH, fname, "toc.dat");
203 
205  if (tocFH == NULL)
206  pg_fatal("could not open input file \"%s\": %m", fname);
207 
208  ctx->dataFH = tocFH;
209 
210  /*
211  * The TOC of a directory format dump shares the format code of the
212  * tar format.
213  */
214  AH->format = archTar;
215  ReadHead(AH);
216  AH->format = archDirectory;
217  ReadToc(AH);
218 
219  /* Nothing else in the file, so close it again... */
220  if (!EndCompressFileHandle(tocFH))
221  pg_fatal("could not close TOC file: %m");
222  ctx->dataFH = NULL;
223  }
224 }
225 
226 /*
227  * Called by the Archiver when the dumper creates a new TOC entry.
228  *
229  * We determine the filename for this entry.
230 */
231 static void
233 {
234  lclTocEntry *tctx;
235  char fn[MAXPGPATH];
236 
237  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
238  if (strcmp(te->desc, "BLOBS") == 0)
239  tctx->filename = pg_strdup("blobs.toc");
240  else if (te->dataDumper)
241  {
242  snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
243  tctx->filename = pg_strdup(fn);
244  }
245  else
246  tctx->filename = NULL;
247 
248  te->formatData = (void *) tctx;
249 }
250 
251 /*
252  * Called by the Archiver to save any extra format-related TOC entry
253  * data.
254  *
255  * Use the Archiver routines to write data - they are non-endian, and
256  * maintain other important file information.
257  */
258 static void
260 {
261  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
262 
263  /*
264  * A dumpable object has set tctx->filename, any other object has not.
265  * (see _ArchiveEntry).
266  */
267  if (tctx->filename)
268  WriteStr(AH, tctx->filename);
269  else
270  WriteStr(AH, "");
271 }
272 
273 /*
274  * Called by the Archiver to read any extra format-related TOC data.
275  *
276  * Needs to match the order defined in _WriteExtraToc, and should also
277  * use the Archiver input routines.
278  */
279 static void
281 {
282  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
283 
284  if (tctx == NULL)
285  {
286  tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
287  te->formatData = (void *) tctx;
288  }
289 
290  tctx->filename = ReadStr(AH);
291  if (strlen(tctx->filename) == 0)
292  {
293  free(tctx->filename);
294  tctx->filename = NULL;
295  }
296 }
297 
298 /*
299  * Called by the Archiver when restoring an archive to output a comment
300  * that includes useful information about the TOC entry.
301  */
302 static void
304 {
305  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
306 
307  if (AH->public.verbose && tctx->filename)
308  ahprintf(AH, "-- File: %s\n", tctx->filename);
309 }
310 
311 /*
312  * Called by the archiver when saving TABLE DATA (not schema). This routine
313  * should save whatever format-specific information is needed to read
314  * the archive back.
315  *
316  * It is called just prior to the dumper's 'DataDumper' routine being called.
317  *
318  * We create the data file for writing.
319  */
320 static void
322 {
323  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
324  lclContext *ctx = (lclContext *) AH->formatData;
325  char fname[MAXPGPATH];
326 
327  setFilePath(AH, fname, tctx->filename);
328 
329  ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
330 
331  if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
332  pg_fatal("could not open output file \"%s\": %m", fname);
333 }
334 
335 /*
336  * Called by archiver when dumper calls WriteData. This routine is
337  * called for both LO and table data; it is the responsibility of
338  * the format to manage each kind of data using StartLO/StartData.
339  *
340  * It should only be called from within a DataDumper routine.
341  *
342  * We write the data to the open data file.
343  */
344 static void
345 _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
346 {
347  lclContext *ctx = (lclContext *) AH->formatData;
348  CompressFileHandle *CFH = ctx->dataFH;
349 
350  errno = 0;
351  if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
352  {
353  /* if write didn't set errno, assume problem is no disk space */
354  if (errno == 0)
355  errno = ENOSPC;
356  pg_fatal("could not write to output file: %s",
357  CFH->get_error_func(CFH));
358  }
359 }
360 
361 /*
362  * Called by the archiver when a dumper's 'DataDumper' routine has
363  * finished.
364  *
365  * We close the data file.
366  */
367 static void
369 {
370  lclContext *ctx = (lclContext *) AH->formatData;
371 
372  /* Close the file */
374  pg_fatal("could not close data file: %m");
375 
376  ctx->dataFH = NULL;
377 }
378 
379 /*
380  * Print data for a given file (can be a LO as well)
381  */
382 static void
384 {
385  size_t cnt = 0;
386  char *buf;
387  size_t buflen;
388  CompressFileHandle *CFH;
389 
390  if (!filename)
391  return;
392 
394  if (!CFH)
395  pg_fatal("could not open input file \"%s\": %m", filename);
396 
397  buflen = DEFAULT_IO_BUFFER_SIZE;
398  buf = pg_malloc(buflen);
399 
400  while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
401  {
402  ahwrite(buf, 1, cnt, AH);
403  }
404 
405  free(buf);
406  if (!EndCompressFileHandle(CFH))
407  pg_fatal("could not close data file \"%s\": %m", filename);
408 }
409 
410 /*
411  * Print data for a given TOC entry
412 */
413 static void
415 {
416  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
417 
418  if (!tctx->filename)
419  return;
420 
421  if (strcmp(te->desc, "BLOBS") == 0)
422  _LoadLOs(AH);
423  else
424  {
425  char fname[MAXPGPATH];
426 
427  setFilePath(AH, fname, tctx->filename);
428  _PrintFileData(AH, fname);
429  }
430 }
431 
432 static void
434 {
435  Oid oid;
436  lclContext *ctx = (lclContext *) AH->formatData;
437  CompressFileHandle *CFH;
438  char tocfname[MAXPGPATH];
439  char line[MAXPGPATH];
440 
441  StartRestoreLOs(AH);
442 
443  setFilePath(AH, tocfname, "blobs.toc");
444 
445  CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
446 
447  if (ctx->LOsTocFH == NULL)
448  pg_fatal("could not open large object TOC file \"%s\" for input: %m",
449  tocfname);
450 
451  /* Read the LOs TOC file line-by-line, and process each LO */
452  while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
453  {
454  char lofname[MAXPGPATH + 1];
455  char path[MAXPGPATH];
456 
457  /* Can't overflow because line and lofname are the same length */
458  if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
459  pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
460  tocfname, line);
461 
462  StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
463  snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
464  _PrintFileData(AH, path);
465  EndRestoreLO(AH, oid);
466  }
467  if (!CFH->eof_func(CFH))
468  pg_fatal("error reading large object TOC file \"%s\"",
469  tocfname);
470 
471  if (!EndCompressFileHandle(ctx->LOsTocFH))
472  pg_fatal("could not close large object TOC file \"%s\": %m",
473  tocfname);
474 
475  ctx->LOsTocFH = NULL;
476 
477  EndRestoreLOs(AH);
478 }
479 
480 
481 /*
482  * Write a byte of data to the archive.
483  * Called by the archiver to do integer & byte output to the archive.
484  * These routines are only used to read & write the headers & TOC.
485  */
486 static int
487 _WriteByte(ArchiveHandle *AH, const int i)
488 {
489  unsigned char c = (unsigned char) i;
490  lclContext *ctx = (lclContext *) AH->formatData;
491  CompressFileHandle *CFH = ctx->dataFH;
492 
493  errno = 0;
494  if (!CFH->write_func(&c, 1, CFH))
495  {
496  /* if write didn't set errno, assume problem is no disk space */
497  if (errno == 0)
498  errno = ENOSPC;
499  pg_fatal("could not write to output file: %s",
500  CFH->get_error_func(CFH));
501  }
502 
503  return 1;
504 }
505 
506 /*
507  * Read a byte of data from the archive.
508  * Called by the archiver to read bytes & integers from the archive.
509  * These routines are only used to read & write headers & TOC.
510  * EOF should be treated as a fatal error.
511  */
512 static int
514 {
515  lclContext *ctx = (lclContext *) AH->formatData;
516  CompressFileHandle *CFH = ctx->dataFH;
517 
518  return CFH->getc_func(CFH);
519 }
520 
521 /*
522  * Write a buffer of data to the archive.
523  * Called by the archiver to write a block of bytes to the TOC or a data file.
524  */
525 static void
526 _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
527 {
528  lclContext *ctx = (lclContext *) AH->formatData;
529  CompressFileHandle *CFH = ctx->dataFH;
530 
531  errno = 0;
532  if (!CFH->write_func(buf, len, CFH))
533  {
534  /* if write didn't set errno, assume problem is no disk space */
535  if (errno == 0)
536  errno = ENOSPC;
537  pg_fatal("could not write to output file: %s",
538  CFH->get_error_func(CFH));
539  }
540 }
541 
542 /*
543  * Read a block of bytes from the archive.
544  *
545  * Called by the archiver to read a block of bytes from the archive
546  */
547 static void
548 _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
549 {
550  lclContext *ctx = (lclContext *) AH->formatData;
551  CompressFileHandle *CFH = ctx->dataFH;
552 
553  /*
554  * If there was an I/O error, we already exited in readF(), so here we
555  * exit on short reads.
556  */
557  if (!CFH->read_func(buf, len, NULL, CFH))
558  pg_fatal("could not read from input file: end of file");
559 }
560 
561 /*
562  * Close the archive.
563  *
564  * When writing the archive, this is the routine that actually starts
565  * the process of saving it to files. No data should be written prior
566  * to this point, since the user could sort the TOC after creating it.
567  *
568  * If an archive is to be written, this routine must call:
569  * WriteHead to save the archive header
570  * WriteToc to save the TOC entries
571  * WriteDataChunks to save all data & LOs.
572  */
573 static void
575 {
576  lclContext *ctx = (lclContext *) AH->formatData;
577 
578  if (AH->mode == archModeWrite)
579  {
580  CompressFileHandle *tocFH;
581  pg_compress_specification compression_spec = {0};
582  char fname[MAXPGPATH];
583 
584  setFilePath(AH, fname, "toc.dat");
585 
586  /* this will actually fork the processes for a parallel backup */
587  ctx->pstate = ParallelBackupStart(AH);
588 
589  /* The TOC is always created uncompressed */
590  compression_spec.algorithm = PG_COMPRESSION_NONE;
591  tocFH = InitCompressFileHandle(compression_spec);
592  if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
593  pg_fatal("could not open output file \"%s\": %m", fname);
594  ctx->dataFH = tocFH;
595 
596  /*
597  * Write 'tar' in the format field of the toc.dat file. The directory
598  * is compatible with 'tar', so there's no point having a different
599  * format code for it.
600  */
601  AH->format = archTar;
602  WriteHead(AH);
603  AH->format = archDirectory;
604  WriteToc(AH);
605  if (!EndCompressFileHandle(tocFH))
606  pg_fatal("could not close TOC file: %m");
607  WriteDataChunks(AH, ctx->pstate);
608 
609  ParallelBackupEnd(AH, ctx->pstate);
610 
611  /*
612  * In directory mode, there is no need to sync all the entries
613  * individually. Just recurse once through all the files generated.
614  */
615  if (AH->dosync)
616  sync_dir_recurse(ctx->directory, AH->sync_method);
617  }
618  AH->FH = NULL;
619 }
620 
621 /*
622  * Reopen the archive's file handle.
623  */
624 static void
626 {
627  /*
628  * Our TOC is in memory, our data files are opened by each child anyway as
629  * they are separate. We support reopening the archive by just doing
630  * nothing.
631  */
632 }
633 
634 /*
635  * LO support
636  */
637 
638 /*
639  * Called by the archiver when starting to save all BLOB DATA (not schema).
640  * It is called just prior to the dumper's DataDumper routine.
641  *
642  * We open the large object TOC file here, so that we can append a line to
643  * it for each LO.
644  */
645 static void
647 {
648  lclContext *ctx = (lclContext *) AH->formatData;
649  pg_compress_specification compression_spec = {0};
650  char fname[MAXPGPATH];
651 
652  setFilePath(AH, fname, "blobs.toc");
653 
654  /* The LO TOC file is never compressed */
655  compression_spec.algorithm = PG_COMPRESSION_NONE;
656  ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
657  if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
658  pg_fatal("could not open output file \"%s\": %m", fname);
659 }
660 
661 /*
662  * Called by the archiver when we're about to start dumping a LO.
663  *
664  * We create a file to write the LO to.
665  */
666 static void
668 {
669  lclContext *ctx = (lclContext *) AH->formatData;
670  char fname[MAXPGPATH];
671 
672  snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
673 
675  if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
676  pg_fatal("could not open output file \"%s\": %m", fname);
677 }
678 
679 /*
680  * Called by the archiver when the dumper is finished writing a LO.
681  *
682  * We close the LO file and write an entry to the LO TOC file for it.
683  */
684 static void
686 {
687  lclContext *ctx = (lclContext *) AH->formatData;
688  CompressFileHandle *CFH = ctx->LOsTocFH;
689  char buf[50];
690  int len;
691 
692  /* Close the BLOB data file itself */
694  pg_fatal("could not close LO data file: %m");
695  ctx->dataFH = NULL;
696 
697  /* register the LO in blobs.toc */
698  len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
699  if (!CFH->write_func(buf, len, CFH))
700  {
701  /* if write didn't set errno, assume problem is no disk space */
702  if (errno == 0)
703  errno = ENOSPC;
704  pg_fatal("could not write to LOs TOC file: %s",
705  CFH->get_error_func(CFH));
706  }
707 }
708 
709 /*
710  * Called by the archiver when finishing saving all BLOB DATA.
711  *
712  * We close the LOs TOC file.
713  */
714 static void
716 {
717  lclContext *ctx = (lclContext *) AH->formatData;
718 
720  pg_fatal("could not close LOs TOC file: %m");
721  ctx->LOsTocFH = NULL;
722 }
723 
724 /*
725  * Gets a relative file name and prepends the output directory, writing the
726  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
727  * big. Can't use a static char[MAXPGPATH] inside the function because we run
728  * multithreaded on Windows.
729  */
730 static void
731 setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
732 {
733  lclContext *ctx = (lclContext *) AH->formatData;
734  char *dname;
735 
736  dname = ctx->directory;
737 
738  if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
739  pg_fatal("file name too long: \"%s\"", dname);
740 
741  strcpy(buf, dname);
742  strcat(buf, "/");
743  strcat(buf, relativeFilename);
744 }
745 
746 /*
747  * Prepare for parallel restore.
748  *
749  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
750  * TOC entries' dataLength fields with appropriate values to guide the
751  * ordering of restore jobs. The source of said data is format-dependent,
752  * as is the exact meaning of the values.
753  *
754  * A format module might also choose to do other setup here.
755  */
756 static void
758 {
759  TocEntry *te;
760 
761  for (te = AH->toc->next; te != AH->toc; te = te->next)
762  {
763  lclTocEntry *tctx = (lclTocEntry *) te->formatData;
764  char fname[MAXPGPATH];
765  struct stat st;
766 
767  /*
768  * A dumpable object has set tctx->filename, any other object has not.
769  * (see _ArchiveEntry).
770  */
771  if (tctx->filename == NULL)
772  continue;
773 
774  /* We may ignore items not due to be restored */
775  if ((te->reqs & REQ_DATA) == 0)
776  continue;
777 
778  /*
779  * Stat the file and, if successful, put its size in dataLength. When
780  * using compression, the physical file size might not be a very good
781  * guide to the amount of work involved in restoring the file, but we
782  * only need an approximate indicator of that.
783  */
784  setFilePath(AH, fname, tctx->filename);
785 
786  if (stat(fname, &st) == 0)
787  te->dataLength = st.st_size;
789  {
791  strlcat(fname, ".gz", sizeof(fname));
793  strlcat(fname, ".lz4", sizeof(fname));
795  strlcat(fname, ".zst", sizeof(fname));
796 
797  if (stat(fname, &st) == 0)
798  te->dataLength = st.st_size;
799  }
800 
801  /*
802  * If this is the BLOBS entry, what we stat'd was blobs.toc, which
803  * most likely is a lot smaller than the actual blob data. We don't
804  * have a cheap way to estimate how much smaller, but fortunately it
805  * doesn't matter too much as long as we get the LOs processed
806  * reasonably early. Arbitrarily scale up by a factor of 1K.
807  */
808  if (strcmp(te->desc, "BLOBS") == 0)
809  te->dataLength *= 1024;
810  }
811 }
812 
813 /*
814  * Clone format-specific fields during parallel restoration.
815  */
816 static void
818 {
819  lclContext *ctx = (lclContext *) AH->formatData;
820 
821  AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
822  memcpy(AH->formatData, ctx, sizeof(lclContext));
823  ctx = (lclContext *) AH->formatData;
824 
825  /*
826  * Note: we do not make a local lo_buf because we expect at most one BLOBS
827  * entry per archive, so no parallelism is possible. Likewise,
828  * TOC-entry-local state isn't an issue because any one TOC entry is
829  * touched by just one worker child.
830  */
831 
832  /*
833  * We also don't copy the ParallelState pointer (pstate), only the leader
834  * process ever writes to it.
835  */
836 }
837 
838 static void
840 {
841  lclContext *ctx = (lclContext *) AH->formatData;
842 
843  free(ctx);
844 }
845 
846 /*
847  * This function is executed in the child of a parallel backup for a
848  * directory-format archive and dumps the actual data for one TOC entry.
849  */
850 static int
852 {
853  /*
854  * This function returns void. We either fail and die horribly or
855  * succeed... A failure will be detected by the parent when the child dies
856  * unexpectedly.
857  */
859 
860  return 0;
861 }
862 
863 /*
864  * This function is executed in the child of a parallel restore from a
865  * directory-format archive and restores the actual data for one TOC entry.
866  */
867 static int
869 {
870  return parallel_restore(AH, te);
871 }
void ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate)
Definition: parallel.c:1059
ParallelState * ParallelBackupStart(ArchiveHandle *AH)
Definition: parallel.c:897
#define PG_BINARY_R
Definition: c.h:1262
#define CppAsString2(x)
Definition: c.h:314
#define PG_BINARY_W
Definition: c.h:1263
bool EndCompressFileHandle(CompressFileHandle *CFH)
Definition: compress_io.c:289
CompressFileHandle * InitDiscoverCompressFileHandle(const char *path, const char *mode)
Definition: compress_io.c:241
CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compression_spec)
Definition: compress_io.c:195
#define DEFAULT_IO_BUFFER_SIZE
Definition: compress_io.h:27
@ PG_COMPRESSION_GZIP
Definition: compression.h:24
@ PG_COMPRESSION_LZ4
Definition: compression.h:25
@ PG_COMPRESSION_NONE
Definition: compression.h:23
@ PG_COMPRESSION_ZSTD
Definition: compression.h:26
int closedir(DIR *)
Definition: dirent.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define free(a)
Definition: header.h:65
int i
Definition: isn.c:73
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
@ archModeWrite
Definition: pg_backup.h:51
@ archTar
Definition: pg_backup.h:43
@ archDirectory
Definition: pg_backup.h:45
void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te)
void WriteHead(ArchiveHandle *AH)
void StartRestoreLOs(ArchiveHandle *AH)
void WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
int ahprintf(ArchiveHandle *AH, const char *fmt,...)
char * ReadStr(ArchiveHandle *AH)
void ReadHead(ArchiveHandle *AH)
void ReadToc(ArchiveHandle *AH)
void EndRestoreLO(ArchiveHandle *AH, Oid oid)
void WriteToc(ArchiveHandle *AH)
void EndRestoreLOs(ArchiveHandle *AH)
void StartRestoreLO(ArchiveHandle *AH, Oid oid, bool drop)
int parallel_restore(ArchiveHandle *AH, TocEntry *te)
size_t WriteStr(ArchiveHandle *AH, const char *c)
#define LOBBUFSIZE
#define REQ_DATA
static void _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void _StartData(ArchiveHandle *AH, TocEntry *te)
static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
static void _CloseArchive(ArchiveHandle *AH)
static void _DeClone(ArchiveHandle *AH)
static void _ReopenArchive(ArchiveHandle *AH)
static void _StartLOs(ArchiveHandle *AH, TocEntry *te)
static void _LoadLOs(ArchiveHandle *AH)
static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void _EndLOs(ArchiveHandle *AH, TocEntry *te)
static int _WriteByte(ArchiveHandle *AH, const int i)
static void _PrepParallelRestore(ArchiveHandle *AH)
static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
static int _ReadByte(ArchiveHandle *AH)
static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
static void _EndData(ArchiveHandle *AH, TocEntry *te)
static void _Clone(ArchiveHandle *AH)
static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
void InitArchiveFmt_Directory(ArchiveHandle *AH)
static void _PrintFileData(ArchiveHandle *AH, char *filename)
static void setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
#define pg_fatal(...)
#define MAXPGPATH
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:121
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
size_t strlcat(char *dst, const char *src, size_t siz)
Definition: strlcat.c:33
unsigned int Oid
Definition: postgres_ext.h:31
char * c
int verbose
Definition: pg_backup.h:215
RestoreOptions * ropt
Definition: pg_backup.h:213
bool(* open_write_func)(const char *path, const char *mode, CompressFileHandle *CFH)
Definition: compress_io.h:122
int(* getc_func)(CompressFileHandle *CFH)
Definition: compress_io.h:161
bool(* read_func)(void *ptr, size_t size, size_t *rsize, CompressFileHandle *CFH)
Definition: compress_io.h:131
Definition: dirent.c:26
ArchiveFormat format
struct _tocEntry * toc
DeClonePtrType DeClonePtr
EndLOsPtrType EndLOsPtr
DataDirSyncMethod sync_method
ReadExtraTocPtrType ReadExtraTocPtr
WorkerJobDumpPtrType WorkerJobDumpPtr
StartLOsPtrType StartLOsPtr
ArchiveEntryPtrType ArchiveEntryPtr
pg_compress_specification compression_spec
WriteDataPtrType WriteDataPtr
StartLOPtrType StartLOPtr
ClonePtrType ClonePtr
WriteBufPtrType WriteBufPtr
PrepParallelRestorePtrType PrepParallelRestorePtr
EndLOPtrType EndLOPtr
WriteExtraTocPtrType WriteExtraTocPtr
ReadBytePtrType ReadBytePtr
WorkerJobRestorePtrType WorkerJobRestorePtr
PrintTocDataPtrType PrintTocDataPtr
WriteBytePtrType WriteBytePtr
ReadBufPtrType ReadBufPtr
PrintExtraTocPtrType PrintExtraTocPtr
StartDataPtrType StartDataPtr
ReopenPtrType ReopenPtr
EndDataPtrType EndDataPtr
ClosePtrType ClosePtr
DataDumperPtr dataDumper
pgoff_t dataLength
struct _tocEntry * next
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
ParallelState * pstate
CompressFileHandle * LOsTocFH
CompressFileHandle * dataFH
pg_compress_algorithm algorithm
Definition: compression.h:34
unsigned short st_mode
Definition: win32_port.h:268
static void * fn(void *arg)
Definition: thread-alloc.c:119
#define stat
Definition: win32_port.h:284
#define S_ISDIR(m)
Definition: win32_port.h:325
#define mkdir(a, b)
Definition: win32_port.h:80