PostgreSQL Source Code  git master
Macros | Functions
copydir.c File Reference
#include "postgres.h"
#include <fcntl.h>
#include <unistd.h>
#include "common/file_utils.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/copydir.h"
#include "storage/fd.h"
Include dependency graph for copydir.c:

Go to the source code of this file.

Macros

#define COPY_BUF_SIZE   (8 * BLCKSZ)
 
#define FLUSH_DISTANCE   (1024 * 1024)
 

Functions

void copydir (const char *fromdir, const char *todir, bool recurse)
 
void copy_file (const char *fromfile, const char *tofile)
 

Macro Definition Documentation

◆ COPY_BUF_SIZE

#define COPY_BUF_SIZE   (8 * BLCKSZ)

◆ FLUSH_DISTANCE

#define FLUSH_DISTANCE   (1024 * 1024)

Function Documentation

◆ copy_file()

void copy_file ( const char *  fromfile,
const char *  tofile 
)

Definition at line 117 of file copydir.c.

118 {
119  char *buffer;
120  int srcfd;
121  int dstfd;
122  int nbytes;
123  off_t offset;
124  off_t flush_offset;
125 
126  /* Size of copy buffer (read and write requests) */
127 #define COPY_BUF_SIZE (8 * BLCKSZ)
128 
129  /*
130  * Size of data flush requests. It seems beneficial on most platforms to
131  * do this every 1MB or so. But macOS, at least with early releases of
132  * APFS, is really unfriendly to small mmap/msync requests, so there do it
133  * only every 32MB.
134  */
135 #if defined(__darwin__)
136 #define FLUSH_DISTANCE (32 * 1024 * 1024)
137 #else
138 #define FLUSH_DISTANCE (1024 * 1024)
139 #endif
140 
141  /* Use palloc to ensure we get a maxaligned buffer */
142  buffer = palloc(COPY_BUF_SIZE);
143 
144  /*
145  * Open the files
146  */
147  srcfd = OpenTransientFile(fromfile, O_RDONLY | PG_BINARY);
148  if (srcfd < 0)
149  ereport(ERROR,
150  (errcode_for_file_access(),
151  errmsg("could not open file \"%s\": %m", fromfile)));
152 
153  dstfd = OpenTransientFile(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY);
154  if (dstfd < 0)
155  ereport(ERROR,
156  (errcode_for_file_access(),
157  errmsg("could not create file \"%s\": %m", tofile)));
158 
159  /*
160  * Do the data copying.
161  */
162  flush_offset = 0;
163  for (offset = 0;; offset += nbytes)
164  {
165  /* If we got a cancel signal during the copy of the file, quit */
166  CHECK_FOR_INTERRUPTS();
167 
168  /*
169  * We fsync the files later, but during the copy, flush them every so
170  * often to avoid spamming the cache and hopefully get the kernel to
171  * start writing them out before the fsync comes.
172  */
173  if (offset - flush_offset >= FLUSH_DISTANCE)
174  {
175  pg_flush_data(dstfd, flush_offset, offset - flush_offset);
176  flush_offset = offset;
177  }
178 
179  pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_READ);
180  nbytes = read(srcfd, buffer, COPY_BUF_SIZE);
181  pgstat_report_wait_end();
182  if (nbytes < 0)
183  ereport(ERROR,
184  (errcode_for_file_access(),
185  errmsg("could not read file \"%s\": %m", fromfile)));
186  if (nbytes == 0)
187  break;
188  errno = 0;
189  pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_WRITE);
190  if ((int) write(dstfd, buffer, nbytes) != nbytes)
191  {
192  /* if write didn't set errno, assume problem is no disk space */
193  if (errno == 0)
194  errno = ENOSPC;
195  ereport(ERROR,
196  (errcode_for_file_access(),
197  errmsg("could not write to file \"%s\": %m", tofile)));
198  }
199  pgstat_report_wait_end();
200  }
201 
202  if (offset > flush_offset)
203  pg_flush_data(dstfd, flush_offset, offset - flush_offset);
204 
205  if (CloseTransientFile(dstfd) != 0)
206  ereport(ERROR,
207  (errcode_for_file_access(),
208  errmsg("could not close file \"%s\": %m", tofile)));
209 
210  if (CloseTransientFile(srcfd) != 0)
211  ereport(ERROR,
212  (errcode_for_file_access(),
213  errmsg("could not close file \"%s\": %m", fromfile)));
214 
215  pfree(buffer);
216 }
PG_BINARY
#define PG_BINARY
Definition: c.h:1278
FLUSH_DISTANCE
#define FLUSH_DISTANCE
COPY_BUF_SIZE
#define COPY_BUF_SIZE
errcode_for_file_access
int errcode_for_file_access(void)
Definition: elog.c:881
errmsg
int errmsg(const char *fmt,...)
Definition: elog.c:1069
ERROR
#define ERROR
Definition: elog.h:39
ereport
#define ereport(elevel,...)
Definition: elog.h:149
pg_flush_data
void pg_flush_data(int fd, off_t offset, off_t nbytes)
Definition: fd.c:463
CloseTransientFile
int CloseTransientFile(int fd)
Definition: fd.c:2706
OpenTransientFile
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2530
dstfd
static int dstfd
Definition: file_ops.c:31
write
#define write(a, b, c)
Definition: win32.h:14
read
#define read(a, b, c)
Definition: win32.h:13
pfree
void pfree(void *pointer)
Definition: mcxt.c:1456
palloc
void * palloc(Size size)
Definition: mcxt.c:1226
CHECK_FOR_INTERRUPTS
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:121
WAIT_EVENT_COPY_FILE_READ
@ WAIT_EVENT_COPY_FILE_READ
Definition: wait_event.h:175
WAIT_EVENT_COPY_FILE_WRITE
@ WAIT_EVENT_COPY_FILE_WRITE
Definition: wait_event.h:176
pgstat_report_wait_start
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:271
pgstat_report_wait_end
static void pgstat_report_wait_end(void)
Definition: wait_event.h:287

References CHECK_FOR_INTERRUPTS, CloseTransientFile(), COPY_BUF_SIZE, dstfd, ereport, errcode_for_file_access(), errmsg(), ERROR, FLUSH_DISTANCE, OpenTransientFile(), palloc(), pfree(), PG_BINARY, pg_flush_data(), pgstat_report_wait_end(), pgstat_report_wait_start(), read, WAIT_EVENT_COPY_FILE_READ, WAIT_EVENT_COPY_FILE_WRITE, and write.

Referenced by basic_archive_file_internal(), copydir(), and ResetUnloggedRelationsInDbspaceDir().

◆ copydir()

void copydir ( const char *  fromdir,
const char *  todir,
bool  recurse 
)

Definition at line 37 of file copydir.c.

38 {
39  DIR *xldir;
40  struct dirent *xlde;
41  char fromfile[MAXPGPATH * 2];
42  char tofile[MAXPGPATH * 2];
43 
44  if (MakePGDirectory(todir) != 0)
45  ereport(ERROR,
46  (errcode_for_file_access(),
47  errmsg("could not create directory \"%s\": %m", todir)));
48 
49  xldir = AllocateDir(fromdir);
50 
51  while ((xlde = ReadDir(xldir, fromdir)) != NULL)
52  {
53  PGFileType xlde_type;
54 
55  /* If we got a cancel signal during the copy of the directory, quit */
56  CHECK_FOR_INTERRUPTS();
57 
58  if (strcmp(xlde->d_name, ".") == 0 ||
59  strcmp(xlde->d_name, "..") == 0)
60  continue;
61 
62  snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, xlde->d_name);
63  snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
64 
65  xlde_type = get_dirent_type(fromfile, xlde, false, ERROR);
66 
67  if (xlde_type == PGFILETYPE_DIR)
68  {
69  /* recurse to handle subdirectories */
70  if (recurse)
71  copydir(fromfile, tofile, true);
72  }
73  else if (xlde_type == PGFILETYPE_REG)
74  copy_file(fromfile, tofile);
75  }
76  FreeDir(xldir);
77 
78  /*
79  * Be paranoid here and fsync all files to ensure the copy is really done.
80  * But if fsync is disabled, we're done.
81  */
82  if (!enableFsync)
83  return;
84 
85  xldir = AllocateDir(todir);
86 
87  while ((xlde = ReadDir(xldir, todir)) != NULL)
88  {
89  if (strcmp(xlde->d_name, ".") == 0 ||
90  strcmp(xlde->d_name, "..") == 0)
91  continue;
92 
93  snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
94 
95  /*
96  * We don't need to sync subdirectories here since the recursive
97  * copydir will do it before it returns
98  */
99  if (get_dirent_type(tofile, xlde, false, ERROR) == PGFILETYPE_REG)
100  fsync_fname(tofile, false);
101  }
102  FreeDir(xldir);
103 
104  /*
105  * It's important to fsync the destination directory itself as individual
106  * file fsyncs don't guarantee that the directory entry for the file is
107  * synced. Recent versions of ext4 have made the window much wider but
108  * it's been true for ext3 and other filesystems in the past.
109  */
110  fsync_fname(todir, true);
111 }
copy_file
void copy_file(const char *fromfile, const char *tofile)
Definition: copydir.c:117
copydir
void copydir(const char *fromdir, const char *todir, bool recurse)
Definition: copydir.c:37
ReadDir
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2806
MakePGDirectory
int MakePGDirectory(const char *directoryName)
Definition: fd.c:3810
FreeDir
int FreeDir(DIR *dir)
Definition: fd.c:2858
fsync_fname
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:667
AllocateDir
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2740
get_dirent_type
PGFileType get_dirent_type(const char *path, const struct dirent *de, bool look_through_symlinks, int elevel)
Definition: file_utils.c:406
PGFileType
PGFileType
Definition: file_utils.h:19
PGFILETYPE_DIR
@ PGFILETYPE_DIR
Definition: file_utils.h:23
PGFILETYPE_REG
@ PGFILETYPE_REG
Definition: file_utils.h:22
enableFsync
bool enableFsync
Definition: globals.c:123
MAXPGPATH
#define MAXPGPATH
Definition: pg_config_manual.h:115
snprintf
#define snprintf
Definition: port.h:238
DIR
Definition: dirent.c:26
dirent
Definition: dirent.h:10
dirent::d_name
char d_name[MAX_PATH]
Definition: dirent.h:15

References AllocateDir(), CHECK_FOR_INTERRUPTS, copy_file(), dirent::d_name, enableFsync, ereport, errcode_for_file_access(), errmsg(), ERROR, FreeDir(), fsync_fname(), get_dirent_type(), MakePGDirectory(), MAXPGPATH, PGFILETYPE_DIR, PGFILETYPE_REG, ReadDir(), and snprintf.

Referenced by CreateDatabaseUsingFileCopy(), dbase_redo(), and movedb().