PostgreSQL Source Code git master
Macros | Functions | Variables
copydir.c File Reference
#include "postgres.h"
#include <fcntl.h>
#include <unistd.h>
#include "common/file_utils.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/copydir.h"
#include "storage/fd.h"
Include dependency graph for copydir.c:

Go to the source code of this file.

Macros

#define COPY_BUF_SIZE   (8 * BLCKSZ)
 
#define FLUSH_DISTANCE   (1024 * 1024)
 

Functions

static void clone_file (const char *fromfile, const char *tofile)
 
void copydir (const char *fromdir, const char *todir, bool recurse)
 
void copy_file (const char *fromfile, const char *tofile)
 

Variables

int file_copy_method = FILE_COPY_METHOD_COPY
 

Macro Definition Documentation

◆ COPY_BUF_SIZE

#define COPY_BUF_SIZE   (8 * BLCKSZ)

◆ FLUSH_DISTANCE

#define FLUSH_DISTANCE   (1024 * 1024)

Function Documentation

◆ clone_file()

static void clone_file ( const char *  fromfile,
const char *  tofile 
)
static

Definition at line 238 of file copydir.c.

239{
240#if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
241 if (copyfile(fromfile, tofile, NULL, COPYFILE_CLONE_FORCE) < 0)
242 ereport(ERROR,
243 (errcode_for_file_access(),
244 errmsg("could not clone file \"%s\" to \"%s\": %m",
245 fromfile, tofile)));
246#elif defined(HAVE_COPY_FILE_RANGE)
247 int srcfd;
248 int dstfd;
249 ssize_t nbytes;
250
251 srcfd = OpenTransientFile(fromfile, O_RDONLY | PG_BINARY);
252 if (srcfd < 0)
253 ereport(ERROR,
254 (errcode_for_file_access(),
255 errmsg("could not open file \"%s\": %m", fromfile)));
256
257 dstfd = OpenTransientFile(tofile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY);
258 if (dstfd < 0)
259 ereport(ERROR,
260 (errcode_for_file_access(),
261 errmsg("could not create file \"%s\": %m", tofile)));
262
263 do
264 {
265 /*
266 * Don't copy too much at once, so we can check for interrupts from
267 * time to time if it falls back to a slow copy.
268 */
269 CHECK_FOR_INTERRUPTS();
270 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_COPY);
271 nbytes = copy_file_range(srcfd, NULL, dstfd, NULL, 1024 * 1024, 0);
272 if (nbytes < 0 && errno != EINTR)
273 ereport(ERROR,
274 (errcode_for_file_access(),
275 errmsg("could not clone file \"%s\" to \"%s\": %m",
276 fromfile, tofile)));
277 pgstat_report_wait_end();
278 }
279 while (nbytes != 0);
280
281 if (CloseTransientFile(dstfd) != 0)
282 ereport(ERROR,
283 (errcode_for_file_access(),
284 errmsg("could not close file \"%s\": %m", tofile)));
285
286 if (CloseTransientFile(srcfd) != 0)
287 ereport(ERROR,
288 (errcode_for_file_access(),
289 errmsg("could not close file \"%s\": %m", fromfile)));
290#else
291 /* If there is no CLONE support this function should not be called. */
292 pg_unreachable();
293#endif
294}
PG_BINARY
#define PG_BINARY
Definition: c.h:1244
pg_unreachable
#define pg_unreachable()
Definition: c.h:332
errcode_for_file_access
int errcode_for_file_access(void)
Definition: elog.c:877
errmsg
int errmsg(const char *fmt,...)
Definition: elog.c:1071
ERROR
#define ERROR
Definition: elog.h:39
ereport
#define ereport(elevel,...)
Definition: elog.h:149
CloseTransientFile
int CloseTransientFile(int fd)
Definition: fd.c:2871
OpenTransientFile
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2694
dstfd
static int dstfd
Definition: file_ops.c:31
CHECK_FOR_INTERRUPTS
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:122
pgstat_report_wait_start
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:69
pgstat_report_wait_end
static void pgstat_report_wait_end(void)
Definition: wait_event.h:85
EINTR
#define EINTR
Definition: win32_port.h:364

References CHECK_FOR_INTERRUPTS, CloseTransientFile(), dstfd, EINTR, ereport, errcode_for_file_access(), errmsg(), ERROR, OpenTransientFile(), PG_BINARY, pg_unreachable, pgstat_report_wait_end(), and pgstat_report_wait_start().

Referenced by copydir().

◆ copy_file()

void copy_file ( const char *  fromfile,
const char *  tofile 
)

Definition at line 133 of file copydir.c.

134{
135 char *buffer;
136 int srcfd;
137 int dstfd;
138 int nbytes;
139 off_t offset;
140 off_t flush_offset;
141
142 /* Size of copy buffer (read and write requests) */
143#define COPY_BUF_SIZE (8 * BLCKSZ)
144
145 /*
146 * Size of data flush requests. It seems beneficial on most platforms to
147 * do this every 1MB or so. But macOS, at least with early releases of
148 * APFS, is really unfriendly to small mmap/msync requests, so there do it
149 * only every 32MB.
150 */
151#if defined(__darwin__)
152#define FLUSH_DISTANCE (32 * 1024 * 1024)
153#else
154#define FLUSH_DISTANCE (1024 * 1024)
155#endif
156
157 /* Use palloc to ensure we get a maxaligned buffer */
158 buffer = palloc(COPY_BUF_SIZE);
159
160 /*
161 * Open the files
162 */
163 srcfd = OpenTransientFile(fromfile, O_RDONLY | PG_BINARY);
164 if (srcfd < 0)
165 ereport(ERROR,
166 (errcode_for_file_access(),
167 errmsg("could not open file \"%s\": %m", fromfile)));
168
169 dstfd = OpenTransientFile(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY);
170 if (dstfd < 0)
171 ereport(ERROR,
172 (errcode_for_file_access(),
173 errmsg("could not create file \"%s\": %m", tofile)));
174
175 /*
176 * Do the data copying.
177 */
178 flush_offset = 0;
179 for (offset = 0;; offset += nbytes)
180 {
181 /* If we got a cancel signal during the copy of the file, quit */
182 CHECK_FOR_INTERRUPTS();
183
184 /*
185 * We fsync the files later, but during the copy, flush them every so
186 * often to avoid spamming the cache and hopefully get the kernel to
187 * start writing them out before the fsync comes.
188 */
189 if (offset - flush_offset >= FLUSH_DISTANCE)
190 {
191 pg_flush_data(dstfd, flush_offset, offset - flush_offset);
192 flush_offset = offset;
193 }
194
195 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_READ);
196 nbytes = read(srcfd, buffer, COPY_BUF_SIZE);
197 pgstat_report_wait_end();
198 if (nbytes < 0)
199 ereport(ERROR,
200 (errcode_for_file_access(),
201 errmsg("could not read file \"%s\": %m", fromfile)));
202 if (nbytes == 0)
203 break;
204 errno = 0;
205 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_WRITE);
206 if ((int) write(dstfd, buffer, nbytes) != nbytes)
207 {
208 /* if write didn't set errno, assume problem is no disk space */
209 if (errno == 0)
210 errno = ENOSPC;
211 ereport(ERROR,
212 (errcode_for_file_access(),
213 errmsg("could not write to file \"%s\": %m", tofile)));
214 }
215 pgstat_report_wait_end();
216 }
217
218 if (offset > flush_offset)
219 pg_flush_data(dstfd, flush_offset, offset - flush_offset);
220
221 if (CloseTransientFile(dstfd) != 0)
222 ereport(ERROR,
223 (errcode_for_file_access(),
224 errmsg("could not close file \"%s\": %m", tofile)));
225
226 if (CloseTransientFile(srcfd) != 0)
227 ereport(ERROR,
228 (errcode_for_file_access(),
229 errmsg("could not close file \"%s\": %m", fromfile)));
230
231 pfree(buffer);
232}
FLUSH_DISTANCE
#define FLUSH_DISTANCE
COPY_BUF_SIZE
#define COPY_BUF_SIZE
pg_flush_data
void pg_flush_data(int fd, off_t offset, off_t nbytes)
Definition: fd.c:525
write
#define write(a, b, c)
Definition: win32.h:14
read
#define read(a, b, c)
Definition: win32.h:13
pfree
void pfree(void *pointer)
Definition: mcxt.c:1528
palloc
void * palloc(Size size)
Definition: mcxt.c:1321

References CHECK_FOR_INTERRUPTS, CloseTransientFile(), COPY_BUF_SIZE, dstfd, ereport, errcode_for_file_access(), errmsg(), ERROR, FLUSH_DISTANCE, OpenTransientFile(), palloc(), pfree(), PG_BINARY, pg_flush_data(), pgstat_report_wait_end(), pgstat_report_wait_start(), read, and write.

Referenced by basic_archive_file(), copydir(), process_directory_recursively(), reconstruct_from_incremental_file(), and ResetUnloggedRelationsInDbspaceDir().

◆ copydir()

void copydir ( const char *  fromdir,
const char *  todir,
bool  recurse 
)

Definition at line 48 of file copydir.c.

49{
50 DIR *xldir;
51 struct dirent *xlde;
52 char fromfile[MAXPGPATH * 2];
53 char tofile[MAXPGPATH * 2];
54
55 if (MakePGDirectory(todir) != 0)
56 ereport(ERROR,
57 (errcode_for_file_access(),
58 errmsg("could not create directory \"%s\": %m", todir)));
59
60 xldir = AllocateDir(fromdir);
61
62 while ((xlde = ReadDir(xldir, fromdir)) != NULL)
63 {
64 PGFileType xlde_type;
65
66 /* If we got a cancel signal during the copy of the directory, quit */
67 CHECK_FOR_INTERRUPTS();
68
69 if (strcmp(xlde->d_name, ".") == 0 ||
70 strcmp(xlde->d_name, "..") == 0)
71 continue;
72
73 snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, xlde->d_name);
74 snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
75
76 xlde_type = get_dirent_type(fromfile, xlde, false, ERROR);
77
78 if (xlde_type == PGFILETYPE_DIR)
79 {
80 /* recurse to handle subdirectories */
81 if (recurse)
82 copydir(fromfile, tofile, true);
83 }
84 else if (xlde_type == PGFILETYPE_REG)
85 {
86 if (file_copy_method == FILE_COPY_METHOD_CLONE)
87 clone_file(fromfile, tofile);
88 else
89 copy_file(fromfile, tofile);
90 }
91 }
92 FreeDir(xldir);
93
94 /*
95 * Be paranoid here and fsync all files to ensure the copy is really done.
96 * But if fsync is disabled, we're done.
97 */
98 if (!enableFsync)
99 return;
100
101 xldir = AllocateDir(todir);
102
103 while ((xlde = ReadDir(xldir, todir)) != NULL)
104 {
105 if (strcmp(xlde->d_name, ".") == 0 ||
106 strcmp(xlde->d_name, "..") == 0)
107 continue;
108
109 snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
110
111 /*
112 * We don't need to sync subdirectories here since the recursive
113 * copydir will do it before it returns
114 */
115 if (get_dirent_type(tofile, xlde, false, ERROR) == PGFILETYPE_REG)
116 fsync_fname(tofile, false);
117 }
118 FreeDir(xldir);
119
120 /*
121 * It's important to fsync the destination directory itself as individual
122 * file fsyncs don't guarantee that the directory entry for the file is
123 * synced. Recent versions of ext4 have made the window much wider but
124 * it's been true for ext3 and other filesystems in the past.
125 */
126 fsync_fname(todir, true);
127}
file_copy_method
int file_copy_method
Definition: copydir.c:34
clone_file
static void clone_file(const char *fromfile, const char *tofile)
Definition: copydir.c:238
copy_file
void copy_file(const char *fromfile, const char *tofile)
Definition: copydir.c:133
copydir
void copydir(const char *fromdir, const char *todir, bool recurse)
Definition: copydir.c:48
FILE_COPY_METHOD_CLONE
@ FILE_COPY_METHOD_CLONE
Definition: copydir.h:19
MakePGDirectory
int MakePGDirectory(const char *directoryName)
Definition: fd.c:3978
FreeDir
int FreeDir(DIR *dir)
Definition: fd.c:3025
fsync_fname
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:756
AllocateDir
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2907
ReadDir
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2973
get_dirent_type
PGFileType get_dirent_type(const char *path, const struct dirent *de, bool look_through_symlinks, int elevel)
Definition: file_utils.c:547
PGFileType
PGFileType
Definition: file_utils.h:19
PGFILETYPE_DIR
@ PGFILETYPE_DIR
Definition: file_utils.h:23
PGFILETYPE_REG
@ PGFILETYPE_REG
Definition: file_utils.h:22
enableFsync
bool enableFsync
Definition: globals.c:129
MAXPGPATH
#define MAXPGPATH
Definition: pg_config_manual.h:100
snprintf
#define snprintf
Definition: port.h:239
DIR
Definition: dirent.c:26
dirent
Definition: dirent.h:10
dirent::d_name
char d_name[MAX_PATH]
Definition: dirent.h:15

References AllocateDir(), CHECK_FOR_INTERRUPTS, clone_file(), copy_file(), copydir(), dirent::d_name, enableFsync, ereport, errcode_for_file_access(), errmsg(), ERROR, file_copy_method, FILE_COPY_METHOD_CLONE, FreeDir(), fsync_fname(), get_dirent_type(), MakePGDirectory(), MAXPGPATH, PGFILETYPE_DIR, PGFILETYPE_REG, ReadDir(), and snprintf.

Referenced by copydir(), CreateDatabaseUsingFileCopy(), dbase_redo(), and movedb().

Variable Documentation

◆ file_copy_method

int file_copy_method = FILE_COPY_METHOD_COPY

Definition at line 34 of file copydir.c.

Referenced by copydir().