PostgreSQL Source Code git master
Loading...
Searching...
No Matches
Macros | Functions | Variables
copydir.c File Reference
#include "postgres.h"
#include <fcntl.h>
#include <unistd.h>
#include "common/file_utils.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/wait_event.h"
Include dependency graph for copydir.c:

Go to the source code of this file.

Macros

#define COPY_BUF_SIZE   (8 * BLCKSZ)
 
#define FLUSH_DISTANCE   (1024 * 1024)
 

Functions

static void clone_file (const char *fromfile, const char *tofile)
 
void copydir (const char *fromdir, const char *todir, bool recurse)
 
void copy_file (const char *fromfile, const char *tofile)
 

Variables

int file_copy_method = FILE_COPY_METHOD_COPY
 

Macro Definition Documentation

◆ COPY_BUF_SIZE

#define COPY_BUF_SIZE   (8 * BLCKSZ)

◆ FLUSH_DISTANCE

#define FLUSH_DISTANCE   (1024 * 1024)

Function Documentation

◆ clone_file()

static void clone_file ( const char *  fromfile,
const char *  tofile 
)
static

Definition at line 239 of file copydir.c.

240{
241#if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
242 if (copyfile(fromfile, tofile, NULL, COPYFILE_CLONE_FORCE) < 0)
243 ereport(ERROR,
244 (errcode_for_file_access(),
245 errmsg("could not clone file \"%s\" to \"%s\": %m",
246 fromfile, tofile)));
247#elif defined(HAVE_COPY_FILE_RANGE)
248 int srcfd;
249 int dstfd;
250 ssize_t nbytes;
251
252 srcfd = OpenTransientFile(fromfile, O_RDONLY | PG_BINARY);
253 if (srcfd < 0)
254 ereport(ERROR,
255 (errcode_for_file_access(),
256 errmsg("could not open file \"%s\": %m", fromfile)));
257
258 dstfd = OpenTransientFile(tofile, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY);
259 if (dstfd < 0)
260 ereport(ERROR,
261 (errcode_for_file_access(),
262 errmsg("could not create file \"%s\": %m", tofile)));
263
264 do
265 {
266 /*
267 * Don't copy too much at once, so we can check for interrupts from
268 * time to time if it falls back to a slow copy.
269 */
270 CHECK_FOR_INTERRUPTS();
271 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_COPY);
272 nbytes = copy_file_range(srcfd, NULL, dstfd, NULL, 1024 * 1024, 0);
273 if (nbytes < 0 && errno != EINTR)
274 ereport(ERROR,
275 (errcode_for_file_access(),
276 errmsg("could not clone file \"%s\" to \"%s\": %m",
277 fromfile, tofile)));
278 pgstat_report_wait_end();
279 }
280 while (nbytes != 0);
281
282 if (CloseTransientFile(dstfd) != 0)
283 ereport(ERROR,
284 (errcode_for_file_access(),
285 errmsg("could not close file \"%s\": %m", tofile)));
286
287 if (CloseTransientFile(srcfd) != 0)
288 ereport(ERROR,
289 (errcode_for_file_access(),
290 errmsg("could not close file \"%s\": %m", fromfile)));
291#else
292 /* If there is no CLONE support this function should not be called. */
293 pg_unreachable();
294#endif
295}
PG_BINARY
#define PG_BINARY
Definition c.h:1376
pg_unreachable
#define pg_unreachable()
Definition c.h:361
errcode_for_file_access
int errcode_for_file_access(void)
Definition elog.c:897
ERROR
#define ERROR
Definition elog.h:39
ereport
#define ereport(elevel,...)
Definition elog.h:150
CloseTransientFile
int CloseTransientFile(int fd)
Definition fd.c:2855
OpenTransientFile
int OpenTransientFile(const char *fileName, int fileFlags)
Definition fd.c:2678
dstfd
static int dstfd
Definition file_ops.c:31
CHECK_FOR_INTERRUPTS
#define CHECK_FOR_INTERRUPTS()
Definition miscadmin.h:123
errmsg
static char * errmsg
Definition oauth_hook_client.c:61
fb
static int fb(int x)
Definition preproc-init.c:92
pgstat_report_wait_start
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition wait_event.h:69
pgstat_report_wait_end
static void pgstat_report_wait_end(void)
Definition wait_event.h:85
EINTR
#define EINTR
Definition win32_port.h:361

References CHECK_FOR_INTERRUPTS, CloseTransientFile(), dstfd, EINTR, ereport, errcode_for_file_access(), errmsg, ERROR, fb(), OpenTransientFile(), PG_BINARY, pg_unreachable, pgstat_report_wait_end(), and pgstat_report_wait_start().

Referenced by copydir().

◆ copy_file()

void copy_file ( const char *  fromfile,
const char *  tofile 
)

Definition at line 134 of file copydir.c.

135{
136 char *buffer;
137 int srcfd;
138 int dstfd;
139 int nbytes;
140 off_t offset;
141 off_t flush_offset;
142
143 /* Size of copy buffer (read and write requests) */
144#define COPY_BUF_SIZE (8 * BLCKSZ)
145
146 /*
147 * Size of data flush requests. It seems beneficial on most platforms to
148 * do this every 1MB or so. But macOS, at least with early releases of
149 * APFS, is really unfriendly to small mmap/msync requests, so there do it
150 * only every 32MB.
151 */
152#if defined(__darwin__)
153#define FLUSH_DISTANCE (32 * 1024 * 1024)
154#else
155#define FLUSH_DISTANCE (1024 * 1024)
156#endif
157
158 /* Use palloc to ensure we get a maxaligned buffer */
159 buffer = palloc(COPY_BUF_SIZE);
160
161 /*
162 * Open the files
163 */
164 srcfd = OpenTransientFile(fromfile, O_RDONLY | PG_BINARY);
165 if (srcfd < 0)
166 ereport(ERROR,
167 (errcode_for_file_access(),
168 errmsg("could not open file \"%s\": %m", fromfile)));
169
170 dstfd = OpenTransientFile(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY);
171 if (dstfd < 0)
172 ereport(ERROR,
173 (errcode_for_file_access(),
174 errmsg("could not create file \"%s\": %m", tofile)));
175
176 /*
177 * Do the data copying.
178 */
179 flush_offset = 0;
180 for (offset = 0;; offset += nbytes)
181 {
182 /* If we got a cancel signal during the copy of the file, quit */
183 CHECK_FOR_INTERRUPTS();
184
185 /*
186 * We fsync the files later, but during the copy, flush them every so
187 * often to avoid spamming the cache and hopefully get the kernel to
188 * start writing them out before the fsync comes.
189 */
190 if (offset - flush_offset >= FLUSH_DISTANCE)
191 {
192 pg_flush_data(dstfd, flush_offset, offset - flush_offset);
193 flush_offset = offset;
194 }
195
196 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_READ);
197 nbytes = read(srcfd, buffer, COPY_BUF_SIZE);
198 pgstat_report_wait_end();
199 if (nbytes < 0)
200 ereport(ERROR,
201 (errcode_for_file_access(),
202 errmsg("could not read file \"%s\": %m", fromfile)));
203 if (nbytes == 0)
204 break;
205 errno = 0;
206 pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_WRITE);
207 if ((int) write(dstfd, buffer, nbytes) != nbytes)
208 {
209 /* if write didn't set errno, assume problem is no disk space */
210 if (errno == 0)
211 errno = ENOSPC;
212 ereport(ERROR,
213 (errcode_for_file_access(),
214 errmsg("could not write to file \"%s\": %m", tofile)));
215 }
216 pgstat_report_wait_end();
217 }
218
219 if (offset > flush_offset)
220 pg_flush_data(dstfd, flush_offset, offset - flush_offset);
221
222 if (CloseTransientFile(dstfd) != 0)
223 ereport(ERROR,
224 (errcode_for_file_access(),
225 errmsg("could not close file \"%s\": %m", tofile)));
226
227 if (CloseTransientFile(srcfd) != 0)
228 ereport(ERROR,
229 (errcode_for_file_access(),
230 errmsg("could not close file \"%s\": %m", fromfile)));
231
232 pfree(buffer);
233}
FLUSH_DISTANCE
#define FLUSH_DISTANCE
COPY_BUF_SIZE
#define COPY_BUF_SIZE
pg_flush_data
void pg_flush_data(int fd, pgoff_t offset, pgoff_t nbytes)
Definition fd.c:526
write
#define write(a, b, c)
Definition win32.h:14
read
#define read(a, b, c)
Definition win32.h:13
pfree
void pfree(void *pointer)
Definition mcxt.c:1616
palloc
void * palloc(Size size)
Definition mcxt.c:1387

References CHECK_FOR_INTERRUPTS, CloseTransientFile(), COPY_BUF_SIZE, dstfd, ereport, errcode_for_file_access(), errmsg, ERROR, fb(), FLUSH_DISTANCE, OpenTransientFile(), palloc(), pfree(), PG_BINARY, pg_flush_data(), pgstat_report_wait_end(), pgstat_report_wait_start(), read, and write.

Referenced by basic_archive_file(), copydir(), process_directory_recursively(), reconstruct_from_incremental_file(), and ResetUnloggedRelationsInDbspaceDir().

◆ copydir()

void copydir ( const char *  fromdir,
const char *  todir,
bool  recurse 
)

Definition at line 49 of file copydir.c.

50{
51 DIR *xldir;
52 struct dirent *xlde;
53 char fromfile[MAXPGPATH * 2];
54 char tofile[MAXPGPATH * 2];
55
56 if (MakePGDirectory(todir) != 0)
57 ereport(ERROR,
58 (errcode_for_file_access(),
59 errmsg("could not create directory \"%s\": %m", todir)));
60
61 xldir = AllocateDir(fromdir);
62
63 while ((xlde = ReadDir(xldir, fromdir)) != NULL)
64 {
65 PGFileType xlde_type;
66
67 /* If we got a cancel signal during the copy of the directory, quit */
68 CHECK_FOR_INTERRUPTS();
69
70 if (strcmp(xlde->d_name, ".") == 0 ||
71 strcmp(xlde->d_name, "..") == 0)
72 continue;
73
74 snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, xlde->d_name);
75 snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
76
77 xlde_type = get_dirent_type(fromfile, xlde, false, ERROR);
78
79 if (xlde_type == PGFILETYPE_DIR)
80 {
81 /* recurse to handle subdirectories */
82 if (recurse)
83 copydir(fromfile, tofile, true);
84 }
85 else if (xlde_type == PGFILETYPE_REG)
86 {
87 if (file_copy_method == FILE_COPY_METHOD_CLONE)
88 clone_file(fromfile, tofile);
89 else
90 copy_file(fromfile, tofile);
91 }
92 }
93 FreeDir(xldir);
94
95 /*
96 * Be paranoid here and fsync all files to ensure the copy is really done.
97 * But if fsync is disabled, we're done.
98 */
99 if (!enableFsync)
100 return;
101
102 xldir = AllocateDir(todir);
103
104 while ((xlde = ReadDir(xldir, todir)) != NULL)
105 {
106 if (strcmp(xlde->d_name, ".") == 0 ||
107 strcmp(xlde->d_name, "..") == 0)
108 continue;
109
110 snprintf(tofile, sizeof(tofile), "%s/%s", todir, xlde->d_name);
111
112 /*
113 * We don't need to sync subdirectories here since the recursive
114 * copydir will do it before it returns
115 */
116 if (get_dirent_type(tofile, xlde, false, ERROR) == PGFILETYPE_REG)
117 fsync_fname(tofile, false);
118 }
119 FreeDir(xldir);
120
121 /*
122 * It's important to fsync the destination directory itself as individual
123 * file fsyncs don't guarantee that the directory entry for the file is
124 * synced. Recent versions of ext4 have made the window much wider but
125 * it's been true for ext3 and other filesystems in the past.
126 */
127 fsync_fname(todir, true);
128}
file_copy_method
int file_copy_method
Definition copydir.c:35
clone_file
static void clone_file(const char *fromfile, const char *tofile)
Definition copydir.c:239
copy_file
void copy_file(const char *fromfile, const char *tofile)
Definition copydir.c:134
copydir
void copydir(const char *fromdir, const char *todir, bool recurse)
Definition copydir.c:49
FILE_COPY_METHOD_CLONE
@ FILE_COPY_METHOD_CLONE
Definition copydir.h:19
MakePGDirectory
int MakePGDirectory(const char *directoryName)
Definition fd.c:3963
FreeDir
int FreeDir(DIR *dir)
Definition fd.c:3009
fsync_fname
void fsync_fname(const char *fname, bool isdir)
Definition fd.c:757
AllocateDir
DIR * AllocateDir(const char *dirname)
Definition fd.c:2891
ReadDir
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition fd.c:2957
get_dirent_type
PGFileType get_dirent_type(const char *path, const struct dirent *de, bool look_through_symlinks, int elevel)
Definition file_utils.c:547
PGFileType
PGFileType
Definition file_utils.h:19
PGFILETYPE_DIR
@ PGFILETYPE_DIR
Definition file_utils.h:23
PGFILETYPE_REG
@ PGFILETYPE_REG
Definition file_utils.h:22
enableFsync
bool enableFsync
Definition globals.c:129
MAXPGPATH
#define MAXPGPATH
Definition pg_config_manual.h:105
snprintf
#define snprintf
Definition port.h:260
DIR
Definition dirent.c:26

References AllocateDir(), CHECK_FOR_INTERRUPTS, clone_file(), copy_file(), copydir(), enableFsync, ereport, errcode_for_file_access(), errmsg, ERROR, fb(), file_copy_method, FILE_COPY_METHOD_CLONE, FreeDir(), fsync_fname(), get_dirent_type(), MakePGDirectory(), MAXPGPATH, PGFILETYPE_DIR, PGFILETYPE_REG, ReadDir(), and snprintf.

Referenced by copydir(), CreateDatabaseUsingFileCopy(), dbase_redo(), and movedb().

Variable Documentation

◆ file_copy_method

int file_copy_method = FILE_COPY_METHOD_COPY

Definition at line 35 of file copydir.c.

Referenced by copydir().