PostgreSQL Source Code  git master
file_ops.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * file_ops.c
4  * Helper functions for operating on files.
5  *
6  * Most of the functions in this file are helper functions for writing to
7  * the target data directory. The functions check the --dry-run flag, and
8  * do nothing if it's enabled. You should avoid accessing the target files
9  * directly but if you do, make sure you honor the --dry-run mode!
10  *
11  * Portions Copyright (c) 2013-2020, PostgreSQL Global Development Group
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres_fe.h"
16 
17 #include <sys/stat.h>
18 #include <dirent.h>
19 #include <fcntl.h>
20 #include <unistd.h>
21 
22 #include "common/file_perm.h"
23 #include "common/file_utils.h"
24 #include "file_ops.h"
25 #include "filemap.h"
26 #include "pg_rewind.h"
27 
28 /*
29  * Currently open target file.
30  */
31 static int dstfd = -1;
32 static char dstpath[MAXPGPATH] = "";
33 
34 static void create_target_dir(const char *path);
35 static void remove_target_dir(const char *path);
36 static void create_target_symlink(const char *path, const char *link);
37 static void remove_target_symlink(const char *path);
38 
39 static void recurse_dir(const char *datadir, const char *parentpath,
41 
42 /*
43  * Open a target file for writing. If 'trunc' is true and the file already
44  * exists, it will be truncated.
45  */
46 void
47 open_target_file(const char *path, bool trunc)
48 {
49  int mode;
50 
51  if (dry_run)
52  return;
53 
54  if (dstfd != -1 && !trunc &&
55  strcmp(path, &dstpath[strlen(datadir_target) + 1]) == 0)
56  return; /* already open */
57 
59 
60  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
61 
62  mode = O_WRONLY | O_CREAT | PG_BINARY;
63  if (trunc)
64  mode |= O_TRUNC;
65  dstfd = open(dstpath, mode, pg_file_create_mode);
66  if (dstfd < 0)
67  pg_fatal("could not open target file \"%s\": %m",
68  dstpath);
69 }
70 
71 /*
72  * Close target file, if it's open.
73  */
74 void
76 {
77  if (dstfd == -1)
78  return;
79 
80  if (close(dstfd) != 0)
81  pg_fatal("could not close target file \"%s\": %m",
82  dstpath);
83 
84  dstfd = -1;
85 }
86 
87 void
88 write_target_range(char *buf, off_t begin, size_t size)
89 {
90  size_t writeleft;
91  char *p;
92 
93  /* update progress report */
94  fetch_done += size;
95  progress_report(false);
96 
97  if (dry_run)
98  return;
99 
100  if (lseek(dstfd, begin, SEEK_SET) == -1)
101  pg_fatal("could not seek in target file \"%s\": %m",
102  dstpath);
103 
104  writeleft = size;
105  p = buf;
106  while (writeleft > 0)
107  {
108  ssize_t writelen;
109 
110  errno = 0;
111  writelen = write(dstfd, p, writeleft);
112  if (writelen < 0)
113  {
114  /* if write didn't set errno, assume problem is no disk space */
115  if (errno == 0)
116  errno = ENOSPC;
117  pg_fatal("could not write file \"%s\": %m",
118  dstpath);
119  }
120 
121  p += writelen;
122  writeleft -= writelen;
123  }
124 
125  /* keep the file open, in case we need to copy more blocks in it */
126 }
127 
128 
129 void
131 {
132  Assert(entry->action == FILE_ACTION_REMOVE);
133  Assert(entry->target_exists);
134 
135  switch (entry->target_type)
136  {
137  case FILE_TYPE_DIRECTORY:
138  remove_target_dir(entry->path);
139  break;
140 
141  case FILE_TYPE_REGULAR:
142  remove_target_file(entry->path, false);
143  break;
144 
145  case FILE_TYPE_SYMLINK:
146  remove_target_symlink(entry->path);
147  break;
148 
149  case FILE_TYPE_UNDEFINED:
150  pg_fatal("undefined file type for \"%s\"", entry->path);
151  break;
152  }
153 }
154 
155 void
157 {
158  Assert(entry->action == FILE_ACTION_CREATE);
159  Assert(!entry->target_exists);
160 
161  switch (entry->source_type)
162  {
163  case FILE_TYPE_DIRECTORY:
164  create_target_dir(entry->path);
165  break;
166 
167  case FILE_TYPE_SYMLINK:
169  break;
170 
171  case FILE_TYPE_REGULAR:
172  /* can't happen. Regular files are created with open_target_file. */
173  pg_fatal("invalid action (CREATE) for regular file");
174  break;
175 
176  case FILE_TYPE_UNDEFINED:
177  pg_fatal("undefined file type for \"%s\"", entry->path);
178  break;
179  }
180 }
181 
182 /*
183  * Remove a file from target data directory. If missing_ok is true, it
184  * is fine for the target file to not exist.
185  */
186 void
187 remove_target_file(const char *path, bool missing_ok)
188 {
189  char dstpath[MAXPGPATH];
190 
191  if (dry_run)
192  return;
193 
194  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
195  if (unlink(dstpath) != 0)
196  {
197  if (errno == ENOENT && missing_ok)
198  return;
199 
200  pg_fatal("could not remove file \"%s\": %m",
201  dstpath);
202  }
203 }
204 
205 void
206 truncate_target_file(const char *path, off_t newsize)
207 {
208  char dstpath[MAXPGPATH];
209  int fd;
210 
211  if (dry_run)
212  return;
213 
214  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
215 
216  fd = open(dstpath, O_WRONLY, pg_file_create_mode);
217  if (fd < 0)
218  pg_fatal("could not open file \"%s\" for truncation: %m",
219  dstpath);
220 
221  if (ftruncate(fd, newsize) != 0)
222  pg_fatal("could not truncate file \"%s\" to %u: %m",
223  dstpath, (unsigned int) newsize);
224 
225  close(fd);
226 }
227 
228 static void
229 create_target_dir(const char *path)
230 {
231  char dstpath[MAXPGPATH];
232 
233  if (dry_run)
234  return;
235 
236  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
237  if (mkdir(dstpath, pg_dir_create_mode) != 0)
238  pg_fatal("could not create directory \"%s\": %m",
239  dstpath);
240 }
241 
242 static void
243 remove_target_dir(const char *path)
244 {
245  char dstpath[MAXPGPATH];
246 
247  if (dry_run)
248  return;
249 
250  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
251  if (rmdir(dstpath) != 0)
252  pg_fatal("could not remove directory \"%s\": %m",
253  dstpath);
254 }
255 
256 static void
257 create_target_symlink(const char *path, const char *link)
258 {
259  char dstpath[MAXPGPATH];
260 
261  if (dry_run)
262  return;
263 
264  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
265  if (symlink(link, dstpath) != 0)
266  pg_fatal("could not create symbolic link at \"%s\": %m",
267  dstpath);
268 }
269 
270 static void
271 remove_target_symlink(const char *path)
272 {
273  char dstpath[MAXPGPATH];
274 
275  if (dry_run)
276  return;
277 
278  snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
279  if (unlink(dstpath) != 0)
280  pg_fatal("could not remove symbolic link \"%s\": %m",
281  dstpath);
282 }
283 
284 /*
285  * Sync target data directory to ensure that modifications are safely on disk.
286  *
287  * We do this once, for the whole data directory, for performance reasons. At
288  * the end of pg_rewind's run, the kernel is likely to already have flushed
289  * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass
290  * approach (only initiating writeback in the first pass), which often reduces
291  * the overall amount of IO noticeably.
292  */
293 void
295 {
296  if (!do_sync || dry_run)
297  return;
298 
299  fsync_pgdata(datadir_target, PG_VERSION_NUM);
300 }
301 
302 
303 /*
304  * Read a file into memory. The file to be read is <datadir>/<path>.
305  * The file contents are returned in a malloc'd buffer, and *filesize
306  * is set to the length of the file.
307  *
308  * The returned buffer is always zero-terminated; the size of the returned
309  * buffer is actually *filesize + 1. That's handy when reading a text file.
310  * This function can be used to read binary files as well, you can just
311  * ignore the zero-terminator in that case.
312  */
313 char *
314 slurpFile(const char *datadir, const char *path, size_t *filesize)
315 {
316  int fd;
317  char *buffer;
318  struct stat statbuf;
319  char fullpath[MAXPGPATH];
320  int len;
321  int r;
322 
323  snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
324 
325  if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
326  pg_fatal("could not open file \"%s\" for reading: %m",
327  fullpath);
328 
329  if (fstat(fd, &statbuf) < 0)
330  pg_fatal("could not open file \"%s\" for reading: %m",
331  fullpath);
332 
333  len = statbuf.st_size;
334 
335  buffer = pg_malloc(len + 1);
336 
337  r = read(fd, buffer, len);
338  if (r != len)
339  {
340  if (r < 0)
341  pg_fatal("could not read file \"%s\": %m",
342  fullpath);
343  else
344  pg_fatal("could not read file \"%s\": read %d of %zu",
345  fullpath, r, (Size) len);
346  }
347  close(fd);
348 
349  /* Zero-terminate the buffer. */
350  buffer[len] = '\0';
351 
352  if (filesize)
353  *filesize = len;
354  return buffer;
355 }
356 
357 /*
358  * Traverse through all files in a data directory, calling 'callback'
359  * for each file.
360  */
361 void
363 {
364  recurse_dir(datadir, NULL, callback);
365 }
366 
367 /*
368  * recursive part of traverse_datadir
369  *
370  * parentpath is the current subdirectory's path relative to datadir,
371  * or NULL at the top level.
372  */
373 static void
374 recurse_dir(const char *datadir, const char *parentpath,
376 {
377  DIR *xldir;
378  struct dirent *xlde;
379  char fullparentpath[MAXPGPATH];
380 
381  if (parentpath)
382  snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
383  else
384  snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
385 
386  xldir = opendir(fullparentpath);
387  if (xldir == NULL)
388  pg_fatal("could not open directory \"%s\": %m",
389  fullparentpath);
390 
391  while (errno = 0, (xlde = readdir(xldir)) != NULL)
392  {
393  struct stat fst;
394  char fullpath[MAXPGPATH * 2];
395  char path[MAXPGPATH * 2];
396 
397  if (strcmp(xlde->d_name, ".") == 0 ||
398  strcmp(xlde->d_name, "..") == 0)
399  continue;
400 
401  snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
402 
403  if (lstat(fullpath, &fst) < 0)
404  {
405  if (errno == ENOENT)
406  {
407  /*
408  * File doesn't exist anymore. This is ok, if the new primary
409  * is running and the file was just removed. If it was a data
410  * file, there should be a WAL record of the removal. If it
411  * was something else, it couldn't have been anyway.
412  *
413  * TODO: But complain if we're processing the target dir!
414  */
415  }
416  else
417  pg_fatal("could not stat file \"%s\": %m",
418  fullpath);
419  }
420 
421  if (parentpath)
422  snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
423  else
424  snprintf(path, sizeof(path), "%s", xlde->d_name);
425 
426  if (S_ISREG(fst.st_mode))
427  callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
428  else if (S_ISDIR(fst.st_mode))
429  {
430  callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
431  /* recurse to handle subdirectories */
432  recurse_dir(datadir, path, callback);
433  }
434 #ifndef WIN32
435  else if (S_ISLNK(fst.st_mode))
436 #else
437  else if (pgwin32_is_junction(fullpath))
438 #endif
439  {
440 #if defined(HAVE_READLINK) || defined(WIN32)
441  char link_target[MAXPGPATH];
442  int len;
443 
444  len = readlink(fullpath, link_target, sizeof(link_target));
445  if (len < 0)
446  pg_fatal("could not read symbolic link \"%s\": %m",
447  fullpath);
448  if (len >= sizeof(link_target))
449  pg_fatal("symbolic link \"%s\" target is too long",
450  fullpath);
451  link_target[len] = '\0';
452 
453  callback(path, FILE_TYPE_SYMLINK, 0, link_target);
454 
455  /*
456  * If it's a symlink within pg_tblspc, we need to recurse into it,
457  * to process all the tablespaces. We also follow a symlink if
458  * it's for pg_wal. Symlinks elsewhere are ignored.
459  */
460  if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
461  strcmp(path, "pg_wal") == 0)
462  recurse_dir(datadir, path, callback);
463 #else
464  pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform",
465  fullpath);
466 #endif /* HAVE_READLINK */
467  }
468  }
469 
470  if (errno)
471  pg_fatal("could not read directory \"%s\": %m",
472  fullparentpath);
473 
474  if (closedir(xldir))
475  pg_fatal("could not close directory \"%s\": %m",
476  fullparentpath);
477 }
static PgChecksumMode mode
Definition: pg_checksums.c:61
char * datadir_target
Definition: pg_rewind.c:59
static int dstfd
Definition: file_ops.c:31
void open_target_file(const char *path, bool trunc)
Definition: file_ops.c:47
int pg_file_create_mode
Definition: file_perm.c:19
void write_target_range(char *buf, off_t begin, size_t size)
Definition: file_ops.c:88
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define write(a, b, c)
Definition: win32.h:14
bool dry_run
Definition: pg_rewind.c:66
void(* process_file_callback_t)(const char *path, file_type_t type, size_t size, const char *link_target)
Definition: file_ops.h:26
void traverse_datadir(const char *datadir, process_file_callback_t callback)
Definition: file_ops.c:362
void remove_target_file(const char *path, bool missing_ok)
Definition: file_ops.c:187
int closedir(DIR *)
Definition: dirent.c:123
#define pg_fatal(...)
Definition: pg_rewind.h:37
Definition: dirent.h:9
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1267
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:206
#define fstat
Definition: win32_port.h:274
char * source_link_target
Definition: filemap.h:76
static void create_target_symlink(const char *path, const char *link)
Definition: file_ops.c:257
Definition: dirent.c:25
file_type_t target_type
Definition: filemap.h:60
#define MAXPGPATH
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:48
DIR * opendir(const char *)
Definition: dirent.c:33
file_action_t action
Definition: filemap.h:81
static char * buf
Definition: pg_test_fsync.c:68
#define symlink(oldpath, newpath)
Definition: win32_port.h:227
bool target_exists
Definition: filemap.h:59
#define readlink(path, buf, size)
Definition: win32_port.h:228
file_type_t source_type
Definition: filemap.h:74
static void remove_target_symlink(const char *path)
Definition: file_ops.c:271
char * datadir
__int64 st_size
Definition: win32_port.h:265
const char * path
Definition: filemap.h:53
static char dstpath[MAXPGPATH]
Definition: file_ops.c:32
#define S_ISREG(m)
Definition: win32_port.h:319
static void recurse_dir(const char *datadir, const char *parentpath, process_file_callback_t callback)
Definition: file_ops.c:374
static void remove_target_dir(const char *path)
Definition: file_ops.c:243
int pg_dir_create_mode
Definition: file_perm.c:18
unsigned short st_mode
Definition: win32_port.h:260
void sync_target_dir(void)
Definition: file_ops.c:294
static void create_target_dir(const char *path)
Definition: file_ops.c:229
void remove_target(file_entry_t *entry)
Definition: file_ops.c:130
#define Assert(condition)
Definition: c.h:800
static bool do_sync
Definition: initdb.c:142
size_t Size
Definition: c.h:528
struct dirent * readdir(DIR *)
Definition: dirent.c:78
#define S_ISDIR(m)
Definition: win32_port.h:316
#define lstat(path, sb)
Definition: win32_port.h:276
void close_target_file(void)
Definition: file_ops.c:75
Definition: filemap.h:49
void create_target(file_entry_t *entry)
Definition: file_ops.c:156
char d_name[MAX_PATH]
Definition: dirent.h:15
#define mkdir(a, b)
Definition: win32_port.h:63
uint64 fetch_done
Definition: pg_rewind.c:76
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:215
#define read(a, b, c)
Definition: win32.h:13
bool pgwin32_is_junction(const char *path)
static void progress_report(int tablespacenum, const char *filename, bool force, bool finished)
char * slurpFile(const char *datadir, const char *path, size_t *filesize)
Definition: file_ops.c:314
#define ftruncate(a, b)
Definition: win32_port.h:65