PostgreSQL Source Code  git master
copy_fetch.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy_fetch.c
4  * Functions for using a data directory as the source.
5  *
6  * Portions Copyright (c) 2013-2018, PostgreSQL Global Development Group
7  *
8  *-------------------------------------------------------------------------
9  */
10 #include "postgres_fe.h"
11 
12 #include <sys/stat.h>
13 #include <dirent.h>
14 #include <fcntl.h>
15 #include <unistd.h>
16 
17 #include "datapagemap.h"
18 #include "fetch.h"
19 #include "file_ops.h"
20 #include "filemap.h"
21 #include "logging.h"
22 #include "pg_rewind.h"
23 
24 static void recurse_dir(const char *datadir, const char *path,
26 
27 static void execute_pagemap(datapagemap_t *pagemap, const char *path);
28 
29 /*
30  * Traverse through all files in a data directory, calling 'callback'
31  * for each file.
32  */
33 void
35 {
36  recurse_dir(datadir, NULL, callback);
37 }
38 
39 /*
40  * recursive part of traverse_datadir
41  *
42  * parentpath is the current subdirectory's path relative to datadir,
43  * or NULL at the top level.
44  */
45 static void
46 recurse_dir(const char *datadir, const char *parentpath,
48 {
49  DIR *xldir;
50  struct dirent *xlde;
51  char fullparentpath[MAXPGPATH];
52 
53  if (parentpath)
54  snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
55  else
56  snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
57 
58  xldir = opendir(fullparentpath);
59  if (xldir == NULL)
60  pg_fatal("could not open directory \"%s\": %s\n",
61  fullparentpath, strerror(errno));
62 
63  while (errno = 0, (xlde = readdir(xldir)) != NULL)
64  {
65  struct stat fst;
66  char fullpath[MAXPGPATH * 2];
67  char path[MAXPGPATH * 2];
68 
69  if (strcmp(xlde->d_name, ".") == 0 ||
70  strcmp(xlde->d_name, "..") == 0)
71  continue;
72 
73  snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
74 
75  if (lstat(fullpath, &fst) < 0)
76  {
77  if (errno == ENOENT)
78  {
79  /*
80  * File doesn't exist anymore. This is ok, if the new master
81  * is running and the file was just removed. If it was a data
82  * file, there should be a WAL record of the removal. If it
83  * was something else, it couldn't have been anyway.
84  *
85  * TODO: But complain if we're processing the target dir!
86  */
87  }
88  else
89  pg_fatal("could not stat file \"%s\": %s\n",
90  fullpath, strerror(errno));
91  }
92 
93  if (parentpath)
94  snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
95  else
96  snprintf(path, sizeof(path), "%s", xlde->d_name);
97 
98  if (S_ISREG(fst.st_mode))
99  callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
100  else if (S_ISDIR(fst.st_mode))
101  {
102  callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
103  /* recurse to handle subdirectories */
104  recurse_dir(datadir, path, callback);
105  }
106 #ifndef WIN32
107  else if (S_ISLNK(fst.st_mode))
108 #else
109  else if (pgwin32_is_junction(fullpath))
110 #endif
111  {
112 #if defined(HAVE_READLINK) || defined(WIN32)
113  char link_target[MAXPGPATH];
114  int len;
115 
116  len = readlink(fullpath, link_target, sizeof(link_target));
117  if (len < 0)
118  pg_fatal("could not read symbolic link \"%s\": %s\n",
119  fullpath, strerror(errno));
120  if (len >= sizeof(link_target))
121  pg_fatal("symbolic link \"%s\" target is too long\n",
122  fullpath);
123  link_target[len] = '\0';
124 
125  callback(path, FILE_TYPE_SYMLINK, 0, link_target);
126 
127  /*
128  * If it's a symlink within pg_tblspc, we need to recurse into it,
129  * to process all the tablespaces. We also follow a symlink if
130  * it's for pg_wal. Symlinks elsewhere are ignored.
131  */
132  if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
133  strcmp(path, "pg_wal") == 0)
134  recurse_dir(datadir, path, callback);
135 #else
136  pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
137  fullpath);
138 #endif /* HAVE_READLINK */
139  }
140  }
141 
142  if (errno)
143  pg_fatal("could not read directory \"%s\": %s\n",
144  fullparentpath, strerror(errno));
145 
146  if (closedir(xldir))
147  pg_fatal("could not close directory \"%s\": %s\n",
148  fullparentpath, strerror(errno));
149 }
150 
151 /*
152  * Copy a file from source to target, between 'begin' and 'end' offsets.
153  *
154  * If 'trunc' is true, any existing file with the same name is truncated.
155  */
156 static void
157 rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
158 {
159  char buf[BLCKSZ];
160  char srcpath[MAXPGPATH];
161  int srcfd;
162 
163  snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
164 
165  srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
166  if (srcfd < 0)
167  pg_fatal("could not open source file \"%s\": %s\n",
168  srcpath, strerror(errno));
169 
170  if (lseek(srcfd, begin, SEEK_SET) == -1)
171  pg_fatal("could not seek in source file: %s\n", strerror(errno));
172 
173  open_target_file(path, trunc);
174 
175  while (end - begin > 0)
176  {
177  int readlen;
178  int len;
179 
180  if (end - begin > sizeof(buf))
181  len = sizeof(buf);
182  else
183  len = end - begin;
184 
185  readlen = read(srcfd, buf, len);
186 
187  if (readlen < 0)
188  pg_fatal("could not read file \"%s\": %s\n",
189  srcpath, strerror(errno));
190  else if (readlen == 0)
191  pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath);
192 
193  write_target_range(buf, begin, readlen);
194  begin += readlen;
195  }
196 
197  if (close(srcfd) != 0)
198  pg_fatal("could not close file \"%s\": %s\n", srcpath, strerror(errno));
199 }
200 
201 /*
202  * Copy all relation data files from datadir_source to datadir_target, which
203  * are marked in the given data page map.
204  */
205 void
207 {
208  file_entry_t *entry;
209  int i;
210 
211  for (i = 0; i < map->narray; i++)
212  {
213  entry = map->array[i];
214  execute_pagemap(&entry->pagemap, entry->path);
215 
216  switch (entry->action)
217  {
218  case FILE_ACTION_NONE:
219  /* ok, do nothing.. */
220  break;
221 
222  case FILE_ACTION_COPY:
223  rewind_copy_file_range(entry->path, 0, entry->newsize, true);
224  break;
225 
227  truncate_target_file(entry->path, entry->newsize);
228  break;
229 
231  rewind_copy_file_range(entry->path, entry->oldsize,
232  entry->newsize, false);
233  break;
234 
235  case FILE_ACTION_CREATE:
236  create_target(entry);
237  break;
238 
239  case FILE_ACTION_REMOVE:
240  remove_target(entry);
241  break;
242  }
243  }
244 
246 }
247 
248 static void
249 execute_pagemap(datapagemap_t *pagemap, const char *path)
250 {
252  BlockNumber blkno;
253  off_t offset;
254 
255  iter = datapagemap_iterate(pagemap);
256  while (datapagemap_next(iter, &blkno))
257  {
258  offset = blkno * BLCKSZ;
259  rewind_copy_file_range(path, offset, offset + BLCKSZ, false);
260  /* Ok, this block has now been copied from new data dir to old */
261  }
262  pg_free(iter);
263 }
static void rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
Definition: copy_fetch.c:157
void open_target_file(const char *path, bool trunc)
Definition: file_ops.c:43
void(* process_file_callback_t)(const char *path, file_type_t type, size_t size, const char *link_target)
Definition: fetch.h:41
void write_target_range(char *buf, off_t begin, size_t size)
Definition: file_ops.c:84
file_entry_t ** array
Definition: filemap.h:79
static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback)
Definition: copy_fetch.c:46
size_t newsize
Definition: filemap.h:51
int closedir(DIR *)
Definition: dirent.c:111
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
uint32 BlockNumber
Definition: block.h:31
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
Definition: dirent.h:9
datapagemap_t pagemap
Definition: filemap.h:54
#define PG_BINARY
Definition: c.h:1080
int narray
Definition: filemap.h:80
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:192
Definition: dirent.c:25
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
#define MAXPGPATH
static void execute_pagemap(datapagemap_t *pagemap, const char *path)
Definition: copy_fetch.c:249
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:48
DIR * opendir(const char *)
Definition: dirent.c:33
file_action_t action
Definition: filemap.h:47
static char * buf
Definition: pg_test_fsync.c:67
#define readlink(path, buf, size)
Definition: win32_port.h:233
size_t oldsize
Definition: filemap.h:50
char * datadir
void copy_executeFileMap(filemap_t *map)
Definition: copy_fetch.c:206
void traverse_datadir(const char *datadir, process_file_callback_t callback)
Definition: copy_fetch.c:34
#define S_ISREG(m)
Definition: win32_port.h:310
#define stat(a, b)
Definition: win32_port.h:266
char * datadir_source
Definition: pg_rewind.c:52
void remove_target(file_entry_t *entry)
Definition: file_ops.c:126
struct dirent * readdir(DIR *)
Definition: dirent.c:77
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
char * path
Definition: filemap.h:44
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define S_ISDIR(m)
Definition: win32_port.h:307
#define lstat(path, sb)
Definition: win32_port.h:255
void close_target_file(void)
Definition: file_ops.c:71
int i
const char * strerror(int errnum)
Definition: strerror.c:19
Definition: filemap.h:42
void create_target(file_entry_t *entry)
Definition: file_ops.c:147
char d_name[MAX_PATH]
Definition: dirent.h:14
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
bool pgwin32_is_junction(const char *path)