PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy_fetch.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy_fetch.c
4  * Functions for using a data directory as the source.
5  *
6  * Portions Copyright (c) 2013-2017, PostgreSQL Global Development Group
7  *
8  *-------------------------------------------------------------------------
9  */
10 #include "postgres_fe.h"
11 
12 #include <sys/types.h>
13 #include <sys/stat.h>
14 #include <dirent.h>
15 #include <fcntl.h>
16 #include <unistd.h>
17 #include <string.h>
18 
19 #include "datapagemap.h"
20 #include "fetch.h"
21 #include "file_ops.h"
22 #include "filemap.h"
23 #include "logging.h"
24 #include "pg_rewind.h"
25 
26 #include "catalog/catalog.h"
27 
28 static void recurse_dir(const char *datadir, const char *path,
30 
31 static void execute_pagemap(datapagemap_t *pagemap, const char *path);
32 
33 /*
34  * Traverse through all files in a data directory, calling 'callback'
35  * for each file.
36  */
37 void
39 {
40  recurse_dir(datadir, NULL, callback);
41 }
42 
43 /*
44  * recursive part of traverse_datadir
45  *
46  * parentpath is the current subdirectory's path relative to datadir,
47  * or NULL at the top level.
48  */
49 static void
50 recurse_dir(const char *datadir, const char *parentpath,
52 {
53  DIR *xldir;
54  struct dirent *xlde;
55  char fullparentpath[MAXPGPATH];
56 
57  if (parentpath)
58  snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
59  else
60  snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
61 
62  xldir = opendir(fullparentpath);
63  if (xldir == NULL)
64  pg_fatal("could not open directory \"%s\": %s\n",
65  fullparentpath, strerror(errno));
66 
67  while (errno = 0, (xlde = readdir(xldir)) != NULL)
68  {
69  struct stat fst;
70  char fullpath[MAXPGPATH];
71  char path[MAXPGPATH];
72 
73  if (strcmp(xlde->d_name, ".") == 0 ||
74  strcmp(xlde->d_name, "..") == 0)
75  continue;
76 
77  snprintf(fullpath, MAXPGPATH, "%s/%s", fullparentpath, xlde->d_name);
78 
79  if (lstat(fullpath, &fst) < 0)
80  {
81  if (errno == ENOENT)
82  {
83  /*
84  * File doesn't exist anymore. This is ok, if the new master
85  * is running and the file was just removed. If it was a data
86  * file, there should be a WAL record of the removal. If it
87  * was something else, it couldn't have been anyway.
88  *
89  * TODO: But complain if we're processing the target dir!
90  */
91  }
92  else
93  pg_fatal("could not stat file \"%s\": %s\n",
94  fullpath, strerror(errno));
95  }
96 
97  if (parentpath)
98  snprintf(path, MAXPGPATH, "%s/%s", parentpath, xlde->d_name);
99  else
100  snprintf(path, MAXPGPATH, "%s", xlde->d_name);
101 
102  if (S_ISREG(fst.st_mode))
103  callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
104  else if (S_ISDIR(fst.st_mode))
105  {
106  callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
107  /* recurse to handle subdirectories */
108  recurse_dir(datadir, path, callback);
109  }
110 #ifndef WIN32
111  else if (S_ISLNK(fst.st_mode))
112 #else
113  else if (pgwin32_is_junction(fullpath))
114 #endif
115  {
116 #if defined(HAVE_READLINK) || defined(WIN32)
117  char link_target[MAXPGPATH];
118  int len;
119 
120  len = readlink(fullpath, link_target, sizeof(link_target));
121  if (len < 0)
122  pg_fatal("could not read symbolic link \"%s\": %s\n",
123  fullpath, strerror(errno));
124  if (len >= sizeof(link_target))
125  pg_fatal("symbolic link \"%s\" target is too long\n",
126  fullpath);
127  link_target[len] = '\0';
128 
129  callback(path, FILE_TYPE_SYMLINK, 0, link_target);
130 
131  /*
132  * If it's a symlink within pg_tblspc, we need to recurse into it,
133  * to process all the tablespaces. We also follow a symlink if
134  * it's for pg_wal. Symlinks elsewhere are ignored.
135  */
136  if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
137  strcmp(path, "pg_wal") == 0)
138  recurse_dir(datadir, path, callback);
139 #else
140  pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
141  fullpath);
142 #endif /* HAVE_READLINK */
143  }
144  }
145 
146  if (errno)
147  pg_fatal("could not read directory \"%s\": %s\n",
148  fullparentpath, strerror(errno));
149 
150  if (closedir(xldir))
151  pg_fatal("could not close directory \"%s\": %s\n",
152  fullparentpath, strerror(errno));
153 }
154 
155 /*
156  * Copy a file from source to target, between 'begin' and 'end' offsets.
157  *
158  * If 'trunc' is true, any existing file with the same name is truncated.
159  */
160 static void
161 copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
162 {
163  char buf[BLCKSZ];
164  char srcpath[MAXPGPATH];
165  int srcfd;
166 
167  snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
168 
169  srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
170  if (srcfd < 0)
171  pg_fatal("could not open source file \"%s\": %s\n",
172  srcpath, strerror(errno));
173 
174  if (lseek(srcfd, begin, SEEK_SET) == -1)
175  pg_fatal("could not seek in source file: %s\n", strerror(errno));
176 
177  open_target_file(path, trunc);
178 
179  while (end - begin > 0)
180  {
181  int readlen;
182  int len;
183 
184  if (end - begin > sizeof(buf))
185  len = sizeof(buf);
186  else
187  len = end - begin;
188 
189  readlen = read(srcfd, buf, len);
190 
191  if (readlen < 0)
192  pg_fatal("could not read file \"%s\": %s\n",
193  srcpath, strerror(errno));
194  else if (readlen == 0)
195  pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath);
196 
197  write_target_range(buf, begin, readlen);
198  begin += readlen;
199  }
200 
201  if (close(srcfd) != 0)
202  pg_fatal("could not close file \"%s\": %s\n", srcpath, strerror(errno));
203 }
204 
205 /*
206  * Copy all relation data files from datadir_source to datadir_target, which
207  * are marked in the given data page map.
208  */
209 void
211 {
212  file_entry_t *entry;
213  int i;
214 
215  for (i = 0; i < map->narray; i++)
216  {
217  entry = map->array[i];
218  execute_pagemap(&entry->pagemap, entry->path);
219 
220  switch (entry->action)
221  {
222  case FILE_ACTION_NONE:
223  /* ok, do nothing.. */
224  break;
225 
226  case FILE_ACTION_COPY:
227  copy_file_range(entry->path, 0, entry->newsize, true);
228  break;
229 
231  truncate_target_file(entry->path, entry->newsize);
232  break;
233 
235  copy_file_range(entry->path, entry->oldsize, entry->newsize, false);
236  break;
237 
238  case FILE_ACTION_CREATE:
239  create_target(entry);
240  break;
241 
242  case FILE_ACTION_REMOVE:
243  remove_target(entry);
244  break;
245  }
246  }
247 
249 }
250 
251 static void
252 execute_pagemap(datapagemap_t *pagemap, const char *path)
253 {
255  BlockNumber blkno;
256  off_t offset;
257 
258  iter = datapagemap_iterate(pagemap);
259  while (datapagemap_next(iter, &blkno))
260  {
261  offset = blkno * BLCKSZ;
262  copy_file_range(path, offset, offset + BLCKSZ, false);
263  /* Ok, this block has now been copied from new data dir to old */
264  }
265  pg_free(iter);
266 }
void open_target_file(const char *path, bool trunc)
Definition: file_ops.c:44
void write_target_range(char *buf, off_t begin, size_t size)
Definition: file_ops.c:85
file_entry_t ** array
Definition: filemap.h:79
static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback)
Definition: copy_fetch.c:50
size_t newsize
Definition: filemap.h:51
int closedir(DIR *)
Definition: dirent.c:113
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
uint32 BlockNumber
Definition: block.h:31
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
void(* process_file_callback_t)(const char *path, file_type_t type, size_t size, const char *link_target)
Definition: fetch.h:43
Definition: dirent.h:9
datapagemap_t pagemap
Definition: filemap.h:54
#define PG_BINARY
Definition: c.h:1037
int narray
Definition: filemap.h:80
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:184
Definition: dirent.c:25
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
#define MAXPGPATH
static void execute_pagemap(datapagemap_t *pagemap, const char *path)
Definition: copy_fetch.c:252
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:49
DIR * opendir(const char *)
Definition: dirent.c:33
file_action_t action
Definition: filemap.h:47
static char * buf
Definition: pg_test_fsync.c:65
size_t oldsize
Definition: filemap.h:50
char * datadir
void copy_executeFileMap(filemap_t *map)
Definition: copy_fetch.c:210
void traverse_datadir(const char *datadir, process_file_callback_t callback)
Definition: copy_fetch.c:38
char * datadir_source
Definition: pg_rewind.c:50
#define NULL
Definition: c.h:226
void remove_target(file_entry_t *entry)
Definition: file_ops.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
char * path
Definition: filemap.h:44
void pg_free(void *ptr)
Definition: fe_memutils.c:105
void close_target_file(void)
Definition: file_ops.c:72
int i
const char * strerror(int errnum)
Definition: strerror.c:19
Definition: filemap.h:42
void create_target(file_entry_t *entry)
Definition: file_ops.c:148
char d_name[MAX_PATH]
Definition: dirent.h:14
#define close(a)
Definition: win32.h:17
static void copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
Definition: copy_fetch.c:161
#define lstat(path, sb)
Definition: win32.h:272
#define read(a, b, c)
Definition: win32.h:18