PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy_fetch.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy_fetch.c
4  * Functions for using a data directory as the source.
5  *
6  * Portions Copyright (c) 2013-2017, PostgreSQL Global Development Group
7  *
8  *-------------------------------------------------------------------------
9  */
10 #include "postgres_fe.h"
11 
12 #include <sys/stat.h>
13 #include <dirent.h>
14 #include <fcntl.h>
15 #include <unistd.h>
16 
17 #include "datapagemap.h"
18 #include "fetch.h"
19 #include "file_ops.h"
20 #include "filemap.h"
21 #include "logging.h"
22 #include "pg_rewind.h"
23 
24 #include "catalog/catalog.h"
25 
26 static void recurse_dir(const char *datadir, const char *path,
28 
29 static void execute_pagemap(datapagemap_t *pagemap, const char *path);
30 
31 /*
32  * Traverse through all files in a data directory, calling 'callback'
33  * for each file.
34  */
35 void
37 {
38  recurse_dir(datadir, NULL, callback);
39 }
40 
41 /*
42  * recursive part of traverse_datadir
43  *
44  * parentpath is the current subdirectory's path relative to datadir,
45  * or NULL at the top level.
46  */
47 static void
48 recurse_dir(const char *datadir, const char *parentpath,
50 {
51  DIR *xldir;
52  struct dirent *xlde;
53  char fullparentpath[MAXPGPATH];
54 
55  if (parentpath)
56  snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
57  else
58  snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
59 
60  xldir = opendir(fullparentpath);
61  if (xldir == NULL)
62  pg_fatal("could not open directory \"%s\": %s\n",
63  fullparentpath, strerror(errno));
64 
65  while (errno = 0, (xlde = readdir(xldir)) != NULL)
66  {
67  struct stat fst;
68  char fullpath[MAXPGPATH * 2];
69  char path[MAXPGPATH * 2];
70 
71  if (strcmp(xlde->d_name, ".") == 0 ||
72  strcmp(xlde->d_name, "..") == 0)
73  continue;
74 
75  snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
76 
77  if (lstat(fullpath, &fst) < 0)
78  {
79  if (errno == ENOENT)
80  {
81  /*
82  * File doesn't exist anymore. This is ok, if the new master
83  * is running and the file was just removed. If it was a data
84  * file, there should be a WAL record of the removal. If it
85  * was something else, it couldn't have been anyway.
86  *
87  * TODO: But complain if we're processing the target dir!
88  */
89  }
90  else
91  pg_fatal("could not stat file \"%s\": %s\n",
92  fullpath, strerror(errno));
93  }
94 
95  if (parentpath)
96  snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
97  else
98  snprintf(path, sizeof(path), "%s", xlde->d_name);
99 
100  if (S_ISREG(fst.st_mode))
101  callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
102  else if (S_ISDIR(fst.st_mode))
103  {
104  callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
105  /* recurse to handle subdirectories */
106  recurse_dir(datadir, path, callback);
107  }
108 #ifndef WIN32
109  else if (S_ISLNK(fst.st_mode))
110 #else
111  else if (pgwin32_is_junction(fullpath))
112 #endif
113  {
114 #if defined(HAVE_READLINK) || defined(WIN32)
115  char link_target[MAXPGPATH];
116  int len;
117 
118  len = readlink(fullpath, link_target, sizeof(link_target));
119  if (len < 0)
120  pg_fatal("could not read symbolic link \"%s\": %s\n",
121  fullpath, strerror(errno));
122  if (len >= sizeof(link_target))
123  pg_fatal("symbolic link \"%s\" target is too long\n",
124  fullpath);
125  link_target[len] = '\0';
126 
127  callback(path, FILE_TYPE_SYMLINK, 0, link_target);
128 
129  /*
130  * If it's a symlink within pg_tblspc, we need to recurse into it,
131  * to process all the tablespaces. We also follow a symlink if
132  * it's for pg_wal. Symlinks elsewhere are ignored.
133  */
134  if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
135  strcmp(path, "pg_wal") == 0)
136  recurse_dir(datadir, path, callback);
137 #else
138  pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
139  fullpath);
140 #endif /* HAVE_READLINK */
141  }
142  }
143 
144  if (errno)
145  pg_fatal("could not read directory \"%s\": %s\n",
146  fullparentpath, strerror(errno));
147 
148  if (closedir(xldir))
149  pg_fatal("could not close directory \"%s\": %s\n",
150  fullparentpath, strerror(errno));
151 }
152 
153 /*
154  * Copy a file from source to target, between 'begin' and 'end' offsets.
155  *
156  * If 'trunc' is true, any existing file with the same name is truncated.
157  */
158 static void
159 copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
160 {
161  char buf[BLCKSZ];
162  char srcpath[MAXPGPATH];
163  int srcfd;
164 
165  snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
166 
167  srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
168  if (srcfd < 0)
169  pg_fatal("could not open source file \"%s\": %s\n",
170  srcpath, strerror(errno));
171 
172  if (lseek(srcfd, begin, SEEK_SET) == -1)
173  pg_fatal("could not seek in source file: %s\n", strerror(errno));
174 
175  open_target_file(path, trunc);
176 
177  while (end - begin > 0)
178  {
179  int readlen;
180  int len;
181 
182  if (end - begin > sizeof(buf))
183  len = sizeof(buf);
184  else
185  len = end - begin;
186 
187  readlen = read(srcfd, buf, len);
188 
189  if (readlen < 0)
190  pg_fatal("could not read file \"%s\": %s\n",
191  srcpath, strerror(errno));
192  else if (readlen == 0)
193  pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath);
194 
195  write_target_range(buf, begin, readlen);
196  begin += readlen;
197  }
198 
199  if (close(srcfd) != 0)
200  pg_fatal("could not close file \"%s\": %s\n", srcpath, strerror(errno));
201 }
202 
203 /*
204  * Copy all relation data files from datadir_source to datadir_target, which
205  * are marked in the given data page map.
206  */
207 void
209 {
210  file_entry_t *entry;
211  int i;
212 
213  for (i = 0; i < map->narray; i++)
214  {
215  entry = map->array[i];
216  execute_pagemap(&entry->pagemap, entry->path);
217 
218  switch (entry->action)
219  {
220  case FILE_ACTION_NONE:
221  /* ok, do nothing.. */
222  break;
223 
224  case FILE_ACTION_COPY:
225  copy_file_range(entry->path, 0, entry->newsize, true);
226  break;
227 
229  truncate_target_file(entry->path, entry->newsize);
230  break;
231 
233  copy_file_range(entry->path, entry->oldsize, entry->newsize, false);
234  break;
235 
236  case FILE_ACTION_CREATE:
237  create_target(entry);
238  break;
239 
240  case FILE_ACTION_REMOVE:
241  remove_target(entry);
242  break;
243  }
244  }
245 
247 }
248 
249 static void
250 execute_pagemap(datapagemap_t *pagemap, const char *path)
251 {
253  BlockNumber blkno;
254  off_t offset;
255 
256  iter = datapagemap_iterate(pagemap);
257  while (datapagemap_next(iter, &blkno))
258  {
259  offset = blkno * BLCKSZ;
260  copy_file_range(path, offset, offset + BLCKSZ, false);
261  /* Ok, this block has now been copied from new data dir to old */
262  }
263  pg_free(iter);
264 }
void open_target_file(const char *path, bool trunc)
Definition: file_ops.c:43
void write_target_range(char *buf, off_t begin, size_t size)
Definition: file_ops.c:84
file_entry_t ** array
Definition: filemap.h:79
static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback)
Definition: copy_fetch.c:48
size_t newsize
Definition: filemap.h:51
int closedir(DIR *)
Definition: dirent.c:113
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
uint32 BlockNumber
Definition: block.h:31
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
void(* process_file_callback_t)(const char *path, file_type_t type, size_t size, const char *link_target)
Definition: fetch.h:41
Definition: dirent.h:9
datapagemap_t pagemap
Definition: filemap.h:54
#define PG_BINARY
Definition: c.h:1038
int narray
Definition: filemap.h:80
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:183
Definition: dirent.c:25
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
#define MAXPGPATH
static void execute_pagemap(datapagemap_t *pagemap, const char *path)
Definition: copy_fetch.c:250
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:48
DIR * opendir(const char *)
Definition: dirent.c:33
file_action_t action
Definition: filemap.h:47
static char * buf
Definition: pg_test_fsync.c:66
size_t oldsize
Definition: filemap.h:50
char * datadir
void copy_executeFileMap(filemap_t *map)
Definition: copy_fetch.c:208
void traverse_datadir(const char *datadir, process_file_callback_t callback)
Definition: copy_fetch.c:36
char * datadir_source
Definition: pg_rewind.c:50
#define NULL
Definition: c.h:229
void remove_target(file_entry_t *entry)
Definition: file_ops.c:126
struct dirent * readdir(DIR *)
Definition: dirent.c:78
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
char * path
Definition: filemap.h:44
void pg_free(void *ptr)
Definition: fe_memutils.c:105
void close_target_file(void)
Definition: file_ops.c:71
int i
const char * strerror(int errnum)
Definition: strerror.c:19
Definition: filemap.h:42
void create_target(file_entry_t *entry)
Definition: file_ops.c:147
char d_name[MAX_PATH]
Definition: dirent.h:14
#define close(a)
Definition: win32.h:12
static void copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
Definition: copy_fetch.c:159
#define lstat(path, sb)
Definition: win32.h:262
#define read(a, b, c)
Definition: win32.h:13