PostgreSQL Source Code  git master
file.c
Go to the documentation of this file.
1 /*
2  * file.c
3  *
4  * file system operations
5  *
6  * Copyright (c) 2010-2018, PostgreSQL Global Development Group
7  * src/bin/pg_upgrade/file.c
8  */
9 
10 #include "postgres_fe.h"
11 
12 #include "access/visibilitymap.h"
13 #include "common/file_perm.h"
14 #include "pg_upgrade.h"
15 #include "storage/bufpage.h"
16 #include "storage/checksum.h"
17 #include "storage/checksum_impl.h"
18 
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 
22 
23 #ifdef WIN32
24 static int win32_pghardlink(const char *src, const char *dst);
25 #endif
26 
27 
28 /*
29  * copyFile()
30  *
31  * Copies a relation file from src to dst.
32  * schemaName/relName are relation's SQL name (used for error messages only).
33  */
34 void
35 copyFile(const char *src, const char *dst,
36  const char *schemaName, const char *relName)
37 {
38 #ifndef WIN32
39  int src_fd;
40  int dest_fd;
41  char *buffer;
42 
43  if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
44  pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s\n",
45  schemaName, relName, src, strerror(errno));
46 
47  if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
48  pg_file_create_mode)) < 0)
49  pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s\n",
50  schemaName, relName, dst, strerror(errno));
51 
52  /* copy in fairly large chunks for best efficiency */
53 #define COPY_BUF_SIZE (50 * BLCKSZ)
54 
55  buffer = (char *) pg_malloc(COPY_BUF_SIZE);
56 
57  /* perform data copying i.e read src source, write to destination */
58  while (true)
59  {
60  ssize_t nbytes = read(src_fd, buffer, COPY_BUF_SIZE);
61 
62  if (nbytes < 0)
63  pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s\n",
64  schemaName, relName, src, strerror(errno));
65 
66  if (nbytes == 0)
67  break;
68 
69  errno = 0;
70  if (write(dest_fd, buffer, nbytes) != nbytes)
71  {
72  /* if write didn't set errno, assume problem is no disk space */
73  if (errno == 0)
74  errno = ENOSPC;
75  pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s\n",
76  schemaName, relName, dst, strerror(errno));
77  }
78  }
79 
80  pg_free(buffer);
81  close(src_fd);
82  close(dest_fd);
83 
84 #else /* WIN32 */
85 
86  if (CopyFile(src, dst, true) == 0)
87  {
88  _dosmaperr(GetLastError());
89  pg_fatal("error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
90  schemaName, relName, src, dst, strerror(errno));
91  }
92 
93 #endif /* WIN32 */
94 }
95 
96 
97 /*
98  * linkFile()
99  *
100  * Hard-links a relation file from src to dst.
101  * schemaName/relName are relation's SQL name (used for error messages only).
102  */
103 void
104 linkFile(const char *src, const char *dst,
105  const char *schemaName, const char *relName)
106 {
107  if (pg_link_file(src, dst) < 0)
108  pg_fatal("error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
109  schemaName, relName, src, dst, strerror(errno));
110 }
111 
112 
113 /*
114  * rewriteVisibilityMap()
115  *
116  * Transform a visibility map file, copying from src to dst.
117  * schemaName/relName are relation's SQL name (used for error messages only).
118  *
119  * In versions of PostgreSQL prior to catversion 201603011, PostgreSQL's
120  * visibility map included one bit per heap page; it now includes two.
121  * When upgrading a cluster from before that time to a current PostgreSQL
122  * version, we could refuse to copy visibility maps from the old cluster
123  * to the new cluster; the next VACUUM would recreate them, but at the
124  * price of scanning the entire table. So, instead, we rewrite the old
125  * visibility maps in the new format. That way, the all-visible bits
126  * remain set for the pages for which they were set previously. The
127  * all-frozen bits are never set by this conversion; we leave that to VACUUM.
128  */
129 void
130 rewriteVisibilityMap(const char *fromfile, const char *tofile,
131  const char *schemaName, const char *relName)
132 {
133  int src_fd;
134  int dst_fd;
135  char *buffer;
136  char *new_vmbuf;
137  ssize_t totalBytesRead = 0;
138  ssize_t src_filesize;
139  int rewriteVmBytesPerPage;
140  BlockNumber new_blkno = 0;
141  struct stat statbuf;
142 
143  /* Compute number of old-format bytes per new page */
144  rewriteVmBytesPerPage = (BLCKSZ - SizeOfPageHeaderData) / 2;
145 
146  if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0)
147  pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s\n",
148  schemaName, relName, fromfile, strerror(errno));
149 
150  if (fstat(src_fd, &statbuf) != 0)
151  pg_fatal("error while copying relation \"%s.%s\": could not stat file \"%s\": %s\n",
152  schemaName, relName, fromfile, strerror(errno));
153 
154  if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
155  pg_file_create_mode)) < 0)
156  pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s\n",
157  schemaName, relName, tofile, strerror(errno));
158 
159  /* Save old file size */
160  src_filesize = statbuf.st_size;
161 
162  /*
163  * Malloc the work buffers, rather than making them local arrays, to
164  * ensure adequate alignment.
165  */
166  buffer = (char *) pg_malloc(BLCKSZ);
167  new_vmbuf = (char *) pg_malloc(BLCKSZ);
168 
169  /*
170  * Turn each visibility map page into 2 pages one by one. Each new page
171  * has the same page header as the old one. If the last section of the
172  * last page is empty, we skip it, mostly to avoid turning one-page
173  * visibility maps for small relations into two pages needlessly.
174  */
175  while (totalBytesRead < src_filesize)
176  {
177  ssize_t bytesRead;
178  char *old_cur;
179  char *old_break;
180  char *old_blkend;
181  PageHeaderData pageheader;
182  bool old_lastblk;
183 
184  if ((bytesRead = read(src_fd, buffer, BLCKSZ)) != BLCKSZ)
185  {
186  if (bytesRead < 0)
187  pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s\n",
188  schemaName, relName, fromfile, strerror(errno));
189  else
190  pg_fatal("error while copying relation \"%s.%s\": partial page found in file \"%s\"\n",
191  schemaName, relName, fromfile);
192  }
193 
194  totalBytesRead += BLCKSZ;
195  old_lastblk = (totalBytesRead == src_filesize);
196 
197  /* Save the page header data */
198  memcpy(&pageheader, buffer, SizeOfPageHeaderData);
199 
200  /*
201  * These old_* variables point to old visibility map page. old_cur
202  * points to current position on old page. old_blkend points to end of
203  * old block. old_break is the end+1 position on the old page for the
204  * data that will be transferred to the current new page.
205  */
206  old_cur = buffer + SizeOfPageHeaderData;
207  old_blkend = buffer + bytesRead;
208  old_break = old_cur + rewriteVmBytesPerPage;
209 
210  while (old_break <= old_blkend)
211  {
212  char *new_cur;
213  bool empty = true;
214  bool old_lastpart;
215 
216  /* First, copy old page header to new page */
217  memcpy(new_vmbuf, &pageheader, SizeOfPageHeaderData);
218 
219  /* Rewriting the last part of the last old page? */
220  old_lastpart = old_lastblk && (old_break == old_blkend);
221 
222  new_cur = new_vmbuf + SizeOfPageHeaderData;
223 
224  /* Process old page bytes one by one, and turn it into new page. */
225  while (old_cur < old_break)
226  {
227  uint8 byte = *(uint8 *) old_cur;
228  uint16 new_vmbits = 0;
229  int i;
230 
231  /* Generate new format bits while keeping old information */
232  for (i = 0; i < BITS_PER_BYTE; i++)
233  {
234  if (byte & (1 << i))
235  {
236  empty = false;
237  new_vmbits |=
239  }
240  }
241 
242  /* Copy new visibility map bytes to new-format page */
243  new_cur[0] = (char) (new_vmbits & 0xFF);
244  new_cur[1] = (char) (new_vmbits >> 8);
245 
246  old_cur++;
247  new_cur += BITS_PER_HEAPBLOCK;
248  }
249 
250  /* If the last part of the last page is empty, skip writing it */
251  if (old_lastpart && empty)
252  break;
253 
254  /* Set new checksum for visibility map page, if enabled */
256  ((PageHeader) new_vmbuf)->pd_checksum =
257  pg_checksum_page(new_vmbuf, new_blkno);
258 
259  errno = 0;
260  if (write(dst_fd, new_vmbuf, BLCKSZ) != BLCKSZ)
261  {
262  /* if write didn't set errno, assume problem is no disk space */
263  if (errno == 0)
264  errno = ENOSPC;
265  pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s\n",
266  schemaName, relName, tofile, strerror(errno));
267  }
268 
269  /* Advance for next new page */
270  old_break += rewriteVmBytesPerPage;
271  new_blkno++;
272  }
273  }
274 
275  /* Clean up */
276  pg_free(buffer);
277  pg_free(new_vmbuf);
278  close(dst_fd);
279  close(src_fd);
280 }
281 
282 void
284 {
285  char existing_file[MAXPGPATH];
286  char new_link_file[MAXPGPATH];
287 
288  snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
289  snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.linktest", new_cluster.pgdata);
290  unlink(new_link_file); /* might fail */
291 
292  if (pg_link_file(existing_file, new_link_file) < 0)
293  pg_fatal("could not create hard link between old and new data directories: %s\n"
294  "In link mode the old and new data directories must be on the same file system.\n",
295  strerror(errno));
296 
297  unlink(new_link_file);
298 }
299 
300 #ifdef WIN32
301 /* implementation of pg_link_file() on Windows */
302 static int
303 win32_pghardlink(const char *src, const char *dst)
304 {
305  /*
306  * CreateHardLinkA returns zero for failure
307  * http://msdn.microsoft.com/en-us/library/aa363860(VS.85).aspx
308  */
309  if (CreateHardLinkA(dst, src, NULL) == 0)
310  {
311  _dosmaperr(GetLastError());
312  return -1;
313  }
314  else
315  return 0;
316 }
317 #endif
int pg_file_create_mode
Definition: file_perm.c:19
void check_hard_link(void)
Definition: file.c:283
#define pg_link_file
Definition: pg_upgrade.h:74
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
ControlData controldata
Definition: pg_upgrade.h:264
#define BITS_PER_BYTE
#define write(a, b, c)
Definition: win32.h:14
void rewriteVisibilityMap(const char *fromfile, const char *tofile, const char *schemaName, const char *relName)
Definition: file.c:130
unsigned char uint8
Definition: c.h:323
void _dosmaperr(unsigned long)
Definition: win32error.c:171
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
uint32 BlockNumber
Definition: block.h:31
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
#define SizeOfPageHeaderData
Definition: bufpage.h:212
#define PG_BINARY
Definition: c.h:1080
void copyFile(const char *src, const char *dst, const char *schemaName, const char *relName)
Definition: file.c:35
ClusterInfo new_cluster
Definition: pg_upgrade.c:57
#define COPY_BUF_SIZE
unsigned short uint16
Definition: c.h:324
#define MAXPGPATH
bool data_checksum_version
Definition: pg_upgrade.h:229
void linkFile(const char *src, const char *dst, const char *schemaName, const char *relName)
Definition: file.c:104
ClusterInfo old_cluster
Definition: pg_upgrade.c:57
#define BITS_PER_HEAPBLOCK
Definition: visibilitymap.h:23
#define byte(x, n)
Definition: rijndael.c:68
#define stat(a, b)
Definition: win32_port.h:266
PageHeaderData * PageHeader
Definition: bufpage.h:162
WalTimeSample buffer[LAG_TRACKER_BUFFER_SIZE]
Definition: walsender.c:215
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define VISIBILITYMAP_ALL_VISIBLE
Definition: visibilitymap.h:26
char * pgdata
Definition: pg_upgrade.h:266
int i
const char * strerror(int errnum)
Definition: strerror.c:19
#define close(a)
Definition: win32.h:12
uint16 pg_checksum_page(char *page, BlockNumber blkno)
#define read(a, b, c)
Definition: win32.h:13