PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
fetch.h File Reference
#include "c.h"
#include "access/xlogdefs.h"
#include "filemap.h"
Include dependency graph for fetch.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Typedefs

typedef void(* process_file_callback_t )(const char *path, file_type_t type, size_t size, const char *link_target)
 

Functions

void fetchSourceFileList (void)
 
char * fetchFile (char *filename, size_t *filesize)
 
void executeFileMap (void)
 
void libpqProcessFileList (void)
 
char * libpqGetFile (const char *filename, size_t *filesize)
 
void libpq_executeFileMap (filemap_t *map)
 
void libpqConnect (const char *connstr)
 
XLogRecPtr libpqGetCurrentXlogInsertLocation (void)
 
void copy_executeFileMap (filemap_t *map)
 
void traverse_datadir (const char *datadir, process_file_callback_t callback)
 

Typedef Documentation

typedef void(* process_file_callback_t)(const char *path, file_type_t type, size_t size, const char *link_target)

Definition at line 43 of file fetch.h.

Function Documentation

void copy_executeFileMap ( filemap_t map)

Definition at line 208 of file copy_fetch.c.

References file_entry_t::action, filemap_t::array, close_target_file(), copy_file_range(), create_target(), execute_pagemap(), FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, FILE_ACTION_TRUNCATE, i, filemap_t::narray, file_entry_t::newsize, file_entry_t::oldsize, file_entry_t::pagemap, file_entry_t::path, remove_target(), and truncate_target_file().

Referenced by executeFileMap().

209 {
210  file_entry_t *entry;
211  int i;
212 
213  for (i = 0; i < map->narray; i++)
214  {
215  entry = map->array[i];
216  execute_pagemap(&entry->pagemap, entry->path);
217 
218  switch (entry->action)
219  {
220  case FILE_ACTION_NONE:
221  /* ok, do nothing.. */
222  break;
223 
224  case FILE_ACTION_COPY:
225  copy_file_range(entry->path, 0, entry->newsize, true);
226  break;
227 
229  truncate_target_file(entry->path, entry->newsize);
230  break;
231 
233  copy_file_range(entry->path, entry->oldsize, entry->newsize, false);
234  break;
235 
236  case FILE_ACTION_CREATE:
237  create_target(entry);
238  break;
239 
240  case FILE_ACTION_REMOVE:
241  remove_target(entry);
242  break;
243  }
244  }
245 
247 }
file_entry_t ** array
Definition: filemap.h:79
size_t newsize
Definition: filemap.h:51
datapagemap_t pagemap
Definition: filemap.h:54
int narray
Definition: filemap.h:80
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:183
static void execute_pagemap(datapagemap_t *pagemap, const char *path)
Definition: copy_fetch.c:250
file_action_t action
Definition: filemap.h:47
size_t oldsize
Definition: filemap.h:50
void remove_target(file_entry_t *entry)
Definition: file_ops.c:126
char * path
Definition: filemap.h:44
void close_target_file(void)
Definition: file_ops.c:71
int i
Definition: filemap.h:42
void create_target(file_entry_t *entry)
Definition: file_ops.c:147
static void copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
Definition: copy_fetch.c:159
void executeFileMap ( void  )

Definition at line 40 of file fetch.c.

References copy_executeFileMap(), datadir_source, filemap, and libpq_executeFileMap().

Referenced by main().

41 {
42  if (datadir_source)
44  else
46 }
filemap_t * filemap
Definition: filemap.c:25
void copy_executeFileMap(filemap_t *map)
Definition: copy_fetch.c:208
char * datadir_source
Definition: pg_rewind.c:50
void libpq_executeFileMap(filemap_t *map)
Definition: libpq_fetch.c:411
char* fetchFile ( char *  filename,
size_t *  filesize 
)

Definition at line 54 of file fetch.c.

References datadir_source, libpqGetFile(), and slurpFile().

Referenced by getTimelineHistory(), and main().

55 {
56  if (datadir_source)
57  return slurpFile(datadir_source, filename, filesize);
58  else
59  return libpqGetFile(filename, filesize);
60 }
char * datadir_source
Definition: pg_rewind.c:50
char * libpqGetFile(const char *filename, size_t *filesize)
Definition: libpq_fetch.c:340
static char * filename
Definition: pg_dumpall.c:80
char * slurpFile(const char *datadir, const char *path, size_t *filesize)
Definition: file_ops.c:276
void fetchSourceFileList ( void  )

Definition at line 28 of file fetch.c.

References datadir_source, libpqProcessFileList(), process_source_file(), and traverse_datadir().

Referenced by main().

29 {
30  if (datadir_source)
32  else
34 }
void traverse_datadir(const char *datadir, process_file_callback_t callback)
Definition: copy_fetch.c:36
char * datadir_source
Definition: pg_rewind.c:50
void libpqProcessFileList(void)
Definition: libpq_fetch.c:149
void process_source_file(const char *path, file_type_t type, size_t newsize, const char *link_target)
Definition: filemap.c:60
void libpq_executeFileMap ( filemap_t map)

Definition at line 411 of file libpq_fetch.c.

References file_entry_t::action, filemap_t::array, create_target(), execute_pagemap(), fetch_file_range(), FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, FILE_ACTION_TRUNCATE, i, filemap_t::narray, file_entry_t::newsize, NULL, file_entry_t::oldsize, open_target_file(), file_entry_t::pagemap, file_entry_t::path, pg_fatal(), PGRES_COMMAND_OK, PGRES_COPY_IN, PQclear(), PQerrorMessage(), PQexec(), PQgetResult(), PQputCopyEnd(), PQresultErrorMessage(), PQresultStatus(), receiveFileChunks(), remove_target(), and truncate_target_file().

Referenced by executeFileMap().

412 {
413  file_entry_t *entry;
414  const char *sql;
415  PGresult *res;
416  int i;
417 
418  /*
419  * First create a temporary table, and load it with the blocks that we
420  * need to fetch.
421  */
422  sql = "CREATE TEMPORARY TABLE fetchchunks(path text, begin int4, len int4);";
423  res = PQexec(conn, sql);
424 
425  if (PQresultStatus(res) != PGRES_COMMAND_OK)
426  pg_fatal("could not create temporary table: %s",
427  PQresultErrorMessage(res));
428  PQclear(res);
429 
430  sql = "COPY fetchchunks FROM STDIN";
431  res = PQexec(conn, sql);
432 
433  if (PQresultStatus(res) != PGRES_COPY_IN)
434  pg_fatal("could not send file list: %s",
435  PQresultErrorMessage(res));
436  PQclear(res);
437 
438  for (i = 0; i < map->narray; i++)
439  {
440  entry = map->array[i];
441 
442  /* If this is a relation file, copy the modified blocks */
443  execute_pagemap(&entry->pagemap, entry->path);
444 
445  switch (entry->action)
446  {
447  case FILE_ACTION_NONE:
448  /* nothing else to do */
449  break;
450 
451  case FILE_ACTION_COPY:
452  /* Truncate the old file out of the way, if any */
453  open_target_file(entry->path, true);
454  fetch_file_range(entry->path, 0, entry->newsize);
455  break;
456 
458  truncate_target_file(entry->path, entry->newsize);
459  break;
460 
462  fetch_file_range(entry->path, entry->oldsize, entry->newsize);
463  break;
464 
465  case FILE_ACTION_REMOVE:
466  remove_target(entry);
467  break;
468 
469  case FILE_ACTION_CREATE:
470  create_target(entry);
471  break;
472  }
473  }
474 
475  if (PQputCopyEnd(conn, NULL) != 1)
476  pg_fatal("could not send end-of-COPY: %s",
478 
479  while ((res = PQgetResult(conn)) != NULL)
480  {
481  if (PQresultStatus(res) != PGRES_COMMAND_OK)
482  pg_fatal("unexpected result while sending file list: %s",
483  PQresultErrorMessage(res));
484  PQclear(res);
485  }
486 
487  /*
488  * We've now copied the list of file ranges that we need to fetch to the
489  * temporary table. Now, actually fetch all of those ranges.
490  */
491  sql =
492  "SELECT path, begin, \n"
493  " pg_read_binary_file(path, begin, len, true) AS chunk\n"
494  "FROM fetchchunks\n";
495 
496  receiveFileChunks(sql);
497 }
static void fetch_file_range(const char *path, unsigned int begin, unsigned int end)
Definition: libpq_fetch.c:383
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:5959
void open_target_file(const char *path, bool trunc)
Definition: file_ops.c:43
file_entry_t ** array
Definition: filemap.h:79
size_t newsize
Definition: filemap.h:51
static void execute_pagemap(datapagemap_t *pagemap, const char *path)
Definition: libpq_fetch.c:500
int PQputCopyEnd(PGconn *conn, const char *errormsg)
Definition: fe-exec.c:2288
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
datapagemap_t pagemap
Definition: filemap.h:54
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
int narray
Definition: filemap.h:80
void truncate_target_file(const char *path, off_t newsize)
Definition: file_ops.c:183
file_action_t action
Definition: filemap.h:47
static void receiveFileChunks(const char *sql)
Definition: libpq_fetch.c:234
size_t oldsize
Definition: filemap.h:50
static PGconn * conn
Definition: libpq_fetch.c:32
void PQclear(PGresult *res)
Definition: fe-exec.c:650
#define NULL
Definition: c.h:226
void remove_target(file_entry_t *entry)
Definition: file_ops.c:126
char * path
Definition: filemap.h:44
char * PQresultErrorMessage(const PGresult *res)
Definition: fe-exec.c:2612
int i
Definition: filemap.h:42
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1846
void create_target(file_entry_t *entry)
Definition: file_ops.c:147
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1702
void libpqConnect ( const char *  connstr)

Definition at line 48 of file libpq_fetch.c.

References CONNECTION_BAD, pg_fatal(), pg_free(), pg_log(), PG_PROGRESS, PGRES_COMMAND_OK, PQclear(), PQconnectdb(), PQerrorMessage(), PQexec(), PQresultErrorMessage(), PQresultStatus(), PQstatus(), and run_simple_query().

Referenced by main().

49 {
50  char *str;
51  PGresult *res;
52 
55  pg_fatal("could not connect to server: %s",
57 
58  pg_log(PG_PROGRESS, "connected to server\n");
59 
60  /*
61  * Check that the server is not in hot standby mode. There is no
62  * fundamental reason that couldn't be made to work, but it doesn't
63  * currently because we use a temporary table. Better to check for it
64  * explicitly than error out, for a better error message.
65  */
66  str = run_simple_query("SELECT pg_is_in_recovery()");
67  if (strcmp(str, "f") != 0)
68  pg_fatal("source server must not be in recovery mode\n");
69  pg_free(str);
70 
71  /*
72  * Also check that full_page_writes is enabled. We can get torn pages if
73  * a page is modified while we read it with pg_read_binary_file(), and we
74  * rely on full page images to fix them.
75  */
76  str = run_simple_query("SHOW full_page_writes");
77  if (strcmp(str, "on") != 0)
78  pg_fatal("full_page_writes must be enabled in the source server\n");
79  pg_free(str);
80 
81  /*
82  * Although we don't do any "real" updates, we do work with a temporary
83  * table. We don't care about synchronous commit for that. It doesn't
84  * otherwise matter much, but if the server is using synchronous
85  * replication, and replication isn't working for some reason, we don't
86  * want to get stuck, waiting for it to start working again.
87  */
88  res = PQexec(conn, "SET synchronous_commit = off");
89  if (PQresultStatus(res) != PGRES_COMMAND_OK)
90  pg_fatal("could not set up connection context: %s",
92  PQclear(res);
93 }
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:5959
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
static char * run_simple_query(const char *sql)
Definition: libpq_fetch.c:100
static PGconn * conn
Definition: libpq_fetch.c:32
void pg_log(eLogType type, const char *fmt,...)
Definition: logging.c:69
void PQclear(PGresult *res)
Definition: fe-exec.c:650
void pg_free(void *ptr)
Definition: fe_memutils.c:105
char * PQresultErrorMessage(const PGresult *res)
Definition: fe-exec.c:2612
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1846
ConnStatusType PQstatus(const PGconn *conn)
Definition: fe-connect.c:5906
PGconn * PQconnectdb(const char *conninfo)
Definition: fe-connect.c:526
static char * connstr
Definition: pg_dumpall.c:63
XLogRecPtr libpqGetCurrentXlogInsertLocation ( void  )

Definition at line 126 of file libpq_fetch.c.

References pg_fatal(), pg_free(), run_simple_query(), and val.

Referenced by main().

127 {
128  XLogRecPtr result;
129  uint32 hi;
130  uint32 lo;
131  char *val;
132 
133  val = run_simple_query("SELECT pg_current_wal_insert_location()");
134 
135  if (sscanf(val, "%X/%X", &hi, &lo) != 2)
136  pg_fatal("unrecognized result \"%s\" for current WAL insert location\n", val);
137 
138  result = ((uint64) hi) << 32 | lo;
139 
140  pg_free(val);
141 
142  return result;
143 }
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
static char * run_simple_query(const char *sql)
Definition: libpq_fetch.c:100
unsigned int uint32
Definition: c.h:265
uint64 XLogRecPtr
Definition: xlogdefs.h:21
void pg_free(void *ptr)
Definition: fe_memutils.c:105
long val
Definition: informix.c:689
char* libpqGetFile ( const char *  filename,
size_t *  filesize 
)

Definition at line 340 of file libpq_fetch.c.

References filename, NULL, PG_DEBUG, pg_fatal(), pg_log(), pg_malloc(), PGRES_TUPLES_OK, PQclear(), PQexecParams(), PQgetisnull(), PQgetlength(), PQgetvalue(), PQntuples(), PQresultErrorMessage(), and PQresultStatus().

Referenced by fetchFile().

341 {
342  PGresult *res;
343  char *result;
344  int len;
345  const char *paramValues[1];
346 
347  paramValues[0] = filename;
348  res = PQexecParams(conn, "SELECT pg_read_binary_file($1)",
349  1, NULL, paramValues, NULL, NULL, 1);
350 
351  if (PQresultStatus(res) != PGRES_TUPLES_OK)
352  pg_fatal("could not fetch remote file \"%s\": %s",
354 
355  /* sanity check the result set */
356  if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0))
357  pg_fatal("unexpected result set while fetching remote file \"%s\"\n",
358  filename);
359 
360  /* Read result to local variables */
361  len = PQgetlength(res, 0, 0);
362  result = pg_malloc(len + 1);
363  memcpy(result, PQgetvalue(res, 0, 0), len);
364  result[len] = '\0';
365 
366  PQclear(res);
367 
368  pg_log(PG_DEBUG, "fetched file \"%s\", length %d\n", filename, len);
369 
370  if (filesize)
371  *filesize = len;
372  return result;
373 }
int PQgetlength(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3078
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3067
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2673
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
static PGconn * conn
Definition: libpq_fetch.c:32
void pg_log(eLogType type, const char *fmt,...)
Definition: logging.c:69
void PQclear(PGresult *res)
Definition: fe-exec.c:650
#define NULL
Definition: c.h:226
static char * filename
Definition: pg_dumpall.c:80
char * PQresultErrorMessage(const PGresult *res)
Definition: fe-exec.c:2612
PGresult * PQexecParams(PGconn *conn, const char *command, int nParams, const Oid *paramTypes, const char *const *paramValues, const int *paramLengths, const int *paramFormats, int resultFormat)
Definition: fe-exec.c:1860
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3092
void libpqProcessFileList ( void  )

Definition at line 149 of file libpq_fetch.c.

References FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, i, pg_fatal(), PGRES_TUPLES_OK, PQclear(), PQexec(), PQgetisnull(), PQgetvalue(), PQnfields(), PQntuples(), PQresultErrorMessage(), PQresultStatus(), and process_source_file().

Referenced by fetchSourceFileList().

150 {
151  PGresult *res;
152  const char *sql;
153  int i;
154 
155  /*
156  * Create a recursive directory listing of the whole data directory.
157  *
158  * The WITH RECURSIVE part does most of the work. The second part gets the
159  * targets of the symlinks in pg_tblspc directory.
160  *
161  * XXX: There is no backend function to get a symbolic link's target in
162  * general, so if the admin has put any custom symbolic links in the data
163  * directory, they won't be copied correctly.
164  */
165  sql =
166  "WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
167  " SELECT '' AS path, filename, size, isdir FROM\n"
168  " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n"
169  " pg_stat_file(fn.filename, true) AS this\n"
170  " UNION ALL\n"
171  " SELECT parent.path || parent.filename || '/' AS path,\n"
172  " fn, this.size, this.isdir\n"
173  " FROM files AS parent,\n"
174  " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n"
175  " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n"
176  " WHERE parent.isdir = 't'\n"
177  ")\n"
178  "SELECT path || filename, size, isdir,\n"
179  " pg_tablespace_location(pg_tablespace.oid) AS link_target\n"
180  "FROM files\n"
181  "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n"
182  " AND oid::text = files.filename\n";
183  res = PQexec(conn, sql);
184 
185  if (PQresultStatus(res) != PGRES_TUPLES_OK)
186  pg_fatal("could not fetch file list: %s",
187  PQresultErrorMessage(res));
188 
189  /* sanity check the result set */
190  if (PQnfields(res) != 4)
191  pg_fatal("unexpected result set while fetching file list\n");
192 
193  /* Read result to local variables */
194  for (i = 0; i < PQntuples(res); i++)
195  {
196  char *path = PQgetvalue(res, i, 0);
197  int filesize = atoi(PQgetvalue(res, i, 1));
198  bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0);
199  char *link_target = PQgetvalue(res, i, 3);
200  file_type_t type;
201 
202  if (PQgetisnull(res, 0, 1))
203  {
204  /*
205  * The file was removed from the server while the query was
206  * running. Ignore it.
207  */
208  continue;
209  }
210 
211  if (link_target[0])
212  type = FILE_TYPE_SYMLINK;
213  else if (isdir)
214  type = FILE_TYPE_DIRECTORY;
215  else
216  type = FILE_TYPE_REGULAR;
217 
218  process_source_file(path, type, filesize, link_target);
219  }
220  PQclear(res);
221 }
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2681
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3067
void pg_fatal(const char *fmt,...)
Definition: logging.c:83
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2673
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
static PGconn * conn
Definition: libpq_fetch.c:32
void PQclear(PGresult *res)
Definition: fe-exec.c:650
void process_source_file(const char *path, file_type_t type, size_t newsize, const char *link_target)
Definition: filemap.c:60
char * PQresultErrorMessage(const PGresult *res)
Definition: fe-exec.c:2612
int i
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1846
file_type_t
Definition: filemap.h:35
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3092
void traverse_datadir ( const char *  datadir,
process_file_callback_t  callback 
)

Definition at line 36 of file copy_fetch.c.

References NULL, and recurse_dir().

Referenced by fetchSourceFileList(), and main().

37 {
39 }
static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback)
Definition: copy_fetch.c:48
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:49
char * datadir
#define NULL
Definition: c.h:226