PostgreSQL Source Code  git master
streamutil.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * streamutil.c - utility functions for pg_basebackup, pg_receivewal and
4  * pg_recvlogical
5  *
6  * Author: Magnus Hagander <magnus@hagander.net>
7  *
8  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
9  *
10  * IDENTIFICATION
11  * src/bin/pg_basebackup/streamutil.c
12  *-------------------------------------------------------------------------
13  */
14 
15 #include "postgres_fe.h"
16 
17 #include <sys/time.h>
18 #include <unistd.h>
19 
20 /* local includes */
21 #include "receivelog.h"
22 #include "streamutil.h"
23 
24 #include "access/xlog_internal.h"
25 #include "common/fe_memutils.h"
26 #include "datatype/timestamp.h"
27 #include "port/pg_bswap.h"
28 #include "pqexpbuffer.h"
29 
30 #define ERRCODE_DUPLICATE_OBJECT "42710"
31 
33 
34 /* SHOW command for replication connection was introduced in version 10 */
35 #define MINIMUM_VERSION_FOR_SHOW_CMD 100000
36 
37 const char *progname;
38 char *connection_string = NULL;
39 char *dbhost = NULL;
40 char *dbuser = NULL;
41 char *dbport = NULL;
42 char *dbname = NULL;
43 int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */
44 static bool have_password = false;
45 static char password[100];
46 PGconn *conn = NULL;
47 
48 /*
49  * Connect to the server. Returns a valid PGconn pointer if connected,
50  * or NULL on non-permanent error. On permanent error, the function will
51  * call exit(1) directly.
52  */
53 PGconn *
55 {
56  PGconn *tmpconn;
57  int argcount = 7; /* dbname, replication, fallback_app_name,
58  * host, user, port, password */
59  int i;
60  const char **keywords;
61  const char **values;
62  const char *tmpparam;
63  bool need_password;
64  PQconninfoOption *conn_opts = NULL;
65  PQconninfoOption *conn_opt;
66  char *err_msg = NULL;
67 
68  /* pg_recvlogical uses dbname only; others use connection_string only. */
69  Assert(dbname == NULL || connection_string == NULL);
70 
71  /*
72  * Merge the connection info inputs given in form of connection string,
73  * options and default values (dbname=replication, replication=true, etc.)
74  * Explicitly discard any dbname value in the connection string;
75  * otherwise, PQconnectdbParams() would interpret that value as being
76  * itself a connection string.
77  */
78  i = 0;
80  {
81  conn_opts = PQconninfoParse(connection_string, &err_msg);
82  if (conn_opts == NULL)
83  {
84  fprintf(stderr, "%s: %s", progname, err_msg);
85  exit(1);
86  }
87 
88  for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
89  {
90  if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
91  strcmp(conn_opt->keyword, "dbname") != 0)
92  argcount++;
93  }
94 
95  keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
96  values = pg_malloc0((argcount + 1) * sizeof(*values));
97 
98  for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
99  {
100  if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
101  strcmp(conn_opt->keyword, "dbname") != 0)
102  {
103  keywords[i] = conn_opt->keyword;
104  values[i] = conn_opt->val;
105  i++;
106  }
107  }
108  }
109  else
110  {
111  keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
112  values = pg_malloc0((argcount + 1) * sizeof(*values));
113  }
114 
115  keywords[i] = "dbname";
116  values[i] = dbname == NULL ? "replication" : dbname;
117  i++;
118  keywords[i] = "replication";
119  values[i] = dbname == NULL ? "true" : "database";
120  i++;
121  keywords[i] = "fallback_application_name";
122  values[i] = progname;
123  i++;
124 
125  if (dbhost)
126  {
127  keywords[i] = "host";
128  values[i] = dbhost;
129  i++;
130  }
131  if (dbuser)
132  {
133  keywords[i] = "user";
134  values[i] = dbuser;
135  i++;
136  }
137  if (dbport)
138  {
139  keywords[i] = "port";
140  values[i] = dbport;
141  i++;
142  }
143 
144  /* If -W was given, force prompt for password, but only the first time */
145  need_password = (dbgetpassword == 1 && !have_password);
146 
147  do
148  {
149  /* Get a new password if appropriate */
150  if (need_password)
151  {
152  simple_prompt("Password: ", password, sizeof(password), false);
153  have_password = true;
154  need_password = false;
155  }
156 
157  /* Use (or reuse, on a subsequent connection) password if we have it */
158  if (have_password)
159  {
160  keywords[i] = "password";
161  values[i] = password;
162  }
163  else
164  {
165  keywords[i] = NULL;
166  values[i] = NULL;
167  }
168 
169  tmpconn = PQconnectdbParams(keywords, values, true);
170 
171  /*
172  * If there is too little memory even to allocate the PGconn object
173  * and PQconnectdbParams returns NULL, we call exit(1) directly.
174  */
175  if (!tmpconn)
176  {
177  fprintf(stderr, _("%s: could not connect to server\n"),
178  progname);
179  exit(1);
180  }
181 
182  /* If we need a password and -w wasn't given, loop back and get one */
183  if (PQstatus(tmpconn) == CONNECTION_BAD &&
184  PQconnectionNeedsPassword(tmpconn) &&
185  dbgetpassword != -1)
186  {
187  PQfinish(tmpconn);
188  need_password = true;
189  }
190  }
191  while (need_password);
192 
193  if (PQstatus(tmpconn) != CONNECTION_OK)
194  {
195  fprintf(stderr, _("%s: could not connect to server: %s"),
196  progname, PQerrorMessage(tmpconn));
197  PQfinish(tmpconn);
198  free(values);
199  free(keywords);
200  if (conn_opts)
201  PQconninfoFree(conn_opts);
202  return NULL;
203  }
204 
205  /* Connection ok! */
206  free(values);
207  free(keywords);
208  if (conn_opts)
209  PQconninfoFree(conn_opts);
210 
211  /*
212  * Ensure we have the same value of integer_datetimes (now always "on") as
213  * the server we are connecting to.
214  */
215  tmpparam = PQparameterStatus(tmpconn, "integer_datetimes");
216  if (!tmpparam)
217  {
218  fprintf(stderr,
219  _("%s: could not determine server setting for integer_datetimes\n"),
220  progname);
221  PQfinish(tmpconn);
222  exit(1);
223  }
224 
225  if (strcmp(tmpparam, "on") != 0)
226  {
227  fprintf(stderr,
228  _("%s: integer_datetimes compile flag does not match server\n"),
229  progname);
230  PQfinish(tmpconn);
231  exit(1);
232  }
233 
234  return tmpconn;
235 }
236 
237 /*
238  * From version 10, explicitly set wal segment size using SHOW wal_segment_size
239  * since ControlFile is not accessible here.
240  */
241 bool
243 {
244  PGresult *res;
245  char xlog_unit[3];
246  int xlog_val,
247  multiplier = 1;
248 
249  /* check connection existence */
250  Assert(conn != NULL);
251 
252  /* for previous versions set the default xlog seg size */
254  {
256  return true;
257  }
258 
259  res = PQexec(conn, "SHOW wal_segment_size");
260  if (PQresultStatus(res) != PGRES_TUPLES_OK)
261  {
262  fprintf(stderr, _("%s: could not send replication command \"%s\": %s\n"),
263  progname, "SHOW wal_segment_size", PQerrorMessage(conn));
264 
265  PQclear(res);
266  return false;
267  }
268  if (PQntuples(res) != 1 || PQnfields(res) < 1)
269  {
270  fprintf(stderr,
271  _("%s: could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
272  progname, PQntuples(res), PQnfields(res), 1, 1);
273 
274  PQclear(res);
275  return false;
276  }
277 
278  /* fetch xlog value and unit from the result */
279  if (sscanf(PQgetvalue(res, 0, 0), "%d%s", &xlog_val, xlog_unit) != 2)
280  {
281  fprintf(stderr, _("%s: WAL segment size could not be parsed\n"),
282  progname);
283  return false;
284  }
285 
286  /* set the multiplier based on unit to convert xlog_val to bytes */
287  if (strcmp(xlog_unit, "MB") == 0)
288  multiplier = 1024 * 1024;
289  else if (strcmp(xlog_unit, "GB") == 0)
290  multiplier = 1024 * 1024 * 1024;
291 
292  /* convert and set WalSegSz */
293  WalSegSz = xlog_val * multiplier;
294 
296  {
297  fprintf(stderr,
298  _("%s: WAL segment size must be a power of two between 1MB and 1GB, but the remote server reported a value of %d bytes\n"),
299  progname, WalSegSz);
300  return false;
301  }
302 
303  PQclear(res);
304  return true;
305 }
306 
307 /*
308  * Run IDENTIFY_SYSTEM through a given connection and give back to caller
309  * some result information if requested:
310  * - System identifier
311  * - Current timeline ID
312  * - Start LSN position
313  * - Database name (NULL in servers prior to 9.4)
314  */
315 bool
316 RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli,
317  XLogRecPtr *startpos, char **db_name)
318 {
319  PGresult *res;
320  uint32 hi,
321  lo;
322 
323  /* Check connection existence */
324  Assert(conn != NULL);
325 
326  res = PQexec(conn, "IDENTIFY_SYSTEM");
327  if (PQresultStatus(res) != PGRES_TUPLES_OK)
328  {
329  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
330  progname, "IDENTIFY_SYSTEM", PQerrorMessage(conn));
331 
332  PQclear(res);
333  return false;
334  }
335  if (PQntuples(res) != 1 || PQnfields(res) < 3)
336  {
337  fprintf(stderr,
338  _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
339  progname, PQntuples(res), PQnfields(res), 1, 3);
340 
341  PQclear(res);
342  return false;
343  }
344 
345  /* Get system identifier */
346  if (sysid != NULL)
347  *sysid = pg_strdup(PQgetvalue(res, 0, 0));
348 
349  /* Get timeline ID to start streaming from */
350  if (starttli != NULL)
351  *starttli = atoi(PQgetvalue(res, 0, 1));
352 
353  /* Get LSN start position if necessary */
354  if (startpos != NULL)
355  {
356  if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &hi, &lo) != 2)
357  {
358  fprintf(stderr,
359  _("%s: could not parse write-ahead log location \"%s\"\n"),
360  progname, PQgetvalue(res, 0, 2));
361 
362  PQclear(res);
363  return false;
364  }
365  *startpos = ((uint64) hi) << 32 | lo;
366  }
367 
368  /* Get database name, only available in 9.4 and newer versions */
369  if (db_name != NULL)
370  {
371  *db_name = NULL;
372  if (PQserverVersion(conn) >= 90400)
373  {
374  if (PQnfields(res) < 4)
375  {
376  fprintf(stderr,
377  _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
378  progname, PQntuples(res), PQnfields(res), 1, 4);
379 
380  PQclear(res);
381  return false;
382  }
383  if (!PQgetisnull(res, 0, 3))
384  *db_name = pg_strdup(PQgetvalue(res, 0, 3));
385  }
386  }
387 
388  PQclear(res);
389  return true;
390 }
391 
392 /*
393  * Create a replication slot for the given connection. This function
394  * returns true in case of success.
395  */
396 bool
397 CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin,
398  bool is_temporary, bool is_physical, bool reserve_wal,
399  bool slot_exists_ok)
400 {
401  PQExpBuffer query;
402  PGresult *res;
403 
404  query = createPQExpBuffer();
405 
406  Assert((is_physical && plugin == NULL) ||
407  (!is_physical && plugin != NULL));
408  Assert(slot_name != NULL);
409 
410  /* Build query */
411  appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\"", slot_name);
412  if (is_temporary)
413  appendPQExpBuffer(query, " TEMPORARY");
414  if (is_physical)
415  {
416  appendPQExpBuffer(query, " PHYSICAL");
417  if (reserve_wal)
418  appendPQExpBuffer(query, " RESERVE_WAL");
419  }
420  else
421  {
422  appendPQExpBuffer(query, " LOGICAL \"%s\"", plugin);
423  if (PQserverVersion(conn) >= 100000)
424  /* pg_recvlogical doesn't use an exported snapshot, so suppress */
425  appendPQExpBuffer(query, " NOEXPORT_SNAPSHOT");
426  }
427 
428  res = PQexec(conn, query->data);
429  if (PQresultStatus(res) != PGRES_TUPLES_OK)
430  {
431  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
432 
433  if (slot_exists_ok &&
434  sqlstate &&
435  strcmp(sqlstate, ERRCODE_DUPLICATE_OBJECT) == 0)
436  {
437  destroyPQExpBuffer(query);
438  PQclear(res);
439  return true;
440  }
441  else
442  {
443  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
444  progname, query->data, PQerrorMessage(conn));
445 
446  destroyPQExpBuffer(query);
447  PQclear(res);
448  return false;
449  }
450  }
451 
452  if (PQntuples(res) != 1 || PQnfields(res) != 4)
453  {
454  fprintf(stderr,
455  _("%s: could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields\n"),
456  progname, slot_name,
457  PQntuples(res), PQnfields(res), 1, 4);
458 
459  destroyPQExpBuffer(query);
460  PQclear(res);
461  return false;
462  }
463 
464  destroyPQExpBuffer(query);
465  PQclear(res);
466  return true;
467 }
468 
469 /*
470  * Drop a replication slot for the given connection. This function
471  * returns true in case of success.
472  */
473 bool
474 DropReplicationSlot(PGconn *conn, const char *slot_name)
475 {
476  PQExpBuffer query;
477  PGresult *res;
478 
479  Assert(slot_name != NULL);
480 
481  query = createPQExpBuffer();
482 
483  /* Build query */
484  appendPQExpBuffer(query, "DROP_REPLICATION_SLOT \"%s\"",
485  slot_name);
486  res = PQexec(conn, query->data);
487  if (PQresultStatus(res) != PGRES_COMMAND_OK)
488  {
489  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
490  progname, query->data, PQerrorMessage(conn));
491 
492  destroyPQExpBuffer(query);
493  PQclear(res);
494  return false;
495  }
496 
497  if (PQntuples(res) != 0 || PQnfields(res) != 0)
498  {
499  fprintf(stderr,
500  _("%s: could not drop replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields\n"),
501  progname, slot_name,
502  PQntuples(res), PQnfields(res), 0, 0);
503 
504  destroyPQExpBuffer(query);
505  PQclear(res);
506  return false;
507  }
508 
509  destroyPQExpBuffer(query);
510  PQclear(res);
511  return true;
512 }
513 
514 
515 /*
516  * Frontend version of GetCurrentTimestamp(), since we are not linked with
517  * backend code.
518  */
521 {
522  TimestampTz result;
523  struct timeval tp;
524 
525  gettimeofday(&tp, NULL);
526 
527  result = (TimestampTz) tp.tv_sec -
529  result = (result * USECS_PER_SEC) + tp.tv_usec;
530 
531  return result;
532 }
533 
534 /*
535  * Frontend version of TimestampDifference(), since we are not linked with
536  * backend code.
537  */
538 void
540  long *secs, int *microsecs)
541 {
542  TimestampTz diff = stop_time - start_time;
543 
544  if (diff <= 0)
545  {
546  *secs = 0;
547  *microsecs = 0;
548  }
549  else
550  {
551  *secs = (long) (diff / USECS_PER_SEC);
552  *microsecs = (int) (diff % USECS_PER_SEC);
553  }
554 }
555 
556 /*
557  * Frontend version of TimestampDifferenceExceeds(), since we are not
558  * linked with backend code.
559  */
560 bool
562  TimestampTz stop_time,
563  int msec)
564 {
565  TimestampTz diff = stop_time - start_time;
566 
567  return (diff >= msec * INT64CONST(1000));
568 }
569 
570 /*
571  * Converts an int64 to network byte order.
572  */
573 void
574 fe_sendint64(int64 i, char *buf)
575 {
576  uint64 n64 = pg_hton64(i);
577 
578  memcpy(buf, &n64, sizeof(n64));
579 }
580 
581 /*
582  * Converts an int64 from network byte order to native format.
583  */
584 int64
586 {
587  uint64 n64;
588 
589  memcpy(&n64, buf, sizeof(n64));
590 
591  return pg_ntoh64(n64);
592 }
#define IsValidWalSegSize(size)
Definition: xlog_internal.h:97
static char password[100]
Definition: streamutil.c:45
static const char * plugin
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2732
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6106
int gettimeofday(struct timeval *tp, struct timezone *tzp)
Definition: gettimeofday.c:105
uint32 TimeLineID
Definition: xlogdefs.h:45
TimestampTz feGetCurrentTimestamp(void)
Definition: streamutil.c:520
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3118
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6071
#define pg_hton64(x)
Definition: pg_bswap.h:122
#define USECS_PER_SEC
Definition: timestamp.h:94
int64 TimestampTz
Definition: timestamp.h:39
#define DEFAULT_XLOG_SEG_SIZE
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:316
void PQfinish(PGconn *conn)
Definition: fe-connect.c:3629
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6096
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2724
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
bool feTimestampDifferenceExceeds(TimestampTz start_time, TimestampTz stop_time, int msec)
Definition: streamutil.c:561
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2647
PGconn * PQconnectdbParams(const char *const *keywords, const char *const *values, int expand_dbname)
Definition: fe-connect.c:529
static time_t start_time
Definition: pg_ctl.c:95
#define MINIMUM_VERSION_FOR_SHOW_CMD
Definition: streamutil.c:35
static bool slot_exists_ok
Definition: pg_receivewal.c:41
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:113
PQconninfoOption * PQconninfoParse(const char *conninfo, char **errmsg)
Definition: fe-connect.c:4726
char * connection_string
Definition: streamutil.c:38
#define SECS_PER_DAY
Definition: timestamp.h:86
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:242
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:46
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:262
void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs)
Definition: streamutil.c:539
static char * buf
Definition: pg_test_fsync.c:67
void simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo)
Definition: sprompt.c:37
void PQconninfoFree(PQconninfoOption *connOptions)
Definition: fe-connect.c:5956
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
unsigned int uint32
Definition: c.h:296
int dbgetpassword
Definition: streamutil.c:43
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:71
char * dbport
Definition: streamutil.c:41
void PQclear(PGresult *res)
Definition: fe-exec.c:671
#define free(a)
Definition: header.h:65
static bool have_password
Definition: streamutil.c:44
const char * progname
Definition: streamutil.c:37
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2709
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:670
char * dbhost
Definition: streamutil.c:39
char * dbname
Definition: streamutil.c:42
static XLogRecPtr startpos
bool CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, bool is_temporary, bool is_physical, bool reserve_wal, bool slot_exists_ok)
Definition: streamutil.c:397
uint32 WalSegSz
Definition: streamutil.c:32
static Datum values[MAXATTR]
Definition: bootstrap.c:164
int PQconnectionNeedsPassword(const PGconn *conn)
Definition: fe-connect.c:6140
int64 fe_recvint64(char *buf)
Definition: streamutil.c:585
char * dbuser
Definition: streamutil.c:40
int i
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1897
bool DropReplicationSlot(PGconn *conn, const char *slot_name)
Definition: streamutil.c:474
#define UNIX_EPOCH_JDATE
Definition: timestamp.h:162
#define POSTGRES_EPOCH_JDATE
Definition: timestamp.h:163
void fe_sendint64(int64 i, char *buf)
Definition: streamutil.c:574
PGconn * GetConnection(void)
Definition: streamutil.c:54
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3143
ConnStatusType PQstatus(const PGconn *conn)
Definition: fe-connect.c:6053
#define ERRCODE_DUPLICATE_OBJECT
Definition: streamutil.c:30
#define _(x)
Definition: elog.c:84
#define pg_ntoh64(x)
Definition: pg_bswap.h:126