PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
streamutil.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * streamutil.c - utility functions for pg_basebackup and pg_receivelog
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/streamutil.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <stdio.h>
17 #include <string.h>
18 #include <sys/time.h>
19 #include <sys/types.h>
20 #include <unistd.h>
21 
22 /* for ntohl/htonl */
23 #include <netinet/in.h>
24 #include <arpa/inet.h>
25 
26 /* local includes */
27 #include "receivelog.h"
28 #include "streamutil.h"
29 
30 #include "pqexpbuffer.h"
31 #include "common/fe_memutils.h"
32 #include "datatype/timestamp.h"
33 
34 #define ERRCODE_DUPLICATE_OBJECT "42710"
35 
36 const char *progname;
38 char *dbhost = NULL;
39 char *dbuser = NULL;
40 char *dbport = NULL;
41 char *dbname = NULL;
42 int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */
43 static bool have_password = false;
44 static char password[100];
46 
47 /*
48  * Connect to the server. Returns a valid PGconn pointer if connected,
49  * or NULL on non-permanent error. On permanent error, the function will
50  * call exit(1) directly.
51  */
52 PGconn *
54 {
55  PGconn *tmpconn;
56  int argcount = 7; /* dbname, replication, fallback_app_name,
57  * host, user, port, password */
58  int i;
59  const char **keywords;
60  const char **values;
61  const char *tmpparam;
62  bool need_password;
63  PQconninfoOption *conn_opts = NULL;
64  PQconninfoOption *conn_opt;
65  char *err_msg = NULL;
66 
67  /* pg_recvlogical uses dbname only; others use connection_string only. */
69 
70  /*
71  * Merge the connection info inputs given in form of connection string,
72  * options and default values (dbname=replication, replication=true, etc.)
73  * Explicitly discard any dbname value in the connection string;
74  * otherwise, PQconnectdbParams() would interpret that value as being
75  * itself a connection string.
76  */
77  i = 0;
79  {
80  conn_opts = PQconninfoParse(connection_string, &err_msg);
81  if (conn_opts == NULL)
82  {
83  fprintf(stderr, "%s: %s", progname, err_msg);
84  exit(1);
85  }
86 
87  for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
88  {
89  if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
90  strcmp(conn_opt->keyword, "dbname") != 0)
91  argcount++;
92  }
93 
94  keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
95  values = pg_malloc0((argcount + 1) * sizeof(*values));
96 
97  for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
98  {
99  if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
100  strcmp(conn_opt->keyword, "dbname") != 0)
101  {
102  keywords[i] = conn_opt->keyword;
103  values[i] = conn_opt->val;
104  i++;
105  }
106  }
107  }
108  else
109  {
110  keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
111  values = pg_malloc0((argcount + 1) * sizeof(*values));
112  }
113 
114  keywords[i] = "dbname";
115  values[i] = dbname == NULL ? "replication" : dbname;
116  i++;
117  keywords[i] = "replication";
118  values[i] = dbname == NULL ? "true" : "database";
119  i++;
120  keywords[i] = "fallback_application_name";
121  values[i] = progname;
122  i++;
123 
124  if (dbhost)
125  {
126  keywords[i] = "host";
127  values[i] = dbhost;
128  i++;
129  }
130  if (dbuser)
131  {
132  keywords[i] = "user";
133  values[i] = dbuser;
134  i++;
135  }
136  if (dbport)
137  {
138  keywords[i] = "port";
139  values[i] = dbport;
140  i++;
141  }
142 
143  /* If -W was given, force prompt for password, but only the first time */
144  need_password = (dbgetpassword == 1 && !have_password);
145 
146  do
147  {
148  /* Get a new password if appropriate */
149  if (need_password)
150  {
151  simple_prompt("Password: ", password, sizeof(password), false);
152  have_password = true;
153  need_password = false;
154  }
155 
156  /* Use (or reuse, on a subsequent connection) password if we have it */
157  if (have_password)
158  {
159  keywords[i] = "password";
160  values[i] = password;
161  }
162  else
163  {
164  keywords[i] = NULL;
165  values[i] = NULL;
166  }
167 
168  tmpconn = PQconnectdbParams(keywords, values, true);
169 
170  /*
171  * If there is too little memory even to allocate the PGconn object
172  * and PQconnectdbParams returns NULL, we call exit(1) directly.
173  */
174  if (!tmpconn)
175  {
176  fprintf(stderr, _("%s: could not connect to server\n"),
177  progname);
178  exit(1);
179  }
180 
181  /* If we need a password and -w wasn't given, loop back and get one */
182  if (PQstatus(tmpconn) == CONNECTION_BAD &&
183  PQconnectionNeedsPassword(tmpconn) &&
184  dbgetpassword != -1)
185  {
186  PQfinish(tmpconn);
187  need_password = true;
188  }
189  }
190  while (need_password);
191 
192  if (PQstatus(tmpconn) != CONNECTION_OK)
193  {
194  fprintf(stderr, _("%s: could not connect to server: %s"),
195  progname, PQerrorMessage(tmpconn));
196  PQfinish(tmpconn);
197  free(values);
198  free(keywords);
199  if (conn_opts)
200  PQconninfoFree(conn_opts);
201  return NULL;
202  }
203 
204  /* Connection ok! */
205  free(values);
206  free(keywords);
207  if (conn_opts)
208  PQconninfoFree(conn_opts);
209 
210  /*
211  * Ensure we have the same value of integer_datetimes (now always "on") as
212  * the server we are connecting to.
213  */
214  tmpparam = PQparameterStatus(tmpconn, "integer_datetimes");
215  if (!tmpparam)
216  {
217  fprintf(stderr,
218  _("%s: could not determine server setting for integer_datetimes\n"),
219  progname);
220  PQfinish(tmpconn);
221  exit(1);
222  }
223 
224  if (strcmp(tmpparam, "on") != 0)
225  {
226  fprintf(stderr,
227  _("%s: integer_datetimes compile flag does not match server\n"),
228  progname);
229  PQfinish(tmpconn);
230  exit(1);
231  }
232 
233  return tmpconn;
234 }
235 
236 /*
237  * Run IDENTIFY_SYSTEM through a given connection and give back to caller
238  * some result information if requested:
239  * - System identifier
240  * - Current timeline ID
241  * - Start LSN position
242  * - Database name (NULL in servers prior to 9.4)
243  */
244 bool
245 RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli,
246  XLogRecPtr *startpos, char **db_name)
247 {
248  PGresult *res;
249  uint32 hi,
250  lo;
251 
252  /* Check connection existence */
253  Assert(conn != NULL);
254 
255  res = PQexec(conn, "IDENTIFY_SYSTEM");
256  if (PQresultStatus(res) != PGRES_TUPLES_OK)
257  {
258  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
259  progname, "IDENTIFY_SYSTEM", PQerrorMessage(conn));
260 
261  PQclear(res);
262  return false;
263  }
264  if (PQntuples(res) != 1 || PQnfields(res) < 3)
265  {
266  fprintf(stderr,
267  _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
268  progname, PQntuples(res), PQnfields(res), 1, 3);
269 
270  PQclear(res);
271  return false;
272  }
273 
274  /* Get system identifier */
275  if (sysid != NULL)
276  *sysid = pg_strdup(PQgetvalue(res, 0, 0));
277 
278  /* Get timeline ID to start streaming from */
279  if (starttli != NULL)
280  *starttli = atoi(PQgetvalue(res, 0, 1));
281 
282  /* Get LSN start position if necessary */
283  if (startpos != NULL)
284  {
285  if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &hi, &lo) != 2)
286  {
287  fprintf(stderr,
288  _("%s: could not parse transaction log location \"%s\"\n"),
289  progname, PQgetvalue(res, 0, 2));
290 
291  PQclear(res);
292  return false;
293  }
294  *startpos = ((uint64) hi) << 32 | lo;
295  }
296 
297  /* Get database name, only available in 9.4 and newer versions */
298  if (db_name != NULL)
299  {
300  *db_name = NULL;
301  if (PQserverVersion(conn) >= 90400)
302  {
303  if (PQnfields(res) < 4)
304  {
305  fprintf(stderr,
306  _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
307  progname, PQntuples(res), PQnfields(res), 1, 4);
308 
309  PQclear(res);
310  return false;
311  }
312  if (!PQgetisnull(res, 0, 3))
313  *db_name = pg_strdup(PQgetvalue(res, 0, 3));
314  }
315  }
316 
317  PQclear(res);
318  return true;
319 }
320 
321 /*
322  * Create a replication slot for the given connection. This function
323  * returns true in case of success as well as the start position
324  * obtained after the slot creation.
325  */
326 bool
327 CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin,
328  bool is_physical, bool slot_exists_ok)
329 {
330  PQExpBuffer query;
331  PGresult *res;
332 
333  query = createPQExpBuffer();
334 
335  Assert((is_physical && plugin == NULL) ||
336  (!is_physical && plugin != NULL));
337  Assert(slot_name != NULL);
338 
339  /* Build query */
340  if (is_physical)
341  appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\" PHYSICAL",
342  slot_name);
343  else
344  appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\" LOGICAL \"%s\"",
345  slot_name, plugin);
346 
347  res = PQexec(conn, query->data);
348  if (PQresultStatus(res) != PGRES_TUPLES_OK)
349  {
350  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
351 
352  if (slot_exists_ok &&
353  sqlstate &&
354  strcmp(sqlstate, ERRCODE_DUPLICATE_OBJECT) == 0)
355  {
356  destroyPQExpBuffer(query);
357  PQclear(res);
358  return true;
359  }
360  else
361  {
362  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
363  progname, query->data, PQerrorMessage(conn));
364 
365  destroyPQExpBuffer(query);
366  PQclear(res);
367  return false;
368  }
369  }
370 
371  if (PQntuples(res) != 1 || PQnfields(res) != 4)
372  {
373  fprintf(stderr,
374  _("%s: could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields\n"),
375  progname, slot_name,
376  PQntuples(res), PQnfields(res), 1, 4);
377 
378  destroyPQExpBuffer(query);
379  PQclear(res);
380  return false;
381  }
382 
383  destroyPQExpBuffer(query);
384  PQclear(res);
385  return true;
386 }
387 
388 /*
389  * Drop a replication slot for the given connection. This function
390  * returns true in case of success.
391  */
392 bool
393 DropReplicationSlot(PGconn *conn, const char *slot_name)
394 {
395  PQExpBuffer query;
396  PGresult *res;
397 
398  Assert(slot_name != NULL);
399 
400  query = createPQExpBuffer();
401 
402  /* Build query */
403  appendPQExpBuffer(query, "DROP_REPLICATION_SLOT \"%s\"",
404  slot_name);
405  res = PQexec(conn, query->data);
406  if (PQresultStatus(res) != PGRES_COMMAND_OK)
407  {
408  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
409  progname, query->data, PQerrorMessage(conn));
410 
411  destroyPQExpBuffer(query);
412  PQclear(res);
413  return false;
414  }
415 
416  if (PQntuples(res) != 0 || PQnfields(res) != 0)
417  {
418  fprintf(stderr,
419  _("%s: could not drop replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields\n"),
420  progname, slot_name,
421  PQntuples(res), PQnfields(res), 0, 0);
422 
423  destroyPQExpBuffer(query);
424  PQclear(res);
425  return false;
426  }
427 
428  destroyPQExpBuffer(query);
429  PQclear(res);
430  return true;
431 }
432 
433 
434 /*
435  * Frontend version of GetCurrentTimestamp(), since we are not linked with
436  * backend code.
437  */
440 {
441  TimestampTz result;
442  struct timeval tp;
443 
444  gettimeofday(&tp, NULL);
445 
446  result = (TimestampTz) tp.tv_sec -
448  result = (result * USECS_PER_SEC) + tp.tv_usec;
449 
450  return result;
451 }
452 
453 /*
454  * Frontend version of TimestampDifference(), since we are not linked with
455  * backend code.
456  */
457 void
459  long *secs, int *microsecs)
460 {
461  TimestampTz diff = stop_time - start_time;
462 
463  if (diff <= 0)
464  {
465  *secs = 0;
466  *microsecs = 0;
467  }
468  else
469  {
470  *secs = (long) (diff / USECS_PER_SEC);
471  *microsecs = (int) (diff % USECS_PER_SEC);
472  }
473 }
474 
475 /*
476  * Frontend version of TimestampDifferenceExceeds(), since we are not
477  * linked with backend code.
478  */
479 bool
481  TimestampTz stop_time,
482  int msec)
483 {
484  TimestampTz diff = stop_time - start_time;
485 
486  return (diff >= msec * INT64CONST(1000));
487 }
488 
489 /*
490  * Converts an int64 to network byte order.
491  */
492 void
493 fe_sendint64(int64 i, char *buf)
494 {
495  uint32 n32;
496 
497  /* High order half first, since we're doing MSB-first */
498  n32 = (uint32) (i >> 32);
499  n32 = htonl(n32);
500  memcpy(&buf[0], &n32, 4);
501 
502  /* Now the low order half */
503  n32 = (uint32) i;
504  n32 = htonl(n32);
505  memcpy(&buf[4], &n32, 4);
506 }
507 
508 /*
509  * Converts an int64 from network byte order to native format.
510  */
511 int64
513 {
514  int64 result;
515  uint32 h32;
516  uint32 l32;
517 
518  memcpy(&h32, buf, 4);
519  memcpy(&l32, buf + 4, 4);
520  h32 = ntohl(h32);
521  l32 = ntohl(l32);
522 
523  result = h32;
524  result <<= 32;
525  result |= l32;
526 
527  return result;
528 }
static char password[100]
Definition: streamutil.c:44
static const char * plugin
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2681
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:5960
int gettimeofday(struct timeval *tp, struct timezone *tzp)
Definition: gettimeofday.c:105
uint32 TimeLineID
Definition: xlogdefs.h:45
TimestampTz feGetCurrentTimestamp(void)
Definition: streamutil.c:439
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3067
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:5925
#define USECS_PER_SEC
Definition: timestamp.h:94
int64 TimestampTz
Definition: timestamp.h:39
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:245
void PQfinish(PGconn *conn)
Definition: fe-connect.c:3517
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:5950
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2673
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:53
bool feTimestampDifferenceExceeds(TimestampTz start_time, TimestampTz stop_time, int msec)
Definition: streamutil.c:480
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
bool CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, bool is_physical, bool slot_exists_ok)
Definition: streamutil.c:327
PGconn * PQconnectdbParams(const char *const *keywords, const char *const *values, int expand_dbname)
Definition: fe-connect.c:471
static time_t start_time
Definition: pg_ctl.c:94
static bool slot_exists_ok
Definition: pg_receivewal.c:42
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:113
PQconninfoOption * PQconninfoParse(const char *conninfo, char **errmsg)
Definition: fe-connect.c:4604
char * connection_string
Definition: streamutil.c:37
#define SECS_PER_DAY
Definition: timestamp.h:86
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:45
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:262
void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs)
Definition: streamutil.c:458
static char * buf
Definition: pg_test_fsync.c:65
void simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo)
Definition: sprompt.c:37
void PQconninfoFree(PQconninfoOption *connOptions)
Definition: fe-connect.c:5810
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
unsigned int uint32
Definition: c.h:265
int dbgetpassword
Definition: streamutil.c:42
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:71
char * dbport
Definition: streamutil.c:40
void PQclear(PGresult *res)
Definition: fe-exec.c:650
#define free(a)
Definition: header.h:60
static bool have_password
Definition: streamutil.c:43
const char * progname
Definition: streamutil.c:36
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2658
#define NULL
Definition: c.h:226
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:671
#define INT64CONST(x)
Definition: c.h:307
char * dbhost
Definition: streamutil.c:38
char * dbname
Definition: streamutil.c:41
static XLogRecPtr startpos
static Datum values[MAXATTR]
Definition: bootstrap.c:162
int PQconnectionNeedsPassword(const PGconn *conn)
Definition: fe-connect.c:5994
int64 fe_recvint64(char *buf)
Definition: streamutil.c:512
char * dbuser
Definition: streamutil.c:39
int i
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1846
bool DropReplicationSlot(PGconn *conn, const char *slot_name)
Definition: streamutil.c:393
#define UNIX_EPOCH_JDATE
Definition: timestamp.h:162
#define POSTGRES_EPOCH_JDATE
Definition: timestamp.h:163
void fe_sendint64(int64 i, char *buf)
Definition: streamutil.c:493
PGconn * GetConnection(void)
Definition: streamutil.c:53
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3092
ConnStatusType PQstatus(const PGconn *conn)
Definition: fe-connect.c:5907
#define ERRCODE_DUPLICATE_OBJECT
Definition: streamutil.c:34
#define _(x)
Definition: elog.c:84