PostgreSQL Source Code  git master
pg_recvlogical.c File Reference
#include "postgres_fe.h"
#include <dirent.h>
#include <sys/stat.h>
#include <unistd.h>
#include "access/xlog_internal.h"
#include "common/fe_memutils.h"
#include "common/file_perm.h"
#include "common/logging.h"
#include "getopt_long.h"
#include "libpq-fe.h"
#include "libpq/pqsignal.h"
#include "pqexpbuffer.h"
#include "streamutil.h"
Include dependency graph for pg_recvlogical.c:

Go to the source code of this file.

Macros

#define RECONNECT_SLEEP_TIME   5
 

Functions

static void usage (void)
 
static void StreamLogicalLog (void)
 
static bool flushAndSendFeedback (PGconn *conn, TimestampTz *now)
 
static void prepareToTerminate (PGconn *conn, XLogRecPtr endpos, bool keepalive, XLogRecPtr lsn)
 
static bool sendFeedback (PGconn *conn, TimestampTz now, bool force, bool replyRequested)
 
static void disconnect_atexit (void)
 
static bool OutputFsync (TimestampTz now)
 
static void sigint_handler (int signum)
 
static void sighup_handler (int signum)
 
int main (int argc, char **argv)
 

Variables

static char * outfile = NULL
 
static int verbose = 0
 
static int noloop = 0
 
static int standby_message_timeout = 10 * 1000
 
static int fsync_interval = 10 * 1000
 
static XLogRecPtr startpos = InvalidXLogRecPtr
 
static XLogRecPtr endpos = InvalidXLogRecPtr
 
static bool do_create_slot = false
 
static bool slot_exists_ok = false
 
static bool do_start_slot = false
 
static bool do_drop_slot = false
 
static char * replication_slot = NULL
 
static char ** options
 
static size_t noptions = 0
 
static const char * plugin = "test_decoding"
 
static int outfd = -1
 
static volatile sig_atomic_t time_to_abort = false
 
static volatile sig_atomic_t output_reopen = false
 
static bool output_isfile
 
static TimestampTz output_last_fsync = -1
 
static bool output_needs_fsync = false
 
static XLogRecPtr output_written_lsn = InvalidXLogRecPtr
 
static XLogRecPtr output_fsync_lsn = InvalidXLogRecPtr
 

Macro Definition Documentation

◆ RECONNECT_SLEEP_TIME

#define RECONNECT_SLEEP_TIME   5

Definition at line 33 of file pg_recvlogical.c.

Referenced by main().

Function Documentation

◆ disconnect_atexit()

static void disconnect_atexit ( void  )
static

Definition at line 166 of file pg_recvlogical.c.

References conn, and PQfinish().

Referenced by main().

167 {
168  if (conn != NULL)
169  PQfinish(conn);
170 }
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4170
PGconn * conn
Definition: streamutil.c:54

◆ flushAndSendFeedback()

static bool flushAndSendFeedback ( PGconn conn,
TimestampTz now 
)
static

Definition at line 1022 of file pg_recvlogical.c.

References feGetCurrentTimestamp(), OutputFsync(), and sendFeedback().

Referenced by StreamLogicalLog().

1023 {
1024  /* flush data to disk, so that we send a recent flush pointer */
1025  if (!OutputFsync(*now))
1026  return false;
1028  if (!sendFeedback(conn, *now, true, false))
1029  return false;
1030 
1031  return true;
1032 }
TimestampTz feGetCurrentTimestamp(void)
Definition: streamutil.c:608
static bool OutputFsync(TimestampTz now)
static bool sendFeedback(PGconn *conn, TimestampTz now, bool force, bool replyRequested)
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1538

◆ main()

int main ( int  argc,
char **  argv 
)

Definition at line 673 of file pg_recvlogical.c.

References _, conn, CreateReplicationSlot(), dbgetpassword, dbhost, dbname, dbport, dbuser, disconnect_atexit(), do_create_slot, do_drop_slot, do_start_slot, DropReplicationSlot(), endpos, fprintf, fsync_interval, get_progname(), GetConnection(), getopt_long(), InvalidXLogRecPtr, no_argument, noloop, noptions, optarg, optind, outfile, pg_log_error, pg_log_info, pg_logging_init(), pg_mode_mask, pg_realloc(), pg_strdup(), PG_TEXTDOMAIN, pg_usleep(), plugin, pqsignal(), progname, RECONNECT_SLEEP_TIME, replication_slot, required_argument, RunIdentifySystem(), set_pglocale_pgservice(), SIGHUP, sighup_handler(), sigint_handler(), slot_exists_ok, standby_message_timeout, startpos, StreamLogicalLog(), time_to_abort, usage(), val, and verbose.

674 {
675  static struct option long_options[] = {
676 /* general options */
677  {"file", required_argument, NULL, 'f'},
678  {"fsync-interval", required_argument, NULL, 'F'},
679  {"no-loop", no_argument, NULL, 'n'},
680  {"verbose", no_argument, NULL, 'v'},
681  {"version", no_argument, NULL, 'V'},
682  {"help", no_argument, NULL, '?'},
683 /* connection options */
684  {"dbname", required_argument, NULL, 'd'},
685  {"host", required_argument, NULL, 'h'},
686  {"port", required_argument, NULL, 'p'},
687  {"username", required_argument, NULL, 'U'},
688  {"no-password", no_argument, NULL, 'w'},
689  {"password", no_argument, NULL, 'W'},
690 /* replication options */
691  {"startpos", required_argument, NULL, 'I'},
692  {"endpos", required_argument, NULL, 'E'},
693  {"option", required_argument, NULL, 'o'},
694  {"plugin", required_argument, NULL, 'P'},
695  {"status-interval", required_argument, NULL, 's'},
696  {"slot", required_argument, NULL, 'S'},
697 /* action */
698  {"create-slot", no_argument, NULL, 1},
699  {"start", no_argument, NULL, 2},
700  {"drop-slot", no_argument, NULL, 3},
701  {"if-not-exists", no_argument, NULL, 4},
702  {NULL, 0, NULL, 0}
703  };
704  int c;
705  int option_index;
706  uint32 hi,
707  lo;
708  char *db_name;
709 
710  pg_logging_init(argv[0]);
711  progname = get_progname(argv[0]);
712  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
713 
714  if (argc > 1)
715  {
716  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
717  {
718  usage();
719  exit(0);
720  }
721  else if (strcmp(argv[1], "-V") == 0 ||
722  strcmp(argv[1], "--version") == 0)
723  {
724  puts("pg_recvlogical (PostgreSQL) " PG_VERSION);
725  exit(0);
726  }
727  }
728 
729  while ((c = getopt_long(argc, argv, "E:f:F:nvd:h:p:U:wWI:o:P:s:S:",
730  long_options, &option_index)) != -1)
731  {
732  switch (c)
733  {
734 /* general options */
735  case 'f':
737  break;
738  case 'F':
739  fsync_interval = atoi(optarg) * 1000;
740  if (fsync_interval < 0)
741  {
742  pg_log_error("invalid fsync interval \"%s\"", optarg);
743  exit(1);
744  }
745  break;
746  case 'n':
747  noloop = 1;
748  break;
749  case 'v':
750  verbose++;
751  break;
752 /* connection options */
753  case 'd':
755  break;
756  case 'h':
758  break;
759  case 'p':
760  if (atoi(optarg) <= 0)
761  {
762  pg_log_error("invalid port number \"%s\"", optarg);
763  exit(1);
764  }
766  break;
767  case 'U':
769  break;
770  case 'w':
771  dbgetpassword = -1;
772  break;
773  case 'W':
774  dbgetpassword = 1;
775  break;
776 /* replication options */
777  case 'I':
778  if (sscanf(optarg, "%X/%X", &hi, &lo) != 2)
779  {
780  pg_log_error("could not parse start position \"%s\"", optarg);
781  exit(1);
782  }
783  startpos = ((uint64) hi) << 32 | lo;
784  break;
785  case 'E':
786  if (sscanf(optarg, "%X/%X", &hi, &lo) != 2)
787  {
788  pg_log_error("could not parse end position \"%s\"", optarg);
789  exit(1);
790  }
791  endpos = ((uint64) hi) << 32 | lo;
792  break;
793  case 'o':
794  {
795  char *data = pg_strdup(optarg);
796  char *val = strchr(data, '=');
797 
798  if (val != NULL)
799  {
800  /* remove =; separate data from val */
801  *val = '\0';
802  val++;
803  }
804 
805  noptions += 1;
806  options = pg_realloc(options, sizeof(char *) * noptions * 2);
807 
808  options[(noptions - 1) * 2] = data;
809  options[(noptions - 1) * 2 + 1] = val;
810  }
811 
812  break;
813  case 'P':
815  break;
816  case 's':
817  standby_message_timeout = atoi(optarg) * 1000;
818  if (standby_message_timeout < 0)
819  {
820  pg_log_error("invalid status interval \"%s\"", optarg);
821  exit(1);
822  }
823  break;
824  case 'S':
826  break;
827 /* action */
828  case 1:
829  do_create_slot = true;
830  break;
831  case 2:
832  do_start_slot = true;
833  break;
834  case 3:
835  do_drop_slot = true;
836  break;
837  case 4:
838  slot_exists_ok = true;
839  break;
840 
841  default:
842 
843  /*
844  * getopt_long already emitted a complaint
845  */
846  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
847  progname);
848  exit(1);
849  }
850  }
851 
852  /*
853  * Any non-option arguments?
854  */
855  if (optind < argc)
856  {
857  pg_log_error("too many command-line arguments (first is \"%s\")",
858  argv[optind]);
859  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
860  progname);
861  exit(1);
862  }
863 
864  /*
865  * Required arguments
866  */
867  if (replication_slot == NULL)
868  {
869  pg_log_error("no slot specified");
870  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
871  progname);
872  exit(1);
873  }
874 
875  if (do_start_slot && outfile == NULL)
876  {
877  pg_log_error("no target file specified");
878  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
879  progname);
880  exit(1);
881  }
882 
883  if (!do_drop_slot && dbname == NULL)
884  {
885  pg_log_error("no database specified");
886  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
887  progname);
888  exit(1);
889  }
890 
892  {
893  pg_log_error("at least one action needs to be specified");
894  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
895  progname);
896  exit(1);
897  }
898 
900  {
901  pg_log_error("cannot use --create-slot or --start together with --drop-slot");
902  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
903  progname);
904  exit(1);
905  }
906 
908  {
909  pg_log_error("cannot use --create-slot or --drop-slot together with --startpos");
910  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
911  progname);
912  exit(1);
913  }
914 
916  {
917  pg_log_error("--endpos may only be specified with --start");
918  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
919  progname);
920  exit(1);
921  }
922 
923 #ifndef WIN32
924  pqsignal(SIGINT, sigint_handler);
926 #endif
927 
928  /*
929  * Obtain a connection to server. This is not really necessary but it
930  * helps to get more precise error messages about authentication, required
931  * GUC parameters and such.
932  */
933  conn = GetConnection();
934  if (!conn)
935  /* Error message already written in GetConnection() */
936  exit(1);
937  atexit(disconnect_atexit);
938 
939  /*
940  * Run IDENTIFY_SYSTEM to make sure we connected using a database specific
941  * replication connection.
942  */
943  if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name))
944  exit(1);
945 
946  if (db_name == NULL)
947  {
948  pg_log_error("could not establish database-specific replication connection");
949  exit(1);
950  }
951 
952  /*
953  * Set umask so that directories/files are created with the same
954  * permissions as directories/files in the source data directory.
955  *
956  * pg_mode_mask is set to owner-only by default and then updated in
957  * GetConnection() where we get the mode from the server-side with
958  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
959  */
960  umask(pg_mode_mask);
961 
962  /* Drop a replication slot. */
963  if (do_drop_slot)
964  {
965  if (verbose)
966  pg_log_info("dropping replication slot \"%s\"", replication_slot);
967 
969  exit(1);
970  }
971 
972  /* Create a replication slot. */
973  if (do_create_slot)
974  {
975  if (verbose)
976  pg_log_info("creating replication slot \"%s\"", replication_slot);
977 
979  false, false, slot_exists_ok))
980  exit(1);
982  }
983 
984  if (!do_start_slot)
985  exit(0);
986 
987  /* Stream loop */
988  while (true)
989  {
991  if (time_to_abort)
992  {
993  /*
994  * We've been Ctrl-C'ed or reached an exit limit condition. That's
995  * not an error, so exit without an errorcode.
996  */
997  exit(0);
998  }
999  else if (noloop)
1000  {
1001  pg_log_error("disconnected");
1002  exit(1);
1003  }
1004  else
1005  {
1006  /* translator: check source for value for %d */
1007  pg_log_info("disconnected; waiting %d seconds to try again",
1009  pg_usleep(RECONNECT_SLEEP_TIME * 1000000);
1010  }
1011  }
1012 }
static const char * plugin
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
static volatile sig_atomic_t time_to_abort
static void sigint_handler(int signum)
static int fsync_interval
#define RECONNECT_SLEEP_TIME
const char * get_progname(const char *argv0)
Definition: path.c:453
static void DropReplicationSlot(DropReplicationSlotCmd *cmd)
Definition: walsender.c:1123
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static void sighup_handler(int signum)
void pg_logging_init(const char *argv0)
Definition: logging.c:81
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:409
static char * replication_slot
const char * progname
Definition: pg_standby.c:36
#define fprintf
Definition: port.h:197
static int noloop
static int verbose
static XLogRecPtr endpos
void pg_usleep(long microsec)
Definition: signal.c:53
#define required_argument
Definition: getopt_long.h:25
int optind
Definition: getopt.c:50
PGconn * conn
Definition: streamutil.c:54
static bool slot_exists_ok
char * c
static void disconnect_atexit(void)
#define SIGHUP
Definition: win32_port.h:153
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
unsigned int uint32
Definition: c.h:374
void * pg_realloc(void *ptr, size_t size)
Definition: fe_memutils.c:65
int dbgetpassword
Definition: streamutil.c:51
#define no_argument
Definition: getopt_long.h:24
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1155
static void usage(void)
char * dbport
Definition: streamutil.c:49
static bool do_drop_slot
pqsigfunc pqsignal(int signum, pqsigfunc handler)
Definition: signal.c:170
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
char * dbhost
Definition: streamutil.c:47
char * dbname
Definition: streamutil.c:50
static XLogRecPtr startpos
static void StreamLogicalLog(void)
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:921
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:434
char * dbuser
Definition: streamutil.c:48
char * optarg
Definition: getopt.c:52
static bool do_start_slot
static char * outfile
static size_t noptions
#define _(x)
Definition: elog.c:88
int pg_mode_mask
Definition: file_perm.c:25
long val
Definition: informix.c:664
#define pg_log_info(...)
Definition: logging.h:87
static bool do_create_slot
static int standby_message_timeout

◆ OutputFsync()

static bool OutputFsync ( TimestampTz  now)
static

Definition at line 173 of file pg_recvlogical.c.

References fsync, fsync_interval, now(), outfd, outfile, output_fsync_lsn, output_isfile, output_last_fsync, output_needs_fsync, output_written_lsn, and pg_log_fatal.

Referenced by flushAndSendFeedback(), and StreamLogicalLog().

174 {
176 
178 
179  if (fsync_interval <= 0)
180  return true;
181 
182  if (!output_needs_fsync)
183  return true;
184 
185  output_needs_fsync = false;
186 
187  /* can only fsync if it's a regular file */
188  if (!output_isfile)
189  return true;
190 
191  if (fsync(outfd) != 0)
192  {
193  pg_log_fatal("could not fsync file \"%s\": %m", outfile);
194  exit(1);
195  }
196 
197  return true;
198 }
static XLogRecPtr output_fsync_lsn
static int fsync_interval
#define fsync(fd)
Definition: win32_port.h:62
static bool output_isfile
static int outfd
static XLogRecPtr output_written_lsn
static bool output_needs_fsync
static TimestampTz output_last_fsync
static char * outfile
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1538
#define pg_log_fatal(...)
Definition: logging.h:75

◆ prepareToTerminate()

static void prepareToTerminate ( PGconn conn,
XLogRecPtr  endpos,
bool  keepalive,
XLogRecPtr  lsn 
)
static

Definition at line 1039 of file pg_recvlogical.c.

References pg_log_info, PQflush(), PQputCopyEnd(), and verbose.

Referenced by StreamLogicalLog().

1040 {
1041  (void) PQputCopyEnd(conn, NULL);
1042  (void) PQflush(conn);
1043 
1044  if (verbose)
1045  {
1046  if (keepalive)
1047  pg_log_info("end position %X/%X reached by keepalive",
1048  (uint32) (endpos >> 32), (uint32) endpos);
1049  else
1050  pg_log_info("end position %X/%X reached by WAL record at %X/%X",
1051  (uint32) (endpos >> 32), (uint32) (endpos),
1052  (uint32) (lsn >> 32), (uint32) lsn);
1053  }
1054 }
int PQputCopyEnd(PGconn *conn, const char *errormsg)
Definition: fe-exec.c:2384
static int verbose
static XLogRecPtr endpos
int PQflush(PGconn *conn)
Definition: fe-exec.c:3283
unsigned int uint32
Definition: c.h:374
#define pg_log_info(...)
Definition: logging.h:87

◆ sendFeedback()

static bool sendFeedback ( PGconn conn,
TimestampTz  now,
bool  force,
bool  replyRequested 
)
static

Definition at line 114 of file pg_recvlogical.c.

References fe_sendint64(), InvalidXLogRecPtr, output_fsync_lsn, output_written_lsn, pg_log_error, pg_log_info, PQerrorMessage(), PQflush(), PQputCopyData(), replication_slot, startpos, and verbose.

Referenced by flushAndSendFeedback(), and StreamLogicalLog().

115 {
116  static XLogRecPtr last_written_lsn = InvalidXLogRecPtr;
117  static XLogRecPtr last_fsync_lsn = InvalidXLogRecPtr;
118 
119  char replybuf[1 + 8 + 8 + 8 + 8 + 1];
120  int len = 0;
121 
122  /*
123  * we normally don't want to send superfluous feedback, but if it's
124  * because of a timeout we need to, otherwise wal_sender_timeout will kill
125  * us.
126  */
127  if (!force &&
128  last_written_lsn == output_written_lsn &&
129  last_fsync_lsn == output_fsync_lsn)
130  return true;
131 
132  if (verbose)
133  pg_log_info("confirming write up to %X/%X, flush to %X/%X (slot %s)",
137 
138  replybuf[len] = 'r';
139  len += 1;
140  fe_sendint64(output_written_lsn, &replybuf[len]); /* write */
141  len += 8;
142  fe_sendint64(output_fsync_lsn, &replybuf[len]); /* flush */
143  len += 8;
144  fe_sendint64(InvalidXLogRecPtr, &replybuf[len]); /* apply */
145  len += 8;
146  fe_sendint64(now, &replybuf[len]); /* sendTime */
147  len += 8;
148  replybuf[len] = replyRequested ? 1 : 0; /* replyRequested */
149  len += 1;
150 
152  last_written_lsn = output_written_lsn;
153  last_fsync_lsn = output_fsync_lsn;
154 
155  if (PQputCopyData(conn, replybuf, len) <= 0 || PQflush(conn))
156  {
157  pg_log_error("could not send feedback packet: %s",
158  PQerrorMessage(conn));
159  return false;
160  }
161 
162  return true;
163 }
int PQputCopyData(PGconn *conn, const char *buffer, int nbytes)
Definition: fe-exec.c:2317
static XLogRecPtr output_fsync_lsn
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6672
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
#define pg_log_error(...)
Definition: logging.h:79
static char * replication_slot
static int verbose
int PQflush(PGconn *conn)
Definition: fe-exec.c:3283
unsigned int uint32
Definition: c.h:374
static XLogRecPtr output_written_lsn
uint64 XLogRecPtr
Definition: xlogdefs.h:21
static XLogRecPtr startpos
void fe_sendint64(int64 i, char *buf)
Definition: streamutil.c:662
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1538
#define pg_log_info(...)
Definition: logging.h:87

◆ sighup_handler()

static void sighup_handler ( int  signum)
static

Definition at line 665 of file pg_recvlogical.c.

References output_reopen.

Referenced by main().

666 {
667  output_reopen = true;
668 }
static volatile sig_atomic_t output_reopen

◆ sigint_handler()

static void sigint_handler ( int  signum)
static

Definition at line 656 of file pg_recvlogical.c.

References time_to_abort.

Referenced by main().

657 {
658  time_to_abort = true;
659 }
static volatile sig_atomic_t time_to_abort

◆ StreamLogicalLog()

static void StreamLogicalLog ( void  )
static

Definition at line 204 of file pg_recvlogical.c.

References appendPQExpBuffer(), appendPQExpBufferChar(), appendPQExpBufferStr(), close, conn, copybuf, createPQExpBuffer(), PQExpBufferData::data, destroyPQExpBuffer(), EINTR, endpos, error(), fe_recvint64(), feGetCurrentTimestamp(), feTimestampDifference(), feTimestampDifferenceExceeds(), flushAndSendFeedback(), fsync_interval, GetConnection(), i, InvalidXLogRecPtr, Max, noptions, now(), outfd, outfile, output_fsync_lsn, output_isfile, output_last_fsync, output_needs_fsync, output_reopen, output_written_lsn, OutputFsync(), PG_BINARY, pg_log_error, pg_log_info, PGRES_COMMAND_OK, PGRES_COPY_BOTH, PGRES_COPY_OUT, PQclear(), PQconsumeInput(), PQerrorMessage(), PQexec(), PQfinish(), PQfreemem(), PQgetCopyData(), PQgetResult(), PQresultErrorMessage(), PQresultStatus(), PQsocket(), prepareToTerminate(), replication_slot, resetPQExpBuffer(), S_IRUSR, S_ISREG, S_IWUSR, select, sendFeedback(), standby_message_timeout, startpos, stat, generate_unaccent_rules::stdout, time_to_abort, verbose, and write.

Referenced by main().

205 {
206  PGresult *res;
207  char *copybuf = NULL;
208  TimestampTz last_status = -1;
209  int i;
210  PQExpBuffer query;
211 
214 
215  query = createPQExpBuffer();
216 
217  /*
218  * Connect in replication mode to the server
219  */
220  if (!conn)
221  conn = GetConnection();
222  if (!conn)
223  /* Error message already written in GetConnection() */
224  return;
225 
226  /*
227  * Start the replication
228  */
229  if (verbose)
230  pg_log_info("starting log streaming at %X/%X (slot %s)",
231  (uint32) (startpos >> 32), (uint32) startpos,
233 
234  /* Initiate the replication stream at specified location */
235  appendPQExpBuffer(query, "START_REPLICATION SLOT \"%s\" LOGICAL %X/%X",
237 
238  /* print options if there are any */
239  if (noptions)
240  appendPQExpBufferStr(query, " (");
241 
242  for (i = 0; i < noptions; i++)
243  {
244  /* separator */
245  if (i > 0)
246  appendPQExpBufferStr(query, ", ");
247 
248  /* write option name */
249  appendPQExpBuffer(query, "\"%s\"", options[(i * 2)]);
250 
251  /* write option value if specified */
252  if (options[(i * 2) + 1] != NULL)
253  appendPQExpBuffer(query, " '%s'", options[(i * 2) + 1]);
254  }
255 
256  if (noptions)
257  appendPQExpBufferChar(query, ')');
258 
259  res = PQexec(conn, query->data);
260  if (PQresultStatus(res) != PGRES_COPY_BOTH)
261  {
262  pg_log_error("could not send replication command \"%s\": %s",
263  query->data, PQresultErrorMessage(res));
264  PQclear(res);
265  goto error;
266  }
267  PQclear(res);
268  resetPQExpBuffer(query);
269 
270  if (verbose)
271  pg_log_info("streaming initiated");
272 
273  while (!time_to_abort)
274  {
275  int r;
276  int bytes_left;
277  int bytes_written;
279  int hdr_len;
280  XLogRecPtr cur_record_lsn = InvalidXLogRecPtr;
281 
282  if (copybuf != NULL)
283  {
284  PQfreemem(copybuf);
285  copybuf = NULL;
286  }
287 
288  /*
289  * Potentially send a status message to the primary.
290  */
291  now = feGetCurrentTimestamp();
292 
293  if (outfd != -1 &&
296  {
297  if (!OutputFsync(now))
298  goto error;
299  }
300 
301  if (standby_message_timeout > 0 &&
302  feTimestampDifferenceExceeds(last_status, now,
304  {
305  /* Time to send feedback! */
306  if (!sendFeedback(conn, now, true, false))
307  goto error;
308 
309  last_status = now;
310  }
311 
312  /* got SIGHUP, close output file */
313  if (outfd != -1 && output_reopen && strcmp(outfile, "-") != 0)
314  {
315  now = feGetCurrentTimestamp();
316  if (!OutputFsync(now))
317  goto error;
318  close(outfd);
319  outfd = -1;
320  }
321  output_reopen = false;
322 
323  /* open the output file, if not open yet */
324  if (outfd == -1)
325  {
326  struct stat statbuf;
327 
328  if (strcmp(outfile, "-") == 0)
329  outfd = fileno(stdout);
330  else
331  outfd = open(outfile, O_CREAT | O_APPEND | O_WRONLY | PG_BINARY,
332  S_IRUSR | S_IWUSR);
333  if (outfd == -1)
334  {
335  pg_log_error("could not open log file \"%s\": %m", outfile);
336  goto error;
337  }
338 
339  if (fstat(outfd, &statbuf) != 0)
340  pg_log_error("could not stat file \"%s\": %m", outfile);
341 
342  output_isfile = S_ISREG(statbuf.st_mode) && !isatty(outfd);
343  }
344 
345  r = PQgetCopyData(conn, &copybuf, 1);
346  if (r == 0)
347  {
348  /*
349  * In async mode, and no data available. We block on reading but
350  * not more than the specified timeout, so that we can send a
351  * response back to the client.
352  */
353  fd_set input_mask;
354  TimestampTz message_target = 0;
355  TimestampTz fsync_target = 0;
356  struct timeval timeout;
357  struct timeval *timeoutptr = NULL;
358 
359  if (PQsocket(conn) < 0)
360  {
361  pg_log_error("invalid socket: %s", PQerrorMessage(conn));
362  goto error;
363  }
364 
365  FD_ZERO(&input_mask);
366  FD_SET(PQsocket(conn), &input_mask);
367 
368  /* Compute when we need to wakeup to send a keepalive message. */
370  message_target = last_status + (standby_message_timeout - 1) *
371  ((int64) 1000);
372 
373  /* Compute when we need to wakeup to fsync the output file. */
375  fsync_target = output_last_fsync + (fsync_interval - 1) *
376  ((int64) 1000);
377 
378  /* Now compute when to wakeup. */
379  if (message_target > 0 || fsync_target > 0)
380  {
381  TimestampTz targettime;
382  long secs;
383  int usecs;
384 
385  targettime = message_target;
386 
387  if (fsync_target > 0 && fsync_target < targettime)
388  targettime = fsync_target;
389 
391  targettime,
392  &secs,
393  &usecs);
394  if (secs <= 0)
395  timeout.tv_sec = 1; /* Always sleep at least 1 sec */
396  else
397  timeout.tv_sec = secs;
398  timeout.tv_usec = usecs;
399  timeoutptr = &timeout;
400  }
401 
402  r = select(PQsocket(conn) + 1, &input_mask, NULL, NULL, timeoutptr);
403  if (r == 0 || (r < 0 && errno == EINTR))
404  {
405  /*
406  * Got a timeout or signal. Continue the loop and either
407  * deliver a status packet to the server or just go back into
408  * blocking.
409  */
410  continue;
411  }
412  else if (r < 0)
413  {
414  pg_log_error("select() failed: %m");
415  goto error;
416  }
417 
418  /* Else there is actually data on the socket */
419  if (PQconsumeInput(conn) == 0)
420  {
421  pg_log_error("could not receive data from WAL stream: %s",
423  goto error;
424  }
425  continue;
426  }
427 
428  /* End of copy stream */
429  if (r == -1)
430  break;
431 
432  /* Failure while reading the copy stream */
433  if (r == -2)
434  {
435  pg_log_error("could not read COPY data: %s",
437  goto error;
438  }
439 
440  /* Check the message type. */
441  if (copybuf[0] == 'k')
442  {
443  int pos;
444  bool replyRequested;
445  XLogRecPtr walEnd;
446  bool endposReached = false;
447 
448  /*
449  * Parse the keepalive message, enclosed in the CopyData message.
450  * We just check if the server requested a reply, and ignore the
451  * rest.
452  */
453  pos = 1; /* skip msgtype 'k' */
454  walEnd = fe_recvint64(&copybuf[pos]);
456 
457  pos += 8; /* read walEnd */
458 
459  pos += 8; /* skip sendTime */
460 
461  if (r < pos + 1)
462  {
463  pg_log_error("streaming header too small: %d", r);
464  goto error;
465  }
466  replyRequested = copybuf[pos];
467 
468  if (endpos != InvalidXLogRecPtr && walEnd >= endpos)
469  {
470  /*
471  * If there's nothing to read on the socket until a keepalive
472  * we know that the server has nothing to send us; and if
473  * walEnd has passed endpos, we know nothing else can have
474  * committed before endpos. So we can bail out now.
475  */
476  endposReached = true;
477  }
478 
479  /* Send a reply, if necessary */
480  if (replyRequested || endposReached)
481  {
482  if (!flushAndSendFeedback(conn, &now))
483  goto error;
484  last_status = now;
485  }
486 
487  if (endposReached)
488  {
490  time_to_abort = true;
491  break;
492  }
493 
494  continue;
495  }
496  else if (copybuf[0] != 'w')
497  {
498  pg_log_error("unrecognized streaming header: \"%c\"",
499  copybuf[0]);
500  goto error;
501  }
502 
503  /*
504  * Read the header of the XLogData message, enclosed in the CopyData
505  * message. We only need the WAL location field (dataStart), the rest
506  * of the header is ignored.
507  */
508  hdr_len = 1; /* msgtype 'w' */
509  hdr_len += 8; /* dataStart */
510  hdr_len += 8; /* walEnd */
511  hdr_len += 8; /* sendTime */
512  if (r < hdr_len + 1)
513  {
514  pg_log_error("streaming header too small: %d", r);
515  goto error;
516  }
517 
518  /* Extract WAL location for this block */
519  cur_record_lsn = fe_recvint64(&copybuf[1]);
520 
521  if (endpos != InvalidXLogRecPtr && cur_record_lsn > endpos)
522  {
523  /*
524  * We've read past our endpoint, so prepare to go away being
525  * cautious about what happens to our output data.
526  */
527  if (!flushAndSendFeedback(conn, &now))
528  goto error;
529  prepareToTerminate(conn, endpos, false, cur_record_lsn);
530  time_to_abort = true;
531  break;
532  }
533 
534  output_written_lsn = Max(cur_record_lsn, output_written_lsn);
535 
536  bytes_left = r - hdr_len;
537  bytes_written = 0;
538 
539  /* signal that a fsync is needed */
540  output_needs_fsync = true;
541 
542  while (bytes_left)
543  {
544  int ret;
545 
546  ret = write(outfd,
547  copybuf + hdr_len + bytes_written,
548  bytes_left);
549 
550  if (ret < 0)
551  {
552  pg_log_error("could not write %u bytes to log file \"%s\": %m",
553  bytes_left, outfile);
554  goto error;
555  }
556 
557  /* Write was successful, advance our position */
558  bytes_written += ret;
559  bytes_left -= ret;
560  }
561 
562  if (write(outfd, "\n", 1) != 1)
563  {
564  pg_log_error("could not write %u bytes to log file \"%s\": %m",
565  1, outfile);
566  goto error;
567  }
568 
569  if (endpos != InvalidXLogRecPtr && cur_record_lsn == endpos)
570  {
571  /* endpos was exactly the record we just processed, we're done */
572  if (!flushAndSendFeedback(conn, &now))
573  goto error;
574  prepareToTerminate(conn, endpos, false, cur_record_lsn);
575  time_to_abort = true;
576  break;
577  }
578  }
579 
580  res = PQgetResult(conn);
581  if (PQresultStatus(res) == PGRES_COPY_OUT)
582  {
583  PQclear(res);
584 
585  /*
586  * We're doing a client-initiated clean exit and have sent CopyDone to
587  * the server. Drain any messages, so we don't miss a last-minute
588  * ErrorResponse. The walsender stops generating XLogData records once
589  * it sees CopyDone, so expect this to finish quickly. After CopyDone,
590  * it's too late for sendFeedback(), even if this were to take a long
591  * time. Hence, use synchronous-mode PQgetCopyData().
592  */
593  while (1)
594  {
595  int r;
596 
597  if (copybuf != NULL)
598  {
599  PQfreemem(copybuf);
600  copybuf = NULL;
601  }
602  r = PQgetCopyData(conn, &copybuf, 0);
603  if (r == -1)
604  break;
605  if (r == -2)
606  {
607  pg_log_error("could not read COPY data: %s",
609  time_to_abort = false; /* unclean exit */
610  goto error;
611  }
612  }
613 
614  res = PQgetResult(conn);
615  }
616  if (PQresultStatus(res) != PGRES_COMMAND_OK)
617  {
618  pg_log_error("unexpected termination of replication stream: %s",
619  PQresultErrorMessage(res));
620  goto error;
621  }
622  PQclear(res);
623 
624  if (outfd != -1 && strcmp(outfile, "-") != 0)
625  {
627 
628  /* no need to jump to error on failure here, we're finishing anyway */
629  OutputFsync(t);
630 
631  if (close(outfd) != 0)
632  pg_log_error("could not close file \"%s\": %m", outfile);
633  }
634  outfd = -1;
635 error:
636  if (copybuf != NULL)
637  {
638  PQfreemem(copybuf);
639  copybuf = NULL;
640  }
641  destroyPQExpBuffer(query);
642  PQfinish(conn);
643  conn = NULL;
644 }
static XLogRecPtr output_fsync_lsn
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6672
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
TimestampTz feGetCurrentTimestamp(void)
Definition: streamutil.c:608
static volatile sig_atomic_t time_to_abort
static void error(void)
Definition: sql-dyntest.c:147
static int fsync_interval
#define write(a, b, c)
Definition: win32.h:14
static bool OutputFsync(TimestampTz now)
int64 TimestampTz
Definition: timestamp.h:39
#define pg_log_error(...)
Definition: logging.h:79
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:369
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4170
static char * replication_slot
static bool flushAndSendFeedback(PGconn *conn, TimestampTz *now)
bool feTimestampDifferenceExceeds(TimestampTz start_time, TimestampTz stop_time, int msec)
Definition: streamutil.c:649
#define PG_BINARY
Definition: c.h:1211
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2692
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2473
static int verbose
static XLogRecPtr endpos
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:116
static bool output_isfile
#define S_IWUSR
Definition: win32_port.h:263
PGconn * conn
Definition: streamutil.c:54
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:267
void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs)
Definition: streamutil.c:627
static int outfd
#define select(n, r, w, e, timeout)
Definition: win32_port.h:436
unsigned int uint32
Definition: c.h:374
static XLogRecPtr output_written_lsn
static bool output_needs_fsync
#define S_ISREG(m)
Definition: win32_port.h:299
static void prepareToTerminate(PGconn *conn, XLogRecPtr endpos, bool keepalive, XLogRecPtr lsn)
#define stat(a, b)
Definition: win32_port.h:255
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:74
static bool sendFeedback(PGconn *conn, TimestampTz now, bool force, bool replyRequested)
int PQconsumeInput(PGconn *conn)
Definition: fe-exec.c:1704
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:380
StringInfo copybuf
Definition: tablesync.c:108
void PQclear(PGresult *res)
Definition: fe-exec.c:694
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
#define Max(x, y)
Definition: c.h:921
uint64 XLogRecPtr
Definition: xlogdefs.h:21
static XLogRecPtr startpos
#define S_IRUSR
Definition: win32_port.h:260
char * PQresultErrorMessage(const PGresult *res)
Definition: fe-exec.c:2708
int64 fe_recvint64(char *buf)
Definition: streamutil.c:673
int i
static TimestampTz output_last_fsync
PGresult * PQexec(PGconn *conn, const char *query)
Definition: fe-exec.c:1939
static char * outfile
#define close(a)
Definition: win32.h:12
#define EINTR
Definition: win32_port.h:323
void resetPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:148
static size_t noptions
void PQfreemem(void *ptr)
Definition: fe-exec.c:3296
int PQsocket(const PGconn *conn)
Definition: fe-connect.c:6690
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1538
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1778
#define pg_log_info(...)
Definition: logging.h:87
static volatile sig_atomic_t output_reopen
static int standby_message_timeout

◆ usage()

static void usage ( void  )
static

Definition at line 71 of file pg_recvlogical.c.

References _, fsync_interval, plugin, printf, progname, and standby_message_timeout.

Referenced by main().

72 {
73  printf(_("%s controls PostgreSQL logical decoding streams.\n\n"),
74  progname);
75  printf(_("Usage:\n"));
76  printf(_(" %s [OPTION]...\n"), progname);
77  printf(_("\nAction to be performed:\n"));
78  printf(_(" --create-slot create a new replication slot (for the slot's name see --slot)\n"));
79  printf(_(" --drop-slot drop the replication slot (for the slot's name see --slot)\n"));
80  printf(_(" --start start streaming in a replication slot (for the slot's name see --slot)\n"));
81  printf(_("\nOptions:\n"));
82  printf(_(" -E, --endpos=LSN exit after receiving the specified LSN\n"));
83  printf(_(" -f, --file=FILE receive log into this file, - for stdout\n"));
84  printf(_(" -F --fsync-interval=SECS\n"
85  " time between fsyncs to the output file (default: %d)\n"), (fsync_interval / 1000));
86  printf(_(" --if-not-exists do not error if slot already exists when creating a slot\n"));
87  printf(_(" -I, --startpos=LSN where in an existing slot should the streaming start\n"));
88  printf(_(" -n, --no-loop do not loop on connection lost\n"));
89  printf(_(" -o, --option=NAME[=VALUE]\n"
90  " pass option NAME with optional value VALUE to the\n"
91  " output plugin\n"));
92  printf(_(" -P, --plugin=PLUGIN use output plugin PLUGIN (default: %s)\n"), plugin);
93  printf(_(" -s, --status-interval=SECS\n"
94  " time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
95  printf(_(" -S, --slot=SLOTNAME name of the logical replication slot\n"));
96  printf(_(" -v, --verbose output verbose messages\n"));
97  printf(_(" -V, --version output version information, then exit\n"));
98  printf(_(" -?, --help show this help, then exit\n"));
99  printf(_("\nConnection options:\n"));
100  printf(_(" -d, --dbname=DBNAME database to connect to\n"));
101  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
102  printf(_(" -p, --port=PORT database server port number\n"));
103  printf(_(" -U, --username=NAME connect as specified database user\n"));
104  printf(_(" -w, --no-password never prompt for password\n"));
105  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
106  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
107  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
108 }
static const char * plugin
static int fsync_interval
#define printf(...)
Definition: port.h:199
const char * progname
Definition: pg_standby.c:36
#define _(x)
Definition: elog.c:88
static int standby_message_timeout

Variable Documentation

◆ do_create_slot

bool do_create_slot = false
static

Definition at line 43 of file pg_recvlogical.c.

Referenced by main().

◆ do_drop_slot

bool do_drop_slot = false
static

Definition at line 46 of file pg_recvlogical.c.

Referenced by main().

◆ do_start_slot

bool do_start_slot = false
static

Definition at line 45 of file pg_recvlogical.c.

Referenced by main().

◆ endpos

XLogRecPtr endpos = InvalidXLogRecPtr
static

Definition at line 42 of file pg_recvlogical.c.

Referenced by main(), and StreamLogicalLog().

◆ fsync_interval

int fsync_interval = 10 * 1000
static

Definition at line 40 of file pg_recvlogical.c.

Referenced by main(), OutputFsync(), StreamLogicalLog(), and usage().

◆ noloop

int noloop = 0
static

Definition at line 38 of file pg_recvlogical.c.

Referenced by main().

◆ noptions

◆ options

◆ outfd

int outfd = -1
static

Definition at line 55 of file pg_recvlogical.c.

Referenced by OutputFsync(), and StreamLogicalLog().

◆ outfile

char* outfile = NULL
static

◆ output_fsync_lsn

XLogRecPtr output_fsync_lsn = InvalidXLogRecPtr
static

Definition at line 62 of file pg_recvlogical.c.

Referenced by OutputFsync(), sendFeedback(), and StreamLogicalLog().

◆ output_isfile

bool output_isfile
static

Definition at line 58 of file pg_recvlogical.c.

Referenced by OutputFsync(), and StreamLogicalLog().

◆ output_last_fsync

TimestampTz output_last_fsync = -1
static

Definition at line 59 of file pg_recvlogical.c.

Referenced by OutputFsync(), and StreamLogicalLog().

◆ output_needs_fsync

bool output_needs_fsync = false
static

Definition at line 60 of file pg_recvlogical.c.

Referenced by OutputFsync(), and StreamLogicalLog().

◆ output_reopen

volatile sig_atomic_t output_reopen = false
static

Definition at line 57 of file pg_recvlogical.c.

Referenced by sighup_handler(), and StreamLogicalLog().

◆ output_written_lsn

XLogRecPtr output_written_lsn = InvalidXLogRecPtr
static

Definition at line 61 of file pg_recvlogical.c.

Referenced by OutputFsync(), sendFeedback(), and StreamLogicalLog().

◆ plugin

const char* plugin = "test_decoding"
static

◆ replication_slot

char* replication_slot = NULL
static

Definition at line 47 of file pg_recvlogical.c.

Referenced by main(), sendFeedback(), and StreamLogicalLog().

◆ slot_exists_ok

bool slot_exists_ok = false
static

Definition at line 44 of file pg_recvlogical.c.

Referenced by main().

◆ standby_message_timeout

int standby_message_timeout = 10 * 1000
static

Definition at line 39 of file pg_recvlogical.c.

Referenced by main(), StreamLogicalLog(), and usage().

◆ startpos

◆ time_to_abort

volatile sig_atomic_t time_to_abort = false
static

Definition at line 56 of file pg_recvlogical.c.

Referenced by main(), sigint_handler(), and StreamLogicalLog().

◆ verbose

int verbose = 0
static

Definition at line 37 of file pg_recvlogical.c.

Referenced by main(), prepareToTerminate(), sendFeedback(), and StreamLogicalLog().