pg_createsubscriber.c

Go to the documentation of this file.

/*-------------------------------------------------------------------------
 *
 * pg_createsubscriber.c
 *    Create a new logical replica from a standby server
 *
 * Copyright (C) 2024, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *    src/bin/pg_basebackup/pg_createsubscriber.c
 *
 *-------------------------------------------------------------------------
 */
 
#include "postgres_fe.h"
 
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <time.h>
 
#include "common/connect.h"
#include "common/controldata_utils.h"
#include "common/logging.h"
#include "common/pg_prng.h"
#include "common/restricted_token.h"
#include "fe_utils/recovery_gen.h"
#include "fe_utils/simple_list.h"
#include "fe_utils/string_utils.h"
#include "getopt_long.h"
 
#define DEFAULT_SUB_PORT    "50432"
 
/* Command-line options */
struct CreateSubscriberOptions
{
    char       *config_file;    /* configuration file */
    char       *pub_conninfo_str;   /* publisher connection string */
    char       *socket_dir;     /* directory for Unix-domain socket, if any */
    char       *sub_port;       /* subscriber port number */
    const char *sub_username;   /* subscriber username */
    SimpleStringList database_names;    /* list of database names */
    SimpleStringList pub_names; /* list of publication names */
    SimpleStringList sub_names; /* list of subscription names */
    SimpleStringList replslot_names;    /* list of replication slot names */
    int         recovery_timeout;   /* stop recovery after this time */
};
 
struct LogicalRepInfo
{
    char       *dbname;         /* database name */
    char       *pubconninfo;    /* publisher connection string */
    char       *subconninfo;    /* subscriber connection string */
    char       *pubname;        /* publication name */
    char       *subname;        /* subscription name */
    char       *replslotname;   /* replication slot name */
 
    bool        made_replslot;  /* replication slot was created */
    bool        made_publication;   /* publication was created */
};
 
static void cleanup_objects_atexit(void);
static void usage();
static char *get_base_conninfo(const char *conninfo, char **dbname);
static char *get_sub_conninfo(const struct CreateSubscriberOptions *opt);
static char *get_exec_path(const char *argv0, const char *progname);
static void check_data_directory(const char *datadir);
static char *concat_conninfo_dbname(const char *conninfo, const char *dbname);
static struct LogicalRepInfo *store_pub_sub_info(const struct CreateSubscriberOptions *opt,
                                                 const char *pub_base_conninfo,
                                                 const char *sub_base_conninfo);
static PGconn *connect_database(const char *conninfo, bool exit_on_error);
static void disconnect_database(PGconn *conn, bool exit_on_error);
static uint64 get_primary_sysid(const char *conninfo);
static uint64 get_standby_sysid(const char *datadir);
static void modify_subscriber_sysid(const struct CreateSubscriberOptions *opt);
static bool server_is_in_recovery(PGconn *conn);
static char *generate_object_name(PGconn *conn);
static void check_publisher(const struct LogicalRepInfo *dbinfo);
static char *setup_publisher(struct LogicalRepInfo *dbinfo);
static void check_subscriber(const struct LogicalRepInfo *dbinfo);
static void setup_subscriber(struct LogicalRepInfo *dbinfo,
                             const char *consistent_lsn);
static void setup_recovery(const struct LogicalRepInfo *dbinfo, const char *datadir,
                           const char *lsn);
static void drop_primary_replication_slot(struct LogicalRepInfo *dbinfo,
                                          const char *slotname);
static void drop_failover_replication_slots(struct LogicalRepInfo *dbinfo);
static char *create_logical_replication_slot(PGconn *conn,
                                             struct LogicalRepInfo *dbinfo);
static void drop_replication_slot(PGconn *conn, struct LogicalRepInfo *dbinfo,
                                  const char *slot_name);
static void pg_ctl_status(const char *pg_ctl_cmd, int rc);
static void start_standby_server(const struct CreateSubscriberOptions *opt,
                                 bool restricted_access,
                                 bool restrict_logical_worker);
static void stop_standby_server(const char *datadir);
static void wait_for_end_recovery(const char *conninfo,
                                  const struct CreateSubscriberOptions *opt);
static void create_publication(PGconn *conn, struct LogicalRepInfo *dbinfo);
static void drop_publication(PGconn *conn, struct LogicalRepInfo *dbinfo);
static void create_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo);
static void set_replication_progress(PGconn *conn, const struct LogicalRepInfo *dbinfo,
                                     const char *lsn);
static void enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo);
static void check_and_drop_existing_subscriptions(PGconn *conn,
                                                  const struct LogicalRepInfo *dbinfo);
static void drop_existing_subscriptions(PGconn *conn, const char *subname,
                                        const char *dbname);
 
#define USEC_PER_SEC    1000000
#define WAIT_INTERVAL   1       /* 1 second */
 
static const char *progname;
 
static char *primary_slot_name = NULL;
static bool dry_run = false;
 
static bool success = false;
 
static struct LogicalRepInfo *dbinfo;
static int  num_dbs = 0;        /* number of specified databases */
static int  num_pubs = 0;       /* number of specified publications */
static int  num_subs = 0;       /* number of specified subscriptions */
static int  num_replslots = 0;  /* number of specified replication slots */
 
static pg_prng_state prng_state;
 
static char *pg_ctl_path = NULL;
static char *pg_resetwal_path = NULL;
 
/* standby / subscriber data directory */
static char *subscriber_dir = NULL;
 
static bool recovery_ended = false;
static bool standby_running = false;
 
enum WaitPMResult
{
    POSTMASTER_READY,
    POSTMASTER_STILL_STARTING
};
 
 
/*
 * Cleanup objects that were created by pg_createsubscriber if there is an
 * error.
 *
 * Publications and replication slots are created on primary. Depending on the
 * step it failed, it should remove the already created objects if it is
 * possible (sometimes it won't work due to a connection issue).
 * There is no cleanup on the target server. The steps on the target server are
 * executed *after* promotion, hence, at this point, a failure means recreate
 * the physical replica and start again.
 */
static void
cleanup_objects_atexit(void)
{
    if (success)
        return;
 
    /*
     * If the server is promoted, there is no way to use the current setup
     * again. Warn the user that a new replication setup should be done before
     * trying again.
     */
    if (recovery_ended)
    {
        pg_log_warning("failed after the end of recovery");
        pg_log_warning_hint("The target server cannot be used as a physical replica anymore.  "
                            "You must recreate the physical replica before continuing.");
    }
 
    for (int i = 0; i < num_dbs; i++)
    {
        if (dbinfo[i].made_publication || dbinfo[i].made_replslot)
        {
            PGconn     *conn;
 
            conn = connect_database(dbinfo[i].pubconninfo, false);
            if (conn != NULL)
            {
                if (dbinfo[i].made_publication)
                    drop_publication(conn, &dbinfo[i]);
                if (dbinfo[i].made_replslot)
                    drop_replication_slot(conn, &dbinfo[i], dbinfo[i].replslotname);
                disconnect_database(conn, false);
            }
            else
            {
                /*
                 * If a connection could not be established, inform the user
                 * that some objects were left on primary and should be
                 * removed before trying again.
                 */
                if (dbinfo[i].made_publication)
                {
                    pg_log_warning("publication \"%s\" created in database \"%s\" on primary was left behind",
                                   dbinfo[i].pubname, dbinfo[i].dbname);
                    pg_log_warning_hint("Drop this publication before trying again.");
                }
                if (dbinfo[i].made_replslot)
                {
                    pg_log_warning("replication slot \"%s\" created in database \"%s\" on primary was left behind",
                                   dbinfo[i].replslotname, dbinfo[i].dbname);
                    pg_log_warning_hint("Drop this replication slot soon to avoid retention of WAL files.");
                }
            }
        }
    }
 
    if (standby_running)
        stop_standby_server(subscriber_dir);
}
 
static void
usage(void)
{
    printf(_("%s creates a new logical replica from a standby server.\n\n"),
           progname);
    printf(_("Usage:\n"));
    printf(_("  %s [OPTION]...\n"), progname);
    printf(_("\nOptions:\n"));
    printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
    printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
    printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
    printf(_("  -p, --subscriber-port=PORT      subscriber port number (default %s)\n"), DEFAULT_SUB_PORT);
    printf(_("  -P, --publisher-server=CONNSTR  publisher connection string\n"));
    printf(_("  -s, --socketdir=DIR             socket directory to use (default current dir.)\n"));
    printf(_("  -t, --recovery-timeout=SECS     seconds to wait for recovery to end\n"));
    printf(_("  -U, --subscriber-username=NAME  user name for subscriber connection\n"));
    printf(_("  -v, --verbose                   output verbose messages\n"));
    printf(_("      --config-file=FILENAME      use specified main server configuration\n"
             "                                  file when running target cluster\n"));
    printf(_("      --publication=NAME          publication name\n"));
    printf(_("      --replication-slot=NAME     replication slot name\n"));
    printf(_("      --subscription=NAME         subscription name\n"));
    printf(_("  -V, --version                   output version information, then exit\n"));
    printf(_("  -?, --help                      show this help, then exit\n"));
    printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
    printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
 
/*
 * Subroutine to append "keyword=value" to a connection string,
 * with proper quoting of the value.  (We assume keywords don't need that.)
 */
static void
appendConnStrItem(PQExpBuffer buf, const char *keyword, const char *val)
{
    if (buf->len > 0)
        appendPQExpBufferChar(buf, ' ');
    appendPQExpBufferStr(buf, keyword);
    appendPQExpBufferChar(buf, '=');
    appendConnStrVal(buf, val);
}
 
/*
 * Validate a connection string. Returns a base connection string that is a
 * connection string without a database name.
 *
 * Since we might process multiple databases, each database name will be
 * appended to this base connection string to provide a final connection
 * string. If the second argument (dbname) is not null, returns dbname if the
 * provided connection string contains it.
 *
 * It is the caller's responsibility to free the returned connection string and
 * dbname.
 */
static char *
get_base_conninfo(const char *conninfo, char **dbname)
{
    PQExpBuffer buf;
    PQconninfoOption *conn_opts;
    PQconninfoOption *conn_opt;
    char       *errmsg = NULL;
    char       *ret;
 
    conn_opts = PQconninfoParse(conninfo, &errmsg);
    if (conn_opts == NULL)
    {
        pg_log_error("could not parse connection string: %s", errmsg);
        PQfreemem(errmsg);
        return NULL;
    }
 
    buf = createPQExpBuffer();
    for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
    {
        if (conn_opt->val != NULL && conn_opt->val[0] != '\0')
        {
            if (strcmp(conn_opt->keyword, "dbname") == 0)
            {
                if (dbname)
                    *dbname = pg_strdup(conn_opt->val);
                continue;
            }
            appendConnStrItem(buf, conn_opt->keyword, conn_opt->val);
        }
    }
 
    ret = pg_strdup(buf->data);
 
    destroyPQExpBuffer(buf);
    PQconninfoFree(conn_opts);
 
    return ret;
}
 
/*
 * Build a subscriber connection string. Only a few parameters are supported
 * since it starts a server with restricted access.
 */
static char *
get_sub_conninfo(const struct CreateSubscriberOptions *opt)
{
    PQExpBuffer buf = createPQExpBuffer();
    char       *ret;
 
    appendConnStrItem(buf, "port", opt->sub_port);
#if !defined(WIN32)
    appendConnStrItem(buf, "host", opt->socket_dir);
#endif
    if (opt->sub_username != NULL)
        appendConnStrItem(buf, "user", opt->sub_username);
    appendConnStrItem(buf, "fallback_application_name", progname);
 
    ret = pg_strdup(buf->data);
 
    destroyPQExpBuffer(buf);
 
    return ret;
}
 
/*
 * Verify if a PostgreSQL binary (progname) is available in the same directory as
 * pg_createsubscriber and it has the same version.  It returns the absolute
 * path of the progname.
 */
static char *
get_exec_path(const char *argv0, const char *progname)
{
    char       *versionstr;
    char       *exec_path;
    int         ret;
 
    versionstr = psprintf("%s (PostgreSQL) %s\n", progname, PG_VERSION);
    exec_path = pg_malloc(MAXPGPATH);
    ret = find_other_exec(argv0, progname, versionstr, exec_path);
 
    if (ret < 0)
    {
        char        full_path[MAXPGPATH];
 
        if (find_my_exec(argv0, full_path) < 0)
            strlcpy(full_path, progname, sizeof(full_path));
 
        if (ret == -1)
            pg_fatal("program \"%s\" is needed by %s but was not found in the same directory as \"%s\"",
                     progname, "pg_createsubscriber", full_path);
        else
            pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s",
                     progname, full_path, "pg_createsubscriber");
    }
 
    pg_log_debug("%s path is:  %s", progname, exec_path);
 
    return exec_path;
}
 
/*
 * Is it a cluster directory? These are preliminary checks. It is far from
 * making an accurate check. If it is not a clone from the publisher, it will
 * eventually fail in a future step.
 */
static void
check_data_directory(const char *datadir)
{
    struct stat statbuf;
    char        versionfile[MAXPGPATH];
 
    pg_log_info("checking if directory \"%s\" is a cluster data directory",
                datadir);
 
    if (stat(datadir, &statbuf) != 0)
    {
        if (errno == ENOENT)
            pg_fatal("data directory \"%s\" does not exist", datadir);
        else
            pg_fatal("could not access directory \"%s\": %m", datadir);
    }
 
    snprintf(versionfile, MAXPGPATH, "%s/PG_VERSION", datadir);
    if (stat(versionfile, &statbuf) != 0 && errno == ENOENT)
    {
        pg_fatal("directory \"%s\" is not a database cluster directory",
                 datadir);
    }
}
 
/*
 * Append database name into a base connection string.
 *
 * dbname is the only parameter that changes so it is not included in the base
 * connection string. This function concatenates dbname to build a "real"
 * connection string.
 */
static char *
concat_conninfo_dbname(const char *conninfo, const char *dbname)
{
    PQExpBuffer buf = createPQExpBuffer();
    char       *ret;
 
    Assert(conninfo != NULL);
 
    appendPQExpBufferStr(buf, conninfo);
    appendConnStrItem(buf, "dbname", dbname);
 
    ret = pg_strdup(buf->data);
    destroyPQExpBuffer(buf);
 
    return ret;
}
 
/*
 * Store publication and subscription information.
 *
 * If publication, replication slot and subscription names were specified,
 * store it here. Otherwise, a generated name will be assigned to the object in
 * setup_publisher().
 */
static struct LogicalRepInfo *
store_pub_sub_info(const struct CreateSubscriberOptions *opt,
                   const char *pub_base_conninfo,
                   const char *sub_base_conninfo)
{
    struct LogicalRepInfo *dbinfo;
    SimpleStringListCell *pubcell = NULL;
    SimpleStringListCell *subcell = NULL;
    SimpleStringListCell *replslotcell = NULL;
    int         i = 0;
 
    dbinfo = pg_malloc_array(struct LogicalRepInfo, num_dbs);
 
    if (num_pubs > 0)
        pubcell = opt->pub_names.head;
    if (num_subs > 0)
        subcell = opt->sub_names.head;
    if (num_replslots > 0)
        replslotcell = opt->replslot_names.head;
 
    for (SimpleStringListCell *cell = opt->database_names.head; cell; cell = cell->next)
    {
        char       *conninfo;
 
        /* Fill publisher attributes */
        conninfo = concat_conninfo_dbname(pub_base_conninfo, cell->val);
        dbinfo[i].pubconninfo = conninfo;
        dbinfo[i].dbname = cell->val;
        if (num_pubs > 0)
            dbinfo[i].pubname = pubcell->val;
        else
            dbinfo[i].pubname = NULL;
        if (num_replslots > 0)
            dbinfo[i].replslotname = replslotcell->val;
        else
            dbinfo[i].replslotname = NULL;
        dbinfo[i].made_replslot = false;
        dbinfo[i].made_publication = false;
        /* Fill subscriber attributes */
        conninfo = concat_conninfo_dbname(sub_base_conninfo, cell->val);
        dbinfo[i].subconninfo = conninfo;
        if (num_subs > 0)
            dbinfo[i].subname = subcell->val;
        else
            dbinfo[i].subname = NULL;
        /* Other fields will be filled later */
 
        pg_log_debug("publisher(%d): publication: %s ; replication slot: %s ; connection string: %s", i,
                     dbinfo[i].pubname ? dbinfo[i].pubname : "(auto)",
                     dbinfo[i].replslotname ? dbinfo[i].replslotname : "(auto)",
                     dbinfo[i].pubconninfo);
        pg_log_debug("subscriber(%d): subscription: %s ; connection string: %s", i,
                     dbinfo[i].subname ? dbinfo[i].subname : "(auto)",
                     dbinfo[i].subconninfo);
 
        if (num_pubs > 0)
            pubcell = pubcell->next;
        if (num_subs > 0)
            subcell = subcell->next;
        if (num_replslots > 0)
            replslotcell = replslotcell->next;
 
        i++;
    }
 
    return dbinfo;
}
 
/*
 * Open a new connection. If exit_on_error is true, it has an undesired
 * condition and it should exit immediately.
 */
static PGconn *
connect_database(const char *conninfo, bool exit_on_error)
{
    PGconn     *conn;
    PGresult   *res;
 
    conn = PQconnectdb(conninfo);
    if (PQstatus(conn) != CONNECTION_OK)
    {
        pg_log_error("connection to database failed: %s",
                     PQerrorMessage(conn));
        PQfinish(conn);
 
        if (exit_on_error)
            exit(1);
        return NULL;
    }
 
    /* Secure search_path */
    res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL);
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not clear \"search_path\": %s",
                     PQresultErrorMessage(res));
        PQclear(res);
        PQfinish(conn);
 
        if (exit_on_error)
            exit(1);
        return NULL;
    }
    PQclear(res);
 
    return conn;
}
 
/*
 * Close the connection. If exit_on_error is true, it has an undesired
 * condition and it should exit immediately.
 */
static void
disconnect_database(PGconn *conn, bool exit_on_error)
{
    Assert(conn != NULL);
 
    PQfinish(conn);
 
    if (exit_on_error)
        exit(1);
}
 
/*
 * Obtain the system identifier using the provided connection. It will be used
 * to compare if a data directory is a clone of another one.
 */
static uint64
get_primary_sysid(const char *conninfo)
{
    PGconn     *conn;
    PGresult   *res;
    uint64      sysid;
 
    pg_log_info("getting system identifier from publisher");
 
    conn = connect_database(conninfo, true);
 
    res = PQexec(conn, "SELECT system_identifier FROM pg_catalog.pg_control_system()");
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not get system identifier: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
    if (PQntuples(res) != 1)
    {
        pg_log_error("could not get system identifier: got %d rows, expected %d row",
                     PQntuples(res), 1);
        disconnect_database(conn, true);
    }
 
    sysid = strtou64(PQgetvalue(res, 0, 0), NULL, 10);
 
    pg_log_info("system identifier is %llu on publisher",
                (unsigned long long) sysid);
 
    PQclear(res);
    disconnect_database(conn, false);
 
    return sysid;
}
 
/*
 * Obtain the system identifier from control file. It will be used to compare
 * if a data directory is a clone of another one. This routine is used locally
 * and avoids a connection.
 */
static uint64
get_standby_sysid(const char *datadir)
{
    ControlFileData *cf;
    bool        crc_ok;
    uint64      sysid;
 
    pg_log_info("getting system identifier from subscriber");
 
    cf = get_controlfile(datadir, &crc_ok);
    if (!crc_ok)
        pg_fatal("control file appears to be corrupt");
 
    sysid = cf->system_identifier;
 
    pg_log_info("system identifier is %llu on subscriber",
                (unsigned long long) sysid);
 
    pg_free(cf);
 
    return sysid;
}
 
/*
 * Modify the system identifier. Since a standby server preserves the system
 * identifier, it makes sense to change it to avoid situations in which WAL
 * files from one of the systems might be used in the other one.
 */
static void
modify_subscriber_sysid(const struct CreateSubscriberOptions *opt)
{
    ControlFileData *cf;
    bool        crc_ok;
    struct timeval tv;
 
    char       *cmd_str;
 
    pg_log_info("modifying system identifier of subscriber");
 
    cf = get_controlfile(subscriber_dir, &crc_ok);
    if (!crc_ok)
        pg_fatal("control file appears to be corrupt");
 
    /*
     * Select a new system identifier.
     *
     * XXX this code was extracted from BootStrapXLOG().
     */
    gettimeofday(&tv, NULL);
    cf->system_identifier = ((uint64) tv.tv_sec) << 32;
    cf->system_identifier |= ((uint64) tv.tv_usec) << 12;
    cf->system_identifier |= getpid() & 0xFFF;
 
    if (!dry_run)
        update_controlfile(subscriber_dir, cf, true);
 
    pg_log_info("system identifier is %llu on subscriber",
                (unsigned long long) cf->system_identifier);
 
    pg_log_info("running pg_resetwal on the subscriber");
 
    cmd_str = psprintf("\"%s\" -D \"%s\" > \"%s\"", pg_resetwal_path,
                       subscriber_dir, DEVNULL);
 
    pg_log_debug("pg_resetwal command is: %s", cmd_str);
 
    if (!dry_run)
    {
        int         rc = system(cmd_str);
 
        if (rc == 0)
            pg_log_info("subscriber successfully changed the system identifier");
        else
            pg_fatal("could not change system identifier of subscriber: %s", wait_result_to_str(rc));
    }
 
    pg_free(cf);
}
 
/*
 * Generate an object name using a prefix, database oid and a random integer.
 * It is used in case the user does not specify an object name (publication,
 * subscription, replication slot).
 */
static char *
generate_object_name(PGconn *conn)
{
    PGresult   *res;
    Oid         oid;
    uint32      rand;
    char       *objname;
 
    res = PQexec(conn,
                 "SELECT oid FROM pg_catalog.pg_database "
                 "WHERE datname = pg_catalog.current_database()");
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain database OID: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    if (PQntuples(res) != 1)
    {
        pg_log_error("could not obtain database OID: got %d rows, expected %d row",
                     PQntuples(res), 1);
        disconnect_database(conn, true);
    }
 
    /* Database OID */
    oid = strtoul(PQgetvalue(res, 0, 0), NULL, 10);
 
    PQclear(res);
 
    /* Random unsigned integer */
    rand = pg_prng_uint32(&prng_state);
 
    /*
     * Build the object name. The name must not exceed NAMEDATALEN - 1. This
     * current schema uses a maximum of 40 characters (20 + 10 + 1 + 8 +
     * '\0').
     */
    objname = psprintf("pg_createsubscriber_%u_%x", oid, rand);
 
    return objname;
}
 
/*
 * Create the publications and replication slots in preparation for logical
 * replication. Returns the LSN from latest replication slot. It will be the
 * replication start point that is used to adjust the subscriptions (see
 * set_replication_progress).
 */
static char *
setup_publisher(struct LogicalRepInfo *dbinfo)
{
    char       *lsn = NULL;
 
    pg_prng_seed(&prng_state, (uint64) (getpid() ^ time(NULL)));
 
    for (int i = 0; i < num_dbs; i++)
    {
        PGconn     *conn;
        char       *genname = NULL;
 
        conn = connect_database(dbinfo[i].pubconninfo, true);
 
        /*
         * If an object name was not specified as command-line options, assign
         * a generated object name. The replication slot has a different rule.
         * The subscription name is assigned to the replication slot name if
         * no replication slot is specified. It follows the same rule as
         * CREATE SUBSCRIPTION.
         */
        if (num_pubs == 0 || num_subs == 0 || num_replslots == 0)
            genname = generate_object_name(conn);
        if (num_pubs == 0)
            dbinfo[i].pubname = pg_strdup(genname);
        if (num_subs == 0)
            dbinfo[i].subname = pg_strdup(genname);
        if (num_replslots == 0)
            dbinfo[i].replslotname = pg_strdup(dbinfo[i].subname);
 
        /*
         * Create publication on publisher. This step should be executed
         * *before* promoting the subscriber to avoid any transactions between
         * consistent LSN and the new publication rows (such transactions
         * wouldn't see the new publication rows resulting in an error).
         */
        create_publication(conn, &dbinfo[i]);
 
        /* Create replication slot on publisher */
        if (lsn)
            pg_free(lsn);
        lsn = create_logical_replication_slot(conn, &dbinfo[i]);
        if (lsn != NULL || dry_run)
            pg_log_info("create replication slot \"%s\" on publisher",
                        dbinfo[i].replslotname);
        else
            exit(1);
 
        /*
         * Since we are using the LSN returned by the last replication slot as
         * recovery_target_lsn, this LSN is ahead of the current WAL position
         * and the recovery waits until the publisher writes a WAL record to
         * reach the target and ends the recovery. On idle systems, this wait
         * time is unpredictable and could lead to failure in promoting the
         * subscriber. To avoid that, insert a harmless WAL record.
         */
        if (i == num_dbs - 1 && !dry_run)
        {
            PGresult   *res;
 
            res = PQexec(conn, "SELECT pg_log_standby_snapshot()");
            if (PQresultStatus(res) != PGRES_TUPLES_OK)
            {
                pg_log_error("could not write an additional WAL record: %s",
                             PQresultErrorMessage(res));
                disconnect_database(conn, true);
            }
            PQclear(res);
        }
 
        disconnect_database(conn, false);
    }
 
    return lsn;
}
 
/*
 * Is recovery still in progress?
 */
static bool
server_is_in_recovery(PGconn *conn)
{
    PGresult   *res;
    int         ret;
 
    res = PQexec(conn, "SELECT pg_catalog.pg_is_in_recovery()");
 
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain recovery progress: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
 
    ret = strcmp("t", PQgetvalue(res, 0, 0));
 
    PQclear(res);
 
    return ret == 0;
}
 
/*
 * Is the primary server ready for logical replication?
 *
 * XXX Does it not allow a synchronous replica?
 */
static void
check_publisher(const struct LogicalRepInfo *dbinfo)
{
    PGconn     *conn;
    PGresult   *res;
    bool        failed = false;
 
    char       *wal_level;
    int         max_repslots;
    int         cur_repslots;
    int         max_walsenders;
    int         cur_walsenders;
    int         max_prepared_transactions;
 
    pg_log_info("checking settings on publisher");
 
    conn = connect_database(dbinfo[0].pubconninfo, true);
 
    /*
     * If the primary server is in recovery (i.e. cascading replication),
     * objects (publication) cannot be created because it is read only.
     */
    if (server_is_in_recovery(conn))
    {
        pg_log_error("primary server cannot be in recovery");
        disconnect_database(conn, true);
    }
 
    /*------------------------------------------------------------------------
     * Logical replication requires a few parameters to be set on publisher.
     * Since these parameters are not a requirement for physical replication,
     * we should check it to make sure it won't fail.
     *
     * - wal_level = logical
     * - max_replication_slots >= current + number of dbs to be converted
     * - max_wal_senders >= current + number of dbs to be converted
     * -----------------------------------------------------------------------
     */
    res = PQexec(conn,
                 "SELECT pg_catalog.current_setting('wal_level'),"
                 " pg_catalog.current_setting('max_replication_slots'),"
                 " (SELECT count(*) FROM pg_catalog.pg_replication_slots),"
                 " pg_catalog.current_setting('max_wal_senders'),"
                 " (SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE backend_type = 'walsender'),"
                 " pg_catalog.current_setting('max_prepared_transactions')");
 
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain publisher settings: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    wal_level = pg_strdup(PQgetvalue(res, 0, 0));
    max_repslots = atoi(PQgetvalue(res, 0, 1));
    cur_repslots = atoi(PQgetvalue(res, 0, 2));
    max_walsenders = atoi(PQgetvalue(res, 0, 3));
    cur_walsenders = atoi(PQgetvalue(res, 0, 4));
    max_prepared_transactions = atoi(PQgetvalue(res, 0, 5));
 
    PQclear(res);
 
    pg_log_debug("publisher: wal_level: %s", wal_level);
    pg_log_debug("publisher: max_replication_slots: %d", max_repslots);
    pg_log_debug("publisher: current replication slots: %d", cur_repslots);
    pg_log_debug("publisher: max_wal_senders: %d", max_walsenders);
    pg_log_debug("publisher: current wal senders: %d", cur_walsenders);
    pg_log_debug("publisher: max_prepared_transactions: %d",
                 max_prepared_transactions);
 
    disconnect_database(conn, false);
 
    if (strcmp(wal_level, "logical") != 0)
    {
        pg_log_error("publisher requires \"wal_level\" >= \"logical\"");
        failed = true;
    }
 
    if (max_repslots - cur_repslots < num_dbs)
    {
        pg_log_error("publisher requires %d replication slots, but only %d remain",
                     num_dbs, max_repslots - cur_repslots);
        pg_log_error_hint("Increase the configuration parameter \"%s\" to at least %d.",
                          "max_replication_slots", cur_repslots + num_dbs);
        failed = true;
    }
 
    if (max_walsenders - cur_walsenders < num_dbs)
    {
        pg_log_error("publisher requires %d WAL sender processes, but only %d remain",
                     num_dbs, max_walsenders - cur_walsenders);
        pg_log_error_hint("Increase the configuration parameter \"%s\" to at least %d.",
                          "max_wal_senders", cur_walsenders + num_dbs);
        failed = true;
    }
 
    if (max_prepared_transactions != 0)
    {
        pg_log_warning("two_phase option will not be enabled for replication slots");
        pg_log_warning_detail("Subscriptions will be created with the two_phase option disabled.  "
                              "Prepared transactions will be replicated at COMMIT PREPARED.");
    }
 
    pg_free(wal_level);
 
    if (failed)
        exit(1);
}
 
/*
 * Is the standby server ready for logical replication?
 *
 * XXX Does it not allow a time-delayed replica?
 *
 * XXX In a cascaded replication scenario (P -> S -> C), if the target server
 * is S, it cannot detect there is a replica (server C) because server S starts
 * accepting only local connections and server C cannot connect to it. Hence,
 * there is not a reliable way to provide a suitable error saying the server C
 * will be broken at the end of this process (due to pg_resetwal).
 */
static void
check_subscriber(const struct LogicalRepInfo *dbinfo)
{
    PGconn     *conn;
    PGresult   *res;
    bool        failed = false;
 
    int         max_lrworkers;
    int         max_repslots;
    int         max_wprocs;
 
    pg_log_info("checking settings on subscriber");
 
    conn = connect_database(dbinfo[0].subconninfo, true);
 
    /* The target server must be a standby */
    if (!server_is_in_recovery(conn))
    {
        pg_log_error("target server must be a standby");
        disconnect_database(conn, true);
    }
 
    /*------------------------------------------------------------------------
     * Logical replication requires a few parameters to be set on subscriber.
     * Since these parameters are not a requirement for physical replication,
     * we should check it to make sure it won't fail.
     *
     * - max_replication_slots >= number of dbs to be converted
     * - max_logical_replication_workers >= number of dbs to be converted
     * - max_worker_processes >= 1 + number of dbs to be converted
     *------------------------------------------------------------------------
     */
    res = PQexec(conn,
                 "SELECT setting FROM pg_catalog.pg_settings WHERE name IN ("
                 "'max_logical_replication_workers', "
                 "'max_replication_slots', "
                 "'max_worker_processes', "
                 "'primary_slot_name') "
                 "ORDER BY name");
 
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain subscriber settings: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    max_lrworkers = atoi(PQgetvalue(res, 0, 0));
    max_repslots = atoi(PQgetvalue(res, 1, 0));
    max_wprocs = atoi(PQgetvalue(res, 2, 0));
    if (strcmp(PQgetvalue(res, 3, 0), "") != 0)
        primary_slot_name = pg_strdup(PQgetvalue(res, 3, 0));
 
    pg_log_debug("subscriber: max_logical_replication_workers: %d",
                 max_lrworkers);
    pg_log_debug("subscriber: max_replication_slots: %d", max_repslots);
    pg_log_debug("subscriber: max_worker_processes: %d", max_wprocs);
    if (primary_slot_name)
        pg_log_debug("subscriber: primary_slot_name: %s", primary_slot_name);
 
    PQclear(res);
 
    disconnect_database(conn, false);
 
    if (max_repslots < num_dbs)
    {
        pg_log_error("subscriber requires %d replication slots, but only %d remain",
                     num_dbs, max_repslots);
        pg_log_error_hint("Increase the configuration parameter \"%s\" to at least %d.",
                          "max_replication_slots", num_dbs);
        failed = true;
    }
 
    if (max_lrworkers < num_dbs)
    {
        pg_log_error("subscriber requires %d logical replication workers, but only %d remain",
                     num_dbs, max_lrworkers);
        pg_log_error_hint("Increase the configuration parameter \"%s\" to at least %d.",
                          "max_logical_replication_workers", num_dbs);
        failed = true;
    }
 
    if (max_wprocs < num_dbs + 1)
    {
        pg_log_error("subscriber requires %d worker processes, but only %d remain",
                     num_dbs + 1, max_wprocs);
        pg_log_error_hint("Increase the configuration parameter \"%s\" to at least %d.",
                          "max_worker_processes", num_dbs + 1);
        failed = true;
    }
 
    if (failed)
        exit(1);
}
 
/*
 * Drop a specified subscription. This is to avoid duplicate subscriptions on
 * the primary (publisher node) and the newly created subscriber. We
 * shouldn't drop the associated slot as that would be used by the publisher
 * node.
 */
static void
drop_existing_subscriptions(PGconn *conn, const char *subname, const char *dbname)
{
    PQExpBuffer query = createPQExpBuffer();
    PGresult   *res;
 
    Assert(conn != NULL);
 
    /*
     * Construct a query string. These commands are allowed to be executed
     * within a transaction.
     */
    appendPQExpBuffer(query, "ALTER SUBSCRIPTION %s DISABLE;",
                      subname);
    appendPQExpBuffer(query, " ALTER SUBSCRIPTION %s SET (slot_name = NONE);",
                      subname);
    appendPQExpBuffer(query, " DROP SUBSCRIPTION %s;", subname);
 
    pg_log_info("dropping subscription \"%s\" in database \"%s\"",
                subname, dbname);
 
    if (!dry_run)
    {
        res = PQexec(conn, query->data);
 
        if (PQresultStatus(res) != PGRES_COMMAND_OK)
        {
            pg_log_error("could not drop subscription \"%s\": %s",
                         subname, PQresultErrorMessage(res));
            disconnect_database(conn, true);
        }
 
        PQclear(res);
    }
 
    destroyPQExpBuffer(query);
}
 
/*
 * Retrieve and drop the pre-existing subscriptions.
 */
static void
check_and_drop_existing_subscriptions(PGconn *conn,
                                      const struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer query = createPQExpBuffer();
    char       *dbname;
    PGresult   *res;
 
    Assert(conn != NULL);
 
    dbname = PQescapeLiteral(conn, dbinfo->dbname, strlen(dbinfo->dbname));
 
    appendPQExpBuffer(query,
                      "SELECT s.subname FROM pg_catalog.pg_subscription s "
                      "INNER JOIN pg_catalog.pg_database d ON (s.subdbid = d.oid) "
                      "WHERE d.datname = %s",
                      dbname);
    res = PQexec(conn, query->data);
 
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain pre-existing subscriptions: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    for (int i = 0; i < PQntuples(res); i++)
        drop_existing_subscriptions(conn, PQgetvalue(res, i, 0),
                                    dbinfo->dbname);
 
    PQclear(res);
    destroyPQExpBuffer(query);
}
 
/*
 * Create the subscriptions, adjust the initial location for logical
 * replication and enable the subscriptions. That's the last step for logical
 * replication setup.
 */
static void
setup_subscriber(struct LogicalRepInfo *dbinfo, const char *consistent_lsn)
{
    for (int i = 0; i < num_dbs; i++)
    {
        PGconn     *conn;
 
        /* Connect to subscriber. */
        conn = connect_database(dbinfo[i].subconninfo, true);
 
        /*
         * We don't need the pre-existing subscriptions on the newly formed
         * subscriber. They can connect to other publisher nodes and either
         * get some unwarranted data or can lead to ERRORs in connecting to
         * such nodes.
         */
        check_and_drop_existing_subscriptions(conn, &dbinfo[i]);
 
        /*
         * Since the publication was created before the consistent LSN, it is
         * available on the subscriber when the physical replica is promoted.
         * Remove publications from the subscriber because it has no use.
         */
        drop_publication(conn, &dbinfo[i]);
 
        create_subscription(conn, &dbinfo[i]);
 
        /* Set the replication progress to the correct LSN */
        set_replication_progress(conn, &dbinfo[i], consistent_lsn);
 
        /* Enable subscription */
        enable_subscription(conn, &dbinfo[i]);
 
        disconnect_database(conn, false);
    }
}
 
/*
 * Write the required recovery parameters.
 */
static void
setup_recovery(const struct LogicalRepInfo *dbinfo, const char *datadir, const char *lsn)
{
    PGconn     *conn;
    PQExpBuffer recoveryconfcontents;
 
    /*
     * Despite of the recovery parameters will be written to the subscriber,
     * use a publisher connection. The primary_conninfo is generated using the
     * connection settings.
     */
    conn = connect_database(dbinfo[0].pubconninfo, true);
 
    /*
     * Write recovery parameters.
     *
     * The subscriber is not running yet. In dry run mode, the recovery
     * parameters *won't* be written. An invalid LSN is used for printing
     * purposes. Additional recovery parameters are added here. It avoids
     * unexpected behavior such as end of recovery as soon as a consistent
     * state is reached (recovery_target) and failure due to multiple recovery
     * targets (name, time, xid, LSN).
     */
    recoveryconfcontents = GenerateRecoveryConfig(conn, NULL, NULL);
    appendPQExpBuffer(recoveryconfcontents, "recovery_target = ''\n");
    appendPQExpBuffer(recoveryconfcontents,
                      "recovery_target_timeline = 'latest'\n");
    appendPQExpBuffer(recoveryconfcontents,
                      "recovery_target_inclusive = true\n");
    appendPQExpBuffer(recoveryconfcontents,
                      "recovery_target_action = promote\n");
    appendPQExpBuffer(recoveryconfcontents, "recovery_target_name = ''\n");
    appendPQExpBuffer(recoveryconfcontents, "recovery_target_time = ''\n");
    appendPQExpBuffer(recoveryconfcontents, "recovery_target_xid = ''\n");
 
    if (dry_run)
    {
        appendPQExpBuffer(recoveryconfcontents, "# dry run mode");
        appendPQExpBuffer(recoveryconfcontents,
                          "recovery_target_lsn = '%X/%X'\n",
                          LSN_FORMAT_ARGS((XLogRecPtr) InvalidXLogRecPtr));
    }
    else
    {
        appendPQExpBuffer(recoveryconfcontents, "recovery_target_lsn = '%s'\n",
                          lsn);
        WriteRecoveryConfig(conn, datadir, recoveryconfcontents);
    }
    disconnect_database(conn, false);
 
    pg_log_debug("recovery parameters:\n%s", recoveryconfcontents->data);
}
 
/*
 * Drop physical replication slot on primary if the standby was using it. After
 * the transformation, it has no use.
 *
 * XXX we might not fail here. Instead, we provide a warning so the user
 * eventually drops this replication slot later.
 */
static void
drop_primary_replication_slot(struct LogicalRepInfo *dbinfo, const char *slotname)
{
    PGconn     *conn;
 
    /* Replication slot does not exist, do nothing */
    if (!primary_slot_name)
        return;
 
    conn = connect_database(dbinfo[0].pubconninfo, false);
    if (conn != NULL)
    {
        drop_replication_slot(conn, &dbinfo[0], slotname);
        disconnect_database(conn, false);
    }
    else
    {
        pg_log_warning("could not drop replication slot \"%s\" on primary",
                       slotname);
        pg_log_warning_hint("Drop this replication slot soon to avoid retention of WAL files.");
    }
}
 
/*
 * Drop failover replication slots on subscriber. After the transformation,
 * they have no use.
 *
 * XXX We do not fail here. Instead, we provide a warning so the user can drop
 * them later.
 */
static void
drop_failover_replication_slots(struct LogicalRepInfo *dbinfo)
{
    PGconn     *conn;
    PGresult   *res;
 
    conn = connect_database(dbinfo[0].subconninfo, false);
    if (conn != NULL)
    {
        /* Get failover replication slot names */
        res = PQexec(conn,
                     "SELECT slot_name FROM pg_catalog.pg_replication_slots WHERE failover");
 
        if (PQresultStatus(res) == PGRES_TUPLES_OK)
        {
            /* Remove failover replication slots from subscriber */
            for (int i = 0; i < PQntuples(res); i++)
                drop_replication_slot(conn, &dbinfo[0], PQgetvalue(res, i, 0));
        }
        else
        {
            pg_log_warning("could not obtain failover replication slot information: %s",
                           PQresultErrorMessage(res));
            pg_log_warning_hint("Drop the failover replication slots on subscriber soon to avoid retention of WAL files.");
        }
 
        PQclear(res);
        disconnect_database(conn, false);
    }
    else
    {
        pg_log_warning("could not drop failover replication slot");
        pg_log_warning_hint("Drop the failover replication slots on subscriber soon to avoid retention of WAL files.");
    }
}
 
/*
 * Create a logical replication slot and returns a LSN.
 *
 * CreateReplicationSlot() is not used because it does not provide the one-row
 * result set that contains the LSN.
 */
static char *
create_logical_replication_slot(PGconn *conn, struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res = NULL;
    const char *slot_name = dbinfo->replslotname;
    char       *slot_name_esc;
    char       *lsn = NULL;
 
    Assert(conn != NULL);
 
    pg_log_info("creating the replication slot \"%s\" in database \"%s\"",
                slot_name, dbinfo->dbname);
 
    slot_name_esc = PQescapeLiteral(conn, slot_name, strlen(slot_name));
 
    appendPQExpBuffer(str,
                      "SELECT lsn FROM pg_catalog.pg_create_logical_replication_slot(%s, 'pgoutput', false, false, false)",
                      slot_name_esc);
 
    pg_free(slot_name_esc);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_TUPLES_OK)
        {
            pg_log_error("could not create replication slot \"%s\" in database \"%s\": %s",
                         slot_name, dbinfo->dbname,
                         PQresultErrorMessage(res));
            PQclear(res);
            destroyPQExpBuffer(str);
            return NULL;
        }
 
        lsn = pg_strdup(PQgetvalue(res, 0, 0));
        PQclear(res);
    }
 
    /* For cleanup purposes */
    dbinfo->made_replslot = true;
 
    destroyPQExpBuffer(str);
 
    return lsn;
}
 
static void
drop_replication_slot(PGconn *conn, struct LogicalRepInfo *dbinfo,
                      const char *slot_name)
{
    PQExpBuffer str = createPQExpBuffer();
    char       *slot_name_esc;
    PGresult   *res;
 
    Assert(conn != NULL);
 
    pg_log_info("dropping the replication slot \"%s\" in database \"%s\"",
                slot_name, dbinfo->dbname);
 
    slot_name_esc = PQescapeLiteral(conn, slot_name, strlen(slot_name));
 
    appendPQExpBuffer(str, "SELECT pg_catalog.pg_drop_replication_slot(%s)", slot_name_esc);
 
    pg_free(slot_name_esc);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_TUPLES_OK)
        {
            pg_log_error("could not drop replication slot \"%s\" in database \"%s\": %s",
                         slot_name, dbinfo->dbname, PQresultErrorMessage(res));
            dbinfo->made_replslot = false;  /* don't try again. */
        }
 
        PQclear(res);
    }
 
    destroyPQExpBuffer(str);
}
 
/*
 * Reports a suitable message if pg_ctl fails.
 */
static void
pg_ctl_status(const char *pg_ctl_cmd, int rc)
{
    if (rc != 0)
    {
        if (WIFEXITED(rc))
        {
            pg_log_error("pg_ctl failed with exit code %d", WEXITSTATUS(rc));
        }
        else if (WIFSIGNALED(rc))
        {
#if defined(WIN32)
            pg_log_error("pg_ctl was terminated by exception 0x%X",
                         WTERMSIG(rc));
            pg_log_error_detail("See C include file \"ntstatus.h\" for a description of the hexadecimal value.");
#else
            pg_log_error("pg_ctl was terminated by signal %d: %s",
                         WTERMSIG(rc), pg_strsignal(WTERMSIG(rc)));
#endif
        }
        else
        {
            pg_log_error("pg_ctl exited with unrecognized status %d", rc);
        }
 
        pg_log_error_detail("The failed command was: %s", pg_ctl_cmd);
        exit(1);
    }
}
 
static void
start_standby_server(const struct CreateSubscriberOptions *opt, bool restricted_access,
                     bool restrict_logical_worker)
{
    PQExpBuffer pg_ctl_cmd = createPQExpBuffer();
    int         rc;
 
    appendPQExpBuffer(pg_ctl_cmd, "\"%s\" start -D ", pg_ctl_path);
    appendShellString(pg_ctl_cmd, subscriber_dir);
    appendPQExpBuffer(pg_ctl_cmd, " -s -o \"-c sync_replication_slots=off\"");
    if (restricted_access)
    {
        appendPQExpBuffer(pg_ctl_cmd, " -o \"-p %s\"", opt->sub_port);
#if !defined(WIN32)
 
        /*
         * An empty listen_addresses list means the server does not listen on
         * any IP interfaces; only Unix-domain sockets can be used to connect
         * to the server. Prevent external connections to minimize the chance
         * of failure.
         */
        appendPQExpBufferStr(pg_ctl_cmd, " -o \"-c listen_addresses='' -c unix_socket_permissions=0700");
        if (opt->socket_dir)
            appendPQExpBuffer(pg_ctl_cmd, " -c unix_socket_directories='%s'",
                              opt->socket_dir);
        appendPQExpBufferChar(pg_ctl_cmd, '"');
#endif
    }
    if (opt->config_file != NULL)
        appendPQExpBuffer(pg_ctl_cmd, " -o \"-c config_file=%s\"",
                          opt->config_file);
 
    /* Suppress to start logical replication if requested */
    if (restrict_logical_worker)
        appendPQExpBuffer(pg_ctl_cmd, " -o \"-c max_logical_replication_workers=0\"");
 
    pg_log_debug("pg_ctl command is: %s", pg_ctl_cmd->data);
    rc = system(pg_ctl_cmd->data);
    pg_ctl_status(pg_ctl_cmd->data, rc);
    standby_running = true;
    destroyPQExpBuffer(pg_ctl_cmd);
    pg_log_info("server was started");
}
 
static void
stop_standby_server(const char *datadir)
{
    char       *pg_ctl_cmd;
    int         rc;
 
    pg_ctl_cmd = psprintf("\"%s\" stop -D \"%s\" -s", pg_ctl_path,
                          datadir);
    pg_log_debug("pg_ctl command is: %s", pg_ctl_cmd);
    rc = system(pg_ctl_cmd);
    pg_ctl_status(pg_ctl_cmd, rc);
    standby_running = false;
    pg_log_info("server was stopped");
}
 
/*
 * Returns after the server finishes the recovery process.
 *
 * If recovery_timeout option is set, terminate abnormally without finishing
 * the recovery process. By default, it waits forever.
 *
 * XXX Is the recovery process still in progress? When recovery process has a
 * better progress reporting mechanism, it should be added here.
 */
static void
wait_for_end_recovery(const char *conninfo, const struct CreateSubscriberOptions *opt)
{
    PGconn     *conn;
    int         status = POSTMASTER_STILL_STARTING;
    int         timer = 0;
 
    pg_log_info("waiting for the target server to reach the consistent state");
 
    conn = connect_database(conninfo, true);
 
    for (;;)
    {
        bool        in_recovery = server_is_in_recovery(conn);
 
        /*
         * Does the recovery process finish? In dry run mode, there is no
         * recovery mode. Bail out as the recovery process has ended.
         */
        if (!in_recovery || dry_run)
        {
            status = POSTMASTER_READY;
            recovery_ended = true;
            break;
        }
 
        /* Bail out after recovery_timeout seconds if this option is set */
        if (opt->recovery_timeout > 0 && timer >= opt->recovery_timeout)
        {
            stop_standby_server(subscriber_dir);
            pg_log_error("recovery timed out");
            disconnect_database(conn, true);
        }
 
        /* Keep waiting */
        pg_usleep(WAIT_INTERVAL * USEC_PER_SEC);
 
        timer += WAIT_INTERVAL;
    }
 
    disconnect_database(conn, false);
 
    if (status == POSTMASTER_STILL_STARTING)
        pg_fatal("server did not end recovery");
 
    pg_log_info("target server reached the consistent state");
    pg_log_info_hint("If pg_createsubscriber fails after this point, you must recreate the physical replica before continuing.");
}
 
/*
 * Create a publication that includes all tables in the database.
 */
static void
create_publication(PGconn *conn, struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res;
    char       *ipubname_esc;
    char       *spubname_esc;
 
    Assert(conn != NULL);
 
    ipubname_esc = PQescapeIdentifier(conn, dbinfo->pubname, strlen(dbinfo->pubname));
    spubname_esc = PQescapeLiteral(conn, dbinfo->pubname, strlen(dbinfo->pubname));
 
    /* Check if the publication already exists */
    appendPQExpBuffer(str,
                      "SELECT 1 FROM pg_catalog.pg_publication "
                      "WHERE pubname = %s",
                      spubname_esc);
    res = PQexec(conn, str->data);
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain publication information: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    if (PQntuples(res) == 1)
    {
        /*
         * Unfortunately, if it reaches this code path, it will always fail
         * (unless you decide to change the existing publication name). That's
         * bad but it is very unlikely that the user will choose a name with
         * pg_createsubscriber_ prefix followed by the exact database oid and
         * a random number.
         */
        pg_log_error("publication \"%s\" already exists", dbinfo->pubname);
        pg_log_error_hint("Consider renaming this publication before continuing.");
        disconnect_database(conn, true);
    }
 
    PQclear(res);
    resetPQExpBuffer(str);
 
    pg_log_info("creating publication \"%s\" in database \"%s\"",
                dbinfo->pubname, dbinfo->dbname);
 
    appendPQExpBuffer(str, "CREATE PUBLICATION %s FOR ALL TABLES",
                      ipubname_esc);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_COMMAND_OK)
        {
            pg_log_error("could not create publication \"%s\" in database \"%s\": %s",
                         dbinfo->pubname, dbinfo->dbname, PQresultErrorMessage(res));
            disconnect_database(conn, true);
        }
        PQclear(res);
    }
 
    /* For cleanup purposes */
    dbinfo->made_publication = true;
 
    pg_free(ipubname_esc);
    pg_free(spubname_esc);
    destroyPQExpBuffer(str);
}
 
/*
 * Remove publication if it couldn't finish all steps.
 */
static void
drop_publication(PGconn *conn, struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res;
    char       *pubname_esc;
 
    Assert(conn != NULL);
 
    pubname_esc = PQescapeIdentifier(conn, dbinfo->pubname, strlen(dbinfo->pubname));
 
    pg_log_info("dropping publication \"%s\" in database \"%s\"",
                dbinfo->pubname, dbinfo->dbname);
 
    appendPQExpBuffer(str, "DROP PUBLICATION %s", pubname_esc);
 
    pg_free(pubname_esc);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_COMMAND_OK)
        {
            pg_log_error("could not drop publication \"%s\" in database \"%s\": %s",
                         dbinfo->pubname, dbinfo->dbname, PQresultErrorMessage(res));
            dbinfo->made_publication = false;   /* don't try again. */
 
            /*
             * Don't disconnect and exit here. This routine is used by primary
             * (cleanup publication / replication slot due to an error) and
             * subscriber (remove the replicated publications). In both cases,
             * it can continue and provide instructions for the user to remove
             * it later if cleanup fails.
             */
        }
        PQclear(res);
    }
 
    destroyPQExpBuffer(str);
}
 
/*
 * Create a subscription with some predefined options.
 *
 * A replication slot was already created in a previous step. Let's use it.  It
 * is not required to copy data. The subscription will be created but it will
 * not be enabled now. That's because the replication progress must be set and
 * the replication origin name (one of the function arguments) contains the
 * subscription OID in its name. Once the subscription is created,
 * set_replication_progress() can obtain the chosen origin name and set up its
 * initial location.
 */
static void
create_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res;
    char       *pubname_esc;
    char       *subname_esc;
    char       *pubconninfo_esc;
    char       *replslotname_esc;
 
    Assert(conn != NULL);
 
    pubname_esc = PQescapeIdentifier(conn, dbinfo->pubname, strlen(dbinfo->pubname));
    subname_esc = PQescapeIdentifier(conn, dbinfo->subname, strlen(dbinfo->subname));
    pubconninfo_esc = PQescapeLiteral(conn, dbinfo->pubconninfo, strlen(dbinfo->pubconninfo));
    replslotname_esc = PQescapeLiteral(conn, dbinfo->replslotname, strlen(dbinfo->replslotname));
 
    pg_log_info("creating subscription \"%s\" in database \"%s\"",
                dbinfo->subname, dbinfo->dbname);
 
    appendPQExpBuffer(str,
                      "CREATE SUBSCRIPTION %s CONNECTION %s PUBLICATION %s "
                      "WITH (create_slot = false, enabled = false, "
                      "slot_name = %s, copy_data = false)",
                      subname_esc, pubconninfo_esc, pubname_esc, replslotname_esc);
 
    pg_free(pubname_esc);
    pg_free(subname_esc);
    pg_free(pubconninfo_esc);
    pg_free(replslotname_esc);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_COMMAND_OK)
        {
            pg_log_error("could not create subscription \"%s\" in database \"%s\": %s",
                         dbinfo->subname, dbinfo->dbname, PQresultErrorMessage(res));
            disconnect_database(conn, true);
        }
        PQclear(res);
    }
 
    destroyPQExpBuffer(str);
}
 
/*
 * Sets the replication progress to the consistent LSN.
 *
 * The subscriber caught up to the consistent LSN provided by the last
 * replication slot that was created. The goal is to set up the initial
 * location for the logical replication that is the exact LSN that the
 * subscriber was promoted. Once the subscription is enabled it will start
 * streaming from that location onwards.  In dry run mode, the subscription OID
 * and LSN are set to invalid values for printing purposes.
 */
static void
set_replication_progress(PGconn *conn, const struct LogicalRepInfo *dbinfo, const char *lsn)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res;
    Oid         suboid;
    char       *subname;
    char       *dbname;
    char       *originname;
    char       *lsnstr;
 
    Assert(conn != NULL);
 
    subname = PQescapeLiteral(conn, dbinfo->subname, strlen(dbinfo->subname));
    dbname = PQescapeLiteral(conn, dbinfo->dbname, strlen(dbinfo->dbname));
 
    appendPQExpBuffer(str,
                      "SELECT s.oid FROM pg_catalog.pg_subscription s "
                      "INNER JOIN pg_catalog.pg_database d ON (s.subdbid = d.oid) "
                      "WHERE s.subname = %s AND d.datname = %s",
                      subname, dbname);
 
    res = PQexec(conn, str->data);
    if (PQresultStatus(res) != PGRES_TUPLES_OK)
    {
        pg_log_error("could not obtain subscription OID: %s",
                     PQresultErrorMessage(res));
        disconnect_database(conn, true);
    }
 
    if (PQntuples(res) != 1 && !dry_run)
    {
        pg_log_error("could not obtain subscription OID: got %d rows, expected %d row",
                     PQntuples(res), 1);
        disconnect_database(conn, true);
    }
 
    if (dry_run)
    {
        suboid = InvalidOid;
        lsnstr = psprintf("%X/%X", LSN_FORMAT_ARGS((XLogRecPtr) InvalidXLogRecPtr));
    }
    else
    {
        suboid = strtoul(PQgetvalue(res, 0, 0), NULL, 10);
        lsnstr = psprintf("%s", lsn);
    }
 
    PQclear(res);
 
    /*
     * The origin name is defined as pg_%u. %u is the subscription OID. See
     * ApplyWorkerMain().
     */
    originname = psprintf("pg_%u", suboid);
 
    pg_log_info("setting the replication progress (node name \"%s\", LSN %s) in database \"%s\"",
                originname, lsnstr, dbinfo->dbname);
 
    resetPQExpBuffer(str);
    appendPQExpBuffer(str,
                      "SELECT pg_catalog.pg_replication_origin_advance('%s', '%s')",
                      originname, lsnstr);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_TUPLES_OK)
        {
            pg_log_error("could not set replication progress for subscription \"%s\": %s",
                         dbinfo->subname, PQresultErrorMessage(res));
            disconnect_database(conn, true);
        }
        PQclear(res);
    }
 
    pg_free(subname);
    pg_free(dbname);
    pg_free(originname);
    pg_free(lsnstr);
    destroyPQExpBuffer(str);
}
 
/*
 * Enables the subscription.
 *
 * The subscription was created in a previous step but it was disabled. After
 * adjusting the initial logical replication location, enable the subscription.
 */
static void
enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
{
    PQExpBuffer str = createPQExpBuffer();
    PGresult   *res;
    char       *subname;
 
    Assert(conn != NULL);
 
    subname = PQescapeIdentifier(conn, dbinfo->subname, strlen(dbinfo->subname));
 
    pg_log_info("enabling subscription \"%s\" in database \"%s\"",
                dbinfo->subname, dbinfo->dbname);
 
    appendPQExpBuffer(str, "ALTER SUBSCRIPTION %s ENABLE", subname);
 
    pg_log_debug("command is: %s", str->data);
 
    if (!dry_run)
    {
        res = PQexec(conn, str->data);
        if (PQresultStatus(res) != PGRES_COMMAND_OK)
        {
            pg_log_error("could not enable subscription \"%s\": %s",
                         dbinfo->subname, PQresultErrorMessage(res));
            disconnect_database(conn, true);
        }
 
        PQclear(res);
    }
 
    pg_free(subname);
    destroyPQExpBuffer(str);
}
 
int
main(int argc, char **argv)
{
    static struct option long_options[] =
    {
        {"database", required_argument, NULL, 'd'},
        {"pgdata", required_argument, NULL, 'D'},
        {"dry-run", no_argument, NULL, 'n'},
        {"subscriber-port", required_argument, NULL, 'p'},
        {"publisher-server", required_argument, NULL, 'P'},
        {"socketdir", required_argument, NULL, 's'},
        {"recovery-timeout", required_argument, NULL, 't'},
        {"subscriber-username", required_argument, NULL, 'U'},
        {"verbose", no_argument, NULL, 'v'},
        {"version", no_argument, NULL, 'V'},
        {"help", no_argument, NULL, '?'},
        {"config-file", required_argument, NULL, 1},
        {"publication", required_argument, NULL, 2},
        {"replication-slot", required_argument, NULL, 3},
        {"subscription", required_argument, NULL, 4},
        {NULL, 0, NULL, 0}
    };
 
    struct CreateSubscriberOptions opt = {0};
 
    int         c;
    int         option_index;
 
    char       *pub_base_conninfo;
    char       *sub_base_conninfo;
    char       *dbname_conninfo = NULL;
 
    uint64      pub_sysid;
    uint64      sub_sysid;
    struct stat statbuf;
 
    char       *consistent_lsn;
 
    char        pidfile[MAXPGPATH];
 
    pg_logging_init(argv[0]);
    pg_logging_set_level(PG_LOG_WARNING);
    progname = get_progname(argv[0]);
    set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
 
    if (argc > 1)
    {
        if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
        {
            usage();
            exit(0);
        }
        else if (strcmp(argv[1], "-V") == 0
                 || strcmp(argv[1], "--version") == 0)
        {
            puts("pg_createsubscriber (PostgreSQL) " PG_VERSION);
            exit(0);
        }
    }
 
    /* Default settings */
    subscriber_dir = NULL;
    opt.config_file = NULL;
    opt.pub_conninfo_str = NULL;
    opt.socket_dir = NULL;
    opt.sub_port = DEFAULT_SUB_PORT;
    opt.sub_username = NULL;
    opt.database_names = (SimpleStringList)
    {
        0
    };
    opt.recovery_timeout = 0;
 
    /*
     * Don't allow it to be run as root. It uses pg_ctl which does not allow
     * it either.
     */
#ifndef WIN32
    if (geteuid() == 0)
    {
        pg_log_error("cannot be executed by \"root\"");
        pg_log_error_hint("You must run %s as the PostgreSQL superuser.",
                          progname);
        exit(1);
    }
#endif
 
    get_restricted_token();
 
    while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
                            long_options, &option_index)) != -1)
    {
        switch (c)
        {
            case 'd':
                if (!simple_string_list_member(&opt.database_names, optarg))
                {
                    simple_string_list_append(&opt.database_names, optarg);
                    num_dbs++;
                }
                else
                {
                    pg_log_error("database \"%s\" specified more than once", optarg);
                    exit(1);
                }
                break;
            case 'D':
                subscriber_dir = pg_strdup(optarg);
                canonicalize_path(subscriber_dir);
                break;
            case 'n':
                dry_run = true;
                break;
            case 'p':
                opt.sub_port = pg_strdup(optarg);
                break;
            case 'P':
                opt.pub_conninfo_str = pg_strdup(optarg);
                break;
            case 's':
                opt.socket_dir = pg_strdup(optarg);
                canonicalize_path(opt.socket_dir);
                break;
            case 't':
                opt.recovery_timeout = atoi(optarg);
                break;
            case 'U':
                opt.sub_username = pg_strdup(optarg);
                break;
            case 'v':
                pg_logging_increase_verbosity();
                break;
            case 1:
                opt.config_file = pg_strdup(optarg);
                break;
            case 2:
                if (!simple_string_list_member(&opt.pub_names, optarg))
                {
                    simple_string_list_append(&opt.pub_names, optarg);
                    num_pubs++;
                }
                else
                {
                    pg_log_error("publication \"%s\" specified more than once", optarg);
                    exit(1);
                }
                break;
            case 3:
                if (!simple_string_list_member(&opt.replslot_names, optarg))
                {
                    simple_string_list_append(&opt.replslot_names, optarg);
                    num_replslots++;
                }
                else
                {
                    pg_log_error("replication slot \"%s\" specified more than once", optarg);
                    exit(1);
                }
                break;
            case 4:
                if (!simple_string_list_member(&opt.sub_names, optarg))
                {
                    simple_string_list_append(&opt.sub_names, optarg);
                    num_subs++;
                }
                else
                {
                    pg_log_error("subscription \"%s\" specified more than once", optarg);
                    exit(1);
                }
                break;
            default:
                /* getopt_long already emitted a complaint */
                pg_log_error_hint("Try \"%s --help\" for more information.", progname);
                exit(1);
        }
    }
 
    /* Any non-option arguments? */
    if (optind < argc)
    {
        pg_log_error("too many command-line arguments (first is \"%s\")",
                     argv[optind]);
        pg_log_error_hint("Try \"%s --help\" for more information.", progname);
        exit(1);
    }
 
    /* Required arguments */
    if (subscriber_dir == NULL)
    {
        pg_log_error("no subscriber data directory specified");
        pg_log_error_hint("Try \"%s --help\" for more information.", progname);
        exit(1);
    }
 
    /* If socket directory is not provided, use the current directory */
    if (opt.socket_dir == NULL)
    {
        char        cwd[MAXPGPATH];
 
        if (!getcwd(cwd, MAXPGPATH))
            pg_fatal("could not determine current directory");
        opt.socket_dir = pg_strdup(cwd);
        canonicalize_path(opt.socket_dir);
    }
 
    /*
     * Parse connection string. Build a base connection string that might be
     * reused by multiple databases.
     */
    if (opt.pub_conninfo_str == NULL)
    {
        /*
         * TODO use primary_conninfo (if available) from subscriber and
         * extract publisher connection string. Assume that there are
         * identical entries for physical and logical replication. If there is
         * not, we would fail anyway.
         */
        pg_log_error("no publisher connection string specified");
        pg_log_error_hint("Try \"%s --help\" for more information.", progname);
        exit(1);
    }
    pg_log_info("validating publisher connection string");
    pub_base_conninfo = get_base_conninfo(opt.pub_conninfo_str,
                                          &dbname_conninfo);
    if (pub_base_conninfo == NULL)
        exit(1);
 
    pg_log_info("validating subscriber connection string");
    sub_base_conninfo = get_sub_conninfo(&opt);
 
    if (opt.database_names.head == NULL)
    {
        pg_log_info("no database was specified");
 
        /*
         * If --database option is not provided, try to obtain the dbname from
         * the publisher conninfo. If dbname parameter is not available, error
         * out.
         */
        if (dbname_conninfo)
        {
            simple_string_list_append(&opt.database_names, dbname_conninfo);
            num_dbs++;
 
            pg_log_info("database name \"%s\" was extracted from the publisher connection string",
                        dbname_conninfo);
        }
        else
        {
            pg_log_error("no database name specified");
            pg_log_error_hint("Try \"%s --help\" for more information.",
                              progname);
            exit(1);
        }
    }
 
    /* Number of object names must match number of databases */
    if (num_pubs > 0 && num_pubs != num_dbs)
    {
        pg_log_error("wrong number of publication names specified");
        pg_log_error_detail("The number of specified publication names (%d) must match the number of specified database names (%d).",
                            num_pubs, num_dbs);
        exit(1);
    }
    if (num_subs > 0 && num_subs != num_dbs)
    {
        pg_log_error("wrong number of subscription names specified");
        pg_log_error_detail("The number of specified subscription names (%d) must match the number of specified database names (%d).",
                            num_subs, num_dbs);
        exit(1);
    }
    if (num_replslots > 0 && num_replslots != num_dbs)
    {
        pg_log_error("wrong number of replication slot names specified");
        pg_log_error_detail("The number of specified replication slot names (%d) must match the number of specified database names (%d).",
                            num_replslots, num_dbs);
        exit(1);
    }
 
    /* Get the absolute path of pg_ctl and pg_resetwal on the subscriber */
    pg_ctl_path = get_exec_path(argv[0], "pg_ctl");
    pg_resetwal_path = get_exec_path(argv[0], "pg_resetwal");
 
    /* Rudimentary check for a data directory */
    check_data_directory(subscriber_dir);
 
    /*
     * Store database information for publisher and subscriber. It should be
     * called before atexit() because its return is used in the
     * cleanup_objects_atexit().
     */
    dbinfo = store_pub_sub_info(&opt, pub_base_conninfo, sub_base_conninfo);
 
    /* Register a function to clean up objects in case of failure */
    atexit(cleanup_objects_atexit);
 
    /*
     * Check if the subscriber data directory has the same system identifier
     * than the publisher data directory.
     */
    pub_sysid = get_primary_sysid(dbinfo[0].pubconninfo);
    sub_sysid = get_standby_sysid(subscriber_dir);
    if (pub_sysid != sub_sysid)
        pg_fatal("subscriber data directory is not a copy of the source database cluster");
 
    /* Subscriber PID file */
    snprintf(pidfile, MAXPGPATH, "%s/postmaster.pid", subscriber_dir);
 
    /*
     * The standby server must not be running. If the server is started under
     * service manager and pg_createsubscriber stops it, the service manager
     * might react to this action and start the server again. Therefore,
     * refuse to proceed if the server is running to avoid possible failures.
     */
    if (stat(pidfile, &statbuf) == 0)
    {
        pg_log_error("standby server is running");
        pg_log_error_hint("Stop the standby server and try again.");
        exit(1);
    }
 
    /*
     * Start a short-lived standby server with temporary parameters (provided
     * by command-line options). The goal is to avoid connections during the
     * transformation steps.
     */
    pg_log_info("starting the standby server with command-line options");
    start_standby_server(&opt, true, false);
 
    /* Check if the standby server is ready for logical replication */
    check_subscriber(dbinfo);
 
    /* Check if the primary server is ready for logical replication */
    check_publisher(dbinfo);
 
    /*
     * Stop the target server. The recovery process requires that the server
     * reaches a consistent state before targeting the recovery stop point.
     * Make sure a consistent state is reached (stop the target server
     * guarantees it) *before* creating the replication slots in
     * setup_publisher().
     */
    pg_log_info("stopping the subscriber");
    stop_standby_server(subscriber_dir);
 
    /* Create the required objects for each database on publisher */
    consistent_lsn = setup_publisher(dbinfo);
 
    /* Write the required recovery parameters */
    setup_recovery(dbinfo, subscriber_dir, consistent_lsn);
 
    /*
     * Start subscriber so the recovery parameters will take effect. Wait
     * until accepting connections. We don't want to start logical replication
     * during setup.
     */
    pg_log_info("starting the subscriber");
    start_standby_server(&opt, true, true);
 
    /* Waiting the subscriber to be promoted */
    wait_for_end_recovery(dbinfo[0].subconninfo, &opt);
 
    /*
     * Create the subscription for each database on subscriber. It does not
     * enable it immediately because it needs to adjust the replication start
     * point to the LSN reported by setup_publisher().  It also cleans up
     * publications created by this tool and replication to the standby.
     */
    setup_subscriber(dbinfo, consistent_lsn);
 
    /* Remove primary_slot_name if it exists on primary */
    drop_primary_replication_slot(dbinfo, primary_slot_name);
 
    /* Remove failover replication slots if they exist on subscriber */
    drop_failover_replication_slots(dbinfo);
 
    /* Stop the subscriber */
    pg_log_info("stopping the subscriber");
    stop_standby_server(subscriber_dir);
 
    /* Change system identifier from subscriber */
    modify_subscriber_sysid(&opt);
 
    success = true;
 
    pg_log_info("Done!");
 
    return 0;
}