xlogarchive.c File Reference

#include "postgres.h"
#include <sys/stat.h>
#include <sys/wait.h>
#include <signal.h>
#include <unistd.h>
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
#include "storage/pmsignal.h"

Include dependency graph for xlogarchive.c:

Go to the source code of this file.

Functions
bool	RestoreArchivedFile (char *path, const char *xlogfname, const char *recovername, off_t expectedSize, bool cleanupEnabled)
void	ExecuteRecoveryCommand (const char *command, const char *commandName, bool failOnSignal)
void	KeepFileRestoredFromArchive (const char *path, const char *xlogfname)
void	XLogArchiveNotify (const char *xlog)
void	XLogArchiveNotifySeg (XLogSegNo segno)
void	XLogArchiveForceDone (const char *xlog)
bool	XLogArchiveCheckDone (const char *xlog)
bool	XLogArchiveIsBusy (const char *xlog)
bool	XLogArchiveIsReadyOrDone (const char *xlog)
bool	XLogArchiveIsReady (const char *xlog)
void	XLogArchiveCleanup (const char *xlog)

Function Documentation

ExecuteRecoveryCommand()

void ExecuteRecoveryCommand	(	const char *	command,
		const char *	commandName,
		bool	failOnSignal
	)

Definition at line 287 of file xlogarchive.c.

References Assert, DEBUG3, ereport, errmsg(), errmsg_internal(), FATAL, GetOldestRestartPoint(), MAXPGPATH, strlcpy(), wait_result_is_any_signal(), wait_result_to_str(), wal_segment_size, WARNING, XLByteToSeg, and XLogFileName.

Referenced by CreateRestartPoint(), and StartupXLOG().

{
    char        xlogRecoveryCmd[MAXPGPATH];
    char        lastRestartPointFname[MAXPGPATH];
    char       *dp;
    char       *endp;
    const char *sp;
    int         rc;
    XLogSegNo   restartSegNo;
    XLogRecPtr  restartRedoPtr;
    TimeLineID  restartTli;

    Assert(command && commandName);

    /*
     * Calculate the archive file cutoff point for use during log shipping
     * replication. All files earlier than this point can be deleted from the
     * archive, though there is no requirement to do so.
     */
    GetOldestRestartPoint(&restartRedoPtr, &restartTli);
    XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
    XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
                 wal_segment_size);

    /*
     * construct the command to be executed
     */
    dp = xlogRecoveryCmd;
    endp = xlogRecoveryCmd + MAXPGPATH - 1;
    *endp = '\0';

    for (sp = command; *sp; sp++)
    {
        if (*sp == '%')
        {
            switch (sp[1])
            {
                case 'r':
                    /* %r: filename of last restartpoint */
                    sp++;
                    strlcpy(dp, lastRestartPointFname, endp - dp);
                    dp += strlen(dp);
                    break;
                case '%':
                    /* convert %% to a single % */
                    sp++;
                    if (dp < endp)
                        *dp++ = *sp;
                    break;
                default:
                    /* otherwise treat the % as not special */
                    if (dp < endp)
                        *dp++ = *sp;
                    break;
            }
        }
        else
        {
            if (dp < endp)
                *dp++ = *sp;
        }
    }
    *dp = '\0';

    ereport(DEBUG3,
            (errmsg_internal("executing %s \"%s\"", commandName, command)));

    /*
     * execute the constructed command
     */
    rc = system(xlogRecoveryCmd);
    if (rc != 0)
    {
        /*
         * If the failure was due to any sort of signal, it's best to punt and
         * abort recovery.  See comments in RestoreArchivedFile().
         */
        ereport((failOnSignal && wait_result_is_any_signal(rc, true)) ? FATAL : WARNING,
        /*------
           translator: First %s represents a postgresql.conf parameter name like
          "recovery_end_command", the 2nd is the value of that parameter, the
          third an already translated error message. */
                (errmsg("%s \"%s\": %s", commandName,
                        command, wait_result_to_str(rc))));
    }
}

KeepFileRestoredFromArchive()

void KeepFileRestoredFromArchive	(	const char *	path,
		const char *	xlogfname
	)

Definition at line 381 of file xlogarchive.c.

References ARCHIVE_MODE_ALWAYS, durable_rename(), ereport, errcode_for_file_access(), errmsg(), ERROR, FATAL, MAXPGPATH, snprintf, stat, strlcpy(), WalSndRqstFileReload(), WalSndWakeup(), XLogArchiveForceDone(), XLogArchiveMode, XLogArchiveNotify(), XLOGDIR, and xlogfpath.

Referenced by readTimeLineHistory(), restoreTimeLineHistoryFiles(), and XLogFileRead().

{
    char        xlogfpath[MAXPGPATH];
    bool        reload = false;
    struct stat statbuf;

    snprintf(xlogfpath, MAXPGPATH, XLOGDIR "/%s", xlogfname);

    if (stat(xlogfpath, &statbuf) == 0)
    {
        char        oldpath[MAXPGPATH];

#ifdef WIN32
        static unsigned int deletedcounter = 1;

        /*
         * On Windows, if another process (e.g a walsender process) holds the
         * file open in FILE_SHARE_DELETE mode, unlink will succeed, but the
         * file will still show up in directory listing until the last handle
         * is closed, and we cannot rename the new file in its place until
         * that. To avoid that problem, rename the old file to a temporary
         * name first. Use a counter to create a unique filename, because the
         * same file might be restored from the archive multiple times, and a
         * walsender could still be holding onto an old deleted version of it.
         */
        snprintf(oldpath, MAXPGPATH, "%s.deleted%u",
                 xlogfpath, deletedcounter++);
        if (rename(xlogfpath, oldpath) != 0)
        {
            ereport(ERROR,
                    (errcode_for_file_access(),
                     errmsg("could not rename file \"%s\" to \"%s\": %m",
                            xlogfpath, oldpath)));
        }
#else
        /* same-size buffers, so this never truncates */
        strlcpy(oldpath, xlogfpath, MAXPGPATH);
#endif
        if (unlink(oldpath) != 0)
            ereport(FATAL,
                    (errcode_for_file_access(),
                     errmsg("could not remove file \"%s\": %m",
                            xlogfpath)));
        reload = true;
    }

    durable_rename(path, xlogfpath, ERROR);

    /*
     * Create .done file forcibly to prevent the restored segment from being
     * archived again later.
     */
    if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS)
        XLogArchiveForceDone(xlogfname);
    else
        XLogArchiveNotify(xlogfname);

    /*
     * If the existing file was replaced, since walsenders might have it open,
     * request them to reload a currently-open segment. This is only required
     * for WAL segments, walsenders don't hold other files open, but there's
     * no harm in doing this too often, and we don't know what kind of a file
     * we're dealing with here.
     */
    if (reload)
        WalSndRqstFileReload();

    /*
     * Signal walsender that new WAL has arrived. Again, this isn't necessary
     * if we restored something other than a WAL segment, but it does no harm
     * either.
     */
    WalSndWakeup();
}

RestoreArchivedFile()

bool RestoreArchivedFile	(	char *	path,
		const char *	xlogfname,
		const char *	recovername,
		off_t	expectedSize,
		bool	cleanupEnabled
	)

Definition at line 53 of file xlogarchive.c.

References ArchiveRecoveryRequested, Assert, BuildRestoreCommand(), DEBUG1, DEBUG2, DEBUG3, elevel, elog, ereport, errcode_for_file_access(), errdetail(), errmsg(), errmsg_internal(), ERROR, FATAL, GetOldestRestartPoint(), LOG, MAXPGPATH, pfree(), PostRestoreCommand(), PreRestoreCommand(), proc_exit(), recoveryRestoreCommand, snprintf, stat::st_size, StandbyMode, stat, wait_result_is_any_signal(), wait_result_is_signal(), wait_result_to_str(), wal_segment_size, XLByteToSeg, XLOGDIR, and XLogFileName.

Referenced by existsTimeLineHistory(), readTimeLineHistory(), restoreTimeLineHistoryFiles(), SimpleXLogPageRead(), writeTimeLineHistory(), and XLogFileRead().

{
    char        xlogpath[MAXPGPATH];
    char       *xlogRestoreCmd;
    char        lastRestartPointFname[MAXPGPATH];
    int         rc;
    struct stat stat_buf;
    XLogSegNo   restartSegNo;
    XLogRecPtr  restartRedoPtr;
    TimeLineID  restartTli;

    /*
     * Ignore restore_command when not in archive recovery (meaning we are in
     * crash recovery).
     */
    if (!ArchiveRecoveryRequested)
        goto not_available;

    /* In standby mode, restore_command might not be supplied */
    if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)
        goto not_available;

    /*
     * When doing archive recovery, we always prefer an archived log file even
     * if a file of the same name exists in XLOGDIR.  The reason is that the
     * file in XLOGDIR could be an old, un-filled or partly-filled version
     * that was copied and restored as part of backing up $PGDATA.
     *
     * We could try to optimize this slightly by checking the local copy
     * lastchange timestamp against the archived copy, but we have no API to
     * do this, nor can we guarantee that the lastchange timestamp was
     * preserved correctly when we copied to archive. Our aim is robustness,
     * so we elect not to do this.
     *
     * If we cannot obtain the log file from the archive, however, we will try
     * to use the XLOGDIR file if it exists.  This is so that we can make use
     * of log segments that weren't yet transferred to the archive.
     *
     * Notice that we don't actually overwrite any files when we copy back
     * from archive because the restore_command may inadvertently restore
     * inappropriate xlogs, or they may be corrupt, so we may wish to fallback
     * to the segments remaining in current XLOGDIR later. The
     * copy-from-archive filename is always the same, ensuring that we don't
     * run out of disk space on long recoveries.
     */
    snprintf(xlogpath, MAXPGPATH, XLOGDIR "/%s", recovername);

    /*
     * Make sure there is no existing file named recovername.
     */
    if (stat(xlogpath, &stat_buf) != 0)
    {
        if (errno != ENOENT)
            ereport(FATAL,
                    (errcode_for_file_access(),
                     errmsg("could not stat file \"%s\": %m",
                            xlogpath)));
    }
    else
    {
        if (unlink(xlogpath) != 0)
            ereport(FATAL,
                    (errcode_for_file_access(),
                     errmsg("could not remove file \"%s\": %m",
                            xlogpath)));
    }

    /*
     * Calculate the archive file cutoff point for use during log shipping
     * replication. All files earlier than this point can be deleted from the
     * archive, though there is no requirement to do so.
     *
     * If cleanup is not enabled, initialise this with the filename of
     * InvalidXLogRecPtr, which will prevent the deletion of any WAL files
     * from the archive because of the alphabetic sorting property of WAL
     * filenames.
     *
     * Once we have successfully located the redo pointer of the checkpoint
     * from which we start recovery we never request a file prior to the redo
     * pointer of the last restartpoint. When redo begins we know that we have
     * successfully located it, so there is no need for additional status
     * flags to signify the point when we can begin deleting WAL files from
     * the archive.
     */
    if (cleanupEnabled)
    {
        GetOldestRestartPoint(&restartRedoPtr, &restartTli);
        XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
        XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
                     wal_segment_size);
        /* we shouldn't need anything earlier than last restart point */
        Assert(strcmp(lastRestartPointFname, xlogfname) <= 0);
    }
    else
        XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);

    /* Build the restore command to execute */
    xlogRestoreCmd = BuildRestoreCommand(recoveryRestoreCommand,
                                         xlogpath, xlogfname,
                                         lastRestartPointFname);
    if (xlogRestoreCmd == NULL)
        elog(ERROR, "could not build restore command \"%s\"",
             recoveryRestoreCommand);

    ereport(DEBUG3,
            (errmsg_internal("executing restore command \"%s\"",
                             xlogRestoreCmd)));

    /*
     * Check signals before restore command and reset afterwards.
     */
    PreRestoreCommand();

    /*
     * Copy xlog from archival storage to XLOGDIR
     */
    rc = system(xlogRestoreCmd);

    PostRestoreCommand();
    pfree(xlogRestoreCmd);

    if (rc == 0)
    {
        /*
         * command apparently succeeded, but let's make sure the file is
         * really there now and has the correct size.
         */
        if (stat(xlogpath, &stat_buf) == 0)
        {
            if (expectedSize > 0 && stat_buf.st_size != expectedSize)
            {
                int         elevel;

                /*
                 * If we find a partial file in standby mode, we assume it's
                 * because it's just being copied to the archive, and keep
                 * trying.
                 *
                 * Otherwise treat a wrong-sized file as FATAL to ensure the
                 * DBA would notice it, but is that too strong? We could try
                 * to plow ahead with a local copy of the file ... but the
                 * problem is that there probably isn't one, and we'd
                 * incorrectly conclude we've reached the end of WAL and we're
                 * done recovering ...
                 */
                if (StandbyMode && stat_buf.st_size < expectedSize)
                    elevel = DEBUG1;
                else
                    elevel = FATAL;
                ereport(elevel,
                        (errmsg("archive file \"%s\" has wrong size: %lld instead of %lld",
                                xlogfname,
                                (long long int) stat_buf.st_size,
                                (long long int) expectedSize)));
                return false;
            }
            else
            {
                ereport(LOG,
                        (errmsg("restored log file \"%s\" from archive",
                                xlogfname)));
                strcpy(path, xlogpath);
                return true;
            }
        }
        else
        {
            /* stat failed */
            int         elevel = (errno == ENOENT) ? LOG : FATAL;

            ereport(elevel,
                    (errcode_for_file_access(),
                     errmsg("could not stat file \"%s\": %m", xlogpath),
                     errdetail("restore_command returned a zero exit status, but stat() failed.")));
        }
    }

    /*
     * Remember, we rollforward UNTIL the restore fails so failure here is
     * just part of the process... that makes it difficult to determine
     * whether the restore failed because there isn't an archive to restore,
     * or because the administrator has specified the restore program
     * incorrectly.  We have to assume the former.
     *
     * However, if the failure was due to any sort of signal, it's best to
     * punt and abort recovery.  (If we "return false" here, upper levels will
     * assume that recovery is complete and start up the database!) It's
     * essential to abort on child SIGINT and SIGQUIT, because per spec
     * system() ignores SIGINT and SIGQUIT while waiting; if we see one of
     * those it's a good bet we should have gotten it too.
     *
     * On SIGTERM, assume we have received a fast shutdown request, and exit
     * cleanly. It's pure chance whether we receive the SIGTERM first, or the
     * child process. If we receive it first, the signal handler will call
     * proc_exit, otherwise we do it here. If we or the child process received
     * SIGTERM for any other reason than a fast shutdown request, postmaster
     * will perform an immediate shutdown when it sees us exiting
     * unexpectedly.
     *
     * We treat hard shell errors such as "command not found" as fatal, too.
     */
    if (wait_result_is_signal(rc, SIGTERM))
        proc_exit(1);

    ereport(wait_result_is_any_signal(rc, true) ? FATAL : DEBUG2,
            (errmsg("could not restore file \"%s\" from archive: %s",
                    xlogfname, wait_result_to_str(rc))));

not_available:

    /*
     * if an archived file is not available, there might still be a version of
     * this file in XLOGDIR, so return that as the filename to open.
     *
     * In many recovery scenarios we expect this to fail also, but if so that
     * just means we've reached the end of WAL.
     */
    snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlogfname);
    return false;
}

XLogArchiveCheckDone()

bool XLogArchiveCheckDone

(

const char *

xlog

)

Definition at line 572 of file xlogarchive.c.

References GetRecoveryState(), MAXPGPATH, RECOVERY_STATE_ARCHIVE, stat, StatusFilePath, XLogArchiveNotify(), XLogArchivingActive, and XLogArchivingAlways.

Referenced by CleanupBackupHistory(), and RemoveOldXlogFiles().

{
    char        archiveStatusPath[MAXPGPATH];
    struct stat stat_buf;

    /* The file is always deletable if archive_mode is "off". */
    if (!XLogArchivingActive())
        return true;

    /*
     * During archive recovery, the file is deletable if archive_mode is not
     * "always".
     */
    if (!XLogArchivingAlways() &&
        GetRecoveryState() == RECOVERY_STATE_ARCHIVE)
        return true;

    /*
     * At this point of the logic, note that we are either a primary with
     * archive_mode set to "on" or "always", or a standby with archive_mode
     * set to "always".
     */

    /* First check for .done --- this means archiver is done with it */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    /* check for .ready --- this means archiver is still busy with it */
    StatusFilePath(archiveStatusPath, xlog, ".ready");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return false;

    /* Race condition --- maybe archiver just finished, so recheck */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    /* Retry creation of the .ready file */
    XLogArchiveNotify(xlog);
    return false;
}

XLogArchiveCleanup()

void XLogArchiveCleanup

(

const char *

xlog

)

Definition at line 719 of file xlogarchive.c.

References MAXPGPATH, and StatusFilePath.

Referenced by CleanupBackupHistory(), exitArchiveRecovery(), RemoveXlogFile(), and StartupXLOG().

{
    char        archiveStatusPath[MAXPGPATH];

    /* Remove the .done file */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    unlink(archiveStatusPath);
    /* should we complain about failure? */

    /* Remove the .ready file if present --- normally it shouldn't be */
    StatusFilePath(archiveStatusPath, xlog, ".ready");
    unlink(archiveStatusPath);
    /* should we complain about failure? */
}

XLogArchiveForceDone()

void XLogArchiveForceDone

(

const char *

xlog

)

Definition at line 517 of file xlogarchive.c.

References AllocateFile(), durable_rename(), ereport, errcode_for_file_access(), errmsg(), fd(), FreeFile(), LOG, MAXPGPATH, stat, StatusFilePath, and WARNING.

Referenced by KeepFileRestoredFromArchive(), WalRcvFetchTimeLineHistoryFiles(), WalReceiverMain(), and XLogWalRcvWrite().

{
    char        archiveReady[MAXPGPATH];
    char        archiveDone[MAXPGPATH];
    struct stat stat_buf;
    FILE       *fd;

    /* Exit if already known done */
    StatusFilePath(archiveDone, xlog, ".done");
    if (stat(archiveDone, &stat_buf) == 0)
        return;

    /* If .ready exists, rename it to .done */
    StatusFilePath(archiveReady, xlog, ".ready");
    if (stat(archiveReady, &stat_buf) == 0)
    {
        (void) durable_rename(archiveReady, archiveDone, WARNING);
        return;
    }

    /* insert an otherwise empty file called <XLOG>.done */
    fd = AllocateFile(archiveDone, "w");
    if (fd == NULL)
    {
        ereport(LOG,
                (errcode_for_file_access(),
                 errmsg("could not create archive status file \"%s\": %m",
                        archiveDone)));
        return;
    }
    if (FreeFile(fd))
    {
        ereport(LOG,
                (errcode_for_file_access(),
                 errmsg("could not write archive status file \"%s\": %m",
                        archiveDone)));
        return;
    }
}

XLogArchiveIsBusy()

bool XLogArchiveIsBusy

(

const char *

xlog

)

Definition at line 626 of file xlogarchive.c.

References MAXPGPATH, snprintf, stat, StatusFilePath, and XLOGDIR.

Referenced by do_pg_stop_backup().

{
    char        archiveStatusPath[MAXPGPATH];
    struct stat stat_buf;

    /* First check for .done --- this means archiver is done with it */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return false;

    /* check for .ready --- this means archiver is still busy with it */
    StatusFilePath(archiveStatusPath, xlog, ".ready");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    /* Race condition --- maybe archiver just finished, so recheck */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return false;

    /*
     * Check to see if the WAL file has been removed by checkpoint, which
     * implies it has already been archived, and explains why we can't see a
     * status file for it.
     */
    snprintf(archiveStatusPath, MAXPGPATH, XLOGDIR "/%s", xlog);
    if (stat(archiveStatusPath, &stat_buf) != 0 &&
        errno == ENOENT)
        return false;

    return true;
}

XLogArchiveIsReady()

bool XLogArchiveIsReady

(

const char *

xlog

)

Definition at line 701 of file xlogarchive.c.

References MAXPGPATH, stat, and StatusFilePath.

Referenced by RemoveNonParentXlogFiles().

{
    char        archiveStatusPath[MAXPGPATH];
    struct stat stat_buf;

    StatusFilePath(archiveStatusPath, xlog, ".ready");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    return false;
}

XLogArchiveIsReadyOrDone()

bool XLogArchiveIsReadyOrDone

(

const char *

xlog

)

Definition at line 671 of file xlogarchive.c.

References MAXPGPATH, stat, and StatusFilePath.

Referenced by StartupXLOG().

{
    char        archiveStatusPath[MAXPGPATH];
    struct stat stat_buf;

    /* First check for .done --- this means archiver is done with it */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    /* check for .ready --- this means archiver is still busy with it */
    StatusFilePath(archiveStatusPath, xlog, ".ready");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    /* Race condition --- maybe archiver just finished, so recheck */
    StatusFilePath(archiveStatusPath, xlog, ".done");
    if (stat(archiveStatusPath, &stat_buf) == 0)
        return true;

    return false;
}

XLogArchiveNotify()

void XLogArchiveNotify

(

const char *

xlog

)

Definition at line 467 of file xlogarchive.c.

References AllocateFile(), ereport, errcode_for_file_access(), errmsg(), fd(), FreeFile(), IsUnderPostmaster, LOG, MAXPGPATH, PMSIGNAL_WAKEN_ARCHIVER, SendPostmasterSignal(), and StatusFilePath.

Referenced by KeepFileRestoredFromArchive(), StartupXLOG(), WalRcvFetchTimeLineHistoryFiles(), WalReceiverMain(), writeTimeLineHistory(), XLogArchiveCheckDone(), XLogArchiveNotifySeg(), and XLogWalRcvWrite().

{
    char        archiveStatusPath[MAXPGPATH];
    FILE       *fd;

    /* insert an otherwise empty file called <XLOG>.ready */
    StatusFilePath(archiveStatusPath, xlog, ".ready");
    fd = AllocateFile(archiveStatusPath, "w");
    if (fd == NULL)
    {
        ereport(LOG,
                (errcode_for_file_access(),
                 errmsg("could not create archive status file \"%s\": %m",
                        archiveStatusPath)));
        return;
    }
    if (FreeFile(fd))
    {
        ereport(LOG,
                (errcode_for_file_access(),
                 errmsg("could not write archive status file \"%s\": %m",
                        archiveStatusPath)));
        return;
    }

    /* Notify archiver that it's got something to do */
    if (IsUnderPostmaster)
        SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER);
}

XLogArchiveNotifySeg()

void XLogArchiveNotifySeg

(

XLogSegNo

segno

)

Definition at line 501 of file xlogarchive.c.

References MAXFNAMELEN, ThisTimeLineID, wal_segment_size, XLogArchiveNotify(), and XLogFileName.

Referenced by XLogWrite().

{
    char        xlog[MAXFNAMELEN];

    XLogFileName(xlog, ThisTimeLineID, segno, wal_segment_size);
    XLogArchiveNotify(xlog);
}