#include "postgres_fe.h"
#include <sys/stat.h>
#include <unistd.h>
#include "access/xlog_internal.h"
#include "catalog/pg_tablespace_d.h"
#include "common/file_utils.h"
#include "common/hashfn_unstable.h"
#include "common/string.h"
#include "datapagemap.h"
#include "filemap.h"
#include "pg_rewind.h"
#include "lib/simplehash.h"

Include dependency graph for filemap.c:

Data Structures
struct	keepwal_entry

struct	exclude_list_item

Macros
#define	SH_PREFIX filehash

#define	SH_ELEMENT_TYPE file_entry_t

#define	SH_KEY_TYPE const char *

#define	SH_KEY path

#define	SH_HASH_KEY(tb, key) hash_string(key)

#define	SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)

#define	SH_SCOPE static inline

#define	SH_RAW_ALLOCATOR pg_malloc0

#define	SH_DECLARE

#define	SH_DEFINE

#define	FILEHASH_INITIAL_SIZE 1000

#define	SH_PREFIX keepwal

#define	SH_ELEMENT_TYPE keepwal_entry

#define	SH_KEY_TYPE const char *

#define	SH_KEY path

#define	SH_HASH_KEY(tb, key) hash_string(key)

#define	SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)

#define	SH_SCOPE static inline

#define	SH_RAW_ALLOCATOR pg_malloc0

#define	SH_DECLARE

#define	SH_DEFINE

#define	KEEPWAL_INITIAL_SIZE 1000

Typedefs
typedef struct keepwal_entry	keepwal_entry

Functions
static file_content_type_t	getFileContentType (const char *path)

static char *	datasegpath (RelFileLocator rlocator, ForkNumber forknum, BlockNumber segno)

static file_entry_t *	insert_filehash_entry (const char *path)

static file_entry_t *	lookup_filehash_entry (const char *path)

static bool	keepwal_entry_exists (const char *path)

static int	final_filemap_cmp (const void a, const void b)

static bool	check_file_excluded (const char *path, bool is_source)

void	filehash_init (void)

void	keepwal_init (void)

void	keepwal_add_entry (const char *path)

void	process_source_file (const char path, file_type_t type, size_t size, const char link_target)

void	process_target_file (const char path, file_type_t type, size_t size, const char link_target)

void	process_target_wal_block_change (ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)

static const char *	action_to_str (file_action_t action)

void	calculate_totals (filemap_t *filemap)

void	print_filemap (filemap_t *filemap)

static file_action_t	decide_wal_file_action (const char *fname, XLogSegNo last_common_segno, size_t source_size, size_t target_size)

static file_action_t	decide_file_action (file_entry_t *entry, XLogSegNo last_common_segno)

filemap_t *	decide_file_actions (XLogSegNo last_common_segno)

Variables
static filehash_hash *	filehash

static keepwal_hash *	keepwal = NULL

static const char *const	excludeDirContents []

static const struct exclude_list_item	excludeFiles []

Macro Definition Documentation

◆ FILEHASH_INITIAL_SIZE

#define FILEHASH_INITIAL_SIZE 1000

Definition at line 54 of file filemap.c.

◆ KEEPWAL_INITIAL_SIZE

#define KEEPWAL_INITIAL_SIZE 1000

Definition at line 86 of file filemap.c.

◆ SH_DECLARE [1/2]

#define SH_DECLARE

Definition at line 50 of file filemap.c.

◆ SH_DECLARE [2/2]

#define SH_DECLARE

Definition at line 50 of file filemap.c.

◆ SH_DEFINE [1/2]

#define SH_DEFINE

Definition at line 51 of file filemap.c.

◆ SH_DEFINE [2/2]

#define SH_DEFINE

Definition at line 51 of file filemap.c.

◆ SH_ELEMENT_TYPE [1/2]

#define SH_ELEMENT_TYPE file_entry_t

Definition at line 43 of file filemap.c.

◆ SH_ELEMENT_TYPE [2/2]

#define SH_ELEMENT_TYPE keepwal_entry

Definition at line 43 of file filemap.c.

◆ SH_EQUAL [1/2]

#define SH_EQUAL	(	tb,
		a,
		b
	)	(strcmp(a, b) == 0)

Definition at line 47 of file filemap.c.

◆ SH_EQUAL [2/2]

#define SH_EQUAL	(	tb,
		a,
		b
	)	(strcmp(a, b) == 0)

Definition at line 47 of file filemap.c.

◆ SH_HASH_KEY [1/2]

#define SH_HASH_KEY	(	tb,
		key
	)	hash_string(key)

Definition at line 46 of file filemap.c.

◆ SH_HASH_KEY [2/2]

#define SH_HASH_KEY	(	tb,
		key
	)	hash_string(key)

Definition at line 46 of file filemap.c.

◆ SH_KEY [1/2]

#define SH_KEY path

Definition at line 45 of file filemap.c.

◆ SH_KEY [2/2]

#define SH_KEY path

Definition at line 45 of file filemap.c.

◆ SH_KEY_TYPE [1/2]

#define SH_KEY_TYPE const char *

Definition at line 44 of file filemap.c.

◆ SH_KEY_TYPE [2/2]

#define SH_KEY_TYPE const char *

Definition at line 44 of file filemap.c.

◆ SH_PREFIX [1/2]

#define SH_PREFIX filehash

Definition at line 42 of file filemap.c.

◆ SH_PREFIX [2/2]

#define SH_PREFIX keepwal

Definition at line 42 of file filemap.c.

◆ SH_RAW_ALLOCATOR [1/2]

#define SH_RAW_ALLOCATOR pg_malloc0

Definition at line 49 of file filemap.c.

◆ SH_RAW_ALLOCATOR [2/2]

#define SH_RAW_ALLOCATOR pg_malloc0

Definition at line 49 of file filemap.c.

◆ SH_SCOPE [1/2]

#define SH_SCOPE static inline

Definition at line 48 of file filemap.c.

◆ SH_SCOPE [2/2]

#define SH_SCOPE static inline

Definition at line 48 of file filemap.c.

Typedef Documentation

◆ keepwal_entry

typedef struct keepwal_entry keepwal_entry

Function Documentation

◆ action_to_str()

static const char * action_to_str ( file_action_t action )

static

Definition at line 474 of file filemap.c.

{
    switch (action)
    {
        case FILE_ACTION_NONE:
            return "NONE";
        case FILE_ACTION_COPY:
            return "COPY";
        case FILE_ACTION_TRUNCATE:
            return "TRUNCATE";
        case FILE_ACTION_COPY_TAIL:
            return "COPY_TAIL";
        case FILE_ACTION_CREATE:
            return "CREATE";
        case FILE_ACTION_REMOVE:
            return "REMOVE";
 
        default:
            return "unknown";
    }
}

References FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, and FILE_ACTION_TRUNCATE.

Referenced by print_filemap().

◆ calculate_totals()

void calculate_totals ( filemap_t * filemap )

Definition at line 500 of file filemap.c.

{
    file_entry_t *entry;
    int         i;
 
    filemap->total_size = 0;
    filemap->fetch_size = 0;
 
    for (i = 0; i < filemap->nentries; i++)
    {
        entry = filemap->entries[i];
 
        if (entry->source_type != FILE_TYPE_REGULAR)
            continue;
 
        filemap->total_size += entry->source_size;
 
        if (entry->action == FILE_ACTION_COPY)
        {
            filemap->fetch_size += entry->source_size;
            continue;
        }
 
        if (entry->action == FILE_ACTION_COPY_TAIL)
            filemap->fetch_size += (entry->source_size - entry->target_size);
 
        if (entry->target_pages_to_overwrite.bitmapsize > 0)
        {
            datapagemap_iterator_t *iter;
            BlockNumber blk;
 
            iter = datapagemap_iterate(&entry->target_pages_to_overwrite);
            while (datapagemap_next(iter, &blk))
                filemap->fetch_size += BLCKSZ;
 
            pg_free(iter);
        }
    }
}

References file_entry_t::action, datapagemap::bitmapsize, datapagemap_iterate(), datapagemap_next(), fb(), FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_TYPE_REGULAR, i, pg_free(), file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_size.

Referenced by main().

◆ check_file_excluded()

static bool check_file_excluded	(	const char *	path,
		bool	is_source
	)

static

Definition at line 410 of file filemap.c.

{
    char        localpath[MAXPGPATH];
    int         excludeIdx;
    const char *filename;
 
    /*
     * Skip all temporary files, .../pgsql_tmp/... and .../pgsql_tmp.*
     */
    if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL ||
        strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
    {
        return true;
    }
 
    /* check individual files... */
    for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
    {
        int         cmplen = strlen(excludeFiles[excludeIdx].name);
 
        filename = last_dir_separator(path);
        if (filename == NULL)
            filename = path;
        else
            filename++;
 
        if (!excludeFiles[excludeIdx].match_prefix)
            cmplen++;
        if (strncmp(filename, excludeFiles[excludeIdx].name, cmplen) == 0)
        {
            if (is_source)
                pg_log_debug("entry \"%s\" excluded from source file list",
                             path);
            else
                pg_log_debug("entry \"%s\" excluded from target file list",
                             path);
            return true;
        }
    }
 
    /*
     * ... And check some directories.  Note that this includes any contents
     * within the directories themselves.
     */
    for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
    {
        snprintf(localpath, sizeof(localpath), "%s/",
                 excludeDirContents[excludeIdx]);
        if (strstr(path, localpath) == path)
        {
            if (is_source)
                pg_log_debug("entry \"%s\" excluded from source file list",
                             path);
            else
                pg_log_debug("entry \"%s\" excluded from target file list",
                             path);
            return true;
        }
    }
 
    return false;
}

References excludeDirContents, excludeFiles, fb(), filename, last_dir_separator(), exclude_list_item::match_prefix, MAXPGPATH, exclude_list_item::name, name, pg_log_debug, PG_TEMP_FILE_PREFIX, PG_TEMP_FILES_DIR, and snprintf.

Referenced by decide_file_action().

◆ datasegpath()

static char * datasegpath	(	RelFileLocator	rlocator,
		ForkNumber	forknum,
		BlockNumber	segno
	)

static

Definition at line 669 of file filemap.c.

{
    RelPathStr  path;
    char       *segpath;
 
    path = relpathperm(rlocator, forknum);
    if (segno > 0)
    {
        segpath = psprintf("%s.%u", path.str, segno);
        return segpath;
    }
    else
        return pstrdup(path.str);
}

References fb(), psprintf(), pstrdup(), relpathperm, and RelPathStr::str.

Referenced by getFileContentType(), and process_target_wal_block_change().

◆ decide_file_action()

static file_action_t decide_file_action	(	file_entry_t *	entry,
		XLogSegNo	last_common_segno
	)

static

Definition at line 749 of file filemap.c.

{
    const char *path = entry->path;
 
    /*
     * Don't touch the control file. It is handled specially, after copying
     * all the other files.
     */
    if (strcmp(path, XLOG_CONTROL_FILE) == 0)
        return FILE_ACTION_NONE;
 
    /* Skip macOS system files */
    if (strstr(path, ".DS_Store") != NULL)
        return FILE_ACTION_NONE;
 
    /*
     * Remove all files matching the exclusion filters in the target.
     */
    if (check_file_excluded(path, true))
    {
        if (entry->target_exists)
            return FILE_ACTION_REMOVE;
        else
            return FILE_ACTION_NONE;
    }
 
    /*
     * Handle cases where the file is missing from one of the systems.
     */
    if (!entry->target_exists && entry->source_exists)
    {
        /*
         * File exists in source, but not in target. Copy it in toto. (If it's
         * a relation data file, WAL replay after rewinding should re-create
         * it anyway. But there's no harm in copying it now.)
         */
        switch (entry->source_type)
        {
            case FILE_TYPE_DIRECTORY:
            case FILE_TYPE_SYMLINK:
                return FILE_ACTION_CREATE;
            case FILE_TYPE_REGULAR:
                return FILE_ACTION_COPY;
            case FILE_TYPE_UNDEFINED:
                pg_fatal("unknown file type for \"%s\"", entry->path);
                break;
        }
    }
    else if (entry->target_exists && !entry->source_exists)
    {
        /*
         * For files that exist in target but not in source, we check the
         * keepwal hash table; any files listed therein must not be removed.
         */
        if (keepwal_entry_exists(path))
        {
            pg_log_debug("Not removing file \"%s\" because it is required for recovery", path);
            return FILE_ACTION_NONE;
        }
        return FILE_ACTION_REMOVE;
    }
    else if (!entry->target_exists && !entry->source_exists)
    {
        /*
         * Doesn't exist in either server. Why does it have an entry in the
         * first place??
         */
        Assert(false);
        return FILE_ACTION_NONE;
    }
 
    /*
     * Otherwise, the file exists on both systems
     */
    Assert(entry->target_exists && entry->source_exists);
 
    if (entry->source_type != entry->target_type)
    {
        /* But it's a different kind of object. Strange.. */
        pg_fatal("file \"%s\" is of different type in source and target", entry->path);
    }
 
    /*
     * PG_VERSION files should be identical on both systems, but avoid
     * overwriting them for paranoia.
     */
    if (pg_str_endswith(entry->path, "PG_VERSION"))
        return FILE_ACTION_NONE;
 
    switch (entry->source_type)
    {
        case FILE_TYPE_DIRECTORY:
            return FILE_ACTION_NONE;
 
        case FILE_TYPE_SYMLINK:
 
            /*
             * XXX: Should we check if it points to the same target?
             */
            return FILE_ACTION_NONE;
 
        case FILE_TYPE_REGULAR:
            if (entry->content_type == FILE_CONTENT_TYPE_WAL)
            {
                /* Handle WAL segment file */
                const char *filename = last_dir_separator(entry->path);
 
                if (filename == NULL)
                    filename = entry->path;
                else
                    filename++; /* Skip the separator */
 
                return decide_wal_file_action(filename, last_common_segno,
                                              entry->source_size,
                                              entry->target_size);
            }
            else if (entry->content_type != FILE_CONTENT_TYPE_RELATION)
            {
                /*
                 * It's a non-data file that we have no special processing
                 * for. Copy it in toto.
                 */
                return FILE_ACTION_COPY;
            }
            else
            {
                /*
                 * It's a data file that exists in both systems.
                 *
                 * If it's larger in target, we can truncate it. There will
                 * also be a WAL record of the truncation in the source
                 * system, so WAL replay would eventually truncate the target
                 * too, but we might as well do it now.
                 *
                 * If it's smaller in the target, it means that it has been
                 * truncated in the target, or enlarged in the source, or
                 * both. If it was truncated in the target, we need to copy
                 * the missing tail from the source system. If it was enlarged
                 * in the source system, there will be WAL records in the
                 * source system for the new blocks, so we wouldn't need to
                 * copy them here. But we don't know which scenario we're
                 * dealing with, and there's no harm in copying the missing
                 * blocks now, so do it now.
                 *
                 * If it's the same size, do nothing here. Any blocks modified
                 * in the target will be copied based on parsing the target
                 * system's WAL, and any blocks modified in the source will be
                 * updated after rewinding, when the source system's WAL is
                 * replayed.
                 */
                if (entry->target_size < entry->source_size)
                    return FILE_ACTION_COPY_TAIL;
                else if (entry->target_size > entry->source_size)
                    return FILE_ACTION_TRUNCATE;
                else
                    return FILE_ACTION_NONE;
            }
            break;
 
        case FILE_TYPE_UNDEFINED:
            pg_fatal("unknown file type for \"%s\"", path);
            break;
    }
 
    /* unreachable */
    pg_fatal("could not decide what to do with file \"%s\"", path);
}

References Assert, check_file_excluded(), file_entry_t::content_type, decide_wal_file_action(), fb(), FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, FILE_ACTION_TRUNCATE, FILE_CONTENT_TYPE_RELATION, FILE_CONTENT_TYPE_WAL, FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, FILE_TYPE_UNDEFINED, filename, keepwal_entry_exists(), last_dir_separator(), file_entry_t::path, pg_fatal, pg_log_debug, pg_str_endswith(), file_entry_t::source_exists, file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_exists, file_entry_t::target_size, file_entry_t::target_type, and XLOG_CONTROL_FILE.

Referenced by decide_file_actions().

◆ decide_file_actions()

filemap_t * decide_file_actions ( XLogSegNo last_common_segno )

Definition at line 924 of file filemap.c.

{
    int         i;
    filehash_iterator it;
    file_entry_t *entry;
    filemap_t  *filemap;
 
    filehash_start_iterate(filehash, &it);
    while ((entry = filehash_iterate(filehash, &it)) != NULL)
    {
        entry->action = decide_file_action(entry, last_common_segno);
    }
 
    /*
     * Turn the hash table into an array, and sort in the order that the
     * actions should be performed.
     */
    filemap = pg_malloc(offsetof(filemap_t, entries) +
                        filehash->members * sizeof(file_entry_t *));
    filemap->nentries = filehash->members;
    filehash_start_iterate(filehash, &it);
    i = 0;
    while ((entry = filehash_iterate(filehash, &it)) != NULL)
    {
        filemap->entries[i++] = entry;
    }
 
    qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
          final_filemap_cmp);
 
    return filemap;
}

References file_entry_t::action, decide_file_action(), fb(), filehash, final_filemap_cmp(), i, pg_malloc(), and qsort.

Referenced by main().

◆ decide_wal_file_action()

static file_action_t decide_wal_file_action	(	const char *	fname,
		XLogSegNo	last_common_segno,
		size_t	source_size,
		size_t	target_size
	)

static

Definition at line 719 of file filemap.c.

{
    TimeLineID  file_tli;
    XLogSegNo   file_segno;
 
    /* Get current WAL segment number given current segment file name */
    XLogFromFileName(fname, &file_tli, &file_segno, WalSegSz);
 
    /*
     * Avoid copying files before the last common segment.
     *
     * These files exist on the source and the target servers, so they should
     * be identical and located strictly before the segment that contains the
     * LSN where target and source servers have diverged.
     *
     * While we are on it, double-check the size of each file and copy the
     * file if they do not match, in case.
     */
    if (file_segno < last_common_segno &&
        source_size == target_size)
        return FILE_ACTION_NONE;
 
    return FILE_ACTION_COPY;
}

References fb(), FILE_ACTION_COPY, FILE_ACTION_NONE, WalSegSz, and XLogFromFileName().

Referenced by decide_file_action().

◆ filehash_init()

void filehash_init ( void )

Definition at line 197 of file filemap.c.

{
    filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
}

References fb(), filehash, and FILEHASH_INITIAL_SIZE.

Referenced by main().

◆ final_filemap_cmp()

static int final_filemap_cmp	(	const void *	a,
		const void *	b
	)

static

Definition at line 695 of file filemap.c.

{
    file_entry_t *fa = *((file_entry_t *const *) a);
    file_entry_t *fb = *((file_entry_t *const *) b);
 
    if (fa->action > fb->action)
        return 1;
    if (fa->action < fb->action)
        return -1;
 
    if (fa->action == FILE_ACTION_REMOVE)
        return strcmp(fb->path, fa->path);
    else
        return strcmp(fa->path, fb->path);
}

References a, b, fa(), fb(), and FILE_ACTION_REMOVE.

Referenced by decide_file_actions().

◆ getFileContentType()

static file_content_type_t getFileContentType ( const char * path )

static

Definition at line 568 of file filemap.c.

{
    RelFileLocator rlocator;
    unsigned int segNo;
    int         nmatch;
    file_content_type_t result = FILE_CONTENT_TYPE_OTHER;
 
    /* Check if it is a WAL file. */
    if (strncmp("pg_wal/", path, 7) == 0)
    {
        const char *filename = path + 7;    /* Skip "pg_wal/" */
 
        if (IsXLogFileName(filename))
            return FILE_CONTENT_TYPE_WAL;
        else
            return FILE_CONTENT_TYPE_OTHER;
    }
 
    /*----
     * Does it look like a relation data file?
     *
     * For our purposes, only files belonging to the main fork are considered
     * relation files. Other forks are always copied in toto, because we
     * cannot reliably track changes to them, because WAL only contains block
     * references for the main fork.
     *
     * Relation data files can be in one of the following directories:
     *
     * global/
     *      shared relations
     *
     * base/<db oid>/
     *      regular relations, default tablespace
     *
     * pg_tblspc/<tblspc oid>/<tblspc version>/
     *      within a non-default tablespace (the name of the directory
     *      depends on version)
     *
     * And the relation data files themselves have a filename like:
     *
     * <oid>.<segment number>
     *
     *----
     */
    rlocator.spcOid = InvalidOid;
    rlocator.dbOid = InvalidOid;
    rlocator.relNumber = InvalidRelFileNumber;
    segNo = 0;
    result = FILE_CONTENT_TYPE_OTHER;
 
    nmatch = sscanf(path, "global/%u.%u", &rlocator.relNumber, &segNo);
    if (nmatch == 1 || nmatch == 2)
    {
        rlocator.spcOid = GLOBALTABLESPACE_OID;
        rlocator.dbOid = 0;
        result = FILE_CONTENT_TYPE_RELATION;
    }
    else
    {
        nmatch = sscanf(path, "base/%u/%u.%u",
                        &rlocator.dbOid, &rlocator.relNumber, &segNo);
        if (nmatch == 2 || nmatch == 3)
        {
            rlocator.spcOid = DEFAULTTABLESPACE_OID;
            result = FILE_CONTENT_TYPE_RELATION;
        }
        else
        {
            nmatch = sscanf(path, "pg_tblspc/%u/" TABLESPACE_VERSION_DIRECTORY "/%u/%u.%u",
                            &rlocator.spcOid, &rlocator.dbOid, &rlocator.relNumber,
                            &segNo);
            if (nmatch == 3 || nmatch == 4)
                result = FILE_CONTENT_TYPE_RELATION;
        }
    }
 
    /*
     * The sscanf tests above can match files that have extra characters at
     * the end. To eliminate such cases, cross-check that GetRelationPath
     * creates the exact same filename, when passed the RelFileLocator
     * information we extracted from the filename.
     */
    if (result == FILE_CONTENT_TYPE_RELATION)
    {
        char       *check_path = datasegpath(rlocator, MAIN_FORKNUM, segNo);
 
        if (strcmp(check_path, path) != 0)
            result = FILE_CONTENT_TYPE_OTHER;
 
        pfree(check_path);
    }
 
    return result;
}

References datasegpath(), RelFileLocator::dbOid, fb(), FILE_CONTENT_TYPE_OTHER, FILE_CONTENT_TYPE_RELATION, FILE_CONTENT_TYPE_WAL, filename, InvalidOid, InvalidRelFileNumber, IsXLogFileName(), MAIN_FORKNUM, pfree(), RelFileLocator::relNumber, RelFileLocator::spcOid, and TABLESPACE_VERSION_DIRECTORY.

Referenced by insert_filehash_entry(), and process_source_file().

◆ insert_filehash_entry()

static file_entry_t * insert_filehash_entry ( const char * path )

static

Definition at line 204 of file filemap.c.

{
    file_entry_t *entry;
    bool        found;
 
    entry = filehash_insert(filehash, path, &found);
    if (!found)
    {
        entry->path = pg_strdup(path);
        entry->content_type = getFileContentType(path);
 
        entry->target_exists = false;
        entry->target_type = FILE_TYPE_UNDEFINED;
        entry->target_size = 0;
        entry->target_link_target = NULL;
        entry->target_pages_to_overwrite.bitmap = NULL;
        entry->target_pages_to_overwrite.bitmapsize = 0;
 
        entry->source_exists = false;
        entry->source_type = FILE_TYPE_UNDEFINED;
        entry->source_size = 0;
        entry->source_link_target = NULL;
 
        entry->action = FILE_ACTION_UNDECIDED;
    }
 
    return entry;
}

References file_entry_t::action, datapagemap::bitmap, datapagemap::bitmapsize, file_entry_t::content_type, fb(), FILE_ACTION_UNDECIDED, FILE_TYPE_UNDEFINED, filehash, getFileContentType(), file_entry_t::path, pg_strdup(), file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and file_entry_t::target_type.

Referenced by process_source_file(), and process_target_file().

◆ keepwal_add_entry()

void keepwal_add_entry ( const char * path )

Definition at line 251 of file filemap.c.

{
    keepwal_entry *entry;
    bool        found;
 
    /* Should only be called with keepwal initialized */
    Assert(keepwal != NULL);
 
    entry = keepwal_insert(keepwal, path, &found);
 
    if (!found)
        entry->path = pg_strdup(path);
}

References Assert, fb(), keepwal, keepwal_entry::path, and pg_strdup().

Referenced by findLastCheckpoint().

◆ keepwal_entry_exists()

static bool keepwal_entry_exists ( const char * path )

static

Definition at line 267 of file filemap.c.

{
    return keepwal_lookup(keepwal, path) != NULL;
}

References fb(), and keepwal.

Referenced by decide_file_action().

◆ keepwal_init()

void keepwal_init ( void )

Definition at line 243 of file filemap.c.

{
    /* An initial hash size out of thin air */
    keepwal = keepwal_create(KEEPWAL_INITIAL_SIZE, NULL);
}

References fb(), keepwal, and KEEPWAL_INITIAL_SIZE.

Referenced by main().

◆ lookup_filehash_entry()

static file_entry_t * lookup_filehash_entry ( const char * path )

static

Definition at line 234 of file filemap.c.

{
    return filehash_lookup(filehash, path);
}

References fb(), and filehash.

Referenced by process_target_wal_block_change().

◆ print_filemap()

void print_filemap ( filemap_t * filemap )

Definition at line 541 of file filemap.c.

{
    file_entry_t *entry;
    int         i;
 
    for (i = 0; i < filemap->nentries; i++)
    {
        entry = filemap->entries[i];
 
        if (entry->action != FILE_ACTION_NONE ||
            entry->content_type == FILE_CONTENT_TYPE_WAL ||
            entry->target_pages_to_overwrite.bitmapsize > 0)
        {
            pg_log_debug("%s (%s)", entry->path,
                         action_to_str(entry->action));
 
            if (entry->target_pages_to_overwrite.bitmapsize > 0)
                datapagemap_print(&entry->target_pages_to_overwrite);
        }
    }
    fflush(stdout);
}

References file_entry_t::action, action_to_str(), datapagemap::bitmapsize, file_entry_t::content_type, datapagemap_print(), fb(), FILE_ACTION_NONE, FILE_CONTENT_TYPE_WAL, i, file_entry_t::path, pg_log_debug, and file_entry_t::target_pages_to_overwrite.

Referenced by main().

◆ process_source_file()

void process_source_file	(	const char *	path,
		file_type_t	type,
		size_t	size,
		const char *	link_target
	)

Definition at line 280 of file filemap.c.

{
    file_entry_t *entry;
 
    /*
     * Pretend that pg_wal is a directory, even if it's really a symlink. We
     * don't want to mess with the symlink itself, nor complain if it's a
     * symlink in source but not in target or vice versa.
     */
    if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
        type = FILE_TYPE_DIRECTORY;
 
    /*
     * sanity check: a filename that looks like a data file better be a
     * regular file
     */
    if (type != FILE_TYPE_REGULAR && getFileContentType(path) == FILE_CONTENT_TYPE_RELATION)
        pg_fatal("data file \"%s\" in source is not a regular file", path);
 
    /* Remember this source file */
    entry = insert_filehash_entry(path);
    if (entry->source_exists)
        pg_fatal("duplicate source file \"%s\"", path);
    entry->source_exists = true;
    entry->source_type = type;
    entry->source_size = size;
    entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
}

References fb(), FILE_CONTENT_TYPE_RELATION, FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, getFileContentType(), insert_filehash_entry(), pg_fatal, pg_strdup(), file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, and type.

Referenced by main().

◆ process_target_file()

void process_target_file	(	const char *	path,
		file_type_t	type,
		size_t	size,
		const char *	link_target
	)

Definition at line 316 of file filemap.c.

{
    file_entry_t *entry;
 
    /*
     * Do not apply any exclusion filters here.  This has advantage to remove
     * from the target data folder all paths which have been filtered out from
     * the source data folder when processing the source files.
     */
 
    /*
     * Like in process_source_file, pretend that pg_wal is always a directory.
     */
    if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
        type = FILE_TYPE_DIRECTORY;
 
    /* Remember this target file */
    entry = insert_filehash_entry(path);
    if (entry->target_exists)
        pg_fatal("duplicate source file \"%s\"", path);
    entry->target_exists = true;
    entry->target_type = type;
    entry->target_size = size;
    entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
}

References fb(), FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK, insert_filehash_entry(), pg_fatal, pg_strdup(), file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_size, file_entry_t::target_type, and type.

Referenced by main().

◆ process_target_wal_block_change()

void process_target_wal_block_change	(	ForkNumber	forknum,
		RelFileLocator	rlocator,
		BlockNumber	blkno
	)

Definition at line 353 of file filemap.c.

{
    char       *path;
    file_entry_t *entry;
    BlockNumber blkno_inseg;
    int         segno;
 
    segno = blkno / RELSEG_SIZE;
    blkno_inseg = blkno % RELSEG_SIZE;
 
    path = datasegpath(rlocator, forknum, segno);
    entry = lookup_filehash_entry(path);
    pfree(path);
 
    /*
     * If the block still exists in both systems, remember it. Otherwise we
     * can safely ignore it.
     *
     * If the block is beyond the EOF in the source system, or the file
     * doesn't exist in the source at all, we're going to truncate/remove it
     * away from the target anyway. Likewise, if it doesn't exist in the
     * target anymore, we will copy it over with the "tail" from the source
     * system, anyway.
     *
     * It is possible to find WAL for a file that doesn't exist on either
     * system anymore. It means that the relation was dropped later in the
     * target system, and independently on the source system too, or that it
     * was created and dropped in the target system and it never existed in
     * the source. Either way, we can safely ignore it.
     */
    if (entry)
    {
        Assert(entry->content_type == FILE_CONTENT_TYPE_RELATION);
 
        if (entry->target_exists)
        {
            if (entry->target_type != FILE_TYPE_REGULAR)
                pg_fatal("unexpected page modification for non-regular file \"%s\"",
                         entry->path);
 
            if (entry->source_exists)
            {
                off_t       end_offset;
 
                end_offset = (blkno_inseg + 1) * BLCKSZ;
                if (end_offset <= entry->source_size && end_offset <= entry->target_size)
                    datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
            }
        }
    }
}

References Assert, file_entry_t::content_type, datapagemap_add(), datasegpath(), fb(), FILE_CONTENT_TYPE_RELATION, FILE_TYPE_REGULAR, lookup_filehash_entry(), file_entry_t::path, pfree(), pg_fatal, file_entry_t::source_exists, file_entry_t::target_exists, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_type.

Referenced by extractPageInfo().

Variable Documentation

◆ excludeDirContents

const char* const excludeDirContents[]

static

Initial value:

=
{
    
    "pg_stat_tmp",              
 
    "pg_replslot",              
 
    "pg_dynshmem",              
 
    "pg_notify",
 
    "pg_serial",
 
    "pg_snapshots",
 
    "pg_subtrans",
 
    NULL
}

Definition at line 117 of file filemap.c.

{
    /*
     * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
     * because extensions like pg_stat_statements store data there.
     */
    "pg_stat_tmp",              /* defined as PG_STAT_TMP_DIR */
 
    /*
     * It is generally not useful to backup the contents of this directory
     * even if the intention is to restore to another primary. See backup.sgml
     * for a more detailed description.
     */
    "pg_replslot",              /* defined as PG_REPLSLOT_DIR */
 
    /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
    "pg_dynshmem",              /* defined as PG_DYNSHMEM_DIR */
 
    /* Contents removed on startup, see AsyncShmemInit(). */
    "pg_notify",
 
    /*
     * Old contents are loaded for possible debugging but are not required for
     * normal operation, see SerialInit().
     */
    "pg_serial",
 
    /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
    "pg_snapshots",
 
    /* Contents zeroed on startup, see StartupSUBTRANS(). */
    "pg_subtrans",
 
    /* end of list */
    NULL
};

Referenced by check_file_excluded().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]

static

Initial value:

=
{
    
    {"postgresql.auto.conf.tmp", false},    
 
    {"current_logfiles.tmp", false},    
 
    {"pg_internal.init", true}, 
 
    {"backup_label", false},    
    {"tablespace_map", false},  
 
    {"backup_manifest", false},
 
    {"postmaster.pid", false},
    {"postmaster.opts", false},
 
    {NULL, false}
}

Definition at line 158 of file filemap.c.

{
    /* Skip auto conf temporary file. */
    {"postgresql.auto.conf.tmp", false},    /* defined as PG_AUTOCONF_FILENAME */
 
    /* Skip current log file temporary file */
    {"current_logfiles.tmp", false},    /* defined as
                                         * LOG_METAINFO_DATAFILE_TMP */
 
    /* Skip relation cache because it is rebuilt on startup */
    {"pg_internal.init", true}, /* defined as RELCACHE_INIT_FILENAME */
 
    /*
     * If there is a backup_label or tablespace_map file, it indicates that a
     * recovery failed and this cluster probably can't be rewound, but exclude
     * them anyway if they are found.
     */
    {"backup_label", false},    /* defined as BACKUP_LABEL_FILE */
    {"tablespace_map", false},  /* defined as TABLESPACE_MAP */
 
    /*
     * If there's a backup_manifest, it belongs to a backup that was used to
     * start this server. It is *not* correct for this backup. Our
     * backup_manifest is injected into the backup separately if users want
     * it.
     */
    {"backup_manifest", false},
 
    {"postmaster.pid", false},
    {"postmaster.opts", false},
 
    /* end of list */
    {NULL, false}
};

Referenced by check_file_excluded().

◆ filehash

filehash_hash* filehash

static

Definition at line 56 of file filemap.c.

Referenced by decide_file_actions(), filehash_init(), insert_filehash_entry(), and lookup_filehash_entry().

◆ keepwal

keepwal_hash* keepwal = NULL

static

Definition at line 89 of file filemap.c.

Referenced by keepwal_add_entry(), keepwal_entry_exists(), and keepwal_init().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ FILEHASH_INITIAL_SIZE

◆ KEEPWAL_INITIAL_SIZE

◆ SH_DECLARE [1/2]

◆ SH_DECLARE [2/2]

◆ SH_DEFINE [1/2]

◆ SH_DEFINE [2/2]

◆ SH_ELEMENT_TYPE [1/2]

◆ SH_ELEMENT_TYPE [2/2]

◆ SH_EQUAL [1/2]

◆ SH_EQUAL [2/2]

◆ SH_HASH_KEY [1/2]

◆ SH_HASH_KEY [2/2]

◆ SH_KEY [1/2]

◆ SH_KEY [2/2]

◆ SH_KEY_TYPE [1/2]

◆ SH_KEY_TYPE [2/2]

◆ SH_PREFIX [1/2]

◆ SH_PREFIX [2/2]

◆ SH_RAW_ALLOCATOR [1/2]

◆ SH_RAW_ALLOCATOR [2/2]

◆ SH_SCOPE [1/2]

◆ SH_SCOPE [2/2]

Typedef Documentation

◆ keepwal_entry

Function Documentation

◆ action_to_str()

◆ calculate_totals()

◆ check_file_excluded()

◆ datasegpath()

◆ decide_file_action()

◆ decide_file_actions()

◆ decide_wal_file_action()

◆ filehash_init()

◆ final_filemap_cmp()

◆ getFileContentType()

◆ insert_filehash_entry()

◆ keepwal_add_entry()

◆ keepwal_entry_exists()

◆ keepwal_init()

◆ lookup_filehash_entry()

◆ print_filemap()

◆ process_source_file()

◆ process_target_file()

◆ process_target_wal_block_change()

Variable Documentation

◆ excludeDirContents

◆ excludeFiles

◆ filehash

◆ keepwal