filemap.h File Reference

#include "datapagemap.h"
#include "storage/block.h"
#include "storage/relfilelocator.h"
#include "access/xlogdefs.h"

Include dependency graph for filemap.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	file_entry_t
struct	filemap_t

Typedefs
typedef struct file_entry_t	file_entry_t
typedef struct filemap_t	filemap_t

Enumerations
enum	file_action_t {   FILE_ACTION_UNDECIDED = 0 , FILE_ACTION_CREATE , FILE_ACTION_COPY , FILE_ACTION_COPY_TAIL ,   FILE_ACTION_NONE , FILE_ACTION_TRUNCATE , FILE_ACTION_REMOVE }
enum	file_type_t { FILE_TYPE_UNDEFINED = 0 , FILE_TYPE_REGULAR , FILE_TYPE_DIRECTORY , FILE_TYPE_SYMLINK }
enum	file_content_type_t { FILE_CONTENT_TYPE_OTHER = 0 , FILE_CONTENT_TYPE_RELATION , FILE_CONTENT_TYPE_WAL }

Functions
void	filehash_init (void)
void	process_source_file (const char *path, file_type_t type, size_t size, const char *link_target)
void	process_target_file (const char *path, file_type_t type, size_t size, const char *link_target)
void	process_target_wal_block_change (ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)
filemap_t *	decide_file_actions (XLogSegNo last_common_segno)
void	calculate_totals (filemap_t *filemap)
void	print_filemap (filemap_t *filemap)
void	keepwal_init (void)
void	keepwal_add_entry (const char *path)

Typedef Documentation

file_entry_t

typedef struct file_entry_t file_entry_t

filemap_t

typedef struct filemap_t filemap_t

Enumeration Type Documentation

file_action_t

enum file_action_t

Enumerator
FILE_ACTION_UNDECIDED
FILE_ACTION_CREATE
FILE_ACTION_COPY
FILE_ACTION_COPY_TAIL
FILE_ACTION_NONE
FILE_ACTION_TRUNCATE
FILE_ACTION_REMOVE

Definition at line 17 of file filemap.h.

{
    FILE_ACTION_UNDECIDED = 0,  /* not decided yet */
 
    FILE_ACTION_CREATE,         /* create local directory or symbolic link */
    FILE_ACTION_COPY,           /* copy whole file, overwriting if exists */
    FILE_ACTION_COPY_TAIL,      /* copy tail from 'source_size' to
                                 * 'target_size' */
    FILE_ACTION_NONE,           /* no action (we might still copy modified
                                 * blocks based on the parsed WAL) */
    FILE_ACTION_TRUNCATE,       /* truncate local file to 'newsize' bytes */
    FILE_ACTION_REMOVE,         /* remove local file / directory / symlink */
} file_action_t;

file_content_type_t

enum file_content_type_t

Enumerator
FILE_CONTENT_TYPE_OTHER
FILE_CONTENT_TYPE_RELATION
FILE_CONTENT_TYPE_WAL

Definition at line 40 of file filemap.h.

{
    FILE_CONTENT_TYPE_OTHER = 0,
    FILE_CONTENT_TYPE_RELATION,
    FILE_CONTENT_TYPE_WAL
} file_content_type_t;

file_type_t

enum file_type_t

Enumerator
FILE_TYPE_UNDEFINED
FILE_TYPE_REGULAR
FILE_TYPE_DIRECTORY
FILE_TYPE_SYMLINK

Definition at line 31 of file filemap.h.

{
    FILE_TYPE_UNDEFINED = 0,
 
    FILE_TYPE_REGULAR,
    FILE_TYPE_DIRECTORY,
    FILE_TYPE_SYMLINK,
} file_type_t;

Function Documentation

calculate_totals()

void calculate_totals

(

filemap_t *

filemap

)

Definition at line 500 of file filemap.c.

{
    file_entry_t *entry;
    int         i;
 
    filemap->total_size = 0;
    filemap->fetch_size = 0;
 
    for (i = 0; i < filemap->nentries; i++)
    {
        entry = filemap->entries[i];
 
        if (entry->source_type != FILE_TYPE_REGULAR)
            continue;
 
        filemap->total_size += entry->source_size;
 
        if (entry->action == FILE_ACTION_COPY)
        {
            filemap->fetch_size += entry->source_size;
            continue;
        }
 
        if (entry->action == FILE_ACTION_COPY_TAIL)
            filemap->fetch_size += (entry->source_size - entry->target_size);
 
        if (entry->target_pages_to_overwrite.bitmapsize > 0)
        {
            datapagemap_iterator_t *iter;
            BlockNumber blk;
 
            iter = datapagemap_iterate(&entry->target_pages_to_overwrite);
            while (datapagemap_next(iter, &blk))
                filemap->fetch_size += BLCKSZ;
 
            pg_free(iter);
        }
    }
}

References file_entry_t::action, datapagemap::bitmapsize, datapagemap_iterate(), datapagemap_next(), filemap_t::entries, filemap_t::fetch_size, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_TYPE_REGULAR, i, filemap_t::nentries, pg_free(), file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and filemap_t::total_size.

Referenced by main().

decide_file_actions()

filemap_t * decide_file_actions

(

XLogSegNo

last_common_segno

)

Definition at line 924 of file filemap.c.

{
    int         i;
    filehash_iterator it;
    file_entry_t *entry;
    filemap_t  *filemap;
 
    filehash_start_iterate(filehash, &it);
    while ((entry = filehash_iterate(filehash, &it)) != NULL)
    {
        entry->action = decide_file_action(entry, last_common_segno);
    }
 
    /*
     * Turn the hash table into an array, and sort in the order that the
     * actions should be performed.
     */
    filemap = pg_malloc(offsetof(filemap_t, entries) +
                        filehash->members * sizeof(file_entry_t *));
    filemap->nentries = filehash->members;
    filehash_start_iterate(filehash, &it);
    i = 0;
    while ((entry = filehash_iterate(filehash, &it)) != NULL)
    {
        filemap->entries[i++] = entry;
    }
 
    qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
          final_filemap_cmp);
 
    return filemap;
}

References file_entry_t::action, decide_file_action(), filemap_t::entries, filehash, final_filemap_cmp(), i, filemap_t::nentries, pg_malloc(), and qsort.

Referenced by main().

filehash_init()

void filehash_init

(

void

)

Definition at line 197 of file filemap.c.

{
    filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
}

References filehash, and FILEHASH_INITIAL_SIZE.

Referenced by main().

keepwal_add_entry()

void keepwal_add_entry

(

const char *

path

)

Definition at line 251 of file filemap.c.

{
    keepwal_entry *entry;
    bool        found;
 
    /* Should only be called with keepwal initialized */
    Assert(keepwal != NULL);
 
    entry = keepwal_insert(keepwal, path, &found);
 
    if (!found)
        entry->path = pg_strdup(path);
}

References Assert(), keepwal, keepwal_entry::path, and pg_strdup().

Referenced by findLastCheckpoint().

keepwal_init()

void keepwal_init

(

void

)

Definition at line 243 of file filemap.c.

{
    /* An initial hash size out of thin air */
    keepwal = keepwal_create(KEEPWAL_INITIAL_SIZE, NULL);
}

References keepwal, and KEEPWAL_INITIAL_SIZE.

Referenced by main().

print_filemap()

void print_filemap

(

filemap_t *

filemap

)

Definition at line 541 of file filemap.c.

{
    file_entry_t *entry;
    int         i;
 
    for (i = 0; i < filemap->nentries; i++)
    {
        entry = filemap->entries[i];
 
        if (entry->action != FILE_ACTION_NONE ||
            entry->content_type == FILE_CONTENT_TYPE_WAL ||
            entry->target_pages_to_overwrite.bitmapsize > 0)
        {
            pg_log_debug("%s (%s)", entry->path,
                         action_to_str(entry->action));
 
            if (entry->target_pages_to_overwrite.bitmapsize > 0)
                datapagemap_print(&entry->target_pages_to_overwrite);
        }
    }
    fflush(stdout);
}

References file_entry_t::action, action_to_str(), datapagemap::bitmapsize, file_entry_t::content_type, datapagemap_print(), filemap_t::entries, FILE_ACTION_NONE, FILE_CONTENT_TYPE_WAL, i, filemap_t::nentries, file_entry_t::path, pg_log_debug, generate_unaccent_rules::stdout, and file_entry_t::target_pages_to_overwrite.

Referenced by main().

process_source_file()

void process_source_file	(	const char *	path,
		file_type_t	type,
		size_t	size,
		const char *	link_target
	)

Definition at line 280 of file filemap.c.

{
    file_entry_t *entry;
 
    /*
     * Pretend that pg_wal is a directory, even if it's really a symlink. We
     * don't want to mess with the symlink itself, nor complain if it's a
     * symlink in source but not in target or vice versa.
     */
    if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
        type = FILE_TYPE_DIRECTORY;
 
    /*
     * sanity check: a filename that looks like a data file better be a
     * regular file
     */
    if (type != FILE_TYPE_REGULAR && getFileContentType(path) == FILE_CONTENT_TYPE_RELATION)
        pg_fatal("data file \"%s\" in source is not a regular file", path);
 
    /* Remember this source file */
    entry = insert_filehash_entry(path);
    if (entry->source_exists)
        pg_fatal("duplicate source file \"%s\"", path);
    entry->source_exists = true;
    entry->source_type = type;
    entry->source_size = size;
    entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
}

References FILE_CONTENT_TYPE_RELATION, FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, getFileContentType(), insert_filehash_entry(), pg_fatal, pg_strdup(), file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, and type.

Referenced by main().

process_target_file()

void process_target_file	(	const char *	path,
		file_type_t	type,
		size_t	size,
		const char *	link_target
	)

Definition at line 316 of file filemap.c.

{
    file_entry_t *entry;
 
    /*
     * Do not apply any exclusion filters here.  This has advantage to remove
     * from the target data folder all paths which have been filtered out from
     * the source data folder when processing the source files.
     */
 
    /*
     * Like in process_source_file, pretend that pg_wal is always a directory.
     */
    if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
        type = FILE_TYPE_DIRECTORY;
 
    /* Remember this target file */
    entry = insert_filehash_entry(path);
    if (entry->target_exists)
        pg_fatal("duplicate source file \"%s\"", path);
    entry->target_exists = true;
    entry->target_type = type;
    entry->target_size = size;
    entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
}

References FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK, insert_filehash_entry(), pg_fatal, pg_strdup(), file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_size, file_entry_t::target_type, and type.

Referenced by main().

process_target_wal_block_change()

void process_target_wal_block_change	(	ForkNumber	forknum,
		RelFileLocator	rlocator,
		BlockNumber	blkno
	)

Definition at line 353 of file filemap.c.

{
    char       *path;
    file_entry_t *entry;
    BlockNumber blkno_inseg;
    int         segno;
 
    segno = blkno / RELSEG_SIZE;
    blkno_inseg = blkno % RELSEG_SIZE;
 
    path = datasegpath(rlocator, forknum, segno);
    entry = lookup_filehash_entry(path);
    pfree(path);
 
    /*
     * If the block still exists in both systems, remember it. Otherwise we
     * can safely ignore it.
     *
     * If the block is beyond the EOF in the source system, or the file
     * doesn't exist in the source at all, we're going to truncate/remove it
     * away from the target anyway. Likewise, if it doesn't exist in the
     * target anymore, we will copy it over with the "tail" from the source
     * system, anyway.
     *
     * It is possible to find WAL for a file that doesn't exist on either
     * system anymore. It means that the relation was dropped later in the
     * target system, and independently on the source system too, or that it
     * was created and dropped in the target system and it never existed in
     * the source. Either way, we can safely ignore it.
     */
    if (entry)
    {
        Assert(entry->content_type == FILE_CONTENT_TYPE_RELATION);
 
        if (entry->target_exists)
        {
            if (entry->target_type != FILE_TYPE_REGULAR)
                pg_fatal("unexpected page modification for non-regular file \"%s\"",
                         entry->path);
 
            if (entry->source_exists)
            {
                off_t       end_offset;
 
                end_offset = (blkno_inseg + 1) * BLCKSZ;
                if (end_offset <= entry->source_size && end_offset <= entry->target_size)
                    datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
            }
        }
    }
}

References Assert(), file_entry_t::content_type, datapagemap_add(), datasegpath(), FILE_CONTENT_TYPE_RELATION, FILE_TYPE_REGULAR, lookup_filehash_entry(), file_entry_t::path, pfree(), pg_fatal, file_entry_t::source_exists, file_entry_t::target_exists, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_type.

Referenced by extractPageInfo().