PostgreSQL Source Code  git master
filemap.h File Reference
#include "datapagemap.h"
#include "storage/block.h"
#include "storage/relfilenode.h"
Include dependency graph for filemap.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  file_entry_t
 
struct  filemap_t
 

Typedefs

typedef struct file_entry_t file_entry_t
 
typedef struct filemap_t filemap_t
 

Enumerations

enum  file_action_t {
  FILE_ACTION_UNDECIDED = 0, FILE_ACTION_CREATE, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL,
  FILE_ACTION_NONE, FILE_ACTION_TRUNCATE, FILE_ACTION_REMOVE
}
 
enum  file_type_t { FILE_TYPE_UNDEFINED = 0, FILE_TYPE_REGULAR, FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK }
 

Functions

void filehash_init (void)
 
void process_source_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_wal_block_change (ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
 
filemap_tdecide_file_actions (void)
 
void calculate_totals (filemap_t *filemap)
 
void print_filemap (filemap_t *filemap)
 

Typedef Documentation

◆ file_entry_t

typedef struct file_entry_t file_entry_t

◆ filemap_t

typedef struct filemap_t filemap_t

Enumeration Type Documentation

◆ file_action_t

Enumerator
FILE_ACTION_UNDECIDED 
FILE_ACTION_CREATE 
FILE_ACTION_COPY 
FILE_ACTION_COPY_TAIL 
FILE_ACTION_NONE 
FILE_ACTION_TRUNCATE 
FILE_ACTION_REMOVE 

Definition at line 16 of file filemap.h.

17 {
18  FILE_ACTION_UNDECIDED = 0, /* not decided yet */
19 
20  FILE_ACTION_CREATE, /* create local directory or symbolic link */
21  FILE_ACTION_COPY, /* copy whole file, overwriting if exists */
22  FILE_ACTION_COPY_TAIL, /* copy tail from 'source_size' to
23  * 'target_size' */
24  FILE_ACTION_NONE, /* no action (we might still copy modified
25  * blocks based on the parsed WAL) */
26  FILE_ACTION_TRUNCATE, /* truncate local file to 'newsize' bytes */
27  FILE_ACTION_REMOVE /* remove local file / directory / symlink */
file_action_t
Definition: filemap.h:16

◆ file_type_t

Enumerator
FILE_TYPE_UNDEFINED 
FILE_TYPE_REGULAR 
FILE_TYPE_DIRECTORY 
FILE_TYPE_SYMLINK 

Definition at line 30 of file filemap.h.

Function Documentation

◆ calculate_totals()

void calculate_totals ( filemap_t filemap)

Definition at line 439 of file filemap.c.

References file_entry_t::action, datapagemap::bitmapsize, datapagemap_iterate(), datapagemap_next(), filemap_t::entries, filemap_t::fetch_size, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_TYPE_REGULAR, i, filemap_t::nentries, pg_free(), file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and filemap_t::total_size.

Referenced by main().

440 {
441  file_entry_t *entry;
442  int i;
443 
444  filemap->total_size = 0;
445  filemap->fetch_size = 0;
446 
447  for (i = 0; i < filemap->nentries; i++)
448  {
449  entry = filemap->entries[i];
450 
451  if (entry->source_type != FILE_TYPE_REGULAR)
452  continue;
453 
454  filemap->total_size += entry->source_size;
455 
456  if (entry->action == FILE_ACTION_COPY)
457  {
458  filemap->fetch_size += entry->source_size;
459  continue;
460  }
461 
462  if (entry->action == FILE_ACTION_COPY_TAIL)
463  filemap->fetch_size += (entry->source_size - entry->target_size);
464 
465  if (entry->target_pages_to_overwrite.bitmapsize > 0)
466  {
468  BlockNumber blk;
469 
471  while (datapagemap_next(iter, &blk))
472  filemap->fetch_size += BLCKSZ;
473 
474  pg_free(iter);
475  }
476  }
477 }
uint32 BlockNumber
Definition: block.h:31
uint64 fetch_size
Definition: filemap.h:93
size_t target_size
Definition: filemap.h:61
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
file_action_t action
Definition: filemap.h:81
file_type_t source_type
Definition: filemap.h:74
size_t source_size
Definition: filemap.h:75
uint64 total_size
Definition: filemap.h:92
datapagemap_t target_pages_to_overwrite
Definition: filemap.h:68
int bitmapsize
Definition: datapagemap.h:18
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
void pg_free(void *ptr)
Definition: fe_memutils.c:105
int nentries
Definition: filemap.h:95
int i
Definition: filemap.h:49
file_entry_t * entries[FLEXIBLE_ARRAY_MEMBER]
Definition: filemap.h:96

◆ decide_file_actions()

filemap_t* decide_file_actions ( void  )

Definition at line 789 of file filemap.c.

References file_entry_t::action, decide_file_action(), filemap_t::entries, filehash, final_filemap_cmp(), i, filemap_t::nentries, offsetof, pg_malloc(), and qsort.

Referenced by main().

790 {
791  int i;
792  filehash_iterator it;
793  file_entry_t *entry;
794  filemap_t *filemap;
795 
796  filehash_start_iterate(filehash, &it);
797  while ((entry = filehash_iterate(filehash, &it)) != NULL)
798  {
799  entry->action = decide_file_action(entry);
800  }
801 
802  /*
803  * Turn the hash table into an array, and sort in the order that the
804  * actions should be performed.
805  */
806  filemap = pg_malloc(offsetof(filemap_t, entries) +
807  filehash->members * sizeof(file_entry_t *));
808  filemap->nentries = filehash->members;
809  filehash_start_iterate(filehash, &it);
810  i = 0;
811  while ((entry = filehash_iterate(filehash, &it)) != NULL)
812  {
813  filemap->entries[i++] = entry;
814  }
815 
816  qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
818 
819  return filemap;
820 }
static int final_filemap_cmp(const void *a, const void *b)
Definition: filemap.c:620
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static file_action_t decide_file_action(file_entry_t *entry)
Definition: filemap.c:640
file_action_t action
Definition: filemap.h:81
static filehash_hash * filehash
Definition: filemap.c:56
int nentries
Definition: filemap.h:95
int i
Definition: filemap.h:49
file_entry_t * entries[FLEXIBLE_ARRAY_MEMBER]
Definition: filemap.h:96
#define qsort(a, b, c, d)
Definition: port.h:504
#define offsetof(type, field)
Definition: c.h:727

◆ filehash_init()

void filehash_init ( void  )

Definition at line 169 of file filemap.c.

References filehash, and FILEHASH_INITIAL_SIZE.

Referenced by main().

170 {
171  filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
172 }
static filehash_hash * filehash
Definition: filemap.c:56
#define FILEHASH_INITIAL_SIZE
Definition: filemap.c:54

◆ print_filemap()

void print_filemap ( filemap_t filemap)

Definition at line 480 of file filemap.c.

References file_entry_t::action, action_to_str(), datapagemap::bitmapsize, datapagemap_print(), filemap_t::entries, FILE_ACTION_NONE, i, filemap_t::nentries, file_entry_t::path, pg_log_debug, generate_unaccent_rules::stdout, and file_entry_t::target_pages_to_overwrite.

Referenced by main().

481 {
482  file_entry_t *entry;
483  int i;
484 
485  for (i = 0; i < filemap->nentries; i++)
486  {
487  entry = filemap->entries[i];
488  if (entry->action != FILE_ACTION_NONE ||
490  {
491  pg_log_debug("%s (%s)", entry->path,
492  action_to_str(entry->action));
493 
494  if (entry->target_pages_to_overwrite.bitmapsize > 0)
496  }
497  }
498  fflush(stdout);
499 }
#define pg_log_debug(...)
Definition: logging.h:92
file_action_t action
Definition: filemap.h:81
static const char * action_to_str(file_action_t action)
Definition: filemap.c:413
const char * path
Definition: filemap.h:53
datapagemap_t target_pages_to_overwrite
Definition: filemap.h:68
void datapagemap_print(datapagemap_t *map)
Definition: datapagemap.c:117
int bitmapsize
Definition: datapagemap.h:18
int nentries
Definition: filemap.h:95
int i
Definition: filemap.h:49
file_entry_t * entries[FLEXIBLE_ARRAY_MEMBER]
Definition: filemap.h:96

◆ process_source_file()

void process_source_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 219 of file filemap.c.

References FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, insert_filehash_entry(), isRelDataFile(), pg_fatal, pg_strdup(), file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, and generate_unaccent_rules::type.

Referenced by libpq_traverse_files(), local_traverse_files(), and main().

221 {
222  file_entry_t *entry;
223 
224  /*
225  * Pretend that pg_wal is a directory, even if it's really a symlink. We
226  * don't want to mess with the symlink itself, nor complain if it's a
227  * symlink in source but not in target or vice versa.
228  */
229  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
231 
232  /*
233  * sanity check: a filename that looks like a data file better be a
234  * regular file
235  */
236  if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
237  pg_fatal("data file \"%s\" in source is not a regular file", path);
238 
239  /* Remember this source file */
240  entry = insert_filehash_entry(path);
241  if (entry->source_exists)
242  pg_fatal("duplicate source file \"%s\"", path);
243  entry->source_exists = true;
244  entry->source_type = type;
245  entry->source_size = size;
246  entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
247 }
static file_entry_t * insert_filehash_entry(const char *path)
Definition: filemap.c:176
#define pg_fatal(...)
Definition: pg_rewind.h:37
char * source_link_target
Definition: filemap.h:76
file_type_t source_type
Definition: filemap.h:74
size_t source_size
Definition: filemap.h:75
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static bool isRelDataFile(const char *path)
Definition: filemap.c:510
Definition: filemap.h:49
bool source_exists
Definition: filemap.h:73

◆ process_target_file()

void process_target_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 255 of file filemap.c.

References FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK, insert_filehash_entry(), pg_fatal, pg_strdup(), file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_size, file_entry_t::target_type, and generate_unaccent_rules::type.

Referenced by main().

257 {
258  file_entry_t *entry;
259 
260  /*
261  * Do not apply any exclusion filters here. This has advantage to remove
262  * from the target data folder all paths which have been filtered out from
263  * the source data folder when processing the source files.
264  */
265 
266  /*
267  * Like in process_source_file, pretend that pg_wal is always a directory.
268  */
269  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
271 
272  /* Remember this target file */
273  entry = insert_filehash_entry(path);
274  if (entry->target_exists)
275  pg_fatal("duplicate source file \"%s\"", path);
276  entry->target_exists = true;
277  entry->target_type = type;
278  entry->target_size = size;
279  entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
280 }
static file_entry_t * insert_filehash_entry(const char *path)
Definition: filemap.c:176
#define pg_fatal(...)
Definition: pg_rewind.h:37
size_t target_size
Definition: filemap.h:61
file_type_t target_type
Definition: filemap.h:60
bool target_exists
Definition: filemap.h:59
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
char * target_link_target
Definition: filemap.h:62
Definition: filemap.h:49

◆ process_target_wal_block_change()

void process_target_wal_block_change ( ForkNumber  forknum,
RelFileNode  rnode,
BlockNumber  blkno 
)

Definition at line 292 of file filemap.c.

References Assert, datapagemap_add(), datasegpath(), FILE_TYPE_REGULAR, file_entry_t::isrelfile, lookup_filehash_entry(), file_entry_t::path, pfree(), pg_fatal, file_entry_t::source_exists, file_entry_t::target_exists, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_type.

Referenced by extractPageInfo().

294 {
295  char *path;
296  file_entry_t *entry;
297  BlockNumber blkno_inseg;
298  int segno;
299 
300  segno = blkno / RELSEG_SIZE;
301  blkno_inseg = blkno % RELSEG_SIZE;
302 
303  path = datasegpath(rnode, forknum, segno);
304  entry = lookup_filehash_entry(path);
305  pfree(path);
306 
307  /*
308  * If the block still exists in both systems, remember it. Otherwise we
309  * can safely ignore it.
310  *
311  * If the block is beyond the EOF in the source system, or the file
312  * doesn't exist in the source at all, we're going to truncate/remove it
313  * away from the target anyway. Likewise, if it doesn't exist in the
314  * target anymore, we will copy it over with the "tail" from the source
315  * system, anyway.
316  *
317  * It is possible to find WAL for a file that doesn't exist on either
318  * system anymore. It means that the relation was dropped later in the
319  * target system, and independently on the source system too, or that it
320  * was created and dropped in the target system and it never existed in
321  * the source. Either way, we can safely ignore it.
322  */
323  if (entry)
324  {
325  Assert(entry->isrelfile);
326 
327  if (entry->target_exists)
328  {
329  if (entry->target_type != FILE_TYPE_REGULAR)
330  pg_fatal("unexpected page modification for non-regular file \"%s\"",
331  entry->path);
332 
333  if (entry->source_exists)
334  {
335  off_t end_offset;
336 
337  end_offset = (blkno_inseg + 1) * BLCKSZ;
338  if (end_offset <= entry->source_size && end_offset <= entry->target_size)
339  datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
340  }
341  }
342  }
343 }
void datapagemap_add(datapagemap_t *map, BlockNumber blkno)
Definition: datapagemap.c:32
static file_entry_t * lookup_filehash_entry(const char *path)
Definition: filemap.c:206
#define pg_fatal(...)
Definition: pg_rewind.h:37
uint32 BlockNumber
Definition: block.h:31
static char * datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno)
Definition: filemap.c:593
bool isrelfile
Definition: filemap.h:54
void pfree(void *pointer)
Definition: mcxt.c:1169
file_type_t target_type
Definition: filemap.h:60
bool target_exists
Definition: filemap.h:59
const char * path
Definition: filemap.h:53
datapagemap_t target_pages_to_overwrite
Definition: filemap.h:68
#define Assert(condition)
Definition: c.h:804
Definition: filemap.h:49
bool source_exists
Definition: filemap.h:73