PostgreSQL Source Code  git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
filemap.h File Reference
#include "datapagemap.h"
#include "storage/block.h"
#include "storage/relfilelocator.h"
Include dependency graph for filemap.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  file_entry_t
 
struct  filemap_t
 

Typedefs

typedef struct file_entry_t file_entry_t
 
typedef struct filemap_t filemap_t
 

Enumerations

enum  file_action_t {
  FILE_ACTION_UNDECIDED = 0 , FILE_ACTION_CREATE , FILE_ACTION_COPY , FILE_ACTION_COPY_TAIL ,
  FILE_ACTION_NONE , FILE_ACTION_TRUNCATE , FILE_ACTION_REMOVE
}
 
enum  file_type_t { FILE_TYPE_UNDEFINED = 0 , FILE_TYPE_REGULAR , FILE_TYPE_DIRECTORY , FILE_TYPE_SYMLINK }
 

Functions

void filehash_init (void)
 
void process_source_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_wal_block_change (ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)
 
filemap_tdecide_file_actions (void)
 
void calculate_totals (filemap_t *filemap)
 
void print_filemap (filemap_t *filemap)
 
void keepwal_init (void)
 
void keepwal_add_entry (const char *path)
 

Typedef Documentation

◆ file_entry_t

typedef struct file_entry_t file_entry_t

◆ filemap_t

typedef struct filemap_t filemap_t

Enumeration Type Documentation

◆ file_action_t

Enumerator
FILE_ACTION_UNDECIDED 
FILE_ACTION_CREATE 
FILE_ACTION_COPY 
FILE_ACTION_COPY_TAIL 
FILE_ACTION_NONE 
FILE_ACTION_TRUNCATE 
FILE_ACTION_REMOVE 

Definition at line 16 of file filemap.h.

17 {
18  FILE_ACTION_UNDECIDED = 0, /* not decided yet */
19 
20  FILE_ACTION_CREATE, /* create local directory or symbolic link */
21  FILE_ACTION_COPY, /* copy whole file, overwriting if exists */
22  FILE_ACTION_COPY_TAIL, /* copy tail from 'source_size' to
23  * 'target_size' */
24  FILE_ACTION_NONE, /* no action (we might still copy modified
25  * blocks based on the parsed WAL) */
26  FILE_ACTION_TRUNCATE, /* truncate local file to 'newsize' bytes */
27  FILE_ACTION_REMOVE, /* remove local file / directory / symlink */
file_action_t
Definition: filemap.h:17
@ FILE_ACTION_REMOVE
Definition: filemap.h:27
@ FILE_ACTION_COPY
Definition: filemap.h:21
@ FILE_ACTION_NONE
Definition: filemap.h:24
@ FILE_ACTION_COPY_TAIL
Definition: filemap.h:22
@ FILE_ACTION_UNDECIDED
Definition: filemap.h:18
@ FILE_ACTION_TRUNCATE
Definition: filemap.h:26
@ FILE_ACTION_CREATE
Definition: filemap.h:20

◆ file_type_t

Enumerator
FILE_TYPE_UNDEFINED 
FILE_TYPE_REGULAR 
FILE_TYPE_DIRECTORY 
FILE_TYPE_SYMLINK 

Definition at line 30 of file filemap.h.

31 {
33 
37 } file_type_t;
file_type_t
Definition: filemap.h:31
@ FILE_TYPE_UNDEFINED
Definition: filemap.h:32
@ FILE_TYPE_REGULAR
Definition: filemap.h:34
@ FILE_TYPE_SYMLINK
Definition: filemap.h:36
@ FILE_TYPE_DIRECTORY
Definition: filemap.h:35

Function Documentation

◆ calculate_totals()

void calculate_totals ( filemap_t filemap)

Definition at line 499 of file filemap.c.

500 {
501  file_entry_t *entry;
502  int i;
503 
504  filemap->total_size = 0;
505  filemap->fetch_size = 0;
506 
507  for (i = 0; i < filemap->nentries; i++)
508  {
509  entry = filemap->entries[i];
510 
511  if (entry->source_type != FILE_TYPE_REGULAR)
512  continue;
513 
514  filemap->total_size += entry->source_size;
515 
516  if (entry->action == FILE_ACTION_COPY)
517  {
518  filemap->fetch_size += entry->source_size;
519  continue;
520  }
521 
522  if (entry->action == FILE_ACTION_COPY_TAIL)
523  filemap->fetch_size += (entry->source_size - entry->target_size);
524 
525  if (entry->target_pages_to_overwrite.bitmapsize > 0)
526  {
528  BlockNumber blk;
529 
531  while (datapagemap_next(iter, &blk))
532  filemap->fetch_size += BLCKSZ;
533 
534  pg_free(iter);
535  }
536  }
537 }
uint32 BlockNumber
Definition: block.h:31
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
void pg_free(void *ptr)
Definition: fe_memutils.c:105
int i
Definition: isn.c:72
int bitmapsize
Definition: datapagemap.h:17
Definition: filemap.h:50
datapagemap_t target_pages_to_overwrite
Definition: filemap.h:68
size_t source_size
Definition: filemap.h:75
file_type_t source_type
Definition: filemap.h:74
size_t target_size
Definition: filemap.h:61
file_action_t action
Definition: filemap.h:81
file_entry_t * entries[FLEXIBLE_ARRAY_MEMBER]
Definition: filemap.h:96
int nentries
Definition: filemap.h:95
uint64 total_size
Definition: filemap.h:92
uint64 fetch_size
Definition: filemap.h:93

References file_entry_t::action, datapagemap::bitmapsize, datapagemap_iterate(), datapagemap_next(), filemap_t::entries, filemap_t::fetch_size, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_TYPE_REGULAR, i, filemap_t::nentries, pg_free(), file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and filemap_t::total_size.

Referenced by main().

◆ decide_file_actions()

filemap_t* decide_file_actions ( void  )

Definition at line 861 of file filemap.c.

862 {
863  int i;
864  filehash_iterator it;
865  file_entry_t *entry;
866  filemap_t *filemap;
867 
868  filehash_start_iterate(filehash, &it);
869  while ((entry = filehash_iterate(filehash, &it)) != NULL)
870  {
871  entry->action = decide_file_action(entry);
872  }
873 
874  /*
875  * Turn the hash table into an array, and sort in the order that the
876  * actions should be performed.
877  */
878  filemap = pg_malloc(offsetof(filemap_t, entries) +
879  filehash->members * sizeof(file_entry_t *));
880  filemap->nentries = filehash->members;
881  filehash_start_iterate(filehash, &it);
882  i = 0;
883  while ((entry = filehash_iterate(filehash, &it)) != NULL)
884  {
885  filemap->entries[i++] = entry;
886  }
887 
888  qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
890 
891  return filemap;
892 }
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static filehash_hash * filehash
Definition: filemap.c:55
static file_action_t decide_file_action(file_entry_t *entry)
Definition: filemap.c:700
static int final_filemap_cmp(const void *a, const void *b)
Definition: filemap.c:680
#define qsort(a, b, c, d)
Definition: port.h:447

References file_entry_t::action, decide_file_action(), filemap_t::entries, filehash, final_filemap_cmp(), i, filemap_t::nentries, pg_malloc(), and qsort.

Referenced by main().

◆ filehash_init()

void filehash_init ( void  )

Definition at line 196 of file filemap.c.

197 {
198  filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
199 }
#define FILEHASH_INITIAL_SIZE
Definition: filemap.c:53

References filehash, and FILEHASH_INITIAL_SIZE.

Referenced by main().

◆ keepwal_add_entry()

void keepwal_add_entry ( const char *  path)

Definition at line 250 of file filemap.c.

251 {
252  keepwal_entry *entry;
253  bool found;
254 
255  /* Should only be called with keepwal initialized */
256  Assert(keepwal != NULL);
257 
258  entry = keepwal_insert(keepwal, path, &found);
259 
260  if (!found)
261  entry->path = pg_strdup(path);
262 }
#define Assert(condition)
Definition: c.h:863
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static keepwal_hash * keepwal
Definition: filemap.c:88
Definition: filemap.c:68
const char * path
Definition: filemap.c:69

References Assert, keepwal, keepwal_entry::path, and pg_strdup().

Referenced by findLastCheckpoint().

◆ keepwal_init()

void keepwal_init ( void  )

Definition at line 242 of file filemap.c.

243 {
244  /* An initial hash size out of thin air */
245  keepwal = keepwal_create(KEEPWAL_INITIAL_SIZE, NULL);
246 }
#define KEEPWAL_INITIAL_SIZE
Definition: filemap.c:85

References keepwal, and KEEPWAL_INITIAL_SIZE.

Referenced by main().

◆ print_filemap()

void print_filemap ( filemap_t filemap)

Definition at line 540 of file filemap.c.

541 {
542  file_entry_t *entry;
543  int i;
544 
545  for (i = 0; i < filemap->nentries; i++)
546  {
547  entry = filemap->entries[i];
548  if (entry->action != FILE_ACTION_NONE ||
550  {
551  pg_log_debug("%s (%s)", entry->path,
552  action_to_str(entry->action));
553 
554  if (entry->target_pages_to_overwrite.bitmapsize > 0)
556  }
557  }
558  fflush(stdout);
559 }
void datapagemap_print(datapagemap_t *map)
Definition: datapagemap.c:117
static const char * action_to_str(file_action_t action)
Definition: filemap.c:473
static void const char fflush(stdout)
#define pg_log_debug(...)
Definition: logging.h:133
const char * path
Definition: filemap.h:53

References file_entry_t::action, action_to_str(), datapagemap::bitmapsize, datapagemap_print(), filemap_t::entries, fflush(), FILE_ACTION_NONE, i, filemap_t::nentries, file_entry_t::path, pg_log_debug, generate_unaccent_rules::stdout, and file_entry_t::target_pages_to_overwrite.

Referenced by main().

◆ process_source_file()

void process_source_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 279 of file filemap.c.

281 {
282  file_entry_t *entry;
283 
284  /*
285  * Pretend that pg_wal is a directory, even if it's really a symlink. We
286  * don't want to mess with the symlink itself, nor complain if it's a
287  * symlink in source but not in target or vice versa.
288  */
289  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
291 
292  /*
293  * sanity check: a filename that looks like a data file better be a
294  * regular file
295  */
296  if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
297  pg_fatal("data file \"%s\" in source is not a regular file", path);
298 
299  /* Remember this source file */
300  entry = insert_filehash_entry(path);
301  if (entry->source_exists)
302  pg_fatal("duplicate source file \"%s\"", path);
303  entry->source_exists = true;
304  entry->source_type = type;
305  entry->source_size = size;
306  entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
307 }
static bool isRelDataFile(const char *path)
Definition: filemap.c:570
static file_entry_t * insert_filehash_entry(const char *path)
Definition: filemap.c:203
#define pg_fatal(...)
static pg_noinline void Size size
Definition: slab.c:607
bool source_exists
Definition: filemap.h:73
char * source_link_target
Definition: filemap.h:76
const char * type

References FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, insert_filehash_entry(), isRelDataFile(), pg_fatal, pg_strdup(), size, file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, and type.

Referenced by main().

◆ process_target_file()

void process_target_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 315 of file filemap.c.

317 {
318  file_entry_t *entry;
319 
320  /*
321  * Do not apply any exclusion filters here. This has advantage to remove
322  * from the target data folder all paths which have been filtered out from
323  * the source data folder when processing the source files.
324  */
325 
326  /*
327  * Like in process_source_file, pretend that pg_wal is always a directory.
328  */
329  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
331 
332  /* Remember this target file */
333  entry = insert_filehash_entry(path);
334  if (entry->target_exists)
335  pg_fatal("duplicate source file \"%s\"", path);
336  entry->target_exists = true;
337  entry->target_type = type;
338  entry->target_size = size;
339  entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
340 }
bool target_exists
Definition: filemap.h:59
char * target_link_target
Definition: filemap.h:62
file_type_t target_type
Definition: filemap.h:60

References FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK, insert_filehash_entry(), pg_fatal, pg_strdup(), size, file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_size, file_entry_t::target_type, and type.

Referenced by main().

◆ process_target_wal_block_change()

void process_target_wal_block_change ( ForkNumber  forknum,
RelFileLocator  rlocator,
BlockNumber  blkno 
)

Definition at line 352 of file filemap.c.

354 {
355  char *path;
356  file_entry_t *entry;
357  BlockNumber blkno_inseg;
358  int segno;
359 
360  segno = blkno / RELSEG_SIZE;
361  blkno_inseg = blkno % RELSEG_SIZE;
362 
363  path = datasegpath(rlocator, forknum, segno);
364  entry = lookup_filehash_entry(path);
365  pfree(path);
366 
367  /*
368  * If the block still exists in both systems, remember it. Otherwise we
369  * can safely ignore it.
370  *
371  * If the block is beyond the EOF in the source system, or the file
372  * doesn't exist in the source at all, we're going to truncate/remove it
373  * away from the target anyway. Likewise, if it doesn't exist in the
374  * target anymore, we will copy it over with the "tail" from the source
375  * system, anyway.
376  *
377  * It is possible to find WAL for a file that doesn't exist on either
378  * system anymore. It means that the relation was dropped later in the
379  * target system, and independently on the source system too, or that it
380  * was created and dropped in the target system and it never existed in
381  * the source. Either way, we can safely ignore it.
382  */
383  if (entry)
384  {
385  Assert(entry->isrelfile);
386 
387  if (entry->target_exists)
388  {
389  if (entry->target_type != FILE_TYPE_REGULAR)
390  pg_fatal("unexpected page modification for non-regular file \"%s\"",
391  entry->path);
392 
393  if (entry->source_exists)
394  {
395  off_t end_offset;
396 
397  end_offset = (blkno_inseg + 1) * BLCKSZ;
398  if (end_offset <= entry->source_size && end_offset <= entry->target_size)
399  datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
400  }
401  }
402  }
403 }
void datapagemap_add(datapagemap_t *map, BlockNumber blkno)
Definition: datapagemap.c:32
static char * datasegpath(RelFileLocator rlocator, ForkNumber forknum, BlockNumber segno)
Definition: filemap.c:653
static file_entry_t * lookup_filehash_entry(const char *path)
Definition: filemap.c:233
void pfree(void *pointer)
Definition: mcxt.c:1521
bool isrelfile
Definition: filemap.h:54

References Assert, datapagemap_add(), datasegpath(), FILE_TYPE_REGULAR, file_entry_t::isrelfile, lookup_filehash_entry(), file_entry_t::path, pfree(), pg_fatal, file_entry_t::source_exists, file_entry_t::target_exists, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_type.

Referenced by extractPageInfo().