PostgreSQL Source Code  git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
filemap.c File Reference
#include "postgres_fe.h"
#include <sys/stat.h>
#include <unistd.h>
#include "catalog/pg_tablespace_d.h"
#include "common/file_utils.h"
#include "common/hashfn_unstable.h"
#include "common/string.h"
#include "datapagemap.h"
#include "filemap.h"
#include "pg_rewind.h"
#include "lib/simplehash.h"
Include dependency graph for filemap.c:

Go to the source code of this file.

Data Structures

struct  keepwal_entry
 
struct  exclude_list_item
 

Macros

#define SH_PREFIX   filehash
 
#define SH_ELEMENT_TYPE   file_entry_t
 
#define SH_KEY_TYPE   const char *
 
#define SH_KEY   path
 
#define SH_HASH_KEY(tb, key)   hash_string(key)
 
#define SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
 
#define SH_SCOPE   static inline
 
#define SH_RAW_ALLOCATOR   pg_malloc0
 
#define SH_DECLARE
 
#define SH_DEFINE
 
#define FILEHASH_INITIAL_SIZE   1000
 
#define SH_PREFIX   keepwal
 
#define SH_ELEMENT_TYPE   keepwal_entry
 
#define SH_KEY_TYPE   const char *
 
#define SH_KEY   path
 
#define SH_HASH_KEY(tb, key)   hash_string(key)
 
#define SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
 
#define SH_SCOPE   static inline
 
#define SH_RAW_ALLOCATOR   pg_malloc0
 
#define SH_DECLARE
 
#define SH_DEFINE
 
#define KEEPWAL_INITIAL_SIZE   1000
 

Typedefs

typedef struct keepwal_entry keepwal_entry
 

Functions

static bool isRelDataFile (const char *path)
 
static char * datasegpath (RelFileLocator rlocator, ForkNumber forknum, BlockNumber segno)
 
static file_entry_tinsert_filehash_entry (const char *path)
 
static file_entry_tlookup_filehash_entry (const char *path)
 
static bool keepwal_entry_exists (const char *path)
 
static int final_filemap_cmp (const void *a, const void *b)
 
static bool check_file_excluded (const char *path, bool is_source)
 
void filehash_init (void)
 
void keepwal_init (void)
 
void keepwal_add_entry (const char *path)
 
void process_source_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_file (const char *path, file_type_t type, size_t size, const char *link_target)
 
void process_target_wal_block_change (ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno)
 
static const char * action_to_str (file_action_t action)
 
void calculate_totals (filemap_t *filemap)
 
void print_filemap (filemap_t *filemap)
 
static file_action_t decide_file_action (file_entry_t *entry)
 
filemap_tdecide_file_actions (void)
 

Variables

static filehash_hash * filehash
 
static keepwal_hash * keepwal = NULL
 
static const char *const excludeDirContents []
 
static const struct exclude_list_item excludeFiles []
 

Macro Definition Documentation

◆ FILEHASH_INITIAL_SIZE

#define FILEHASH_INITIAL_SIZE   1000

Definition at line 53 of file filemap.c.

◆ KEEPWAL_INITIAL_SIZE

#define KEEPWAL_INITIAL_SIZE   1000

Definition at line 85 of file filemap.c.

◆ SH_DECLARE [1/2]

#define SH_DECLARE

Definition at line 81 of file filemap.c.

◆ SH_DECLARE [2/2]

#define SH_DECLARE

Definition at line 81 of file filemap.c.

◆ SH_DEFINE [1/2]

#define SH_DEFINE

Definition at line 82 of file filemap.c.

◆ SH_DEFINE [2/2]

#define SH_DEFINE

Definition at line 82 of file filemap.c.

◆ SH_ELEMENT_TYPE [1/2]

#define SH_ELEMENT_TYPE   file_entry_t

Definition at line 74 of file filemap.c.

◆ SH_ELEMENT_TYPE [2/2]

#define SH_ELEMENT_TYPE   keepwal_entry

Definition at line 74 of file filemap.c.

◆ SH_EQUAL [1/2]

#define SH_EQUAL (   tb,
  a,
  b 
)    (strcmp(a, b) == 0)

Definition at line 78 of file filemap.c.

◆ SH_EQUAL [2/2]

#define SH_EQUAL (   tb,
  a,
  b 
)    (strcmp(a, b) == 0)

Definition at line 78 of file filemap.c.

◆ SH_HASH_KEY [1/2]

#define SH_HASH_KEY (   tb,
  key 
)    hash_string(key)

Definition at line 77 of file filemap.c.

◆ SH_HASH_KEY [2/2]

#define SH_HASH_KEY (   tb,
  key 
)    hash_string(key)

Definition at line 77 of file filemap.c.

◆ SH_KEY [1/2]

#define SH_KEY   path

Definition at line 76 of file filemap.c.

◆ SH_KEY [2/2]

#define SH_KEY   path

Definition at line 76 of file filemap.c.

◆ SH_KEY_TYPE [1/2]

#define SH_KEY_TYPE   const char *

Definition at line 75 of file filemap.c.

◆ SH_KEY_TYPE [2/2]

#define SH_KEY_TYPE   const char *

Definition at line 75 of file filemap.c.

◆ SH_PREFIX [1/2]

#define SH_PREFIX   filehash

Definition at line 73 of file filemap.c.

◆ SH_PREFIX [2/2]

#define SH_PREFIX   keepwal

Definition at line 73 of file filemap.c.

◆ SH_RAW_ALLOCATOR [1/2]

#define SH_RAW_ALLOCATOR   pg_malloc0

Definition at line 80 of file filemap.c.

◆ SH_RAW_ALLOCATOR [2/2]

#define SH_RAW_ALLOCATOR   pg_malloc0

Definition at line 80 of file filemap.c.

◆ SH_SCOPE [1/2]

#define SH_SCOPE   static inline

Definition at line 79 of file filemap.c.

◆ SH_SCOPE [2/2]

#define SH_SCOPE   static inline

Definition at line 79 of file filemap.c.

Typedef Documentation

◆ keepwal_entry

typedef struct keepwal_entry keepwal_entry

Function Documentation

◆ action_to_str()

static const char* action_to_str ( file_action_t  action)
static

Definition at line 473 of file filemap.c.

474 {
475  switch (action)
476  {
477  case FILE_ACTION_NONE:
478  return "NONE";
479  case FILE_ACTION_COPY:
480  return "COPY";
482  return "TRUNCATE";
484  return "COPY_TAIL";
485  case FILE_ACTION_CREATE:
486  return "CREATE";
487  case FILE_ACTION_REMOVE:
488  return "REMOVE";
489 
490  default:
491  return "unknown";
492  }
493 }
@ FILE_ACTION_REMOVE
Definition: filemap.h:27
@ FILE_ACTION_COPY
Definition: filemap.h:21
@ FILE_ACTION_NONE
Definition: filemap.h:24
@ FILE_ACTION_COPY_TAIL
Definition: filemap.h:22
@ FILE_ACTION_TRUNCATE
Definition: filemap.h:26
@ FILE_ACTION_CREATE
Definition: filemap.h:20

References generate_unaccent_rules::action, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, and FILE_ACTION_TRUNCATE.

Referenced by print_filemap().

◆ calculate_totals()

void calculate_totals ( filemap_t filemap)

Definition at line 499 of file filemap.c.

500 {
501  file_entry_t *entry;
502  int i;
503 
504  filemap->total_size = 0;
505  filemap->fetch_size = 0;
506 
507  for (i = 0; i < filemap->nentries; i++)
508  {
509  entry = filemap->entries[i];
510 
511  if (entry->source_type != FILE_TYPE_REGULAR)
512  continue;
513 
514  filemap->total_size += entry->source_size;
515 
516  if (entry->action == FILE_ACTION_COPY)
517  {
518  filemap->fetch_size += entry->source_size;
519  continue;
520  }
521 
522  if (entry->action == FILE_ACTION_COPY_TAIL)
523  filemap->fetch_size += (entry->source_size - entry->target_size);
524 
525  if (entry->target_pages_to_overwrite.bitmapsize > 0)
526  {
528  BlockNumber blk;
529 
531  while (datapagemap_next(iter, &blk))
532  filemap->fetch_size += BLCKSZ;
533 
534  pg_free(iter);
535  }
536  }
537 }
uint32 BlockNumber
Definition: block.h:31
bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno)
Definition: datapagemap.c:87
datapagemap_iterator_t * datapagemap_iterate(datapagemap_t *map)
Definition: datapagemap.c:75
void pg_free(void *ptr)
Definition: fe_memutils.c:105
@ FILE_TYPE_REGULAR
Definition: filemap.h:34
int i
Definition: isn.c:72
int bitmapsize
Definition: datapagemap.h:17
Definition: filemap.h:50
datapagemap_t target_pages_to_overwrite
Definition: filemap.h:68
size_t source_size
Definition: filemap.h:75
file_type_t source_type
Definition: filemap.h:74
size_t target_size
Definition: filemap.h:61
file_action_t action
Definition: filemap.h:81
file_entry_t * entries[FLEXIBLE_ARRAY_MEMBER]
Definition: filemap.h:96
int nentries
Definition: filemap.h:95
uint64 total_size
Definition: filemap.h:92
uint64 fetch_size
Definition: filemap.h:93

References file_entry_t::action, datapagemap::bitmapsize, datapagemap_iterate(), datapagemap_next(), filemap_t::entries, filemap_t::fetch_size, FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_TYPE_REGULAR, i, filemap_t::nentries, pg_free(), file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and filemap_t::total_size.

Referenced by main().

◆ check_file_excluded()

static bool check_file_excluded ( const char *  path,
bool  is_source 
)
static

Definition at line 409 of file filemap.c.

410 {
411  char localpath[MAXPGPATH];
412  int excludeIdx;
413  const char *filename;
414 
415  /*
416  * Skip all temporary files, .../pgsql_tmp/... and .../pgsql_tmp.*
417  */
418  if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL ||
419  strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
420  {
421  return true;
422  }
423 
424  /* check individual files... */
425  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
426  {
427  int cmplen = strlen(excludeFiles[excludeIdx].name);
428 
430  if (filename == NULL)
431  filename = path;
432  else
433  filename++;
434 
435  if (!excludeFiles[excludeIdx].match_prefix)
436  cmplen++;
437  if (strncmp(filename, excludeFiles[excludeIdx].name, cmplen) == 0)
438  {
439  if (is_source)
440  pg_log_debug("entry \"%s\" excluded from source file list",
441  path);
442  else
443  pg_log_debug("entry \"%s\" excluded from target file list",
444  path);
445  return true;
446  }
447  }
448 
449  /*
450  * ... And check some directories. Note that this includes any contents
451  * within the directories themselves.
452  */
453  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
454  {
455  snprintf(localpath, sizeof(localpath), "%s/",
456  excludeDirContents[excludeIdx]);
457  if (strstr(path, localpath) == path)
458  {
459  if (is_source)
460  pg_log_debug("entry \"%s\" excluded from source file list",
461  path);
462  else
463  pg_log_debug("entry \"%s\" excluded from target file list",
464  path);
465  return true;
466  }
467  }
468 
469  return false;
470 }
#define PG_TEMP_FILES_DIR
Definition: file_utils.h:62
#define PG_TEMP_FILE_PREFIX
Definition: file_utils.h:63
static const struct exclude_list_item excludeFiles[]
Definition: filemap.c:157
static const char *const excludeDirContents[]
Definition: filemap.c:116
#define pg_log_debug(...)
Definition: logging.h:133
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:119
char * last_dir_separator(const char *filename)
Definition: path.c:140
#define snprintf
Definition: port.h:238
const char * name
Definition: basebackup.c:139
const char * name

References excludeDirContents, excludeFiles, filename, last_dir_separator(), exclude_list_item::match_prefix, MAXPGPATH, exclude_list_item::name, name, pg_log_debug, PG_TEMP_FILE_PREFIX, PG_TEMP_FILES_DIR, and snprintf.

Referenced by decide_file_action().

◆ datasegpath()

static char * datasegpath ( RelFileLocator  rlocator,
ForkNumber  forknum,
BlockNumber  segno 
)
static

Definition at line 653 of file filemap.c.

654 {
655  char *path;
656  char *segpath;
657 
658  path = relpathperm(rlocator, forknum);
659  if (segno > 0)
660  {
661  segpath = psprintf("%s.%u", path, segno);
662  pfree(path);
663  return segpath;
664  }
665  else
666  return path;
667 }
void pfree(void *pointer)
Definition: mcxt.c:1521
char * psprintf(const char *fmt,...)
Definition: psprintf.c:43
#define relpathperm(rlocator, forknum)
Definition: relpath.h:98

References pfree(), psprintf(), and relpathperm.

Referenced by isRelDataFile(), and process_target_wal_block_change().

◆ decide_file_action()

static file_action_t decide_file_action ( file_entry_t entry)
static

Definition at line 700 of file filemap.c.

701 {
702  const char *path = entry->path;
703 
704  /*
705  * Don't touch the control file. It is handled specially, after copying
706  * all the other files.
707  */
708  if (strcmp(path, "global/pg_control") == 0)
709  return FILE_ACTION_NONE;
710 
711  /* Skip macOS system files */
712  if (strstr(path, ".DS_Store") != NULL)
713  return FILE_ACTION_NONE;
714 
715  /*
716  * Remove all files matching the exclusion filters in the target.
717  */
718  if (check_file_excluded(path, true))
719  {
720  if (entry->target_exists)
721  return FILE_ACTION_REMOVE;
722  else
723  return FILE_ACTION_NONE;
724  }
725 
726  /*
727  * Handle cases where the file is missing from one of the systems.
728  */
729  if (!entry->target_exists && entry->source_exists)
730  {
731  /*
732  * File exists in source, but not in target. Copy it in toto. (If it's
733  * a relation data file, WAL replay after rewinding should re-create
734  * it anyway. But there's no harm in copying it now.)
735  */
736  switch (entry->source_type)
737  {
738  case FILE_TYPE_DIRECTORY:
739  case FILE_TYPE_SYMLINK:
740  return FILE_ACTION_CREATE;
741  case FILE_TYPE_REGULAR:
742  return FILE_ACTION_COPY;
743  case FILE_TYPE_UNDEFINED:
744  pg_fatal("unknown file type for \"%s\"", entry->path);
745  break;
746  }
747  }
748  else if (entry->target_exists && !entry->source_exists)
749  {
750  /*
751  * For files that exist in target but not in source, we check the
752  * keepwal hash table; any files listed therein must not be removed.
753  */
754  if (keepwal_entry_exists(path))
755  {
756  pg_log_debug("Not removing file \"%s\" because it is required for recovery", path);
757  return FILE_ACTION_NONE;
758  }
759  return FILE_ACTION_REMOVE;
760  }
761  else if (!entry->target_exists && !entry->source_exists)
762  {
763  /*
764  * Doesn't exist in either server. Why does it have an entry in the
765  * first place??
766  */
767  Assert(false);
768  return FILE_ACTION_NONE;
769  }
770 
771  /*
772  * Otherwise, the file exists on both systems
773  */
774  Assert(entry->target_exists && entry->source_exists);
775 
776  if (entry->source_type != entry->target_type)
777  {
778  /* But it's a different kind of object. Strange.. */
779  pg_fatal("file \"%s\" is of different type in source and target", entry->path);
780  }
781 
782  /*
783  * PG_VERSION files should be identical on both systems, but avoid
784  * overwriting them for paranoia.
785  */
786  if (pg_str_endswith(entry->path, "PG_VERSION"))
787  return FILE_ACTION_NONE;
788 
789  switch (entry->source_type)
790  {
791  case FILE_TYPE_DIRECTORY:
792  return FILE_ACTION_NONE;
793 
794  case FILE_TYPE_SYMLINK:
795 
796  /*
797  * XXX: Should we check if it points to the same target?
798  */
799  return FILE_ACTION_NONE;
800 
801  case FILE_TYPE_REGULAR:
802  if (!entry->isrelfile)
803  {
804  /*
805  * It's a non-data file that we have no special processing
806  * for. Copy it in toto.
807  */
808  return FILE_ACTION_COPY;
809  }
810  else
811  {
812  /*
813  * It's a data file that exists in both systems.
814  *
815  * If it's larger in target, we can truncate it. There will
816  * also be a WAL record of the truncation in the source
817  * system, so WAL replay would eventually truncate the target
818  * too, but we might as well do it now.
819  *
820  * If it's smaller in the target, it means that it has been
821  * truncated in the target, or enlarged in the source, or
822  * both. If it was truncated in the target, we need to copy
823  * the missing tail from the source system. If it was enlarged
824  * in the source system, there will be WAL records in the
825  * source system for the new blocks, so we wouldn't need to
826  * copy them here. But we don't know which scenario we're
827  * dealing with, and there's no harm in copying the missing
828  * blocks now, so do it now.
829  *
830  * If it's the same size, do nothing here. Any blocks modified
831  * in the target will be copied based on parsing the target
832  * system's WAL, and any blocks modified in the source will be
833  * updated after rewinding, when the source system's WAL is
834  * replayed.
835  */
836  if (entry->target_size < entry->source_size)
837  return FILE_ACTION_COPY_TAIL;
838  else if (entry->target_size > entry->source_size)
839  return FILE_ACTION_TRUNCATE;
840  else
841  return FILE_ACTION_NONE;
842  }
843  break;
844 
845  case FILE_TYPE_UNDEFINED:
846  pg_fatal("unknown file type for \"%s\"", path);
847  break;
848  }
849 
850  /* unreachable */
851  pg_fatal("could not decide what to do with file \"%s\"", path);
852 }
#define Assert(condition)
Definition: c.h:863
static bool keepwal_entry_exists(const char *path)
Definition: filemap.c:266
static bool check_file_excluded(const char *path, bool is_source)
Definition: filemap.c:409
@ FILE_TYPE_UNDEFINED
Definition: filemap.h:32
@ FILE_TYPE_SYMLINK
Definition: filemap.h:36
@ FILE_TYPE_DIRECTORY
Definition: filemap.h:35
#define pg_fatal(...)
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
const char * path
Definition: filemap.h:53
bool source_exists
Definition: filemap.h:73
bool target_exists
Definition: filemap.h:59
file_type_t target_type
Definition: filemap.h:60
bool isrelfile
Definition: filemap.h:54

References Assert, check_file_excluded(), FILE_ACTION_COPY, FILE_ACTION_COPY_TAIL, FILE_ACTION_CREATE, FILE_ACTION_NONE, FILE_ACTION_REMOVE, FILE_ACTION_TRUNCATE, FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, FILE_TYPE_UNDEFINED, file_entry_t::isrelfile, keepwal_entry_exists(), file_entry_t::path, pg_fatal, pg_log_debug, pg_str_endswith(), file_entry_t::source_exists, file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_exists, file_entry_t::target_size, and file_entry_t::target_type.

Referenced by decide_file_actions().

◆ decide_file_actions()

filemap_t* decide_file_actions ( void  )

Definition at line 861 of file filemap.c.

862 {
863  int i;
864  filehash_iterator it;
865  file_entry_t *entry;
866  filemap_t *filemap;
867 
868  filehash_start_iterate(filehash, &it);
869  while ((entry = filehash_iterate(filehash, &it)) != NULL)
870  {
871  entry->action = decide_file_action(entry);
872  }
873 
874  /*
875  * Turn the hash table into an array, and sort in the order that the
876  * actions should be performed.
877  */
878  filemap = pg_malloc(offsetof(filemap_t, entries) +
879  filehash->members * sizeof(file_entry_t *));
880  filemap->nentries = filehash->members;
881  filehash_start_iterate(filehash, &it);
882  i = 0;
883  while ((entry = filehash_iterate(filehash, &it)) != NULL)
884  {
885  filemap->entries[i++] = entry;
886  }
887 
888  qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *),
890 
891  return filemap;
892 }
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static filehash_hash * filehash
Definition: filemap.c:55
static file_action_t decide_file_action(file_entry_t *entry)
Definition: filemap.c:700
static int final_filemap_cmp(const void *a, const void *b)
Definition: filemap.c:680
#define qsort(a, b, c, d)
Definition: port.h:447

References file_entry_t::action, decide_file_action(), filemap_t::entries, filehash, final_filemap_cmp(), i, filemap_t::nentries, pg_malloc(), and qsort.

Referenced by main().

◆ filehash_init()

void filehash_init ( void  )

Definition at line 196 of file filemap.c.

197 {
198  filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL);
199 }
#define FILEHASH_INITIAL_SIZE
Definition: filemap.c:53

References filehash, and FILEHASH_INITIAL_SIZE.

Referenced by main().

◆ final_filemap_cmp()

static int final_filemap_cmp ( const void *  a,
const void *  b 
)
static

Definition at line 680 of file filemap.c.

681 {
682  file_entry_t *fa = *((file_entry_t **) a);
683  file_entry_t *fb = *((file_entry_t **) b);
684 
685  if (fa->action > fb->action)
686  return 1;
687  if (fa->action < fb->action)
688  return -1;
689 
690  if (fa->action == FILE_ACTION_REMOVE)
691  return strcmp(fb->path, fa->path);
692  else
693  return strcmp(fa->path, fb->path);
694 }
int b
Definition: isn.c:69
int a
Definition: isn.c:68
static int fa(void)
Definition: preproc-init.c:85
static int fb(int x)
Definition: preproc-init.c:92

References a, b, fa(), fb(), and FILE_ACTION_REMOVE.

Referenced by decide_file_actions().

◆ insert_filehash_entry()

static file_entry_t * insert_filehash_entry ( const char *  path)
static

Definition at line 203 of file filemap.c.

204 {
205  file_entry_t *entry;
206  bool found;
207 
208  entry = filehash_insert(filehash, path, &found);
209  if (!found)
210  {
211  entry->path = pg_strdup(path);
212  entry->isrelfile = isRelDataFile(path);
213 
214  entry->target_exists = false;
216  entry->target_size = 0;
217  entry->target_link_target = NULL;
218  entry->target_pages_to_overwrite.bitmap = NULL;
220 
221  entry->source_exists = false;
223  entry->source_size = 0;
224  entry->source_link_target = NULL;
225 
226  entry->action = FILE_ACTION_UNDECIDED;
227  }
228 
229  return entry;
230 }
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static bool isRelDataFile(const char *path)
Definition: filemap.c:570
@ FILE_ACTION_UNDECIDED
Definition: filemap.h:18
char * bitmap
Definition: datapagemap.h:16
char * source_link_target
Definition: filemap.h:76
char * target_link_target
Definition: filemap.h:62

References file_entry_t::action, datapagemap::bitmap, datapagemap::bitmapsize, FILE_ACTION_UNDECIDED, FILE_TYPE_UNDEFINED, filehash, isRelDataFile(), file_entry_t::isrelfile, file_entry_t::path, pg_strdup(), file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_pages_to_overwrite, file_entry_t::target_size, and file_entry_t::target_type.

Referenced by process_source_file(), and process_target_file().

◆ isRelDataFile()

static bool isRelDataFile ( const char *  path)
static

Definition at line 570 of file filemap.c.

571 {
572  RelFileLocator rlocator;
573  unsigned int segNo;
574  int nmatch;
575  bool matched;
576 
577  /*----
578  * Relation data files can be in one of the following directories:
579  *
580  * global/
581  * shared relations
582  *
583  * base/<db oid>/
584  * regular relations, default tablespace
585  *
586  * pg_tblspc/<tblspc oid>/<tblspc version>/
587  * within a non-default tablespace (the name of the directory
588  * depends on version)
589  *
590  * And the relation data files themselves have a filename like:
591  *
592  * <oid>.<segment number>
593  *
594  *----
595  */
596  rlocator.spcOid = InvalidOid;
597  rlocator.dbOid = InvalidOid;
598  rlocator.relNumber = InvalidRelFileNumber;
599  segNo = 0;
600  matched = false;
601 
602  nmatch = sscanf(path, "global/%u.%u", &rlocator.relNumber, &segNo);
603  if (nmatch == 1 || nmatch == 2)
604  {
605  rlocator.spcOid = GLOBALTABLESPACE_OID;
606  rlocator.dbOid = 0;
607  matched = true;
608  }
609  else
610  {
611  nmatch = sscanf(path, "base/%u/%u.%u",
612  &rlocator.dbOid, &rlocator.relNumber, &segNo);
613  if (nmatch == 2 || nmatch == 3)
614  {
615  rlocator.spcOid = DEFAULTTABLESPACE_OID;
616  matched = true;
617  }
618  else
619  {
620  nmatch = sscanf(path, "pg_tblspc/%u/" TABLESPACE_VERSION_DIRECTORY "/%u/%u.%u",
621  &rlocator.spcOid, &rlocator.dbOid, &rlocator.relNumber,
622  &segNo);
623  if (nmatch == 3 || nmatch == 4)
624  matched = true;
625  }
626  }
627 
628  /*
629  * The sscanf tests above can match files that have extra characters at
630  * the end. To eliminate such cases, cross-check that GetRelationPath
631  * creates the exact same filename, when passed the RelFileLocator
632  * information we extracted from the filename.
633  */
634  if (matched)
635  {
636  char *check_path = datasegpath(rlocator, MAIN_FORKNUM, segNo);
637 
638  if (strcmp(check_path, path) != 0)
639  matched = false;
640 
641  pfree(check_path);
642  }
643 
644  return matched;
645 }
static char * datasegpath(RelFileLocator rlocator, ForkNumber forknum, BlockNumber segno)
Definition: filemap.c:653
#define InvalidOid
Definition: postgres_ext.h:36
@ MAIN_FORKNUM
Definition: relpath.h:58
#define InvalidRelFileNumber
Definition: relpath.h:26
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:33
RelFileNumber relNumber

References datasegpath(), RelFileLocator::dbOid, InvalidOid, InvalidRelFileNumber, MAIN_FORKNUM, pfree(), RelFileLocator::relNumber, RelFileLocator::spcOid, and TABLESPACE_VERSION_DIRECTORY.

Referenced by insert_filehash_entry(), and process_source_file().

◆ keepwal_add_entry()

void keepwal_add_entry ( const char *  path)

Definition at line 250 of file filemap.c.

251 {
252  keepwal_entry *entry;
253  bool found;
254 
255  /* Should only be called with keepwal initialized */
256  Assert(keepwal != NULL);
257 
258  entry = keepwal_insert(keepwal, path, &found);
259 
260  if (!found)
261  entry->path = pg_strdup(path);
262 }
static keepwal_hash * keepwal
Definition: filemap.c:88
Definition: filemap.c:68
const char * path
Definition: filemap.c:69

References Assert, keepwal, keepwal_entry::path, and pg_strdup().

Referenced by findLastCheckpoint().

◆ keepwal_entry_exists()

static bool keepwal_entry_exists ( const char *  path)
static

Definition at line 266 of file filemap.c.

267 {
268  return keepwal_lookup(keepwal, path) != NULL;
269 }

References keepwal.

Referenced by decide_file_action().

◆ keepwal_init()

void keepwal_init ( void  )

Definition at line 242 of file filemap.c.

243 {
244  /* An initial hash size out of thin air */
245  keepwal = keepwal_create(KEEPWAL_INITIAL_SIZE, NULL);
246 }
#define KEEPWAL_INITIAL_SIZE
Definition: filemap.c:85

References keepwal, and KEEPWAL_INITIAL_SIZE.

Referenced by main().

◆ lookup_filehash_entry()

static file_entry_t * lookup_filehash_entry ( const char *  path)
static

Definition at line 233 of file filemap.c.

234 {
235  return filehash_lookup(filehash, path);
236 }

References filehash.

Referenced by process_target_wal_block_change().

◆ print_filemap()

void print_filemap ( filemap_t filemap)

Definition at line 540 of file filemap.c.

541 {
542  file_entry_t *entry;
543  int i;
544 
545  for (i = 0; i < filemap->nentries; i++)
546  {
547  entry = filemap->entries[i];
548  if (entry->action != FILE_ACTION_NONE ||
550  {
551  pg_log_debug("%s (%s)", entry->path,
552  action_to_str(entry->action));
553 
554  if (entry->target_pages_to_overwrite.bitmapsize > 0)
556  }
557  }
558  fflush(stdout);
559 }
void datapagemap_print(datapagemap_t *map)
Definition: datapagemap.c:117
static const char * action_to_str(file_action_t action)
Definition: filemap.c:473
static void const char fflush(stdout)

References file_entry_t::action, action_to_str(), datapagemap::bitmapsize, datapagemap_print(), filemap_t::entries, fflush(), FILE_ACTION_NONE, i, filemap_t::nentries, file_entry_t::path, pg_log_debug, generate_unaccent_rules::stdout, and file_entry_t::target_pages_to_overwrite.

Referenced by main().

◆ process_source_file()

void process_source_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 279 of file filemap.c.

281 {
282  file_entry_t *entry;
283 
284  /*
285  * Pretend that pg_wal is a directory, even if it's really a symlink. We
286  * don't want to mess with the symlink itself, nor complain if it's a
287  * symlink in source but not in target or vice versa.
288  */
289  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
291 
292  /*
293  * sanity check: a filename that looks like a data file better be a
294  * regular file
295  */
296  if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
297  pg_fatal("data file \"%s\" in source is not a regular file", path);
298 
299  /* Remember this source file */
300  entry = insert_filehash_entry(path);
301  if (entry->source_exists)
302  pg_fatal("duplicate source file \"%s\"", path);
303  entry->source_exists = true;
304  entry->source_type = type;
305  entry->source_size = size;
306  entry->source_link_target = link_target ? pg_strdup(link_target) : NULL;
307 }
static file_entry_t * insert_filehash_entry(const char *path)
Definition: filemap.c:203
static pg_noinline void Size size
Definition: slab.c:607
const char * type

References FILE_TYPE_DIRECTORY, FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, insert_filehash_entry(), isRelDataFile(), pg_fatal, pg_strdup(), size, file_entry_t::source_exists, file_entry_t::source_link_target, file_entry_t::source_size, file_entry_t::source_type, and type.

Referenced by main().

◆ process_target_file()

void process_target_file ( const char *  path,
file_type_t  type,
size_t  size,
const char *  link_target 
)

Definition at line 315 of file filemap.c.

317 {
318  file_entry_t *entry;
319 
320  /*
321  * Do not apply any exclusion filters here. This has advantage to remove
322  * from the target data folder all paths which have been filtered out from
323  * the source data folder when processing the source files.
324  */
325 
326  /*
327  * Like in process_source_file, pretend that pg_wal is always a directory.
328  */
329  if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK)
331 
332  /* Remember this target file */
333  entry = insert_filehash_entry(path);
334  if (entry->target_exists)
335  pg_fatal("duplicate source file \"%s\"", path);
336  entry->target_exists = true;
337  entry->target_type = type;
338  entry->target_size = size;
339  entry->target_link_target = link_target ? pg_strdup(link_target) : NULL;
340 }

References FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK, insert_filehash_entry(), pg_fatal, pg_strdup(), size, file_entry_t::target_exists, file_entry_t::target_link_target, file_entry_t::target_size, file_entry_t::target_type, and type.

Referenced by main().

◆ process_target_wal_block_change()

void process_target_wal_block_change ( ForkNumber  forknum,
RelFileLocator  rlocator,
BlockNumber  blkno 
)

Definition at line 352 of file filemap.c.

354 {
355  char *path;
356  file_entry_t *entry;
357  BlockNumber blkno_inseg;
358  int segno;
359 
360  segno = blkno / RELSEG_SIZE;
361  blkno_inseg = blkno % RELSEG_SIZE;
362 
363  path = datasegpath(rlocator, forknum, segno);
364  entry = lookup_filehash_entry(path);
365  pfree(path);
366 
367  /*
368  * If the block still exists in both systems, remember it. Otherwise we
369  * can safely ignore it.
370  *
371  * If the block is beyond the EOF in the source system, or the file
372  * doesn't exist in the source at all, we're going to truncate/remove it
373  * away from the target anyway. Likewise, if it doesn't exist in the
374  * target anymore, we will copy it over with the "tail" from the source
375  * system, anyway.
376  *
377  * It is possible to find WAL for a file that doesn't exist on either
378  * system anymore. It means that the relation was dropped later in the
379  * target system, and independently on the source system too, or that it
380  * was created and dropped in the target system and it never existed in
381  * the source. Either way, we can safely ignore it.
382  */
383  if (entry)
384  {
385  Assert(entry->isrelfile);
386 
387  if (entry->target_exists)
388  {
389  if (entry->target_type != FILE_TYPE_REGULAR)
390  pg_fatal("unexpected page modification for non-regular file \"%s\"",
391  entry->path);
392 
393  if (entry->source_exists)
394  {
395  off_t end_offset;
396 
397  end_offset = (blkno_inseg + 1) * BLCKSZ;
398  if (end_offset <= entry->source_size && end_offset <= entry->target_size)
399  datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg);
400  }
401  }
402  }
403 }
void datapagemap_add(datapagemap_t *map, BlockNumber blkno)
Definition: datapagemap.c:32
static file_entry_t * lookup_filehash_entry(const char *path)
Definition: filemap.c:233

References Assert, datapagemap_add(), datasegpath(), FILE_TYPE_REGULAR, file_entry_t::isrelfile, lookup_filehash_entry(), file_entry_t::path, pfree(), pg_fatal, file_entry_t::source_exists, file_entry_t::target_exists, file_entry_t::target_pages_to_overwrite, and file_entry_t::target_type.

Referenced by extractPageInfo().

Variable Documentation

◆ excludeDirContents

const char* const excludeDirContents[]
static
Initial value:
=
{
"pg_stat_tmp",
"pg_replslot",
"pg_dynshmem",
"pg_notify",
"pg_serial",
"pg_snapshots",
"pg_subtrans",
NULL
}

Definition at line 116 of file filemap.c.

Referenced by check_file_excluded().

◆ excludeFiles

const struct exclude_list_item excludeFiles[]
static
Initial value:
=
{
{"postgresql.auto.conf.tmp", false},
{"current_logfiles.tmp", false},
{"pg_internal.init", true},
{"backup_label", false},
{"tablespace_map", false},
{"backup_manifest", false},
{"postmaster.pid", false},
{"postmaster.opts", false},
{NULL, false}
}

Definition at line 116 of file filemap.c.

Referenced by check_file_excluded().

◆ filehash

filehash_hash* filehash
static

◆ keepwal

keepwal_hash* keepwal = NULL
static

Definition at line 88 of file filemap.c.

Referenced by keepwal_add_entry(), keepwal_entry_exists(), and keepwal_init().