PostgreSQL Source Code  git master
pg_upgrade.h
Go to the documentation of this file.
1 /*
2  * pg_upgrade.h
3  *
4  * Copyright (c) 2010-2017, PostgreSQL Global Development Group
5  * src/bin/pg_upgrade/pg_upgrade.h
6  */
7 
8 #include <unistd.h>
9 #include <assert.h>
10 #include <sys/stat.h>
11 #include <sys/time.h>
12 
13 #include "libpq-fe.h"
14 
15 /* Use port in the private/dynamic port number range */
16 #define DEF_PGUPORT 50432
17 
18 /* Allocate for null byte */
19 #define USER_NAME_SIZE 128
20 
21 #define MAX_STRING 1024
22 #define LINE_ALLOC 4096
23 #define QUERY_ALLOC 8192
24 
25 #define MIGRATOR_API_VERSION 1
26 
27 #define MESSAGE_WIDTH 60
28 
29 #define GET_MAJOR_VERSION(v) ((v) / 100)
30 
31 /* contains both global db information and CREATE DATABASE commands */
32 #define GLOBALS_DUMP_FILE "pg_upgrade_dump_globals.sql"
33 #define DB_DUMP_FILE_MASK "pg_upgrade_dump_%u.custom"
34 
35 #define DB_DUMP_LOG_FILE_MASK "pg_upgrade_dump_%u.log"
36 #define SERVER_LOG_FILE "pg_upgrade_server.log"
37 #define UTILITY_LOG_FILE "pg_upgrade_utility.log"
38 #define INTERNAL_LOG_FILE "pg_upgrade_internal.log"
39 
40 extern char *output_files[];
41 
42 /*
43  * WIN32 files do not accept writes from multiple processes
44  *
45  * On Win32, we can't send both pg_upgrade output and command output to the
46  * same file because we get the error: "The process cannot access the file
47  * because it is being used by another process." so send the pg_ctl
48  * command-line output to a new file, rather than into the server log file.
49  * Ideally we could use UTILITY_LOG_FILE for this, but some Windows platforms
50  * keep the pg_ctl output file open by the running postmaster, even after
51  * pg_ctl exits.
52  *
53  * We could use the Windows pgwin32_open() flags to allow shared file
54  * writes but is unclear how all other tools would use those flags, so
55  * we just avoid it and log a little differently on Windows; we adjust
56  * the error message appropriately.
57  */
58 #ifndef WIN32
59 #define SERVER_START_LOG_FILE SERVER_LOG_FILE
60 #define SERVER_STOP_LOG_FILE SERVER_LOG_FILE
61 #else
62 #define SERVER_START_LOG_FILE "pg_upgrade_server_start.log"
63 /*
64  * "pg_ctl start" keeps SERVER_START_LOG_FILE and SERVER_LOG_FILE open
65  * while the server is running, so we use UTILITY_LOG_FILE for "pg_ctl
66  * stop".
67  */
68 #define SERVER_STOP_LOG_FILE UTILITY_LOG_FILE
69 #endif
70 
71 
72 #ifndef WIN32
73 #define pg_mv_file rename
74 #define pg_link_file link
75 #define PATH_SEPARATOR '/'
76 #define PATH_QUOTE '\''
77 #define RM_CMD "rm -f"
78 #define RMDIR_CMD "rm -rf"
79 #define SCRIPT_PREFIX "./"
80 #define SCRIPT_EXT "sh"
81 #define ECHO_QUOTE "'"
82 #define ECHO_BLANK ""
83 #else
84 #define pg_mv_file pgrename
85 #define pg_link_file win32_pghardlink
86 #define PATH_SEPARATOR '\\'
87 #define PATH_QUOTE '"'
88 #define RM_CMD "DEL /q"
89 #define RMDIR_CMD "RMDIR /s/q"
90 #define SCRIPT_PREFIX ""
91 #define SCRIPT_EXT "bat"
92 #define EXE_EXT ".exe"
93 #define ECHO_QUOTE ""
94 #define ECHO_BLANK "."
95 #endif
96 
97 
98 /*
99  * postmaster/postgres -b (binary_upgrade) flag added during PG 9.1
100  * development
101  */
102 #define BINARY_UPGRADE_SERVER_FLAG_CAT_VER 201104251
103 
104 /*
105  * Visibility map changed with this 9.2 commit,
106  * 8f9fe6edce358f7904e0db119416b4d1080a83aa; pick later catalog version.
107  */
108 #define VISIBILITY_MAP_CRASHSAFE_CAT_VER 201107031
109 
110 /*
111  * The format of visibility map is changed with this 9.6 commit,
112  */
113 #define VISIBILITY_MAP_FROZEN_BIT_CAT_VER 201603011
114 
115 /*
116  * pg_multixact format changed in 9.3 commit 0ac5ad5134f2769ccbaefec73844f85,
117  * ("Improve concurrency of foreign key locking") which also updated catalog
118  * version to this value. pg_upgrade behavior depends on whether old and new
119  * server versions are both newer than this, or only the new one is.
120  */
121 #define MULTIXACT_FORMATCHANGE_CAT_VER 201301231
122 
123 /*
124  * large object chunk size added to pg_controldata,
125  * commit 5f93c37805e7485488480916b4585e098d3cc883
126  */
127 #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942
128 
129 /*
130  * change in JSONB format during 9.4 beta
131  */
132 #define JSONB_FORMAT_CHANGE_CAT_VER 201409291
133 
134 
135 /*
136  * Each relation is represented by a relinfo structure.
137  */
138 typedef struct
139 {
140  /* Can't use NAMEDATALEN; not guaranteed to be same on client */
141  char *nspname; /* namespace name */
142  char *relname; /* relation name */
143  Oid reloid; /* relation OID */
144  Oid relfilenode; /* relation relfile node */
145  Oid indtable; /* if index, OID of its table, else 0 */
146  Oid toastheap; /* if toast table, OID of base table, else 0 */
147  char *tablespace; /* tablespace path; "" for cluster default */
148  bool nsp_alloc; /* should nspname be freed? */
149  bool tblsp_alloc; /* should tablespace be freed? */
150 } RelInfo;
151 
152 typedef struct
153 {
155  int nrels;
156 } RelInfoArr;
157 
158 /*
159  * The following structure represents a relation mapping.
160  */
161 typedef struct
162 {
163  const char *old_tablespace;
164  const char *new_tablespace;
169 
170  /*
171  * old/new relfilenodes might differ for pg_largeobject(_metadata) indexes
172  * due to VACUUM FULL or REINDEX. Other relfilenodes are preserved.
173  */
176  /* the rest are used only for logging and error reporting */
177  char *nspname; /* namespaces */
178  char *relname;
179 } FileNameMap;
180 
181 /*
182  * Structure to store database information
183  */
184 typedef struct
185 {
186  Oid db_oid; /* oid of the database */
187  char *db_name; /* database name */
188  char db_tablespace[MAXPGPATH]; /* database default tablespace
189  * path */
190  char *db_collate;
191  char *db_ctype;
193  RelInfoArr rel_arr; /* array of all user relinfos */
194 } DbInfo;
195 
196 typedef struct
197 {
198  DbInfo *dbs; /* array of db infos */
199  int ndbs; /* number of db infos */
200 } DbInfoArr;
201 
202 /*
203  * The following structure is used to hold pg_control information.
204  * Rather than using the backend's control structure we use our own
205  * structure to avoid pg_control version issues between releases.
206  */
207 typedef struct
208 {
211  char nextxlogfile[25];
230 } ControlData;
231 
232 /*
233  * Enumeration to denote link modes
234  */
235 typedef enum
236 {
239 } transferMode;
240 
241 /*
242  * Enumeration to denote pg_log modes
243  */
244 typedef enum
245 {
251 } eLogType;
252 
253 
254 typedef long pgpid_t;
255 
256 
257 /*
258  * cluster
259  *
260  * information about each cluster
261  */
262 typedef struct
263 {
264  ControlData controldata; /* pg_control information */
265  DbInfoArr dbarr; /* dbinfos array */
266  char *pgdata; /* pathname for cluster's $PGDATA directory */
267  char *pgconfig; /* pathname for cluster's config file
268  * directory */
269  char *bindir; /* pathname for cluster's executable directory */
270  char *pgopts; /* options to pass to the server, like pg_ctl
271  * -o */
272  char *sockdir; /* directory for Unix Domain socket, if any */
273  unsigned short port; /* port number where postmaster is waiting */
274  uint32 major_version; /* PG_VERSION of cluster */
275  char major_version_str[64]; /* string PG_VERSION of cluster */
276  uint32 bin_version; /* version returned from pg_ctl */
277  const char *tablespace_suffix; /* directory specification */
278 } ClusterInfo;
279 
280 
281 /*
282  * LogOpts
283 */
284 typedef struct
285 {
286  FILE *internal; /* internal log FILE */
287  bool verbose; /* true -> be verbose in messages */
288  bool retain; /* retain log files on success */
289 } LogOpts;
290 
291 
292 /*
293  * UserOpts
294 */
295 typedef struct
296 {
297  bool check; /* true -> ask user for permission to make
298  * changes */
299  transferMode transfer_mode; /* copy files or link them? */
300  int jobs;
301 } UserOpts;
302 
303 
304 /*
305  * OSInfo
306  */
307 typedef struct
308 {
309  const char *progname; /* complete pathname for this program */
310  char *exec_path; /* full path to my executable */
311  char *user; /* username for clusters */
312  bool user_specified; /* user specified on command-line */
313  char **old_tablespaces; /* tablespaces */
315  char **libraries; /* loadable libraries */
318 } OSInfo;
319 
320 
321 /*
322  * Global variables
323  */
324 extern LogOpts log_opts;
325 extern UserOpts user_opts;
326 extern ClusterInfo old_cluster,
327  new_cluster;
328 extern OSInfo os_info;
329 
330 
331 /* check.c */
332 
333 void output_check_banner(bool live_check);
334 void check_and_dump_old_cluster(bool live_check);
335 void check_new_cluster(void);
336 void report_clusters_compatible(void);
338 void output_completion_banner(char *analyze_script_file_name,
339  char *deletion_script_file_name);
340 void check_cluster_versions(void);
341 void check_cluster_compatibility(bool live_check);
342 void create_script_for_old_cluster_deletion(char **deletion_script_file_name);
343 void create_script_for_cluster_analyze(char **analyze_script_file_name);
344 
345 
346 /* controldata.c */
347 
348 void get_control_data(ClusterInfo *cluster, bool live_check);
349 void check_control_data(ControlData *oldctrl, ControlData *newctrl);
350 void disable_old_cluster(void);
351 
352 
353 /* dump.c */
354 
355 void generate_old_dump(void);
356 
357 
358 /* exec.c */
359 
360 #define EXEC_PSQL_ARGS "--echo-queries --set ON_ERROR_STOP=on --no-psqlrc --dbname=template1"
361 
362 bool exec_prog(const char *log_file, const char *opt_log_file,
363  bool throw_error, const char *fmt,...) pg_attribute_printf(4, 5);
364 void verify_directories(void);
365 bool pid_lock_file_exists(const char *datadir);
366 
367 
368 /* file.c */
369 
370 void copyFile(const char *src, const char *dst,
371  const char *schemaName, const char *relName);
372 void linkFile(const char *src, const char *dst,
373  const char *schemaName, const char *relName);
374 void rewriteVisibilityMap(const char *fromfile, const char *tofile,
375  const char *schemaName, const char *relName);
376 void check_hard_link(void);
377 FILE *fopen_priv(const char *path, const char *mode);
378 
379 /* function.c */
380 
381 void get_loadable_libraries(void);
382 void check_loadable_libraries(void);
383 
384 /* info.c */
385 
387  DbInfo *new_db, int *nmaps, const char *old_pgdata,
388  const char *new_pgdata);
390 void print_maps(FileNameMap *maps, int n,
391  const char *db_name);
392 
393 /* option.c */
394 
395 void parseCommandLine(int argc, char *argv[]);
396 void adjust_data_dir(ClusterInfo *cluster);
397 void get_sock_dir(ClusterInfo *cluster, bool live_check);
398 
399 /* relfilenode.c */
400 
401 void transfer_all_new_tablespaces(DbInfoArr *old_db_arr,
402  DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata);
403 void transfer_all_new_dbs(DbInfoArr *old_db_arr,
404  DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata,
405  char *old_tablespace);
406 
407 /* tablespace.c */
408 
409 void init_tablespaces(void);
410 
411 
412 /* server.c */
413 
414 PGconn *connectToServer(ClusterInfo *cluster, const char *db_name);
415 PGresult *executeQueryOrDie(PGconn *conn, const char *fmt,...) pg_attribute_printf(2, 3);
416 
417 char *cluster_conn_opts(ClusterInfo *cluster);
418 
419 bool start_postmaster(ClusterInfo *cluster, bool throw_error);
420 void stop_postmaster(bool fast);
422 void check_pghost_envvar(void);
423 
424 
425 /* util.c */
426 
427 char *quote_identifier(const char *s);
428 int get_user_info(char **user_name_p);
429 void check_ok(void);
430 void report_status(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3);
431 void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3);
432 void pg_fatal(const char *fmt,...) pg_attribute_printf(1, 2) pg_attribute_noreturn();
433 void end_progress_output(void);
434 void prep_status(const char *fmt,...) pg_attribute_printf(1, 2);
435 void check_ok(void);
436 unsigned int str2uint(const char *str);
437 void pg_putenv(const char *var, const char *val);
438 
439 
440 /* version.c */
441 
443  bool check_mode);
447  bool check_mode);
448 
449 /* parallel.c */
450 void parallel_exec_prog(const char *log_file, const char *opt_log_file,
451  const char *fmt,...) pg_attribute_printf(3, 4);
452 void parallel_transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr,
453  char *old_pgdata, char *new_pgdata,
454  char *old_tablespace);
455 bool reap_child(bool wait_for_child);
void copyFile(const char *src, const char *dst, const char *schemaName, const char *relName)
Definition: file.c:34
uint32 get_major_server_version(ClusterInfo *cluster)
Definition: server.c:155
bool date_is_int
Definition: pg_upgrade.h:227
void transfer_all_new_tablespaces(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata)
Definition: relfilenode.c:30
Oid old_relfilenode
Definition: pg_upgrade.h:174
void report_clusters_compatible(void)
Definition: check.c:162
uint32 major_version
Definition: pg_upgrade.h:274
bool void verify_directories(void)
Definition: exec.c:252
unsigned int str2uint(const char *str)
Definition: util.c:241
void transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata, char *old_tablespace)
Definition: relfilenode.c:81
ClusterInfo old_cluster
Definition: pg_upgrade.c:56
void check_new_cluster(void)
Definition: check.c:143
ControlData controldata
Definition: pg_upgrade.h:264
long pgpid_t
Definition: pg_upgrade.h:254
uint32 chkpnt_nxtoid
Definition: pg_upgrade.h:214
static char * log_file
Definition: pg_ctl.c:87
int num_libraries
Definition: pg_upgrade.h:316
int get_user_info(char **user_name_p)
Definition: util.c:212
uint32 walsz
Definition: pg_upgrade.h:221
void rewriteVisibilityMap(const char *fromfile, const char *tofile, const char *schemaName, const char *relName)
Definition: file.c:129
void check_cluster_compatibility(bool live_check)
Definition: check.c:275
PGresult * executeQueryOrDie(PGconn *conn, const char *fmt,...) pg_attribute_printf(2
bool exec_prog(const char *log_file, const char *opt_log_file, bool throw_error, const char *fmt,...) pg_attribute_printf(4
void end_progress_output(void)
Definition: util.c:44
char * relname
Definition: pg_upgrade.h:142
void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode)
Definition: version.c:297
unsigned short port
Definition: pg_upgrade.h:273
void void void pg_fatal(const char *fmt,...) pg_attribute_printf(1
bool tblsp_alloc
Definition: pg_upgrade.h:149
PGresult char * cluster_conn_opts(ClusterInfo *cluster)
Definition: server.c:88
uint32 index
Definition: pg_upgrade.h:224
LogOpts log_opts
Definition: util.c:18
Oid new_db_oid
Definition: pg_upgrade.h:168
Oid reloid
Definition: pg_upgrade.h:143
unsigned int Oid
Definition: postgres_ext.h:31
void generate_old_dump(void)
Definition: dump.c:18
int jobs
Definition: pg_upgrade.h:300
char * nspname
Definition: pg_upgrade.h:141
Oid db_oid
Definition: pg_upgrade.h:186
bool user_specified
Definition: pg_upgrade.h:312
#define pg_attribute_printf(f, a)
Definition: c.h:131
char * pgopts
Definition: pg_upgrade.h:270
void stop_postmaster(bool fast)
Definition: server.c:325
FILE * fopen_priv(const char *path, const char *mode)
Definition: file.c:321
uint32 chkpnt_nxtmxoff
Definition: pg_upgrade.h:216
void check_pghost_envvar(void)
Definition: server.c:352
void check_loadable_libraries(void)
Definition: function.c:203
char * db_ctype
Definition: pg_upgrade.h:191
uint32 blocksz
Definition: pg_upgrade.h:219
uint32 chkpnt_nxtxid
Definition: pg_upgrade.h:212
void get_control_data(ClusterInfo *cluster, bool live_check)
Definition: controldata.c:34
char * pgconfig
Definition: pg_upgrade.h:267
const char * old_tablespace
Definition: pg_upgrade.h:163
void print_maps(FileNameMap *maps, int n, const char *db_name)
Definition: info.c:285
Oid new_relfilenode
Definition: pg_upgrade.h:175
eLogType
Definition: logging.h:22
PGconn * conn
Definition: streamutil.c:46
uint32 chkpnt_nxtepoch
Definition: pg_upgrade.h:213
#define MAXPGPATH
RelInfo * rels
Definition: pg_upgrade.h:154
Oid relfilenode
Definition: pg_upgrade.h:144
bool reap_child(bool wait_for_child)
Definition: parallel.c:290
void disable_old_cluster(void)
Definition: controldata.c:609
char ** libraries
Definition: pg_upgrade.h:315
bool data_checksum_version
Definition: pg_upgrade.h:229
ClusterInfo new_cluster
Definition: pg_upgrade.c:56
void create_script_for_old_cluster_deletion(char **deletion_script_file_name)
Definition: check.c:517
void void void pg_attribute_noreturn()
void prep_status(const char *fmt,...) pg_attribute_printf(1
void new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster, bool check_mode)
Definition: version.c:25
uint32 ident
Definition: pg_upgrade.h:223
void check_control_data(ControlData *oldctrl, ControlData *newctrl)
Definition: controldata.c:552
char * datadir
char * output_files[]
Definition: pg_upgrade.c:60
unsigned int uint32
Definition: c.h:296
void cluster(ClusterStmt *stmt, bool isTopLevel)
Definition: cluster.c:106
uint32 align
Definition: pg_upgrade.h:218
void check_ok(void)
Definition: util.c:172
void output_completion_banner(char *analyze_script_file_name, char *deletion_script_file_name)
Definition: check.c:202
PGconn * connectToServer(ClusterInfo *cluster, const char *db_name)
Definition: server.c:27
const char * old_tablespace_suffix
Definition: pg_upgrade.h:165
transferMode transfer_mode
Definition: pg_upgrade.h:299
uint32 chkpnt_oldstMulti
Definition: pg_upgrade.h:217
uint32 walseg
Definition: pg_upgrade.h:222
Oid indtable
Definition: pg_upgrade.h:145
bool verbose
Definition: pg_upgrade.h:287
char * bindir
Definition: pg_upgrade.h:269
void linkFile(const char *src, const char *dst, const char *schemaName, const char *relName)
Definition: file.c:103
void void parallel_transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata, char *old_tablespace)
Definition: parallel.c:178
void check_cluster_versions(void)
Definition: check.c:233
uint32 cat_ver
Definition: pg_upgrade.h:210
char ** old_tablespaces
Definition: pg_upgrade.h:313
uint32 chkpnt_nxtmulti
Definition: pg_upgrade.h:215
transferMode
Definition: pg_upgrade.h:235
uint32 largesz
Definition: pg_upgrade.h:220
void get_sock_dir(ClusterInfo *cluster, bool live_check)
Definition: option.c:447
const char * tablespace_suffix
Definition: pg_upgrade.h:277
void check_hard_link(void)
Definition: file.c:282
uint32 bin_version
Definition: pg_upgrade.h:276
DbInfoArr dbarr
Definition: pg_upgrade.h:265
void void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2
void output_check_banner(bool live_check)
Definition: check.c:61
FileNameMap * gen_db_file_maps(DbInfo *old_db, DbInfo *new_db, int *nmaps, const char *old_pgdata, const char *new_pgdata)
Definition: info.c:41
Oid old_db_oid
Definition: pg_upgrade.h:167
const char * progname
Definition: pg_upgrade.h:309
uint32 toast
Definition: pg_upgrade.h:225
RelInfoArr rel_arr
Definition: pg_upgrade.h:193
bool pid_lock_file_exists(const char *datadir)
Definition: exec.c:221
OSInfo os_info
Definition: pg_upgrade.c:58
static const pg_conv_map maps[]
void get_loadable_libraries(void)
Definition: function.c:49
void parallel_exec_prog(const char *log_file, const char *opt_log_file, const char *fmt,...) pg_attribute_printf(3
uint32 large_object
Definition: pg_upgrade.h:226
bool check
Definition: pg_upgrade.h:297
char * quote_identifier(const char *s)
Definition: ruleutils.c:10412
char * nspname
Definition: pg_upgrade.h:177
int db_encoding
Definition: pg_upgrade.h:192
uint32 ctrl_ver
Definition: pg_upgrade.h:209
ClusterInfo * running_cluster
Definition: pg_upgrade.h:317
void get_db_and_rel_infos(ClusterInfo *cluster)
Definition: info.c:311
void adjust_data_dir(ClusterInfo *cluster)
Definition: option.c:384
void init_tablespaces(void)
Definition: tablespace.c:19
char * pgdata
Definition: pg_upgrade.h:266
char * sockdir
Definition: pg_upgrade.h:272
UserOpts user_opts
Definition: option.c:29
int num_old_tablespaces
Definition: pg_upgrade.h:314
void report_status(eLogType type, const char *fmt,...) pg_attribute_printf(2
char * tablespace
Definition: pg_upgrade.h:147
char * db_name
Definition: pg_upgrade.h:187
bool nsp_alloc
Definition: pg_upgrade.h:148
void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster)
Definition: version.c:111
void parseCommandLine(int argc, char *argv[])
Definition: option.c:38
char * db_collate
Definition: pg_upgrade.h:190
bool start_postmaster(ClusterInfo *cluster, bool throw_error)
Definition: server.c:194
void issue_warnings_and_set_wal_level(void)
Definition: check.c:179
void pg_putenv(const char *var, const char *val)
Definition: util.c:254
DbInfo * dbs
Definition: pg_upgrade.h:198
eLogType
Definition: pg_upgrade.h:244
bool retain
Definition: pg_upgrade.h:288
char * user
Definition: pg_upgrade.h:311
void create_script_for_cluster_analyze(char **analyze_script_file_name)
Definition: check.c:427
const char * new_tablespace
Definition: pg_upgrade.h:164
long val
Definition: informix.c:689
bool float8_pass_by_value
Definition: pg_upgrade.h:228
const char * new_tablespace_suffix
Definition: pg_upgrade.h:166
char * relname
Definition: pg_upgrade.h:178
Oid toastheap
Definition: pg_upgrade.h:146
void check_and_dump_old_cluster(bool live_check)
Definition: check.c:79
char * exec_path
Definition: pg_upgrade.h:310
void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster)
Definition: version.c:208