PostgreSQL Source Code  git master
reinit.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * reinit.c
4  * Reinitialization of unlogged relations
5  *
6  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  * IDENTIFICATION
10  * src/backend/storage/file/reinit.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 
15 #include "postgres.h"
16 
17 #include <unistd.h>
18 
19 #include "common/relpath.h"
20 #include "postmaster/startup.h"
21 #include "storage/copydir.h"
22 #include "storage/fd.h"
23 #include "storage/reinit.h"
24 #include "utils/hsearch.h"
25 #include "utils/memutils.h"
26 
27 static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
28  int op);
29 static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
30  int op);
31 
32 typedef struct
33 {
34  Oid reloid; /* hash key */
36 
37 /*
38  * Reset unlogged relations from before the last restart.
39  *
40  * If op includes UNLOGGED_RELATION_CLEANUP, we remove all forks of any
41  * relation with an "init" fork, except for the "init" fork itself.
42  *
43  * If op includes UNLOGGED_RELATION_INIT, we copy the "init" fork to the main
44  * fork.
45  */
46 void
48 {
49  char temp_path[MAXPGPATH + 10 + sizeof(TABLESPACE_VERSION_DIRECTORY)];
50  DIR *spc_dir;
51  struct dirent *spc_de;
52  MemoryContext tmpctx,
53  oldctx;
54 
55  /* Log it. */
56  elog(DEBUG1, "resetting unlogged relations: cleanup %d init %d",
57  (op & UNLOGGED_RELATION_CLEANUP) != 0,
58  (op & UNLOGGED_RELATION_INIT) != 0);
59 
60  /*
61  * Just to be sure we don't leak any memory, let's create a temporary
62  * memory context for this operation.
63  */
65  "ResetUnloggedRelations",
67  oldctx = MemoryContextSwitchTo(tmpctx);
68 
69  /* Prepare to report progress resetting unlogged relations. */
71 
72  /*
73  * First process unlogged files in pg_default ($PGDATA/base)
74  */
76 
77  /*
78  * Cycle through directories for all non-default tablespaces.
79  */
80  spc_dir = AllocateDir("pg_tblspc");
81 
82  while ((spc_de = ReadDir(spc_dir, "pg_tblspc")) != NULL)
83  {
84  if (strcmp(spc_de->d_name, ".") == 0 ||
85  strcmp(spc_de->d_name, "..") == 0)
86  continue;
87 
88  snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s",
91  }
92 
93  FreeDir(spc_dir);
94 
95  /*
96  * Restore memory context.
97  */
98  MemoryContextSwitchTo(oldctx);
99  MemoryContextDelete(tmpctx);
100 }
101 
102 /*
103  * Process one tablespace directory for ResetUnloggedRelations
104  */
105 static void
106 ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
107 {
108  DIR *ts_dir;
109  struct dirent *de;
110  char dbspace_path[MAXPGPATH * 2];
111 
112  ts_dir = AllocateDir(tsdirname);
113 
114  /*
115  * If we get ENOENT on a tablespace directory, log it and return. This
116  * can happen if a previous DROP TABLESPACE crashed between removing the
117  * tablespace directory and removing the symlink in pg_tblspc. We don't
118  * really want to prevent database startup in that scenario, so let it
119  * pass instead. Any other type of error will be reported by ReadDir
120  * (causing a startup failure).
121  */
122  if (ts_dir == NULL && errno == ENOENT)
123  {
124  ereport(LOG,
126  errmsg("could not open directory \"%s\": %m",
127  tsdirname)));
128  return;
129  }
130 
131  while ((de = ReadDir(ts_dir, tsdirname)) != NULL)
132  {
133  /*
134  * We're only interested in the per-database directories, which have
135  * numeric names. Note that this code will also (properly) ignore "."
136  * and "..".
137  */
138  if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
139  continue;
140 
141  snprintf(dbspace_path, sizeof(dbspace_path), "%s/%s",
142  tsdirname, de->d_name);
143 
144  if (op & UNLOGGED_RELATION_INIT)
145  ereport_startup_progress("resetting unlogged relations (init), elapsed time: %ld.%02d s, current path: %s",
146  dbspace_path);
147  else if (op & UNLOGGED_RELATION_CLEANUP)
148  ereport_startup_progress("resetting unlogged relations (cleanup), elapsed time: %ld.%02d s, current path: %s",
149  dbspace_path);
150 
151  ResetUnloggedRelationsInDbspaceDir(dbspace_path, op);
152  }
153 
154  FreeDir(ts_dir);
155 }
156 
157 /*
158  * Process one per-dbspace directory for ResetUnloggedRelations
159  */
160 static void
161 ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
162 {
163  DIR *dbspace_dir;
164  struct dirent *de;
165  char rm_path[MAXPGPATH * 2];
166 
167  /* Caller must specify at least one operation. */
169 
170  /*
171  * Cleanup is a two-pass operation. First, we go through and identify all
172  * the files with init forks. Then, we go through again and nuke
173  * everything with the same OID except the init fork.
174  */
175  if ((op & UNLOGGED_RELATION_CLEANUP) != 0)
176  {
177  HTAB *hash;
178  HASHCTL ctl;
179 
180  /*
181  * It's possible that someone could create a ton of unlogged relations
182  * in the same database & tablespace, so we'd better use a hash table
183  * rather than an array or linked list to keep track of which files
184  * need to be reset. Otherwise, this cleanup operation would be
185  * O(n^2).
186  */
187  ctl.keysize = sizeof(Oid);
188  ctl.entrysize = sizeof(unlogged_relation_entry);
190  hash = hash_create("unlogged relation OIDs", 32, &ctl,
192 
193  /* Scan the directory. */
194  dbspace_dir = AllocateDir(dbspacedirname);
195  while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
196  {
197  ForkNumber forkNum;
198  int relnumchars;
200 
201  /* Skip anything that doesn't look like a relation data file. */
202  if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
203  &forkNum))
204  continue;
205 
206  /* Also skip it unless this is the init fork. */
207  if (forkNum != INIT_FORKNUM)
208  continue;
209 
210  /*
211  * Put the OID portion of the name into the hash table, if it
212  * isn't already.
213  */
214  ent.reloid = atooid(de->d_name);
215  (void) hash_search(hash, &ent, HASH_ENTER, NULL);
216  }
217 
218  /* Done with the first pass. */
219  FreeDir(dbspace_dir);
220 
221  /*
222  * If we didn't find any init forks, there's no point in continuing;
223  * we can bail out now.
224  */
225  if (hash_get_num_entries(hash) == 0)
226  {
228  return;
229  }
230 
231  /*
232  * Now, make a second pass and remove anything that matches.
233  */
234  dbspace_dir = AllocateDir(dbspacedirname);
235  while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
236  {
237  ForkNumber forkNum;
238  int relnumchars;
240 
241  /* Skip anything that doesn't look like a relation data file. */
242  if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
243  &forkNum))
244  continue;
245 
246  /* We never remove the init fork. */
247  if (forkNum == INIT_FORKNUM)
248  continue;
249 
250  /*
251  * See whether the OID portion of the name shows up in the hash
252  * table. If so, nuke it!
253  */
254  ent.reloid = atooid(de->d_name);
255  if (hash_search(hash, &ent, HASH_FIND, NULL))
256  {
257  snprintf(rm_path, sizeof(rm_path), "%s/%s",
258  dbspacedirname, de->d_name);
259  if (unlink(rm_path) < 0)
260  ereport(ERROR,
262  errmsg("could not remove file \"%s\": %m",
263  rm_path)));
264  else
265  elog(DEBUG2, "unlinked file \"%s\"", rm_path);
266  }
267  }
268 
269  /* Cleanup is complete. */
270  FreeDir(dbspace_dir);
272  }
273 
274  /*
275  * Initialization happens after cleanup is complete: we copy each init
276  * fork file to the corresponding main fork file. Note that if we are
277  * asked to do both cleanup and init, we may never get here: if the
278  * cleanup code determines that there are no init forks in this dbspace,
279  * it will return before we get to this point.
280  */
281  if ((op & UNLOGGED_RELATION_INIT) != 0)
282  {
283  /* Scan the directory. */
284  dbspace_dir = AllocateDir(dbspacedirname);
285  while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
286  {
287  ForkNumber forkNum;
288  int relnumchars;
289  char relnumbuf[OIDCHARS + 1];
290  char srcpath[MAXPGPATH * 2];
291  char dstpath[MAXPGPATH];
292 
293  /* Skip anything that doesn't look like a relation data file. */
294  if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
295  &forkNum))
296  continue;
297 
298  /* Also skip it unless this is the init fork. */
299  if (forkNum != INIT_FORKNUM)
300  continue;
301 
302  /* Construct source pathname. */
303  snprintf(srcpath, sizeof(srcpath), "%s/%s",
304  dbspacedirname, de->d_name);
305 
306  /* Construct destination pathname. */
307  memcpy(relnumbuf, de->d_name, relnumchars);
308  relnumbuf[relnumchars] = '\0';
309  snprintf(dstpath, sizeof(dstpath), "%s/%s%s",
310  dbspacedirname, relnumbuf, de->d_name + relnumchars + 1 +
311  strlen(forkNames[INIT_FORKNUM]));
312 
313  /* OK, we're ready to perform the actual copy. */
314  elog(DEBUG2, "copying %s to %s", srcpath, dstpath);
315  copy_file(srcpath, dstpath);
316  }
317 
318  FreeDir(dbspace_dir);
319 
320  /*
321  * copy_file() above has already called pg_flush_data() on the files
322  * it created. Now we need to fsync those files, because a checkpoint
323  * won't do it for us while we're in recovery. We do this in a
324  * separate pass to allow the kernel to perform all the flushes
325  * (especially the metadata ones) at once.
326  */
327  dbspace_dir = AllocateDir(dbspacedirname);
328  while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL)
329  {
330  ForkNumber forkNum;
331  int relnumchars;
332  char relnumbuf[OIDCHARS + 1];
333  char mainpath[MAXPGPATH];
334 
335  /* Skip anything that doesn't look like a relation data file. */
336  if (!parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
337  &forkNum))
338  continue;
339 
340  /* Also skip it unless this is the init fork. */
341  if (forkNum != INIT_FORKNUM)
342  continue;
343 
344  /* Construct main fork pathname. */
345  memcpy(relnumbuf, de->d_name, relnumchars);
346  relnumbuf[relnumchars] = '\0';
347  snprintf(mainpath, sizeof(mainpath), "%s/%s%s",
348  dbspacedirname, relnumbuf, de->d_name + relnumchars + 1 +
349  strlen(forkNames[INIT_FORKNUM]));
350 
351  fsync_fname(mainpath, false);
352  }
353 
354  FreeDir(dbspace_dir);
355 
356  /*
357  * Lastly, fsync the database directory itself, ensuring the
358  * filesystem remembers the file creations and deletions we've done.
359  * We don't bother with this during a call that does only
360  * UNLOGGED_RELATION_CLEANUP, because if recovery crashes before we
361  * get to doing UNLOGGED_RELATION_INIT, we'll redo the cleanup step
362  * too at the next startup attempt.
363  */
364  fsync_fname(dbspacedirname, true);
365  }
366 }
367 
368 /*
369  * Basic parsing of putative relation filenames.
370  *
371  * This function returns true if the file appears to be in the correct format
372  * for a non-temporary relation and false otherwise.
373  *
374  * NB: If this function returns true, the caller is entitled to assume that
375  * *relnumchars has been set to a value no more than OIDCHARS, and thus
376  * that a buffer of OIDCHARS+1 characters is sufficient to hold the
377  * RelFileNumber portion of the filename. This is critical to protect against
378  * a possible buffer overrun.
379  */
380 bool
381 parse_filename_for_nontemp_relation(const char *name, int *relnumchars,
382  ForkNumber *fork)
383 {
384  int pos;
385 
386  /* Look for a non-empty string of digits (that isn't too long). */
387  for (pos = 0; isdigit((unsigned char) name[pos]); ++pos)
388  ;
389  if (pos == 0 || pos > OIDCHARS)
390  return false;
391  *relnumchars = pos;
392 
393  /* Check for a fork name. */
394  if (name[pos] != '_')
395  *fork = MAIN_FORKNUM;
396  else
397  {
398  int forkchar;
399 
400  forkchar = forkname_chars(&name[pos + 1], fork);
401  if (forkchar <= 0)
402  return false;
403  pos += forkchar + 1;
404  }
405 
406  /* Check for a segment number. */
407  if (name[pos] == '.')
408  {
409  int segchar;
410 
411  for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar)
412  ;
413  if (segchar <= 1)
414  return false;
415  pos += segchar;
416  }
417 
418  /* Now we should be at the end. */
419  if (name[pos] != '\0')
420  return false;
421  return true;
422 }
void begin_startup_progress_phase(void)
Definition: startup.c:321
void copy_file(char *fromfile, char *tofile)
Definition: copydir.c:117
void hash_destroy(HTAB *hashp)
Definition: dynahash.c:863
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:953
HTAB * hash_create(const char *tabname, long nelem, const HASHCTL *info, int flags)
Definition: dynahash.c:350
long hash_get_num_entries(HTAB *hashp)
Definition: dynahash.c:1377
int errcode_for_file_access(void)
Definition: elog.c:718
int errmsg(const char *fmt,...)
Definition: elog.c:906
#define LOG
Definition: elog.h:27
#define DEBUG2
Definition: elog.h:25
#define DEBUG1
Definition: elog.h:26
#define ERROR
Definition: elog.h:35
#define ereport(elevel,...)
Definition: elog.h:145
const char * name
Definition: encode.c:561
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2709
int FreeDir(DIR *dir)
Definition: fd.c:2761
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:662
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2643
static char dstpath[MAXPGPATH]
Definition: file_ops.c:32
@ HASH_FIND
Definition: hsearch.h:113
@ HASH_ENTER
Definition: hsearch.h:114
#define HASH_CONTEXT
Definition: hsearch.h:102
#define HASH_ELEM
Definition: hsearch.h:95
#define HASH_BLOBS
Definition: hsearch.h:97
Assert(fmt[strlen(fmt) - 1] !='\n')
MemoryContext CurrentMemoryContext
Definition: mcxt.c:124
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:376
#define AllocSetContextCreate
Definition: memutils.h:129
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:153
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:135
#define MAXPGPATH
#define snprintf
Definition: port.h:238
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
static unsigned hash(unsigned *uv, int n)
Definition: rege_dfa.c:715
static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
Definition: reinit.c:106
void ResetUnloggedRelations(int op)
Definition: reinit.c:47
bool parse_filename_for_nontemp_relation(const char *name, int *relnumchars, ForkNumber *fork)
Definition: reinit.c:381
static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
Definition: reinit.c:161
#define UNLOGGED_RELATION_INIT
Definition: reinit.h:27
#define UNLOGGED_RELATION_CLEANUP
Definition: reinit.h:26
int forkname_chars(const char *str, ForkNumber *fork)
Definition: relpath.c:81
const char *const forkNames[]
Definition: relpath.c:33
ForkNumber
Definition: relpath.h:48
@ MAIN_FORKNUM
Definition: relpath.h:50
@ INIT_FORKNUM
Definition: relpath.h:53
#define OIDCHARS
Definition: relpath.h:37
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:33
#define ereport_startup_progress(msg,...)
Definition: startup.h:18
Definition: dirent.c:26
Size keysize
Definition: hsearch.h:75
Size entrysize
Definition: hsearch.h:76
MemoryContext hcxt
Definition: hsearch.h:86
Definition: dynahash.c:220
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
Definition: reinit.c:33
Oid reloid
Definition: reinit.c:34