PostgreSQL Source Code  git master
sharedfileset.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * sharedfileset.c
4  * Shared temporary file management.
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  * IDENTIFICATION
10  * src/backend/storage/file/sharedfileset.c
11  *
12  * SharedFileSets provide a temporary namespace (think directory) so that
13  * files can be discovered by name, and a shared ownership semantics so that
14  * shared files survive until the last user detaches.
15  *
16  * SharedFileSets can be used by backends when the temporary files need to be
17  * opened/closed multiple times and the underlying files need to survive across
18  * transactions.
19  *
20  *-------------------------------------------------------------------------
21  */
22 
23 #include "postgres.h"
24 
25 #include <limits.h>
26 
27 #include "catalog/pg_tablespace.h"
28 #include "commands/tablespace.h"
29 #include "common/hashfn.h"
30 #include "miscadmin.h"
31 #include "storage/dsm.h"
32 #include "storage/ipc.h"
33 #include "storage/sharedfileset.h"
34 #include "utils/builtins.h"
35 
36 static List *filesetlist = NIL;
37 
38 static void SharedFileSetOnDetach(dsm_segment *segment, Datum datum);
40 static void SharedFileSetPath(char *path, SharedFileSet *fileset, Oid tablespace);
41 static void SharedFilePath(char *path, SharedFileSet *fileset, const char *name);
42 static Oid ChooseTablespace(const SharedFileSet *fileset, const char *name);
43 
44 /*
45  * Initialize a space for temporary files that can be opened by other backends.
46  * Other backends must attach to it before accessing it. Associate this
47  * SharedFileSet with 'seg'. Any contained files will be deleted when the
48  * last backend detaches.
49  *
50  * We can also use this interface if the temporary files are used only by
51  * single backend but the files need to be opened and closed multiple times
52  * and also the underlying files need to survive across transactions. For
53  * such cases, dsm segment 'seg' should be passed as NULL. Callers are
54  * expected to explicitly remove such files by using SharedFileSetDelete/
55  * SharedFileSetDeleteAll or we remove such files on proc exit.
56  *
57  * Files will be distributed over the tablespaces configured in
58  * temp_tablespaces.
59  *
60  * Under the covers the set is one or more directories which will eventually
61  * be deleted.
62  */
63 void
65 {
66  static uint32 counter = 0;
67 
68  SpinLockInit(&fileset->mutex);
69  fileset->refcnt = 1;
70  fileset->creator_pid = MyProcPid;
71  fileset->number = counter;
72  counter = (counter + 1) % INT_MAX;
73 
74  /* Capture the tablespace OIDs so that all backends agree on them. */
76  fileset->ntablespaces =
77  GetTempTablespaces(&fileset->tablespaces[0],
78  lengthof(fileset->tablespaces));
79  if (fileset->ntablespaces == 0)
80  {
81  /* If the GUC is empty, use current database's default tablespace */
82  fileset->tablespaces[0] = MyDatabaseTableSpace;
83  fileset->ntablespaces = 1;
84  }
85  else
86  {
87  int i;
88 
89  /*
90  * An entry of InvalidOid means use the default tablespace for the
91  * current database. Replace that now, to be sure that all users of
92  * the SharedFileSet agree on what to do.
93  */
94  for (i = 0; i < fileset->ntablespaces; i++)
95  {
96  if (fileset->tablespaces[i] == InvalidOid)
98  }
99  }
100 
101  /* Register our cleanup callback. */
102  if (seg)
104  else
105  {
106  static bool registered_cleanup = false;
107 
108  if (!registered_cleanup)
109  {
110  /*
111  * We must not have registered any fileset before registering the
112  * fileset clean up.
113  */
114  Assert(filesetlist == NIL);
116  registered_cleanup = true;
117  }
118 
119  filesetlist = lcons((void *) fileset, filesetlist);
120  }
121 }
122 
123 /*
124  * Attach to a set of directories that was created with SharedFileSetInit.
125  */
126 void
128 {
129  bool success;
130 
131  SpinLockAcquire(&fileset->mutex);
132  if (fileset->refcnt == 0)
133  success = false;
134  else
135  {
136  ++fileset->refcnt;
137  success = true;
138  }
139  SpinLockRelease(&fileset->mutex);
140 
141  if (!success)
142  ereport(ERROR,
143  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
144  errmsg("could not attach to a SharedFileSet that is already destroyed")));
145 
146  /* Register our cleanup callback. */
148 }
149 
150 /*
151  * Create a new file in the given set.
152  */
153 File
154 SharedFileSetCreate(SharedFileSet *fileset, const char *name)
155 {
156  char path[MAXPGPATH];
157  File file;
158 
159  SharedFilePath(path, fileset, name);
160  file = PathNameCreateTemporaryFile(path, false);
161 
162  /* If we failed, see if we need to create the directory on demand. */
163  if (file <= 0)
164  {
165  char tempdirpath[MAXPGPATH];
166  char filesetpath[MAXPGPATH];
167  Oid tablespace = ChooseTablespace(fileset, name);
168 
169  TempTablespacePath(tempdirpath, tablespace);
170  SharedFileSetPath(filesetpath, fileset, tablespace);
171  PathNameCreateTemporaryDir(tempdirpath, filesetpath);
172  file = PathNameCreateTemporaryFile(path, true);
173  }
174 
175  return file;
176 }
177 
178 /*
179  * Open a file that was created with SharedFileSetCreate(), possibly in
180  * another backend.
181  */
182 File
183 SharedFileSetOpen(SharedFileSet *fileset, const char *name, int mode)
184 {
185  char path[MAXPGPATH];
186  File file;
187 
188  SharedFilePath(path, fileset, name);
189  file = PathNameOpenTemporaryFile(path, mode);
190 
191  return file;
192 }
193 
194 /*
195  * Delete a file that was created with SharedFileSetCreate().
196  * Return true if the file existed, false if didn't.
197  */
198 bool
199 SharedFileSetDelete(SharedFileSet *fileset, const char *name,
200  bool error_on_failure)
201 {
202  char path[MAXPGPATH];
203 
204  SharedFilePath(path, fileset, name);
205 
206  return PathNameDeleteTemporaryFile(path, error_on_failure);
207 }
208 
209 /*
210  * Delete all files in the set.
211  */
212 void
214 {
215  char dirpath[MAXPGPATH];
216  int i;
217 
218  /*
219  * Delete the directory we created in each tablespace. Doesn't fail
220  * because we use this in error cleanup paths, but can generate LOG
221  * message on IO error.
222  */
223  for (i = 0; i < fileset->ntablespaces; ++i)
224  {
225  SharedFileSetPath(dirpath, fileset, fileset->tablespaces[i]);
227  }
228 
229  /* Unregister the shared fileset */
230  SharedFileSetUnregister(fileset);
231 }
232 
233 /*
234  * Callback function that will be invoked when this backend detaches from a
235  * DSM segment holding a SharedFileSet that it has created or attached to. If
236  * we are the last to detach, then try to remove the directories and
237  * everything in them. We can't raise an error on failures, because this runs
238  * in error cleanup paths.
239  */
240 static void
242 {
243  bool unlink_all = false;
244  SharedFileSet *fileset = (SharedFileSet *) DatumGetPointer(datum);
245 
246  SpinLockAcquire(&fileset->mutex);
247  Assert(fileset->refcnt > 0);
248  if (--fileset->refcnt == 0)
249  unlink_all = true;
250  SpinLockRelease(&fileset->mutex);
251 
252  /*
253  * If we are the last to detach, we delete the directory in all
254  * tablespaces. Note that we are still actually attached for the rest of
255  * this function so we can safely access its data.
256  */
257  if (unlink_all)
258  SharedFileSetDeleteAll(fileset);
259 }
260 
261 /*
262  * Callback function that will be invoked on the process exit. This will
263  * process the list of all the registered sharedfilesets and delete the
264  * underlying files.
265  */
266 static void
268 {
269  /*
270  * Remove all the pending shared fileset entries. We don't use foreach() here
271  * because SharedFileSetDeleteAll will remove the current element in
272  * filesetlist. Though we have used foreach_delete_current() to remove the
273  * element from filesetlist it could only fix up the state of one of the
274  * loops, see SharedFileSetUnregister.
275  */
276  while (list_length(filesetlist) > 0)
277  {
278  SharedFileSet *fileset = (SharedFileSet *) linitial(filesetlist);
279 
280  SharedFileSetDeleteAll(fileset);
281  }
282 
283  filesetlist = NIL;
284 }
285 
286 /*
287  * Unregister the shared fileset entry registered for cleanup on proc exit.
288  */
289 void
291 {
292  ListCell *l;
293 
294  /*
295  * If the caller is following the dsm based cleanup then we don't maintain
296  * the filesetlist so return.
297  */
298  if (filesetlist == NIL)
299  return;
300 
301  foreach(l, filesetlist)
302  {
303  SharedFileSet *fileset = (SharedFileSet *) lfirst(l);
304 
305  /* Remove the entry from the list */
306  if (input_fileset == fileset)
307  {
308  filesetlist = foreach_delete_current(filesetlist, l);
309  return;
310  }
311  }
312 
313  /* Should have found a match */
314  Assert(false);
315 }
316 
317 /*
318  * Build the path for the directory holding the files backing a SharedFileSet
319  * in a given tablespace.
320  */
321 static void
323 {
324  char tempdirpath[MAXPGPATH];
325 
326  TempTablespacePath(tempdirpath, tablespace);
327  snprintf(path, MAXPGPATH, "%s/%s%lu.%u.sharedfileset",
328  tempdirpath, PG_TEMP_FILE_PREFIX,
329  (unsigned long) fileset->creator_pid, fileset->number);
330 }
331 
332 /*
333  * Sorting hat to determine which tablespace a given shared temporary file
334  * belongs in.
335  */
336 static Oid
337 ChooseTablespace(const SharedFileSet *fileset, const char *name)
338 {
339  uint32 hash = hash_any((const unsigned char *) name, strlen(name));
340 
341  return fileset->tablespaces[hash % fileset->ntablespaces];
342 }
343 
344 /*
345  * Compute the full path of a file in a SharedFileSet.
346  */
347 static void
348 SharedFilePath(char *path, SharedFileSet *fileset, const char *name)
349 {
350  char dirpath[MAXPGPATH];
351 
352  SharedFileSetPath(dirpath, fileset, ChooseTablespace(fileset, name));
353  snprintf(path, MAXPGPATH, "%s/%s", dirpath, name);
354 }
#define NIL
Definition: pg_list.h:65
static void SharedFileSetPath(char *path, SharedFileSet *fileset, Oid tablespace)
static PgChecksumMode mode
Definition: pg_checksums.c:61
bool PathNameDeleteTemporaryFile(const char *path, bool error_on_failure)
Definition: fd.c:1780
int MyProcPid
Definition: globals.c:40
File PathNameCreateTemporaryFile(const char *path, bool error_on_failure)
Definition: fd.c:1713
void SharedFileSetInit(SharedFileSet *fileset, dsm_segment *seg)
Definition: sharedfileset.c:64
void SharedFileSetUnregister(SharedFileSet *input_fileset)
bool SharedFileSetDelete(SharedFileSet *fileset, const char *name, bool error_on_failure)
void on_proc_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:305
int GetTempTablespaces(Oid *tableSpaces, int numSpaces)
Definition: fd.c:2832
#define PointerGetDatum(X)
Definition: postgres.h:556
static void SharedFileSetDeleteOnProcExit(int status, Datum arg)
#define SpinLockInit(lock)
Definition: spin.h:60
int errcode(int sqlerrcode)
Definition: elog.c:610
void PathNameDeleteTemporaryDir(const char *dirname)
Definition: fd.c:1545
#define lengthof(array)
Definition: c.h:675
void TempTablespacePath(char *path, Oid tablespace)
Definition: fd.c:1631
unsigned int Oid
Definition: postgres_ext.h:31
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
Definition: dsm.c:1091
Oid MyDatabaseTableSpace
Definition: globals.c:87
#define foreach_delete_current(lst, cell)
Definition: pg_list.h:368
#define SpinLockAcquire(lock)
Definition: spin.h:62
#define linitial(l)
Definition: pg_list.h:195
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
void PrepareTempTablespaces(void)
Definition: tablespace.c:1326
static List * filesetlist
Definition: sharedfileset.c:36
#define MAXPGPATH
File PathNameOpenTemporaryFile(const char *path, int mode)
Definition: fd.c:1751
char * tablespace
Definition: pgbench.c:189
unsigned int uint32
Definition: c.h:374
static Datum hash_any(const unsigned char *k, int keylen)
Definition: hashfn.h:31
#define SpinLockRelease(lock)
Definition: spin.h:64
uintptr_t Datum
Definition: postgres.h:367
#define InvalidOid
Definition: postgres_ext.h:36
#define ereport(elevel,...)
Definition: elog.h:144
List * lcons(void *datum, List *list)
Definition: list.c:453
void SharedFileSetDeleteAll(SharedFileSet *fileset)
#define Assert(condition)
Definition: c.h:745
#define lfirst(lc)
Definition: pg_list.h:190
static int list_length(const List *l)
Definition: pg_list.h:169
Oid tablespaces[8]
Definition: sharedfileset.h:32
const char * name
Definition: encode.c:561
#define DatumGetPointer(X)
Definition: postgres.h:549
File SharedFileSetCreate(SharedFileSet *fileset, const char *name)
int errmsg(const char *fmt,...)
Definition: elog.c:824
void SharedFileSetAttach(SharedFileSet *fileset, dsm_segment *seg)
static void SharedFilePath(char *path, SharedFileSet *fileset, const char *name)
static Oid ChooseTablespace(const SharedFileSet *fileset, const char *name)
int i
void * arg
File SharedFileSetOpen(SharedFileSet *fileset, const char *name, int mode)
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:227
static bool success
Definition: initdb.c:162
void PathNameCreateTemporaryDir(const char *basedir, const char *directory)
Definition: fd.c:1514
Definition: pg_list.h:50
static unsigned hash(unsigned *uv, int n)
Definition: rege_dfa.c:541
#define snprintf
Definition: port.h:193
int File
Definition: fd.h:49
static void SharedFileSetOnDetach(dsm_segment *segment, Datum datum)