80 #include <sys/types.h>
104 #if defined(HAVE_SYNC_FILE_RANGE)
105 #define PG_FLUSH_DATA_WORKS 1
106 #elif !defined(WIN32) && defined(MS_ASYNC)
107 #define PG_FLUSH_DATA_WORKS 1
108 #elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
109 #define PG_FLUSH_DATA_WORKS 1
127 #define NUM_RESERVED_FDS 10
136 #define FD_MINFREE 48
170 int _do_db_save_errno = errno; \
172 errno = _do_db_save_errno; \
179 #define VFD_CLOSED (-1)
181 #define FileIsValid(file) \
182 ((file) > 0 && (file) < (int) SizeVfdCache && VfdCache[file].fileName != NULL)
184 #define FileIsNotOpen(file) (VfdCache[file].fd == VFD_CLOSED)
187 #define FD_DELETE_AT_CLOSE (1 << 0)
188 #define FD_CLOSE_AT_EOXACT (1 << 1)
189 #define FD_TEMP_FILE_LIMIT (1 << 2)
234 #ifdef USE_ASSERT_CHECKING
235 static bool temporary_files_allowed =
false;
339 static void walkdir(
const char *path,
340 void (*
action) (
const char *fname,
bool isdir,
int elevel),
341 bool process_symlinks,
343 #ifdef PG_FLUSH_DATA_WORKS
344 static void pre_sync_fname(
const char *fname,
bool isdir,
int elevel);
358 #if !defined(WIN32) && defined(USE_ASSERT_CHECKING)
382 int desc_flags = fcntl(
fd, F_GETFL);
389 Assert((desc_flags & (O_RDWR | O_WRONLY)) == 0);
391 Assert((desc_flags & (O_RDWR | O_WRONLY)) != 0);
397 #if defined(HAVE_FSYNC_WRITETHROUGH) && !defined(FSYNC_WRITETHROUGH_IS_FSYNC)
429 #elif defined(F_FULLFSYNC)
430 return (fcntl(
fd, F_FULLFSYNC, 0) == -1) ? -1 : 0;
473 #if defined(HAVE_SYNC_FILE_RANGE)
476 static bool not_implemented_by_kernel =
false;
478 if (not_implemented_by_kernel)
490 rc = sync_file_range(
fd, offset, nbytes,
491 SYNC_FILE_RANGE_WRITE);
504 not_implemented_by_kernel =
true;
511 errmsg(
"could not flush dirty data: %m")));
517 #if !defined(WIN32) && defined(MS_ASYNC)
520 static int pagesize = 0;
534 if (offset == 0 && nbytes == 0)
536 nbytes = lseek(
fd, 0, SEEK_END);
541 errmsg(
"could not determine dirty data size: %m")));
554 pagesize = sysconf(_SC_PAGESIZE);
558 nbytes = (nbytes / pagesize) * pagesize;
569 if (nbytes <= (off_t) SSIZE_MAX)
570 p = mmap(NULL, nbytes, PROT_READ, MAP_SHARED,
fd, offset);
578 rc = msync(p, (
size_t) nbytes, MS_ASYNC);
583 errmsg(
"could not flush dirty data: %m")));
587 rc = munmap(p, (
size_t) nbytes);
593 errmsg(
"could not munmap() while flushing data: %m")));
600 #if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
612 rc = posix_fadvise(
fd, offset, nbytes, POSIX_FADV_DONTNEED);
619 errmsg(
"could not flush dirty data: %m")));
651 return truncate(path, length);
709 errmsg(
"could not open file \"%s\": %m", newfile)));
726 errmsg(
"could not fsync file \"%s\": %m", newfile)));
734 errmsg(
"could not close file \"%s\": %m", newfile)));
740 if (rename(oldfile, newfile) < 0)
744 errmsg(
"could not rename file \"%s\" to \"%s\": %m",
780 if (unlink(fname) < 0)
784 errmsg(
"could not remove file \"%s\": %m",
817 (
errcode(ERRCODE_OUT_OF_MEMORY),
818 errmsg(
"out of memory")));
842 Assert(!temporary_files_allowed);
850 #ifdef USE_ASSERT_CHECKING
851 temporary_files_allowed =
true;
878 #ifdef HAVE_GETRLIMIT
880 int getrlimit_status;
884 fd = (
int *)
palloc(size *
sizeof(
int));
886 #ifdef HAVE_GETRLIMIT
887 getrlimit_status = getrlimit(RLIMIT_NOFILE, &rlim);
888 if (getrlimit_status != 0)
897 #ifdef HAVE_GETRLIMIT
903 if (getrlimit_status == 0 && highestfd >= rlim.rlim_cur - 1)
911 if (errno != EMFILE && errno != ENFILE)
912 elog(
WARNING,
"duplicating stderr file descriptor failed after %d successes: %m", used);
923 if (highestfd < thisfd)
926 if (used >= max_to_probe)
931 for (
j = 0;
j < used;
j++)
942 *already_open = highestfd + 1 - used;
964 &usable_fds, &already_open);
978 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
979 errmsg(
"insufficient file descriptors available to start server process"),
980 errdetail(
"System allows %d, server needs at least %d.",
984 elog(
DEBUG2,
"max_safe_fds = %d, usable_fds = %d, already_open = %d",
1020 #ifdef PG_O_DIRECT_USE_F_NOCACHE
1037 "PG_O_DIRECT value collides with standard flag");
1040 fd = open(fileName, fileFlags, fileMode);
1045 #ifdef PG_O_DIRECT_USE_F_NOCACHE
1048 if (fcntl(
fd, F_NOCACHE, 1) < 0)
1050 int save_errno = errno;
1062 if (errno == EMFILE || errno == ENFILE)
1064 int save_errno = errno;
1067 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
1068 errmsg(
"out of file descriptors: %m; release and retry")));
1152 #if defined(FDDEBUG)
1210 "could not close file \"%s\": %m", vfdP->
fileName);
1339 if (newCacheSize < 32)
1346 if (newVfdCache == NULL)
1348 (
errcode(ERRCODE_OUT_OF_MEMORY),
1349 errmsg(
"out of memory")));
1413 if (returnValue != 0)
1416 else if (
VfdCache[0].lruLessRecently != file)
1442 (
errmsg(
"temporary file: path \"%s\", size %lu",
1443 path, (
unsigned long) size)));
1468 FileInvalidate(
File file)
1501 fileName, fileFlags, fileMode));
1506 fnamecopy = strdup(fileName);
1507 if (fnamecopy == NULL)
1509 (
errcode(ERRCODE_OUT_OF_MEMORY),
1510 errmsg(
"out of memory")));
1530 int save_errno = errno;
1543 vfdP->
fileFlags = fileFlags & ~(O_CREAT | O_TRUNC | O_EXCL);
1570 if (errno == EEXIST)
1581 errmsg(
"cannot create temporary directory \"%s\": %m",
1588 errmsg(
"cannot create temporary subdirectory \"%s\": %m",
1599 struct stat statbuf;
1602 if (
stat(dirname, &statbuf) != 0 && errno == ENOENT)
1634 Assert(temporary_files_allowed);
1668 DEFAULTTABLESPACE_OID,
1722 snprintf(tempfilepath,
sizeof(tempfilepath),
"%s/%s%d.%ld",
1730 O_RDWR | O_CREAT | O_TRUNC |
PG_BINARY);
1744 O_RDWR | O_CREAT | O_TRUNC |
PG_BINARY);
1745 if (file <= 0 && rejectError)
1746 elog(
ERROR,
"could not create temporary file \"%s\": %m",
1771 Assert(temporary_files_allowed);
1782 if (error_on_failure)
1785 errmsg(
"could not create temporary file \"%s\": %m",
1811 Assert(temporary_files_allowed);
1818 if (file <= 0 && errno != ENOENT)
1821 errmsg(
"could not open temporary file \"%s\": %m",
1840 struct stat filestats;
1844 if (
stat(path, &filestats) != 0)
1854 if (stat_errno == ENOENT)
1857 if (unlink(path) < 0)
1859 if (errno != ENOENT)
1862 errmsg(
"could not unlink temporary file \"%s\": %m",
1867 if (stat_errno == 0)
1874 errmsg(
"could not stat file \"%s\": %m", path)));
1905 "could not close file \"%s\": %m", vfdP->
fileName);
1927 struct stat filestats;
1953 if (stat_errno == 0)
1986 #if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
1993 (int64) offset, amount));
2000 returnCode = posix_fadvise(
VfdCache[file].
fd, offset, amount,
2001 POSIX_FADV_WILLNEED);
2020 (int64) offset, (int64) nbytes));
2056 returnCode =
pg_pread(vfdP->
fd, buffer, amount, offset);
2069 DWORD
error = GetLastError();
2073 case ERROR_NO_SYSTEM_RESOURCES:
2120 off_t past_write = offset + amount;
2126 newTotal += past_write - vfdP->
fileSize;
2129 (
errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
2130 errmsg(
"temporary file size exceeds temp_file_limit (%dkB)",
2142 if (returnCode != amount && errno == 0)
2145 if (returnCode >= 0)
2152 off_t past_write = offset + amount;
2167 DWORD
error = GetLastError();
2171 case ERROR_NO_SYSTEM_RESOURCES:
2223 return lseek(
VfdCache[file].
fd, 0, SEEK_END);
2244 if (returnCode == 0 &&
VfdCache[file].fileSize > offset)
2329 if (newDescs == NULL)
2331 (
errcode(ERRCODE_OUT_OF_MEMORY),
2332 errmsg(
"out of memory")));
2355 if (newDescs == NULL)
2394 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2395 errmsg(
"exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"",
2402 if ((file = fopen(
name,
mode)) != NULL)
2413 if (errno == EMFILE || errno == ENFILE)
2415 int save_errno = errno;
2418 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2419 errmsg(
"out of file descriptors: %m; release and retry")));
2447 DO_DB(
elog(
LOG,
"OpenTransientFile: Allocated %d (%s)",
2453 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2454 errmsg(
"exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"",
2498 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2499 errmsg(
"exceeded maxAllocatedDescs (%d) while trying to execute command \"%s\"",
2509 file = popen(command,
mode);
2524 if (errno == EMFILE || errno == ENFILE)
2527 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2528 errmsg(
"out of file descriptors: %m; release and retry")));
2563 elog(
ERROR,
"AllocateDesc kind not recognized");
2598 elog(
WARNING,
"file passed to FreeFile was not obtained from AllocateFile");
2600 return fclose(file);
2626 elog(
WARNING,
"fd passed to CloseTransientFile was not obtained from OpenTransientFile");
2654 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2655 errmsg(
"exceeded maxAllocatedDescs (%d) while trying to open directory \"%s\"",
2662 if ((dir =
opendir(dirname)) != NULL)
2673 if (errno == EMFILE || errno == ENFILE)
2675 int save_errno = errno;
2678 (
errcode(ERRCODE_INSUFFICIENT_RESOURCES),
2679 errmsg(
"out of file descriptors: %m; release and retry")));
2734 errmsg(
"could not open directory \"%s\": %m",
2740 if ((dent =
readdir(dir)) != NULL)
2746 errmsg(
"could not read directory \"%s\": %m",
2782 elog(
WARNING,
"dir passed to FreeDir was not obtained from AllocateDir");
2808 elog(
WARNING,
"file passed to ClosePipeStream was not obtained from OpenPipeStream");
2810 return pclose(file);
2985 #ifdef USE_ASSERT_CHECKING
2986 temporary_files_allowed =
false;
3033 "temporary file %s not closed at end-of-transaction",
3045 elog(
WARNING,
"%d temporary files and directories not closed at end-of-transaction",
3095 if (strcmp(spc_de->
d_name,
".") == 0 ||
3096 strcmp(spc_de->
d_name,
"..") == 0)
3099 snprintf(temp_path,
sizeof(temp_path),
"pg_tblspc/%s/%s/%s",
3103 snprintf(temp_path,
sizeof(temp_path),
"pg_tblspc/%s/%s",
3142 if (temp_dir == NULL && errno == ENOENT && missing_ok)
3147 if (strcmp(temp_de->
d_name,
".") == 0 ||
3148 strcmp(temp_de->
d_name,
"..") == 0)
3151 snprintf(rm_path,
sizeof(rm_path),
"%s/%s",
3152 tmpdirname, temp_de->
d_name);
3168 if (rmdir(rm_path) < 0)
3171 errmsg(
"could not remove directory \"%s\": %m",
3176 if (unlink(rm_path) < 0)
3179 errmsg(
"could not remove file \"%s\": %m",
3185 (
errmsg(
"unexpected file found in temporary-files directory: \"%s\"",
3209 if (strspn(de->
d_name,
"0123456789") != strlen(de->
d_name))
3212 snprintf(dbspace_path,
sizeof(dbspace_path),
"%s/%s",
3235 snprintf(rm_path,
sizeof(rm_path),
"%s/%s",
3236 dbspacedirname, de->
d_name);
3238 if (unlink(rm_path) < 0)
3241 errmsg(
"could not remove file \"%s\": %m",
3260 for (pos = 1; isdigit((
unsigned char)
name[pos]); ++pos)
3262 if (pos == 1 ||
name[pos] !=
'_')
3266 for (savepos = ++pos; isdigit((
unsigned char)
name[pos]); ++pos)
3272 if (
name[pos] ==
'_')
3278 pos += forkchar + 1;
3280 if (
name[pos] ==
'.')
3284 for (segchar = 1; isdigit((
unsigned char)
name[pos + segchar]); ++segchar)
3292 if (
name[pos] !=
'\0')
3299 do_syncfs(
const char *path)
3311 errmsg(
"could not open file \"%s\": %m", path)));
3317 errmsg(
"could not synchronize file system for file \"%s\": %m", path)));
3347 bool xlog_is_symlink;
3357 xlog_is_symlink =
false;
3362 if (
lstat(
"pg_wal", &st) < 0)
3365 errmsg(
"could not stat file \"%s\": %m",
3368 xlog_is_symlink =
true;
3396 if (strcmp(de->
d_name,
".") == 0 || strcmp(de->
d_name,
"..") == 0)
3404 if (xlog_is_symlink)
3405 do_syncfs(
"pg_wal");
3410 #ifdef PG_FLUSH_DATA_WORKS
3420 if (xlog_is_symlink)
3438 if (xlog_is_symlink)
3460 void (*
action) (
const char *fname,
bool isdir,
int elevel),
3461 bool process_symlinks,
3475 if (strcmp(de->
d_name,
".") == 0 ||
3476 strcmp(de->
d_name,
"..") == 0)
3484 (*action) (
subpath,
false, elevel);
3509 (*action) (path,
true, elevel);
3519 #ifdef PG_FLUSH_DATA_WORKS
3522 pre_sync_fname(
const char *fname,
bool isdir,
int elevel)
3537 if (errno == EACCES)
3541 errmsg(
"could not open file \"%s\": %m", fname)));
3554 errmsg(
"could not close file \"%s\": %m", fname)));
3577 if (rmdir(fname) != 0 && errno != ENOENT)
3580 errmsg(
"could not remove directory \"%s\": %m", fname)));
3623 if (
fd < 0 && isdir && (errno == EISDIR || errno == EACCES))
3625 else if (
fd < 0 && ignore_perm && errno == EACCES)
3631 errmsg(
"could not open file \"%s\": %m", fname)));
3641 if (returncode != 0 && !(isdir && (errno == EBADF || errno == EINVAL)))
3652 errmsg(
"could not fsync file \"%s\": %m", fname)));
3660 errmsg(
"could not close file \"%s\": %m", fname)));
3686 if (strlen(parentpath) == 0)
void begin_startup_progress_phase(void)
#define MemSet(start, val, len)
#define StaticAssertStmt(condition, errmessage)
int fdatasync(int fildes)
#define OidIsValid(objectId)
elog(ERROR, "%s: %s", p2, msg)
struct dirent * readdir(DIR *)
DIR * opendir(const char *)
int errcode_for_file_access(void)
int errdetail(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
struct dirent * ReadDir(DIR *dir, const char *dirname)
int max_files_per_process
void pg_flush_data(int fd, off_t offset, off_t nbytes)
int FileGetRawDesc(File file)
int MakePGDirectory(const char *directoryName)
int recovery_init_sync_method
void FileWriteback(File file, off_t offset, off_t nbytes, uint32 wait_event_info)
int pg_fsync_no_writethrough(int fd)
static int numTempTableSpaces
static bool ReleaseLruFile(void)
FILE * AllocateFile(const char *name, const char *mode)
#define FD_DELETE_AT_CLOSE
int BasicOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode)
static int maxAllocatedDescs
static void Delete(File file)
static int FreeDesc(AllocateDesc *desc)
static long tempFileCounter
int durable_rename(const char *oldfile, const char *newfile, int elevel)
int GetTempTablespaces(Oid *tableSpaces, int numSpaces)
static int numAllocatedDescs
File PathNameOpenTemporaryFile(const char *path, int mode)
static void LruDelete(File file)
#define FileIsValid(file)
int FileSync(File file, uint32 wait_event_info)
int CloseTransientFile(int fd)
int BasicOpenFile(const char *fileName, int fileFlags)
static File AllocateVfd(void)
File PathNameCreateTemporaryFile(const char *path, bool error_on_failure)
void PathNameDeleteTemporaryDir(const char *dirname)
int ClosePipeStream(FILE *file)
void AtEOXact_Files(bool isCommit)
int FileGetRawFlags(File file)
static int nextTempTableSpace
#define FD_CLOSE_AT_EOXACT
int fsync_fname_ext(const char *fname, bool isdir, bool ignore_perm, int elevel)
static void unlink_if_exists_fname(const char *fname, bool isdir, int elevel)
static void RemovePgTempRelationFiles(const char *tsdirname)
mode_t FileGetRawMode(File file)
static AllocateDesc * allocatedDescs
static void count_usable_fds(int max_to_probe, int *usable_fds, int *already_open)
static int FileAccess(File file)
static void FreeVfd(File file)
int pg_fsync_writethrough(int fd)
void FileClose(File file)
FILE * OpenPipeStream(const char *command, const char *mode)
void ReleaseExternalFD(void)
#define FD_TEMP_FILE_LIMIT
void RemovePgTempFilesInDir(const char *tmpdirname, bool missing_ok, bool unlink_all)
void RemovePgTempFiles(void)
#define FileIsNotOpen(file)
bool TempTablespacesAreSet(void)
void fsync_fname(const char *fname, bool isdir)
int FilePrefetch(File file, off_t offset, off_t amount, uint32 wait_event_info)
int data_sync_elevel(int elevel)
File PathNameOpenFile(const char *fileName, int fileFlags)
static void Insert(File file)
int FileWrite(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info)
Oid GetNextTempTableSpace(void)
File PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode)
static void datadir_fsync_fname(const char *fname, bool isdir, int elevel)
static void ReportTemporaryFileUsage(const char *path, off_t size)
static File OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError)
bool AcquireExternalFD(void)
static void RegisterTemporaryFile(File file)
int FileRead(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info)
struct dirent * ReadDirExtended(DIR *dir, const char *dirname, int elevel)
static Oid * tempTableSpaces
static bool reserveAllocatedDesc(void)
void InitFileAccess(void)
static void RemovePgTempRelationFilesInDbspace(const char *dbspacedirname)
File OpenTemporaryFile(bool interXact)
int durable_unlink(const char *fname, int elevel)
static uint64 temporary_files_size
void ReserveExternalFD(void)
char * FilePathName(File file)
bool looks_like_temp_rel_name(const char *name)
bool PathNameDeleteTemporaryFile(const char *path, bool error_on_failure)
void set_max_safe_fds(void)
static void CleanupTempFiles(bool isCommit, bool isProcExit)
static bool have_xact_temporary_files
static int LruInsert(File file)
static int numExternalFDs
static int fsync_parent_path(const char *fname, int elevel)
void PathNameCreateTemporaryDir(const char *basedir, const char *directory)
void AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid, SubTransactionId parentSubid)
int OpenTransientFile(const char *fileName, int fileFlags)
void InitTemporaryFileAccess(void)
int OpenTransientFilePerm(const char *fileName, int fileFlags, mode_t fileMode)
static void ReleaseLruFiles(void)
void SyncDataDirectory(void)
off_t FileSize(File file)
int FileTruncate(File file, off_t offset, uint32 wait_event_info)
static void BeforeShmemExit_Files(int code, Datum arg)
static void walkdir(const char *path, void(*action)(const char *fname, bool isdir, int elevel), bool process_symlinks, int elevel)
int pg_truncate(const char *path, off_t length)
void SetTempTablespaces(Oid *tableSpaces, int numSpaces)
DIR * AllocateDir(const char *dirname)
void TempTablespacePath(char *path, Oid tablespace)
@ RECOVERY_INIT_SYNC_METHOD_SYNCFS
@ RECOVERY_INIT_SYNC_METHOD_FSYNC
PGFileType get_dirent_type(const char *path, const struct dirent *de, bool look_through_symlinks, int elevel)
void before_shmem_exit(pg_on_exit_callback function, Datum arg)
static void const char fflush(stdout)
Assert(fmt[strlen(fmt) - 1] !='\n')
Datum subpath(PG_FUNCTION_ARGS)
void pfree(void *pointer)
void * repalloc(void *pointer, Size size)
#define CHECK_FOR_INTERRUPTS()
static PgChecksumMode mode
#define PG_TEMP_FILES_DIR
#define PG_TEMP_FILE_PREFIX
uint64 pg_prng_uint64_range(pg_prng_state *state, uint64 rmin, uint64 rmax)
pg_prng_state pg_global_prng_state
void pgstat_report_tempfile(size_t filesize)
void get_parent_directory(char *path)
pqsigfunc pqsignal(int signo, pqsigfunc func)
size_t strlcpy(char *dst, const char *src, size_t siz)
static int fd(const char *x, int i)
int forkname_chars(const char *str, ForkNumber *fork)
#define TABLESPACE_VERSION_DIRECTORY
void ResourceOwnerEnlargeFiles(ResourceOwner owner)
void ResourceOwnerForgetFile(ResourceOwner owner, File file)
ResourceOwner CurrentResourceOwner
void ResourceOwnerRememberFile(ResourceOwner owner, File file)
void pg_usleep(long microsec)
#define ereport_startup_progress(msg,...)
SubTransactionId create_subid
union AllocateDesc::@19 desc
static void pgstat_report_wait_start(uint32 wait_event_info)
static void pgstat_report_wait_end(void)
void _dosmaperr(unsigned long)
SubTransactionId GetCurrentSubTransactionId(void)
#define SYNC_METHOD_FSYNC_WRITETHROUGH
static const char * directory