PostgreSQL Source Code  git master
walmethods.c File Reference
#include "postgres_fe.h"
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include "pgtar.h"
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "receivelog.h"
#include "streamutil.h"
Include dependency graph for walmethods.c:

Go to the source code of this file.

Data Structures

struct  DirectoryMethodData
 
struct  DirectoryMethodFile
 
struct  TarMethodFile
 
struct  TarMethodData
 

Macros

#define ZLIB_OUT_SIZE   4096
 
#define tar_clear_error()   tar_data->lasterror[0] = '\0'
 
#define tar_set_error(msg)   strlcpy(tar_data->lasterror, _(msg), sizeof(tar_data->lasterror))
 

Typedefs

typedef struct DirectoryMethodData DirectoryMethodData
 
typedef struct DirectoryMethodFile DirectoryMethodFile
 
typedef struct TarMethodFile TarMethodFile
 
typedef struct TarMethodData TarMethodData
 

Functions

static const char * dir_getlasterror (void)
 
static Walfile dir_open_for_write (const char *pathname, const char *temp_suffix, size_t pad_to_size)
 
static ssize_t dir_write (Walfile f, const void *buf, size_t count)
 
static off_t dir_get_current_pos (Walfile f)
 
static int dir_close (Walfile f, WalCloseMethod method)
 
static int dir_sync (Walfile f)
 
static ssize_t dir_get_file_size (const char *pathname)
 
static bool dir_existsfile (const char *pathname)
 
static bool dir_finish (void)
 
WalWriteMethodCreateWalDirectoryMethod (const char *basedir, int compression, bool sync)
 
void FreeWalDirectoryMethod (void)
 
static const char * tar_getlasterror (void)
 
static ssize_t tar_write (Walfile f, const void *buf, size_t count)
 
static bool tar_write_padding_data (TarMethodFile *f, size_t bytes)
 
static Walfile tar_open_for_write (const char *pathname, const char *temp_suffix, size_t pad_to_size)
 
static ssize_t tar_get_file_size (const char *pathname)
 
static off_t tar_get_current_pos (Walfile f)
 
static int tar_sync (Walfile f)
 
static int tar_close (Walfile f, WalCloseMethod method)
 
static bool tar_existsfile (const char *pathname)
 
static bool tar_finish (void)
 
WalWriteMethodCreateWalTarMethod (const char *tarbase, int compression, bool sync)
 
void FreeWalTarMethod (void)
 

Variables

static DirectoryMethodDatadir_data = NULL
 
static TarMethodDatatar_data = NULL
 

Macro Definition Documentation

◆ tar_clear_error

#define tar_clear_error ( )    tar_data->lasterror[0] = '\0'

◆ tar_set_error

#define tar_set_error (   msg)    strlcpy(tar_data->lasterror, _(msg), sizeof(tar_data->lasterror))

Definition at line 411 of file walmethods.c.

Referenced by tar_close(), tar_finish(), tar_getlasterror(), and tar_open_for_write().

◆ ZLIB_OUT_SIZE

#define ZLIB_OUT_SIZE   4096

Typedef Documentation

◆ DirectoryMethodData

◆ DirectoryMethodFile

◆ TarMethodData

typedef struct TarMethodData TarMethodData

◆ TarMethodFile

typedef struct TarMethodFile TarMethodFile

Function Documentation

◆ CreateWalDirectoryMethod()

WalWriteMethod* CreateWalDirectoryMethod ( const char *  basedir,
int  compression,
bool  sync 
)

Definition at line 350 of file walmethods.c.

References DirectoryMethodData::basedir, WalWriteMethod::close, DirectoryMethodData::compression, dir_close(), dir_existsfile(), dir_finish(), dir_get_current_pos(), dir_get_file_size(), dir_getlasterror(), dir_open_for_write(), dir_sync(), dir_write(), WalWriteMethod::existsfile, WalWriteMethod::finish, WalWriteMethod::get_current_pos, WalWriteMethod::get_file_size, WalWriteMethod::getlasterror, WalWriteMethod::open_for_write, pg_malloc0(), pg_strdup(), DirectoryMethodData::sync, WalWriteMethod::sync, and WalWriteMethod::write.

Referenced by LogStreamerMain(), and StreamLog().

351 {
352  WalWriteMethod *method;
353 
354  method = pg_malloc0(sizeof(WalWriteMethod));
356  method->write = dir_write;
359  method->close = dir_close;
360  method->sync = dir_sync;
361  method->existsfile = dir_existsfile;
362  method->finish = dir_finish;
363  method->getlasterror = dir_getlasterror;
364 
366  dir_data->compression = compression;
368  dir_data->sync = sync;
369 
370  return method;
371 }
static off_t dir_get_current_pos(Walfile f)
Definition: walmethods.c:206
int(* close)(Walfile f, WalCloseMethod method)
Definition: walmethods.h:47
int(* sync)(Walfile f)
Definition: walmethods.h:67
static char * basedir
Definition: pg_basebackup.c:82
const char *(* getlasterror)(void)
Definition: walmethods.h:78
off_t(* get_current_pos)(Walfile f)
Definition: walmethods.h:62
static bool dir_finish(void)
Definition: walmethods.c:334
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
ssize_t(* write)(Walfile f, const void *buf, size_t count)
Definition: walmethods.h:59
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static ssize_t dir_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:187
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
static int dir_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:215
bool(* finish)(void)
Definition: walmethods.h:75
bool(* existsfile)(const char *pathname)
Definition: walmethods.h:50
Walfile(* open_for_write)(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.h:41
static Walfile dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.c:73
ssize_t(* get_file_size)(const char *pathname)
Definition: walmethods.h:53
static const char * dir_getlasterror(void)
Definition: walmethods.c:66
static ssize_t dir_get_file_size(const char *pathname)
Definition: walmethods.c:303
static bool dir_existsfile(const char *pathname)
Definition: walmethods.c:318
static int dir_sync(Walfile f)
Definition: walmethods.c:284

◆ CreateWalTarMethod()

WalWriteMethod* CreateWalTarMethod ( const char *  tarbase,
int  compression,
bool  sync 
)

Definition at line 983 of file walmethods.c.

References WalWriteMethod::close, DirectoryMethodData::compression, TarMethodData::compression, WalWriteMethod::existsfile, TarMethodData::fd, WalWriteMethod::finish, WalWriteMethod::get_current_pos, WalWriteMethod::get_file_size, WalWriteMethod::getlasterror, WalWriteMethod::open_for_write, pg_malloc(), pg_malloc0(), sprintf, DirectoryMethodData::sync, WalWriteMethod::sync, TarMethodData::sync, tar_close(), tar_existsfile(), tar_finish(), tar_get_current_pos(), tar_get_file_size(), tar_getlasterror(), tar_open_for_write(), tar_sync(), tar_write(), TarMethodData::tarfilename, WalWriteMethod::write, and ZLIB_OUT_SIZE.

Referenced by LogStreamerMain().

984 {
985  WalWriteMethod *method;
986  const char *suffix = (compression != 0) ? ".tar.gz" : ".tar";
987 
988  method = pg_malloc0(sizeof(WalWriteMethod));
990  method->write = tar_write;
993  method->close = tar_close;
994  method->sync = tar_sync;
995  method->existsfile = tar_existsfile;
996  method->finish = tar_finish;
997  method->getlasterror = tar_getlasterror;
998 
999  tar_data = pg_malloc0(sizeof(TarMethodData));
1000  tar_data->tarfilename = pg_malloc0(strlen(tarbase) + strlen(suffix) + 1);
1001  sprintf(tar_data->tarfilename, "%s%s", tarbase, suffix);
1002  tar_data->fd = -1;
1003  tar_data->compression = compression;
1004  tar_data->sync = sync;
1005 #ifdef HAVE_LIBZ
1006  if (compression)
1007  tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
1008 #endif
1009 
1010  return method;
1011 }
int(* close)(Walfile f, WalCloseMethod method)
Definition: walmethods.h:47
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static bool tar_finish(void)
Definition: walmethods.c:886
int(* sync)(Walfile f)
Definition: walmethods.h:67
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:481
const char *(* getlasterror)(void)
Definition: walmethods.h:78
static int tar_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:718
off_t(* get_current_pos)(Walfile f)
Definition: walmethods.h:62
#define sprintf
Definition: port.h:194
static const char * tar_getlasterror(void)
Definition: walmethods.c:414
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
static Walfile tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.c:532
ssize_t(* write)(Walfile f, const void *buf, size_t count)
Definition: walmethods.h:59
static ssize_t tar_get_file_size(const char *pathname)
Definition: walmethods.c:680
static bool tar_existsfile(const char *pathname)
Definition: walmethods.c:878
static off_t tar_get_current_pos(Walfile f)
Definition: walmethods.c:690
char * tarfilename
Definition: walmethods.c:397
#define ZLIB_OUT_SIZE
Definition: walmethods.c:32
bool(* finish)(void)
Definition: walmethods.h:75
bool(* existsfile)(const char *pathname)
Definition: walmethods.h:50
Walfile(* open_for_write)(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.h:41
ssize_t(* get_file_size)(const char *pathname)
Definition: walmethods.h:53
static TarMethodData * tar_data
Definition: walmethods.c:408
static int tar_sync(Walfile f)
Definition: walmethods.c:699

◆ dir_close()

static int dir_close ( Walfile  f,
WalCloseMethod  method 
)
static

Definition at line 215 of file walmethods.c.

References Assert, DirectoryMethodData::basedir, close, CLOSE_NORMAL, CLOSE_UNLINK, DirectoryMethodData::compression, durable_rename(), DirectoryMethodFile::fd, fsync_fname(), fsync_parent_path(), DirectoryMethodFile::fullpath, MAXPGPATH, DirectoryMethodFile::pathname, pg_free(), snprintf, DirectoryMethodData::sync, and DirectoryMethodFile::temp_suffix.

Referenced by CreateWalDirectoryMethod().

216 {
217  int r;
219  static char tmppath[MAXPGPATH];
220  static char tmppath2[MAXPGPATH];
221 
222  Assert(f != NULL);
223 
224 #ifdef HAVE_LIBZ
225  if (dir_data->compression > 0)
226  r = gzclose(df->gzfp);
227  else
228 #endif
229  r = close(df->fd);
230 
231  if (r == 0)
232  {
233  /* Build path to the current version of the file */
234  if (method == CLOSE_NORMAL && df->temp_suffix)
235  {
236  /*
237  * If we have a temp prefix, normal operation is to rename the
238  * file.
239  */
240  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
241  dir_data->basedir, df->pathname,
242  dir_data->compression > 0 ? ".gz" : "",
243  df->temp_suffix);
244  snprintf(tmppath2, sizeof(tmppath2), "%s/%s%s",
245  dir_data->basedir, df->pathname,
246  dir_data->compression > 0 ? ".gz" : "");
247  r = durable_rename(tmppath, tmppath2);
248  }
249  else if (method == CLOSE_UNLINK)
250  {
251  /* Unlink the file once it's closed */
252  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
253  dir_data->basedir, df->pathname,
254  dir_data->compression > 0 ? ".gz" : "",
255  df->temp_suffix ? df->temp_suffix : "");
256  r = unlink(tmppath);
257  }
258  else
259  {
260  /*
261  * Else either CLOSE_NORMAL and no temp suffix, or
262  * CLOSE_NO_RENAME. In this case, fsync the file and containing
263  * directory if sync mode is requested.
264  */
265  if (dir_data->sync)
266  {
267  r = fsync_fname(df->fullpath, false);
268  if (r == 0)
269  r = fsync_parent_path(df->fullpath);
270  }
271  }
272  }
273 
274  pg_free(df->pathname);
275  pg_free(df->fullpath);
276  if (df->temp_suffix)
277  pg_free(df->temp_suffix);
278  pg_free(df);
279 
280  return r;
281 }
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:580
#define MAXPGPATH
int durable_rename(const char *oldfile, const char *newfile, int elevel)
Definition: fd.c:606
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define Assert(condition)
Definition: c.h:732
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:192
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3419

◆ dir_existsfile()

static bool dir_existsfile ( const char *  pathname)
static

Definition at line 318 of file walmethods.c.

References DirectoryMethodData::basedir, close, fd(), MAXPGPATH, PG_BINARY, and snprintf.

Referenced by CreateWalDirectoryMethod().

319 {
320  static char tmppath[MAXPGPATH];
321  int fd;
322 
323  snprintf(tmppath, sizeof(tmppath), "%s/%s",
324  dir_data->basedir, pathname);
325 
326  fd = open(tmppath, O_RDONLY | PG_BINARY, 0);
327  if (fd < 0)
328  return false;
329  close(fd);
330  return true;
331 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1191
#define MAXPGPATH
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:192

◆ dir_finish()

static bool dir_finish ( void  )
static

Definition at line 334 of file walmethods.c.

References DirectoryMethodData::basedir, fsync_fname(), and DirectoryMethodData::sync.

Referenced by CreateWalDirectoryMethod().

335 {
336  if (dir_data->sync)
337  {
338  /*
339  * Files are fsynced when they are closed, but we need to fsync the
340  * directory entry here as well.
341  */
342  if (fsync_fname(dir_data->basedir, true) != 0)
343  return false;
344  }
345  return true;
346 }
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:580
static DirectoryMethodData * dir_data
Definition: walmethods.c:48

◆ dir_get_current_pos()

static off_t dir_get_current_pos ( Walfile  f)
static

Definition at line 206 of file walmethods.c.

References Assert.

Referenced by CreateWalDirectoryMethod().

207 {
208  Assert(f != NULL);
209 
210  /* Use a cached value to prevent lots of reseeks */
211  return ((DirectoryMethodFile *) f)->currpos;
212 }
#define Assert(condition)
Definition: c.h:732

◆ dir_get_file_size()

static ssize_t dir_get_file_size ( const char *  pathname)
static

Definition at line 303 of file walmethods.c.

References DirectoryMethodData::basedir, MAXPGPATH, snprintf, and stat.

Referenced by CreateWalDirectoryMethod().

304 {
305  struct stat statbuf;
306  static char tmppath[MAXPGPATH];
307 
308  snprintf(tmppath, sizeof(tmppath), "%s/%s",
309  dir_data->basedir, pathname);
310 
311  if (stat(tmppath, &statbuf) != 0)
312  return -1;
313 
314  return statbuf.st_size;
315 }
#define MAXPGPATH
#define stat(a, b)
Definition: win32_port.h:255
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define snprintf
Definition: port.h:192

◆ dir_getlasterror()

static const char* dir_getlasterror ( void  )
static

Definition at line 66 of file walmethods.c.

References strerror.

Referenced by CreateWalDirectoryMethod().

67 {
68  /* Directory method always sets errno, so just use strerror */
69  return strerror(errno);
70 }
#define strerror
Definition: port.h:205

◆ dir_open_for_write()

static Walfile dir_open_for_write ( const char *  pathname,
const char *  temp_suffix,
size_t  pad_to_size 
)
static

Definition at line 73 of file walmethods.c.

References DirectoryMethodData::basedir, generate_unaccent_rules::bytes(), close, DirectoryMethodData::compression, DirectoryMethodFile::currpos, PGAlignedXLogBlock::data, DirectoryMethodFile::fd, fd(), fsync_fname(), fsync_parent_path(), DirectoryMethodFile::fullpath, MAXPGPATH, DirectoryMethodFile::pathname, PG_BINARY, pg_file_create_mode, pg_malloc0(), pg_strdup(), snprintf, DirectoryMethodData::sync, DirectoryMethodFile::temp_suffix, and write.

Referenced by CreateWalDirectoryMethod().

74 {
75  static char tmppath[MAXPGPATH];
76  int fd;
78 #ifdef HAVE_LIBZ
79  gzFile gzfp = NULL;
80 #endif
81 
82  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
83  dir_data->basedir, pathname,
84  dir_data->compression > 0 ? ".gz" : "",
85  temp_suffix ? temp_suffix : "");
86 
87  /*
88  * Open a file for non-compressed as well as compressed files. Tracking
89  * the file descriptor is important for dir_sync() method as gzflush()
90  * does not do any system calls to fsync() to make changes permanent on
91  * disk.
92  */
93  fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, pg_file_create_mode);
94  if (fd < 0)
95  return NULL;
96 
97 #ifdef HAVE_LIBZ
98  if (dir_data->compression > 0)
99  {
100  gzfp = gzdopen(fd, "wb");
101  if (gzfp == NULL)
102  {
103  close(fd);
104  return NULL;
105  }
106 
107  if (gzsetparams(gzfp, dir_data->compression,
108  Z_DEFAULT_STRATEGY) != Z_OK)
109  {
110  gzclose(gzfp);
111  return NULL;
112  }
113  }
114 #endif
115 
116  /* Do pre-padding on non-compressed files */
117  if (pad_to_size && dir_data->compression == 0)
118  {
119  PGAlignedXLogBlock zerobuf;
120  int bytes;
121 
122  memset(zerobuf.data, 0, XLOG_BLCKSZ);
123  for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ)
124  {
125  errno = 0;
126  if (write(fd, zerobuf.data, XLOG_BLCKSZ) != XLOG_BLCKSZ)
127  {
128  int save_errno = errno;
129 
130  close(fd);
131 
132  /*
133  * If write didn't set errno, assume problem is no disk space.
134  */
135  errno = save_errno ? save_errno : ENOSPC;
136  return NULL;
137  }
138  }
139 
140  if (lseek(fd, 0, SEEK_SET) != 0)
141  {
142  int save_errno = errno;
143 
144  close(fd);
145  errno = save_errno;
146  return NULL;
147  }
148  }
149 
150  /*
151  * fsync WAL file and containing directory, to ensure the file is
152  * persistently created and zeroed (if padded). That's particularly
153  * important when using synchronous mode, where the file is modified and
154  * fsynced in-place, without a directory fsync.
155  */
156  if (dir_data->sync)
157  {
158  if (fsync_fname(tmppath, false) != 0 ||
159  fsync_parent_path(tmppath) != 0)
160  {
161 #ifdef HAVE_LIBZ
162  if (dir_data->compression > 0)
163  gzclose(gzfp);
164  else
165 #endif
166  close(fd);
167  return NULL;
168  }
169  }
170 
171  f = pg_malloc0(sizeof(DirectoryMethodFile));
172 #ifdef HAVE_LIBZ
173  if (dir_data->compression > 0)
174  f->gzfp = gzfp;
175 #endif
176  f->fd = fd;
177  f->currpos = 0;
178  f->pathname = pg_strdup(pathname);
179  f->fullpath = pg_strdup(tmppath);
180  if (temp_suffix)
181  f->temp_suffix = pg_strdup(temp_suffix);
182 
183  return f;
184 }
int pg_file_create_mode
Definition: file_perm.c:19
#define write(a, b, c)
Definition: win32.h:14
def bytes(source, encoding='ascii', errors='strict')
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:580
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1191
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
#define MAXPGPATH
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define close(a)
Definition: win32.h:12
char data[XLOG_BLCKSZ]
Definition: c.h:1068
#define snprintf
Definition: port.h:192
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3419

◆ dir_sync()

static int dir_sync ( Walfile  f)
static

Definition at line 284 of file walmethods.c.

References Assert, DirectoryMethodData::compression, fd(), fsync, and DirectoryMethodData::sync.

Referenced by CreateWalDirectoryMethod().

285 {
286  Assert(f != NULL);
287 
288  if (!dir_data->sync)
289  return 0;
290 
291 #ifdef HAVE_LIBZ
292  if (dir_data->compression > 0)
293  {
294  if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK)
295  return -1;
296  }
297 #endif
298 
299  return fsync(((DirectoryMethodFile *) f)->fd);
300 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define fsync(fd)
Definition: win32_port.h:63
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define Assert(condition)
Definition: c.h:732

◆ dir_write()

static ssize_t dir_write ( Walfile  f,
const void *  buf,
size_t  count 
)
static

Definition at line 187 of file walmethods.c.

References Assert, DirectoryMethodData::compression, DirectoryMethodFile::currpos, DirectoryMethodFile::fd, and write.

Referenced by CreateWalDirectoryMethod().

188 {
189  ssize_t r;
191 
192  Assert(f != NULL);
193 
194 #ifdef HAVE_LIBZ
195  if (dir_data->compression > 0)
196  r = (ssize_t) gzwrite(df->gzfp, buf, count);
197  else
198 #endif
199  r = write(df->fd, buf, count);
200  if (r > 0)
201  df->currpos += r;
202  return r;
203 }
#define write(a, b, c)
Definition: win32.h:14
static char * buf
Definition: pg_test_fsync.c:68
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
#define Assert(condition)
Definition: c.h:732

◆ FreeWalDirectoryMethod()

void FreeWalDirectoryMethod ( void  )

Definition at line 374 of file walmethods.c.

References DirectoryMethodData::basedir, and pg_free().

Referenced by LogStreamerMain(), and StreamLog().

375 {
377  pg_free(dir_data);
378 }
static DirectoryMethodData * dir_data
Definition: walmethods.c:48
void pg_free(void *ptr)
Definition: fe_memutils.c:105

◆ FreeWalTarMethod()

void FreeWalTarMethod ( void  )

Definition at line 1014 of file walmethods.c.

References TarMethodData::compression, pg_free(), and TarMethodData::tarfilename.

Referenced by LogStreamerMain().

1015 {
1017 #ifdef HAVE_LIBZ
1018  if (tar_data->compression)
1019  pg_free(tar_data->zlibOut);
1020 #endif
1021  pg_free(tar_data);
1022 }
char * tarfilename
Definition: walmethods.c:397
void pg_free(void *ptr)
Definition: fe_memutils.c:105
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_close()

static int tar_close ( Walfile  f,
WalCloseMethod  method 
)
static

Definition at line 718 of file walmethods.c.

References Assert, CLOSE_NORMAL, CLOSE_UNLINK, TarMethodData::compression, TarMethodData::currentfile, TarMethodFile::currpos, TarMethodData::fd, ftruncate, header(), TarMethodFile::header, MemSet, TarMethodFile::ofs_start, TarMethodFile::pad_to_size, TarMethodFile::pathname, pg_free(), print_tar_number(), strlcpy(), tar_clear_error, tar_get_current_pos(), tar_set_error, tar_sync(), tar_write(), tar_write_padding_data(), tarChecksum(), and write.

Referenced by CreateWalTarMethod(), and tar_finish().

719 {
720  ssize_t filesize;
721  int padding;
722  TarMethodFile *tf = (TarMethodFile *) f;
723 
724  Assert(f != NULL);
725  tar_clear_error();
726 
727  if (method == CLOSE_UNLINK)
728  {
729  if (tar_data->compression)
730  {
731  tar_set_error("unlink not supported with compression");
732  return -1;
733  }
734 
735  /*
736  * Unlink the file that we just wrote to the tar. We do this by
737  * truncating it to the start of the header. This is safe as we only
738  * allow writing of the very last file.
739  */
740  if (ftruncate(tar_data->fd, tf->ofs_start) != 0)
741  return -1;
742 
743  pg_free(tf->pathname);
744  pg_free(tf);
745  tar_data->currentfile = NULL;
746 
747  return 0;
748  }
749 
750  /*
751  * Pad the file itself with zeroes if necessary. Note that this is
752  * different from the tar format padding -- this is the padding we asked
753  * for when the file was opened.
754  */
755  if (tf->pad_to_size)
756  {
757  if (tar_data->compression)
758  {
759  /*
760  * A compressed tarfile is padded on close since we cannot know
761  * the size of the compressed output until the end.
762  */
763  size_t sizeleft = tf->pad_to_size - tf->currpos;
764 
765  if (sizeleft)
766  {
767  if (!tar_write_padding_data(tf, sizeleft))
768  return -1;
769  }
770  }
771  else
772  {
773  /*
774  * An uncompressed tarfile was padded on creation, so just adjust
775  * the current position as if we seeked to the end.
776  */
777  tf->currpos = tf->pad_to_size;
778  }
779  }
780 
781  /*
782  * Get the size of the file, and pad the current data up to the nearest
783  * 512 byte boundary.
784  */
785  filesize = tar_get_current_pos(f);
786  padding = ((filesize + 511) & ~511) - filesize;
787  if (padding)
788  {
789  char zerobuf[512];
790 
791  MemSet(zerobuf, 0, padding);
792  if (tar_write(f, zerobuf, padding) != padding)
793  return -1;
794  }
795 
796 
797 #ifdef HAVE_LIBZ
798  if (tar_data->compression)
799  {
800  /* Flush the current buffer */
801  if (!tar_write_compressed_data(NULL, 0, true))
802  {
803  errno = EINVAL;
804  return -1;
805  }
806  }
807 #endif
808 
809  /*
810  * Now go back and update the header with the correct filesize and
811  * possibly also renaming the file. We overwrite the entire current header
812  * when done, including the checksum.
813  */
814  print_tar_number(&(tf->header[124]), 12, filesize);
815 
816  if (method == CLOSE_NORMAL)
817 
818  /*
819  * We overwrite it with what it was before if we have no tempname,
820  * since we're going to write the buffer anyway.
821  */
822  strlcpy(&(tf->header[0]), tf->pathname, 100);
823 
824  print_tar_number(&(tf->header[148]), 8, tarChecksum(((TarMethodFile *) f)->header));
825  if (lseek(tar_data->fd, tf->ofs_start, SEEK_SET) != ((TarMethodFile *) f)->ofs_start)
826  return -1;
827  if (!tar_data->compression)
828  {
829  errno = 0;
830  if (write(tar_data->fd, tf->header, 512) != 512)
831  {
832  /* if write didn't set errno, assume problem is no disk space */
833  if (errno == 0)
834  errno = ENOSPC;
835  return -1;
836  }
837  }
838 #ifdef HAVE_LIBZ
839  else
840  {
841  /* Turn off compression */
842  if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
843  {
844  tar_set_error("could not change compression parameters");
845  return -1;
846  }
847 
848  /* Overwrite the header, assuming the size will be the same */
849  if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true))
850  return -1;
851 
852  /* Turn compression back on */
853  if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK)
854  {
855  tar_set_error("could not change compression parameters");
856  return -1;
857  }
858  }
859 #endif
860 
861  /* Move file pointer back down to end, so we can write the next file */
862  if (lseek(tar_data->fd, 0, SEEK_END) < 0)
863  return -1;
864 
865  /* Always fsync on close, so the padding gets fsynced */
866  if (tar_sync(f) < 0)
867  exit(1);
868 
869  /* Clean up and done */
870  pg_free(tf->pathname);
871  pg_free(tf);
872  tar_data->currentfile = NULL;
873 
874  return 0;
875 }
#define write(a, b, c)
Definition: win32.h:14
char * pathname
Definition: walmethods.c:391
#define MemSet(start, val, len)
Definition: c.h:955
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:481
size_t pad_to_size
Definition: walmethods.c:392
char header[512]
Definition: walmethods.c:390
#define tar_set_error(msg)
Definition: walmethods.c:411
static bool tar_write_padding_data(TarMethodFile *f, size_t bytes)
Definition: walmethods.c:512
static off_t tar_get_current_pos(Walfile f)
Definition: walmethods.c:690
int tarChecksum(char *header)
Definition: tar.c:88
TarMethodFile * currentfile
Definition: walmethods.c:401
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
#define Assert(condition)
Definition: c.h:732
void print_tar_number(char *s, int len, uint64 val)
Definition: tar.c:20
#define tar_clear_error()
Definition: walmethods.c:410
void pg_free(void *ptr)
Definition: fe_memutils.c:105
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:210
off_t ofs_start
Definition: walmethods.c:388
static TarMethodData * tar_data
Definition: walmethods.c:408
static int tar_sync(Walfile f)
Definition: walmethods.c:699
#define ftruncate(a, b)
Definition: win32_port.h:60

◆ tar_existsfile()

static bool tar_existsfile ( const char *  pathname)
static

Definition at line 878 of file walmethods.c.

References tar_clear_error.

Referenced by CreateWalTarMethod().

879 {
880  tar_clear_error();
881  /* We only deal with new tarfiles, so nothing externally created exists */
882  return false;
883 }
#define tar_clear_error()
Definition: walmethods.c:410

◆ tar_finish()

static bool tar_finish ( void  )
static

Definition at line 886 of file walmethods.c.

References close, CLOSE_NORMAL, TarMethodData::compression, TarMethodData::currentfile, TarMethodData::fd, fsync, fsync_fname(), fsync_parent_path(), MemSet, TarMethodData::sync, tar_clear_error, tar_close(), tar_set_error, TarMethodData::tarfilename, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

887 {
888  char zerobuf[1024];
889 
890  tar_clear_error();
891 
892  if (tar_data->currentfile)
893  {
895  return false;
896  }
897 
898  /* A tarfile always ends with two empty blocks */
899  MemSet(zerobuf, 0, sizeof(zerobuf));
900  if (!tar_data->compression)
901  {
902  errno = 0;
903  if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf))
904  {
905  /* if write didn't set errno, assume problem is no disk space */
906  if (errno == 0)
907  errno = ENOSPC;
908  return false;
909  }
910  }
911 #ifdef HAVE_LIBZ
912  else
913  {
914  if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false))
915  return false;
916 
917  /* Also flush all data to make sure the gzip stream is finished */
918  tar_data->zp->next_in = NULL;
919  tar_data->zp->avail_in = 0;
920  while (true)
921  {
922  int r;
923 
924  r = deflate(tar_data->zp, Z_FINISH);
925 
926  if (r == Z_STREAM_ERROR)
927  {
928  tar_set_error("could not compress data");
929  return false;
930  }
931  if (tar_data->zp->avail_out < ZLIB_OUT_SIZE)
932  {
933  size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out;
934 
935  errno = 0;
936  if (write(tar_data->fd, tar_data->zlibOut, len) != len)
937  {
938  /*
939  * If write didn't set errno, assume problem is no disk
940  * space.
941  */
942  if (errno == 0)
943  errno = ENOSPC;
944  return false;
945  }
946  }
947  if (r == Z_STREAM_END)
948  break;
949  }
950 
951  if (deflateEnd(tar_data->zp) != Z_OK)
952  {
953  tar_set_error("could not close compression stream");
954  return false;
955  }
956  }
957 #endif
958 
959  /* sync the empty blocks as well, since they're after the last file */
960  if (tar_data->sync)
961  {
962  if (fsync(tar_data->fd) != 0)
963  return false;
964  }
965 
966  if (close(tar_data->fd) != 0)
967  return false;
968 
969  tar_data->fd = -1;
970 
971  if (tar_data->sync)
972  {
973  if (fsync_fname(tar_data->tarfilename, false) != 0)
974  return false;
976  return false;
977  }
978 
979  return true;
980 }
#define write(a, b, c)
Definition: win32.h:14
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:580
#define MemSet(start, val, len)
Definition: c.h:955
static int tar_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:718
#define fsync(fd)
Definition: win32_port.h:63
#define tar_set_error(msg)
Definition: walmethods.c:411
char * tarfilename
Definition: walmethods.c:397
TarMethodFile * currentfile
Definition: walmethods.c:401
#define tar_clear_error()
Definition: walmethods.c:410
#define ZLIB_OUT_SIZE
Definition: walmethods.c:32
#define close(a)
Definition: win32.h:12
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3419
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_get_current_pos()

static off_t tar_get_current_pos ( Walfile  f)
static

Definition at line 690 of file walmethods.c.

References Assert, and tar_clear_error.

Referenced by CreateWalTarMethod(), and tar_close().

691 {
692  Assert(f != NULL);
693  tar_clear_error();
694 
695  return ((TarMethodFile *) f)->currpos;
696 }
#define Assert(condition)
Definition: c.h:732
#define tar_clear_error()
Definition: walmethods.c:410

◆ tar_get_file_size()

static ssize_t tar_get_file_size ( const char *  pathname)
static

Definition at line 680 of file walmethods.c.

References tar_clear_error.

Referenced by CreateWalTarMethod().

681 {
682  tar_clear_error();
683 
684  /* Currently not used, so not supported */
685  errno = ENOSYS;
686  return -1;
687 }
#define tar_clear_error()
Definition: walmethods.c:410

◆ tar_getlasterror()

static const char* tar_getlasterror ( void  )
static

Definition at line 414 of file walmethods.c.

References buf, TarMethodData::fd, TarMethodData::lasterror, strerror, tar_set_error, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

415 {
416  /*
417  * If a custom error is set, return that one. Otherwise, assume errno is
418  * set and return that one.
419  */
420  if (tar_data->lasterror[0])
421  return tar_data->lasterror;
422  return strerror(errno);
423 }
#define strerror
Definition: port.h:205
char lasterror[1024]
Definition: walmethods.c:402
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_open_for_write()

static Walfile tar_open_for_write ( const char *  pathname,
const char *  temp_suffix,
size_t  pad_to_size 
)
static

Definition at line 532 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::currentfile, TarMethodFile::currpos, TarMethodData::fd, TarMethodFile::header, MAXPGPATH, TarMethodFile::ofs_start, TarMethodFile::pad_to_size, TarMethodFile::pathname, PG_BINARY, pg_file_create_mode, pg_free(), pg_malloc(), pg_malloc0(), pg_strdup(), S_IRUSR, S_IWUSR, snprintf, tar_clear_error, TAR_OK, tar_set_error, tar_write_padding_data(), tarCreateHeader(), TarMethodData::tarfilename, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

533 {
534  int save_errno;
535  static char tmppath[MAXPGPATH];
536 
537  tar_clear_error();
538 
539  if (tar_data->fd < 0)
540  {
541  /*
542  * We open the tar file only when we first try to write to it.
543  */
544  tar_data->fd = open(tar_data->tarfilename,
545  O_WRONLY | O_CREAT | PG_BINARY,
547  if (tar_data->fd < 0)
548  return NULL;
549 
550 #ifdef HAVE_LIBZ
551  if (tar_data->compression)
552  {
553  tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream));
554  tar_data->zp->zalloc = Z_NULL;
555  tar_data->zp->zfree = Z_NULL;
556  tar_data->zp->opaque = Z_NULL;
557  tar_data->zp->next_out = tar_data->zlibOut;
558  tar_data->zp->avail_out = ZLIB_OUT_SIZE;
559 
560  /*
561  * Initialize deflation library. Adding the magic value 16 to the
562  * default 15 for the windowBits parameter makes the output be
563  * gzip instead of zlib.
564  */
565  if (deflateInit2(tar_data->zp, tar_data->compression, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK)
566  {
567  pg_free(tar_data->zp);
568  tar_data->zp = NULL;
569  tar_set_error("could not initialize compression library");
570  return NULL;
571  }
572  }
573 #endif
574 
575  /* There's no tar header itself, the file starts with regular files */
576  }
577 
578  Assert(tar_data->currentfile == NULL);
579  if (tar_data->currentfile != NULL)
580  {
581  tar_set_error("implementation error: tar files can't have more than one open file");
582  return NULL;
583  }
584 
586 
587  snprintf(tmppath, sizeof(tmppath), "%s%s",
588  pathname, temp_suffix ? temp_suffix : "");
589 
590  /* Create a header with size set to 0 - we will fill out the size on close */
591  if (tarCreateHeader(tar_data->currentfile->header, tmppath, NULL, 0, S_IRUSR | S_IWUSR, 0, 0, time(NULL)) != TAR_OK)
592  {
594  tar_data->currentfile = NULL;
595  tar_set_error("could not create tar header");
596  return NULL;
597  }
598 
599 #ifdef HAVE_LIBZ
600  if (tar_data->compression)
601  {
602  /* Flush existing data */
603  if (!tar_write_compressed_data(NULL, 0, true))
604  return NULL;
605 
606  /* Turn off compression for header */
607  if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
608  {
609  tar_set_error("could not change compression parameters");
610  return NULL;
611  }
612  }
613 #endif
614 
615  tar_data->currentfile->ofs_start = lseek(tar_data->fd, 0, SEEK_CUR);
616  if (tar_data->currentfile->ofs_start == -1)
617  {
618  save_errno = errno;
620  tar_data->currentfile = NULL;
621  errno = save_errno;
622  return NULL;
623  }
625 
626  if (!tar_data->compression)
627  {
628  errno = 0;
629  if (write(tar_data->fd, tar_data->currentfile->header, 512) != 512)
630  {
631  save_errno = errno;
633  tar_data->currentfile = NULL;
634  /* if write didn't set errno, assume problem is no disk space */
635  errno = save_errno ? save_errno : ENOSPC;
636  return NULL;
637  }
638  }
639 #ifdef HAVE_LIBZ
640  else
641  {
642  /* Write header through the zlib APIs but with no compression */
643  if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true))
644  return NULL;
645 
646  /* Re-enable compression for the rest of the file */
647  if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK)
648  {
649  tar_set_error("could not change compression parameters");
650  return NULL;
651  }
652  }
653 #endif
654 
655  tar_data->currentfile->pathname = pg_strdup(pathname);
656 
657  /*
658  * Uncompressed files are padded on creation, but for compression we can't
659  * do that
660  */
661  if (pad_to_size)
662  {
663  tar_data->currentfile->pad_to_size = pad_to_size;
664  if (!tar_data->compression)
665  {
666  /* Uncompressed, so pad now */
668  /* Seek back to start */
669  if (lseek(tar_data->fd, tar_data->currentfile->ofs_start + 512, SEEK_SET) != tar_data->currentfile->ofs_start + 512)
670  return NULL;
671 
673  }
674  }
675 
676  return tar_data->currentfile;
677 }
int pg_file_create_mode
Definition: file_perm.c:19
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define write(a, b, c)
Definition: win32.h:14
char * pathname
Definition: walmethods.c:391
Definition: pgtar.h:17
#define PG_BINARY
Definition: c.h:1191
size_t pad_to_size
Definition: walmethods.c:392
char header[512]
Definition: walmethods.c:390
#define S_IWUSR
Definition: win32_port.h:263
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
#define MAXPGPATH
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
z_stream * z_streamp
#define tar_set_error(msg)
Definition: walmethods.c:411
static bool tar_write_padding_data(TarMethodFile *f, size_t bytes)
Definition: walmethods.c:512
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
char * tarfilename
Definition: walmethods.c:397
TarMethodFile * currentfile
Definition: walmethods.c:401
#define Assert(condition)
Definition: c.h:732
#define tar_clear_error()
Definition: walmethods.c:410
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define ZLIB_OUT_SIZE
Definition: walmethods.c:32
#define S_IRUSR
Definition: win32_port.h:260
#define snprintf
Definition: port.h:192
off_t ofs_start
Definition: walmethods.c:388
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_sync()

static int tar_sync ( Walfile  f)
static

Definition at line 699 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::fd, fsync, TarMethodData::sync, and tar_clear_error.

Referenced by CreateWalTarMethod(), and tar_close().

700 {
701  Assert(f != NULL);
702  tar_clear_error();
703 
704  if (!tar_data->sync)
705  return 0;
706 
707  /*
708  * Always sync the whole tarfile, because that's all we can do. This makes
709  * no sense on compressed files, so just ignore those.
710  */
711  if (tar_data->compression)
712  return 0;
713 
714  return fsync(tar_data->fd);
715 }
#define fsync(fd)
Definition: win32_port.h:63
#define Assert(condition)
Definition: c.h:732
#define tar_clear_error()
Definition: walmethods.c:410
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_write()

static ssize_t tar_write ( Walfile  f,
const void *  buf,
size_t  count 
)
static

Definition at line 481 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::fd, tar_clear_error, unconstify, and write.

Referenced by CreateWalTarMethod(), tar_close(), and tar_write_padding_data().

482 {
483  ssize_t r;
484 
485  Assert(f != NULL);
486  tar_clear_error();
487 
488  /* Tarfile will always be positioned at the end */
489  if (!tar_data->compression)
490  {
491  r = write(tar_data->fd, buf, count);
492  if (r > 0)
493  ((TarMethodFile *) f)->currpos += r;
494  return r;
495  }
496 #ifdef HAVE_LIBZ
497  else
498  {
499  if (!tar_write_compressed_data(unconstify(void *, buf), count, false))
500  return -1;
501  ((TarMethodFile *) f)->currpos += count;
502  return count;
503  }
504 #else
505  else
506  /* Can't happen - compression enabled with no libz */
507  return -1;
508 #endif
509 }
#define write(a, b, c)
Definition: win32.h:14
static char * buf
Definition: pg_test_fsync.c:68
#define unconstify(underlying_type, expr)
Definition: c.h:1163
#define Assert(condition)
Definition: c.h:732
#define tar_clear_error()
Definition: walmethods.c:410
static TarMethodData * tar_data
Definition: walmethods.c:408

◆ tar_write_padding_data()

static bool tar_write_padding_data ( TarMethodFile f,
size_t  bytes 
)
static

Definition at line 512 of file walmethods.c.

References generate_unaccent_rules::bytes(), PGAlignedXLogBlock::data, Min, and tar_write().

Referenced by tar_close(), and tar_open_for_write().

513 {
514  PGAlignedXLogBlock zerobuf;
515  size_t bytesleft = bytes;
516 
517  memset(zerobuf.data, 0, XLOG_BLCKSZ);
518  while (bytesleft)
519  {
520  size_t bytestowrite = Min(bytesleft, XLOG_BLCKSZ);
521  ssize_t r = tar_write(f, zerobuf.data, bytestowrite);
522 
523  if (r < 0)
524  return false;
525  bytesleft -= r;
526  }
527 
528  return true;
529 }
def bytes(source, encoding='ascii', errors='strict')
#define Min(x, y)
Definition: c.h:904
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:481
char data[XLOG_BLCKSZ]
Definition: c.h:1068

Variable Documentation

◆ dir_data

DirectoryMethodData* dir_data = NULL
static

Definition at line 48 of file walmethods.c.

◆ tar_data

TarMethodData* tar_data = NULL
static

Definition at line 408 of file walmethods.c.