PostgreSQL Source Code  git master
walmethods.c File Reference
#include "postgres_fe.h"
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "pgtar.h"
#include "receivelog.h"
#include "streamutil.h"
Include dependency graph for walmethods.c:

Go to the source code of this file.

Data Structures

struct  DirectoryMethodData
 
struct  DirectoryMethodFile
 
struct  TarMethodFile
 
struct  TarMethodData
 

Macros

#define ZLIB_OUT_SIZE   4096
 
#define tar_clear_error()   tar_data->lasterror[0] = '\0'
 
#define tar_set_error(msg)   strlcpy(tar_data->lasterror, _(msg), sizeof(tar_data->lasterror))
 

Typedefs

typedef struct DirectoryMethodData DirectoryMethodData
 
typedef struct DirectoryMethodFile DirectoryMethodFile
 
typedef struct TarMethodFile TarMethodFile
 
typedef struct TarMethodData TarMethodData
 

Functions

static const char * dir_getlasterror (void)
 
static Walfile dir_open_for_write (const char *pathname, const char *temp_suffix, size_t pad_to_size)
 
static ssize_t dir_write (Walfile f, const void *buf, size_t count)
 
static off_t dir_get_current_pos (Walfile f)
 
static int dir_close (Walfile f, WalCloseMethod method)
 
static int dir_sync (Walfile f)
 
static ssize_t dir_get_file_size (const char *pathname)
 
static bool dir_existsfile (const char *pathname)
 
static bool dir_finish (void)
 
WalWriteMethodCreateWalDirectoryMethod (const char *basedir, int compression, bool sync)
 
void FreeWalDirectoryMethod (void)
 
static const char * tar_getlasterror (void)
 
static ssize_t tar_write (Walfile f, const void *buf, size_t count)
 
static bool tar_write_padding_data (TarMethodFile *f, size_t bytes)
 
static Walfile tar_open_for_write (const char *pathname, const char *temp_suffix, size_t pad_to_size)
 
static ssize_t tar_get_file_size (const char *pathname)
 
static off_t tar_get_current_pos (Walfile f)
 
static int tar_sync (Walfile f)
 
static int tar_close (Walfile f, WalCloseMethod method)
 
static bool tar_existsfile (const char *pathname)
 
static bool tar_finish (void)
 
WalWriteMethodCreateWalTarMethod (const char *tarbase, int compression, bool sync)
 
void FreeWalTarMethod (void)
 

Variables

static DirectoryMethodDatadir_data = NULL
 
static TarMethodDatatar_data = NULL
 

Macro Definition Documentation

◆ tar_clear_error

#define tar_clear_error ( )    tar_data->lasterror[0] = '\0'

◆ tar_set_error

#define tar_set_error (   msg)    strlcpy(tar_data->lasterror, _(msg), sizeof(tar_data->lasterror))

Definition at line 410 of file walmethods.c.

Referenced by tar_close(), tar_finish(), tar_getlasterror(), and tar_open_for_write().

◆ ZLIB_OUT_SIZE

#define ZLIB_OUT_SIZE   4096

Typedef Documentation

◆ DirectoryMethodData

◆ DirectoryMethodFile

◆ TarMethodData

typedef struct TarMethodData TarMethodData

◆ TarMethodFile

typedef struct TarMethodFile TarMethodFile

Function Documentation

◆ CreateWalDirectoryMethod()

WalWriteMethod* CreateWalDirectoryMethod ( const char *  basedir,
int  compression,
bool  sync 
)

Definition at line 349 of file walmethods.c.

References DirectoryMethodData::basedir, WalWriteMethod::close, DirectoryMethodData::compression, dir_close(), dir_existsfile(), dir_finish(), dir_get_current_pos(), dir_get_file_size(), dir_getlasterror(), dir_open_for_write(), dir_sync(), dir_write(), WalWriteMethod::existsfile, WalWriteMethod::finish, WalWriteMethod::get_current_pos, WalWriteMethod::get_file_size, WalWriteMethod::getlasterror, WalWriteMethod::open_for_write, pg_malloc0(), pg_strdup(), DirectoryMethodData::sync, WalWriteMethod::sync, and WalWriteMethod::write.

Referenced by LogStreamerMain(), and StreamLog().

350 {
351  WalWriteMethod *method;
352 
353  method = pg_malloc0(sizeof(WalWriteMethod));
355  method->write = dir_write;
358  method->close = dir_close;
359  method->sync = dir_sync;
360  method->existsfile = dir_existsfile;
361  method->finish = dir_finish;
362  method->getlasterror = dir_getlasterror;
363 
365  dir_data->compression = compression;
367  dir_data->sync = sync;
368 
369  return method;
370 }
static off_t dir_get_current_pos(Walfile f)
Definition: walmethods.c:205
int(* close)(Walfile f, WalCloseMethod method)
Definition: walmethods.h:47
int(* sync)(Walfile f)
Definition: walmethods.h:67
static char * basedir
const char *(* getlasterror)(void)
Definition: walmethods.h:78
off_t(* get_current_pos)(Walfile f)
Definition: walmethods.h:62
static bool dir_finish(void)
Definition: walmethods.c:333
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
ssize_t(* write)(Walfile f, const void *buf, size_t count)
Definition: walmethods.h:59
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static ssize_t dir_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:186
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
static int dir_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:214
bool(* finish)(void)
Definition: walmethods.h:75
bool(* existsfile)(const char *pathname)
Definition: walmethods.h:50
Walfile(* open_for_write)(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.h:41
static Walfile dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.c:72
ssize_t(* get_file_size)(const char *pathname)
Definition: walmethods.h:53
static const char * dir_getlasterror(void)
Definition: walmethods.c:65
static ssize_t dir_get_file_size(const char *pathname)
Definition: walmethods.c:302
static bool dir_existsfile(const char *pathname)
Definition: walmethods.c:317
static int dir_sync(Walfile f)
Definition: walmethods.c:283

◆ CreateWalTarMethod()

WalWriteMethod* CreateWalTarMethod ( const char *  tarbase,
int  compression,
bool  sync 
)

Definition at line 987 of file walmethods.c.

References WalWriteMethod::close, DirectoryMethodData::compression, TarMethodData::compression, WalWriteMethod::existsfile, TarMethodData::fd, WalWriteMethod::finish, WalWriteMethod::get_current_pos, WalWriteMethod::get_file_size, WalWriteMethod::getlasterror, WalWriteMethod::open_for_write, pg_malloc(), pg_malloc0(), sprintf, DirectoryMethodData::sync, WalWriteMethod::sync, TarMethodData::sync, tar_close(), tar_existsfile(), tar_finish(), tar_get_current_pos(), tar_get_file_size(), tar_getlasterror(), tar_open_for_write(), tar_sync(), tar_write(), TarMethodData::tarfilename, WalWriteMethod::write, and ZLIB_OUT_SIZE.

Referenced by LogStreamerMain().

988 {
989  WalWriteMethod *method;
990  const char *suffix = (compression != 0) ? ".tar.gz" : ".tar";
991 
992  method = pg_malloc0(sizeof(WalWriteMethod));
994  method->write = tar_write;
997  method->close = tar_close;
998  method->sync = tar_sync;
999  method->existsfile = tar_existsfile;
1000  method->finish = tar_finish;
1001  method->getlasterror = tar_getlasterror;
1002 
1003  tar_data = pg_malloc0(sizeof(TarMethodData));
1004  tar_data->tarfilename = pg_malloc0(strlen(tarbase) + strlen(suffix) + 1);
1005  sprintf(tar_data->tarfilename, "%s%s", tarbase, suffix);
1006  tar_data->fd = -1;
1007  tar_data->compression = compression;
1008  tar_data->sync = sync;
1009 #ifdef HAVE_LIBZ
1010  if (compression)
1011  tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
1012 #endif
1013 
1014  return method;
1015 }
int(* close)(Walfile f, WalCloseMethod method)
Definition: walmethods.h:47
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static bool tar_finish(void)
Definition: walmethods.c:890
int(* sync)(Walfile f)
Definition: walmethods.h:67
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:480
const char *(* getlasterror)(void)
Definition: walmethods.h:78
static int tar_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:721
off_t(* get_current_pos)(Walfile f)
Definition: walmethods.h:62
#define sprintf
Definition: port.h:195
static const char * tar_getlasterror(void)
Definition: walmethods.c:413
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
static Walfile tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.c:531
ssize_t(* write)(Walfile f, const void *buf, size_t count)
Definition: walmethods.h:59
static ssize_t tar_get_file_size(const char *pathname)
Definition: walmethods.c:683
static bool tar_existsfile(const char *pathname)
Definition: walmethods.c:882
static off_t tar_get_current_pos(Walfile f)
Definition: walmethods.c:693
char * tarfilename
Definition: walmethods.c:396
#define ZLIB_OUT_SIZE
Definition: walmethods.c:31
bool(* finish)(void)
Definition: walmethods.h:75
bool(* existsfile)(const char *pathname)
Definition: walmethods.h:50
Walfile(* open_for_write)(const char *pathname, const char *temp_suffix, size_t pad_to_size)
Definition: walmethods.h:41
ssize_t(* get_file_size)(const char *pathname)
Definition: walmethods.h:53
static TarMethodData * tar_data
Definition: walmethods.c:407
static int tar_sync(Walfile f)
Definition: walmethods.c:702

◆ dir_close()

static int dir_close ( Walfile  f,
WalCloseMethod  method 
)
static

Definition at line 214 of file walmethods.c.

References Assert, DirectoryMethodData::basedir, close, CLOSE_NORMAL, CLOSE_UNLINK, DirectoryMethodData::compression, durable_rename(), DirectoryMethodFile::fd, fsync_fname(), fsync_parent_path(), DirectoryMethodFile::fullpath, MAXPGPATH, DirectoryMethodFile::pathname, pg_free(), snprintf, DirectoryMethodData::sync, and DirectoryMethodFile::temp_suffix.

Referenced by CreateWalDirectoryMethod().

215 {
216  int r;
218  static char tmppath[MAXPGPATH];
219  static char tmppath2[MAXPGPATH];
220 
221  Assert(f != NULL);
222 
223 #ifdef HAVE_LIBZ
224  if (dir_data->compression > 0)
225  r = gzclose(df->gzfp);
226  else
227 #endif
228  r = close(df->fd);
229 
230  if (r == 0)
231  {
232  /* Build path to the current version of the file */
233  if (method == CLOSE_NORMAL && df->temp_suffix)
234  {
235  /*
236  * If we have a temp prefix, normal operation is to rename the
237  * file.
238  */
239  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
240  dir_data->basedir, df->pathname,
241  dir_data->compression > 0 ? ".gz" : "",
242  df->temp_suffix);
243  snprintf(tmppath2, sizeof(tmppath2), "%s/%s%s",
244  dir_data->basedir, df->pathname,
245  dir_data->compression > 0 ? ".gz" : "");
246  r = durable_rename(tmppath, tmppath2);
247  }
248  else if (method == CLOSE_UNLINK)
249  {
250  /* Unlink the file once it's closed */
251  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
252  dir_data->basedir, df->pathname,
253  dir_data->compression > 0 ? ".gz" : "",
254  df->temp_suffix ? df->temp_suffix : "");
255  r = unlink(tmppath);
256  }
257  else
258  {
259  /*
260  * Else either CLOSE_NORMAL and no temp suffix, or
261  * CLOSE_NO_RENAME. In this case, fsync the file and containing
262  * directory if sync mode is requested.
263  */
264  if (dir_data->sync)
265  {
266  r = fsync_fname(df->fullpath, false);
267  if (r == 0)
268  r = fsync_parent_path(df->fullpath);
269  }
270  }
271  }
272 
273  pg_free(df->pathname);
274  pg_free(df->fullpath);
275  if (df->temp_suffix)
276  pg_free(df->temp_suffix);
277  pg_free(df);
278 
279  return r;
280 }
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:632
#define MAXPGPATH
int durable_rename(const char *oldfile, const char *newfile, int elevel)
Definition: fd.c:658
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define Assert(condition)
Definition: c.h:745
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:193
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3542

◆ dir_existsfile()

static bool dir_existsfile ( const char *  pathname)
static

Definition at line 317 of file walmethods.c.

References DirectoryMethodData::basedir, close, fd(), MAXPGPATH, PG_BINARY, and snprintf.

Referenced by CreateWalDirectoryMethod().

318 {
319  static char tmppath[MAXPGPATH];
320  int fd;
321 
322  snprintf(tmppath, sizeof(tmppath), "%s/%s",
323  dir_data->basedir, pathname);
324 
325  fd = open(tmppath, O_RDONLY | PG_BINARY, 0);
326  if (fd < 0)
327  return false;
328  close(fd);
329  return true;
330 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1240
#define MAXPGPATH
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:193

◆ dir_finish()

static bool dir_finish ( void  )
static

Definition at line 333 of file walmethods.c.

References DirectoryMethodData::basedir, fsync_fname(), and DirectoryMethodData::sync.

Referenced by CreateWalDirectoryMethod().

334 {
335  if (dir_data->sync)
336  {
337  /*
338  * Files are fsynced when they are closed, but we need to fsync the
339  * directory entry here as well.
340  */
341  if (fsync_fname(dir_data->basedir, true) != 0)
342  return false;
343  }
344  return true;
345 }
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:632
static DirectoryMethodData * dir_data
Definition: walmethods.c:47

◆ dir_get_current_pos()

static off_t dir_get_current_pos ( Walfile  f)
static

Definition at line 205 of file walmethods.c.

References Assert.

Referenced by CreateWalDirectoryMethod().

206 {
207  Assert(f != NULL);
208 
209  /* Use a cached value to prevent lots of reseeks */
210  return ((DirectoryMethodFile *) f)->currpos;
211 }
#define Assert(condition)
Definition: c.h:745

◆ dir_get_file_size()

static ssize_t dir_get_file_size ( const char *  pathname)
static

Definition at line 302 of file walmethods.c.

References DirectoryMethodData::basedir, MAXPGPATH, snprintf, and stat.

Referenced by CreateWalDirectoryMethod().

303 {
304  struct stat statbuf;
305  static char tmppath[MAXPGPATH];
306 
307  snprintf(tmppath, sizeof(tmppath), "%s/%s",
308  dir_data->basedir, pathname);
309 
310  if (stat(tmppath, &statbuf) != 0)
311  return -1;
312 
313  return statbuf.st_size;
314 }
#define MAXPGPATH
#define stat(a, b)
Definition: win32_port.h:255
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define snprintf
Definition: port.h:193

◆ dir_getlasterror()

static const char* dir_getlasterror ( void  )
static

Definition at line 65 of file walmethods.c.

References strerror.

Referenced by CreateWalDirectoryMethod().

66 {
67  /* Directory method always sets errno, so just use strerror */
68  return strerror(errno);
69 }
#define strerror
Definition: port.h:206

◆ dir_open_for_write()

static Walfile dir_open_for_write ( const char *  pathname,
const char *  temp_suffix,
size_t  pad_to_size 
)
static

Definition at line 72 of file walmethods.c.

References DirectoryMethodData::basedir, generate_unaccent_rules::bytes(), close, DirectoryMethodData::compression, DirectoryMethodFile::currpos, PGAlignedXLogBlock::data, DirectoryMethodFile::fd, fd(), fsync_fname(), fsync_parent_path(), DirectoryMethodFile::fullpath, MAXPGPATH, DirectoryMethodFile::pathname, PG_BINARY, pg_file_create_mode, pg_malloc0(), pg_strdup(), snprintf, DirectoryMethodData::sync, DirectoryMethodFile::temp_suffix, and write.

Referenced by CreateWalDirectoryMethod().

73 {
74  static char tmppath[MAXPGPATH];
75  int fd;
77 #ifdef HAVE_LIBZ
78  gzFile gzfp = NULL;
79 #endif
80 
81  snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s",
82  dir_data->basedir, pathname,
83  dir_data->compression > 0 ? ".gz" : "",
84  temp_suffix ? temp_suffix : "");
85 
86  /*
87  * Open a file for non-compressed as well as compressed files. Tracking
88  * the file descriptor is important for dir_sync() method as gzflush()
89  * does not do any system calls to fsync() to make changes permanent on
90  * disk.
91  */
92  fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, pg_file_create_mode);
93  if (fd < 0)
94  return NULL;
95 
96 #ifdef HAVE_LIBZ
97  if (dir_data->compression > 0)
98  {
99  gzfp = gzdopen(fd, "wb");
100  if (gzfp == NULL)
101  {
102  close(fd);
103  return NULL;
104  }
105 
106  if (gzsetparams(gzfp, dir_data->compression,
107  Z_DEFAULT_STRATEGY) != Z_OK)
108  {
109  gzclose(gzfp);
110  return NULL;
111  }
112  }
113 #endif
114 
115  /* Do pre-padding on non-compressed files */
116  if (pad_to_size && dir_data->compression == 0)
117  {
118  PGAlignedXLogBlock zerobuf;
119  int bytes;
120 
121  memset(zerobuf.data, 0, XLOG_BLCKSZ);
122  for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ)
123  {
124  errno = 0;
125  if (write(fd, zerobuf.data, XLOG_BLCKSZ) != XLOG_BLCKSZ)
126  {
127  int save_errno = errno;
128 
129  close(fd);
130 
131  /*
132  * If write didn't set errno, assume problem is no disk space.
133  */
134  errno = save_errno ? save_errno : ENOSPC;
135  return NULL;
136  }
137  }
138 
139  if (lseek(fd, 0, SEEK_SET) != 0)
140  {
141  int save_errno = errno;
142 
143  close(fd);
144  errno = save_errno;
145  return NULL;
146  }
147  }
148 
149  /*
150  * fsync WAL file and containing directory, to ensure the file is
151  * persistently created and zeroed (if padded). That's particularly
152  * important when using synchronous mode, where the file is modified and
153  * fsynced in-place, without a directory fsync.
154  */
155  if (dir_data->sync)
156  {
157  if (fsync_fname(tmppath, false) != 0 ||
158  fsync_parent_path(tmppath) != 0)
159  {
160 #ifdef HAVE_LIBZ
161  if (dir_data->compression > 0)
162  gzclose(gzfp);
163  else
164 #endif
165  close(fd);
166  return NULL;
167  }
168  }
169 
170  f = pg_malloc0(sizeof(DirectoryMethodFile));
171 #ifdef HAVE_LIBZ
172  if (dir_data->compression > 0)
173  f->gzfp = gzfp;
174 #endif
175  f->fd = fd;
176  f->currpos = 0;
177  f->pathname = pg_strdup(pathname);
178  f->fullpath = pg_strdup(tmppath);
179  if (temp_suffix)
180  f->temp_suffix = pg_strdup(temp_suffix);
181 
182  return f;
183 }
int pg_file_create_mode
Definition: file_perm.c:19
#define write(a, b, c)
Definition: win32.h:14
def bytes(source, encoding='ascii', errors='strict')
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:632
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1240
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
#define MAXPGPATH
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define close(a)
Definition: win32.h:12
char data[XLOG_BLCKSZ]
Definition: c.h:1119
#define snprintf
Definition: port.h:193
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3542

◆ dir_sync()

static int dir_sync ( Walfile  f)
static

Definition at line 283 of file walmethods.c.

References Assert, DirectoryMethodData::compression, fd(), fsync, and DirectoryMethodData::sync.

Referenced by CreateWalDirectoryMethod().

284 {
285  Assert(f != NULL);
286 
287  if (!dir_data->sync)
288  return 0;
289 
290 #ifdef HAVE_LIBZ
291  if (dir_data->compression > 0)
292  {
293  if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK)
294  return -1;
295  }
296 #endif
297 
298  return fsync(((DirectoryMethodFile *) f)->fd);
299 }
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define fsync(fd)
Definition: win32_port.h:62
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define Assert(condition)
Definition: c.h:745

◆ dir_write()

static ssize_t dir_write ( Walfile  f,
const void *  buf,
size_t  count 
)
static

Definition at line 186 of file walmethods.c.

References Assert, DirectoryMethodData::compression, DirectoryMethodFile::currpos, DirectoryMethodFile::fd, and write.

Referenced by CreateWalDirectoryMethod().

187 {
188  ssize_t r;
190 
191  Assert(f != NULL);
192 
193 #ifdef HAVE_LIBZ
194  if (dir_data->compression > 0)
195  r = (ssize_t) gzwrite(df->gzfp, buf, count);
196  else
197 #endif
198  r = write(df->fd, buf, count);
199  if (r > 0)
200  df->currpos += r;
201  return r;
202 }
#define write(a, b, c)
Definition: win32.h:14
static char * buf
Definition: pg_test_fsync.c:67
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
#define Assert(condition)
Definition: c.h:745

◆ FreeWalDirectoryMethod()

void FreeWalDirectoryMethod ( void  )

Definition at line 373 of file walmethods.c.

References DirectoryMethodData::basedir, and pg_free().

Referenced by LogStreamerMain(), and StreamLog().

374 {
376  pg_free(dir_data);
377 }
static DirectoryMethodData * dir_data
Definition: walmethods.c:47
void pg_free(void *ptr)
Definition: fe_memutils.c:105

◆ FreeWalTarMethod()

void FreeWalTarMethod ( void  )

Definition at line 1018 of file walmethods.c.

References TarMethodData::compression, pg_free(), and TarMethodData::tarfilename.

Referenced by LogStreamerMain().

1019 {
1021 #ifdef HAVE_LIBZ
1022  if (tar_data->compression)
1023  pg_free(tar_data->zlibOut);
1024 #endif
1025  pg_free(tar_data);
1026 }
char * tarfilename
Definition: walmethods.c:396
void pg_free(void *ptr)
Definition: fe_memutils.c:105
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_close()

static int tar_close ( Walfile  f,
WalCloseMethod  method 
)
static

Definition at line 721 of file walmethods.c.

References Assert, CLOSE_NORMAL, CLOSE_UNLINK, TarMethodData::compression, TarMethodData::currentfile, TarMethodFile::currpos, TarMethodData::fd, ftruncate, header(), TarMethodFile::header, MemSet, TarMethodFile::ofs_start, TarMethodFile::pad_to_size, TarMethodFile::pathname, pg_free(), print_tar_number(), strlcpy(), TAR_BLOCK_SIZE, tar_clear_error, tar_get_current_pos(), tar_set_error, tar_sync(), tar_write(), tar_write_padding_data(), tarChecksum(), tarPaddingBytesRequired(), and write.

Referenced by CreateWalTarMethod(), and tar_finish().

722 {
723  ssize_t filesize;
724  int padding;
725  TarMethodFile *tf = (TarMethodFile *) f;
726 
727  Assert(f != NULL);
728  tar_clear_error();
729 
730  if (method == CLOSE_UNLINK)
731  {
732  if (tar_data->compression)
733  {
734  tar_set_error("unlink not supported with compression");
735  return -1;
736  }
737 
738  /*
739  * Unlink the file that we just wrote to the tar. We do this by
740  * truncating it to the start of the header. This is safe as we only
741  * allow writing of the very last file.
742  */
743  if (ftruncate(tar_data->fd, tf->ofs_start) != 0)
744  return -1;
745 
746  pg_free(tf->pathname);
747  pg_free(tf);
748  tar_data->currentfile = NULL;
749 
750  return 0;
751  }
752 
753  /*
754  * Pad the file itself with zeroes if necessary. Note that this is
755  * different from the tar format padding -- this is the padding we asked
756  * for when the file was opened.
757  */
758  if (tf->pad_to_size)
759  {
760  if (tar_data->compression)
761  {
762  /*
763  * A compressed tarfile is padded on close since we cannot know
764  * the size of the compressed output until the end.
765  */
766  size_t sizeleft = tf->pad_to_size - tf->currpos;
767 
768  if (sizeleft)
769  {
770  if (!tar_write_padding_data(tf, sizeleft))
771  return -1;
772  }
773  }
774  else
775  {
776  /*
777  * An uncompressed tarfile was padded on creation, so just adjust
778  * the current position as if we seeked to the end.
779  */
780  tf->currpos = tf->pad_to_size;
781  }
782  }
783 
784  /*
785  * Get the size of the file, and pad out to a multiple of the tar block
786  * size.
787  */
788  filesize = tar_get_current_pos(f);
789  padding = tarPaddingBytesRequired(filesize);
790  if (padding)
791  {
792  char zerobuf[TAR_BLOCK_SIZE];
793 
794  MemSet(zerobuf, 0, padding);
795  if (tar_write(f, zerobuf, padding) != padding)
796  return -1;
797  }
798 
799 
800 #ifdef HAVE_LIBZ
801  if (tar_data->compression)
802  {
803  /* Flush the current buffer */
804  if (!tar_write_compressed_data(NULL, 0, true))
805  {
806  errno = EINVAL;
807  return -1;
808  }
809  }
810 #endif
811 
812  /*
813  * Now go back and update the header with the correct filesize and
814  * possibly also renaming the file. We overwrite the entire current header
815  * when done, including the checksum.
816  */
817  print_tar_number(&(tf->header[124]), 12, filesize);
818 
819  if (method == CLOSE_NORMAL)
820 
821  /*
822  * We overwrite it with what it was before if we have no tempname,
823  * since we're going to write the buffer anyway.
824  */
825  strlcpy(&(tf->header[0]), tf->pathname, 100);
826 
827  print_tar_number(&(tf->header[148]), 8, tarChecksum(((TarMethodFile *) f)->header));
828  if (lseek(tar_data->fd, tf->ofs_start, SEEK_SET) != ((TarMethodFile *) f)->ofs_start)
829  return -1;
830  if (!tar_data->compression)
831  {
832  errno = 0;
834  {
835  /* if write didn't set errno, assume problem is no disk space */
836  if (errno == 0)
837  errno = ENOSPC;
838  return -1;
839  }
840  }
841 #ifdef HAVE_LIBZ
842  else
843  {
844  /* Turn off compression */
845  if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
846  {
847  tar_set_error("could not change compression parameters");
848  return -1;
849  }
850 
851  /* Overwrite the header, assuming the size will be the same */
852  if (!tar_write_compressed_data(tar_data->currentfile->header,
853  TAR_BLOCK_SIZE, true))
854  return -1;
855 
856  /* Turn compression back on */
857  if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK)
858  {
859  tar_set_error("could not change compression parameters");
860  return -1;
861  }
862  }
863 #endif
864 
865  /* Move file pointer back down to end, so we can write the next file */
866  if (lseek(tar_data->fd, 0, SEEK_END) < 0)
867  return -1;
868 
869  /* Always fsync on close, so the padding gets fsynced */
870  if (tar_sync(f) < 0)
871  exit(1);
872 
873  /* Clean up and done */
874  pg_free(tf->pathname);
875  pg_free(tf);
876  tar_data->currentfile = NULL;
877 
878  return 0;
879 }
#define write(a, b, c)
Definition: win32.h:14
char * pathname
Definition: walmethods.c:390
#define MemSet(start, val, len)
Definition: c.h:978
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:480
size_t pad_to_size
Definition: walmethods.c:391
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define tar_set_error(msg)
Definition: walmethods.c:410
static bool tar_write_padding_data(TarMethodFile *f, size_t bytes)
Definition: walmethods.c:511
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
static off_t tar_get_current_pos(Walfile f)
Definition: walmethods.c:693
int tarChecksum(char *header)
Definition: tar.c:90
TarMethodFile * currentfile
Definition: walmethods.c:400
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
#define Assert(condition)
Definition: c.h:745
char header[TAR_BLOCK_SIZE]
Definition: walmethods.c:389
void print_tar_number(char *s, int len, uint64 val)
Definition: tar.c:22
#define tar_clear_error()
Definition: walmethods.c:409
void pg_free(void *ptr)
Definition: fe_memutils.c:105
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:208
off_t ofs_start
Definition: walmethods.c:387
static TarMethodData * tar_data
Definition: walmethods.c:407
static int tar_sync(Walfile f)
Definition: walmethods.c:702
#define ftruncate(a, b)
Definition: win32_port.h:59

◆ tar_existsfile()

static bool tar_existsfile ( const char *  pathname)
static

Definition at line 882 of file walmethods.c.

References tar_clear_error.

Referenced by CreateWalTarMethod().

883 {
884  tar_clear_error();
885  /* We only deal with new tarfiles, so nothing externally created exists */
886  return false;
887 }
#define tar_clear_error()
Definition: walmethods.c:409

◆ tar_finish()

static bool tar_finish ( void  )
static

Definition at line 890 of file walmethods.c.

References close, CLOSE_NORMAL, TarMethodData::compression, TarMethodData::currentfile, TarMethodData::fd, fsync, fsync_fname(), fsync_parent_path(), MemSet, TarMethodData::sync, tar_clear_error, tar_close(), tar_set_error, TarMethodData::tarfilename, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

891 {
892  char zerobuf[1024];
893 
894  tar_clear_error();
895 
896  if (tar_data->currentfile)
897  {
899  return false;
900  }
901 
902  /* A tarfile always ends with two empty blocks */
903  MemSet(zerobuf, 0, sizeof(zerobuf));
904  if (!tar_data->compression)
905  {
906  errno = 0;
907  if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf))
908  {
909  /* if write didn't set errno, assume problem is no disk space */
910  if (errno == 0)
911  errno = ENOSPC;
912  return false;
913  }
914  }
915 #ifdef HAVE_LIBZ
916  else
917  {
918  if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false))
919  return false;
920 
921  /* Also flush all data to make sure the gzip stream is finished */
922  tar_data->zp->next_in = NULL;
923  tar_data->zp->avail_in = 0;
924  while (true)
925  {
926  int r;
927 
928  r = deflate(tar_data->zp, Z_FINISH);
929 
930  if (r == Z_STREAM_ERROR)
931  {
932  tar_set_error("could not compress data");
933  return false;
934  }
935  if (tar_data->zp->avail_out < ZLIB_OUT_SIZE)
936  {
937  size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out;
938 
939  errno = 0;
940  if (write(tar_data->fd, tar_data->zlibOut, len) != len)
941  {
942  /*
943  * If write didn't set errno, assume problem is no disk
944  * space.
945  */
946  if (errno == 0)
947  errno = ENOSPC;
948  return false;
949  }
950  }
951  if (r == Z_STREAM_END)
952  break;
953  }
954 
955  if (deflateEnd(tar_data->zp) != Z_OK)
956  {
957  tar_set_error("could not close compression stream");
958  return false;
959  }
960  }
961 #endif
962 
963  /* sync the empty blocks as well, since they're after the last file */
964  if (tar_data->sync)
965  {
966  if (fsync(tar_data->fd) != 0)
967  return false;
968  }
969 
970  if (close(tar_data->fd) != 0)
971  return false;
972 
973  tar_data->fd = -1;
974 
975  if (tar_data->sync)
976  {
977  if (fsync_fname(tar_data->tarfilename, false) != 0)
978  return false;
980  return false;
981  }
982 
983  return true;
984 }
#define write(a, b, c)
Definition: win32.h:14
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:632
#define MemSet(start, val, len)
Definition: c.h:978
static int tar_close(Walfile f, WalCloseMethod method)
Definition: walmethods.c:721
#define fsync(fd)
Definition: win32_port.h:62
#define tar_set_error(msg)
Definition: walmethods.c:410
char * tarfilename
Definition: walmethods.c:396
TarMethodFile * currentfile
Definition: walmethods.c:400
#define tar_clear_error()
Definition: walmethods.c:409
#define ZLIB_OUT_SIZE
Definition: walmethods.c:31
#define close(a)
Definition: win32.h:12
static int fsync_parent_path(const char *fname, int elevel)
Definition: fd.c:3542
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_get_current_pos()

static off_t tar_get_current_pos ( Walfile  f)
static

Definition at line 693 of file walmethods.c.

References Assert, and tar_clear_error.

Referenced by CreateWalTarMethod(), and tar_close().

694 {
695  Assert(f != NULL);
696  tar_clear_error();
697 
698  return ((TarMethodFile *) f)->currpos;
699 }
#define Assert(condition)
Definition: c.h:745
#define tar_clear_error()
Definition: walmethods.c:409

◆ tar_get_file_size()

static ssize_t tar_get_file_size ( const char *  pathname)
static

Definition at line 683 of file walmethods.c.

References tar_clear_error.

Referenced by CreateWalTarMethod().

684 {
685  tar_clear_error();
686 
687  /* Currently not used, so not supported */
688  errno = ENOSYS;
689  return -1;
690 }
#define tar_clear_error()
Definition: walmethods.c:409

◆ tar_getlasterror()

static const char* tar_getlasterror ( void  )
static

Definition at line 413 of file walmethods.c.

References buf, TarMethodData::fd, TarMethodData::lasterror, strerror, tar_set_error, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

414 {
415  /*
416  * If a custom error is set, return that one. Otherwise, assume errno is
417  * set and return that one.
418  */
419  if (tar_data->lasterror[0])
420  return tar_data->lasterror;
421  return strerror(errno);
422 }
#define strerror
Definition: port.h:206
char lasterror[1024]
Definition: walmethods.c:401
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_open_for_write()

static Walfile tar_open_for_write ( const char *  pathname,
const char *  temp_suffix,
size_t  pad_to_size 
)
static

Definition at line 531 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::currentfile, TarMethodFile::currpos, TarMethodData::fd, TarMethodFile::header, MAXPGPATH, TarMethodFile::ofs_start, TarMethodFile::pad_to_size, TarMethodFile::pathname, PG_BINARY, pg_file_create_mode, pg_free(), pg_malloc(), pg_malloc0(), pg_strdup(), S_IRUSR, S_IWUSR, snprintf, TAR_BLOCK_SIZE, tar_clear_error, TAR_OK, tar_set_error, tar_write_padding_data(), tarCreateHeader(), TarMethodData::tarfilename, write, and ZLIB_OUT_SIZE.

Referenced by CreateWalTarMethod().

532 {
533  int save_errno;
534  static char tmppath[MAXPGPATH];
535 
536  tar_clear_error();
537 
538  if (tar_data->fd < 0)
539  {
540  /*
541  * We open the tar file only when we first try to write to it.
542  */
543  tar_data->fd = open(tar_data->tarfilename,
544  O_WRONLY | O_CREAT | PG_BINARY,
546  if (tar_data->fd < 0)
547  return NULL;
548 
549 #ifdef HAVE_LIBZ
550  if (tar_data->compression)
551  {
552  tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream));
553  tar_data->zp->zalloc = Z_NULL;
554  tar_data->zp->zfree = Z_NULL;
555  tar_data->zp->opaque = Z_NULL;
556  tar_data->zp->next_out = tar_data->zlibOut;
557  tar_data->zp->avail_out = ZLIB_OUT_SIZE;
558 
559  /*
560  * Initialize deflation library. Adding the magic value 16 to the
561  * default 15 for the windowBits parameter makes the output be
562  * gzip instead of zlib.
563  */
564  if (deflateInit2(tar_data->zp, tar_data->compression, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK)
565  {
566  pg_free(tar_data->zp);
567  tar_data->zp = NULL;
568  tar_set_error("could not initialize compression library");
569  return NULL;
570  }
571  }
572 #endif
573 
574  /* There's no tar header itself, the file starts with regular files */
575  }
576 
577  Assert(tar_data->currentfile == NULL);
578  if (tar_data->currentfile != NULL)
579  {
580  tar_set_error("implementation error: tar files can't have more than one open file");
581  return NULL;
582  }
583 
585 
586  snprintf(tmppath, sizeof(tmppath), "%s%s",
587  pathname, temp_suffix ? temp_suffix : "");
588 
589  /* Create a header with size set to 0 - we will fill out the size on close */
590  if (tarCreateHeader(tar_data->currentfile->header, tmppath, NULL, 0, S_IRUSR | S_IWUSR, 0, 0, time(NULL)) != TAR_OK)
591  {
593  tar_data->currentfile = NULL;
594  tar_set_error("could not create tar header");
595  return NULL;
596  }
597 
598 #ifdef HAVE_LIBZ
599  if (tar_data->compression)
600  {
601  /* Flush existing data */
602  if (!tar_write_compressed_data(NULL, 0, true))
603  return NULL;
604 
605  /* Turn off compression for header */
606  if (deflateParams(tar_data->zp, 0, 0) != Z_OK)
607  {
608  tar_set_error("could not change compression parameters");
609  return NULL;
610  }
611  }
612 #endif
613 
614  tar_data->currentfile->ofs_start = lseek(tar_data->fd, 0, SEEK_CUR);
615  if (tar_data->currentfile->ofs_start == -1)
616  {
617  save_errno = errno;
619  tar_data->currentfile = NULL;
620  errno = save_errno;
621  return NULL;
622  }
624 
625  if (!tar_data->compression)
626  {
627  errno = 0;
630  {
631  save_errno = errno;
633  tar_data->currentfile = NULL;
634  /* if write didn't set errno, assume problem is no disk space */
635  errno = save_errno ? save_errno : ENOSPC;
636  return NULL;
637  }
638  }
639 #ifdef HAVE_LIBZ
640  else
641  {
642  /* Write header through the zlib APIs but with no compression */
643  if (!tar_write_compressed_data(tar_data->currentfile->header,
644  TAR_BLOCK_SIZE, true))
645  return NULL;
646 
647  /* Re-enable compression for the rest of the file */
648  if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK)
649  {
650  tar_set_error("could not change compression parameters");
651  return NULL;
652  }
653  }
654 #endif
655 
656  tar_data->currentfile->pathname = pg_strdup(pathname);
657 
658  /*
659  * Uncompressed files are padded on creation, but for compression we can't
660  * do that
661  */
662  if (pad_to_size)
663  {
664  tar_data->currentfile->pad_to_size = pad_to_size;
665  if (!tar_data->compression)
666  {
667  /* Uncompressed, so pad now */
669  /* Seek back to start */
670  if (lseek(tar_data->fd,
672  SEEK_SET) != tar_data->currentfile->ofs_start + TAR_BLOCK_SIZE)
673  return NULL;
674 
676  }
677  }
678 
679  return tar_data->currentfile;
680 }
int pg_file_create_mode
Definition: file_perm.c:19
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
#define write(a, b, c)
Definition: win32.h:14
char * pathname
Definition: walmethods.c:390
Definition: pgtar.h:21
#define PG_BINARY
Definition: c.h:1240
size_t pad_to_size
Definition: walmethods.c:391
#define S_IWUSR
Definition: win32_port.h:263
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
#define MAXPGPATH
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
z_stream * z_streamp
#define tar_set_error(msg)
Definition: walmethods.c:410
static bool tar_write_padding_data(TarMethodFile *f, size_t bytes)
Definition: walmethods.c:511
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
char * tarfilename
Definition: walmethods.c:396
TarMethodFile * currentfile
Definition: walmethods.c:400
#define Assert(condition)
Definition: c.h:745
char header[TAR_BLOCK_SIZE]
Definition: walmethods.c:389
#define tar_clear_error()
Definition: walmethods.c:409
void pg_free(void *ptr)
Definition: fe_memutils.c:105
#define ZLIB_OUT_SIZE
Definition: walmethods.c:31
#define S_IRUSR
Definition: win32_port.h:260
#define snprintf
Definition: port.h:193
off_t ofs_start
Definition: walmethods.c:387
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_sync()

static int tar_sync ( Walfile  f)
static

Definition at line 702 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::fd, fsync, TarMethodData::sync, and tar_clear_error.

Referenced by CreateWalTarMethod(), and tar_close().

703 {
704  Assert(f != NULL);
705  tar_clear_error();
706 
707  if (!tar_data->sync)
708  return 0;
709 
710  /*
711  * Always sync the whole tarfile, because that's all we can do. This makes
712  * no sense on compressed files, so just ignore those.
713  */
714  if (tar_data->compression)
715  return 0;
716 
717  return fsync(tar_data->fd);
718 }
#define fsync(fd)
Definition: win32_port.h:62
#define Assert(condition)
Definition: c.h:745
#define tar_clear_error()
Definition: walmethods.c:409
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_write()

static ssize_t tar_write ( Walfile  f,
const void *  buf,
size_t  count 
)
static

Definition at line 480 of file walmethods.c.

References Assert, TarMethodData::compression, TarMethodData::fd, tar_clear_error, unconstify, and write.

Referenced by CreateWalTarMethod(), tar_close(), and tar_write_padding_data().

481 {
482  ssize_t r;
483 
484  Assert(f != NULL);
485  tar_clear_error();
486 
487  /* Tarfile will always be positioned at the end */
488  if (!tar_data->compression)
489  {
490  r = write(tar_data->fd, buf, count);
491  if (r > 0)
492  ((TarMethodFile *) f)->currpos += r;
493  return r;
494  }
495 #ifdef HAVE_LIBZ
496  else
497  {
498  if (!tar_write_compressed_data(unconstify(void *, buf), count, false))
499  return -1;
500  ((TarMethodFile *) f)->currpos += count;
501  return count;
502  }
503 #else
504  else
505  /* Can't happen - compression enabled with no libz */
506  return -1;
507 #endif
508 }
#define write(a, b, c)
Definition: win32.h:14
static char * buf
Definition: pg_test_fsync.c:67
#define unconstify(underlying_type, expr)
Definition: c.h:1212
#define Assert(condition)
Definition: c.h:745
#define tar_clear_error()
Definition: walmethods.c:409
static TarMethodData * tar_data
Definition: walmethods.c:407

◆ tar_write_padding_data()

static bool tar_write_padding_data ( TarMethodFile f,
size_t  bytes 
)
static

Definition at line 511 of file walmethods.c.

References generate_unaccent_rules::bytes(), PGAlignedXLogBlock::data, Min, and tar_write().

Referenced by tar_close(), and tar_open_for_write().

512 {
513  PGAlignedXLogBlock zerobuf;
514  size_t bytesleft = bytes;
515 
516  memset(zerobuf.data, 0, XLOG_BLCKSZ);
517  while (bytesleft)
518  {
519  size_t bytestowrite = Min(bytesleft, XLOG_BLCKSZ);
520  ssize_t r = tar_write(f, zerobuf.data, bytestowrite);
521 
522  if (r < 0)
523  return false;
524  bytesleft -= r;
525  }
526 
527  return true;
528 }
def bytes(source, encoding='ascii', errors='strict')
#define Min(x, y)
Definition: c.h:927
static ssize_t tar_write(Walfile f, const void *buf, size_t count)
Definition: walmethods.c:480
char data[XLOG_BLCKSZ]
Definition: c.h:1119

Variable Documentation

◆ dir_data

DirectoryMethodData* dir_data = NULL
static

Definition at line 47 of file walmethods.c.

◆ tar_data

TarMethodData* tar_data = NULL
static

Definition at line 407 of file walmethods.c.