PostgreSQL Source Code  git master
buffile.c File Reference
#include "postgres.h"
#include "executor/instrument.h"
#include "pgstat.h"
#include "storage/fd.h"
#include "storage/buffile.h"
#include "storage/buf_internals.h"
#include "utils/resowner.h"
Include dependency graph for buffile.c:

Go to the source code of this file.

Data Structures

struct  BufFile
 

Macros

#define MAX_PHYSICAL_FILESIZE   0x40000000
 
#define BUFFILE_SEG_SIZE   (MAX_PHYSICAL_FILESIZE / BLCKSZ)
 

Functions

static BufFilemakeBufFile (File firstfile)
 
static void extendBufFile (BufFile *file)
 
static void BufFileLoadBuffer (BufFile *file)
 
static void BufFileDumpBuffer (BufFile *file)
 
static int BufFileFlush (BufFile *file)
 
BufFileBufFileCreateTemp (bool interXact)
 
void BufFileClose (BufFile *file)
 
size_t BufFileRead (BufFile *file, void *ptr, size_t size)
 
size_t BufFileWrite (BufFile *file, void *ptr, size_t size)
 
int BufFileSeek (BufFile *file, int fileno, off_t offset, int whence)
 
void BufFileTell (BufFile *file, int *fileno, off_t *offset)
 
int BufFileSeekBlock (BufFile *file, long blknum)
 

Macro Definition Documentation

◆ BUFFILE_SEG_SIZE

#define BUFFILE_SEG_SIZE   (MAX_PHYSICAL_FILESIZE / BLCKSZ)

Definition at line 52 of file buffile.c.

Referenced by BufFileSeekBlock().

◆ MAX_PHYSICAL_FILESIZE

#define MAX_PHYSICAL_FILESIZE   0x40000000

Definition at line 51 of file buffile.c.

Referenced by BufFileDumpBuffer(), BufFileLoadBuffer(), and BufFileSeek().

Function Documentation

◆ BufFileClose()

void BufFileClose ( BufFile file)

Definition at line 199 of file buffile.c.

References BufFileFlush(), FileClose(), BufFile::files, i, BufFile::numFiles, BufFile::offsets, and pfree().

Referenced by ExecHashJoinNewBatch(), ExecHashTableDestroy(), gistFreeBuildBuffers(), LogicalTapeSetClose(), tuplestore_clear(), and tuplestore_end().

200 {
201  int i;
202 
203  /* flush any unwritten data */
204  BufFileFlush(file);
205  /* close the underlying file(s) (with delete if it's a temp file) */
206  for (i = 0; i < file->numFiles; i++)
207  FileClose(file->files[i]);
208  /* release the buffer space */
209  pfree(file->files);
210  pfree(file->offsets);
211  pfree(file);
212 }
off_t * offsets
Definition: buffile.c:64
static int BufFileFlush(BufFile *file)
Definition: buffile.c:458
int numFiles
Definition: buffile.c:61
void pfree(void *pointer)
Definition: mcxt.c:949
File * files
Definition: buffile.c:63
void FileClose(File file)
Definition: fd.c:1522
int i

◆ BufFileCreateTemp()

BufFile* BufFileCreateTemp ( bool  interXact)

Definition at line 164 of file buffile.c.

References Assert, BufFile::isInterXact, makeBufFile(), and OpenTemporaryFile().

Referenced by ExecHashJoinSaveTuple(), gistInitBuildBuffers(), LogicalTapeSetCreate(), and tuplestore_puttuple_common().

165 {
166  BufFile *file;
167  File pfile;
168 
169  pfile = OpenTemporaryFile(interXact);
170  Assert(pfile >= 0);
171 
172  file = makeBufFile(pfile);
173  file->isInterXact = interXact;
174 
175  return file;
176 }
bool isInterXact
Definition: buffile.c:71
static BufFile * makeBufFile(File firstfile)
Definition: buffile.c:104
#define Assert(condition)
Definition: c.h:670
File OpenTemporaryFile(bool interXact)
Definition: fd.c:1396
int File
Definition: fd.h:49

◆ BufFileDumpBuffer()

static void BufFileDumpBuffer ( BufFile file)
static

Definition at line 275 of file buffile.c.

References Assert, BufFile::buffer, BufFile::curFile, BufFile::curOffset, BufFile::dirty, extendBufFile(), BufFile::files, FileSeek(), FileWrite(), MAX_PHYSICAL_FILESIZE, BufFile::nbytes, BufFile::numFiles, BufFile::offsets, pgBufferUsage, BufFile::pos, BufferUsage::temp_blks_written, WAIT_EVENT_BUFFILE_WRITE, and wpos.

Referenced by BufFileFlush(), and BufFileWrite().

276 {
277  int wpos = 0;
278  int bytestowrite;
279  File thisfile;
280 
281  /*
282  * Unlike BufFileLoadBuffer, we must dump the whole buffer even if it
283  * crosses a component-file boundary; so we need a loop.
284  */
285  while (wpos < file->nbytes)
286  {
287  off_t availbytes;
288 
289  /*
290  * Advance to next component file if necessary and possible.
291  */
292  if (file->curOffset >= MAX_PHYSICAL_FILESIZE)
293  {
294  while (file->curFile + 1 >= file->numFiles)
295  extendBufFile(file);
296  file->curFile++;
297  file->curOffset = 0L;
298  }
299 
300  /*
301  * Enforce per-file size limit only for temp files, else just try to
302  * write as much as asked...
303  */
304  bytestowrite = file->nbytes - wpos;
305  availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
306 
307  if ((off_t) bytestowrite > availbytes)
308  bytestowrite = (int) availbytes;
309 
310  /*
311  * May need to reposition physical file.
312  */
313  thisfile = file->files[file->curFile];
314  if (file->curOffset != file->offsets[file->curFile])
315  {
316  if (FileSeek(thisfile, file->curOffset, SEEK_SET) != file->curOffset)
317  return; /* seek failed, give up */
318  file->offsets[file->curFile] = file->curOffset;
319  }
320  bytestowrite = FileWrite(thisfile,
321  file->buffer + wpos,
322  bytestowrite,
324  if (bytestowrite <= 0)
325  return; /* failed to write */
326  file->offsets[file->curFile] += bytestowrite;
327  file->curOffset += bytestowrite;
328  wpos += bytestowrite;
329 
331  }
332  file->dirty = false;
333 
334  /*
335  * At this point, curOffset has been advanced to the end of the buffer,
336  * ie, its original value + nbytes. We need to make it point to the
337  * logical file position, ie, original value + pos, in case that is less
338  * (as could happen due to a small backwards seek in a dirty buffer!)
339  */
340  file->curOffset -= (file->nbytes - file->pos);
341  if (file->curOffset < 0) /* handle possible segment crossing */
342  {
343  file->curFile--;
344  Assert(file->curFile >= 0);
346  }
347 
348  /*
349  * Now we can set the buffer empty without changing the logical position
350  */
351  file->pos = 0;
352  file->nbytes = 0;
353 }
off_t * offsets
Definition: buffile.c:64
int nbytes
Definition: buffile.c:88
#define MAX_PHYSICAL_FILESIZE
Definition: buffile.c:51
long temp_blks_written
Definition: instrument.h:30
static void extendBufFile(BufFile *file)
Definition: buffile.c:128
int numFiles
Definition: buffile.c:61
int FileWrite(File file, char *buffer, int amount, uint32 wait_event_info)
Definition: fd.c:1738
File * files
Definition: buffile.c:63
#define wpos(wep)
Definition: tsrank.c:27
off_t curOffset
Definition: buffile.c:86
int curFile
Definition: buffile.c:85
#define Assert(condition)
Definition: c.h:670
bool dirty
Definition: buffile.c:72
char buffer[BLCKSZ]
Definition: buffile.c:89
off_t FileSeek(File file, off_t offset, int whence)
Definition: fd.c:1880
int File
Definition: fd.h:49
BufferUsage pgBufferUsage
Definition: instrument.c:20
int pos
Definition: buffile.c:87

◆ BufFileFlush()

static int BufFileFlush ( BufFile file)
static

Definition at line 458 of file buffile.c.

References BufFileDumpBuffer(), and BufFile::dirty.

Referenced by BufFileClose(), BufFileRead(), and BufFileSeek().

459 {
460  if (file->dirty)
461  {
462  BufFileDumpBuffer(file);
463  if (file->dirty)
464  return EOF;
465  }
466 
467  return 0;
468 }
static void BufFileDumpBuffer(BufFile *file)
Definition: buffile.c:275
bool dirty
Definition: buffile.c:72

◆ BufFileLoadBuffer()

static void BufFileLoadBuffer ( BufFile file)
static

Definition at line 222 of file buffile.c.

References BufFile::buffer, BufFile::curFile, BufFile::curOffset, FileRead(), BufFile::files, FileSeek(), MAX_PHYSICAL_FILESIZE, BufFile::nbytes, BufFile::numFiles, BufFile::offsets, pgBufferUsage, BufferUsage::temp_blks_read, and WAIT_EVENT_BUFFILE_READ.

Referenced by BufFileRead().

223 {
224  File thisfile;
225 
226  /*
227  * Advance to next component file if necessary and possible.
228  *
229  * This path can only be taken if there is more than one component, so it
230  * won't interfere with reading a non-temp file that is over
231  * MAX_PHYSICAL_FILESIZE.
232  */
233  if (file->curOffset >= MAX_PHYSICAL_FILESIZE &&
234  file->curFile + 1 < file->numFiles)
235  {
236  file->curFile++;
237  file->curOffset = 0L;
238  }
239 
240  /*
241  * May need to reposition physical file.
242  */
243  thisfile = file->files[file->curFile];
244  if (file->curOffset != file->offsets[file->curFile])
245  {
246  if (FileSeek(thisfile, file->curOffset, SEEK_SET) != file->curOffset)
247  return; /* seek failed, read nothing */
248  file->offsets[file->curFile] = file->curOffset;
249  }
250 
251  /*
252  * Read whatever we can get, up to a full bufferload.
253  */
254  file->nbytes = FileRead(thisfile,
255  file->buffer,
256  sizeof(file->buffer),
258  if (file->nbytes < 0)
259  file->nbytes = 0;
260  file->offsets[file->curFile] += file->nbytes;
261  /* we choose not to advance curOffset here */
262 
263  if (file->nbytes > 0)
265 }
off_t * offsets
Definition: buffile.c:64
int nbytes
Definition: buffile.c:88
#define MAX_PHYSICAL_FILESIZE
Definition: buffile.c:51
int numFiles
Definition: buffile.c:61
long temp_blks_read
Definition: instrument.h:29
File * files
Definition: buffile.c:63
off_t curOffset
Definition: buffile.c:86
int FileRead(File file, char *buffer, int amount, uint32 wait_event_info)
Definition: fd.c:1674
int curFile
Definition: buffile.c:85
char buffer[BLCKSZ]
Definition: buffile.c:89
off_t FileSeek(File file, off_t offset, int whence)
Definition: fd.c:1880
int File
Definition: fd.h:49
BufferUsage pgBufferUsage
Definition: instrument.c:20

◆ BufFileRead()

size_t BufFileRead ( BufFile file,
void *  ptr,
size_t  size 
)

Definition at line 361 of file buffile.c.

References Assert, BufFile::buffer, BufFileFlush(), BufFileLoadBuffer(), BufFile::curOffset, BufFile::dirty, BufFile::nbytes, and BufFile::pos.

Referenced by ExecHashJoinGetSavedTuple(), getlen(), ltsReadBlock(), ReadTempFileBlock(), and readtup_heap().

362 {
363  size_t nread = 0;
364  size_t nthistime;
365 
366  if (file->dirty)
367  {
368  if (BufFileFlush(file) != 0)
369  return 0; /* could not flush... */
370  Assert(!file->dirty);
371  }
372 
373  while (size > 0)
374  {
375  if (file->pos >= file->nbytes)
376  {
377  /* Try to load more data into buffer. */
378  file->curOffset += file->pos;
379  file->pos = 0;
380  file->nbytes = 0;
381  BufFileLoadBuffer(file);
382  if (file->nbytes <= 0)
383  break; /* no more data available */
384  }
385 
386  nthistime = file->nbytes - file->pos;
387  if (nthistime > size)
388  nthistime = size;
389  Assert(nthistime > 0);
390 
391  memcpy(ptr, file->buffer + file->pos, nthistime);
392 
393  file->pos += nthistime;
394  ptr = (void *) ((char *) ptr + nthistime);
395  size -= nthistime;
396  nread += nthistime;
397  }
398 
399  return nread;
400 }
int nbytes
Definition: buffile.c:88
static int BufFileFlush(BufFile *file)
Definition: buffile.c:458
off_t curOffset
Definition: buffile.c:86
#define Assert(condition)
Definition: c.h:670
bool dirty
Definition: buffile.c:72
char buffer[BLCKSZ]
Definition: buffile.c:89
static void BufFileLoadBuffer(BufFile *file)
Definition: buffile.c:222
int pos
Definition: buffile.c:87

◆ BufFileSeek()

int BufFileSeek ( BufFile file,
int  fileno,
off_t  offset,
int  whence 
)

Definition at line 481 of file buffile.c.

References BufFileFlush(), BufFile::curFile, BufFile::curOffset, elog, ERROR, MAX_PHYSICAL_FILESIZE, BufFile::nbytes, BufFile::numFiles, and BufFile::pos.

Referenced by BufFileSeekBlock(), ExecHashJoinNewBatch(), tuplestore_copy_read_pointer(), tuplestore_gettuple(), tuplestore_puttuple_common(), tuplestore_rescan(), and tuplestore_select_read_pointer().

482 {
483  int newFile;
484  off_t newOffset;
485 
486  switch (whence)
487  {
488  case SEEK_SET:
489  if (fileno < 0)
490  return EOF;
491  newFile = fileno;
492  newOffset = offset;
493  break;
494  case SEEK_CUR:
495 
496  /*
497  * Relative seek considers only the signed offset, ignoring
498  * fileno. Note that large offsets (> 1 gig) risk overflow in this
499  * add, unless we have 64-bit off_t.
500  */
501  newFile = file->curFile;
502  newOffset = (file->curOffset + file->pos) + offset;
503  break;
504 #ifdef NOT_USED
505  case SEEK_END:
506  /* could be implemented, not needed currently */
507  break;
508 #endif
509  default:
510  elog(ERROR, "invalid whence: %d", whence);
511  return EOF;
512  }
513  while (newOffset < 0)
514  {
515  if (--newFile < 0)
516  return EOF;
517  newOffset += MAX_PHYSICAL_FILESIZE;
518  }
519  if (newFile == file->curFile &&
520  newOffset >= file->curOffset &&
521  newOffset <= file->curOffset + file->nbytes)
522  {
523  /*
524  * Seek is to a point within existing buffer; we can just adjust
525  * pos-within-buffer, without flushing buffer. Note this is OK
526  * whether reading or writing, but buffer remains dirty if we were
527  * writing.
528  */
529  file->pos = (int) (newOffset - file->curOffset);
530  return 0;
531  }
532  /* Otherwise, must reposition buffer, so flush any dirty data */
533  if (BufFileFlush(file) != 0)
534  return EOF;
535 
536  /*
537  * At this point and no sooner, check for seek past last segment. The
538  * above flush could have created a new segment, so checking sooner would
539  * not work (at least not with this code).
540  */
541 
542  /* convert seek to "start of next seg" to "end of last seg" */
543  if (newFile == file->numFiles && newOffset == 0)
544  {
545  newFile--;
546  newOffset = MAX_PHYSICAL_FILESIZE;
547  }
548  while (newOffset > MAX_PHYSICAL_FILESIZE)
549  {
550  if (++newFile >= file->numFiles)
551  return EOF;
552  newOffset -= MAX_PHYSICAL_FILESIZE;
553  }
554  if (newFile >= file->numFiles)
555  return EOF;
556  /* Seek is OK! */
557  file->curFile = newFile;
558  file->curOffset = newOffset;
559  file->pos = 0;
560  file->nbytes = 0;
561  return 0;
562 }
int nbytes
Definition: buffile.c:88
static int BufFileFlush(BufFile *file)
Definition: buffile.c:458
#define MAX_PHYSICAL_FILESIZE
Definition: buffile.c:51
int numFiles
Definition: buffile.c:61
#define ERROR
Definition: elog.h:43
off_t curOffset
Definition: buffile.c:86
int curFile
Definition: buffile.c:85
#define elog
Definition: elog.h:219
int pos
Definition: buffile.c:87

◆ BufFileSeekBlock()

int BufFileSeekBlock ( BufFile file,
long  blknum 
)

Definition at line 583 of file buffile.c.

References BUFFILE_SEG_SIZE, BufFileSeek(), BufFile::curFile, BufFile::curOffset, and BufFile::pos.

Referenced by ltsReadBlock(), ltsWriteBlock(), ReadTempFileBlock(), and WriteTempFileBlock().

584 {
585  return BufFileSeek(file,
586  (int) (blknum / BUFFILE_SEG_SIZE),
587  (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
588  SEEK_SET);
589 }
int BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
Definition: buffile.c:481
#define BUFFILE_SEG_SIZE
Definition: buffile.c:52

◆ BufFileTell()

void BufFileTell ( BufFile file,
int *  fileno,
off_t *  offset 
)

Definition at line 565 of file buffile.c.

References BufFile::curFile, BufFile::curOffset, and BufFile::pos.

Referenced by dumptuples(), tuplestore_copy_read_pointer(), tuplestore_gettuple(), tuplestore_puttuple_common(), and tuplestore_select_read_pointer().

566 {
567  *fileno = file->curFile;
568  *offset = file->curOffset + file->pos;
569 }
off_t curOffset
Definition: buffile.c:86
int curFile
Definition: buffile.c:85
int pos
Definition: buffile.c:87

◆ BufFileWrite()

size_t BufFileWrite ( BufFile file,
void *  ptr,
size_t  size 
)

Definition at line 408 of file buffile.c.

References Assert, BufFile::buffer, BufFileDumpBuffer(), BufFile::curOffset, BufFile::dirty, BufFile::nbytes, and BufFile::pos.

Referenced by ExecHashJoinSaveTuple(), ltsWriteBlock(), WriteTempFileBlock(), and writetup_heap().

409 {
410  size_t nwritten = 0;
411  size_t nthistime;
412 
413  while (size > 0)
414  {
415  if (file->pos >= BLCKSZ)
416  {
417  /* Buffer full, dump it out */
418  if (file->dirty)
419  {
420  BufFileDumpBuffer(file);
421  if (file->dirty)
422  break; /* I/O error */
423  }
424  else
425  {
426  /* Hmm, went directly from reading to writing? */
427  file->curOffset += file->pos;
428  file->pos = 0;
429  file->nbytes = 0;
430  }
431  }
432 
433  nthistime = BLCKSZ - file->pos;
434  if (nthistime > size)
435  nthistime = size;
436  Assert(nthistime > 0);
437 
438  memcpy(file->buffer + file->pos, ptr, nthistime);
439 
440  file->dirty = true;
441  file->pos += nthistime;
442  if (file->nbytes < file->pos)
443  file->nbytes = file->pos;
444  ptr = (void *) ((char *) ptr + nthistime);
445  size -= nthistime;
446  nwritten += nthistime;
447  }
448 
449  return nwritten;
450 }
int nbytes
Definition: buffile.c:88
static void BufFileDumpBuffer(BufFile *file)
Definition: buffile.c:275
off_t curOffset
Definition: buffile.c:86
#define Assert(condition)
Definition: c.h:670
bool dirty
Definition: buffile.c:72
char buffer[BLCKSZ]
Definition: buffile.c:89
int pos
Definition: buffile.c:87

◆ extendBufFile()

static void extendBufFile ( BufFile file)
static

Definition at line 128 of file buffile.c.

References Assert, CurrentResourceOwner, BufFile::files, BufFile::isInterXact, BufFile::numFiles, BufFile::offsets, OpenTemporaryFile(), repalloc(), and BufFile::resowner.

Referenced by BufFileDumpBuffer().

129 {
130  File pfile;
131  ResourceOwner oldowner;
132 
133  /* Be sure to associate the file with the BufFile's resource owner */
134  oldowner = CurrentResourceOwner;
136 
137  pfile = OpenTemporaryFile(file->isInterXact);
138  Assert(pfile >= 0);
139 
140  CurrentResourceOwner = oldowner;
141 
142  file->files = (File *) repalloc(file->files,
143  (file->numFiles + 1) * sizeof(File));
144  file->offsets = (off_t *) repalloc(file->offsets,
145  (file->numFiles + 1) * sizeof(off_t));
146  file->files[file->numFiles] = pfile;
147  file->offsets[file->numFiles] = 0L;
148  file->numFiles++;
149 }
off_t * offsets
Definition: buffile.c:64
ResourceOwner CurrentResourceOwner
Definition: resowner.c:138
int numFiles
Definition: buffile.c:61
ResourceOwner resowner
Definition: buffile.c:79
bool isInterXact
Definition: buffile.c:71
File * files
Definition: buffile.c:63
#define Assert(condition)
Definition: c.h:670
File OpenTemporaryFile(bool interXact)
Definition: fd.c:1396
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:962
int File
Definition: fd.h:49

◆ makeBufFile()

static BufFile * makeBufFile ( File  firstfile)
static

Definition at line 104 of file buffile.c.

References BufFile::curFile, BufFile::curOffset, CurrentResourceOwner, BufFile::dirty, BufFile::files, BufFile::isInterXact, BufFile::nbytes, BufFile::numFiles, BufFile::offsets, palloc(), BufFile::pos, and BufFile::resowner.

Referenced by BufFileCreateTemp().

105 {
106  BufFile *file = (BufFile *) palloc(sizeof(BufFile));
107 
108  file->numFiles = 1;
109  file->files = (File *) palloc(sizeof(File));
110  file->files[0] = firstfile;
111  file->offsets = (off_t *) palloc(sizeof(off_t));
112  file->offsets[0] = 0L;
113  file->isInterXact = false;
114  file->dirty = false;
116  file->curFile = 0;
117  file->curOffset = 0L;
118  file->pos = 0;
119  file->nbytes = 0;
120 
121  return file;
122 }
off_t * offsets
Definition: buffile.c:64
int nbytes
Definition: buffile.c:88
ResourceOwner CurrentResourceOwner
Definition: resowner.c:138
int numFiles
Definition: buffile.c:61
ResourceOwner resowner
Definition: buffile.c:79
bool isInterXact
Definition: buffile.c:71
File * files
Definition: buffile.c:63
off_t curOffset
Definition: buffile.c:86
int curFile
Definition: buffile.c:85
bool dirty
Definition: buffile.c:72
void * palloc(Size size)
Definition: mcxt.c:848
int File
Definition: fd.h:49
int pos
Definition: buffile.c:87