astreamer.h File Reference

#include "common/compression.h"
#include "lib/stringinfo.h"
#include "pqexpbuffer.h"

Include dependency graph for astreamer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	astreamer_member
struct	astreamer
struct	astreamer_ops

Typedefs
typedef struct astreamer	astreamer
typedef struct astreamer_ops	astreamer_ops

Enumerations
enum	astreamer_archive_context {   ASTREAMER_UNKNOWN , ASTREAMER_MEMBER_HEADER , ASTREAMER_MEMBER_CONTENTS , ASTREAMER_MEMBER_TRAILER ,   ASTREAMER_ARCHIVE_TRAILER }

Functions
static void	astreamer_content (astreamer *streamer, astreamer_member *member, const char *data, int len, astreamer_archive_context context)
static void	astreamer_finalize (astreamer *streamer)
static void	astreamer_free (astreamer *streamer)
static void	astreamer_buffer_bytes (astreamer *streamer, const char **data, int *len, int nbytes)
static bool	astreamer_buffer_until (astreamer *streamer, const char **data, int *len, int target_bytes)
astreamer *	astreamer_plain_writer_new (char *pathname, FILE *file)
astreamer *	astreamer_gzip_writer_new (char *pathname, FILE *file, pg_compress_specification *compress)
astreamer *	astreamer_extractor_new (const char *basepath, const char *(*link_map)(const char *), void(*report_output_file)(const char *))
astreamer *	astreamer_gzip_decompressor_new (astreamer *next)
astreamer *	astreamer_lz4_compressor_new (astreamer *next, pg_compress_specification *compress)
astreamer *	astreamer_lz4_decompressor_new (astreamer *next)
astreamer *	astreamer_zstd_compressor_new (astreamer *next, pg_compress_specification *compress)
astreamer *	astreamer_zstd_decompressor_new (astreamer *next)
astreamer *	astreamer_tar_parser_new (astreamer *next)
astreamer *	astreamer_tar_terminator_new (astreamer *next)
astreamer *	astreamer_tar_archiver_new (astreamer *next)

Typedef Documentation

astreamer

typedef struct astreamer astreamer

Definition at line 40 of file astreamer.h.

astreamer_ops

typedef struct astreamer_ops astreamer_ops

Definition at line 41 of file astreamer.h.

Enumeration Type Documentation

astreamer_archive_context

enum astreamer_archive_context

Enumerator
ASTREAMER_UNKNOWN
ASTREAMER_MEMBER_HEADER
ASTREAMER_MEMBER_CONTENTS
ASTREAMER_MEMBER_TRAILER
ASTREAMER_ARCHIVE_TRAILER

Definition at line 62 of file astreamer.h.

{
    ASTREAMER_UNKNOWN,
    ASTREAMER_MEMBER_HEADER,
    ASTREAMER_MEMBER_CONTENTS,
    ASTREAMER_MEMBER_TRAILER,
    ASTREAMER_ARCHIVE_TRAILER,
} astreamer_archive_context;

Function Documentation

astreamer_buffer_bytes()

static void astreamer_buffer_bytes ( astreamer *  streamer, const char **  data, int *  len, int  nbytes  )

inlinestatic

Definition at line 168 of file astreamer.h.

{
    Assert(nbytes <= *len);
 
    appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
    *len -= nbytes;
    *data += nbytes;
}

References appendBinaryStringInfo(), Assert, astreamer::bbs_buffer, data, and len.

Referenced by astreamer_buffer_until(), and astreamer_tar_parser_content().

astreamer_buffer_until()

static bool astreamer_buffer_until ( astreamer *  streamer, const char **  data, int *  len, int  target_bytes  )

inlinestatic

Definition at line 186 of file astreamer.h.

{
    int         buflen = streamer->bbs_buffer.len;
 
    if (buflen >= target_bytes)
    {
        /* Target length already reached; nothing to do. */
        return true;
    }
 
    if (buflen + *len < target_bytes)
    {
        /* Not enough data to reach target length; buffer all of it. */
        astreamer_buffer_bytes(streamer, data, len, *len);
        return false;
    }
 
    /* Buffer just enough to reach the target length. */
    astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
    return true;
}

References astreamer_buffer_bytes(), astreamer::bbs_buffer, data, fb(), StringInfoData::len, and len.

Referenced by astreamer_tar_parser_content().

astreamer_content()

static void astreamer_content ( astreamer *  streamer, astreamer_member *  member, const char *  data, int  len, astreamer_archive_context  context  )

inlinestatic

Definition at line 137 of file astreamer.h.

{
    Assert(streamer != NULL);
    streamer->bbs_ops->content(streamer, member, data, len, context);
}

References Assert, astreamer::bbs_ops, astreamer_ops::content, data, fb(), and len.

Referenced by astreamer_inject_file(), astreamer_recovery_injector_content(), astreamer_tar_archiver_content(), astreamer_tar_header(), astreamer_tar_parser_content(), astreamer_tar_parser_finalize(), astreamer_tar_terminator_content(), astreamer_tar_terminator_finalize(), read_archive_file(), ReceiveArchiveStreamChunk(), ReceiveTarCopyChunk(), and verify_tar_file().

astreamer_extractor_new()

astreamer * astreamer_extractor_new ( const char *  basepath, const char *(*)(const char *)  link_map, void(*)(const char *)  report_output_file  )

extern

Definition at line 186 of file astreamer_file.c.

{
    astreamer_extractor *streamer;
 
    streamer = palloc0_object(astreamer_extractor);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_extractor_ops;
    streamer->basepath = pstrdup(basepath);
    streamer->link_map = link_map;
    streamer->report_output_file = report_output_file;
 
    return &streamer->base;
}

References astreamer_extractor_ops, astreamer_extractor::base, astreamer_extractor::basepath, astreamer::bbs_ops, astreamer_extractor::link_map, palloc0_object, pstrdup(), and astreamer_extractor::report_output_file.

Referenced by CreateBackupStreamer().

astreamer_finalize()

static void astreamer_finalize ( astreamer *  streamer )

inlinestatic

Definition at line 147 of file astreamer.h.

{
    Assert(streamer != NULL);
    streamer->bbs_ops->finalize(streamer);
}

References Assert, astreamer::bbs_ops, fb(), and astreamer_ops::finalize.

Referenced by astreamer_recovery_injector_finalize(), astreamer_tar_archiver_finalize(), astreamer_tar_parser_finalize(), astreamer_tar_terminator_finalize(), read_archive_file(), ReceiveArchiveStream(), ReceiveArchiveStreamChunk(), ReceiveTarFile(), and verify_tar_backup().

astreamer_free()

static void astreamer_free ( astreamer *  streamer )

inlinestatic

Definition at line 155 of file astreamer.h.

{
    Assert(streamer != NULL);
    streamer->bbs_ops->free(streamer);
}

References Assert, astreamer::bbs_ops, fb(), and astreamer_ops::free.

Referenced by astreamer_recovery_injector_free(), astreamer_tar_archiver_free(), astreamer_tar_parser_free(), astreamer_tar_terminator_free(), free_archive_reader(), ReceiveArchiveStream(), ReceiveArchiveStreamChunk(), ReceiveTarFile(), and verify_tar_backup().

astreamer_gzip_decompressor_new()

astreamer * astreamer_gzip_decompressor_new ( astreamer *  next )

extern

Definition at line 236 of file astreamer_gzip.c.

{
#ifdef HAVE_LIBZ
    astreamer_gzip_decompressor *streamer;
    z_stream   *zs;
 
    Assert(next != NULL);
 
    streamer = palloc0_object(astreamer_gzip_decompressor);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_gzip_decompressor_ops;
 
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    /* Use a buffer size comparable to the other decompressors */
    enlargeStringInfo(&streamer->base.bbs_buffer, 256 * 1024 - 1);
 
    /* Initialize internal stream state for decompression */
    zs = &streamer->zstream;
    zs->zalloc = gzip_palloc;
    zs->zfree = gzip_pfree;
    zs->next_out = (uint8 *) streamer->base.bbs_buffer.data;
    zs->avail_out = streamer->base.bbs_buffer.maxlen;
 
    /*
     * Data compression was initialized using deflateInit2 to request a gzip
     * header. Similarly, we are using inflateInit2 to initialize data
     * decompression.
     *
     * Per the documentation for inflateInit2, the second argument is
     * "windowBits" and its value must be greater than or equal to the value
     * provided while compressing the data, so we are using the maximum
     * possible value for safety.
     */
    if (inflateInit2(zs, 15 + 16) != Z_OK)
        pg_fatal("could not initialize compression library");
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "gzip");
    return NULL;                /* keep compiler quiet */
#endif
}

References Assert, enlargeStringInfo(), fb(), initStringInfo(), next, palloc0_object, and pg_fatal.

Referenced by create_archive_verifier(), CreateBackupStreamer(), and init_archive_reader().

astreamer_gzip_writer_new()

astreamer * astreamer_gzip_writer_new ( char *  pathname, FILE *  file, pg_compress_specification *  compress  )

extern

Definition at line 99 of file astreamer_gzip.c.

{
#ifdef HAVE_LIBZ
    astreamer_gzip_writer *streamer;
 
    streamer = palloc0_object(astreamer_gzip_writer);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_gzip_writer_ops;
 
    streamer->pathname = pstrdup(pathname);
 
    if (file == NULL)
    {
        streamer->gzfile = gzopen(pathname, "wb");
        if (streamer->gzfile == NULL)
            pg_fatal("could not create compressed file \"%s\": %m",
                     pathname);
    }
    else
    {
        /*
         * We must dup the file handle so that gzclose doesn't break the
         * caller's FILE.  See comment for astreamer_gzip_writer_finalize.
         */
        int         fd = dup(fileno(file));
 
        if (fd < 0)
            pg_fatal("could not duplicate stdout: %m");
 
        streamer->gzfile = gzdopen(fd, "wb");
        if (streamer->gzfile == NULL)
            pg_fatal("could not open output file: %m");
    }
 
    if (gzsetparams(streamer->gzfile, compress->level, Z_DEFAULT_STRATEGY) != Z_OK)
        pg_fatal("could not set compression level %d: %s",
                 compress->level, get_gz_error(streamer->gzfile));
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "gzip");
    return NULL;                /* keep compiler quiet */
#endif
}

References fb(), fd(), pg_compress_specification::level, palloc0_object, pg_fatal, and pstrdup().

Referenced by CreateBackupStreamer().

astreamer_lz4_compressor_new()

astreamer * astreamer_lz4_compressor_new ( astreamer *  next, pg_compress_specification *  compress  )

extern

Definition at line 72 of file astreamer_lz4.c.

{
#ifdef USE_LZ4
    astreamer_lz4_frame *streamer;
    LZ4F_errorCode_t ctxError;
    LZ4F_preferences_t *prefs;
 
    Assert(next != NULL);
 
    streamer = palloc0_object(astreamer_lz4_frame);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_lz4_compressor_ops;
 
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    streamer->header_written = false;
 
    /* Initialize stream compression preferences */
    prefs = &streamer->prefs;
    memset(prefs, 0, sizeof(LZ4F_preferences_t));
    prefs->frameInfo.blockSizeID = LZ4F_max256KB;
    prefs->compressionLevel = compress->level;
 
    ctxError = LZ4F_createCompressionContext(&streamer->cctx, LZ4F_VERSION);
    if (LZ4F_isError(ctxError))
        pg_fatal("could not create lz4 compression context: %s",
                 LZ4F_getErrorName(ctxError));
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "LZ4");
    return NULL;                /* keep compiler quiet */
#endif
}

References Assert, fb(), initStringInfo(), pg_compress_specification::level, next, palloc0_object, and pg_fatal.

Referenced by CreateBackupStreamer().

astreamer_lz4_decompressor_new()

astreamer * astreamer_lz4_decompressor_new ( astreamer *  next )

extern

Definition at line 277 of file astreamer_lz4.c.

{
#ifdef USE_LZ4
    astreamer_lz4_frame *streamer;
    LZ4F_errorCode_t ctxError;
 
    Assert(next != NULL);
 
    streamer = palloc0_object(astreamer_lz4_frame);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_lz4_decompressor_ops;
 
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    /* Use a buffer size comparable to the compressor's */
    enlargeStringInfo(&streamer->base.bbs_buffer, 256 * 1024 - 1);
 
    /* Initialize internal stream state for decompression */
    ctxError = LZ4F_createDecompressionContext(&streamer->dctx, LZ4F_VERSION);
    if (LZ4F_isError(ctxError))
        pg_fatal("could not initialize compression library: %s",
                 LZ4F_getErrorName(ctxError));
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "LZ4");
    return NULL;                /* keep compiler quiet */
#endif
}

References Assert, enlargeStringInfo(), fb(), initStringInfo(), next, palloc0_object, and pg_fatal.

Referenced by create_archive_verifier(), CreateBackupStreamer(), and init_archive_reader().

astreamer_plain_writer_new()

astreamer * astreamer_plain_writer_new ( char *  pathname, FILE *  file  )

extern

Definition at line 81 of file astreamer_file.c.

{
    astreamer_plain_writer *streamer;
 
    streamer = palloc0_object(astreamer_plain_writer);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_plain_writer_ops;
 
    streamer->pathname = pstrdup(pathname);
    streamer->file = file;
 
    if (file == NULL)
    {
        streamer->file = fopen(pathname, "wb");
        if (streamer->file == NULL)
            pg_fatal("could not create file \"%s\": %m", pathname);
        streamer->should_close_file = true;
    }
 
    return &streamer->base;
}

References astreamer_plain_writer_ops, astreamer_plain_writer::base, astreamer::bbs_ops, fb(), astreamer_plain_writer::file, palloc0_object, astreamer_plain_writer::pathname, pg_fatal, pstrdup(), and astreamer_plain_writer::should_close_file.

Referenced by CreateBackupStreamer().

astreamer_tar_archiver_new()

astreamer * astreamer_tar_archiver_new ( astreamer *  next )

extern

Definition at line 388 of file astreamer_tar.c.

{
    astreamer_tar_archiver *streamer;
 
    streamer = palloc0_object(astreamer_tar_archiver);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_tar_archiver_ops;
    streamer->base.bbs_next = next;
 
    return &streamer->base;
}

References astreamer_tar_archiver_ops, astreamer_tar_archiver::base, astreamer::bbs_next, astreamer::bbs_ops, next, and palloc0_object.

Referenced by CreateBackupStreamer().

astreamer_tar_parser_new()

astreamer * astreamer_tar_parser_new ( astreamer *  next )

extern

Definition at line 93 of file astreamer_tar.c.

{
    astreamer_tar_parser *streamer;
 
    streamer = palloc0_object(astreamer_tar_parser);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_tar_parser_ops;
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    streamer->next_context = ASTREAMER_MEMBER_HEADER;
 
    return &streamer->base;
}

References ASTREAMER_MEMBER_HEADER, astreamer_tar_parser_ops, astreamer_tar_parser::base, astreamer::bbs_buffer, astreamer::bbs_next, astreamer::bbs_ops, initStringInfo(), next, astreamer_tar_parser::next_context, and palloc0_object.

Referenced by create_archive_verifier(), CreateBackupStreamer(), and init_archive_reader().

astreamer_tar_terminator_new()

astreamer * astreamer_tar_terminator_new ( astreamer *  next )

extern

Definition at line 494 of file astreamer_tar.c.

{
    astreamer  *streamer;
 
    streamer = palloc0_object(astreamer);
    *((const astreamer_ops **) &streamer->bbs_ops) =
        &astreamer_tar_terminator_ops;
    streamer->bbs_next = next;
 
    return streamer;
}

References astreamer_tar_terminator_ops, astreamer::bbs_next, astreamer::bbs_ops, next, and palloc0_object.

Referenced by CreateBackupStreamer().

astreamer_zstd_compressor_new()

astreamer * astreamer_zstd_compressor_new ( astreamer *  next, pg_compress_specification *  compress  )

extern

Definition at line 70 of file astreamer_zstd.c.

{
#ifdef USE_ZSTD
    astreamer_zstd_frame *streamer;
    size_t      ret;
 
    Assert(next != NULL);
 
    streamer = palloc0_object(astreamer_zstd_frame);
 
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_zstd_compressor_ops;
 
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_CStreamOutSize());
 
    streamer->cctx = ZSTD_createCCtx();
    if (!streamer->cctx)
        pg_fatal("could not create zstd compression context");
 
    /* Set compression level */
    ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel,
                                 compress->level);
    if (ZSTD_isError(ret))
        pg_fatal("could not set zstd compression level to %d: %s",
                 compress->level, ZSTD_getErrorName(ret));
 
    /* Set # of workers, if specified */
    if ((compress->options & PG_COMPRESSION_OPTION_WORKERS) != 0)
    {
        /*
         * On older versions of libzstd, this option does not exist, and
         * trying to set it will fail. Similarly for newer versions if they
         * are compiled without threading support.
         */
        ret = ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_nbWorkers,
                                     compress->workers);
        if (ZSTD_isError(ret))
            pg_fatal("could not set compression worker count to %d: %s",
                     compress->workers, ZSTD_getErrorName(ret));
    }
 
    if ((compress->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) != 0)
    {
        ret = ZSTD_CCtx_setParameter(streamer->cctx,
                                     ZSTD_c_enableLongDistanceMatching,
                                     compress->long_distance);
        if (ZSTD_isError(ret))
            pg_fatal("could not enable long-distance mode: %s",
                     ZSTD_getErrorName(ret));
    }
 
    /* Initialize the ZSTD output buffer. */
    streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
    streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
    streamer->zstd_outBuf.pos = 0;
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "ZSTD");
    return NULL;                /* keep compiler quiet */
#endif
}

References Assert, enlargeStringInfo(), fb(), initStringInfo(), pg_compress_specification::level, pg_compress_specification::long_distance, next, pg_compress_specification::options, palloc0_object, PG_COMPRESSION_OPTION_LONG_DISTANCE, PG_COMPRESSION_OPTION_WORKERS, pg_fatal, and pg_compress_specification::workers.

Referenced by CreateBackupStreamer().

astreamer_zstd_decompressor_new()

astreamer * astreamer_zstd_decompressor_new ( astreamer *  next )

extern

Definition at line 259 of file astreamer_zstd.c.

{
#ifdef USE_ZSTD
    astreamer_zstd_frame *streamer;
 
    Assert(next != NULL);
 
    streamer = palloc0_object(astreamer_zstd_frame);
    *((const astreamer_ops **) &streamer->base.bbs_ops) =
        &astreamer_zstd_decompressor_ops;
 
    streamer->base.bbs_next = next;
    initStringInfo(&streamer->base.bbs_buffer);
    enlargeStringInfo(&streamer->base.bbs_buffer, ZSTD_DStreamOutSize());
 
    streamer->dctx = ZSTD_createDCtx();
    if (!streamer->dctx)
        pg_fatal("could not create zstd decompression context");
 
    /* Initialize the ZSTD output buffer. */
    streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data;
    streamer->zstd_outBuf.size = streamer->base.bbs_buffer.maxlen;
    streamer->zstd_outBuf.pos = 0;
 
    return &streamer->base;
#else
    pg_fatal("this build does not support compression with %s", "ZSTD");
    return NULL;                /* keep compiler quiet */
#endif
}

References Assert, enlargeStringInfo(), fb(), initStringInfo(), next, palloc0_object, and pg_fatal.

Referenced by create_archive_verifier(), CreateBackupStreamer(), and init_archive_reader().