toast_compression.c File Reference

#include "postgres.h"
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"

Include dependency graph for toast_compression.c:

Go to the source code of this file.

Macros
#define	NO_COMPRESSION_SUPPORT(method)

Functions
struct varlena *	pglz_compress_datum (const struct varlena *value)
struct varlena *	pglz_decompress_datum (const struct varlena *value)
struct varlena *	pglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
struct varlena *	lz4_compress_datum (const struct varlena *value)
struct varlena *	lz4_decompress_datum (const struct varlena *value)
struct varlena *	lz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
ToastCompressionId	toast_get_compression_id (struct varlena *attr)
char	CompressionNameToMethod (const char *compression)
const char *	GetCompressionMethodName (char method)

Variables
int	default_toast_compression = TOAST_PGLZ_COMPRESSION

Macro Definition Documentation

NO_COMPRESSION_SUPPORT

#define NO_COMPRESSION_SUPPORT

(

method

)

Value:

    ereport(ERROR, \
            (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
             errmsg("compression method %s not supported", method), \
             errdetail("This functionality requires the server to be built with %s support.", method)))

Definition at line 28 of file toast_compression.c.

{
    int32       valsize,
                len;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * No point in wasting a palloc cycle if value size is outside the allowed
     * range for compression.
     */
    if (valsize < PGLZ_strategy_default->min_input_size ||
        valsize > PGLZ_strategy_default->max_input_size)
        return NULL;
 
    /*
     * Figure out the maximum possible size of the pglz output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
                                    VARHDRSZ_COMPRESSED);
 
    len = pglz_compress(VARDATA_ANY(value),
                        valsize,
                        (char *) tmp + VARHDRSZ_COMPRESSED,
                        NULL);
    if (len < 0)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
}
 
/*
 * Decompress a varlena that was compressed using PGLZ.
 */
struct varlena *
pglz_decompress_datum(const struct varlena *value)
{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((const char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}
 
/*
 * Decompress part of a varlena that was compressed using PGLZ.
 */
struct varlena *
pglz_decompress_datum_slice(const struct varlena *value,
                            int32 slicelength)
{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((const char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              slicelength, false);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}
 
/*
 * Compress a varlena using LZ4.
 *
 * Returns the compressed varlena, or NULL if compression fails.
 */
struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       valsize;
    int32       len;
    int32       max_size;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * Figure out the maximum possible size of the LZ4 output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    max_size = LZ4_compressBound(valsize);
    tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
 
    len = LZ4_compress_default(VARDATA_ANY(value),
                               (char *) tmp + VARHDRSZ_COMPRESSED,
                               valsize, max_size);
    if (len <= 0)
        elog(ERROR, "lz4 compression failed");
 
    /* data is incompressible so just free the memory and return NULL */
    if (len > valsize)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
#endif
}
 
/*
 * Decompress a varlena that was compressed using LZ4.
 */
struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe((const char *) value + VARHDRSZ_COMPRESSED,
                                  VARDATA(result),
                                  VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                  VARDATA_COMPRESSED_GET_EXTSIZE(value));
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}
 
/*
 * Decompress part of a varlena that was compressed using LZ4.
 */
struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* slice decompression not supported prior to 1.8.3 */
    if (LZ4_versionNumber() < 10803)
        return lz4_decompress_datum(value);
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe_partial((const char *) value + VARHDRSZ_COMPRESSED,
                                          VARDATA(result),
                                          VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                          slicelength,
                                          slicelength);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}
 
/*
 * Extract compression ID from a varlena.
 *
 * Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed.
 */
ToastCompressionId
toast_get_compression_id(struct varlena *attr)
{
    ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
 
    /*
     * If it is stored externally then fetch the compression method id from
     * the external toast pointer.  If compressed inline, fetch it from the
     * toast compression header.
     */
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
            cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
    }
    else if (VARATT_IS_COMPRESSED(attr))
        cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 
    return cmid;
}
 
/*
 * CompressionNameToMethod - Get compression method from compression name
 *
 * Search in the available built-in methods.  If the compression not found
 * in the built-in methods then return InvalidCompressionMethod.
 */
char
CompressionNameToMethod(const char *compression)
{
    if (strcmp(compression, "pglz") == 0)
        return TOAST_PGLZ_COMPRESSION;
    else if (strcmp(compression, "lz4") == 0)
    {
#ifndef USE_LZ4
        NO_COMPRESSION_SUPPORT("lz4");
#endif
        return TOAST_LZ4_COMPRESSION;
    }
 
    return InvalidCompressionMethod;
}
 
/*
 * GetCompressionMethodName - Get compression method name
 */
const char *
GetCompressionMethodName(char method)
{
    switch (method)
    {
        case TOAST_PGLZ_COMPRESSION:
            return "pglz";
        case TOAST_LZ4_COMPRESSION:
            return "lz4";
        default:
            elog(ERROR, "invalid compression method %c", method);
            return NULL;        /* keep compiler quiet */
    }
}

Function Documentation

CompressionNameToMethod()

char CompressionNameToMethod

(

const char *

compression

)

Definition at line 285 of file toast_compression.c.

{
    if (strcmp(compression, "pglz") == 0)
        return TOAST_PGLZ_COMPRESSION;
    else if (strcmp(compression, "lz4") == 0)
    {
#ifndef USE_LZ4
        NO_COMPRESSION_SUPPORT("lz4");
#endif
        return TOAST_LZ4_COMPRESSION;
    }
 
    return InvalidCompressionMethod;
}

References fb(), InvalidCompressionMethod, NO_COMPRESSION_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression().

GetCompressionMethodName()

const char * GetCompressionMethodName

(

char

method

)

Definition at line 304 of file toast_compression.c.

{
    switch (method)
    {
        case TOAST_PGLZ_COMPRESSION:
            return "pglz";
        case TOAST_LZ4_COMPRESSION:
            return "lz4";
        default:
            elog(ERROR, "invalid compression method %c", method);
            return NULL;        /* keep compiler quiet */
    }
}

References elog, ERROR, fb(), TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by getAttributesList(), MergeAttributes(), and transformTableLikeClause().

lz4_compress_datum()

struct varlena * lz4_compress_datum

(

const struct varlena *

value

)

Definition at line 139 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       valsize;
    int32       len;
    int32       max_size;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * Figure out the maximum possible size of the LZ4 output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    max_size = LZ4_compressBound(valsize);
    tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
 
    len = LZ4_compress_default(VARDATA_ANY(value),
                               (char *) tmp + VARHDRSZ_COMPRESSED,
                               valsize, max_size);
    if (len <= 0)
        elog(ERROR, "lz4 compression failed");
 
    /* data is incompressible so just free the memory and return NULL */
    if (len > valsize)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
#endif
}

References elog, ERROR, fb(), len, NO_COMPRESSION_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED(), value, VARDATA_ANY(), VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR().

Referenced by toast_compress_datum().

lz4_decompress_datum()

struct varlena * lz4_decompress_datum

(

const struct varlena *

value

)

Definition at line 182 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe((const char *) value + VARHDRSZ_COMPRESSED,
                                  VARDATA(result),
                                  VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                  VARDATA_COMPRESSED_GET_EXTSIZE(value));
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, fb(), NO_COMPRESSION_SUPPORT, palloc(), SET_VARSIZE(), value, VARDATA(), VARDATA_COMPRESSED_GET_EXTSIZE(), VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE().

Referenced by lz4_decompress_datum_slice(), and toast_decompress_datum().

lz4_decompress_datum_slice()

struct varlena * lz4_decompress_datum_slice	(	const struct varlena *	value,
		int32	slicelength
	)

Definition at line 215 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_COMPRESSION_SUPPORT("lz4");
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* slice decompression not supported prior to 1.8.3 */
    if (LZ4_versionNumber() < 10803)
        return lz4_decompress_datum(value);
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe_partial((const char *) value + VARHDRSZ_COMPRESSED,
                                          VARDATA(result),
                                          VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                          slicelength,
                                          slicelength);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, fb(), lz4_decompress_datum(), NO_COMPRESSION_SUPPORT, palloc(), SET_VARSIZE(), value, VARDATA(), VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE().

Referenced by toast_decompress_datum_slice().

pglz_compress_datum()

struct varlena * pglz_compress_datum

(

const struct varlena *

value

)

Definition at line 40 of file toast_compression.c.

{
    int32       valsize,
                len;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * No point in wasting a palloc cycle if value size is outside the allowed
     * range for compression.
     */
    if (valsize < PGLZ_strategy_default->min_input_size ||
        valsize > PGLZ_strategy_default->max_input_size)
        return NULL;
 
    /*
     * Figure out the maximum possible size of the pglz output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
                                    VARHDRSZ_COMPRESSED);
 
    len = pglz_compress(VARDATA_ANY(value),
                        valsize,
                        (char *) tmp + VARHDRSZ_COMPRESSED,
                        NULL);
    if (len < 0)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
}

References fb(), len, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED(), value, VARDATA_ANY(), VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR().

Referenced by toast_compress_datum().

pglz_decompress_datum()

struct varlena * pglz_decompress_datum

(

const struct varlena *

value

)

Definition at line 82 of file toast_compression.c.

{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((const char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, fb(), palloc(), pglz_decompress(), SET_VARSIZE(), value, VARDATA(), VARDATA_COMPRESSED_GET_EXTSIZE(), VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE().

Referenced by toast_decompress_datum().

pglz_decompress_datum_slice()

struct varlena * pglz_decompress_datum_slice	(	const struct varlena *	value,
		int32	slicelength
	)

Definition at line 109 of file toast_compression.c.

{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((const char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              slicelength, false);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, fb(), palloc(), pglz_decompress(), SET_VARSIZE(), value, VARDATA(), VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE().

Referenced by toast_decompress_datum_slice().

toast_get_compression_id()

ToastCompressionId toast_get_compression_id

(

struct varlena *

attr

)

Definition at line 254 of file toast_compression.c.

{
    ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
 
    /*
     * If it is stored externally then fetch the compression method id from
     * the external toast pointer.  If compressed inline, fetch it from the
     * toast compression header.
     */
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
            cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
    }
    else if (VARATT_IS_COMPRESSED(attr))
        cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 
    return cmid;
}

References fb(), TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD(), VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED(), VARATT_IS_COMPRESSED(), VARATT_IS_EXTERNAL_ONDISK(), and VARDATA_COMPRESSED_GET_COMPRESS_METHOD().

Referenced by pg_column_compression().

Variable Documentation

default_toast_compression

int default_toast_compression = TOAST_PGLZ_COMPRESSION

Definition at line 26 of file toast_compression.c.

Referenced by toast_compress_datum().