toast_compression.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	TOAST_PGLZ_COMPRESSION   'p'
#define	TOAST_LZ4_COMPRESSION   'l'
#define	InvalidCompressionMethod   '\0'
#define	CompressionMethodIsValid(cm)   ((cm) != InvalidCompressionMethod)

Typedefs
typedef enum ToastCompressionId	ToastCompressionId

Enumerations
enum	ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0 , TOAST_LZ4_COMPRESSION_ID = 1 , TOAST_INVALID_COMPRESSION_ID = 2 }

Functions
struct varlena *	pglz_compress_datum (const struct varlena *value)
struct varlena *	pglz_decompress_datum (const struct varlena *value)
struct varlena *	pglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
struct varlena *	lz4_compress_datum (const struct varlena *value)
struct varlena *	lz4_decompress_datum (const struct varlena *value)
struct varlena *	lz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
ToastCompressionId	toast_get_compression_id (struct varlena *attr)
char	CompressionNameToMethod (const char *compression)
const char *	GetCompressionMethodName (char method)

Variables
PGDLLIMPORT int	default_toast_compression

Macro Definition Documentation

CompressionMethodIsValid

#define CompressionMethodIsValid

(

cm

)

((cm) != InvalidCompressionMethod)

Definition at line 53 of file toast_compression.h.

InvalidCompressionMethod

#define InvalidCompressionMethod   '\0'

Definition at line 51 of file toast_compression.h.

TOAST_LZ4_COMPRESSION

#define TOAST_LZ4_COMPRESSION   'l'

Definition at line 50 of file toast_compression.h.

TOAST_PGLZ_COMPRESSION

#define TOAST_PGLZ_COMPRESSION   'p'

Definition at line 49 of file toast_compression.h.

Typedef Documentation

ToastCompressionId

typedef enum ToastCompressionId ToastCompressionId

Enumeration Type Documentation

ToastCompressionId

enum ToastCompressionId

Enumerator
TOAST_PGLZ_COMPRESSION_ID
TOAST_LZ4_COMPRESSION_ID
TOAST_INVALID_COMPRESSION_ID

Definition at line 37 of file toast_compression.h.

{
    TOAST_PGLZ_COMPRESSION_ID = 0,
    TOAST_LZ4_COMPRESSION_ID = 1,
    TOAST_INVALID_COMPRESSION_ID = 2,
} ToastCompressionId;

Function Documentation

CompressionNameToMethod()

char CompressionNameToMethod

(

const char *

compression

)

Definition at line 285 of file toast_compression.c.

{
    if (strcmp(compression, "pglz") == 0)
        return TOAST_PGLZ_COMPRESSION;
    else if (strcmp(compression, "lz4") == 0)
    {
#ifndef USE_LZ4
        NO_LZ4_SUPPORT();
#endif
        return TOAST_LZ4_COMPRESSION;
    }
 
    return InvalidCompressionMethod;
}

References InvalidCompressionMethod, NO_LZ4_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression().

GetCompressionMethodName()

const char* GetCompressionMethodName

(

char

method

)

Definition at line 304 of file toast_compression.c.

{
    switch (method)
    {
        case TOAST_PGLZ_COMPRESSION:
            return "pglz";
        case TOAST_LZ4_COMPRESSION:
            return "lz4";
        default:
            elog(ERROR, "invalid compression method %c", method);
            return NULL;        /* keep compiler quiet */
    }
}

References elog, ERROR, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by MergeAttributes(), and transformTableLikeClause().

lz4_compress_datum()

struct varlena* lz4_compress_datum

(

const struct varlena *

value

)

Definition at line 139 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_LZ4_SUPPORT();
    return NULL;                /* keep compiler quiet */
#else
    int32       valsize;
    int32       len;
    int32       max_size;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * Figure out the maximum possible size of the LZ4 output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    max_size = LZ4_compressBound(valsize);
    tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
 
    len = LZ4_compress_default(VARDATA_ANY(value),
                               (char *) tmp + VARHDRSZ_COMPRESSED,
                               valsize, max_size);
    if (len <= 0)
        elog(ERROR, "lz4 compression failed");
 
    /* data is incompressible so just free the memory and return NULL */
    if (len > valsize)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
#endif
}

References elog, ERROR, len, NO_LZ4_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

lz4_decompress_datum()

struct varlena* lz4_decompress_datum

(

const struct varlena *

value

)

Definition at line 182 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_LZ4_SUPPORT();
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
                                  VARDATA(result),
                                  VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                  VARDATA_COMPRESSED_GET_EXTSIZE(value));
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by lz4_decompress_datum_slice(), and toast_decompress_datum().

lz4_decompress_datum_slice()

struct varlena* lz4_decompress_datum_slice	(	const struct varlena *	value,
		int32	slicelength
	)

Definition at line 215 of file toast_compression.c.

{
#ifndef USE_LZ4
    NO_LZ4_SUPPORT();
    return NULL;                /* keep compiler quiet */
#else
    int32       rawsize;
    struct varlena *result;
 
    /* slice decompression not supported prior to 1.8.3 */
    if (LZ4_versionNumber() < 10803)
        return lz4_decompress_datum(value);
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
                                          VARDATA(result),
                                          VARSIZE(value) - VARHDRSZ_COMPRESSED,
                                          slicelength,
                                          slicelength);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed lz4 data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
#endif
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, lz4_decompress_datum(), NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

pglz_compress_datum()

struct varlena* pglz_compress_datum

(

const struct varlena *

value

)

Definition at line 40 of file toast_compression.c.

{
    int32       valsize,
                len;
    struct varlena *tmp = NULL;
 
    valsize = VARSIZE_ANY_EXHDR(value);
 
    /*
     * No point in wasting a palloc cycle if value size is outside the allowed
     * range for compression.
     */
    if (valsize < PGLZ_strategy_default->min_input_size ||
        valsize > PGLZ_strategy_default->max_input_size)
        return NULL;
 
    /*
     * Figure out the maximum possible size of the pglz output, add the bytes
     * that will be needed for varlena overhead, and allocate that amount.
     */
    tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
                                    VARHDRSZ_COMPRESSED);
 
    len = pglz_compress(VARDATA_ANY(value),
                        valsize,
                        (char *) tmp + VARHDRSZ_COMPRESSED,
                        NULL);
    if (len < 0)
    {
        pfree(tmp);
        return NULL;
    }
 
    SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
 
    return tmp;
}

References len, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

pglz_decompress_datum()

struct varlena* pglz_decompress_datum

(

const struct varlena *

value

)

Definition at line 82 of file toast_compression.c.

{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum().

pglz_decompress_datum_slice()

struct varlena* pglz_decompress_datum_slice	(	const struct varlena *	value,
		int32	slicelength
	)

Definition at line 109 of file toast_compression.c.

{
    struct varlena *result;
    int32       rawsize;
 
    /* allocate memory for the uncompressed data */
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
 
    /* decompress the data */
    rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
                              VARSIZE(value) - VARHDRSZ_COMPRESSED,
                              VARDATA(result),
                              slicelength, false);
    if (rawsize < 0)
        ereport(ERROR,
                (errcode(ERRCODE_DATA_CORRUPTED),
                 errmsg_internal("compressed pglz data is corrupt")));
 
    SET_VARSIZE(result, rawsize + VARHDRSZ);
 
    return result;
}

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

toast_get_compression_id()

ToastCompressionId toast_get_compression_id

(

struct varlena *

attr

)

Definition at line 254 of file toast_compression.c.

{
    ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
 
    /*
     * If it is stored externally then fetch the compression method id from
     * the external toast pointer.  If compressed inline, fetch it from the
     * toast compression header.
     */
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
            cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
    }
    else if (VARATT_IS_COMPRESSED(attr))
        cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
 
    return cmid;
}

References TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARDATA_COMPRESSED_GET_COMPRESS_METHOD.

Referenced by pg_column_compression().

Variable Documentation

default_toast_compression

PGDLLIMPORT int default_toast_compression

extern

Definition at line 26 of file toast_compression.c.

Referenced by toast_compress_datum().