detoast.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
#define	TOAST_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
#define	INDIRECT_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))

Functions
struct varlena *	detoast_external_attr (struct varlena *attr)
struct varlena *	detoast_attr (struct varlena *attr)
struct varlena *	detoast_attr_slice (struct varlena *attr, int32 sliceoffset, int32 slicelength)
Size	toast_raw_datum_size (Datum value)
Size	toast_datum_size (Datum value)

Macro Definition Documentation

INDIRECT_POINTER_SIZE

#define INDIRECT_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))

Definition at line 34 of file detoast.h.

TOAST_POINTER_SIZE

#define TOAST_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_external))

Definition at line 31 of file detoast.h.

VARATT_EXTERNAL_GET_POINTER

#define VARATT_EXTERNAL_GET_POINTER	(		toast_pointer,
			attr
	)

Value:

do { \
    varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
    Assert(VARATT_IS_EXTERNAL(attre)); \
    Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
    memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)

Definition at line 22 of file detoast.h.

Function Documentation

detoast_attr()

struct varlena * detoast_attr

(

struct varlena *

attr

)

Definition at line 116 of file detoast.c.

{
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        /*
         * This is an externally stored datum --- fetch it back from there
         */
        attr = toast_fetch_datum(attr);
        /* If it's compressed, decompress it */
        if (VARATT_IS_COMPRESSED(attr))
        {
            struct varlena *tmp = attr;
 
            attr = toast_decompress_datum(tmp);
            pfree(tmp);
        }
    }
    else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
    {
        /*
         * This is an indirect pointer --- dereference it
         */
        struct varatt_indirect redirect;
 
        VARATT_EXTERNAL_GET_POINTER(redirect, attr);
        attr = (struct varlena *) redirect.pointer;
 
        /* nested indirect Datums aren't allowed */
        Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
 
        /* recurse in case value is still extended in some other way */
        attr = detoast_attr(attr);
 
        /* if it isn't, we'd better copy it */
        if (attr == (struct varlena *) redirect.pointer)
        {
            struct varlena *result;
 
            result = (struct varlena *) palloc(VARSIZE_ANY(attr));
            memcpy(result, attr, VARSIZE_ANY(attr));
            attr = result;
        }
    }
    else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
    {
        /*
         * This is an expanded-object pointer --- get flat format
         */
        attr = detoast_external_attr(attr);
        /* flatteners are not allowed to produce compressed/short output */
        Assert(!VARATT_IS_EXTENDED(attr));
    }
    else if (VARATT_IS_COMPRESSED(attr))
    {
        /*
         * This is a compressed value inside of the main tuple
         */
        attr = toast_decompress_datum(attr);
    }
    else if (VARATT_IS_SHORT(attr))
    {
        /*
         * This is a short-header varlena --- convert to 4-byte header format
         */
        Size        data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT;
        Size        new_size = data_size + VARHDRSZ;
        struct varlena *new_attr;
 
        new_attr = (struct varlena *) palloc(new_size);
        SET_VARSIZE(new_attr, new_size);
        memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size);
        attr = new_attr;
    }
 
    return attr;
}

References Assert(), detoast_attr(), detoast_external_attr(), palloc(), pfree(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTENDED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE_ANY, and VARSIZE_SHORT.

Referenced by detoast_attr(), getdatafield(), pg_detoast_datum(), pg_detoast_datum_copy(), pg_detoast_datum_packed(), toast_flatten_tuple_to_datum(), and toast_tuple_init().

detoast_attr_slice()

struct varlena * detoast_attr_slice	(	struct varlena *	attr,
		int32	sliceoffset,
		int32	slicelength
	)

Definition at line 205 of file detoast.c.

{
    struct varlena *preslice;
    struct varlena *result;
    char       *attrdata;
    int32       slicelimit;
    int32       attrsize;
 
    if (sliceoffset < 0)
        elog(ERROR, "invalid sliceoffset: %d", sliceoffset);
 
    /*
     * Compute slicelimit = offset + length, or -1 if we must fetch all of the
     * value.  In case of integer overflow, we must fetch all.
     */
    if (slicelength < 0)
        slicelimit = -1;
    else if (pg_add_s32_overflow(sliceoffset, slicelength, &slicelimit))
        slicelength = slicelimit = -1;
 
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        /* fast path for non-compressed external datums */
        if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
            return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
 
        /*
         * For compressed values, we need to fetch enough slices to decompress
         * at least the requested part (when a prefix is requested).
         * Otherwise, just fetch all slices.
         */
        if (slicelimit >= 0)
        {
            int32       max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
 
            /*
             * Determine maximum amount of compressed data needed for a prefix
             * of a given length (after decompression).
             *
             * At least for now, if it's LZ4 data, we'll have to fetch the
             * whole thing, because there doesn't seem to be an API call to
             * determine how much compressed data we need to be sure of being
             * able to decompress the required slice.
             */
            if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
                TOAST_PGLZ_COMPRESSION_ID)
                max_size = pglz_maximum_compressed_size(slicelimit, max_size);
 
            /*
             * Fetch enough compressed slices (compressed marker will get set
             * automatically).
             */
            preslice = toast_fetch_datum_slice(attr, 0, max_size);
        }
        else
            preslice = toast_fetch_datum(attr);
    }
    else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
    {
        struct varatt_indirect redirect;
 
        VARATT_EXTERNAL_GET_POINTER(redirect, attr);
 
        /* nested indirect Datums aren't allowed */
        Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer));
 
        return detoast_attr_slice(redirect.pointer,
                                  sliceoffset, slicelength);
    }
    else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
    {
        /* pass it off to detoast_external_attr to flatten */
        preslice = detoast_external_attr(attr);
    }
    else
        preslice = attr;
 
    Assert(!VARATT_IS_EXTERNAL(preslice));
 
    if (VARATT_IS_COMPRESSED(preslice))
    {
        struct varlena *tmp = preslice;
 
        /* Decompress enough to encompass the slice and the offset */
        if (slicelimit >= 0)
            preslice = toast_decompress_datum_slice(tmp, slicelimit);
        else
            preslice = toast_decompress_datum(tmp);
 
        if (tmp != attr)
            pfree(tmp);
    }
 
    if (VARATT_IS_SHORT(preslice))
    {
        attrdata = VARDATA_SHORT(preslice);
        attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
    }
    else
    {
        attrdata = VARDATA(preslice);
        attrsize = VARSIZE(preslice) - VARHDRSZ;
    }
 
    /* slicing of datum for compressed cases and plain value */
 
    if (sliceoffset >= attrsize)
    {
        sliceoffset = 0;
        slicelength = 0;
    }
    else if (slicelength < 0 || slicelimit > attrsize)
        slicelength = attrsize - sliceoffset;
 
    result = (struct varlena *) palloc(slicelength + VARHDRSZ);
    SET_VARSIZE(result, slicelength + VARHDRSZ);
 
    memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
 
    if (preslice != attr)
        pfree(preslice);
 
    return result;
}

References Assert(), detoast_attr_slice(), detoast_external_attr(), elog, ERROR, palloc(), pfree(), pg_add_s32_overflow(), pglz_maximum_compressed_size(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_decompress_datum_slice(), toast_fetch_datum(), toast_fetch_datum_slice(), TOAST_PGLZ_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by detoast_attr_slice(), and pg_detoast_datum_slice().

detoast_external_attr()

struct varlena * detoast_external_attr

(

struct varlena *

attr

)

Definition at line 45 of file detoast.c.

{
    struct varlena *result;
 
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        /*
         * This is an external stored plain value
         */
        result = toast_fetch_datum(attr);
    }
    else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
    {
        /*
         * This is an indirect pointer --- dereference it
         */
        struct varatt_indirect redirect;
 
        VARATT_EXTERNAL_GET_POINTER(redirect, attr);
        attr = (struct varlena *) redirect.pointer;
 
        /* nested indirect Datums aren't allowed */
        Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
 
        /* recurse if value is still external in some other way */
        if (VARATT_IS_EXTERNAL(attr))
            return detoast_external_attr(attr);
 
        /*
         * Copy into the caller's memory context, in case caller tries to
         * pfree the result.
         */
        result = (struct varlena *) palloc(VARSIZE_ANY(attr));
        memcpy(result, attr, VARSIZE_ANY(attr));
    }
    else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
    {
        /*
         * This is an expanded-object pointer --- get flat format
         */
        ExpandedObjectHeader *eoh;
        Size        resultsize;
 
        eoh = DatumGetEOHP(PointerGetDatum(attr));
        resultsize = EOH_get_flat_size(eoh);
        result = (struct varlena *) palloc(resultsize);
        EOH_flatten_into(eoh, result, resultsize);
    }
    else
    {
        /*
         * This is a plain value inside of the main tuple - why am I called?
         */
        result = attr;
    }
 
    return result;
}

References Assert(), DatumGetEOHP(), detoast_external_attr(), EOH_flatten_into(), EOH_get_flat_size(), palloc(), varatt_indirect::pointer, PointerGetDatum(), toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, and VARSIZE_ANY.

Referenced by assign_simple_var(), brin_form_tuple(), detoast_attr(), detoast_attr_slice(), detoast_external_attr(), expanded_record_set_field_internal(), expanded_record_set_fields(), index_form_tuple_context(), make_tuple_indirect(), toast_build_flattened_tuple(), toast_flatten_tuple(), toast_tuple_init(), and tstoreReceiveSlot_detoast().

toast_datum_size()

Size toast_datum_size

(

Datum

value

)

Definition at line 601 of file detoast.c.

{
    struct varlena *attr = (struct varlena *) DatumGetPointer(value);
    Size        result;
 
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        /*
         * Attribute is stored externally - return the extsize whether
         * compressed or not.  We do not count the size of the toast pointer
         * ... should we?
         */
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
        result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
    }
    else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
    {
        struct varatt_indirect toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        /* nested indirect Datums aren't allowed */
        Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
 
        return toast_datum_size(PointerGetDatum(toast_pointer.pointer));
    }
    else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
    {
        result = EOH_get_flat_size(DatumGetEOHP(value));
    }
    else if (VARATT_IS_SHORT(attr))
    {
        result = VARSIZE_SHORT(attr);
    }
    else
    {
        /*
         * Attribute is stored inline either compressed or not, just calculate
         * the size of the datum in either case.
         */
        result = VARSIZE(attr);
    }
    return result;
}

References Assert(), DatumGetEOHP(), DatumGetPointer(), EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum(), toast_datum_size(), value, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by pg_column_size(), and toast_datum_size().

toast_raw_datum_size()

Size toast_raw_datum_size

(

Datum

value

)

Definition at line 545 of file detoast.c.

{
    struct varlena *attr = (struct varlena *) DatumGetPointer(value);
    Size        result;
 
    if (VARATT_IS_EXTERNAL_ONDISK(attr))
    {
        /* va_rawsize is the size of the original datum -- including header */
        struct varatt_external toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
        result = toast_pointer.va_rawsize;
    }
    else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
    {
        struct varatt_indirect toast_pointer;
 
        VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
        /* nested indirect Datums aren't allowed */
        Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer));
 
        return toast_raw_datum_size(PointerGetDatum(toast_pointer.pointer));
    }
    else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
    {
        result = EOH_get_flat_size(DatumGetEOHP(value));
    }
    else if (VARATT_IS_COMPRESSED(attr))
    {
        /* here, va_rawsize is just the payload size */
        result = VARDATA_COMPRESSED_GET_EXTSIZE(attr) + VARHDRSZ;
    }
    else if (VARATT_IS_SHORT(attr))
    {
        /*
         * we have to normalize the header length to VARHDRSZ or else the
         * callers of this function will be confused.
         */
        result = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT + VARHDRSZ;
    }
    else
    {
        /* plain untoasted datum */
        result = VARSIZE(attr);
    }
    return result;
}

References Assert(), DatumGetEOHP(), DatumGetPointer(), EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum(), toast_raw_datum_size(), varatt_external::va_rawsize, value, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by bpcharoctetlen(), build_sorted_items(), byteaeq(), byteane(), byteaoctetlen(), compute_array_stats(), compute_distinct_stats(), compute_scalar_stats(), datum_image_eq(), datum_image_hash(), record_image_cmp(), text_length(), text_starts_with(), texteq(), textne(), textoctetlen(), and toast_raw_datum_size().