PostgreSQL Source Code  git master
extended_stats_internal.h File Reference
Include dependency graph for extended_stats_internal.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  StdAnalyzeData
 
struct  ScalarItem
 
struct  DimensionInfo
 
struct  MultiSortSupportData
 
struct  SortItem
 
struct  StatsBuildData
 

Typedefs

typedef struct DimensionInfo DimensionInfo
 
typedef struct MultiSortSupportData MultiSortSupportData
 
typedef MultiSortSupportDataMultiSortSupport
 
typedef struct SortItem SortItem
 
typedef struct StatsBuildData StatsBuildData
 

Functions

MVNDistinctstatext_ndistinct_build (double totalrows, StatsBuildData *data)
 
byteastatext_ndistinct_serialize (MVNDistinct *ndistinct)
 
MVNDistinctstatext_ndistinct_deserialize (bytea *data)
 
MVDependenciesstatext_dependencies_build (StatsBuildData *data)
 
byteastatext_dependencies_serialize (MVDependencies *dependencies)
 
MVDependenciesstatext_dependencies_deserialize (bytea *data)
 
MCVListstatext_mcv_build (StatsBuildData *data, double totalrows, int stattarget)
 
byteastatext_mcv_serialize (MCVList *mcv, VacAttrStats **stats)
 
MCVListstatext_mcv_deserialize (bytea *data)
 
MultiSortSupport multi_sort_init (int ndims)
 
void multi_sort_add_dimension (MultiSortSupport mss, int sortdim, Oid oper, Oid collation)
 
int multi_sort_compare (const void *a, const void *b, void *arg)
 
int multi_sort_compare_dim (int dim, const SortItem *a, const SortItem *b, MultiSortSupport mss)
 
int multi_sort_compare_dims (int start, int end, const SortItem *a, const SortItem *b, MultiSortSupport mss)
 
int compare_scalars_simple (const void *a, const void *b, void *arg)
 
int compare_datums_simple (Datum a, Datum b, SortSupport ssup)
 
AttrNumberbuild_attnums_array (Bitmapset *attrs, int nexprs, int *numattrs)
 
SortItembuild_sorted_items (StatsBuildData *data, int *nitems, MultiSortSupport mss, int numattrs, AttrNumber *attnums)
 
bool examine_opclause_args (List *args, Node **exprp, Const **cstp, bool *expronleftp)
 
Selectivity mcv_combine_selectivities (Selectivity simple_sel, Selectivity mcv_sel, Selectivity mcv_basesel, Selectivity mcv_totalsel)
 
Selectivity mcv_clauselist_selectivity (PlannerInfo *root, StatisticExtInfo *stat, List *clauses, int varRelid, JoinType jointype, SpecialJoinInfo *sjinfo, RelOptInfo *rel, Selectivity *basesel, Selectivity *totalsel)
 
Selectivity mcv_clause_selectivity_or (PlannerInfo *root, StatisticExtInfo *stat, MCVList *mcv, Node *clause, bool **or_matches, Selectivity *basesel, Selectivity *overlap_mcvsel, Selectivity *overlap_basesel, Selectivity *totalsel)
 

Typedef Documentation

◆ DimensionInfo

typedef struct DimensionInfo DimensionInfo

◆ MultiSortSupport

Definition at line 51 of file extended_stats_internal.h.

◆ MultiSortSupportData

◆ SortItem

typedef struct SortItem SortItem

◆ StatsBuildData

Function Documentation

◆ build_attnums_array()

AttrNumber* build_attnums_array ( Bitmapset attrs,
int  nexprs,
int *  numattrs 
)

Definition at line 965 of file extended_stats.c.

References Assert, attnum, AttributeNumberIsValid, bms_next_member(), bms_num_members(), i, MaxAttrNumber, and palloc().

966 {
967  int i,
968  j;
969  AttrNumber *attnums;
970  int num = bms_num_members(attrs);
971 
972  if (numattrs)
973  *numattrs = num;
974 
975  /* build attnums from the bitmapset */
976  attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * num);
977  i = 0;
978  j = -1;
979  while ((j = bms_next_member(attrs, j)) >= 0)
980  {
981  int attnum = (j - nexprs);
982 
983  /*
984  * Make sure the bitmap contains only user-defined attributes. As
985  * bitmaps can't contain negative values, this can be violated in two
986  * ways. Firstly, the bitmap might contain 0 as a member, and secondly
987  * the integer value might be larger than MaxAttrNumber.
988  */
990  Assert(attnum <= MaxAttrNumber);
991  Assert(attnum >= (-nexprs));
992 
993  attnums[i++] = (AttrNumber) attnum;
994 
995  /* protect against overflows */
996  Assert(i <= num);
997  }
998 
999  return attnums;
1000 }
#define MaxAttrNumber
Definition: attnum.h:24
int bms_next_member(const Bitmapset *a, int prevbit)
Definition: bitmapset.c:1043
int bms_num_members(const Bitmapset *a)
Definition: bitmapset.c:646
#define AttributeNumberIsValid(attributeNumber)
Definition: attnum.h:34
int16 attnum
Definition: pg_attribute.h:83
#define Assert(condition)
Definition: c.h:804
void * palloc(Size size)
Definition: mcxt.c:1062
int i
int16 AttrNumber
Definition: attnum.h:21

◆ build_sorted_items()

SortItem* build_sorted_items ( StatsBuildData data,
int *  nitems,
MultiSortSupport  mss,
int  numattrs,
AttrNumber attnums 
)

Definition at line 1010 of file extended_stats.c.

References Assert, attlen, attnum, StatsBuildData::attnums, VacAttrStats::attrtypid, get_typlen(), i, idx(), multi_sort_compare(), StatsBuildData::nattnums, StatsBuildData::nulls, StatsBuildData::numrows, palloc(), palloc0(), pfree(), PG_DETOAST_DATUM, PointerGetDatum, qsort_arg(), StatsBuildData::stats, toast_raw_datum_size(), value, StatsBuildData::values, values, and WIDTH_THRESHOLD.

Referenced by dependency_degree(), and statext_mcv_build().

1013 {
1014  int i,
1015  j,
1016  len,
1017  nrows;
1018  int nvalues = data->numrows * numattrs;
1019 
1020  SortItem *items;
1021  Datum *values;
1022  bool *isnull;
1023  char *ptr;
1024  int *typlen;
1025 
1026  /* Compute the total amount of memory we need (both items and values). */
1027  len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
1028 
1029  /* Allocate the memory and split it into the pieces. */
1030  ptr = palloc0(len);
1031 
1032  /* items to sort */
1033  items = (SortItem *) ptr;
1034  ptr += data->numrows * sizeof(SortItem);
1035 
1036  /* values and null flags */
1037  values = (Datum *) ptr;
1038  ptr += nvalues * sizeof(Datum);
1039 
1040  isnull = (bool *) ptr;
1041  ptr += nvalues * sizeof(bool);
1042 
1043  /* make sure we consumed the whole buffer exactly */
1044  Assert((ptr - (char *) items) == len);
1045 
1046  /* fix the pointers to Datum and bool arrays */
1047  nrows = 0;
1048  for (i = 0; i < data->numrows; i++)
1049  {
1050  items[nrows].values = &values[nrows * numattrs];
1051  items[nrows].isnull = &isnull[nrows * numattrs];
1052 
1053  nrows++;
1054  }
1055 
1056  /* build a local cache of typlen for all attributes */
1057  typlen = (int *) palloc(sizeof(int) * data->nattnums);
1058  for (i = 0; i < data->nattnums; i++)
1059  typlen[i] = get_typlen(data->stats[i]->attrtypid);
1060 
1061  nrows = 0;
1062  for (i = 0; i < data->numrows; i++)
1063  {
1064  bool toowide = false;
1065 
1066  /* load the values/null flags from sample rows */
1067  for (j = 0; j < numattrs; j++)
1068  {
1069  Datum value;
1070  bool isnull;
1071  int attlen;
1072  AttrNumber attnum = attnums[j];
1073 
1074  int idx;
1075 
1076  /* match attnum to the pre-calculated data */
1077  for (idx = 0; idx < data->nattnums; idx++)
1078  {
1079  if (attnum == data->attnums[idx])
1080  break;
1081  }
1082 
1083  Assert(idx < data->nattnums);
1084 
1085  value = data->values[idx][i];
1086  isnull = data->nulls[idx][i];
1087  attlen = typlen[idx];
1088 
1089  /*
1090  * If this is a varlena value, check if it's too wide and if yes
1091  * then skip the whole item. Otherwise detoast the value.
1092  *
1093  * XXX It may happen that we've already detoasted some preceding
1094  * values for the current item. We don't bother to cleanup those
1095  * on the assumption that those are small (below WIDTH_THRESHOLD)
1096  * and will be discarded at the end of analyze.
1097  */
1098  if ((!isnull) && (attlen == -1))
1099  {
1101  {
1102  toowide = true;
1103  break;
1104  }
1105 
1106  value = PointerGetDatum(PG_DETOAST_DATUM(value));
1107  }
1108 
1109  items[nrows].values[j] = value;
1110  items[nrows].isnull[j] = isnull;
1111  }
1112 
1113  if (toowide)
1114  continue;
1115 
1116  nrows++;
1117  }
1118 
1119  /* store the actual number of items (ignoring the too-wide ones) */
1120  *nitems = nrows;
1121 
1122  /* all items were too wide */
1123  if (nrows == 0)
1124  {
1125  /* everything is allocated as a single chunk */
1126  pfree(items);
1127  return NULL;
1128  }
1129 
1130  /* do the sort, using the multi-sort */
1131  qsort_arg((void *) items, nrows, sizeof(SortItem),
1132  multi_sort_compare, mss);
1133 
1134  return items;
1135 }
static struct @142 value
#define PointerGetDatum(X)
Definition: postgres.h:600
Datum idx(PG_FUNCTION_ARGS)
Definition: _int_op.c:259
void pfree(void *pointer)
Definition: mcxt.c:1169
Oid attrtypid
Definition: vacuum.h:124
Size toast_raw_datum_size(Datum value)
Definition: detoast.c:545
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
int16 attlen
Definition: pg_attribute.h:68
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
struct SortItem SortItem
#define WIDTH_THRESHOLD
VacAttrStats ** stats
int16 attnum
Definition: pg_attribute.h:83
#define Assert(condition)
Definition: c.h:804
int multi_sort_compare(const void *a, const void *b, void *arg)
static Datum values[MAXATTR]
Definition: bootstrap.c:166
int16 get_typlen(Oid typid)
Definition: lsyscache.c:2144
void * palloc(Size size)
Definition: mcxt.c:1062
int i
#define PG_DETOAST_DATUM(datum)
Definition: fmgr.h:240
int16 AttrNumber
Definition: attnum.h:21
unsigned char bool
Definition: c.h:391

◆ compare_datums_simple()

int compare_datums_simple ( Datum  a,
Datum  b,
SortSupport  ssup 
)

Definition at line 951 of file extended_stats.c.

References ApplySortComparator().

Referenced by compare_scalars_simple(), and statext_mcv_serialize().

952 {
953  return ApplySortComparator(a, false, b, false, ssup);
954 }
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ compare_scalars_simple()

int compare_scalars_simple ( const void *  a,
const void *  b,
void *  arg 
)

Definition at line 943 of file extended_stats.c.

References compare_datums_simple().

Referenced by statext_mcv_serialize().

944 {
945  return compare_datums_simple(*(Datum *) a,
946  *(Datum *) b,
947  (SortSupport) arg);
948 }
int compare_datums_simple(Datum a, Datum b, SortSupport ssup)
uintptr_t Datum
Definition: postgres.h:411
void * arg

◆ examine_opclause_args()

bool examine_opclause_args ( List args,
Node **  exprp,
Const **  cstp,
bool expronleftp 
)

Definition at line 2021 of file extended_stats.c.

References arg, Assert, IsA, linitial, list_length(), and lsecond.

Referenced by mcv_get_match_bitmap(), and statext_is_compatible_clause_internal().

2023 {
2024  Node *expr;
2025  Const *cst;
2026  bool expronleft;
2027  Node *leftop,
2028  *rightop;
2029 
2030  /* enforced by statext_is_compatible_clause_internal */
2031  Assert(list_length(args) == 2);
2032 
2033  leftop = linitial(args);
2034  rightop = lsecond(args);
2035 
2036  /* strip RelabelType from either side of the expression */
2037  if (IsA(leftop, RelabelType))
2038  leftop = (Node *) ((RelabelType *) leftop)->arg;
2039 
2040  if (IsA(rightop, RelabelType))
2041  rightop = (Node *) ((RelabelType *) rightop)->arg;
2042 
2043  if (IsA(rightop, Const))
2044  {
2045  expr = (Node *) leftop;
2046  cst = (Const *) rightop;
2047  expronleft = true;
2048  }
2049  else if (IsA(leftop, Const))
2050  {
2051  expr = (Node *) rightop;
2052  cst = (Const *) leftop;
2053  expronleft = false;
2054  }
2055  else
2056  return false;
2057 
2058  /* return pointers to the extracted parts if requested */
2059  if (exprp)
2060  *exprp = expr;
2061 
2062  if (cstp)
2063  *cstp = cst;
2064 
2065  if (expronleftp)
2066  *expronleftp = expronleft;
2067 
2068  return true;
2069 }
#define IsA(nodeptr, _type_)
Definition: nodes.h:590
Definition: nodes.h:539
#define lsecond(l)
Definition: pg_list.h:179
#define linitial(l)
Definition: pg_list.h:174
#define Assert(condition)
Definition: c.h:804
static int list_length(const List *l)
Definition: pg_list.h:149
void * arg

◆ mcv_clause_selectivity_or()

Selectivity mcv_clause_selectivity_or ( PlannerInfo root,
StatisticExtInfo stat,
MCVList mcv,
Node clause,
bool **  or_matches,
Selectivity basesel,
Selectivity overlap_mcvsel,
Selectivity overlap_basesel,
Selectivity totalsel 
)

Definition at line 2107 of file mcv.c.

References MCVItem::base_frequency, StatisticExtInfo::exprs, MCVItem::frequency, i, MCVList::items, StatisticExtInfo::keys, list_make1, mcv_get_match_bitmap(), MCVList::nitems, palloc0(), and pfree().

Referenced by statext_mcv_clauselist_selectivity().

2111 {
2112  Selectivity s = 0.0;
2113  bool *new_matches;
2114  int i;
2115 
2116  /* build the OR-matches bitmap, if not built already */
2117  if (*or_matches == NULL)
2118  *or_matches = palloc0(sizeof(bool) * mcv->nitems);
2119 
2120  /* build the match bitmap for the new clause */
2121  new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys,
2122  stat->exprs, mcv, false);
2123 
2124  /*
2125  * Sum the frequencies for all the MCV items matching this clause and also
2126  * those matching the overlap between this clause and any of the preceding
2127  * clauses as described above.
2128  */
2129  *basesel = 0.0;
2130  *overlap_mcvsel = 0.0;
2131  *overlap_basesel = 0.0;
2132  *totalsel = 0.0;
2133  for (i = 0; i < mcv->nitems; i++)
2134  {
2135  *totalsel += mcv->items[i].frequency;
2136 
2137  if (new_matches[i])
2138  {
2139  s += mcv->items[i].frequency;
2140  *basesel += mcv->items[i].base_frequency;
2141 
2142  if ((*or_matches)[i])
2143  {
2144  *overlap_mcvsel += mcv->items[i].frequency;
2145  *overlap_basesel += mcv->items[i].base_frequency;
2146  }
2147  }
2148 
2149  /* update the OR-matches bitmap for the next clause */
2150  (*or_matches)[i] = (*or_matches)[i] || new_matches[i];
2151  }
2152 
2153  pfree(new_matches);
2154 
2155  return s;
2156 }
uint32 nitems
Definition: statistics.h:91
static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Bitmapset *keys, List *exprs, MCVList *mcvlist, bool is_or)
Definition: mcv.c:1603
double Selectivity
Definition: nodes.h:672
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
#define list_make1(x1)
Definition: pg_list.h:206
void pfree(void *pointer)
Definition: mcxt.c:1169
void * palloc0(Size size)
Definition: mcxt.c:1093
double base_frequency
Definition: statistics.h:81
Bitmapset * keys
Definition: pathnodes.h:931
int i
double frequency
Definition: statistics.h:80

◆ mcv_clauselist_selectivity()

Selectivity mcv_clauselist_selectivity ( PlannerInfo root,
StatisticExtInfo stat,
List clauses,
int  varRelid,
JoinType  jointype,
SpecialJoinInfo sjinfo,
RelOptInfo rel,
Selectivity basesel,
Selectivity totalsel 
)

Definition at line 2030 of file mcv.c.

References MCVItem::base_frequency, StatisticExtInfo::exprs, MCVItem::frequency, i, MCVList::items, StatisticExtInfo::keys, mcv_get_match_bitmap(), MCVList::nitems, statext_mcv_load(), and StatisticExtInfo::statOid.

Referenced by statext_mcv_clauselist_selectivity().

2035 {
2036  int i;
2037  MCVList *mcv;
2038  Selectivity s = 0.0;
2039 
2040  /* match/mismatch bitmap for each MCV item */
2041  bool *matches = NULL;
2042 
2043  /* load the MCV list stored in the statistics object */
2044  mcv = statext_mcv_load(stat->statOid);
2045 
2046  /* build a match bitmap for the clauses */
2047  matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs,
2048  mcv, false);
2049 
2050  /* sum frequencies for all the matching MCV items */
2051  *basesel = 0.0;
2052  *totalsel = 0.0;
2053  for (i = 0; i < mcv->nitems; i++)
2054  {
2055  *totalsel += mcv->items[i].frequency;
2056 
2057  if (matches[i] != false)
2058  {
2059  *basesel += mcv->items[i].base_frequency;
2060  s += mcv->items[i].frequency;
2061  }
2062  }
2063 
2064  return s;
2065 }
uint32 nitems
Definition: statistics.h:91
static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Bitmapset *keys, List *exprs, MCVList *mcvlist, bool is_or)
Definition: mcv.c:1603
double Selectivity
Definition: nodes.h:672
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
MCVList * statext_mcv_load(Oid mvoid)
Definition: mcv.c:560
double base_frequency
Definition: statistics.h:81
Bitmapset * keys
Definition: pathnodes.h:931
int i
double frequency
Definition: statistics.h:80

◆ mcv_combine_selectivities()

Selectivity mcv_combine_selectivities ( Selectivity  simple_sel,
Selectivity  mcv_sel,
Selectivity  mcv_basesel,
Selectivity  mcv_totalsel 
)

Definition at line 1988 of file mcv.c.

References CLAMP_PROBABILITY.

Referenced by statext_mcv_clauselist_selectivity().

1992 {
1993  Selectivity other_sel;
1994  Selectivity sel;
1995 
1996  /* estimated selectivity of values not covered by MCV matches */
1997  other_sel = simple_sel - mcv_basesel;
1998  CLAMP_PROBABILITY(other_sel);
1999 
2000  /* this non-MCV selectivity cannot exceed 1 - mcv_totalsel */
2001  if (other_sel > 1.0 - mcv_totalsel)
2002  other_sel = 1.0 - mcv_totalsel;
2003 
2004  /* overall selectivity is the sum of the MCV and non-MCV parts */
2005  sel = mcv_sel + other_sel;
2006  CLAMP_PROBABILITY(sel);
2007 
2008  return sel;
2009 }
double Selectivity
Definition: nodes.h:672
#define CLAMP_PROBABILITY(p)
Definition: selfuncs.h:63

◆ multi_sort_add_dimension()

void multi_sort_add_dimension ( MultiSortSupport  mss,
int  sortdim,
Oid  oper,
Oid  collation 
)

Definition at line 875 of file extended_stats.c.

References CurrentMemoryContext, PrepareSortSupportFromOrderingOp(), MultiSortSupportData::ssup, SortSupportData::ssup_collation, SortSupportData::ssup_cxt, and SortSupportData::ssup_nulls_first.

Referenced by build_mss(), dependency_degree(), and ndistinct_for_combination().

877 {
878  SortSupport ssup = &mss->ssup[sortdim];
879 
881  ssup->ssup_collation = collation;
882  ssup->ssup_nulls_first = false;
883 
885 }
bool ssup_nulls_first
Definition: sortsupport.h:75
void PrepareSortSupportFromOrderingOp(Oid orderingOp, SortSupport ssup)
Definition: sortsupport.c:135
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
MemoryContext ssup_cxt
Definition: sortsupport.h:66
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
Operator oper(ParseState *pstate, List *opname, Oid ltypeId, Oid rtypeId, bool noError, int location)
Definition: parse_oper.c:382

◆ multi_sort_compare()

int multi_sort_compare ( const void *  a,
const void *  b,
void *  arg 
)

Definition at line 889 of file extended_stats.c.

References ApplySortComparator(), compare(), i, SortItem::isnull, MultiSortSupportData::ndims, MultiSortSupportData::ssup, and SortItem::values.

Referenced by build_distinct_groups(), build_sorted_items(), count_distinct_groups(), ndistinct_for_combination(), and statext_mcv_build().

890 {
892  SortItem *ia = (SortItem *) a;
893  SortItem *ib = (SortItem *) b;
894  int i;
895 
896  for (i = 0; i < mss->ndims; i++)
897  {
898  int compare;
899 
900  compare = ApplySortComparator(ia->values[i], ia->isnull[i],
901  ib->values[i], ib->isnull[i],
902  &mss->ssup[i]);
903 
904  if (compare != 0)
905  return compare;
906  }
907 
908  /* equal by default */
909  return 0;
910 }
static int compare(const void *arg1, const void *arg2)
Definition: geqo_pool.c:145
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
int i
void * arg
MultiSortSupportData * MultiSortSupport
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_compare_dim()

int multi_sort_compare_dim ( int  dim,
const SortItem a,
const SortItem b,
MultiSortSupport  mss 
)

Definition at line 914 of file extended_stats.c.

References ApplySortComparator(), SortItem::isnull, MultiSortSupportData::ssup, and SortItem::values.

Referenced by dependency_degree().

916 {
917  return ApplySortComparator(a->values[dim], a->isnull[dim],
918  b->values[dim], b->isnull[dim],
919  &mss->ssup[dim]);
920 }
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_compare_dims()

int multi_sort_compare_dims ( int  start,
int  end,
const SortItem a,
const SortItem b,
MultiSortSupport  mss 
)

Definition at line 923 of file extended_stats.c.

References ApplySortComparator(), SortItem::isnull, MultiSortSupportData::ssup, and SortItem::values.

Referenced by dependency_degree().

926 {
927  int dim;
928 
929  for (dim = start; dim <= end; dim++)
930  {
931  int r = ApplySortComparator(a->values[dim], a->isnull[dim],
932  b->values[dim], b->isnull[dim],
933  &mss->ssup[dim]);
934 
935  if (r != 0)
936  return r;
937  }
938 
939  return 0;
940 }
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_init()

MultiSortSupport multi_sort_init ( int  ndims)

Definition at line 856 of file extended_stats.c.

References Assert, MultiSortSupportData::ndims, offsetof, and palloc0().

Referenced by build_mss(), dependency_degree(), and ndistinct_for_combination().

857 {
858  MultiSortSupport mss;
859 
860  Assert(ndims >= 2);
861 
863  + sizeof(SortSupportData) * ndims);
864 
865  mss->ndims = ndims;
866 
867  return mss;
868 }
struct SortSupportData SortSupportData
void * palloc0(Size size)
Definition: mcxt.c:1093
#define Assert(condition)
Definition: c.h:804
MultiSortSupportData * MultiSortSupport
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_build()

MVDependencies* statext_dependencies_build ( StatsBuildData data)

Definition at line 355 of file dependencies.c.

References Assert, StatsBuildData::attnums, MVDependency::attributes, MVDependency::degree, DependencyGeneratorData::dependencies, dependency_degree(), DependencyGenerator_free(), DependencyGenerator_init(), DependencyGenerator_next(), MVDependencies::deps, i, DependencyGeneratorData::k, MVDependencies::magic, StatsBuildData::nattnums, MVDependency::nattributes, MVDependencies::ndeps, offsetof, palloc0(), repalloc(), STATS_DEPS_MAGIC, STATS_DEPS_TYPE_BASIC, and MVDependencies::type.

Referenced by BuildRelationExtStatistics().

356 {
357  int i,
358  k;
359 
360  /* result */
361  MVDependencies *dependencies = NULL;
362 
363  Assert(data->nattnums >= 2);
364 
365  /*
366  * We'll try build functional dependencies starting from the smallest ones
367  * covering just 2 columns, to the largest ones, covering all columns
368  * included in the statistics object. We start from the smallest ones
369  * because we want to be able to skip already implied ones.
370  */
371  for (k = 2; k <= data->nattnums; k++)
372  {
373  AttrNumber *dependency; /* array with k elements */
374 
375  /* prepare a DependencyGenerator of variation */
377 
378  /* generate all possible variations of k values (out of n) */
379  while ((dependency = DependencyGenerator_next(DependencyGenerator)))
380  {
381  double degree;
382  MVDependency *d;
383 
384  /* compute how valid the dependency seems */
385  degree = dependency_degree(data, k, dependency);
386 
387  /*
388  * if the dependency seems entirely invalid, don't store it
389  */
390  if (degree == 0.0)
391  continue;
392 
393  d = (MVDependency *) palloc0(offsetof(MVDependency, attributes)
394  + k * sizeof(AttrNumber));
395 
396  /* copy the dependency (and keep the indexes into stxkeys) */
397  d->degree = degree;
398  d->nattributes = k;
399  for (i = 0; i < k; i++)
400  d->attributes[i] = data->attnums[dependency[i]];
401 
402  /* initialize the list of dependencies */
403  if (dependencies == NULL)
404  {
405  dependencies
406  = (MVDependencies *) palloc0(sizeof(MVDependencies));
407 
408  dependencies->magic = STATS_DEPS_MAGIC;
409  dependencies->type = STATS_DEPS_TYPE_BASIC;
410  dependencies->ndeps = 0;
411  }
412 
413  dependencies->ndeps++;
414  dependencies = (MVDependencies *) repalloc(dependencies,
415  offsetof(MVDependencies, deps)
416  + dependencies->ndeps * sizeof(MVDependency *));
417 
418  dependencies->deps[dependencies->ndeps - 1] = d;
419  }
420 
421  /*
422  * we're done with variations of k elements, so free the
423  * DependencyGenerator
424  */
425  DependencyGenerator_free(DependencyGenerator);
426  }
427 
428  return dependencies;
429 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define STATS_DEPS_MAGIC
Definition: statistics.h:43
static double dependency_degree(StatsBuildData *data, int k, AttrNumber *dependency)
Definition: dependencies.c:222
#define STATS_DEPS_TYPE_BASIC
Definition: statistics.h:44
AttrNumber nattributes
Definition: statistics.h:53
static DependencyGenerator DependencyGenerator_init(int n, int k)
Definition: dependencies.c:173
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
static AttrNumber * DependencyGenerator_next(DependencyGenerator state)
Definition: dependencies.c:205
int i
static void DependencyGenerator_free(DependencyGenerator state)
Definition: dependencies.c:196
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_deserialize()

MVDependencies* statext_dependencies_deserialize ( bytea data)

Definition at line 491 of file dependencies.c.

References Assert, MVDependency::attributes, MVDependency::degree, DependencyGeneratorData::dependencies, MVDependencies::deps, elog, ERROR, i, DependencyGeneratorData::k, MVDependencies::magic, MVDependency::nattributes, MVDependencies::ndeps, offsetof, palloc0(), repalloc(), SizeOfHeader, SizeOfItem, STATS_DEPS_MAGIC, STATS_DEPS_TYPE_BASIC, STATS_MAX_DIMENSIONS, MVDependencies::type, VARDATA_ANY, VARSIZE_ANY, and VARSIZE_ANY_EXHDR.

Referenced by pg_dependencies_out(), and statext_dependencies_load().

492 {
493  int i;
494  Size min_expected_size;
495  MVDependencies *dependencies;
496  char *tmp;
497 
498  if (data == NULL)
499  return NULL;
500 
501  if (VARSIZE_ANY_EXHDR(data) < SizeOfHeader)
502  elog(ERROR, "invalid MVDependencies size %zd (expected at least %zd)",
504 
505  /* read the MVDependencies header */
506  dependencies = (MVDependencies *) palloc0(sizeof(MVDependencies));
507 
508  /* initialize pointer to the data part (skip the varlena header) */
509  tmp = VARDATA_ANY(data);
510 
511  /* read the header fields and perform basic sanity checks */
512  memcpy(&dependencies->magic, tmp, sizeof(uint32));
513  tmp += sizeof(uint32);
514  memcpy(&dependencies->type, tmp, sizeof(uint32));
515  tmp += sizeof(uint32);
516  memcpy(&dependencies->ndeps, tmp, sizeof(uint32));
517  tmp += sizeof(uint32);
518 
519  if (dependencies->magic != STATS_DEPS_MAGIC)
520  elog(ERROR, "invalid dependency magic %d (expected %d)",
521  dependencies->magic, STATS_DEPS_MAGIC);
522 
523  if (dependencies->type != STATS_DEPS_TYPE_BASIC)
524  elog(ERROR, "invalid dependency type %d (expected %d)",
525  dependencies->type, STATS_DEPS_TYPE_BASIC);
526 
527  if (dependencies->ndeps == 0)
528  elog(ERROR, "invalid zero-length item array in MVDependencies");
529 
530  /* what minimum bytea size do we expect for those parameters */
531  min_expected_size = SizeOfItem(dependencies->ndeps);
532 
533  if (VARSIZE_ANY_EXHDR(data) < min_expected_size)
534  elog(ERROR, "invalid dependencies size %zd (expected at least %zd)",
535  VARSIZE_ANY_EXHDR(data), min_expected_size);
536 
537  /* allocate space for the MCV items */
538  dependencies = repalloc(dependencies, offsetof(MVDependencies, deps)
539  + (dependencies->ndeps * sizeof(MVDependency *)));
540 
541  for (i = 0; i < dependencies->ndeps; i++)
542  {
543  double degree;
544  AttrNumber k;
545  MVDependency *d;
546 
547  /* degree of validity */
548  memcpy(&degree, tmp, sizeof(double));
549  tmp += sizeof(double);
550 
551  /* number of attributes */
552  memcpy(&k, tmp, sizeof(AttrNumber));
553  tmp += sizeof(AttrNumber);
554 
555  /* is the number of attributes valid? */
556  Assert((k >= 2) && (k <= STATS_MAX_DIMENSIONS));
557 
558  /* now that we know the number of attributes, allocate the dependency */
559  d = (MVDependency *) palloc0(offsetof(MVDependency, attributes)
560  + (k * sizeof(AttrNumber)));
561 
562  d->degree = degree;
563  d->nattributes = k;
564 
565  /* copy attribute numbers */
566  memcpy(d->attributes, tmp, sizeof(AttrNumber) * d->nattributes);
567  tmp += sizeof(AttrNumber) * d->nattributes;
568 
569  dependencies->deps[i] = d;
570 
571  /* still within the bytea */
572  Assert(tmp <= ((char *) data + VARSIZE_ANY(data)));
573  }
574 
575  /* we should have consumed the whole bytea exactly */
576  Assert(tmp == ((char *) data + VARSIZE_ANY(data)));
577 
578  return dependencies;
579 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define STATS_DEPS_MAGIC
Definition: statistics.h:43
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define STATS_DEPS_TYPE_BASIC
Definition: statistics.h:44
#define SizeOfItem(natts)
Definition: dependencies.c:41
AttrNumber nattributes
Definition: statistics.h:53
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
size_t Size
Definition: c.h:540
#define SizeOfHeader
Definition: dependencies.c:38
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
#define elog(elevel,...)
Definition: elog.h:232
int i
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_serialize()

bytea* statext_dependencies_serialize ( MVDependencies dependencies)

Definition at line 436 of file dependencies.c.

References Assert, MVDependency::attributes, MVDependency::degree, MVDependencies::deps, i, MVDependencies::magic, MVDependency::nattributes, MVDependencies::ndeps, output(), palloc0(), SET_VARSIZE, SizeOfHeader, SizeOfItem, MVDependencies::type, VARDATA, and VARHDRSZ.

Referenced by statext_store().

437 {
438  int i;
439  bytea *output;
440  char *tmp;
441  Size len;
442 
443  /* we need to store ndeps, with a number of attributes for each one */
444  len = VARHDRSZ + SizeOfHeader;
445 
446  /* and also include space for the actual attribute numbers and degrees */
447  for (i = 0; i < dependencies->ndeps; i++)
448  len += SizeOfItem(dependencies->deps[i]->nattributes);
449 
450  output = (bytea *) palloc0(len);
451  SET_VARSIZE(output, len);
452 
453  tmp = VARDATA(output);
454 
455  /* Store the base struct values (magic, type, ndeps) */
456  memcpy(tmp, &dependencies->magic, sizeof(uint32));
457  tmp += sizeof(uint32);
458  memcpy(tmp, &dependencies->type, sizeof(uint32));
459  tmp += sizeof(uint32);
460  memcpy(tmp, &dependencies->ndeps, sizeof(uint32));
461  tmp += sizeof(uint32);
462 
463  /* store number of attributes and attribute numbers for each dependency */
464  for (i = 0; i < dependencies->ndeps; i++)
465  {
466  MVDependency *d = dependencies->deps[i];
467 
468  memcpy(tmp, &d->degree, sizeof(double));
469  tmp += sizeof(double);
470 
471  memcpy(tmp, &d->nattributes, sizeof(AttrNumber));
472  tmp += sizeof(AttrNumber);
473 
474  memcpy(tmp, d->attributes, sizeof(AttrNumber) * d->nattributes);
475  tmp += sizeof(AttrNumber) * d->nattributes;
476 
477  /* protect against overflow */
478  Assert(tmp <= ((char *) output + len));
479  }
480 
481  /* make sure we've produced exactly the right amount of data */
482  Assert(tmp == ((char *) output + len));
483 
484  return output;
485 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define VARDATA(PTR)
Definition: postgres.h:315
static void output(uint64 loop_count)
#define VARHDRSZ
Definition: c.h:627
#define SizeOfItem(natts)
Definition: dependencies.c:41
AttrNumber nattributes
Definition: statistics.h:53
unsigned int uint32
Definition: c.h:441
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
size_t Size
Definition: c.h:540
#define SizeOfHeader
Definition: dependencies.c:38
int i
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21

◆ statext_mcv_build()

MCVList* statext_mcv_build ( StatsBuildData data,
double  totalrows,
int  stattarget 
)

Definition at line 184 of file mcv.c.

References Assert, StatsBuildData::attnums, VacAttrStats::attrtypid, MCVItem::base_frequency, bsearch_arg(), build_column_frequencies(), build_distinct_groups(), build_mss(), build_sorted_items(), SortItem::count, MCVItem::frequency, get_mincount_for_mcv_list(), i, SortItem::isnull, MCVItem::isnull, MCVList::items, sort-test::key, MCVList::magic, multi_sort_compare(), StatsBuildData::nattnums, MCVList::ndimensions, MultiSortSupportData::ndims, MCVList::nitems, StatsBuildData::numrows, offsetof, palloc(), palloc0(), pfree(), MultiSortSupportData::ssup, StatsBuildData::stats, STATS_MCV_MAGIC, STATS_MCV_TYPE_BASIC, MCVList::type, MCVList::types, SortItem::values, and MCVItem::values.

Referenced by BuildRelationExtStatistics().

185 {
186  int i,
187  numattrs,
188  numrows,
189  ngroups,
190  nitems;
191  double mincount;
192  SortItem *items;
193  SortItem *groups;
194  MCVList *mcvlist = NULL;
195  MultiSortSupport mss;
196 
197  /* comparator for all the columns */
198  mss = build_mss(data);
199 
200  /* sort the rows */
201  items = build_sorted_items(data, &nitems, mss,
202  data->nattnums, data->attnums);
203 
204  if (!items)
205  return NULL;
206 
207  /* for convenience */
208  numattrs = data->nattnums;
209  numrows = data->numrows;
210 
211  /* transform the sorted rows into groups (sorted by frequency) */
212  groups = build_distinct_groups(nitems, items, mss, &ngroups);
213 
214  /*
215  * Maximum number of MCV items to store, based on the statistics target we
216  * computed for the statistics object (from target set for the object
217  * itself, attributes and the system default). In any case, we can't keep
218  * more groups than we have available.
219  */
220  nitems = stattarget;
221  if (nitems > ngroups)
222  nitems = ngroups;
223 
224  /*
225  * Decide how many items to keep in the MCV list. We can't use the same
226  * algorithm as per-column MCV lists, because that only considers the
227  * actual group frequency - but we're primarily interested in how the
228  * actual frequency differs from the base frequency (product of simple
229  * per-column frequencies, as if the columns were independent).
230  *
231  * Using the same algorithm might exclude items that are close to the
232  * "average" frequency of the sample. But that does not say whether the
233  * observed frequency is close to the base frequency or not. We also need
234  * to consider unexpectedly uncommon items (again, compared to the base
235  * frequency), and the single-column algorithm does not have to.
236  *
237  * We simply decide how many items to keep by computing minimum count
238  * using get_mincount_for_mcv_list() and then keep all items that seem to
239  * be more common than that.
240  */
241  mincount = get_mincount_for_mcv_list(numrows, totalrows);
242 
243  /*
244  * Walk the groups until we find the first group with a count below the
245  * mincount threshold (the index of that group is the number of groups we
246  * want to keep).
247  */
248  for (i = 0; i < nitems; i++)
249  {
250  if (groups[i].count < mincount)
251  {
252  nitems = i;
253  break;
254  }
255  }
256 
257  /*
258  * At this point we know the number of items for the MCV list. There might
259  * be none (for uniform distribution with many groups), and in that case
260  * there will be no MCV list. Otherwise construct the MCV list.
261  */
262  if (nitems > 0)
263  {
264  int j;
265  SortItem key;
266  MultiSortSupport tmp;
267 
268  /* frequencies for values in each attribute */
269  SortItem **freqs;
270  int *nfreqs;
271 
272  /* used to search values */
274  + sizeof(SortSupportData));
275 
276  /* compute frequencies for values in each column */
277  nfreqs = (int *) palloc0(sizeof(int) * numattrs);
278  freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
279 
280  /*
281  * Allocate the MCV list structure, set the global parameters.
282  */
283  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) +
284  sizeof(MCVItem) * nitems);
285 
286  mcvlist->magic = STATS_MCV_MAGIC;
287  mcvlist->type = STATS_MCV_TYPE_BASIC;
288  mcvlist->ndimensions = numattrs;
289  mcvlist->nitems = nitems;
290 
291  /* store info about data type OIDs */
292  for (i = 0; i < numattrs; i++)
293  mcvlist->types[i] = data->stats[i]->attrtypid;
294 
295  /* Copy the first chunk of groups into the result. */
296  for (i = 0; i < nitems; i++)
297  {
298  /* just pointer to the proper place in the list */
299  MCVItem *item = &mcvlist->items[i];
300 
301  item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
302  item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
303 
304  /* copy values for the group */
305  memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
306  memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
307 
308  /* groups should be sorted by frequency in descending order */
309  Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
310 
311  /* group frequency */
312  item->frequency = (double) groups[i].count / numrows;
313 
314  /* base frequency, if the attributes were independent */
315  item->base_frequency = 1.0;
316  for (j = 0; j < numattrs; j++)
317  {
318  SortItem *freq;
319 
320  /* single dimension */
321  tmp->ndims = 1;
322  tmp->ssup[0] = mss->ssup[j];
323 
324  /* fill search key */
325  key.values = &groups[i].values[j];
326  key.isnull = &groups[i].isnull[j];
327 
328  freq = (SortItem *) bsearch_arg(&key, freqs[j], nfreqs[j],
329  sizeof(SortItem),
330  multi_sort_compare, tmp);
331 
332  item->base_frequency *= ((double) freq->count) / numrows;
333  }
334  }
335 
336  pfree(nfreqs);
337  pfree(freqs);
338  }
339 
340  pfree(items);
341  pfree(groups);
342 
343  return mcvlist;
344 }
static SortItem * build_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss, int *ndistinct)
Definition: mcv.c:426
uint32 nitems
Definition: statistics.h:91
static SortItem ** build_column_frequencies(SortItem *groups, int ngroups, MultiSortSupport mss, int *ncounts)
Definition: mcv.c:492
Oid types[STATS_MAX_DIMENSIONS]
Definition: statistics.h:93
uint32 magic
Definition: statistics.h:89
AttrNumber ndimensions
Definition: statistics.h:92
struct MCVItem MCVItem
Datum * values
Definition: statistics.h:83
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
static double get_mincount_for_mcv_list(int samplerows, double totalrows)
Definition: mcv.c:152
void pfree(void *pointer)
Definition: mcxt.c:1169
Oid attrtypid
Definition: vacuum.h:124
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
SortItem * build_sorted_items(StatsBuildData *data, int *nitems, MultiSortSupport mss, int numattrs, AttrNumber *attnums)
uint32 type
Definition: statistics.h:90
#define STATS_MCV_TYPE_BASIC
Definition: statistics.h:67
struct SortSupportData SortSupportData
static MultiSortSupport build_mss(StatsBuildData *data)
Definition: mcv.c:351
void * bsearch_arg(const void *key, const void *base, size_t nmemb, size_t size, int(*compar)(const void *, const void *, void *), void *arg)
Definition: bsearch_arg.c:55
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
VacAttrStats ** stats
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
#define STATS_MCV_MAGIC
Definition: statistics.h:66
int multi_sort_compare(const void *a, const void *b, void *arg)
bool * isnull
Definition: statistics.h:82
void * palloc(Size size)
Definition: mcxt.c:1062
int i
MultiSortSupportData * MultiSortSupport
#define offsetof(type, field)
Definition: c.h:727
double frequency
Definition: statistics.h:80

◆ statext_mcv_deserialize()

MCVList* statext_mcv_deserialize ( bytea data)

Definition at line 996 of file mcv.c.

References Assert, MCVItem::base_frequency, elog, ERROR, fetch_att, MCVItem::frequency, i, MCVItem::isnull, MCVList::items, MCVList::magic, MAXALIGN, MinSizeOfMCVList, DimensionInfo::nbytes, DimensionInfo::nbytes_aligned, MCVList::ndimensions, MCVList::nitems, DimensionInfo::nvalues, offsetof, palloc(), palloc0(), pfree(), PG_USED_FOR_ASSERTS_ONLY, PointerGetDatum, repalloc(), SET_VARSIZE, SizeOfMCVList, STATS_MAX_DIMENSIONS, STATS_MCV_MAGIC, STATS_MCV_TYPE_BASIC, STATS_MCVLIST_MAX_ITEMS, MCVList::type, MCVList::types, DimensionInfo::typlen, MCVItem::values, VARDATA, VARDATA_ANY, VARHDRSZ, and VARSIZE_ANY.

Referenced by pg_stats_ext_mcvlist_items(), and statext_mcv_load().

997 {
998  int dim,
999  i;
1000  Size expected_size;
1001  MCVList *mcvlist;
1002  char *raw;
1003  char *ptr;
1004  char *endptr PG_USED_FOR_ASSERTS_ONLY;
1005 
1006  int ndims,
1007  nitems;
1008  DimensionInfo *info = NULL;
1009 
1010  /* local allocation buffer (used only for deserialization) */
1011  Datum **map = NULL;
1012 
1013  /* MCV list */
1014  Size mcvlen;
1015 
1016  /* buffer used for the result */
1017  Size datalen;
1018  char *dataptr;
1019  char *valuesptr;
1020  char *isnullptr;
1021 
1022  if (data == NULL)
1023  return NULL;
1024 
1025  /*
1026  * We can't possibly deserialize a MCV list if there's not even a complete
1027  * header. We need an explicit formula here, because we serialize the
1028  * header fields one by one, so we need to ignore struct alignment.
1029  */
1030  if (VARSIZE_ANY(data) < MinSizeOfMCVList)
1031  elog(ERROR, "invalid MCV size %zd (expected at least %zu)",
1032  VARSIZE_ANY(data), MinSizeOfMCVList);
1033 
1034  /* read the MCV list header */
1035  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items));
1036 
1037  /* pointer to the data part (skip the varlena header) */
1038  raw = (char *) data;
1039  ptr = VARDATA_ANY(raw);
1040  endptr = (char *) raw + VARSIZE_ANY(data);
1041 
1042  /* get the header and perform further sanity checks */
1043  memcpy(&mcvlist->magic, ptr, sizeof(uint32));
1044  ptr += sizeof(uint32);
1045 
1046  memcpy(&mcvlist->type, ptr, sizeof(uint32));
1047  ptr += sizeof(uint32);
1048 
1049  memcpy(&mcvlist->nitems, ptr, sizeof(uint32));
1050  ptr += sizeof(uint32);
1051 
1052  memcpy(&mcvlist->ndimensions, ptr, sizeof(AttrNumber));
1053  ptr += sizeof(AttrNumber);
1054 
1055  if (mcvlist->magic != STATS_MCV_MAGIC)
1056  elog(ERROR, "invalid MCV magic %u (expected %u)",
1057  mcvlist->magic, STATS_MCV_MAGIC);
1058 
1059  if (mcvlist->type != STATS_MCV_TYPE_BASIC)
1060  elog(ERROR, "invalid MCV type %u (expected %u)",
1061  mcvlist->type, STATS_MCV_TYPE_BASIC);
1062 
1063  if (mcvlist->ndimensions == 0)
1064  elog(ERROR, "invalid zero-length dimension array in MCVList");
1065  else if ((mcvlist->ndimensions > STATS_MAX_DIMENSIONS) ||
1066  (mcvlist->ndimensions < 0))
1067  elog(ERROR, "invalid length (%d) dimension array in MCVList",
1068  mcvlist->ndimensions);
1069 
1070  if (mcvlist->nitems == 0)
1071  elog(ERROR, "invalid zero-length item array in MCVList");
1072  else if (mcvlist->nitems > STATS_MCVLIST_MAX_ITEMS)
1073  elog(ERROR, "invalid length (%u) item array in MCVList",
1074  mcvlist->nitems);
1075 
1076  nitems = mcvlist->nitems;
1077  ndims = mcvlist->ndimensions;
1078 
1079  /*
1080  * Check amount of data including DimensionInfo for all dimensions and
1081  * also the serialized items (including uint16 indexes). Also, walk
1082  * through the dimension information and add it to the sum.
1083  */
1084  expected_size = SizeOfMCVList(ndims, nitems);
1085 
1086  /*
1087  * Check that we have at least the dimension and info records, along with
1088  * the items. We don't know the size of the serialized values yet. We need
1089  * to do this check first, before accessing the dimension info.
1090  */
1091  if (VARSIZE_ANY(data) < expected_size)
1092  elog(ERROR, "invalid MCV size %zd (expected %zu)",
1093  VARSIZE_ANY(data), expected_size);
1094 
1095  /* Now copy the array of type Oids. */
1096  memcpy(mcvlist->types, ptr, sizeof(Oid) * ndims);
1097  ptr += (sizeof(Oid) * ndims);
1098 
1099  /* Now it's safe to access the dimension info. */
1100  info = palloc(ndims * sizeof(DimensionInfo));
1101 
1102  memcpy(info, ptr, ndims * sizeof(DimensionInfo));
1103  ptr += (ndims * sizeof(DimensionInfo));
1104 
1105  /* account for the value arrays */
1106  for (dim = 0; dim < ndims; dim++)
1107  {
1108  /*
1109  * XXX I wonder if we can/should rely on asserts here. Maybe those
1110  * checks should be done every time?
1111  */
1112  Assert(info[dim].nvalues >= 0);
1113  Assert(info[dim].nbytes >= 0);
1114 
1115  expected_size += info[dim].nbytes;
1116  }
1117 
1118  /*
1119  * Now we know the total expected MCV size, including all the pieces
1120  * (header, dimension info. items and deduplicated data). So do the final
1121  * check on size.
1122  */
1123  if (VARSIZE_ANY(data) != expected_size)
1124  elog(ERROR, "invalid MCV size %zd (expected %zu)",
1125  VARSIZE_ANY(data), expected_size);
1126 
1127  /*
1128  * We need an array of Datum values for each dimension, so that we can
1129  * easily translate the uint16 indexes later. We also need a top-level
1130  * array of pointers to those per-dimension arrays.
1131  *
1132  * While allocating the arrays for dimensions, compute how much space we
1133  * need for a copy of the by-ref data, as we can't simply point to the
1134  * original values (it might go away).
1135  */
1136  datalen = 0; /* space for by-ref data */
1137  map = (Datum **) palloc(ndims * sizeof(Datum *));
1138 
1139  for (dim = 0; dim < ndims; dim++)
1140  {
1141  map[dim] = (Datum *) palloc(sizeof(Datum) * info[dim].nvalues);
1142 
1143  /* space needed for a copy of data for by-ref types */
1144  datalen += info[dim].nbytes_aligned;
1145  }
1146 
1147  /*
1148  * Now resize the MCV list so that the allocation includes all the data.
1149  *
1150  * Allocate space for a copy of the data, as we can't simply reference the
1151  * serialized data - it's not aligned properly, and it may disappear while
1152  * we're still using the MCV list, e.g. due to catcache release.
1153  *
1154  * We do care about alignment here, because we will allocate all the
1155  * pieces at once, but then use pointers to different parts.
1156  */
1157  mcvlen = MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1158 
1159  /* arrays of values and isnull flags for all MCV items */
1160  mcvlen += nitems * MAXALIGN(sizeof(Datum) * ndims);
1161  mcvlen += nitems * MAXALIGN(sizeof(bool) * ndims);
1162 
1163  /* we don't quite need to align this, but it makes some asserts easier */
1164  mcvlen += MAXALIGN(datalen);
1165 
1166  /* now resize the deserialized MCV list, and compute pointers to parts */
1167  mcvlist = repalloc(mcvlist, mcvlen);
1168 
1169  /* pointer to the beginning of values/isnull arrays */
1170  valuesptr = (char *) mcvlist
1171  + MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1172 
1173  isnullptr = valuesptr + (nitems * MAXALIGN(sizeof(Datum) * ndims));
1174 
1175  dataptr = isnullptr + (nitems * MAXALIGN(sizeof(bool) * ndims));
1176 
1177  /*
1178  * Build mapping (index => value) for translating the serialized data into
1179  * the in-memory representation.
1180  */
1181  for (dim = 0; dim < ndims; dim++)
1182  {
1183  /* remember start position in the input array */
1184  char *start PG_USED_FOR_ASSERTS_ONLY = ptr;
1185 
1186  if (info[dim].typbyval)
1187  {
1188  /* for by-val types we simply copy data into the mapping */
1189  for (i = 0; i < info[dim].nvalues; i++)
1190  {
1191  Datum v = 0;
1192 
1193  memcpy(&v, ptr, info[dim].typlen);
1194  ptr += info[dim].typlen;
1195 
1196  map[dim][i] = fetch_att(&v, true, info[dim].typlen);
1197 
1198  /* no under/overflow of input array */
1199  Assert(ptr <= (start + info[dim].nbytes));
1200  }
1201  }
1202  else
1203  {
1204  /* for by-ref types we need to also make a copy of the data */
1205 
1206  /* passed by reference, but fixed length (name, tid, ...) */
1207  if (info[dim].typlen > 0)
1208  {
1209  for (i = 0; i < info[dim].nvalues; i++)
1210  {
1211  memcpy(dataptr, ptr, info[dim].typlen);
1212  ptr += info[dim].typlen;
1213 
1214  /* just point into the array */
1215  map[dim][i] = PointerGetDatum(dataptr);
1216  dataptr += MAXALIGN(info[dim].typlen);
1217  }
1218  }
1219  else if (info[dim].typlen == -1)
1220  {
1221  /* varlena */
1222  for (i = 0; i < info[dim].nvalues; i++)
1223  {
1224  uint32 len;
1225 
1226  /* read the uint32 length */
1227  memcpy(&len, ptr, sizeof(uint32));
1228  ptr += sizeof(uint32);
1229 
1230  /* the length is data-only */
1231  SET_VARSIZE(dataptr, len + VARHDRSZ);
1232  memcpy(VARDATA(dataptr), ptr, len);
1233  ptr += len;
1234 
1235  /* just point into the array */
1236  map[dim][i] = PointerGetDatum(dataptr);
1237 
1238  /* skip to place of the next deserialized value */
1239  dataptr += MAXALIGN(len + VARHDRSZ);
1240  }
1241  }
1242  else if (info[dim].typlen == -2)
1243  {
1244  /* cstring */
1245  for (i = 0; i < info[dim].nvalues; i++)
1246  {
1247  uint32 len;
1248 
1249  memcpy(&len, ptr, sizeof(uint32));
1250  ptr += sizeof(uint32);
1251 
1252  memcpy(dataptr, ptr, len);
1253  ptr += len;
1254 
1255  /* just point into the array */
1256  map[dim][i] = PointerGetDatum(dataptr);
1257  dataptr += MAXALIGN(len);
1258  }
1259  }
1260 
1261  /* no under/overflow of input array */
1262  Assert(ptr <= (start + info[dim].nbytes));
1263 
1264  /* no overflow of the output mcv value */
1265  Assert(dataptr <= ((char *) mcvlist + mcvlen));
1266  }
1267 
1268  /* check we consumed input data for this dimension exactly */
1269  Assert(ptr == (start + info[dim].nbytes));
1270  }
1271 
1272  /* we should have also filled the MCV list exactly */
1273  Assert(dataptr == ((char *) mcvlist + mcvlen));
1274 
1275  /* deserialize the MCV items and translate the indexes to Datums */
1276  for (i = 0; i < nitems; i++)
1277  {
1278  MCVItem *item = &mcvlist->items[i];
1279 
1280  item->values = (Datum *) valuesptr;
1281  valuesptr += MAXALIGN(sizeof(Datum) * ndims);
1282 
1283  item->isnull = (bool *) isnullptr;
1284  isnullptr += MAXALIGN(sizeof(bool) * ndims);
1285 
1286  memcpy(item->isnull, ptr, sizeof(bool) * ndims);
1287  ptr += sizeof(bool) * ndims;
1288 
1289  memcpy(&item->frequency, ptr, sizeof(double));
1290  ptr += sizeof(double);
1291 
1292  memcpy(&item->base_frequency, ptr, sizeof(double));
1293  ptr += sizeof(double);
1294 
1295  /* finally translate the indexes (for non-NULL only) */
1296  for (dim = 0; dim < ndims; dim++)
1297  {
1298  uint16 index;
1299 
1300  memcpy(&index, ptr, sizeof(uint16));
1301  ptr += sizeof(uint16);
1302 
1303  if (item->isnull[dim])
1304  continue;
1305 
1306  item->values[dim] = map[dim][index];
1307  }
1308 
1309  /* check we're not overflowing the input */
1310  Assert(ptr <= endptr);
1311  }
1312 
1313  /* check that we processed all the data */
1314  Assert(ptr == endptr);
1315 
1316  /* release the buffers used for mapping */
1317  for (dim = 0; dim < ndims; dim++)
1318  pfree(map[dim]);
1319 
1320  pfree(map);
1321 
1322  return mcvlist;
1323 }
uint32 nitems
Definition: statistics.h:91
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define VARDATA(PTR)
Definition: postgres.h:315
Oid types[STATS_MAX_DIMENSIONS]
Definition: statistics.h:93
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARHDRSZ
Definition: c.h:627
#define MinSizeOfMCVList
Definition: mcv.c:63
struct DimensionInfo DimensionInfo
uint32 magic
Definition: statistics.h:89
AttrNumber ndimensions
Definition: statistics.h:92
struct MCVItem MCVItem
Datum * values
Definition: statistics.h:83
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
unsigned int Oid
Definition: postgres_ext.h:31
Definition: type.h:89
unsigned short uint16
Definition: c.h:440
void pfree(void *pointer)
Definition: mcxt.c:1169
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
#define SizeOfMCVList(ndims, nitems)
Definition: mcv.c:72
uint32 type
Definition: statistics.h:90
#define STATS_MCV_TYPE_BASIC
Definition: statistics.h:67
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
#define STATS_MCVLIST_MAX_ITEMS
Definition: statistics.h:70
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
#define STATS_MCV_MAGIC
Definition: statistics.h:66
size_t Size
Definition: c.h:540
#define MAXALIGN(LEN)
Definition: c.h:757
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
bool * isnull
Definition: statistics.h:82
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
void * palloc(Size size)
Definition: mcxt.c:1062
#define fetch_att(T, attbyval, attlen)
Definition: tupmacs.h:75
#define elog(elevel,...)
Definition: elog.h:232
int i
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727
double frequency
Definition: statistics.h:80
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
unsigned char bool
Definition: c.h:391

◆ statext_mcv_serialize()

bytea* statext_mcv_serialize ( MCVList mcv,
VacAttrStats **  stats 
)

Definition at line 621 of file mcv.c.

References Assert, VacAttrStats::attrcollid, VacAttrStats::attrtype, MCVItem::base_frequency, bsearch_arg(), compare_datums_simple(), compare_scalars_simple(), CurrentMemoryContext, DatumGetCString, DatumGetPointer, MCVItem::frequency, i, MCVItem::isnull, ITEM_SIZE, MCVList::items, lookup_type_cache(), TypeCacheEntry::lt_opr, MCVList::magic, MAXALIGN, DimensionInfo::nbytes, DimensionInfo::nbytes_aligned, MCVList::ndimensions, MCVList::nitems, DimensionInfo::nvalues, palloc0(), pfree(), PG_DETOAST_DATUM, PG_UINT16_MAX, PG_USED_FOR_ASSERTS_ONLY, PointerGetDatum, PrepareSortSupportFromOrderingOp(), qsort_arg(), SET_VARSIZE, SortSupportData::ssup_collation, SortSupportData::ssup_cxt, SortSupportData::ssup_nulls_first, store_att_byval, DimensionInfo::typbyval, MCVList::type, TYPECACHE_LT_OPR, MCVList::types, DimensionInfo::typlen, value, MCVItem::values, values, VARDATA, VARDATA_ANY, VARHDRSZ, and VARSIZE_ANY_EXHDR.

Referenced by statext_store().

622 {
623  int i;
624  int dim;
625  int ndims = mcvlist->ndimensions;
626 
627  SortSupport ssup;
628  DimensionInfo *info;
629 
630  Size total_length;
631 
632  /* serialized items (indexes into arrays, etc.) */
633  bytea *raw;
634  char *ptr;
635  char *endptr PG_USED_FOR_ASSERTS_ONLY;
636 
637  /* values per dimension (and number of non-NULL values) */
638  Datum **values = (Datum **) palloc0(sizeof(Datum *) * ndims);
639  int *counts = (int *) palloc0(sizeof(int) * ndims);
640 
641  /*
642  * We'll include some rudimentary information about the attribute types
643  * (length, by-val flag), so that we don't have to look them up while
644  * deserializating the MCV list (we already have the type OID in the
645  * header). This is safe, because when changing type of the attribute the
646  * statistics gets dropped automatically. We need to store the info about
647  * the arrays of deduplicated values anyway.
648  */
649  info = (DimensionInfo *) palloc0(sizeof(DimensionInfo) * ndims);
650 
651  /* sort support data for all attributes included in the MCV list */
652  ssup = (SortSupport) palloc0(sizeof(SortSupportData) * ndims);
653 
654  /* collect and deduplicate values for each dimension (attribute) */
655  for (dim = 0; dim < ndims; dim++)
656  {
657  int ndistinct;
658  TypeCacheEntry *typentry;
659 
660  /*
661  * Lookup the LT operator (can't get it from stats extra_data, as we
662  * don't know how to interpret that - scalar vs. array etc.).
663  */
664  typentry = lookup_type_cache(stats[dim]->attrtypid, TYPECACHE_LT_OPR);
665 
666  /* copy important info about the data type (length, by-value) */
667  info[dim].typlen = stats[dim]->attrtype->typlen;
668  info[dim].typbyval = stats[dim]->attrtype->typbyval;
669 
670  /* allocate space for values in the attribute and collect them */
671  values[dim] = (Datum *) palloc0(sizeof(Datum) * mcvlist->nitems);
672 
673  for (i = 0; i < mcvlist->nitems; i++)
674  {
675  /* skip NULL values - we don't need to deduplicate those */
676  if (mcvlist->items[i].isnull[dim])
677  continue;
678 
679  /* append the value at the end */
680  values[dim][counts[dim]] = mcvlist->items[i].values[dim];
681  counts[dim] += 1;
682  }
683 
684  /* if there are just NULL values in this dimension, we're done */
685  if (counts[dim] == 0)
686  continue;
687 
688  /* sort and deduplicate the data */
689  ssup[dim].ssup_cxt = CurrentMemoryContext;
690  ssup[dim].ssup_collation = stats[dim]->attrcollid;
691  ssup[dim].ssup_nulls_first = false;
692 
693  PrepareSortSupportFromOrderingOp(typentry->lt_opr, &ssup[dim]);
694 
695  qsort_arg(values[dim], counts[dim], sizeof(Datum),
696  compare_scalars_simple, &ssup[dim]);
697 
698  /*
699  * Walk through the array and eliminate duplicate values, but keep the
700  * ordering (so that we can do bsearch later). We know there's at
701  * least one item as (counts[dim] != 0), so we can skip the first
702  * element.
703  */
704  ndistinct = 1; /* number of distinct values */
705  for (i = 1; i < counts[dim]; i++)
706  {
707  /* expect sorted array */
708  Assert(compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]) <= 0);
709 
710  /* if the value is the same as the previous one, we can skip it */
711  if (!compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]))
712  continue;
713 
714  values[dim][ndistinct] = values[dim][i];
715  ndistinct += 1;
716  }
717 
718  /* we must not exceed PG_UINT16_MAX, as we use uint16 indexes */
719  Assert(ndistinct <= PG_UINT16_MAX);
720 
721  /*
722  * Store additional info about the attribute - number of deduplicated
723  * values, and also size of the serialized data. For fixed-length data
724  * types this is trivial to compute, for varwidth types we need to
725  * actually walk the array and sum the sizes.
726  */
727  info[dim].nvalues = ndistinct;
728 
729  if (info[dim].typbyval) /* by-value data types */
730  {
731  info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
732 
733  /*
734  * We copy the data into the MCV item during deserialization, so
735  * we don't need to allocate any extra space.
736  */
737  info[dim].nbytes_aligned = 0;
738  }
739  else if (info[dim].typlen > 0) /* fixed-length by-ref */
740  {
741  /*
742  * We don't care about alignment in the serialized data, so we
743  * pack the data as much as possible. But we also track how much
744  * data will be needed after deserialization, and in that case we
745  * need to account for alignment of each item.
746  *
747  * Note: As the items are fixed-length, we could easily compute
748  * this during deserialization, but we do it here anyway.
749  */
750  info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
751  info[dim].nbytes_aligned = info[dim].nvalues * MAXALIGN(info[dim].typlen);
752  }
753  else if (info[dim].typlen == -1) /* varlena */
754  {
755  info[dim].nbytes = 0;
756  info[dim].nbytes_aligned = 0;
757  for (i = 0; i < info[dim].nvalues; i++)
758  {
759  Size len;
760 
761  /*
762  * For varlena values, we detoast the values and store the
763  * length and data separately. We don't bother with alignment
764  * here, which means that during deserialization we need to
765  * copy the fields and only access the copies.
766  */
767  values[dim][i] = PointerGetDatum(PG_DETOAST_DATUM(values[dim][i]));
768 
769  /* serialized length (uint32 length + data) */
770  len = VARSIZE_ANY_EXHDR(values[dim][i]);
771  info[dim].nbytes += sizeof(uint32); /* length */
772  info[dim].nbytes += len; /* value (no header) */
773 
774  /*
775  * During deserialization we'll build regular varlena values
776  * with full headers, and we need to align them properly.
777  */
778  info[dim].nbytes_aligned += MAXALIGN(VARHDRSZ + len);
779  }
780  }
781  else if (info[dim].typlen == -2) /* cstring */
782  {
783  info[dim].nbytes = 0;
784  info[dim].nbytes_aligned = 0;
785  for (i = 0; i < info[dim].nvalues; i++)
786  {
787  Size len;
788 
789  /*
790  * For cstring, we do similar thing as for varlena - first we
791  * store the length as uint32 and then the data. We don't care
792  * about alignment, which means that during deserialization we
793  * need to copy the fields and only access the copies.
794  */
795 
796  /* c-strings include terminator, so +1 byte */
797  len = strlen(DatumGetCString(values[dim][i])) + 1;
798  info[dim].nbytes += sizeof(uint32); /* length */
799  info[dim].nbytes += len; /* value */
800 
801  /* space needed for properly aligned deserialized copies */
802  info[dim].nbytes_aligned += MAXALIGN(len);
803  }
804  }
805 
806  /* we know (count>0) so there must be some data */
807  Assert(info[dim].nbytes > 0);
808  }
809 
810  /*
811  * Now we can finally compute how much space we'll actually need for the
812  * whole serialized MCV list (varlena header, MCV header, dimension info
813  * for each attribute, deduplicated values and items).
814  */
815  total_length = (3 * sizeof(uint32)) /* magic + type + nitems */
816  + sizeof(AttrNumber) /* ndimensions */
817  + (ndims * sizeof(Oid)); /* attribute types */
818 
819  /* dimension info */
820  total_length += ndims * sizeof(DimensionInfo);
821 
822  /* add space for the arrays of deduplicated values */
823  for (i = 0; i < ndims; i++)
824  total_length += info[i].nbytes;
825 
826  /*
827  * And finally account for the items (those are fixed-length, thanks to
828  * replacing values with uint16 indexes into the deduplicated arrays).
829  */
830  total_length += mcvlist->nitems * ITEM_SIZE(dim);
831 
832  /*
833  * Allocate space for the whole serialized MCV list (we'll skip bytes, so
834  * we set them to zero to make the result more compressible).
835  */
836  raw = (bytea *) palloc0(VARHDRSZ + total_length);
837  SET_VARSIZE(raw, VARHDRSZ + total_length);
838 
839  ptr = VARDATA(raw);
840  endptr = ptr + total_length;
841 
842  /* copy the MCV list header fields, one by one */
843  memcpy(ptr, &mcvlist->magic, sizeof(uint32));
844  ptr += sizeof(uint32);
845 
846  memcpy(ptr, &mcvlist->type, sizeof(uint32));
847  ptr += sizeof(uint32);
848 
849  memcpy(ptr, &mcvlist->nitems, sizeof(uint32));
850  ptr += sizeof(uint32);
851 
852  memcpy(ptr, &mcvlist->ndimensions, sizeof(AttrNumber));
853  ptr += sizeof(AttrNumber);
854 
855  memcpy(ptr, mcvlist->types, sizeof(Oid) * ndims);
856  ptr += (sizeof(Oid) * ndims);
857 
858  /* store information about the attributes (data amounts, ...) */
859  memcpy(ptr, info, sizeof(DimensionInfo) * ndims);
860  ptr += sizeof(DimensionInfo) * ndims;
861 
862  /* Copy the deduplicated values for all attributes to the output. */
863  for (dim = 0; dim < ndims; dim++)
864  {
865  /* remember the starting point for Asserts later */
866  char *start PG_USED_FOR_ASSERTS_ONLY = ptr;
867 
868  for (i = 0; i < info[dim].nvalues; i++)
869  {
870  Datum value = values[dim][i];
871 
872  if (info[dim].typbyval) /* passed by value */
873  {
874  Datum tmp;
875 
876  /*
877  * For values passed by value, we need to copy just the
878  * significant bytes - we can't use memcpy directly, as that
879  * assumes little endian behavior. store_att_byval does
880  * almost what we need, but it requires properly aligned
881  * buffer - the output buffer does not guarantee that. So we
882  * simply use a local Datum variable (which guarantees proper
883  * alignment), and then copy the value from it.
884  */
885  store_att_byval(&tmp, value, info[dim].typlen);
886 
887  memcpy(ptr, &tmp, info[dim].typlen);
888  ptr += info[dim].typlen;
889  }
890  else if (info[dim].typlen > 0) /* passed by reference */
891  {
892  /* no special alignment needed, treated as char array */
893  memcpy(ptr, DatumGetPointer(value), info[dim].typlen);
894  ptr += info[dim].typlen;
895  }
896  else if (info[dim].typlen == -1) /* varlena */
897  {
899 
900  /* copy the length */
901  memcpy(ptr, &len, sizeof(uint32));
902  ptr += sizeof(uint32);
903 
904  /* data from the varlena value (without the header) */
905  memcpy(ptr, VARDATA_ANY(DatumGetPointer(value)), len);
906  ptr += len;
907  }
908  else if (info[dim].typlen == -2) /* cstring */
909  {
910  uint32 len = (uint32) strlen(DatumGetCString(value)) + 1;
911 
912  /* copy the length */
913  memcpy(ptr, &len, sizeof(uint32));
914  ptr += sizeof(uint32);
915 
916  /* value */
917  memcpy(ptr, DatumGetCString(value), len);
918  ptr += len;
919  }
920 
921  /* no underflows or overflows */
922  Assert((ptr > start) && ((ptr - start) <= info[dim].nbytes));
923  }
924 
925  /* we should get exactly nbytes of data for this dimension */
926  Assert((ptr - start) == info[dim].nbytes);
927  }
928 
929  /* Serialize the items, with uint16 indexes instead of the values. */
930  for (i = 0; i < mcvlist->nitems; i++)
931  {
932  MCVItem *mcvitem = &mcvlist->items[i];
933 
934  /* don't write beyond the allocated space */
935  Assert(ptr <= (endptr - ITEM_SIZE(dim)));
936 
937  /* copy NULL and frequency flags into the serialized MCV */
938  memcpy(ptr, mcvitem->isnull, sizeof(bool) * ndims);
939  ptr += sizeof(bool) * ndims;
940 
941  memcpy(ptr, &mcvitem->frequency, sizeof(double));
942  ptr += sizeof(double);
943 
944  memcpy(ptr, &mcvitem->base_frequency, sizeof(double));
945  ptr += sizeof(double);
946 
947  /* store the indexes last */
948  for (dim = 0; dim < ndims; dim++)
949  {
950  uint16 index = 0;
951  Datum *value;
952 
953  /* do the lookup only for non-NULL values */
954  if (!mcvitem->isnull[dim])
955  {
956  value = (Datum *) bsearch_arg(&mcvitem->values[dim], values[dim],
957  info[dim].nvalues, sizeof(Datum),
958  compare_scalars_simple, &ssup[dim]);
959 
960  Assert(value != NULL); /* serialization or deduplication
961  * error */
962 
963  /* compute index within the deduplicated array */
964  index = (uint16) (value - values[dim]);
965 
966  /* check the index is within expected bounds */
967  Assert(index < info[dim].nvalues);
968  }
969 
970  /* copy the index into the serialized MCV */
971  memcpy(ptr, &index, sizeof(uint16));
972  ptr += sizeof(uint16);
973  }
974 
975  /* make sure we don't overflow the allocated value */
976  Assert(ptr <= endptr);
977  }
978 
979  /* at this point we expect to match the total_length exactly */
980  Assert(ptr == endptr);
981 
982  pfree(values);
983  pfree(counts);
984 
985  return raw;
986 }
struct SortSupportData * SortSupport
Definition: sortsupport.h:58
bool ssup_nulls_first
Definition: sortsupport.h:75
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define VARDATA(PTR)
Definition: postgres.h:315
static struct @142 value
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARHDRSZ
Definition: c.h:627
void PrepareSortSupportFromOrderingOp(Oid orderingOp, SortSupport ssup)
Definition: sortsupport.c:135
struct DimensionInfo DimensionInfo
Datum * values
Definition: statistics.h:83
unsigned int Oid
Definition: postgres_ext.h:31
Definition: type.h:89
unsigned short uint16
Definition: c.h:440
void pfree(void *pointer)
Definition: mcxt.c:1169
#define PG_UINT16_MAX
Definition: c.h:522
#define DatumGetCString(X)
Definition: postgres.h:610
MemoryContext ssup_cxt
Definition: sortsupport.h:66
int compare_scalars_simple(const void *a, const void *b, void *arg)
unsigned int uint32
Definition: c.h:441
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
int compare_datums_simple(Datum a, Datum b, SortSupport ssup)
#define store_att_byval(T, newdatum, attlen)
Definition: tupmacs.h:226
void * bsearch_arg(const void *key, const void *base, size_t nmemb, size_t size, int(*compar)(const void *, const void *, void *), void *arg)
Definition: bsearch_arg.c:55
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
TypeCacheEntry * lookup_type_cache(Oid type_id, int flags)
Definition: typcache.c:338
#define ITEM_SIZE(ndims)
Definition: mcv.c:57
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
size_t Size
Definition: c.h:540
#define MAXALIGN(LEN)
Definition: c.h:757
Oid attrcollid
Definition: vacuum.h:127
#define DatumGetPointer(X)
Definition: postgres.h:593
bool * isnull
Definition: statistics.h:82
static Datum values[MAXATTR]
Definition: bootstrap.c:166
Form_pg_type attrtype
Definition: vacuum.h:126
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
int i
#define TYPECACHE_LT_OPR
Definition: typcache.h:137
#define PG_DETOAST_DATUM(datum)
Definition: fmgr.h:240
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21
double frequency
Definition: statistics.h:80
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
unsigned char bool
Definition: c.h:391

◆ statext_ndistinct_build()

MVNDistinct* statext_ndistinct_build ( double  totalrows,
StatsBuildData data 
)

Definition at line 89 of file mvdistinct.c.

References Assert, StatsBuildData::attnums, AttributeNumberIsValid, MVNDistinctItem::attributes, generator_free(), generator_init(), generator_next(), MVNDistinct::items, CombinationGenerator::k, MVNDistinct::magic, StatsBuildData::nattnums, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, ndistinct_for_combination(), MVNDistinct::nitems, num_combinations(), offsetof, palloc(), STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, and MVNDistinct::type.

Referenced by BuildRelationExtStatistics().

90 {
91  MVNDistinct *result;
92  int k;
93  int itemcnt;
94  int numattrs = data->nattnums;
95  int numcombs = num_combinations(numattrs);
96 
97  result = palloc(offsetof(MVNDistinct, items) +
98  numcombs * sizeof(MVNDistinctItem));
99  result->magic = STATS_NDISTINCT_MAGIC;
101  result->nitems = numcombs;
102 
103  itemcnt = 0;
104  for (k = 2; k <= numattrs; k++)
105  {
106  int *combination;
108 
109  /* generate combinations of K out of N elements */
110  generator = generator_init(numattrs, k);
111 
112  while ((combination = generator_next(generator)))
113  {
114  MVNDistinctItem *item = &result->items[itemcnt];
115  int j;
116 
117  item->attributes = palloc(sizeof(AttrNumber) * k);
118  item->nattributes = k;
119 
120  /* translate the indexes to attnums */
121  for (j = 0; j < k; j++)
122  {
123  item->attributes[j] = data->attnums[combination[j]];
124 
126  }
127 
128  item->ndistinct =
129  ndistinct_for_combination(totalrows, data, k, combination);
130 
131  itemcnt++;
132  Assert(itemcnt <= result->nitems);
133  }
134 
135  generator_free(generator);
136  }
137 
138  /* must consume exactly the whole output array */
139  Assert(itemcnt == result->nitems);
140 
141  return result;
142 }
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
double ndistinct
Definition: statistics.h:28
static int * generator_next(CombinationGenerator *state)
Definition: mvdistinct.c:627
static double ndistinct_for_combination(double totalrows, StatsBuildData *data, int k, int *combination)
Definition: mvdistinct.c:425
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
#define AttributeNumberIsValid(attributeNumber)
Definition: attnum.h:34
static void generator_free(CombinationGenerator *state)
Definition: mvdistinct.c:642
AttrNumber * attributes
Definition: statistics.h:30
uint32 type
Definition: statistics.h:37
#define Assert(condition)
Definition: c.h:804
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
void * palloc(Size size)
Definition: mcxt.c:1062
static int num_combinations(int n)
Definition: mvdistinct.c:575
int16 AttrNumber
Definition: attnum.h:21
static CombinationGenerator * generator_init(int n, int k)
Definition: mvdistinct.c:589
#define offsetof(type, field)
Definition: c.h:727

◆ statext_ndistinct_deserialize()

MVNDistinct* statext_ndistinct_deserialize ( bytea data)

Definition at line 250 of file mvdistinct.c.

References Assert, MVNDistinctItem::attributes, elog, ERROR, i, MVNDistinct::items, MVNDistinct::magic, MAXALIGN, MinSizeOfItems, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, MVNDistinct::nitems, offsetof, palloc(), palloc0(), SizeOfHeader, STATS_MAX_DIMENSIONS, STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, MVNDistinct::type, VARDATA_ANY, VARSIZE_ANY, and VARSIZE_ANY_EXHDR.

Referenced by pg_ndistinct_out(), and statext_ndistinct_load().

251 {
252  int i;
253  Size minimum_size;
254  MVNDistinct ndist;
255  MVNDistinct *ndistinct;
256  char *tmp;
257 
258  if (data == NULL)
259  return NULL;
260 
261  /* we expect at least the basic fields of MVNDistinct struct */
262  if (VARSIZE_ANY_EXHDR(data) < SizeOfHeader)
263  elog(ERROR, "invalid MVNDistinct size %zd (expected at least %zd)",
265 
266  /* initialize pointer to the data part (skip the varlena header) */
267  tmp = VARDATA_ANY(data);
268 
269  /* read the header fields and perform basic sanity checks */
270  memcpy(&ndist.magic, tmp, sizeof(uint32));
271  tmp += sizeof(uint32);
272  memcpy(&ndist.type, tmp, sizeof(uint32));
273  tmp += sizeof(uint32);
274  memcpy(&ndist.nitems, tmp, sizeof(uint32));
275  tmp += sizeof(uint32);
276 
277  if (ndist.magic != STATS_NDISTINCT_MAGIC)
278  elog(ERROR, "invalid ndistinct magic %08x (expected %08x)",
280  if (ndist.type != STATS_NDISTINCT_TYPE_BASIC)
281  elog(ERROR, "invalid ndistinct type %d (expected %d)",
283  if (ndist.nitems == 0)
284  elog(ERROR, "invalid zero-length item array in MVNDistinct");
285 
286  /* what minimum bytea size do we expect for those parameters */
287  minimum_size = MinSizeOfItems(ndist.nitems);
288  if (VARSIZE_ANY_EXHDR(data) < minimum_size)
289  elog(ERROR, "invalid MVNDistinct size %zd (expected at least %zd)",
290  VARSIZE_ANY_EXHDR(data), minimum_size);
291 
292  /*
293  * Allocate space for the ndistinct items (no space for each item's
294  * attnos: those live in bitmapsets allocated separately)
295  */
296  ndistinct = palloc0(MAXALIGN(offsetof(MVNDistinct, items)) +
297  (ndist.nitems * sizeof(MVNDistinctItem)));
298  ndistinct->magic = ndist.magic;
299  ndistinct->type = ndist.type;
300  ndistinct->nitems = ndist.nitems;
301 
302  for (i = 0; i < ndistinct->nitems; i++)
303  {
304  MVNDistinctItem *item = &ndistinct->items[i];
305 
306  /* ndistinct value */
307  memcpy(&item->ndistinct, tmp, sizeof(double));
308  tmp += sizeof(double);
309 
310  /* number of attributes */
311  memcpy(&item->nattributes, tmp, sizeof(int));
312  tmp += sizeof(int);
313  Assert((item->nattributes >= 2) && (item->nattributes <= STATS_MAX_DIMENSIONS));
314 
315  item->attributes
316  = (AttrNumber *) palloc(item->nattributes * sizeof(AttrNumber));
317 
318  memcpy(item->attributes, tmp, sizeof(AttrNumber) * item->nattributes);
319  tmp += sizeof(AttrNumber) * item->nattributes;
320 
321  /* still within the bytea */
322  Assert(tmp <= ((char *) data + VARSIZE_ANY(data)));
323  }
324 
325  /* we should have consumed the whole bytea exactly */
326  Assert(tmp == ((char *) data + VARSIZE_ANY(data)));
327 
328  return ndistinct;
329 }
#define MinSizeOfItems(nitems)
Definition: mvdistinct.c:56
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
double ndistinct
Definition: statistics.h:28
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
void * palloc0(Size size)
Definition: mcxt.c:1093
AttrNumber * attributes
Definition: statistics.h:30
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
uint32 type
Definition: statistics.h:37
#define SizeOfHeader
Definition: mvdistinct.c:46
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
#define MAXALIGN(LEN)
Definition: c.h:757
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
int i
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_ndistinct_serialize()

bytea* statext_ndistinct_serialize ( MVNDistinct ndistinct)

Definition at line 179 of file mvdistinct.c.

References Assert, MVNDistinctItem::attributes, i, MVNDistinct::items, MVNDistinct::magic, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, MVNDistinct::nitems, output(), palloc(), SET_VARSIZE, SizeOfHeader, SizeOfItem, STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, MVNDistinct::type, VARDATA, and VARHDRSZ.

Referenced by statext_store().

180 {
181  int i;
182  bytea *output;
183  char *tmp;
184  Size len;
185 
186  Assert(ndistinct->magic == STATS_NDISTINCT_MAGIC);
187  Assert(ndistinct->type == STATS_NDISTINCT_TYPE_BASIC);
188 
189  /*
190  * Base size is size of scalar fields in the struct, plus one base struct
191  * for each item, including number of items for each.
192  */
193  len = VARHDRSZ + SizeOfHeader;
194 
195  /* and also include space for the actual attribute numbers */
196  for (i = 0; i < ndistinct->nitems; i++)
197  {
198  int nmembers;
199 
200  nmembers = ndistinct->items[i].nattributes;
201  Assert(nmembers >= 2);
202 
203  len += SizeOfItem(nmembers);
204  }
205 
206  output = (bytea *) palloc(len);
207  SET_VARSIZE(output, len);
208 
209  tmp = VARDATA(output);
210 
211  /* Store the base struct values (magic, type, nitems) */
212  memcpy(tmp, &ndistinct->magic, sizeof(uint32));
213  tmp += sizeof(uint32);
214  memcpy(tmp, &ndistinct->type, sizeof(uint32));
215  tmp += sizeof(uint32);
216  memcpy(tmp, &ndistinct->nitems, sizeof(uint32));
217  tmp += sizeof(uint32);
218 
219  /*
220  * store number of attributes and attribute numbers for each entry
221  */
222  for (i = 0; i < ndistinct->nitems; i++)
223  {
224  MVNDistinctItem item = ndistinct->items[i];
225  int nmembers = item.nattributes;
226 
227  memcpy(tmp, &item.ndistinct, sizeof(double));
228  tmp += sizeof(double);
229  memcpy(tmp, &nmembers, sizeof(int));
230  tmp += sizeof(int);
231 
232  memcpy(tmp, item.attributes, sizeof(AttrNumber) * nmembers);
233  tmp += nmembers * sizeof(AttrNumber);
234 
235  /* protect against overflows */
236  Assert(tmp <= ((char *) output + len));
237  }
238 
239  /* check we used exactly the expected space */
240  Assert(tmp == ((char *) output + len));
241 
242  return output;
243 }
#define VARDATA(PTR)
Definition: postgres.h:315
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
static void output(uint64 loop_count)
#define VARHDRSZ
Definition: c.h:627
double ndistinct
Definition: statistics.h:28
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
#define SizeOfItem(natts)
Definition: mvdistinct.c:49
unsigned int uint32
Definition: c.h:441
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
AttrNumber * attributes
Definition: statistics.h:30
uint32 type
Definition: statistics.h:37
#define SizeOfHeader
Definition: mvdistinct.c:46
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
void * palloc(Size size)
Definition: mcxt.c:1062
int i
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21