PostgreSQL Source Code  git master
extended_stats_internal.h File Reference
Include dependency graph for extended_stats_internal.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  StdAnalyzeData
 
struct  ScalarItem
 
struct  DimensionInfo
 
struct  MultiSortSupportData
 
struct  SortItem
 
struct  StatsBuildData
 

Typedefs

typedef struct DimensionInfo DimensionInfo
 
typedef struct MultiSortSupportData MultiSortSupportData
 
typedef MultiSortSupportDataMultiSortSupport
 
typedef struct SortItem SortItem
 
typedef struct StatsBuildData StatsBuildData
 

Functions

MVNDistinctstatext_ndistinct_build (double totalrows, StatsBuildData *data)
 
byteastatext_ndistinct_serialize (MVNDistinct *ndistinct)
 
MVNDistinctstatext_ndistinct_deserialize (bytea *data)
 
MVDependenciesstatext_dependencies_build (StatsBuildData *data)
 
byteastatext_dependencies_serialize (MVDependencies *dependencies)
 
MVDependenciesstatext_dependencies_deserialize (bytea *data)
 
MCVListstatext_mcv_build (StatsBuildData *data, double totalrows, int stattarget)
 
byteastatext_mcv_serialize (MCVList *mcv, VacAttrStats **stats)
 
MCVListstatext_mcv_deserialize (bytea *data)
 
MultiSortSupport multi_sort_init (int ndims)
 
void multi_sort_add_dimension (MultiSortSupport mss, int sortdim, Oid oper, Oid collation)
 
int multi_sort_compare (const void *a, const void *b, void *arg)
 
int multi_sort_compare_dim (int dim, const SortItem *a, const SortItem *b, MultiSortSupport mss)
 
int multi_sort_compare_dims (int start, int end, const SortItem *a, const SortItem *b, MultiSortSupport mss)
 
int compare_scalars_simple (const void *a, const void *b, void *arg)
 
int compare_datums_simple (Datum a, Datum b, SortSupport ssup)
 
AttrNumberbuild_attnums_array (Bitmapset *attrs, int nexprs, int *numattrs)
 
SortItembuild_sorted_items (StatsBuildData *data, int *nitems, MultiSortSupport mss, int numattrs, AttrNumber *attnums)
 
bool examine_opclause_args (List *args, Node **exprp, Const **cstp, bool *expronleftp)
 
Selectivity mcv_combine_selectivities (Selectivity simple_sel, Selectivity mcv_sel, Selectivity mcv_basesel, Selectivity mcv_totalsel)
 
Selectivity mcv_clauselist_selectivity (PlannerInfo *root, StatisticExtInfo *stat, List *clauses, int varRelid, JoinType jointype, SpecialJoinInfo *sjinfo, RelOptInfo *rel, Selectivity *basesel, Selectivity *totalsel)
 
Selectivity mcv_clause_selectivity_or (PlannerInfo *root, StatisticExtInfo *stat, MCVList *mcv, Node *clause, bool **or_matches, Selectivity *basesel, Selectivity *overlap_mcvsel, Selectivity *overlap_basesel, Selectivity *totalsel)
 

Typedef Documentation

◆ DimensionInfo

typedef struct DimensionInfo DimensionInfo

◆ MultiSortSupport

Definition at line 51 of file extended_stats_internal.h.

◆ MultiSortSupportData

◆ SortItem

typedef struct SortItem SortItem

◆ StatsBuildData

Function Documentation

◆ build_attnums_array()

AttrNumber* build_attnums_array ( Bitmapset attrs,
int  nexprs,
int *  numattrs 
)

Definition at line 965 of file extended_stats.c.

References Assert, attnum, AttributeNumberIsValid, bms_next_member(), bms_num_members(), i, MaxAttrNumber, and palloc().

966 {
967  int i,
968  j;
969  AttrNumber *attnums;
970  int num = bms_num_members(attrs);
971 
972  if (numattrs)
973  *numattrs = num;
974 
975  /* build attnums from the bitmapset */
976  attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * num);
977  i = 0;
978  j = -1;
979  while ((j = bms_next_member(attrs, j)) >= 0)
980  {
981  int attnum = (j - nexprs);
982 
983  /*
984  * Make sure the bitmap contains only user-defined attributes. As
985  * bitmaps can't contain negative values, this can be violated in two
986  * ways. Firstly, the bitmap might contain 0 as a member, and secondly
987  * the integer value might be larger than MaxAttrNumber.
988  */
990  Assert(attnum <= MaxAttrNumber);
991  Assert(attnum >= (-nexprs));
992 
993  attnums[i++] = (AttrNumber) attnum;
994 
995  /* protect against overflows */
996  Assert(i <= num);
997  }
998 
999  return attnums;
1000 }
#define MaxAttrNumber
Definition: attnum.h:24
int bms_next_member(const Bitmapset *a, int prevbit)
Definition: bitmapset.c:1043
int bms_num_members(const Bitmapset *a)
Definition: bitmapset.c:646
#define AttributeNumberIsValid(attributeNumber)
Definition: attnum.h:34
int16 attnum
Definition: pg_attribute.h:83
#define Assert(condition)
Definition: c.h:804
void * palloc(Size size)
Definition: mcxt.c:1062
int i
int16 AttrNumber
Definition: attnum.h:21

◆ build_sorted_items()

SortItem* build_sorted_items ( StatsBuildData data,
int *  nitems,
MultiSortSupport  mss,
int  numattrs,
AttrNumber attnums 
)

Definition at line 1010 of file extended_stats.c.

References Assert, attlen, attnum, StatsBuildData::attnums, VacAttrStats::attrtypid, get_typlen(), i, idx(), multi_sort_compare(), StatsBuildData::nattnums, StatsBuildData::nulls, StatsBuildData::numrows, palloc(), palloc0(), pfree(), PG_DETOAST_DATUM, PointerGetDatum, qsort_arg(), StatsBuildData::stats, toast_raw_datum_size(), value, StatsBuildData::values, values, and WIDTH_THRESHOLD.

Referenced by dependency_degree(), and statext_mcv_build().

1013 {
1014  int i,
1015  j,
1016  len,
1017  nrows;
1018  int nvalues = data->numrows * numattrs;
1019 
1020  SortItem *items;
1021  Datum *values;
1022  bool *isnull;
1023  char *ptr;
1024  int *typlen;
1025 
1026  /* Compute the total amount of memory we need (both items and values). */
1027  len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
1028 
1029  /* Allocate the memory and split it into the pieces. */
1030  ptr = palloc0(len);
1031 
1032  /* items to sort */
1033  items = (SortItem *) ptr;
1034  ptr += data->numrows * sizeof(SortItem);
1035 
1036  /* values and null flags */
1037  values = (Datum *) ptr;
1038  ptr += nvalues * sizeof(Datum);
1039 
1040  isnull = (bool *) ptr;
1041  ptr += nvalues * sizeof(bool);
1042 
1043  /* make sure we consumed the whole buffer exactly */
1044  Assert((ptr - (char *) items) == len);
1045 
1046  /* fix the pointers to Datum and bool arrays */
1047  nrows = 0;
1048  for (i = 0; i < data->numrows; i++)
1049  {
1050  items[nrows].values = &values[nrows * numattrs];
1051  items[nrows].isnull = &isnull[nrows * numattrs];
1052 
1053  nrows++;
1054  }
1055 
1056  /* build a local cache of typlen for all attributes */
1057  typlen = (int *) palloc(sizeof(int) * data->nattnums);
1058  for (i = 0; i < data->nattnums; i++)
1059  typlen[i] = get_typlen(data->stats[i]->attrtypid);
1060 
1061  nrows = 0;
1062  for (i = 0; i < data->numrows; i++)
1063  {
1064  bool toowide = false;
1065 
1066  /* load the values/null flags from sample rows */
1067  for (j = 0; j < numattrs; j++)
1068  {
1069  Datum value;
1070  bool isnull;
1071  int attlen;
1072  AttrNumber attnum = attnums[j];
1073 
1074  int idx;
1075 
1076  /* match attnum to the pre-calculated data */
1077  for (idx = 0; idx < data->nattnums; idx++)
1078  {
1079  if (attnum == data->attnums[idx])
1080  break;
1081  }
1082 
1083  Assert(idx < data->nattnums);
1084 
1085  value = data->values[idx][i];
1086  isnull = data->nulls[idx][i];
1087  attlen = typlen[idx];
1088 
1089  /*
1090  * If this is a varlena value, check if it's too wide and if yes
1091  * then skip the whole item. Otherwise detoast the value.
1092  *
1093  * XXX It may happen that we've already detoasted some preceding
1094  * values for the current item. We don't bother to cleanup those
1095  * on the assumption that those are small (below WIDTH_THRESHOLD)
1096  * and will be discarded at the end of analyze.
1097  */
1098  if ((!isnull) && (attlen == -1))
1099  {
1101  {
1102  toowide = true;
1103  break;
1104  }
1105 
1106  value = PointerGetDatum(PG_DETOAST_DATUM(value));
1107  }
1108 
1109  items[nrows].values[j] = value;
1110  items[nrows].isnull[j] = isnull;
1111  }
1112 
1113  if (toowide)
1114  continue;
1115 
1116  nrows++;
1117  }
1118 
1119  /* store the actual number of items (ignoring the too-wide ones) */
1120  *nitems = nrows;
1121 
1122  /* all items were too wide */
1123  if (nrows == 0)
1124  {
1125  /* everything is allocated as a single chunk */
1126  pfree(items);
1127  return NULL;
1128  }
1129 
1130  /* do the sort, using the multi-sort */
1131  qsort_arg((void *) items, nrows, sizeof(SortItem),
1132  multi_sort_compare, mss);
1133 
1134  return items;
1135 }
#define PointerGetDatum(X)
Definition: postgres.h:600
Datum idx(PG_FUNCTION_ARGS)
Definition: _int_op.c:259
void pfree(void *pointer)
Definition: mcxt.c:1169
Oid attrtypid
Definition: vacuum.h:124
Size toast_raw_datum_size(Datum value)
Definition: detoast.c:545
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
int16 attlen
Definition: pg_attribute.h:68
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
struct SortItem SortItem
#define WIDTH_THRESHOLD
VacAttrStats ** stats
int16 attnum
Definition: pg_attribute.h:83
static struct @143 value
#define Assert(condition)
Definition: c.h:804
int multi_sort_compare(const void *a, const void *b, void *arg)
static Datum values[MAXATTR]
Definition: bootstrap.c:156
int16 get_typlen(Oid typid)
Definition: lsyscache.c:2144
void * palloc(Size size)
Definition: mcxt.c:1062
int i
#define PG_DETOAST_DATUM(datum)
Definition: fmgr.h:240
int16 AttrNumber
Definition: attnum.h:21
unsigned char bool
Definition: c.h:391

◆ compare_datums_simple()

int compare_datums_simple ( Datum  a,
Datum  b,
SortSupport  ssup 
)

Definition at line 951 of file extended_stats.c.

References ApplySortComparator().

Referenced by compare_scalars_simple(), and statext_mcv_serialize().

952 {
953  return ApplySortComparator(a, false, b, false, ssup);
954 }
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ compare_scalars_simple()

int compare_scalars_simple ( const void *  a,
const void *  b,
void *  arg 
)

Definition at line 943 of file extended_stats.c.

References compare_datums_simple().

Referenced by statext_mcv_serialize().

944 {
945  return compare_datums_simple(*(Datum *) a,
946  *(Datum *) b,
947  (SortSupport) arg);
948 }
int compare_datums_simple(Datum a, Datum b, SortSupport ssup)
uintptr_t Datum
Definition: postgres.h:411
void * arg

◆ examine_opclause_args()

bool examine_opclause_args ( List args,
Node **  exprp,
Const **  cstp,
bool expronleftp 
)

Definition at line 2021 of file extended_stats.c.

References arg, Assert, IsA, linitial, list_length(), and lsecond.

Referenced by mcv_get_match_bitmap(), and statext_is_compatible_clause_internal().

2023 {
2024  Node *expr;
2025  Const *cst;
2026  bool expronleft;
2027  Node *leftop,
2028  *rightop;
2029 
2030  /* enforced by statext_is_compatible_clause_internal */
2031  Assert(list_length(args) == 2);
2032 
2033  leftop = linitial(args);
2034  rightop = lsecond(args);
2035 
2036  /* strip RelabelType from either side of the expression */
2037  if (IsA(leftop, RelabelType))
2038  leftop = (Node *) ((RelabelType *) leftop)->arg;
2039 
2040  if (IsA(rightop, RelabelType))
2041  rightop = (Node *) ((RelabelType *) rightop)->arg;
2042 
2043  if (IsA(rightop, Const))
2044  {
2045  expr = (Node *) leftop;
2046  cst = (Const *) rightop;
2047  expronleft = true;
2048  }
2049  else if (IsA(leftop, Const))
2050  {
2051  expr = (Node *) rightop;
2052  cst = (Const *) leftop;
2053  expronleft = false;
2054  }
2055  else
2056  return false;
2057 
2058  /* return pointers to the extracted parts if requested */
2059  if (exprp)
2060  *exprp = expr;
2061 
2062  if (cstp)
2063  *cstp = cst;
2064 
2065  if (expronleftp)
2066  *expronleftp = expronleft;
2067 
2068  return true;
2069 }
#define IsA(nodeptr, _type_)
Definition: nodes.h:587
Definition: nodes.h:536
#define lsecond(l)
Definition: pg_list.h:179
#define linitial(l)
Definition: pg_list.h:174
#define Assert(condition)
Definition: c.h:804
static int list_length(const List *l)
Definition: pg_list.h:149
void * arg

◆ mcv_clause_selectivity_or()

Selectivity mcv_clause_selectivity_or ( PlannerInfo root,
StatisticExtInfo stat,
MCVList mcv,
Node clause,
bool **  or_matches,
Selectivity basesel,
Selectivity overlap_mcvsel,
Selectivity overlap_basesel,
Selectivity totalsel 
)

Definition at line 2110 of file mcv.c.

References MCVItem::base_frequency, StatisticExtInfo::exprs, MCVItem::frequency, i, MCVList::items, StatisticExtInfo::keys, list_make1, mcv_get_match_bitmap(), MCVList::nitems, palloc0(), and pfree().

Referenced by statext_mcv_clauselist_selectivity().

2114 {
2115  Selectivity s = 0.0;
2116  bool *new_matches;
2117  int i;
2118 
2119  /* build the OR-matches bitmap, if not built already */
2120  if (*or_matches == NULL)
2121  *or_matches = palloc0(sizeof(bool) * mcv->nitems);
2122 
2123  /* build the match bitmap for the new clause */
2124  new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys,
2125  stat->exprs, mcv, false);
2126 
2127  /*
2128  * Sum the frequencies for all the MCV items matching this clause and also
2129  * those matching the overlap between this clause and any of the preceding
2130  * clauses as described above.
2131  */
2132  *basesel = 0.0;
2133  *overlap_mcvsel = 0.0;
2134  *overlap_basesel = 0.0;
2135  *totalsel = 0.0;
2136  for (i = 0; i < mcv->nitems; i++)
2137  {
2138  *totalsel += mcv->items[i].frequency;
2139 
2140  if (new_matches[i])
2141  {
2142  s += mcv->items[i].frequency;
2143  *basesel += mcv->items[i].base_frequency;
2144 
2145  if ((*or_matches)[i])
2146  {
2147  *overlap_mcvsel += mcv->items[i].frequency;
2148  *overlap_basesel += mcv->items[i].base_frequency;
2149  }
2150  }
2151 
2152  /* update the OR-matches bitmap for the next clause */
2153  (*or_matches)[i] = (*or_matches)[i] || new_matches[i];
2154  }
2155 
2156  pfree(new_matches);
2157 
2158  return s;
2159 }
uint32 nitems
Definition: statistics.h:91
static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Bitmapset *keys, List *exprs, MCVList *mcvlist, bool is_or)
Definition: mcv.c:1606
double Selectivity
Definition: nodes.h:669
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
#define list_make1(x1)
Definition: pg_list.h:206
void pfree(void *pointer)
Definition: mcxt.c:1169
void * palloc0(Size size)
Definition: mcxt.c:1093
double base_frequency
Definition: statistics.h:81
Bitmapset * keys
Definition: pathnodes.h:939
int i
double frequency
Definition: statistics.h:80

◆ mcv_clauselist_selectivity()

Selectivity mcv_clauselist_selectivity ( PlannerInfo root,
StatisticExtInfo stat,
List clauses,
int  varRelid,
JoinType  jointype,
SpecialJoinInfo sjinfo,
RelOptInfo rel,
Selectivity basesel,
Selectivity totalsel 
)

Definition at line 2033 of file mcv.c.

References MCVItem::base_frequency, StatisticExtInfo::exprs, MCVItem::frequency, i, MCVList::items, StatisticExtInfo::keys, mcv_get_match_bitmap(), MCVList::nitems, statext_mcv_load(), and StatisticExtInfo::statOid.

Referenced by statext_mcv_clauselist_selectivity().

2038 {
2039  int i;
2040  MCVList *mcv;
2041  Selectivity s = 0.0;
2042 
2043  /* match/mismatch bitmap for each MCV item */
2044  bool *matches = NULL;
2045 
2046  /* load the MCV list stored in the statistics object */
2047  mcv = statext_mcv_load(stat->statOid);
2048 
2049  /* build a match bitmap for the clauses */
2050  matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs,
2051  mcv, false);
2052 
2053  /* sum frequencies for all the matching MCV items */
2054  *basesel = 0.0;
2055  *totalsel = 0.0;
2056  for (i = 0; i < mcv->nitems; i++)
2057  {
2058  *totalsel += mcv->items[i].frequency;
2059 
2060  if (matches[i] != false)
2061  {
2062  *basesel += mcv->items[i].base_frequency;
2063  s += mcv->items[i].frequency;
2064  }
2065  }
2066 
2067  return s;
2068 }
uint32 nitems
Definition: statistics.h:91
static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Bitmapset *keys, List *exprs, MCVList *mcvlist, bool is_or)
Definition: mcv.c:1606
double Selectivity
Definition: nodes.h:669
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
MCVList * statext_mcv_load(Oid mvoid)
Definition: mcv.c:562
double base_frequency
Definition: statistics.h:81
Bitmapset * keys
Definition: pathnodes.h:939
int i
double frequency
Definition: statistics.h:80

◆ mcv_combine_selectivities()

Selectivity mcv_combine_selectivities ( Selectivity  simple_sel,
Selectivity  mcv_sel,
Selectivity  mcv_basesel,
Selectivity  mcv_totalsel 
)

Definition at line 1991 of file mcv.c.

References CLAMP_PROBABILITY.

Referenced by statext_mcv_clauselist_selectivity().

1995 {
1996  Selectivity other_sel;
1997  Selectivity sel;
1998 
1999  /* estimated selectivity of values not covered by MCV matches */
2000  other_sel = simple_sel - mcv_basesel;
2001  CLAMP_PROBABILITY(other_sel);
2002 
2003  /* this non-MCV selectivity cannot exceed 1 - mcv_totalsel */
2004  if (other_sel > 1.0 - mcv_totalsel)
2005  other_sel = 1.0 - mcv_totalsel;
2006 
2007  /* overall selectivity is the sum of the MCV and non-MCV parts */
2008  sel = mcv_sel + other_sel;
2009  CLAMP_PROBABILITY(sel);
2010 
2011  return sel;
2012 }
double Selectivity
Definition: nodes.h:669
#define CLAMP_PROBABILITY(p)
Definition: selfuncs.h:63

◆ multi_sort_add_dimension()

void multi_sort_add_dimension ( MultiSortSupport  mss,
int  sortdim,
Oid  oper,
Oid  collation 
)

Definition at line 875 of file extended_stats.c.

References CurrentMemoryContext, PrepareSortSupportFromOrderingOp(), MultiSortSupportData::ssup, SortSupportData::ssup_collation, SortSupportData::ssup_cxt, and SortSupportData::ssup_nulls_first.

Referenced by build_mss(), dependency_degree(), and ndistinct_for_combination().

877 {
878  SortSupport ssup = &mss->ssup[sortdim];
879 
881  ssup->ssup_collation = collation;
882  ssup->ssup_nulls_first = false;
883 
885 }
bool ssup_nulls_first
Definition: sortsupport.h:75
void PrepareSortSupportFromOrderingOp(Oid orderingOp, SortSupport ssup)
Definition: sortsupport.c:135
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
MemoryContext ssup_cxt
Definition: sortsupport.h:66
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
Operator oper(ParseState *pstate, List *opname, Oid ltypeId, Oid rtypeId, bool noError, int location)
Definition: parse_oper.c:382

◆ multi_sort_compare()

int multi_sort_compare ( const void *  a,
const void *  b,
void *  arg 
)

Definition at line 889 of file extended_stats.c.

References ApplySortComparator(), compare(), i, SortItem::isnull, MultiSortSupportData::ndims, MultiSortSupportData::ssup, and SortItem::values.

Referenced by build_distinct_groups(), build_sorted_items(), count_distinct_groups(), ndistinct_for_combination(), and statext_mcv_build().

890 {
892  SortItem *ia = (SortItem *) a;
893  SortItem *ib = (SortItem *) b;
894  int i;
895 
896  for (i = 0; i < mss->ndims; i++)
897  {
898  int compare;
899 
900  compare = ApplySortComparator(ia->values[i], ia->isnull[i],
901  ib->values[i], ib->isnull[i],
902  &mss->ssup[i]);
903 
904  if (compare != 0)
905  return compare;
906  }
907 
908  /* equal by default */
909  return 0;
910 }
static int compare(const void *arg1, const void *arg2)
Definition: geqo_pool.c:145
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
int i
void * arg
MultiSortSupportData * MultiSortSupport
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_compare_dim()

int multi_sort_compare_dim ( int  dim,
const SortItem a,
const SortItem b,
MultiSortSupport  mss 
)

Definition at line 914 of file extended_stats.c.

References ApplySortComparator(), SortItem::isnull, MultiSortSupportData::ssup, and SortItem::values.

Referenced by dependency_degree().

916 {
917  return ApplySortComparator(a->values[dim], a->isnull[dim],
918  b->values[dim], b->isnull[dim],
919  &mss->ssup[dim]);
920 }
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_compare_dims()

int multi_sort_compare_dims ( int  start,
int  end,
const SortItem a,
const SortItem b,
MultiSortSupport  mss 
)

Definition at line 923 of file extended_stats.c.

References ApplySortComparator(), SortItem::isnull, MultiSortSupportData::ssup, and SortItem::values.

Referenced by dependency_degree().

926 {
927  int dim;
928 
929  for (dim = start; dim <= end; dim++)
930  {
931  int r = ApplySortComparator(a->values[dim], a->isnull[dim],
932  b->values[dim], b->isnull[dim],
933  &mss->ssup[dim]);
934 
935  if (r != 0)
936  return r;
937  }
938 
939  return 0;
940 }
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
static int ApplySortComparator(Datum datum1, bool isNull1, Datum datum2, bool isNull2, SortSupport ssup)
Definition: sortsupport.h:200

◆ multi_sort_init()

MultiSortSupport multi_sort_init ( int  ndims)

Definition at line 856 of file extended_stats.c.

References Assert, MultiSortSupportData::ndims, offsetof, and palloc0().

Referenced by build_mss(), dependency_degree(), and ndistinct_for_combination().

857 {
858  MultiSortSupport mss;
859 
860  Assert(ndims >= 2);
861 
863  + sizeof(SortSupportData) * ndims);
864 
865  mss->ndims = ndims;
866 
867  return mss;
868 }
struct SortSupportData SortSupportData
void * palloc0(Size size)
Definition: mcxt.c:1093
#define Assert(condition)
Definition: c.h:804
MultiSortSupportData * MultiSortSupport
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_build()

MVDependencies* statext_dependencies_build ( StatsBuildData data)

Definition at line 355 of file dependencies.c.

References Assert, StatsBuildData::attnums, MVDependency::attributes, MVDependency::degree, DependencyGeneratorData::dependencies, dependency_degree(), DependencyGenerator_free(), DependencyGenerator_init(), DependencyGenerator_next(), MVDependencies::deps, i, DependencyGeneratorData::k, MVDependencies::magic, StatsBuildData::nattnums, MVDependency::nattributes, MVDependencies::ndeps, offsetof, palloc0(), repalloc(), STATS_DEPS_MAGIC, STATS_DEPS_TYPE_BASIC, and MVDependencies::type.

Referenced by BuildRelationExtStatistics().

356 {
357  int i,
358  k;
359 
360  /* result */
361  MVDependencies *dependencies = NULL;
362 
363  Assert(data->nattnums >= 2);
364 
365  /*
366  * We'll try build functional dependencies starting from the smallest ones
367  * covering just 2 columns, to the largest ones, covering all columns
368  * included in the statistics object. We start from the smallest ones
369  * because we want to be able to skip already implied ones.
370  */
371  for (k = 2; k <= data->nattnums; k++)
372  {
373  AttrNumber *dependency; /* array with k elements */
374 
375  /* prepare a DependencyGenerator of variation */
377 
378  /* generate all possible variations of k values (out of n) */
379  while ((dependency = DependencyGenerator_next(DependencyGenerator)))
380  {
381  double degree;
382  MVDependency *d;
383 
384  /* compute how valid the dependency seems */
385  degree = dependency_degree(data, k, dependency);
386 
387  /*
388  * if the dependency seems entirely invalid, don't store it
389  */
390  if (degree == 0.0)
391  continue;
392 
393  d = (MVDependency *) palloc0(offsetof(MVDependency, attributes)
394  + k * sizeof(AttrNumber));
395 
396  /* copy the dependency (and keep the indexes into stxkeys) */
397  d->degree = degree;
398  d->nattributes = k;
399  for (i = 0; i < k; i++)
400  d->attributes[i] = data->attnums[dependency[i]];
401 
402  /* initialize the list of dependencies */
403  if (dependencies == NULL)
404  {
405  dependencies
406  = (MVDependencies *) palloc0(sizeof(MVDependencies));
407 
408  dependencies->magic = STATS_DEPS_MAGIC;
409  dependencies->type = STATS_DEPS_TYPE_BASIC;
410  dependencies->ndeps = 0;
411  }
412 
413  dependencies->ndeps++;
414  dependencies = (MVDependencies *) repalloc(dependencies,
415  offsetof(MVDependencies, deps)
416  + dependencies->ndeps * sizeof(MVDependency *));
417 
418  dependencies->deps[dependencies->ndeps - 1] = d;
419  }
420 
421  /*
422  * we're done with variations of k elements, so free the
423  * DependencyGenerator
424  */
425  DependencyGenerator_free(DependencyGenerator);
426  }
427 
428  return dependencies;
429 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define STATS_DEPS_MAGIC
Definition: statistics.h:43
static double dependency_degree(StatsBuildData *data, int k, AttrNumber *dependency)
Definition: dependencies.c:222
#define STATS_DEPS_TYPE_BASIC
Definition: statistics.h:44
AttrNumber nattributes
Definition: statistics.h:53
static DependencyGenerator DependencyGenerator_init(int n, int k)
Definition: dependencies.c:173
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
static AttrNumber * DependencyGenerator_next(DependencyGenerator state)
Definition: dependencies.c:205
int i
static void DependencyGenerator_free(DependencyGenerator state)
Definition: dependencies.c:196
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_deserialize()

MVDependencies* statext_dependencies_deserialize ( bytea data)

Definition at line 491 of file dependencies.c.

References Assert, MVDependency::attributes, MVDependency::degree, DependencyGeneratorData::dependencies, MVDependencies::deps, elog, ERROR, i, DependencyGeneratorData::k, MVDependencies::magic, MVDependency::nattributes, MVDependencies::ndeps, offsetof, palloc0(), repalloc(), SizeOfHeader, SizeOfItem, STATS_DEPS_MAGIC, STATS_DEPS_TYPE_BASIC, STATS_MAX_DIMENSIONS, MVDependencies::type, VARDATA_ANY, VARSIZE_ANY, and VARSIZE_ANY_EXHDR.

Referenced by pg_dependencies_out(), and statext_dependencies_load().

492 {
493  int i;
494  Size min_expected_size;
495  MVDependencies *dependencies;
496  char *tmp;
497 
498  if (data == NULL)
499  return NULL;
500 
501  if (VARSIZE_ANY_EXHDR(data) < SizeOfHeader)
502  elog(ERROR, "invalid MVDependencies size %zd (expected at least %zd)",
504 
505  /* read the MVDependencies header */
506  dependencies = (MVDependencies *) palloc0(sizeof(MVDependencies));
507 
508  /* initialize pointer to the data part (skip the varlena header) */
509  tmp = VARDATA_ANY(data);
510 
511  /* read the header fields and perform basic sanity checks */
512  memcpy(&dependencies->magic, tmp, sizeof(uint32));
513  tmp += sizeof(uint32);
514  memcpy(&dependencies->type, tmp, sizeof(uint32));
515  tmp += sizeof(uint32);
516  memcpy(&dependencies->ndeps, tmp, sizeof(uint32));
517  tmp += sizeof(uint32);
518 
519  if (dependencies->magic != STATS_DEPS_MAGIC)
520  elog(ERROR, "invalid dependency magic %d (expected %d)",
521  dependencies->magic, STATS_DEPS_MAGIC);
522 
523  if (dependencies->type != STATS_DEPS_TYPE_BASIC)
524  elog(ERROR, "invalid dependency type %d (expected %d)",
525  dependencies->type, STATS_DEPS_TYPE_BASIC);
526 
527  if (dependencies->ndeps == 0)
528  elog(ERROR, "invalid zero-length item array in MVDependencies");
529 
530  /* what minimum bytea size do we expect for those parameters */
531  min_expected_size = SizeOfItem(dependencies->ndeps);
532 
533  if (VARSIZE_ANY_EXHDR(data) < min_expected_size)
534  elog(ERROR, "invalid dependencies size %zd (expected at least %zd)",
535  VARSIZE_ANY_EXHDR(data), min_expected_size);
536 
537  /* allocate space for the MCV items */
538  dependencies = repalloc(dependencies, offsetof(MVDependencies, deps)
539  + (dependencies->ndeps * sizeof(MVDependency *)));
540 
541  for (i = 0; i < dependencies->ndeps; i++)
542  {
543  double degree;
544  AttrNumber k;
545  MVDependency *d;
546 
547  /* degree of validity */
548  memcpy(&degree, tmp, sizeof(double));
549  tmp += sizeof(double);
550 
551  /* number of attributes */
552  memcpy(&k, tmp, sizeof(AttrNumber));
553  tmp += sizeof(AttrNumber);
554 
555  /* is the number of attributes valid? */
556  Assert((k >= 2) && (k <= STATS_MAX_DIMENSIONS));
557 
558  /* now that we know the number of attributes, allocate the dependency */
559  d = (MVDependency *) palloc0(offsetof(MVDependency, attributes)
560  + (k * sizeof(AttrNumber)));
561 
562  d->degree = degree;
563  d->nattributes = k;
564 
565  /* copy attribute numbers */
566  memcpy(d->attributes, tmp, sizeof(AttrNumber) * d->nattributes);
567  tmp += sizeof(AttrNumber) * d->nattributes;
568 
569  dependencies->deps[i] = d;
570 
571  /* still within the bytea */
572  Assert(tmp <= ((char *) data + VARSIZE_ANY(data)));
573  }
574 
575  /* we should have consumed the whole bytea exactly */
576  Assert(tmp == ((char *) data + VARSIZE_ANY(data)));
577 
578  return dependencies;
579 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define STATS_DEPS_MAGIC
Definition: statistics.h:43
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define STATS_DEPS_TYPE_BASIC
Definition: statistics.h:44
#define SizeOfItem(natts)
Definition: dependencies.c:41
AttrNumber nattributes
Definition: statistics.h:53
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
size_t Size
Definition: c.h:540
#define SizeOfHeader
Definition: dependencies.c:38
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
#define elog(elevel,...)
Definition: elog.h:232
int i
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_dependencies_serialize()

bytea* statext_dependencies_serialize ( MVDependencies dependencies)

Definition at line 436 of file dependencies.c.

References Assert, MVDependency::attributes, MVDependency::degree, MVDependencies::deps, i, MVDependencies::magic, MVDependency::nattributes, MVDependencies::ndeps, output(), palloc0(), SET_VARSIZE, SizeOfHeader, SizeOfItem, MVDependencies::type, VARDATA, and VARHDRSZ.

Referenced by statext_store().

437 {
438  int i;
439  bytea *output;
440  char *tmp;
441  Size len;
442 
443  /* we need to store ndeps, with a number of attributes for each one */
444  len = VARHDRSZ + SizeOfHeader;
445 
446  /* and also include space for the actual attribute numbers and degrees */
447  for (i = 0; i < dependencies->ndeps; i++)
448  len += SizeOfItem(dependencies->deps[i]->nattributes);
449 
450  output = (bytea *) palloc0(len);
451  SET_VARSIZE(output, len);
452 
453  tmp = VARDATA(output);
454 
455  /* Store the base struct values (magic, type, ndeps) */
456  memcpy(tmp, &dependencies->magic, sizeof(uint32));
457  tmp += sizeof(uint32);
458  memcpy(tmp, &dependencies->type, sizeof(uint32));
459  tmp += sizeof(uint32);
460  memcpy(tmp, &dependencies->ndeps, sizeof(uint32));
461  tmp += sizeof(uint32);
462 
463  /* store number of attributes and attribute numbers for each dependency */
464  for (i = 0; i < dependencies->ndeps; i++)
465  {
466  MVDependency *d = dependencies->deps[i];
467 
468  memcpy(tmp, &d->degree, sizeof(double));
469  tmp += sizeof(double);
470 
471  memcpy(tmp, &d->nattributes, sizeof(AttrNumber));
472  tmp += sizeof(AttrNumber);
473 
474  memcpy(tmp, d->attributes, sizeof(AttrNumber) * d->nattributes);
475  tmp += sizeof(AttrNumber) * d->nattributes;
476 
477  /* protect against overflow */
478  Assert(tmp <= ((char *) output + len));
479  }
480 
481  /* make sure we've produced exactly the right amount of data */
482  Assert(tmp == ((char *) output + len));
483 
484  return output;
485 }
AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:54
MVDependency * deps[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:62
#define VARDATA(PTR)
Definition: postgres.h:315
static void output(uint64 loop_count)
#define VARHDRSZ
Definition: c.h:627
#define SizeOfItem(natts)
Definition: dependencies.c:41
AttrNumber nattributes
Definition: statistics.h:53
unsigned int uint32
Definition: c.h:441
void * palloc0(Size size)
Definition: mcxt.c:1093
uint32 magic
Definition: statistics.h:59
uint32 ndeps
Definition: statistics.h:61
#define Assert(condition)
Definition: c.h:804
double degree
Definition: statistics.h:52
size_t Size
Definition: c.h:540
#define SizeOfHeader
Definition: dependencies.c:38
int i
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21

◆ statext_mcv_build()

MCVList* statext_mcv_build ( StatsBuildData data,
double  totalrows,
int  stattarget 
)

Definition at line 184 of file mcv.c.

References Assert, StatsBuildData::attnums, VacAttrStats::attrtypid, MCVItem::base_frequency, bsearch_arg(), build_column_frequencies(), build_distinct_groups(), build_mss(), build_sorted_items(), SortItem::count, MCVItem::frequency, get_mincount_for_mcv_list(), i, SortItem::isnull, MCVItem::isnull, MCVList::items, sort-test::key, MCVList::magic, multi_sort_compare(), StatsBuildData::nattnums, MCVList::ndimensions, MultiSortSupportData::ndims, MCVList::nitems, StatsBuildData::numrows, offsetof, palloc(), palloc0(), pfree(), MultiSortSupportData::ssup, StatsBuildData::stats, STATS_MCV_MAGIC, STATS_MCV_TYPE_BASIC, MCVList::type, MCVList::types, SortItem::values, and MCVItem::values.

Referenced by BuildRelationExtStatistics().

185 {
186  int i,
187  numattrs,
188  numrows,
189  ngroups,
190  nitems;
191  double mincount;
192  SortItem *items;
193  SortItem *groups;
194  MCVList *mcvlist = NULL;
195  MultiSortSupport mss;
196 
197  /* comparator for all the columns */
198  mss = build_mss(data);
199 
200  /* sort the rows */
201  items = build_sorted_items(data, &nitems, mss,
202  data->nattnums, data->attnums);
203 
204  if (!items)
205  return NULL;
206 
207  /* for convenience */
208  numattrs = data->nattnums;
209  numrows = data->numrows;
210 
211  /* transform the sorted rows into groups (sorted by frequency) */
212  groups = build_distinct_groups(nitems, items, mss, &ngroups);
213 
214  /*
215  * The maximum number of MCV items to store, based on the statistics
216  * target we computed for the statistics object (from the target set for
217  * the object itself, attributes and the system default). In any case, we
218  * can't keep more groups than we have available.
219  */
220  nitems = stattarget;
221  if (nitems > ngroups)
222  nitems = ngroups;
223 
224  /*
225  * Decide how many items to keep in the MCV list. We can't use the same
226  * algorithm as per-column MCV lists, because that only considers the
227  * actual group frequency - but we're primarily interested in how the
228  * actual frequency differs from the base frequency (product of simple
229  * per-column frequencies, as if the columns were independent).
230  *
231  * Using the same algorithm might exclude items that are close to the
232  * "average" frequency of the sample. But that does not say whether the
233  * observed frequency is close to the base frequency or not. We also need
234  * to consider unexpectedly uncommon items (again, compared to the base
235  * frequency), and the single-column algorithm does not have to.
236  *
237  * We simply decide how many items to keep by computing the minimum count
238  * using get_mincount_for_mcv_list() and then keep all items that seem to
239  * be more common than that.
240  */
241  mincount = get_mincount_for_mcv_list(numrows, totalrows);
242 
243  /*
244  * Walk the groups until we find the first group with a count below the
245  * mincount threshold (the index of that group is the number of groups we
246  * want to keep).
247  */
248  for (i = 0; i < nitems; i++)
249  {
250  if (groups[i].count < mincount)
251  {
252  nitems = i;
253  break;
254  }
255  }
256 
257  /*
258  * At this point, we know the number of items for the MCV list. There
259  * might be none (for uniform distribution with many groups), and in that
260  * case, there will be no MCV list. Otherwise, construct the MCV list.
261  */
262  if (nitems > 0)
263  {
264  int j;
265  SortItem key;
266  MultiSortSupport tmp;
267 
268  /* frequencies for values in each attribute */
269  SortItem **freqs;
270  int *nfreqs;
271 
272  /* used to search values */
274  + sizeof(SortSupportData));
275 
276  /* compute frequencies for values in each column */
277  nfreqs = (int *) palloc0(sizeof(int) * numattrs);
278  freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
279 
280  /*
281  * Allocate the MCV list structure, set the global parameters.
282  */
283  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) +
284  sizeof(MCVItem) * nitems);
285 
286  mcvlist->magic = STATS_MCV_MAGIC;
287  mcvlist->type = STATS_MCV_TYPE_BASIC;
288  mcvlist->ndimensions = numattrs;
289  mcvlist->nitems = nitems;
290 
291  /* store info about data type OIDs */
292  for (i = 0; i < numattrs; i++)
293  mcvlist->types[i] = data->stats[i]->attrtypid;
294 
295  /* Copy the first chunk of groups into the result. */
296  for (i = 0; i < nitems; i++)
297  {
298  /* just pointer to the proper place in the list */
299  MCVItem *item = &mcvlist->items[i];
300 
301  item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
302  item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
303 
304  /* copy values for the group */
305  memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
306  memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
307 
308  /* groups should be sorted by frequency in descending order */
309  Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
310 
311  /* group frequency */
312  item->frequency = (double) groups[i].count / numrows;
313 
314  /* base frequency, if the attributes were independent */
315  item->base_frequency = 1.0;
316  for (j = 0; j < numattrs; j++)
317  {
318  SortItem *freq;
319 
320  /* single dimension */
321  tmp->ndims = 1;
322  tmp->ssup[0] = mss->ssup[j];
323 
324  /* fill search key */
325  key.values = &groups[i].values[j];
326  key.isnull = &groups[i].isnull[j];
327 
328  freq = (SortItem *) bsearch_arg(&key, freqs[j], nfreqs[j],
329  sizeof(SortItem),
330  multi_sort_compare, tmp);
331 
332  item->base_frequency *= ((double) freq->count) / numrows;
333  }
334  }
335 
336  pfree(nfreqs);
337  pfree(freqs);
338  }
339 
340  pfree(items);
341  pfree(groups);
342 
343  return mcvlist;
344 }
static SortItem * build_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss, int *ndistinct)
Definition: mcv.c:428
uint32 nitems
Definition: statistics.h:91
static SortItem ** build_column_frequencies(SortItem *groups, int ngroups, MultiSortSupport mss, int *ncounts)
Definition: mcv.c:494
Oid types[STATS_MAX_DIMENSIONS]
Definition: statistics.h:93
uint32 magic
Definition: statistics.h:89
AttrNumber ndimensions
Definition: statistics.h:92
struct MCVItem MCVItem
Datum * values
Definition: statistics.h:83
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
static double get_mincount_for_mcv_list(int samplerows, double totalrows)
Definition: mcv.c:152
void pfree(void *pointer)
Definition: mcxt.c:1169
Oid attrtypid
Definition: vacuum.h:124
SortSupportData ssup[FLEXIBLE_ARRAY_MEMBER]
SortItem * build_sorted_items(StatsBuildData *data, int *nitems, MultiSortSupport mss, int numattrs, AttrNumber *attnums)
uint32 type
Definition: statistics.h:90
#define STATS_MCV_TYPE_BASIC
Definition: statistics.h:67
struct SortSupportData SortSupportData
static MultiSortSupport build_mss(StatsBuildData *data)
Definition: mcv.c:351
void * bsearch_arg(const void *key, const void *base, size_t nmemb, size_t size, int(*compar)(const void *, const void *, void *), void *arg)
Definition: bsearch_arg.c:55
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
VacAttrStats ** stats
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
#define STATS_MCV_MAGIC
Definition: statistics.h:66
int multi_sort_compare(const void *a, const void *b, void *arg)
bool * isnull
Definition: statistics.h:82
void * palloc(Size size)
Definition: mcxt.c:1062
int i
MultiSortSupportData * MultiSortSupport
#define offsetof(type, field)
Definition: c.h:727
double frequency
Definition: statistics.h:80

◆ statext_mcv_deserialize()

MCVList* statext_mcv_deserialize ( bytea data)

Definition at line 999 of file mcv.c.

References Assert, MCVItem::base_frequency, elog, ERROR, fetch_att, MCVItem::frequency, i, MCVItem::isnull, MCVList::items, MCVList::magic, MAXALIGN, MinSizeOfMCVList, DimensionInfo::nbytes, DimensionInfo::nbytes_aligned, MCVList::ndimensions, MCVList::nitems, DimensionInfo::nvalues, offsetof, palloc(), palloc0(), pfree(), PG_USED_FOR_ASSERTS_ONLY, PointerGetDatum, repalloc(), SET_VARSIZE, SizeOfMCVList, STATS_MAX_DIMENSIONS, STATS_MCV_MAGIC, STATS_MCV_TYPE_BASIC, STATS_MCVLIST_MAX_ITEMS, MCVList::type, MCVList::types, DimensionInfo::typlen, MCVItem::values, VARDATA, VARDATA_ANY, VARHDRSZ, and VARSIZE_ANY.

Referenced by pg_stats_ext_mcvlist_items(), and statext_mcv_load().

1000 {
1001  int dim,
1002  i;
1003  Size expected_size;
1004  MCVList *mcvlist;
1005  char *raw;
1006  char *ptr;
1007  char *endptr PG_USED_FOR_ASSERTS_ONLY;
1008 
1009  int ndims,
1010  nitems;
1011  DimensionInfo *info = NULL;
1012 
1013  /* local allocation buffer (used only for deserialization) */
1014  Datum **map = NULL;
1015 
1016  /* MCV list */
1017  Size mcvlen;
1018 
1019  /* buffer used for the result */
1020  Size datalen;
1021  char *dataptr;
1022  char *valuesptr;
1023  char *isnullptr;
1024 
1025  if (data == NULL)
1026  return NULL;
1027 
1028  /*
1029  * We can't possibly deserialize a MCV list if there's not even a complete
1030  * header. We need an explicit formula here, because we serialize the
1031  * header fields one by one, so we need to ignore struct alignment.
1032  */
1033  if (VARSIZE_ANY(data) < MinSizeOfMCVList)
1034  elog(ERROR, "invalid MCV size %zd (expected at least %zu)",
1035  VARSIZE_ANY(data), MinSizeOfMCVList);
1036 
1037  /* read the MCV list header */
1038  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items));
1039 
1040  /* pointer to the data part (skip the varlena header) */
1041  raw = (char *) data;
1042  ptr = VARDATA_ANY(raw);
1043  endptr = (char *) raw + VARSIZE_ANY(data);
1044 
1045  /* get the header and perform further sanity checks */
1046  memcpy(&mcvlist->magic, ptr, sizeof(uint32));
1047  ptr += sizeof(uint32);
1048 
1049  memcpy(&mcvlist->type, ptr, sizeof(uint32));
1050  ptr += sizeof(uint32);
1051 
1052  memcpy(&mcvlist->nitems, ptr, sizeof(uint32));
1053  ptr += sizeof(uint32);
1054 
1055  memcpy(&mcvlist->ndimensions, ptr, sizeof(AttrNumber));
1056  ptr += sizeof(AttrNumber);
1057 
1058  if (mcvlist->magic != STATS_MCV_MAGIC)
1059  elog(ERROR, "invalid MCV magic %u (expected %u)",
1060  mcvlist->magic, STATS_MCV_MAGIC);
1061 
1062  if (mcvlist->type != STATS_MCV_TYPE_BASIC)
1063  elog(ERROR, "invalid MCV type %u (expected %u)",
1064  mcvlist->type, STATS_MCV_TYPE_BASIC);
1065 
1066  if (mcvlist->ndimensions == 0)
1067  elog(ERROR, "invalid zero-length dimension array in MCVList");
1068  else if ((mcvlist->ndimensions > STATS_MAX_DIMENSIONS) ||
1069  (mcvlist->ndimensions < 0))
1070  elog(ERROR, "invalid length (%d) dimension array in MCVList",
1071  mcvlist->ndimensions);
1072 
1073  if (mcvlist->nitems == 0)
1074  elog(ERROR, "invalid zero-length item array in MCVList");
1075  else if (mcvlist->nitems > STATS_MCVLIST_MAX_ITEMS)
1076  elog(ERROR, "invalid length (%u) item array in MCVList",
1077  mcvlist->nitems);
1078 
1079  nitems = mcvlist->nitems;
1080  ndims = mcvlist->ndimensions;
1081 
1082  /*
1083  * Check amount of data including DimensionInfo for all dimensions and
1084  * also the serialized items (including uint16 indexes). Also, walk
1085  * through the dimension information and add it to the sum.
1086  */
1087  expected_size = SizeOfMCVList(ndims, nitems);
1088 
1089  /*
1090  * Check that we have at least the dimension and info records, along with
1091  * the items. We don't know the size of the serialized values yet. We need
1092  * to do this check first, before accessing the dimension info.
1093  */
1094  if (VARSIZE_ANY(data) < expected_size)
1095  elog(ERROR, "invalid MCV size %zd (expected %zu)",
1096  VARSIZE_ANY(data), expected_size);
1097 
1098  /* Now copy the array of type Oids. */
1099  memcpy(mcvlist->types, ptr, sizeof(Oid) * ndims);
1100  ptr += (sizeof(Oid) * ndims);
1101 
1102  /* Now it's safe to access the dimension info. */
1103  info = palloc(ndims * sizeof(DimensionInfo));
1104 
1105  memcpy(info, ptr, ndims * sizeof(DimensionInfo));
1106  ptr += (ndims * sizeof(DimensionInfo));
1107 
1108  /* account for the value arrays */
1109  for (dim = 0; dim < ndims; dim++)
1110  {
1111  /*
1112  * XXX I wonder if we can/should rely on asserts here. Maybe those
1113  * checks should be done every time?
1114  */
1115  Assert(info[dim].nvalues >= 0);
1116  Assert(info[dim].nbytes >= 0);
1117 
1118  expected_size += info[dim].nbytes;
1119  }
1120 
1121  /*
1122  * Now we know the total expected MCV size, including all the pieces
1123  * (header, dimension info. items and deduplicated data). So do the final
1124  * check on size.
1125  */
1126  if (VARSIZE_ANY(data) != expected_size)
1127  elog(ERROR, "invalid MCV size %zd (expected %zu)",
1128  VARSIZE_ANY(data), expected_size);
1129 
1130  /*
1131  * We need an array of Datum values for each dimension, so that we can
1132  * easily translate the uint16 indexes later. We also need a top-level
1133  * array of pointers to those per-dimension arrays.
1134  *
1135  * While allocating the arrays for dimensions, compute how much space we
1136  * need for a copy of the by-ref data, as we can't simply point to the
1137  * original values (it might go away).
1138  */
1139  datalen = 0; /* space for by-ref data */
1140  map = (Datum **) palloc(ndims * sizeof(Datum *));
1141 
1142  for (dim = 0; dim < ndims; dim++)
1143  {
1144  map[dim] = (Datum *) palloc(sizeof(Datum) * info[dim].nvalues);
1145 
1146  /* space needed for a copy of data for by-ref types */
1147  datalen += info[dim].nbytes_aligned;
1148  }
1149 
1150  /*
1151  * Now resize the MCV list so that the allocation includes all the data.
1152  *
1153  * Allocate space for a copy of the data, as we can't simply reference the
1154  * serialized data - it's not aligned properly, and it may disappear while
1155  * we're still using the MCV list, e.g. due to catcache release.
1156  *
1157  * We do care about alignment here, because we will allocate all the
1158  * pieces at once, but then use pointers to different parts.
1159  */
1160  mcvlen = MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1161 
1162  /* arrays of values and isnull flags for all MCV items */
1163  mcvlen += nitems * MAXALIGN(sizeof(Datum) * ndims);
1164  mcvlen += nitems * MAXALIGN(sizeof(bool) * ndims);
1165 
1166  /* we don't quite need to align this, but it makes some asserts easier */
1167  mcvlen += MAXALIGN(datalen);
1168 
1169  /* now resize the deserialized MCV list, and compute pointers to parts */
1170  mcvlist = repalloc(mcvlist, mcvlen);
1171 
1172  /* pointer to the beginning of values/isnull arrays */
1173  valuesptr = (char *) mcvlist
1174  + MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1175 
1176  isnullptr = valuesptr + (nitems * MAXALIGN(sizeof(Datum) * ndims));
1177 
1178  dataptr = isnullptr + (nitems * MAXALIGN(sizeof(bool) * ndims));
1179 
1180  /*
1181  * Build mapping (index => value) for translating the serialized data into
1182  * the in-memory representation.
1183  */
1184  for (dim = 0; dim < ndims; dim++)
1185  {
1186  /* remember start position in the input array */
1187  char *start PG_USED_FOR_ASSERTS_ONLY = ptr;
1188 
1189  if (info[dim].typbyval)
1190  {
1191  /* for by-val types we simply copy data into the mapping */
1192  for (i = 0; i < info[dim].nvalues; i++)
1193  {
1194  Datum v = 0;
1195 
1196  memcpy(&v, ptr, info[dim].typlen);
1197  ptr += info[dim].typlen;
1198 
1199  map[dim][i] = fetch_att(&v, true, info[dim].typlen);
1200 
1201  /* no under/overflow of input array */
1202  Assert(ptr <= (start + info[dim].nbytes));
1203  }
1204  }
1205  else
1206  {
1207  /* for by-ref types we need to also make a copy of the data */
1208 
1209  /* passed by reference, but fixed length (name, tid, ...) */
1210  if (info[dim].typlen > 0)
1211  {
1212  for (i = 0; i < info[dim].nvalues; i++)
1213  {
1214  memcpy(dataptr, ptr, info[dim].typlen);
1215  ptr += info[dim].typlen;
1216 
1217  /* just point into the array */
1218  map[dim][i] = PointerGetDatum(dataptr);
1219  dataptr += MAXALIGN(info[dim].typlen);
1220  }
1221  }
1222  else if (info[dim].typlen == -1)
1223  {
1224  /* varlena */
1225  for (i = 0; i < info[dim].nvalues; i++)
1226  {
1227  uint32 len;
1228 
1229  /* read the uint32 length */
1230  memcpy(&len, ptr, sizeof(uint32));
1231  ptr += sizeof(uint32);
1232 
1233  /* the length is data-only */
1234  SET_VARSIZE(dataptr, len + VARHDRSZ);
1235  memcpy(VARDATA(dataptr), ptr, len);
1236  ptr += len;
1237 
1238  /* just point into the array */
1239  map[dim][i] = PointerGetDatum(dataptr);
1240 
1241  /* skip to place of the next deserialized value */
1242  dataptr += MAXALIGN(len + VARHDRSZ);
1243  }
1244  }
1245  else if (info[dim].typlen == -2)
1246  {
1247  /* cstring */
1248  for (i = 0; i < info[dim].nvalues; i++)
1249  {
1250  uint32 len;
1251 
1252  memcpy(&len, ptr, sizeof(uint32));
1253  ptr += sizeof(uint32);
1254 
1255  memcpy(dataptr, ptr, len);
1256  ptr += len;
1257 
1258  /* just point into the array */
1259  map[dim][i] = PointerGetDatum(dataptr);
1260  dataptr += MAXALIGN(len);
1261  }
1262  }
1263 
1264  /* no under/overflow of input array */
1265  Assert(ptr <= (start + info[dim].nbytes));
1266 
1267  /* no overflow of the output mcv value */
1268  Assert(dataptr <= ((char *) mcvlist + mcvlen));
1269  }
1270 
1271  /* check we consumed input data for this dimension exactly */
1272  Assert(ptr == (start + info[dim].nbytes));
1273  }
1274 
1275  /* we should have also filled the MCV list exactly */
1276  Assert(dataptr == ((char *) mcvlist + mcvlen));
1277 
1278  /* deserialize the MCV items and translate the indexes to Datums */
1279  for (i = 0; i < nitems; i++)
1280  {
1281  MCVItem *item = &mcvlist->items[i];
1282 
1283  item->values = (Datum *) valuesptr;
1284  valuesptr += MAXALIGN(sizeof(Datum) * ndims);
1285 
1286  item->isnull = (bool *) isnullptr;
1287  isnullptr += MAXALIGN(sizeof(bool) * ndims);
1288 
1289  memcpy(item->isnull, ptr, sizeof(bool) * ndims);
1290  ptr += sizeof(bool) * ndims;
1291 
1292  memcpy(&item->frequency, ptr, sizeof(double));
1293  ptr += sizeof(double);
1294 
1295  memcpy(&item->base_frequency, ptr, sizeof(double));
1296  ptr += sizeof(double);
1297 
1298  /* finally translate the indexes (for non-NULL only) */
1299  for (dim = 0; dim < ndims; dim++)
1300  {
1301  uint16 index;
1302 
1303  memcpy(&index, ptr, sizeof(uint16));
1304  ptr += sizeof(uint16);
1305 
1306  if (item->isnull[dim])
1307  continue;
1308 
1309  item->values[dim] = map[dim][index];
1310  }
1311 
1312  /* check we're not overflowing the input */
1313  Assert(ptr <= endptr);
1314  }
1315 
1316  /* check that we processed all the data */
1317  Assert(ptr == endptr);
1318 
1319  /* release the buffers used for mapping */
1320  for (dim = 0; dim < ndims; dim++)
1321  pfree(map[dim]);
1322 
1323  pfree(map);
1324 
1325  return mcvlist;
1326 }
uint32 nitems
Definition: statistics.h:91
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define VARDATA(PTR)
Definition: postgres.h:315
Oid types[STATS_MAX_DIMENSIONS]
Definition: statistics.h:93
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARHDRSZ
Definition: c.h:627
#define MinSizeOfMCVList
Definition: mcv.c:63
struct DimensionInfo DimensionInfo
uint32 magic
Definition: statistics.h:89
AttrNumber ndimensions
Definition: statistics.h:92
struct MCVItem MCVItem
Datum * values
Definition: statistics.h:83
MCVItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:94
unsigned int Oid
Definition: postgres_ext.h:31
Definition: type.h:89
unsigned short uint16
Definition: c.h:440
void pfree(void *pointer)
Definition: mcxt.c:1169
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
#define SizeOfMCVList(ndims, nitems)
Definition: mcv.c:72
uint32 type
Definition: statistics.h:90
#define STATS_MCV_TYPE_BASIC
Definition: statistics.h:67
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
#define STATS_MCVLIST_MAX_ITEMS
Definition: statistics.h:70
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
#define STATS_MCV_MAGIC
Definition: statistics.h:66
size_t Size
Definition: c.h:540
#define MAXALIGN(LEN)
Definition: c.h:757
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
bool * isnull
Definition: statistics.h:82
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
void * palloc(Size size)
Definition: mcxt.c:1062
#define fetch_att(T, attbyval, attlen)
Definition: tupmacs.h:75
#define elog(elevel,...)
Definition: elog.h:232
int i
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727
double frequency
Definition: statistics.h:80
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
unsigned char bool
Definition: c.h:391

◆ statext_mcv_serialize()

bytea* statext_mcv_serialize ( MCVList mcv,
VacAttrStats **  stats 
)

Definition at line 624 of file mcv.c.

References Assert, VacAttrStats::attrcollid, VacAttrStats::attrtype, MCVItem::base_frequency, bsearch_arg(), compare_datums_simple(), compare_scalars_simple(), CurrentMemoryContext, DatumGetCString, DatumGetPointer, MCVItem::frequency, i, MCVItem::isnull, ITEM_SIZE, MCVList::items, lookup_type_cache(), TypeCacheEntry::lt_opr, MCVList::magic, MAXALIGN, DimensionInfo::nbytes, DimensionInfo::nbytes_aligned, MCVList::ndimensions, MCVList::nitems, DimensionInfo::nvalues, palloc0(), pfree(), PG_DETOAST_DATUM, PG_UINT16_MAX, PG_USED_FOR_ASSERTS_ONLY, PointerGetDatum, PrepareSortSupportFromOrderingOp(), qsort_arg(), SET_VARSIZE, SortSupportData::ssup_collation, SortSupportData::ssup_cxt, SortSupportData::ssup_nulls_first, store_att_byval, DimensionInfo::typbyval, MCVList::type, TYPECACHE_LT_OPR, MCVList::types, DimensionInfo::typlen, value, MCVItem::values, values, VARDATA, VARDATA_ANY, VARHDRSZ, and VARSIZE_ANY_EXHDR.

Referenced by statext_store().

625 {
626  int i;
627  int dim;
628  int ndims = mcvlist->ndimensions;
629 
630  SortSupport ssup;
631  DimensionInfo *info;
632 
633  Size total_length;
634 
635  /* serialized items (indexes into arrays, etc.) */
636  bytea *raw;
637  char *ptr;
638  char *endptr PG_USED_FOR_ASSERTS_ONLY;
639 
640  /* values per dimension (and number of non-NULL values) */
641  Datum **values = (Datum **) palloc0(sizeof(Datum *) * ndims);
642  int *counts = (int *) palloc0(sizeof(int) * ndims);
643 
644  /*
645  * We'll include some rudimentary information about the attribute types
646  * (length, by-val flag), so that we don't have to look them up while
647  * deserializing the MCV list (we already have the type OID in the
648  * header). This is safe because when changing the type of the attribute
649  * the statistics gets dropped automatically. We need to store the info
650  * about the arrays of deduplicated values anyway.
651  */
652  info = (DimensionInfo *) palloc0(sizeof(DimensionInfo) * ndims);
653 
654  /* sort support data for all attributes included in the MCV list */
655  ssup = (SortSupport) palloc0(sizeof(SortSupportData) * ndims);
656 
657  /* collect and deduplicate values for each dimension (attribute) */
658  for (dim = 0; dim < ndims; dim++)
659  {
660  int ndistinct;
661  TypeCacheEntry *typentry;
662 
663  /*
664  * Lookup the LT operator (can't get it from stats extra_data, as we
665  * don't know how to interpret that - scalar vs. array etc.).
666  */
667  typentry = lookup_type_cache(stats[dim]->attrtypid, TYPECACHE_LT_OPR);
668 
669  /* copy important info about the data type (length, by-value) */
670  info[dim].typlen = stats[dim]->attrtype->typlen;
671  info[dim].typbyval = stats[dim]->attrtype->typbyval;
672 
673  /* allocate space for values in the attribute and collect them */
674  values[dim] = (Datum *) palloc0(sizeof(Datum) * mcvlist->nitems);
675 
676  for (i = 0; i < mcvlist->nitems; i++)
677  {
678  /* skip NULL values - we don't need to deduplicate those */
679  if (mcvlist->items[i].isnull[dim])
680  continue;
681 
682  /* append the value at the end */
683  values[dim][counts[dim]] = mcvlist->items[i].values[dim];
684  counts[dim] += 1;
685  }
686 
687  /* if there are just NULL values in this dimension, we're done */
688  if (counts[dim] == 0)
689  continue;
690 
691  /* sort and deduplicate the data */
692  ssup[dim].ssup_cxt = CurrentMemoryContext;
693  ssup[dim].ssup_collation = stats[dim]->attrcollid;
694  ssup[dim].ssup_nulls_first = false;
695 
696  PrepareSortSupportFromOrderingOp(typentry->lt_opr, &ssup[dim]);
697 
698  qsort_arg(values[dim], counts[dim], sizeof(Datum),
699  compare_scalars_simple, &ssup[dim]);
700 
701  /*
702  * Walk through the array and eliminate duplicate values, but keep the
703  * ordering (so that we can do a binary search later). We know there's
704  * at least one item as (counts[dim] != 0), so we can skip the first
705  * element.
706  */
707  ndistinct = 1; /* number of distinct values */
708  for (i = 1; i < counts[dim]; i++)
709  {
710  /* expect sorted array */
711  Assert(compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]) <= 0);
712 
713  /* if the value is the same as the previous one, we can skip it */
714  if (!compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]))
715  continue;
716 
717  values[dim][ndistinct] = values[dim][i];
718  ndistinct += 1;
719  }
720 
721  /* we must not exceed PG_UINT16_MAX, as we use uint16 indexes */
722  Assert(ndistinct <= PG_UINT16_MAX);
723 
724  /*
725  * Store additional info about the attribute - number of deduplicated
726  * values, and also size of the serialized data. For fixed-length data
727  * types this is trivial to compute, for varwidth types we need to
728  * actually walk the array and sum the sizes.
729  */
730  info[dim].nvalues = ndistinct;
731 
732  if (info[dim].typbyval) /* by-value data types */
733  {
734  info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
735 
736  /*
737  * We copy the data into the MCV item during deserialization, so
738  * we don't need to allocate any extra space.
739  */
740  info[dim].nbytes_aligned = 0;
741  }
742  else if (info[dim].typlen > 0) /* fixed-length by-ref */
743  {
744  /*
745  * We don't care about alignment in the serialized data, so we
746  * pack the data as much as possible. But we also track how much
747  * data will be needed after deserialization, and in that case we
748  * need to account for alignment of each item.
749  *
750  * Note: As the items are fixed-length, we could easily compute
751  * this during deserialization, but we do it here anyway.
752  */
753  info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
754  info[dim].nbytes_aligned = info[dim].nvalues * MAXALIGN(info[dim].typlen);
755  }
756  else if (info[dim].typlen == -1) /* varlena */
757  {
758  info[dim].nbytes = 0;
759  info[dim].nbytes_aligned = 0;
760  for (i = 0; i < info[dim].nvalues; i++)
761  {
762  Size len;
763 
764  /*
765  * For varlena values, we detoast the values and store the
766  * length and data separately. We don't bother with alignment
767  * here, which means that during deserialization we need to
768  * copy the fields and only access the copies.
769  */
770  values[dim][i] = PointerGetDatum(PG_DETOAST_DATUM(values[dim][i]));
771 
772  /* serialized length (uint32 length + data) */
773  len = VARSIZE_ANY_EXHDR(values[dim][i]);
774  info[dim].nbytes += sizeof(uint32); /* length */
775  info[dim].nbytes += len; /* value (no header) */
776 
777  /*
778  * During deserialization we'll build regular varlena values
779  * with full headers, and we need to align them properly.
780  */
781  info[dim].nbytes_aligned += MAXALIGN(VARHDRSZ + len);
782  }
783  }
784  else if (info[dim].typlen == -2) /* cstring */
785  {
786  info[dim].nbytes = 0;
787  info[dim].nbytes_aligned = 0;
788  for (i = 0; i < info[dim].nvalues; i++)
789  {
790  Size len;
791 
792  /*
793  * cstring is handled similar to varlena - first we store the
794  * length as uint32 and then the data. We don't care about
795  * alignment, which means that during deserialization we need
796  * to copy the fields and only access the copies.
797  */
798 
799  /* c-strings include terminator, so +1 byte */
800  len = strlen(DatumGetCString(values[dim][i])) + 1;
801  info[dim].nbytes += sizeof(uint32); /* length */
802  info[dim].nbytes += len; /* value */
803 
804  /* space needed for properly aligned deserialized copies */
805  info[dim].nbytes_aligned += MAXALIGN(len);
806  }
807  }
808 
809  /* we know (count>0) so there must be some data */
810  Assert(info[dim].nbytes > 0);
811  }
812 
813  /*
814  * Now we can finally compute how much space we'll actually need for the
815  * whole serialized MCV list (varlena header, MCV header, dimension info
816  * for each attribute, deduplicated values and items).
817  */
818  total_length = (3 * sizeof(uint32)) /* magic + type + nitems */
819  + sizeof(AttrNumber) /* ndimensions */
820  + (ndims * sizeof(Oid)); /* attribute types */
821 
822  /* dimension info */
823  total_length += ndims * sizeof(DimensionInfo);
824 
825  /* add space for the arrays of deduplicated values */
826  for (i = 0; i < ndims; i++)
827  total_length += info[i].nbytes;
828 
829  /*
830  * And finally account for the items (those are fixed-length, thanks to
831  * replacing values with uint16 indexes into the deduplicated arrays).
832  */
833  total_length += mcvlist->nitems * ITEM_SIZE(dim);
834 
835  /*
836  * Allocate space for the whole serialized MCV list (we'll skip bytes, so
837  * we set them to zero to make the result more compressible).
838  */
839  raw = (bytea *) palloc0(VARHDRSZ + total_length);
840  SET_VARSIZE(raw, VARHDRSZ + total_length);
841 
842  ptr = VARDATA(raw);
843  endptr = ptr + total_length;
844 
845  /* copy the MCV list header fields, one by one */
846  memcpy(ptr, &mcvlist->magic, sizeof(uint32));
847  ptr += sizeof(uint32);
848 
849  memcpy(ptr, &mcvlist->type, sizeof(uint32));
850  ptr += sizeof(uint32);
851 
852  memcpy(ptr, &mcvlist->nitems, sizeof(uint32));
853  ptr += sizeof(uint32);
854 
855  memcpy(ptr, &mcvlist->ndimensions, sizeof(AttrNumber));
856  ptr += sizeof(AttrNumber);
857 
858  memcpy(ptr, mcvlist->types, sizeof(Oid) * ndims);
859  ptr += (sizeof(Oid) * ndims);
860 
861  /* store information about the attributes (data amounts, ...) */
862  memcpy(ptr, info, sizeof(DimensionInfo) * ndims);
863  ptr += sizeof(DimensionInfo) * ndims;
864 
865  /* Copy the deduplicated values for all attributes to the output. */
866  for (dim = 0; dim < ndims; dim++)
867  {
868  /* remember the starting point for Asserts later */
869  char *start PG_USED_FOR_ASSERTS_ONLY = ptr;
870 
871  for (i = 0; i < info[dim].nvalues; i++)
872  {
873  Datum value = values[dim][i];
874 
875  if (info[dim].typbyval) /* passed by value */
876  {
877  Datum tmp;
878 
879  /*
880  * For byval types, we need to copy just the significant bytes
881  * - we can't use memcpy directly, as that assumes
882  * little-endian behavior. store_att_byval does almost what
883  * we need, but it requires a properly aligned buffer - the
884  * output buffer does not guarantee that. So we simply use a
885  * local Datum variable (which guarantees proper alignment),
886  * and then copy the value from it.
887  */
888  store_att_byval(&tmp, value, info[dim].typlen);
889 
890  memcpy(ptr, &tmp, info[dim].typlen);
891  ptr += info[dim].typlen;
892  }
893  else if (info[dim].typlen > 0) /* passed by reference */
894  {
895  /* no special alignment needed, treated as char array */
896  memcpy(ptr, DatumGetPointer(value), info[dim].typlen);
897  ptr += info[dim].typlen;
898  }
899  else if (info[dim].typlen == -1) /* varlena */
900  {
902 
903  /* copy the length */
904  memcpy(ptr, &len, sizeof(uint32));
905  ptr += sizeof(uint32);
906 
907  /* data from the varlena value (without the header) */
908  memcpy(ptr, VARDATA_ANY(DatumGetPointer(value)), len);
909  ptr += len;
910  }
911  else if (info[dim].typlen == -2) /* cstring */
912  {
913  uint32 len = (uint32) strlen(DatumGetCString(value)) + 1;
914 
915  /* copy the length */
916  memcpy(ptr, &len, sizeof(uint32));
917  ptr += sizeof(uint32);
918 
919  /* value */
920  memcpy(ptr, DatumGetCString(value), len);
921  ptr += len;
922  }
923 
924  /* no underflows or overflows */
925  Assert((ptr > start) && ((ptr - start) <= info[dim].nbytes));
926  }
927 
928  /* we should get exactly nbytes of data for this dimension */
929  Assert((ptr - start) == info[dim].nbytes);
930  }
931 
932  /* Serialize the items, with uint16 indexes instead of the values. */
933  for (i = 0; i < mcvlist->nitems; i++)
934  {
935  MCVItem *mcvitem = &mcvlist->items[i];
936 
937  /* don't write beyond the allocated space */
938  Assert(ptr <= (endptr - ITEM_SIZE(dim)));
939 
940  /* copy NULL and frequency flags into the serialized MCV */
941  memcpy(ptr, mcvitem->isnull, sizeof(bool) * ndims);
942  ptr += sizeof(bool) * ndims;
943 
944  memcpy(ptr, &mcvitem->frequency, sizeof(double));
945  ptr += sizeof(double);
946 
947  memcpy(ptr, &mcvitem->base_frequency, sizeof(double));
948  ptr += sizeof(double);
949 
950  /* store the indexes last */
951  for (dim = 0; dim < ndims; dim++)
952  {
953  uint16 index = 0;
954  Datum *value;
955 
956  /* do the lookup only for non-NULL values */
957  if (!mcvitem->isnull[dim])
958  {
959  value = (Datum *) bsearch_arg(&mcvitem->values[dim], values[dim],
960  info[dim].nvalues, sizeof(Datum),
961  compare_scalars_simple, &ssup[dim]);
962 
963  Assert(value != NULL); /* serialization or deduplication
964  * error */
965 
966  /* compute index within the deduplicated array */
967  index = (uint16) (value - values[dim]);
968 
969  /* check the index is within expected bounds */
970  Assert(index < info[dim].nvalues);
971  }
972 
973  /* copy the index into the serialized MCV */
974  memcpy(ptr, &index, sizeof(uint16));
975  ptr += sizeof(uint16);
976  }
977 
978  /* make sure we don't overflow the allocated value */
979  Assert(ptr <= endptr);
980  }
981 
982  /* at this point we expect to match the total_length exactly */
983  Assert(ptr == endptr);
984 
985  pfree(values);
986  pfree(counts);
987 
988  return raw;
989 }
struct SortSupportData * SortSupport
Definition: sortsupport.h:58
bool ssup_nulls_first
Definition: sortsupport.h:75
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define VARDATA(PTR)
Definition: postgres.h:315
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARHDRSZ
Definition: c.h:627
void PrepareSortSupportFromOrderingOp(Oid orderingOp, SortSupport ssup)
Definition: sortsupport.c:135
struct DimensionInfo DimensionInfo
Datum * values
Definition: statistics.h:83
unsigned int Oid
Definition: postgres_ext.h:31
Definition: type.h:89
unsigned short uint16
Definition: c.h:440
void pfree(void *pointer)
Definition: mcxt.c:1169
#define PG_UINT16_MAX
Definition: c.h:522
#define DatumGetCString(X)
Definition: postgres.h:610
MemoryContext ssup_cxt
Definition: sortsupport.h:66
int compare_scalars_simple(const void *a, const void *b, void *arg)
unsigned int uint32
Definition: c.h:441
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
int compare_datums_simple(Datum a, Datum b, SortSupport ssup)
#define store_att_byval(T, newdatum, attlen)
Definition: tupmacs.h:226
void * bsearch_arg(const void *key, const void *base, size_t nmemb, size_t size, int(*compar)(const void *, const void *, void *), void *arg)
Definition: bsearch_arg.c:55
void * palloc0(Size size)
Definition: mcxt.c:1093
uintptr_t Datum
Definition: postgres.h:411
TypeCacheEntry * lookup_type_cache(Oid type_id, int flags)
Definition: typcache.c:339
#define ITEM_SIZE(ndims)
Definition: mcv.c:57
static struct @143 value
#define Assert(condition)
Definition: c.h:804
double base_frequency
Definition: statistics.h:81
size_t Size
Definition: c.h:540
#define MAXALIGN(LEN)
Definition: c.h:757
Oid attrcollid
Definition: vacuum.h:127
#define DatumGetPointer(X)
Definition: postgres.h:593
bool * isnull
Definition: statistics.h:82
static Datum values[MAXATTR]
Definition: bootstrap.c:156
Form_pg_type attrtype
Definition: vacuum.h:126
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
int i
#define TYPECACHE_LT_OPR
Definition: typcache.h:137
#define PG_DETOAST_DATUM(datum)
Definition: fmgr.h:240
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21
double frequency
Definition: statistics.h:80
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
unsigned char bool
Definition: c.h:391

◆ statext_ndistinct_build()

MVNDistinct* statext_ndistinct_build ( double  totalrows,
StatsBuildData data 
)

Definition at line 89 of file mvdistinct.c.

References Assert, StatsBuildData::attnums, AttributeNumberIsValid, MVNDistinctItem::attributes, generator_free(), generator_init(), generator_next(), MVNDistinct::items, CombinationGenerator::k, MVNDistinct::magic, StatsBuildData::nattnums, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, ndistinct_for_combination(), MVNDistinct::nitems, num_combinations(), offsetof, palloc(), STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, and MVNDistinct::type.

Referenced by BuildRelationExtStatistics().

90 {
91  MVNDistinct *result;
92  int k;
93  int itemcnt;
94  int numattrs = data->nattnums;
95  int numcombs = num_combinations(numattrs);
96 
97  result = palloc(offsetof(MVNDistinct, items) +
98  numcombs * sizeof(MVNDistinctItem));
99  result->magic = STATS_NDISTINCT_MAGIC;
101  result->nitems = numcombs;
102 
103  itemcnt = 0;
104  for (k = 2; k <= numattrs; k++)
105  {
106  int *combination;
108 
109  /* generate combinations of K out of N elements */
110  generator = generator_init(numattrs, k);
111 
112  while ((combination = generator_next(generator)))
113  {
114  MVNDistinctItem *item = &result->items[itemcnt];
115  int j;
116 
117  item->attributes = palloc(sizeof(AttrNumber) * k);
118  item->nattributes = k;
119 
120  /* translate the indexes to attnums */
121  for (j = 0; j < k; j++)
122  {
123  item->attributes[j] = data->attnums[combination[j]];
124 
126  }
127 
128  item->ndistinct =
129  ndistinct_for_combination(totalrows, data, k, combination);
130 
131  itemcnt++;
132  Assert(itemcnt <= result->nitems);
133  }
134 
135  generator_free(generator);
136  }
137 
138  /* must consume exactly the whole output array */
139  Assert(itemcnt == result->nitems);
140 
141  return result;
142 }
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
double ndistinct
Definition: statistics.h:28
static int * generator_next(CombinationGenerator *state)
Definition: mvdistinct.c:627
static double ndistinct_for_combination(double totalrows, StatsBuildData *data, int k, int *combination)
Definition: mvdistinct.c:425
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
#define AttributeNumberIsValid(attributeNumber)
Definition: attnum.h:34
static void generator_free(CombinationGenerator *state)
Definition: mvdistinct.c:642
AttrNumber * attributes
Definition: statistics.h:30
uint32 type
Definition: statistics.h:37
#define Assert(condition)
Definition: c.h:804
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
void * palloc(Size size)
Definition: mcxt.c:1062
static int num_combinations(int n)
Definition: mvdistinct.c:575
int16 AttrNumber
Definition: attnum.h:21
static CombinationGenerator * generator_init(int n, int k)
Definition: mvdistinct.c:589
#define offsetof(type, field)
Definition: c.h:727

◆ statext_ndistinct_deserialize()

MVNDistinct* statext_ndistinct_deserialize ( bytea data)

Definition at line 250 of file mvdistinct.c.

References Assert, MVNDistinctItem::attributes, elog, ERROR, i, MVNDistinct::items, MVNDistinct::magic, MAXALIGN, MinSizeOfItems, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, MVNDistinct::nitems, offsetof, palloc(), palloc0(), SizeOfHeader, STATS_MAX_DIMENSIONS, STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, MVNDistinct::type, VARDATA_ANY, VARSIZE_ANY, and VARSIZE_ANY_EXHDR.

Referenced by pg_ndistinct_out(), and statext_ndistinct_load().

251 {
252  int i;
253  Size minimum_size;
254  MVNDistinct ndist;
255  MVNDistinct *ndistinct;
256  char *tmp;
257 
258  if (data == NULL)
259  return NULL;
260 
261  /* we expect at least the basic fields of MVNDistinct struct */
262  if (VARSIZE_ANY_EXHDR(data) < SizeOfHeader)
263  elog(ERROR, "invalid MVNDistinct size %zd (expected at least %zd)",
265 
266  /* initialize pointer to the data part (skip the varlena header) */
267  tmp = VARDATA_ANY(data);
268 
269  /* read the header fields and perform basic sanity checks */
270  memcpy(&ndist.magic, tmp, sizeof(uint32));
271  tmp += sizeof(uint32);
272  memcpy(&ndist.type, tmp, sizeof(uint32));
273  tmp += sizeof(uint32);
274  memcpy(&ndist.nitems, tmp, sizeof(uint32));
275  tmp += sizeof(uint32);
276 
277  if (ndist.magic != STATS_NDISTINCT_MAGIC)
278  elog(ERROR, "invalid ndistinct magic %08x (expected %08x)",
280  if (ndist.type != STATS_NDISTINCT_TYPE_BASIC)
281  elog(ERROR, "invalid ndistinct type %d (expected %d)",
283  if (ndist.nitems == 0)
284  elog(ERROR, "invalid zero-length item array in MVNDistinct");
285 
286  /* what minimum bytea size do we expect for those parameters */
287  minimum_size = MinSizeOfItems(ndist.nitems);
288  if (VARSIZE_ANY_EXHDR(data) < minimum_size)
289  elog(ERROR, "invalid MVNDistinct size %zd (expected at least %zd)",
290  VARSIZE_ANY_EXHDR(data), minimum_size);
291 
292  /*
293  * Allocate space for the ndistinct items (no space for each item's
294  * attnos: those live in bitmapsets allocated separately)
295  */
296  ndistinct = palloc0(MAXALIGN(offsetof(MVNDistinct, items)) +
297  (ndist.nitems * sizeof(MVNDistinctItem)));
298  ndistinct->magic = ndist.magic;
299  ndistinct->type = ndist.type;
300  ndistinct->nitems = ndist.nitems;
301 
302  for (i = 0; i < ndistinct->nitems; i++)
303  {
304  MVNDistinctItem *item = &ndistinct->items[i];
305 
306  /* ndistinct value */
307  memcpy(&item->ndistinct, tmp, sizeof(double));
308  tmp += sizeof(double);
309 
310  /* number of attributes */
311  memcpy(&item->nattributes, tmp, sizeof(int));
312  tmp += sizeof(int);
313  Assert((item->nattributes >= 2) && (item->nattributes <= STATS_MAX_DIMENSIONS));
314 
315  item->attributes
316  = (AttrNumber *) palloc(item->nattributes * sizeof(AttrNumber));
317 
318  memcpy(item->attributes, tmp, sizeof(AttrNumber) * item->nattributes);
319  tmp += sizeof(AttrNumber) * item->nattributes;
320 
321  /* still within the bytea */
322  Assert(tmp <= ((char *) data + VARSIZE_ANY(data)));
323  }
324 
325  /* we should have consumed the whole bytea exactly */
326  Assert(tmp == ((char *) data + VARSIZE_ANY(data)));
327 
328  return ndistinct;
329 }
#define MinSizeOfItems(nitems)
Definition: mvdistinct.c:56
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
double ndistinct
Definition: statistics.h:28
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
#define ERROR
Definition: elog.h:46
unsigned int uint32
Definition: c.h:441
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
void * palloc0(Size size)
Definition: mcxt.c:1093
AttrNumber * attributes
Definition: statistics.h:30
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
uint32 type
Definition: statistics.h:37
#define SizeOfHeader
Definition: mvdistinct.c:46
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
#define MAXALIGN(LEN)
Definition: c.h:757
#define STATS_MAX_DIMENSIONS
Definition: statistics.h:19
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
int i
int16 AttrNumber
Definition: attnum.h:21
#define offsetof(type, field)
Definition: c.h:727

◆ statext_ndistinct_serialize()

bytea* statext_ndistinct_serialize ( MVNDistinct ndistinct)

Definition at line 179 of file mvdistinct.c.

References Assert, MVNDistinctItem::attributes, i, MVNDistinct::items, MVNDistinct::magic, MVNDistinctItem::nattributes, MVNDistinctItem::ndistinct, MVNDistinct::nitems, output(), palloc(), SET_VARSIZE, SizeOfHeader, SizeOfItem, STATS_NDISTINCT_MAGIC, STATS_NDISTINCT_TYPE_BASIC, MVNDistinct::type, VARDATA, and VARHDRSZ.

Referenced by statext_store().

180 {
181  int i;
182  bytea *output;
183  char *tmp;
184  Size len;
185 
186  Assert(ndistinct->magic == STATS_NDISTINCT_MAGIC);
187  Assert(ndistinct->type == STATS_NDISTINCT_TYPE_BASIC);
188 
189  /*
190  * Base size is size of scalar fields in the struct, plus one base struct
191  * for each item, including number of items for each.
192  */
193  len = VARHDRSZ + SizeOfHeader;
194 
195  /* and also include space for the actual attribute numbers */
196  for (i = 0; i < ndistinct->nitems; i++)
197  {
198  int nmembers;
199 
200  nmembers = ndistinct->items[i].nattributes;
201  Assert(nmembers >= 2);
202 
203  len += SizeOfItem(nmembers);
204  }
205 
206  output = (bytea *) palloc(len);
207  SET_VARSIZE(output, len);
208 
209  tmp = VARDATA(output);
210 
211  /* Store the base struct values (magic, type, nitems) */
212  memcpy(tmp, &ndistinct->magic, sizeof(uint32));
213  tmp += sizeof(uint32);
214  memcpy(tmp, &ndistinct->type, sizeof(uint32));
215  tmp += sizeof(uint32);
216  memcpy(tmp, &ndistinct->nitems, sizeof(uint32));
217  tmp += sizeof(uint32);
218 
219  /*
220  * store number of attributes and attribute numbers for each entry
221  */
222  for (i = 0; i < ndistinct->nitems; i++)
223  {
224  MVNDistinctItem item = ndistinct->items[i];
225  int nmembers = item.nattributes;
226 
227  memcpy(tmp, &item.ndistinct, sizeof(double));
228  tmp += sizeof(double);
229  memcpy(tmp, &nmembers, sizeof(int));
230  tmp += sizeof(int);
231 
232  memcpy(tmp, item.attributes, sizeof(AttrNumber) * nmembers);
233  tmp += nmembers * sizeof(AttrNumber);
234 
235  /* protect against overflows */
236  Assert(tmp <= ((char *) output + len));
237  }
238 
239  /* check we used exactly the expected space */
240  Assert(tmp == ((char *) output + len));
241 
242  return output;
243 }
#define VARDATA(PTR)
Definition: postgres.h:315
MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]
Definition: statistics.h:39
static void output(uint64 loop_count)
#define VARHDRSZ
Definition: c.h:627
double ndistinct
Definition: statistics.h:28
#define STATS_NDISTINCT_TYPE_BASIC
Definition: statistics.h:23
#define SizeOfItem(natts)
Definition: mvdistinct.c:49
unsigned int uint32
Definition: c.h:441
uint32 nitems
Definition: statistics.h:38
uint32 magic
Definition: statistics.h:36
AttrNumber * attributes
Definition: statistics.h:30
uint32 type
Definition: statistics.h:37
#define SizeOfHeader
Definition: mvdistinct.c:46
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define STATS_NDISTINCT_MAGIC
Definition: statistics.h:22
void * palloc(Size size)
Definition: mcxt.c:1062
int i
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
int16 AttrNumber
Definition: attnum.h:21