PostgreSQL Source Code  git master
hashsort.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * hashsort.c
4  * Sort tuples for insertion into a new hash index.
5  *
6  * When building a very large hash index, we pre-sort the tuples by bucket
7  * number to improve locality of access to the index, and thereby avoid
8  * thrashing. We use tuplesort.c to sort the given index tuples into order.
9  *
10  * Note: if the number of rows in the table has been underestimated,
11  * bucket splits may occur during the index build. In that case we'd
12  * be inserting into two or more buckets for each possible masked-off
13  * hash code value. That's no big problem though, since we'll still have
14  * plenty of locality of access.
15  *
16  *
17  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
18  * Portions Copyright (c) 1994, Regents of the University of California
19  *
20  * IDENTIFICATION
21  * src/backend/access/hash/hashsort.c
22  *
23  *-------------------------------------------------------------------------
24  */
25 
26 #include "postgres.h"
27 
28 #include "access/hash.h"
29 #include "commands/progress.h"
30 #include "miscadmin.h"
31 #include "pgstat.h"
32 #include "port/pg_bitutils.h"
33 #include "utils/tuplesort.h"
34 
35 
36 /*
37  * Status record for spooling/sorting phase.
38  */
39 struct HSpool
40 {
41  Tuplesortstate *sortstate; /* state data for tuplesort.c */
43 
44  /*
45  * We sort the hash keys based on the buckets they belong to. Below masks
46  * are used in _hash_hashkey2bucket to determine the bucket of given hash
47  * key.
48  */
52 };
53 
54 
55 /*
56  * create and initialize a spool structure
57  */
58 HSpool *
60 {
61  HSpool *hspool = (HSpool *) palloc0(sizeof(HSpool));
62 
63  hspool->index = index;
64 
65  /*
66  * Determine the bitmask for hash code values. Since there are currently
67  * num_buckets buckets in the index, the appropriate mask can be computed
68  * as follows.
69  *
70  * NOTE : This hash mask calculation should be in sync with similar
71  * calculation in _hash_init_metabuffer.
72  */
73  hspool->high_mask = pg_nextpower2_32(num_buckets + 1) - 1;
74  hspool->low_mask = (hspool->high_mask >> 1);
75  hspool->max_buckets = num_buckets - 1;
76 
77  /*
78  * We size the sort area as maintenance_work_mem rather than work_mem to
79  * speed index creation. This should be OK since a single backend can't
80  * run multiple index creations in parallel.
81  */
83  index,
84  hspool->high_mask,
85  hspool->low_mask,
86  hspool->max_buckets,
88  NULL,
89  false);
90 
91  return hspool;
92 }
93 
94 /*
95  * clean up a spool structure and its substructures.
96  */
97 void
99 {
100  tuplesort_end(hspool->sortstate);
101  pfree(hspool);
102 }
103 
104 /*
105  * spool an index entry into the sort file.
106  */
107 void
108 _h_spool(HSpool *hspool, ItemPointer self, Datum *values, bool *isnull)
109 {
111  self, values, isnull);
112 }
113 
114 /*
115  * given a spool loaded by successive calls to _h_spool,
116  * create an entire index.
117  */
118 void
119 _h_indexbuild(HSpool *hspool, Relation heapRel)
120 {
121  IndexTuple itup;
122  int64 tups_done = 0;
123 #ifdef USE_ASSERT_CHECKING
124  uint32 hashkey = 0;
125 #endif
126 
128 
129  while ((itup = tuplesort_getindextuple(hspool->sortstate, true)) != NULL)
130  {
131  /*
132  * Technically, it isn't critical that hash keys be found in sorted
133  * order, since this sorting is only used to increase locality of
134  * access as a performance optimization. It still seems like a good
135  * idea to test tuplesort.c's handling of hash index tuple sorts
136  * through an assertion, though.
137  */
138 #ifdef USE_ASSERT_CHECKING
139  uint32 lasthashkey = hashkey;
140 
142  hspool->max_buckets, hspool->high_mask,
143  hspool->low_mask);
144  Assert(hashkey >= lasthashkey);
145 #endif
146 
147  _hash_doinsert(hspool->index, itup, heapRel);
148 
150  ++tups_done);
151  }
152 }
IndexTuple tuplesort_getindextuple(Tuplesortstate *state, bool forward)
Definition: tuplesort.c:2389
void _hash_doinsert(Relation rel, IndexTuple itup, Relation heapRel)
Definition: hashinsert.c:36
void tuplesort_performsort(Tuplesortstate *state)
Definition: tuplesort.c:1964
Bucket _hash_hashkey2bucket(uint32 hashkey, uint32 maxbucket, uint32 highmask, uint32 lowmask)
Definition: hashutil.c:126
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3231
Tuplesortstate * tuplesort_begin_index_hash(Relation heapRel, Relation indexRel, uint32 high_mask, uint32 low_mask, uint32 max_buckets, int workMem, SortCoordinate coordinate, bool randomAccess)
Definition: tuplesort.c:1125
Tuplesortstate * sortstate
Definition: hashsort.c:41
uint32 low_mask
Definition: hashsort.c:50
Definition: type.h:89
uint32 high_mask
Definition: hashsort.c:49
void pfree(void *pointer)
Definition: mcxt.c:1056
uint32 _hash_get_indextuple_hashkey(IndexTuple itup)
Definition: hashutil.c:292
void _h_spool(HSpool *hspool, ItemPointer self, Datum *values, bool *isnull)
Definition: hashsort.c:108
void _h_indexbuild(HSpool *hspool, Relation heapRel)
Definition: hashsort.c:119
static uint32 pg_nextpower2_32(uint32 num)
Definition: pg_bitutils.h:146
unsigned int uint32
Definition: c.h:367
void * palloc0(Size size)
Definition: mcxt.c:980
uintptr_t Datum
Definition: postgres.h:367
uint32 max_buckets
Definition: hashsort.c:51
Relation index
Definition: hashsort.c:42
void tuplesort_putindextuplevalues(Tuplesortstate *state, Relation rel, ItemPointer self, Datum *values, bool *isnull)
Definition: tuplesort.c:1651
int maintenance_work_mem
Definition: globals.c:122
#define Assert(condition)
Definition: c.h:738
static Datum values[MAXATTR]
Definition: bootstrap.c:167
void _h_spooldestroy(HSpool *hspool)
Definition: hashsort.c:98
#define PROGRESS_CREATEIDX_TUPLES_DONE
Definition: progress.h:85
HSpool * _h_spoolinit(Relation heap, Relation index, uint32 num_buckets)
Definition: hashsort.c:59
void tuplesort_end(Tuplesortstate *state)
Definition: tuplesort.c:1388