68 #define DSA_INITIAL_SEGMENT_SIZE ((size_t) (1 * 1024 * 1024))
77 #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2
84 #if SIZEOF_DSA_POINTER == 4
85 #define DSA_OFFSET_WIDTH 27
87 #define DSA_OFFSET_WIDTH 40
94 #define DSA_MAX_SEGMENTS \
95 Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
98 #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
101 #define DSA_MAX_SEGMENT_SIZE ((size_t) 1 << DSA_OFFSET_WIDTH)
104 #define DSA_PAGES_PER_SUPERBLOCK 16
111 #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
114 #define DSA_MAKE_POINTER(segment_number, offset) \
115 (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
118 #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
121 #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
127 #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
133 #define DSA_NUM_SEGMENT_BINS 16
154 #define DSA_AREA_LOCK(area) (&area->control->lock)
155 #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
224 #define NextFreeObjectIndex(object) (* (uint16 *) (object))
249 8, 16, 24, 32, 40, 48, 56, 64,
254 1280, 1560, 1816, 2048,
255 2616, 3120, 3640, 4096,
256 5456, 6552, 7280, 8192
258 #define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)
261 #define DSA_SCLASS_BLOCK_OF_SPANS 0
262 #define DSA_SCLASS_SPAN_LARGE 1
271 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
272 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
273 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
274 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
275 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
276 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
277 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
278 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
280 #define DSA_SIZE_CLASS_MAP_QUANTUM 8
288 #define DSA_FULLNESS_CLASSES 4
340 #define DsaAreaPoolToDsaPointer(area, p) \
341 DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
393 #define DSA_SPAN_NOTHING_FREE ((uint16) -1)
394 #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
397 #define get_segment_index(area, segment_map_ptr) \
398 (segment_map_ptr - &area->segment_maps[0])
404 int fromclass,
int toclass);
535 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
536 errmsg(
"could not attach to dynamic shared area")));
627 if (--control->
refcnt == 0)
696 elog(
ERROR,
"invalid DSA memory alloc request size %zu",
size);
719 (
errcode(ERRCODE_OUT_OF_MEMORY),
721 errdetail(
"Failed on DSA request of size %zu.",
730 if (segment_map == NULL)
732 if (segment_map == NULL)
741 (
errcode(ERRCODE_OUT_OF_MEMORY),
743 errdetail(
"Failed on DSA request of size %zu.",
757 "dsa_allocate could not find %zu free pages", npages);
766 init_span(area, span_pointer, pool, start_pointer, npages,
768 segment_map->
pagemap[first_page] = span_pointer;
775 return start_pointer;
799 uint16 mid = (min + max) / 2;
802 if (class_size <
size)
822 (
errcode(ERRCODE_OUT_OF_MEMORY),
856 span_pointer = segment_map->
pagemap[pageno];
870 #ifdef CLOBBER_FREED_MEMORY
894 #ifdef CLOBBER_FREED_MEMORY
895 memset(
object, 0x7f,
size);
901 Assert(
object >= superblock);
1081 span_pointer = pool->
spans[1];
1090 span_pointer =
next;
1114 fprintf(stderr,
" max_total_segment_size: %zu\n",
1116 fprintf(stderr,
" total_segment_size: %zu\n",
1120 fprintf(stderr,
" segment bins:\n");
1129 " segment bin %zu (no contiguous free pages):\n",
i);
1132 " segment bin %zu (at least %d contiguous pages free):\n",
1143 " segment index %zu, usable_pages = %zu, "
1144 "contiguous_pages = %zu, mapped at %p\n",
1167 fprintf(stderr,
" pool for blocks of span objects:\n");
1169 fprintf(stderr,
" pool for large object spans:\n");
1172 " pool for size class %zu (object size %hu bytes):\n",
1177 fprintf(stderr,
" fullness class %zu is empty\n",
j);
1182 fprintf(stderr,
" fullness class %zu:\n",
j);
1189 " span descriptor at "
1192 ", pages = %zu, objects free = %hu/%hu\n",
1239 size_t usable_pages;
1241 size_t metadata_bytes;
1246 elog(
ERROR,
"dsa_area space must be at least %zu, but %zu provided",
1266 memset(place, 0,
sizeof(*control));
1274 control->
handle = control_handle;
1301 segment_map->
segment = control_segment;
1316 if (usable_pages > 0)
1353 segment_map->
segment = segment;
1364 if (control->
refcnt == 0)
1368 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1369 errmsg(
"could not attach to dynamic shared area")));
1407 pool->
spans[1] = span_pointer;
1409 span->
start = start;
1445 span_pointer = pool->
spans[fromclass];
1461 pool->
spans[toclass] = span_pointer;
1515 block = span->
start;
1598 span_pointer = pool->
spans[fclass];
1610 next_span_pointer = span->
nextspan;
1627 if (tfclass < fclass)
1630 if (pool->
spans[fclass] == span_pointer)
1635 if (nextspan != NULL)
1646 if (nextspan != NULL)
1651 pool->
spans[tfclass] = span_pointer;
1663 span_pointer = next_span_pointer;
1707 if (segment_map == NULL)
1710 if (segment_map == NULL)
1723 "dsa_allocate could not find %zu free pages for superblock",
1742 span_pointer = start_pointer;
1746 init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1747 for (
i = 0;
i < npages; ++
i)
1748 segment_map->
pagemap[first_page +
i] = span_pointer;
1786 "dsa_area could not attach to a segment that has been freed");
1792 if (segment == NULL)
1793 elog(
ERROR,
"dsa_area could not attach to segment");
1795 segment_map->
segment = segment;
1887 segment_map->
header = NULL;
1893 if (segment_map->
header != NULL)
1943 pool->
spans[fclass]);
1949 pool->
spans[fclass] = span_pointer;
2034 size_t threshold = (size_t) 1 << (bin - 1);
2043 size_t contiguous_pages;
2046 next_segment_index = segment_map->
header->
next;
2050 if (contiguous_pages >= threshold && contiguous_pages < npages)
2052 segment_index = next_segment_index;
2057 if (contiguous_pages < threshold)
2068 if (contiguous_pages >= npages)
2072 segment_index = next_segment_index;
2089 size_t metadata_bytes;
2092 size_t usable_pages;
2152 if (requested_pages > usable_pages)
2158 usable_pages = requested_pages;
2184 if (segment == NULL)
2204 segment_map->
segment = segment;
2237 next->header->prev = new_index;
2259 size_t freed_segment_counter;
2295 size_t freed_segment_counter;
2327 if (segment_map->
header->
bin == new_bin)
2345 next->header->prev = segment_index;
#define pg_read_barrier()
static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
static void check_for_freed_segments(dsa_area *area)
dsa_area * dsa_create_in_place(void *place, size_t size, int tranche_id, dsm_segment *segment)
static const uint16 dsa_size_classes[]
#define DSA_EXTRACT_SEGMENT_NUMBER(dp)
#define DSA_AREA_LOCK(area)
#define DSA_NUM_SEGMENTS_AT_EACH_SIZE
static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)
static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)
#define DSA_SEGMENT_INDEX_NONE
#define DSA_SEGMENT_HEADER_MAGIC
void dsa_trim(dsa_area *area)
#define DSA_SPAN_NOTHING_FREE
#define DSA_MAKE_POINTER(segment_number, offset)
#define get_segment_index(area, segment_map_ptr)
void dsa_on_shmem_exit_release_in_place(int code, Datum place)
#define DSA_INITIAL_SEGMENT_SIZE
void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
static dsa_pointer alloc_object(dsa_area *area, int size_class)
#define DSA_PAGES_PER_SUPERBLOCK
#define DSA_SIZE_CLASS_MAP_QUANTUM
static size_t contiguous_pages_to_segment_bin(size_t n)
static const uint8 dsa_size_class_map[]
dsa_pointer dsa_allocate_extended(dsa_area *area, size_t size, int flags)
static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)
size_t dsa_get_total_size(dsa_area *area)
dsa_area * dsa_attach(dsa_handle handle)
#define DSA_SUPERBLOCK_SIZE
#define DsaAreaPoolToDsaPointer(area, p)
void * dsa_get_address(dsa_area *area, dsa_pointer dp)
static void check_for_freed_segments_locked(dsa_area *area)
#define DSA_EXTRACT_OFFSET(dp)
static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
static void rebin_segment(dsa_area *area, dsa_segment_map *segment_map)
#define DSA_SCLASS_LOCK(area, sclass)
void dsa_release_in_place(void *place)
static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)
void dsa_set_size_limit(dsa_area *area, size_t limit)
#define DSA_SCLASS_BLOCK_OF_SPANS
static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)
static void unlink_span(dsa_area *area, dsa_area_span *span)
dsa_area * dsa_create(int tranche_id)
#define DSA_SCLASS_SPAN_LARGE
#define DSA_NUM_SIZE_CLASSES
void dsa_unpin(dsa_area *area)
void dsa_pin_mapping(dsa_area *area)
static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
#define NextFreeObjectIndex(object)
void dsa_dump(dsa_area *area)
dsa_handle dsa_get_handle(dsa_area *area)
static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)
void dsa_detach(dsa_area *area)
static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)
dsa_area * dsa_attach_in_place(void *place, dsm_segment *segment)
#define DSA_MAX_SEGMENT_SIZE
#define DSA_FULLNESS_CLASSES
void dsa_free(dsa_area *area, dsa_pointer dp)
#define DSA_NUM_SEGMENT_BINS
static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment)
size_t dsa_minimum_size(void)
void dsa_pin(dsa_area *area)
#define DSA_POINTER_FORMAT
#define InvalidDsaPointer
#define DSA_HANDLE_INVALID
#define DsaPointerIsValid(x)
dsm_handle dsm_segment_handle(dsm_segment *seg)
void * dsm_segment_address(dsm_segment *seg)
void dsm_detach(dsm_segment *seg)
void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)
dsm_segment * dsm_attach(dsm_handle h)
dsm_segment * dsm_create(Size size, int flags)
void dsm_pin_mapping(dsm_segment *seg)
void dsm_unpin_segment(dsm_handle handle)
void dsm_pin_segment(dsm_segment *seg)
#define DSM_HANDLE_INVALID
int errdetail(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)
void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)
void FreePageManagerInitialize(FreePageManager *fpm, char *base)
#define fpm_size_to_pages(sz)
Assert(fmt[strlen(fmt) - 1] !='\n')
bool LWLockHeldByMe(LWLock *lock)
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
void LWLockRelease(LWLock *lock)
void LWLockInitialize(LWLock *lock, int tranche_id)
void pfree(void *pointer)
#define AllocHugeSizeIsValid(size)
#define AllocSizeIsValid(size)
#define pg_leftmost_one_pos_size_t
static Datum PointerGetDatum(const void *X)
static Pointer DatumGetPointer(Datum X)
ResourceOwner CurrentResourceOwner
static pg_noinline void Size size
dsa_segment_header segment_header
size_t total_segment_size
dsa_segment_index high_segment_index
size_t max_total_segment_size
dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]
dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]
size_t freed_segment_counter
dsm_handle segment_handles[DSA_MAX_SEGMENTS]
dsa_pointer spans[DSA_FULLNESS_CLASSES]
dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]
dsa_segment_index high_segment_index
size_t freed_segment_counter
dsa_area_control * control
dsa_segment_header * header