#include "port/atomics.h"
#include "storage/dsm.h"

Include dependency graph for dsa.h:

This graph shows which files directly or indirectly include this file:

Macros
#define	SIZEOF_DSA_POINTER 8

#define	dsa_pointer_atomic_init pg_atomic_init_u64

#define	dsa_pointer_atomic_read pg_atomic_read_u64

#define	dsa_pointer_atomic_write pg_atomic_write_u64

#define	dsa_pointer_atomic_fetch_add pg_atomic_fetch_add_u64

#define	dsa_pointer_atomic_compare_exchange pg_atomic_compare_exchange_u64

#define	DSA_POINTER_FORMAT "%016" INT64_MODIFIER "x"

#define	DSA_ALLOC_HUGE 0x01 /* allow huge allocation (> 1 GB) */

#define	DSA_ALLOC_NO_OOM 0x02 /* no failure if out-of-memory */

#define	DSA_ALLOC_ZERO 0x04 /* zero allocated memory */

#define	InvalidDsaPointer ((dsa_pointer) 0)

#define	DsaPointerIsValid(x) ((x) != InvalidDsaPointer)

#define	dsa_allocate(area, size) dsa_allocate_extended(area, size, 0)

#define	dsa_allocate0(area, size) dsa_allocate_extended(area, size, DSA_ALLOC_ZERO)

#define	DSA_HANDLE_INVALID ((dsa_handle) DSM_HANDLE_INVALID)

Typedefs
typedef struct dsa_area	dsa_area

typedef uint64	dsa_pointer

typedef pg_atomic_uint64	dsa_pointer_atomic

typedef dsm_handle	dsa_handle

Functions
dsa_area *	dsa_create (int tranche_id)

dsa_area *	dsa_create_in_place (void place, size_t size, int tranche_id, dsm_segment segment)

dsa_area *	dsa_attach (dsa_handle handle)

dsa_area *	dsa_attach_in_place (void place, dsm_segment segment)

void	dsa_release_in_place (void *place)

void	dsa_on_dsm_detach_release_in_place (dsm_segment *, Datum)

void	dsa_on_shmem_exit_release_in_place (int, Datum)

void	dsa_pin_mapping (dsa_area *area)

void	dsa_detach (dsa_area *area)

void	dsa_pin (dsa_area *area)

void	dsa_unpin (dsa_area *area)

void	dsa_set_size_limit (dsa_area *area, size_t limit)

size_t	dsa_minimum_size (void)

dsa_handle	dsa_get_handle (dsa_area *area)

dsa_pointer	dsa_allocate_extended (dsa_area *area, size_t size, int flags)

void	dsa_free (dsa_area *area, dsa_pointer dp)

void *	dsa_get_address (dsa_area *area, dsa_pointer dp)

void	dsa_trim (dsa_area *area)

void	dsa_dump (dsa_area *area)

Macro Definition Documentation

◆ DSA_ALLOC_HUGE

#define DSA_ALLOC_HUGE 0x01 /* allow huge allocation (> 1 GB) */

Definition at line 73 of file dsa.h.

◆ DSA_ALLOC_NO_OOM

#define DSA_ALLOC_NO_OOM 0x02 /* no failure if out-of-memory */

Definition at line 74 of file dsa.h.

◆ DSA_ALLOC_ZERO

#define DSA_ALLOC_ZERO 0x04 /* zero allocated memory */

Definition at line 75 of file dsa.h.

◆ dsa_allocate

#define dsa_allocate	(	area,
		size
	)	dsa_allocate_extended(area, size, 0)

Definition at line 84 of file dsa.h.

◆ dsa_allocate0

#define dsa_allocate0	(	area,
		size
	)	dsa_allocate_extended(area, size, DSA_ALLOC_ZERO)

Definition at line 88 of file dsa.h.

◆ DSA_HANDLE_INVALID

#define DSA_HANDLE_INVALID ((dsa_handle) DSM_HANDLE_INVALID)

Definition at line 103 of file dsa.h.

◆ dsa_pointer_atomic_compare_exchange

#define dsa_pointer_atomic_compare_exchange pg_atomic_compare_exchange_u64

Definition at line 68 of file dsa.h.

◆ dsa_pointer_atomic_fetch_add

#define dsa_pointer_atomic_fetch_add pg_atomic_fetch_add_u64

Definition at line 67 of file dsa.h.

◆ dsa_pointer_atomic_init

#define dsa_pointer_atomic_init pg_atomic_init_u64

Definition at line 64 of file dsa.h.

◆ dsa_pointer_atomic_read

#define dsa_pointer_atomic_read pg_atomic_read_u64

Definition at line 65 of file dsa.h.

◆ dsa_pointer_atomic_write

#define dsa_pointer_atomic_write pg_atomic_write_u64

Definition at line 66 of file dsa.h.

◆ DSA_POINTER_FORMAT

#define DSA_POINTER_FORMAT "%016" INT64_MODIFIER "x"

Definition at line 69 of file dsa.h.

◆ DsaPointerIsValid

#define DsaPointerIsValid ( x ) ((x) != InvalidDsaPointer)

Definition at line 81 of file dsa.h.

◆ InvalidDsaPointer

#define InvalidDsaPointer ((dsa_pointer) 0)

Definition at line 78 of file dsa.h.

◆ SIZEOF_DSA_POINTER

#define SIZEOF_DSA_POINTER 8

Definition at line 42 of file dsa.h.

Typedef Documentation

◆ dsa_area

typedef struct dsa_area dsa_area

Definition at line 1 of file dsa.h.

◆ dsa_handle

typedef dsm_handle dsa_handle

Definition at line 100 of file dsa.h.

◆ dsa_pointer

typedef uint64 dsa_pointer

Definition at line 62 of file dsa.h.

◆ dsa_pointer_atomic

typedef pg_atomic_uint64 dsa_pointer_atomic

Definition at line 63 of file dsa.h.

Function Documentation

◆ dsa_allocate_extended()

dsa_pointer dsa_allocate_extended	(	dsa_area *	area,
		size_t	size,
		int	flags
	)

Definition at line 678 of file dsa.c.

 {
     uint16      size_class;
     dsa_pointer start_pointer;
     dsa_segment_map *segment_map;
     dsa_pointer result;
  
     Assert(size > 0);
  
     /* Sanity check on huge individual allocation size. */
     if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
         ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
         elog(ERROR, "invalid DSA memory alloc request size %zu", size);
  
     /*
      * If bigger than the largest size class, just grab a run of pages from
      * the free page manager, instead of allocating an object from a pool.
      * There will still be a span, but it's a special class of span that
      * manages this whole allocation and simply gives all pages back to the
      * free page manager when dsa_free is called.
      */
     if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
     {
         size_t      npages = fpm_size_to_pages(size);
         size_t      first_page;
         dsa_pointer span_pointer;
         dsa_area_pool *pool = &area->control->pools[DSA_SCLASS_SPAN_LARGE];
  
         /* Obtain a span object. */
         span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
         if (!DsaPointerIsValid(span_pointer))
         {
             /* Raise error unless asked not to. */
             if ((flags & DSA_ALLOC_NO_OOM) == 0)
                 ereport(ERROR,
                         (errcode(ERRCODE_OUT_OF_MEMORY),
                          errmsg("out of memory"),
                          errdetail("Failed on DSA request of size %zu.",
                                    size)));
             return InvalidDsaPointer;
         }
  
         LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
  
         /* Find a segment from which to allocate. */
         segment_map = get_best_segment(area, npages);
         if (segment_map == NULL)
             segment_map = make_new_segment(area, npages);
         if (segment_map == NULL)
         {
             /* Can't make any more segments: game over. */
             LWLockRelease(DSA_AREA_LOCK(area));
             dsa_free(area, span_pointer);
  
             /* Raise error unless asked not to. */
             if ((flags & DSA_ALLOC_NO_OOM) == 0)
                 ereport(ERROR,
                         (errcode(ERRCODE_OUT_OF_MEMORY),
                          errmsg("out of memory"),
                          errdetail("Failed on DSA request of size %zu.",
                                    size)));
             return InvalidDsaPointer;
         }
  
         /*
          * Ask the free page manager for a run of pages.  This should always
          * succeed, since both get_best_segment and make_new_segment should
          * only return a non-NULL pointer if it actually contains enough
          * contiguous freespace.  If it does fail, something in our backend
          * private state is out of whack, so use FATAL to kill the process.
          */
         if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
             elog(FATAL,
                  "dsa_allocate could not find %zu free pages", npages);
         LWLockRelease(DSA_AREA_LOCK(area));
  
         start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
                                          first_page * FPM_PAGE_SIZE);
  
         /* Initialize span and pagemap. */
         LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                       LW_EXCLUSIVE);
         init_span(area, span_pointer, pool, start_pointer, npages,
                   DSA_SCLASS_SPAN_LARGE);
         segment_map->pagemap[first_page] = span_pointer;
         LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
  
         /* Zero-initialize the memory if requested. */
         if ((flags & DSA_ALLOC_ZERO) != 0)
             memset(dsa_get_address(area, start_pointer), 0, size);
  
         return start_pointer;
     }
  
     /* Map allocation to a size class. */
     if (size < lengthof(dsa_size_class_map) * DSA_SIZE_CLASS_MAP_QUANTUM)
     {
         int         mapidx;
  
         /* For smaller sizes we have a lookup table... */
         mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
                   DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
         size_class = dsa_size_class_map[mapidx];
     }
     else
     {
         uint16      min;
         uint16      max;
  
         /* ... and for the rest we search by binary chop. */
         min = dsa_size_class_map[lengthof(dsa_size_class_map) - 1];
         max = lengthof(dsa_size_classes) - 1;
  
         while (min < max)
         {
             uint16      mid = (min + max) / 2;
             uint16      class_size = dsa_size_classes[mid];
  
             if (class_size < size)
                 min = mid + 1;
             else
                 max = mid;
         }
  
         size_class = min;
     }
     Assert(size <= dsa_size_classes[size_class]);
     Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
  
     /* Attempt to allocate an object from the appropriate pool. */
     result = alloc_object(area, size_class);
  
     /* Check for failure to allocate. */
     if (!DsaPointerIsValid(result))
     {
         /* Raise error unless asked not to. */
         if ((flags & DSA_ALLOC_NO_OOM) == 0)
             ereport(ERROR,
                     (errcode(ERRCODE_OUT_OF_MEMORY),
                      errmsg("out of memory"),
                      errdetail("Failed on DSA request of size %zu.", size)));
         return InvalidDsaPointer;
     }
  
     /* Zero-initialize the memory if requested. */
     if ((flags & DSA_ALLOC_ZERO) != 0)
         memset(dsa_get_address(area, result), 0, size);
  
     return result;
 }

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert(), dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog(), ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, and dsa_area_control::pools.

Referenced by dshash_create(), and pagetable_allocate().

◆ dsa_attach()

dsa_area* dsa_attach ( dsa_handle handle )

Definition at line 519 of file dsa.c.

 {
     dsm_segment *segment;
     dsa_area   *area;
  
     /*
      * An area handle is really a DSM segment handle for the first segment, so
      * we go ahead and attach to that.
      */
     segment = dsm_attach(handle);
     if (segment == NULL)
         ereport(ERROR,
                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                  errmsg("could not attach to dynamic shared area")));
  
     area = attach_internal(dsm_segment_address(segment), segment, handle);
  
     /* Clean up when the control segment detaches. */
     on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                   PointerGetDatum(dsm_segment_address(segment)));
  
     return area;
 }

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum().

Referenced by logicalrep_launcher_attach_dshmem().

◆ dsa_attach_in_place()

dsa_area* dsa_attach_in_place	(	void *	place,
		dsm_segment *	segment
	)

Definition at line 554 of file dsa.c.

 {
     dsa_area   *area;
  
     area = attach_internal(place, NULL, DSA_HANDLE_INVALID);
  
     /*
      * Clean up when the control segment detaches, if a containing DSM segment
      * was provided.
      */
     if (segment != NULL)
         on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                       PointerGetDatum(place));
  
     return area;
 }

References attach_internal(), DSA_HANDLE_INVALID, dsa_on_dsm_detach_release_in_place(), on_dsm_detach(), and PointerGetDatum().

Referenced by AttachSession(), ParallelQueryMain(), and pgstat_attach_shmem().

◆ dsa_create()

dsa_area* dsa_create ( int tranche_id )

Definition at line 433 of file dsa.c.

 {
     dsm_segment *segment;
     dsa_area   *area;
  
     /*
      * Create the DSM segment that will hold the shared control object and the
      * first segment of usable space.
      */
     segment = dsm_create(DSA_INITIAL_SEGMENT_SIZE, 0);
  
     /*
      * All segments backing this area are pinned, so that DSA can explicitly
      * control their lifetime (otherwise a newly created segment belonging to
      * this area might be freed when the only backend that happens to have it
      * mapped in ends, corrupting the area).
      */
     dsm_pin_segment(segment);
  
     /* Create a new DSA area with the control object in this segment. */
     area = create_internal(dsm_segment_address(segment),
                            DSA_INITIAL_SEGMENT_SIZE,
                            tranche_id,
                            dsm_segment_handle(segment), segment);
  
     /* Clean up when the control segment detaches. */
     on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                   PointerGetDatum(dsm_segment_address(segment)));
  
     return area;
 }

References create_internal(), DSA_INITIAL_SEGMENT_SIZE, dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum().

Referenced by logicalrep_launcher_attach_dshmem().

◆ dsa_create_in_place()

dsa_area* dsa_create_in_place	(	void *	place,
		size_t	size,
		int	tranche_id,
		dsm_segment *	segment
	)

Definition at line 482 of file dsa.c.

 {
     dsa_area   *area;
  
     area = create_internal(place, size, tranche_id,
                            DSM_HANDLE_INVALID, NULL);
  
     /*
      * Clean up when the control segment detaches, if a containing DSM segment
      * was provided.
      */
     if (segment != NULL)
         on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                       PointerGetDatum(place));
  
     return area;
 }

References create_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), and PointerGetDatum().

Referenced by ExecInitParallelPlan(), GetSessionDsmHandle(), and StatsShmemInit().

◆ dsa_detach()

void dsa_detach ( dsa_area * area )

Definition at line 1932 of file dsa.c.

 {
     int         i;
  
     /* Detach from all segments. */
     for (i = 0; i <= area->high_segment_index; ++i)
         if (area->segment_maps[i].segment != NULL)
             dsm_detach(area->segment_maps[i].segment);
  
     /*
      * Note that 'detaching' (= detaching from DSM segments) doesn't include
      * 'releasing' (= adjusting the reference count).  It would be nice to
      * combine these operations, but client code might never get around to
      * calling dsa_detach because of an error path, and a detach hook on any
      * particular segment is too late to detach other segments in the area
      * without risking a 'leak' warning in the non-error path.
      */
  
     /* Free the backend-local area object. */
     pfree(area);
 }

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by DetachSession(), ExecParallelCleanup(), ParallelQueryMain(), pgstat_detach_shmem(), and StatsShmemInit().

◆ dsa_dump()

void dsa_dump ( dsa_area * area )

Definition at line 1082 of file dsa.c.

 {
     size_t      i,
                 j;
  
     /*
      * Note: This gives an inconsistent snapshot as it acquires and releases
      * individual locks as it goes...
      */
  
     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     check_for_freed_segments_locked(area);
     fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
     fprintf(stderr, "  max_total_segment_size: %zu\n",
             area->control->max_total_segment_size);
     fprintf(stderr, "  total_segment_size: %zu\n",
             area->control->total_segment_size);
     fprintf(stderr, "  refcnt: %d\n", area->control->refcnt);
     fprintf(stderr, "  pinned: %c\n", area->control->pinned ? 't' : 'f');
     fprintf(stderr, "  segment bins:\n");
     for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
     {
         if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
         {
             dsa_segment_index segment_index;
  
             fprintf(stderr,
                     "    segment bin %zu (at least %d contiguous pages free):\n",
                     i, 1 << (i - 1));
             segment_index = area->control->segment_bins[i];
             while (segment_index != DSA_SEGMENT_INDEX_NONE)
             {
                 dsa_segment_map *segment_map;
  
                 segment_map =
                     get_segment_by_index(area, segment_index);
  
                 fprintf(stderr,
                         "      segment index %zu, usable_pages = %zu, "
                         "contiguous_pages = %zu, mapped at %p\n",
                         segment_index,
                         segment_map->header->usable_pages,
                         fpm_largest(segment_map->fpm),
                         segment_map->mapped_address);
                 segment_index = segment_map->header->next;
             }
         }
     }
     LWLockRelease(DSA_AREA_LOCK(area));
  
     fprintf(stderr, "  pools:\n");
     for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
     {
         bool        found = false;
  
         LWLockAcquire(DSA_SCLASS_LOCK(area, i), LW_EXCLUSIVE);
         for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
             if (DsaPointerIsValid(area->control->pools[i].spans[j]))
                 found = true;
         if (found)
         {
             if (i == DSA_SCLASS_BLOCK_OF_SPANS)
                 fprintf(stderr, "    pool for blocks of span objects:\n");
             else if (i == DSA_SCLASS_SPAN_LARGE)
                 fprintf(stderr, "    pool for large object spans:\n");
             else
                 fprintf(stderr,
                         "    pool for size class %zu (object size %hu bytes):\n",
                         i, dsa_size_classes[i]);
             for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
             {
                 if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
                     fprintf(stderr, "      fullness class %zu is empty\n", j);
                 else
                 {
                     dsa_pointer span_pointer = area->control->pools[i].spans[j];
  
                     fprintf(stderr, "      fullness class %zu:\n", j);
                     while (DsaPointerIsValid(span_pointer))
                     {
                         dsa_area_span *span;
  
                         span = dsa_get_address(area, span_pointer);
                         fprintf(stderr,
                                 "        span descriptor at "
                                 DSA_POINTER_FORMAT ", superblock at "
                                 DSA_POINTER_FORMAT
                                 ", pages = %zu, objects free = %hu/%hu\n",
                                 span_pointer, span->start, span->npages,
                                 span->nallocatable, span->nmax);
                         span_pointer = span->nextspan;
                     }
                 }
             }
         }
         LWLockRelease(DSA_SCLASS_LOCK(area, i));
     }
 }

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, j, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

◆ dsa_free()

void dsa_free	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 833 of file dsa.c.

 {
     dsa_segment_map *segment_map;
     int         pageno;
     dsa_pointer span_pointer;
     dsa_area_span *span;
     char       *superblock;
     char       *object;
     size_t      size;
     int         size_class;
  
     /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
     check_for_freed_segments(area);
  
     /* Locate the object, span and pool. */
     segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
     pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
     span_pointer = segment_map->pagemap[pageno];
     span = dsa_get_address(area, span_pointer);
     superblock = dsa_get_address(area, span->start);
     object = dsa_get_address(area, dp);
     size_class = span->size_class;
     size = dsa_size_classes[size_class];
  
     /*
      * Special case for large objects that live in a special span: we return
      * those pages directly to the free page manager and free the span.
      */
     if (span->size_class == DSA_SCLASS_SPAN_LARGE)
     {
  
 #ifdef CLOBBER_FREED_MEMORY
         memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
 #endif
  
         /* Give pages back to free page manager. */
         LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
         FreePageManagerPut(segment_map->fpm,
                            DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
                            span->npages);
  
         /* Move segment to appropriate bin if necessary. */
         rebin_segment(area, segment_map);
         LWLockRelease(DSA_AREA_LOCK(area));
  
         /* Unlink span. */
         LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                       LW_EXCLUSIVE);
         unlink_span(area, span);
         LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
         /* Free the span object so it can be reused. */
         dsa_free(area, span_pointer);
         return;
     }
  
 #ifdef CLOBBER_FREED_MEMORY
     memset(object, 0x7f, size);
 #endif
  
     LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
  
     /* Put the object on the span's freelist. */
     Assert(object >= superblock);
     Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
     Assert((object - superblock) % size == 0);
     NextFreeObjectIndex(object) = span->firstfree;
     span->firstfree = (object - superblock) / size;
     ++span->nallocatable;
  
     /*
      * See if the span needs to moved to a different fullness class, or be
      * freed so its pages can be given back to the segment.
      */
     if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
     {
         /*
          * The block was completely full and is located in the
          * highest-numbered fullness class, which is never scanned for free
          * chunks.  We must move it to the next-lower fullness class.
          */
         unlink_span(area, span);
         add_span_to_fullness_class(area, span, span_pointer,
                                    DSA_FULLNESS_CLASSES - 2);
  
         /*
          * If this is the only span, and there is no active span, then we
          * should probably move this span to fullness class 1.  (Otherwise if
          * you allocate exactly all the objects in the only span, it moves to
          * class 3, then you free them all, it moves to 2, and then is given
          * back, leaving no active span).
          */
     }
     else if (span->nallocatable == span->nmax &&
              (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
     {
         /*
          * This entire block is free, and it's not the active block for this
          * size class.  Return the memory to the free page manager. We don't
          * do this for the active block to prevent hysteresis: if we
          * repeatedly allocate and free the only chunk in the active block, it
          * will be very inefficient if we deallocate and reallocate the block
          * every time.
          */
         destroy_superblock(area, span_pointer);
     }
  
     LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
 }

References add_span_to_fullness_class(), Assert(), check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, rebin_segment(), dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), pagetable_free(), pgstat_free_entry(), resize(), and tbm_free_shared_area().

◆ dsa_get_address()

void* dsa_get_address	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 949 of file dsa.c.

 {
     dsa_segment_index index;
     size_t      offset;
  
     /* Convert InvalidDsaPointer to NULL. */
     if (!DsaPointerIsValid(dp))
         return NULL;
  
     /* Process any requests to detach from freed segments. */
     check_for_freed_segments(area);
  
     /* Break the dsa_pointer into its components. */
     index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
     offset = DSA_EXTRACT_OFFSET(dp);
     Assert(index < DSA_MAX_SEGMENTS);
  
     /* Check if we need to cause this segment to be mapped in. */
     if (unlikely(area->segment_maps[index].mapped_address == NULL))
     {
         /* Call for effect (we don't need the result). */
         get_segment_by_index(area, index);
     }
  
     return area->segment_maps[index].mapped_address + offset;
 }

References Assert(), check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

◆ dsa_get_handle()

dsa_handle dsa_get_handle ( dsa_area * area )

Definition at line 507 of file dsa.c.

 {
     Assert(area->control->handle != DSA_HANDLE_INVALID);
     return area->control->handle;
 }

References Assert(), dsa_area::control, DSA_HANDLE_INVALID, and dsa_area_control::handle.

Referenced by logicalrep_launcher_attach_dshmem().

◆ dsa_minimum_size()

size_t dsa_minimum_size ( void )

Definition at line 1186 of file dsa.c.

 {
     size_t      size;
     int         pages = 0;
  
     size = MAXALIGN(sizeof(dsa_area_control)) +
         MAXALIGN(sizeof(FreePageManager));
  
     /* Figure out how many pages we need, including the page map... */
     while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
     {
         ++pages;
         size += sizeof(dsa_pointer);
     }
  
     return pages * FPM_PAGE_SIZE;
 }

References FPM_PAGE_SIZE, and MAXALIGN.

Referenced by create_internal(), ExecInitParallelPlan(), and pgstat_dsa_init_size().

◆ dsa_on_dsm_detach_release_in_place()

void dsa_on_dsm_detach_release_in_place	(	dsm_segment *	segment,
		Datum	place
	)

Definition at line 585 of file dsa.c.

 {
     dsa_release_in_place(DatumGetPointer(place));
 }

References DatumGetPointer(), and dsa_release_in_place().

Referenced by dsa_attach(), dsa_attach_in_place(), dsa_create(), and dsa_create_in_place().

◆ dsa_on_shmem_exit_release_in_place()

void dsa_on_shmem_exit_release_in_place	(	int	code,
		Datum	place
	)

Definition at line 599 of file dsa.c.

 {
     dsa_release_in_place(DatumGetPointer(place));
 }

References DatumGetPointer(), and dsa_release_in_place().

◆ dsa_pin()

void dsa_pin ( dsa_area * area )

Definition at line 982 of file dsa.c.

 {
     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     if (area->control->pinned)
     {
         LWLockRelease(DSA_AREA_LOCK(area));
         elog(ERROR, "dsa_area already pinned");
     }
     area->control->pinned = true;
     ++area->control->refcnt;
     LWLockRelease(DSA_AREA_LOCK(area));
 }

References dsa_area::control, DSA_AREA_LOCK, elog(), ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Referenced by logicalrep_launcher_attach_dshmem(), and StatsShmemInit().

◆ dsa_pin_mapping()

void dsa_pin_mapping ( dsa_area * area )

Definition at line 644 of file dsa.c.

 {
     int         i;
  
     Assert(!area->mapping_pinned);
     area->mapping_pinned = true;
  
     for (i = 0; i <= area->high_segment_index; ++i)
         if (area->segment_maps[i].segment != NULL)
             dsm_pin_mapping(area->segment_maps[i].segment);
 }

References Assert(), dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::mapping_pinned, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), GetSessionDsmHandle(), logicalrep_launcher_attach_dshmem(), and pgstat_attach_shmem().

◆ dsa_release_in_place()

void dsa_release_in_place ( void * place )

Definition at line 614 of file dsa.c.

 {
     dsa_area_control *control = (dsa_area_control *) place;
     int         i;
  
     LWLockAcquire(&control->lock, LW_EXCLUSIVE);
     Assert(control->segment_header.magic ==
            (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
     Assert(control->refcnt > 0);
     if (--control->refcnt == 0)
     {
         for (i = 0; i <= control->high_segment_index; ++i)
         {
             dsm_handle  handle;
  
             handle = control->segment_handles[i];
             if (handle != DSM_HANDLE_INVALID)
                 dsm_unpin_segment(handle);
         }
     }
     LWLockRelease(&control->lock);
 }

References Assert(), DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), and dsa_on_shmem_exit_release_in_place().

◆ dsa_set_size_limit()

void dsa_set_size_limit	(	dsa_area *	area,
		size_t	limit
	)

Definition at line 1025 of file dsa.c.

 {
     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     area->control->max_total_segment_size = limit;
     LWLockRelease(DSA_AREA_LOCK(area));
 }

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::max_total_segment_size.

Referenced by StatsShmemInit().

◆ dsa_trim()

void dsa_trim ( dsa_area * area )

Definition at line 1037 of file dsa.c.

 {
     int         size_class;
  
     /*
      * Trim in reverse pool order so we get to the spans-of-spans last, just
      * in case any become entirely free while processing all the other pools.
      */
     for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
     {
         dsa_area_pool *pool = &area->control->pools[size_class];
         dsa_pointer span_pointer;
  
         if (size_class == DSA_SCLASS_SPAN_LARGE)
         {
             /* Large object frees give back segments aggressively already. */
             continue;
         }
  
         /*
          * Search fullness class 1 only.  That is where we expect to find an
          * entirely empty superblock (entirely empty superblocks in other
          * fullness classes are returned to the free page map by dsa_free).
          */
         LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
         span_pointer = pool->spans[1];
         while (DsaPointerIsValid(span_pointer))
         {
             dsa_area_span *span = dsa_get_address(area, span_pointer);
             dsa_pointer next = span->nextspan;
  
             if (span->nallocatable == span->nmax)
                 destroy_superblock(area, span_pointer);
  
             span_pointer = next;
         }
         LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
     }
 }

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

◆ dsa_unpin()

void dsa_unpin ( dsa_area * area )

Definition at line 1001 of file dsa.c.

 {
     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     Assert(area->control->refcnt > 1);
     if (!area->control->pinned)
     {
         LWLockRelease(DSA_AREA_LOCK(area));
         elog(ERROR, "dsa_area not pinned");
     }
     area->control->pinned = false;
     --area->control->refcnt;
     LWLockRelease(DSA_AREA_LOCK(area));
 }

References Assert(), dsa_area::control, DSA_AREA_LOCK, elog(), ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Macros

Typedefs

Functions

Macro Definition Documentation

◆ DSA_ALLOC_HUGE

◆ DSA_ALLOC_NO_OOM

◆ DSA_ALLOC_ZERO

◆ dsa_allocate

◆ dsa_allocate0

◆ DSA_HANDLE_INVALID

◆ dsa_pointer_atomic_compare_exchange

◆ dsa_pointer_atomic_fetch_add

◆ dsa_pointer_atomic_init

◆ dsa_pointer_atomic_read

◆ dsa_pointer_atomic_write

◆ DSA_POINTER_FORMAT

◆ DsaPointerIsValid

◆ InvalidDsaPointer

◆ SIZEOF_DSA_POINTER

Typedef Documentation

◆ dsa_area

◆ dsa_handle

◆ dsa_pointer

◆ dsa_pointer_atomic

Function Documentation

◆ dsa_allocate_extended()

◆ dsa_attach()

◆ dsa_attach_in_place()

◆ dsa_create()

◆ dsa_create_in_place()

◆ dsa_detach()

◆ dsa_dump()

◆ dsa_free()

◆ dsa_get_address()

◆ dsa_get_handle()

◆ dsa_minimum_size()

◆ dsa_on_dsm_detach_release_in_place()

◆ dsa_on_shmem_exit_release_in_place()

◆ dsa_pin()

◆ dsa_pin_mapping()

◆ dsa_release_in_place()

◆ dsa_set_size_limit()

◆ dsa_trim()

◆ dsa_unpin()