#include "storage/aio_types.h"
#include "storage/procnumber.h"

Include dependency graph for aio.h:

This graph shows which files directly or indirectly include this file:

Data Structures
union	PgAioOpData

struct	PgAioTargetInfo

struct	PgAioHandleCallbacks

Macros
#define	DEFAULT_IO_METHOD IOMETHOD_WORKER

#define	PGAIO_OP_COUNT (PGAIO_OP_WRITEV + 1)

#define	PGAIO_TID_COUNT (PGAIO_TID_SMGR + 1)

#define	PGAIO_HCB_MAX PGAIO_HCB_LOCAL_BUFFER_READV

#define	PGAIO_HANDLE_MAX_CALLBACKS 4

Typedefs
typedef enum IoMethod	IoMethod

typedef enum PgAioHandleFlags	PgAioHandleFlags

typedef enum PgAioOp	PgAioOp

typedef enum PgAioTargetID	PgAioTargetID

typedef enum PgAioHandleCallbackID	PgAioHandleCallbackID

typedef void(*	PgAioHandleCallbackStage) (PgAioHandle *ioh, uint8 cb_flags)

typedef PgAioResult(*	PgAioHandleCallbackComplete) (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_flags)

typedef void(*	PgAioHandleCallbackReport) (PgAioResult result, const PgAioTargetData *target_data, int elevel)

Enumerations
enum	IoMethod { IOMETHOD_SYNC = 0 , IOMETHOD_WORKER }

enum	PgAioHandleFlags { PGAIO_HF_REFERENCES_LOCAL = 1 << 1 , PGAIO_HF_SYNCHRONOUS = 1 << 0 , PGAIO_HF_BUFFERED = 1 << 2 }

enum	PgAioOp { PGAIO_OP_INVALID = 0 , PGAIO_OP_READV , PGAIO_OP_WRITEV }

enum	PgAioTargetID { PGAIO_TID_INVALID = 0 , PGAIO_TID_SMGR }

enum	PgAioHandleCallbackID { PGAIO_HCB_INVALID = 0 , PGAIO_HCB_MD_READV , PGAIO_HCB_SHARED_BUFFER_READV , PGAIO_HCB_LOCAL_BUFFER_READV }

Functions
	StaticAssertDecl (PGAIO_HCB_MAX<=(1<< PGAIO_RESULT_ID_BITS), "PGAIO_HCB_MAX is too big for PGAIO_RESULT_ID_BITS")

PgAioHandle *	pgaio_io_acquire (struct ResourceOwnerData resowner, PgAioReturn ret)

PgAioHandle *	pgaio_io_acquire_nb (struct ResourceOwnerData resowner, PgAioReturn ret)

void	pgaio_io_release (PgAioHandle *ioh)

void	pgaio_io_release_resowner (struct dlist_node *ioh_node, bool on_error)

void	pgaio_io_set_flag (PgAioHandle *ioh, PgAioHandleFlags flag)

int	pgaio_io_get_id (PgAioHandle *ioh)

ProcNumber	pgaio_io_get_owner (PgAioHandle *ioh)

void	pgaio_io_get_wref (PgAioHandle ioh, PgAioWaitRef iow)

int	pgaio_io_get_iovec (PgAioHandle ioh, struct iovec *iov)

PgAioOp	pgaio_io_get_op (PgAioHandle *ioh)

PgAioOpData *	pgaio_io_get_op_data (PgAioHandle *ioh)

void	pgaio_io_start_readv (PgAioHandle *ioh, int fd, int iovcnt, uint64 offset)

void	pgaio_io_start_writev (PgAioHandle *ioh, int fd, int iovcnt, uint64 offset)

void	pgaio_io_set_target (PgAioHandle *ioh, PgAioTargetID targetid)

bool	pgaio_io_has_target (PgAioHandle *ioh)

PgAioTargetData *	pgaio_io_get_target_data (PgAioHandle *ioh)

char *	pgaio_io_get_target_description (PgAioHandle *ioh)

void	pgaio_io_register_callbacks (PgAioHandle *ioh, PgAioHandleCallbackID cb_id, uint8 cb_data)

void	pgaio_io_set_handle_data_64 (PgAioHandle ioh, uint64 data, uint8 len)

void	pgaio_io_set_handle_data_32 (PgAioHandle ioh, uint32 data, uint8 len)

uint64 *	pgaio_io_get_handle_data (PgAioHandle ioh, uint8 len)

void	pgaio_wref_clear (PgAioWaitRef *iow)

bool	pgaio_wref_valid (PgAioWaitRef *iow)

int	pgaio_wref_get_id (PgAioWaitRef *iow)

void	pgaio_wref_wait (PgAioWaitRef *iow)

bool	pgaio_wref_check_done (PgAioWaitRef *iow)

void	pgaio_result_report (PgAioResult result, const PgAioTargetData *target_data, int elevel)

void	pgaio_enter_batchmode (void)

void	pgaio_exit_batchmode (void)

void	pgaio_submit_staged (void)

bool	pgaio_have_staged (void)

void	pgaio_closing_fd (int fd)

Variables
PGDLLIMPORT int	io_method

PGDLLIMPORT int	io_max_concurrency

Macro Definition Documentation

◆ DEFAULT_IO_METHOD

#define DEFAULT_IO_METHOD IOMETHOD_WORKER

Definition at line 42 of file aio.h.

◆ PGAIO_HANDLE_MAX_CALLBACKS

#define PGAIO_HANDLE_MAX_CALLBACKS 4

Definition at line 267 of file aio.h.

◆ PGAIO_HCB_MAX

#define PGAIO_HCB_MAX PGAIO_HCB_LOCAL_BUFFER_READV

Definition at line 203 of file aio.h.

◆ PGAIO_OP_COUNT

#define PGAIO_OP_COUNT (PGAIO_OP_WRITEV + 1)

Definition at line 107 of file aio.h.

◆ PGAIO_TID_COUNT

#define PGAIO_TID_COUNT (PGAIO_TID_SMGR + 1)

Definition at line 123 of file aio.h.

Typedef Documentation

◆ IoMethod

typedef enum IoMethod IoMethod

◆ PgAioHandleCallbackComplete

typedef PgAioResult(* PgAioHandleCallbackComplete) (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_flags)

Definition at line 209 of file aio.h.

◆ PgAioHandleCallbackID

typedef enum PgAioHandleCallbackID PgAioHandleCallbackID

◆ PgAioHandleCallbackReport

typedef void(* PgAioHandleCallbackReport) (PgAioResult result, const PgAioTargetData *target_data, int elevel)

Definition at line 210 of file aio.h.

◆ PgAioHandleCallbackStage

typedef void(* PgAioHandleCallbackStage) (PgAioHandle *ioh, uint8 cb_flags)

Definition at line 208 of file aio.h.

◆ PgAioHandleFlags

typedef enum PgAioHandleFlags PgAioHandleFlags

◆ PgAioOp

typedef enum PgAioOp PgAioOp

◆ PgAioTargetID

typedef enum PgAioTargetID PgAioTargetID

Enumeration Type Documentation

◆ IoMethod

enum IoMethod

Enumerator
IOMETHOD_SYNC
IOMETHOD_WORKER

Definition at line 32 of file aio.h.

{
    IOMETHOD_SYNC = 0,
    IOMETHOD_WORKER,
#ifdef IOMETHOD_IO_URING_ENABLED
    IOMETHOD_IO_URING,
#endif
} IoMethod;

◆ PgAioHandleCallbackID

enum PgAioHandleCallbackID

Enumerator
PGAIO_HCB_INVALID
PGAIO_HCB_MD_READV
PGAIO_HCB_SHARED_BUFFER_READV
PGAIO_HCB_LOCAL_BUFFER_READV

Definition at line 192 of file aio.h.

{
    PGAIO_HCB_INVALID = 0,
 
    PGAIO_HCB_MD_READV,
 
    PGAIO_HCB_SHARED_BUFFER_READV,
 
    PGAIO_HCB_LOCAL_BUFFER_READV,
} PgAioHandleCallbackID;

◆ PgAioHandleFlags

enum PgAioHandleFlags

Enumerator
PGAIO_HF_REFERENCES_LOCAL
PGAIO_HF_SYNCHRONOUS
PGAIO_HF_BUFFERED

Definition at line 48 of file aio.h.

{
    /*
     * The IO references backend local memory.
     *
     * This needs to be set on an IO whenever the IO references process-local
     * memory. Some IO methods do not support executing IO that references
     * process local memory and thus need to fall back to executing IO
     * synchronously for IOs with this flag set.
     *
     * Required for correctness.
     */
    PGAIO_HF_REFERENCES_LOCAL = 1 << 1,
 
    /*
     * Hint that IO will be executed synchronously.
     *
     * This can make it a bit cheaper to execute synchronous IO via the AIO
     * interface, to avoid needing an AIO and non-AIO version of code.
     *
     * Advantageous to set, if applicable, but not required for correctness.
     */
    PGAIO_HF_SYNCHRONOUS = 1 << 0,
 
    /*
     * IO is using buffered IO, used to control heuristic in some IO methods.
     *
     * Advantageous to set, if applicable, but not required for correctness.
     */
    PGAIO_HF_BUFFERED = 1 << 2,
} PgAioHandleFlags;

◆ PgAioOp

enum PgAioOp

Enumerator
PGAIO_OP_INVALID
PGAIO_OP_READV
PGAIO_OP_WRITEV

Definition at line 87 of file aio.h.

{
    /* intentionally the zero value, to help catch zeroed memory etc */
    PGAIO_OP_INVALID = 0,
 
    PGAIO_OP_READV,
    PGAIO_OP_WRITEV,
 
} PgAioOp;

◆ PgAioTargetID

enum PgAioTargetID

Enumerator
PGAIO_TID_INVALID
PGAIO_TID_SMGR

Definition at line 116 of file aio.h.

{
    /* intentionally the zero value, to help catch zeroed memory etc */
    PGAIO_TID_INVALID = 0,
    PGAIO_TID_SMGR,
} PgAioTargetID;

Function Documentation

◆ pgaio_closing_fd()

void pgaio_closing_fd ( int fd )

Definition at line 1117 of file aio.c.

{
    /*
     * Might be called before AIO is initialized or in a subprocess that
     * doesn't use AIO.
     */
    if (!pgaio_my_backend)
        return;
 
    /*
     * For now just submit all staged IOs - we could be more selective, but
     * it's probably not worth it.
     */
    pgaio_submit_staged();
 
    /*
     * If requested by the IO method, wait for all IOs that use the
     * to-be-closed FD.
     */
    if (pgaio_method_ops->wait_on_fd_before_close)
    {
        /*
         * As waiting for one IO to complete may complete multiple IOs, we
         * can't just use a mutable list iterator. The maximum number of
         * in-flight IOs is fairly small, so just restart the loop after
         * waiting for an IO.
         */
        while (!dclist_is_empty(&pgaio_my_backend->in_flight_ios))
        {
            dlist_iter  iter;
            PgAioHandle *ioh = NULL;
 
            dclist_foreach(iter, &pgaio_my_backend->in_flight_ios)
            {
                ioh = dclist_container(PgAioHandle, node, iter.cur);
 
                if (pgaio_io_uses_fd(ioh, fd))
                    break;
                else
                    ioh = NULL;
            }
 
            if (!ioh)
                break;
 
            /* see comment in pgaio_io_wait_for_free() about raciness */
            pgaio_io_wait(ioh, ioh->generation);
        }
    }
}

References dlist_iter::cur, dclist_container, dclist_foreach, dclist_is_empty(), fd(), PgAioHandle::generation, PgAioBackend::in_flight_ios, pgaio_io_uses_fd(), pgaio_io_wait(), pgaio_method_ops, pgaio_my_backend, pgaio_submit_staged(), and IoMethodOps::wait_on_fd_before_close.

Referenced by CloseTransientFile(), FileClose(), FreeDesc(), and LruDelete().

◆ pgaio_enter_batchmode()

void pgaio_enter_batchmode ( void )

Definition at line 978 of file aio.c.

{
    if (pgaio_my_backend->in_batchmode)
        elog(ERROR, "starting batch while batch already in progress");
    pgaio_my_backend->in_batchmode = true;
}

References elog, ERROR, PgAioBackend::in_batchmode, and pgaio_my_backend.

Referenced by batch_start(), read_rel_block_ll(), and read_stream_look_ahead().

◆ pgaio_exit_batchmode()

void pgaio_exit_batchmode ( void )

Definition at line 989 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode);
 
    pgaio_submit_staged();
    pgaio_my_backend->in_batchmode = false;
}

References Assert(), PgAioBackend::in_batchmode, pgaio_my_backend, and pgaio_submit_staged().

Referenced by batch_end(), read_rel_block_ll(), and read_stream_look_ahead().

◆ pgaio_have_staged()

bool pgaio_have_staged ( void )

Definition at line 1004 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode ||
           pgaio_my_backend->num_staged_ios == 0);
    return pgaio_my_backend->num_staged_ios > 0;
}

References Assert(), PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, and pgaio_my_backend.

Referenced by ReadBuffersCanStartIO().

◆ pgaio_io_acquire()

PgAioHandle * pgaio_io_acquire	(	struct ResourceOwnerData *	resowner,
		PgAioReturn *	ret
	)

Definition at line 173 of file aio.c.

{
    PgAioHandle *h;
 
    while (true)
    {
        h = pgaio_io_acquire_nb(resowner, ret);
 
        if (h != NULL)
            return h;
 
        /*
         * Evidently all handles by this backend are in use. Just wait for
         * some to complete.
         */
        pgaio_io_wait_for_free();
    }
}

References pgaio_io_acquire_nb(), and pgaio_io_wait_for_free().

Referenced by AsyncReadBuffers(), handle_get(), handle_get_and_error(), handle_get_release(), handle_get_twice(), and read_rel_block_ll().

◆ pgaio_io_acquire_nb()

PgAioHandle * pgaio_io_acquire_nb	(	struct ResourceOwnerData *	resowner,
		PgAioReturn *	ret
	)

Definition at line 199 of file aio.c.

{
    if (pgaio_my_backend->num_staged_ios >= PGAIO_SUBMIT_BATCH_SIZE)
    {
        Assert(pgaio_my_backend->num_staged_ios == PGAIO_SUBMIT_BATCH_SIZE);
        pgaio_submit_staged();
    }
 
    if (pgaio_my_backend->handed_out_io)
        elog(ERROR, "API violation: Only one IO can be handed out");
 
    if (!dclist_is_empty(&pgaio_my_backend->idle_ios))
    {
        dlist_node *ion = dclist_pop_head_node(&pgaio_my_backend->idle_ios);
        PgAioHandle *ioh = dclist_container(PgAioHandle, node, ion);
 
        Assert(ioh->state == PGAIO_HS_IDLE);
        Assert(ioh->owner_procno == MyProcNumber);
 
        pgaio_io_update_state(ioh, PGAIO_HS_HANDED_OUT);
        pgaio_my_backend->handed_out_io = ioh;
 
        if (resowner)
            pgaio_io_resowner_register(ioh);
 
        if (ret)
        {
            ioh->report_return = ret;
            ret->result.status = PGAIO_RS_UNKNOWN;
        }
 
        return ioh;
    }
 
    return NULL;
}

References Assert(), dclist_container, dclist_is_empty(), dclist_pop_head_node(), elog, ERROR, PgAioBackend::handed_out_io, PgAioBackend::idle_ios, MyProcNumber, PgAioBackend::num_staged_ios, PgAioHandle::owner_procno, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, pgaio_io_resowner_register(), pgaio_io_update_state(), pgaio_my_backend, PGAIO_RS_UNKNOWN, PGAIO_SUBMIT_BATCH_SIZE, pgaio_submit_staged(), PgAioHandle::report_return, PgAioReturn::result, PgAioHandle::state, and PgAioResult::status.

Referenced by AsyncReadBuffers(), and pgaio_io_acquire().

◆ pgaio_io_get_handle_data()

uint64 * pgaio_io_get_handle_data	(	PgAioHandle *	ioh,
		uint8 *	len
	)

Definition at line 154 of file aio_callback.c.

{
    Assert(ioh->handle_data_len > 0);
 
    *len = ioh->handle_data_len;
 
    return &pgaio_ctl->handle_data[ioh->iovec_off];
}

References Assert(), PgAioCtl::handle_data, PgAioHandle::handle_data_len, PgAioHandle::iovec_off, len, and pgaio_ctl.

Referenced by buffer_readv_complete(), and buffer_stage_common().

◆ pgaio_io_get_id()

int pgaio_io_get_id ( PgAioHandle * ioh )

Definition at line 330 of file aio.c.

{
    Assert(ioh >= pgaio_ctl->io_handles &&
           ioh < (pgaio_ctl->io_handles + pgaio_ctl->io_handle_count));
    return ioh - pgaio_ctl->io_handles;
}

References Assert(), PgAioCtl::io_handle_count, PgAioCtl::io_handles, and pgaio_ctl.

Referenced by pg_get_aios(), pgaio_io_wait_for_free(), and pgaio_worker_submission_queue_insert().

◆ pgaio_io_get_iovec()

int pgaio_io_get_iovec	(	PgAioHandle *	ioh,
		struct iovec **	iov
	)

Definition at line 42 of file aio_io.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
 
    *iov = &pgaio_ctl->iovecs[ioh->iovec_off];
 
    return PG_IOV_MAX;
}

References Assert(), PgAioHandle::iovec_off, PgAioCtl::iovecs, PG_IOV_MAX, pgaio_ctl, PGAIO_HS_HANDED_OUT, and PgAioHandle::state.

Referenced by mdstartreadv().

◆ pgaio_io_get_op()

PgAioOp pgaio_io_get_op ( PgAioHandle * ioh )

Definition at line 52 of file aio_io.c.

{
    return ioh->op;
}

References PgAioHandle::op.

Referenced by smgr_aio_reopen().

◆ pgaio_io_get_op_data()

PgAioOpData * pgaio_io_get_op_data ( PgAioHandle * ioh )

Definition at line 58 of file aio_io.c.

{
    return &ioh->op_data;
}

References PgAioHandle::op_data.

Referenced by smgr_aio_reopen().

◆ pgaio_io_get_owner()

ProcNumber pgaio_io_get_owner ( PgAioHandle * ioh )

Definition at line 343 of file aio.c.

{
    return ioh->owner_procno;
}

References PgAioHandle::owner_procno.

Referenced by buffer_readv_complete(), and smgr_aio_reopen().

◆ pgaio_io_get_target_data()

PgAioTargetData * pgaio_io_get_target_data ( PgAioHandle * ioh )

Definition at line 72 of file aio_target.c.

{
    return &ioh->target_data;
}

References PgAioHandle::target_data.

Referenced by buffer_readv_complete(), md_readv_complete(), pgaio_io_set_target_smgr(), shared_buffer_readv_complete_local(), and smgr_aio_reopen().

◆ pgaio_io_get_target_description()

char * pgaio_io_get_target_description ( PgAioHandle * ioh )

Definition at line 83 of file aio_target.c.

{
    return pgaio_target_info[ioh->target]->describe_identity(&ioh->target_data);
}

References PgAioTargetInfo::describe_identity, pgaio_target_info, PgAioHandle::target, and PgAioHandle::target_data.

Referenced by pg_get_aios().

◆ pgaio_io_get_wref()

void pgaio_io_get_wref	(	PgAioHandle *	ioh,
		PgAioWaitRef *	iow
	)

Definition at line 354 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT ||
           ioh->state == PGAIO_HS_DEFINED ||
           ioh->state == PGAIO_HS_STAGED);
    Assert(ioh->generation != 0);
 
    iow->aio_index = ioh - pgaio_ctl->io_handles;
    iow->generation_upper = (uint32) (ioh->generation >> 32);
    iow->generation_lower = (uint32) ioh->generation;
}

References PgAioWaitRef::aio_index, Assert(), PgAioHandle::generation, PgAioWaitRef::generation_lower, PgAioWaitRef::generation_upper, PgAioCtl::io_handles, pgaio_ctl, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_STAGED, and PgAioHandle::state.

Referenced by AsyncReadBuffers(), buffer_stage_common(), and read_rel_block_ll().

◆ pgaio_io_has_target()

bool pgaio_io_has_target ( PgAioHandle * ioh )

Definition at line 40 of file aio_target.c.

{
    return ioh->target != PGAIO_TID_INVALID;
}

References PGAIO_TID_INVALID, and PgAioHandle::target.

Referenced by pgaio_io_before_start(), and pgaio_io_stage().

◆ pgaio_io_register_callbacks()

void pgaio_io_register_callbacks	(	PgAioHandle *	ioh,
		PgAioHandleCallbackID	cb_id,
		uint8	cb_data
	)

Definition at line 86 of file aio_callback.c.

{
    const PgAioHandleCallbacksEntry *ce = &aio_handle_cbs[cb_id];
 
    Assert(cb_id <= PGAIO_HCB_MAX);
    if (cb_id >= lengthof(aio_handle_cbs))
        elog(ERROR, "callback %d is out of range", cb_id);
    if (aio_handle_cbs[cb_id].cb->complete_shared == NULL &&
        aio_handle_cbs[cb_id].cb->complete_local == NULL)
        elog(ERROR, "callback %d does not have a completion callback", cb_id);
    if (ioh->num_callbacks >= PGAIO_HANDLE_MAX_CALLBACKS)
        elog(PANIC, "too many callbacks, the max is %d",
             PGAIO_HANDLE_MAX_CALLBACKS);
    ioh->callbacks[ioh->num_callbacks] = cb_id;
    ioh->callbacks_data[ioh->num_callbacks] = cb_data;
 
    pgaio_debug_io(DEBUG3, ioh,
                   "adding cb #%d, id %d/%s",
                   ioh->num_callbacks + 1,
                   cb_id, ce->name);
 
    ioh->num_callbacks++;
}

References aio_handle_cbs, Assert(), PgAioHandle::callbacks, PgAioHandle::callbacks_data, PgAioHandleCallbacksEntry::cb, PgAioHandleCallbacks::complete_local, DEBUG3, elog, ERROR, lengthof, PgAioHandleCallbacksEntry::name, PgAioHandle::num_callbacks, PANIC, pgaio_debug_io, PGAIO_HANDLE_MAX_CALLBACKS, and PGAIO_HCB_MAX.

Referenced by AsyncReadBuffers(), mdstartreadv(), and read_rel_block_ll().

◆ pgaio_io_release()

void pgaio_io_release ( PgAioHandle * ioh )

Definition at line 242 of file aio.c.

{
    if (ioh == pgaio_my_backend->handed_out_io)
    {
        Assert(ioh->state == PGAIO_HS_HANDED_OUT);
        Assert(ioh->resowner);
 
        pgaio_my_backend->handed_out_io = NULL;
        pgaio_io_reclaim(ioh);
    }
    else
    {
        elog(ERROR, "release in unexpected state");
    }
}

References Assert(), elog, ERROR, PgAioBackend::handed_out_io, PGAIO_HS_HANDED_OUT, pgaio_io_reclaim(), pgaio_my_backend, PgAioHandle::resowner, and PgAioHandle::state.

Referenced by AsyncReadBuffers(), handle_get_release(), and handle_release_last().

◆ pgaio_io_release_resowner()

void pgaio_io_release_resowner	(	struct dlist_node *	ioh_node,
		bool	on_error
	)

Definition at line 262 of file aio.c.

{
    PgAioHandle *ioh = dlist_container(PgAioHandle, resowner_node, ioh_node);
 
    Assert(ioh->resowner);
 
    ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node);
    ioh->resowner = NULL;
 
    switch (ioh->state)
    {
        case PGAIO_HS_IDLE:
            elog(ERROR, "unexpected");
            break;
        case PGAIO_HS_HANDED_OUT:
            Assert(ioh == pgaio_my_backend->handed_out_io || pgaio_my_backend->handed_out_io == NULL);
 
            if (ioh == pgaio_my_backend->handed_out_io)
            {
                pgaio_my_backend->handed_out_io = NULL;
                if (!on_error)
                    elog(WARNING, "leaked AIO handle");
            }
 
            pgaio_io_reclaim(ioh);
            break;
        case PGAIO_HS_DEFINED:
        case PGAIO_HS_STAGED:
            if (!on_error)
                elog(WARNING, "AIO handle was not submitted");
            pgaio_submit_staged();
            break;
        case PGAIO_HS_SUBMITTED:
        case PGAIO_HS_COMPLETED_IO:
        case PGAIO_HS_COMPLETED_SHARED:
        case PGAIO_HS_COMPLETED_LOCAL:
            /* this is expected to happen */
            break;
    }
 
    /*
     * Need to unregister the reporting of the IO's result, the memory it's
     * referencing likely has gone away.
     */
    if (ioh->report_return)
        ioh->report_return = NULL;
}

References Assert(), dlist_container, elog, ERROR, PgAioBackend::handed_out_io, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, PGAIO_HS_STAGED, PGAIO_HS_SUBMITTED, pgaio_io_reclaim(), pgaio_my_backend, pgaio_submit_staged(), PgAioHandle::report_return, ResourceOwnerForgetAioHandle(), PgAioHandle::resowner, PgAioHandle::resowner_node, PgAioHandle::state, and WARNING.

Referenced by ResourceOwnerReleaseInternal().

◆ pgaio_io_set_flag()

void pgaio_io_set_flag	(	PgAioHandle *	ioh,
		PgAioHandleFlags	flag
	)

Definition at line 318 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
 
    ioh->flags |= flag;
}

References Assert(), flag(), PgAioHandle::flags, PGAIO_HS_HANDED_OUT, and PgAioHandle::state.

Referenced by AsyncReadBuffers(), mdstartreadv(), and read_rel_block_ll().

◆ pgaio_io_set_handle_data_32()

void pgaio_io_set_handle_data_32	(	PgAioHandle *	ioh,
		uint32 *	data,
		uint8	len
	)

Definition at line 139 of file aio_callback.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->handle_data_len == 0);
    Assert(len <= PG_IOV_MAX);
 
    for (int i = 0; i < len; i++)
        pgaio_ctl->handle_data[ioh->iovec_off + i] = data[i];
    ioh->handle_data_len = len;
}

References Assert(), data, PgAioCtl::handle_data, PgAioHandle::handle_data_len, i, PgAioHandle::iovec_off, len, PG_IOV_MAX, pgaio_ctl, PGAIO_HS_HANDED_OUT, and PgAioHandle::state.

Referenced by AsyncReadBuffers(), and read_rel_block_ll().

◆ pgaio_io_set_handle_data_64()

void pgaio_io_set_handle_data_64	(	PgAioHandle *	ioh,
		uint64 *	data,
		uint8	len
	)

Definition at line 122 of file aio_callback.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->handle_data_len == 0);
    Assert(len <= PG_IOV_MAX);
 
    for (int i = 0; i < len; i++)
        pgaio_ctl->handle_data[ioh->iovec_off + i] = data[i];
    ioh->handle_data_len = len;
}

References Assert(), data, PgAioCtl::handle_data, PgAioHandle::handle_data_len, i, PgAioHandle::iovec_off, len, PG_IOV_MAX, pgaio_ctl, PGAIO_HS_HANDED_OUT, and PgAioHandle::state.

◆ pgaio_io_set_target()

void pgaio_io_set_target	(	PgAioHandle *	ioh,
		PgAioTargetID	targetid
	)

Definition at line 63 of file aio_target.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->target == PGAIO_TID_INVALID);
 
    ioh->target = targetid;
}

References Assert(), PGAIO_HS_HANDED_OUT, PGAIO_TID_INVALID, PgAioHandle::state, and PgAioHandle::target.

Referenced by pgaio_io_set_target_smgr().

◆ pgaio_io_start_readv()

void pgaio_io_start_readv	(	PgAioHandle *	ioh,
		int	fd,
		int	iovcnt,
		uint64	offset
	)

Definition at line 78 of file aio_io.c.

{
    pgaio_io_before_start(ioh);
 
    ioh->op_data.read.fd = fd;
    ioh->op_data.read.offset = offset;
    ioh->op_data.read.iov_length = iovcnt;
 
    pgaio_io_stage(ioh, PGAIO_OP_READV);
}

References PgAioOpData::fd, fd(), PgAioOpData::iov_length, PgAioOpData::offset, PgAioHandle::op_data, pgaio_io_before_start(), pgaio_io_stage(), PGAIO_OP_READV, and PgAioOpData::read.

Referenced by FileStartReadV().

◆ pgaio_io_start_writev()

void pgaio_io_start_writev	(	PgAioHandle *	ioh,
		int	fd,
		int	iovcnt,
		uint64	offset
	)

Definition at line 91 of file aio_io.c.

{
    pgaio_io_before_start(ioh);
 
    ioh->op_data.write.fd = fd;
    ioh->op_data.write.offset = offset;
    ioh->op_data.write.iov_length = iovcnt;
 
    pgaio_io_stage(ioh, PGAIO_OP_WRITEV);
}

References PgAioOpData::fd, fd(), PgAioOpData::iov_length, PgAioOpData::offset, PgAioHandle::op_data, pgaio_io_before_start(), pgaio_io_stage(), PGAIO_OP_WRITEV, and PgAioOpData::write.

◆ pgaio_result_report()

void pgaio_result_report	(	PgAioResult	result,
		const PgAioTargetData *	target_data,
		int	elevel
	)

Definition at line 171 of file aio_callback.c.

{
    PgAioHandleCallbackID cb_id = result.id;
    const PgAioHandleCallbacksEntry *ce = &aio_handle_cbs[cb_id];
 
    Assert(result.status != PGAIO_RS_UNKNOWN);
    Assert(result.status != PGAIO_RS_OK);
 
    if (ce->cb->report == NULL)
        elog(ERROR, "callback %d/%s does not have report callback",
             result.id, ce->name);
 
    ce->cb->report(result, target_data, elevel);
}

References aio_handle_cbs, Assert(), PgAioHandleCallbacksEntry::cb, elog, ERROR, PgAioResult::id, PgAioHandleCallbacksEntry::name, PGAIO_RS_OK, PGAIO_RS_UNKNOWN, PgAioHandleCallbacks::report, and PgAioResult::status.

Referenced by buffer_readv_complete(), buffer_readv_complete_one(), md_readv_complete(), ProcessReadBuffersResult(), and read_rel_block_ll().

◆ pgaio_submit_staged()

void pgaio_submit_staged ( void )

Definition at line 1020 of file aio.c.

{
    int         total_submitted = 0;
    int         did_submit;
 
    if (pgaio_my_backend->num_staged_ios == 0)
        return;
 
 
    START_CRIT_SECTION();
 
    did_submit = pgaio_method_ops->submit(pgaio_my_backend->num_staged_ios,
                                          pgaio_my_backend->staged_ios);
 
    END_CRIT_SECTION();
 
    total_submitted += did_submit;
 
    Assert(total_submitted == did_submit);
 
    pgaio_my_backend->num_staged_ios = 0;
 
    pgaio_debug(DEBUG4,
                "aio: submitted %d IOs",
                total_submitted);
}

References Assert(), DEBUG4, END_CRIT_SECTION, PgAioBackend::num_staged_ios, pgaio_debug, pgaio_method_ops, pgaio_my_backend, PgAioBackend::staged_ios, START_CRIT_SECTION, and IoMethodOps::submit.

Referenced by AsyncReadBuffers(), pgaio_closing_fd(), pgaio_error_cleanup(), pgaio_exit_batchmode(), pgaio_io_acquire_nb(), pgaio_io_release_resowner(), pgaio_io_stage(), pgaio_io_wait_for_free(), and ReadBuffersCanStartIO().

◆ pgaio_wref_check_done()

bool pgaio_wref_check_done ( PgAioWaitRef * iow )

Definition at line 907 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandleState state;
    bool        am_owner;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    if (pgaio_io_was_recycled(ioh, ref_generation, &state))
        return true;
 
    if (state == PGAIO_HS_IDLE)
        return true;
 
    am_owner = ioh->owner_procno == MyProcNumber;
 
    if (state == PGAIO_HS_COMPLETED_SHARED ||
        state == PGAIO_HS_COMPLETED_LOCAL)
    {
        if (am_owner)
            pgaio_io_reclaim(ioh);
        return true;
    }
 
    /*
     * XXX: It likely would be worth checking in with the io method, to give
     * the IO method a chance to check if there are completion events queued.
     */
 
    return false;
}

References MyProcNumber, PgAioHandle::owner_procno, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_IDLE, pgaio_io_from_wref(), pgaio_io_reclaim(), and pgaio_io_was_recycled().

Referenced by WaitReadBuffers().

◆ pgaio_wref_clear()

void pgaio_wref_clear ( PgAioWaitRef * iow )

Definition at line 866 of file aio.c.

{
    iow->aio_index = PG_UINT32_MAX;
}

References PgAioWaitRef::aio_index, and PG_UINT32_MAX.

Referenced by AsyncReadBuffers(), BufferManagerShmemInit(), InitLocalBuffers(), StartReadBuffersImpl(), TerminateBufferIO(), and TerminateLocalBufferIO().

◆ pgaio_wref_get_id()

int pgaio_wref_get_id ( PgAioWaitRef * iow )

Definition at line 882 of file aio.c.

{
    Assert(pgaio_wref_valid(iow));
    return iow->aio_index;
}

References PgAioWaitRef::aio_index, Assert(), and pgaio_wref_valid().

◆ pgaio_wref_valid()

bool pgaio_wref_valid ( PgAioWaitRef * iow )

Definition at line 873 of file aio.c.

{
    return iow->aio_index != PG_UINT32_MAX;
}

References PgAioWaitRef::aio_index, and PG_UINT32_MAX.

Referenced by InvalidateLocalBuffer(), pgaio_wref_get_id(), ProcessReadBuffersResult(), StartLocalBufferIO(), WaitIO(), and WaitReadBuffers().

◆ pgaio_wref_wait()

void pgaio_wref_wait ( PgAioWaitRef * iow )

Definition at line 893 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    pgaio_io_wait(ioh, ref_generation);
}

References pgaio_io_from_wref(), and pgaio_io_wait().

Referenced by InvalidateLocalBuffer(), read_rel_block_ll(), StartLocalBufferIO(), WaitIO(), and WaitReadBuffers().

◆ StaticAssertDecl()

StaticAssertDecl	(	PGAIO_HCB_MAX<=	1<< PGAIO_RESULT_ID_BITS,
		"PGAIO_HCB_MAX is too big for PGAIO_RESULT_ID_BITS"
	)

Variable Documentation

◆ io_max_concurrency

PGDLLIMPORT int io_max_concurrency

extern

Definition at line 78 of file aio.c.

Referenced by AioHandleDataShmemSize(), AioHandleIOVShmemSize(), AioHandleShmemSize(), AioShmemInit(), AioShmemSize(), and pgaio_io_wait_for_free().

◆ io_method

PGDLLIMPORT int io_method

extern

Definition at line 77 of file aio.c.

Referenced by pgaio_workers_enabled(), read_stream_begin_impl(), StartReadBuffersImpl(), and WaitReadBuffers().

Data Structures

Macros

Typedefs

Enumerations

Functions

Variables

Macro Definition Documentation

◆ DEFAULT_IO_METHOD

◆ PGAIO_HANDLE_MAX_CALLBACKS

◆ PGAIO_HCB_MAX

◆ PGAIO_OP_COUNT

◆ PGAIO_TID_COUNT

Typedef Documentation

◆ IoMethod

◆ PgAioHandleCallbackComplete

◆ PgAioHandleCallbackID

◆ PgAioHandleCallbackReport

◆ PgAioHandleCallbackStage

◆ PgAioHandleFlags

◆ PgAioOp

◆ PgAioTargetID

Enumeration Type Documentation

◆ IoMethod

◆ PgAioHandleCallbackID

◆ PgAioHandleFlags

◆ PgAioOp

◆ PgAioTargetID

Function Documentation

◆ pgaio_closing_fd()

◆ pgaio_enter_batchmode()

◆ pgaio_exit_batchmode()

◆ pgaio_have_staged()

◆ pgaio_io_acquire()

◆ pgaio_io_acquire_nb()

◆ pgaio_io_get_handle_data()

◆ pgaio_io_get_id()

◆ pgaio_io_get_iovec()

◆ pgaio_io_get_op()

◆ pgaio_io_get_op_data()

◆ pgaio_io_get_owner()

◆ pgaio_io_get_target_data()

◆ pgaio_io_get_target_description()

◆ pgaio_io_get_wref()

◆ pgaio_io_has_target()

◆ pgaio_io_register_callbacks()

◆ pgaio_io_release()

◆ pgaio_io_release_resowner()

◆ pgaio_io_set_flag()

◆ pgaio_io_set_handle_data_32()

◆ pgaio_io_set_handle_data_64()

◆ pgaio_io_set_target()

◆ pgaio_io_start_readv()

◆ pgaio_io_start_writev()

◆ pgaio_result_report()

◆ pgaio_submit_staged()

◆ pgaio_wref_check_done()

◆ pgaio_wref_clear()

◆ pgaio_wref_get_id()

◆ pgaio_wref_valid()

◆ pgaio_wref_wait()

◆ StaticAssertDecl()

Variable Documentation

◆ io_max_concurrency

◆ io_method