aio.h File Reference

#include "storage/aio_types.h"
#include "storage/procnumber.h"

Include dependency graph for aio.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
union	PgAioOpData
struct	PgAioTargetInfo
struct	PgAioHandleCallbacks

Macros
#define	DEFAULT_IO_METHOD   IOMETHOD_WORKER
#define	PGAIO_OP_COUNT   (PGAIO_OP_WRITEV + 1)
#define	PGAIO_TID_COUNT   (PGAIO_TID_SMGR + 1)
#define	PGAIO_HCB_MAX   PGAIO_HCB_LOCAL_BUFFER_READV
#define	PGAIO_HANDLE_MAX_CALLBACKS   4

Typedefs
typedef enum IoMethod	IoMethod
typedef enum PgAioHandleFlags	PgAioHandleFlags
typedef enum PgAioOp	PgAioOp
typedef enum PgAioTargetID	PgAioTargetID
typedef enum PgAioHandleCallbackID	PgAioHandleCallbackID
typedef void(*	PgAioHandleCallbackStage) (PgAioHandle *ioh, uint8 cb_flags)
typedef PgAioResult(*	PgAioHandleCallbackComplete) (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_flags)
typedef void(*	PgAioHandleCallbackReport) (PgAioResult result, const PgAioTargetData *target_data, int elevel)

Enumerations
enum	IoMethod { IOMETHOD_SYNC = 0 , IOMETHOD_WORKER }
enum	PgAioHandleFlags { PGAIO_HF_REFERENCES_LOCAL = 1 << 1 , PGAIO_HF_SYNCHRONOUS = 1 << 0 , PGAIO_HF_BUFFERED = 1 << 2 }
enum	PgAioOp { PGAIO_OP_INVALID = 0 , PGAIO_OP_READV , PGAIO_OP_WRITEV }
enum	PgAioTargetID { PGAIO_TID_INVALID = 0 , PGAIO_TID_SMGR }
enum	PgAioHandleCallbackID { PGAIO_HCB_INVALID = 0 , PGAIO_HCB_MD_READV , PGAIO_HCB_SHARED_BUFFER_READV , PGAIO_HCB_LOCAL_BUFFER_READV }

Functions
	StaticAssertDecl (PGAIO_HCB_MAX<(1<< PGAIO_RESULT_ID_BITS), "PGAIO_HCB_MAX is too big for PGAIO_RESULT_ID_BITS")
PgAioHandle *	pgaio_io_acquire (struct ResourceOwnerData *resowner, PgAioReturn *ret)
PgAioHandle *	pgaio_io_acquire_nb (struct ResourceOwnerData *resowner, PgAioReturn *ret)
void	pgaio_io_release (PgAioHandle *ioh)
void	pgaio_io_release_resowner (struct dlist_node *ioh_node, bool on_error)
void	pgaio_io_set_flag (PgAioHandle *ioh, PgAioHandleFlags flag)
int	pgaio_io_get_id (PgAioHandle *ioh)
ProcNumber	pgaio_io_get_owner (PgAioHandle *ioh)
void	pgaio_io_get_wref (PgAioHandle *ioh, PgAioWaitRef *iow)
int	pgaio_io_get_iovec (PgAioHandle *ioh, struct iovec **iov)
PgAioOp	pgaio_io_get_op (PgAioHandle *ioh)
PgAioOpData *	pgaio_io_get_op_data (PgAioHandle *ioh)
void	pgaio_io_start_readv (PgAioHandle *ioh, int fd, int iovcnt, uint64 offset)
void	pgaio_io_start_writev (PgAioHandle *ioh, int fd, int iovcnt, uint64 offset)
void	pgaio_io_set_target (PgAioHandle *ioh, PgAioTargetID targetid)
bool	pgaio_io_has_target (PgAioHandle *ioh)
PgAioTargetData *	pgaio_io_get_target_data (PgAioHandle *ioh)
char *	pgaio_io_get_target_description (PgAioHandle *ioh)
void	pgaio_io_register_callbacks (PgAioHandle *ioh, PgAioHandleCallbackID cb_id, uint8 cb_data)
void	pgaio_io_set_handle_data_64 (PgAioHandle *ioh, uint64 *data, uint8 len)
void	pgaio_io_set_handle_data_32 (PgAioHandle *ioh, uint32 *data, uint8 len)
uint64 *	pgaio_io_get_handle_data (PgAioHandle *ioh, uint8 *len)
void	pgaio_wref_clear (PgAioWaitRef *iow)
bool	pgaio_wref_valid (PgAioWaitRef *iow)
int	pgaio_wref_get_id (PgAioWaitRef *iow)
void	pgaio_wref_wait (PgAioWaitRef *iow)
bool	pgaio_wref_check_done (PgAioWaitRef *iow)
void	pgaio_result_report (PgAioResult result, const PgAioTargetData *target_data, int elevel)
void	pgaio_enter_batchmode (void)
void	pgaio_exit_batchmode (void)
void	pgaio_submit_staged (void)
bool	pgaio_have_staged (void)
void	pgaio_closing_fd (int fd)

Variables
PGDLLIMPORT int	io_method
PGDLLIMPORT int	io_max_concurrency

Macro Definition Documentation

DEFAULT_IO_METHOD

#define DEFAULT_IO_METHOD   IOMETHOD_WORKER

Definition at line 42 of file aio.h.

PGAIO_HANDLE_MAX_CALLBACKS

#define PGAIO_HANDLE_MAX_CALLBACKS   4

Definition at line 267 of file aio.h.

PGAIO_HCB_MAX

#define PGAIO_HCB_MAX   PGAIO_HCB_LOCAL_BUFFER_READV

Definition at line 203 of file aio.h.

PGAIO_OP_COUNT

#define PGAIO_OP_COUNT   (PGAIO_OP_WRITEV + 1)

Definition at line 107 of file aio.h.

PGAIO_TID_COUNT

#define PGAIO_TID_COUNT   (PGAIO_TID_SMGR + 1)

Definition at line 123 of file aio.h.

Typedef Documentation

IoMethod

typedef enum IoMethod IoMethod

PgAioHandleCallbackComplete

typedef PgAioResult(* PgAioHandleCallbackComplete) (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_flags)

Definition at line 209 of file aio.h.

PgAioHandleCallbackID

typedef enum PgAioHandleCallbackID PgAioHandleCallbackID

PgAioHandleCallbackReport

typedef void(* PgAioHandleCallbackReport) (PgAioResult result, const PgAioTargetData *target_data, int elevel)

Definition at line 210 of file aio.h.

PgAioHandleCallbackStage

typedef void(* PgAioHandleCallbackStage) (PgAioHandle *ioh, uint8 cb_flags)

Definition at line 208 of file aio.h.

PgAioHandleFlags

typedef enum PgAioHandleFlags PgAioHandleFlags

PgAioOp

typedef enum PgAioOp PgAioOp

PgAioTargetID

typedef enum PgAioTargetID PgAioTargetID

Enumeration Type Documentation

IoMethod

enum IoMethod

Enumerator
IOMETHOD_SYNC
IOMETHOD_WORKER

Definition at line 32 of file aio.h.

{
    IOMETHOD_SYNC = 0,
    IOMETHOD_WORKER,
#ifdef IOMETHOD_IO_URING_ENABLED
    IOMETHOD_IO_URING,
#endif
}           IoMethod;

PgAioHandleCallbackID

enum PgAioHandleCallbackID

Enumerator
PGAIO_HCB_INVALID
PGAIO_HCB_MD_READV
PGAIO_HCB_SHARED_BUFFER_READV
PGAIO_HCB_LOCAL_BUFFER_READV

Definition at line 192 of file aio.h.

{
    PGAIO_HCB_INVALID = 0,
 
    PGAIO_HCB_MD_READV,
 
    PGAIO_HCB_SHARED_BUFFER_READV,
 
    PGAIO_HCB_LOCAL_BUFFER_READV,
} PgAioHandleCallbackID;

PgAioHandleFlags

enum PgAioHandleFlags

Enumerator
PGAIO_HF_REFERENCES_LOCAL
PGAIO_HF_SYNCHRONOUS
PGAIO_HF_BUFFERED

Definition at line 48 of file aio.h.

{
    /*
     * The IO references backend local memory.
     *
     * This needs to be set on an IO whenever the IO references process-local
     * memory. Some IO methods do not support executing IO that references
     * process local memory and thus need to fall back to executing IO
     * synchronously for IOs with this flag set.
     *
     * Required for correctness.
     */
    PGAIO_HF_REFERENCES_LOCAL = 1 << 1,
 
    /*
     * Hint that IO will be executed synchronously.
     *
     * This can make it a bit cheaper to execute synchronous IO via the AIO
     * interface, to avoid needing an AIO and non-AIO version of code.
     *
     * Advantageous to set, if applicable, but not required for correctness.
     */
    PGAIO_HF_SYNCHRONOUS = 1 << 0,
 
    /*
     * IO is using buffered IO, used to control heuristic in some IO methods.
     *
     * Advantageous to set, if applicable, but not required for correctness.
     */
    PGAIO_HF_BUFFERED = 1 << 2,
} PgAioHandleFlags;

PgAioOp

enum PgAioOp

Enumerator
PGAIO_OP_INVALID
PGAIO_OP_READV
PGAIO_OP_WRITEV

Definition at line 87 of file aio.h.

{
    /* intentionally the zero value, to help catch zeroed memory etc */
    PGAIO_OP_INVALID = 0,
 
    PGAIO_OP_READV,
    PGAIO_OP_WRITEV,
 
} PgAioOp;

PgAioTargetID

enum PgAioTargetID

Enumerator
PGAIO_TID_INVALID
PGAIO_TID_SMGR

Definition at line 116 of file aio.h.

{
    /* intentionally the zero value, to help catch zeroed memory etc */
    PGAIO_TID_INVALID = 0,
    PGAIO_TID_SMGR,
} PgAioTargetID;

Function Documentation

pgaio_closing_fd()

void pgaio_closing_fd ( int  fd )

extern

Definition at line 1220 of file aio.c.

{
    /*
     * Might be called before AIO is initialized or in a subprocess that
     * doesn't use AIO.
     */
    if (!pgaio_my_backend)
        return;
 
    /*
     * For now just submit all staged IOs - we could be more selective, but
     * it's probably not worth it.
     */
    if (pgaio_my_backend->num_staged_ios > 0)
    {
        pgaio_debug(DEBUG2,
                    "submitting %d IOs before FD %d gets closed",
                    pgaio_my_backend->num_staged_ios, fd);
        pgaio_submit_staged();
    }
 
    /*
     * If requested by the IO method, wait for all IOs that use the
     * to-be-closed FD.
     */
    if (pgaio_method_ops->wait_on_fd_before_close)
    {
        /*
         * As waiting for one IO to complete may complete multiple IOs, we
         * can't just use a mutable list iterator. The maximum number of
         * in-flight IOs is fairly small, so just restart the loop after
         * waiting for an IO.
         */
        while (!dclist_is_empty(&pgaio_my_backend->in_flight_ios))
        {
            dlist_iter  iter;
            PgAioHandle *ioh = NULL;
            uint64      generation;
 
            dclist_foreach(iter, &pgaio_my_backend->in_flight_ios)
            {
                ioh = dclist_container(PgAioHandle, node, iter.cur);
 
                generation = ioh->generation;
 
                if (pgaio_io_uses_fd(ioh, fd))
                    break;
                else
                    ioh = NULL;
            }
 
            if (!ioh)
                break;
 
            pgaio_debug_io(DEBUG2, ioh,
                           "waiting for IO before FD %d gets closed, %u in-flight IOs",
                           fd, dclist_count(&pgaio_my_backend->in_flight_ios));
 
            /* see comment in pgaio_io_wait_for_free() about raciness */
            pgaio_io_wait(ioh, generation);
        }
    }
}

References dlist_iter::cur, dclist_container, dclist_count(), dclist_foreach, dclist_is_empty(), DEBUG2, fb(), fd(), PgAioBackend::in_flight_ios, PgAioBackend::num_staged_ios, pgaio_debug, pgaio_debug_io, pgaio_io_uses_fd(), pgaio_io_wait(), pgaio_method_ops, pgaio_my_backend, pgaio_submit_staged(), and IoMethodOps::wait_on_fd_before_close.

Referenced by CloseTransientFile(), FileClose(), FreeDesc(), and LruDelete().

pgaio_enter_batchmode()

void pgaio_enter_batchmode ( void  )

extern

Definition at line 1081 of file aio.c.

{
    if (pgaio_my_backend->in_batchmode)
        elog(ERROR, "starting batch while batch already in progress");
    pgaio_my_backend->in_batchmode = true;
}

References elog, ERROR, PgAioBackend::in_batchmode, and pgaio_my_backend.

Referenced by batch_start(), read_rel_block_ll(), and read_stream_look_ahead().

pgaio_exit_batchmode()

void pgaio_exit_batchmode ( void  )

extern

Definition at line 1092 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode);
 
    pgaio_submit_staged();
    pgaio_my_backend->in_batchmode = false;
}

References Assert, PgAioBackend::in_batchmode, pgaio_my_backend, and pgaio_submit_staged().

Referenced by batch_end(), read_rel_block_ll(), and read_stream_look_ahead().

pgaio_have_staged()

bool pgaio_have_staged ( void  )

extern

Definition at line 1107 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode ||
           pgaio_my_backend->num_staged_ios == 0);
    return pgaio_my_backend->num_staged_ios > 0;
}

References Assert, PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, and pgaio_my_backend.

Referenced by ReadBuffersCanStartIO().

pgaio_io_acquire()

PgAioHandle * pgaio_io_acquire ( struct ResourceOwnerData *  resowner, PgAioReturn *  ret  )

extern

Definition at line 162 of file aio.c.

{
    PgAioHandle *h;
 
    while (true)
    {
        h = pgaio_io_acquire_nb(resowner, ret);
 
        if (h != NULL)
            return h;
 
        /*
         * Evidently all handles by this backend are in use. Just wait for
         * some to complete.
         */
        pgaio_io_wait_for_free();
    }
}

References fb(), pgaio_io_acquire_nb(), and pgaio_io_wait_for_free().

Referenced by AsyncReadBuffers(), handle_get(), handle_get_and_error(), handle_get_release(), handle_get_twice(), and read_rel_block_ll().

pgaio_io_acquire_nb()

PgAioHandle * pgaio_io_acquire_nb ( struct ResourceOwnerData *  resowner, PgAioReturn *  ret  )

extern

Definition at line 188 of file aio.c.

{
    PgAioHandle *ioh = NULL;
 
    if (pgaio_my_backend->num_staged_ios >= PGAIO_SUBMIT_BATCH_SIZE)
    {
        Assert(pgaio_my_backend->num_staged_ios == PGAIO_SUBMIT_BATCH_SIZE);
        pgaio_submit_staged();
    }
 
    if (pgaio_my_backend->handed_out_io)
        elog(ERROR, "API violation: Only one IO can be handed out");
 
    /*
     * Probably not needed today, as interrupts should not process this IO,
     * but...
     */
    HOLD_INTERRUPTS();
 
    if (!dclist_is_empty(&pgaio_my_backend->idle_ios))
    {
        dlist_node *ion = dclist_pop_head_node(&pgaio_my_backend->idle_ios);
 
        ioh = dclist_container(PgAioHandle, node, ion);
 
        Assert(ioh->state == PGAIO_HS_IDLE);
        Assert(ioh->owner_procno == MyProcNumber);
 
        pgaio_io_update_state(ioh, PGAIO_HS_HANDED_OUT);
        pgaio_my_backend->handed_out_io = ioh;
 
        if (resowner)
            pgaio_io_resowner_register(ioh, resowner);
 
        if (ret)
        {
            ioh->report_return = ret;
            ret->result.status = PGAIO_RS_UNKNOWN;
        }
    }
 
    RESUME_INTERRUPTS();
 
    return ioh;
}

References Assert, dclist_container, dclist_is_empty(), dclist_pop_head_node(), elog, ERROR, fb(), PgAioBackend::handed_out_io, HOLD_INTERRUPTS, PgAioBackend::idle_ios, MyProcNumber, PgAioBackend::num_staged_ios, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, pgaio_io_resowner_register(), pgaio_io_update_state(), pgaio_my_backend, PGAIO_RS_UNKNOWN, PGAIO_SUBMIT_BATCH_SIZE, pgaio_submit_staged(), PgAioReturn::result, RESUME_INTERRUPTS, and PgAioResult::status.

Referenced by AsyncReadBuffers(), and pgaio_io_acquire().

pgaio_io_get_handle_data()

uint64 * pgaio_io_get_handle_data ( PgAioHandle *  ioh, uint8 *  len  )

extern

Definition at line 156 of file aio_callback.c.

{
    Assert(ioh->handle_data_len > 0);
 
    *len = ioh->handle_data_len;
 
    return &pgaio_ctl->handle_data[ioh->iovec_off];
}

References Assert, fb(), PgAioCtl::handle_data, len, and pgaio_ctl.

Referenced by buffer_readv_complete(), and buffer_stage_common().

pgaio_io_get_id()

int pgaio_io_get_id ( PgAioHandle *  ioh )

extern

Definition at line 342 of file aio.c.

{
    Assert(ioh >= pgaio_ctl->io_handles &&
           ioh < (pgaio_ctl->io_handles + pgaio_ctl->io_handle_count));
    return ioh - pgaio_ctl->io_handles;
}

References Assert, fb(), PgAioCtl::io_handle_count, PgAioCtl::io_handles, and pgaio_ctl.

Referenced by pg_get_aios(), pgaio_io_wait(), pgaio_io_wait_for_free(), and pgaio_worker_submission_queue_insert().

pgaio_io_get_iovec()

int pgaio_io_get_iovec ( PgAioHandle *  ioh, struct iovec **  iov  )

extern

Definition at line 42 of file aio_io.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
 
    *iov = &pgaio_ctl->iovecs[ioh->iovec_off];
 
    return PG_IOV_MAX;
}

References Assert, fb(), PgAioCtl::iovecs, PG_IOV_MAX, pgaio_ctl, and PGAIO_HS_HANDED_OUT.

Referenced by mdstartreadv().

pgaio_io_get_op()

PgAioOp pgaio_io_get_op ( PgAioHandle *  ioh )

extern

Definition at line 52 of file aio_io.c.

{
    return ioh->op;
}

References fb().

Referenced by smgr_aio_reopen().

pgaio_io_get_op_data()

PgAioOpData * pgaio_io_get_op_data ( PgAioHandle *  ioh )

extern

Definition at line 58 of file aio_io.c.

{
    return &ioh->op_data;
}

References fb().

Referenced by smgr_aio_reopen().

pgaio_io_get_owner()

ProcNumber pgaio_io_get_owner ( PgAioHandle *  ioh )

extern

Definition at line 355 of file aio.c.

{
    return ioh->owner_procno;
}

References fb(), and PgAioHandle::owner_procno.

Referenced by buffer_readv_complete(), and smgr_aio_reopen().

pgaio_io_get_target_data()

PgAioTargetData * pgaio_io_get_target_data ( PgAioHandle *  ioh )

extern

Definition at line 73 of file aio_target.c.

{
    return &ioh->target_data;
}

References fb().

Referenced by buffer_readv_complete(), md_readv_complete(), pgaio_io_set_target_smgr(), shared_buffer_readv_complete_local(), and smgr_aio_reopen().

pgaio_io_get_target_description()

char * pgaio_io_get_target_description ( PgAioHandle *  ioh )

extern

Definition at line 84 of file aio_target.c.

{
    /* disallow INVALID, there wouldn't be a description */
    Assert(ioh->target > PGAIO_TID_INVALID && ioh->target < PGAIO_TID_COUNT);
 
    return pgaio_target_info[ioh->target]->describe_identity(&ioh->target_data);
}

References Assert, PgAioTargetInfo::describe_identity, fb(), pgaio_target_info, PGAIO_TID_COUNT, and PGAIO_TID_INVALID.

Referenced by pg_get_aios().

pgaio_io_get_wref()

void pgaio_io_get_wref ( PgAioHandle *  ioh, PgAioWaitRef *  iow  )

extern

Definition at line 366 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT ||
           ioh->state == PGAIO_HS_DEFINED ||
           ioh->state == PGAIO_HS_STAGED);
    Assert(ioh->generation != 0);
 
    iow->aio_index = ioh - pgaio_ctl->io_handles;
    iow->generation_upper = (uint32) (ioh->generation >> 32);
    iow->generation_lower = (uint32) ioh->generation;
}

References Assert, fb(), PgAioHandle::generation, PgAioCtl::io_handles, pgaio_ctl, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, and PGAIO_HS_STAGED.

Referenced by AsyncReadBuffers(), buffer_stage_common(), and read_rel_block_ll().

pgaio_io_has_target()

bool pgaio_io_has_target ( PgAioHandle *  ioh )

extern

Definition at line 40 of file aio_target.c.

{
    return ioh->target != PGAIO_TID_INVALID;
}

References fb(), and PGAIO_TID_INVALID.

Referenced by pgaio_io_before_start(), and pgaio_io_stage().

pgaio_io_register_callbacks()

void pgaio_io_register_callbacks ( PgAioHandle *  ioh, PgAioHandleCallbackID  cb_id, uint8  cb_data  )

extern

Definition at line 86 of file aio_callback.c.

{
    const PgAioHandleCallbacksEntry *ce = &aio_handle_cbs[cb_id];
 
    Assert(cb_id <= PGAIO_HCB_MAX);
    if (cb_id >= lengthof(aio_handle_cbs))
        elog(ERROR, "callback %d is out of range", cb_id);
    if (aio_handle_cbs[cb_id].cb->complete_shared == NULL &&
        aio_handle_cbs[cb_id].cb->complete_local == NULL)
        elog(ERROR, "callback %d does not have a completion callback", cb_id);
    if (ioh->num_callbacks >= PGAIO_HANDLE_MAX_CALLBACKS)
        elog(PANIC, "too many callbacks, the max is %d",
             PGAIO_HANDLE_MAX_CALLBACKS);
    ioh->callbacks[ioh->num_callbacks] = cb_id;
    ioh->callbacks_data[ioh->num_callbacks] = cb_data;
 
    pgaio_debug_io(DEBUG3, ioh,
                   "adding cb #%d, id %d/%s",
                   ioh->num_callbacks + 1,
                   cb_id, ce->name);
 
    ioh->num_callbacks++;
}

References aio_handle_cbs, Assert, PgAioHandleCallbacksEntry::cb, PgAioHandleCallbacks::complete_local, DEBUG3, elog, ERROR, fb(), lengthof, PANIC, pgaio_debug_io, PGAIO_HANDLE_MAX_CALLBACKS, and PGAIO_HCB_MAX.

Referenced by AsyncReadBuffers(), mdstartreadv(), and read_rel_block_ll().

pgaio_io_release()

void pgaio_io_release ( PgAioHandle *  ioh )

extern

Definition at line 240 of file aio.c.

{
    if (ioh == pgaio_my_backend->handed_out_io)
    {
        Assert(ioh->state == PGAIO_HS_HANDED_OUT);
        Assert(ioh->resowner);
 
        pgaio_my_backend->handed_out_io = NULL;
 
        /*
         * Note that no interrupts are processed between the handed_out_io
         * check and the call to reclaim - that's important as otherwise an
         * interrupt could have already reclaimed the handle.
         */
        pgaio_io_reclaim(ioh);
    }
    else
    {
        elog(ERROR, "release in unexpected state");
    }
}

References Assert, elog, ERROR, fb(), PgAioBackend::handed_out_io, PGAIO_HS_HANDED_OUT, pgaio_io_reclaim(), and pgaio_my_backend.

Referenced by AsyncReadBuffers(), handle_get_release(), and handle_release_last().

pgaio_io_release_resowner()

void pgaio_io_release_resowner ( struct dlist_node *  ioh_node, bool  on_error  )

extern

Definition at line 266 of file aio.c.

{
    PgAioHandle *ioh = dlist_container(PgAioHandle, resowner_node, ioh_node);
 
    Assert(ioh->resowner);
 
    /*
     * Otherwise an interrupt, in the middle of releasing the IO, could end up
     * trying to wait for the IO, leading to state confusion.
     */
    HOLD_INTERRUPTS();
 
    ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node);
    ioh->resowner = NULL;
 
    switch ((PgAioHandleState) ioh->state)
    {
        case PGAIO_HS_IDLE:
            elog(ERROR, "unexpected");
            break;
        case PGAIO_HS_HANDED_OUT:
            Assert(ioh == pgaio_my_backend->handed_out_io || pgaio_my_backend->handed_out_io == NULL);
 
            if (ioh == pgaio_my_backend->handed_out_io)
            {
                pgaio_my_backend->handed_out_io = NULL;
                if (!on_error)
                    elog(WARNING, "leaked AIO handle");
            }
 
            pgaio_io_reclaim(ioh);
            break;
        case PGAIO_HS_DEFINED:
        case PGAIO_HS_STAGED:
            if (!on_error)
                elog(WARNING, "AIO handle was not submitted");
            pgaio_submit_staged();
            break;
        case PGAIO_HS_SUBMITTED:
        case PGAIO_HS_COMPLETED_IO:
        case PGAIO_HS_COMPLETED_SHARED:
        case PGAIO_HS_COMPLETED_LOCAL:
            /* this is expected to happen */
            break;
    }
 
    /*
     * Need to unregister the reporting of the IO's result, the memory it's
     * referencing likely has gone away.
     */
    if (ioh->report_return)
        ioh->report_return = NULL;
 
    RESUME_INTERRUPTS();
}

References Assert, dlist_container, elog, ERROR, fb(), PgAioBackend::handed_out_io, HOLD_INTERRUPTS, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, PGAIO_HS_STAGED, PGAIO_HS_SUBMITTED, pgaio_io_reclaim(), pgaio_my_backend, pgaio_submit_staged(), ResourceOwnerForgetAioHandle(), RESUME_INTERRUPTS, and WARNING.

Referenced by ResourceOwnerReleaseInternal().

pgaio_io_set_flag()

void pgaio_io_set_flag ( PgAioHandle *  ioh, PgAioHandleFlags  flag  )

extern

Definition at line 330 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
 
    ioh->flags |= flag;
}

References Assert, fb(), flag(), and PGAIO_HS_HANDED_OUT.

Referenced by AsyncReadBuffers(), mdstartreadv(), and read_rel_block_ll().

pgaio_io_set_handle_data_32()

void pgaio_io_set_handle_data_32 ( PgAioHandle *  ioh, uint32 *  data, uint8  len  )

extern

Definition at line 140 of file aio_callback.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->handle_data_len == 0);
    Assert(len <= PG_IOV_MAX);
    Assert(len <= io_max_combine_limit);
 
    for (int i = 0; i < len; i++)
        pgaio_ctl->handle_data[ioh->iovec_off + i] = data[i];
    ioh->handle_data_len = len;
}

References Assert, data, fb(), PgAioCtl::handle_data, i, io_max_combine_limit, len, PG_IOV_MAX, pgaio_ctl, and PGAIO_HS_HANDED_OUT.

Referenced by AsyncReadBuffers(), and read_rel_block_ll().

pgaio_io_set_handle_data_64()

void pgaio_io_set_handle_data_64 ( PgAioHandle *  ioh, uint64 *  data, uint8  len  )

extern

Definition at line 122 of file aio_callback.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->handle_data_len == 0);
    Assert(len <= PG_IOV_MAX);
    Assert(len <= io_max_combine_limit);
 
    for (int i = 0; i < len; i++)
        pgaio_ctl->handle_data[ioh->iovec_off + i] = data[i];
    ioh->handle_data_len = len;
}

References Assert, data, fb(), PgAioCtl::handle_data, i, io_max_combine_limit, len, PG_IOV_MAX, pgaio_ctl, and PGAIO_HS_HANDED_OUT.

pgaio_io_set_target()

void pgaio_io_set_target ( PgAioHandle *  ioh, PgAioTargetID  targetid  )

extern

Definition at line 64 of file aio_target.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(ioh->target == PGAIO_TID_INVALID);
 
    ioh->target = targetid;
}

References Assert, fb(), PGAIO_HS_HANDED_OUT, and PGAIO_TID_INVALID.

Referenced by pgaio_io_set_target_smgr().

pgaio_io_start_readv()

void pgaio_io_start_readv ( PgAioHandle *  ioh, int  fd, int  iovcnt, uint64  offset  )

extern

Definition at line 78 of file aio_io.c.

{
    pgaio_io_before_start(ioh);
 
    ioh->op_data.read.fd = fd;
    ioh->op_data.read.offset = offset;
    ioh->op_data.read.iov_length = iovcnt;
 
    pgaio_io_stage(ioh, PGAIO_OP_READV);
}

References fb(), fd(), pgaio_io_before_start(), pgaio_io_stage(), and PGAIO_OP_READV.

Referenced by FileStartReadV().

pgaio_io_start_writev()

void pgaio_io_start_writev ( PgAioHandle *  ioh, int  fd, int  iovcnt, uint64  offset  )

extern

Definition at line 91 of file aio_io.c.

{
    pgaio_io_before_start(ioh);
 
    ioh->op_data.write.fd = fd;
    ioh->op_data.write.offset = offset;
    ioh->op_data.write.iov_length = iovcnt;
 
    pgaio_io_stage(ioh, PGAIO_OP_WRITEV);
}

References fb(), fd(), pgaio_io_before_start(), pgaio_io_stage(), and PGAIO_OP_WRITEV.

pgaio_result_report()

void pgaio_result_report ( PgAioResult  result, const PgAioTargetData *  target_data, int  elevel  )

extern

Definition at line 173 of file aio_callback.c.

{
    PgAioHandleCallbackID cb_id = result.id;
    const PgAioHandleCallbacksEntry *ce = &aio_handle_cbs[cb_id];
 
    Assert(result.status != PGAIO_RS_UNKNOWN);
    Assert(result.status != PGAIO_RS_OK);
 
    if (ce->cb->report == NULL)
        elog(ERROR, "callback %d/%s does not have report callback",
             result.id, ce->name);
 
    ce->cb->report(result, target_data, elevel);
}

References aio_handle_cbs, Assert, elog, ERROR, fb(), PgAioResult::id, PGAIO_RS_OK, PGAIO_RS_UNKNOWN, and PgAioResult::status.

Referenced by buffer_readv_complete(), buffer_readv_complete_one(), md_readv_complete(), ProcessReadBuffersResult(), and read_rel_block_ll().

pgaio_submit_staged()

void pgaio_submit_staged ( void  )

extern

Definition at line 1123 of file aio.c.

{
    int         total_submitted = 0;
    int         did_submit;
 
    if (pgaio_my_backend->num_staged_ios == 0)
        return;
 
 
    START_CRIT_SECTION();
 
    did_submit = pgaio_method_ops->submit(pgaio_my_backend->num_staged_ios,
                                          pgaio_my_backend->staged_ios);
 
    END_CRIT_SECTION();
 
    total_submitted += did_submit;
 
    Assert(total_submitted == did_submit);
 
    pgaio_my_backend->num_staged_ios = 0;
 
    pgaio_debug(DEBUG4,
                "aio: submitted %d IOs",
                total_submitted);
}

References Assert, DEBUG4, END_CRIT_SECTION, fb(), PgAioBackend::num_staged_ios, pgaio_debug, pgaio_method_ops, pgaio_my_backend, PgAioBackend::staged_ios, START_CRIT_SECTION, and IoMethodOps::submit.

Referenced by AsyncReadBuffers(), pgaio_closing_fd(), pgaio_error_cleanup(), pgaio_exit_batchmode(), pgaio_io_acquire_nb(), pgaio_io_release_resowner(), pgaio_io_stage(), pgaio_io_wait_for_free(), and ReadBuffersCanStartIO().

pgaio_wref_check_done()

bool pgaio_wref_check_done ( PgAioWaitRef *  iow )

extern

Definition at line 1005 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandleState state;
    bool        am_owner;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    if (pgaio_io_was_recycled(ioh, ref_generation, &state))
        return true;
 
    if (state == PGAIO_HS_IDLE)
        return true;
 
    am_owner = ioh->owner_procno == MyProcNumber;
 
    if (state == PGAIO_HS_COMPLETED_SHARED ||
        state == PGAIO_HS_COMPLETED_LOCAL)
    {
        /*
         * Note that no interrupts are processed between
         * pgaio_io_was_recycled() and this check - that's important as
         * otherwise an interrupt could have already reclaimed the handle.
         */
        if (am_owner)
            pgaio_io_reclaim(ioh);
        return true;
    }
 
    /*
     * XXX: It likely would be worth checking in with the io method, to give
     * the IO method a chance to check if there are completion events queued.
     */
 
    return false;
}

References fb(), MyProcNumber, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_IDLE, pgaio_io_from_wref(), pgaio_io_reclaim(), and pgaio_io_was_recycled().

Referenced by WaitReadBuffers().

pgaio_wref_clear()

void pgaio_wref_clear ( PgAioWaitRef *  iow )

extern

Definition at line 964 of file aio.c.

{
    iow->aio_index = PG_UINT32_MAX;
}

References fb(), and PG_UINT32_MAX.

Referenced by AsyncReadBuffers(), BufferManagerShmemInit(), InitLocalBuffers(), StartReadBuffersImpl(), TerminateBufferIO(), and TerminateLocalBufferIO().

pgaio_wref_get_id()

int pgaio_wref_get_id ( PgAioWaitRef *  iow )

extern

Definition at line 980 of file aio.c.

{
    Assert(pgaio_wref_valid(iow));
    return iow->aio_index;
}

References Assert, fb(), and pgaio_wref_valid().

pgaio_wref_valid()

bool pgaio_wref_valid ( PgAioWaitRef *  iow )

extern

Definition at line 971 of file aio.c.

{
    return iow->aio_index != PG_UINT32_MAX;
}

References fb(), and PG_UINT32_MAX.

Referenced by InvalidateLocalBuffer(), pgaio_wref_get_id(), ProcessReadBuffersResult(), StartLocalBufferIO(), WaitIO(), and WaitReadBuffers().

pgaio_wref_wait()

void pgaio_wref_wait ( PgAioWaitRef *  iow )

extern

Definition at line 991 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    pgaio_io_wait(ioh, ref_generation);
}

References fb(), pgaio_io_from_wref(), and pgaio_io_wait().

Referenced by InvalidateLocalBuffer(), read_rel_block_ll(), StartLocalBufferIO(), WaitIO(), and WaitReadBuffers().

StaticAssertDecl()

StaticAssertDecl

(

)

Variable Documentation

io_max_concurrency

PGDLLIMPORT int io_max_concurrency

extern

Definition at line 75 of file aio.c.

Referenced by AioHandleDataShmemSize(), AioHandleIOVShmemSize(), AioHandleShmemSize(), AioShmemInit(), AioShmemSize(), and pgaio_io_wait_for_free().

io_method

PGDLLIMPORT int io_method

extern

Definition at line 74 of file aio.c.

Referenced by pgaio_workers_enabled(), read_stream_begin_impl(), StartReadBuffersImpl(), and WaitReadBuffers().