aio.c File Reference

#include "postgres.h"
#include "lib/ilist.h"
#include "miscadmin.h"
#include "port/atomics.h"
#include "storage/aio.h"
#include "storage/aio_internal.h"
#include "storage/aio_subsys.h"
#include "utils/guc.h"
#include "utils/guc_hooks.h"
#include "utils/resowner.h"
#include "utils/wait_event_types.h"

Include dependency graph for aio.c:

Go to the source code of this file.

Macros
#define	PGAIO_HS_TOSTR_CASE(sym)   case PGAIO_HS_##sym: return #sym

Functions
static void	pgaio_io_update_state (PgAioHandle *ioh, PgAioHandleState new_state)
static void	pgaio_io_reclaim (PgAioHandle *ioh)
static void	pgaio_io_resowner_register (PgAioHandle *ioh)
static void	pgaio_io_wait_for_free (void)
static PgAioHandle *	pgaio_io_from_wref (PgAioWaitRef *iow, uint64 *ref_generation)
static const char *	pgaio_io_state_get_name (PgAioHandleState s)
static void	pgaio_io_wait (PgAioHandle *ioh, uint64 ref_generation)
PgAioHandle *	pgaio_io_acquire (struct ResourceOwnerData *resowner, PgAioReturn *ret)
PgAioHandle *	pgaio_io_acquire_nb (struct ResourceOwnerData *resowner, PgAioReturn *ret)
void	pgaio_io_release (PgAioHandle *ioh)
void	pgaio_io_release_resowner (dlist_node *ioh_node, bool on_error)
void	pgaio_io_set_flag (PgAioHandle *ioh, PgAioHandleFlags flag)
int	pgaio_io_get_id (PgAioHandle *ioh)
ProcNumber	pgaio_io_get_owner (PgAioHandle *ioh)
void	pgaio_io_get_wref (PgAioHandle *ioh, PgAioWaitRef *iow)
void	pgaio_io_stage (PgAioHandle *ioh, PgAioOp op)
bool	pgaio_io_needs_synchronous_execution (PgAioHandle *ioh)
void	pgaio_io_prepare_submit (PgAioHandle *ioh)
void	pgaio_io_process_completion (PgAioHandle *ioh, int result)
bool	pgaio_io_was_recycled (PgAioHandle *ioh, uint64 ref_generation, PgAioHandleState *state)
const char *	pgaio_io_get_state_name (PgAioHandle *ioh)
const char *	pgaio_result_status_string (PgAioResultStatus rs)
void	pgaio_wref_clear (PgAioWaitRef *iow)
bool	pgaio_wref_valid (PgAioWaitRef *iow)
int	pgaio_wref_get_id (PgAioWaitRef *iow)
void	pgaio_wref_wait (PgAioWaitRef *iow)
bool	pgaio_wref_check_done (PgAioWaitRef *iow)
void	pgaio_enter_batchmode (void)
void	pgaio_exit_batchmode (void)
bool	pgaio_have_staged (void)
void	pgaio_submit_staged (void)
void	pgaio_error_cleanup (void)
void	AtEOXact_Aio (bool is_commit)
void	pgaio_closing_fd (int fd)
void	pgaio_shutdown (int code, Datum arg)
void	assign_io_method (int newval, void *extra)
bool	check_io_max_concurrency (int *newval, void **extra, GucSource source)

Variables
const struct config_enum_entry	io_method_options []
int	io_method = DEFAULT_IO_METHOD
int	io_max_concurrency = -1
PgAioCtl *	pgaio_ctl
PgAioBackend *	pgaio_my_backend
static const IoMethodOps *const	pgaio_method_ops_table []
const IoMethodOps *	pgaio_method_ops

Macro Definition Documentation

PGAIO_HS_TOSTR_CASE

#define PGAIO_HS_TOSTR_CASE

(

sym

)

case PGAIO_HS_##sym: return #sym

Function Documentation

assign_io_method()

void assign_io_method	(	int	newval,
		void *	extra
	)

Definition at line 1156 of file aio.c.

{
    Assert(pgaio_method_ops_table[newval] != NULL);
    Assert(newval < lengthof(io_method_options));
 
    pgaio_method_ops = pgaio_method_ops_table[newval];
}

References Assert(), io_method_options, lengthof, newval, pgaio_method_ops, and pgaio_method_ops_table.

AtEOXact_Aio()

void AtEOXact_Aio

(

bool

is_commit

)

Definition at line 1080 of file aio.c.

{
    /*
     * We should never be in batch mode at transactional boundaries. In case
     * an error was thrown while in batch mode, pgaio_error_cleanup() should
     * have exited batchmode.
     *
     * In case we are in batchmode somehow, make sure to submit all staged
     * IOs, other backends may need them to complete to continue.
     */
    if (pgaio_my_backend->in_batchmode)
    {
        pgaio_error_cleanup();
        elog(WARNING, "open AIO batch at end of (sub-)transaction");
    }
 
    /*
     * As we aren't in batchmode, there shouldn't be any unsubmitted IOs.
     */
    Assert(pgaio_my_backend->num_staged_ios == 0);
}

References Assert(), elog, PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, pgaio_error_cleanup(), pgaio_my_backend, and WARNING.

Referenced by AbortSubTransaction(), AbortTransaction(), CommitTransaction(), pgaio_shutdown(), and PrepareTransaction().

check_io_max_concurrency()

bool check_io_max_concurrency	(	int *	newval,
		void **	extra,
		GucSource	source
	)

Definition at line 1165 of file aio.c.

{
    if (*newval == -1)
    {
        /*
         * Auto-tuning will be applied later during startup, as auto-tuning
         * depends on the value of various GUCs.
         */
        return true;
    }
    else if (*newval == 0)
    {
        GUC_check_errdetail("Only -1 or values bigger than 0 are valid.");
        return false;
    }
 
    return true;
}

References GUC_check_errdetail, and newval.

pgaio_closing_fd()

void pgaio_closing_fd

(

int

fd

)

Definition at line 1107 of file aio.c.

{
    /*
     * Might be called before AIO is initialized or in a subprocess that
     * doesn't use AIO.
     */
    if (!pgaio_my_backend)
        return;
 
    /*
     * For now just submit all staged IOs - we could be more selective, but
     * it's probably not worth it.
     */
    pgaio_submit_staged();
}

References pgaio_my_backend, and pgaio_submit_staged().

pgaio_enter_batchmode()

void pgaio_enter_batchmode

(

void

)

Definition at line 968 of file aio.c.

{
    if (pgaio_my_backend->in_batchmode)
        elog(ERROR, "starting batch while batch already in progress");
    pgaio_my_backend->in_batchmode = true;
}

References elog, ERROR, PgAioBackend::in_batchmode, and pgaio_my_backend.

pgaio_error_cleanup()

void pgaio_error_cleanup

(

void

)

Definition at line 1052 of file aio.c.

{
    /*
     * It is possible that code errored out after pgaio_enter_batchmode() but
     * before pgaio_exit_batchmode() was called. In that case we need to
     * submit the IO now.
     */
    if (pgaio_my_backend->in_batchmode)
    {
        pgaio_my_backend->in_batchmode = false;
 
        pgaio_submit_staged();
    }
 
    /*
     * As we aren't in batchmode, there shouldn't be any unsubmitted IOs.
     */
    Assert(pgaio_my_backend->num_staged_ios == 0);
}

References Assert(), PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, pgaio_my_backend, and pgaio_submit_staged().

Referenced by AbortSubTransaction(), AbortTransaction(), AtEOXact_Aio(), AutoVacLauncherMain(), BackgroundWriterMain(), CheckpointerMain(), pgarch_archiveXlog(), WalSndErrorCleanup(), WalSummarizerMain(), and WalWriterMain().

pgaio_exit_batchmode()

void pgaio_exit_batchmode

(

void

)

Definition at line 979 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode);
 
    pgaio_submit_staged();
    pgaio_my_backend->in_batchmode = false;
}

References Assert(), PgAioBackend::in_batchmode, pgaio_my_backend, and pgaio_submit_staged().

pgaio_have_staged()

bool pgaio_have_staged

(

void

)

Definition at line 994 of file aio.c.

{
    Assert(pgaio_my_backend->in_batchmode ||
           pgaio_my_backend->num_staged_ios == 0);
    return pgaio_my_backend->num_staged_ios > 0;
}

References Assert(), PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, and pgaio_my_backend.

pgaio_io_acquire()

PgAioHandle * pgaio_io_acquire	(	struct ResourceOwnerData *	resowner,
		PgAioReturn *	ret
	)

Definition at line 165 of file aio.c.

{
    PgAioHandle *h;
 
    while (true)
    {
        h = pgaio_io_acquire_nb(resowner, ret);
 
        if (h != NULL)
            return h;
 
        /*
         * Evidently all handles by this backend are in use. Just wait for
         * some to complete.
         */
        pgaio_io_wait_for_free();
    }
}

References pgaio_io_acquire_nb(), and pgaio_io_wait_for_free().

pgaio_io_acquire_nb()

PgAioHandle * pgaio_io_acquire_nb	(	struct ResourceOwnerData *	resowner,
		PgAioReturn *	ret
	)

Definition at line 191 of file aio.c.

{
    if (pgaio_my_backend->num_staged_ios >= PGAIO_SUBMIT_BATCH_SIZE)
    {
        Assert(pgaio_my_backend->num_staged_ios == PGAIO_SUBMIT_BATCH_SIZE);
        pgaio_submit_staged();
    }
 
    if (pgaio_my_backend->handed_out_io)
        elog(ERROR, "API violation: Only one IO can be handed out");
 
    if (!dclist_is_empty(&pgaio_my_backend->idle_ios))
    {
        dlist_node *ion = dclist_pop_head_node(&pgaio_my_backend->idle_ios);
        PgAioHandle *ioh = dclist_container(PgAioHandle, node, ion);
 
        Assert(ioh->state == PGAIO_HS_IDLE);
        Assert(ioh->owner_procno == MyProcNumber);
 
        pgaio_io_update_state(ioh, PGAIO_HS_HANDED_OUT);
        pgaio_my_backend->handed_out_io = ioh;
 
        if (resowner)
            pgaio_io_resowner_register(ioh);
 
        if (ret)
        {
            ioh->report_return = ret;
            ret->result.status = PGAIO_RS_UNKNOWN;
        }
 
        return ioh;
    }
 
    return NULL;
}

References Assert(), dclist_container, dclist_is_empty(), dclist_pop_head_node(), elog, ERROR, PgAioBackend::handed_out_io, PgAioBackend::idle_ios, MyProcNumber, PgAioBackend::num_staged_ios, PgAioHandle::owner_procno, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, pgaio_io_resowner_register(), pgaio_io_update_state(), pgaio_my_backend, PGAIO_RS_UNKNOWN, PGAIO_SUBMIT_BATCH_SIZE, pgaio_submit_staged(), PgAioHandle::report_return, PgAioReturn::result, PgAioHandle::state, and PgAioResult::status.

Referenced by pgaio_io_acquire().

pgaio_io_from_wref()

static PgAioHandle * pgaio_io_from_wref ( PgAioWaitRef *  iow, uint64 *  ref_generation  )

static

Definition at line 785 of file aio.c.

{
    PgAioHandle *ioh;
 
    Assert(iow->aio_index < pgaio_ctl->io_handle_count);
 
    ioh = &pgaio_ctl->io_handles[iow->aio_index];
 
    *ref_generation = ((uint64) iow->generation_upper) << 32 |
        iow->generation_lower;
 
    Assert(*ref_generation != 0);
 
    return ioh;
}

References PgAioWaitRef::aio_index, Assert(), PgAioWaitRef::generation_lower, PgAioWaitRef::generation_upper, PgAioCtl::io_handle_count, PgAioCtl::io_handles, and pgaio_ctl.

Referenced by pgaio_wref_check_done(), and pgaio_wref_wait().

pgaio_io_get_id()

int pgaio_io_get_id

(

PgAioHandle *

ioh

)

Definition at line 322 of file aio.c.

{
    Assert(ioh >= pgaio_ctl->io_handles &&
           ioh < (pgaio_ctl->io_handles + pgaio_ctl->io_handle_count));
    return ioh - pgaio_ctl->io_handles;
}

References Assert(), PgAioCtl::io_handle_count, PgAioCtl::io_handles, and pgaio_ctl.

Referenced by pgaio_io_wait_for_free(), and pgaio_worker_submission_queue_insert().

pgaio_io_get_owner()

ProcNumber pgaio_io_get_owner

(

PgAioHandle *

ioh

)

Definition at line 335 of file aio.c.

{
    return ioh->owner_procno;
}

References PgAioHandle::owner_procno.

pgaio_io_get_state_name()

const char * pgaio_io_get_state_name

(

PgAioHandle *

ioh

)

Definition at line 822 of file aio.c.

{
    return pgaio_io_state_get_name(ioh->state);
}

References pgaio_io_state_get_name(), and PgAioHandle::state.

pgaio_io_get_wref()

void pgaio_io_get_wref	(	PgAioHandle *	ioh,
		PgAioWaitRef *	iow
	)

Definition at line 346 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT ||
           ioh->state == PGAIO_HS_DEFINED ||
           ioh->state == PGAIO_HS_STAGED);
    Assert(ioh->generation != 0);
 
    iow->aio_index = ioh - pgaio_ctl->io_handles;
    iow->generation_upper = (uint32) (ioh->generation >> 32);
    iow->generation_lower = (uint32) ioh->generation;
}

References PgAioWaitRef::aio_index, Assert(), PgAioHandle::generation, PgAioWaitRef::generation_lower, PgAioWaitRef::generation_upper, PgAioCtl::io_handles, pgaio_ctl, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_STAGED, and PgAioHandle::state.

pgaio_io_needs_synchronous_execution()

bool pgaio_io_needs_synchronous_execution

(

PgAioHandle *

ioh

)

Definition at line 447 of file aio.c.

{
    /*
     * If the caller said to execute the IO synchronously, do so.
     *
     * XXX: We could optimize the logic when to execute synchronously by first
     * checking if there are other IOs in flight and only synchronously
     * executing if not. Unclear whether that'll be sufficiently common to be
     * worth worrying about.
     */
    if (ioh->flags & PGAIO_HF_SYNCHRONOUS)
        return true;
 
    /* Check if the IO method requires synchronous execution of IO */
    if (pgaio_method_ops->needs_synchronous_execution)
        return pgaio_method_ops->needs_synchronous_execution(ioh);
 
    return false;
}

References PgAioHandle::flags, IoMethodOps::needs_synchronous_execution, PGAIO_HF_SYNCHRONOUS, and pgaio_method_ops.

Referenced by pgaio_io_stage().

pgaio_io_prepare_submit()

void pgaio_io_prepare_submit

(

PgAioHandle *

ioh

)

Definition at line 474 of file aio.c.

{
    pgaio_io_update_state(ioh, PGAIO_HS_SUBMITTED);
 
    dclist_push_tail(&pgaio_my_backend->in_flight_ios, &ioh->node);
}

References dclist_push_tail(), PgAioBackend::in_flight_ios, PgAioHandle::node, PGAIO_HS_SUBMITTED, pgaio_io_update_state(), and pgaio_my_backend.

Referenced by pgaio_io_stage(), and pgaio_worker_submit().

pgaio_io_process_completion()

void pgaio_io_process_completion	(	PgAioHandle *	ioh,
		int	result
	)

Definition at line 492 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_SUBMITTED);
 
    Assert(CritSectionCount > 0);
 
    ioh->result = result;
 
    pgaio_io_update_state(ioh, PGAIO_HS_COMPLETED_IO);
 
    pgaio_io_call_inj(ioh, "AIO_PROCESS_COMPLETION_BEFORE_SHARED");
 
    pgaio_io_call_complete_shared(ioh);
 
    pgaio_io_update_state(ioh, PGAIO_HS_COMPLETED_SHARED);
 
    /* condition variable broadcast ensures state is visible before wakeup */
    ConditionVariableBroadcast(&ioh->cv);
 
    /* contains call to pgaio_io_call_complete_local() */
    if (ioh->owner_procno == MyProcNumber)
        pgaio_io_reclaim(ioh);
}

References Assert(), ConditionVariableBroadcast(), CritSectionCount, PgAioHandle::cv, MyProcNumber, PgAioHandle::owner_procno, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_SUBMITTED, pgaio_io_call_complete_shared(), pgaio_io_call_inj, pgaio_io_reclaim(), pgaio_io_update_state(), PgAioHandle::result, and PgAioHandle::state.

Referenced by IoWorkerMain(), and pgaio_io_perform_synchronously().

pgaio_io_reclaim()

static void pgaio_io_reclaim ( PgAioHandle *  ioh )

static

Definition at line 621 of file aio.c.

{
    /* This is only ok if it's our IO */
    Assert(ioh->owner_procno == MyProcNumber);
    Assert(ioh->state != PGAIO_HS_IDLE);
 
    /*
     * It's a bit ugly, but right now the easiest place to put the execution
     * of shared completion callbacks is this function, as we need to execute
     * local callbacks just before reclaiming at multiple callsites.
     */
    if (ioh->state == PGAIO_HS_COMPLETED_SHARED)
    {
        pgaio_io_call_complete_local(ioh);
        pgaio_io_update_state(ioh, PGAIO_HS_COMPLETED_LOCAL);
    }
 
    pgaio_debug_io(DEBUG4, ioh,
                   "reclaiming: distilled_result: (status %s, id %u, error_data %d), raw_result: %d",
                   pgaio_result_status_string(ioh->distilled_result.status),
                   ioh->distilled_result.id,
                   ioh->distilled_result.error_data,
                   ioh->result);
 
    /* if the IO has been defined, we might need to do more work */
    if (ioh->state != PGAIO_HS_HANDED_OUT)
    {
        dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node);
 
        if (ioh->report_return)
        {
            ioh->report_return->result = ioh->distilled_result;
            ioh->report_return->target_data = ioh->target_data;
        }
    }
 
    if (ioh->resowner)
    {
        ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node);
        ioh->resowner = NULL;
    }
 
    Assert(!ioh->resowner);
 
    ioh->op = PGAIO_OP_INVALID;
    ioh->target = PGAIO_TID_INVALID;
    ioh->flags = 0;
    ioh->num_callbacks = 0;
    ioh->handle_data_len = 0;
    ioh->report_return = NULL;
    ioh->result = 0;
    ioh->distilled_result.status = PGAIO_RS_UNKNOWN;
 
    /* XXX: the barrier is probably superfluous */
    pg_write_barrier();
    ioh->generation++;
 
    pgaio_io_update_state(ioh, PGAIO_HS_IDLE);
 
    /*
     * We push the IO to the head of the idle IO list, that seems more cache
     * efficient in cases where only a few IOs are used.
     */
    dclist_push_head(&pgaio_my_backend->idle_ios, &ioh->node);
}

References Assert(), dclist_delete_from(), dclist_push_head(), DEBUG4, PgAioHandle::distilled_result, PgAioResult::error_data, PgAioHandle::flags, PgAioHandle::generation, PgAioHandle::handle_data_len, PgAioResult::id, PgAioBackend::idle_ios, PgAioBackend::in_flight_ios, MyProcNumber, PgAioHandle::node, PgAioHandle::num_callbacks, PgAioHandle::op, PgAioHandle::owner_procno, pg_write_barrier, pgaio_debug_io, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, pgaio_io_call_complete_local(), pgaio_io_update_state(), pgaio_my_backend, PGAIO_OP_INVALID, pgaio_result_status_string(), PGAIO_RS_UNKNOWN, PGAIO_TID_INVALID, PgAioHandle::report_return, ResourceOwnerForgetAioHandle(), PgAioHandle::resowner, PgAioHandle::resowner_node, PgAioHandle::result, PgAioReturn::result, PgAioHandle::state, PgAioResult::status, PgAioHandle::target, PgAioHandle::target_data, and PgAioReturn::target_data.

Referenced by pgaio_io_process_completion(), pgaio_io_release(), pgaio_io_release_resowner(), pgaio_io_wait(), pgaio_io_wait_for_free(), and pgaio_wref_check_done().

pgaio_io_release()

void pgaio_io_release

(

PgAioHandle *

ioh

)

Definition at line 234 of file aio.c.

{
    if (ioh == pgaio_my_backend->handed_out_io)
    {
        Assert(ioh->state == PGAIO_HS_HANDED_OUT);
        Assert(ioh->resowner);
 
        pgaio_my_backend->handed_out_io = NULL;
        pgaio_io_reclaim(ioh);
    }
    else
    {
        elog(ERROR, "release in unexpected state");
    }
}

References Assert(), elog, ERROR, PgAioBackend::handed_out_io, PGAIO_HS_HANDED_OUT, pgaio_io_reclaim(), pgaio_my_backend, PgAioHandle::resowner, and PgAioHandle::state.

pgaio_io_release_resowner()

void pgaio_io_release_resowner	(	dlist_node *	ioh_node,
		bool	on_error
	)

Definition at line 254 of file aio.c.

{
    PgAioHandle *ioh = dlist_container(PgAioHandle, resowner_node, ioh_node);
 
    Assert(ioh->resowner);
 
    ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node);
    ioh->resowner = NULL;
 
    switch (ioh->state)
    {
        case PGAIO_HS_IDLE:
            elog(ERROR, "unexpected");
            break;
        case PGAIO_HS_HANDED_OUT:
            Assert(ioh == pgaio_my_backend->handed_out_io || pgaio_my_backend->handed_out_io == NULL);
 
            if (ioh == pgaio_my_backend->handed_out_io)
            {
                pgaio_my_backend->handed_out_io = NULL;
                if (!on_error)
                    elog(WARNING, "leaked AIO handle");
            }
 
            pgaio_io_reclaim(ioh);
            break;
        case PGAIO_HS_DEFINED:
        case PGAIO_HS_STAGED:
            if (!on_error)
                elog(WARNING, "AIO handle was not submitted");
            pgaio_submit_staged();
            break;
        case PGAIO_HS_SUBMITTED:
        case PGAIO_HS_COMPLETED_IO:
        case PGAIO_HS_COMPLETED_SHARED:
        case PGAIO_HS_COMPLETED_LOCAL:
            /* this is expected to happen */
            break;
    }
 
    /*
     * Need to unregister the reporting of the IO's result, the memory it's
     * referencing likely has gone away.
     */
    if (ioh->report_return)
        ioh->report_return = NULL;
}

References Assert(), dlist_container, elog, ERROR, PgAioBackend::handed_out_io, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, PGAIO_HS_STAGED, PGAIO_HS_SUBMITTED, pgaio_io_reclaim(), pgaio_my_backend, pgaio_submit_staged(), PgAioHandle::report_return, ResourceOwnerForgetAioHandle(), PgAioHandle::resowner, PgAioHandle::resowner_node, PgAioHandle::state, and WARNING.

Referenced by ResourceOwnerReleaseInternal().

pgaio_io_resowner_register()

static void pgaio_io_resowner_register ( PgAioHandle *  ioh )

static

Definition at line 382 of file aio.c.

{
    Assert(!ioh->resowner);
    Assert(CurrentResourceOwner);
 
    ResourceOwnerRememberAioHandle(CurrentResourceOwner, &ioh->resowner_node);
    ioh->resowner = CurrentResourceOwner;
}

References Assert(), CurrentResourceOwner, ResourceOwnerRememberAioHandle(), PgAioHandle::resowner, and PgAioHandle::resowner_node.

Referenced by pgaio_io_acquire_nb().

pgaio_io_set_flag()

void pgaio_io_set_flag	(	PgAioHandle *	ioh,
		PgAioHandleFlags	flag
	)

Definition at line 310 of file aio.c.

{
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
 
    ioh->flags |= flag;
}

References Assert(), flag(), PgAioHandle::flags, PGAIO_HS_HANDED_OUT, and PgAioHandle::state.

pgaio_io_stage()

void pgaio_io_stage	(	PgAioHandle *	ioh,
		PgAioOp	op
	)

Definition at line 397 of file aio.c.

{
    bool        needs_synchronous;
 
    Assert(ioh->state == PGAIO_HS_HANDED_OUT);
    Assert(pgaio_my_backend->handed_out_io == ioh);
    Assert(pgaio_io_has_target(ioh));
 
    ioh->op = op;
    ioh->result = 0;
 
    pgaio_io_update_state(ioh, PGAIO_HS_DEFINED);
 
    /* allow a new IO to be staged */
    pgaio_my_backend->handed_out_io = NULL;
 
    pgaio_io_call_stage(ioh);
 
    pgaio_io_update_state(ioh, PGAIO_HS_STAGED);
 
    /*
     * Synchronous execution has to be executed, well, synchronously, so check
     * that first.
     */
    needs_synchronous = pgaio_io_needs_synchronous_execution(ioh);
 
    pgaio_debug_io(DEBUG3, ioh,
                   "prepared (synchronous: %d, in_batch: %d)",
                   needs_synchronous, pgaio_my_backend->in_batchmode);
 
    if (!needs_synchronous)
    {
        pgaio_my_backend->staged_ios[pgaio_my_backend->num_staged_ios++] = ioh;
        Assert(pgaio_my_backend->num_staged_ios <= PGAIO_SUBMIT_BATCH_SIZE);
 
        /*
         * Unless code explicitly opted into batching IOs, submit the IO
         * immediately.
         */
        if (!pgaio_my_backend->in_batchmode)
            pgaio_submit_staged();
    }
    else
    {
        pgaio_io_prepare_submit(ioh);
        pgaio_io_perform_synchronously(ioh);
    }
}

References Assert(), DEBUG3, PgAioBackend::handed_out_io, PgAioBackend::in_batchmode, PgAioBackend::num_staged_ios, PgAioHandle::op, pgaio_debug_io, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_STAGED, pgaio_io_call_stage(), pgaio_io_has_target(), pgaio_io_needs_synchronous_execution(), pgaio_io_perform_synchronously(), pgaio_io_prepare_submit(), pgaio_io_update_state(), pgaio_my_backend, PGAIO_SUBMIT_BATCH_SIZE, pgaio_submit_staged(), PgAioHandle::result, PgAioBackend::staged_ios, and PgAioHandle::state.

Referenced by pgaio_io_prep_readv(), and pgaio_io_prep_writev().

pgaio_io_state_get_name()

static const char * pgaio_io_state_get_name ( PgAioHandleState  s )

static

Definition at line 802 of file aio.c.

{
#define PGAIO_HS_TOSTR_CASE(sym) case PGAIO_HS_##sym: return #sym
    switch (s)
    {
            PGAIO_HS_TOSTR_CASE(IDLE);
            PGAIO_HS_TOSTR_CASE(HANDED_OUT);
            PGAIO_HS_TOSTR_CASE(DEFINED);
            PGAIO_HS_TOSTR_CASE(STAGED);
            PGAIO_HS_TOSTR_CASE(SUBMITTED);
            PGAIO_HS_TOSTR_CASE(COMPLETED_IO);
            PGAIO_HS_TOSTR_CASE(COMPLETED_SHARED);
            PGAIO_HS_TOSTR_CASE(COMPLETED_LOCAL);
    }
#undef PGAIO_HS_TOSTR_CASE
 
    return NULL;                /* silence compiler */
}

References PGAIO_HS_TOSTR_CASE.

Referenced by pgaio_io_get_state_name(), and pgaio_io_update_state().

pgaio_io_update_state()

static void pgaio_io_update_state ( PgAioHandle *  ioh, PgAioHandleState  new_state  )

inlinestatic

Definition at line 366 of file aio.c.

{
    pgaio_debug_io(DEBUG5, ioh,
                   "updating state to %s",
                   pgaio_io_state_get_name(new_state));
 
    /*
     * Ensure the changes signified by the new state are visible before the
     * new state becomes visible.
     */
    pg_write_barrier();
 
    ioh->state = new_state;
}

References DEBUG5, pg_write_barrier, pgaio_debug_io, pgaio_io_state_get_name(), and PgAioHandle::state.

Referenced by pgaio_io_acquire_nb(), pgaio_io_prepare_submit(), pgaio_io_process_completion(), pgaio_io_reclaim(), and pgaio_io_stage().

pgaio_io_wait()

static void pgaio_io_wait ( PgAioHandle *  ioh, uint64  ref_generation  )

static

Definition at line 536 of file aio.c.

{
    PgAioHandleState state;
    bool        am_owner;
 
    am_owner = ioh->owner_procno == MyProcNumber;
 
    if (pgaio_io_was_recycled(ioh, ref_generation, &state))
        return;
 
    if (am_owner)
    {
        if (state != PGAIO_HS_SUBMITTED
            && state != PGAIO_HS_COMPLETED_IO
            && state != PGAIO_HS_COMPLETED_SHARED
            && state != PGAIO_HS_COMPLETED_LOCAL)
        {
            elog(PANIC, "waiting for own IO in wrong state: %d",
                 state);
        }
    }
 
    while (true)
    {
        if (pgaio_io_was_recycled(ioh, ref_generation, &state))
            return;
 
        switch (state)
        {
            case PGAIO_HS_IDLE:
            case PGAIO_HS_HANDED_OUT:
                elog(ERROR, "IO in wrong state: %d", state);
                break;
 
            case PGAIO_HS_SUBMITTED:
 
                /*
                 * If we need to wait via the IO method, do so now. Don't
                 * check via the IO method if the issuing backend is executing
                 * the IO synchronously.
                 */
                if (pgaio_method_ops->wait_one && !(ioh->flags & PGAIO_HF_SYNCHRONOUS))
                {
                    pgaio_method_ops->wait_one(ioh, ref_generation);
                    continue;
                }
                /* fallthrough */
 
                /* waiting for owner to submit */
            case PGAIO_HS_DEFINED:
            case PGAIO_HS_STAGED:
                /* waiting for reaper to complete */
                /* fallthrough */
            case PGAIO_HS_COMPLETED_IO:
                /* shouldn't be able to hit this otherwise */
                Assert(IsUnderPostmaster);
                /* ensure we're going to get woken up */
                ConditionVariablePrepareToSleep(&ioh->cv);
 
                while (!pgaio_io_was_recycled(ioh, ref_generation, &state))
                {
                    if (state == PGAIO_HS_COMPLETED_SHARED ||
                        state == PGAIO_HS_COMPLETED_LOCAL)
                        break;
                    ConditionVariableSleep(&ioh->cv, WAIT_EVENT_AIO_IO_COMPLETION);
                }
 
                ConditionVariableCancelSleep();
                break;
 
            case PGAIO_HS_COMPLETED_SHARED:
            case PGAIO_HS_COMPLETED_LOCAL:
                /* see above */
                if (am_owner)
                    pgaio_io_reclaim(ioh);
                return;
        }
    }
}

References Assert(), ConditionVariableCancelSleep(), ConditionVariablePrepareToSleep(), ConditionVariableSleep(), PgAioHandle::cv, elog, ERROR, PgAioHandle::flags, IsUnderPostmaster, MyProcNumber, PgAioHandle::owner_procno, PANIC, PGAIO_HF_SYNCHRONOUS, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, PGAIO_HS_STAGED, PGAIO_HS_SUBMITTED, pgaio_io_reclaim(), pgaio_io_was_recycled(), pgaio_method_ops, and IoMethodOps::wait_one.

Referenced by pgaio_io_wait_for_free(), pgaio_shutdown(), and pgaio_wref_wait().

pgaio_io_wait_for_free()

static void pgaio_io_wait_for_free ( void  )

static

Definition at line 693 of file aio.c.

{
    int         reclaimed = 0;
 
    pgaio_debug(DEBUG2, "waiting for self with %d pending",
                pgaio_my_backend->num_staged_ios);
 
    /*
     * First check if any of our IOs actually have completed - when using
     * worker, that'll often be the case. We could do so as part of the loop
     * below, but that'd potentially lead us to wait for some IO submitted
     * before.
     */
    for (int i = 0; i < io_max_concurrency; i++)
    {
        PgAioHandle *ioh = &pgaio_ctl->io_handles[pgaio_my_backend->io_handle_off + i];
 
        if (ioh->state == PGAIO_HS_COMPLETED_SHARED)
        {
            pgaio_io_reclaim(ioh);
            reclaimed++;
        }
    }
 
    if (reclaimed > 0)
        return;
 
    /*
     * If we have any unsubmitted IOs, submit them now. We'll start waiting in
     * a second, so it's better they're in flight. This also addresses the
     * edge-case that all IOs are unsubmitted.
     */
    if (pgaio_my_backend->num_staged_ios > 0)
        pgaio_submit_staged();
 
    if (dclist_count(&pgaio_my_backend->in_flight_ios) == 0)
        elog(ERROR, "no free IOs despite no in-flight IOs");
 
    /*
     * Wait for the oldest in-flight IO to complete.
     *
     * XXX: Reusing the general IO wait is suboptimal, we don't need to wait
     * for that specific IO to complete, we just need *any* IO to complete.
     */
    {
        PgAioHandle *ioh = dclist_head_element(PgAioHandle, node,
                                               &pgaio_my_backend->in_flight_ios);
 
        switch (ioh->state)
        {
                /* should not be in in-flight list */
            case PGAIO_HS_IDLE:
            case PGAIO_HS_DEFINED:
            case PGAIO_HS_HANDED_OUT:
            case PGAIO_HS_STAGED:
            case PGAIO_HS_COMPLETED_LOCAL:
                elog(ERROR, "shouldn't get here with io:%d in state %d",
                     pgaio_io_get_id(ioh), ioh->state);
                break;
 
            case PGAIO_HS_COMPLETED_IO:
            case PGAIO_HS_SUBMITTED:
                pgaio_debug_io(DEBUG2, ioh,
                               "waiting for free io with %d in flight",
                               dclist_count(&pgaio_my_backend->in_flight_ios));
 
                /*
                 * In a more general case this would be racy, because the
                 * generation could increase after we read ioh->state above.
                 * But we are only looking at IOs by the current backend and
                 * the IO can only be recycled by this backend.
                 */
                pgaio_io_wait(ioh, ioh->generation);
                break;
 
            case PGAIO_HS_COMPLETED_SHARED:
                /* it's possible that another backend just finished this IO */
                pgaio_io_reclaim(ioh);
                break;
        }
 
        if (dclist_count(&pgaio_my_backend->idle_ios) == 0)
            elog(PANIC, "no idle IO after waiting for IO to terminate");
        return;
    }
}

References dclist_count(), dclist_head_element, DEBUG2, elog, ERROR, PgAioHandle::generation, i, PgAioBackend::idle_ios, PgAioBackend::in_flight_ios, PgAioBackend::io_handle_off, PgAioCtl::io_handles, io_max_concurrency, PgAioBackend::num_staged_ios, PANIC, pgaio_ctl, pgaio_debug, pgaio_debug_io, PGAIO_HS_COMPLETED_IO, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_DEFINED, PGAIO_HS_HANDED_OUT, PGAIO_HS_IDLE, PGAIO_HS_STAGED, PGAIO_HS_SUBMITTED, pgaio_io_get_id(), pgaio_io_reclaim(), pgaio_io_wait(), pgaio_my_backend, pgaio_submit_staged(), and PgAioHandle::state.

Referenced by pgaio_io_acquire().

pgaio_io_was_recycled()

bool pgaio_io_was_recycled	(	PgAioHandle *	ioh,
		uint64	ref_generation,
		PgAioHandleState *	state
	)

Definition at line 523 of file aio.c.

{
    *state = ioh->state;
    pg_read_barrier();
 
    return ioh->generation != ref_generation;
}

References PgAioHandle::generation, pg_read_barrier, and PgAioHandle::state.

Referenced by pgaio_io_wait(), and pgaio_wref_check_done().

pgaio_result_status_string()

const char * pgaio_result_status_string

(

PgAioResultStatus

rs

)

Definition at line 828 of file aio.c.

{
    switch (rs)
    {
        case PGAIO_RS_UNKNOWN:
            return "UNKNOWN";
        case PGAIO_RS_OK:
            return "OK";
        case PGAIO_RS_PARTIAL:
            return "PARTIAL";
        case PGAIO_RS_ERROR:
            return "ERROR";
    }
 
    return NULL;                /* silence compiler */
}

References PGAIO_RS_ERROR, PGAIO_RS_OK, PGAIO_RS_PARTIAL, and PGAIO_RS_UNKNOWN.

Referenced by pgaio_io_call_complete_local(), pgaio_io_call_complete_shared(), and pgaio_io_reclaim().

pgaio_shutdown()

void pgaio_shutdown	(	int	code,
		Datum	arg
	)

Definition at line 1127 of file aio.c.

{
    Assert(pgaio_my_backend);
    Assert(!pgaio_my_backend->handed_out_io);
 
    /* first clean up resources as we would at a transaction boundary */
    AtEOXact_Aio(code == 0);
 
    /*
     * Before exiting, make sure that all IOs are finished. That has two main
     * purposes:
     *
     * - Some kernel-level AIO mechanisms don't deal well with the issuer of
     * an AIO exiting before IO completed
     *
     * - It'd be confusing to see partially finished IOs in stats views etc
     */
    while (!dclist_is_empty(&pgaio_my_backend->in_flight_ios))
    {
        PgAioHandle *ioh = dclist_head_element(PgAioHandle, node, &pgaio_my_backend->in_flight_ios);
 
        /* see comment in pgaio_io_wait_for_free() about raciness */
        pgaio_io_wait(ioh, ioh->generation);
    }
 
    pgaio_my_backend = NULL;
}

References Assert(), AtEOXact_Aio(), dclist_head_element, dclist_is_empty(), PgAioHandle::generation, PgAioBackend::handed_out_io, PgAioBackend::in_flight_ios, pgaio_io_wait(), and pgaio_my_backend.

Referenced by pgaio_init_backend().

pgaio_submit_staged()

void pgaio_submit_staged

(

void

)

Definition at line 1010 of file aio.c.

{
    int         total_submitted = 0;
    int         did_submit;
 
    if (pgaio_my_backend->num_staged_ios == 0)
        return;
 
 
    START_CRIT_SECTION();
 
    did_submit = pgaio_method_ops->submit(pgaio_my_backend->num_staged_ios,
                                          pgaio_my_backend->staged_ios);
 
    END_CRIT_SECTION();
 
    total_submitted += did_submit;
 
    Assert(total_submitted == did_submit);
 
    pgaio_my_backend->num_staged_ios = 0;
 
    pgaio_debug(DEBUG4,
                "aio: submitted %d IOs",
                total_submitted);
}

References Assert(), DEBUG4, END_CRIT_SECTION, PgAioBackend::num_staged_ios, pgaio_debug, pgaio_method_ops, pgaio_my_backend, PgAioBackend::staged_ios, START_CRIT_SECTION, and IoMethodOps::submit.

Referenced by pgaio_closing_fd(), pgaio_error_cleanup(), pgaio_exit_batchmode(), pgaio_io_acquire_nb(), pgaio_io_release_resowner(), pgaio_io_stage(), and pgaio_io_wait_for_free().

pgaio_wref_check_done()

bool pgaio_wref_check_done

(

PgAioWaitRef *

iow

)

Definition at line 897 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandleState state;
    bool        am_owner;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    if (pgaio_io_was_recycled(ioh, ref_generation, &state))
        return true;
 
    if (state == PGAIO_HS_IDLE)
        return true;
 
    am_owner = ioh->owner_procno == MyProcNumber;
 
    if (state == PGAIO_HS_COMPLETED_SHARED ||
        state == PGAIO_HS_COMPLETED_LOCAL)
    {
        if (am_owner)
            pgaio_io_reclaim(ioh);
        return true;
    }
 
    /*
     * XXX: It likely would be worth checking in with the io method, to give
     * the IO method a chance to check if there are completion events queued.
     */
 
    return false;
}

References MyProcNumber, PgAioHandle::owner_procno, PGAIO_HS_COMPLETED_LOCAL, PGAIO_HS_COMPLETED_SHARED, PGAIO_HS_IDLE, pgaio_io_from_wref(), pgaio_io_reclaim(), and pgaio_io_was_recycled().

pgaio_wref_clear()

void pgaio_wref_clear

(

PgAioWaitRef *

iow

)

Definition at line 856 of file aio.c.

{
    iow->aio_index = PG_UINT32_MAX;
}

References PgAioWaitRef::aio_index, and PG_UINT32_MAX.

pgaio_wref_get_id()

int pgaio_wref_get_id

(

PgAioWaitRef *

iow

)

Definition at line 872 of file aio.c.

{
    Assert(pgaio_wref_valid(iow));
    return iow->aio_index;
}

References PgAioWaitRef::aio_index, Assert(), and pgaio_wref_valid().

pgaio_wref_valid()

bool pgaio_wref_valid

(

PgAioWaitRef *

iow

)

Definition at line 863 of file aio.c.

{
    return iow->aio_index != PG_UINT32_MAX;
}

References PgAioWaitRef::aio_index, and PG_UINT32_MAX.

Referenced by pgaio_wref_get_id().

pgaio_wref_wait()

void pgaio_wref_wait

(

PgAioWaitRef *

iow

)

Definition at line 883 of file aio.c.

{
    uint64      ref_generation;
    PgAioHandle *ioh;
 
    ioh = pgaio_io_from_wref(iow, &ref_generation);
 
    pgaio_io_wait(ioh, ref_generation);
}

References pgaio_io_from_wref(), and pgaio_io_wait().

Variable Documentation

io_max_concurrency

int io_max_concurrency = -1

Definition at line 73 of file aio.c.

Referenced by AioHandleDataShmemSize(), AioHandleIOVShmemSize(), AioHandleShmemSize(), AioShmemInit(), AioShmemSize(), and pgaio_io_wait_for_free().

io_method

int io_method = DEFAULT_IO_METHOD

Definition at line 72 of file aio.c.

Referenced by pgaio_workers_enabled().

io_method_options

const struct config_enum_entry io_method_options[]

Initial value:

= {
    {"sync", IOMETHOD_SYNC, false},
    {"worker", IOMETHOD_WORKER, false},
    {NULL, 0, false}
}

Definition at line 65 of file aio.c.

Referenced by assign_io_method().

pgaio_ctl

PgAioCtl* pgaio_ctl

Definition at line 76 of file aio.c.

Referenced by AioShmemInit(), IoWorkerMain(), pgaio_init_backend(), pgaio_io_from_wref(), pgaio_io_get_handle_data(), pgaio_io_get_id(), pgaio_io_get_iovec(), pgaio_io_get_wref(), pgaio_io_perform_synchronously(), pgaio_io_set_handle_data_32(), pgaio_io_set_handle_data_64(), and pgaio_io_wait_for_free().

pgaio_method_ops

const IoMethodOps* pgaio_method_ops

Definition at line 88 of file aio.c.

Referenced by AioShmemInit(), AioShmemSize(), assign_io_method(), pgaio_init_backend(), pgaio_io_needs_synchronous_execution(), pgaio_io_wait(), and pgaio_submit_staged().

pgaio_method_ops_table

const IoMethodOps* const pgaio_method_ops_table[]

static

Initial value:

= {
    [IOMETHOD_SYNC] = &pgaio_sync_ops,
    [IOMETHOD_WORKER] = &pgaio_worker_ops,
}

Definition at line 82 of file aio.c.

Referenced by assign_io_method().

pgaio_my_backend

PgAioBackend* pgaio_my_backend

Definition at line 79 of file aio.c.

Referenced by AtEOXact_Aio(), pgaio_closing_fd(), pgaio_enter_batchmode(), pgaio_error_cleanup(), pgaio_exit_batchmode(), pgaio_have_staged(), pgaio_init_backend(), pgaio_io_acquire_nb(), pgaio_io_before_prep(), pgaio_io_prepare_submit(), pgaio_io_reclaim(), pgaio_io_release(), pgaio_io_release_resowner(), pgaio_io_stage(), pgaio_io_wait_for_free(), pgaio_shutdown(), and pgaio_submit_staged().