method_worker.c File Reference

#include "postgres.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "port/pg_bitutils.h"
#include "postmaster/auxprocess.h"
#include "postmaster/interrupt.h"
#include "storage/aio.h"
#include "storage/aio_internal.h"
#include "storage/aio_subsys.h"
#include "storage/io_worker.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/proc.h"
#include "tcop/tcopprot.h"
#include "utils/ps_status.h"
#include "utils/wait_event.h"

Include dependency graph for method_worker.c:

Go to the source code of this file.

Data Structures
struct	AioWorkerSubmissionQueue
struct	AioWorkerSlot
struct	AioWorkerControl

Macros
#define	IO_WORKER_WAKEUP_FANOUT   2

Typedefs
typedef struct AioWorkerSubmissionQueue	AioWorkerSubmissionQueue
typedef struct AioWorkerSlot	AioWorkerSlot
typedef struct AioWorkerControl	AioWorkerControl

Functions
static size_t	pgaio_worker_shmem_size (void)
static void	pgaio_worker_shmem_init (bool first_time)
static bool	pgaio_worker_needs_synchronous_execution (PgAioHandle *ioh)
static int	pgaio_worker_submit (uint16 num_staged_ios, PgAioHandle **staged_ios)
static size_t	pgaio_worker_queue_shmem_size (int *queue_size)
static size_t	pgaio_worker_control_shmem_size (void)
static int	pgaio_choose_idle_worker (void)
static bool	pgaio_worker_submission_queue_insert (PgAioHandle *ioh)
static uint32	pgaio_worker_submission_queue_consume (void)
static uint32	pgaio_worker_submission_queue_depth (void)
static void	pgaio_worker_submit_internal (int nios, PgAioHandle *ios[])
static void	pgaio_worker_die (int code, Datum arg)
static void	pgaio_worker_register (void)
void	IoWorkerMain (const void *startup_data, size_t startup_data_len)
bool	pgaio_workers_enabled (void)

Variables
const IoMethodOps	pgaio_worker_ops
int	io_workers = 3
static int	io_worker_queue_size = 64
static int	MyIoWorkerId
static AioWorkerSubmissionQueue *	io_worker_submission_queue
static AioWorkerControl *	io_worker_control

Macro Definition Documentation

IO_WORKER_WAKEUP_FANOUT

#define IO_WORKER_WAKEUP_FANOUT   2

Definition at line 50 of file method_worker.c.

Typedef Documentation

AioWorkerControl

typedef struct AioWorkerControl AioWorkerControl

AioWorkerSlot

typedef struct AioWorkerSlot AioWorkerSlot

AioWorkerSubmissionQueue

typedef struct AioWorkerSubmissionQueue AioWorkerSubmissionQueue

Function Documentation

IoWorkerMain()

void IoWorkerMain	(	const void *	startup_data,
		size_t	startup_data_len
	)

Definition at line 361 of file method_worker.c.

{
    sigjmp_buf  local_sigjmp_buf;
    PgAioHandle *volatile error_ioh = NULL;
    volatile int error_errno = 0;
    char        cmd[128];
 
    MyBackendType = B_IO_WORKER;
    AuxiliaryProcessMainCommon();
 
    pqsignal(SIGHUP, SignalHandlerForConfigReload);
    pqsignal(SIGINT, die);      /* to allow manually triggering worker restart */
 
    /*
     * Ignore SIGTERM, will get explicit shutdown via SIGUSR2 later in the
     * shutdown sequence, similar to checkpointer.
     */
    pqsignal(SIGTERM, SIG_IGN);
    /* SIGQUIT handler was already set up by InitPostmasterChild */
    pqsignal(SIGALRM, SIG_IGN);
    pqsignal(SIGPIPE, SIG_IGN);
    pqsignal(SIGUSR1, procsignal_sigusr1_handler);
    pqsignal(SIGUSR2, SignalHandlerForShutdownRequest);
 
    /* also registers a shutdown callback to unregister */
    pgaio_worker_register();
 
    sprintf(cmd, "%d", MyIoWorkerId);
    set_ps_display(cmd);
 
    /* see PostgresMain() */
    if (sigsetjmp(local_sigjmp_buf, 1) != 0)
    {
        error_context_stack = NULL;
        HOLD_INTERRUPTS();
 
        EmitErrorReport();
 
        /*
         * In the - very unlikely - case that the IO failed in a way that
         * raises an error we need to mark the IO as failed.
         *
         * Need to do just enough error recovery so that we can mark the IO as
         * failed and then exit (postmaster will start a new worker).
         */
        LWLockReleaseAll();
 
        if (error_ioh != NULL)
        {
            /* should never fail without setting error_errno */
            Assert(error_errno != 0);
 
            errno = error_errno;
 
            START_CRIT_SECTION();
            pgaio_io_process_completion(error_ioh, -error_errno);
            END_CRIT_SECTION();
        }
 
        proc_exit(1);
    }
 
    /* We can now handle ereport(ERROR) */
    PG_exception_stack = &local_sigjmp_buf;
 
    sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
 
    while (!ShutdownRequestPending)
    {
        uint32      io_index;
        Latch      *latches[IO_WORKER_WAKEUP_FANOUT];
        int         nlatches = 0;
        int         nwakeups = 0;
        int         worker;
 
        /* Try to get a job to do. */
        LWLockAcquire(AioWorkerSubmissionQueueLock, LW_EXCLUSIVE);
        if ((io_index = pgaio_worker_submission_queue_consume()) == UINT32_MAX)
        {
            /*
             * Nothing to do.  Mark self idle.
             *
             * XXX: Invent some kind of back pressure to reduce useless
             * wakeups?
             */
            io_worker_control->idle_worker_mask |= (UINT64_C(1) << MyIoWorkerId);
        }
        else
        {
            /* Got one.  Clear idle flag. */
            io_worker_control->idle_worker_mask &= ~(UINT64_C(1) << MyIoWorkerId);
 
            /* See if we can wake up some peers. */
            nwakeups = Min(pgaio_worker_submission_queue_depth(),
                           IO_WORKER_WAKEUP_FANOUT);
            for (int i = 0; i < nwakeups; ++i)
            {
                if ((worker = pgaio_choose_idle_worker()) < 0)
                    break;
                latches[nlatches++] = io_worker_control->workers[worker].latch;
            }
        }
        LWLockRelease(AioWorkerSubmissionQueueLock);
 
        for (int i = 0; i < nlatches; ++i)
            SetLatch(latches[i]);
 
        if (io_index != UINT32_MAX)
        {
            PgAioHandle *ioh = NULL;
 
            ioh = &pgaio_ctl->io_handles[io_index];
            error_ioh = ioh;
 
            pgaio_debug_io(DEBUG4, ioh,
                           "worker %d processing IO",
                           MyIoWorkerId);
 
            /*
             * Prevent interrupts between pgaio_io_reopen() and
             * pgaio_io_perform_synchronously() that otherwise could lead to
             * the FD getting closed in that window.
             */
            HOLD_INTERRUPTS();
 
            /*
             * It's very unlikely, but possible, that reopen fails. E.g. due
             * to memory allocations failing or file permissions changing or
             * such.  In that case we need to fail the IO.
             *
             * There's not really a good errno we can report here.
             */
            error_errno = ENOENT;
            pgaio_io_reopen(ioh);
 
            /*
             * To be able to exercise the reopen-fails path, allow injection
             * points to trigger a failure at this point.
             */
            pgaio_io_call_inj(ioh, "AIO_WORKER_AFTER_REOPEN");
 
            error_errno = 0;
            error_ioh = NULL;
 
            /*
             * We don't expect this to ever fail with ERROR or FATAL, no need
             * to keep error_ioh set to the IO.
             * pgaio_io_perform_synchronously() contains a critical section to
             * ensure we don't accidentally fail.
             */
            pgaio_io_perform_synchronously(ioh);
 
            RESUME_INTERRUPTS();
        }
        else
        {
            WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, -1,
                      WAIT_EVENT_IO_WORKER_MAIN);
            ResetLatch(MyLatch);
        }
 
        CHECK_FOR_INTERRUPTS();
    }
 
    proc_exit(0);
}

References Assert(), AuxiliaryProcessMainCommon(), B_IO_WORKER, CHECK_FOR_INTERRUPTS, DEBUG4, die, EmitErrorReport(), END_CRIT_SECTION, error_context_stack, HOLD_INTERRUPTS, i, AioWorkerControl::idle_worker_mask, PgAioCtl::io_handles, io_worker_control, IO_WORKER_WAKEUP_FANOUT, AioWorkerSlot::latch, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), LWLockReleaseAll(), Min, MyBackendType, MyIoWorkerId, MyLatch, PG_exception_stack, pgaio_choose_idle_worker(), pgaio_ctl, pgaio_debug_io, pgaio_io_call_inj, pgaio_io_perform_synchronously(), pgaio_io_process_completion(), pgaio_io_reopen(), pgaio_worker_register(), pgaio_worker_submission_queue_consume(), pgaio_worker_submission_queue_depth(), pqsignal, proc_exit(), procsignal_sigusr1_handler(), ResetLatch(), RESUME_INTERRUPTS, set_ps_display(), SetLatch(), ShutdownRequestPending, SIGALRM, SIGHUP, SignalHandlerForConfigReload(), SignalHandlerForShutdownRequest(), SIGPIPE, SIGUSR1, SIGUSR2, sprintf, START_CRIT_SECTION, UnBlockSig, WaitLatch(), WL_EXIT_ON_PM_DEATH, WL_LATCH_SET, and AioWorkerControl::workers.

pgaio_choose_idle_worker()

static int pgaio_choose_idle_worker ( void  )

static

Definition at line 163 of file method_worker.c.

{
    int         worker;
 
    if (io_worker_control->idle_worker_mask == 0)
        return -1;
 
    /* Find the lowest bit position, and clear it. */
    worker = pg_rightmost_one_pos64(io_worker_control->idle_worker_mask);
    io_worker_control->idle_worker_mask &= ~(UINT64_C(1) << worker);
 
    return worker;
}

References AioWorkerControl::idle_worker_mask, io_worker_control, and pg_rightmost_one_pos64().

Referenced by IoWorkerMain(), and pgaio_worker_submit_internal().

pgaio_worker_control_shmem_size()

static size_t pgaio_worker_control_shmem_size ( void  )

static

Definition at line 112 of file method_worker.c.

{
    return offsetof(AioWorkerControl, workers) +
        sizeof(AioWorkerSlot) * MAX_IO_WORKERS;
}

References MAX_IO_WORKERS.

Referenced by pgaio_worker_shmem_init(), and pgaio_worker_shmem_size().

pgaio_worker_die()

static void pgaio_worker_die ( int  code, Datum  arg  )

static

Definition at line 311 of file method_worker.c.

{
    LWLockAcquire(AioWorkerSubmissionQueueLock, LW_EXCLUSIVE);
    Assert(io_worker_control->workers[MyIoWorkerId].in_use);
    Assert(io_worker_control->workers[MyIoWorkerId].latch == MyLatch);
 
    io_worker_control->workers[MyIoWorkerId].in_use = false;
    io_worker_control->workers[MyIoWorkerId].latch = NULL;
    LWLockRelease(AioWorkerSubmissionQueueLock);
}

References Assert(), AioWorkerSlot::in_use, io_worker_control, AioWorkerSlot::latch, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), MyIoWorkerId, MyLatch, and AioWorkerControl::workers.

Referenced by pgaio_worker_register().

pgaio_worker_needs_synchronous_execution()

static bool pgaio_worker_needs_synchronous_execution ( PgAioHandle *  ioh )

static

Definition at line 232 of file method_worker.c.

{
    return
        !IsUnderPostmaster
        || ioh->flags & PGAIO_HF_REFERENCES_LOCAL
        || !pgaio_io_can_reopen(ioh);
}

References PgAioHandle::flags, IsUnderPostmaster, PGAIO_HF_REFERENCES_LOCAL, and pgaio_io_can_reopen().

Referenced by pgaio_worker_submit_internal().

pgaio_worker_queue_shmem_size()

static size_t pgaio_worker_queue_shmem_size ( int *  queue_size )

static

Definition at line 102 of file method_worker.c.

{
    /* Round size up to next power of two so we can make a mask. */
    *queue_size = pg_nextpower2_32(io_worker_queue_size);
 
    return offsetof(AioWorkerSubmissionQueue, ios) +
        sizeof(uint32) * *queue_size;
}

References io_worker_queue_size, and pg_nextpower2_32().

Referenced by pgaio_worker_shmem_init(), and pgaio_worker_shmem_size().

pgaio_worker_register()

static void pgaio_worker_register ( void  )

static

Definition at line 327 of file method_worker.c.

{
    MyIoWorkerId = -1;
 
    /*
     * XXX: This could do with more fine-grained locking. But it's also not
     * very common for the number of workers to change at the moment...
     */
    LWLockAcquire(AioWorkerSubmissionQueueLock, LW_EXCLUSIVE);
 
    for (int i = 0; i < MAX_IO_WORKERS; ++i)
    {
        if (!io_worker_control->workers[i].in_use)
        {
            Assert(io_worker_control->workers[i].latch == NULL);
            io_worker_control->workers[i].in_use = true;
            MyIoWorkerId = i;
            break;
        }
        else
            Assert(io_worker_control->workers[i].latch != NULL);
    }
 
    if (MyIoWorkerId == -1)
        elog(ERROR, "couldn't find a free worker slot");
 
    io_worker_control->idle_worker_mask |= (UINT64_C(1) << MyIoWorkerId);
    io_worker_control->workers[MyIoWorkerId].latch = MyLatch;
    LWLockRelease(AioWorkerSubmissionQueueLock);
 
    on_shmem_exit(pgaio_worker_die, 0);
}

References Assert(), elog, ERROR, i, AioWorkerControl::idle_worker_mask, AioWorkerSlot::in_use, io_worker_control, AioWorkerSlot::latch, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), MAX_IO_WORKERS, MyIoWorkerId, MyLatch, on_shmem_exit(), pgaio_worker_die(), and AioWorkerControl::workers.

Referenced by IoWorkerMain().

pgaio_worker_shmem_init()

static void pgaio_worker_shmem_init ( bool  first_time )

static

Definition at line 131 of file method_worker.c.

{
    bool        found;
    int         queue_size;
 
    io_worker_submission_queue =
        ShmemInitStruct("AioWorkerSubmissionQueue",
                        pgaio_worker_queue_shmem_size(&queue_size),
                        &found);
    if (!found)
    {
        io_worker_submission_queue->size = queue_size;
        io_worker_submission_queue->head = 0;
        io_worker_submission_queue->tail = 0;
    }
 
    io_worker_control =
        ShmemInitStruct("AioWorkerControl",
                        pgaio_worker_control_shmem_size(),
                        &found);
    if (!found)
    {
        io_worker_control->idle_worker_mask = 0;
        for (int i = 0; i < MAX_IO_WORKERS; ++i)
        {
            io_worker_control->workers[i].latch = NULL;
            io_worker_control->workers[i].in_use = false;
        }
    }
}

References AioWorkerSubmissionQueue::head, i, AioWorkerControl::idle_worker_mask, AioWorkerSlot::in_use, io_worker_control, io_worker_submission_queue, AioWorkerSlot::latch, MAX_IO_WORKERS, pgaio_worker_control_shmem_size(), pgaio_worker_queue_shmem_size(), ShmemInitStruct(), AioWorkerSubmissionQueue::size, AioWorkerSubmissionQueue::tail, and AioWorkerControl::workers.

pgaio_worker_shmem_size()

static size_t pgaio_worker_shmem_size ( void  )

static

Definition at line 119 of file method_worker.c.

{
    size_t      sz;
    int         queue_size;
 
    sz = pgaio_worker_queue_shmem_size(&queue_size);
    sz = add_size(sz, pgaio_worker_control_shmem_size());
 
    return sz;
}

References add_size(), pgaio_worker_control_shmem_size(), and pgaio_worker_queue_shmem_size().

pgaio_worker_submission_queue_consume()

static uint32 pgaio_worker_submission_queue_consume ( void  )

static

Definition at line 199 of file method_worker.c.

{
    AioWorkerSubmissionQueue *queue;
    uint32      result;
 
    queue = io_worker_submission_queue;
    if (queue->tail == queue->head)
        return UINT32_MAX;      /* empty */
 
    result = queue->ios[queue->tail];
    queue->tail = (queue->tail + 1) & (queue->size - 1);
 
    return result;
}

References AioWorkerSubmissionQueue::head, io_worker_submission_queue, AioWorkerSubmissionQueue::ios, AioWorkerSubmissionQueue::size, and AioWorkerSubmissionQueue::tail.

Referenced by IoWorkerMain().

pgaio_worker_submission_queue_depth()

static uint32 pgaio_worker_submission_queue_depth ( void  )

static

Definition at line 215 of file method_worker.c.

{
    uint32      head;
    uint32      tail;
 
    head = io_worker_submission_queue->head;
    tail = io_worker_submission_queue->tail;
 
    if (tail > head)
        head += io_worker_submission_queue->size;
 
    Assert(head >= tail);
 
    return head - tail;
}

References Assert(), AioWorkerSubmissionQueue::head, io_worker_submission_queue, AioWorkerSubmissionQueue::size, and AioWorkerSubmissionQueue::tail.

Referenced by IoWorkerMain().

pgaio_worker_submission_queue_insert()

static bool pgaio_worker_submission_queue_insert ( PgAioHandle *  ioh )

static

Definition at line 178 of file method_worker.c.

{
    AioWorkerSubmissionQueue *queue;
    uint32      new_head;
 
    queue = io_worker_submission_queue;
    new_head = (queue->head + 1) & (queue->size - 1);
    if (new_head == queue->tail)
    {
        pgaio_debug(DEBUG3, "io queue is full, at %u elements",
                    io_worker_submission_queue->size);
        return false;           /* full */
    }
 
    queue->ios[queue->head] = pgaio_io_get_id(ioh);
    queue->head = new_head;
 
    return true;
}

References DEBUG3, AioWorkerSubmissionQueue::head, io_worker_submission_queue, AioWorkerSubmissionQueue::ios, pgaio_debug, pgaio_io_get_id(), AioWorkerSubmissionQueue::size, and AioWorkerSubmissionQueue::tail.

Referenced by pgaio_worker_submit_internal().

pgaio_worker_submit()

static int pgaio_worker_submit ( uint16  num_staged_ios, PgAioHandle **  staged_ios  )

static

Definition at line 292 of file method_worker.c.

{
    for (int i = 0; i < num_staged_ios; i++)
    {
        PgAioHandle *ioh = staged_ios[i];
 
        pgaio_io_prepare_submit(ioh);
    }
 
    pgaio_worker_submit_internal(num_staged_ios, staged_ios);
 
    return num_staged_ios;
}

References i, pgaio_io_prepare_submit(), and pgaio_worker_submit_internal().

pgaio_worker_submit_internal()

static void pgaio_worker_submit_internal ( int  nios, PgAioHandle *  ios[]  )

static

Definition at line 241 of file method_worker.c.

{
    PgAioHandle *synchronous_ios[PGAIO_SUBMIT_BATCH_SIZE];
    int         nsync = 0;
    Latch      *wakeup = NULL;
    int         worker;
 
    Assert(nios <= PGAIO_SUBMIT_BATCH_SIZE);
 
    LWLockAcquire(AioWorkerSubmissionQueueLock, LW_EXCLUSIVE);
    for (int i = 0; i < nios; ++i)
    {
        Assert(!pgaio_worker_needs_synchronous_execution(ios[i]));
        if (!pgaio_worker_submission_queue_insert(ios[i]))
        {
            /*
             * We'll do it synchronously, but only after we've sent as many as
             * we can to workers, to maximize concurrency.
             */
            synchronous_ios[nsync++] = ios[i];
            continue;
        }
 
        if (wakeup == NULL)
        {
            /* Choose an idle worker to wake up if we haven't already. */
            worker = pgaio_choose_idle_worker();
            if (worker >= 0)
                wakeup = io_worker_control->workers[worker].latch;
 
            pgaio_debug_io(DEBUG4, ios[i],
                           "choosing worker %d",
                           worker);
        }
    }
    LWLockRelease(AioWorkerSubmissionQueueLock);
 
    if (wakeup)
        SetLatch(wakeup);
 
    /* Run whatever is left synchronously. */
    if (nsync > 0)
    {
        for (int i = 0; i < nsync; ++i)
        {
            pgaio_io_perform_synchronously(synchronous_ios[i]);
        }
    }
}

References Assert(), DEBUG4, i, io_worker_control, AioWorkerSlot::latch, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pgaio_choose_idle_worker(), pgaio_debug_io, pgaio_io_perform_synchronously(), PGAIO_SUBMIT_BATCH_SIZE, pgaio_worker_needs_synchronous_execution(), pgaio_worker_submission_queue_insert(), SetLatch(), wakeup, and AioWorkerControl::workers.

Referenced by pgaio_worker_submit().

pgaio_workers_enabled()

bool pgaio_workers_enabled

(

void

)

Definition at line 529 of file method_worker.c.

{
    return io_method == IOMETHOD_WORKER;
}

References io_method, and IOMETHOD_WORKER.

Referenced by maybe_adjust_io_workers().

Variable Documentation

io_worker_control

AioWorkerControl* io_worker_control

static

Definition at line 98 of file method_worker.c.

Referenced by IoWorkerMain(), pgaio_choose_idle_worker(), pgaio_worker_die(), pgaio_worker_register(), pgaio_worker_shmem_init(), and pgaio_worker_submit_internal().

io_worker_queue_size

int io_worker_queue_size = 64

static

Definition at line 95 of file method_worker.c.

Referenced by pgaio_worker_queue_shmem_size().

io_worker_submission_queue

AioWorkerSubmissionQueue* io_worker_submission_queue

static

Definition at line 97 of file method_worker.c.

Referenced by pgaio_worker_shmem_init(), pgaio_worker_submission_queue_consume(), pgaio_worker_submission_queue_depth(), and pgaio_worker_submission_queue_insert().

io_workers

int io_workers = 3

Definition at line 92 of file method_worker.c.

Referenced by maybe_adjust_io_workers().

MyIoWorkerId

int MyIoWorkerId

static

Definition at line 96 of file method_worker.c.

Referenced by IoWorkerMain(), pgaio_worker_die(), and pgaio_worker_register().

pgaio_worker_ops

const IoMethodOps pgaio_worker_ops

Initial value:

= {
    .shmem_size = pgaio_worker_shmem_size,
    .shmem_init = pgaio_worker_shmem_init,
 
    .needs_synchronous_execution = pgaio_worker_needs_synchronous_execution,
    .submit = pgaio_worker_submit,
}

Definition at line 82 of file method_worker.c.