pg_crc32c_armv8.c

Go to the documentation of this file.

/*-------------------------------------------------------------------------
 *
 * pg_crc32c_armv8.c
 *    Compute CRC-32C checksum using ARMv8 CRC Extension instructions
 *
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 *
 * IDENTIFICATION
 *    src/port/pg_crc32c_armv8.c
 *
 *-------------------------------------------------------------------------
 */
#include "c.h"
 
#ifdef _MSC_VER
#include <intrin.h>
#else
#include <arm_acle.h>
#endif
 
#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
#include <arm_neon.h>
#endif
 
#include "port/pg_crc32c.h"
 
pg_crc32c
pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len)
{
    const unsigned char *p = data;
    const unsigned char *pend = p + len;
 
    /*
     * ARMv8 doesn't require alignment, but aligned memory access is
     * significantly faster. Process leading bytes so that the loop below
     * starts with a pointer aligned to eight bytes.
     */
    if (!PointerIsAligned(p, uint16) &&
        p + 1 <= pend)
    {
        crc = __crc32cb(crc, *p);
        p += 1;
    }
    if (!PointerIsAligned(p, uint32) &&
        p + 2 <= pend)
    {
        crc = __crc32ch(crc, *(const uint16 *) p);
        p += 2;
    }
    if (!PointerIsAligned(p, uint64) &&
        p + 4 <= pend)
    {
        crc = __crc32cw(crc, *(const uint32 *) p);
        p += 4;
    }
 
    /* Process eight bytes at a time, as far as we can. */
    while (p + 8 <= pend)
    {
        crc = __crc32cd(crc, *(const uint64 *) p);
        p += 8;
    }
 
    /* Process remaining 0-7 bytes. */
    if (p + 4 <= pend)
    {
        crc = __crc32cw(crc, *(const uint32 *) p);
        p += 4;
    }
    if (p + 2 <= pend)
    {
        crc = __crc32ch(crc, *(const uint16 *) p);
        p += 2;
    }
    if (p < pend)
    {
        crc = __crc32cb(crc, *p);
    }
 
    return crc;
}
 
#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
 
/*
 * Note: There is no copyright notice in the following generated code.
 *
 * We have modified the output to
 *   - match our function declaration
 *   - match whitespace to our project style
 *   - be more friendly for pgindent
 *   - exit early for small inputs
 */
 
/* Generated by https://github.com/corsix/fast-crc32/ using: */
/* ./generate -i neon -p crc32c -a v4e */
/* MIT licensed */
 
pg_attribute_target("+crypto")
static inline
uint64x2_t
clmul_lo_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
{
    uint64x2_t  r;
 
__asm("pmull %0.1q, %2.1d, %3.1d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
    return r;
}
 
pg_attribute_target("+crypto")
static inline
uint64x2_t
clmul_hi_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
{
    uint64x2_t  r;
 
__asm("pmull2 %0.1q, %2.2d, %3.2d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
    return r;
}
 
pg_attribute_target("+crypto")
pg_crc32c
pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len)
{
    /* adjust names to match generated code */
    pg_crc32c   crc0 = crc;
    const char *buf = data;
 
    /*
     * Immediately fall back to the scalar path if the vector path is not
     * guaranteed to perform at least one iteration after the alignment
     * preamble.
     */
    if (len < 5 * sizeof(uint64x2_t))
        return pg_comp_crc32c_armv8(crc, data, len);
 
    /* align to 16 bytes */
    for (; (uintptr_t) buf & 7; --len)
    {
        crc0 = __crc32cb(crc0, *buf++);
    }
    if ((uintptr_t) buf & 8)
    {
        crc0 = __crc32cd(crc0, *(const uint64_t *) buf);
        buf += 8;
        len -= 8;
    }
 
    Assert(len >= 64);
 
    {
        const char *end = buf + len;
        const char *limit = buf + len - 64;
 
        /* First vector chunk. */
        uint64x2_t  x0 = vld1q_u64((const uint64_t *) buf),
                    y0;
        uint64x2_t  x1 = vld1q_u64((const uint64_t *) (buf + 16)),
                    y1;
        uint64x2_t  x2 = vld1q_u64((const uint64_t *) (buf + 32)),
                    y2;
        uint64x2_t  x3 = vld1q_u64((const uint64_t *) (buf + 48)),
                    y3;
        uint64x2_t  k;
 
        {
            static const uint64_t pg_attribute_aligned(16) k_[] = {0x740eef02, 0x9e4addf8};
 
            k = vld1q_u64(k_);
        }
 
        /*
         * pgindent complained of unmatched parens, so the following has been
         * re-written with intrinsics:
         *
         * x0 = veorq_u64((uint64x2_t) {crc0, 0}, x0);
         */
        x0 = veorq_u64((uint64x2_t) vsetq_lane_u64(crc0, vdupq_n_u64(0), 0), x0);
        buf += 64;
 
        /* Main loop. */
        while (buf <= limit)
        {
            y0 = clmul_lo_e(x0, k, vld1q_u64((const uint64_t *) buf)), x0 = clmul_hi_e(x0, k, y0);
            y1 = clmul_lo_e(x1, k, vld1q_u64((const uint64_t *) (buf + 16))), x1 = clmul_hi_e(x1, k, y1);
            y2 = clmul_lo_e(x2, k, vld1q_u64((const uint64_t *) (buf + 32))), x2 = clmul_hi_e(x2, k, y2);
            y3 = clmul_lo_e(x3, k, vld1q_u64((const uint64_t *) (buf + 48))), x3 = clmul_hi_e(x3, k, y3);
            buf += 64;
        }
 
        /* Reduce x0 ... x3 to just x0. */
        {
            static const uint64_t pg_attribute_aligned(16) k_[] = {0xf20c0dfe, 0x493c7d27};
 
            k = vld1q_u64(k_);
        }
        y0 = clmul_lo_e(x0, k, x1), x0 = clmul_hi_e(x0, k, y0);
        y2 = clmul_lo_e(x2, k, x3), x2 = clmul_hi_e(x2, k, y2);
        {
            static const uint64_t pg_attribute_aligned(16) k_[] = {0x3da6d0cb, 0xba4fc28e};
 
            k = vld1q_u64(k_);
        }
        y0 = clmul_lo_e(x0, k, x2), x0 = clmul_hi_e(x0, k, y0);
 
        /* Reduce 128 bits to 32 bits, and multiply by x^32. */
        crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0));
        crc0 = __crc32cd(crc0, vgetq_lane_u64(x0, 1));
        len = end - buf;
    }
 
    return pg_comp_crc32c_armv8(crc0, buf, len);
}
 
#endif