unicode_norm.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Enumerations
enum	UnicodeNormalizationForm { UNICODE_NFC = 0 , UNICODE_NFD = 1 , UNICODE_NFKC = 2 , UNICODE_NFKD = 3 }
enum	UnicodeNormalizationQC { UNICODE_NORM_QC_NO = 0 , UNICODE_NORM_QC_YES = 1 , UNICODE_NORM_QC_MAYBE = -1 }

Functions
char32_t *	unicode_normalize (UnicodeNormalizationForm form, const char32_t *input)
UnicodeNormalizationQC	unicode_is_normalized_quickcheck (UnicodeNormalizationForm form, const char32_t *input)

Enumeration Type Documentation

UnicodeNormalizationForm

enum UnicodeNormalizationForm

Enumerator
UNICODE_NFC
UNICODE_NFD
UNICODE_NFKC
UNICODE_NFKD

Definition at line 17 of file unicode_norm.h.

{
    UNICODE_NFC = 0,
    UNICODE_NFD = 1,
    UNICODE_NFKC = 2,
    UNICODE_NFKD = 3,
} UnicodeNormalizationForm;

UnicodeNormalizationQC

enum UnicodeNormalizationQC

Enumerator
UNICODE_NORM_QC_NO
UNICODE_NORM_QC_YES
UNICODE_NORM_QC_MAYBE

Definition at line 26 of file unicode_norm.h.

{
    UNICODE_NORM_QC_NO = 0,
    UNICODE_NORM_QC_YES = 1,
    UNICODE_NORM_QC_MAYBE = -1,
} UnicodeNormalizationQC;

Function Documentation

unicode_is_normalized_quickcheck()

UnicodeNormalizationQC unicode_is_normalized_quickcheck ( UnicodeNormalizationForm  form, const char32_t *  input  )

extern

Definition at line 615 of file unicode_norm.c.

{
    uint8       lastCanonicalClass = 0;
    UnicodeNormalizationQC result = UNICODE_NORM_QC_YES;
 
    /*
     * For the "D" forms, we don't run the quickcheck.  We don't include the
     * lookup tables for those because they are huge, checking for these
     * particular forms is less common, and running the slow path is faster
     * for the "D" forms than the "C" forms because you don't need to
     * recompose, which is slow.
     */
    if (form == UNICODE_NFD || form == UNICODE_NFKD)
        return UNICODE_NORM_QC_MAYBE;
 
    for (const char32_t *p = input; *p; p++)
    {
        char32_t    ch = *p;
        uint8       canonicalClass;
        UnicodeNormalizationQC check;
 
        canonicalClass = get_canonical_class(ch);
        if (lastCanonicalClass > canonicalClass && canonicalClass != 0)
            return UNICODE_NORM_QC_NO;
 
        check = qc_is_allowed(form, ch);
        if (check == UNICODE_NORM_QC_NO)
            return UNICODE_NORM_QC_NO;
        else if (check == UNICODE_NORM_QC_MAYBE)
            result = UNICODE_NORM_QC_MAYBE;
 
        lastCanonicalClass = canonicalClass;
    }
    return result;
}

References fb(), get_canonical_class(), input, qc_is_allowed(), result, UNICODE_NFD, UNICODE_NFKD, UNICODE_NORM_QC_MAYBE, UNICODE_NORM_QC_NO, and UNICODE_NORM_QC_YES.

Referenced by unicode_is_normalized().

unicode_normalize()

char32_t * unicode_normalize ( UnicodeNormalizationForm  form, const char32_t *  input  )

extern

Definition at line 401 of file unicode_norm.c.

{
    bool        compat = (form == UNICODE_NFKC || form == UNICODE_NFKD);
    bool        recompose = (form == UNICODE_NFC || form == UNICODE_NFKC);
    char32_t   *decomp_chars;
    char32_t   *recomp_chars;
    int         decomp_size,
                current_size;
    int         count;
    const char32_t *p;
 
    /* variables for recomposition */
    int         last_class;
    int         starter_pos;
    int         target_pos;
    uint32      starter_ch;
 
    /* First, do character decomposition */
 
    /*
     * Calculate how many characters long the decomposed version will be.
     *
     * Some characters decompose to quite a few code points, so that the
     * decomposed version's size could overrun MaxAllocSize, and even 32-bit
     * size_t, even though the input string presumably fits in that.  In
     * frontend we want to just return NULL in that case, so monitor the sum
     * and exit early once we'd need more than MaxAllocSize bytes.
     */
    decomp_size = 0;
    for (p = input; *p; p++)
    {
        decomp_size += get_decomposed_size(*p, compat);
        if (unlikely(decomp_size > MaxAllocSize / sizeof(char32_t)))
        {
#ifndef FRONTEND
            /* Exit loop and let palloc() throw error below */
            break;
#else
            /* Just return NULL with no explicit error */
            return NULL;
#endif
        }
    }
 
    decomp_chars = (char32_t *) ALLOC((decomp_size + 1) * sizeof(char32_t));
    if (decomp_chars == NULL)
        return NULL;
 
    /*
     * Now fill in each entry recursively. This needs a second pass on the
     * decomposition table.
     */
    current_size = 0;
    for (p = input; *p; p++)
        decompose_code(*p, compat, &decomp_chars, &current_size);
    decomp_chars[decomp_size] = '\0';
    Assert(decomp_size == current_size);
 
    /* Leave if there is nothing to decompose */
    if (decomp_size == 0)
        return decomp_chars;
 
    /*
     * Now apply canonical ordering.
     */
    for (count = 1; count < decomp_size; count++)
    {
        char32_t    prev = decomp_chars[count - 1];
        char32_t    next = decomp_chars[count];
        char32_t    tmp;
        const uint8 prevClass = get_canonical_class(prev);
        const uint8 nextClass = get_canonical_class(next);
 
        /*
         * Per Unicode (https://www.unicode.org/reports/tr15/tr15-18.html)
         * annex 4, a sequence of two adjacent characters in a string is an
         * exchangeable pair if the combining class (from the Unicode
         * Character Database) for the first character is greater than the
         * combining class for the second, and the second is not a starter.  A
         * character is a starter if its combining class is 0.
         */
        if (prevClass == 0 || nextClass == 0)
            continue;
 
        if (prevClass <= nextClass)
            continue;
 
        /* exchange can happen */
        tmp = decomp_chars[count - 1];
        decomp_chars[count - 1] = decomp_chars[count];
        decomp_chars[count] = tmp;
 
        /* backtrack to check again */
        if (count > 1)
            count -= 2;
    }
 
    if (!recompose)
        return decomp_chars;
 
    /*
     * The last phase of NFC and NFKC is the recomposition of the reordered
     * Unicode string using combining classes. The recomposed string cannot be
     * longer than the decomposed one, so make the allocation of the output
     * string based on that assumption.
     */
    recomp_chars = (char32_t *) ALLOC((decomp_size + 1) * sizeof(char32_t));
    if (!recomp_chars)
    {
        FREE(decomp_chars);
        return NULL;
    }
 
    last_class = -1;            /* this eliminates a special check */
    starter_pos = 0;
    target_pos = 1;
    starter_ch = recomp_chars[0] = decomp_chars[0];
 
    for (count = 1; count < decomp_size; count++)
    {
        char32_t    ch = decomp_chars[count];
        int         ch_class = get_canonical_class(ch);
        char32_t    composite;
 
        if (last_class < ch_class &&
            recompose_code(starter_ch, ch, &composite))
        {
            recomp_chars[starter_pos] = composite;
            starter_ch = composite;
        }
        else if (ch_class == 0)
        {
            starter_pos = target_pos;
            starter_ch = ch;
            last_class = -1;
            recomp_chars[target_pos++] = ch;
        }
        else
        {
            last_class = ch_class;
            recomp_chars[target_pos++] = ch;
        }
    }
    recomp_chars[target_pos] = (char32_t) '\0';
 
    FREE(decomp_chars);
 
    return recomp_chars;
}

References ALLOC, Assert, compat, current_size, decompose_code(), fb(), FREE, get_canonical_class(), get_decomposed_size(), input, MaxAllocSize, next, recompose_code(), UNICODE_NFC, UNICODE_NFKC, UNICODE_NFKD, and unlikely.

Referenced by main(), pg_saslprep(), unicode_is_normalized(), and unicode_normalize_func().