wchar.c File Reference

#include "c.h"
#include <limits.h>
#include "mb/pg_wchar.h"
#include "utils/ascii.h"
#include "common/unicode_nonspacing_table.h"
#include "common/unicode_east_asian_fw_table.h"

Include dependency graph for wchar.c:

Go to the source code of this file.

Data Structures
struct	mbinterval

Macros
#define	NONUTF8_INVALID_BYTE0   (0x8d)
#define	NONUTF8_INVALID_BYTE1   (' ')
#define	IS_EUC_RANGE_VALID(c)   ((c) >= 0xa1 && (c) <= 0xfe)
#define	pg_euccn_verifychar   pg_euckr_verifychar
#define	pg_euccn_verifystr   pg_euckr_verifystr
#define	ERR   0
#define	BGN   11
#define	CS1   16
#define	CS2   1
#define	CS3   5
#define	P3A   6 /* Lead was E0, check for 3-byte overlong */
#define	P3B   20 /* Lead was ED, check for surrogate */
#define	P4A   25 /* Lead was F0, check for 4-byte overlong */
#define	P4B   30 /* Lead was F4, check for too-large */
#define	END   BGN
#define	ASC   (END << BGN)
#define	L2A   (CS1 << BGN)
#define	L3A   (P3A << BGN)
#define	L3B   (CS2 << BGN)
#define	L3C   (P3B << BGN)
#define	L4A   (P4A << BGN)
#define	L4B   (CS3 << BGN)
#define	L4C   (P4B << BGN)
#define	CR1   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4B)
#define	CR2   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4A)
#define	CR3   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3A) | (CS2 << P4A)
#define	ILL   ERR
#define	STRIDE_LENGTH   (2 * sizeof(Vector8))

Functions
static int	pg_ascii2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_ascii_mblen (const unsigned char *s)
static int	pg_ascii_dsplen (const unsigned char *s)
static int	pg_euc2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euc_mblen (const unsigned char *s)
static int	pg_euc_dsplen (const unsigned char *s)
static int	pg_eucjp2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_eucjp_mblen (const unsigned char *s)
static int	pg_eucjp_dsplen (const unsigned char *s)
static int	pg_euckr2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euckr_mblen (const unsigned char *s)
static int	pg_euckr_dsplen (const unsigned char *s)
static int	pg_euccn2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euccn_mblen (const unsigned char *s)
static int	pg_euccn_dsplen (const unsigned char *s)
static int	pg_euctw2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euctw_mblen (const unsigned char *s)
static int	pg_euctw_dsplen (const unsigned char *s)
static int	pg_wchar2euc_with_len (const pg_wchar *from, unsigned char *to, int len)
static int	pg_johab_mblen (const unsigned char *s)
static int	pg_johab_dsplen (const unsigned char *s)
static int	pg_utf2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_wchar2utf_with_len (const pg_wchar *from, unsigned char *to, int len)
int	pg_utf_mblen (const unsigned char *s)
static int	mbbisearch (pg_wchar ucs, const struct mbinterval *table, int max)
static int	ucs_wcwidth (pg_wchar ucs)
static int	pg_utf_dsplen (const unsigned char *s)
static int	pg_mule2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_wchar2mule_with_len (const pg_wchar *from, unsigned char *to, int len)
int	pg_mule_mblen (const unsigned char *s)
static int	pg_mule_dsplen (const unsigned char *s)
static int	pg_latin12wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_wchar2single_with_len (const pg_wchar *from, unsigned char *to, int len)
static int	pg_latin1_mblen (const unsigned char *s)
static int	pg_latin1_dsplen (const unsigned char *s)
static int	pg_sjis_mblen (const unsigned char *s)
static int	pg_sjis_dsplen (const unsigned char *s)
static int	pg_big5_mblen (const unsigned char *s)
static int	pg_big5_dsplen (const unsigned char *s)
static int	pg_gbk_mblen (const unsigned char *s)
static int	pg_gbk_dsplen (const unsigned char *s)
static int	pg_uhc_mblen (const unsigned char *s)
static int	pg_uhc_dsplen (const unsigned char *s)
static int	pg_gb18030_mblen (const unsigned char *s)
static int	pg_gb18030_dsplen (const unsigned char *s)
static int	pg_ascii_verifychar (const unsigned char *s, int len)
static int	pg_ascii_verifystr (const unsigned char *s, int len)
static int	pg_eucjp_verifychar (const unsigned char *s, int len)
static int	pg_eucjp_verifystr (const unsigned char *s, int len)
static int	pg_euckr_verifychar (const unsigned char *s, int len)
static int	pg_euckr_verifystr (const unsigned char *s, int len)
static int	pg_euctw_verifychar (const unsigned char *s, int len)
static int	pg_euctw_verifystr (const unsigned char *s, int len)
static int	pg_johab_verifychar (const unsigned char *s, int len)
static int	pg_johab_verifystr (const unsigned char *s, int len)
static int	pg_mule_verifychar (const unsigned char *s, int len)
static int	pg_mule_verifystr (const unsigned char *s, int len)
static int	pg_latin1_verifychar (const unsigned char *s, int len)
static int	pg_latin1_verifystr (const unsigned char *s, int len)
static int	pg_sjis_verifychar (const unsigned char *s, int len)
static int	pg_sjis_verifystr (const unsigned char *s, int len)
static int	pg_big5_verifychar (const unsigned char *s, int len)
static int	pg_big5_verifystr (const unsigned char *s, int len)
static int	pg_gbk_verifychar (const unsigned char *s, int len)
static int	pg_gbk_verifystr (const unsigned char *s, int len)
static int	pg_uhc_verifychar (const unsigned char *s, int len)
static int	pg_uhc_verifystr (const unsigned char *s, int len)
static int	pg_gb18030_verifychar (const unsigned char *s, int len)
static int	pg_gb18030_verifystr (const unsigned char *s, int len)
static int	pg_utf8_verifychar (const unsigned char *s, int len)
static void	utf8_advance (const unsigned char *s, uint32 *state, int len)
static int	pg_utf8_verifystr (const unsigned char *s, int len)
bool	pg_utf8_islegal (const unsigned char *source, int length)
void	pg_encoding_set_invalid (int encoding, char *dst)
int	pg_encoding_mblen (int encoding, const char *mbstr)
int	pg_encoding_mblen_or_incomplete (int encoding, const char *mbstr, size_t remaining)
int	pg_encoding_mblen_bounded (int encoding, const char *mbstr)
int	pg_encoding_dsplen (int encoding, const char *mbstr)
int	pg_encoding_verifymbchar (int encoding, const char *mbstr, int len)
int	pg_encoding_verifymbstr (int encoding, const char *mbstr, int len)
int	pg_encoding_max_length (int encoding)

Variables
static const uint32	Utf8Transition [256]
const pg_wchar_tbl	pg_wchar_table []

Macro Definition Documentation

ASC

#define ASC   (END << BGN)

Definition at line 1797 of file wchar.c.

BGN

#define BGN   11

Definition at line 1781 of file wchar.c.

CR1

#define CR1   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4B)

Definition at line 1809 of file wchar.c.

CR2

#define CR2   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4A)

Definition at line 1810 of file wchar.c.

CR3

#define CR3   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3A) | (CS2 << P4A)

Definition at line 1811 of file wchar.c.

CS1

#define CS1   16

Definition at line 1783 of file wchar.c.

CS2

#define CS2   1

Definition at line 1784 of file wchar.c.

CS3

#define CS3   5

Definition at line 1785 of file wchar.c.

END

#define END   BGN

Definition at line 1792 of file wchar.c.

ERR

#define ERR   0

Definition at line 1779 of file wchar.c.

ILL

#define ILL   ERR

Definition at line 1813 of file wchar.c.

IS_EUC_RANGE_VALID

#define IS_EUC_RANGE_VALID

(

c

)

((c) >= 0xa1 && (c) <= 0xfe)

Definition at line 1079 of file wchar.c.

L2A

#define L2A   (CS1 << BGN)

Definition at line 1799 of file wchar.c.

L3A

#define L3A   (P3A << BGN)

Definition at line 1801 of file wchar.c.

L3B

#define L3B   (CS2 << BGN)

Definition at line 1802 of file wchar.c.

L3C

#define L3C   (P3B << BGN)

Definition at line 1803 of file wchar.c.

L4A

#define L4A   (P4A << BGN)

Definition at line 1805 of file wchar.c.

L4B

#define L4B   (CS3 << BGN)

Definition at line 1806 of file wchar.c.

L4C

#define L4C   (P4B << BGN)

Definition at line 1807 of file wchar.c.

NONUTF8_INVALID_BYTE0

#define NONUTF8_INVALID_BYTE0   (0x8d)

Definition at line 36 of file wchar.c.

NONUTF8_INVALID_BYTE1

#define NONUTF8_INVALID_BYTE1   (' ')

Definition at line 37 of file wchar.c.

P3A

#define P3A   6 /* Lead was E0, check for 3-byte overlong */

Definition at line 1787 of file wchar.c.

P3B

#define P3B   20 /* Lead was ED, check for surrogate */

Definition at line 1788 of file wchar.c.

P4A

#define P4A   25 /* Lead was F0, check for 4-byte overlong */

Definition at line 1789 of file wchar.c.

P4B

#define P4B   30 /* Lead was F4, check for too-large */

Definition at line 1790 of file wchar.c.

pg_euccn_verifychar

#define pg_euccn_verifychar   pg_euckr_verifychar

Definition at line 1224 of file wchar.c.

pg_euccn_verifystr

#define pg_euccn_verifystr   pg_euckr_verifystr

Definition at line 1225 of file wchar.c.

STRIDE_LENGTH

#define STRIDE_LENGTH   (2 * sizeof(Vector8))

Function Documentation

mbbisearch()

static int mbbisearch ( pg_wchar  ucs, const struct mbinterval *  table, int  max  )

static

Definition at line 581 of file wchar.c.

{
    int         min = 0;
    int         mid;
 
    if (ucs < table[0].first || ucs > table[max].last)
        return 0;
    while (max >= min)
    {
        mid = (min + max) / 2;
        if (ucs > table[mid].last)
            min = mid + 1;
        else if (ucs < table[mid].first)
            max = mid - 1;
        else
            return 1;
    }
 
    return 0;
}

References fb(), and table.

Referenced by ucs_wcwidth().

pg_ascii2wchar_with_len()

static int pg_ascii2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 70 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_ascii_dsplen()

static int pg_ascii_dsplen ( const unsigned char *  s )

static

Definition at line 91 of file wchar.c.

{
    if (*s == '\0')
        return 0;
    if (*s < 0x20 || *s == 0x7f)
        return -1;
 
    return 1;
}

Referenced by pg_big5_dsplen(), pg_euc_dsplen(), pg_euccn_dsplen(), pg_eucjp_dsplen(), pg_euctw_dsplen(), pg_gb18030_dsplen(), pg_gbk_dsplen(), pg_latin1_dsplen(), pg_sjis_dsplen(), and pg_uhc_dsplen().

pg_ascii_mblen()

static int pg_ascii_mblen ( const unsigned char *  s )

static

Definition at line 85 of file wchar.c.

{
    return 1;
}

pg_ascii_verifychar()

static int pg_ascii_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1063 of file wchar.c.

{
    return 1;
}

pg_ascii_verifystr()

static int pg_ascii_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1069 of file wchar.c.

{
    const unsigned char *nullpos = memchr(s, 0, len);
 
    if (nullpos == NULL)
        return len;
    else
        return nullpos - s;
}

References fb(), and len.

pg_big5_dsplen()

static int pg_big5_dsplen ( const unsigned char *  s )

static

Definition at line 934 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_big5_mblen()

static int pg_big5_mblen ( const unsigned char *  s )

static

Definition at line 922 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_big5_verifychar().

pg_big5_verifychar()

static int pg_big5_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1479 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_big5_mblen(s);
 
    if (len < l)
        return -1;
 
    if (l == 2 &&
        s[0] == NONUTF8_INVALID_BYTE0 &&
        s[1] == NONUTF8_INVALID_BYTE1)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References fb(), len, NONUTF8_INVALID_BYTE0, NONUTF8_INVALID_BYTE1, and pg_big5_mblen().

Referenced by pg_big5_verifystr().

pg_big5_verifystr()

static int pg_big5_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1504 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_big5_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_big5_verifychar(), and start.

pg_encoding_dsplen()

int pg_encoding_dsplen	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2176 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].dsplen((const unsigned char *) mbstr) :
            pg_wchar_table[PG_SQL_ASCII].dsplen((const unsigned char *) mbstr));
}

References pg_wchar_tbl::dsplen, encoding, fb(), PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by PQdsplen(), and reportErrorPosition().

pg_encoding_max_length()

int pg_encoding_max_length

(

int

encoding

)

Definition at line 2213 of file wchar.c.

{
    Assert(PG_VALID_ENCODING(encoding));
 
    /*
     * Check for the encoding despite the assert, due to some mingw versions
     * otherwise issuing bogus warnings.
     */
    return PG_VALID_ENCODING(encoding) ?
        pg_wchar_table[encoding].maxmblen :
        pg_wchar_table[PG_SQL_ASCII].maxmblen;
}

References Assert, encoding, pg_wchar_tbl::maxmblen, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by ascii(), chr(), CopyConvertBuf(), pg_encoding_mbcliplen(), pg_encoding_set_invalid(), pg_verify_mbstr_len(), reportErrorPosition(), test_enc_setup(), and type_maximum_size().

pg_encoding_mblen()

int pg_encoding_mblen	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2135 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mblen((const unsigned char *) mbstr) :
            pg_wchar_table[PG_SQL_ASCII].mblen((const unsigned char *) mbstr));
}

References encoding, fb(), pg_wchar_tbl::mblen, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), fmtIdEnc(), pg_encoding_mblen_bounded(), pg_encoding_mblen_or_incomplete(), PQescapeInternal(), PQmblen(), PQmblenBounded(), and test_enc_setup().

pg_encoding_mblen_bounded()

int pg_encoding_mblen_bounded	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2167 of file wchar.c.

{
    return strnlen(mbstr, pg_encoding_mblen(encoding, mbstr));
}

References encoding, fb(), and pg_encoding_mblen().

pg_encoding_mblen_or_incomplete()

int pg_encoding_mblen_or_incomplete	(	int	encoding,
		const char *	mbstr,
		size_t	remaining
	)

Definition at line 2147 of file wchar.c.

{
    /*
     * Define zero remaining as too few, even for single-byte encodings.
     * pg_gb18030_mblen() reads one or two bytes; single-byte encodings read
     * zero; others read one.
     */
    if (remaining < 1 ||
        (encoding == PG_GB18030 && IS_HIGHBIT_SET(*mbstr) && remaining < 2))
        return INT_MAX;
    return pg_encoding_mblen(encoding, mbstr);
}

References encoding, fb(), IS_HIGHBIT_SET, pg_encoding_mblen(), PG_GB18030, and remaining.

Referenced by PQescapeInternal(), PQescapeStringInternal(), report_invalid_encoding(), and report_untranslatable_char().

pg_encoding_set_invalid()

void pg_encoding_set_invalid	(	int	encoding,
		char *	dst
	)

Definition at line 2051 of file wchar.c.

{
    Assert(pg_encoding_max_length(encoding) > 1);
 
    dst[0] = (encoding == PG_UTF8 ? 0xc0 : NONUTF8_INVALID_BYTE0);
    dst[1] = NONUTF8_INVALID_BYTE1;
}

References Assert, encoding, fb(), NONUTF8_INVALID_BYTE0, NONUTF8_INVALID_BYTE1, pg_encoding_max_length(), and PG_UTF8.

Referenced by appendStringLiteral(), fmtIdEnc(), PQescapeStringInternal(), and test_enc_setup().

pg_encoding_verifymbchar()

int pg_encoding_verifymbchar	(	int	encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 2189 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mbverifychar((const unsigned char *) mbstr, len) :
            pg_wchar_table[PG_SQL_ASCII].mbverifychar((const unsigned char *) mbstr, len));
}

References encoding, fb(), len, pg_wchar_tbl::mbverifychar, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by appendStringLiteral(), big52euc_tw(), big52mic(), euc_jis_20042shift_jis_2004(), euc_jp2mic(), euc_jp2sjis(), euc_kr2mic(), euc_tw2big5(), euc_tw2mic(), fmtIdEnc(), LocalToUtf(), mic2big5(), mic2euc_jp(), mic2euc_kr(), mic2euc_tw(), mic2sjis(), PQescapeStringInternal(), shift_jis_20042euc_jis_2004(), and sjis2euc_jp().

pg_encoding_verifymbstr()

int pg_encoding_verifymbstr	(	int	encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 2202 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mbverifystr((const unsigned char *) mbstr, len) :
            pg_wchar_table[PG_SQL_ASCII].mbverifystr((const unsigned char *) mbstr, len));
}

References encoding, fb(), len, pg_wchar_tbl::mbverifystr, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by add_file_to_manifest(), CopyConvertBuf(), handle_oauth_sasl_error(), parse_oauth_json(), PQescapeInternal(), test_enc_conversion(), test_enc_setup(), and test_one_vector_escape().

pg_euc2wchar_with_len()

static int pg_euc2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 105 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 2)   /* JIS X 0201 (so called "1 byte
                                         * KANA") */
        {
            from++;
            *to = (SS2 << 8) | *from++;
            len -= 2;
        }
        else if (*from == SS3 && len >= 3)  /* JIS X 0212 KANJI */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* JIS X 0208 KANJI */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else                    /* must be ASCII */
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp2wchar_with_len(), and pg_euckr2wchar_with_len().

pg_euc_dsplen()

static int pg_euc_dsplen ( const unsigned char *  s )

inlinestatic

Definition at line 160 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

Referenced by pg_euckr_dsplen(), and pg_johab_dsplen().

pg_euc_mblen()

static int pg_euc_mblen ( const unsigned char *  s )

inlinestatic

Definition at line 144 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 3;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp_mblen(), pg_euckr_mblen(), and pg_johab_mblen().

pg_euccn2wchar_with_len()

static int pg_euccn2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 232 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 3)   /* code set 2 (unused?) */
        {
            from++;
            *to = (SS2 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (*from == SS3 && len >= 3)  /* code set 3 (unused ?) */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 1 */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euccn_dsplen()

static int pg_euccn_dsplen ( const unsigned char *  s )

static

Definition at line 283 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_euccn_mblen()

static int pg_euccn_mblen ( const unsigned char *  s )

static

Definition at line 271 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, and len.

pg_eucjp2wchar_with_len()

static int pg_eucjp2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 179 of file wchar.c.

{
    return pg_euc2wchar_with_len(from, to, len);
}

References len, and pg_euc2wchar_with_len().

pg_eucjp_dsplen()

static int pg_eucjp_dsplen ( const unsigned char *  s )

static

Definition at line 191 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 1;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

pg_eucjp_mblen()

static int pg_eucjp_mblen ( const unsigned char *  s )

static

Definition at line 185 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

pg_eucjp_verifychar()

static int pg_eucjp_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1082 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    switch (c1)
    {
        case SS2:               /* JIS X 0201 */
            l = 2;
            if (l > len)
                return -1;
            c2 = *s++;
            if (c2 < 0xa1 || c2 > 0xdf)
                return -1;
            break;
 
        case SS3:               /* JIS X 0212 */
            l = 3;
            if (l > len)
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            break;
 
        default:
            if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
            {
                l = 2;
                if (l > len)
                    return -1;
                if (!IS_EUC_RANGE_VALID(c1))
                    return -1;
                c2 = *s++;
                if (!IS_EUC_RANGE_VALID(c2))
                    return -1;
            }
            else
                /* must be ASCII */
            {
                l = 1;
            }
            break;
    }
 
    return l;
}

References fb(), IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp_verifystr().

pg_eucjp_verifystr()

static int pg_eucjp_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1137 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_eucjp_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_eucjp_verifychar(), and start.

pg_euckr2wchar_with_len()

static int pg_euckr2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 210 of file wchar.c.

{
    return pg_euc2wchar_with_len(from, to, len);
}

References len, and pg_euc2wchar_with_len().

pg_euckr_dsplen()

static int pg_euckr_dsplen ( const unsigned char *  s )

static

Definition at line 222 of file wchar.c.

{
    return pg_euc_dsplen(s);
}

References pg_euc_dsplen().

pg_euckr_mblen()

static int pg_euckr_mblen ( const unsigned char *  s )

static

Definition at line 216 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

pg_euckr_verifychar()

static int pg_euckr_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1166 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    if (IS_HIGHBIT_SET(c1))
    {
        l = 2;
        if (l > len)
            return -1;
        if (!IS_EUC_RANGE_VALID(c1))
            return -1;
        c2 = *s++;
        if (!IS_EUC_RANGE_VALID(c2))
            return -1;
    }
    else
        /* must be ASCII */
    {
        l = 1;
    }
 
    return l;
}

References fb(), IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, and len.

Referenced by pg_euckr_verifystr().

pg_euckr_verifystr()

static int pg_euckr_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1195 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_euckr_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_euckr_verifychar(), and start.

pg_euctw2wchar_with_len()

static int pg_euctw2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 299 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 4)   /* code set 2 */
        {
            from++;
            *to = (((uint32) SS2) << 24) | (*from++ << 16);
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 4;
        }
        else if (*from == SS3 && len >= 3)  /* code set 3 (unused?) */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 2 */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euctw_dsplen()

static int pg_euctw_dsplen ( const unsigned char *  s )

static

Definition at line 355 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

pg_euctw_mblen()

static int pg_euctw_mblen ( const unsigned char *  s )

static

Definition at line 339 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 4;
    else if (*s == SS3)
        len = 3;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euctw_verifychar()

static int pg_euctw_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1228 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    switch (c1)
    {
        case SS2:               /* CNS 11643 Plane 1-7 */
            l = 4;
            if (l > len)
                return -1;
            c2 = *s++;
            if (c2 < 0xa1 || c2 > 0xa7)
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            break;
 
        case SS3:               /* unused */
            return -1;
 
        default:
            if (IS_HIGHBIT_SET(c1)) /* CNS 11643 Plane 1 */
            {
                l = 2;
                if (l > len)
                    return -1;
                /* no further range check on c1? */
                c2 = *s++;
                if (!IS_EUC_RANGE_VALID(c2))
                    return -1;
            }
            else
                /* must be ASCII */
            {
                l = 1;
            }
            break;
    }
    return l;
}

References fb(), IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_euctw_verifystr().

pg_euctw_verifystr()

static int pg_euctw_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1278 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_euctw_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_euctw_verifychar(), and start.

pg_gb18030_dsplen()

static int pg_gb18030_dsplen ( const unsigned char *  s )

static

Definition at line 1029 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);   /* ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_gb18030_mblen()

static int pg_gb18030_mblen ( const unsigned char *  s )

static

Definition at line 1015 of file wchar.c.

{
    int         len;
 
    if (!IS_HIGHBIT_SET(*s))
        len = 1;                /* ASCII */
    else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
        len = 4;
    else
        len = 2;
    return len;
}

References IS_HIGHBIT_SET, and len.

pg_gb18030_verifychar()

static int pg_gb18030_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1641 of file wchar.c.

{
    int         l;
 
    if (!IS_HIGHBIT_SET(*s))
        l = 1;                  /* ASCII */
    else if (len >= 4 && *(s + 1) >= 0x30 && *(s + 1) <= 0x39)
    {
        /* Should be 4-byte, validate remaining bytes */
        if (*s >= 0x81 && *s <= 0xfe &&
            *(s + 2) >= 0x81 && *(s + 2) <= 0xfe &&
            *(s + 3) >= 0x30 && *(s + 3) <= 0x39)
            l = 4;
        else
            l = -1;
    }
    else if (len >= 2 && *s >= 0x81 && *s <= 0xfe)
    {
        /* Should be 2-byte, validate */
        if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) ||
            (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
            l = 2;
        else
            l = -1;
    }
    else
        l = -1;
    return l;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_gb18030_verifystr().

pg_gb18030_verifystr()

static int pg_gb18030_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1672 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_gb18030_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_gb18030_verifychar(), and start.

pg_gbk_dsplen()

static int pg_gbk_dsplen ( const unsigned char *  s )

static

Definition at line 961 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_gbk_mblen()

static int pg_gbk_mblen ( const unsigned char *  s )

static

Definition at line 949 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_gbk_verifychar().

pg_gbk_verifychar()

static int pg_gbk_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1533 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_gbk_mblen(s);
 
    if (len < l)
        return -1;
 
    if (l == 2 &&
        s[0] == NONUTF8_INVALID_BYTE0 &&
        s[1] == NONUTF8_INVALID_BYTE1)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References fb(), len, NONUTF8_INVALID_BYTE0, NONUTF8_INVALID_BYTE1, and pg_gbk_mblen().

Referenced by pg_gbk_verifystr().

pg_gbk_verifystr()

static int pg_gbk_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1558 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_gbk_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_gbk_verifychar(), and start.

pg_johab_dsplen()

static int pg_johab_dsplen ( const unsigned char *  s )

static

Definition at line 429 of file wchar.c.

{
    return pg_euc_dsplen(s);
}

References pg_euc_dsplen().

pg_johab_mblen()

static int pg_johab_mblen ( const unsigned char *  s )

static

Definition at line 423 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

Referenced by pg_johab_verifychar().

pg_johab_verifychar()

static int pg_johab_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1307 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c;
 
    l = mbl = pg_johab_mblen(s);
 
    if (len < l)
        return -1;
 
    if (!IS_HIGHBIT_SET(*s))
        return mbl;
 
    while (--l > 0)
    {
        c = *++s;
        if (!IS_EUC_RANGE_VALID(c))
            return -1;
    }
    return mbl;
}

References fb(), IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, and pg_johab_mblen().

Referenced by pg_johab_verifystr().

pg_johab_verifystr()

static int pg_johab_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1331 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_johab_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_johab_verifychar(), and start.

pg_latin12wchar_with_len()

static int pg_latin12wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 839 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_latin1_dsplen()

static int pg_latin1_dsplen ( const unsigned char *  s )

static

Definition at line 882 of file wchar.c.

{
    return pg_ascii_dsplen(s);
}

References pg_ascii_dsplen().

pg_latin1_mblen()

static int pg_latin1_mblen ( const unsigned char *  s )

static

Definition at line 876 of file wchar.c.

{
    return 1;
}

pg_latin1_verifychar()

static int pg_latin1_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1410 of file wchar.c.

{
    return 1;
}

pg_latin1_verifystr()

static int pg_latin1_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1416 of file wchar.c.

{
    const unsigned char *nullpos = memchr(s, 0, len);
 
    if (nullpos == NULL)
        return len;
    else
        return nullpos - s;
}

References fb(), and len.

pg_mule2wchar_with_len()

static int pg_mule2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 674 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (IS_LC1(*from) && len >= 2)
        {
            *to = *from++ << 16;
            *to |= *from++;
            len -= 2;
        }
        else if (IS_LCPRV1(*from) && len >= 3)
        {
            from++;
            *to = *from++ << 16;
            *to |= *from++;
            len -= 3;
        }
        else if (IS_LC2(*from) && len >= 3)
        {
            *to = *from++ << 16;
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 3;
        }
        else if (IS_LCPRV2(*from) && len >= 4)
        {
            from++;
            *to = *from++ << 16;
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 4;
        }
        else
        {                       /* assume ASCII */
            *to = (unsigned char) *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References fb(), IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

pg_mule_dsplen()

static int pg_mule_dsplen ( const unsigned char *  s )

static

Definition at line 811 of file wchar.c.

{
    int         len;
 
    /*
     * Note: it's not really appropriate to assume that all multibyte charsets
     * are double-wide on screen.  But this seems an okay approximation for
     * the MULE charsets we currently support.
     */
 
    if (IS_LC1(*s))
        len = 1;
    else if (IS_LCPRV1(*s))
        len = 1;
    else if (IS_LC2(*s))
        len = 2;
    else if (IS_LCPRV2(*s))
        len = 2;
    else
        len = 1;                /* assume ASCII */
 
    return len;
}

References IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

pg_mule_mblen()

int pg_mule_mblen

(

const unsigned char *

s

)

Definition at line 793 of file wchar.c.

{
    int         len;
 
    if (IS_LC1(*s))
        len = 2;
    else if (IS_LCPRV1(*s))
        len = 3;
    else if (IS_LC2(*s))
        len = 3;
    else if (IS_LCPRV2(*s))
        len = 4;
    else
        len = 1;                /* assume ASCII */
    return len;
}

References IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

Referenced by mic2latin(), mic2latin_with_table(), and pg_mule_verifychar().

pg_mule_verifychar()

static int pg_mule_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1360 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c;
 
    l = mbl = pg_mule_mblen(s);
 
    if (len < l)
        return -1;
 
    while (--l > 0)
    {
        c = *++s;
        if (!IS_HIGHBIT_SET(c))
            return -1;
    }
    return mbl;
}

References fb(), IS_HIGHBIT_SET, len, and pg_mule_mblen().

Referenced by pg_mule_verifystr().

pg_mule_verifystr()

static int pg_mule_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1381 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_mule_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_mule_verifychar(), and start.

pg_sjis_dsplen()

static int pg_sjis_dsplen ( const unsigned char *  s )

static

Definition at line 905 of file wchar.c.

{
    int         len;
 
    if (*s >= 0xa1 && *s <= 0xdf)
        len = 1;                /* 1 byte kana? */
    else if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_sjis_mblen()

static int pg_sjis_mblen ( const unsigned char *  s )

static

Definition at line 891 of file wchar.c.

{
    int         len;
 
    if (*s >= 0xa1 && *s <= 0xdf)
        len = 1;                /* 1 byte kana? */
    else if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_sjis_verifychar().

pg_sjis_verifychar()

static int pg_sjis_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1427 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c1,
                c2;
 
    l = mbl = pg_sjis_mblen(s);
 
    if (len < l)
        return -1;
 
    if (l == 1)                 /* pg_sjis_mblen already verified it */
        return mbl;
 
    c1 = *s++;
    c2 = *s;
    if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
        return -1;
    return mbl;
}

References fb(), ISSJISHEAD, ISSJISTAIL, len, and pg_sjis_mblen().

Referenced by pg_sjis_verifystr().

pg_sjis_verifystr()

static int pg_sjis_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1450 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_sjis_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_sjis_verifychar(), and start.

pg_uhc_dsplen()

static int pg_uhc_dsplen ( const unsigned char *  s )

static

Definition at line 988 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* 2byte? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_uhc_mblen()

static int pg_uhc_mblen ( const unsigned char *  s )

static

Definition at line 976 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* 2byte? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_uhc_verifychar().

pg_uhc_verifychar()

static int pg_uhc_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1587 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_uhc_mblen(s);
 
    if (len < l)
        return -1;
 
    if (l == 2 &&
        s[0] == NONUTF8_INVALID_BYTE0 &&
        s[1] == NONUTF8_INVALID_BYTE1)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References fb(), len, NONUTF8_INVALID_BYTE0, NONUTF8_INVALID_BYTE1, and pg_uhc_mblen().

Referenced by pg_uhc_verifystr().

pg_uhc_verifystr()

static int pg_uhc_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1612 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_uhc_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, pg_uhc_verifychar(), and start.

pg_utf2wchar_with_len()

static int pg_utf2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 441 of file wchar.c.

{
    int         cnt = 0;
    uint32      c1,
                c2,
                c3,
                c4;
 
    while (len > 0 && *from)
    {
        if ((*from & 0x80) == 0)
        {
            *to = *from++;
            len--;
        }
        else if ((*from & 0xe0) == 0xc0)
        {
            if (len < 2)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x1f;
            c2 = *from++ & 0x3f;
            *to = (c1 << 6) | c2;
            len -= 2;
        }
        else if ((*from & 0xf0) == 0xe0)
        {
            if (len < 3)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x0f;
            c2 = *from++ & 0x3f;
            c3 = *from++ & 0x3f;
            *to = (c1 << 12) | (c2 << 6) | c3;
            len -= 3;
        }
        else if ((*from & 0xf8) == 0xf0)
        {
            if (len < 4)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x07;
            c2 = *from++ & 0x3f;
            c3 = *from++ & 0x3f;
            c4 = *from++ & 0x3f;
            *to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
            len -= 4;
        }
        else
        {
            /* treat a bogus char as length 1; not ours to raise error */
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References fb(), and len.

pg_utf8_islegal()

bool pg_utf8_islegal	(	const unsigned char *	source,
		int	length
	)

Definition at line 1989 of file wchar.c.

{
    unsigned char a;
 
    switch (length)
    {
        default:
            /* reject lengths 5 and 6 for now */
            return false;
        case 4:
            a = source[3];
            if (a < 0x80 || a > 0xBF)
                return false;
            /* FALL THRU */
        case 3:
            a = source[2];
            if (a < 0x80 || a > 0xBF)
                return false;
            /* FALL THRU */
        case 2:
            a = source[1];
            switch (*source)
            {
                case 0xE0:
                    if (a < 0xA0 || a > 0xBF)
                        return false;
                    break;
                case 0xED:
                    if (a < 0x80 || a > 0x9F)
                        return false;
                    break;
                case 0xF0:
                    if (a < 0x90 || a > 0xBF)
                        return false;
                    break;
                case 0xF4:
                    if (a < 0x80 || a > 0x8F)
                        return false;
                    break;
                default:
                    if (a < 0x80 || a > 0xBF)
                        return false;
                    break;
            }
            /* FALL THRU */
        case 1:
            a = *source;
            if (a >= 0x80 && a < 0xC2)
                return false;
            if (a > 0xF4)
                return false;
            break;
    }
    return true;
}

References a, fb(), and source.

Referenced by chr(), pg_utf8_string_len(), pg_utf8_verifychar(), utf8_to_iso8859_1(), and UtfToLocal().

pg_utf8_verifychar()

static int pg_utf8_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1701 of file wchar.c.

{
    int         l;
 
    if ((*s & 0x80) == 0)
    {
        if (*s == '\0')
            return -1;
        return 1;
    }
    else if ((*s & 0xe0) == 0xc0)
        l = 2;
    else if ((*s & 0xf0) == 0xe0)
        l = 3;
    else if ((*s & 0xf8) == 0xf0)
        l = 4;
    else
        l = 1;
 
    if (l > len)
        return -1;
 
    if (!pg_utf8_islegal(s, l))
        return -1;
 
    return l;
}

References len, and pg_utf8_islegal().

Referenced by pg_utf8_verifystr().

pg_utf8_verifystr()

static int pg_utf8_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1891 of file wchar.c.

{
    const unsigned char *start = s;
    const int   orig_len = len;
    uint32      state = BGN;
 
/*
 * With a stride of two vector widths, gcc will unroll the loop. Even if
 * the compiler can unroll a longer loop, it's not worth it because we
 * must fall back to the byte-wise algorithm if we find any non-ASCII.
 */
#define STRIDE_LENGTH (2 * sizeof(Vector8))
 
    if (len >= STRIDE_LENGTH)
    {
        while (len >= STRIDE_LENGTH)
        {
            /*
             * If the chunk is all ASCII, we can skip the full UTF-8 check,
             * but we must first check for a non-END state, which means the
             * previous chunk ended in the middle of a multibyte sequence.
             */
            if (state != END || !is_valid_ascii(s, STRIDE_LENGTH))
                utf8_advance(s, &state, STRIDE_LENGTH);
 
            s += STRIDE_LENGTH;
            len -= STRIDE_LENGTH;
        }
 
        /* The error state persists, so we only need to check for it here. */
        if (state == ERR)
        {
            /*
             * Start over from the beginning with the slow path so we can
             * count the valid bytes.
             */
            len = orig_len;
            s = start;
        }
        else if (state != END)
        {
            /*
             * The fast path exited in the middle of a multibyte sequence.
             * Walk backwards to find the leading byte so that the slow path
             * can resume checking from there. We must always backtrack at
             * least one byte, since the current byte could be e.g. an ASCII
             * byte after a 2-byte lead, which is invalid.
             */
            do
            {
                Assert(s > start);
                s--;
                len++;
                Assert(IS_HIGHBIT_SET(*s));
            } while (pg_utf_mblen(s) <= 1);
        }
    }
 
    /* check remaining bytes */
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_utf8_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References Assert, BGN, END, ERR, fb(), IS_HIGHBIT_SET, is_valid_ascii(), len, pg_utf8_verifychar(), pg_utf_mblen, start, STRIDE_LENGTH, and utf8_advance().

pg_utf_dsplen()

static int pg_utf_dsplen ( const unsigned char *  s )

static

Definition at line 662 of file wchar.c.

{
    return ucs_wcwidth(utf8_to_unicode(s));
}

References ucs_wcwidth(), and utf8_to_unicode().

pg_utf_mblen()

int pg_utf_mblen

(

const unsigned char *

s

)

Definition at line 538 of file wchar.c.

{
    int         len;
 
    if ((*s & 0x80) == 0)
        len = 1;
    else if ((*s & 0xe0) == 0xc0)
        len = 2;
    else if ((*s & 0xf0) == 0xe0)
        len = 3;
    else if ((*s & 0xf8) == 0xf0)
        len = 4;
#ifdef NOT_USED
    else if ((*s & 0xfc) == 0xf8)
        len = 5;
    else if ((*s & 0xfe) == 0xfc)
        len = 6;
#endif
    else
        len = 1;
    return len;
}

References len.

pg_wchar2euc_with_len()

static int pg_wchar2euc_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 377 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        unsigned char c;
 
        if ((c = (*from >> 24)))
        {
            *to++ = c;
            *to++ = (*from >> 16) & 0xff;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else if ((c = (*from >> 16)))
        {
            *to++ = c;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if ((c = (*from >> 8)))
        {
            *to++ = c;
            *to++ = *from & 0xff;
            cnt += 2;
        }
        else
        {
            *to++ = *from;
            cnt++;
        }
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References len.

pg_wchar2mule_with_len()

static int pg_wchar2mule_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 727 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        unsigned char lb;
 
        lb = (*from >> 16) & 0xff;
        if (IS_LC1(lb))
        {
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 2;
        }
        else if (IS_LC2(lb))
        {
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV1_A_RANGE(lb))
        {
            *to++ = LCPRV1_A;
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV1_B_RANGE(lb))
        {
            *to++ = LCPRV1_B;
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV2_A_RANGE(lb))
        {
            *to++ = LCPRV2_A;
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else if (IS_LCPRV2_B_RANGE(lb))
        {
            *to++ = LCPRV2_B;
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else
        {
            *to++ = *from & 0xff;
            cnt += 1;
        }
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References IS_LC1, IS_LC2, IS_LCPRV1_A_RANGE, IS_LCPRV1_B_RANGE, IS_LCPRV2_A_RANGE, IS_LCPRV2_B_RANGE, LCPRV1_A, LCPRV1_B, LCPRV2_A, LCPRV2_B, and len.

pg_wchar2single_with_len()

static int pg_wchar2single_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 861 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_wchar2utf_with_len()

static int pg_wchar2utf_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 507 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        int         char_len;
 
        unicode_to_utf8(*from, to);
        char_len = pg_utf_mblen(to);
        cnt += char_len;
        to += char_len;
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References fb(), len, pg_utf_mblen, and unicode_to_utf8().

ucs_wcwidth()

static int ucs_wcwidth ( pg_wchar  ucs )

static

Definition at line 628 of file wchar.c.

{
#include "common/unicode_nonspacing_table.h"
#include "common/unicode_east_asian_fw_table.h"
 
    /* test for 8-bit control characters */
    if (ucs == 0)
        return 0;
 
    if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
        return -1;
 
    /*
     * binary search in table of non-spacing characters
     *
     * XXX: In the official Unicode sources, it is possible for a character to
     * be described as both non-spacing and wide at the same time. As of
     * Unicode 13.0, treating the non-spacing property as the determining
     * factor for display width leads to the correct behavior, so do that
     * search first.
     */
    if (mbbisearch(ucs, nonspacing,
                   sizeof(nonspacing) / sizeof(struct mbinterval) - 1))
        return 0;
 
    /* binary search in table of wide characters */
    if (mbbisearch(ucs, east_asian_fw,
                   sizeof(east_asian_fw) / sizeof(struct mbinterval) - 1))
        return 2;
 
    return 1;
}

References east_asian_fw, fb(), mbbisearch(), and nonspacing.

Referenced by pg_utf_dsplen().

utf8_advance()

static void utf8_advance ( const unsigned char *  s, uint32 *  state, int  len  )

static

Definition at line 1873 of file wchar.c.

{
    /* Note: We deliberately don't check the state's value here. */
    while (len > 0)
    {
        /*
         * It's important that the mask value is 31: In most instruction sets,
         * a shift by a 32-bit operand is understood to be a shift by its mod
         * 32, so the compiler should elide the mask operation.
         */
        *state = Utf8Transition[*s++] >> (*state & 31);
        len--;
    }
 
    *state &= 31;
}

References len, and Utf8Transition.

Referenced by pg_utf8_verifystr().

Variable Documentation

pg_wchar_table

const pg_wchar_tbl pg_wchar_table[]

Definition at line 2064 of file wchar.c.

                                      {
    [PG_SQL_ASCII] = {pg_ascii2wchar_with_len, pg_wchar2single_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifychar, pg_ascii_verifystr, 1},
    [PG_EUC_JP] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
    [PG_EUC_CN] = {pg_euccn2wchar_with_len, pg_wchar2euc_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifychar, pg_euccn_verifystr, 2},
    [PG_EUC_KR] = {pg_euckr2wchar_with_len, pg_wchar2euc_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifychar, pg_euckr_verifystr, 3},
    [PG_EUC_TW] = {pg_euctw2wchar_with_len, pg_wchar2euc_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifychar, pg_euctw_verifystr, 4},
    [PG_EUC_JIS_2004] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
    [PG_UTF8] = {pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifychar, pg_utf8_verifystr, 4},
    [PG_MULE_INTERNAL] = {pg_mule2wchar_with_len, pg_wchar2mule_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifychar, pg_mule_verifystr, 4},
    [PG_LATIN1] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN2] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN3] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN4] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN9] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_LATIN10] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1256] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1258] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN866] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN874] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_KOI8R] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1251] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1252] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_ISO_8859_5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_ISO_8859_6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_ISO_8859_7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_ISO_8859_8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1250] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1253] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1254] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1255] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_WIN1257] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_KOI8U] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
    [PG_SJIS] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
    [PG_BIG5] = {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifychar, pg_big5_verifystr, 2},
    [PG_GBK] = {0, 0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifychar, pg_gbk_verifystr, 2},
    [PG_UHC] = {0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifychar, pg_uhc_verifystr, 2},
    [PG_GB18030] = {0, 0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifychar, pg_gb18030_verifystr, 4},
    [PG_JOHAB] = {0, 0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifychar, pg_johab_verifystr, 3},
    [PG_SHIFT_JIS_2004] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
};

Referenced by pg_database_encoding_max_length(), pg_dsplen(), pg_encoding_dsplen(), pg_encoding_max_length(), pg_encoding_max_length_sql(), pg_encoding_mb2wchar_with_len(), pg_encoding_mbcliplen(), pg_encoding_mblen(), pg_encoding_verifymbchar(), pg_encoding_verifymbstr(), pg_encoding_wchar2mb_with_len(), pg_generic_charinc(), pg_mb2wchar(), pg_mb2wchar_with_len(), pg_mblen(), pg_verify_mbstr(), pg_verify_mbstr_len(), pg_wchar2mb(), and pg_wchar2mb_with_len().

Utf8Transition

const uint32 Utf8Transition[256]

static

Definition at line 1815 of file wchar.c.

{
    /* ASCII */
 
    ILL, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
 
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
 
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
 
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
 
    /* continuation bytes */
 
    /* 80..8F */
    CR1, CR1, CR1, CR1, CR1, CR1, CR1, CR1,
    CR1, CR1, CR1, CR1, CR1, CR1, CR1, CR1,
 
    /* 90..9F */
    CR2, CR2, CR2, CR2, CR2, CR2, CR2, CR2,
    CR2, CR2, CR2, CR2, CR2, CR2, CR2, CR2,
 
    /* A0..BF */
    CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
    CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
    CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
    CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
 
    /* leading bytes */
 
    /* C0..DF */
    ILL, ILL, L2A, L2A, L2A, L2A, L2A, L2A,
    L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
    L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
    L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
 
    /* E0..EF */
    L3A, L3B, L3B, L3B, L3B, L3B, L3B, L3B,
    L3B, L3B, L3B, L3B, L3B, L3C, L3B, L3B,
 
    /* F0..FF */
    L4A, L4B, L4B, L4B, L4C, ILL, ILL, ILL,
    ILL, ILL, ILL, ILL, ILL, ILL, ILL, ILL
};

Referenced by utf8_advance().