wchar.c File Reference

#include "c.h"
#include "mb/pg_wchar.h"
#include "common/unicode_nonspacing_table.h"
#include "common/unicode_east_asian_fw_table.h"

Include dependency graph for wchar.c:

Go to the source code of this file.

Data Structures
struct	mbinterval

Macros
#define	IS_EUC_RANGE_VALID(c)   ((c) >= 0xa1 && (c) <= 0xfe)
#define	pg_euccn_verifychar   pg_euckr_verifychar
#define	pg_euccn_verifystr   pg_euckr_verifystr
#define	ERR   0
#define	BGN   11
#define	CS1   16
#define	CS2   1
#define	CS3   5
#define	P3A   6 /* Lead was E0, check for 3-byte overlong */
#define	P3B   20 /* Lead was ED, check for surrogate */
#define	P4A   25 /* Lead was F0, check for 4-byte overlong */
#define	P4B   30 /* Lead was F4, check for too-large */
#define	END   BGN
#define	ASC   (END << BGN)
#define	L2A   (CS1 << BGN)
#define	L3A   (P3A << BGN)
#define	L3B   (CS2 << BGN)
#define	L3C   (P3B << BGN)
#define	L4A   (P4A << BGN)
#define	L4B   (CS3 << BGN)
#define	L4C   (P4B << BGN)
#define	CR1   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4B)
#define	CR2   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4A)
#define	CR3   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3A) | (CS2 << P4A)
#define	ILL   ERR
#define	STRIDE_LENGTH   (2 * sizeof(Vector8))

Functions
static int	pg_ascii2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_ascii_mblen (const unsigned char *s)
static int	pg_ascii_dsplen (const unsigned char *s)
static int	pg_euc2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euc_mblen (const unsigned char *s)
static int	pg_euc_dsplen (const unsigned char *s)
static int	pg_eucjp2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_eucjp_mblen (const unsigned char *s)
static int	pg_eucjp_dsplen (const unsigned char *s)
static int	pg_euckr2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euckr_mblen (const unsigned char *s)
static int	pg_euckr_dsplen (const unsigned char *s)
static int	pg_euccn2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euccn_mblen (const unsigned char *s)
static int	pg_euccn_dsplen (const unsigned char *s)
static int	pg_euctw2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_euctw_mblen (const unsigned char *s)
static int	pg_euctw_dsplen (const unsigned char *s)
static int	pg_wchar2euc_with_len (const pg_wchar *from, unsigned char *to, int len)
static int	pg_johab_mblen (const unsigned char *s)
static int	pg_johab_dsplen (const unsigned char *s)
static int	pg_utf2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
unsigned char *	unicode_to_utf8 (pg_wchar c, unsigned char *utf8string)
static int	pg_wchar2utf_with_len (const pg_wchar *from, unsigned char *to, int len)
int	pg_utf_mblen (const unsigned char *s)
static int	mbbisearch (pg_wchar ucs, const struct mbinterval *table, int max)
static int	ucs_wcwidth (pg_wchar ucs)
pg_wchar	utf8_to_unicode (const unsigned char *c)
static int	pg_utf_dsplen (const unsigned char *s)
static int	pg_mule2wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_wchar2mule_with_len (const pg_wchar *from, unsigned char *to, int len)
int	pg_mule_mblen (const unsigned char *s)
static int	pg_mule_dsplen (const unsigned char *s)
static int	pg_latin12wchar_with_len (const unsigned char *from, pg_wchar *to, int len)
static int	pg_wchar2single_with_len (const pg_wchar *from, unsigned char *to, int len)
static int	pg_latin1_mblen (const unsigned char *s)
static int	pg_latin1_dsplen (const unsigned char *s)
static int	pg_sjis_mblen (const unsigned char *s)
static int	pg_sjis_dsplen (const unsigned char *s)
static int	pg_big5_mblen (const unsigned char *s)
static int	pg_big5_dsplen (const unsigned char *s)
static int	pg_gbk_mblen (const unsigned char *s)
static int	pg_gbk_dsplen (const unsigned char *s)
static int	pg_uhc_mblen (const unsigned char *s)
static int	pg_uhc_dsplen (const unsigned char *s)
static int	pg_gb18030_mblen (const unsigned char *s)
static int	pg_gb18030_dsplen (const unsigned char *s)
static int	pg_ascii_verifychar (const unsigned char *s, int len)
static int	pg_ascii_verifystr (const unsigned char *s, int len)
static int	pg_eucjp_verifychar (const unsigned char *s, int len)
static int	pg_eucjp_verifystr (const unsigned char *s, int len)
static int	pg_euckr_verifychar (const unsigned char *s, int len)
static int	pg_euckr_verifystr (const unsigned char *s, int len)
static int	pg_euctw_verifychar (const unsigned char *s, int len)
static int	pg_euctw_verifystr (const unsigned char *s, int len)
static int	pg_johab_verifychar (const unsigned char *s, int len)
static int	pg_johab_verifystr (const unsigned char *s, int len)
static int	pg_mule_verifychar (const unsigned char *s, int len)
static int	pg_mule_verifystr (const unsigned char *s, int len)
static int	pg_latin1_verifychar (const unsigned char *s, int len)
static int	pg_latin1_verifystr (const unsigned char *s, int len)
static int	pg_sjis_verifychar (const unsigned char *s, int len)
static int	pg_sjis_verifystr (const unsigned char *s, int len)
static int	pg_big5_verifychar (const unsigned char *s, int len)
static int	pg_big5_verifystr (const unsigned char *s, int len)
static int	pg_gbk_verifychar (const unsigned char *s, int len)
static int	pg_gbk_verifystr (const unsigned char *s, int len)
static int	pg_uhc_verifychar (const unsigned char *s, int len)
static int	pg_uhc_verifystr (const unsigned char *s, int len)
static int	pg_gb18030_verifychar (const unsigned char *s, int len)
static int	pg_gb18030_verifystr (const unsigned char *s, int len)
static int	pg_utf8_verifychar (const unsigned char *s, int len)
static void	utf8_advance (const unsigned char *s, uint32 *state, int len)
static int	pg_utf8_verifystr (const unsigned char *s, int len)
bool	pg_utf8_islegal (const unsigned char *source, int length)
int	pg_encoding_mblen (int encoding, const char *mbstr)
int	pg_encoding_mblen_bounded (int encoding, const char *mbstr)
int	pg_encoding_dsplen (int encoding, const char *mbstr)
int	pg_encoding_verifymbchar (int encoding, const char *mbstr, int len)
int	pg_encoding_verifymbstr (int encoding, const char *mbstr, int len)
int	pg_encoding_max_length (int encoding)

Variables
static const uint32	Utf8Transition [256]
const pg_wchar_tbl	pg_wchar_table []

Macro Definition Documentation

ASC

#define ASC   (END << BGN)

Definition at line 1821 of file wchar.c.

BGN

#define BGN   11

Definition at line 1805 of file wchar.c.

CR1

#define CR1   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4B)

Definition at line 1833 of file wchar.c.

CR2

#define CR2   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4A)

Definition at line 1834 of file wchar.c.

CR3

#define CR3   (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3A) | (CS2 << P4A)

Definition at line 1835 of file wchar.c.

CS1

#define CS1   16

Definition at line 1807 of file wchar.c.

CS2

#define CS2   1

Definition at line 1808 of file wchar.c.

CS3

#define CS3   5

Definition at line 1809 of file wchar.c.

END

#define END   BGN

Definition at line 1816 of file wchar.c.

ERR

#define ERR   0

Definition at line 1803 of file wchar.c.

ILL

#define ILL   ERR

Definition at line 1837 of file wchar.c.

IS_EUC_RANGE_VALID

#define IS_EUC_RANGE_VALID

(

c

)

((c) >= 0xa1 && (c) <= 0xfe)

Definition at line 1118 of file wchar.c.

L2A

#define L2A   (CS1 << BGN)

Definition at line 1823 of file wchar.c.

L3A

#define L3A   (P3A << BGN)

Definition at line 1825 of file wchar.c.

L3B

#define L3B   (CS2 << BGN)

Definition at line 1826 of file wchar.c.

L3C

#define L3C   (P3B << BGN)

Definition at line 1827 of file wchar.c.

L4A

#define L4A   (P4A << BGN)

Definition at line 1829 of file wchar.c.

L4B

#define L4B   (CS3 << BGN)

Definition at line 1830 of file wchar.c.

L4C

#define L4C   (P4B << BGN)

Definition at line 1831 of file wchar.c.

P3A

#define P3A   6 /* Lead was E0, check for 3-byte overlong */

Definition at line 1811 of file wchar.c.

P3B

#define P3B   20 /* Lead was ED, check for surrogate */

Definition at line 1812 of file wchar.c.

P4A

#define P4A   25 /* Lead was F0, check for 4-byte overlong */

Definition at line 1813 of file wchar.c.

P4B

#define P4B   30 /* Lead was F4, check for too-large */

Definition at line 1814 of file wchar.c.

pg_euccn_verifychar

#define pg_euccn_verifychar   pg_euckr_verifychar

Definition at line 1263 of file wchar.c.

pg_euccn_verifystr

#define pg_euccn_verifystr   pg_euckr_verifystr

Definition at line 1264 of file wchar.c.

STRIDE_LENGTH

#define STRIDE_LENGTH   (2 * sizeof(Vector8))

Function Documentation

mbbisearch()

static int mbbisearch ( pg_wchar  ucs, const struct mbinterval *  table, int  max  )

static

Definition at line 592 of file wchar.c.

{
    int         min = 0;
    int         mid;
 
    if (ucs < table[0].first || ucs > table[max].last)
        return 0;
    while (max >= min)
    {
        mid = (min + max) / 2;
        if (ucs > table[mid].last)
            min = mid + 1;
        else if (ucs < table[mid].first)
            max = mid - 1;
        else
            return 1;
    }
 
    return 0;
}

Referenced by ucs_wcwidth().

pg_ascii2wchar_with_len()

static int pg_ascii2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 48 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_ascii_dsplen()

static int pg_ascii_dsplen ( const unsigned char *  s )

static

Definition at line 69 of file wchar.c.

{
    if (*s == '\0')
        return 0;
    if (*s < 0x20 || *s == 0x7f)
        return -1;
 
    return 1;
}

Referenced by pg_big5_dsplen(), pg_euc_dsplen(), pg_euccn_dsplen(), pg_eucjp_dsplen(), pg_euctw_dsplen(), pg_gb18030_dsplen(), pg_gbk_dsplen(), pg_latin1_dsplen(), pg_sjis_dsplen(), and pg_uhc_dsplen().

pg_ascii_mblen()

static int pg_ascii_mblen ( const unsigned char *  s )

static

Definition at line 63 of file wchar.c.

{
    return 1;
}

pg_ascii_verifychar()

static int pg_ascii_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1102 of file wchar.c.

{
    return 1;
}

pg_ascii_verifystr()

static int pg_ascii_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1108 of file wchar.c.

{
    const unsigned char *nullpos = memchr(s, 0, len);
 
    if (nullpos == NULL)
        return len;
    else
        return nullpos - s;
}

References len.

pg_big5_dsplen()

static int pg_big5_dsplen ( const unsigned char *  s )

static

Definition at line 973 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_big5_mblen()

static int pg_big5_mblen ( const unsigned char *  s )

static

Definition at line 961 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_big5_verifychar().

pg_big5_verifychar()

static int pg_big5_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1518 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_big5_mblen(s);
 
    if (len < l)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References len, and pg_big5_mblen().

Referenced by pg_big5_verifystr().

pg_big5_verifystr()

static int pg_big5_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1538 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_big5_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_big5_verifychar().

pg_encoding_dsplen()

int pg_encoding_dsplen	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2151 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].dsplen((const unsigned char *) mbstr) :
            pg_wchar_table[PG_SQL_ASCII].dsplen((const unsigned char *) mbstr));
}

References encoding, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by PQdsplen(), and reportErrorPosition().

pg_encoding_max_length()

int pg_encoding_max_length

(

int

encoding

)

Definition at line 2188 of file wchar.c.

{
    Assert(PG_VALID_ENCODING(encoding));
 
    return pg_wchar_table[encoding].maxmblen;
}

References Assert(), encoding, pg_wchar_tbl::maxmblen, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by ascii(), chr(), CopyConvertBuf(), pg_encoding_mbcliplen(), pg_verify_mbstr_len(), reportErrorPosition(), and type_maximum_size().

pg_encoding_mblen()

int pg_encoding_mblen	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2130 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mblen((const unsigned char *) mbstr) :
            pg_wchar_table[PG_SQL_ASCII].mblen((const unsigned char *) mbstr));
}

References encoding, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), pg_encoding_mblen_bounded(), PQescapeInternal(), PQescapeStringInternal(), PQmblen(), PQmblenBounded(), report_invalid_encoding(), and report_untranslatable_char().

pg_encoding_mblen_bounded()

int pg_encoding_mblen_bounded	(	int	encoding,
		const char *	mbstr
	)

Definition at line 2142 of file wchar.c.

{
    return strnlen(mbstr, pg_encoding_mblen(encoding, mbstr));
}

References encoding, pg_encoding_mblen(), and strnlen().

pg_encoding_verifymbchar()

int pg_encoding_verifymbchar	(	int	encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 2164 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mbverifychar((const unsigned char *) mbstr, len) :
            pg_wchar_table[PG_SQL_ASCII].mbverifychar((const unsigned char *) mbstr, len));
}

References encoding, len, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by big52euc_tw(), big52mic(), euc_jis_20042shift_jis_2004(), euc_jp2mic(), euc_jp2sjis(), euc_kr2mic(), euc_tw2big5(), euc_tw2mic(), LocalToUtf(), mic2big5(), mic2euc_jp(), mic2euc_kr(), mic2euc_tw(), mic2sjis(), shift_jis_20042euc_jis_2004(), and sjis2euc_jp().

pg_encoding_verifymbstr()

int pg_encoding_verifymbstr	(	int	encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 2177 of file wchar.c.

{
    return (PG_VALID_ENCODING(encoding) ?
            pg_wchar_table[encoding].mbverifystr((const unsigned char *) mbstr, len) :
            pg_wchar_table[PG_SQL_ASCII].mbverifystr((const unsigned char *) mbstr, len));
}

References encoding, len, PG_SQL_ASCII, PG_VALID_ENCODING, and pg_wchar_table.

Referenced by CopyConvertBuf(), and test_enc_conversion().

pg_euc2wchar_with_len()

static int pg_euc2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 83 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 2)   /* JIS X 0201 (so called "1 byte
                                         * KANA") */
        {
            from++;
            *to = (SS2 << 8) | *from++;
            len -= 2;
        }
        else if (*from == SS3 && len >= 3)  /* JIS X 0212 KANJI */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* JIS X 0208 KANJI */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else                    /* must be ASCII */
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp2wchar_with_len(), and pg_euckr2wchar_with_len().

pg_euc_dsplen()

static int pg_euc_dsplen ( const unsigned char *  s )

inlinestatic

Definition at line 138 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

Referenced by pg_euckr_dsplen(), and pg_johab_dsplen().

pg_euc_mblen()

static int pg_euc_mblen ( const unsigned char *  s )

inlinestatic

Definition at line 122 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 3;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp_mblen(), pg_euckr_mblen(), and pg_johab_mblen().

pg_euccn2wchar_with_len()

static int pg_euccn2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 210 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 3)   /* code set 2 (unused?) */
        {
            from++;
            *to = (SS2 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (*from == SS3 && len >= 3)  /* code set 3 (unused ?) */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 1 */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euccn_dsplen()

static int pg_euccn_dsplen ( const unsigned char *  s )

static

Definition at line 261 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_euccn_mblen()

static int pg_euccn_mblen ( const unsigned char *  s )

static

Definition at line 249 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, and len.

pg_eucjp2wchar_with_len()

static int pg_eucjp2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 157 of file wchar.c.

{
    return pg_euc2wchar_with_len(from, to, len);
}

References len, and pg_euc2wchar_with_len().

pg_eucjp_dsplen()

static int pg_eucjp_dsplen ( const unsigned char *  s )

static

Definition at line 169 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 1;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

pg_eucjp_mblen()

static int pg_eucjp_mblen ( const unsigned char *  s )

static

Definition at line 163 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

pg_eucjp_verifychar()

static int pg_eucjp_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1121 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    switch (c1)
    {
        case SS2:               /* JIS X 0201 */
            l = 2;
            if (l > len)
                return -1;
            c2 = *s++;
            if (c2 < 0xa1 || c2 > 0xdf)
                return -1;
            break;
 
        case SS3:               /* JIS X 0212 */
            l = 3;
            if (l > len)
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            break;
 
        default:
            if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
            {
                l = 2;
                if (l > len)
                    return -1;
                if (!IS_EUC_RANGE_VALID(c1))
                    return -1;
                c2 = *s++;
                if (!IS_EUC_RANGE_VALID(c2))
                    return -1;
            }
            else
                /* must be ASCII */
            {
                l = 1;
            }
            break;
    }
 
    return l;
}

References IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_eucjp_verifystr().

pg_eucjp_verifystr()

static int pg_eucjp_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1176 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_eucjp_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_eucjp_verifychar().

pg_euckr2wchar_with_len()

static int pg_euckr2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 188 of file wchar.c.

{
    return pg_euc2wchar_with_len(from, to, len);
}

References len, and pg_euc2wchar_with_len().

pg_euckr_dsplen()

static int pg_euckr_dsplen ( const unsigned char *  s )

static

Definition at line 200 of file wchar.c.

{
    return pg_euc_dsplen(s);
}

References pg_euc_dsplen().

pg_euckr_mblen()

static int pg_euckr_mblen ( const unsigned char *  s )

static

Definition at line 194 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

pg_euckr_verifychar()

static int pg_euckr_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1205 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    if (IS_HIGHBIT_SET(c1))
    {
        l = 2;
        if (l > len)
            return -1;
        if (!IS_EUC_RANGE_VALID(c1))
            return -1;
        c2 = *s++;
        if (!IS_EUC_RANGE_VALID(c2))
            return -1;
    }
    else
        /* must be ASCII */
    {
        l = 1;
    }
 
    return l;
}

References IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, and len.

Referenced by pg_euckr_verifystr().

pg_euckr_verifystr()

static int pg_euckr_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1234 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_euckr_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_euckr_verifychar().

pg_euctw2wchar_with_len()

static int pg_euctw2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 277 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (*from == SS2 && len >= 4)   /* code set 2 */
        {
            from++;
            *to = (((uint32) SS2) << 24) | (*from++ << 16);
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 4;
        }
        else if (*from == SS3 && len >= 3)  /* code set 3 (unused?) */
        {
            from++;
            *to = (SS3 << 16) | (*from++ << 8);
            *to |= *from++;
            len -= 3;
        }
        else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 2 */
        {
            *to = *from++ << 8;
            *to |= *from++;
            len -= 2;
        }
        else
        {
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euctw_dsplen()

static int pg_euctw_dsplen ( const unsigned char *  s )

static

Definition at line 333 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 2;
    else if (*s == SS3)
        len = 2;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);
    return len;
}

References IS_HIGHBIT_SET, len, pg_ascii_dsplen(), SS2, and SS3.

pg_euctw_mblen()

static int pg_euctw_mblen ( const unsigned char *  s )

static

Definition at line 317 of file wchar.c.

{
    int         len;
 
    if (*s == SS2)
        len = 4;
    else if (*s == SS3)
        len = 3;
    else if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = 1;
    return len;
}

References IS_HIGHBIT_SET, len, SS2, and SS3.

pg_euctw_verifychar()

static int pg_euctw_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1267 of file wchar.c.

{
    int         l;
    unsigned char c1,
                c2;
 
    c1 = *s++;
 
    switch (c1)
    {
        case SS2:               /* CNS 11643 Plane 1-7 */
            l = 4;
            if (l > len)
                return -1;
            c2 = *s++;
            if (c2 < 0xa1 || c2 > 0xa7)
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                return -1;
            break;
 
        case SS3:               /* unused */
            return -1;
 
        default:
            if (IS_HIGHBIT_SET(c1)) /* CNS 11643 Plane 1 */
            {
                l = 2;
                if (l > len)
                    return -1;
                /* no further range check on c1? */
                c2 = *s++;
                if (!IS_EUC_RANGE_VALID(c2))
                    return -1;
            }
            else
                /* must be ASCII */
            {
                l = 1;
            }
            break;
    }
    return l;
}

References IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, SS2, and SS3.

Referenced by pg_euctw_verifystr().

pg_euctw_verifystr()

static int pg_euctw_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1317 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_euctw_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_euctw_verifychar().

pg_gb18030_dsplen()

static int pg_gb18030_dsplen ( const unsigned char *  s )

static

Definition at line 1068 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;
    else
        len = pg_ascii_dsplen(s);   /* ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_gb18030_mblen()

static int pg_gb18030_mblen ( const unsigned char *  s )

static

Definition at line 1054 of file wchar.c.

{
    int         len;
 
    if (!IS_HIGHBIT_SET(*s))
        len = 1;                /* ASCII */
    else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
        len = 4;
    else
        len = 2;
    return len;
}

References IS_HIGHBIT_SET, and len.

pg_gb18030_verifychar()

static int pg_gb18030_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1665 of file wchar.c.

{
    int         l;
 
    if (!IS_HIGHBIT_SET(*s))
        l = 1;                  /* ASCII */
    else if (len >= 4 && *(s + 1) >= 0x30 && *(s + 1) <= 0x39)
    {
        /* Should be 4-byte, validate remaining bytes */
        if (*s >= 0x81 && *s <= 0xfe &&
            *(s + 2) >= 0x81 && *(s + 2) <= 0xfe &&
            *(s + 3) >= 0x30 && *(s + 3) <= 0x39)
            l = 4;
        else
            l = -1;
    }
    else if (len >= 2 && *s >= 0x81 && *s <= 0xfe)
    {
        /* Should be 2-byte, validate */
        if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) ||
            (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
            l = 2;
        else
            l = -1;
    }
    else
        l = -1;
    return l;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_gb18030_verifystr().

pg_gb18030_verifystr()

static int pg_gb18030_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1696 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_gb18030_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_gb18030_verifychar().

pg_gbk_dsplen()

static int pg_gbk_dsplen ( const unsigned char *  s )

static

Definition at line 1000 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_gbk_mblen()

static int pg_gbk_mblen ( const unsigned char *  s )

static

Definition at line 988 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_gbk_verifychar().

pg_gbk_verifychar()

static int pg_gbk_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1567 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_gbk_mblen(s);
 
    if (len < l)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References len, and pg_gbk_mblen().

Referenced by pg_gbk_verifystr().

pg_gbk_verifystr()

static int pg_gbk_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1587 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_gbk_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_gbk_verifychar().

pg_johab_dsplen()

static int pg_johab_dsplen ( const unsigned char *  s )

static

Definition at line 407 of file wchar.c.

{
    return pg_euc_dsplen(s);
}

References pg_euc_dsplen().

pg_johab_mblen()

static int pg_johab_mblen ( const unsigned char *  s )

static

Definition at line 401 of file wchar.c.

{
    return pg_euc_mblen(s);
}

References pg_euc_mblen().

Referenced by pg_johab_verifychar().

pg_johab_verifychar()

static int pg_johab_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1346 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c;
 
    l = mbl = pg_johab_mblen(s);
 
    if (len < l)
        return -1;
 
    if (!IS_HIGHBIT_SET(*s))
        return mbl;
 
    while (--l > 0)
    {
        c = *++s;
        if (!IS_EUC_RANGE_VALID(c))
            return -1;
    }
    return mbl;
}

References IS_EUC_RANGE_VALID, IS_HIGHBIT_SET, len, and pg_johab_mblen().

Referenced by pg_johab_verifystr().

pg_johab_verifystr()

static int pg_johab_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1370 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_johab_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_johab_verifychar().

pg_latin12wchar_with_len()

static int pg_latin12wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 878 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_latin1_dsplen()

static int pg_latin1_dsplen ( const unsigned char *  s )

static

Definition at line 921 of file wchar.c.

{
    return pg_ascii_dsplen(s);
}

References pg_ascii_dsplen().

pg_latin1_mblen()

static int pg_latin1_mblen ( const unsigned char *  s )

static

Definition at line 915 of file wchar.c.

{
    return 1;
}

pg_latin1_verifychar()

static int pg_latin1_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1449 of file wchar.c.

{
    return 1;
}

pg_latin1_verifystr()

static int pg_latin1_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1455 of file wchar.c.

{
    const unsigned char *nullpos = memchr(s, 0, len);
 
    if (nullpos == NULL)
        return len;
    else
        return nullpos - s;
}

References len.

pg_mule2wchar_with_len()

static int pg_mule2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 713 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        if (IS_LC1(*from) && len >= 2)
        {
            *to = *from++ << 16;
            *to |= *from++;
            len -= 2;
        }
        else if (IS_LCPRV1(*from) && len >= 3)
        {
            from++;
            *to = *from++ << 16;
            *to |= *from++;
            len -= 3;
        }
        else if (IS_LC2(*from) && len >= 3)
        {
            *to = *from++ << 16;
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 3;
        }
        else if (IS_LCPRV2(*from) && len >= 4)
        {
            from++;
            *to = *from++ << 16;
            *to |= *from++ << 8;
            *to |= *from++;
            len -= 4;
        }
        else
        {                       /* assume ASCII */
            *to = (unsigned char) *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

pg_mule_dsplen()

static int pg_mule_dsplen ( const unsigned char *  s )

static

Definition at line 850 of file wchar.c.

{
    int         len;
 
    /*
     * Note: it's not really appropriate to assume that all multibyte charsets
     * are double-wide on screen.  But this seems an okay approximation for
     * the MULE charsets we currently support.
     */
 
    if (IS_LC1(*s))
        len = 1;
    else if (IS_LCPRV1(*s))
        len = 1;
    else if (IS_LC2(*s))
        len = 2;
    else if (IS_LCPRV2(*s))
        len = 2;
    else
        len = 1;                /* assume ASCII */
 
    return len;
}

References IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

pg_mule_mblen()

int pg_mule_mblen

(

const unsigned char *

s

)

Definition at line 832 of file wchar.c.

{
    int         len;
 
    if (IS_LC1(*s))
        len = 2;
    else if (IS_LCPRV1(*s))
        len = 3;
    else if (IS_LC2(*s))
        len = 3;
    else if (IS_LCPRV2(*s))
        len = 4;
    else
        len = 1;                /* assume ASCII */
    return len;
}

References IS_LC1, IS_LC2, IS_LCPRV1, IS_LCPRV2, and len.

Referenced by mic2latin(), mic2latin_with_table(), and pg_mule_verifychar().

pg_mule_verifychar()

static int pg_mule_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1399 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c;
 
    l = mbl = pg_mule_mblen(s);
 
    if (len < l)
        return -1;
 
    while (--l > 0)
    {
        c = *++s;
        if (!IS_HIGHBIT_SET(c))
            return -1;
    }
    return mbl;
}

References IS_HIGHBIT_SET, len, and pg_mule_mblen().

Referenced by pg_mule_verifystr().

pg_mule_verifystr()

static int pg_mule_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1420 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_mule_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_mule_verifychar().

pg_sjis_dsplen()

static int pg_sjis_dsplen ( const unsigned char *  s )

static

Definition at line 944 of file wchar.c.

{
    int         len;
 
    if (*s >= 0xa1 && *s <= 0xdf)
        len = 1;                /* 1 byte kana? */
    else if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_sjis_mblen()

static int pg_sjis_mblen ( const unsigned char *  s )

static

Definition at line 930 of file wchar.c.

{
    int         len;
 
    if (*s >= 0xa1 && *s <= 0xdf)
        len = 1;                /* 1 byte kana? */
    else if (IS_HIGHBIT_SET(*s))
        len = 2;                /* kanji? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_sjis_verifychar().

pg_sjis_verifychar()

static int pg_sjis_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1466 of file wchar.c.

{
    int         l,
                mbl;
    unsigned char c1,
                c2;
 
    l = mbl = pg_sjis_mblen(s);
 
    if (len < l)
        return -1;
 
    if (l == 1)                 /* pg_sjis_mblen already verified it */
        return mbl;
 
    c1 = *s++;
    c2 = *s;
    if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
        return -1;
    return mbl;
}

References ISSJISHEAD, ISSJISTAIL, len, and pg_sjis_mblen().

Referenced by pg_sjis_verifystr().

pg_sjis_verifystr()

static int pg_sjis_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1489 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_sjis_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_sjis_verifychar().

pg_uhc_dsplen()

static int pg_uhc_dsplen ( const unsigned char *  s )

static

Definition at line 1027 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* 2byte? */
    else
        len = pg_ascii_dsplen(s);   /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, len, and pg_ascii_dsplen().

pg_uhc_mblen()

static int pg_uhc_mblen ( const unsigned char *  s )

static

Definition at line 1015 of file wchar.c.

{
    int         len;
 
    if (IS_HIGHBIT_SET(*s))
        len = 2;                /* 2byte? */
    else
        len = 1;                /* should be ASCII */
    return len;
}

References IS_HIGHBIT_SET, and len.

Referenced by pg_uhc_verifychar().

pg_uhc_verifychar()

static int pg_uhc_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1616 of file wchar.c.

{
    int         l,
                mbl;
 
    l = mbl = pg_uhc_mblen(s);
 
    if (len < l)
        return -1;
 
    while (--l > 0)
    {
        if (*++s == '\0')
            return -1;
    }
 
    return mbl;
}

References len, and pg_uhc_mblen().

Referenced by pg_uhc_verifystr().

pg_uhc_verifystr()

static int pg_uhc_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1636 of file wchar.c.

{
    const unsigned char *start = s;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_uhc_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References IS_HIGHBIT_SET, len, and pg_uhc_verifychar().

pg_utf2wchar_with_len()

static int pg_utf2wchar_with_len ( const unsigned char *  from, pg_wchar *  to, int  len  )

static

Definition at line 419 of file wchar.c.

{
    int         cnt = 0;
    uint32      c1,
                c2,
                c3,
                c4;
 
    while (len > 0 && *from)
    {
        if ((*from & 0x80) == 0)
        {
            *to = *from++;
            len--;
        }
        else if ((*from & 0xe0) == 0xc0)
        {
            if (len < 2)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x1f;
            c2 = *from++ & 0x3f;
            *to = (c1 << 6) | c2;
            len -= 2;
        }
        else if ((*from & 0xf0) == 0xe0)
        {
            if (len < 3)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x0f;
            c2 = *from++ & 0x3f;
            c3 = *from++ & 0x3f;
            *to = (c1 << 12) | (c2 << 6) | c3;
            len -= 3;
        }
        else if ((*from & 0xf8) == 0xf0)
        {
            if (len < 4)
                break;          /* drop trailing incomplete char */
            c1 = *from++ & 0x07;
            c2 = *from++ & 0x3f;
            c3 = *from++ & 0x3f;
            c4 = *from++ & 0x3f;
            *to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
            len -= 4;
        }
        else
        {
            /* treat a bogus char as length 1; not ours to raise error */
            *to = *from++;
            len--;
        }
        to++;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_utf8_islegal()

bool pg_utf8_islegal	(	const unsigned char *	source,
		int	length
	)

Definition at line 2013 of file wchar.c.

{
    unsigned char a;
 
    switch (length)
    {
        default:
            /* reject lengths 5 and 6 for now */
            return false;
        case 4:
            a = source[3];
            if (a < 0x80 || a > 0xBF)
                return false;
            /* FALL THRU */
        case 3:
            a = source[2];
            if (a < 0x80 || a > 0xBF)
                return false;
            /* FALL THRU */
        case 2:
            a = source[1];
            switch (*source)
            {
                case 0xE0:
                    if (a < 0xA0 || a > 0xBF)
                        return false;
                    break;
                case 0xED:
                    if (a < 0x80 || a > 0x9F)
                        return false;
                    break;
                case 0xF0:
                    if (a < 0x90 || a > 0xBF)
                        return false;
                    break;
                case 0xF4:
                    if (a < 0x80 || a > 0x8F)
                        return false;
                    break;
                default:
                    if (a < 0x80 || a > 0xBF)
                        return false;
                    break;
            }
            /* FALL THRU */
        case 1:
            a = *source;
            if (a >= 0x80 && a < 0xC2)
                return false;
            if (a > 0xF4)
                return false;
            break;
    }
    return true;
}

References a, and source.

Referenced by chr(), pg_utf8_string_len(), pg_utf8_verifychar(), utf8_to_iso8859_1(), and UtfToLocal().

pg_utf8_verifychar()

static int pg_utf8_verifychar ( const unsigned char *  s, int  len  )

static

Definition at line 1725 of file wchar.c.

{
    int         l;
 
    if ((*s & 0x80) == 0)
    {
        if (*s == '\0')
            return -1;
        return 1;
    }
    else if ((*s & 0xe0) == 0xc0)
        l = 2;
    else if ((*s & 0xf0) == 0xe0)
        l = 3;
    else if ((*s & 0xf8) == 0xf0)
        l = 4;
    else
        l = 1;
 
    if (l > len)
        return -1;
 
    if (!pg_utf8_islegal(s, l))
        return -1;
 
    return l;
}

References len, and pg_utf8_islegal().

Referenced by pg_utf8_verifystr().

pg_utf8_verifystr()

static int pg_utf8_verifystr ( const unsigned char *  s, int  len  )

static

Definition at line 1915 of file wchar.c.

{
    const unsigned char *start = s;
    const int   orig_len = len;
    uint32      state = BGN;
 
/*
 * With a stride of two vector widths, gcc will unroll the loop. Even if
 * the compiler can unroll a longer loop, it's not worth it because we
 * must fall back to the byte-wise algorithm if we find any non-ASCII.
 */
#define STRIDE_LENGTH (2 * sizeof(Vector8))
 
    if (len >= STRIDE_LENGTH)
    {
        while (len >= STRIDE_LENGTH)
        {
            /*
             * If the chunk is all ASCII, we can skip the full UTF-8 check,
             * but we must first check for a non-END state, which means the
             * previous chunk ended in the middle of a multibyte sequence.
             */
            if (state != END || !is_valid_ascii(s, STRIDE_LENGTH))
                utf8_advance(s, &state, STRIDE_LENGTH);
 
            s += STRIDE_LENGTH;
            len -= STRIDE_LENGTH;
        }
 
        /* The error state persists, so we only need to check for it here. */
        if (state == ERR)
        {
            /*
             * Start over from the beginning with the slow path so we can
             * count the valid bytes.
             */
            len = orig_len;
            s = start;
        }
        else if (state != END)
        {
            /*
             * The fast path exited in the middle of a multibyte sequence.
             * Walk backwards to find the leading byte so that the slow path
             * can resume checking from there. We must always backtrack at
             * least one byte, since the current byte could be e.g. an ASCII
             * byte after a 2-byte lead, which is invalid.
             */
            do
            {
                Assert(s > start);
                s--;
                len++;
                Assert(IS_HIGHBIT_SET(*s));
            } while (pg_utf_mblen(s) <= 1);
        }
    }
 
    /* check remaining bytes */
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*s))
        {
            if (*s == '\0')
                break;
            l = 1;
        }
        else
        {
            l = pg_utf8_verifychar(s, len);
            if (l == -1)
                break;
        }
        s += l;
        len -= l;
    }
 
    return s - start;
}

References Assert(), BGN, END, ERR, IS_HIGHBIT_SET, is_valid_ascii(), len, pg_utf8_verifychar(), pg_utf_mblen(), STRIDE_LENGTH, and utf8_advance().

pg_utf_dsplen()

static int pg_utf_dsplen ( const unsigned char *  s )

static

Definition at line 701 of file wchar.c.

{
    return ucs_wcwidth(utf8_to_unicode(s));
}

References ucs_wcwidth(), and utf8_to_unicode().

pg_utf_mblen()

int pg_utf_mblen

(

const unsigned char *

s

)

Definition at line 549 of file wchar.c.

{
    int         len;
 
    if ((*s & 0x80) == 0)
        len = 1;
    else if ((*s & 0xe0) == 0xc0)
        len = 2;
    else if ((*s & 0xf0) == 0xe0)
        len = 3;
    else if ((*s & 0xf8) == 0xf0)
        len = 4;
#ifdef NOT_USED
    else if ((*s & 0xfc) == 0xf8)
        len = 5;
    else if ((*s & 0xfe) == 0xfc)
        len = 6;
#endif
    else
        len = 1;
    return len;
}

References len.

Referenced by json_lex_string(), pg_saslprep(), pg_unicode_to_server(), pg_unicode_to_server_noerror(), pg_utf8_string_len(), pg_utf8_verifystr(), pg_wchar2utf_with_len(), read_char(), unicode_is_normalized(), unicode_normalize_func(), utf8_to_iso8859_1(), and UtfToLocal().

pg_wchar2euc_with_len()

static int pg_wchar2euc_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 355 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        unsigned char c;
 
        if ((c = (*from >> 24)))
        {
            *to++ = c;
            *to++ = (*from >> 16) & 0xff;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else if ((c = (*from >> 16)))
        {
            *to++ = c;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if ((c = (*from >> 8)))
        {
            *to++ = c;
            *to++ = *from & 0xff;
            cnt += 2;
        }
        else
        {
            *to++ = *from;
            cnt++;
        }
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References len.

pg_wchar2mule_with_len()

static int pg_wchar2mule_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 766 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        unsigned char lb;
 
        lb = (*from >> 16) & 0xff;
        if (IS_LC1(lb))
        {
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 2;
        }
        else if (IS_LC2(lb))
        {
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV1_A_RANGE(lb))
        {
            *to++ = LCPRV1_A;
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV1_B_RANGE(lb))
        {
            *to++ = LCPRV1_B;
            *to++ = lb;
            *to++ = *from & 0xff;
            cnt += 3;
        }
        else if (IS_LCPRV2_A_RANGE(lb))
        {
            *to++ = LCPRV2_A;
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else if (IS_LCPRV2_B_RANGE(lb))
        {
            *to++ = LCPRV2_B;
            *to++ = lb;
            *to++ = (*from >> 8) & 0xff;
            *to++ = *from & 0xff;
            cnt += 4;
        }
        else
        {
            *to++ = *from & 0xff;
            cnt += 1;
        }
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References IS_LC1, IS_LC2, IS_LCPRV1_A_RANGE, IS_LCPRV1_B_RANGE, IS_LCPRV2_A_RANGE, IS_LCPRV2_B_RANGE, LCPRV1_A, LCPRV1_B, LCPRV2_A, LCPRV2_B, and len.

pg_wchar2single_with_len()

static int pg_wchar2single_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 900 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        *to++ = *from++;
        len--;
        cnt++;
    }
    *to = 0;
    return cnt;
}

References len.

pg_wchar2utf_with_len()

static int pg_wchar2utf_with_len ( const pg_wchar *  from, unsigned char *  to, int  len  )

static

Definition at line 518 of file wchar.c.

{
    int         cnt = 0;
 
    while (len > 0 && *from)
    {
        int         char_len;
 
        unicode_to_utf8(*from, to);
        char_len = pg_utf_mblen(to);
        cnt += char_len;
        to += char_len;
        from++;
        len--;
    }
    *to = 0;
    return cnt;
}

References len, pg_utf_mblen(), and unicode_to_utf8().

ucs_wcwidth()

static int ucs_wcwidth ( pg_wchar  ucs )

static

Definition at line 639 of file wchar.c.

{
#include "common/unicode_nonspacing_table.h"
#include "common/unicode_east_asian_fw_table.h"
 
    /* test for 8-bit control characters */
    if (ucs == 0)
        return 0;
 
    if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
        return -1;
 
    /*
     * binary search in table of non-spacing characters
     *
     * XXX: In the official Unicode sources, it is possible for a character to
     * be described as both non-spacing and wide at the same time. As of
     * Unicode 13.0, treating the non-spacing property as the determining
     * factor for display width leads to the correct behavior, so do that
     * search first.
     */
    if (mbbisearch(ucs, nonspacing,
                   sizeof(nonspacing) / sizeof(struct mbinterval) - 1))
        return 0;
 
    /* binary search in table of wide characters */
    if (mbbisearch(ucs, east_asian_fw,
                   sizeof(east_asian_fw) / sizeof(struct mbinterval) - 1))
        return 2;
 
    return 1;
}

References east_asian_fw, mbbisearch(), and nonspacing.

Referenced by pg_utf_dsplen().

unicode_to_utf8()

unsigned char* unicode_to_utf8	(	pg_wchar	c,
		unsigned char *	utf8string
	)

Definition at line 483 of file wchar.c.

{
    if (c <= 0x7F)
    {
        utf8string[0] = c;
    }
    else if (c <= 0x7FF)
    {
        utf8string[0] = 0xC0 | ((c >> 6) & 0x1F);
        utf8string[1] = 0x80 | (c & 0x3F);
    }
    else if (c <= 0xFFFF)
    {
        utf8string[0] = 0xE0 | ((c >> 12) & 0x0F);
        utf8string[1] = 0x80 | ((c >> 6) & 0x3F);
        utf8string[2] = 0x80 | (c & 0x3F);
    }
    else
    {
        utf8string[0] = 0xF0 | ((c >> 18) & 0x07);
        utf8string[1] = 0x80 | ((c >> 12) & 0x3F);
        utf8string[2] = 0x80 | ((c >> 6) & 0x3F);
        utf8string[3] = 0x80 | (c & 0x3F);
    }
 
    return utf8string;
}

Referenced by json_lex_string(), pg_saslprep(), pg_unicode_to_server(), pg_unicode_to_server_noerror(), pg_wchar2utf_with_len(), and unicode_normalize_func().

utf8_advance()

static void utf8_advance ( const unsigned char *  s, uint32 *  state, int  len  )

static

Definition at line 1897 of file wchar.c.

{
    /* Note: We deliberately don't check the state's value here. */
    while (len > 0)
    {
        /*
         * It's important that the mask value is 31: In most instruction sets,
         * a shift by a 32-bit operand is understood to be a shift by its mod
         * 32, so the compiler should elide the mask operation.
         */
        *state = Utf8Transition[*s++] >> (*state & 31);
        len--;
    }
 
    *state &= 31;
}

References len, and Utf8Transition.

Referenced by pg_utf8_verifystr().

utf8_to_unicode()

pg_wchar utf8_to_unicode

(

const unsigned char *

c

)

Definition at line 679 of file wchar.c.

{
    if ((*c & 0x80) == 0)
        return (pg_wchar) c[0];
    else if ((*c & 0xe0) == 0xc0)
        return (pg_wchar) (((c[0] & 0x1f) << 6) |
                           (c[1] & 0x3f));
    else if ((*c & 0xf0) == 0xe0)
        return (pg_wchar) (((c[0] & 0x0f) << 12) |
                           ((c[1] & 0x3f) << 6) |
                           (c[2] & 0x3f));
    else if ((*c & 0xf8) == 0xf0)
        return (pg_wchar) (((c[0] & 0x07) << 18) |
                           ((c[1] & 0x3f) << 12) |
                           ((c[2] & 0x3f) << 6) |
                           (c[3] & 0x3f));
    else
        /* that is an invalid code on purpose */
        return 0xffffffff;
}

Referenced by pg_saslprep(), pg_utf_dsplen(), read_char(), unicode_is_normalized(), and unicode_normalize_func().

Variable Documentation

pg_wchar_table

const pg_wchar_tbl pg_wchar_table[]

Definition at line 2076 of file wchar.c.

Referenced by pg_database_encoding_max_length(), pg_dsplen(), pg_encoding_dsplen(), pg_encoding_max_length(), pg_encoding_max_length_sql(), pg_encoding_mb2wchar_with_len(), pg_encoding_mbcliplen(), pg_encoding_mblen(), pg_encoding_verifymbchar(), pg_encoding_verifymbstr(), pg_encoding_wchar2mb_with_len(), pg_generic_charinc(), pg_mb2wchar(), pg_mb2wchar_with_len(), pg_mblen(), pg_verify_mbstr(), pg_verify_mbstr_len(), pg_wchar2mb(), and pg_wchar2mb_with_len().

Utf8Transition

const uint32 Utf8Transition[256]

static

Definition at line 1839 of file wchar.c.

Referenced by utf8_advance().