conv.c File Reference

#include "postgres.h"
#include "mb/pg_wchar.h"

Include dependency graph for conv.c:

Go to the source code of this file.

Functions
int	local2local (const unsigned char *l, unsigned char *p, int len, int src_encoding, int dest_encoding, const unsigned char *tab, bool noError)
int	latin2mic (const unsigned char *l, unsigned char *p, int len, int lc, int encoding, bool noError)
int	mic2latin (const unsigned char *mic, unsigned char *p, int len, int lc, int encoding, bool noError)
int	latin2mic_with_table (const unsigned char *l, unsigned char *p, int len, int lc, int encoding, const unsigned char *tab, bool noError)
int	mic2latin_with_table (const unsigned char *mic, unsigned char *p, int len, int lc, int encoding, const unsigned char *tab, bool noError)
static int	compare3 (const void *p1, const void *p2)
static int	compare4 (const void *p1, const void *p2)
static unsigned char *	store_coded_char (unsigned char *dest, uint32 code)
static uint32	pg_mb_radix_conv (const pg_mb_radix_tree *rt, int l, unsigned char b1, unsigned char b2, unsigned char b3, unsigned char b4)
int	UtfToLocal (const unsigned char *utf, int len, unsigned char *iso, const pg_mb_radix_tree *map, const pg_utf_to_local_combined *cmap, int cmapsize, utf_local_conversion_func conv_func, int encoding, bool noError)
int	LocalToUtf (const unsigned char *iso, int len, unsigned char *utf, const pg_mb_radix_tree *map, const pg_local_to_utf_combined *cmap, int cmapsize, utf_local_conversion_func conv_func, int encoding, bool noError)

Function Documentation

compare3()

static int compare3 ( const void *  p1, const void *  p2  )

static

Definition at line 320 of file conv.c.

{
    uint32      s1,
                s2,
                d1,
                d2;
 
    s1 = *(const uint32 *) p1;
    s2 = *((const uint32 *) p1 + 1);
    d1 = ((const pg_utf_to_local_combined *) p2)->utf1;
    d2 = ((const pg_utf_to_local_combined *) p2)->utf2;
    return (s1 > d1 || (s1 == d1 && s2 > d2)) ? 1 : ((s1 == d1 && s2 == d2) ? 0 : -1);
}

References s1, and s2.

Referenced by UtfToLocal().

compare4()

static int compare4 ( const void *  p1, const void *  p2  )

static

Definition at line 339 of file conv.c.

{
    uint32      v1,
                v2;
 
    v1 = *(const uint32 *) p1;
    v2 = ((const pg_local_to_utf_combined *) p2)->code;
    return (v1 > v2) ? 1 : ((v1 == v2) ? 0 : -1);
}

Referenced by LocalToUtf().

latin2mic()

int latin2mic	(	const unsigned char *	l,
		unsigned char *	p,
		int	len,
		int	lc,
		int	encoding,
		bool	noError
	)

Definition at line 89 of file conv.c.

{
    const unsigned char *start = l;
    int         c1;
 
    while (len > 0)
    {
        c1 = *l;
        if (c1 == 0)
        {
            if (noError)
                break;
            report_invalid_encoding(encoding, (const char *) l, len);
        }
        if (IS_HIGHBIT_SET(c1))
            *p++ = lc;
        *p++ = c1;
        l++;
        len--;
    }
    *p = '\0';
 
    return l - start;
}

References encoding, IS_HIGHBIT_SET, len, report_invalid_encoding(), and start.

Referenced by koi8r_to_mic(), latin1_to_mic(), latin2_to_mic(), latin3_to_mic(), and latin4_to_mic().

latin2mic_with_table()

int latin2mic_with_table	(	const unsigned char *	l,
		unsigned char *	p,
		int	len,
		int	lc,
		int	encoding,
		const unsigned char *	tab,
		bool	noError
	)

Definition at line 194 of file conv.c.

{
    const unsigned char *start = l;
    unsigned char c1,
                c2;
 
    while (len > 0)
    {
        c1 = *l;
        if (c1 == 0)
        {
            if (noError)
                break;
            report_invalid_encoding(encoding, (const char *) l, len);
        }
        if (!IS_HIGHBIT_SET(c1))
            *p++ = c1;
        else
        {
            c2 = tab[c1 - HIGHBIT];
            if (c2)
            {
                *p++ = lc;
                *p++ = c2;
            }
            else
            {
                if (noError)
                    break;
                report_untranslatable_char(encoding, PG_MULE_INTERNAL,
                                           (const char *) l, len);
            }
        }
        l++;
        len--;
    }
    *p = '\0';
 
    return l - start;
}

References encoding, HIGHBIT, IS_HIGHBIT_SET, len, PG_MULE_INTERNAL, report_invalid_encoding(), report_untranslatable_char(), and start.

Referenced by iso_to_mic(), win1250_to_mic(), win1251_to_mic(), and win866_to_mic().

local2local()

int local2local	(	const unsigned char *	l,
		unsigned char *	p,
		int	len,
		int	src_encoding,
		int	dest_encoding,
		const unsigned char *	tab,
		bool	noError
	)

Definition at line 33 of file conv.c.

{
    const unsigned char *start = l;
    unsigned char c1,
                c2;
 
    while (len > 0)
    {
        c1 = *l;
        if (c1 == 0)
        {
            if (noError)
                break;
            report_invalid_encoding(src_encoding, (const char *) l, len);
        }
        if (!IS_HIGHBIT_SET(c1))
            *p++ = c1;
        else
        {
            c2 = tab[c1 - HIGHBIT];
            if (c2)
                *p++ = c2;
            else
            {
                if (noError)
                    break;
                report_untranslatable_char(src_encoding, dest_encoding,
                                           (const char *) l, len);
            }
        }
        l++;
        len--;
    }
    *p = '\0';
 
    return l - start;
}

References HIGHBIT, IS_HIGHBIT_SET, len, report_invalid_encoding(), report_untranslatable_char(), and start.

Referenced by iso_to_koi8r(), iso_to_win1251(), iso_to_win866(), koi8r_to_iso(), koi8r_to_win1251(), koi8r_to_win866(), latin2_to_win1250(), win1250_to_latin2(), win1251_to_iso(), win1251_to_koi8r(), win1251_to_win866(), win866_to_iso(), win866_to_koi8r(), and win866_to_win1251().

LocalToUtf()

int LocalToUtf	(	const unsigned char *	iso,
		int	len,
		unsigned char *	utf,
		const pg_mb_radix_tree *	map,
		const pg_local_to_utf_combined *	cmap,
		int	cmapsize,
		utf_local_conversion_func	conv_func,
		int	encoding,
		bool	noError
	)

Definition at line 717 of file conv.c.

{
    uint32      iiso;
    int         l;
    const pg_local_to_utf_combined *cp;
    const unsigned char *start = iso;
 
    if (!PG_VALID_ENCODING(encoding))
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                 errmsg("invalid encoding number: %d", encoding)));
 
    for (; len > 0; len -= l)
    {
        unsigned char b1 = 0;
        unsigned char b2 = 0;
        unsigned char b3 = 0;
        unsigned char b4 = 0;
 
        /* "break" cases all represent errors */
        if (*iso == '\0')
            break;
 
        if (!IS_HIGHBIT_SET(*iso))
        {
            /* ASCII case is easy, assume it's one-to-one conversion */
            *utf++ = *iso++;
            l = 1;
            continue;
        }
 
        l = pg_encoding_verifymbchar(encoding, (const char *) iso, len);
        if (l < 0)
            break;
 
        /* collect coded char of length l */
        if (l == 1)
            b4 = *iso++;
        else if (l == 2)
        {
            b3 = *iso++;
            b4 = *iso++;
        }
        else if (l == 3)
        {
            b2 = *iso++;
            b3 = *iso++;
            b4 = *iso++;
        }
        else if (l == 4)
        {
            b1 = *iso++;
            b2 = *iso++;
            b3 = *iso++;
            b4 = *iso++;
        }
        else
        {
            elog(ERROR, "unsupported character length %d", l);
            iiso = 0;           /* keep compiler quiet */
        }
        iiso = (b1 << 24 | b2 << 16 | b3 << 8 | b4);
 
        if (map)
        {
            uint32      converted = pg_mb_radix_conv(map, l, b1, b2, b3, b4);
 
            if (converted)
            {
                utf = store_coded_char(utf, converted);
                continue;
            }
 
            /* If there's a combined character map, try that */
            if (cmap)
            {
                cp = bsearch(&iiso, cmap, cmapsize,
                             sizeof(pg_local_to_utf_combined), compare4);
 
                if (cp)
                {
                    utf = store_coded_char(utf, cp->utf1);
                    utf = store_coded_char(utf, cp->utf2);
                    continue;
                }
            }
        }
 
        /* if there's a conversion function, try that */
        if (conv_func)
        {
            uint32      converted = (*conv_func) (iiso);
 
            if (converted)
            {
                utf = store_coded_char(utf, converted);
                continue;
            }
        }
 
        /* failed to translate this character */
        iso -= l;
        if (noError)
            break;
        report_untranslatable_char(encoding, PG_UTF8,
                                   (const char *) iso, len);
    }
 
    /* if we broke out of loop early, must be invalid input */
    if (len > 0 && !noError)
        report_invalid_encoding(encoding, (const char *) iso, len);
 
    *utf = '\0';
 
    return iso - start;
}

References compare4(), elog, encoding, ereport, errcode(), errmsg(), ERROR, IS_HIGHBIT_SET, len, pg_encoding_verifymbchar(), pg_mb_radix_conv(), PG_UTF8, PG_VALID_ENCODING, report_invalid_encoding(), report_untranslatable_char(), start, store_coded_char(), pg_local_to_utf_combined::utf1, and pg_local_to_utf_combined::utf2.

Referenced by big5_to_utf8(), euc_cn_to_utf8(), euc_jis_2004_to_utf8(), euc_jp_to_utf8(), euc_kr_to_utf8(), euc_tw_to_utf8(), gb18030_to_utf8(), gbk_to_utf8(), iso8859_to_utf8(), johab_to_utf8(), koi8r_to_utf8(), koi8u_to_utf8(), shift_jis_2004_to_utf8(), sjis_to_utf8(), uhc_to_utf8(), and win_to_utf8().

mic2latin()

int mic2latin	(	const unsigned char *	mic,
		unsigned char *	p,
		int	len,
		int	lc,
		int	encoding,
		bool	noError
	)

Definition at line 127 of file conv.c.

{
    const unsigned char *start = mic;
    int         c1;
 
    while (len > 0)
    {
        c1 = *mic;
        if (c1 == 0)
        {
            if (noError)
                break;
            report_invalid_encoding(PG_MULE_INTERNAL, (const char *) mic, len);
        }
        if (!IS_HIGHBIT_SET(c1))
        {
            /* easy for ASCII */
            *p++ = c1;
            mic++;
            len--;
        }
        else
        {
            int         l = pg_mule_mblen(mic);
 
            if (len < l)
            {
                if (noError)
                    break;
                report_invalid_encoding(PG_MULE_INTERNAL, (const char *) mic,
                                        len);
            }
            if (l != 2 || c1 != lc || !IS_HIGHBIT_SET(mic[1]))
            {
                if (noError)
                    break;
                report_untranslatable_char(PG_MULE_INTERNAL, encoding,
                                           (const char *) mic, len);
            }
            *p++ = mic[1];
            mic += 2;
            len -= 2;
        }
    }
    *p = '\0';
 
    return mic - start;
}

References encoding, IS_HIGHBIT_SET, len, PG_MULE_INTERNAL, pg_mule_mblen(), report_invalid_encoding(), report_untranslatable_char(), and start.

Referenced by mic_to_koi8r(), mic_to_latin1(), mic_to_latin2(), mic_to_latin3(), and mic_to_latin4().

mic2latin_with_table()

int mic2latin_with_table	(	const unsigned char *	mic,
		unsigned char *	p,
		int	len,
		int	lc,
		int	encoding,
		const unsigned char *	tab,
		bool	noError
	)

Definition at line 257 of file conv.c.

{
    const unsigned char *start = mic;
    unsigned char c1,
                c2;
 
    while (len > 0)
    {
        c1 = *mic;
        if (c1 == 0)
        {
            if (noError)
                break;
            report_invalid_encoding(PG_MULE_INTERNAL, (const char *) mic, len);
        }
        if (!IS_HIGHBIT_SET(c1))
        {
            /* easy for ASCII */
            *p++ = c1;
            mic++;
            len--;
        }
        else
        {
            int         l = pg_mule_mblen(mic);
 
            if (len < l)
            {
                if (noError)
                    break;
                report_invalid_encoding(PG_MULE_INTERNAL, (const char *) mic,
                                        len);
            }
            if (l != 2 || c1 != lc || !IS_HIGHBIT_SET(mic[1]) ||
                (c2 = tab[mic[1] - HIGHBIT]) == 0)
            {
                if (noError)
                    break;
                report_untranslatable_char(PG_MULE_INTERNAL, encoding,
                                           (const char *) mic, len);
                break;          /* keep compiler quiet */
            }
            *p++ = c2;
            mic += 2;
            len -= 2;
        }
    }
    *p = '\0';
 
    return mic - start;
}

References encoding, HIGHBIT, IS_HIGHBIT_SET, len, PG_MULE_INTERNAL, pg_mule_mblen(), report_invalid_encoding(), report_untranslatable_char(), and start.

Referenced by mic_to_iso(), mic_to_win1250(), mic_to_win1251(), and mic_to_win866().

pg_mb_radix_conv()

static uint32 pg_mb_radix_conv ( const pg_mb_radix_tree *  rt, int  l, unsigned char  b1, unsigned char  b2, unsigned char  b3, unsigned char  b4  )

inlinestatic

Definition at line 373 of file conv.c.

{
    if (l == 4)
    {
        /* 4-byte code */
 
        /* check code validity */
        if (b1 < rt->b4_1_lower || b1 > rt->b4_1_upper ||
            b2 < rt->b4_2_lower || b2 > rt->b4_2_upper ||
            b3 < rt->b4_3_lower || b3 > rt->b4_3_upper ||
            b4 < rt->b4_4_lower || b4 > rt->b4_4_upper)
            return 0;
 
        /* perform lookup */
        if (rt->chars32)
        {
            uint32      idx = rt->b4root;
 
            idx = rt->chars32[b1 + idx - rt->b4_1_lower];
            idx = rt->chars32[b2 + idx - rt->b4_2_lower];
            idx = rt->chars32[b3 + idx - rt->b4_3_lower];
            return rt->chars32[b4 + idx - rt->b4_4_lower];
        }
        else
        {
            uint16      idx = rt->b4root;
 
            idx = rt->chars16[b1 + idx - rt->b4_1_lower];
            idx = rt->chars16[b2 + idx - rt->b4_2_lower];
            idx = rt->chars16[b3 + idx - rt->b4_3_lower];
            return rt->chars16[b4 + idx - rt->b4_4_lower];
        }
    }
    else if (l == 3)
    {
        /* 3-byte code */
 
        /* check code validity */
        if (b2 < rt->b3_1_lower || b2 > rt->b3_1_upper ||
            b3 < rt->b3_2_lower || b3 > rt->b3_2_upper ||
            b4 < rt->b3_3_lower || b4 > rt->b3_3_upper)
            return 0;
 
        /* perform lookup */
        if (rt->chars32)
        {
            uint32      idx = rt->b3root;
 
            idx = rt->chars32[b2 + idx - rt->b3_1_lower];
            idx = rt->chars32[b3 + idx - rt->b3_2_lower];
            return rt->chars32[b4 + idx - rt->b3_3_lower];
        }
        else
        {
            uint16      idx = rt->b3root;
 
            idx = rt->chars16[b2 + idx - rt->b3_1_lower];
            idx = rt->chars16[b3 + idx - rt->b3_2_lower];
            return rt->chars16[b4 + idx - rt->b3_3_lower];
        }
    }
    else if (l == 2)
    {
        /* 2-byte code */
 
        /* check code validity - first byte */
        if (b3 < rt->b2_1_lower || b3 > rt->b2_1_upper ||
            b4 < rt->b2_2_lower || b4 > rt->b2_2_upper)
            return 0;
 
        /* perform lookup */
        if (rt->chars32)
        {
            uint32      idx = rt->b2root;
 
            idx = rt->chars32[b3 + idx - rt->b2_1_lower];
            return rt->chars32[b4 + idx - rt->b2_2_lower];
        }
        else
        {
            uint16      idx = rt->b2root;
 
            idx = rt->chars16[b3 + idx - rt->b2_1_lower];
            return rt->chars16[b4 + idx - rt->b2_2_lower];
        }
    }
    else if (l == 1)
    {
        /* 1-byte code */
 
        /* check code validity - first byte */
        if (b4 < rt->b1_lower || b4 > rt->b1_upper)
            return 0;
 
        /* perform lookup */
        if (rt->chars32)
            return rt->chars32[b4 + rt->b1root - rt->b1_lower];
        else
            return rt->chars16[b4 + rt->b1root - rt->b1_lower];
    }
    return 0;                   /* shouldn't happen */
}

References pg_mb_radix_tree::b1_lower, pg_mb_radix_tree::b1_upper, pg_mb_radix_tree::b1root, pg_mb_radix_tree::b2_1_lower, pg_mb_radix_tree::b2_1_upper, pg_mb_radix_tree::b2_2_lower, pg_mb_radix_tree::b2_2_upper, pg_mb_radix_tree::b2root, pg_mb_radix_tree::b3_1_lower, pg_mb_radix_tree::b3_1_upper, pg_mb_radix_tree::b3_2_lower, pg_mb_radix_tree::b3_2_upper, pg_mb_radix_tree::b3_3_lower, pg_mb_radix_tree::b3_3_upper, pg_mb_radix_tree::b3root, pg_mb_radix_tree::b4_1_lower, pg_mb_radix_tree::b4_1_upper, pg_mb_radix_tree::b4_2_lower, pg_mb_radix_tree::b4_2_upper, pg_mb_radix_tree::b4_3_lower, pg_mb_radix_tree::b4_3_upper, pg_mb_radix_tree::b4_4_lower, pg_mb_radix_tree::b4_4_upper, pg_mb_radix_tree::b4root, pg_mb_radix_tree::chars16, pg_mb_radix_tree::chars32, and idx().

Referenced by LocalToUtf(), and UtfToLocal().

store_coded_char()

static unsigned char * store_coded_char ( unsigned char *  dest, uint32  code  )

inlinestatic

Definition at line 353 of file conv.c.

{
    if (code & 0xff000000)
        *dest++ = code >> 24;
    if (code & 0x00ff0000)
        *dest++ = code >> 16;
    if (code & 0x0000ff00)
        *dest++ = code >> 8;
    if (code & 0x000000ff)
        *dest++ = code;
    return dest;
}

References generate_unaccent_rules::dest.

Referenced by LocalToUtf(), and UtfToLocal().

UtfToLocal()

int UtfToLocal	(	const unsigned char *	utf,
		int	len,
		unsigned char *	iso,
		const pg_mb_radix_tree *	map,
		const pg_utf_to_local_combined *	cmap,
		int	cmapsize,
		utf_local_conversion_func	conv_func,
		int	encoding,
		bool	noError
	)

Definition at line 507 of file conv.c.

{
    uint32      iutf;
    int         l;
    const pg_utf_to_local_combined *cp;
    const unsigned char *start = utf;
 
    if (!PG_VALID_ENCODING(encoding))
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                 errmsg("invalid encoding number: %d", encoding)));
 
    for (; len > 0; len -= l)
    {
        unsigned char b1 = 0;
        unsigned char b2 = 0;
        unsigned char b3 = 0;
        unsigned char b4 = 0;
 
        /* "break" cases all represent errors */
        if (*utf == '\0')
            break;
 
        l = pg_utf_mblen(utf);
        if (len < l)
            break;
 
        if (!pg_utf8_islegal(utf, l))
            break;
 
        if (l == 1)
        {
            /* ASCII case is easy, assume it's one-to-one conversion */
            *iso++ = *utf++;
            continue;
        }
 
        /* collect coded char of length l */
        if (l == 2)
        {
            b3 = *utf++;
            b4 = *utf++;
        }
        else if (l == 3)
        {
            b2 = *utf++;
            b3 = *utf++;
            b4 = *utf++;
        }
        else if (l == 4)
        {
            b1 = *utf++;
            b2 = *utf++;
            b3 = *utf++;
            b4 = *utf++;
        }
        else
        {
            elog(ERROR, "unsupported character length %d", l);
            iutf = 0;           /* keep compiler quiet */
        }
        iutf = (b1 << 24 | b2 << 16 | b3 << 8 | b4);
 
        /* First, try with combined map if possible */
        if (cmap && len > l)
        {
            const unsigned char *utf_save = utf;
            int         len_save = len;
            int         l_save = l;
 
            /* collect next character, same as above */
            len -= l;
 
            l = pg_utf_mblen(utf);
            if (len < l)
            {
                /* need more data to decide if this is a combined char */
                utf -= l_save;
                break;
            }
 
            if (!pg_utf8_islegal(utf, l))
            {
                if (!noError)
                    report_invalid_encoding(PG_UTF8, (const char *) utf, len);
                utf -= l_save;
                break;
            }
 
            /* We assume ASCII character cannot be in combined map */
            if (l > 1)
            {
                uint32      iutf2;
                uint32      cutf[2];
 
                if (l == 2)
                {
                    iutf2 = *utf++ << 8;
                    iutf2 |= *utf++;
                }
                else if (l == 3)
                {
                    iutf2 = *utf++ << 16;
                    iutf2 |= *utf++ << 8;
                    iutf2 |= *utf++;
                }
                else if (l == 4)
                {
                    iutf2 = *utf++ << 24;
                    iutf2 |= *utf++ << 16;
                    iutf2 |= *utf++ << 8;
                    iutf2 |= *utf++;
                }
                else
                {
                    elog(ERROR, "unsupported character length %d", l);
                    iutf2 = 0;  /* keep compiler quiet */
                }
 
                cutf[0] = iutf;
                cutf[1] = iutf2;
 
                cp = bsearch(cutf, cmap, cmapsize,
                             sizeof(pg_utf_to_local_combined), compare3);
 
                if (cp)
                {
                    iso = store_coded_char(iso, cp->code);
                    continue;
                }
            }
 
            /* fail, so back up to reprocess second character next time */
            utf = utf_save;
            len = len_save;
            l = l_save;
        }
 
        /* Now check ordinary map */
        if (map)
        {
            uint32      converted = pg_mb_radix_conv(map, l, b1, b2, b3, b4);
 
            if (converted)
            {
                iso = store_coded_char(iso, converted);
                continue;
            }
        }
 
        /* if there's a conversion function, try that */
        if (conv_func)
        {
            uint32      converted = (*conv_func) (iutf);
 
            if (converted)
            {
                iso = store_coded_char(iso, converted);
                continue;
            }
        }
 
        /* failed to translate this character */
        utf -= l;
        if (noError)
            break;
        report_untranslatable_char(PG_UTF8, encoding,
                                   (const char *) utf, len);
    }
 
    /* if we broke out of loop early, must be invalid input */
    if (len > 0 && !noError)
        report_invalid_encoding(PG_UTF8, (const char *) utf, len);
 
    *iso = '\0';
 
    return utf - start;
}

References pg_utf_to_local_combined::code, compare3(), elog, encoding, ereport, errcode(), errmsg(), ERROR, len, pg_mb_radix_conv(), PG_UTF8, pg_utf8_islegal(), pg_utf_mblen, PG_VALID_ENCODING, report_invalid_encoding(), report_untranslatable_char(), start, and store_coded_char().

Referenced by utf8_to_big5(), utf8_to_euc_cn(), utf8_to_euc_jis_2004(), utf8_to_euc_jp(), utf8_to_euc_kr(), utf8_to_euc_tw(), utf8_to_gb18030(), utf8_to_gbk(), utf8_to_iso8859(), utf8_to_johab(), utf8_to_koi8r(), utf8_to_koi8u(), utf8_to_shift_jis_2004(), utf8_to_sjis(), utf8_to_uhc(), and utf8_to_win().