scan.l File Reference

#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include "common/string.h"
#include "gramparse.h"
#include "nodes/miscnodes.h"
#include "parser/parser.h"
#include "parser/scansup.h"
#include "port/pg_bitutils.h"
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "parser/kwlist.h"

Include dependency graph for scan.l:

Go to the source code of this file.

Macros
#define	fprintf(file, fmt, msg)   fprintf_to_ereport(fmt, msg)
#define	PG_KEYWORD(kwname, value, category, collabel)   value,
#define	YYSTYPE   core_YYSTYPE
#define	SET_YYLLOC()   (*(yylloc) = yytext - yyextra->scanbuf)
#define	ADVANCE_YYLLOC(delta)   ( *(yylloc) += (delta) )
#define	PUSH_YYLLOC()   (yyextra->save_yylloc = *(yylloc))
#define	POP_YYLLOC()   (*(yylloc) = yyextra->save_yylloc)
#define	startlit()   ( yyextra->literallen = 0 )
#define	yyerror(msg)   scanner_yyerror(msg, yyscanner)
#define	lexer_errposition()   scanner_errposition(*(yylloc), yyscanner)
#define	yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)
#define	yylloc   (((struct yyguts_t *) yyscanner)->yylloc_r)
#define	yyleng   (((struct yyguts_t *) yyscanner)->yyleng_r)

Functions
static void	fprintf_to_ereport (const char *fmt, const char *msg)
static void	addlit (char *ytext, int yleng, core_yyscan_t yyscanner)
static void	addlitchar (unsigned char ychar, core_yyscan_t yyscanner)
static char *	litbufdup (core_yyscan_t yyscanner)
static unsigned char	unescape_single_char (unsigned char c, core_yyscan_t yyscanner)
static int	process_integer_literal (const char *token, YYSTYPE *lval, int base)
static void	addunicode (char32_t c, yyscan_t yyscanner)
static void	check_string_escape_warning (unsigned char ychar, core_yyscan_t yyscanner)
static void	check_escape_warning (core_yyscan_t yyscanner)
int	yylex (YYSTYPE *yylval_param, YYLTYPE *yylloc_param, yyscan_t yyscanner)
int	scanner_errposition (int location, core_yyscan_t yyscanner)
static void	scb_error_callback (void *arg)
void	setup_scanner_errposition_callback (ScannerCallbackState *scbstate, core_yyscan_t yyscanner, int location)
void	cancel_scanner_errposition_callback (ScannerCallbackState *scbstate)
void	scanner_yyerror (const char *message, core_yyscan_t yyscanner)
core_yyscan_t	scanner_init (const char *str, core_yy_extra_type *yyext, const ScanKeywordList *keywordlist, const uint16 *keyword_tokens)
void	scanner_finish (core_yyscan_t yyscanner)
void *	core_yyalloc (yy_size_t bytes, core_yyscan_t yyscanner)
void *	core_yyrealloc (void *ptr, yy_size_t bytes, core_yyscan_t yyscanner)
void	core_yyfree (void *ptr, core_yyscan_t yyscanner)

Variables
int	backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING
bool	escape_string_warning = true
bool	standard_conforming_strings = true
const uint16	ScanKeywordTokens []

Macro Definition Documentation

ADVANCE_YYLLOC

#define ADVANCE_YYLLOC

(

delta

)

( *(yylloc) += (delta) )

Definition at line 104 of file scan.l.

fprintf

#define fprintf	(		file,
			fmt,
			msg
	)		fprintf_to_ereport(fmt, msg)

Definition at line 54 of file scan.l.

lexer_errposition

#define lexer_errposition

(

)

scanner_errposition(*(yylloc), yyscanner)

Definition at line 128 of file scan.l.

PG_KEYWORD

#define PG_KEYWORD	(		kwname,
			value,
			category,
			collabel
	)		value,

Definition at line 79 of file scan.l.

POP_YYLLOC

#define POP_YYLLOC

(

)

(*(yylloc) = yyextra->save_yylloc)

Definition at line 116 of file scan.l.

PUSH_YYLLOC

#define PUSH_YYLLOC

(

)

(yyextra->save_yylloc = *(yylloc))

Definition at line 115 of file scan.l.

SET_YYLLOC

#define SET_YYLLOC

(

)

(*(yylloc) = yytext - yyextra->scanbuf)

Definition at line 99 of file scan.l.

startlit

#define startlit

(

)

( yyextra->literallen = 0 )

Definition at line 118 of file scan.l.

yyerror

#define yyerror

(

msg

)

scanner_yyerror(msg, yyscanner)

Definition at line 126 of file scan.l.

yyextra

#define yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)

Definition at line 1118 of file scan.l.

yyleng

#define yyleng   (((struct yyguts_t *) yyscanner)->yyleng_r)

Definition at line 1124 of file scan.l.

yylloc

#define yylloc   (((struct yyguts_t *) yyscanner)->yylloc_r)

Definition at line 1122 of file scan.l.

YYSTYPE

#define YYSTYPE   core_YYSTYPE

Definition at line 90 of file scan.l.

Function Documentation

addlit()

static void addlit ( char *  ytext, int  yleng, core_yyscan_t  yyscanner  )

static

Definition at line 1311 of file scan.l.

{
    /* enlarge buffer if needed */
    if ((yyextra->literallen + yleng) >= yyextra->literalalloc)
    {
        yyextra->literalalloc = pg_nextpower2_32(yyextra->literallen + yleng + 1);
        yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
                                                yyextra->literalalloc);
    }
    /* append new data */
    memcpy(yyextra->literalbuf + yyextra->literallen, ytext, yleng);
    yyextra->literallen += yleng;
}

References pg_nextpower2_32(), repalloc(), and yyextra.

Referenced by addunicode().

addlitchar()

static void addlitchar ( unsigned char  ychar, core_yyscan_t  yyscanner  )

static

Definition at line 1327 of file scan.l.

{
    /* enlarge buffer if needed */
    if ((yyextra->literallen + 1) >= yyextra->literalalloc)
    {
        yyextra->literalalloc *= 2;
        yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
                                                yyextra->literalalloc);
    }
    /* append new data */
    yyextra->literalbuf[yyextra->literallen] = ychar;
    yyextra->literallen += 1;
}

References repalloc(), and yyextra.

addunicode()

static void addunicode ( char32_t  c, yyscan_t  yyscanner  )

static

Definition at line 1379 of file scan.l.

{
    ScannerCallbackState scbstate;
    char        buf[MAX_UNICODE_EQUIVALENT_STRING + 1];
 
    if (!is_valid_unicode_codepoint(c))
        yyerror("invalid Unicode escape value");
 
    /*
     * We expect that pg_unicode_to_server() will complain about any
     * unconvertible code point, so we don't have to set saw_non_ascii.
     */
    setup_scanner_errposition_callback(&scbstate, yyscanner, *(yylloc));
    pg_unicode_to_server(c, (unsigned char *) buf);
    cancel_scanner_errposition_callback(&scbstate);
    addlit(buf, strlen(buf), yyscanner);
}

References addlit(), buf, cancel_scanner_errposition_callback(), is_valid_unicode_codepoint(), MAX_UNICODE_EQUIVALENT_STRING, pg_unicode_to_server(), setup_scanner_errposition_callback(), yyerror, and yylloc.

cancel_scanner_errposition_callback()

void cancel_scanner_errposition_callback

(

ScannerCallbackState *

scbstate

)

Definition at line 1203 of file scan.l.

{
    /* Pop the error context stack */
    error_context_stack = scbstate->errcallback.previous;
}

References ScannerCallbackState::errcallback, error_context_stack, and ErrorContextCallback::previous.

Referenced by addunicode(), and str_udeescape().

check_escape_warning()

static void check_escape_warning ( core_yyscan_t  yyscanner )

static

Definition at line 1451 of file scan.l.

{
    if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
        ereport(WARNING,
                (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
                 errmsg("nonstandard use of escape in a string literal"),
                 errhint("Use the escape string syntax for escapes, e.g., E'\\r\\n'."),
                 lexer_errposition()));
    yyextra->warn_on_first_escape = false;  /* warn only once per string */
}

References ereport, errcode(), errhint(), errmsg(), lexer_errposition, WARNING, and yyextra.

Referenced by check_string_escape_warning().

check_string_escape_warning()

static void check_string_escape_warning ( unsigned char  ychar, core_yyscan_t  yyscanner  )

static

Definition at line 1424 of file scan.l.

{
    if (ychar == '\'')
    {
        if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
            ereport(WARNING,
                    (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
                     errmsg("nonstandard use of \\' in a string literal"),
                     errhint("Use '' to write quotes in strings, or use the escape string syntax (E'...')."),
                     lexer_errposition()));
        yyextra->warn_on_first_escape = false;  /* warn only once per string */
    }
    else if (ychar == '\\')
    {
        if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
            ereport(WARNING,
                    (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
                     errmsg("nonstandard use of \\\\ in a string literal"),
                     errhint("Use the escape string syntax for backslashes, e.g., E'\\\\'."),
                     lexer_errposition()));
        yyextra->warn_on_first_escape = false;  /* warn only once per string */
    }
    else
        check_escape_warning(yyscanner);
}

References check_escape_warning(), ereport, errcode(), errhint(), errmsg(), lexer_errposition, WARNING, and yyextra.

core_yyalloc()

void * core_yyalloc	(	yy_size_t	bytes,
		core_yyscan_t	yyscanner
	)

Definition at line 1468 of file scan.l.

{
    return palloc(bytes);
}

References palloc().

core_yyfree()

void core_yyfree	(	void *	ptr,
		core_yyscan_t	yyscanner
	)

Definition at line 1483 of file scan.l.

{
    if (ptr)
        pfree(ptr);
}

References pfree().

core_yyrealloc()

void * core_yyrealloc	(	void *	ptr,
		yy_size_t	bytes,
		core_yyscan_t	yyscanner
	)

Definition at line 1474 of file scan.l.

{
    if (ptr)
        return repalloc(ptr, bytes);
    else
        return palloc(bytes);
}

References palloc(), and repalloc().

fprintf_to_ereport()

static void fprintf_to_ereport ( const char *  fmt, const char *  msg  )

static

Definition at line 57 of file scan.l.

{
    ereport(ERROR, (errmsg_internal("%s", msg)));
}

References ereport, errmsg_internal(), and ERROR.

litbufdup()

static char * litbufdup ( core_yyscan_t  yyscanner )

static

Definition at line 1346 of file scan.l.

{
    int         llen = yyextra->literallen;
    char       *new;
 
    new = palloc(llen + 1);
    memcpy(new, yyextra->literalbuf, llen);
    new[llen] = '\0';
    return new;
}

References palloc(), and yyextra.

process_integer_literal()

static int process_integer_literal ( const char *  token, YYSTYPE *  lval, int  base  )

static

Definition at line 1362 of file scan.l.

{
    ErrorSaveContext escontext = {T_ErrorSaveContext};
    int32       val;
 
    val = pg_strtoint32_safe(token, (Node *) &escontext);
    if (escontext.error_occurred)
    {
        /* integer too large (or contains decimal pt), treat it as a float */
        lval->str = pstrdup(token);
        return FCONST;
    }
    lval->ival = val;
    return ICONST;
}

References ErrorSaveContext::error_occurred, pg_strtoint32_safe(), pstrdup(), and val.

scanner_errposition()

int scanner_errposition	(	int	location,
		core_yyscan_t	yyscanner
	)

Definition at line 1140 of file scan.l.

{
    int         pos;
 
    if (location < 0)
        return 0;               /* no-op if location is unknown */
 
    /* Convert byte offset to character number */
    pos = pg_mbstrlen_with_len(yyextra->scanbuf, location) + 1;
    /* And pass it to the ereport mechanism */
    return errposition(pos);
}

References errposition(), pg_mbstrlen_with_len(), and yyextra.

Referenced by scb_error_callback(), and str_udeescape().

scanner_finish()

void scanner_finish

(

core_yyscan_t

yyscanner

)

Definition at line 1291 of file scan.l.

{
    /*
     * We don't bother to call yylex_destroy(), because all it would do is
     * pfree a small amount of control storage.  It's cheaper to leak the
     * storage until the parsing context is destroyed.  The amount of space
     * involved is usually negligible compared to the output parse tree
     * anyway.
     *
     * We do bother to pfree the scanbuf and literal buffer, but only if they
     * represent a nontrivial amount of space.  The 8K cutoff is arbitrary.
     */
    if (yyextra->scanbuflen >= 8192)
        pfree(yyextra->scanbuf);
    if (yyextra->literalalloc >= 8192)
        pfree(yyextra->literalbuf);
}

References pfree(), and yyextra.

Referenced by fill_in_constant_lengths(), plpgsql_scanner_finish(), and raw_parser().

scanner_init()

core_yyscan_t scanner_init	(	const char *	str,
		core_yy_extra_type *	yyext,
		const ScanKeywordList *	keywordlist,
		const uint16 *	keyword_tokens
	)

Definition at line 1249 of file scan.l.

{
    Size        slen = strlen(str);
    yyscan_t    scanner;
 
    if (yylex_init(&scanner) != 0)
        elog(ERROR, "yylex_init() failed: %m");
 
    core_yyset_extra(yyext, scanner);
 
    yyext->keywordlist = keywordlist;
    yyext->keyword_tokens = keyword_tokens;
 
    yyext->backslash_quote = backslash_quote;
    yyext->escape_string_warning = escape_string_warning;
    yyext->standard_conforming_strings = standard_conforming_strings;
 
    /*
     * Make a scan buffer with special termination needed by flex.
     */
    yyext->scanbuf = (char *) palloc(slen + 2);
    yyext->scanbuflen = slen;
    memcpy(yyext->scanbuf, str, slen);
    yyext->scanbuf[slen] = yyext->scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
    yy_scan_buffer(yyext->scanbuf, slen + 2, scanner);
 
    /* initialize literal buffer to a reasonable but expansible size */
    yyext->literalalloc = 1024;
    yyext->literalbuf = (char *) palloc(yyext->literalalloc);
    yyext->literallen = 0;
 
    return scanner;
}

References backslash_quote, core_yy_extra_type::backslash_quote, elog, ERROR, escape_string_warning, core_yy_extra_type::escape_string_warning, core_yy_extra_type::keyword_tokens, core_yy_extra_type::keywordlist, core_yy_extra_type::literalalloc, core_yy_extra_type::literalbuf, core_yy_extra_type::literallen, palloc(), core_yy_extra_type::scanbuf, core_yy_extra_type::scanbuflen, standard_conforming_strings, core_yy_extra_type::standard_conforming_strings, and str.

Referenced by fill_in_constant_lengths(), plpgsql_scanner_init(), and raw_parser().

scanner_yyerror()

void scanner_yyerror	(	const char *	message,
		core_yyscan_t	yyscanner
	)

Definition at line 1222 of file scan.l.

{
    const char *loc = yyextra->scanbuf + *yylloc;
 
    if (*loc == YY_END_OF_BUFFER_CHAR)
    {
        ereport(ERROR,
                (errcode(ERRCODE_SYNTAX_ERROR),
        /* translator: %s is typically the translation of "syntax error" */
                 errmsg("%s at end of input", _(message)),
                 lexer_errposition()));
    }
    else
    {
        ereport(ERROR,
                (errcode(ERRCODE_SYNTAX_ERROR),
        /* translator: first %s is typically the translation of "syntax error" */
                 errmsg("%s at or near \"%s\"", _(message), loc),
                 lexer_errposition()));
    }
}

References _, ereport, errcode(), errmsg(), ERROR, lexer_errposition, yyextra, and yylloc.

Referenced by base_yylex().

scb_error_callback()

static void scb_error_callback ( void *  arg )

static

Definition at line 1161 of file scan.l.

{
    ScannerCallbackState *scbstate = (ScannerCallbackState *) arg;
 
    if (geterrcode() != ERRCODE_QUERY_CANCELED)
        (void) scanner_errposition(scbstate->location, scbstate->yyscanner);
}

References arg, geterrcode(), ScannerCallbackState::location, scanner_errposition(), and ScannerCallbackState::yyscanner.

Referenced by setup_scanner_errposition_callback().

setup_scanner_errposition_callback()

void setup_scanner_errposition_callback	(	ScannerCallbackState *	scbstate,
		core_yyscan_t	yyscanner,
		int	location
	)

Definition at line 1186 of file scan.l.

{
    /* Setup error traceback support for ereport() */
    scbstate->yyscanner = yyscanner;
    scbstate->location = location;
    scbstate->errcallback.callback = scb_error_callback;
    scbstate->errcallback.arg = scbstate;
    scbstate->errcallback.previous = error_context_stack;
    error_context_stack = &scbstate->errcallback;
}

References ErrorContextCallback::arg, ErrorContextCallback::callback, ScannerCallbackState::errcallback, error_context_stack, ScannerCallbackState::location, ErrorContextCallback::previous, scb_error_callback(), and ScannerCallbackState::yyscanner.

Referenced by addunicode(), and str_udeescape().

unescape_single_char()

static unsigned char unescape_single_char ( unsigned char  c, core_yyscan_t  yyscanner  )

static

Definition at line 1398 of file scan.l.

{
    switch (c)
    {
        case 'b':
            return '\b';
        case 'f':
            return '\f';
        case 'n':
            return '\n';
        case 'r':
            return '\r';
        case 't':
            return '\t';
        case 'v':
            return '\v';
        default:
            /* check for backslash followed by non-7-bit-ASCII */
            if (c == '\0' || IS_HIGHBIT_SET(c))
                yyextra->saw_non_ascii = true;
 
            return c;
    }
}

References IS_HIGHBIT_SET, and yyextra.

yylex()

int yylex	(	YYSTYPE *	yylval_param,
		YYLTYPE *	yylloc_param,
		yyscan_t	yyscanner
	)

Definition at line 441 of file scan.l.

{whitespace}    {
                    /* ignore */
                }
 
{xcstart}       {
                    /* Set location in case of syntax error in comment */
                    SET_YYLLOC();
                    yyextra->xcdepth = 0;
                    BEGIN(xc);
                    /* Put back any characters past slash-star; see above */
                    yyless(2);
                }
 
<xc>{
{xcstart}       {
                    (yyextra->xcdepth)++;
                    /* Put back any characters past slash-star; see above */
                    yyless(2);
                }
 
{xcstop}        {
                    if (yyextra->xcdepth <= 0)
                        BEGIN(INITIAL);
                    else
                        (yyextra->xcdepth)--;
                }
 
{xcinside}      {
                    /* ignore */
                }
 
{op_chars}      {
                    /* ignore */
                }
 
\*+             {
                    /* ignore */
                }
 
<<EOF>>         {
                    yyerror("unterminated /* comment");
                }
} /* <xc> */
 
{xbstart}       {
                    /* Binary bit type.
                     * At some point we should simply pass the string
                     * forward to the parser and label it there.
                     * In the meantime, place a leading "b" on the string
                     * to mark it for the input routine as a binary string.
                     */
                    SET_YYLLOC();
                    BEGIN(xb);
                    startlit();
                    addlitchar('b', yyscanner);
                }
<xh>{xhinside}  |
<xb>{xbinside}  {
                    addlit(yytext, yyleng, yyscanner);
                }
<xb><<EOF>>     { yyerror("unterminated bit string literal"); }
 
{xhstart}       {
                    /* Hexadecimal bit type.
                     * At some point we should simply pass the string
                     * forward to the parser and label it there.
                     * In the meantime, place a leading "x" on the string
                     * to mark it for the input routine as a hex string.
                     */
                    SET_YYLLOC();
                    BEGIN(xh);
                    startlit();
                    addlitchar('x', yyscanner);
                }
<xh><<EOF>>     { yyerror("unterminated hexadecimal string literal"); }
 
{xnstart}       {
                    /* National character.
                     * We will pass this along as a normal character string,
                     * but preceded with an internally-generated "NCHAR".
                     */
                    int     kwnum;
 
                    SET_YYLLOC();
                    yyless(1);  /* eat only 'n' this time */
 
                    kwnum = ScanKeywordLookup("nchar",
                                              yyextra->keywordlist);
                    if (kwnum >= 0)
                    {
                        yylval->keyword = GetScanKeyword(kwnum,
                                                         yyextra->keywordlist);
                        return yyextra->keyword_tokens[kwnum];
                    }
                    else
                    {
                        /* If NCHAR isn't a keyword, just return "n" */
                        yylval->str = pstrdup("n");
                        return IDENT;
                    }
                }
 
{xqstart}       {
                    yyextra->warn_on_first_escape = true;
                    yyextra->saw_non_ascii = false;
                    SET_YYLLOC();
                    if (yyextra->standard_conforming_strings)
                        BEGIN(xq);
                    else
                        BEGIN(xe);
                    startlit();
                }
{xestart}       {
                    yyextra->warn_on_first_escape = false;
                    yyextra->saw_non_ascii = false;
                    SET_YYLLOC();
                    BEGIN(xe);
                    startlit();
                }
{xusstart}      {
                    SET_YYLLOC();
                    if (!yyextra->standard_conforming_strings)
                        ereport(ERROR,
                                (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                                 errmsg("unsafe use of string constant with Unicode escapes"),
                                 errdetail("String constants with Unicode escapes cannot be used when \"standard_conforming_strings\" is off."),
                                 lexer_errposition()));
                    BEGIN(xus);
                    startlit();
                }
 
<xb,xh,xq,xe,xus>{quote} {
                    /*
                     * When we are scanning a quoted string and see an end
                     * quote, we must look ahead for a possible continuation.
                     * If we don't see one, we know the end quote was in fact
                     * the end of the string.  To reduce the lexer table size,
                     * we use a single "xqs" state to do the lookahead for all
                     * types of strings.
                     */
                    yyextra->state_before_str_stop = YYSTATE;
                    BEGIN(xqs);
                }
<xqs>{quotecontinue} {
                    /*
                     * Found a quote continuation, so return to the in-quote
                     * state and continue scanning the literal.  Nothing is
                     * added to the literal's contents.
                     */
                    BEGIN(yyextra->state_before_str_stop);
                }
<xqs>{quotecontinuefail} |
<xqs>{other} |
<xqs><<EOF>>    {
                    /*
                     * Failed to see a quote continuation.  Throw back
                     * everything after the end quote, and handle the string
                     * according to the state we were in previously.
                     */
                    yyless(0);
                    BEGIN(INITIAL);
 
                    switch (yyextra->state_before_str_stop)
                    {
                        case xb:
                            yylval->str = litbufdup(yyscanner);
                            return BCONST;
                        case xh:
                            yylval->str = litbufdup(yyscanner);
                            return XCONST;
                        case xq:
                        case xe:
                            /*
                             * Check that the data remains valid, if it might
                             * have been made invalid by unescaping any chars.
                             */
                            if (yyextra->saw_non_ascii)
                                pg_verifymbstr(yyextra->literalbuf,
                                               yyextra->literallen,
                                               false);
                            yylval->str = litbufdup(yyscanner);
                            return SCONST;
                        case xus:
                            yylval->str = litbufdup(yyscanner);
                            return USCONST;
                        default:
                            yyerror("unhandled previous state in xqs");
                    }
                }
 
<xq,xe,xus>{xqdouble} {
                    addlitchar('\'', yyscanner);
                }
<xq,xus>{xqinside}  {
                    addlit(yytext, yyleng, yyscanner);
                }
<xe>{xeinside}  {
                    addlit(yytext, yyleng, yyscanner);
                }
<xe>{xeunicode} {
                    char32_t    c = strtoul(yytext + 2, NULL, 16);
 
                    /*
                     * For consistency with other productions, issue any
                     * escape warning with cursor pointing to start of string.
                     * We might want to change that, someday.
                     */
                    check_escape_warning(yyscanner);
 
                    /* Remember start of overall string token ... */
                    PUSH_YYLLOC();
                    /* ... and set the error cursor to point at this esc seq */
                    SET_YYLLOC();
 
                    if (is_utf16_surrogate_first(c))
                    {
                        yyextra->utf16_first_part = c;
                        BEGIN(xeu);
                    }
                    else if (is_utf16_surrogate_second(c))
                        yyerror("invalid Unicode surrogate pair");
                    else
                        addunicode(c, yyscanner);
 
                    /* Restore yylloc to be start of string token */
                    POP_YYLLOC();
                }
<xeu>{xeunicode} {
                    char32_t    c = strtoul(yytext + 2, NULL, 16);
 
                    /* Remember start of overall string token ... */
                    PUSH_YYLLOC();
                    /* ... and set the error cursor to point at this esc seq */
                    SET_YYLLOC();
 
                    if (!is_utf16_surrogate_second(c))
                        yyerror("invalid Unicode surrogate pair");
 
                    c = surrogate_pair_to_codepoint(yyextra->utf16_first_part, c);
 
                    addunicode(c, yyscanner);
 
                    /* Restore yylloc to be start of string token */
                    POP_YYLLOC();
 
                    BEGIN(xe);
                }
<xeu>. |
<xeu>\n |
<xeu><<EOF>>    {
                    /* Set the error cursor to point at missing esc seq */
                    SET_YYLLOC();
                    yyerror("invalid Unicode surrogate pair");
                }
<xe,xeu>{xeunicodefail} {
                    /* Set the error cursor to point at malformed esc seq */
                    SET_YYLLOC();
                    ereport(ERROR,
                            (errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE),
                             errmsg("invalid Unicode escape"),
                             errhint("Unicode escapes must be \\uXXXX or \\UXXXXXXXX."),
                             lexer_errposition()));
                }
<xe>{xeescape}  {
                    if (yytext[1] == '\'')
                    {
                        if (yyextra->backslash_quote == BACKSLASH_QUOTE_OFF ||
                            (yyextra->backslash_quote == BACKSLASH_QUOTE_SAFE_ENCODING &&
                             PG_ENCODING_IS_CLIENT_ONLY(pg_get_client_encoding())))
                            ereport(ERROR,
                                    (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
                                     errmsg("unsafe use of \\' in a string literal"),
                                     errhint("Use '' to write quotes in strings. \\' is insecure in client-only encodings."),
                                     lexer_errposition()));
                    }
                    check_string_escape_warning(yytext[1], yyscanner);
                    addlitchar(unescape_single_char(yytext[1], yyscanner),
                               yyscanner);
                }
<xe>{xeoctesc}  {
                    unsigned char c = strtoul(yytext + 1, NULL, 8);
 
                    check_escape_warning(yyscanner);
                    addlitchar(c, yyscanner);
                    if (c == '\0' || IS_HIGHBIT_SET(c))
                        yyextra->saw_non_ascii = true;
                }
<xe>{xehexesc}  {
                    unsigned char c = strtoul(yytext + 2, NULL, 16);
 
                    check_escape_warning(yyscanner);
                    addlitchar(c, yyscanner);
                    if (c == '\0' || IS_HIGHBIT_SET(c))
                        yyextra->saw_non_ascii = true;
                }
<xe>.           {
                    /* This is only needed for \ just before EOF */
                    addlitchar(yytext[0], yyscanner);
                }
<xq,xe,xus><<EOF>>      { yyerror("unterminated quoted string"); }
 
{dolqdelim}     {
                    SET_YYLLOC();
                    yyextra->dolqstart = pstrdup(yytext);
                    BEGIN(xdolq);
                    startlit();
                }
{dolqfailed}    {
                    SET_YYLLOC();
                    /* throw back all but the initial "$" */
                    yyless(1);
                    /* and treat it as {other} */
                    return yytext[0];
                }
<xdolq>{dolqdelim} {
                    if (strcmp(yytext, yyextra->dolqstart) == 0)
                    {
                        pfree(yyextra->dolqstart);
                        yyextra->dolqstart = NULL;
                        BEGIN(INITIAL);
                        yylval->str = litbufdup(yyscanner);
                        return SCONST;
                    }
                    else
                    {
                        /*
                         * When we fail to match $...$ to dolqstart, transfer
                         * the $... part to the output, but put back the final
                         * $ for rescanning.  Consider $delim$...$junk$delim$
                         */
                        addlit(yytext, yyleng - 1, yyscanner);
                        yyless(yyleng - 1);
                    }
                }
<xdolq>{dolqinside} {
                    addlit(yytext, yyleng, yyscanner);
                }
<xdolq>{dolqfailed} {
                    addlit(yytext, yyleng, yyscanner);
                }
<xdolq>.        {
                    /* This is only needed for $ inside the quoted text */
                    addlitchar(yytext[0], yyscanner);
                }
<xdolq><<EOF>>  { yyerror("unterminated dollar-quoted string"); }
 
{xdstart}       {
                    SET_YYLLOC();
                    BEGIN(xd);
                    startlit();
                }
{xuistart}      {
                    SET_YYLLOC();
                    BEGIN(xui);
                    startlit();
                }
<xd>{xdstop}    {
                    char       *ident;
 
                    BEGIN(INITIAL);
                    if (yyextra->literallen == 0)
                        yyerror("zero-length delimited identifier");
                    ident = litbufdup(yyscanner);
                    if (yyextra->literallen >= NAMEDATALEN)
                        truncate_identifier(ident, yyextra->literallen, true);
                    yylval->str = ident;
                    return IDENT;
                }
<xui>{dquote}   {
                    BEGIN(INITIAL);
                    if (yyextra->literallen == 0)
                        yyerror("zero-length delimited identifier");
                    /* can't truncate till after we de-escape the ident */
                    yylval->str = litbufdup(yyscanner);
                    return UIDENT;
                }
<xd,xui>{xddouble}  {
                    addlitchar('"', yyscanner);
                }
<xd,xui>{xdinside}  {
                    addlit(yytext, yyleng, yyscanner);
                }
<xd,xui><<EOF>>     { yyerror("unterminated quoted identifier"); }
 
{xufailed}  {
                    char       *ident;
 
                    SET_YYLLOC();
                    /* throw back all but the initial u/U */
                    yyless(1);
                    /* and treat it as {identifier} */
                    ident = downcase_truncate_identifier(yytext, yyleng, true);
                    yylval->str = ident;
                    return IDENT;
                }
 
{typecast}      {
                    SET_YYLLOC();
                    return TYPECAST;
                }
 
{dot_dot}       {
                    SET_YYLLOC();
                    return DOT_DOT;
                }
 
{colon_equals}  {
                    SET_YYLLOC();
                    return COLON_EQUALS;
                }
 
{equals_greater} {
                    SET_YYLLOC();
                    return EQUALS_GREATER;
                }
 
{less_equals}   {
                    SET_YYLLOC();
                    return LESS_EQUALS;
                }
 
{greater_equals} {
                    SET_YYLLOC();
                    return GREATER_EQUALS;
                }
 
{less_greater}  {
                    /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
                    SET_YYLLOC();
                    return NOT_EQUALS;
                }
 
{not_equals}    {
                    /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
                    SET_YYLLOC();
                    return NOT_EQUALS;
                }
 
{self}          {
                    SET_YYLLOC();
                    return yytext[0];
                }
 
{operator}      {
                    /*
                     * Check for embedded slash-star or dash-dash; those
                     * are comment starts, so operator must stop there.
                     * Note that slash-star or dash-dash at the first
                     * character will match a prior rule, not this one.
                     */
                    int         nchars = yyleng;
                    char       *slashstar = strstr(yytext, "/*");
                    char       *dashdash = strstr(yytext, "--");
 
                    if (slashstar && dashdash)
                    {
                        /* if both appear, take the first one */
                        if (slashstar > dashdash)
                            slashstar = dashdash;
                    }
                    else if (!slashstar)
                        slashstar = dashdash;
                    if (slashstar)
                        nchars = slashstar - yytext;
 
                    /*
                     * For SQL compatibility, '+' and '-' cannot be the
                     * last char of a multi-char operator unless the operator
                     * contains chars that are not in SQL operators.
                     * The idea is to lex '=-' as two operators, but not
                     * to forbid operator names like '?-' that could not be
                     * sequences of SQL operators.
                     */
                    if (nchars > 1 &&
                        (yytext[nchars - 1] == '+' ||
                         yytext[nchars - 1] == '-'))
                    {
                        int         ic;
 
                        for (ic = nchars - 2; ic >= 0; ic--)
                        {
                            char c = yytext[ic];
                            if (c == '~' || c == '!' || c == '@' ||
                                c == '#' || c == '^' || c == '&' ||
                                c == '|' || c == '`' || c == '?' ||
                                c == '%')
                                break;
                        }
                        if (ic < 0)
                        {
                            /*
                             * didn't find a qualifying character, so remove
                             * all trailing [+-]
                             */
                            do {
                                nchars--;
                            } while (nchars > 1 &&
                                 (yytext[nchars - 1] == '+' ||
                                  yytext[nchars - 1] == '-'));
                        }
                    }
 
                    SET_YYLLOC();
 
                    if (nchars < yyleng)
                    {
                        /* Strip the unwanted chars from the token */
                        yyless(nchars);
                        /*
                         * If what we have left is only one char, and it's
                         * one of the characters matching "self", then
                         * return it as a character token the same way
                         * that the "self" rule would have.
                         */
                        if (nchars == 1 &&
                            strchr(",()[].;:+-*/%^<>=", yytext[0]))
                            return yytext[0];
                        /*
                         * Likewise, if what we have left is two chars, and
                         * those match the tokens ">=", "<=", "=>", "<>" or
                         * "!=", then we must return the appropriate token
                         * rather than the generic Op.
                         */
                        if (nchars == 2)
                        {
                            if (yytext[0] == '=' && yytext[1] == '>')
                                return EQUALS_GREATER;
                            if (yytext[0] == '>' && yytext[1] == '=')
                                return GREATER_EQUALS;
                            if (yytext[0] == '<' && yytext[1] == '=')
                                return LESS_EQUALS;
                            if (yytext[0] == '<' && yytext[1] == '>')
                                return NOT_EQUALS;
                            if (yytext[0] == '!' && yytext[1] == '=')
                                return NOT_EQUALS;
                        }
                    }
 
                    /*
                     * Complain if operator is too long.  Unlike the case
                     * for identifiers, we make this an error not a notice-
                     * and-truncate, because the odds are we are looking at
                     * a syntactic mistake anyway.
                     */
                    if (nchars >= NAMEDATALEN)
                        yyerror("operator too long");
 
                    yylval->str = pstrdup(yytext);
                    return Op;
                }
 
{param}         {
                    ErrorSaveContext escontext = {T_ErrorSaveContext};
                    int32       val;
 
                    SET_YYLLOC();
                    val = pg_strtoint32_safe(yytext + 1, (Node *) &escontext);
                    if (escontext.error_occurred)
                        yyerror("parameter number too large");
                    yylval->ival = val;
                    return PARAM;
                }
{param_junk}    {
                    SET_YYLLOC();
                    yyerror("trailing junk after parameter");
                }
 
{decinteger}    {
                    SET_YYLLOC();
                    return process_integer_literal(yytext, yylval, 10);
                }
{hexinteger}    {
                    SET_YYLLOC();
                    return process_integer_literal(yytext, yylval, 16);
                }
{octinteger}    {
                    SET_YYLLOC();
                    return process_integer_literal(yytext, yylval, 8);
                }
{bininteger}    {
                    SET_YYLLOC();
                    return process_integer_literal(yytext, yylval, 2);
                }
{hexfail}       {
                    SET_YYLLOC();
                    yyerror("invalid hexadecimal integer");
                }
{octfail}       {
                    SET_YYLLOC();
                    yyerror("invalid octal integer");
                }
{binfail}       {
                    SET_YYLLOC();
                    yyerror("invalid binary integer");
                }
{numeric}       {
                    SET_YYLLOC();
                    yylval->str = pstrdup(yytext);
                    return FCONST;
                }
{numericfail}   {
                    /* throw back the .., and treat as integer */
                    yyless(yyleng - 2);
                    SET_YYLLOC();
                    return process_integer_literal(yytext, yylval, 10);
                }
{real}          {
                    SET_YYLLOC();
                    yylval->str = pstrdup(yytext);
                    return FCONST;
                }
{realfail}      {
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }
{integer_junk}  {
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }
{numeric_junk}  {
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }
{real_junk}     {
                    SET_YYLLOC();
                    yyerror("trailing junk after numeric literal");
                }
 
 
{identifier}    {
                    int         kwnum;
                    char       *ident;
 
                    SET_YYLLOC();
 
                    /* Is it a keyword? */
                    kwnum = ScanKeywordLookup(yytext,
                                              yyextra->keywordlist);
                    if (kwnum >= 0)
                    {
                        yylval->keyword = GetScanKeyword(kwnum,
                                                         yyextra->keywordlist);
                        return yyextra->keyword_tokens[kwnum];
                    }
 
                    /*
                     * No.  Convert the identifier to lower case, and truncate
                     * if necessary.
                     */
                    ident = downcase_truncate_identifier(yytext, yyleng, true);
                    yylval->str = ident;
                    return IDENT;
                }
 
{other}         {
                    SET_YYLLOC();
                    return yytext[0];
                }
 
<<EOF>>         {
                    SET_YYLLOC();
                    yyterminate();
                }
 
%%

Variable Documentation

backslash_quote

int backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING

Definition at line 68 of file scan.l.

Referenced by scanner_init().

escape_string_warning

bool escape_string_warning = true

Definition at line 69 of file scan.l.

Referenced by scanner_init().

ScanKeywordTokens

const uint16 ScanKeywordTokens[]

Initial value:

= {
}

Definition at line 81 of file scan.l.

Referenced by fill_in_constant_lengths(), and raw_parser().

standard_conforming_strings

bool standard_conforming_strings = true

Definition at line 70 of file scan.l.

Referenced by scanner_init(), and simple_quote_literal().