PostgreSQL Source Code git master
scan.l File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include "common/string.h"
#include "gramparse.h"
#include "nodes/miscnodes.h"
#include "parser/parser.h"
#include "parser/scansup.h"
#include "port/pg_bitutils.h"
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "parser/kwlist.h"
Include dependency graph for scan.l:

Go to the source code of this file.

Macros

#define fprintf(file, fmt, msg)   fprintf_to_ereport(fmt, msg)
 
#define PG_KEYWORD(kwname, value, category, collabel)   value,
 
#define YYSTYPE   core_YYSTYPE
 
#define SET_YYLLOC()   (*(yylloc) = yytext - yyextra->scanbuf)
 
#define ADVANCE_YYLLOC(delta)   ( *(yylloc) += (delta) )
 
#define PUSH_YYLLOC()   (yyextra->save_yylloc = *(yylloc))
 
#define POP_YYLLOC()   (*(yylloc) = yyextra->save_yylloc)
 
#define startlit()   ( yyextra->literallen = 0 )
 
#define yyerror(msg)   scanner_yyerror(msg, yyscanner)
 
#define lexer_errposition()   scanner_errposition(*(yylloc), yyscanner)
 
#define yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)
 
#define yylloc   (((struct yyguts_t *) yyscanner)->yylloc_r)
 
#define yyleng   (((struct yyguts_t *) yyscanner)->yyleng_r)
 

Functions

static void fprintf_to_ereport (const char *fmt, const char *msg)
 
static void addlit (char *ytext, int yleng, core_yyscan_t yyscanner)
 
static void addlitchar (unsigned char ychar, core_yyscan_t yyscanner)
 
static char * litbufdup (core_yyscan_t yyscanner)
 
static unsigned char unescape_single_char (unsigned char c, core_yyscan_t yyscanner)
 
static int process_integer_literal (const char *token, YYSTYPE *lval, int base)
 
static void addunicode (pg_wchar c, yyscan_t yyscanner)
 
static void check_string_escape_warning (unsigned char ychar, core_yyscan_t yyscanner)
 
static void check_escape_warning (core_yyscan_t yyscanner)
 
int yylex (YYSTYPE *yylval_param, YYLTYPE *yylloc_param, yyscan_t yyscanner)
 
int scanner_errposition (int location, core_yyscan_t yyscanner)
 
static void scb_error_callback (void *arg)
 
void setup_scanner_errposition_callback (ScannerCallbackState *scbstate, core_yyscan_t yyscanner, int location)
 
void cancel_scanner_errposition_callback (ScannerCallbackState *scbstate)
 
void scanner_yyerror (const char *message, core_yyscan_t yyscanner)
 
core_yyscan_t scanner_init (const char *str, core_yy_extra_type *yyext, const ScanKeywordList *keywordlist, const uint16 *keyword_tokens)
 
void scanner_finish (core_yyscan_t yyscanner)
 
void * core_yyalloc (yy_size_t bytes, core_yyscan_t yyscanner)
 
void * core_yyrealloc (void *ptr, yy_size_t bytes, core_yyscan_t yyscanner)
 
void core_yyfree (void *ptr, core_yyscan_t yyscanner)
 

Variables

int backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING
 
bool escape_string_warning = true
 
bool standard_conforming_strings = true
 
const uint16 ScanKeywordTokens []
 

Macro Definition Documentation

◆ ADVANCE_YYLLOC

#define ADVANCE_YYLLOC (   delta)    ( *(yylloc) += (delta) )

Definition at line 104 of file scan.l.

◆ fprintf

#define fprintf (   file,
  fmt,
  msg 
)    fprintf_to_ereport(fmt, msg)

Definition at line 54 of file scan.l.

◆ lexer_errposition

#define lexer_errposition ( )    scanner_errposition(*(yylloc), yyscanner)

Definition at line 128 of file scan.l.

◆ PG_KEYWORD

#define PG_KEYWORD (   kwname,
  value,
  category,
  collabel 
)    value,

Definition at line 79 of file scan.l.

◆ POP_YYLLOC

#define POP_YYLLOC ( )    (*(yylloc) = yyextra->save_yylloc)

Definition at line 116 of file scan.l.

◆ PUSH_YYLLOC

#define PUSH_YYLLOC ( )    (yyextra->save_yylloc = *(yylloc))

Definition at line 115 of file scan.l.

◆ SET_YYLLOC

#define SET_YYLLOC ( )    (*(yylloc) = yytext - yyextra->scanbuf)

Definition at line 99 of file scan.l.

◆ startlit

#define startlit ( )    ( yyextra->literallen = 0 )

Definition at line 118 of file scan.l.

◆ yyerror

#define yyerror (   msg)    scanner_yyerror(msg, yyscanner)

Definition at line 126 of file scan.l.

◆ yyextra

#define yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)

Definition at line 1118 of file scan.l.

◆ yyleng

#define yyleng   (((struct yyguts_t *) yyscanner)->yyleng_r)

Definition at line 1124 of file scan.l.

◆ yylloc

#define yylloc   (((struct yyguts_t *) yyscanner)->yylloc_r)

Definition at line 1122 of file scan.l.

◆ YYSTYPE

#define YYSTYPE   core_YYSTYPE

Definition at line 90 of file scan.l.

Function Documentation

◆ addlit()

static void addlit ( char *  ytext,
int  yleng,
core_yyscan_t  yyscanner 
)
static

Definition at line 1311 of file scan.l.

1312{
1313 /* enlarge buffer if needed */
1314 if ((yyextra->literallen + yleng) >= yyextra->literalalloc)
1315 {
1316 yyextra->literalalloc = pg_nextpower2_32(yyextra->literallen + yleng + 1);
1317 yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
1318 yyextra->literalalloc);
1319 }
1320 /* append new data */
1321 memcpy(yyextra->literalbuf + yyextra->literallen, ytext, yleng);
1322 yyextra->literallen += yleng;
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1541
static uint32 pg_nextpower2_32(uint32 num)
Definition: pg_bitutils.h:189
#define yyextra
Definition: scan.l:1118
1323}

References pg_nextpower2_32(), repalloc(), and yyextra.

Referenced by addunicode().

◆ addlitchar()

static void addlitchar ( unsigned char  ychar,
core_yyscan_t  yyscanner 
)
static

Definition at line 1327 of file scan.l.

1328{
1329 /* enlarge buffer if needed */
1330 if ((yyextra->literallen + 1) >= yyextra->literalalloc)
1331 {
1332 yyextra->literalalloc *= 2;
1333 yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
1334 yyextra->literalalloc);
1335 }
1336 /* append new data */
1337 yyextra->literalbuf[yyextra->literallen] = ychar;
1338 yyextra->literallen += 1;
1339}

References repalloc(), and yyextra.

◆ addunicode()

static void addunicode ( pg_wchar  c,
yyscan_t  yyscanner 
)
static

Definition at line 1379 of file scan.l.

1380{
1381 ScannerCallbackState scbstate;
static char * buf
Definition: pg_test_fsync.c:72
#define MAX_UNICODE_EQUIVALENT_STRING
Definition: pg_wchar.h:329
1383
1385 yyerror("invalid Unicode escape value");
static bool is_valid_unicode_codepoint(pg_wchar c)
Definition: pg_wchar.h:519
char * c
#define yyerror(msg)
Definition: scan.l:126
1386
1387 /*
1388 * We expect that pg_unicode_to_server() will complain about any
1389 * unconvertible code point, so we don't have to set saw_non_ascii.
1390 */
1391 setup_scanner_errposition_callback(&scbstate, yyscanner, *(yylloc));
1392 pg_unicode_to_server(c, (unsigned char *) buf);
1394 addlit(buf, strlen(buf), yyscanner);
void pg_unicode_to_server(pg_wchar c, unsigned char *s)
Definition: mbutils.c:864
#define yylloc
Definition: scan.l:1122
void setup_scanner_errposition_callback(ScannerCallbackState *scbstate, core_yyscan_t yyscanner, int location)
Definition: scan.l:1186
void cancel_scanner_errposition_callback(ScannerCallbackState *scbstate)
Definition: scan.l:1203
static void addlit(char *ytext, int yleng, core_yyscan_t yyscanner)
Definition: scan.l:1311
1395}

References addlit(), buf, cancel_scanner_errposition_callback(), is_valid_unicode_codepoint(), MAX_UNICODE_EQUIVALENT_STRING, pg_unicode_to_server(), setup_scanner_errposition_callback(), yyerror, and yylloc.

◆ cancel_scanner_errposition_callback()

void cancel_scanner_errposition_callback ( ScannerCallbackState scbstate)

Definition at line 1203 of file scan.l.

1204{
1205 /* Pop the error context stack */
ErrorContextCallback * error_context_stack
Definition: elog.c:94
struct ErrorContextCallback * previous
Definition: elog.h:296
ErrorContextCallback errcallback
Definition: scanner.h:128
1207}

References ScannerCallbackState::errcallback, error_context_stack, and ErrorContextCallback::previous.

Referenced by addunicode(), and str_udeescape().

◆ check_escape_warning()

static void check_escape_warning ( core_yyscan_t  yyscanner)
static

Definition at line 1451 of file scan.l.

1452{
1453 if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
1455 (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
1456 errmsg("nonstandard use of escape in a string literal"),
1457 errhint("Use the escape string syntax for escapes, e.g., E'\\r\\n'."),
1459 yyextra->warn_on_first_escape = false; /* warn only once per string */
int errhint(const char *fmt,...)
Definition: elog.c:1317
int errcode(int sqlerrcode)
Definition: elog.c:853
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define WARNING
Definition: elog.h:36
#define ereport(elevel,...)
Definition: elog.h:149
#define lexer_errposition()
Definition: scan.l:128
1460}

References ereport, errcode(), errhint(), errmsg(), lexer_errposition, WARNING, and yyextra.

Referenced by check_string_escape_warning().

◆ check_string_escape_warning()

static void check_string_escape_warning ( unsigned char  ychar,
core_yyscan_t  yyscanner 
)
static

Definition at line 1424 of file scan.l.

1425{
1426 if (ychar == '\'')
1427 {
1428 if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
1430 (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
1431 errmsg("nonstandard use of \\' in a string literal"),
1432 errhint("Use '' to write quotes in strings, or use the escape string syntax (E'...')."),
1434 yyextra->warn_on_first_escape = false; /* warn only once per string */
1435 }
1436 else if (ychar == '\\')
1437 {
1438 if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
1440 (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
1441 errmsg("nonstandard use of \\\\ in a string literal"),
1442 errhint("Use the escape string syntax for backslashes, e.g., E'\\\\'."),
1444 yyextra->warn_on_first_escape = false; /* warn only once per string */
1445 }
1446 else
1447 check_escape_warning(yyscanner);
static void check_escape_warning(core_yyscan_t yyscanner)
Definition: scan.l:1451
1448}

References check_escape_warning(), ereport, errcode(), errhint(), errmsg(), lexer_errposition, WARNING, and yyextra.

◆ core_yyalloc()

void * core_yyalloc ( yy_size_t  bytes,
core_yyscan_t  yyscanner 
)

Definition at line 1468 of file scan.l.

1469{
1470 return palloc(bytes);
void * palloc(Size size)
Definition: mcxt.c:1317
1471}

References palloc().

◆ core_yyfree()

void core_yyfree ( void *  ptr,
core_yyscan_t  yyscanner 
)

Definition at line 1483 of file scan.l.

1484{
1485 if (ptr)
1486 pfree(ptr);
void pfree(void *pointer)
Definition: mcxt.c:1521
1487}

References pfree().

◆ core_yyrealloc()

void * core_yyrealloc ( void *  ptr,
yy_size_t  bytes,
core_yyscan_t  yyscanner 
)

Definition at line 1474 of file scan.l.

1475{
1476 if (ptr)
1477 return repalloc(ptr, bytes);
1478 else
1479 return palloc(bytes);
1480}

References palloc(), and repalloc().

◆ fprintf_to_ereport()

static void fprintf_to_ereport ( const char *  fmt,
const char *  msg 
)
static

Definition at line 57 of file scan.l.

58{
59 ereport(ERROR, (errmsg_internal("%s", msg)));
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1157
#define ERROR
Definition: elog.h:39
60}

References ereport, errmsg_internal(), and ERROR.

◆ litbufdup()

static char * litbufdup ( core_yyscan_t  yyscanner)
static

Definition at line 1346 of file scan.l.

1347{
1348 int llen = yyextra->literallen;
1349 char *new;
1350
1351 new = palloc(llen + 1);
1352 memcpy(new, yyextra->literalbuf, llen);
1353 new[llen] = '\0';
1354 return new;
1355}

References palloc(), and yyextra.

◆ process_integer_literal()

static int process_integer_literal ( const char *  token,
YYSTYPE lval,
int  base 
)
static

Definition at line 1362 of file scan.l.

1363{
1364 ErrorSaveContext escontext = {T_ErrorSaveContext};
1365 int32 val;
int32_t int32
Definition: c.h:484
long val
Definition: informix.c:689
1366
1367 val = pg_strtoint32_safe(token, (Node *) &escontext);
1368 if (escontext.error_occurred)
1369 {
1370 /* integer too large (or contains decimal pt), treat it as a float */
1371 lval->str = pstrdup(token);
1372 return FCONST;
1373 }
1374 lval->ival = val;
1375 return ICONST;
#define token
Definition: indent_globs.h:126
char * pstrdup(const char *in)
Definition: mcxt.c:1696
int32 pg_strtoint32_safe(const char *s, Node *escontext)
Definition: numutils.c:389
bool error_occurred
Definition: miscnodes.h:47
Definition: nodes.h:129
1376}

References ErrorSaveContext::error_occurred, pg_strtoint32_safe(), pstrdup(), token, and val.

◆ scanner_errposition()

int scanner_errposition ( int  location,
core_yyscan_t  yyscanner 
)

Definition at line 1140 of file scan.l.

1141{
1142 int pos;
1143
1144 if (location < 0)
1145 return 0; /* no-op if location is unknown */
1146
1147 /* Convert byte offset to character number */
1148 pos = pg_mbstrlen_with_len(yyextra->scanbuf, location) + 1;
1149 /* And pass it to the ereport mechanism */
1150 return errposition(pos);
int errposition(int cursorpos)
Definition: elog.c:1446
int pg_mbstrlen_with_len(const char *mbstr, int limit)
Definition: mbutils.c:1057
1151}

References errposition(), pg_mbstrlen_with_len(), and yyextra.

Referenced by scb_error_callback(), and str_udeescape().

◆ scanner_finish()

void scanner_finish ( core_yyscan_t  yyscanner)

Definition at line 1291 of file scan.l.

1292{
1293 /*
1294 * We don't bother to call yylex_destroy(), because all it would do is
1295 * pfree a small amount of control storage. It's cheaper to leak the
1296 * storage until the parsing context is destroyed. The amount of space
1297 * involved is usually negligible compared to the output parse tree
1298 * anyway.
1299 *
1300 * We do bother to pfree the scanbuf and literal buffer, but only if they
1301 * represent a nontrivial amount of space. The 8K cutoff is arbitrary.
1302 */
1303 if (yyextra->scanbuflen >= 8192)
1304 pfree(yyextra->scanbuf);
1305 if (yyextra->literalalloc >= 8192)
1306 pfree(yyextra->literalbuf);
1307}

References pfree(), and yyextra.

Referenced by fill_in_constant_lengths(), plpgsql_scanner_finish(), and raw_parser().

◆ scanner_init()

core_yyscan_t scanner_init ( const char *  str,
core_yy_extra_type yyext,
const ScanKeywordList keywordlist,
const uint16 keyword_tokens 
)

Definition at line 1249 of file scan.l.

1253{
1254 Size slen = strlen(str);
1255 yyscan_t scanner;
size_t Size
Definition: c.h:562
void * yyscan_t
Definition: cubedata.h:67
const char * str
1256
1257 if (yylex_init(&scanner) != 0)
1258 elog(ERROR, "yylex_init() failed: %m");
#define elog(elevel,...)
Definition: elog.h:225
1259
1260 core_yyset_extra(yyext, scanner);
1261
1262 yyext->keywordlist = keywordlist;
1263 yyext->keyword_tokens = keyword_tokens;
const ScanKeywordList * keywordlist
Definition: scanner.h:78
const uint16 * keyword_tokens
Definition: scanner.h:79
1264
bool escape_string_warning
Definition: scan.l:69
int backslash_quote
Definition: scan.l:68
bool standard_conforming_strings
Definition: scan.l:70
bool escape_string_warning
Definition: scanner.h:88
bool standard_conforming_strings
Definition: scanner.h:89
1268
1269 /*
1270 * Make a scan buffer with special termination needed by flex.
1271 */
1272 yyext->scanbuf = (char *) palloc(slen + 2);
1273 yyext->scanbuflen = slen;
1274 memcpy(yyext->scanbuf, str, slen);
1275 yyext->scanbuf[slen] = yyext->scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
1276 yy_scan_buffer(yyext->scanbuf, slen + 2, scanner);
char * scanbuf
Definition: scanner.h:72
1277
1278 /* initialize literal buffer to a reasonable but expansible size */
1279 yyext->literalalloc = 1024;
1280 yyext->literalbuf = (char *) palloc(yyext->literalalloc);
1281 yyext->literallen = 0;
char * literalbuf
Definition: scanner.h:98
1282
1283 return scanner;
1284}

References backslash_quote, core_yy_extra_type::backslash_quote, elog, ERROR, escape_string_warning, core_yy_extra_type::escape_string_warning, core_yy_extra_type::keyword_tokens, core_yy_extra_type::keywordlist, core_yy_extra_type::literalalloc, core_yy_extra_type::literalbuf, core_yy_extra_type::literallen, palloc(), core_yy_extra_type::scanbuf, core_yy_extra_type::scanbuflen, standard_conforming_strings, core_yy_extra_type::standard_conforming_strings, and str.

Referenced by fill_in_constant_lengths(), plpgsql_scanner_init(), and raw_parser().

◆ scanner_yyerror()

void scanner_yyerror ( const char *  message,
core_yyscan_t  yyscanner 
)

Definition at line 1222 of file scan.l.

1223{
1224 const char *loc = yyextra->scanbuf + *yylloc;
1225
1226 if (*loc == YY_END_OF_BUFFER_CHAR)
1227 {
1228 ereport(ERROR,
1229 (errcode(ERRCODE_SYNTAX_ERROR),
1230 /* translator: %s is typically the translation of "syntax error" */
1231 errmsg("%s at end of input", _(message)),
1233 }
1234 else
1235 {
1236 ereport(ERROR,
1237 (errcode(ERRCODE_SYNTAX_ERROR),
1238 /* translator: first %s is typically the translation of "syntax error" */
1239 errmsg("%s at or near \"%s\"", _(message), loc),
1241 }
#define _(x)
Definition: elog.c:90
1242}

References _, ereport, errcode(), errmsg(), ERROR, lexer_errposition, yyextra, and yylloc.

Referenced by base_yylex().

◆ scb_error_callback()

static void scb_error_callback ( void *  arg)
static

Definition at line 1161 of file scan.l.

1162{
void * arg
1164
1165 if (geterrcode() != ERRCODE_QUERY_CANCELED)
1166 (void) scanner_errposition(scbstate->location, scbstate->yyscanner);
int geterrcode(void)
Definition: elog.c:1561
int scanner_errposition(int location, core_yyscan_t yyscanner)
Definition: scan.l:1140
core_yyscan_t yyscanner
Definition: scanner.h:126
1167}

References arg, geterrcode(), ScannerCallbackState::location, scanner_errposition(), and ScannerCallbackState::yyscanner.

Referenced by setup_scanner_errposition_callback().

◆ setup_scanner_errposition_callback()

void setup_scanner_errposition_callback ( ScannerCallbackState scbstate,
core_yyscan_t  yyscanner,
int  location 
)

Definition at line 1186 of file scan.l.

1189{
1190 /* Setup error traceback support for ereport() */
1191 scbstate->yyscanner = yyscanner;
1192 scbstate->location = location;
1194 scbstate->errcallback.arg = scbstate;
1196 error_context_stack = &scbstate->errcallback;
static void scb_error_callback(void *arg)
Definition: scan.l:1161
void(* callback)(void *arg)
Definition: elog.h:297
1197}

References ErrorContextCallback::arg, ErrorContextCallback::callback, ScannerCallbackState::errcallback, error_context_stack, ScannerCallbackState::location, ErrorContextCallback::previous, scb_error_callback(), and ScannerCallbackState::yyscanner.

Referenced by addunicode(), and str_udeescape().

◆ unescape_single_char()

static unsigned char unescape_single_char ( unsigned char  c,
core_yyscan_t  yyscanner 
)
static

Definition at line 1398 of file scan.l.

1399{
1400 switch (c)
1401 {
1402 case 'b':
1403 return '\b';
1404 case 'f':
1405 return '\f';
1406 case 'n':
1407 return '\n';
1408 case 'r':
1409 return '\r';
1410 case 't':
1411 return '\t';
1412 case 'v':
1413 return '\v';
1414 default:
1415 /* check for backslash followed by non-7-bit-ASCII */
1416 if (c == '\0' || IS_HIGHBIT_SET(c))
1417 yyextra->saw_non_ascii = true;
#define IS_HIGHBIT_SET(ch)
Definition: c.h:1112
1418
1419 return c;
1420 }
1421}

References IS_HIGHBIT_SET, and yyextra.

◆ yylex()

int yylex ( YYSTYPE yylval_param,
YYLTYPE yylloc_param,
yyscan_t  yyscanner 
)

Definition at line 441 of file scan.l.

443{whitespace} {
444 /* ignore */
445 }
446
447{xcstart} {
448 /* Set location in case of syntax error in comment */
449 SET_YYLLOC();
450 yyextra->xcdepth = 0;
451 BEGIN(xc);
452 /* Put back any characters past slash-star; see above */
453 yyless(2);
454 }
#define SET_YYLLOC()
Definition: scan.l:99
455
456<xc>{
457{xcstart} {
458 (yyextra->xcdepth)++;
459 /* Put back any characters past slash-star; see above */
460 yyless(2);
461 }
462
463{xcstop} {
464 if (yyextra->xcdepth <= 0)
465 BEGIN(INITIAL);
466 else
467 (yyextra->xcdepth)--;
468 }
469
470{xcinside} {
471 /* ignore */
472 }
473
474{op_chars} {
475 /* ignore */
476 }
477
478\*+ {
479 /* ignore */
480 }
481
482<<EOF>> {
483 yyerror("unterminated /* comment");
484 }
485} /* <xc> */
486
487{xbstart} {
488 /* Binary bit type.
489 * At some point we should simply pass the string
490 * forward to the parser and label it there.
491 * In the meantime, place a leading "b" on the string
492 * to mark it for the input routine as a binary string.
493 */
494 SET_YYLLOC();
495 BEGIN(xb);
496 startlit();
497 addlitchar('b', yyscanner);
498 }
static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner)
Definition: scan.l:1327
#define startlit()
Definition: scan.l:118
499<xh>{xhinside} |
500<xb>{xbinside} {
501 addlit(yytext, yyleng, yyscanner);
502 }
#define yyleng
Definition: scan.l:1124
503<xb><<EOF>> { yyerror("unterminated bit string literal"); }
504
505{xhstart} {
506 /* Hexadecimal bit type.
507 * At some point we should simply pass the string
508 * forward to the parser and label it there.
509 * In the meantime, place a leading "x" on the string
510 * to mark it for the input routine as a hex string.
511 */
512 SET_YYLLOC();
513 BEGIN(xh);
514 startlit();
515 addlitchar('x', yyscanner);
516 }
517<xh><<EOF>> { yyerror("unterminated hexadecimal string literal"); }
518
519{xnstart} {
520 /* National character.
521 * We will pass this along as a normal character string,
522 * but preceded with an internally-generated "NCHAR".
523 */
524 int kwnum;
525
526 SET_YYLLOC();
527 yyless(1); /* eat only 'n' this time */
528
529 kwnum = ScanKeywordLookup("nchar",
530 yyextra->keywordlist);
531 if (kwnum >= 0)
532 {
533 yylval->keyword = GetScanKeyword(kwnum,
534 yyextra->keywordlist);
535 return yyextra->keyword_tokens[kwnum];
536 }
537 else
538 {
539 /* If NCHAR isn't a keyword, just return "n" */
540 yylval->str = pstrdup("n");
541 return IDENT;
542 }
543 }
int ScanKeywordLookup(const char *str, const ScanKeywordList *keywords)
Definition: kwlookup.c:38
static const char * GetScanKeyword(int n, const ScanKeywordList *keywords)
Definition: kwlookup.h:39
544
545{xqstart} {
546 yyextra->warn_on_first_escape = true;
547 yyextra->saw_non_ascii = false;
548 SET_YYLLOC();
549 if (yyextra->standard_conforming_strings)
550 BEGIN(xq);
551 else
552 BEGIN(xe);
553 startlit();
554 }
555{xestart} {
556 yyextra->warn_on_first_escape = false;
557 yyextra->saw_non_ascii = false;
558 SET_YYLLOC();
559 BEGIN(xe);
560 startlit();
561 }
562{xusstart} {
563 SET_YYLLOC();
564 if (!yyextra->standard_conforming_strings)
566 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
567 errmsg("unsafe use of string constant with Unicode escapes"),
568 errdetail("String constants with Unicode escapes cannot be used when \"standard_conforming_strings\" is off."),
570 BEGIN(xus);
571 startlit();
572 }
int errdetail(const char *fmt,...)
Definition: elog.c:1203
573
574<xb,xh,xq,xe,xus>{quote} {
575 /*
576 * When we are scanning a quoted string and see an end
577 * quote, we must look ahead for a possible continuation.
578 * If we don't see one, we know the end quote was in fact
579 * the end of the string. To reduce the lexer table size,
580 * we use a single "xqs" state to do the lookahead for all
581 * types of strings.
582 */
583 yyextra->state_before_str_stop = YYSTATE;
584 BEGIN(xqs);
585 }
586<xqs>{quotecontinue} {
587 /*
588 * Found a quote continuation, so return to the in-quote
589 * state and continue scanning the literal. Nothing is
590 * added to the literal's contents.
591 */
592 BEGIN(yyextra->state_before_str_stop);
593 }
594<xqs>{quotecontinuefail} |
595<xqs>{other} |
596<xqs><<EOF>> {
597 /*
598 * Failed to see a quote continuation. Throw back
599 * everything after the end quote, and handle the string
600 * according to the state we were in previously.
601 */
602 yyless(0);
603 BEGIN(INITIAL);
604
605 switch (yyextra->state_before_str_stop)
606 {
607 case xb:
608 yylval->str = litbufdup(yyscanner);
609 return BCONST;
610 case xh:
611 yylval->str = litbufdup(yyscanner);
612 return XCONST;
613 case xq:
614 case xe:
615 /*
616 * Check that the data remains valid, if it might
617 * have been made invalid by unescaping any chars.
618 */
619 if (yyextra->saw_non_ascii)
620 pg_verifymbstr(yyextra->literalbuf,
621 yyextra->literallen,
622 false);
623 yylval->str = litbufdup(yyscanner);
624 return SCONST;
625 case xus:
626 yylval->str = litbufdup(yyscanner);
627 return USCONST;
628 default:
629 yyerror("unhandled previous state in xqs");
630 }
631 }
bool pg_verifymbstr(const char *mbstr, int len, bool noError)
Definition: mbutils.c:1556
static char * litbufdup(core_yyscan_t yyscanner)
Definition: scan.l:1346
632
633<xq,xe,xus>{xqdouble} {
634 addlitchar('\'', yyscanner);
635 }
636<xq,xus>{xqinside} {
637 addlit(yytext, yyleng, yyscanner);
638 }
639<xe>{xeinside} {
640 addlit(yytext, yyleng, yyscanner);
641 }
642<xe>{xeunicode} {
643 pg_wchar c = strtoul(yytext + 2, NULL, 16);
unsigned int pg_wchar
Definition: mbprint.c:31
644
645 /*
646 * For consistency with other productions, issue any
647 * escape warning with cursor pointing to start of string.
648 * We might want to change that, someday.
649 */
650 check_escape_warning(yyscanner);
651
652 /* Remember start of overall string token ... */
653 PUSH_YYLLOC();
654 /* ... and set the error cursor to point at this esc seq */
655 SET_YYLLOC();
#define PUSH_YYLLOC()
Definition: scan.l:115
656
658 {
659 yyextra->utf16_first_part = c;
660 BEGIN(xeu);
661 }
663 yyerror("invalid Unicode surrogate pair");
664 else
665 addunicode(c, yyscanner);
static bool is_utf16_surrogate_first(pg_wchar c)
Definition: pg_wchar.h:525
static bool is_utf16_surrogate_second(pg_wchar c)
Definition: pg_wchar.h:531
static void addunicode(pg_wchar c, yyscan_t yyscanner)
Definition: scan.l:1379
666
667 /* Restore yylloc to be start of string token */
668 POP_YYLLOC();
669 }
#define POP_YYLLOC()
Definition: scan.l:116
670<xeu>{xeunicode} {
671 pg_wchar c = strtoul(yytext + 2, NULL, 16);
672
673 /* Remember start of overall string token ... */
674 PUSH_YYLLOC();
675 /* ... and set the error cursor to point at this esc seq */
676 SET_YYLLOC();
677
679 yyerror("invalid Unicode surrogate pair");
680
681 c = surrogate_pair_to_codepoint(yyextra->utf16_first_part, c);
static pg_wchar surrogate_pair_to_codepoint(pg_wchar first, pg_wchar second)
Definition: pg_wchar.h:537
682
683 addunicode(c, yyscanner);
684
685 /* Restore yylloc to be start of string token */
686 POP_YYLLOC();
687
688 BEGIN(xe);
689 }
690<xeu>. |
691<xeu>\n |
692<xeu><<EOF>> {
693 /* Set the error cursor to point at missing esc seq */
694 SET_YYLLOC();
695 yyerror("invalid Unicode surrogate pair");
696 }
697<xe,xeu>{xeunicodefail} {
698 /* Set the error cursor to point at malformed esc seq */
699 SET_YYLLOC();
701 (errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE),
702 errmsg("invalid Unicode escape"),
703 errhint("Unicode escapes must be \\uXXXX or \\UXXXXXXXX."),
705 }
706<xe>{xeescape} {
707 if (yytext[1] == '\'')
708 {
709 if (yyextra->backslash_quote == BACKSLASH_QUOTE_OFF ||
710 (yyextra->backslash_quote == BACKSLASH_QUOTE_SAFE_ENCODING &&
713 (errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
714 errmsg("unsafe use of \\' in a string literal"),
715 errhint("Use '' to write quotes in strings. \\' is insecure in client-only encodings."),
717 }
718 check_string_escape_warning(yytext[1], yyscanner);
719 addlitchar(unescape_single_char(yytext[1], yyscanner),
720 yyscanner);
721 }
int pg_get_client_encoding(void)
Definition: mbutils.c:336
@ BACKSLASH_QUOTE_SAFE_ENCODING
Definition: parser.h:52
@ BACKSLASH_QUOTE_OFF
Definition: parser.h:50
#define PG_ENCODING_IS_CLIENT_ONLY(_enc)
Definition: pg_wchar.h:284
static unsigned char unescape_single_char(unsigned char c, core_yyscan_t yyscanner)
Definition: scan.l:1398
static void check_string_escape_warning(unsigned char ychar, core_yyscan_t yyscanner)
Definition: scan.l:1424
722<xe>{xeoctesc} {
723 unsigned char c = strtoul(yytext + 1, NULL, 8);
724
725 check_escape_warning(yyscanner);
726 addlitchar(c, yyscanner);
727 if (c == '\0' || IS_HIGHBIT_SET(c))
728 yyextra->saw_non_ascii = true;
729 }
730<xe>{xehexesc} {
731 unsigned char c = strtoul(yytext + 2, NULL, 16);
732
733 check_escape_warning(yyscanner);
734 addlitchar(c, yyscanner);
735 if (c == '\0' || IS_HIGHBIT_SET(c))
736 yyextra->saw_non_ascii = true;
737 }
738<xe>. {
739 /* This is only needed for \ just before EOF */
740 addlitchar(yytext[0], yyscanner);
741 }
742<xq,xe,xus><<EOF>> { yyerror("unterminated quoted string"); }
743
744{dolqdelim} {
745 SET_YYLLOC();
746 yyextra->dolqstart = pstrdup(yytext);
747 BEGIN(xdolq);
748 startlit();
749 }
750{dolqfailed} {
751 SET_YYLLOC();
752 /* throw back all but the initial "$" */
753 yyless(1);
754 /* and treat it as {other} */
755 return yytext[0];
756 }
757<xdolq>{dolqdelim} {
758 if (strcmp(yytext, yyextra->dolqstart) == 0)
759 {
760 pfree(yyextra->dolqstart);
761 yyextra->dolqstart = NULL;
762 BEGIN(INITIAL);
763 yylval->str = litbufdup(yyscanner);
764 return SCONST;
765 }
766 else
767 {
768 /*
769 * When we fail to match $...$ to dolqstart, transfer
770 * the $... part to the output, but put back the final
771 * $ for rescanning. Consider $delim$...$junk$delim$
772 */
773 addlit(yytext, yyleng - 1, yyscanner);
774 yyless(yyleng - 1);
775 }
776 }
777<xdolq>{dolqinside} {
778 addlit(yytext, yyleng, yyscanner);
779 }
780<xdolq>{dolqfailed} {
781 addlit(yytext, yyleng, yyscanner);
782 }
783<xdolq>. {
784 /* This is only needed for $ inside the quoted text */
785 addlitchar(yytext[0], yyscanner);
786 }
787<xdolq><<EOF>> { yyerror("unterminated dollar-quoted string"); }
788
789{xdstart} {
790 SET_YYLLOC();
791 BEGIN(xd);
792 startlit();
793 }
794{xuistart} {
795 SET_YYLLOC();
796 BEGIN(xui);
797 startlit();
798 }
799<xd>{xdstop} {
800 char *ident;
#define ident
Definition: indent_codes.h:47
801
802 BEGIN(INITIAL);
803 if (yyextra->literallen == 0)
804 yyerror("zero-length delimited identifier");
805 ident = litbufdup(yyscanner);
806 if (yyextra->literallen >= NAMEDATALEN)
807 truncate_identifier(ident, yyextra->literallen, true);
808 yylval->str = ident;
809 return IDENT;
810 }
#define NAMEDATALEN
void truncate_identifier(char *ident, int len, bool warn)
Definition: scansup.c:93
811<xui>{dquote} {
812 BEGIN(INITIAL);
813 if (yyextra->literallen == 0)
814 yyerror("zero-length delimited identifier");
815 /* can't truncate till after we de-escape the ident */
816 yylval->str = litbufdup(yyscanner);
817 return UIDENT;
818 }
819<xd,xui>{xddouble} {
820 addlitchar('"', yyscanner);
821 }
822<xd,xui>{xdinside} {
823 addlit(yytext, yyleng, yyscanner);
824 }
825<xd,xui><<EOF>> { yyerror("unterminated quoted identifier"); }
826
827{xufailed} {
828 char *ident;
829
830 SET_YYLLOC();
831 /* throw back all but the initial u/U */
832 yyless(1);
833 /* and treat it as {identifier} */
835 yylval->str = ident;
836 return IDENT;
837 }
char * downcase_truncate_identifier(const char *ident, int len, bool warn)
Definition: scansup.c:37
838
839{typecast} {
840 SET_YYLLOC();
841 return TYPECAST;
842 }
843
844{dot_dot} {
845 SET_YYLLOC();
846 return DOT_DOT;
847 }
848
849{colon_equals} {
850 SET_YYLLOC();
851 return COLON_EQUALS;
852 }
853
854{equals_greater} {
855 SET_YYLLOC();
856 return EQUALS_GREATER;
857 }
858
859{less_equals} {
860 SET_YYLLOC();
861 return LESS_EQUALS;
862 }
863
864{greater_equals} {
865 SET_YYLLOC();
866 return GREATER_EQUALS;
867 }
868
869{less_greater} {
870 /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
871 SET_YYLLOC();
872 return NOT_EQUALS;
873 }
874
875{not_equals} {
876 /* We accept both "<>" and "!=" as meaning NOT_EQUALS */
877 SET_YYLLOC();
878 return NOT_EQUALS;
879 }
880
881{self} {
882 SET_YYLLOC();
883 return yytext[0];
884 }
885
886{operator} {
887 /*
888 * Check for embedded slash-star or dash-dash; those
889 * are comment starts, so operator must stop there.
890 * Note that slash-star or dash-dash at the first
891 * character will match a prior rule, not this one.
892 */
893 int nchars = yyleng;
894 char *slashstar = strstr(yytext, "/*");
895 char *dashdash = strstr(yytext, "--");
896
897 if (slashstar && dashdash)
898 {
899 /* if both appear, take the first one */
900 if (slashstar > dashdash)
901 slashstar = dashdash;
902 }
903 else if (!slashstar)
904 slashstar = dashdash;
905 if (slashstar)
906 nchars = slashstar - yytext;
907
908 /*
909 * For SQL compatibility, '+' and '-' cannot be the
910 * last char of a multi-char operator unless the operator
911 * contains chars that are not in SQL operators.
912 * The idea is to lex '=-' as two operators, but not
913 * to forbid operator names like '?-' that could not be
914 * sequences of SQL operators.
915 */
916 if (nchars > 1 &&
917 (yytext[nchars - 1] == '+' ||
918 yytext[nchars - 1] == '-'))
919 {
920 int ic;
921
922 for (ic = nchars - 2; ic >= 0; ic--)
923 {
924 char c = yytext[ic];
925 if (c == '~' || c == '!' || c == '@' ||
926 c == '#' || c == '^' || c == '&' ||
927 c == '|' || c == '`' || c == '?' ||
928 c == '%')
929 break;
930 }
931 if (ic < 0)
932 {
933 /*
934 * didn't find a qualifying character, so remove
935 * all trailing [+-]
936 */
937 do {
938 nchars--;
939 } while (nchars > 1 &&
940 (yytext[nchars - 1] == '+' ||
941 yytext[nchars - 1] == '-'));
942 }
943 }
944
945 SET_YYLLOC();
946
947 if (nchars < yyleng)
948 {
949 /* Strip the unwanted chars from the token */
950 yyless(nchars);
951 /*
952 * If what we have left is only one char, and it's
953 * one of the characters matching "self", then
954 * return it as a character token the same way
955 * that the "self" rule would have.
956 */
957 if (nchars == 1 &&
958 strchr(",()[].;:+-*/%^<>=", yytext[0]))
959 return yytext[0];
960 /*
961 * Likewise, if what we have left is two chars, and
962 * those match the tokens ">=", "<=", "=>", "<>" or
963 * "!=", then we must return the appropriate token
964 * rather than the generic Op.
965 */
966 if (nchars == 2)
967 {
968 if (yytext[0] == '=' && yytext[1] == '>')
969 return EQUALS_GREATER;
970 if (yytext[0] == '>' && yytext[1] == '=')
971 return GREATER_EQUALS;
972 if (yytext[0] == '<' && yytext[1] == '=')
973 return LESS_EQUALS;
974 if (yytext[0] == '<' && yytext[1] == '>')
975 return NOT_EQUALS;
976 if (yytext[0] == '!' && yytext[1] == '=')
977 return NOT_EQUALS;
978 }
979 }
980
981 /*
982 * Complain if operator is too long. Unlike the case
983 * for identifiers, we make this an error not a notice-
984 * and-truncate, because the odds are we are looking at
985 * a syntactic mistake anyway.
986 */
987 if (nchars >= NAMEDATALEN)
988 yyerror("operator too long");
989
990 yylval->str = pstrdup(yytext);
991 return Op;
992 }
993
994{param} {
995 ErrorSaveContext escontext = {T_ErrorSaveContext};
996 int32 val;
997
998 SET_YYLLOC();
999 val = pg_strtoint32_safe(yytext + 1, (Node *) &escontext);
1000 if (escontext.error_occurred)
1001 yyerror("parameter number too large");
1002 yylval->ival = val;
1003 return PARAM;
1004 }
1005{param_junk} {
1006 SET_YYLLOC();
1007 yyerror("trailing junk after parameter");
1008 }
1009
1010{decinteger} {
1011 SET_YYLLOC();
1012 return process_integer_literal(yytext, yylval, 10);
1013 }
static int process_integer_literal(const char *token, YYSTYPE *lval, int base)
Definition: scan.l:1362
1014{hexinteger} {
1015 SET_YYLLOC();
1016 return process_integer_literal(yytext, yylval, 16);
1017 }
1018{octinteger} {
1019 SET_YYLLOC();
1020 return process_integer_literal(yytext, yylval, 8);
1021 }
1022{bininteger} {
1023 SET_YYLLOC();
1024 return process_integer_literal(yytext, yylval, 2);
1025 }
1026{hexfail} {
1027 SET_YYLLOC();
1028 yyerror("invalid hexadecimal integer");
1029 }
1030{octfail} {
1031 SET_YYLLOC();
1032 yyerror("invalid octal integer");
1033 }
1034{binfail} {
1035 SET_YYLLOC();
1036 yyerror("invalid binary integer");
1037 }
1038{numeric} {
1039 SET_YYLLOC();
1040 yylval->str = pstrdup(yytext);
1041 return FCONST;
1042 }
1043{numericfail} {
1044 /* throw back the .., and treat as integer */
1045 yyless(yyleng - 2);
1046 SET_YYLLOC();
1047 return process_integer_literal(yytext, yylval, 10);
1048 }
1049{real} {
1050 SET_YYLLOC();
1051 yylval->str = pstrdup(yytext);
1052 return FCONST;
1053 }
1054{realfail} {
1055 SET_YYLLOC();
1056 yyerror("trailing junk after numeric literal");
1057 }
1058{integer_junk} {
1059 SET_YYLLOC();
1060 yyerror("trailing junk after numeric literal");
1061 }
1062{numeric_junk} {
1063 SET_YYLLOC();
1064 yyerror("trailing junk after numeric literal");
1065 }
1066{real_junk} {
1067 SET_YYLLOC();
1068 yyerror("trailing junk after numeric literal");
1069 }
1070
1071
1072{identifier} {
1073 int kwnum;
1074 char *ident;
1075
1076 SET_YYLLOC();
1077
1078 /* Is it a keyword? */
1079 kwnum = ScanKeywordLookup(yytext,
1080 yyextra->keywordlist);
1081 if (kwnum >= 0)
1082 {
1083 yylval->keyword = GetScanKeyword(kwnum,
1084 yyextra->keywordlist);
1085 return yyextra->keyword_tokens[kwnum];
1086 }
1087
1088 /*
1089 * No. Convert the identifier to lower case, and truncate
1090 * if necessary.
1091 */
1092 ident = downcase_truncate_identifier(yytext, yyleng, true);
1093 yylval->str = ident;
1094 return IDENT;
1095 }
1096
1097{other} {
1098 SET_YYLLOC();
1099 return yytext[0];
1100 }
1101
1102<<EOF>> {
1103 SET_YYLLOC();
1104 yyterminate();
1105 }
1106
1107%%

Variable Documentation

◆ backslash_quote

int backslash_quote = BACKSLASH_QUOTE_SAFE_ENCODING

Definition at line 68 of file scan.l.

Referenced by scanner_init().

◆ escape_string_warning

bool escape_string_warning = true

Definition at line 69 of file scan.l.

Referenced by scanner_init().

◆ ScanKeywordTokens

const uint16 ScanKeywordTokens[]
Initial value:
= {
}

Definition at line 81 of file scan.l.

Referenced by fill_in_constant_lengths(), and raw_parser().

◆ standard_conforming_strings

bool standard_conforming_strings = true

Definition at line 70 of file scan.l.

Referenced by scanner_init(), and simple_quote_literal().