PostgreSQL Source Code git master
pl_scanner.c File Reference
#include "postgres.h"
#include "mb/pg_wchar.h"
#include "parser/scanner.h"
#include "plpgsql.h"
#include "pl_gram.h"
#include "pl_reserved_kwlist_d.h"
#include "pl_unreserved_kwlist_d.h"
#include "pl_reserved_kwlist.h"
#include "pl_unreserved_kwlist.h"
Include dependency graph for pl_scanner.c:

Go to the source code of this file.

Data Structures

struct  TokenAuxData
 
struct  plpgsql_yy_extra_type
 
struct  yyguts_t
 

Macros

#define PG_KEYWORD(kwname, value)   value,
 
#define AT_STMT_START(prev_token)
 
#define MAX_PUSHBACKS   4
 
#define yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)
 

Functions

static int internal_yylex (TokenAuxData *auxdata, yyscan_t yyscanner)
 
static void push_back_token (int token, TokenAuxData *auxdata, yyscan_t yyscanner)
 
static void location_lineno_init (yyscan_t yyscanner)
 
int plpgsql_yylex (YYSTYPE *yylvalp, YYLTYPE *yyllocp, yyscan_t yyscanner)
 
int plpgsql_token_length (yyscan_t yyscanner)
 
void plpgsql_push_back_token (int token, YYSTYPE *yylvalp, YYLTYPE *yyllocp, yyscan_t yyscanner)
 
bool plpgsql_token_is_unreserved_keyword (int token)
 
void plpgsql_append_source_text (StringInfo buf, int startlocation, int endlocation, yyscan_t yyscanner)
 
int plpgsql_peek (yyscan_t yyscanner)
 
void plpgsql_peek2 (int *tok1_p, int *tok2_p, int *tok1_loc, int *tok2_loc, yyscan_t yyscanner)
 
int plpgsql_scanner_errposition (int location, yyscan_t yyscanner)
 
void plpgsql_yyerror (YYLTYPE *yyllocp, PLpgSQL_stmt_block **plpgsql_parse_result_p, yyscan_t yyscanner, const char *message)
 
int plpgsql_location_to_lineno (int location, yyscan_t yyscanner)
 
int plpgsql_latest_lineno (yyscan_t yyscanner)
 
yyscan_t plpgsql_scanner_init (const char *str)
 
void plpgsql_scanner_finish (yyscan_t yyscanner)
 

Variables

IdentifierLookup plpgsql_IdentifierLookup = IDENTIFIER_LOOKUP_NORMAL
 
static const uint16 ReservedPLKeywordTokens []
 
static const uint16 UnreservedPLKeywordTokens []
 

Macro Definition Documentation

◆ AT_STMT_START

#define AT_STMT_START (   prev_token)
Value:
((prev_token) == ';' || \
(prev_token) == K_BEGIN || \
(prev_token) == K_THEN || \
(prev_token) == K_ELSE || \
(prev_token) == K_LOOP)

Definition at line 82 of file pl_scanner.c.

◆ MAX_PUSHBACKS

#define MAX_PUSHBACKS   4

Definition at line 98 of file pl_scanner.c.

◆ PG_KEYWORD

#define PG_KEYWORD (   kwname,
  value 
)    value,

Definition at line 64 of file pl_scanner.c.

◆ yyextra

#define yyextra   (((struct yyguts_t *) yyscanner)->yyextra_r)

Definition at line 147 of file pl_scanner.c.

Function Documentation

◆ internal_yylex()

static int internal_yylex ( TokenAuxData auxdata,
yyscan_t  yyscanner 
)
static

Definition at line 339 of file pl_scanner.c.

340{
341 int token;
342 const char *yytext;
343
344 if (yyextra->num_pushbacks > 0)
345 {
346 yyextra->num_pushbacks--;
347 token = yyextra->pushback_token[yyextra->num_pushbacks];
348 *auxdata = yyextra->pushback_auxdata[yyextra->num_pushbacks];
349 }
350 else
351 {
352 token = core_yylex(&auxdata->lval.core_yystype,
353 &auxdata->lloc,
354 yyscanner);
355
356 /* remember the length of yytext before it gets changed */
357 yytext = yyextra->core_yy_extra.scanbuf + auxdata->lloc;
358 auxdata->leng = strlen(yytext);
359
360 /* Check for << >> and #, which the core considers operators */
361 if (token == Op)
362 {
363 if (strcmp(auxdata->lval.str, "<<") == 0)
364 token = LESS_LESS;
365 else if (strcmp(auxdata->lval.str, ">>") == 0)
366 token = GREATER_GREATER;
367 else if (strcmp(auxdata->lval.str, "#") == 0)
368 token = '#';
369 }
370
371 /* The core returns PARAM as ival, but we treat it like IDENT */
372 else if (token == PARAM)
373 {
374 auxdata->lval.str = pstrdup(yytext);
375 }
376 }
377
378 return token;
379}
#define token
Definition: indent_globs.h:126
char * pstrdup(const char *in)
Definition: mcxt.c:1696
#define yyextra
Definition: pl_scanner.c:147
int core_yylex(core_YYSTYPE *yylval_param, YYLTYPE *yylloc_param, core_yyscan_t yyscanner)
YYLTYPE lloc
Definition: pl_scanner.c:94
YYSTYPE lval
Definition: pl_scanner.c:93

References core_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, pstrdup(), token, and yyextra.

Referenced by plpgsql_peek(), plpgsql_peek2(), and plpgsql_yylex().

◆ location_lineno_init()

static void location_lineno_init ( yyscan_t  yyscanner)
static

Definition at line 597 of file pl_scanner.c.

598{
599 yyextra->cur_line_start = yyextra->scanorig;
600 yyextra->cur_line_num = 1;
601
602 yyextra->cur_line_end = strchr(yyextra->cur_line_start, '\n');
603}

References yyextra.

Referenced by plpgsql_location_to_lineno(), and plpgsql_scanner_init().

◆ plpgsql_append_source_text()

void plpgsql_append_source_text ( StringInfo  buf,
int  startlocation,
int  endlocation,
yyscan_t  yyscanner 
)

Definition at line 435 of file pl_scanner.c.

438{
439 Assert(startlocation <= endlocation);
440 appendBinaryStringInfo(buf, yyextra->scanorig + startlocation,
441 endlocation - startlocation);
442}
#define Assert(condition)
Definition: c.h:815
static char * buf
Definition: pg_test_fsync.c:72
void appendBinaryStringInfo(StringInfo str, const void *data, int datalen)
Definition: stringinfo.c:281

References appendBinaryStringInfo(), Assert, buf, and yyextra.

◆ plpgsql_latest_lineno()

int plpgsql_latest_lineno ( yyscan_t  yyscanner)

Definition at line 607 of file pl_scanner.c.

608{
609 return yyextra->cur_line_num;
610}

References yyextra.

Referenced by plpgsql_compile_error_callback().

◆ plpgsql_location_to_lineno()

int plpgsql_location_to_lineno ( int  location,
yyscan_t  yyscanner 
)

Definition at line 573 of file pl_scanner.c.

574{
575 const char *loc;
576
577 if (location < 0 || yyextra->scanorig == NULL)
578 return 0; /* garbage in, garbage out */
579 loc = yyextra->scanorig + location;
580
581 /* be correct, but not fast, if input location goes backwards */
582 if (loc < yyextra->cur_line_start)
583 location_lineno_init(yyscanner);
584
585 while (yyextra->cur_line_end != NULL && loc > yyextra->cur_line_end)
586 {
587 yyextra->cur_line_start = yyextra->cur_line_end + 1;
588 yyextra->cur_line_num++;
589 yyextra->cur_line_end = strchr(yyextra->cur_line_start, '\n');
590 }
591
592 return yyextra->cur_line_num;
593}
static void location_lineno_init(yyscan_t yyscanner)
Definition: pl_scanner.c:597

References plpgsql_yy_extra_type::cur_line_start, location_lineno_init(), plpgsql_yy_extra_type::scanorig, and yyextra.

◆ plpgsql_peek()

int plpgsql_peek ( yyscan_t  yyscanner)

Definition at line 452 of file pl_scanner.c.

453{
454 int tok1;
455 TokenAuxData aux1;
456
457 tok1 = internal_yylex(&aux1, yyscanner);
458 push_back_token(tok1, &aux1, yyscanner);
459 return tok1;
460}
static void push_back_token(int token, TokenAuxData *auxdata, yyscan_t yyscanner)
Definition: pl_scanner.c:385
static int internal_yylex(TokenAuxData *auxdata, yyscan_t yyscanner)
Definition: pl_scanner.c:339

References internal_yylex(), and push_back_token().

◆ plpgsql_peek2()

void plpgsql_peek2 ( int *  tok1_p,
int *  tok2_p,
int *  tok1_loc,
int *  tok2_loc,
yyscan_t  yyscanner 
)

Definition at line 471 of file pl_scanner.c.

472{
473 int tok1,
474 tok2;
475 TokenAuxData aux1,
476 aux2;
477
478 tok1 = internal_yylex(&aux1, yyscanner);
479 tok2 = internal_yylex(&aux2, yyscanner);
480
481 *tok1_p = tok1;
482 if (tok1_loc)
483 *tok1_loc = aux1.lloc;
484 *tok2_p = tok2;
485 if (tok2_loc)
486 *tok2_loc = aux2.lloc;
487
488 push_back_token(tok2, &aux2, yyscanner);
489 push_back_token(tok1, &aux1, yyscanner);
490}

References internal_yylex(), TokenAuxData::lloc, and push_back_token().

◆ plpgsql_push_back_token()

void plpgsql_push_back_token ( int  token,
YYSTYPE yylvalp,
YYLTYPE yyllocp,
yyscan_t  yyscanner 
)

Definition at line 401 of file pl_scanner.c.

402{
403 TokenAuxData auxdata;
404
405 auxdata.lval = *yylvalp;
406 auxdata.lloc = *yyllocp;
407 auxdata.leng = yyextra->plpgsql_yyleng;
408 push_back_token(token, &auxdata, yyscanner);
409}

References TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, push_back_token(), token, and yyextra.

◆ plpgsql_scanner_errposition()

int plpgsql_scanner_errposition ( int  location,
yyscan_t  yyscanner 
)

Definition at line 504 of file pl_scanner.c.

505{
506 int pos;
507
508 if (location < 0 || yyextra->scanorig == NULL)
509 return 0; /* no-op if location is unknown */
510
511 /* Convert byte offset to character number */
512 pos = pg_mbstrlen_with_len(yyextra->scanorig, location) + 1;
513 /* And pass it to the ereport mechanism */
514 (void) internalerrposition(pos);
515 /* Also pass the function body string */
516 return internalerrquery(yyextra->scanorig);
517}
int internalerrquery(const char *query)
Definition: elog.c:1482
int internalerrposition(int cursorpos)
Definition: elog.c:1462
int pg_mbstrlen_with_len(const char *mbstr, int limit)
Definition: mbutils.c:1057

References internalerrposition(), internalerrquery(), pg_mbstrlen_with_len(), plpgsql_yy_extra_type::scanorig, and yyextra.

Referenced by plpgsql_yyerror().

◆ plpgsql_scanner_finish()

void plpgsql_scanner_finish ( yyscan_t  yyscanner)

Definition at line 653 of file pl_scanner.c.

654{
655 /* release storage */
656 scanner_finish(yyscanner);
657}
void scanner_finish(core_yyscan_t yyscanner)
Definition: scan.l:1291

References scanner_finish().

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_scanner_init()

yyscan_t plpgsql_scanner_init ( const char *  str)

Definition at line 621 of file pl_scanner.c.

622{
623 yyscan_t yyscanner;
625
626 /* Start up the core scanner */
627 yyscanner = scanner_init(str, (core_yy_extra_type *) yyext,
628 &ReservedPLKeywords, ReservedPLKeywordTokens);
629
630 /*
631 * scanorig points to the original string, which unlike the scanner's
632 * scanbuf won't be modified on-the-fly by flex. Notice that although
633 * yytext points into scanbuf, we rely on being able to apply locations
634 * (offsets from string start) to scanorig as well.
635 */
636 yyext->scanorig = str;
637
638 /* Other setup */
640 yyext->plpgsql_yytoken = 0;
641
642 yyext->num_pushbacks = 0;
643
644 location_lineno_init(yyscanner);
645
646 return yyscanner;
647}
void * yyscan_t
Definition: cubedata.h:67
#define palloc0_object(type)
Definition: fe_memutils.h:75
const char * str
IdentifierLookup plpgsql_IdentifierLookup
Definition: pl_scanner.c:26
static const uint16 ReservedPLKeywordTokens[]
Definition: pl_scanner.c:66
@ IDENTIFIER_LOOKUP_NORMAL
Definition: plpgsql.h:1212
core_yyscan_t scanner_init(const char *str, core_yy_extra_type *yyext, const ScanKeywordList *keywordlist, const uint16 *keyword_tokens)
Definition: scan.l:1249
const char * scanorig
Definition: pl_scanner.c:109

References IDENTIFIER_LOOKUP_NORMAL, location_lineno_init(), plpgsql_yy_extra_type::num_pushbacks, palloc0_object, plpgsql_IdentifierLookup, plpgsql_yy_extra_type::plpgsql_yytoken, ReservedPLKeywordTokens, scanner_init(), plpgsql_yy_extra_type::scanorig, and str.

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_token_is_unreserved_keyword()

bool plpgsql_token_is_unreserved_keyword ( int  token)

Definition at line 418 of file pl_scanner.c.

419{
420 int i;
421
422 for (i = 0; i < lengthof(UnreservedPLKeywordTokens); i++)
423 {
425 return true;
426 }
427 return false;
428}
#define lengthof(array)
Definition: c.h:745
int i
Definition: isn.c:72
static const uint16 UnreservedPLKeywordTokens[]
Definition: pl_scanner.c:70

References i, lengthof, token, and UnreservedPLKeywordTokens.

◆ plpgsql_token_length()

int plpgsql_token_length ( yyscan_t  yyscanner)

Definition at line 327 of file pl_scanner.c.

328{
329 return yyextra->plpgsql_yyleng;
330}

References yyextra.

◆ plpgsql_yyerror()

void plpgsql_yyerror ( YYLTYPE yyllocp,
PLpgSQL_stmt_block **  plpgsql_parse_result_p,
yyscan_t  yyscanner,
const char *  message 
)

Definition at line 534 of file pl_scanner.c.

535{
536 char *yytext = yyextra->core_yy_extra.scanbuf + *yyllocp;
537
538 if (*yytext == '\0')
539 {
541 (errcode(ERRCODE_SYNTAX_ERROR),
542 /* translator: %s is typically the translation of "syntax error" */
543 errmsg("%s at end of input", _(message)),
544 plpgsql_scanner_errposition(*yyllocp, yyscanner)));
545 }
546 else
547 {
548 /*
549 * If we have done any lookahead then flex will have restored the
550 * character after the end-of-token. Zap it again so that we report
551 * only the single token here. This modifies scanbuf but we no longer
552 * care about that.
553 */
554 yytext[yyextra->plpgsql_yyleng] = '\0';
555
557 (errcode(ERRCODE_SYNTAX_ERROR),
558 /* translator: first %s is typically the translation of "syntax error" */
559 errmsg("%s at or near \"%s\"", _(message), yytext),
560 plpgsql_scanner_errposition(*yyllocp, yyscanner)));
561 }
562}
int errcode(int sqlerrcode)
Definition: elog.c:853
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define _(x)
Definition: elog.c:90
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
int plpgsql_scanner_errposition(int location, yyscan_t yyscanner)
Definition: pl_scanner.c:504

References _, ereport, errcode(), errmsg(), ERROR, plpgsql_scanner_errposition(), and yyextra.

◆ plpgsql_yylex()

int plpgsql_yylex ( YYSTYPE yylvalp,
YYLTYPE yyllocp,
yyscan_t  yyscanner 
)

Definition at line 160 of file pl_scanner.c.

161{
162 int tok1;
163 TokenAuxData aux1;
164 int kwnum;
165
166 tok1 = internal_yylex(&aux1, yyscanner);
167 if (tok1 == IDENT || tok1 == PARAM)
168 {
169 int tok2;
170 TokenAuxData aux2;
171
172 tok2 = internal_yylex(&aux2, yyscanner);
173 if (tok2 == '.')
174 {
175 int tok3;
176 TokenAuxData aux3;
177
178 tok3 = internal_yylex(&aux3, yyscanner);
179 if (tok3 == IDENT)
180 {
181 int tok4;
182 TokenAuxData aux4;
183
184 tok4 = internal_yylex(&aux4, yyscanner);
185 if (tok4 == '.')
186 {
187 int tok5;
188 TokenAuxData aux5;
189
190 tok5 = internal_yylex(&aux5, yyscanner);
191 if (tok5 == IDENT)
192 {
193 if (plpgsql_parse_tripword(aux1.lval.str,
194 aux3.lval.str,
195 aux5.lval.str,
196 &aux1.lval.wdatum,
197 &aux1.lval.cword))
198 tok1 = T_DATUM;
199 else
200 tok1 = T_CWORD;
201 /* Adjust token length to include A.B.C */
202 aux1.leng = aux5.lloc - aux1.lloc + aux5.leng;
203 }
204 else
205 {
206 /* not A.B.C, so just process A.B */
207 push_back_token(tok5, &aux5, yyscanner);
208 push_back_token(tok4, &aux4, yyscanner);
209 if (plpgsql_parse_dblword(aux1.lval.str,
210 aux3.lval.str,
211 &aux1.lval.wdatum,
212 &aux1.lval.cword))
213 tok1 = T_DATUM;
214 else
215 tok1 = T_CWORD;
216 /* Adjust token length to include A.B */
217 aux1.leng = aux3.lloc - aux1.lloc + aux3.leng;
218 }
219 }
220 else
221 {
222 /* not A.B.C, so just process A.B */
223 push_back_token(tok4, &aux4, yyscanner);
224 if (plpgsql_parse_dblword(aux1.lval.str,
225 aux3.lval.str,
226 &aux1.lval.wdatum,
227 &aux1.lval.cword))
228 tok1 = T_DATUM;
229 else
230 tok1 = T_CWORD;
231 /* Adjust token length to include A.B */
232 aux1.leng = aux3.lloc - aux1.lloc + aux3.leng;
233 }
234 }
235 else
236 {
237 /* not A.B, so just process A */
238 push_back_token(tok3, &aux3, yyscanner);
239 push_back_token(tok2, &aux2, yyscanner);
240 if (plpgsql_parse_word(aux1.lval.str,
241 yyextra->core_yy_extra.scanbuf + aux1.lloc,
242 true,
243 &aux1.lval.wdatum,
244 &aux1.lval.word))
245 tok1 = T_DATUM;
246 else if (!aux1.lval.word.quoted &&
247 (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
248 &UnreservedPLKeywords)) >= 0)
249 {
250 aux1.lval.keyword = GetScanKeyword(kwnum,
251 &UnreservedPLKeywords);
252 tok1 = UnreservedPLKeywordTokens[kwnum];
253 }
254 else
255 tok1 = T_WORD;
256 }
257 }
258 else
259 {
260 /* not A.B, so just process A */
261 push_back_token(tok2, &aux2, yyscanner);
262
263 /*
264 * See if it matches a variable name, except in the context where
265 * we are at start of statement and the next token isn't
266 * assignment or '['. In that case, it couldn't validly be a
267 * variable name, and skipping the lookup allows variable names to
268 * be used that would conflict with plpgsql or core keywords that
269 * introduce statements (e.g., "comment"). Without this special
270 * logic, every statement-introducing keyword would effectively be
271 * reserved in PL/pgSQL, which would be unpleasant.
272 *
273 * If it isn't a variable name, try to match against unreserved
274 * plpgsql keywords. If not one of those either, it's T_WORD.
275 *
276 * Note: we must call plpgsql_parse_word even if we don't want to
277 * do variable lookup, because it sets up aux1.lval.word for the
278 * non-variable cases.
279 */
280 if (plpgsql_parse_word(aux1.lval.str,
281 yyextra->core_yy_extra.scanbuf + aux1.lloc,
282 (!AT_STMT_START(yyextra->plpgsql_yytoken) ||
283 (tok2 == '=' || tok2 == COLON_EQUALS ||
284 tok2 == '[')),
285 &aux1.lval.wdatum,
286 &aux1.lval.word))
287 tok1 = T_DATUM;
288 else if (!aux1.lval.word.quoted &&
289 (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
290 &UnreservedPLKeywords)) >= 0)
291 {
292 aux1.lval.keyword = GetScanKeyword(kwnum,
293 &UnreservedPLKeywords);
294 tok1 = UnreservedPLKeywordTokens[kwnum];
295 }
296 else
297 tok1 = T_WORD;
298 }
299 }
300 else
301 {
302 /*
303 * Not a potential plpgsql variable name, just return the data.
304 *
305 * Note that we also come through here if the grammar pushed back a
306 * T_DATUM, T_CWORD, T_WORD, or unreserved-keyword token returned by a
307 * previous lookup cycle; thus, pushbacks do not incur extra lookup
308 * work, since we'll never do the above code twice for the same token.
309 * This property also makes it safe to rely on the old value of
310 * plpgsql_yytoken in the is-this-start-of-statement test above.
311 */
312 }
313
314 *yylvalp = aux1.lval;
315 *yyllocp = aux1.lloc;
316 yyextra->plpgsql_yyleng = aux1.leng;
317 yyextra->plpgsql_yytoken = tok1;
318 return tok1;
319}
int ScanKeywordLookup(const char *str, const ScanKeywordList *keywords)
Definition: kwlookup.c:38
static const char * GetScanKeyword(int n, const ScanKeywordList *keywords)
Definition: kwlookup.h:39
bool plpgsql_parse_dblword(char *word1, char *word2, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1461
bool plpgsql_parse_word(char *word1, const char *yytxt, bool lookup, PLwdatum *wdatum, PLword *word)
Definition: pl_comp.c:1406
bool plpgsql_parse_tripword(char *word1, char *word2, char *word3, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1542
#define AT_STMT_START(prev_token)
Definition: pl_scanner.c:82

References AT_STMT_START, GetScanKeyword(), internal_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, plpgsql_parse_dblword(), plpgsql_parse_tripword(), plpgsql_parse_word(), push_back_token(), ScanKeywordLookup(), UnreservedPLKeywordTokens, and yyextra.

◆ push_back_token()

static void push_back_token ( int  token,
TokenAuxData auxdata,
yyscan_t  yyscanner 
)
static

Definition at line 385 of file pl_scanner.c.

386{
387 if (yyextra->num_pushbacks >= MAX_PUSHBACKS)
388 elog(ERROR, "too many tokens pushed back");
389 yyextra->pushback_token[yyextra->num_pushbacks] = token;
390 yyextra->pushback_auxdata[yyextra->num_pushbacks] = *auxdata;
391 yyextra->num_pushbacks++;
392}
#define elog(elevel,...)
Definition: elog.h:225
#define MAX_PUSHBACKS
Definition: pl_scanner.c:98

References elog, ERROR, MAX_PUSHBACKS, token, and yyextra.

Referenced by plpgsql_peek(), plpgsql_peek2(), plpgsql_push_back_token(), and plpgsql_yylex().

Variable Documentation

◆ plpgsql_IdentifierLookup

◆ ReservedPLKeywordTokens

const uint16 ReservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 66 of file pl_scanner.c.

Referenced by plpgsql_scanner_init().

◆ UnreservedPLKeywordTokens

const uint16 UnreservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 70 of file pl_scanner.c.

Referenced by plpgsql_token_is_unreserved_keyword(), and plpgsql_yylex().