PostgreSQL Source Code  git master
pl_scanner.c File Reference
#include "postgres.h"
#include "mb/pg_wchar.h"
#include "parser/scanner.h"
#include "plpgsql.h"
#include "pl_gram.h"
#include "pl_reserved_kwlist_d.h"
#include "pl_unreserved_kwlist_d.h"
#include "pl_reserved_kwlist.h"
#include "pl_unreserved_kwlist.h"
Include dependency graph for pl_scanner.c:

Go to the source code of this file.

Data Structures

struct  TokenAuxData
 

Macros

#define PG_KEYWORD(kwname, value)   value,
 
#define AT_STMT_START(prev_token)
 
#define MAX_PUSHBACKS   4
 

Functions

static int internal_yylex (TokenAuxData *auxdata)
 
static void push_back_token (int token, TokenAuxData *auxdata)
 
static void location_lineno_init (void)
 
int plpgsql_yylex (void)
 
int plpgsql_token_length (void)
 
void plpgsql_push_back_token (int token)
 
bool plpgsql_token_is_unreserved_keyword (int token)
 
void plpgsql_append_source_text (StringInfo buf, int startlocation, int endlocation)
 
int plpgsql_peek (void)
 
void plpgsql_peek2 (int *tok1_p, int *tok2_p, int *tok1_loc, int *tok2_loc)
 
int plpgsql_scanner_errposition (int location)
 
void plpgsql_yyerror (const char *message)
 
int plpgsql_location_to_lineno (int location)
 
int plpgsql_latest_lineno (void)
 
void plpgsql_scanner_init (const char *str)
 
void plpgsql_scanner_finish (void)
 

Variables

IdentifierLookup plpgsql_IdentifierLookup = IDENTIFIER_LOOKUP_NORMAL
 
static const uint16 ReservedPLKeywordTokens []
 
static const uint16 UnreservedPLKeywordTokens []
 
static core_yyscan_t yyscanner = NULL
 
static core_yy_extra_type core_yy
 
static const char * scanorig
 
static int plpgsql_yyleng
 
static int plpgsql_yytoken
 
static int num_pushbacks
 
static int pushback_token [MAX_PUSHBACKS]
 
static TokenAuxData pushback_auxdata [MAX_PUSHBACKS]
 
static const char * cur_line_start
 
static const char * cur_line_end
 
static int cur_line_num
 

Macro Definition Documentation

◆ AT_STMT_START

#define AT_STMT_START (   prev_token)
Value:
((prev_token) == ';' || \
(prev_token) == K_BEGIN || \
(prev_token) == K_THEN || \
(prev_token) == K_ELSE || \
(prev_token) == K_LOOP)

Definition at line 82 of file pl_scanner.c.

◆ MAX_PUSHBACKS

#define MAX_PUSHBACKS   4

Definition at line 119 of file pl_scanner.c.

◆ PG_KEYWORD

#define PG_KEYWORD (   kwname,
  value 
)    value,

Definition at line 64 of file pl_scanner.c.

Function Documentation

◆ internal_yylex()

static int internal_yylex ( TokenAuxData auxdata)
static

Definition at line 325 of file pl_scanner.c.

326 {
327  int token;
328  const char *yytext;
329 
330  if (num_pushbacks > 0)
331  {
332  num_pushbacks--;
334  *auxdata = pushback_auxdata[num_pushbacks];
335  }
336  else
337  {
338  token = core_yylex(&auxdata->lval.core_yystype,
339  &auxdata->lloc,
340  yyscanner);
341 
342  /* remember the length of yytext before it gets changed */
343  yytext = core_yy.scanbuf + auxdata->lloc;
344  auxdata->leng = strlen(yytext);
345 
346  /* Check for << >> and #, which the core considers operators */
347  if (token == Op)
348  {
349  if (strcmp(auxdata->lval.str, "<<") == 0)
350  token = LESS_LESS;
351  else if (strcmp(auxdata->lval.str, ">>") == 0)
352  token = GREATER_GREATER;
353  else if (strcmp(auxdata->lval.str, "#") == 0)
354  token = '#';
355  }
356 
357  /* The core returns PARAM as ival, but we treat it like IDENT */
358  else if (token == PARAM)
359  {
360  auxdata->lval.str = pstrdup(yytext);
361  }
362  }
363 
364  return token;
365 }
#define token
Definition: indent_globs.h:126
char * pstrdup(const char *in)
Definition: mcxt.c:1696
static TokenAuxData pushback_auxdata[MAX_PUSHBACKS]
Definition: pl_scanner.c:123
static core_yyscan_t yyscanner
Definition: pl_scanner.c:106
static int num_pushbacks
Definition: pl_scanner.c:121
static int pushback_token[MAX_PUSHBACKS]
Definition: pl_scanner.c:122
static core_yy_extra_type core_yy
Definition: pl_scanner.c:107
int core_yylex(core_YYSTYPE *yylval_param, YYLTYPE *yylloc_param, core_yyscan_t yyscanner)
YYLTYPE lloc
Definition: pl_scanner.c:94
YYSTYPE lval
Definition: pl_scanner.c:93
char * scanbuf
Definition: scanner.h:72

References core_yy, core_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, num_pushbacks, pstrdup(), pushback_auxdata, pushback_token, core_yy_extra_type::scanbuf, token, and yyscanner.

Referenced by plpgsql_peek(), plpgsql_peek2(), and plpgsql_yylex().

◆ location_lineno_init()

static void location_lineno_init ( void  )
static

Definition at line 579 of file pl_scanner.c.

580 {
582  cur_line_num = 1;
583 
584  cur_line_end = strchr(cur_line_start, '\n');
585 }
static const char * scanorig
Definition: pl_scanner.c:110
static int cur_line_num
Definition: pl_scanner.c:128
static const char * cur_line_end
Definition: pl_scanner.c:127
static const char * cur_line_start
Definition: pl_scanner.c:126

References cur_line_end, cur_line_num, cur_line_start, and scanorig.

Referenced by plpgsql_location_to_lineno(), and plpgsql_scanner_init().

◆ plpgsql_append_source_text()

void plpgsql_append_source_text ( StringInfo  buf,
int  startlocation,
int  endlocation 
)

Definition at line 421 of file pl_scanner.c.

423 {
424  Assert(startlocation <= endlocation);
425  appendBinaryStringInfo(buf, scanorig + startlocation,
426  endlocation - startlocation);
427 }
#define Assert(condition)
Definition: c.h:858
static char * buf
Definition: pg_test_fsync.c:73
void appendBinaryStringInfo(StringInfo str, const void *data, int datalen)
Definition: stringinfo.c:233

References appendBinaryStringInfo(), Assert, buf, and scanorig.

◆ plpgsql_latest_lineno()

int plpgsql_latest_lineno ( void  )

Definition at line 589 of file pl_scanner.c.

590 {
591  return cur_line_num;
592 }

References cur_line_num.

Referenced by plpgsql_compile_error_callback().

◆ plpgsql_location_to_lineno()

int plpgsql_location_to_lineno ( int  location)

Definition at line 555 of file pl_scanner.c.

556 {
557  const char *loc;
558 
559  if (location < 0 || scanorig == NULL)
560  return 0; /* garbage in, garbage out */
561  loc = scanorig + location;
562 
563  /* be correct, but not fast, if input location goes backwards */
564  if (loc < cur_line_start)
566 
567  while (cur_line_end != NULL && loc > cur_line_end)
568  {
570  cur_line_num++;
571  cur_line_end = strchr(cur_line_start, '\n');
572  }
573 
574  return cur_line_num;
575 }
static void location_lineno_init(void)
Definition: pl_scanner.c:579

References cur_line_end, cur_line_num, cur_line_start, location_lineno_init(), and scanorig.

◆ plpgsql_peek()

int plpgsql_peek ( void  )

Definition at line 437 of file pl_scanner.c.

438 {
439  int tok1;
440  TokenAuxData aux1;
441 
442  tok1 = internal_yylex(&aux1);
443  push_back_token(tok1, &aux1);
444  return tok1;
445 }
static void push_back_token(int token, TokenAuxData *auxdata)
Definition: pl_scanner.c:371
static int internal_yylex(TokenAuxData *auxdata)
Definition: pl_scanner.c:325

References internal_yylex(), and push_back_token().

◆ plpgsql_peek2()

void plpgsql_peek2 ( int *  tok1_p,
int *  tok2_p,
int *  tok1_loc,
int *  tok2_loc 
)

Definition at line 456 of file pl_scanner.c.

457 {
458  int tok1,
459  tok2;
460  TokenAuxData aux1,
461  aux2;
462 
463  tok1 = internal_yylex(&aux1);
464  tok2 = internal_yylex(&aux2);
465 
466  *tok1_p = tok1;
467  if (tok1_loc)
468  *tok1_loc = aux1.lloc;
469  *tok2_p = tok2;
470  if (tok2_loc)
471  *tok2_loc = aux2.lloc;
472 
473  push_back_token(tok2, &aux2);
474  push_back_token(tok1, &aux1);
475 }

References internal_yylex(), TokenAuxData::lloc, and push_back_token().

◆ plpgsql_push_back_token()

void plpgsql_push_back_token ( int  token)

Definition at line 387 of file pl_scanner.c.

388 {
389  TokenAuxData auxdata;
390 
391  auxdata.lval = plpgsql_yylval;
392  auxdata.lloc = plpgsql_yylloc;
393  auxdata.leng = plpgsql_yyleng;
394  push_back_token(token, &auxdata);
395 }
static int plpgsql_yyleng
Definition: pl_scanner.c:113

References TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, plpgsql_yyleng, push_back_token(), and token.

◆ plpgsql_scanner_errposition()

int plpgsql_scanner_errposition ( int  location)

Definition at line 489 of file pl_scanner.c.

490 {
491  int pos;
492 
493  if (location < 0 || scanorig == NULL)
494  return 0; /* no-op if location is unknown */
495 
496  /* Convert byte offset to character number */
497  pos = pg_mbstrlen_with_len(scanorig, location) + 1;
498  /* And pass it to the ereport mechanism */
499  (void) internalerrposition(pos);
500  /* Also pass the function body string */
501  return internalerrquery(scanorig);
502 }
int internalerrquery(const char *query)
Definition: elog.c:1484
int internalerrposition(int cursorpos)
Definition: elog.c:1464
int pg_mbstrlen_with_len(const char *mbstr, int limit)
Definition: mbutils.c:1057

References internalerrposition(), internalerrquery(), pg_mbstrlen_with_len(), and scanorig.

Referenced by plpgsql_yyerror().

◆ plpgsql_scanner_finish()

void plpgsql_scanner_finish ( void  )

Definition at line 630 of file pl_scanner.c.

631 {
632  /* release storage */
634  /* avoid leaving any dangling pointers */
635  yyscanner = NULL;
636  scanorig = NULL;
637 }
void scanner_finish(core_yyscan_t yyscanner)

References scanner_finish(), scanorig, and yyscanner.

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_scanner_init()

void plpgsql_scanner_init ( const char *  str)

Definition at line 603 of file pl_scanner.c.

604 {
605  /* Start up the core scanner */
607  &ReservedPLKeywords, ReservedPLKeywordTokens);
608 
609  /*
610  * scanorig points to the original string, which unlike the scanner's
611  * scanbuf won't be modified on-the-fly by flex. Notice that although
612  * yytext points into scanbuf, we rely on being able to apply locations
613  * (offsets from string start) to scanorig as well.
614  */
615  scanorig = str;
616 
617  /* Other setup */
619  plpgsql_yytoken = 0;
620 
621  num_pushbacks = 0;
622 
624 }
const char * str
static int plpgsql_yytoken
Definition: pl_scanner.c:116
IdentifierLookup plpgsql_IdentifierLookup
Definition: pl_scanner.c:26
static const uint16 ReservedPLKeywordTokens[]
Definition: pl_scanner.c:66
@ IDENTIFIER_LOOKUP_NORMAL
Definition: plpgsql.h:1189
core_yyscan_t scanner_init(const char *str, core_yy_extra_type *yyext, const ScanKeywordList *keywordlist, const uint16 *keyword_tokens)

References core_yy, IDENTIFIER_LOOKUP_NORMAL, location_lineno_init(), num_pushbacks, plpgsql_IdentifierLookup, plpgsql_yytoken, ReservedPLKeywordTokens, scanner_init(), scanorig, str, and yyscanner.

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_token_is_unreserved_keyword()

bool plpgsql_token_is_unreserved_keyword ( int  token)

Definition at line 404 of file pl_scanner.c.

405 {
406  int i;
407 
408  for (i = 0; i < lengthof(UnreservedPLKeywordTokens); i++)
409  {
411  return true;
412  }
413  return false;
414 }
#define lengthof(array)
Definition: c.h:788
int i
Definition: isn.c:73
static const uint16 UnreservedPLKeywordTokens[]
Definition: pl_scanner.c:70

References i, lengthof, token, and UnreservedPLKeywordTokens.

◆ plpgsql_token_length()

int plpgsql_token_length ( void  )

Definition at line 313 of file pl_scanner.c.

314 {
315  return plpgsql_yyleng;
316 }

References plpgsql_yyleng.

◆ plpgsql_yyerror()

void plpgsql_yyerror ( const char *  message)

Definition at line 516 of file pl_scanner.c.

517 {
518  char *yytext = core_yy.scanbuf + plpgsql_yylloc;
519 
520  if (*yytext == '\0')
521  {
522  ereport(ERROR,
523  (errcode(ERRCODE_SYNTAX_ERROR),
524  /* translator: %s is typically the translation of "syntax error" */
525  errmsg("%s at end of input", _(message)),
526  plpgsql_scanner_errposition(plpgsql_yylloc)));
527  }
528  else
529  {
530  /*
531  * If we have done any lookahead then flex will have restored the
532  * character after the end-of-token. Zap it again so that we report
533  * only the single token here. This modifies scanbuf but we no longer
534  * care about that.
535  */
536  yytext[plpgsql_yyleng] = '\0';
537 
538  ereport(ERROR,
539  (errcode(ERRCODE_SYNTAX_ERROR),
540  /* translator: first %s is typically the translation of "syntax error" */
541  errmsg("%s at or near \"%s\"", _(message), yytext),
542  plpgsql_scanner_errposition(plpgsql_yylloc)));
543  }
544 }
int errcode(int sqlerrcode)
Definition: elog.c:855
int errmsg(const char *fmt,...)
Definition: elog.c:1072
#define _(x)
Definition: elog.c:90
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
int plpgsql_scanner_errposition(int location)
Definition: pl_scanner.c:489

References _, core_yy, ereport, errcode(), errmsg(), ERROR, plpgsql_scanner_errposition(), plpgsql_yyleng, and core_yy_extra_type::scanbuf.

◆ plpgsql_yylex()

int plpgsql_yylex ( void  )

Definition at line 146 of file pl_scanner.c.

147 {
148  int tok1;
149  TokenAuxData aux1;
150  int kwnum;
151 
152  tok1 = internal_yylex(&aux1);
153  if (tok1 == IDENT || tok1 == PARAM)
154  {
155  int tok2;
156  TokenAuxData aux2;
157 
158  tok2 = internal_yylex(&aux2);
159  if (tok2 == '.')
160  {
161  int tok3;
162  TokenAuxData aux3;
163 
164  tok3 = internal_yylex(&aux3);
165  if (tok3 == IDENT)
166  {
167  int tok4;
168  TokenAuxData aux4;
169 
170  tok4 = internal_yylex(&aux4);
171  if (tok4 == '.')
172  {
173  int tok5;
174  TokenAuxData aux5;
175 
176  tok5 = internal_yylex(&aux5);
177  if (tok5 == IDENT)
178  {
179  if (plpgsql_parse_tripword(aux1.lval.str,
180  aux3.lval.str,
181  aux5.lval.str,
182  &aux1.lval.wdatum,
183  &aux1.lval.cword))
184  tok1 = T_DATUM;
185  else
186  tok1 = T_CWORD;
187  /* Adjust token length to include A.B.C */
188  aux1.leng = aux5.lloc - aux1.lloc + aux5.leng;
189  }
190  else
191  {
192  /* not A.B.C, so just process A.B */
193  push_back_token(tok5, &aux5);
194  push_back_token(tok4, &aux4);
195  if (plpgsql_parse_dblword(aux1.lval.str,
196  aux3.lval.str,
197  &aux1.lval.wdatum,
198  &aux1.lval.cword))
199  tok1 = T_DATUM;
200  else
201  tok1 = T_CWORD;
202  /* Adjust token length to include A.B */
203  aux1.leng = aux3.lloc - aux1.lloc + aux3.leng;
204  }
205  }
206  else
207  {
208  /* not A.B.C, so just process A.B */
209  push_back_token(tok4, &aux4);
210  if (plpgsql_parse_dblword(aux1.lval.str,
211  aux3.lval.str,
212  &aux1.lval.wdatum,
213  &aux1.lval.cword))
214  tok1 = T_DATUM;
215  else
216  tok1 = T_CWORD;
217  /* Adjust token length to include A.B */
218  aux1.leng = aux3.lloc - aux1.lloc + aux3.leng;
219  }
220  }
221  else
222  {
223  /* not A.B, so just process A */
224  push_back_token(tok3, &aux3);
225  push_back_token(tok2, &aux2);
226  if (plpgsql_parse_word(aux1.lval.str,
227  core_yy.scanbuf + aux1.lloc,
228  true,
229  &aux1.lval.wdatum,
230  &aux1.lval.word))
231  tok1 = T_DATUM;
232  else if (!aux1.lval.word.quoted &&
233  (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
234  &UnreservedPLKeywords)) >= 0)
235  {
236  aux1.lval.keyword = GetScanKeyword(kwnum,
237  &UnreservedPLKeywords);
238  tok1 = UnreservedPLKeywordTokens[kwnum];
239  }
240  else
241  tok1 = T_WORD;
242  }
243  }
244  else
245  {
246  /* not A.B, so just process A */
247  push_back_token(tok2, &aux2);
248 
249  /*
250  * See if it matches a variable name, except in the context where
251  * we are at start of statement and the next token isn't
252  * assignment or '['. In that case, it couldn't validly be a
253  * variable name, and skipping the lookup allows variable names to
254  * be used that would conflict with plpgsql or core keywords that
255  * introduce statements (e.g., "comment"). Without this special
256  * logic, every statement-introducing keyword would effectively be
257  * reserved in PL/pgSQL, which would be unpleasant.
258  *
259  * If it isn't a variable name, try to match against unreserved
260  * plpgsql keywords. If not one of those either, it's T_WORD.
261  *
262  * Note: we must call plpgsql_parse_word even if we don't want to
263  * do variable lookup, because it sets up aux1.lval.word for the
264  * non-variable cases.
265  */
266  if (plpgsql_parse_word(aux1.lval.str,
267  core_yy.scanbuf + aux1.lloc,
269  (tok2 == '=' || tok2 == COLON_EQUALS ||
270  tok2 == '[')),
271  &aux1.lval.wdatum,
272  &aux1.lval.word))
273  tok1 = T_DATUM;
274  else if (!aux1.lval.word.quoted &&
275  (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
276  &UnreservedPLKeywords)) >= 0)
277  {
278  aux1.lval.keyword = GetScanKeyword(kwnum,
279  &UnreservedPLKeywords);
280  tok1 = UnreservedPLKeywordTokens[kwnum];
281  }
282  else
283  tok1 = T_WORD;
284  }
285  }
286  else
287  {
288  /*
289  * Not a potential plpgsql variable name, just return the data.
290  *
291  * Note that we also come through here if the grammar pushed back a
292  * T_DATUM, T_CWORD, T_WORD, or unreserved-keyword token returned by a
293  * previous lookup cycle; thus, pushbacks do not incur extra lookup
294  * work, since we'll never do the above code twice for the same token.
295  * This property also makes it safe to rely on the old value of
296  * plpgsql_yytoken in the is-this-start-of-statement test above.
297  */
298  }
299 
300  plpgsql_yylval = aux1.lval;
301  plpgsql_yylloc = aux1.lloc;
302  plpgsql_yyleng = aux1.leng;
303  plpgsql_yytoken = tok1;
304  return tok1;
305 }
int ScanKeywordLookup(const char *str, const ScanKeywordList *keywords)
Definition: kwlookup.c:38
static const char * GetScanKeyword(int n, const ScanKeywordList *keywords)
Definition: kwlookup.h:39
bool plpgsql_parse_dblword(char *word1, char *word2, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1441
bool plpgsql_parse_word(char *word1, const char *yytxt, bool lookup, PLwdatum *wdatum, PLword *word)
Definition: pl_comp.c:1386
bool plpgsql_parse_tripword(char *word1, char *word2, char *word3, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1522
#define AT_STMT_START(prev_token)
Definition: pl_scanner.c:82

References AT_STMT_START, core_yy, GetScanKeyword(), internal_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, plpgsql_parse_dblword(), plpgsql_parse_tripword(), plpgsql_parse_word(), plpgsql_yyleng, plpgsql_yytoken, push_back_token(), core_yy_extra_type::scanbuf, ScanKeywordLookup(), and UnreservedPLKeywordTokens.

◆ push_back_token()

static void push_back_token ( int  token,
TokenAuxData auxdata 
)
static

Definition at line 371 of file pl_scanner.c.

372 {
374  elog(ERROR, "too many tokens pushed back");
376  pushback_auxdata[num_pushbacks] = *auxdata;
377  num_pushbacks++;
378 }
#define elog(elevel,...)
Definition: elog.h:224
#define MAX_PUSHBACKS
Definition: pl_scanner.c:119

References elog, ERROR, MAX_PUSHBACKS, num_pushbacks, pushback_auxdata, pushback_token, and token.

Referenced by plpgsql_peek(), plpgsql_peek2(), plpgsql_push_back_token(), and plpgsql_yylex().

Variable Documentation

◆ core_yy

core_yy_extra_type core_yy
static

Definition at line 107 of file pl_scanner.c.

Referenced by internal_yylex(), plpgsql_scanner_init(), plpgsql_yyerror(), and plpgsql_yylex().

◆ cur_line_end

const char* cur_line_end
static

Definition at line 127 of file pl_scanner.c.

Referenced by location_lineno_init(), and plpgsql_location_to_lineno().

◆ cur_line_num

int cur_line_num
static

◆ cur_line_start

const char* cur_line_start
static

Definition at line 126 of file pl_scanner.c.

Referenced by location_lineno_init(), and plpgsql_location_to_lineno().

◆ num_pushbacks

int num_pushbacks
static

Definition at line 121 of file pl_scanner.c.

Referenced by internal_yylex(), plpgsql_scanner_init(), and push_back_token().

◆ plpgsql_IdentifierLookup

◆ plpgsql_yyleng

int plpgsql_yyleng
static

◆ plpgsql_yytoken

int plpgsql_yytoken
static

Definition at line 116 of file pl_scanner.c.

Referenced by plpgsql_scanner_init(), and plpgsql_yylex().

◆ pushback_auxdata

TokenAuxData pushback_auxdata[MAX_PUSHBACKS]
static

Definition at line 123 of file pl_scanner.c.

Referenced by internal_yylex(), and push_back_token().

◆ pushback_token

int pushback_token[MAX_PUSHBACKS]
static

Definition at line 122 of file pl_scanner.c.

Referenced by internal_yylex(), and push_back_token().

◆ ReservedPLKeywordTokens

const uint16 ReservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 66 of file pl_scanner.c.

Referenced by plpgsql_scanner_init().

◆ scanorig

◆ UnreservedPLKeywordTokens

const uint16 UnreservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 70 of file pl_scanner.c.

Referenced by plpgsql_token_is_unreserved_keyword(), and plpgsql_yylex().

◆ yyscanner