PostgreSQL Source Code  git master
pl_scanner.c File Reference
#include "postgres.h"
#include "mb/pg_wchar.h"
#include "parser/scanner.h"
#include "plpgsql.h"
#include "pl_gram.h"
#include "pl_reserved_kwlist_d.h"
#include "pl_unreserved_kwlist_d.h"
#include "pl_reserved_kwlist.h"
#include "pl_unreserved_kwlist.h"
Include dependency graph for pl_scanner.c:

Go to the source code of this file.

Data Structures

struct  TokenAuxData
 

Macros

#define PG_KEYWORD(kwname, value)   value,
 
#define AT_STMT_START(prev_token)
 
#define MAX_PUSHBACKS   4
 

Functions

static int internal_yylex (TokenAuxData *auxdata)
 
static void push_back_token (int token, TokenAuxData *auxdata)
 
static void location_lineno_init (void)
 
int plpgsql_yylex (void)
 
void plpgsql_push_back_token (int token)
 
bool plpgsql_token_is_unreserved_keyword (int token)
 
void plpgsql_append_source_text (StringInfo buf, int startlocation, int endlocation)
 
int plpgsql_peek (void)
 
void plpgsql_peek2 (int *tok1_p, int *tok2_p, int *tok1_loc, int *tok2_loc)
 
int plpgsql_scanner_errposition (int location)
 
void plpgsql_yyerror (const char *message)
 
int plpgsql_location_to_lineno (int location)
 
int plpgsql_latest_lineno (void)
 
void plpgsql_scanner_init (const char *str)
 
void plpgsql_scanner_finish (void)
 

Variables

IdentifierLookup plpgsql_IdentifierLookup = IDENTIFIER_LOOKUP_NORMAL
 
static const uint16 ReservedPLKeywordTokens []
 
static const uint16 UnreservedPLKeywordTokens []
 
static core_yyscan_t yyscanner = NULL
 
static core_yy_extra_type core_yy
 
static const char * scanorig
 
static int plpgsql_yyleng
 
static int plpgsql_yytoken
 
static int num_pushbacks
 
static int pushback_token [MAX_PUSHBACKS]
 
static TokenAuxData pushback_auxdata [MAX_PUSHBACKS]
 
static const char * cur_line_start
 
static const char * cur_line_end
 
static int cur_line_num
 

Macro Definition Documentation

◆ AT_STMT_START

#define AT_STMT_START (   prev_token)
Value:
((prev_token) == ';' || \
(prev_token) == K_BEGIN || \
(prev_token) == K_THEN || \
(prev_token) == K_ELSE || \
(prev_token) == K_LOOP)

Definition at line 82 of file pl_scanner.c.

◆ MAX_PUSHBACKS

#define MAX_PUSHBACKS   4

Definition at line 119 of file pl_scanner.c.

◆ PG_KEYWORD

#define PG_KEYWORD (   kwname,
  value 
)    value,

Definition at line 64 of file pl_scanner.c.

Function Documentation

◆ internal_yylex()

static int internal_yylex ( TokenAuxData auxdata)
static

Definition at line 308 of file pl_scanner.c.

309 {
310  int token;
311  const char *yytext;
312 
313  if (num_pushbacks > 0)
314  {
315  num_pushbacks--;
317  *auxdata = pushback_auxdata[num_pushbacks];
318  }
319  else
320  {
321  token = core_yylex(&auxdata->lval.core_yystype,
322  &auxdata->lloc,
323  yyscanner);
324 
325  /* remember the length of yytext before it gets changed */
326  yytext = core_yy.scanbuf + auxdata->lloc;
327  auxdata->leng = strlen(yytext);
328 
329  /* Check for << >> and #, which the core considers operators */
330  if (token == Op)
331  {
332  if (strcmp(auxdata->lval.str, "<<") == 0)
333  token = LESS_LESS;
334  else if (strcmp(auxdata->lval.str, ">>") == 0)
335  token = GREATER_GREATER;
336  else if (strcmp(auxdata->lval.str, "#") == 0)
337  token = '#';
338  }
339 
340  /* The core returns PARAM as ival, but we treat it like IDENT */
341  else if (token == PARAM)
342  {
343  auxdata->lval.str = pstrdup(yytext);
344  }
345  }
346 
347  return token;
348 }
#define token
Definition: indent_globs.h:126
char * pstrdup(const char *in)
Definition: mcxt.c:1644
static TokenAuxData pushback_auxdata[MAX_PUSHBACKS]
Definition: pl_scanner.c:123
static core_yyscan_t yyscanner
Definition: pl_scanner.c:106
static int num_pushbacks
Definition: pl_scanner.c:121
static int pushback_token[MAX_PUSHBACKS]
Definition: pl_scanner.c:122
static core_yy_extra_type core_yy
Definition: pl_scanner.c:107
int core_yylex(core_YYSTYPE *yylval_param, YYLTYPE *yylloc_param, core_yyscan_t yyscanner)
YYLTYPE lloc
Definition: pl_scanner.c:94
YYSTYPE lval
Definition: pl_scanner.c:93
char * scanbuf
Definition: scanner.h:72

References core_yy, core_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, num_pushbacks, pstrdup(), pushback_auxdata, pushback_token, core_yy_extra_type::scanbuf, token, and yyscanner.

Referenced by plpgsql_peek(), plpgsql_peek2(), and plpgsql_yylex().

◆ location_lineno_init()

static void location_lineno_init ( void  )
static

Definition at line 562 of file pl_scanner.c.

563 {
565  cur_line_num = 1;
566 
567  cur_line_end = strchr(cur_line_start, '\n');
568 }
static const char * scanorig
Definition: pl_scanner.c:110
static int cur_line_num
Definition: pl_scanner.c:128
static const char * cur_line_end
Definition: pl_scanner.c:127
static const char * cur_line_start
Definition: pl_scanner.c:126

References cur_line_end, cur_line_num, cur_line_start, and scanorig.

Referenced by plpgsql_location_to_lineno(), and plpgsql_scanner_init().

◆ plpgsql_append_source_text()

void plpgsql_append_source_text ( StringInfo  buf,
int  startlocation,
int  endlocation 
)

Definition at line 404 of file pl_scanner.c.

406 {
407  Assert(startlocation <= endlocation);
408  appendBinaryStringInfo(buf, scanorig + startlocation,
409  endlocation - startlocation);
410 }
Assert(fmt[strlen(fmt) - 1] !='\n')
static char * buf
Definition: pg_test_fsync.c:67
void appendBinaryStringInfo(StringInfo str, const void *data, int datalen)
Definition: stringinfo.c:227

References appendBinaryStringInfo(), Assert(), buf, and scanorig.

◆ plpgsql_latest_lineno()

int plpgsql_latest_lineno ( void  )

Definition at line 572 of file pl_scanner.c.

573 {
574  return cur_line_num;
575 }

References cur_line_num.

Referenced by plpgsql_compile_error_callback().

◆ plpgsql_location_to_lineno()

int plpgsql_location_to_lineno ( int  location)

Definition at line 538 of file pl_scanner.c.

539 {
540  const char *loc;
541 
542  if (location < 0 || scanorig == NULL)
543  return 0; /* garbage in, garbage out */
544  loc = scanorig + location;
545 
546  /* be correct, but not fast, if input location goes backwards */
547  if (loc < cur_line_start)
549 
550  while (cur_line_end != NULL && loc > cur_line_end)
551  {
553  cur_line_num++;
554  cur_line_end = strchr(cur_line_start, '\n');
555  }
556 
557  return cur_line_num;
558 }
static void location_lineno_init(void)
Definition: pl_scanner.c:562

References cur_line_end, cur_line_num, cur_line_start, location_lineno_init(), and scanorig.

◆ plpgsql_peek()

int plpgsql_peek ( void  )

Definition at line 420 of file pl_scanner.c.

421 {
422  int tok1;
423  TokenAuxData aux1;
424 
425  tok1 = internal_yylex(&aux1);
426  push_back_token(tok1, &aux1);
427  return tok1;
428 }
static void push_back_token(int token, TokenAuxData *auxdata)
Definition: pl_scanner.c:354
static int internal_yylex(TokenAuxData *auxdata)
Definition: pl_scanner.c:308

References internal_yylex(), and push_back_token().

◆ plpgsql_peek2()

void plpgsql_peek2 ( int *  tok1_p,
int *  tok2_p,
int *  tok1_loc,
int *  tok2_loc 
)

Definition at line 439 of file pl_scanner.c.

440 {
441  int tok1,
442  tok2;
443  TokenAuxData aux1,
444  aux2;
445 
446  tok1 = internal_yylex(&aux1);
447  tok2 = internal_yylex(&aux2);
448 
449  *tok1_p = tok1;
450  if (tok1_loc)
451  *tok1_loc = aux1.lloc;
452  *tok2_p = tok2;
453  if (tok2_loc)
454  *tok2_loc = aux2.lloc;
455 
456  push_back_token(tok2, &aux2);
457  push_back_token(tok1, &aux1);
458 }

References internal_yylex(), TokenAuxData::lloc, and push_back_token().

◆ plpgsql_push_back_token()

void plpgsql_push_back_token ( int  token)

Definition at line 370 of file pl_scanner.c.

371 {
372  TokenAuxData auxdata;
373 
374  auxdata.lval = plpgsql_yylval;
375  auxdata.lloc = plpgsql_yylloc;
376  auxdata.leng = plpgsql_yyleng;
377  push_back_token(token, &auxdata);
378 }
static int plpgsql_yyleng
Definition: pl_scanner.c:113

References TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, plpgsql_yyleng, push_back_token(), and token.

◆ plpgsql_scanner_errposition()

int plpgsql_scanner_errposition ( int  location)

Definition at line 472 of file pl_scanner.c.

473 {
474  int pos;
475 
476  if (location < 0 || scanorig == NULL)
477  return 0; /* no-op if location is unknown */
478 
479  /* Convert byte offset to character number */
480  pos = pg_mbstrlen_with_len(scanorig, location) + 1;
481  /* And pass it to the ereport mechanism */
482  (void) internalerrposition(pos);
483  /* Also pass the function body string */
484  return internalerrquery(scanorig);
485 }
int internalerrquery(const char *query)
Definition: elog.c:1481
int internalerrposition(int cursorpos)
Definition: elog.c:1461
int pg_mbstrlen_with_len(const char *mbstr, int limit)
Definition: mbutils.c:1058

References internalerrposition(), internalerrquery(), pg_mbstrlen_with_len(), and scanorig.

Referenced by plpgsql_yyerror().

◆ plpgsql_scanner_finish()

void plpgsql_scanner_finish ( void  )

Definition at line 613 of file pl_scanner.c.

614 {
615  /* release storage */
617  /* avoid leaving any dangling pointers */
618  yyscanner = NULL;
619  scanorig = NULL;
620 }
void scanner_finish(core_yyscan_t yyscanner)

References scanner_finish(), scanorig, and yyscanner.

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_scanner_init()

void plpgsql_scanner_init ( const char *  str)

Definition at line 586 of file pl_scanner.c.

587 {
588  /* Start up the core scanner */
590  &ReservedPLKeywords, ReservedPLKeywordTokens);
591 
592  /*
593  * scanorig points to the original string, which unlike the scanner's
594  * scanbuf won't be modified on-the-fly by flex. Notice that although
595  * yytext points into scanbuf, we rely on being able to apply locations
596  * (offsets from string start) to scanorig as well.
597  */
598  scanorig = str;
599 
600  /* Other setup */
602  plpgsql_yytoken = 0;
603 
604  num_pushbacks = 0;
605 
607 }
static int plpgsql_yytoken
Definition: pl_scanner.c:116
IdentifierLookup plpgsql_IdentifierLookup
Definition: pl_scanner.c:26
static const uint16 ReservedPLKeywordTokens[]
Definition: pl_scanner.c:66
@ IDENTIFIER_LOOKUP_NORMAL
Definition: plpgsql.h:1189
core_yyscan_t scanner_init(const char *str, core_yy_extra_type *yyext, const ScanKeywordList *keywordlist, const uint16 *keyword_tokens)

References core_yy, IDENTIFIER_LOOKUP_NORMAL, location_lineno_init(), num_pushbacks, plpgsql_IdentifierLookup, plpgsql_yytoken, ReservedPLKeywordTokens, scanner_init(), scanorig, generate_unaccent_rules::str, and yyscanner.

Referenced by do_compile(), and plpgsql_compile_inline().

◆ plpgsql_token_is_unreserved_keyword()

bool plpgsql_token_is_unreserved_keyword ( int  token)

Definition at line 387 of file pl_scanner.c.

388 {
389  int i;
390 
391  for (i = 0; i < lengthof(UnreservedPLKeywordTokens); i++)
392  {
394  return true;
395  }
396  return false;
397 }
#define lengthof(array)
Definition: c.h:777
int i
Definition: isn.c:73
static const uint16 UnreservedPLKeywordTokens[]
Definition: pl_scanner.c:70

References i, lengthof, token, and UnreservedPLKeywordTokens.

◆ plpgsql_yyerror()

void plpgsql_yyerror ( const char *  message)

Definition at line 499 of file pl_scanner.c.

500 {
501  char *yytext = core_yy.scanbuf + plpgsql_yylloc;
502 
503  if (*yytext == '\0')
504  {
505  ereport(ERROR,
506  (errcode(ERRCODE_SYNTAX_ERROR),
507  /* translator: %s is typically the translation of "syntax error" */
508  errmsg("%s at end of input", _(message)),
509  plpgsql_scanner_errposition(plpgsql_yylloc)));
510  }
511  else
512  {
513  /*
514  * If we have done any lookahead then flex will have restored the
515  * character after the end-of-token. Zap it again so that we report
516  * only the single token here. This modifies scanbuf but we no longer
517  * care about that.
518  */
519  yytext[plpgsql_yyleng] = '\0';
520 
521  ereport(ERROR,
522  (errcode(ERRCODE_SYNTAX_ERROR),
523  /* translator: first %s is typically the translation of "syntax error" */
524  errmsg("%s at or near \"%s\"", _(message), yytext),
525  plpgsql_scanner_errposition(plpgsql_yylloc)));
526  }
527 }
int errcode(int sqlerrcode)
Definition: elog.c:858
int errmsg(const char *fmt,...)
Definition: elog.c:1069
#define _(x)
Definition: elog.c:91
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
int plpgsql_scanner_errposition(int location)
Definition: pl_scanner.c:472

References _, core_yy, ereport, errcode(), errmsg(), ERROR, plpgsql_scanner_errposition(), plpgsql_yyleng, and core_yy_extra_type::scanbuf.

◆ plpgsql_yylex()

int plpgsql_yylex ( void  )

Definition at line 146 of file pl_scanner.c.

147 {
148  int tok1;
149  TokenAuxData aux1;
150  int kwnum;
151 
152  tok1 = internal_yylex(&aux1);
153  if (tok1 == IDENT || tok1 == PARAM)
154  {
155  int tok2;
156  TokenAuxData aux2;
157 
158  tok2 = internal_yylex(&aux2);
159  if (tok2 == '.')
160  {
161  int tok3;
162  TokenAuxData aux3;
163 
164  tok3 = internal_yylex(&aux3);
165  if (tok3 == IDENT)
166  {
167  int tok4;
168  TokenAuxData aux4;
169 
170  tok4 = internal_yylex(&aux4);
171  if (tok4 == '.')
172  {
173  int tok5;
174  TokenAuxData aux5;
175 
176  tok5 = internal_yylex(&aux5);
177  if (tok5 == IDENT)
178  {
179  if (plpgsql_parse_tripword(aux1.lval.str,
180  aux3.lval.str,
181  aux5.lval.str,
182  &aux1.lval.wdatum,
183  &aux1.lval.cword))
184  tok1 = T_DATUM;
185  else
186  tok1 = T_CWORD;
187  }
188  else
189  {
190  /* not A.B.C, so just process A.B */
191  push_back_token(tok5, &aux5);
192  push_back_token(tok4, &aux4);
193  if (plpgsql_parse_dblword(aux1.lval.str,
194  aux3.lval.str,
195  &aux1.lval.wdatum,
196  &aux1.lval.cword))
197  tok1 = T_DATUM;
198  else
199  tok1 = T_CWORD;
200  }
201  }
202  else
203  {
204  /* not A.B.C, so just process A.B */
205  push_back_token(tok4, &aux4);
206  if (plpgsql_parse_dblword(aux1.lval.str,
207  aux3.lval.str,
208  &aux1.lval.wdatum,
209  &aux1.lval.cword))
210  tok1 = T_DATUM;
211  else
212  tok1 = T_CWORD;
213  }
214  }
215  else
216  {
217  /* not A.B, so just process A */
218  push_back_token(tok3, &aux3);
219  push_back_token(tok2, &aux2);
220  if (plpgsql_parse_word(aux1.lval.str,
221  core_yy.scanbuf + aux1.lloc,
222  true,
223  &aux1.lval.wdatum,
224  &aux1.lval.word))
225  tok1 = T_DATUM;
226  else if (!aux1.lval.word.quoted &&
227  (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
228  &UnreservedPLKeywords)) >= 0)
229  {
230  aux1.lval.keyword = GetScanKeyword(kwnum,
231  &UnreservedPLKeywords);
232  tok1 = UnreservedPLKeywordTokens[kwnum];
233  }
234  else
235  tok1 = T_WORD;
236  }
237  }
238  else
239  {
240  /* not A.B, so just process A */
241  push_back_token(tok2, &aux2);
242 
243  /*
244  * See if it matches a variable name, except in the context where
245  * we are at start of statement and the next token isn't
246  * assignment or '['. In that case, it couldn't validly be a
247  * variable name, and skipping the lookup allows variable names to
248  * be used that would conflict with plpgsql or core keywords that
249  * introduce statements (e.g., "comment"). Without this special
250  * logic, every statement-introducing keyword would effectively be
251  * reserved in PL/pgSQL, which would be unpleasant.
252  *
253  * If it isn't a variable name, try to match against unreserved
254  * plpgsql keywords. If not one of those either, it's T_WORD.
255  *
256  * Note: we must call plpgsql_parse_word even if we don't want to
257  * do variable lookup, because it sets up aux1.lval.word for the
258  * non-variable cases.
259  */
260  if (plpgsql_parse_word(aux1.lval.str,
261  core_yy.scanbuf + aux1.lloc,
263  (tok2 == '=' || tok2 == COLON_EQUALS ||
264  tok2 == '[')),
265  &aux1.lval.wdatum,
266  &aux1.lval.word))
267  tok1 = T_DATUM;
268  else if (!aux1.lval.word.quoted &&
269  (kwnum = ScanKeywordLookup(aux1.lval.word.ident,
270  &UnreservedPLKeywords)) >= 0)
271  {
272  aux1.lval.keyword = GetScanKeyword(kwnum,
273  &UnreservedPLKeywords);
274  tok1 = UnreservedPLKeywordTokens[kwnum];
275  }
276  else
277  tok1 = T_WORD;
278  }
279  }
280  else
281  {
282  /*
283  * Not a potential plpgsql variable name, just return the data.
284  *
285  * Note that we also come through here if the grammar pushed back a
286  * T_DATUM, T_CWORD, T_WORD, or unreserved-keyword token returned by a
287  * previous lookup cycle; thus, pushbacks do not incur extra lookup
288  * work, since we'll never do the above code twice for the same token.
289  * This property also makes it safe to rely on the old value of
290  * plpgsql_yytoken in the is-this-start-of-statement test above.
291  */
292  }
293 
294  plpgsql_yylval = aux1.lval;
295  plpgsql_yylloc = aux1.lloc;
296  plpgsql_yyleng = aux1.leng;
297  plpgsql_yytoken = tok1;
298  return tok1;
299 }
int ScanKeywordLookup(const char *str, const ScanKeywordList *keywords)
Definition: kwlookup.c:38
static const char * GetScanKeyword(int n, const ScanKeywordList *keywords)
Definition: kwlookup.h:39
bool plpgsql_parse_dblword(char *word1, char *word2, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1441
bool plpgsql_parse_word(char *word1, const char *yytxt, bool lookup, PLwdatum *wdatum, PLword *word)
Definition: pl_comp.c:1386
bool plpgsql_parse_tripword(char *word1, char *word2, char *word3, PLwdatum *wdatum, PLcword *cword)
Definition: pl_comp.c:1522
#define AT_STMT_START(prev_token)
Definition: pl_scanner.c:82

References AT_STMT_START, core_yy, GetScanKeyword(), internal_yylex(), TokenAuxData::leng, TokenAuxData::lloc, TokenAuxData::lval, plpgsql_parse_dblword(), plpgsql_parse_tripword(), plpgsql_parse_word(), plpgsql_yyleng, plpgsql_yytoken, push_back_token(), core_yy_extra_type::scanbuf, ScanKeywordLookup(), and UnreservedPLKeywordTokens.

◆ push_back_token()

static void push_back_token ( int  token,
TokenAuxData auxdata 
)
static

Definition at line 354 of file pl_scanner.c.

355 {
357  elog(ERROR, "too many tokens pushed back");
359  pushback_auxdata[num_pushbacks] = *auxdata;
360  num_pushbacks++;
361 }
#define MAX_PUSHBACKS
Definition: pl_scanner.c:119

References elog(), ERROR, MAX_PUSHBACKS, num_pushbacks, pushback_auxdata, pushback_token, and token.

Referenced by plpgsql_peek(), plpgsql_peek2(), plpgsql_push_back_token(), and plpgsql_yylex().

Variable Documentation

◆ core_yy

core_yy_extra_type core_yy
static

Definition at line 107 of file pl_scanner.c.

Referenced by internal_yylex(), plpgsql_scanner_init(), plpgsql_yyerror(), and plpgsql_yylex().

◆ cur_line_end

const char* cur_line_end
static

Definition at line 127 of file pl_scanner.c.

Referenced by location_lineno_init(), and plpgsql_location_to_lineno().

◆ cur_line_num

int cur_line_num
static

◆ cur_line_start

const char* cur_line_start
static

Definition at line 126 of file pl_scanner.c.

Referenced by location_lineno_init(), and plpgsql_location_to_lineno().

◆ num_pushbacks

int num_pushbacks
static

Definition at line 121 of file pl_scanner.c.

Referenced by internal_yylex(), plpgsql_scanner_init(), and push_back_token().

◆ plpgsql_IdentifierLookup

◆ plpgsql_yyleng

int plpgsql_yyleng
static

Definition at line 113 of file pl_scanner.c.

Referenced by plpgsql_push_back_token(), plpgsql_yyerror(), and plpgsql_yylex().

◆ plpgsql_yytoken

int plpgsql_yytoken
static

Definition at line 116 of file pl_scanner.c.

Referenced by plpgsql_scanner_init(), and plpgsql_yylex().

◆ pushback_auxdata

TokenAuxData pushback_auxdata[MAX_PUSHBACKS]
static

Definition at line 123 of file pl_scanner.c.

Referenced by internal_yylex(), and push_back_token().

◆ pushback_token

int pushback_token[MAX_PUSHBACKS]
static

Definition at line 122 of file pl_scanner.c.

Referenced by internal_yylex(), and push_back_token().

◆ ReservedPLKeywordTokens

const uint16 ReservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 66 of file pl_scanner.c.

Referenced by plpgsql_scanner_init().

◆ scanorig

◆ UnreservedPLKeywordTokens

const uint16 UnreservedPLKeywordTokens[]
static
Initial value:
= {
}

Definition at line 70 of file pl_scanner.c.

Referenced by plpgsql_token_is_unreserved_keyword(), and plpgsql_yylex().

◆ yyscanner