tsvector_parser.c

Go to the documentation of this file.

/*-------------------------------------------------------------------------
 *
 * tsvector_parser.c
 *    Parser for tsvector
 *
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
 *
 *
 * IDENTIFICATION
 *    src/backend/utils/adt/tsvector_parser.c
 *
 *-------------------------------------------------------------------------
 */
 
#include "postgres.h"
 
#include "tsearch/ts_locale.h"
#include "tsearch/ts_utils.h"
 
 
/*
 * Private state of tsvector parser.  Note that tsquery also uses this code to
 * parse its input, hence the boolean flags.  The oprisdelim and is_tsquery
 * flags are both true or both false in current usage, but we keep them
 * separate for clarity.
 *
 * If oprisdelim is set, the following characters are treated as delimiters
 * (in addition to whitespace): ! | & ( )
 *
 * is_tsquery affects *only* the content of error messages.
 *
 * is_web can be true to further modify tsquery parsing.
 *
 * If escontext is an ErrorSaveContext node, then soft errors can be
 * captured there rather than being thrown.
 */
struct TSVectorParseStateData
{
    char       *prsbuf;         /* next input character */
    char       *bufstart;       /* whole string (used only for errors) */
    char       *word;           /* buffer to hold the current word */
    int         len;            /* size in bytes allocated for 'word' */
    int         eml;            /* max bytes per character */
    bool        oprisdelim;     /* treat ! | * ( ) as delimiters? */
    bool        is_tsquery;     /* say "tsquery" not "tsvector" in errors? */
    bool        is_web;         /* we're in websearch_to_tsquery() */
    Node       *escontext;      /* for soft error reporting */
};
 
 
/*
 * Initializes a parser state object for the given input string.
 * A bitmask of flags (see ts_utils.h) and an error context object
 * can be provided as well.
 */
TSVectorParseState
init_tsvector_parser(char *input, int flags, Node *escontext)
{
    TSVectorParseState state;
 
    state = palloc_object(struct TSVectorParseStateData);
    state->prsbuf = input;
    state->bufstart = input;
    state->len = 32;
    state->word = (char *) palloc(state->len);
    state->eml = pg_database_encoding_max_length();
    state->oprisdelim = (flags & P_TSV_OPR_IS_DELIM) != 0;
    state->is_tsquery = (flags & P_TSV_IS_TSQUERY) != 0;
    state->is_web = (flags & P_TSV_IS_WEB) != 0;
    state->escontext = escontext;
 
    return state;
}
 
/*
 * Reinitializes parser to parse 'input', instead of previous input.
 *
 * Note that bufstart (the string reported in errors) is not changed.
 */
void
reset_tsvector_parser(TSVectorParseState state, char *input)
{
    state->prsbuf = input;
}
 
/*
 * Shuts down a tsvector parser.
 */
void
close_tsvector_parser(TSVectorParseState state)
{
    pfree(state->word);
    pfree(state);
}
 
/* increase the size of 'word' if needed to hold one more character */
#define RESIZEPRSBUF \
do { \
    int clen = curpos - state->word; \
    if ( clen + state->eml >= state->len ) \
    { \
        state->len *= 2; \
        state->word = (char *) repalloc(state->word, state->len); \
        curpos = state->word + clen; \
    } \
} while (0)
 
/* Fills gettoken_tsvector's output parameters, and returns true */
#define RETURN_TOKEN \
do { \
    if (pos_ptr != NULL) \
    { \
        *pos_ptr = pos; \
        *poslen = npos; \
    } \
    else if (pos != NULL) \
        pfree(pos); \
    \
    if (strval != NULL) \
        *strval = state->word; \
    if (lenval != NULL) \
        *lenval = curpos - state->word; \
    if (endptr != NULL) \
        *endptr = state->prsbuf; \
    return true; \
} while(0)
 
 
/* State codes used in gettoken_tsvector */
#define WAITWORD        1
#define WAITENDWORD     2
#define WAITNEXTCHAR    3
#define WAITENDCMPLX    4
#define WAITPOSINFO     5
#define INPOSINFO       6
#define WAITPOSDELIM    7
#define WAITCHARCMPLX   8
 
#define PRSSYNTAXERROR return prssyntaxerror(state)
 
static bool
prssyntaxerror(TSVectorParseState state)
{
    errsave(state->escontext,
            (errcode(ERRCODE_SYNTAX_ERROR),
             state->is_tsquery ?
             errmsg("syntax error in tsquery: \"%s\"", state->bufstart) :
             errmsg("syntax error in tsvector: \"%s\"", state->bufstart)));
    /* In soft error situation, return false as convenience for caller */
    return false;
}
 
 
/*
 * Get next token from string being parsed. Returns true if successful,
 * false if end of input string is reached or soft error.
 *
 * On success, these output parameters are filled in:
 *
 * *strval      pointer to token
 * *lenval      length of *strval
 * *pos_ptr     pointer to a palloc'd array of positions and weights
 *              associated with the token. If the caller is not interested
 *              in the information, NULL can be supplied. Otherwise
 *              the caller is responsible for pfreeing the array.
 * *poslen      number of elements in *pos_ptr
 * *endptr      scan resumption point
 *
 * Pass NULL for any unwanted output parameters.
 *
 * If state->escontext is an ErrorSaveContext, then caller must check
 * SOFT_ERROR_OCCURRED() to determine whether a "false" result means
 * error or normal end-of-string.
 */
bool
gettoken_tsvector(TSVectorParseState state,
                  char **strval, int *lenval,
                  WordEntryPos **pos_ptr, int *poslen,
                  char **endptr)
{
    int         oldstate = 0;
    char       *curpos = state->word;
    int         statecode = WAITWORD;
 
    /*
     * pos is for collecting the comma delimited list of positions followed by
     * the actual token.
     */
    WordEntryPos *pos = NULL;
    int         npos = 0;       /* elements of pos used */
    int         posalen = 0;    /* allocated size of pos */
 
    while (1)
    {
        if (statecode == WAITWORD)
        {
            if (*(state->prsbuf) == '\0')
                return false;
            else if (!state->is_web && t_iseq(state->prsbuf, '\''))
                statecode = WAITENDCMPLX;
            else if (!state->is_web && t_iseq(state->prsbuf, '\\'))
            {
                statecode = WAITNEXTCHAR;
                oldstate = WAITENDWORD;
            }
            else if ((state->oprisdelim && ISOPERATOR(state->prsbuf)) ||
                     (state->is_web && t_iseq(state->prsbuf, '"')))
                PRSSYNTAXERROR;
            else if (!isspace((unsigned char) *state->prsbuf))
            {
                curpos += ts_copychar_cstr(curpos, state->prsbuf);
                statecode = WAITENDWORD;
            }
        }
        else if (statecode == WAITNEXTCHAR)
        {
            if (*(state->prsbuf) == '\0')
                ereturn(state->escontext, false,
                        (errcode(ERRCODE_SYNTAX_ERROR),
                         errmsg("there is no escaped character: \"%s\"",
                                state->bufstart)));
            else
            {
                RESIZEPRSBUF;
                curpos += ts_copychar_cstr(curpos, state->prsbuf);
                Assert(oldstate != 0);
                statecode = oldstate;
            }
        }
        else if (statecode == WAITENDWORD)
        {
            if (!state->is_web && t_iseq(state->prsbuf, '\\'))
            {
                statecode = WAITNEXTCHAR;
                oldstate = WAITENDWORD;
            }
            else if (isspace((unsigned char) *state->prsbuf) || *(state->prsbuf) == '\0' ||
                     (state->oprisdelim && ISOPERATOR(state->prsbuf)) ||
                     (state->is_web && t_iseq(state->prsbuf, '"')))
            {
                RESIZEPRSBUF;
                if (curpos == state->word)
                    PRSSYNTAXERROR;
                *(curpos) = '\0';
                RETURN_TOKEN;
            }
            else if (t_iseq(state->prsbuf, ':'))
            {
                if (curpos == state->word)
                    PRSSYNTAXERROR;
                *(curpos) = '\0';
                if (state->oprisdelim)
                    RETURN_TOKEN;
                else
                    statecode = INPOSINFO;
            }
            else
            {
                RESIZEPRSBUF;
                curpos += ts_copychar_cstr(curpos, state->prsbuf);
            }
        }
        else if (statecode == WAITENDCMPLX)
        {
            if (!state->is_web && t_iseq(state->prsbuf, '\''))
            {
                statecode = WAITCHARCMPLX;
            }
            else if (!state->is_web && t_iseq(state->prsbuf, '\\'))
            {
                statecode = WAITNEXTCHAR;
                oldstate = WAITENDCMPLX;
            }
            else if (*(state->prsbuf) == '\0')
                PRSSYNTAXERROR;
            else
            {
                RESIZEPRSBUF;
                curpos += ts_copychar_cstr(curpos, state->prsbuf);
            }
        }
        else if (statecode == WAITCHARCMPLX)
        {
            if (!state->is_web && t_iseq(state->prsbuf, '\''))
            {
                RESIZEPRSBUF;
                curpos += ts_copychar_cstr(curpos, state->prsbuf);
                statecode = WAITENDCMPLX;
            }
            else
            {
                RESIZEPRSBUF;
                *(curpos) = '\0';
                if (curpos == state->word)
                    PRSSYNTAXERROR;
                if (state->oprisdelim)
                {
                    /* state->prsbuf+=pg_mblen_cstr(state->prsbuf); */
                    RETURN_TOKEN;
                }
                else
                    statecode = WAITPOSINFO;
                continue;       /* recheck current character */
            }
        }
        else if (statecode == WAITPOSINFO)
        {
            if (t_iseq(state->prsbuf, ':'))
                statecode = INPOSINFO;
            else
                RETURN_TOKEN;
        }
        else if (statecode == INPOSINFO)
        {
            if (isdigit((unsigned char) *state->prsbuf))
            {
                if (posalen == 0)
                {
                    posalen = 4;
                    pos = palloc_array(WordEntryPos, posalen);
                    npos = 0;
                }
                else if (npos + 1 >= posalen)
                {
                    posalen *= 2;
                    pos = repalloc_array(pos, WordEntryPos, posalen);
                }
                npos++;
                WEP_SETPOS(pos[npos - 1], LIMITPOS(atoi(state->prsbuf)));
                /* we cannot get here in tsquery, so no need for 2 errmsgs */
                if (WEP_GETPOS(pos[npos - 1]) == 0)
                    ereturn(state->escontext, false,
                            (errcode(ERRCODE_SYNTAX_ERROR),
                             errmsg("wrong position info in tsvector: \"%s\"",
                                    state->bufstart)));
                WEP_SETWEIGHT(pos[npos - 1], 0);
                statecode = WAITPOSDELIM;
            }
            else
                PRSSYNTAXERROR;
        }
        else if (statecode == WAITPOSDELIM)
        {
            if (t_iseq(state->prsbuf, ','))
                statecode = INPOSINFO;
            else if (t_iseq(state->prsbuf, 'a') || t_iseq(state->prsbuf, 'A') || t_iseq(state->prsbuf, '*'))
            {
                if (WEP_GETWEIGHT(pos[npos - 1]))
                    PRSSYNTAXERROR;
                WEP_SETWEIGHT(pos[npos - 1], 3);
            }
            else if (t_iseq(state->prsbuf, 'b') || t_iseq(state->prsbuf, 'B'))
            {
                if (WEP_GETWEIGHT(pos[npos - 1]))
                    PRSSYNTAXERROR;
                WEP_SETWEIGHT(pos[npos - 1], 2);
            }
            else if (t_iseq(state->prsbuf, 'c') || t_iseq(state->prsbuf, 'C'))
            {
                if (WEP_GETWEIGHT(pos[npos - 1]))
                    PRSSYNTAXERROR;
                WEP_SETWEIGHT(pos[npos - 1], 1);
            }
            else if (t_iseq(state->prsbuf, 'd') || t_iseq(state->prsbuf, 'D'))
            {
                if (WEP_GETWEIGHT(pos[npos - 1]))
                    PRSSYNTAXERROR;
                WEP_SETWEIGHT(pos[npos - 1], 0);
            }
            else if (isspace((unsigned char) *state->prsbuf) ||
                     *(state->prsbuf) == '\0')
                RETURN_TOKEN;
            else if (!isdigit((unsigned char) *state->prsbuf))
                PRSSYNTAXERROR;
        }
        else                    /* internal error */
            elog(ERROR, "unrecognized state in gettoken_tsvector: %d",
                 statecode);
 
        /* get next char */
        state->prsbuf += pg_mblen_cstr(state->prsbuf);
    }
}