PostgreSQL Source Code  git master
ts_utils.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * ts_utils.h
4  * helper utilities for tsearch
5  *
6  * Copyright (c) 1998-2020, PostgreSQL Global Development Group
7  *
8  * src/include/tsearch/ts_utils.h
9  *
10  *-------------------------------------------------------------------------
11  */
12 #ifndef _PG_TS_UTILS_H_
13 #define _PG_TS_UTILS_H_
14 
15 #include "nodes/pg_list.h"
16 #include "tsearch/ts_public.h"
17 #include "tsearch/ts_type.h"
18 
19 /*
20  * Common parse definitions for tsvector and tsquery
21  */
22 
23 /* tsvector parser support. */
24 
25 struct TSVectorParseStateData; /* opaque struct in tsvector_parser.c */
27 
28 #define P_TSV_OPR_IS_DELIM (1 << 0)
29 #define P_TSV_IS_TSQUERY (1 << 1)
30 #define P_TSV_IS_WEB (1 << 2)
31 
32 extern TSVectorParseState init_tsvector_parser(char *input, int flags);
33 extern void reset_tsvector_parser(TSVectorParseState state, char *input);
34 extern bool gettoken_tsvector(TSVectorParseState state,
35  char **token, int *len,
36  WordEntryPos **pos, int *poslen,
37  char **endptr);
38 extern void close_tsvector_parser(TSVectorParseState state);
39 
40 /* phrase operator begins with '<' */
41 #define ISOPERATOR(x) \
42  ( pg_mblen(x) == 1 && ( *(x) == '!' || \
43  *(x) == '&' || \
44  *(x) == '|' || \
45  *(x) == '(' || \
46  *(x) == ')' || \
47  *(x) == '<' \
48  ) )
49 
50 /* parse_tsquery */
51 
52 struct TSQueryParserStateData; /* private in backend/utils/adt/tsquery.c */
54 
55 typedef void (*PushFunction) (Datum opaque, TSQueryParserState state,
56  char *token, int tokenlen,
57  int16 tokenweights, /* bitmap as described in
58  * QueryOperand struct */
59  bool prefix);
60 
61 #define P_TSQ_PLAIN (1 << 0)
62 #define P_TSQ_WEB (1 << 1)
63 
64 extern TSQuery parse_tsquery(char *buf,
65  PushFunction pushval,
66  Datum opaque,
67  int flags);
68 
69 /* Functions for use by PushFunction implementations */
70 extern void pushValue(TSQueryParserState state,
71  char *strval, int lenval, int16 weight, bool prefix);
72 extern void pushStop(TSQueryParserState state);
73 extern void pushOperator(TSQueryParserState state, int8 oper, int16 distance);
74 
75 /*
76  * parse plain text and lexize words
77  */
78 typedef struct
79 {
82  union
83  {
85 
86  /*
87  * When apos array is used, apos[0] is the number of elements in the
88  * array (excluding apos[0]), and alen is the allocated size of the
89  * array.
90  */
92  } pos;
93  uint16 flags; /* currently, only TSL_PREFIX */
94  char *word;
96 } ParsedWord;
97 
98 typedef struct
99 {
104 } ParsedText;
105 
106 extern void parsetext(Oid cfgId, ParsedText *prs, char *buf, int32 buflen);
107 
108 /*
109  * headline framework, flow in common to generate:
110  * 1 parse text with hlparsetext
111  * 2 parser-specific function to find part
112  * 3 generateHeadline to generate result text
113  */
114 
115 extern void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query,
116  char *buf, int32 buflen);
118 
119 /*
120  * TSQuery execution support
121  *
122  * TS_execute() executes a tsquery against data that can be represented in
123  * various forms. The TSExecuteCallback callback function is called to check
124  * whether a given primitive tsquery value is matched in the data.
125  */
126 
127 /* TS_execute requires ternary logic to handle NOT with phrase matches */
128 typedef enum
129 {
130  TS_NO, /* definitely no match */
131  TS_YES, /* definitely does match */
132  TS_MAYBE /* can't verify match for lack of pos data */
134 
135 /*
136  * struct ExecPhraseData is passed to a TSExecuteCallback function if we need
137  * lexeme position data (because of a phrase-match operator in the tsquery).
138  * The callback should fill in position data when it returns TS_YES (success).
139  * If it cannot return position data, it should leave "data" unchanged and
140  * return TS_MAYBE. The caller of TS_execute() must then arrange for a later
141  * recheck with position data available.
142  *
143  * The reported lexeme positions must be sorted and unique. Callers must only
144  * consult the position bits of the pos array, ie, WEP_GETPOS(data->pos[i]).
145  * This allows the returned "pos" to point directly to the WordEntryPos
146  * portion of a tsvector value. If "allocated" is true then the pos array
147  * is palloc'd workspace and caller may free it when done.
148  *
149  * "negate" means that the pos array contains positions where the query does
150  * not match, rather than positions where it does. "width" is positive when
151  * the match is wider than one lexeme. Neither of these fields normally need
152  * to be touched by TSExecuteCallback functions; they are used for
153  * phrase-search processing within TS_execute.
154  *
155  * All fields of the ExecPhraseData struct are initially zeroed by caller.
156  */
157 typedef struct ExecPhraseData
158 {
159  int npos; /* number of positions reported */
160  bool allocated; /* pos points to palloc'd data? */
161  bool negate; /* positions are where query is NOT matched */
162  WordEntryPos *pos; /* ordered, non-duplicate lexeme positions */
163  int width; /* width of match in lexemes, less 1 */
165 
166 /*
167  * Signature for TSQuery lexeme check functions
168  *
169  * arg: opaque value passed through from caller of TS_execute
170  * val: lexeme to test for presence of
171  * data: to be filled with lexeme positions; NULL if position data not needed
172  *
173  * Return TS_YES if lexeme is present in data, TS_MAYBE if it might be
174  * present, TS_NO if it definitely is not present. If data is not NULL,
175  * it must be filled with lexeme positions if available. If position data
176  * is not available, leave *data as zeroes and return TS_MAYBE, never TS_YES.
177  */
179  ExecPhraseData *data);
180 
181 /*
182  * Flag bits for TS_execute
183  */
184 #define TS_EXEC_EMPTY (0x00)
185 /*
186  * If TS_EXEC_SKIP_NOT is set, then NOT sub-expressions are automatically
187  * evaluated to be true. This was formerly the default behavior. It's now
188  * deprecated because it tends to give silly answers, but some applications
189  * might still have a use for it.
190  */
191 #define TS_EXEC_SKIP_NOT (0x01)
192 /*
193  * If TS_EXEC_PHRASE_NO_POS is set, allow OP_PHRASE to be executed lossily
194  * in the absence of position information: a true result indicates that the
195  * phrase might be present. Without this flag, OP_PHRASE always returns
196  * false if lexeme position information is not available.
197  */
198 #define TS_EXEC_PHRASE_NO_POS (0x02)
199 
200 extern bool TS_execute(QueryItem *curitem, void *arg, uint32 flags,
201  TSExecuteCallback chkcond);
202 extern bool tsquery_requires_match(QueryItem *curitem);
203 
204 /*
205  * to_ts* - text transformation to tsvector, tsquery
206  */
207 extern TSVector make_tsvector(ParsedText *prs);
208 extern int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix);
209 
210 /*
211  * Possible strategy numbers for indexes
212  * TSearchStrategyNumber - (tsvector|text) @@ tsquery
213  * TSearchWithClassStrategyNumber - tsvector @@@ tsquery
214  */
215 #define TSearchStrategyNumber 1
216 #define TSearchWithClassStrategyNumber 2
217 
218 /*
219  * TSQuery Utilities
220  */
221 extern QueryItem *clean_NOT(QueryItem *ptr, int32 *len);
223 
224 typedef struct QTNode
225 {
229  char *word;
231  struct QTNode **child;
232 } QTNode;
233 
234 /* bits in QTNode.flags */
235 #define QTN_NEEDFREE 0x01
236 #define QTN_NOCHANGE 0x02
237 #define QTN_WORDFREE 0x04
238 
239 typedef uint64 TSQuerySign;
240 
241 #define TSQS_SIGLEN (sizeof(TSQuerySign)*BITS_PER_BYTE)
242 
243 #define TSQuerySignGetDatum(X) Int64GetDatum((int64) (X))
244 #define DatumGetTSQuerySign(X) ((TSQuerySign) DatumGetInt64(X))
245 #define PG_RETURN_TSQUERYSIGN(X) return TSQuerySignGetDatum(X)
246 #define PG_GETARG_TSQUERYSIGN(n) DatumGetTSQuerySign(PG_GETARG_DATUM(n))
247 
248 
249 extern QTNode *QT2QTN(QueryItem *in, char *operand);
250 extern TSQuery QTN2QT(QTNode *in);
251 extern void QTNFree(QTNode *in);
252 extern void QTNSort(QTNode *in);
253 extern void QTNTernary(QTNode *in);
254 extern void QTNBinary(QTNode *in);
255 extern int QTNodeCompare(QTNode *an, QTNode *bn);
256 extern QTNode *QTNCopy(QTNode *in);
257 extern void QTNClearFlags(QTNode *in, uint32 flags);
258 extern bool QTNEq(QTNode *a, QTNode *b);
260 extern QTNode *findsubquery(QTNode *root, QTNode *ex, QTNode *subs,
261  bool *isfind);
262 
263 #endif /* _PG_TS_UTILS_H_ */
uint16 WordEntryPos
Definition: ts_type.h:63
QTNode * QTNCopy(QTNode *in)
Definition: tsquery_util.c:395
signed short int16
Definition: c.h:361
TSQuery cleanup_tsquery_stopwords(TSQuery in)
struct TSQueryParserStateData * TSQueryParserState
Definition: ts_utils.h:53
void pushOperator(TSQueryParserState state, int8 oper, int16 distance)
Definition: tsquery.c:545
QTNode * QT2QTN(QueryItem *in, char *operand)
Definition: tsquery_util.c:24
char * word
Definition: ts_utils.h:229
bool QTNEq(QTNode *a, QTNode *b)
Definition: tsquery_util.c:182
QueryItem * clean_NOT(QueryItem *ptr, int32 *len)
TSVectorParseState init_tsvector_parser(char *input, int flags)
struct TSVectorParseStateData * TSVectorParseState
Definition: ts_utils.h:26
uint32 sign
Definition: ts_utils.h:230
bool allocated
Definition: ts_utils.h:160
void QTNClearFlags(QTNode *in, uint32 flags)
Definition: tsquery_util.c:433
uint16 * apos
Definition: ts_utils.h:91
uint16 nvariant
Definition: ts_utils.h:81
void(* PushFunction)(Datum opaque, TSQueryParserState state, char *token, int tokenlen, int16 tokenweights, bool prefix)
Definition: ts_utils.h:55
void QTNTernary(QTNode *in)
Definition: tsquery_util.c:200
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1147
int32 lenwords
Definition: ts_utils.h:101
unsigned int Oid
Definition: postgres_ext.h:31
int32 curwords
Definition: ts_utils.h:102
void QTNBinary(QTNode *in)
Definition: tsquery_util.c:249
signed int int32
Definition: c.h:362
bool tsquery_requires_match(QueryItem *curitem)
Definition: tsvector_op.c:1965
void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int32 buflen)
unsigned short uint16
Definition: c.h:373
TSVector make_tsvector(ParsedText *prs)
Definition: to_tsany.c:156
void pushStop(TSQueryParserState state)
Definition: tsquery.c:630
static char * buf
Definition: pg_test_fsync.c:68
uint64 TSQuerySign
Definition: ts_utils.h:239
WordEntryPos * pos
Definition: ts_utils.h:162
unsigned int uint32
Definition: c.h:374
ParsedWord * words
Definition: ts_utils.h:100
uint16 pos
Definition: ts_utils.h:84
TSQuery QTN2QT(QTNode *in)
Definition: tsquery_util.c:362
signed char int8
Definition: c.h:360
void parsetext(Oid cfgId, ParsedText *prs, char *buf, int32 buflen)
ts_parserstate state
Definition: tsquery.c:81
void reset_tsvector_parser(TSVectorParseState state, char *input)
uintptr_t Datum
Definition: postgres.h:367
bool TS_execute(QueryItem *curitem, void *arg, uint32 flags, TSExecuteCallback chkcond)
Definition: tsvector_op.c:1846
void QTNFree(QTNode *in)
Definition: tsquery_util.c:63
bool gettoken_tsvector(TSVectorParseState state, char **token, int *len, WordEntryPos **pos, int *poslen, char **endptr)
TSTernaryValue(* TSExecuteCallback)(void *arg, QueryOperand *val, ExecPhraseData *data)
Definition: ts_utils.h:178
Definition: regguts.h:298
uint16 len
Definition: ts_utils.h:80
QTNode * findsubquery(QTNode *root, QTNode *ex, QTNode *subs, bool *isfind)
int32 pos
Definition: ts_utils.h:103
void QTNSort(QTNode *in)
Definition: tsquery_util.c:162
struct QTNode ** child
Definition: ts_utils.h:231
struct ExecPhraseData ExecPhraseData
int QTNodeCompare(QTNode *an, QTNode *bn)
Definition: tsquery_util.c:96
uint16 flags
Definition: ts_utils.h:93
uint32 alen
Definition: ts_utils.h:95
QueryItem * valnode
Definition: ts_utils.h:226
text * generateHeadline(HeadlineParsedText *prs)
Definition: ts_parse.c:594
int32 nchild
Definition: ts_utils.h:228
struct QTNode QTNode
void * arg
TSTernaryValue
Definition: ts_utils.h:128
Definition: c.h:562
void pushValue(TSQueryParserState state, char *strval, int lenval, int16 weight, bool prefix)
Definition: tsquery.c:594
Operator oper(ParseState *pstate, List *opname, Oid ltypeId, Oid rtypeId, bool noError, int location)
Definition: parse_oper.c:382
TSQuerySign makeTSQuerySign(TSQuery a)
Definition: tsquery_op.c:249
void close_tsvector_parser(TSVectorParseState state)
long val
Definition: informix.c:664
TSQuery parse_tsquery(char *buf, PushFunction pushval, Datum opaque, int flags)
Definition: tsquery.c:817
uint32 flags
Definition: ts_utils.h:227
char * word
Definition: ts_utils.h:94