PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
ts_utils.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * ts_utils.h
4  * helper utilities for tsearch
5  *
6  * Copyright (c) 1998-2017, PostgreSQL Global Development Group
7  *
8  * src/include/tsearch/ts_utils.h
9  *
10  *-------------------------------------------------------------------------
11  */
12 #ifndef _PG_TS_UTILS_H_
13 #define _PG_TS_UTILS_H_
14 
15 #include "nodes/pg_list.h"
16 #include "tsearch/ts_public.h"
17 #include "tsearch/ts_type.h"
18 
19 /*
20  * Common parse definitions for tsvector and tsquery
21  */
22 
23 /* tsvector parser support. */
24 
25 struct TSVectorParseStateData; /* opaque struct in tsvector_parser.c */
27 
28 extern TSVectorParseState init_tsvector_parser(char *input,
29  bool oprisdelim,
30  bool is_tsquery);
31 extern void reset_tsvector_parser(TSVectorParseState state, char *input);
32 extern bool gettoken_tsvector(TSVectorParseState state,
33  char **token, int *len,
34  WordEntryPos **pos, int *poslen,
35  char **endptr);
36 extern void close_tsvector_parser(TSVectorParseState state);
37 
38 /* parse_tsquery */
39 
40 struct TSQueryParserStateData; /* private in backend/utils/adt/tsquery.c */
42 
43 typedef void (*PushFunction) (Datum opaque, TSQueryParserState state,
44  char *token, int tokenlen,
45  int16 tokenweights, /* bitmap as described
46  * in QueryOperand
47  * struct */
48  bool prefix);
49 
50 extern TSQuery parse_tsquery(char *buf,
51  PushFunction pushval,
52  Datum opaque, bool isplain);
53 
54 /* Functions for use by PushFunction implementations */
55 extern void pushValue(TSQueryParserState state,
56  char *strval, int lenval, int16 weight, bool prefix);
57 extern void pushStop(TSQueryParserState state);
58 extern void pushOperator(TSQueryParserState state, int8 oper, int16 distance);
59 
60 /*
61  * parse plain text and lexize words
62  */
63 typedef struct
64 {
67  union
68  {
70 
71  /*
72  * When apos array is used, apos[0] is the number of elements in the
73  * array (excluding apos[0]), and alen is the allocated size of the
74  * array.
75  */
77  } pos;
78  uint16 flags; /* currently, only TSL_PREFIX */
79  char *word;
81 } ParsedWord;
82 
83 typedef struct
84 {
89 } ParsedText;
90 
91 extern void parsetext(Oid cfgId, ParsedText *prs, char *buf, int32 buflen);
92 
93 /*
94  * headline framework, flow in common to generate:
95  * 1 parse text with hlparsetext
96  * 2 parser-specific function to find part
97  * 3 generateHeadline to generate result text
98  */
99 
100 extern void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query,
101  char *buf, int32 buflen);
103 
104 /*
105  * TSQuery execution support
106  *
107  * TS_execute() executes a tsquery against data that can be represented in
108  * various forms. The TSExecuteCallback callback function is called to check
109  * whether a given primitive tsquery value is matched in the data.
110  */
111 
112 /*
113  * struct ExecPhraseData is passed to a TSExecuteCallback function if we need
114  * lexeme position data (because of a phrase-match operator in the tsquery).
115  * The callback should fill in position data when it returns true (success).
116  * If it cannot return position data, it may leave "data" unchanged, but
117  * then the caller of TS_execute() must pass the TS_EXEC_PHRASE_NO_POS flag
118  * and must arrange for a later recheck with position data available.
119  *
120  * The reported lexeme positions must be sorted and unique. Callers must only
121  * consult the position bits of the pos array, ie, WEP_GETPOS(data->pos[i]).
122  * This allows the returned "pos" to point directly to the WordEntryPos
123  * portion of a tsvector value. If "allocated" is true then the pos array
124  * is palloc'd workspace and caller may free it when done.
125  *
126  * "negate" means that the pos array contains positions where the query does
127  * not match, rather than positions where it does. "width" is positive when
128  * the match is wider than one lexeme. Neither of these fields normally need
129  * to be touched by TSExecuteCallback functions; they are used for
130  * phrase-search processing within TS_execute.
131  *
132  * All fields of the ExecPhraseData struct are initially zeroed by caller.
133  */
134 typedef struct ExecPhraseData
135 {
136  int npos; /* number of positions reported */
137  bool allocated; /* pos points to palloc'd data? */
138  bool negate; /* positions are where query is NOT matched */
139  WordEntryPos *pos; /* ordered, non-duplicate lexeme positions */
140  int width; /* width of match in lexemes, less 1 */
142 
143 /*
144  * Signature for TSQuery lexeme check functions
145  *
146  * arg: opaque value passed through from caller of TS_execute
147  * val: lexeme to test for presence of
148  * data: to be filled with lexeme positions; NULL if position data not needed
149  *
150  * Return TRUE if lexeme is present in data, else FALSE. If data is not
151  * NULL, it should be filled with lexeme positions, but function can leave
152  * it as zeroes if position data is not available.
153  */
155  ExecPhraseData *data);
156 
157 /*
158  * Flag bits for TS_execute
159  */
160 #define TS_EXEC_EMPTY (0x00)
161 /*
162  * If TS_EXEC_CALC_NOT is not set, then NOT expressions are automatically
163  * evaluated to be true. Useful in cases where NOT cannot be accurately
164  * computed (GiST) or it isn't important (ranking). From TS_execute's
165  * perspective, !CALC_NOT means that the TSExecuteCallback function might
166  * return false-positive indications of a lexeme's presence.
167  */
168 #define TS_EXEC_CALC_NOT (0x01)
169 /*
170  * If TS_EXEC_PHRASE_NO_POS is set, allow OP_PHRASE to be executed lossily
171  * in the absence of position information: a TRUE result indicates that the
172  * phrase might be present. Without this flag, OP_PHRASE always returns
173  * false if lexeme position information is not available.
174  */
175 #define TS_EXEC_PHRASE_NO_POS (0x02)
176 
177 extern bool TS_execute(QueryItem *curitem, void *arg, uint32 flags,
178  TSExecuteCallback chkcond);
179 extern bool tsquery_requires_match(QueryItem *curitem);
180 
181 /*
182  * to_ts* - text transformation to tsvector, tsquery
183  */
184 extern TSVector make_tsvector(ParsedText *prs);
185 extern int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix);
186 
187 /*
188  * Possible strategy numbers for indexes
189  * TSearchStrategyNumber - (tsvector|text) @@ tsquery
190  * TSearchWithClassStrategyNumber - tsvector @@@ tsquery
191  */
192 #define TSearchStrategyNumber 1
193 #define TSearchWithClassStrategyNumber 2
194 
195 /*
196  * TSQuery Utilities
197  */
198 extern QueryItem *clean_NOT(QueryItem *ptr, int32 *len);
200 
201 typedef struct QTNode
202 {
206  char *word;
208  struct QTNode **child;
209 } QTNode;
210 
211 /* bits in QTNode.flags */
212 #define QTN_NEEDFREE 0x01
213 #define QTN_NOCHANGE 0x02
214 #define QTN_WORDFREE 0x04
215 
216 typedef uint64 TSQuerySign;
217 
218 #define TSQS_SIGLEN (sizeof(TSQuerySign)*BITS_PER_BYTE)
219 
220 #define TSQuerySignGetDatum(X) Int64GetDatum((int64) (X))
221 #define DatumGetTSQuerySign(X) ((TSQuerySign) DatumGetInt64(X))
222 #define PG_RETURN_TSQUERYSIGN(X) return TSQuerySignGetDatum(X)
223 #define PG_GETARG_TSQUERYSIGN(n) DatumGetTSQuerySign(PG_GETARG_DATUM(n))
224 
225 
226 extern QTNode *QT2QTN(QueryItem *in, char *operand);
227 extern TSQuery QTN2QT(QTNode *in);
228 extern void QTNFree(QTNode *in);
229 extern void QTNSort(QTNode *in);
230 extern void QTNTernary(QTNode *in);
231 extern void QTNBinary(QTNode *in);
232 extern int QTNodeCompare(QTNode *an, QTNode *bn);
233 extern QTNode *QTNCopy(QTNode *in);
234 extern void QTNClearFlags(QTNode *in, uint32 flags);
235 extern bool QTNEq(QTNode *a, QTNode *b);
237 extern QTNode *findsubquery(QTNode *root, QTNode *ex, QTNode *subs,
238  bool *isfind);
239 
240 #endif /* _PG_TS_UTILS_H_ */
uint16 WordEntryPos
Definition: ts_type.h:63
QTNode * QTNCopy(QTNode *in)
Definition: tsquery_util.c:395
signed short int16
Definition: c.h:252
TSQuery cleanup_tsquery_stopwords(TSQuery in)
struct TSQueryParserStateData * TSQueryParserState
Definition: ts_utils.h:41
bool(* TSExecuteCallback)(void *arg, QueryOperand *val, ExecPhraseData *data)
Definition: ts_utils.h:154
void pushOperator(TSQueryParserState state, int8 oper, int16 distance)
Definition: tsquery.c:335
QTNode * QT2QTN(QueryItem *in, char *operand)
Definition: tsquery_util.c:24
char * word
Definition: ts_utils.h:206
bool QTNEq(QTNode *a, QTNode *b)
Definition: tsquery_util.c:182
QueryItem * clean_NOT(QueryItem *ptr, int32 *len)
struct TSVectorParseStateData * TSVectorParseState
Definition: ts_utils.h:26
uint32 sign
Definition: ts_utils.h:207
TSQuery parse_tsquery(char *buf, PushFunction pushval, Datum opaque, bool isplain)
Definition: tsquery.c:605
bool allocated
Definition: ts_utils.h:137
void QTNClearFlags(QTNode *in, uint32 flags)
Definition: tsquery_util.c:433
uint16 * apos
Definition: ts_utils.h:76
uint16 nvariant
Definition: ts_utils.h:66
void QTNTernary(QTNode *in)
Definition: tsquery_util.c:200
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1160
int32 lenwords
Definition: ts_utils.h:86
unsigned int Oid
Definition: postgres_ext.h:31
int32 curwords
Definition: ts_utils.h:87
void QTNBinary(QTNode *in)
Definition: tsquery_util.c:249
char bool
Definition: c.h:199
signed int int32
Definition: c.h:253
bool tsquery_requires_match(QueryItem *curitem)
Definition: tsvector_op.c:1865
TSVectorParseState init_tsvector_parser(char *input, bool oprisdelim, bool is_tsquery)
void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int32 buflen)
unsigned short uint16
Definition: c.h:264
TSVector make_tsvector(ParsedText *prs)
Definition: to_tsany.c:145
void pushStop(TSQueryParserState state)
Definition: tsquery.c:420
static char * buf
Definition: pg_test_fsync.c:65
uint64 TSQuerySign
Definition: ts_utils.h:216
WordEntryPos * pos
Definition: ts_utils.h:139
unsigned int uint32
Definition: c.h:265
ParsedWord * words
Definition: ts_utils.h:85
uint16 pos
Definition: ts_utils.h:69
TSQuery QTN2QT(QTNode *in)
Definition: tsquery_util.c:362
signed char int8
Definition: c.h:251
void parsetext(Oid cfgId, ParsedText *prs, char *buf, int32 buflen)
void reset_tsvector_parser(TSVectorParseState state, char *input)
uintptr_t Datum
Definition: postgres.h:374
bool TS_execute(QueryItem *curitem, void *arg, uint32 flags, TSExecuteCallback chkcond)
Definition: tsvector_op.c:1815
void(* PushFunction)(Datum opaque, TSQueryParserState state, char *token, int tokenlen, int16 tokenweights, bool prefix)
Definition: ts_utils.h:43
void QTNFree(QTNode *in)
Definition: tsquery_util.c:63
bool gettoken_tsvector(TSVectorParseState state, char **token, int *len, WordEntryPos **pos, int *poslen, char **endptr)
Definition: regguts.h:298
uint16 len
Definition: ts_utils.h:65
QTNode * findsubquery(QTNode *root, QTNode *ex, QTNode *subs, bool *isfind)
int32 pos
Definition: ts_utils.h:88
void QTNSort(QTNode *in)
Definition: tsquery_util.c:162
struct QTNode ** child
Definition: ts_utils.h:208
struct ExecPhraseData ExecPhraseData
int QTNodeCompare(QTNode *an, QTNode *bn)
Definition: tsquery_util.c:96
uint16 flags
Definition: ts_utils.h:78
uint32 alen
Definition: ts_utils.h:80
QueryItem * valnode
Definition: ts_utils.h:203
text * generateHeadline(HeadlineParsedText *prs)
Definition: ts_parse.c:598
int32 nchild
Definition: ts_utils.h:205
struct QTNode QTNode
void * arg
Definition: c.h:435
void pushValue(TSQueryParserState state, char *strval, int lenval, int16 weight, bool prefix)
Definition: tsquery.c:384
Operator oper(ParseState *pstate, List *opname, Oid ltypeId, Oid rtypeId, bool noError, int location)
Definition: parse_oper.c:375
TSQuerySign makeTSQuerySign(TSQuery a)
Definition: tsquery_op.c:249
void close_tsvector_parser(TSVectorParseState state)
long val
Definition: informix.c:689
uint32 flags
Definition: ts_utils.h:204
char * word
Definition: ts_utils.h:79