PostgreSQL Source Code  git master
dict_snowball.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * dict_snowball.c
4  * Snowball dictionary
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/snowball/dict_snowball.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include "commands/defrem.h"
16 #include "tsearch/ts_locale.h"
17 #include "tsearch/ts_utils.h"
18 
19 /* Some platforms define MAXINT and/or MININT, causing conflicts */
20 #ifdef MAXINT
21 #undef MAXINT
22 #endif
23 #ifdef MININT
24 #undef MININT
25 #endif
26 
27 /* Now we can include the original Snowball header.h */
74 
76 
78 
80 
81 /* List of supported modules */
82 typedef struct stemmer_module
83 {
84  const char *name;
86  struct SN_env *(*create) (void);
87  void (*close) (struct SN_env *);
88  int (*stem) (struct SN_env *);
90 
91 /* Args: stemmer name, PG code for encoding, Snowball's name for encoding */
92 #define STEMMER_MODULE(name,enc,senc) \
93  {#name, enc, name##_##senc##_create_env, name##_##senc##_close_env, name##_##senc##_stem}
94 
96 {
97  /*
98  * Stemmers list from Snowball distribution
99  */
100  STEMMER_MODULE(basque, PG_LATIN1, ISO_8859_1),
101  STEMMER_MODULE(catalan, PG_LATIN1, ISO_8859_1),
102  STEMMER_MODULE(danish, PG_LATIN1, ISO_8859_1),
103  STEMMER_MODULE(dutch, PG_LATIN1, ISO_8859_1),
104  STEMMER_MODULE(english, PG_LATIN1, ISO_8859_1),
105  STEMMER_MODULE(finnish, PG_LATIN1, ISO_8859_1),
106  STEMMER_MODULE(french, PG_LATIN1, ISO_8859_1),
107  STEMMER_MODULE(german, PG_LATIN1, ISO_8859_1),
108  STEMMER_MODULE(indonesian, PG_LATIN1, ISO_8859_1),
109  STEMMER_MODULE(irish, PG_LATIN1, ISO_8859_1),
110  STEMMER_MODULE(italian, PG_LATIN1, ISO_8859_1),
111  STEMMER_MODULE(norwegian, PG_LATIN1, ISO_8859_1),
112  STEMMER_MODULE(porter, PG_LATIN1, ISO_8859_1),
113  STEMMER_MODULE(portuguese, PG_LATIN1, ISO_8859_1),
114  STEMMER_MODULE(spanish, PG_LATIN1, ISO_8859_1),
115  STEMMER_MODULE(swedish, PG_LATIN1, ISO_8859_1),
116  STEMMER_MODULE(hungarian, PG_LATIN2, ISO_8859_2),
117  STEMMER_MODULE(romanian, PG_LATIN2, ISO_8859_2),
118  STEMMER_MODULE(russian, PG_KOI8R, KOI8_R),
119  STEMMER_MODULE(arabic, PG_UTF8, UTF_8),
120  STEMMER_MODULE(basque, PG_UTF8, UTF_8),
121  STEMMER_MODULE(catalan, PG_UTF8, UTF_8),
122  STEMMER_MODULE(danish, PG_UTF8, UTF_8),
123  STEMMER_MODULE(dutch, PG_UTF8, UTF_8),
124  STEMMER_MODULE(english, PG_UTF8, UTF_8),
125  STEMMER_MODULE(finnish, PG_UTF8, UTF_8),
126  STEMMER_MODULE(french, PG_UTF8, UTF_8),
127  STEMMER_MODULE(german, PG_UTF8, UTF_8),
128  STEMMER_MODULE(greek, PG_UTF8, UTF_8),
129  STEMMER_MODULE(hindi, PG_UTF8, UTF_8),
130  STEMMER_MODULE(hungarian, PG_UTF8, UTF_8),
131  STEMMER_MODULE(indonesian, PG_UTF8, UTF_8),
132  STEMMER_MODULE(irish, PG_UTF8, UTF_8),
133  STEMMER_MODULE(italian, PG_UTF8, UTF_8),
134  STEMMER_MODULE(lithuanian, PG_UTF8, UTF_8),
135  STEMMER_MODULE(nepali, PG_UTF8, UTF_8),
136  STEMMER_MODULE(norwegian, PG_UTF8, UTF_8),
137  STEMMER_MODULE(porter, PG_UTF8, UTF_8),
138  STEMMER_MODULE(portuguese, PG_UTF8, UTF_8),
139  STEMMER_MODULE(romanian, PG_UTF8, UTF_8),
140  STEMMER_MODULE(russian, PG_UTF8, UTF_8),
141  STEMMER_MODULE(spanish, PG_UTF8, UTF_8),
142  STEMMER_MODULE(swedish, PG_UTF8, UTF_8),
143  STEMMER_MODULE(tamil, PG_UTF8, UTF_8),
144  STEMMER_MODULE(turkish, PG_UTF8, UTF_8),
145 
146  /*
147  * Stemmer with PG_SQL_ASCII encoding should be valid for any server
148  * encoding
149  */
150  STEMMER_MODULE(english, PG_SQL_ASCII, ISO_8859_1),
151 
152  {NULL, 0, NULL, NULL, NULL} /* list end marker */
153 };
154 
155 
156 typedef struct DictSnowball
157 {
158  struct SN_env *z;
160  bool needrecode; /* needs recoding before/after call stem */
161  int (*stem) (struct SN_env *z);
162 
163  /*
164  * snowball saves alloced memory between calls, so we should run it in our
165  * private memory context. Note, init function is executed in long lived
166  * context, so we just remember CurrentMemoryContext
167  */
169 } DictSnowball;
170 
171 
172 static void
173 locate_stem_module(DictSnowball *d, const char *lang)
174 {
175  const stemmer_module *m;
176 
177  /*
178  * First, try to find exact match of stemmer module. Stemmer with
179  * PG_SQL_ASCII encoding is treated as working with any server encoding
180  */
181  for (m = stemmer_modules; m->name; m++)
182  {
183  if ((m->enc == PG_SQL_ASCII || m->enc == GetDatabaseEncoding()) &&
184  pg_strcasecmp(m->name, lang) == 0)
185  {
186  d->stem = m->stem;
187  d->z = m->create();
188  d->needrecode = false;
189  return;
190  }
191  }
192 
193  /*
194  * Second, try to find stemmer for needed language for UTF8 encoding.
195  */
196  for (m = stemmer_modules; m->name; m++)
197  {
198  if (m->enc == PG_UTF8 && pg_strcasecmp(m->name, lang) == 0)
199  {
200  d->stem = m->stem;
201  d->z = m->create();
202  d->needrecode = true;
203  return;
204  }
205  }
206 
207  ereport(ERROR,
208  (errcode(ERRCODE_UNDEFINED_OBJECT),
209  errmsg("no Snowball stemmer available for language \"%s\" and encoding \"%s\"",
210  lang, GetDatabaseEncodingName())));
211 }
212 
213 Datum
215 {
216  List *dictoptions = (List *) PG_GETARG_POINTER(0);
217  DictSnowball *d;
218  bool stoploaded = false;
219  ListCell *l;
220 
221  d = (DictSnowball *) palloc0(sizeof(DictSnowball));
222 
223  foreach(l, dictoptions)
224  {
225  DefElem *defel = (DefElem *) lfirst(l);
226 
227  if (strcmp(defel->defname, "stopwords") == 0)
228  {
229  if (stoploaded)
230  ereport(ERROR,
231  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
232  errmsg("multiple StopWords parameters")));
234  stoploaded = true;
235  }
236  else if (strcmp(defel->defname, "language") == 0)
237  {
238  if (d->stem)
239  ereport(ERROR,
240  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
241  errmsg("multiple Language parameters")));
242  locate_stem_module(d, defGetString(defel));
243  }
244  else
245  {
246  ereport(ERROR,
247  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
248  errmsg("unrecognized Snowball parameter: \"%s\"",
249  defel->defname)));
250  }
251  }
252 
253  if (!d->stem)
254  ereport(ERROR,
255  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
256  errmsg("missing Language parameter")));
257 
259 
261 }
262 
263 Datum
265 {
267  char *in = (char *) PG_GETARG_POINTER(1);
268  int32 len = PG_GETARG_INT32(2);
269  char *txt = lowerstr_with_len(in, len);
270  TSLexeme *res = palloc0(sizeof(TSLexeme) * 2);
271 
272  if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
273  {
274  pfree(txt);
275  }
276  else
277  {
278  MemoryContext saveCtx;
279 
280  /*
281  * recode to utf8 if stemmer is utf8 and doesn't match server encoding
282  */
283  if (d->needrecode)
284  {
285  char *recoded;
286 
287  recoded = pg_server_to_any(txt, strlen(txt), PG_UTF8);
288  if (recoded != txt)
289  {
290  pfree(txt);
291  txt = recoded;
292  }
293  }
294 
295  /* see comment about d->dictCtx */
296  saveCtx = MemoryContextSwitchTo(d->dictCtx);
297  SN_set_current(d->z, strlen(txt), (symbol *) txt);
298  d->stem(d->z);
299  MemoryContextSwitchTo(saveCtx);
300 
301  if (d->z->p && d->z->l)
302  {
303  txt = repalloc(txt, d->z->l + 1);
304  memcpy(txt, d->z->p, d->z->l);
305  txt[d->z->l] = '\0';
306  }
307 
308  /* back recode if needed */
309  if (d->needrecode)
310  {
311  char *recoded;
312 
313  recoded = pg_any_to_server(txt, strlen(txt), PG_UTF8);
314  if (recoded != txt)
315  {
316  pfree(txt);
317  txt = recoded;
318  }
319  }
320 
321  res->lexeme = txt;
322  }
323 
324  PG_RETURN_POINTER(res);
325 }
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:360
#define PG_GETARG_INT32(n)
Definition: fmgr.h:269
StopList stoplist
symbol * p
Definition: api.h:15
#define STEMMER_MODULE(name, enc, senc)
Definition: dict_snowball.c:92
int SN_set_current(struct SN_env *z, int size, const symbol *s)
Definition: api.c:58
char * lowerstr_with_len(const char *str, int len)
Definition: ts_locale.c:257
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
int errcode(int sqlerrcode)
Definition: elog.c:610
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:276
char * lowerstr(const char *str)
Definition: ts_locale.c:244
struct DictSnowball DictSnowball
int(* stem)(struct SN_env *z)
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:692
signed int int32
Definition: c.h:362
void pfree(void *pointer)
Definition: mcxt.c:1057
#define ERROR
Definition: elog.h:43
char * defGetString(DefElem *def)
Definition: define.c:49
char * lexeme
Definition: ts_public.h:111
struct SN_env *(* create)(void)
Definition: dict_snowball.c:86
MemoryContext CurrentMemoryContext
Definition: mcxt.c:38
PG_FUNCTION_INFO_V1(dsnowball_init)
Datum dsnowball_lexize(PG_FUNCTION_ARGS)
void readstoplist(const char *fname, StopList *s, char *(*wordop)(const char *))
Definition: ts_utils.c:68
void * palloc0(Size size)
Definition: mcxt.c:981
uintptr_t Datum
Definition: postgres.h:367
int GetDatabaseEncoding(void)
Definition: mbutils.c:1151
#define ereport(elevel,...)
Definition: elog.h:144
void(* close)(struct SN_env *)
Definition: dict_snowball.c:87
static void locate_stem_module(DictSnowball *d, const char *lang)
#define lfirst(lc)
Definition: pg_list.h:190
const char * GetDatabaseEncodingName(void)
Definition: mbutils.c:1157
static const stemmer_module stemmer_modules[]
Definition: dict_snowball.c:95
struct stemmer_module stemmer_module
pg_enc
Definition: pg_wchar.h:238
int(* stem)(struct SN_env *)
Definition: dict_snowball.c:88
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1070
struct SN_env * z
bool searchstoplist(StopList *s, char *key)
Definition: ts_utils.c:141
Datum dsnowball_init(PG_FUNCTION_ARGS)
int errmsg(const char *fmt,...)
Definition: elog.c:824
Definition: api.h:14
PG_MODULE_MAGIC
Definition: dict_snowball.c:75
MemoryContext dictCtx
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
char * defname
Definition: parsenodes.h:733
int l
Definition: api.h:16
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:619
const char * name
Definition: dict_snowball.c:84
Definition: pg_list.h:50
unsigned char symbol
Definition: api.h:2