PostgreSQL Source Code git master
•All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Data Structures | Macros | Typedefs | Functions | Variables
dict_snowball.c File Reference
#include "postgres.h"
#include "catalog/pg_collation_d.h"
#include "commands/defrem.h"
#include "mb/pg_wchar.h"
#include "tsearch/ts_public.h"
#include "utils/formatting.h"
#include "snowball/libstemmer/header.h"
#include "snowball/libstemmer/stem_ISO_8859_1_basque.h"
#include "snowball/libstemmer/stem_ISO_8859_1_catalan.h"
#include "snowball/libstemmer/stem_ISO_8859_1_danish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_dutch.h"
#include "snowball/libstemmer/stem_ISO_8859_1_english.h"
#include "snowball/libstemmer/stem_ISO_8859_1_finnish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_french.h"
#include "snowball/libstemmer/stem_ISO_8859_1_german.h"
#include "snowball/libstemmer/stem_ISO_8859_1_indonesian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_irish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_italian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_norwegian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_porter.h"
#include "snowball/libstemmer/stem_ISO_8859_1_portuguese.h"
#include "snowball/libstemmer/stem_ISO_8859_1_spanish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_swedish.h"
#include "snowball/libstemmer/stem_ISO_8859_2_hungarian.h"
#include "snowball/libstemmer/stem_KOI8_R_russian.h"
#include "snowball/libstemmer/stem_UTF_8_arabic.h"
#include "snowball/libstemmer/stem_UTF_8_armenian.h"
#include "snowball/libstemmer/stem_UTF_8_basque.h"
#include "snowball/libstemmer/stem_UTF_8_catalan.h"
#include "snowball/libstemmer/stem_UTF_8_danish.h"
#include "snowball/libstemmer/stem_UTF_8_dutch.h"
#include "snowball/libstemmer/stem_UTF_8_english.h"
#include "snowball/libstemmer/stem_UTF_8_estonian.h"
#include "snowball/libstemmer/stem_UTF_8_finnish.h"
#include "snowball/libstemmer/stem_UTF_8_french.h"
#include "snowball/libstemmer/stem_UTF_8_german.h"
#include "snowball/libstemmer/stem_UTF_8_greek.h"
#include "snowball/libstemmer/stem_UTF_8_hindi.h"
#include "snowball/libstemmer/stem_UTF_8_hungarian.h"
#include "snowball/libstemmer/stem_UTF_8_indonesian.h"
#include "snowball/libstemmer/stem_UTF_8_irish.h"
#include "snowball/libstemmer/stem_UTF_8_italian.h"
#include "snowball/libstemmer/stem_UTF_8_lithuanian.h"
#include "snowball/libstemmer/stem_UTF_8_nepali.h"
#include "snowball/libstemmer/stem_UTF_8_norwegian.h"
#include "snowball/libstemmer/stem_UTF_8_porter.h"
#include "snowball/libstemmer/stem_UTF_8_portuguese.h"
#include "snowball/libstemmer/stem_UTF_8_romanian.h"
#include "snowball/libstemmer/stem_UTF_8_russian.h"
#include "snowball/libstemmer/stem_UTF_8_serbian.h"
#include "snowball/libstemmer/stem_UTF_8_spanish.h"
#include "snowball/libstemmer/stem_UTF_8_swedish.h"
#include "snowball/libstemmer/stem_UTF_8_tamil.h"
#include "snowball/libstemmer/stem_UTF_8_turkish.h"
#include "snowball/libstemmer/stem_UTF_8_yiddish.h"
Include dependency graph for dict_snowball.c:

Go to the source code of this file.

Data Structures

struct  stemmer_module
 
struct  DictSnowball
 

Macros

#define STEMMER_MODULE(name, enc, senc)    {#name, enc, name##_##senc##_create_env, name##_##senc##_close_env, name##_##senc##_stem}
 

Typedefs

typedef struct stemmer_module stemmer_module
 
typedef struct DictSnowball DictSnowball
 

Functions

 PG_MODULE_MAGIC_EXT (.name="dict_snowball",.version=PG_VERSION)
 
 PG_FUNCTION_INFO_V1 (dsnowball_init)
 
 PG_FUNCTION_INFO_V1 (dsnowball_lexize)
 
static void locate_stem_module (DictSnowball *d, const char *lang)
 
Datum dsnowball_init (PG_FUNCTION_ARGS)
 
Datum dsnowball_lexize (PG_FUNCTION_ARGS)
 

Variables

static const stemmer_module stemmer_modules []
 

Macro Definition Documentation

◆ STEMMER_MODULE

#define STEMMER_MODULE (   name,
  enc,
  senc 
)     {#name, enc, name##_##senc##_create_env, name##_##senc##_close_env, name##_##senc##_stem}

Definition at line 100 of file dict_snowball.c.

Typedef Documentation

◆ DictSnowball

typedef struct DictSnowball DictSnowball

◆ stemmer_module

typedef struct stemmer_module stemmer_module

Function Documentation

◆ dsnowball_init()

Datum dsnowball_init ( PG_FUNCTION_ARGS  )

Definition at line 225 of file dict_snowball.c.

226{
227 List *dictoptions = (List *) PG_GETARG_POINTER(0);
228 DictSnowball *d;
229 bool stoploaded = false;
230 ListCell *l;
231
232 d = (DictSnowball *) palloc0(sizeof(DictSnowball));
233
234 foreach(l, dictoptions)
235 {
236 DefElem *defel = (DefElem *) lfirst(l);
237
238 if (strcmp(defel->defname, "stopwords") == 0)
239 {
240 if (stoploaded)
241 ereport(ERROR,
242 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
243 errmsg("multiple StopWords parameters")));
244 readstoplist(defGetString(defel), &d->stoplist, str_tolower);
245 stoploaded = true;
246 }
247 else if (strcmp(defel->defname, "language") == 0)
248 {
249 if (d->stem)
250 ereport(ERROR,
251 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
252 errmsg("multiple Language parameters")));
253 locate_stem_module(d, defGetString(defel));
254 }
255 else
256 {
257 ereport(ERROR,
258 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
259 errmsg("unrecognized Snowball parameter: \"%s\"",
260 defel->defname)));
261 }
262 }
263
264 if (!d->stem)
265 ereport(ERROR,
266 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
267 errmsg("missing Language parameter")));
268
269 d->dictCtx = CurrentMemoryContext;
270
271 PG_RETURN_POINTER(d);
272}
defGetString
char * defGetString(DefElem *def)
Definition: define.c:35
locate_stem_module
static void locate_stem_module(DictSnowball *d, const char *lang)
Definition: dict_snowball.c:184
errcode
int errcode(int sqlerrcode)
Definition: elog.c:863
errmsg
int errmsg(const char *fmt,...)
Definition: elog.c:1080
ERROR
#define ERROR
Definition: elog.h:39
ereport
#define ereport(elevel,...)
Definition: elog.h:150
PG_GETARG_POINTER
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:276
PG_RETURN_POINTER
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:361
str_tolower
char * str_tolower(const char *buff, size_t nbytes, Oid collid)
Definition: formatting.c:1636
palloc0
void * palloc0(Size size)
Definition: mcxt.c:1395
CurrentMemoryContext
MemoryContext CurrentMemoryContext
Definition: mcxt.c:160
lfirst
#define lfirst(lc)
Definition: pg_list.h:172
DefElem::defname
char * defname
Definition: parsenodes.h:843
DictSnowball::dictCtx
MemoryContext dictCtx
Definition: dict_snowball.c:179
DictSnowball::stoplist
StopList stoplist
Definition: dict_snowball.c:170
DictSnowball::stem
int(* stem)(struct SN_env *z)
Definition: dict_snowball.c:172
List
Definition: pg_list.h:54
readstoplist
void readstoplist(const char *fname, StopList *s, char *(*wordop)(const char *, size_t, Oid))
Definition: ts_utils.c:69

References CurrentMemoryContext, defGetString(), DefElem::defname, DictSnowball::dictCtx, ereport, errcode(), errmsg(), ERROR, SN_env::l, lfirst, locate_stem_module(), palloc0(), PG_GETARG_POINTER, PG_RETURN_POINTER, readstoplist(), DictSnowball::stem, DictSnowball::stoplist, and str_tolower().

◆ dsnowball_lexize()

Datum dsnowball_lexize ( PG_FUNCTION_ARGS  )

Definition at line 275 of file dict_snowball.c.

276{
277 DictSnowball *d = (DictSnowball *) PG_GETARG_POINTER(0);
278 char *in = (char *) PG_GETARG_POINTER(1);
279 int32 len = PG_GETARG_INT32(2);
280 char *txt = str_tolower(in, len, DEFAULT_COLLATION_OID);
281 TSLexeme *res = palloc0(sizeof(TSLexeme) * 2);
282
283 /*
284 * Do not pass strings exceeding 1000 bytes to the stemmer, as they're
285 * surely not words in any human language. This restriction avoids
286 * wasting cycles on stuff like base64-encoded data, and it protects us
287 * against possible inefficiency or misbehavior in the stemmer. (For
288 * example, the Turkish stemmer has an indefinite recursion, so it can
289 * crash on long-enough strings.) However, Snowball dictionaries are
290 * defined to recognize all strings, so we can't reject the string as an
291 * unknown word.
292 */
293 if (len > 1000)
294 {
295 /* return the lexeme lowercased, but otherwise unmodified */
296 res->lexeme = txt;
297 }
298 else if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
299 {
300 /* empty or stopword, so report as stopword */
301 pfree(txt);
302 }
303 else
304 {
305 MemoryContext saveCtx;
306
307 /*
308 * recode to utf8 if stemmer is utf8 and doesn't match server encoding
309 */
310 if (d->needrecode)
311 {
312 char *recoded;
313
314 recoded = pg_server_to_any(txt, strlen(txt), PG_UTF8);
315 if (recoded != txt)
316 {
317 pfree(txt);
318 txt = recoded;
319 }
320 }
321
322 /* see comment about d->dictCtx */
323 saveCtx = MemoryContextSwitchTo(d->dictCtx);
324 SN_set_current(d->z, strlen(txt), (symbol *) txt);
325 d->stem(d->z);
326 MemoryContextSwitchTo(saveCtx);
327
328 if (d->z->p && d->z->l)
329 {
330 txt = repalloc(txt, d->z->l + 1);
331 memcpy(txt, d->z->p, d->z->l);
332 txt[d->z->l] = '\0';
333 }
334
335 /* back recode if needed */
336 if (d->needrecode)
337 {
338 char *recoded;
339
340 recoded = pg_any_to_server(txt, strlen(txt), PG_UTF8);
341 if (recoded != txt)
342 {
343 pfree(txt);
344 txt = recoded;
345 }
346 }
347
348 res->lexeme = txt;
349 }
350
351 PG_RETURN_POINTER(res);
352}
SN_set_current
int SN_set_current(struct SN_env *z, int size, const symbol *s)
Definition: api.c:51
symbol
unsigned char symbol
Definition: api.h:2
int32
int32_t int32
Definition: c.h:538
PG_GETARG_INT32
#define PG_GETARG_INT32(n)
Definition: fmgr.h:269
pg_any_to_server
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:677
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:750
repalloc
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1610
pfree
void pfree(void *pointer)
Definition: mcxt.c:1594
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:124
len
const void size_t len
Definition: pg_crc32c_sse42.c:28
PG_UTF8
@ PG_UTF8
Definition: pg_wchar.h:232
DictSnowball::z
struct SN_env * z
Definition: dict_snowball.c:169
SN_env::p
symbol * p
Definition: api.h:15
SN_env::l
int l
Definition: api.h:16
TSLexeme::lexeme
char * lexeme
Definition: ts_public.h:138
searchstoplist
bool searchstoplist(StopList *s, char *key)
Definition: ts_utils.c:141

References DictSnowball::dictCtx, SN_env::l, len, TSLexeme::lexeme, MemoryContextSwitchTo(), DictSnowball::needrecode, SN_env::p, palloc0(), pfree(), pg_any_to_server(), PG_GETARG_INT32, PG_GETARG_POINTER, PG_RETURN_POINTER, pg_server_to_any(), PG_UTF8, repalloc(), searchstoplist(), SN_set_current(), DictSnowball::stem, DictSnowball::stoplist, str_tolower(), and DictSnowball::z.

◆ locate_stem_module()

static void locate_stem_module ( DictSnowball *  d,
const char *  lang 
)
static

Definition at line 184 of file dict_snowball.c.

185{
186 const stemmer_module *m;
187
188 /*
189 * First, try to find exact match of stemmer module. Stemmer with
190 * PG_SQL_ASCII encoding is treated as working with any server encoding
191 */
192 for (m = stemmer_modules; m->name; m++)
193 {
194 if ((m->enc == PG_SQL_ASCII || m->enc == GetDatabaseEncoding()) &&
195 pg_strcasecmp(m->name, lang) == 0)
196 {
197 d->stem = m->stem;
198 d->z = m->create();
199 d->needrecode = false;
200 return;
201 }
202 }
203
204 /*
205 * Second, try to find stemmer for needed language for UTF8 encoding.
206 */
207 for (m = stemmer_modules; m->name; m++)
208 {
209 if (m->enc == PG_UTF8 && pg_strcasecmp(m->name, lang) == 0)
210 {
211 d->stem = m->stem;
212 d->z = m->create();
213 d->needrecode = true;
214 return;
215 }
216 }
217
218 ereport(ERROR,
219 (errcode(ERRCODE_UNDEFINED_OBJECT),
220 errmsg("no Snowball stemmer available for language \"%s\" and encoding \"%s\"",
221 lang, GetDatabaseEncodingName())));
222}
stemmer_modules
static const stemmer_module stemmer_modules[]
Definition: dict_snowball.c:103
GetDatabaseEncoding
int GetDatabaseEncoding(void)
Definition: mbutils.c:1262
GetDatabaseEncodingName
const char * GetDatabaseEncodingName(void)
Definition: mbutils.c:1268
PG_SQL_ASCII
@ PG_SQL_ASCII
Definition: pg_wchar.h:226
pg_strcasecmp
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
stemmer_module
Definition: dict_snowball.c:91
stemmer_module::create
struct SN_env *(* create)(void)
Definition: dict_snowball.c:94
stemmer_module::name
const char * name
Definition: dict_snowball.c:92
stemmer_module::enc
pg_enc enc
Definition: dict_snowball.c:93
stemmer_module::stem
int(* stem)(struct SN_env *)
Definition: dict_snowball.c:96

References stemmer_module::create, stemmer_module::enc, ereport, errcode(), errmsg(), ERROR, GetDatabaseEncoding(), GetDatabaseEncodingName(), stemmer_module::name, DictSnowball::needrecode, PG_SQL_ASCII, pg_strcasecmp(), PG_UTF8, stemmer_module::stem, DictSnowball::stem, stemmer_modules, and DictSnowball::z.

Referenced by dsnowball_init().

◆ PG_FUNCTION_INFO_V1() [1/2]

PG_FUNCTION_INFO_V1 ( dsnowball_init  )

◆ PG_FUNCTION_INFO_V1() [2/2]

PG_FUNCTION_INFO_V1 ( dsnowball_lexize  )

◆ PG_MODULE_MAGIC_EXT()

PG_MODULE_MAGIC_EXT ( .  name = "dict_snowball",
.  version = PG_VERSION 
)

Variable Documentation

◆ stemmer_modules

const stemmer_module stemmer_modules[]
static

Definition at line 103 of file dict_snowball.c.

Referenced by locate_stem_module().