PostgreSQL Source Code  git master
compression.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * compression.c
4  *
5  * Shared code for compression methods and specifications.
6  *
7  * A compression specification specifies the parameters that should be used
8  * when performing compression with a specific algorithm. The simplest
9  * possible compression specification is an integer, which sets the
10  * compression level.
11  *
12  * Otherwise, a compression specification is a comma-separated list of items,
13  * each having the form keyword or keyword=value.
14  *
15  * Currently, the only supported keywords are "level" and "workers".
16  *
17  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
18  *
19  * IDENTIFICATION
20  * src/common/compression.c
21  *-------------------------------------------------------------------------
22  */
23 
24 #ifndef FRONTEND
25 #include "postgres.h"
26 #else
27 #include "postgres_fe.h"
28 #endif
29 
30 #ifdef USE_ZSTD
31 #include <zstd.h>
32 #endif
33 #ifdef HAVE_LIBZ
34 #include <zlib.h>
35 #endif
36 
37 #include "common/compression.h"
38 
39 static int expect_integer_value(char *keyword, char *value,
41 
42 /*
43  * Look up a compression algorithm by name. Returns true and sets *algorithm
44  * if the name is recognized. Otherwise returns false.
45  */
46 bool
48 {
49  if (strcmp(name, "none") == 0)
50  *algorithm = PG_COMPRESSION_NONE;
51  else if (strcmp(name, "gzip") == 0)
52  *algorithm = PG_COMPRESSION_GZIP;
53  else if (strcmp(name, "lz4") == 0)
54  *algorithm = PG_COMPRESSION_LZ4;
55  else if (strcmp(name, "zstd") == 0)
56  *algorithm = PG_COMPRESSION_ZSTD;
57  else
58  return false;
59  return true;
60 }
61 
62 /*
63  * Get the human-readable name corresponding to a particular compression
64  * algorithm.
65  */
66 const char *
68 {
69  switch (algorithm)
70  {
72  return "none";
74  return "gzip";
75  case PG_COMPRESSION_LZ4:
76  return "lz4";
78  return "zstd";
79  /* no default, to provoke compiler warnings if values are added */
80  }
81  Assert(false);
82  return "???"; /* placate compiler */
83 }
84 
85 /*
86  * Parse a compression specification for a specified algorithm.
87  *
88  * See the file header comments for a brief description of what a compression
89  * specification is expected to look like.
90  *
91  * On return, all fields of the result object will be initialized.
92  * In particular, result->parse_error will be NULL if no errors occurred
93  * during parsing, and will otherwise contain an appropriate error message.
94  * The caller may free this error message string using pfree, if desired.
95  * Note, however, even if there's no parse error, the string might not make
96  * sense: e.g. for gzip, level=12 is not sensible, but it does parse OK.
97  *
98  * The compression level is assigned by default if not directly specified
99  * by the specification.
100  *
101  * Use validate_compress_specification() to find out whether a compression
102  * specification is semantically sensible.
103  */
104 void
107 {
108  int bare_level;
109  char *bare_level_endp;
110 
111  /* Initial setup of result object. */
112  result->algorithm = algorithm;
113  result->options = 0;
114  result->parse_error = NULL;
115 
116  /*
117  * Assign a default level depending on the compression method. This may
118  * be enforced later.
119  */
120  switch (result->algorithm)
121  {
122  case PG_COMPRESSION_NONE:
123  result->level = 0;
124  break;
125  case PG_COMPRESSION_LZ4:
126 #ifdef USE_LZ4
127  result->level = 0; /* fast compression mode */
128 #else
129  result->parse_error =
130  psprintf(_("this build does not support compression with %s"),
131  "LZ4");
132 #endif
133  break;
134  case PG_COMPRESSION_ZSTD:
135 #ifdef USE_ZSTD
136  result->level = ZSTD_CLEVEL_DEFAULT;
137 #else
138  result->parse_error =
139  psprintf(_("this build does not support compression with %s"),
140  "ZSTD");
141 #endif
142  break;
143  case PG_COMPRESSION_GZIP:
144 #ifdef HAVE_LIBZ
145  result->level = Z_DEFAULT_COMPRESSION;
146 #else
147  result->parse_error =
148  psprintf(_("this build does not support compression with %s"),
149  "gzip");
150 #endif
151  break;
152  }
153 
154  /* If there is no specification, we're done already. */
155  if (specification == NULL)
156  return;
157 
158  /* As a special case, the specification can be a bare integer. */
159  bare_level = strtol(specification, &bare_level_endp, 10);
160  if (specification != bare_level_endp && *bare_level_endp == '\0')
161  {
162  result->level = bare_level;
163  return;
164  }
165 
166  /* Look for comma-separated keyword or keyword=value entries. */
167  while (1)
168  {
169  char *kwstart;
170  char *kwend;
171  char *vstart;
172  char *vend;
173  int kwlen;
174  int vlen;
175  bool has_value;
176  char *keyword;
177  char *value;
178 
179  /* Figure start, end, and length of next keyword and any value. */
180  kwstart = kwend = specification;
181  while (*kwend != '\0' && *kwend != ',' && *kwend != '=')
182  ++kwend;
183  kwlen = kwend - kwstart;
184  if (*kwend != '=')
185  {
186  vstart = vend = NULL;
187  vlen = 0;
188  has_value = false;
189  }
190  else
191  {
192  vstart = vend = kwend + 1;
193  while (*vend != '\0' && *vend != ',')
194  ++vend;
195  vlen = vend - vstart;
196  has_value = true;
197  }
198 
199  /* Reject empty keyword. */
200  if (kwlen == 0)
201  {
202  result->parse_error =
203  pstrdup(_("found empty string where a compression option was expected"));
204  break;
205  }
206 
207  /* Extract keyword and value as separate C strings. */
208  keyword = palloc(kwlen + 1);
209  memcpy(keyword, kwstart, kwlen);
210  keyword[kwlen] = '\0';
211  if (!has_value)
212  value = NULL;
213  else
214  {
215  value = palloc(vlen + 1);
216  memcpy(value, vstart, vlen);
217  value[vlen] = '\0';
218  }
219 
220  /* Handle whatever keyword we found. */
221  if (strcmp(keyword, "level") == 0)
222  {
223  result->level = expect_integer_value(keyword, value, result);
224 
225  /*
226  * No need to set a flag in "options", there is a default level
227  * set at least thanks to the logic above.
228  */
229  }
230  else if (strcmp(keyword, "workers") == 0)
231  {
232  result->workers = expect_integer_value(keyword, value, result);
234  }
235  else
236  result->parse_error =
237  psprintf(_("unrecognized compression option: \"%s\""), keyword);
238 
239  /* Release memory, just to be tidy. */
240  pfree(keyword);
241  if (value != NULL)
242  pfree(value);
243 
244  /*
245  * If we got an error or have reached the end of the string, stop.
246  *
247  * If there is no value, then the end of the keyword might have been
248  * the end of the string. If there is a value, then the end of the
249  * keyword cannot have been the end of the string, but the end of the
250  * value might have been.
251  */
252  if (result->parse_error != NULL ||
253  (vend == NULL ? *kwend == '\0' : *vend == '\0'))
254  break;
255 
256  /* Advance to next entry and loop around. */
257  specification = vend == NULL ? kwend + 1 : vend + 1;
258  }
259 }
260 
261 /*
262  * Parse 'value' as an integer and return the result.
263  *
264  * If parsing fails, set result->parse_error to an appropriate message
265  * and return -1.
266  */
267 static int
269 {
270  int ivalue;
271  char *ivalue_endp;
272 
273  if (value == NULL)
274  {
275  result->parse_error =
276  psprintf(_("compression option \"%s\" requires a value"),
277  keyword);
278  return -1;
279  }
280 
281  ivalue = strtol(value, &ivalue_endp, 10);
282  if (ivalue_endp == value || *ivalue_endp != '\0')
283  {
284  result->parse_error =
285  psprintf(_("value for compression option \"%s\" must be an integer"),
286  keyword);
287  return -1;
288  }
289  return ivalue;
290 }
291 
292 /*
293  * Returns NULL if the compression specification string was syntactically
294  * valid and semantically sensible. Otherwise, returns an error message.
295  *
296  * Does not test whether this build of PostgreSQL supports the requested
297  * compression method.
298  */
299 char *
301 {
302  int min_level = 1;
303  int max_level = 1;
304  int default_level = 0;
305 
306  /* If it didn't even parse OK, it's definitely no good. */
307  if (spec->parse_error != NULL)
308  return spec->parse_error;
309 
310  /*
311  * Check that the algorithm expects a compression level and it is within
312  * the legal range for the algorithm.
313  */
314  switch (spec->algorithm)
315  {
316  case PG_COMPRESSION_GZIP:
317  max_level = 9;
318 #ifdef HAVE_LIBZ
319  default_level = Z_DEFAULT_COMPRESSION;
320 #endif
321  break;
322  case PG_COMPRESSION_LZ4:
323  max_level = 12;
324  default_level = 0; /* fast mode */
325  break;
326  case PG_COMPRESSION_ZSTD:
327 #ifdef USE_ZSTD
328  max_level = ZSTD_maxCLevel();
329  min_level = ZSTD_minCLevel();
330  default_level = ZSTD_CLEVEL_DEFAULT;
331 #endif
332  break;
333  case PG_COMPRESSION_NONE:
334  if (spec->level != 0)
335  return psprintf(_("compression algorithm \"%s\" does not accept a compression level"),
337  break;
338  }
339 
340  if ((spec->level < min_level || spec->level > max_level) &&
341  spec->level != default_level)
342  return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"),
344  min_level, max_level, default_level);
345 
346  /*
347  * Of the compression algorithms that we currently support, only zstd
348  * allows parallel workers.
349  */
350  if ((spec->options & PG_COMPRESSION_OPTION_WORKERS) != 0 &&
351  (spec->algorithm != PG_COMPRESSION_ZSTD))
352  {
353  return psprintf(_("compression algorithm \"%s\" does not accept a worker count"),
355  }
356 
357  return NULL;
358 }
359 
360 #ifdef FRONTEND
361 
362 /*
363  * Basic parsing of a value specified through a command-line option, commonly
364  * -Z/--compress.
365  *
366  * The parsing consists of a METHOD:DETAIL string fed later to
367  * parse_compress_specification(). This only extracts METHOD and DETAIL.
368  * If only an integer is found, the method is implied by the value specified.
369  */
370 void
371 parse_compress_options(const char *option, char **algorithm, char **detail)
372 {
373  char *sep;
374  char *endp;
375  long result;
376 
377  /*
378  * Check whether the compression specification consists of a bare integer.
379  *
380  * For backward-compatibility, assume "none" if the integer found is zero
381  * and "gzip" otherwise.
382  */
383  result = strtol(option, &endp, 10);
384  if (*endp == '\0')
385  {
386  if (result == 0)
387  {
388  *algorithm = pstrdup("none");
389  *detail = NULL;
390  }
391  else
392  {
393  *algorithm = pstrdup("gzip");
394  *detail = pstrdup(option);
395  }
396  return;
397  }
398 
399  /*
400  * Check whether there is a compression detail following the algorithm
401  * name.
402  */
403  sep = strchr(option, ':');
404  if (sep == NULL)
405  {
406  *algorithm = pstrdup(option);
407  *detail = NULL;
408  }
409  else
410  {
411  char *alg;
412 
413  alg = palloc((sep - option) + 1);
414  memcpy(alg, option, sep - option);
415  alg[sep - option] = '\0';
416 
417  *algorithm = alg;
418  *detail = pstrdup(sep + 1);
419  }
420 }
421 #endif /* FRONTEND */
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:47
const char * get_compress_algorithm_name(pg_compress_algorithm algorithm)
Definition: compression.c:67
static int expect_integer_value(char *keyword, char *value, pg_compress_specification *result)
Definition: compression.c:268
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:105
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:300
#define PG_COMPRESSION_OPTION_WORKERS
Definition: compression.h:25
pg_compress_algorithm
Definition: compression.h:18
@ PG_COMPRESSION_GZIP
Definition: compression.h:20
@ PG_COMPRESSION_LZ4
Definition: compression.h:21
@ PG_COMPRESSION_NONE
Definition: compression.h:19
@ PG_COMPRESSION_ZSTD
Definition: compression.h:22
void parse_compress_options(const char *option, char **algorithm, char **detail)
#define _(x)
Definition: elog.c:91
const char * name
Definition: encode.c:571
static struct @143 value
Assert(fmt[strlen(fmt) - 1] !='\n')
char * pstrdup(const char *in)
Definition: mcxt.c:1624
void pfree(void *pointer)
Definition: mcxt.c:1436
void * palloc(Size size)
Definition: mcxt.c:1210
#define Z_DEFAULT_COMPRESSION
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
pg_compress_algorithm algorithm
Definition: compression.h:29