PostgreSQL Source Code  git master
pgbench.c
Go to the documentation of this file.
1 /*
2  * pgbench.c
3  *
4  * A simple benchmark program for PostgreSQL
5  * Originally written by Tatsuo Ishii and enhanced by many contributors.
6  *
7  * src/bin/pgbench/pgbench.c
8  * Copyright (c) 2000-2024, PostgreSQL Global Development Group
9  * ALL RIGHTS RESERVED;
10  *
11  * Permission to use, copy, modify, and distribute this software and its
12  * documentation for any purpose, without fee, and without a written agreement
13  * is hereby granted, provided that the above copyright notice and this
14  * paragraph and the following two paragraphs appear in all copies.
15  *
16  * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
17  * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
18  * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
19  * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
20  * POSSIBILITY OF SUCH DAMAGE.
21  *
22  * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIMS ANY WARRANTIES,
23  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
24  * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
25  * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
26  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
27  *
28  */
29 
30 #if defined(WIN32) && FD_SETSIZE < 1024
31 #error FD_SETSIZE needs to have been increased
32 #endif
33 
34 #include "postgres_fe.h"
35 
36 #include <ctype.h>
37 #include <float.h>
38 #include <limits.h>
39 #include <math.h>
40 #include <signal.h>
41 #include <time.h>
42 #include <sys/time.h>
43 #include <sys/resource.h> /* for getrlimit */
44 
45 /* For testing, PGBENCH_USE_SELECT can be defined to force use of that code */
46 #if defined(HAVE_PPOLL) && !defined(PGBENCH_USE_SELECT)
47 #define POLL_USING_PPOLL
48 #ifdef HAVE_POLL_H
49 #include <poll.h>
50 #endif
51 #else /* no ppoll(), so use select() */
52 #define POLL_USING_SELECT
53 #include <sys/select.h>
54 #endif
55 
56 #include "common/int.h"
57 #include "common/logging.h"
58 #include "common/pg_prng.h"
59 #include "common/string.h"
60 #include "common/username.h"
61 #include "fe_utils/cancel.h"
62 #include "fe_utils/conditional.h"
63 #include "fe_utils/option_utils.h"
64 #include "fe_utils/string_utils.h"
65 #include "getopt_long.h"
66 #include "libpq-fe.h"
67 #include "pgbench.h"
68 #include "port/pg_bitutils.h"
69 #include "portability/instr_time.h"
70 
71 /* X/Open (XSI) requires <math.h> to provide M_PI, but core POSIX does not */
72 #ifndef M_PI
73 #define M_PI 3.14159265358979323846
74 #endif
75 
76 #define ERRCODE_T_R_SERIALIZATION_FAILURE "40001"
77 #define ERRCODE_T_R_DEADLOCK_DETECTED "40P01"
78 #define ERRCODE_UNDEFINED_TABLE "42P01"
79 
80 /*
81  * Hashing constants
82  */
83 #define FNV_PRIME UINT64CONST(0x100000001b3)
84 #define FNV_OFFSET_BASIS UINT64CONST(0xcbf29ce484222325)
85 #define MM2_MUL UINT64CONST(0xc6a4a7935bd1e995)
86 #define MM2_MUL_TIMES_8 UINT64CONST(0x35253c9ade8f4ca8)
87 #define MM2_ROT 47
88 
89 /*
90  * Multi-platform socket set implementations
91  */
92 
93 #ifdef POLL_USING_PPOLL
94 #define SOCKET_WAIT_METHOD "ppoll"
95 
96 typedef struct socket_set
97 {
98  int maxfds; /* allocated length of pollfds[] array */
99  int curfds; /* number currently in use */
100  struct pollfd pollfds[FLEXIBLE_ARRAY_MEMBER];
101 } socket_set;
102 
103 #endif /* POLL_USING_PPOLL */
104 
105 #ifdef POLL_USING_SELECT
106 #define SOCKET_WAIT_METHOD "select"
107 
108 typedef struct socket_set
109 {
110  int maxfd; /* largest FD currently set in fds */
111  fd_set fds;
113 
114 #endif /* POLL_USING_SELECT */
115 
116 /*
117  * Multi-platform thread implementations
118  */
119 
120 #ifdef WIN32
121 /* Use Windows threads */
122 #include <windows.h>
123 #define GETERRNO() (_dosmaperr(GetLastError()), errno)
124 #define THREAD_T HANDLE
125 #define THREAD_FUNC_RETURN_TYPE unsigned
126 #define THREAD_FUNC_RETURN return 0
127 #define THREAD_FUNC_CC __stdcall
128 #define THREAD_CREATE(handle, function, arg) \
129  ((*(handle) = (HANDLE) _beginthreadex(NULL, 0, (function), (arg), 0, NULL)) == 0 ? errno : 0)
130 #define THREAD_JOIN(handle) \
131  (WaitForSingleObject(handle, INFINITE) != WAIT_OBJECT_0 ? \
132  GETERRNO() : CloseHandle(handle) ? 0 : GETERRNO())
133 #define THREAD_BARRIER_T SYNCHRONIZATION_BARRIER
134 #define THREAD_BARRIER_INIT(barrier, n) \
135  (InitializeSynchronizationBarrier((barrier), (n), 0) ? 0 : GETERRNO())
136 #define THREAD_BARRIER_WAIT(barrier) \
137  EnterSynchronizationBarrier((barrier), \
138  SYNCHRONIZATION_BARRIER_FLAGS_BLOCK_ONLY)
139 #define THREAD_BARRIER_DESTROY(barrier)
140 #else
141 /* Use POSIX threads */
142 #include "port/pg_pthread.h"
143 #define THREAD_T pthread_t
144 #define THREAD_FUNC_RETURN_TYPE void *
145 #define THREAD_FUNC_RETURN return NULL
146 #define THREAD_FUNC_CC
147 #define THREAD_CREATE(handle, function, arg) \
148  pthread_create((handle), NULL, (function), (arg))
149 #define THREAD_JOIN(handle) \
150  pthread_join((handle), NULL)
151 #define THREAD_BARRIER_T pthread_barrier_t
152 #define THREAD_BARRIER_INIT(barrier, n) \
153  pthread_barrier_init((barrier), NULL, (n))
154 #define THREAD_BARRIER_WAIT(barrier) pthread_barrier_wait((barrier))
155 #define THREAD_BARRIER_DESTROY(barrier) pthread_barrier_destroy((barrier))
156 #endif
157 
158 
159 /********************************************************************
160  * some configurable parameters */
161 
162 #define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
163 #define ALL_INIT_STEPS "dtgGvpf" /* all possible steps */
164 
165 #define LOG_STEP_SECONDS 5 /* seconds between log messages */
166 #define DEFAULT_NXACTS 10 /* default nxacts */
167 
168 #define MIN_GAUSSIAN_PARAM 2.0 /* minimum parameter for gauss */
169 
170 #define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
171 #define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
172 
173 int nxacts = 0; /* number of transactions per client */
174 int duration = 0; /* duration in seconds */
175 int64 end_time = 0; /* when to stop in micro seconds, under -T */
176 
177 /*
178  * scaling factor. for example, scale = 10 will make 1000000 tuples in
179  * pgbench_accounts table.
180  */
181 int scale = 1;
182 
183 /*
184  * fillfactor. for example, fillfactor = 90 will use only 90 percent
185  * space during inserts and leave 10 percent free.
186  */
187 int fillfactor = 100;
188 
189 /*
190  * use unlogged tables?
191  */
192 bool unlogged_tables = false;
193 
194 /*
195  * log sampling rate (1.0 = log everything, 0.0 = option not given)
196  */
197 double sample_rate = 0.0;
198 
199 /*
200  * When threads are throttled to a given rate limit, this is the target delay
201  * to reach that rate in usec. 0 is the default and means no throttling.
202  */
203 double throttle_delay = 0;
204 
205 /*
206  * Transactions which take longer than this limit (in usec) are counted as
207  * late, and reported as such, although they are completed anyway. When
208  * throttling is enabled, execution time slots that are more than this late
209  * are skipped altogether, and counted separately.
210  */
211 int64 latency_limit = 0;
212 
213 /*
214  * tablespace selection
215  */
216 char *tablespace = NULL;
217 char *index_tablespace = NULL;
218 
219 /*
220  * Number of "pgbench_accounts" partitions. 0 is the default and means no
221  * partitioning.
222  */
223 static int partitions = 0;
224 
225 /* partitioning strategy for "pgbench_accounts" */
226 typedef enum
227 {
228  PART_NONE, /* no partitioning */
229  PART_RANGE, /* range partitioning */
230  PART_HASH, /* hash partitioning */
232 
234 static const char *const PARTITION_METHOD[] = {"none", "range", "hash"};
235 
236 /* random seed used to initialize base_random_sequence */
237 int64 random_seed = -1;
238 
239 /*
240  * end of configurable parameters
241  *********************************************************************/
242 
243 #define nbranches 1 /* Makes little sense to change this. Change
244  * -s instead */
245 #define ntellers 10
246 #define naccounts 100000
247 
248 /*
249  * The scale factor at/beyond which 32bit integers are incapable of storing
250  * 64bit values.
251  *
252  * Although the actual threshold is 21474, we use 20000 because it is easier to
253  * document and remember, and isn't that far away from the real threshold.
254  */
255 #define SCALE_32BIT_THRESHOLD 20000
256 
257 bool use_log; /* log transaction latencies to a file */
258 bool use_quiet; /* quiet logging onto stderr */
259 int agg_interval; /* log aggregates instead of individual
260  * transactions */
261 bool per_script_stats = false; /* whether to collect stats per script */
262 int progress = 0; /* thread progress report every this seconds */
263 bool progress_timestamp = false; /* progress report with Unix time */
264 int nclients = 1; /* number of clients */
265 int nthreads = 1; /* number of threads */
266 bool is_connect; /* establish connection for each transaction */
267 bool report_per_command = false; /* report per-command latencies,
268  * retries after errors and failures
269  * (errors without retrying) */
270 int main_pid; /* main process id used in log filename */
271 
272 /*
273  * There are different types of restrictions for deciding that the current
274  * transaction with a serialization/deadlock error can no longer be retried and
275  * should be reported as failed:
276  * - max_tries (--max-tries) can be used to limit the number of tries;
277  * - latency_limit (-L) can be used to limit the total time of tries;
278  * - duration (-T) can be used to limit the total benchmark time.
279  *
280  * They can be combined together, and you need to use at least one of them to
281  * retry the transactions with serialization/deadlock errors. If none of them is
282  * used, the default value of max_tries is 1 and such transactions will not be
283  * retried.
284  */
285 
286 /*
287  * We cannot retry a transaction after the serialization/deadlock error if its
288  * number of tries reaches this maximum; if its value is zero, it is not used.
289  */
290 uint32 max_tries = 1;
291 
292 bool failures_detailed = false; /* whether to group failures in
293  * reports or logs by basic types */
294 
295 const char *pghost = NULL;
296 const char *pgport = NULL;
297 const char *username = NULL;
298 const char *dbName = NULL;
299 char *logfile_prefix = NULL;
300 const char *progname;
301 
302 #define WSEP '@' /* weight separator */
303 
304 volatile sig_atomic_t timer_exceeded = false; /* flag from signal handler */
305 
306 /*
307  * We don't want to allocate variables one by one; for efficiency, add a
308  * constant margin each time it overflows.
309  */
310 #define VARIABLES_ALLOC_MARGIN 8
311 
312 /*
313  * Variable definitions.
314  *
315  * If a variable only has a string value, "svalue" is that value, and value is
316  * "not set". If the value is known, "value" contains the value (in any
317  * variant).
318  *
319  * In this case "svalue" contains the string equivalent of the value, if we've
320  * had occasion to compute that, or NULL if we haven't.
321  */
322 typedef struct
323 {
324  char *name; /* variable's name */
325  char *svalue; /* its value in string form, if known */
326  PgBenchValue value; /* actual variable's value */
327 } Variable;
328 
329 /*
330  * Data structure for client variables.
331  */
332 typedef struct
333 {
334  Variable *vars; /* array of variable definitions */
335  int nvars; /* number of variables */
336 
337  /*
338  * The maximum number of variables that we can currently store in 'vars'
339  * without having to reallocate more space. We must always have max_vars
340  * >= nvars.
341  */
342  int max_vars;
343 
344  bool vars_sorted; /* are variables sorted by name? */
345 } Variables;
346 
347 #define MAX_SCRIPTS 128 /* max number of SQL scripts allowed */
348 #define SHELL_COMMAND_SIZE 256 /* maximum size allowed for shell command */
349 
350 /*
351  * Simple data structure to keep stats about something.
352  *
353  * XXX probably the first value should be kept and used as an offset for
354  * better numerical stability...
355  */
356 typedef struct SimpleStats
357 {
358  int64 count; /* how many values were encountered */
359  double min; /* the minimum seen */
360  double max; /* the maximum seen */
361  double sum; /* sum of values */
362  double sum2; /* sum of squared values */
363 } SimpleStats;
364 
365 /*
366  * The instr_time type is expensive when dealing with time arithmetic. Define
367  * a type to hold microseconds instead. Type int64 is good enough for about
368  * 584500 years.
369  */
370 typedef int64 pg_time_usec_t;
371 
372 /*
373  * Data structure to hold various statistics: per-thread and per-script stats
374  * are maintained and merged together.
375  */
376 typedef struct StatsData
377 {
378  pg_time_usec_t start_time; /* interval start time, for aggregates */
379 
380  /*----------
381  * Transactions are counted depending on their execution and outcome.
382  * First a transaction may have started or not: skipped transactions occur
383  * under --rate and --latency-limit when the client is too late to execute
384  * them. Secondly, a started transaction may ultimately succeed or fail,
385  * possibly after some retries when --max-tries is not one. Thus
386  *
387  * the number of all transactions =
388  * 'skipped' (it was too late to execute them) +
389  * 'cnt' (the number of successful transactions) +
390  * 'failed' (the number of failed transactions).
391  *
392  * A successful transaction can have several unsuccessful tries before a
393  * successful run. Thus
394  *
395  * 'cnt' (the number of successful transactions) =
396  * successfully retried transactions (they got a serialization or a
397  * deadlock error(s), but were
398  * successfully retried from the very
399  * beginning) +
400  * directly successful transactions (they were successfully completed on
401  * the first try).
402  *
403  * A failed transaction is defined as unsuccessfully retried transactions.
404  * It can be one of two types:
405  *
406  * failed (the number of failed transactions) =
407  * 'serialization_failures' (they got a serialization error and were not
408  * successfully retried) +
409  * 'deadlock_failures' (they got a deadlock error and were not
410  * successfully retried).
411  *
412  * If the transaction was retried after a serialization or a deadlock
413  * error this does not guarantee that this retry was successful. Thus
414  *
415  * 'retries' (number of retries) =
416  * number of retries in all retried transactions =
417  * number of retries in (successfully retried transactions +
418  * failed transactions);
419  *
420  * 'retried' (number of all retried transactions) =
421  * successfully retried transactions +
422  * failed transactions.
423  *----------
424  */
425  int64 cnt; /* number of successful transactions, not
426  * including 'skipped' */
427  int64 skipped; /* number of transactions skipped under --rate
428  * and --latency-limit */
429  int64 retries; /* number of retries after a serialization or
430  * a deadlock error in all the transactions */
431  int64 retried; /* number of all transactions that were
432  * retried after a serialization or a deadlock
433  * error (perhaps the last try was
434  * unsuccessful) */
435  int64 serialization_failures; /* number of transactions that were
436  * not successfully retried after a
437  * serialization error */
438  int64 deadlock_failures; /* number of transactions that were not
439  * successfully retried after a deadlock
440  * error */
443 } StatsData;
444 
445 /*
446  * For displaying Unix epoch timestamps, as some time functions may have
447  * another reference.
448  */
450 
451 /*
452  * Error status for errors during script execution.
453  */
454 typedef enum EStatus
455 {
458 
459  /* SQL errors */
463 } EStatus;
464 
465 /*
466  * Transaction status at the end of a command.
467  */
468 typedef enum TStatus
469 {
474 } TStatus;
475 
476 /* Various random sequences are initialized from this one. */
478 
479 /* Synchronization barrier for start and connection */
481 
482 /*
483  * Connection state machine states.
484  */
485 typedef enum
486 {
487  /*
488  * The client must first choose a script to execute. Once chosen, it can
489  * either be throttled (state CSTATE_PREPARE_THROTTLE under --rate), start
490  * right away (state CSTATE_START_TX) or not start at all if the timer was
491  * exceeded (state CSTATE_FINISHED).
492  */
494 
495  /*
496  * CSTATE_START_TX performs start-of-transaction processing. Establishes
497  * a new connection for the transaction in --connect mode, records the
498  * transaction start time, and proceed to the first command.
499  *
500  * Note: once a script is started, it will either error or run till its
501  * end, where it may be interrupted. It is not interrupted while running,
502  * so pgbench --time is to be understood as tx are allowed to start in
503  * that time, and will finish when their work is completed.
504  */
506 
507  /*
508  * In CSTATE_PREPARE_THROTTLE state, we calculate when to begin the next
509  * transaction, and advance to CSTATE_THROTTLE. CSTATE_THROTTLE state
510  * sleeps until that moment, then advances to CSTATE_START_TX, or
511  * CSTATE_FINISHED if the next transaction would start beyond the end of
512  * the run.
513  */
516 
517  /*
518  * We loop through these states, to process each command in the script:
519  *
520  * CSTATE_START_COMMAND starts the execution of a command. On a SQL
521  * command, the command is sent to the server, and we move to
522  * CSTATE_WAIT_RESULT state unless in pipeline mode. On a \sleep
523  * meta-command, the timer is set, and we enter the CSTATE_SLEEP state to
524  * wait for it to expire. Other meta-commands are executed immediately. If
525  * the command about to start is actually beyond the end of the script,
526  * advance to CSTATE_END_TX.
527  *
528  * CSTATE_WAIT_RESULT waits until we get a result set back from the server
529  * for the current command.
530  *
531  * CSTATE_SLEEP waits until the end of \sleep.
532  *
533  * CSTATE_END_COMMAND records the end-of-command timestamp, increments the
534  * command counter, and loops back to CSTATE_START_COMMAND state.
535  *
536  * CSTATE_SKIP_COMMAND is used by conditional branches which are not
537  * executed. It quickly skip commands that do not need any evaluation.
538  * This state can move forward several commands, till there is something
539  * to do or the end of the script.
540  */
546 
547  /*
548  * States for failed commands.
549  *
550  * If the SQL/meta command fails, in CSTATE_ERROR clean up after an error:
551  * (1) clear the conditional stack; (2) if we have an unterminated
552  * (possibly failed) transaction block, send the rollback command to the
553  * server and wait for the result in CSTATE_WAIT_ROLLBACK_RESULT. If
554  * something goes wrong with rolling back, go to CSTATE_ABORTED.
555  *
556  * But if everything is ok we are ready for future transactions: if this
557  * is a serialization or deadlock error and we can re-execute the
558  * transaction from the very beginning, go to CSTATE_RETRY; otherwise go
559  * to CSTATE_FAILURE.
560  *
561  * In CSTATE_RETRY report an error, set the same parameters for the
562  * transaction execution as in the previous tries and process the first
563  * transaction command in CSTATE_START_COMMAND.
564  *
565  * In CSTATE_FAILURE report a failure, set the parameters for the
566  * transaction execution as they were before the first run of this
567  * transaction (except for a random state) and go to CSTATE_END_TX to
568  * complete this transaction.
569  */
574 
575  /*
576  * CSTATE_END_TX performs end-of-transaction processing. It calculates
577  * latency, and logs the transaction. In --connect mode, it closes the
578  * current connection.
579  *
580  * Then either starts over in CSTATE_CHOOSE_SCRIPT, or enters
581  * CSTATE_FINISHED if we have no more work to do.
582  */
584 
585  /*
586  * Final states. CSTATE_ABORTED means that the script execution was
587  * aborted because a command failed, CSTATE_FINISHED means success.
588  */
592 
593 /*
594  * Connection state.
595  */
596 typedef struct
597 {
598  PGconn *con; /* connection handle to DB */
599  int id; /* client No. */
600  ConnectionStateEnum state; /* state machine's current state. */
601  ConditionalStack cstack; /* enclosing conditionals state */
602 
603  /*
604  * Separate randomness for each client. This is used for random functions
605  * PGBENCH_RANDOM_* during the execution of the script.
606  */
607  pg_prng_state cs_func_rs;
608 
609  int use_file; /* index in sql_script for this client */
610  int command; /* command number in script */
611  int num_syncs; /* number of ongoing sync commands */
612 
613  /* client variables */
614  Variables variables;
615 
616  /* various times about current transaction in microseconds */
617  pg_time_usec_t txn_scheduled; /* scheduled start time of transaction */
618  pg_time_usec_t sleep_until; /* scheduled start time of next cmd */
619  pg_time_usec_t txn_begin; /* used for measuring schedule lag times */
620  pg_time_usec_t stmt_begin; /* used for measuring statement latencies */
621 
622  /* whether client prepared each command of each script */
623  bool **prepared;
624 
625  /*
626  * For processing failures and repeating transactions with serialization
627  * or deadlock errors:
628  */
629  EStatus estatus; /* the error status of the current transaction
630  * execution; this is ESTATUS_NO_ERROR if
631  * there were no errors */
632  pg_prng_state random_state; /* random state */
633  uint32 tries; /* how many times have we already tried the
634  * current transaction? */
635 
636  /* per client collected stats */
637  int64 cnt; /* client transaction count, for -t; skipped
638  * and failed transactions are also counted
639  * here */
640 } CState;
641 
642 /*
643  * Thread state
644  */
645 typedef struct
646 {
647  int tid; /* thread id */
648  THREAD_T thread; /* thread handle */
649  CState *state; /* array of CState */
650  int nstate; /* length of state[] */
651 
652  /*
653  * Separate randomness for each thread. Each thread option uses its own
654  * random state to make all of them independent of each other and
655  * therefore deterministic at the thread level.
656  */
657  pg_prng_state ts_choose_rs; /* random state for selecting a script */
658  pg_prng_state ts_throttle_rs; /* random state for transaction throttling */
659  pg_prng_state ts_sample_rs; /* random state for log sampling */
660 
661  int64 throttle_trigger; /* previous/next throttling (us) */
662  FILE *logfile; /* where to log, or NULL */
663 
664  /* per thread collected stats in microseconds */
665  pg_time_usec_t create_time; /* thread creation time */
666  pg_time_usec_t started_time; /* thread is running */
667  pg_time_usec_t bench_start; /* thread is benchmarking */
668  pg_time_usec_t conn_duration; /* cumulated connection and disconnection
669  * delays */
670 
671  StatsData stats;
672  int64 latency_late; /* count executed but late transactions */
673 } TState;
674 
675 /*
676  * queries read from files
677  */
678 #define SQL_COMMAND 1
679 #define META_COMMAND 2
680 
681 /*
682  * max number of backslash command arguments or SQL variables,
683  * including the command or SQL statement itself
684  */
685 #define MAX_ARGS 256
686 
687 typedef enum MetaCommand
688 {
689  META_NONE, /* not a known meta-command */
690  META_SET, /* \set */
691  META_SETSHELL, /* \setshell */
692  META_SHELL, /* \shell */
693  META_SLEEP, /* \sleep */
694  META_GSET, /* \gset */
695  META_ASET, /* \aset */
696  META_IF, /* \if */
697  META_ELIF, /* \elif */
698  META_ELSE, /* \else */
699  META_ENDIF, /* \endif */
700  META_STARTPIPELINE, /* \startpipeline */
701  META_SYNCPIPELINE, /* \syncpipeline */
702  META_ENDPIPELINE, /* \endpipeline */
703 } MetaCommand;
704 
705 typedef enum QueryMode
706 {
707  QUERY_SIMPLE, /* simple query */
708  QUERY_EXTENDED, /* extended query */
709  QUERY_PREPARED, /* extended query with prepared statements */
711 } QueryMode;
712 
714 static const char *const QUERYMODE[] = {"simple", "extended", "prepared"};
715 
716 /*
717  * struct Command represents one command in a script.
718  *
719  * lines The raw, possibly multi-line command text. Variable substitution
720  * not applied.
721  * first_line A short, single-line extract of 'lines', for error reporting.
722  * type SQL_COMMAND or META_COMMAND
723  * meta The type of meta-command, with META_NONE/GSET/ASET if command
724  * is SQL.
725  * argc Number of arguments of the command, 0 if not yet processed.
726  * argv Command arguments, the first of which is the command or SQL
727  * string itself. For SQL commands, after post-processing
728  * argv[0] is the same as 'lines' with variables substituted.
729  * prepname The name that this command is prepared under, in prepare mode
730  * varprefix SQL commands terminated with \gset or \aset have this set
731  * to a non NULL value. If nonempty, it's used to prefix the
732  * variable name that receives the value.
733  * aset do gset on all possible queries of a combined query (\;).
734  * expr Parsed expression, if needed.
735  * stats Time spent in this command.
736  * retries Number of retries after a serialization or deadlock error in the
737  * current command.
738  * failures Number of errors in the current command that were not retried.
739  */
740 typedef struct Command
741 {
743  char *first_line;
744  int type;
746  int argc;
747  char *argv[MAX_ARGS];
748  char *prepname;
749  char *varprefix;
752  int64 retries;
753  int64 failures;
754 } Command;
755 
756 typedef struct ParsedScript
757 {
758  const char *desc; /* script descriptor (eg, file name) */
759  int weight; /* selection weight */
760  Command **commands; /* NULL-terminated array of Commands */
761  StatsData stats; /* total time spent in script */
762 } ParsedScript;
763 
764 static ParsedScript sql_script[MAX_SCRIPTS]; /* SQL script files */
765 static int num_scripts; /* number of scripts in sql_script[] */
766 static int64 total_weight = 0;
767 
768 static bool verbose_errors = false; /* print verbose messages of all errors */
769 
770 static bool exit_on_abort = false; /* exit when any client is aborted */
771 
772 /* Builtin test scripts */
773 typedef struct BuiltinScript
774 {
775  const char *name; /* very short name for -b ... */
776  const char *desc; /* short description */
777  const char *script; /* actual pgbench script */
778 } BuiltinScript;
779 
780 static const BuiltinScript builtin_script[] =
781 {
782  {
783  "tpcb-like",
784  "<builtin: TPC-B (sort of)>",
785  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
786  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
787  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
788  "\\set delta random(-5000, 5000)\n"
789  "BEGIN;\n"
790  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
791  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
792  "UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;\n"
793  "UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;\n"
794  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
795  "END;\n"
796  },
797  {
798  "simple-update",
799  "<builtin: simple update>",
800  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
801  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
802  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
803  "\\set delta random(-5000, 5000)\n"
804  "BEGIN;\n"
805  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
806  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
807  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
808  "END;\n"
809  },
810  {
811  "select-only",
812  "<builtin: select only>",
813  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
814  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
815  }
816 };
817 
818 
819 /* Function prototypes */
820 static void setNullValue(PgBenchValue *pv);
821 static void setBoolValue(PgBenchValue *pv, bool bval);
822 static void setIntValue(PgBenchValue *pv, int64 ival);
823 static void setDoubleValue(PgBenchValue *pv, double dval);
824 static bool evaluateExpr(CState *st, PgBenchExpr *expr,
825  PgBenchValue *retval);
827 static void doLog(TState *thread, CState *st,
828  StatsData *agg, bool skipped, double latency, double lag);
829 static void processXactStats(TState *thread, CState *st, pg_time_usec_t *now,
830  bool skipped, StatsData *agg);
831 static void addScript(const ParsedScript *script);
833 static void finishCon(CState *st);
834 static void setalarm(int seconds);
835 static socket_set *alloc_socket_set(int count);
836 static void free_socket_set(socket_set *sa);
837 static void clear_socket_set(socket_set *sa);
838 static void add_socket_to_set(socket_set *sa, int fd, int idx);
839 static int wait_on_socket_set(socket_set *sa, int64 usecs);
840 static bool socket_has_input(socket_set *sa, int fd, int idx);
841 
842 /* callback used to build rows for COPY during data loading */
843 typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
844 
845 /* callback functions for our flex lexer */
846 static const PsqlScanCallbacks pgbench_callbacks = {
847  NULL, /* don't need get_variable functionality */
848 };
849 
850 static inline pg_time_usec_t
851 pg_time_now(void)
852 {
853  instr_time now;
854 
856 
858 }
859 
860 static inline void
862 {
863  if ((*now) == 0)
864  (*now) = pg_time_now();
865 }
866 
867 #define PG_TIME_GET_DOUBLE(t) (0.000001 * (t))
868 
869 static void
870 usage(void)
871 {
872  printf("%s is a benchmarking tool for PostgreSQL.\n\n"
873  "Usage:\n"
874  " %s [OPTION]... [DBNAME]\n"
875  "\nInitialization options:\n"
876  " -i, --initialize invokes initialization mode\n"
877  " -I, --init-steps=[" ALL_INIT_STEPS "]+ (default \"" DEFAULT_INIT_STEPS "\")\n"
878  " run selected initialization steps, in the specified order\n"
879  " d: drop any existing pgbench tables\n"
880  " t: create the tables used by the standard pgbench scenario\n"
881  " g: generate data, client-side\n"
882  " G: generate data, server-side\n"
883  " v: invoke VACUUM on the standard tables\n"
884  " p: create primary key indexes on the standard tables\n"
885  " f: create foreign keys between the standard tables\n"
886  " -F, --fillfactor=NUM set fill factor\n"
887  " -n, --no-vacuum do not run VACUUM during initialization\n"
888  " -q, --quiet quiet logging (one message each 5 seconds)\n"
889  " -s, --scale=NUM scaling factor\n"
890  " --foreign-keys create foreign key constraints between tables\n"
891  " --index-tablespace=TABLESPACE\n"
892  " create indexes in the specified tablespace\n"
893  " --partition-method=(range|hash)\n"
894  " partition pgbench_accounts with this method (default: range)\n"
895  " --partitions=NUM partition pgbench_accounts into NUM parts (default: 0)\n"
896  " --tablespace=TABLESPACE create tables in the specified tablespace\n"
897  " --unlogged-tables create tables as unlogged tables\n"
898  "\nOptions to select what to run:\n"
899  " -b, --builtin=NAME[@W] add builtin script NAME weighted at W (default: 1)\n"
900  " (use \"-b list\" to list available scripts)\n"
901  " -f, --file=FILENAME[@W] add script FILENAME weighted at W (default: 1)\n"
902  " -N, --skip-some-updates skip updates of pgbench_tellers and pgbench_branches\n"
903  " (same as \"-b simple-update\")\n"
904  " -S, --select-only perform SELECT-only transactions\n"
905  " (same as \"-b select-only\")\n"
906  "\nBenchmarking options:\n"
907  " -c, --client=NUM number of concurrent database clients (default: 1)\n"
908  " -C, --connect establish new connection for each transaction\n"
909  " -D, --define=VARNAME=VALUE\n"
910  " define variable for use by custom script\n"
911  " -j, --jobs=NUM number of threads (default: 1)\n"
912  " -l, --log write transaction times to log file\n"
913  " -L, --latency-limit=NUM count transactions lasting more than NUM ms as late\n"
914  " -M, --protocol=simple|extended|prepared\n"
915  " protocol for submitting queries (default: simple)\n"
916  " -n, --no-vacuum do not run VACUUM before tests\n"
917  " -P, --progress=NUM show thread progress report every NUM seconds\n"
918  " -r, --report-per-command report latencies, failures, and retries per command\n"
919  " -R, --rate=NUM target rate in transactions per second\n"
920  " -s, --scale=NUM report this scale factor in output\n"
921  " -t, --transactions=NUM number of transactions each client runs (default: 10)\n"
922  " -T, --time=NUM duration of benchmark test in seconds\n"
923  " -v, --vacuum-all vacuum all four standard tables before tests\n"
924  " --aggregate-interval=NUM aggregate data over NUM seconds\n"
925  " --exit-on-abort exit when any client is aborted\n"
926  " --failures-detailed report the failures grouped by basic types\n"
927  " --log-prefix=PREFIX prefix for transaction time log file\n"
928  " (default: \"pgbench_log\")\n"
929  " --max-tries=NUM max number of tries to run transaction (default: 1)\n"
930  " --progress-timestamp use Unix epoch timestamps for progress\n"
931  " --random-seed=SEED set random seed (\"time\", \"rand\", integer)\n"
932  " --sampling-rate=NUM fraction of transactions to log (e.g., 0.01 for 1%%)\n"
933  " --show-script=NAME show builtin script code, then exit\n"
934  " --verbose-errors print messages of all errors\n"
935  "\nCommon options:\n"
936  " -d, --debug print debugging output\n"
937  " -h, --host=HOSTNAME database server host or socket directory\n"
938  " -p, --port=PORT database server port number\n"
939  " -U, --username=USERNAME connect as specified database user\n"
940  " -V, --version output version information, then exit\n"
941  " -?, --help show this help, then exit\n"
942  "\n"
943  "Report bugs to <%s>.\n"
944  "%s home page: <%s>\n",
945  progname, progname, PACKAGE_BUGREPORT, PACKAGE_NAME, PACKAGE_URL);
946 }
947 
948 /* return whether str matches "^\s*[-+]?[0-9]+$" */
949 static bool
950 is_an_int(const char *str)
951 {
952  const char *ptr = str;
953 
954  /* skip leading spaces; cast is consistent with strtoint64 */
955  while (*ptr && isspace((unsigned char) *ptr))
956  ptr++;
957 
958  /* skip sign */
959  if (*ptr == '+' || *ptr == '-')
960  ptr++;
961 
962  /* at least one digit */
963  if (*ptr && !isdigit((unsigned char) *ptr))
964  return false;
965 
966  /* eat all digits */
967  while (*ptr && isdigit((unsigned char) *ptr))
968  ptr++;
969 
970  /* must have reached end of string */
971  return *ptr == '\0';
972 }
973 
974 
975 /*
976  * strtoint64 -- convert a string to 64-bit integer
977  *
978  * This function is a slightly modified version of pg_strtoint64() from
979  * src/backend/utils/adt/numutils.c.
980  *
981  * The function returns whether the conversion worked, and if so
982  * "*result" is set to the result.
983  *
984  * If not errorOK, an error message is also printed out on errors.
985  */
986 bool
987 strtoint64(const char *str, bool errorOK, int64 *result)
988 {
989  const char *ptr = str;
990  int64 tmp = 0;
991  bool neg = false;
992 
993  /*
994  * Do our own scan, rather than relying on sscanf which might be broken
995  * for long long.
996  *
997  * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
998  * value as a negative number.
999  */
1000 
1001  /* skip leading spaces */
1002  while (*ptr && isspace((unsigned char) *ptr))
1003  ptr++;
1004 
1005  /* handle sign */
1006  if (*ptr == '-')
1007  {
1008  ptr++;
1009  neg = true;
1010  }
1011  else if (*ptr == '+')
1012  ptr++;
1013 
1014  /* require at least one digit */
1015  if (unlikely(!isdigit((unsigned char) *ptr)))
1016  goto invalid_syntax;
1017 
1018  /* process digits */
1019  while (*ptr && isdigit((unsigned char) *ptr))
1020  {
1021  int8 digit = (*ptr++ - '0');
1022 
1023  if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
1024  unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
1025  goto out_of_range;
1026  }
1027 
1028  /* allow trailing whitespace, but not other trailing chars */
1029  while (*ptr != '\0' && isspace((unsigned char) *ptr))
1030  ptr++;
1031 
1032  if (unlikely(*ptr != '\0'))
1033  goto invalid_syntax;
1034 
1035  if (!neg)
1036  {
1037  if (unlikely(tmp == PG_INT64_MIN))
1038  goto out_of_range;
1039  tmp = -tmp;
1040  }
1041 
1042  *result = tmp;
1043  return true;
1044 
1045 out_of_range:
1046  if (!errorOK)
1047  pg_log_error("value \"%s\" is out of range for type bigint", str);
1048  return false;
1049 
1050 invalid_syntax:
1051  if (!errorOK)
1052  pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
1053  return false;
1054 }
1055 
1056 /* convert string to double, detecting overflows/underflows */
1057 bool
1058 strtodouble(const char *str, bool errorOK, double *dv)
1059 {
1060  char *end;
1061 
1062  errno = 0;
1063  *dv = strtod(str, &end);
1064 
1065  if (unlikely(errno != 0))
1066  {
1067  if (!errorOK)
1068  pg_log_error("value \"%s\" is out of range for type double", str);
1069  return false;
1070  }
1071 
1072  if (unlikely(end == str || *end != '\0'))
1073  {
1074  if (!errorOK)
1075  pg_log_error("invalid input syntax for type double: \"%s\"", str);
1076  return false;
1077  }
1078  return true;
1079 }
1080 
1081 /*
1082  * Initialize a prng state struct.
1083  *
1084  * We derive the seed from base_random_sequence, which must be set up already.
1085  */
1086 static void
1088 {
1090 }
1091 
1092 
1093 /*
1094  * random number generator: uniform distribution from min to max inclusive.
1095  *
1096  * Although the limits are expressed as int64, you can't generate the full
1097  * int64 range in one call, because the difference of the limits mustn't
1098  * overflow int64. This is not checked.
1099  */
1100 static int64
1101 getrand(pg_prng_state *state, int64 min, int64 max)
1102 {
1103  return min + (int64) pg_prng_uint64_range(state, 0, max - min);
1104 }
1105 
1106 /*
1107  * random number generator: exponential distribution from min to max inclusive.
1108  * the parameter is so that the density of probability for the last cut-off max
1109  * value is exp(-parameter).
1110  */
1111 static int64
1112 getExponentialRand(pg_prng_state *state, int64 min, int64 max,
1113  double parameter)
1114 {
1115  double cut,
1116  uniform,
1117  rand;
1118 
1119  /* abort if wrong parameter, but must really be checked beforehand */
1120  Assert(parameter > 0.0);
1121  cut = exp(-parameter);
1122  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1123  uniform = 1.0 - pg_prng_double(state);
1124 
1125  /*
1126  * inner expression in (cut, 1] (if parameter > 0), rand in [0, 1)
1127  */
1128  Assert((1.0 - cut) != 0.0);
1129  rand = -log(cut + (1.0 - cut) * uniform) / parameter;
1130  /* return int64 random number within between min and max */
1131  return min + (int64) ((max - min + 1) * rand);
1132 }
1133 
1134 /* random number generator: gaussian distribution from min to max inclusive */
1135 static int64
1136 getGaussianRand(pg_prng_state *state, int64 min, int64 max,
1137  double parameter)
1138 {
1139  double stdev;
1140  double rand;
1141 
1142  /* abort if parameter is too low, but must really be checked beforehand */
1143  Assert(parameter >= MIN_GAUSSIAN_PARAM);
1144 
1145  /*
1146  * Get normally-distributed random number in the range -parameter <= stdev
1147  * < parameter.
1148  *
1149  * This loop is executed until the number is in the expected range.
1150  *
1151  * As the minimum parameter is 2.0, the probability of looping is low:
1152  * sqrt(-2 ln(r)) <= 2 => r >= e^{-2} ~ 0.135, then when taking the
1153  * average sinus multiplier as 2/pi, we have a 8.6% looping probability in
1154  * the worst case. For a parameter value of 5.0, the looping probability
1155  * is about e^{-5} * 2 / pi ~ 0.43%.
1156  */
1157  do
1158  {
1159  stdev = pg_prng_double_normal(state);
1160  }
1161  while (stdev < -parameter || stdev >= parameter);
1162 
1163  /* stdev is in [-parameter, parameter), normalization to [0,1) */
1164  rand = (stdev + parameter) / (parameter * 2.0);
1165 
1166  /* return int64 random number within between min and max */
1167  return min + (int64) ((max - min + 1) * rand);
1168 }
1169 
1170 /*
1171  * random number generator: generate a value, such that the series of values
1172  * will approximate a Poisson distribution centered on the given value.
1173  *
1174  * Individual results are rounded to integers, though the center value need
1175  * not be one.
1176  */
1177 static int64
1178 getPoissonRand(pg_prng_state *state, double center)
1179 {
1180  /*
1181  * Use inverse transform sampling to generate a value > 0, such that the
1182  * expected (i.e. average) value is the given argument.
1183  */
1184  double uniform;
1185 
1186  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1187  uniform = 1.0 - pg_prng_double(state);
1188 
1189  return (int64) (-log(uniform) * center + 0.5);
1190 }
1191 
1192 /*
1193  * Computing zipfian using rejection method, based on
1194  * "Non-Uniform Random Variate Generation",
1195  * Luc Devroye, p. 550-551, Springer 1986.
1196  *
1197  * This works for s > 1.0, but may perform badly for s very close to 1.0.
1198  */
1199 static int64
1200 computeIterativeZipfian(pg_prng_state *state, int64 n, double s)
1201 {
1202  double b = pow(2.0, s - 1.0);
1203  double x,
1204  t,
1205  u,
1206  v;
1207 
1208  /* Ensure n is sane */
1209  if (n <= 1)
1210  return 1;
1211 
1212  while (true)
1213  {
1214  /* random variates */
1215  u = pg_prng_double(state);
1216  v = pg_prng_double(state);
1217 
1218  x = floor(pow(u, -1.0 / (s - 1.0)));
1219 
1220  t = pow(1.0 + 1.0 / x, s - 1.0);
1221  /* reject if too large or out of bound */
1222  if (v * x * (t - 1.0) / (b - 1.0) <= t / b && x <= n)
1223  break;
1224  }
1225  return (int64) x;
1226 }
1227 
1228 /* random number generator: zipfian distribution from min to max inclusive */
1229 static int64
1230 getZipfianRand(pg_prng_state *state, int64 min, int64 max, double s)
1231 {
1232  int64 n = max - min + 1;
1233 
1234  /* abort if parameter is invalid */
1236 
1237  return min - 1 + computeIterativeZipfian(state, n, s);
1238 }
1239 
1240 /*
1241  * FNV-1a hash function
1242  */
1243 static int64
1244 getHashFnv1a(int64 val, uint64 seed)
1245 {
1246  int64 result;
1247  int i;
1248 
1249  result = FNV_OFFSET_BASIS ^ seed;
1250  for (i = 0; i < 8; ++i)
1251  {
1252  int32 octet = val & 0xff;
1253 
1254  val = val >> 8;
1255  result = result ^ octet;
1256  result = result * FNV_PRIME;
1257  }
1258 
1259  return result;
1260 }
1261 
1262 /*
1263  * Murmur2 hash function
1264  *
1265  * Based on original work of Austin Appleby
1266  * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash2.cpp
1267  */
1268 static int64
1269 getHashMurmur2(int64 val, uint64 seed)
1270 {
1271  uint64 result = seed ^ MM2_MUL_TIMES_8; /* sizeof(int64) */
1272  uint64 k = (uint64) val;
1273 
1274  k *= MM2_MUL;
1275  k ^= k >> MM2_ROT;
1276  k *= MM2_MUL;
1277 
1278  result ^= k;
1279  result *= MM2_MUL;
1280 
1281  result ^= result >> MM2_ROT;
1282  result *= MM2_MUL;
1283  result ^= result >> MM2_ROT;
1284 
1285  return (int64) result;
1286 }
1287 
1288 /*
1289  * Pseudorandom permutation function
1290  *
1291  * For small sizes, this generates each of the (size!) possible permutations
1292  * of integers in the range [0, size) with roughly equal probability. Once
1293  * the size is larger than 20, the number of possible permutations exceeds the
1294  * number of distinct states of the internal pseudorandom number generator,
1295  * and so not all possible permutations can be generated, but the permutations
1296  * chosen should continue to give the appearance of being random.
1297  *
1298  * THIS FUNCTION IS NOT CRYPTOGRAPHICALLY SECURE.
1299  * DO NOT USE FOR SUCH PURPOSE.
1300  */
1301 static int64
1302 permute(const int64 val, const int64 isize, const int64 seed)
1303 {
1304  /* using a high-end PRNG is probably overkill */
1306  uint64 size;
1307  uint64 v;
1308  int masklen;
1309  uint64 mask;
1310  int i;
1311 
1312  if (isize < 2)
1313  return 0; /* nothing to permute */
1314 
1315  /* Initialize prng state using the seed */
1316  pg_prng_seed(&state, (uint64) seed);
1317 
1318  /* Computations are performed on unsigned values */
1319  size = (uint64) isize;
1320  v = (uint64) val % size;
1321 
1322  /* Mask to work modulo largest power of 2 less than or equal to size */
1323  masklen = pg_leftmost_one_pos64(size);
1324  mask = (((uint64) 1) << masklen) - 1;
1325 
1326  /*
1327  * Permute the input value by applying several rounds of pseudorandom
1328  * bijective transformations. The intention here is to distribute each
1329  * input uniformly randomly across the range, and separate adjacent inputs
1330  * approximately uniformly randomly from each other, leading to a fairly
1331  * random overall choice of permutation.
1332  *
1333  * To separate adjacent inputs, we multiply by a random number modulo
1334  * (mask + 1), which is a power of 2. For this to be a bijection, the
1335  * multiplier must be odd. Since this is known to lead to less randomness
1336  * in the lower bits, we also apply a rotation that shifts the topmost bit
1337  * into the least significant bit. In the special cases where size <= 3,
1338  * mask = 1 and each of these operations is actually a no-op, so we also
1339  * XOR the value with a different random number to inject additional
1340  * randomness. Since the size is generally not a power of 2, we apply
1341  * this bijection on overlapping upper and lower halves of the input.
1342  *
1343  * To distribute the inputs uniformly across the range, we then also apply
1344  * a random offset modulo the full range.
1345  *
1346  * Taken together, these operations resemble a modified linear
1347  * congruential generator, as is commonly used in pseudorandom number
1348  * generators. The number of rounds is fairly arbitrary, but six has been
1349  * found empirically to give a fairly good tradeoff between performance
1350  * and uniform randomness. For small sizes it selects each of the (size!)
1351  * possible permutations with roughly equal probability. For larger
1352  * sizes, not all permutations can be generated, but the intended random
1353  * spread is still produced.
1354  */
1355  for (i = 0; i < 6; i++)
1356  {
1357  uint64 m,
1358  r,
1359  t;
1360 
1361  /* Random multiply (by an odd number), XOR and rotate of lower half */
1362  m = (pg_prng_uint64(&state) & mask) | 1;
1363  r = pg_prng_uint64(&state) & mask;
1364  if (v <= mask)
1365  {
1366  v = ((v * m) ^ r) & mask;
1367  v = ((v << 1) & mask) | (v >> (masklen - 1));
1368  }
1369 
1370  /* Random multiply (by an odd number), XOR and rotate of upper half */
1371  m = (pg_prng_uint64(&state) & mask) | 1;
1372  r = pg_prng_uint64(&state) & mask;
1373  t = size - 1 - v;
1374  if (t <= mask)
1375  {
1376  t = ((t * m) ^ r) & mask;
1377  t = ((t << 1) & mask) | (t >> (masklen - 1));
1378  v = size - 1 - t;
1379  }
1380 
1381  /* Random offset */
1382  r = pg_prng_uint64_range(&state, 0, size - 1);
1383  v = (v + r) % size;
1384  }
1385 
1386  return (int64) v;
1387 }
1388 
1389 /*
1390  * Initialize the given SimpleStats struct to all zeroes
1391  */
1392 static void
1394 {
1395  memset(ss, 0, sizeof(SimpleStats));
1396 }
1397 
1398 /*
1399  * Accumulate one value into a SimpleStats struct.
1400  */
1401 static void
1402 addToSimpleStats(SimpleStats *ss, double val)
1403 {
1404  if (ss->count == 0 || val < ss->min)
1405  ss->min = val;
1406  if (ss->count == 0 || val > ss->max)
1407  ss->max = val;
1408  ss->count++;
1409  ss->sum += val;
1410  ss->sum2 += val * val;
1411 }
1412 
1413 /*
1414  * Merge two SimpleStats objects
1415  */
1416 static void
1418 {
1419  if (acc->count == 0 || ss->min < acc->min)
1420  acc->min = ss->min;
1421  if (acc->count == 0 || ss->max > acc->max)
1422  acc->max = ss->max;
1423  acc->count += ss->count;
1424  acc->sum += ss->sum;
1425  acc->sum2 += ss->sum2;
1426 }
1427 
1428 /*
1429  * Initialize a StatsData struct to mostly zeroes, with its start time set to
1430  * the given value.
1431  */
1432 static void
1434 {
1435  sd->start_time = start;
1436  sd->cnt = 0;
1437  sd->skipped = 0;
1438  sd->retries = 0;
1439  sd->retried = 0;
1440  sd->serialization_failures = 0;
1441  sd->deadlock_failures = 0;
1442  initSimpleStats(&sd->latency);
1443  initSimpleStats(&sd->lag);
1444 }
1445 
1446 /*
1447  * Accumulate one additional item into the given stats object.
1448  */
1449 static void
1450 accumStats(StatsData *stats, bool skipped, double lat, double lag,
1451  EStatus estatus, int64 tries)
1452 {
1453  /* Record the skipped transaction */
1454  if (skipped)
1455  {
1456  /* no latency to record on skipped transactions */
1457  stats->skipped++;
1458  return;
1459  }
1460 
1461  /*
1462  * Record the number of retries regardless of whether the transaction was
1463  * successful or failed.
1464  */
1465  if (tries > 1)
1466  {
1467  stats->retries += (tries - 1);
1468  stats->retried++;
1469  }
1470 
1471  switch (estatus)
1472  {
1473  /* Record the successful transaction */
1474  case ESTATUS_NO_ERROR:
1475  stats->cnt++;
1476 
1477  addToSimpleStats(&stats->latency, lat);
1478 
1479  /* and possibly the same for schedule lag */
1480  if (throttle_delay)
1481  addToSimpleStats(&stats->lag, lag);
1482  break;
1483 
1484  /* Record the failed transaction */
1486  stats->serialization_failures++;
1487  break;
1489  stats->deadlock_failures++;
1490  break;
1491  default:
1492  /* internal error which should never occur */
1493  pg_fatal("unexpected error status: %d", estatus);
1494  }
1495 }
1496 
1497 /* call PQexec() and exit() on failure */
1498 static void
1499 executeStatement(PGconn *con, const char *sql)
1500 {
1501  PGresult *res;
1502 
1503  res = PQexec(con, sql);
1505  {
1506  pg_log_error("query failed: %s", PQerrorMessage(con));
1507  pg_log_error_detail("Query was: %s", sql);
1508  exit(1);
1509  }
1510  PQclear(res);
1511 }
1512 
1513 /* call PQexec() and complain, but without exiting, on failure */
1514 static void
1515 tryExecuteStatement(PGconn *con, const char *sql)
1516 {
1517  PGresult *res;
1518 
1519  res = PQexec(con, sql);
1521  {
1522  pg_log_error("%s", PQerrorMessage(con));
1523  pg_log_error_detail("(ignoring this error and continuing anyway)");
1524  }
1525  PQclear(res);
1526 }
1527 
1528 /* set up a connection to the backend */
1529 static PGconn *
1530 doConnect(void)
1531 {
1532  PGconn *conn;
1533  bool new_pass;
1534  static char *password = NULL;
1535 
1536  /*
1537  * Start the connection. Loop until we have a password if requested by
1538  * backend.
1539  */
1540  do
1541  {
1542 #define PARAMS_ARRAY_SIZE 7
1543 
1544  const char *keywords[PARAMS_ARRAY_SIZE];
1545  const char *values[PARAMS_ARRAY_SIZE];
1546 
1547  keywords[0] = "host";
1548  values[0] = pghost;
1549  keywords[1] = "port";
1550  values[1] = pgport;
1551  keywords[2] = "user";
1552  values[2] = username;
1553  keywords[3] = "password";
1554  values[3] = password;
1555  keywords[4] = "dbname";
1556  values[4] = dbName;
1557  keywords[5] = "fallback_application_name";
1558  values[5] = progname;
1559  keywords[6] = NULL;
1560  values[6] = NULL;
1561 
1562  new_pass = false;
1563 
1564  conn = PQconnectdbParams(keywords, values, true);
1565 
1566  if (!conn)
1567  {
1568  pg_log_error("connection to database \"%s\" failed", dbName);
1569  return NULL;
1570  }
1571 
1572  if (PQstatus(conn) == CONNECTION_BAD &&
1574  !password)
1575  {
1576  PQfinish(conn);
1577  password = simple_prompt("Password: ", false);
1578  new_pass = true;
1579  }
1580  } while (new_pass);
1581 
1582  /* check to see that the backend connection was successfully made */
1583  if (PQstatus(conn) == CONNECTION_BAD)
1584  {
1586  PQfinish(conn);
1587  return NULL;
1588  }
1589 
1590  return conn;
1591 }
1592 
1593 /* qsort comparator for Variable array */
1594 static int
1595 compareVariableNames(const void *v1, const void *v2)
1596 {
1597  return strcmp(((const Variable *) v1)->name,
1598  ((const Variable *) v2)->name);
1599 }
1600 
1601 /* Locate a variable by name; returns NULL if unknown */
1602 static Variable *
1603 lookupVariable(Variables *variables, char *name)
1604 {
1605  Variable key;
1606 
1607  /* On some versions of Solaris, bsearch of zero items dumps core */
1608  if (variables->nvars <= 0)
1609  return NULL;
1610 
1611  /* Sort if we have to */
1612  if (!variables->vars_sorted)
1613  {
1614  qsort(variables->vars, variables->nvars, sizeof(Variable),
1616  variables->vars_sorted = true;
1617  }
1618 
1619  /* Now we can search */
1620  key.name = name;
1621  return (Variable *) bsearch(&key,
1622  variables->vars,
1623  variables->nvars,
1624  sizeof(Variable),
1626 }
1627 
1628 /* Get the value of a variable, in string form; returns NULL if unknown */
1629 static char *
1630 getVariable(Variables *variables, char *name)
1631 {
1632  Variable *var;
1633  char stringform[64];
1634 
1635  var = lookupVariable(variables, name);
1636  if (var == NULL)
1637  return NULL; /* not found */
1638 
1639  if (var->svalue)
1640  return var->svalue; /* we have it in string form */
1641 
1642  /* We need to produce a string equivalent of the value */
1643  Assert(var->value.type != PGBT_NO_VALUE);
1644  if (var->value.type == PGBT_NULL)
1645  snprintf(stringform, sizeof(stringform), "NULL");
1646  else if (var->value.type == PGBT_BOOLEAN)
1647  snprintf(stringform, sizeof(stringform),
1648  "%s", var->value.u.bval ? "true" : "false");
1649  else if (var->value.type == PGBT_INT)
1650  snprintf(stringform, sizeof(stringform),
1651  INT64_FORMAT, var->value.u.ival);
1652  else if (var->value.type == PGBT_DOUBLE)
1653  snprintf(stringform, sizeof(stringform),
1654  "%.*g", DBL_DIG, var->value.u.dval);
1655  else /* internal error, unexpected type */
1656  Assert(0);
1657  var->svalue = pg_strdup(stringform);
1658  return var->svalue;
1659 }
1660 
1661 /* Try to convert variable to a value; return false on failure */
1662 static bool
1664 {
1665  size_t slen;
1666 
1667  if (var->value.type != PGBT_NO_VALUE)
1668  return true; /* no work */
1669 
1670  slen = strlen(var->svalue);
1671 
1672  if (slen == 0)
1673  /* what should it do on ""? */
1674  return false;
1675 
1676  if (pg_strcasecmp(var->svalue, "null") == 0)
1677  {
1678  setNullValue(&var->value);
1679  }
1680 
1681  /*
1682  * accept prefixes such as y, ye, n, no... but not for "o". 0/1 are
1683  * recognized later as an int, which is converted to bool if needed.
1684  */
1685  else if (pg_strncasecmp(var->svalue, "true", slen) == 0 ||
1686  pg_strncasecmp(var->svalue, "yes", slen) == 0 ||
1687  pg_strcasecmp(var->svalue, "on") == 0)
1688  {
1689  setBoolValue(&var->value, true);
1690  }
1691  else if (pg_strncasecmp(var->svalue, "false", slen) == 0 ||
1692  pg_strncasecmp(var->svalue, "no", slen) == 0 ||
1693  pg_strcasecmp(var->svalue, "off") == 0 ||
1694  pg_strcasecmp(var->svalue, "of") == 0)
1695  {
1696  setBoolValue(&var->value, false);
1697  }
1698  else if (is_an_int(var->svalue))
1699  {
1700  /* if it looks like an int, it must be an int without overflow */
1701  int64 iv;
1702 
1703  if (!strtoint64(var->svalue, false, &iv))
1704  return false;
1705 
1706  setIntValue(&var->value, iv);
1707  }
1708  else /* type should be double */
1709  {
1710  double dv;
1711 
1712  if (!strtodouble(var->svalue, true, &dv))
1713  {
1714  pg_log_error("malformed variable \"%s\" value: \"%s\"",
1715  var->name, var->svalue);
1716  return false;
1717  }
1718  setDoubleValue(&var->value, dv);
1719  }
1720  return true;
1721 }
1722 
1723 /*
1724  * Check whether a variable's name is allowed.
1725  *
1726  * We allow any non-ASCII character, as well as ASCII letters, digits, and
1727  * underscore.
1728  *
1729  * Keep this in sync with the definitions of variable name characters in
1730  * "src/fe_utils/psqlscan.l", "src/bin/psql/psqlscanslash.l" and
1731  * "src/bin/pgbench/exprscan.l". Also see parseVariable(), below.
1732  *
1733  * Note: this static function is copied from "src/bin/psql/variables.c"
1734  * but changed to disallow variable names starting with a digit.
1735  */
1736 static bool
1737 valid_variable_name(const char *name)
1738 {
1739  const unsigned char *ptr = (const unsigned char *) name;
1740 
1741  /* Mustn't be zero-length */
1742  if (*ptr == '\0')
1743  return false;
1744 
1745  /* must not start with [0-9] */
1746  if (IS_HIGHBIT_SET(*ptr) ||
1747  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1748  "_", *ptr) != NULL)
1749  ptr++;
1750  else
1751  return false;
1752 
1753  /* remaining characters can include [0-9] */
1754  while (*ptr)
1755  {
1756  if (IS_HIGHBIT_SET(*ptr) ||
1757  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1758  "_0123456789", *ptr) != NULL)
1759  ptr++;
1760  else
1761  return false;
1762  }
1763 
1764  return true;
1765 }
1766 
1767 /*
1768  * Make sure there is enough space for 'needed' more variable in the variables
1769  * array.
1770  */
1771 static void
1772 enlargeVariables(Variables *variables, int needed)
1773 {
1774  /* total number of variables required now */
1775  needed += variables->nvars;
1776 
1777  if (variables->max_vars < needed)
1778  {
1779  variables->max_vars = needed + VARIABLES_ALLOC_MARGIN;
1780  variables->vars = (Variable *)
1781  pg_realloc(variables->vars, variables->max_vars * sizeof(Variable));
1782  }
1783 }
1784 
1785 /*
1786  * Lookup a variable by name, creating it if need be.
1787  * Caller is expected to assign a value to the variable.
1788  * Returns NULL on failure (bad name).
1789  */
1790 static Variable *
1791 lookupCreateVariable(Variables *variables, const char *context, char *name)
1792 {
1793  Variable *var;
1794 
1795  var = lookupVariable(variables, name);
1796  if (var == NULL)
1797  {
1798  /*
1799  * Check for the name only when declaring a new variable to avoid
1800  * overhead.
1801  */
1802  if (!valid_variable_name(name))
1803  {
1804  pg_log_error("%s: invalid variable name: \"%s\"", context, name);
1805  return NULL;
1806  }
1807 
1808  /* Create variable at the end of the array */
1809  enlargeVariables(variables, 1);
1810 
1811  var = &(variables->vars[variables->nvars]);
1812 
1813  var->name = pg_strdup(name);
1814  var->svalue = NULL;
1815  /* caller is expected to initialize remaining fields */
1816 
1817  variables->nvars++;
1818  /* we don't re-sort the array till we have to */
1819  variables->vars_sorted = false;
1820  }
1821 
1822  return var;
1823 }
1824 
1825 /* Assign a string value to a variable, creating it if need be */
1826 /* Returns false on failure (bad name) */
1827 static bool
1828 putVariable(Variables *variables, const char *context, char *name,
1829  const char *value)
1830 {
1831  Variable *var;
1832  char *val;
1833 
1834  var = lookupCreateVariable(variables, context, name);
1835  if (!var)
1836  return false;
1837 
1838  /* dup then free, in case value is pointing at this variable */
1839  val = pg_strdup(value);
1840 
1841  free(var->svalue);
1842  var->svalue = val;
1843  var->value.type = PGBT_NO_VALUE;
1844 
1845  return true;
1846 }
1847 
1848 /* Assign a value to a variable, creating it if need be */
1849 /* Returns false on failure (bad name) */
1850 static bool
1851 putVariableValue(Variables *variables, const char *context, char *name,
1852  const PgBenchValue *value)
1853 {
1854  Variable *var;
1855 
1856  var = lookupCreateVariable(variables, context, name);
1857  if (!var)
1858  return false;
1859 
1860  free(var->svalue);
1861  var->svalue = NULL;
1862  var->value = *value;
1863 
1864  return true;
1865 }
1866 
1867 /* Assign an integer value to a variable, creating it if need be */
1868 /* Returns false on failure (bad name) */
1869 static bool
1870 putVariableInt(Variables *variables, const char *context, char *name,
1871  int64 value)
1872 {
1873  PgBenchValue val;
1874 
1875  setIntValue(&val, value);
1876  return putVariableValue(variables, context, name, &val);
1877 }
1878 
1879 /*
1880  * Parse a possible variable reference (:varname).
1881  *
1882  * "sql" points at a colon. If what follows it looks like a valid
1883  * variable name, return a malloc'd string containing the variable name,
1884  * and set *eaten to the number of characters consumed (including the colon).
1885  * Otherwise, return NULL.
1886  */
1887 static char *
1888 parseVariable(const char *sql, int *eaten)
1889 {
1890  int i = 1; /* starting at 1 skips the colon */
1891  char *name;
1892 
1893  /* keep this logic in sync with valid_variable_name() */
1894  if (IS_HIGHBIT_SET(sql[i]) ||
1895  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1896  "_", sql[i]) != NULL)
1897  i++;
1898  else
1899  return NULL;
1900 
1901  while (IS_HIGHBIT_SET(sql[i]) ||
1902  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1903  "_0123456789", sql[i]) != NULL)
1904  i++;
1905 
1906  name = pg_malloc(i);
1907  memcpy(name, &sql[1], i - 1);
1908  name[i - 1] = '\0';
1909 
1910  *eaten = i;
1911  return name;
1912 }
1913 
1914 static char *
1915 replaceVariable(char **sql, char *param, int len, char *value)
1916 {
1917  int valueln = strlen(value);
1918 
1919  if (valueln > len)
1920  {
1921  size_t offset = param - *sql;
1922 
1923  *sql = pg_realloc(*sql, strlen(*sql) - len + valueln + 1);
1924  param = *sql + offset;
1925  }
1926 
1927  if (valueln != len)
1928  memmove(param + valueln, param + len, strlen(param + len) + 1);
1929  memcpy(param, value, valueln);
1930 
1931  return param + valueln;
1932 }
1933 
1934 static char *
1935 assignVariables(Variables *variables, char *sql)
1936 {
1937  char *p,
1938  *name,
1939  *val;
1940 
1941  p = sql;
1942  while ((p = strchr(p, ':')) != NULL)
1943  {
1944  int eaten;
1945 
1946  name = parseVariable(p, &eaten);
1947  if (name == NULL)
1948  {
1949  while (*p == ':')
1950  {
1951  p++;
1952  }
1953  continue;
1954  }
1955 
1956  val = getVariable(variables, name);
1957  free(name);
1958  if (val == NULL)
1959  {
1960  p++;
1961  continue;
1962  }
1963 
1964  p = replaceVariable(&sql, p, eaten, val);
1965  }
1966 
1967  return sql;
1968 }
1969 
1970 static void
1971 getQueryParams(Variables *variables, const Command *command,
1972  const char **params)
1973 {
1974  int i;
1975 
1976  for (i = 0; i < command->argc - 1; i++)
1977  params[i] = getVariable(variables, command->argv[i + 1]);
1978 }
1979 
1980 static char *
1982 {
1983  if (pval->type == PGBT_NO_VALUE)
1984  return "none";
1985  else if (pval->type == PGBT_NULL)
1986  return "null";
1987  else if (pval->type == PGBT_INT)
1988  return "int";
1989  else if (pval->type == PGBT_DOUBLE)
1990  return "double";
1991  else if (pval->type == PGBT_BOOLEAN)
1992  return "boolean";
1993  else
1994  {
1995  /* internal error, should never get there */
1996  Assert(false);
1997  return NULL;
1998  }
1999 }
2000 
2001 /* get a value as a boolean, or tell if there is a problem */
2002 static bool
2003 coerceToBool(PgBenchValue *pval, bool *bval)
2004 {
2005  if (pval->type == PGBT_BOOLEAN)
2006  {
2007  *bval = pval->u.bval;
2008  return true;
2009  }
2010  else /* NULL, INT or DOUBLE */
2011  {
2012  pg_log_error("cannot coerce %s to boolean", valueTypeName(pval));
2013  *bval = false; /* suppress uninitialized-variable warnings */
2014  return false;
2015  }
2016 }
2017 
2018 /*
2019  * Return true or false from an expression for conditional purposes.
2020  * Non zero numerical values are true, zero and NULL are false.
2021  */
2022 static bool
2023 valueTruth(PgBenchValue *pval)
2024 {
2025  switch (pval->type)
2026  {
2027  case PGBT_NULL:
2028  return false;
2029  case PGBT_BOOLEAN:
2030  return pval->u.bval;
2031  case PGBT_INT:
2032  return pval->u.ival != 0;
2033  case PGBT_DOUBLE:
2034  return pval->u.dval != 0.0;
2035  default:
2036  /* internal error, unexpected type */
2037  Assert(0);
2038  return false;
2039  }
2040 }
2041 
2042 /* get a value as an int, tell if there is a problem */
2043 static bool
2044 coerceToInt(PgBenchValue *pval, int64 *ival)
2045 {
2046  if (pval->type == PGBT_INT)
2047  {
2048  *ival = pval->u.ival;
2049  return true;
2050  }
2051  else if (pval->type == PGBT_DOUBLE)
2052  {
2053  double dval = rint(pval->u.dval);
2054 
2055  if (isnan(dval) || !FLOAT8_FITS_IN_INT64(dval))
2056  {
2057  pg_log_error("double to int overflow for %f", dval);
2058  return false;
2059  }
2060  *ival = (int64) dval;
2061  return true;
2062  }
2063  else /* BOOLEAN or NULL */
2064  {
2065  pg_log_error("cannot coerce %s to int", valueTypeName(pval));
2066  return false;
2067  }
2068 }
2069 
2070 /* get a value as a double, or tell if there is a problem */
2071 static bool
2072 coerceToDouble(PgBenchValue *pval, double *dval)
2073 {
2074  if (pval->type == PGBT_DOUBLE)
2075  {
2076  *dval = pval->u.dval;
2077  return true;
2078  }
2079  else if (pval->type == PGBT_INT)
2080  {
2081  *dval = (double) pval->u.ival;
2082  return true;
2083  }
2084  else /* BOOLEAN or NULL */
2085  {
2086  pg_log_error("cannot coerce %s to double", valueTypeName(pval));
2087  return false;
2088  }
2089 }
2090 
2091 /* assign a null value */
2092 static void
2094 {
2095  pv->type = PGBT_NULL;
2096  pv->u.ival = 0;
2097 }
2098 
2099 /* assign a boolean value */
2100 static void
2101 setBoolValue(PgBenchValue *pv, bool bval)
2102 {
2103  pv->type = PGBT_BOOLEAN;
2104  pv->u.bval = bval;
2105 }
2106 
2107 /* assign an integer value */
2108 static void
2109 setIntValue(PgBenchValue *pv, int64 ival)
2110 {
2111  pv->type = PGBT_INT;
2112  pv->u.ival = ival;
2113 }
2114 
2115 /* assign a double value */
2116 static void
2117 setDoubleValue(PgBenchValue *pv, double dval)
2118 {
2119  pv->type = PGBT_DOUBLE;
2120  pv->u.dval = dval;
2121 }
2122 
2123 static bool
2125 {
2126  return func == PGBENCH_AND || func == PGBENCH_OR || func == PGBENCH_CASE;
2127 }
2128 
2129 /* lazy evaluation of some functions */
2130 static bool
2131 evalLazyFunc(CState *st,
2133 {
2134  PgBenchValue a1,
2135  a2;
2136  bool ba1,
2137  ba2;
2138 
2139  Assert(isLazyFunc(func) && args != NULL && args->next != NULL);
2140 
2141  /* args points to first condition */
2142  if (!evaluateExpr(st, args->expr, &a1))
2143  return false;
2144 
2145  /* second condition for AND/OR and corresponding branch for CASE */
2146  args = args->next;
2147 
2148  switch (func)
2149  {
2150  case PGBENCH_AND:
2151  if (a1.type == PGBT_NULL)
2152  {
2153  setNullValue(retval);
2154  return true;
2155  }
2156 
2157  if (!coerceToBool(&a1, &ba1))
2158  return false;
2159 
2160  if (!ba1)
2161  {
2162  setBoolValue(retval, false);
2163  return true;
2164  }
2165 
2166  if (!evaluateExpr(st, args->expr, &a2))
2167  return false;
2168 
2169  if (a2.type == PGBT_NULL)
2170  {
2171  setNullValue(retval);
2172  return true;
2173  }
2174  else if (!coerceToBool(&a2, &ba2))
2175  return false;
2176  else
2177  {
2178  setBoolValue(retval, ba2);
2179  return true;
2180  }
2181 
2182  return true;
2183 
2184  case PGBENCH_OR:
2185 
2186  if (a1.type == PGBT_NULL)
2187  {
2188  setNullValue(retval);
2189  return true;
2190  }
2191 
2192  if (!coerceToBool(&a1, &ba1))
2193  return false;
2194 
2195  if (ba1)
2196  {
2197  setBoolValue(retval, true);
2198  return true;
2199  }
2200 
2201  if (!evaluateExpr(st, args->expr, &a2))
2202  return false;
2203 
2204  if (a2.type == PGBT_NULL)
2205  {
2206  setNullValue(retval);
2207  return true;
2208  }
2209  else if (!coerceToBool(&a2, &ba2))
2210  return false;
2211  else
2212  {
2213  setBoolValue(retval, ba2);
2214  return true;
2215  }
2216 
2217  case PGBENCH_CASE:
2218  /* when true, execute branch */
2219  if (valueTruth(&a1))
2220  return evaluateExpr(st, args->expr, retval);
2221 
2222  /* now args contains next condition or final else expression */
2223  args = args->next;
2224 
2225  /* final else case? */
2226  if (args->next == NULL)
2227  return evaluateExpr(st, args->expr, retval);
2228 
2229  /* no, another when, proceed */
2230  return evalLazyFunc(st, PGBENCH_CASE, args, retval);
2231 
2232  default:
2233  /* internal error, cannot get here */
2234  Assert(0);
2235  break;
2236  }
2237  return false;
2238 }
2239 
2240 /* maximum number of function arguments */
2241 #define MAX_FARGS 16
2242 
2243 /*
2244  * Recursive evaluation of standard functions,
2245  * which do not require lazy evaluation.
2246  */
2247 static bool
2250  PgBenchValue *retval)
2251 {
2252  /* evaluate all function arguments */
2253  int nargs = 0;
2254  PgBenchExprLink *l = args;
2255  bool has_null = false;
2256 
2257  /*
2258  * This value is double braced to workaround GCC bug 53119, which seems to
2259  * exist at least on gcc (Debian 4.7.2-5) 4.7.2, 32-bit.
2260  */
2261  PgBenchValue vargs[MAX_FARGS] = {{0}};
2262 
2263  for (nargs = 0; nargs < MAX_FARGS && l != NULL; nargs++, l = l->next)
2264  {
2265  if (!evaluateExpr(st, l->expr, &vargs[nargs]))
2266  return false;
2267  has_null |= vargs[nargs].type == PGBT_NULL;
2268  }
2269 
2270  if (l != NULL)
2271  {
2272  pg_log_error("too many function arguments, maximum is %d", MAX_FARGS);
2273  return false;
2274  }
2275 
2276  /* NULL arguments */
2277  if (has_null && func != PGBENCH_IS && func != PGBENCH_DEBUG)
2278  {
2279  setNullValue(retval);
2280  return true;
2281  }
2282 
2283  /* then evaluate function */
2284  switch (func)
2285  {
2286  /* overloaded operators */
2287  case PGBENCH_ADD:
2288  case PGBENCH_SUB:
2289  case PGBENCH_MUL:
2290  case PGBENCH_DIV:
2291  case PGBENCH_MOD:
2292  case PGBENCH_EQ:
2293  case PGBENCH_NE:
2294  case PGBENCH_LE:
2295  case PGBENCH_LT:
2296  {
2297  PgBenchValue *lval = &vargs[0],
2298  *rval = &vargs[1];
2299 
2300  Assert(nargs == 2);
2301 
2302  /* overloaded type management, double if some double */
2303  if ((lval->type == PGBT_DOUBLE ||
2304  rval->type == PGBT_DOUBLE) && func != PGBENCH_MOD)
2305  {
2306  double ld,
2307  rd;
2308 
2309  if (!coerceToDouble(lval, &ld) ||
2310  !coerceToDouble(rval, &rd))
2311  return false;
2312 
2313  switch (func)
2314  {
2315  case PGBENCH_ADD:
2316  setDoubleValue(retval, ld + rd);
2317  return true;
2318 
2319  case PGBENCH_SUB:
2320  setDoubleValue(retval, ld - rd);
2321  return true;
2322 
2323  case PGBENCH_MUL:
2324  setDoubleValue(retval, ld * rd);
2325  return true;
2326 
2327  case PGBENCH_DIV:
2328  setDoubleValue(retval, ld / rd);
2329  return true;
2330 
2331  case PGBENCH_EQ:
2332  setBoolValue(retval, ld == rd);
2333  return true;
2334 
2335  case PGBENCH_NE:
2336  setBoolValue(retval, ld != rd);
2337  return true;
2338 
2339  case PGBENCH_LE:
2340  setBoolValue(retval, ld <= rd);
2341  return true;
2342 
2343  case PGBENCH_LT:
2344  setBoolValue(retval, ld < rd);
2345  return true;
2346 
2347  default:
2348  /* cannot get here */
2349  Assert(0);
2350  }
2351  }
2352  else /* we have integer operands, or % */
2353  {
2354  int64 li,
2355  ri,
2356  res;
2357 
2358  if (!coerceToInt(lval, &li) ||
2359  !coerceToInt(rval, &ri))
2360  return false;
2361 
2362  switch (func)
2363  {
2364  case PGBENCH_ADD:
2365  if (pg_add_s64_overflow(li, ri, &res))
2366  {
2367  pg_log_error("bigint add out of range");
2368  return false;
2369  }
2370  setIntValue(retval, res);
2371  return true;
2372 
2373  case PGBENCH_SUB:
2374  if (pg_sub_s64_overflow(li, ri, &res))
2375  {
2376  pg_log_error("bigint sub out of range");
2377  return false;
2378  }
2379  setIntValue(retval, res);
2380  return true;
2381 
2382  case PGBENCH_MUL:
2383  if (pg_mul_s64_overflow(li, ri, &res))
2384  {
2385  pg_log_error("bigint mul out of range");
2386  return false;
2387  }
2388  setIntValue(retval, res);
2389  return true;
2390 
2391  case PGBENCH_EQ:
2392  setBoolValue(retval, li == ri);
2393  return true;
2394 
2395  case PGBENCH_NE:
2396  setBoolValue(retval, li != ri);
2397  return true;
2398 
2399  case PGBENCH_LE:
2400  setBoolValue(retval, li <= ri);
2401  return true;
2402 
2403  case PGBENCH_LT:
2404  setBoolValue(retval, li < ri);
2405  return true;
2406 
2407  case PGBENCH_DIV:
2408  case PGBENCH_MOD:
2409  if (ri == 0)
2410  {
2411  pg_log_error("division by zero");
2412  return false;
2413  }
2414  /* special handling of -1 divisor */
2415  if (ri == -1)
2416  {
2417  if (func == PGBENCH_DIV)
2418  {
2419  /* overflow check (needed for INT64_MIN) */
2420  if (li == PG_INT64_MIN)
2421  {
2422  pg_log_error("bigint div out of range");
2423  return false;
2424  }
2425  else
2426  setIntValue(retval, -li);
2427  }
2428  else
2429  setIntValue(retval, 0);
2430  return true;
2431  }
2432  /* else divisor is not -1 */
2433  if (func == PGBENCH_DIV)
2434  setIntValue(retval, li / ri);
2435  else /* func == PGBENCH_MOD */
2436  setIntValue(retval, li % ri);
2437 
2438  return true;
2439 
2440  default:
2441  /* cannot get here */
2442  Assert(0);
2443  }
2444  }
2445 
2446  Assert(0);
2447  return false; /* NOTREACHED */
2448  }
2449 
2450  /* integer bitwise operators */
2451  case PGBENCH_BITAND:
2452  case PGBENCH_BITOR:
2453  case PGBENCH_BITXOR:
2454  case PGBENCH_LSHIFT:
2455  case PGBENCH_RSHIFT:
2456  {
2457  int64 li,
2458  ri;
2459 
2460  if (!coerceToInt(&vargs[0], &li) || !coerceToInt(&vargs[1], &ri))
2461  return false;
2462 
2463  if (func == PGBENCH_BITAND)
2464  setIntValue(retval, li & ri);
2465  else if (func == PGBENCH_BITOR)
2466  setIntValue(retval, li | ri);
2467  else if (func == PGBENCH_BITXOR)
2468  setIntValue(retval, li ^ ri);
2469  else if (func == PGBENCH_LSHIFT)
2470  setIntValue(retval, li << ri);
2471  else if (func == PGBENCH_RSHIFT)
2472  setIntValue(retval, li >> ri);
2473  else /* cannot get here */
2474  Assert(0);
2475 
2476  return true;
2477  }
2478 
2479  /* logical operators */
2480  case PGBENCH_NOT:
2481  {
2482  bool b;
2483 
2484  if (!coerceToBool(&vargs[0], &b))
2485  return false;
2486 
2487  setBoolValue(retval, !b);
2488  return true;
2489  }
2490 
2491  /* no arguments */
2492  case PGBENCH_PI:
2493  setDoubleValue(retval, M_PI);
2494  return true;
2495 
2496  /* 1 overloaded argument */
2497  case PGBENCH_ABS:
2498  {
2499  PgBenchValue *varg = &vargs[0];
2500 
2501  Assert(nargs == 1);
2502 
2503  if (varg->type == PGBT_INT)
2504  {
2505  int64 i = varg->u.ival;
2506 
2507  setIntValue(retval, i < 0 ? -i : i);
2508  }
2509  else
2510  {
2511  double d = varg->u.dval;
2512 
2513  Assert(varg->type == PGBT_DOUBLE);
2514  setDoubleValue(retval, d < 0.0 ? -d : d);
2515  }
2516 
2517  return true;
2518  }
2519 
2520  case PGBENCH_DEBUG:
2521  {
2522  PgBenchValue *varg = &vargs[0];
2523 
2524  Assert(nargs == 1);
2525 
2526  fprintf(stderr, "debug(script=%d,command=%d): ",
2527  st->use_file, st->command + 1);
2528 
2529  if (varg->type == PGBT_NULL)
2530  fprintf(stderr, "null\n");
2531  else if (varg->type == PGBT_BOOLEAN)
2532  fprintf(stderr, "boolean %s\n", varg->u.bval ? "true" : "false");
2533  else if (varg->type == PGBT_INT)
2534  fprintf(stderr, "int " INT64_FORMAT "\n", varg->u.ival);
2535  else if (varg->type == PGBT_DOUBLE)
2536  fprintf(stderr, "double %.*g\n", DBL_DIG, varg->u.dval);
2537  else /* internal error, unexpected type */
2538  Assert(0);
2539 
2540  *retval = *varg;
2541 
2542  return true;
2543  }
2544 
2545  /* 1 double argument */
2546  case PGBENCH_DOUBLE:
2547  case PGBENCH_SQRT:
2548  case PGBENCH_LN:
2549  case PGBENCH_EXP:
2550  {
2551  double dval;
2552 
2553  Assert(nargs == 1);
2554 
2555  if (!coerceToDouble(&vargs[0], &dval))
2556  return false;
2557 
2558  if (func == PGBENCH_SQRT)
2559  dval = sqrt(dval);
2560  else if (func == PGBENCH_LN)
2561  dval = log(dval);
2562  else if (func == PGBENCH_EXP)
2563  dval = exp(dval);
2564  /* else is cast: do nothing */
2565 
2566  setDoubleValue(retval, dval);
2567  return true;
2568  }
2569 
2570  /* 1 int argument */
2571  case PGBENCH_INT:
2572  {
2573  int64 ival;
2574 
2575  Assert(nargs == 1);
2576 
2577  if (!coerceToInt(&vargs[0], &ival))
2578  return false;
2579 
2580  setIntValue(retval, ival);
2581  return true;
2582  }
2583 
2584  /* variable number of arguments */
2585  case PGBENCH_LEAST:
2586  case PGBENCH_GREATEST:
2587  {
2588  bool havedouble;
2589  int i;
2590 
2591  Assert(nargs >= 1);
2592 
2593  /* need double result if any input is double */
2594  havedouble = false;
2595  for (i = 0; i < nargs; i++)
2596  {
2597  if (vargs[i].type == PGBT_DOUBLE)
2598  {
2599  havedouble = true;
2600  break;
2601  }
2602  }
2603  if (havedouble)
2604  {
2605  double extremum;
2606 
2607  if (!coerceToDouble(&vargs[0], &extremum))
2608  return false;
2609  for (i = 1; i < nargs; i++)
2610  {
2611  double dval;
2612 
2613  if (!coerceToDouble(&vargs[i], &dval))
2614  return false;
2615  if (func == PGBENCH_LEAST)
2616  extremum = Min(extremum, dval);
2617  else
2618  extremum = Max(extremum, dval);
2619  }
2620  setDoubleValue(retval, extremum);
2621  }
2622  else
2623  {
2624  int64 extremum;
2625 
2626  if (!coerceToInt(&vargs[0], &extremum))
2627  return false;
2628  for (i = 1; i < nargs; i++)
2629  {
2630  int64 ival;
2631 
2632  if (!coerceToInt(&vargs[i], &ival))
2633  return false;
2634  if (func == PGBENCH_LEAST)
2635  extremum = Min(extremum, ival);
2636  else
2637  extremum = Max(extremum, ival);
2638  }
2639  setIntValue(retval, extremum);
2640  }
2641  return true;
2642  }
2643 
2644  /* random functions */
2645  case PGBENCH_RANDOM:
2649  {
2650  int64 imin,
2651  imax,
2652  delta;
2653 
2654  Assert(nargs >= 2);
2655 
2656  if (!coerceToInt(&vargs[0], &imin) ||
2657  !coerceToInt(&vargs[1], &imax))
2658  return false;
2659 
2660  /* check random range */
2661  if (unlikely(imin > imax))
2662  {
2663  pg_log_error("empty range given to random");
2664  return false;
2665  }
2666  else if (unlikely(pg_sub_s64_overflow(imax, imin, &delta) ||
2667  pg_add_s64_overflow(delta, 1, &delta)))
2668  {
2669  /* prevent int overflows in random functions */
2670  pg_log_error("random range is too large");
2671  return false;
2672  }
2673 
2674  if (func == PGBENCH_RANDOM)
2675  {
2676  Assert(nargs == 2);
2677  setIntValue(retval, getrand(&st->cs_func_rs, imin, imax));
2678  }
2679  else /* gaussian & exponential */
2680  {
2681  double param;
2682 
2683  Assert(nargs == 3);
2684 
2685  if (!coerceToDouble(&vargs[2], &param))
2686  return false;
2687 
2688  if (func == PGBENCH_RANDOM_GAUSSIAN)
2689  {
2690  if (param < MIN_GAUSSIAN_PARAM)
2691  {
2692  pg_log_error("gaussian parameter must be at least %f (not %f)",
2693  MIN_GAUSSIAN_PARAM, param);
2694  return false;
2695  }
2696 
2697  setIntValue(retval,
2699  imin, imax, param));
2700  }
2701  else if (func == PGBENCH_RANDOM_ZIPFIAN)
2702  {
2703  if (param < MIN_ZIPFIAN_PARAM || param > MAX_ZIPFIAN_PARAM)
2704  {
2705  pg_log_error("zipfian parameter must be in range [%.3f, %.0f] (not %f)",
2707  return false;
2708  }
2709 
2710  setIntValue(retval,
2711  getZipfianRand(&st->cs_func_rs, imin, imax, param));
2712  }
2713  else /* exponential */
2714  {
2715  if (param <= 0.0)
2716  {
2717  pg_log_error("exponential parameter must be greater than zero (not %f)",
2718  param);
2719  return false;
2720  }
2721 
2722  setIntValue(retval,
2724  imin, imax, param));
2725  }
2726  }
2727 
2728  return true;
2729  }
2730 
2731  case PGBENCH_POW:
2732  {
2733  PgBenchValue *lval = &vargs[0];
2734  PgBenchValue *rval = &vargs[1];
2735  double ld,
2736  rd;
2737 
2738  Assert(nargs == 2);
2739 
2740  if (!coerceToDouble(lval, &ld) ||
2741  !coerceToDouble(rval, &rd))
2742  return false;
2743 
2744  setDoubleValue(retval, pow(ld, rd));
2745 
2746  return true;
2747  }
2748 
2749  case PGBENCH_IS:
2750  {
2751  Assert(nargs == 2);
2752 
2753  /*
2754  * note: this simple implementation is more permissive than
2755  * SQL
2756  */
2757  setBoolValue(retval,
2758  vargs[0].type == vargs[1].type &&
2759  vargs[0].u.bval == vargs[1].u.bval);
2760  return true;
2761  }
2762 
2763  /* hashing */
2764  case PGBENCH_HASH_FNV1A:
2765  case PGBENCH_HASH_MURMUR2:
2766  {
2767  int64 val,
2768  seed;
2769 
2770  Assert(nargs == 2);
2771 
2772  if (!coerceToInt(&vargs[0], &val) ||
2773  !coerceToInt(&vargs[1], &seed))
2774  return false;
2775 
2776  if (func == PGBENCH_HASH_MURMUR2)
2777  setIntValue(retval, getHashMurmur2(val, seed));
2778  else if (func == PGBENCH_HASH_FNV1A)
2779  setIntValue(retval, getHashFnv1a(val, seed));
2780  else
2781  /* cannot get here */
2782  Assert(0);
2783 
2784  return true;
2785  }
2786 
2787  case PGBENCH_PERMUTE:
2788  {
2789  int64 val,
2790  size,
2791  seed;
2792 
2793  Assert(nargs == 3);
2794 
2795  if (!coerceToInt(&vargs[0], &val) ||
2796  !coerceToInt(&vargs[1], &size) ||
2797  !coerceToInt(&vargs[2], &seed))
2798  return false;
2799 
2800  if (size <= 0)
2801  {
2802  pg_log_error("permute size parameter must be greater than zero");
2803  return false;
2804  }
2805 
2806  setIntValue(retval, permute(val, size, seed));
2807  return true;
2808  }
2809 
2810  default:
2811  /* cannot get here */
2812  Assert(0);
2813  /* dead code to avoid a compiler warning */
2814  return false;
2815  }
2816 }
2817 
2818 /* evaluate some function */
2819 static bool
2820 evalFunc(CState *st,
2822 {
2823  if (isLazyFunc(func))
2824  return evalLazyFunc(st, func, args, retval);
2825  else
2826  return evalStandardFunc(st, func, args, retval);
2827 }
2828 
2829 /*
2830  * Recursive evaluation of an expression in a pgbench script
2831  * using the current state of variables.
2832  * Returns whether the evaluation was ok,
2833  * the value itself is returned through the retval pointer.
2834  */
2835 static bool
2836 evaluateExpr(CState *st, PgBenchExpr *expr, PgBenchValue *retval)
2837 {
2838  switch (expr->etype)
2839  {
2840  case ENODE_CONSTANT:
2841  {
2842  *retval = expr->u.constant;
2843  return true;
2844  }
2845 
2846  case ENODE_VARIABLE:
2847  {
2848  Variable *var;
2849 
2850  if ((var = lookupVariable(&st->variables, expr->u.variable.varname)) == NULL)
2851  {
2852  pg_log_error("undefined variable \"%s\"", expr->u.variable.varname);
2853  return false;
2854  }
2855 
2856  if (!makeVariableValue(var))
2857  return false;
2858 
2859  *retval = var->value;
2860  return true;
2861  }
2862 
2863  case ENODE_FUNCTION:
2864  return evalFunc(st,
2865  expr->u.function.function,
2866  expr->u.function.args,
2867  retval);
2868 
2869  default:
2870  /* internal error which should never occur */
2871  pg_fatal("unexpected enode type in evaluation: %d", expr->etype);
2872  }
2873 }
2874 
2875 /*
2876  * Convert command name to meta-command enum identifier
2877  */
2879 getMetaCommand(const char *cmd)
2880 {
2881  MetaCommand mc;
2882 
2883  if (cmd == NULL)
2884  mc = META_NONE;
2885  else if (pg_strcasecmp(cmd, "set") == 0)
2886  mc = META_SET;
2887  else if (pg_strcasecmp(cmd, "setshell") == 0)
2888  mc = META_SETSHELL;
2889  else if (pg_strcasecmp(cmd, "shell") == 0)
2890  mc = META_SHELL;
2891  else if (pg_strcasecmp(cmd, "sleep") == 0)
2892  mc = META_SLEEP;
2893  else if (pg_strcasecmp(cmd, "if") == 0)
2894  mc = META_IF;
2895  else if (pg_strcasecmp(cmd, "elif") == 0)
2896  mc = META_ELIF;
2897  else if (pg_strcasecmp(cmd, "else") == 0)
2898  mc = META_ELSE;
2899  else if (pg_strcasecmp(cmd, "endif") == 0)
2900  mc = META_ENDIF;
2901  else if (pg_strcasecmp(cmd, "gset") == 0)
2902  mc = META_GSET;
2903  else if (pg_strcasecmp(cmd, "aset") == 0)
2904  mc = META_ASET;
2905  else if (pg_strcasecmp(cmd, "startpipeline") == 0)
2906  mc = META_STARTPIPELINE;
2907  else if (pg_strcasecmp(cmd, "syncpipeline") == 0)
2908  mc = META_SYNCPIPELINE;
2909  else if (pg_strcasecmp(cmd, "endpipeline") == 0)
2910  mc = META_ENDPIPELINE;
2911  else
2912  mc = META_NONE;
2913  return mc;
2914 }
2915 
2916 /*
2917  * Run a shell command. The result is assigned to the variable if not NULL.
2918  * Return true if succeeded, or false on error.
2919  */
2920 static bool
2921 runShellCommand(Variables *variables, char *variable, char **argv, int argc)
2922 {
2923  char command[SHELL_COMMAND_SIZE];
2924  int i,
2925  len = 0;
2926  FILE *fp;
2927  char res[64];
2928  char *endptr;
2929  int retval;
2930 
2931  /*----------
2932  * Join arguments with whitespace separators. Arguments starting with
2933  * exactly one colon are treated as variables:
2934  * name - append a string "name"
2935  * :var - append a variable named 'var'
2936  * ::name - append a string ":name"
2937  *----------
2938  */
2939  for (i = 0; i < argc; i++)
2940  {
2941  char *arg;
2942  int arglen;
2943 
2944  if (argv[i][0] != ':')
2945  {
2946  arg = argv[i]; /* a string literal */
2947  }
2948  else if (argv[i][1] == ':')
2949  {
2950  arg = argv[i] + 1; /* a string literal starting with colons */
2951  }
2952  else if ((arg = getVariable(variables, argv[i] + 1)) == NULL)
2953  {
2954  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[i]);
2955  return false;
2956  }
2957 
2958  arglen = strlen(arg);
2959  if (len + arglen + (i > 0 ? 1 : 0) >= SHELL_COMMAND_SIZE - 1)
2960  {
2961  pg_log_error("%s: shell command is too long", argv[0]);
2962  return false;
2963  }
2964 
2965  if (i > 0)
2966  command[len++] = ' ';
2967  memcpy(command + len, arg, arglen);
2968  len += arglen;
2969  }
2970 
2971  command[len] = '\0';
2972 
2973  fflush(NULL); /* needed before either system() or popen() */
2974 
2975  /* Fast path for non-assignment case */
2976  if (variable == NULL)
2977  {
2978  if (system(command))
2979  {
2980  if (!timer_exceeded)
2981  pg_log_error("%s: could not launch shell command", argv[0]);
2982  return false;
2983  }
2984  return true;
2985  }
2986 
2987  /* Execute the command with pipe and read the standard output. */
2988  if ((fp = popen(command, "r")) == NULL)
2989  {
2990  pg_log_error("%s: could not launch shell command", argv[0]);
2991  return false;
2992  }
2993  if (fgets(res, sizeof(res), fp) == NULL)
2994  {
2995  if (!timer_exceeded)
2996  pg_log_error("%s: could not read result of shell command", argv[0]);
2997  (void) pclose(fp);
2998  return false;
2999  }
3000  if (pclose(fp) < 0)
3001  {
3002  pg_log_error("%s: could not run shell command: %m", argv[0]);
3003  return false;
3004  }
3005 
3006  /* Check whether the result is an integer and assign it to the variable */
3007  retval = (int) strtol(res, &endptr, 10);
3008  while (*endptr != '\0' && isspace((unsigned char) *endptr))
3009  endptr++;
3010  if (*res == '\0' || *endptr != '\0')
3011  {
3012  pg_log_error("%s: shell command must return an integer (not \"%s\")", argv[0], res);
3013  return false;
3014  }
3015  if (!putVariableInt(variables, "setshell", variable, retval))
3016  return false;
3017 
3018  pg_log_debug("%s: shell parameter name: \"%s\", value: \"%s\"", argv[0], argv[1], res);
3019 
3020  return true;
3021 }
3022 
3023 /*
3024  * Report the abortion of the client when processing SQL commands.
3025  */
3026 static void
3027 commandFailed(CState *st, const char *cmd, const char *message)
3028 {
3029  pg_log_error("client %d aborted in command %d (%s) of script %d; %s",
3030  st->id, st->command, cmd, st->use_file, message);
3031 }
3032 
3033 /*
3034  * Report the error in the command while the script is executing.
3035  */
3036 static void
3037 commandError(CState *st, const char *message)
3038 {
3040  pg_log_info("client %d got an error in command %d (SQL) of script %d; %s",
3041  st->id, st->command, st->use_file, message);
3042 }
3043 
3044 /* return a script number with a weighted choice. */
3045 static int
3046 chooseScript(TState *thread)
3047 {
3048  int i = 0;
3049  int64 w;
3050 
3051  if (num_scripts == 1)
3052  return 0;
3053 
3054  w = getrand(&thread->ts_choose_rs, 0, total_weight - 1);
3055  do
3056  {
3057  w -= sql_script[i++].weight;
3058  } while (w >= 0);
3059 
3060  return i - 1;
3061 }
3062 
3063 /*
3064  * Allocate space for CState->prepared: we need one boolean for each command
3065  * of each script.
3066  */
3067 static void
3069 {
3070  Assert(st->prepared == NULL);
3071 
3072  st->prepared = pg_malloc(sizeof(bool *) * num_scripts);
3073  for (int i = 0; i < num_scripts; i++)
3074  {
3075  ParsedScript *script = &sql_script[i];
3076  int numcmds;
3077 
3078  for (numcmds = 0; script->commands[numcmds] != NULL; numcmds++)
3079  ;
3080  st->prepared[i] = pg_malloc0(sizeof(bool) * numcmds);
3081  }
3082 }
3083 
3084 /*
3085  * Prepare the SQL command from st->use_file at command_num.
3086  */
3087 static void
3088 prepareCommand(CState *st, int command_num)
3089 {
3090  Command *command = sql_script[st->use_file].commands[command_num];
3091 
3092  /* No prepare for non-SQL commands */
3093  if (command->type != SQL_COMMAND)
3094  return;
3095 
3096  if (!st->prepared)
3097  allocCStatePrepared(st);
3098 
3099  if (!st->prepared[st->use_file][command_num])
3100  {
3101  PGresult *res;
3102 
3103  pg_log_debug("client %d preparing %s", st->id, command->prepname);
3104  res = PQprepare(st->con, command->prepname,
3105  command->argv[0], command->argc - 1, NULL);
3107  pg_log_error("%s", PQerrorMessage(st->con));
3108  PQclear(res);
3109  st->prepared[st->use_file][command_num] = true;
3110  }
3111 }
3112 
3113 /*
3114  * Prepare all the commands in the script that come after the \startpipeline
3115  * that's at position st->command, and the first \endpipeline we find.
3116  *
3117  * This sets the ->prepared flag for each relevant command as well as the
3118  * \startpipeline itself, but doesn't move the st->command counter.
3119  */
3120 static void
3122 {
3123  int j;
3124  Command **commands = sql_script[st->use_file].commands;
3125 
3126  Assert(commands[st->command]->type == META_COMMAND &&
3127  commands[st->command]->meta == META_STARTPIPELINE);
3128 
3129  if (!st->prepared)
3130  allocCStatePrepared(st);
3131 
3132  /*
3133  * We set the 'prepared' flag on the \startpipeline itself to flag that we
3134  * don't need to do this next time without calling prepareCommand(), even
3135  * though we don't actually prepare this command.
3136  */
3137  if (st->prepared[st->use_file][st->command])
3138  return;
3139 
3140  for (j = st->command + 1; commands[j] != NULL; j++)
3141  {
3142  if (commands[j]->type == META_COMMAND &&
3143  commands[j]->meta == META_ENDPIPELINE)
3144  break;
3145 
3146  prepareCommand(st, j);
3147  }
3148 
3149  st->prepared[st->use_file][st->command] = true;
3150 }
3151 
3152 /* Send a SQL command, using the chosen querymode */
3153 static bool
3154 sendCommand(CState *st, Command *command)
3155 {
3156  int r;
3157 
3158  if (querymode == QUERY_SIMPLE)
3159  {
3160  char *sql;
3161 
3162  sql = pg_strdup(command->argv[0]);
3163  sql = assignVariables(&st->variables, sql);
3164 
3165  pg_log_debug("client %d sending %s", st->id, sql);
3166  r = PQsendQuery(st->con, sql);
3167  free(sql);
3168  }
3169  else if (querymode == QUERY_EXTENDED)
3170  {
3171  const char *sql = command->argv[0];
3172  const char *params[MAX_ARGS];
3173 
3174  getQueryParams(&st->variables, command, params);
3175 
3176  pg_log_debug("client %d sending %s", st->id, sql);
3177  r = PQsendQueryParams(st->con, sql, command->argc - 1,
3178  NULL, params, NULL, NULL, 0);
3179  }
3180  else if (querymode == QUERY_PREPARED)
3181  {
3182  const char *params[MAX_ARGS];
3183 
3184  prepareCommand(st, st->command);
3185  getQueryParams(&st->variables, command, params);
3186 
3187  pg_log_debug("client %d sending %s", st->id, command->prepname);
3188  r = PQsendQueryPrepared(st->con, command->prepname, command->argc - 1,
3189  params, NULL, NULL, 0);
3190  }
3191  else /* unknown sql mode */
3192  r = 0;
3193 
3194  if (r == 0)
3195  {
3196  pg_log_debug("client %d could not send %s", st->id, command->argv[0]);
3197  return false;
3198  }
3199  else
3200  return true;
3201 }
3202 
3203 /*
3204  * Get the error status from the error code.
3205  */
3206 static EStatus
3207 getSQLErrorStatus(const char *sqlState)
3208 {
3209  if (sqlState != NULL)
3210  {
3211  if (strcmp(sqlState, ERRCODE_T_R_SERIALIZATION_FAILURE) == 0)
3213  else if (strcmp(sqlState, ERRCODE_T_R_DEADLOCK_DETECTED) == 0)
3214  return ESTATUS_DEADLOCK_ERROR;
3215  }
3216 
3217  return ESTATUS_OTHER_SQL_ERROR;
3218 }
3219 
3220 /*
3221  * Returns true if this type of error can be retried.
3222  */
3223 static bool
3224 canRetryError(EStatus estatus)
3225 {
3226  return (estatus == ESTATUS_SERIALIZATION_ERROR ||
3227  estatus == ESTATUS_DEADLOCK_ERROR);
3228 }
3229 
3230 /*
3231  * Process query response from the backend.
3232  *
3233  * If varprefix is not NULL, it's the variable name prefix where to store
3234  * the results of the *last* command (META_GSET) or *all* commands
3235  * (META_ASET).
3236  *
3237  * Returns true if everything is A-OK, false if any error occurs.
3238  */
3239 static bool
3240 readCommandResponse(CState *st, MetaCommand meta, char *varprefix)
3241 {
3242  PGresult *res;
3243  PGresult *next_res;
3244  int qrynum = 0;
3245 
3246  /*
3247  * varprefix should be set only with \gset or \aset, and \endpipeline and
3248  * SQL commands do not need it.
3249  */
3250  Assert((meta == META_NONE && varprefix == NULL) ||
3251  ((meta == META_ENDPIPELINE) && varprefix == NULL) ||
3252  ((meta == META_GSET || meta == META_ASET) && varprefix != NULL));
3253 
3254  res = PQgetResult(st->con);
3255 
3256  while (res != NULL)
3257  {
3258  bool is_last;
3259 
3260  /* peek at the next result to know whether the current is last */
3261  next_res = PQgetResult(st->con);
3262  is_last = (next_res == NULL);
3263 
3264  switch (PQresultStatus(res))
3265  {
3266  case PGRES_COMMAND_OK: /* non-SELECT commands */
3267  case PGRES_EMPTY_QUERY: /* may be used for testing no-op overhead */
3268  if (is_last && meta == META_GSET)
3269  {
3270  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3271  st->id, st->use_file, st->command, qrynum, 0);
3273  goto error;
3274  }
3275  break;
3276 
3277  case PGRES_TUPLES_OK:
3278  if ((is_last && meta == META_GSET) || meta == META_ASET)
3279  {
3280  int ntuples = PQntuples(res);
3281 
3282  if (meta == META_GSET && ntuples != 1)
3283  {
3284  /* under \gset, report the error */
3285  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3286  st->id, st->use_file, st->command, qrynum, PQntuples(res));
3288  goto error;
3289  }
3290  else if (meta == META_ASET && ntuples <= 0)
3291  {
3292  /* coldly skip empty result under \aset */
3293  break;
3294  }
3295 
3296  /* store results into variables */
3297  for (int fld = 0; fld < PQnfields(res); fld++)
3298  {
3299  char *varname = PQfname(res, fld);
3300 
3301  /* allocate varname only if necessary, freed below */
3302  if (*varprefix != '\0')
3303  varname = psprintf("%s%s", varprefix, varname);
3304 
3305  /* store last row result as a string */
3306  if (!putVariable(&st->variables, meta == META_ASET ? "aset" : "gset", varname,
3307  PQgetvalue(res, ntuples - 1, fld)))
3308  {
3309  /* internal error */
3310  pg_log_error("client %d script %d command %d query %d: error storing into variable %s",
3311  st->id, st->use_file, st->command, qrynum, varname);
3313  goto error;
3314  }
3315 
3316  if (*varprefix != '\0')
3317  pg_free(varname);
3318  }
3319  }
3320  /* otherwise the result is simply thrown away by PQclear below */
3321  break;
3322 
3323  case PGRES_PIPELINE_SYNC:
3324  pg_log_debug("client %d pipeline ending, ongoing syncs: %d",
3325  st->id, st->num_syncs);
3326  st->num_syncs--;
3327  if (st->num_syncs == 0 && PQexitPipelineMode(st->con) != 1)
3328  pg_log_error("client %d failed to exit pipeline mode: %s", st->id,
3329  PQerrorMessage(st->con));
3330  break;
3331 
3332  case PGRES_NONFATAL_ERROR:
3333  case PGRES_FATAL_ERROR:
3335  PG_DIAG_SQLSTATE));
3336  if (canRetryError(st->estatus))
3337  {
3338  if (verbose_errors)
3339  commandError(st, PQerrorMessage(st->con));
3340  goto error;
3341  }
3342  /* fall through */
3343 
3344  default:
3345  /* anything else is unexpected */
3346  pg_log_error("client %d script %d aborted in command %d query %d: %s",
3347  st->id, st->use_file, st->command, qrynum,
3348  PQerrorMessage(st->con));
3349  goto error;
3350  }
3351 
3352  PQclear(res);
3353  qrynum++;
3354  res = next_res;
3355  }
3356 
3357  if (qrynum == 0)
3358  {
3359  pg_log_error("client %d command %d: no results", st->id, st->command);
3360  return false;
3361  }
3362 
3363  return true;
3364 
3365 error:
3366  PQclear(res);
3367  PQclear(next_res);
3368  do
3369  {
3370  res = PQgetResult(st->con);
3371  PQclear(res);
3372  } while (res);
3373 
3374  return false;
3375 }
3376 
3377 /*
3378  * Parse the argument to a \sleep command, and return the requested amount
3379  * of delay, in microseconds. Returns true on success, false on error.
3380  */
3381 static bool
3382 evaluateSleep(Variables *variables, int argc, char **argv, int *usecs)
3383 {
3384  char *var;
3385  int usec;
3386 
3387  if (*argv[1] == ':')
3388  {
3389  if ((var = getVariable(variables, argv[1] + 1)) == NULL)
3390  {
3391  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[1] + 1);
3392  return false;
3393  }
3394 
3395  usec = atoi(var);
3396 
3397  /* Raise an error if the value of a variable is not a number */
3398  if (usec == 0 && !isdigit((unsigned char) *var))
3399  {
3400  pg_log_error("%s: invalid sleep time \"%s\" for variable \"%s\"",
3401  argv[0], var, argv[1] + 1);
3402  return false;
3403  }
3404  }
3405  else
3406  usec = atoi(argv[1]);
3407 
3408  if (argc > 2)
3409  {
3410  if (pg_strcasecmp(argv[2], "ms") == 0)
3411  usec *= 1000;
3412  else if (pg_strcasecmp(argv[2], "s") == 0)
3413  usec *= 1000000;
3414  }
3415  else
3416  usec *= 1000000;
3417 
3418  *usecs = usec;
3419  return true;
3420 }
3421 
3422 
3423 /*
3424  * Returns true if the error can be retried.
3425  */
3426 static bool
3428 {
3430 
3431  /* We can only retry serialization or deadlock errors. */
3432  if (!canRetryError(st->estatus))
3433  return false;
3434 
3435  /*
3436  * We must have at least one option to limit the retrying of transactions
3437  * that got an error.
3438  */
3440 
3441  /*
3442  * We cannot retry the error if we have reached the maximum number of
3443  * tries.
3444  */
3445  if (max_tries && st->tries >= max_tries)
3446  return false;
3447 
3448  /*
3449  * We cannot retry the error if we spent too much time on this
3450  * transaction.
3451  */
3452  if (latency_limit)
3453  {
3455  if (*now - st->txn_scheduled > latency_limit)
3456  return false;
3457  }
3458 
3459  /*
3460  * We cannot retry the error if the benchmark duration is over.
3461  */
3462  if (timer_exceeded)
3463  return false;
3464 
3465  /* OK */
3466  return true;
3467 }
3468 
3469 /*
3470  * Read results and discard it until a sync point.
3471  */
3472 static int
3474 {
3475  /* send a sync */
3476  if (!PQpipelineSync(st->con))
3477  {
3478  pg_log_error("client %d aborted: failed to send a pipeline sync",
3479  st->id);
3480  return 0;
3481  }
3482 
3483  /* receive PGRES_PIPELINE_SYNC and null following it */
3484  for (;;)
3485  {
3486  PGresult *res = PQgetResult(st->con);
3487 
3489  {
3490  PQclear(res);
3491  res = PQgetResult(st->con);
3492  Assert(res == NULL);
3493  break;
3494  }
3495  PQclear(res);
3496  }
3497 
3498  /* exit pipeline */
3499  if (PQexitPipelineMode(st->con) != 1)
3500  {
3501  pg_log_error("client %d aborted: failed to exit pipeline mode for rolling back the failed transaction",
3502  st->id);
3503  return 0;
3504  }
3505  return 1;
3506 }
3507 
3508 /*
3509  * Get the transaction status at the end of a command especially for
3510  * checking if we are in a (failed) transaction block.
3511  */
3512 static TStatus
3514 {
3515  PGTransactionStatusType tx_status;
3516 
3517  tx_status = PQtransactionStatus(con);
3518  switch (tx_status)
3519  {
3520  case PQTRANS_IDLE:
3521  return TSTATUS_IDLE;
3522  case PQTRANS_INTRANS:
3523  case PQTRANS_INERROR:
3524  return TSTATUS_IN_BLOCK;
3525  case PQTRANS_UNKNOWN:
3526  /* PQTRANS_UNKNOWN is expected given a broken connection */
3527  if (PQstatus(con) == CONNECTION_BAD)
3528  return TSTATUS_CONN_ERROR;
3529  /* fall through */
3530  case PQTRANS_ACTIVE:
3531  default:
3532 
3533  /*
3534  * We cannot find out whether we are in a transaction block or
3535  * not. Internal error which should never occur.
3536  */
3537  pg_log_error("unexpected transaction status %d", tx_status);
3538  return TSTATUS_OTHER_ERROR;
3539  }
3540 
3541  /* not reached */
3542  Assert(false);
3543  return TSTATUS_OTHER_ERROR;
3544 }
3545 
3546 /*
3547  * Print verbose messages of an error
3548  */
3549 static void
3550 printVerboseErrorMessages(CState *st, pg_time_usec_t *now, bool is_retry)
3551 {
3552  static PQExpBuffer buf = NULL;
3553 
3554  if (buf == NULL)
3555  buf = createPQExpBuffer();
3556  else
3558 
3559  printfPQExpBuffer(buf, "client %d ", st->id);
3560  appendPQExpBufferStr(buf, (is_retry ?
3561  "repeats the transaction after the error" :
3562  "ends the failed transaction"));
3563  appendPQExpBuffer(buf, " (try %u", st->tries);
3564 
3565  /* Print max_tries if it is not unlimited. */
3566  if (max_tries)
3567  appendPQExpBuffer(buf, "/%u", max_tries);
3568 
3569  /*
3570  * If the latency limit is used, print a percentage of the current
3571  * transaction latency from the latency limit.
3572  */
3573  if (latency_limit)
3574  {
3576  appendPQExpBuffer(buf, ", %.3f%% of the maximum time of tries was used",
3577  (100.0 * (*now - st->txn_scheduled) / latency_limit));
3578  }
3579  appendPQExpBufferStr(buf, ")\n");
3580 
3581  pg_log_info("%s", buf->data);
3582 }
3583 
3584 /*
3585  * Advance the state machine of a connection.
3586  */
3587 static void
3588 advanceConnectionState(TState *thread, CState *st, StatsData *agg)
3589 {
3590 
3591  /*
3592  * gettimeofday() isn't free, so we get the current timestamp lazily the
3593  * first time it's needed, and reuse the same value throughout this
3594  * function after that. This also ensures that e.g. the calculated
3595  * latency reported in the log file and in the totals are the same. Zero
3596  * means "not set yet". Reset "now" when we execute shell commands or
3597  * expressions, which might take a non-negligible amount of time, though.
3598  */
3599  pg_time_usec_t now = 0;
3600 
3601  /*
3602  * Loop in the state machine, until we have to wait for a result from the
3603  * server or have to sleep for throttling or \sleep.
3604  *
3605  * Note: In the switch-statement below, 'break' will loop back here,
3606  * meaning "continue in the state machine". Return is used to return to
3607  * the caller, giving the thread the opportunity to advance another
3608  * client.
3609  */
3610  for (;;)
3611  {
3612  Command *command;
3613 
3614  switch (st->state)
3615  {
3616  /* Select transaction (script) to run. */
3617  case CSTATE_CHOOSE_SCRIPT:
3618  st->use_file = chooseScript(thread);
3620 
3621  /* reset transaction variables to default values */
3622  st->estatus = ESTATUS_NO_ERROR;
3623  st->tries = 1;
3624 
3625  pg_log_debug("client %d executing script \"%s\"",
3626  st->id, sql_script[st->use_file].desc);
3627 
3628  /*
3629  * If time is over, we're done; otherwise, get ready to start
3630  * a new transaction, or to get throttled if that's requested.
3631  */
3634  break;
3635 
3636  /* Start new transaction (script) */
3637  case CSTATE_START_TX:
3639 
3640  /* establish connection if needed, i.e. under --connect */
3641  if (st->con == NULL)
3642  {
3643  pg_time_usec_t start = now;
3644 
3645  if ((st->con = doConnect()) == NULL)
3646  {
3647  /*
3648  * as the bench is already running, we do not abort
3649  * the process
3650  */
3651  pg_log_error("client %d aborted while establishing connection", st->id);
3652  st->state = CSTATE_ABORTED;
3653  break;
3654  }
3655 
3656  /* reset now after connection */
3657  now = pg_time_now();
3658 
3659  thread->conn_duration += now - start;
3660 
3661  /* Reset session-local state */
3662  pg_free(st->prepared);
3663  st->prepared = NULL;
3664  }
3665 
3666  /*
3667  * It is the first try to run this transaction. Remember the
3668  * random state: maybe it will get an error and we will need
3669  * to run it again.
3670  */
3671  st->random_state = st->cs_func_rs;
3672 
3673  /* record transaction start time */
3674  st->txn_begin = now;
3675 
3676  /*
3677  * When not throttling, this is also the transaction's
3678  * scheduled start time.
3679  */
3680  if (!throttle_delay)
3681  st->txn_scheduled = now;
3682 
3683  /* Begin with the first command */
3685  st->command = 0;
3686  break;
3687 
3688  /*
3689  * Handle throttling once per transaction by sleeping.
3690  */
3692 
3693  /*
3694  * Generate a delay such that the series of delays will
3695  * approximate a Poisson distribution centered on the
3696  * throttle_delay time.
3697  *
3698  * If transactions are too slow or a given wait is shorter
3699  * than a transaction, the next transaction will start right
3700  * away.
3701  */
3702  Assert(throttle_delay > 0);
3703 
3704  thread->throttle_trigger +=
3706  st->txn_scheduled = thread->throttle_trigger;
3707 
3708  /*
3709  * If --latency-limit is used, and this slot is already late
3710  * so that the transaction will miss the latency limit even if
3711  * it completed immediately, skip this time slot and loop to
3712  * reschedule.
3713  */
3714  if (latency_limit)
3715  {
3717 
3718  if (thread->throttle_trigger < now - latency_limit)
3719  {
3720  processXactStats(thread, st, &now, true, agg);
3721 
3722  /*
3723  * Finish client if -T or -t was exceeded.
3724  *
3725  * Stop counting skipped transactions under -T as soon
3726  * as the timer is exceeded. Because otherwise it can
3727  * take a very long time to count all of them
3728  * especially when quite a lot of them happen with
3729  * unrealistically high rate setting in -R, which
3730  * would prevent pgbench from ending immediately.
3731  * Because of this behavior, note that there is no
3732  * guarantee that all skipped transactions are counted
3733  * under -T though there is under -t. This is OK in
3734  * practice because it's very unlikely to happen with
3735  * realistic setting.
3736  */
3737  if (timer_exceeded || (nxacts > 0 && st->cnt >= nxacts))
3738  st->state = CSTATE_FINISHED;
3739 
3740  /* Go back to top of loop with CSTATE_PREPARE_THROTTLE */
3741  break;
3742  }
3743  }
3744 
3745  /*
3746  * stop client if next transaction is beyond pgbench end of
3747  * execution; otherwise, throttle it.
3748  */
3749  st->state = end_time > 0 && st->txn_scheduled > end_time ?
3751  break;
3752 
3753  /*
3754  * Wait until it's time to start next transaction.
3755  */
3756  case CSTATE_THROTTLE:
3758 
3759  if (now < st->txn_scheduled)
3760  return; /* still sleeping, nothing to do here */
3761 
3762  /* done sleeping, but don't start transaction if we're done */
3764  break;
3765 
3766  /*
3767  * Send a command to server (or execute a meta-command)
3768  */
3769  case CSTATE_START_COMMAND:
3770  command = sql_script[st->use_file].commands[st->command];
3771 
3772  /*
3773  * Transition to script end processing if done, but close up
3774  * shop if a pipeline is open at this point.
3775  */
3776  if (command == NULL)
3777  {
3778  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3779  st->state = CSTATE_END_TX;
3780  else
3781  {
3782  pg_log_error("client %d aborted: end of script reached with pipeline open",
3783  st->id);
3784  st->state = CSTATE_ABORTED;
3785  }
3786 
3787  break;
3788  }
3789 
3790  /* record begin time of next command, and initiate it */
3791  if (report_per_command)
3792  {
3794  st->stmt_begin = now;
3795  }
3796 
3797  /* Execute the command */
3798  if (command->type == SQL_COMMAND)
3799  {
3800  /* disallow \aset and \gset in pipeline mode */
3801  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3802  {
3803  if (command->meta == META_GSET)
3804  {
3805  commandFailed(st, "gset", "\\gset is not allowed in pipeline mode");
3806  st->state = CSTATE_ABORTED;
3807  break;
3808  }
3809  else if (command->meta == META_ASET)
3810  {
3811  commandFailed(st, "aset", "\\aset is not allowed in pipeline mode");
3812  st->state = CSTATE_ABORTED;
3813  break;
3814  }
3815  }
3816 
3817  if (!sendCommand(st, command))
3818  {
3819  commandFailed(st, "SQL", "SQL command send failed");
3820  st->state = CSTATE_ABORTED;
3821  }
3822  else
3823  {
3824  /* Wait for results, unless in pipeline mode */
3825  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3826  st->state = CSTATE_WAIT_RESULT;
3827  else
3828  st->state = CSTATE_END_COMMAND;
3829  }
3830  }
3831  else if (command->type == META_COMMAND)
3832  {
3833  /*-----
3834  * Possible state changes when executing meta commands:
3835  * - on errors CSTATE_ABORTED
3836  * - on sleep CSTATE_SLEEP
3837  * - else CSTATE_END_COMMAND
3838  */
3839  st->state = executeMetaCommand(st, &now);
3840  if (st->state == CSTATE_ABORTED)
3842  }
3843 
3844  /*
3845  * We're now waiting for an SQL command to complete, or
3846  * finished processing a metacommand, or need to sleep, or
3847  * something bad happened.
3848  */
3849  Assert(st->state == CSTATE_WAIT_RESULT ||
3850  st->state == CSTATE_END_COMMAND ||
3851  st->state == CSTATE_SLEEP ||
3852  st->state == CSTATE_ABORTED);
3853  break;
3854 
3855  /*
3856  * non executed conditional branch
3857  */
3858  case CSTATE_SKIP_COMMAND:
3860  /* quickly skip commands until something to do... */
3861  while (true)
3862  {
3863  command = sql_script[st->use_file].commands[st->command];
3864 
3865  /* cannot reach end of script in that state */
3866  Assert(command != NULL);
3867 
3868  /*
3869  * if this is conditional related, update conditional
3870  * state
3871  */
3872  if (command->type == META_COMMAND &&
3873  (command->meta == META_IF ||
3874  command->meta == META_ELIF ||
3875  command->meta == META_ELSE ||
3876  command->meta == META_ENDIF))
3877  {
3878  switch (conditional_stack_peek(st->cstack))
3879  {
3880  case IFSTATE_FALSE:
3881  if (command->meta == META_IF ||
3882  command->meta == META_ELIF)
3883  {
3884  /* we must evaluate the condition */
3886  }
3887  else if (command->meta == META_ELSE)
3888  {
3889  /* we must execute next command */
3893  st->command++;
3894  }
3895  else if (command->meta == META_ENDIF)
3896  {
3899  if (conditional_active(st->cstack))
3901 
3902  /*
3903  * else state remains in
3904  * CSTATE_SKIP_COMMAND
3905  */
3906  st->command++;
3907  }
3908  break;
3909 
3910  case IFSTATE_IGNORED:
3911  case IFSTATE_ELSE_FALSE:
3912  if (command->meta == META_IF)
3914  IFSTATE_IGNORED);
3915  else if (command->meta == META_ENDIF)
3916  {
3919  if (conditional_active(st->cstack))
3921  }
3922  /* could detect "else" & "elif" after "else" */
3923  st->command++;
3924  break;
3925 
3926  case IFSTATE_NONE:
3927  case IFSTATE_TRUE:
3928  case IFSTATE_ELSE_TRUE:
3929  default:
3930 
3931  /*
3932  * inconsistent if inactive, unreachable dead
3933  * code
3934  */
3935  Assert(false);
3936  }
3937  }
3938  else
3939  {
3940  /* skip and consider next */
3941  st->command++;
3942  }
3943 
3944  if (st->state != CSTATE_SKIP_COMMAND)
3945  /* out of quick skip command loop */
3946  break;
3947  }
3948  break;
3949 
3950  /*
3951  * Wait for the current SQL command to complete
3952  */
3953  case CSTATE_WAIT_RESULT:
3954  pg_log_debug("client %d receiving", st->id);
3955 
3956  /*
3957  * Only check for new network data if we processed all data
3958  * fetched prior. Otherwise we end up doing a syscall for each
3959  * individual pipelined query, which has a measurable
3960  * performance impact.
3961  */
3962  if (PQisBusy(st->con) && !PQconsumeInput(st->con))
3963  {
3964  /* there's something wrong */
3965  commandFailed(st, "SQL", "perhaps the backend died while processing");
3966  st->state = CSTATE_ABORTED;
3967  break;
3968  }
3969  if (PQisBusy(st->con))
3970  return; /* don't have the whole result yet */
3971 
3972  /* store or discard the query results */
3973  if (readCommandResponse(st,
3976  {
3977  /*
3978  * outside of pipeline mode: stop reading results.
3979  * pipeline mode: continue reading results until an
3980  * end-of-pipeline response.
3981  */
3982  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
3983  st->state = CSTATE_END_COMMAND;
3984  }
3985  else if (canRetryError(st->estatus))
3986  st->state = CSTATE_ERROR;
3987  else
3988  st->state = CSTATE_ABORTED;
3989  break;
3990 
3991  /*
3992  * Wait until sleep is done. This state is entered after a
3993  * \sleep metacommand. The behavior is similar to
3994  * CSTATE_THROTTLE, but proceeds to CSTATE_START_COMMAND
3995  * instead of CSTATE_START_TX.
3996  */
3997  case CSTATE_SLEEP:
3999  if (now < st->sleep_until)
4000  return; /* still sleeping, nothing to do here */
4001  /* Else done sleeping. */
4002  st->state = CSTATE_END_COMMAND;
4003  break;
4004 
4005  /*
4006  * End of command: record stats and proceed to next command.
4007  */
4008  case CSTATE_END_COMMAND:
4009 
4010  /*
4011  * command completed: accumulate per-command execution times
4012  * in thread-local data structure, if per-command latencies
4013  * are requested.
4014  */
4015  if (report_per_command)
4016  {
4018 
4019  command = sql_script[st->use_file].commands[st->command];
4020  /* XXX could use a mutex here, but we choose not to */
4021  addToSimpleStats(&command->stats,
4023  }
4024 
4025  /* Go ahead with next command, to be executed or skipped */
4026  st->command++;
4027  st->state = conditional_active(st->cstack) ?
4029  break;
4030 
4031  /*
4032  * Clean up after an error.
4033  */
4034  case CSTATE_ERROR:
4035  {
4036  TStatus tstatus;
4037 
4039 
4040  /* Clear the conditional stack */
4042 
4043  /* Read and discard until a sync point in pipeline mode */
4044  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4045  {
4046  if (!discardUntilSync(st))
4047  {
4048  st->state = CSTATE_ABORTED;
4049  break;
4050  }
4051  }
4052 
4053  /*
4054  * Check if we have a (failed) transaction block or not,
4055  * and roll it back if any.
4056  */
4057  tstatus = getTransactionStatus(st->con);
4058  if (tstatus == TSTATUS_IN_BLOCK)
4059  {
4060  /* Try to rollback a (failed) transaction block. */
4061  if (!PQsendQuery(st->con, "ROLLBACK"))
4062  {
4063  pg_log_error("client %d aborted: failed to send sql command for rolling back the failed transaction",
4064  st->id);
4065  st->state = CSTATE_ABORTED;
4066  }
4067  else
4069  }
4070  else if (tstatus == TSTATUS_IDLE)
4071  {
4072  /*
4073  * If time is over, we're done; otherwise, check if we
4074  * can retry the error.
4075  */
4078  }
4079  else
4080  {
4081  if (tstatus == TSTATUS_CONN_ERROR)
4082  pg_log_error("perhaps the backend died while processing");
4083 
4084  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4085  st->state = CSTATE_ABORTED;
4086  }
4087  break;
4088  }
4089 
4090  /*
4091  * Wait for the rollback command to complete
4092  */
4094  {
4095  PGresult *res;
4096 
4097  pg_log_debug("client %d receiving", st->id);
4098  if (!PQconsumeInput(st->con))
4099  {
4100  pg_log_error("client %d aborted while rolling back the transaction after an error; perhaps the backend died while processing",
4101  st->id);
4102  st->state = CSTATE_ABORTED;
4103  break;
4104  }
4105  if (PQisBusy(st->con))
4106  return; /* don't have the whole result yet */
4107 
4108  /*
4109  * Read and discard the query result;
4110  */
4111  res = PQgetResult(st->con);
4112  switch (PQresultStatus(res))
4113  {
4114  case PGRES_COMMAND_OK:
4115  /* OK */
4116  PQclear(res);
4117  /* null must be returned */
4118  res = PQgetResult(st->con);
4119  Assert(res == NULL);
4120 
4121  /*
4122  * If time is over, we're done; otherwise, check
4123  * if we can retry the error.
4124  */
4127  break;
4128  default:
4129  pg_log_error("client %d aborted while rolling back the transaction after an error; %s",
4130  st->id, PQerrorMessage(st->con));
4131  PQclear(res);
4132  st->state = CSTATE_ABORTED;
4133  break;
4134  }
4135  break;
4136  }
4137 
4138  /*
4139  * Retry the transaction after an error.
4140  */
4141  case CSTATE_RETRY:
4142  command = sql_script[st->use_file].commands[st->command];
4143 
4144  /*
4145  * Inform that the transaction will be retried after the
4146  * error.
4147  */
4148  if (verbose_errors)
4149  printVerboseErrorMessages(st, &now, true);
4150 
4151  /* Count tries and retries */
4152  st->tries++;
4153  command->retries++;
4154 
4155  /*
4156  * Reset the random state as they were at the beginning of the
4157  * transaction.
4158  */
4159  st->cs_func_rs = st->random_state;
4160 
4161  /* Process the first transaction command. */
4162  st->command = 0;
4163  st->estatus = ESTATUS_NO_ERROR;
4165  break;
4166 
4167  /*
4168  * Record a failed transaction.
4169  */
4170  case CSTATE_FAILURE:
4171  command = sql_script[st->use_file].commands[st->command];
4172 
4173  /* Accumulate the failure. */
4174  command->failures++;
4175 
4176  /*
4177  * Inform that the failed transaction will not be retried.
4178  */
4179  if (verbose_errors)
4180  printVerboseErrorMessages(st, &now, false);
4181 
4182  /* End the failed transaction. */
4183  st->state = CSTATE_END_TX;
4184  break;
4185 
4186  /*
4187  * End of transaction (end of script, really).
4188  */
4189  case CSTATE_END_TX:
4190  {
4191  TStatus tstatus;
4192 
4193  /* transaction finished: calculate latency and do log */
4194  processXactStats(thread, st, &now, false, agg);
4195 
4196  /*
4197  * missing \endif... cannot happen if CheckConditional was
4198  * okay
4199  */
4201 
4202  /*
4203  * We must complete all the transaction blocks that were
4204  * started in this script.
4205  */
4206  tstatus = getTransactionStatus(st->con);
4207  if (tstatus == TSTATUS_IN_BLOCK)
4208  {
4209  pg_log_error("client %d aborted: end of script reached without completing the last transaction",
4210  st->id);
4211  st->state = CSTATE_ABORTED;
4212  break;
4213  }
4214  else if (tstatus != TSTATUS_IDLE)
4215  {
4216  if (tstatus == TSTATUS_CONN_ERROR)
4217  pg_log_error("perhaps the backend died while processing");
4218 
4219  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4220  st->state = CSTATE_ABORTED;
4221  break;
4222  }
4223 
4224  if (is_connect)
4225  {
4226  pg_time_usec_t start = now;
4227 
4228  pg_time_now_lazy(&start);
4229  finishCon(st);
4230  now = pg_time_now();
4231  thread->conn_duration += now - start;
4232  }
4233 
4234  if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded)
4235  {
4236  /* script completed */
4237  st->state = CSTATE_FINISHED;
4238  break;
4239  }
4240 
4241  /* next transaction (script) */
4243 
4244  /*
4245  * Ensure that we always return on this point, so as to
4246  * avoid an infinite loop if the script only contains meta
4247  * commands.
4248  */
4249  return;
4250  }
4251 
4252  /*
4253  * Final states. Close the connection if it's still open.
4254  */
4255  case CSTATE_ABORTED:
4256  case CSTATE_FINISHED:
4257 
4258  /*
4259  * Don't measure the disconnection delays here even if in
4260  * CSTATE_FINISHED and -C/--connect option is specified.
4261  * Because in this case all the connections that this thread
4262  * established are closed at the end of transactions and the
4263  * disconnection delays should have already been measured at
4264  * that moment.
4265  *
4266  * In CSTATE_ABORTED state, the measurement is no longer
4267  * necessary because we cannot report complete results anyways
4268  * in this case.
4269  */
4270  finishCon(st);
4271  return;
4272  }
4273  }
4274 }
4275 
4276 /*
4277  * Subroutine for advanceConnectionState -- initiate or execute the current
4278  * meta command, and return the next state to set.
4279  *
4280  * *now is updated to the current time, unless the command is expected to
4281  * take no time to execute.
4282  */
4285 {
4286  Command *command = sql_script[st->use_file].commands[st->command];
4287  int argc;
4288  char **argv;
4289 
4290  Assert(command != NULL && command->type == META_COMMAND);
4291 
4292  argc = command->argc;
4293  argv = command->argv;
4294 
4296  {
4298 
4299  initPQExpBuffer(&buf);
4300 
4301  printfPQExpBuffer(&buf, "client %d executing \\%s", st->id, argv[0]);
4302  for (int i = 1; i < argc; i++)
4303  appendPQExpBuffer(&buf, " %s", argv[i]);
4304 
4305  pg_log_debug("%s", buf.data);
4306 
4307  termPQExpBuffer(&buf);
4308  }
4309 
4310  if (command->meta == META_SLEEP)
4311  {
4312  int usec;
4313 
4314  /*
4315  * A \sleep doesn't execute anything, we just get the delay from the
4316  * argument, and enter the CSTATE_SLEEP state. (The per-command
4317  * latency will be recorded in CSTATE_SLEEP state, not here, after the
4318  * delay has elapsed.)
4319  */
4320  if (!evaluateSleep(&st->variables, argc, argv, &usec))
4321  {
4322  commandFailed(st, "sleep", "execution of meta-command failed");
4323  return CSTATE_ABORTED;
4324  }
4325 
4327  st->sleep_until = (*now) + usec;
4328  return CSTATE_SLEEP;
4329  }
4330  else if (command->meta == META_SET)
4331  {
4332  PgBenchExpr *expr = command->expr;
4333  PgBenchValue result;
4334 
4335  if (!evaluateExpr(st, expr, &result))
4336  {
4337  commandFailed(st, argv[0], "evaluation of meta-command failed");
4338  return CSTATE_ABORTED;
4339  }
4340 
4341  if (!putVariableValue(&st->variables, argv[0], argv[1], &result))
4342  {
4343  commandFailed(st, "set", "assignment of meta-command failed");
4344  return CSTATE_ABORTED;
4345  }
4346  }
4347  else if (command->meta == META_IF)
4348  {
4349  /* backslash commands with an expression to evaluate */
4350  PgBenchExpr *expr = command->expr;
4351  PgBenchValue result;
4352  bool cond;
4353 
4354  if (!evaluateExpr(st, expr, &result))
4355  {
4356  commandFailed(st, argv[0], "evaluation of meta-command failed");
4357  return CSTATE_ABORTED;
4358  }
4359 
4360  cond = valueTruth(&result);
4362  }
4363  else if (command->meta == META_ELIF)
4364  {
4365  /* backslash commands with an expression to evaluate */
4366  PgBenchExpr *expr = command->expr;
4367  PgBenchValue result;
4368  bool cond;
4369 
4371  {
4372  /* elif after executed block, skip eval and wait for endif. */
4374  return CSTATE_END_COMMAND;
4375  }
4376 
4377  if (!evaluateExpr(st, expr, &result))
4378  {
4379  commandFailed(st, argv[0], "evaluation of meta-command failed");
4380  return CSTATE_ABORTED;
4381  }
4382 
4383  cond = valueTruth(&result);
4386  }
4387  else if (command->meta == META_ELSE)
4388  {
4389  switch (conditional_stack_peek(st->cstack))
4390  {
4391  case IFSTATE_TRUE:
4393  break;
4394  case IFSTATE_FALSE: /* inconsistent if active */
4395  case IFSTATE_IGNORED: /* inconsistent if active */
4396  case IFSTATE_NONE: /* else without if */
4397  case IFSTATE_ELSE_TRUE: /* else after else */
4398  case IFSTATE_ELSE_FALSE: /* else after else */
4399  default:
4400  /* dead code if conditional check is ok */
4401  Assert(false);
4402  }
4403  }
4404  else if (command->meta == META_ENDIF)
4405  {
4408  }
4409  else if (command->meta == META_SETSHELL)
4410  {
4411  if (!runShellCommand(&st->variables, argv[1], argv + 2, argc - 2))
4412  {
4413  commandFailed(st, "setshell", "execution of meta-command failed");
4414  return CSTATE_ABORTED;
4415  }
4416  }
4417  else if (command->meta == META_SHELL)
4418  {
4419  if (!runShellCommand(&st->variables, NULL, argv + 1, argc - 1))
4420  {
4421  commandFailed(st, "shell", "execution of meta-command failed");
4422  return CSTATE_ABORTED;
4423  }
4424  }
4425  else if (command->meta == META_STARTPIPELINE)
4426  {
4427  /*
4428  * In pipeline mode, we use a workflow based on libpq pipeline
4429  * functions.
4430  */
4431  if (querymode == QUERY_SIMPLE)
4432  {
4433  commandFailed(st, "startpipeline", "cannot use pipeline mode with the simple query protocol");
4434  return CSTATE_ABORTED;
4435  }
4436 
4437  /*
4438  * If we're in prepared-query mode, we need to prepare all the
4439  * commands that are inside the pipeline before we actually start the
4440  * pipeline itself. This solves the problem that running BEGIN
4441  * ISOLATION LEVEL SERIALIZABLE in a pipeline would fail due to a
4442  * snapshot having been acquired by the prepare within the pipeline.
4443  */
4444  if (querymode == QUERY_PREPARED)
4446 
4447  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4448  {
4449  commandFailed(st, "startpipeline", "already in pipeline mode");
4450  return CSTATE_ABORTED;
4451  }
4452  if (PQenterPipelineMode(st->con) == 0)
4453  {
4454  commandFailed(st, "startpipeline", "failed to enter pipeline mode");
4455  return CSTATE_ABORTED;
4456  }
4457  }
4458  else if (command->meta == META_SYNCPIPELINE)
4459  {
4460  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
4461  {
4462  commandFailed(st, "syncpipeline", "not in pipeline mode");
4463  return CSTATE_ABORTED;
4464  }
4465  if (PQsendPipelineSync(st->con) == 0)
4466  {
4467  commandFailed(st, "syncpipeline", "failed to send a pipeline sync");
4468  return CSTATE_ABORTED;
4469  }
4470  st->num_syncs++;
4471  }
4472  else if (command->meta == META_ENDPIPELINE)
4473  {
4474  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
4475  {
4476  commandFailed(st, "endpipeline", "not in pipeline mode");
4477  return CSTATE_ABORTED;
4478  }
4479  if (!PQpipelineSync(st->con))
4480  {
4481  commandFailed(st, "endpipeline", "failed to send a pipeline sync");
4482  return CSTATE_ABORTED;
4483  }
4484  st->num_syncs++;
4485  /* Now wait for the PGRES_PIPELINE_SYNC and exit pipeline mode there */
4486  /* collect pending results before getting out of pipeline mode */
4487  return CSTATE_WAIT_RESULT;
4488  }
4489 
4490  /*
4491  * executing the expression or shell command might have taken a
4492  * non-negligible amount of time, so reset 'now'
4493  */
4494  *now = 0;
4495 
4496  return CSTATE_END_COMMAND;
4497 }
4498 
4499 /*
4500  * Return the number of failed transactions.
4501  */
4502 static int64
4503 getFailures(const StatsData *stats)
4504 {
4505  return (stats->serialization_failures +
4506  stats->deadlock_failures);
4507 }
4508 
4509 /*
4510  * Return a string constant representing the result of a transaction
4511  * that is not successfully processed.
4512  */
4513 static const char *
4514 getResultString(bool skipped, EStatus estatus)
4515 {
4516  if (skipped)
4517  return "skipped";
4518  else if (failures_detailed)
4519  {
4520  switch (estatus)
4521  {
4523  return "serialization";
4525  return "deadlock";
4526  default:
4527  /* internal error which should never occur */
4528  pg_fatal("unexpected error status: %d", estatus);
4529  }
4530  }
4531  else
4532  return "failed";
4533 }
4534 
4535 /*
4536  * Print log entry after completing one transaction.
4537  *
4538  * We print Unix-epoch timestamps in the log, so that entries can be
4539  * correlated against other logs.
4540  *
4541  * XXX We could obtain the time from the caller and just shift it here, to
4542  * avoid the cost of an extra call to pg_time_now().
4543  */
4544 static void
4545 doLog(TState *thread, CState *st,
4546  StatsData *agg, bool skipped, double latency, double lag)
4547 {
4548  FILE *logfile = thread->logfile;
4550 
4551  Assert(use_log);
4552 
4553  /*
4554  * Skip the log entry if sampling is enabled and this row doesn't belong
4555  * to the random sample.
4556  */
4557  if (sample_rate != 0.0 &&
4559  return;
4560 
4561  /* should we aggregate the results or not? */
4562  if (agg_interval > 0)
4563  {
4565 
4566  /*
4567  * Loop until we reach the interval of the current moment, and print
4568  * any empty intervals in between (this may happen with very low tps,
4569  * e.g. --rate=0.1).
4570  */
4571 
4572  while ((next = agg->start_time + agg_interval * INT64CONST(1000000)) <= now)
4573  {
4574  double lag_sum = 0.0;
4575  double lag_sum2 = 0.0;
4576  double lag_min = 0.0;
4577  double lag_max = 0.0;
4578  int64 skipped = 0;
4579  int64 serialization_failures = 0;
4580  int64 deadlock_failures = 0;
4581  int64 retried = 0;
4582  int64 retries = 0;
4583 
4584  /* print aggregated report to logfile */
4585  fprintf(logfile, INT64_FORMAT " " INT64_FORMAT " %.0f %.0f %.0f %.0f",
4586  agg->start_time / 1000000, /* seconds since Unix epoch */
4587  agg->cnt,
4588  agg->latency.sum,
4589  agg->latency.sum2,
4590  agg->latency.min,
4591  agg->latency.max);
4592 
4593  if (throttle_delay)
4594  {
4595  lag_sum = agg->lag.sum;
4596  lag_sum2 = agg->lag.sum2;
4597  lag_min = agg->lag.min;
4598  lag_max = agg->lag.max;
4599  }
4600  fprintf(logfile, " %.0f %.0f %.0f %.0f",
4601  lag_sum,
4602  lag_sum2,
4603  lag_min,
4604  lag_max);
4605 
4606  if (latency_limit)
4607  skipped = agg->skipped;
4608  fprintf(logfile, " " INT64_FORMAT, skipped);
4609 
4610  if (max_tries != 1)
4611  {
4612  retried = agg->retried;
4613  retries = agg->retries;
4614  }
4615  fprintf(logfile, " " INT64_FORMAT " " INT64_FORMAT, retried, retries);
4616 
4617  if (failures_detailed)
4618  {
4619  serialization_failures = agg->serialization_failures;
4620  deadlock_failures = agg->deadlock_failures;
4621  }
4623  serialization_failures,
4624  deadlock_failures);
4625 
4626  fputc('\n', logfile);
4627 
4628  /* reset data and move to next interval */
4629  initStats(agg, next);
4630  }
4631 
4632  /* accumulate the current transaction */
4633  accumStats(agg, skipped, latency, lag, st->estatus, st->tries);
4634  }
4635  else
4636  {
4637  /* no, print raw transactions */
4638  if (!skipped && st->estatus == ESTATUS_NO_ERROR)
4639  fprintf(logfile, "%d " INT64_FORMAT " %.0f %d " INT64_FORMAT " "
4640  INT64_FORMAT,
4641  st->id, st->cnt, latency, st->use_file,
4642  now / 1000000, now % 1000000);
4643  else
4644  fprintf(logfile, "%d " INT64_FORMAT " %s %d " INT64_FORMAT " "
4645  INT64_FORMAT,
4646  st->id, st->cnt, getResultString(skipped, st->estatus),
4647  st->use_file, now / 1000000, now % 1000000);
4648 
4649  if (throttle_delay)
4650  fprintf(logfile, " %.0f", lag);
4651  if (max_tries != 1)
4652  fprintf(logfile, " %u", st->tries - 1);
4653  fputc('\n', logfile);
4654  }
4655 }
4656 
4657 /*
4658  * Accumulate and report statistics at end of a transaction.
4659  *
4660  * (This is also called when a transaction is late and thus skipped.
4661  * Note that even skipped and failed transactions are counted in the CState
4662  * "cnt" field.)
4663  */
4664 static void
4666  bool skipped, StatsData *agg)
4667 {
4668  double latency = 0.0,
4669  lag = 0.0;
4670  bool detailed = progress || throttle_delay || latency_limit ||
4672 
4673  if (detailed && !skipped && st->estatus == ESTATUS_NO_ERROR)
4674  {
4676 
4677  /* compute latency & lag */
4678  latency = (*now) - st->txn_scheduled;
4679  lag = st->txn_begin - st->txn_scheduled;
4680  }
4681 
4682  /* keep detailed thread stats */
4683  accumStats(&thread->stats, skipped, latency, lag, st->estatus, st->tries);
4684 
4685  /* count transactions over the latency limit, if needed */
4686  if (latency_limit && latency > latency_limit)
4687  thread->latency_late++;
4688 
4689  /* client stat is just counting */
4690  st->cnt++;
4691 
4692  if (use_log)
4693  doLog(thread, st, agg, skipped, latency, lag);
4694 
4695  /* XXX could use a mutex here, but we choose not to */
4696  if (per_script_stats)
4697  accumStats(&sql_script[st->use_file].stats, skipped, latency, lag,
4698  st->estatus, st->tries);
4699 }
4700 
4701 
4702 /* discard connections */
4703 static void
4704 disconnect_all(CState *state, int length)
4705 {
4706  int i;
4707 
4708  for (i = 0; i < length; i++)
4709  finishCon(&state[i]);
4710 }
4711 
4712 /*
4713  * Remove old pgbench tables, if any exist
4714  */
4715 static void
4716 initDropTables(PGconn *con)
4717 {
4718  fprintf(stderr, "dropping old tables...\n");
4719 
4720  /*
4721  * We drop all the tables in one command, so that whether there are
4722  * foreign key dependencies or not doesn't matter.
4723  */
4724  executeStatement(con, "drop table if exists "
4725  "pgbench_accounts, "
4726  "pgbench_branches, "
4727  "pgbench_history, "
4728  "pgbench_tellers");
4729 }
4730 
4731 /*
4732  * Create "pgbench_accounts" partitions if needed.
4733  *
4734  * This is the larger table of pgbench default tpc-b like schema
4735  * with a known size, so we choose to partition it.
4736  */
4737 static void
4739 {
4740  PQExpBufferData query;
4741 
4742  /* we must have to create some partitions */
4743  Assert(partitions > 0);
4744 
4745  fprintf(stderr, "creating %d partitions...\n", partitions);
4746 
4747  initPQExpBuffer(&query);
4748 
4749  for (int p = 1; p <= partitions; p++)
4750  {
4752  {
4753  int64 part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
4754 
4755  printfPQExpBuffer(&query,
4756  "create%s table pgbench_accounts_%d\n"
4757  " partition of pgbench_accounts\n"
4758  " for values from (",
4759  unlogged_tables ? " unlogged" : "", p);
4760 
4761  /*
4762  * For RANGE, we use open-ended partitions at the beginning and
4763  * end to allow any valid value for the primary key. Although the
4764  * actual minimum and maximum values can be derived from the
4765  * scale, it is more generic and the performance is better.
4766  */
4767  if (p == 1)
4768  appendPQExpBufferStr(&query, "minvalue");
4769  else
4770  appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
4771 
4772  appendPQExpBufferStr(&query, ") to (");
4773 
4774  if (p < partitions)
4775  appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
4776  else
4777  appendPQExpBufferStr(&query, "maxvalue");
4778 
4779  appendPQExpBufferChar(&query, ')');
4780  }
4781  else if (partition_method == PART_HASH)
4782  printfPQExpBuffer(&query,
4783  "create%s table pgbench_accounts_%d\n"
4784  " partition of pgbench_accounts\n"
4785  " for values with (modulus %d, remainder %d)",
4786  unlogged_tables ? " unlogged" : "", p,
4787  partitions, p - 1);
4788  else /* cannot get there */
4789  Assert(0);
4790 
4791  /*
4792  * Per ddlinfo in initCreateTables, fillfactor is needed on table
4793  * pgbench_accounts.
4794  */
4795  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4796 
4797  executeStatement(con, query.data);
4798  }
4799 
4800  termPQExpBuffer(&query);
4801 }
4802 
4803 /*
4804  * Create pgbench's standard tables
4805  */
4806 static void
4808 {
4809  /*
4810  * Note: TPC-B requires at least 100 bytes per row, and the "filler"
4811  * fields in these table declarations were intended to comply with that.
4812  * The pgbench_accounts table complies with that because the "filler"
4813  * column is set to blank-padded empty string. But for all other tables
4814  * the columns default to NULL and so don't actually take any space. We
4815  * could fix that by giving them non-null default values. However, that
4816  * would completely break comparability of pgbench results with prior
4817  * versions. Since pgbench has never pretended to be fully TPC-B compliant
4818  * anyway, we stick with the historical behavior.
4819  */
4820  struct ddlinfo
4821  {
4822  const char *table; /* table name */
4823  const char *smcols; /* column decls if accountIDs are 32 bits */
4824  const char *bigcols; /* column decls if accountIDs are 64 bits */
4825  int declare_fillfactor;
4826  };
4827  static const struct ddlinfo DDLs[] = {
4828  {
4829  "pgbench_history",
4830  "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)",
4831  "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)",
4832  0
4833  },
4834  {
4835  "pgbench_tellers",
4836  "tid int not null,bid int,tbalance int,filler char(84)",
4837  "tid int not null,bid int,tbalance int,filler char(84)",
4838  1
4839  },
4840  {
4841  "pgbench_accounts",
4842  "aid int not null,bid int,abalance int,filler char(84)",
4843  "aid bigint not null,bid int,abalance int,filler char(84)",
4844  1
4845  },
4846  {
4847  "pgbench_branches",
4848  "bid int not null,bbalance int,filler char(88)",
4849  "bid int not null,bbalance int,filler char(88)",
4850  1
4851  }
4852  };
4853  int i;
4854  PQExpBufferData query;
4855 
4856  fprintf(stderr, "creating tables...\n");
4857 
4858  initPQExpBuffer(&query);
4859 
4860  for (i = 0; i < lengthof(DDLs); i++)
4861  {
4862  const struct ddlinfo *ddl = &DDLs[i];
4863 
4864  /* Construct new create table statement. */
4865  printfPQExpBuffer(&query, "create%s table %s(%s)",
4866  unlogged_tables ? " unlogged" : "",
4867  ddl->table,
4868  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
4869 
4870  /* Partition pgbench_accounts table */
4871  if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
4872  appendPQExpBuffer(&query,
4873  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
4874  else if (ddl->declare_fillfactor)
4875  {
4876  /* fillfactor is only expected on actual tables */
4877  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4878  }
4879 
4880  if (tablespace != NULL)
4881  {
4882  char *escape_tablespace;
4883 
4884  escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
4885  appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
4886  PQfreemem(escape_tablespace);
4887  }
4888 
4889  executeStatement(con, query.data);
4890  }
4891 
4892  termPQExpBuffer(&query);
4893 
4894  if (partition_method != PART_NONE)
4895  createPartitions(con);
4896 }
4897 
4898 /*
4899  * Truncate away any old data, in one command in case there are foreign keys
4900  */
4901 static void
4903 {
4904  executeStatement(con, "truncate table "
4905  "pgbench_accounts, "
4906  "pgbench_branches, "
4907  "pgbench_history, "
4908  "pgbench_tellers");
4909 }
4910 
4911 static void
4912 initBranch(PQExpBufferData *sql, int64 curr)
4913 {
4914  /* "filler" column uses NULL */
4915  printfPQExpBuffer(sql,
4916  INT64_FORMAT "\t0\t\\N\n",
4917  curr + 1);
4918 }
4919 
4920 static void
4921 initTeller(PQExpBufferData *sql, int64 curr)
4922 {
4923  /* "filler" column uses NULL */
4924  printfPQExpBuffer(sql,
4925  INT64_FORMAT "\t" INT64_FORMAT "\t0\t\\N\n",
4926  curr + 1, curr / ntellers + 1);
4927 }
4928 
4929 static void
4930 initAccount(PQExpBufferData *sql, int64 curr)
4931 {
4932  /* "filler" column defaults to blank padded empty string */
4933  printfPQExpBuffer(sql,
4934  INT64_FORMAT "\t" INT64_FORMAT "\t0\t\n",
4935  curr + 1, curr / naccounts + 1);
4936 }
4937 
4938 static void
4939 initPopulateTable(PGconn *con, const char *table, int64 base,
4940  initRowMethod init_row)
4941 {
4942  int n;
4943  int64 k;
4944  int chars = 0;
4945  PGresult *res;
4946  PQExpBufferData sql;
4947  char copy_statement[256];
4948  const char *copy_statement_fmt = "copy %s from stdin";
4949  int64 total = base * scale;
4950 
4951  /* used to track elapsed time and estimate of the remaining time */
4952  pg_time_usec_t start;
4953  int log_interval = 1;
4954 
4955  /* Stay on the same line if reporting to a terminal */
4956  char eol = isatty(fileno(stderr)) ? '\r' : '\n';
4957 
4958  initPQExpBuffer(&sql);
4959 
4960  /*
4961  * Use COPY with FREEZE on v14 and later for all the tables except
4962  * pgbench_accounts when it is partitioned.
4963  */
4964  if (PQserverVersion(con) >= 140000)
4965  {
4966  if (strcmp(table, "pgbench_accounts") != 0 ||
4967  partitions == 0)
4968  copy_statement_fmt = "copy %s from stdin with (freeze on)";
4969  }
4970 
4971  n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table);
4972  if (n >= sizeof(copy_statement))
4973  pg_fatal("invalid buffer size: must be at least %d characters long", n);
4974  else if (n == -1)
4975  pg_fatal("invalid format string");
4976 
4977  res = PQexec(con, copy_statement);
4978 
4980  pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
4981  PQclear(res);
4982 
4983  start = pg_time_now();
4984 
4985  for (k = 0; k < total; k++)
4986  {
4987  int64 j = k + 1;
4988 
4989  init_row(&sql, k);
4990  if (PQputline(con, sql.data))
4991  pg_fatal("PQputline failed");
4992 
4993  if (CancelRequested)
4994  break;
4995 
4996  /*
4997  * If we want to stick with the original logging, print a message each
4998  * 100k inserted rows.
4999  */
5000  if ((!use_quiet) && (j % 100000 == 0))
5001  {
5002  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5003  double remaining_sec = ((double) total - j) * elapsed_sec / j;
5004 
5005  chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)%c",
5006  j, total,
5007  (int) ((j * 100) / total),
5008  table, elapsed_sec, remaining_sec, eol);
5009  }
5010  /* let's not call the timing for each row, but only each 100 rows */
5011  else if (use_quiet && (j % 100 == 0))
5012  {
5013  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5014  double remaining_sec = ((double) total - j) * elapsed_sec / j;
5015 
5016  /* have we reached the next interval (or end)? */
5017  if ((j == total) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
5018  {
5019  chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)%c",
5020  j, total,
5021  (int) ((j * 100) / total),
5022  table, elapsed_sec, remaining_sec, eol);
5023 
5024  /* skip to the next interval */
5025  log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
5026  }
5027  }
5028  }
5029 
5030  if (chars != 0 && eol != '\n')
5031  fprintf(stderr, "%*c\r", chars - 1, ' '); /* Clear the current line */
5032 
5033  if (PQputline(con, "\\.\n"))
5034  pg_fatal("very last PQputline failed");
5035  if (PQendcopy(con))
5036  pg_fatal("PQendcopy failed");
5037 
5038  termPQExpBuffer(&sql);
5039 }
5040 
5041 /*
5042  * Fill the standard tables with some data generated and sent from the client.
5043  *
5044  * The filler column is NULL in pgbench_branches and pgbench_tellers, and is
5045  * a blank-padded string in pgbench_accounts.
5046  */
5047 static void
5049 {
5050  fprintf(stderr, "generating data (client-side)...\n");
5051 
5052  /*
5053  * we do all of this in one transaction to enable the backend's
5054  * data-loading optimizations
5055  */
5056  executeStatement(con, "begin");
5057 
5058  /* truncate away any old data */
5059  initTruncateTables(con);
5060 
5061  /*
5062  * fill branches, tellers, accounts in that order in case foreign keys
5063  * already exist
5064  */
5065  initPopulateTable(con, "pgbench_branches", nbranches, initBranch);
5066  initPopulateTable(con, "pgbench_tellers", ntellers, initTeller);
5067  initPopulateTable(con, "pgbench_accounts", naccounts, initAccount);
5068 
5069  executeStatement(con, "commit");
5070 }
5071 
5072 /*
5073  * Fill the standard tables with some data generated on the server
5074  *
5075  * As already the case with the client-side data generation, the filler
5076  * column defaults to NULL in pgbench_branches and pgbench_tellers,
5077  * and is a blank-padded string in pgbench_accounts.
5078  */
5079 static void
5081 {
5082  PQExpBufferData sql;
5083 
5084  fprintf(stderr, "generating data (server-side)...\n");
5085 
5086  /*
5087  * we do all of this in one transaction to enable the backend's
5088  * data-loading optimizations
5089  */
5090  executeStatement(con, "begin");
5091 
5092  /* truncate away any old data */
5093  initTruncateTables(con);
5094 
5095  initPQExpBuffer(&sql);
5096 
5097  printfPQExpBuffer(&sql,
5098  "insert into pgbench_branches(bid,bbalance) "
5099  "select bid, 0 "
5100  "from generate_series(1, %d) as bid", nbranches * scale);
5101  executeStatement(con, sql.data);
5102 
5103  printfPQExpBuffer(&sql,
5104  "insert into pgbench_tellers(tid,bid,tbalance) "
5105  "select tid, (tid - 1) / %d + 1, 0 "
5106  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
5107  executeStatement(con, sql.data);
5108 
5109  printfPQExpBuffer(&sql,
5110  "insert into pgbench_accounts(aid,bid,abalance,filler) "
5111  "select aid, (aid - 1) / %d + 1, 0, '' "
5112  "from generate_series(1, " INT64_FORMAT ") as aid",
5113  naccounts, (int64) naccounts * scale);
5114  executeStatement(con, sql.data);
5115 
5116  termPQExpBuffer(&sql);
5117 
5118  executeStatement(con, "commit");
5119 }
5120 
5121 /*
5122  * Invoke vacuum on the standard tables
5123  */
5124 static void
5125 initVacuum(PGconn *con)
5126 {
5127  fprintf(stderr, "vacuuming...\n");
5128  executeStatement(con, "vacuum analyze pgbench_branches");
5129  executeStatement(con, "vacuum analyze pgbench_tellers");
5130  executeStatement(con, "vacuum analyze pgbench_accounts");
5131  executeStatement(con, "vacuum analyze pgbench_history");
5132 }
5133 
5134 /*
5135  * Create primary keys on the standard tables
5136  */
5137 static void
5138 initCreatePKeys(PGconn *con)
5139 {
5140  static const char *const DDLINDEXes[] = {
5141  "alter table pgbench_branches add primary key (bid)",
5142  "alter table pgbench_tellers add primary key (tid)",
5143  "alter table pgbench_accounts add primary key (aid)"
5144  };
5145  int i;
5146  PQExpBufferData query;
5147 
5148  fprintf(stderr, "creating primary keys...\n");
5149  initPQExpBuffer(&query);
5150 
5151  for (i = 0; i < lengthof(DDLINDEXes); i++)
5152  {
5153  resetPQExpBuffer(&query);
5154  appendPQExpBufferStr(&query, DDLINDEXes[i]);
5155 
5156  if (index_tablespace != NULL)
5157  {
5158  char *escape_tablespace;
5159 
5160  escape_tablespace = PQescapeIdentifier(con, index_tablespace,
5161  strlen(index_tablespace));
5162  appendPQExpBuffer(&query, " using index tablespace %s", escape_tablespace);
5163  PQfreemem(escape_tablespace);
5164  }
5165 
5166  executeStatement(con, query.data);
5167  }
5168 
5169  termPQExpBuffer(&query);
5170 }
5171 
5172 /*
5173  * Create foreign key constraints between the standard tables
5174  */
5175 static void
5176 initCreateFKeys(PGconn *con)
5177 {
5178  static const char *const DDLKEYs[] = {
5179  "alter table pgbench_tellers add constraint pgbench_tellers_bid_fkey foreign key (bid) references pgbench_branches",
5180  "alter table pgbench_accounts add constraint pgbench_accounts_bid_fkey foreign key (bid) references pgbench_branches",
5181  "alter table pgbench_history add constraint pgbench_history_bid_fkey foreign key (bid) references pgbench_branches",
5182  "alter table pgbench_history add constraint pgbench_history_tid_fkey foreign key (tid) references pgbench_tellers",
5183  "alter table pgbench_history add constraint pgbench_history_aid_fkey foreign key (aid) references pgbench_accounts"
5184  };
5185  int i;
5186 
5187  fprintf(stderr, "creating foreign keys...\n");
5188  for (i = 0; i < lengthof(DDLKEYs); i++)
5189  {
5190  executeStatement(con, DDLKEYs[i]);
5191  }
5192 }
5193 
5194 /*
5195  * Validate an initialization-steps string
5196  *
5197  * (We could just leave it to runInitSteps() to fail if there are wrong
5198  * characters, but since initialization can take awhile, it seems friendlier
5199  * to check during option parsing.)
5200  */
5201 static void
5202 checkInitSteps(const char *initialize_steps)
5203 {
5204  if (initialize_steps[0] == '\0')
5205  pg_fatal("no initialization steps specified");
5206 
5207  for (const char *step = initialize_steps; *step != '\0'; step++)
5208  {
5209  if (strchr(ALL_INIT_STEPS " ", *step) == NULL)
5210  {
5211  pg_log_error("unrecognized initialization step \"%c\"", *step);
5212  pg_log_error_detail("Allowed step characters are: \"" ALL_INIT_STEPS "\".");
5213  exit(1);
5214  }
5215  }
5216 }
5217 
5218 /*
5219  * Invoke each initialization step in the given string
5220  */
5221 static void
5222 runInitSteps(const char *initialize_steps)
5223 {
5224  PQExpBufferData stats;
5225  PGconn *con;
5226  const char *step;
5227  double run_time = 0.0;
5228  bool first = true;
5229 
5230  initPQExpBuffer(&stats);
5231 
5232  if ((con = doConnect()) == NULL)
5233  pg_fatal("could not create connection for initialization");
5234 
5235  setup_cancel_handler(NULL);
5236  SetCancelConn(con);
5237 
5238  for (step = initialize_steps; *step != '\0'; step++)
5239  {
5240  char *op = NULL;
5241  pg_time_usec_t start = pg_time_now();
5242 
5243  switch (*step)
5244  {
5245  case 'd':
5246  op = "drop tables";
5247  initDropTables(con);
5248  break;
5249  case 't':
5250  op = "create tables";
5251  initCreateTables(con);
5252  break;
5253  case 'g':
5254  op = "client-side generate";
5256  break;
5257  case 'G':
5258  op = "server-side generate";
5260  break;
5261  case 'v':
5262  op = "vacuum";
5263  initVacuum(con);
5264  break;
5265  case 'p':
5266  op = "primary keys";
5267  initCreatePKeys(con);
5268  break;
5269  case 'f':
5270  op = "foreign keys";
5271  initCreateFKeys(con);
5272  break;
5273  case ' ':
5274  break; /* ignore */
5275  default:
5276  pg_log_error("unrecognized initialization step \"%c\"", *step);
5277  PQfinish(con);
5278  exit(1);
5279  }
5280 
5281  if (op != NULL)
5282  {
5283  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5284 
5285  if (!first)
5286  appendPQExpBufferStr(&stats, ", ");
5287  else
5288  first = false;
5289 
5290  appendPQExpBuffer(&stats, "%s %.2f s", op, elapsed_sec);
5291 
5292  run_time += elapsed_sec;
5293  }
5294  }
5295 
5296  fprintf(stderr, "done in %.2f s (%s).\n", run_time, stats.data);
5297  ResetCancelConn();
5298  PQfinish(con);
5299  termPQExpBuffer(&stats);
5300 }
5301 
5302 /*
5303  * Extract pgbench table information into global variables scale,
5304  * partition_method and partitions.
5305  */
5306 static void
5307 GetTableInfo(PGconn *con, bool scale_given)
5308 {
5309  PGresult *res;
5310 
5311  /*
5312  * get the scaling factor that should be same as count(*) from
5313  * pgbench_branches if this is not a custom query
5314  */
5315  res = PQexec(con, "select count(*) from pgbench_branches");
5317  {
5318  char *sqlState = PQresultErrorField(res, PG_DIAG_SQLSTATE);
5319 
5320  pg_log_error("could not count number of branches: %s", PQerrorMessage(con));
5321 
5322  if (sqlState && strcmp(sqlState, ERRCODE_UNDEFINED_TABLE) == 0)
5323  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".",
5324  PQdb(con));
5325 
5326  exit(1);
5327  }
5328  scale = atoi(PQgetvalue(res, 0, 0));
5329  if (scale < 0)
5330  pg_fatal("invalid count(*) from pgbench_branches: \"%s\"",
5331  PQgetvalue(res, 0, 0));
5332  PQclear(res);
5333 
5334  /* warn if we override user-given -s switch */
5335  if (scale_given)
5336  pg_log_warning("scale option ignored, using count from pgbench_branches table (%d)",
5337  scale);
5338 
5339  /*
5340  * Get the partition information for the first "pgbench_accounts" table
5341  * found in search_path.
5342  *
5343  * The result is empty if no "pgbench_accounts" is found.
5344  *
5345  * Otherwise, it always returns one row even if the table is not
5346  * partitioned (in which case the partition strategy is NULL).
5347  *
5348  * The number of partitions can be 0 even for partitioned tables, if no
5349  * partition is attached.
5350  *
5351  * We assume no partitioning on any failure, so as to avoid failing on an
5352  * old version without "pg_partitioned_table".
5353  */
5354  res = PQexec(con,
5355  "select o.n, p.partstrat, pg_catalog.count(i.inhparent) "
5356  "from pg_catalog.pg_class as c "
5357  "join pg_catalog.pg_namespace as n on (n.oid = c.relnamespace) "
5358  "cross join lateral (select pg_catalog.array_position(pg_catalog.current_schemas(true), n.nspname)) as o(n) "
5359  "left join pg_catalog.pg_partitioned_table as p on (p.partrelid = c.oid) "
5360  "left join pg_catalog.pg_inherits as i on (c.oid = i.inhparent) "
5361  "where c.relname = 'pgbench_accounts' and o.n is not null "
5362  "group by 1, 2 "
5363  "order by 1 asc "
5364  "limit 1");
5365 
5367  {
5368  /* probably an older version, coldly assume no partitioning */
5370  partitions = 0;
5371  }
5372  else if (PQntuples(res) == 0)
5373  {
5374  /*
5375  * This case is unlikely as pgbench already found "pgbench_branches"
5376  * above to compute the scale.
5377  */
5378  pg_log_error("no pgbench_accounts table found in search_path");
5379  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".", PQdb(con));
5380  exit(1);
5381  }
5382  else /* PQntuples(res) == 1 */
5383  {
5384  /* normal case, extract partition information */
5385  if (PQgetisnull(res, 0, 1))
5387  else
5388  {
5389  char *ps = PQgetvalue(res, 0, 1);
5390 
5391  /* column must be there */
5392  Assert(ps != NULL);
5393 
5394  if (strcmp(ps, "r") == 0)
5396  else if (strcmp(ps, "h") == 0)
5398  else
5399  {
5400  /* possibly a newer version with new partition method */
5401  pg_fatal("unexpected partition method: \"%s\"", ps);
5402  }
5403  }
5404 
5405  partitions = atoi(PQgetvalue(res, 0, 2));
5406  }
5407 
5408  PQclear(res);
5409 }
5410 
5411 /*
5412  * Replace :param with $n throughout the command's SQL text, which
5413  * is a modifiable string in cmd->lines.
5414  */
5415 static bool
5416 parseQuery(Command *cmd)
5417 {
5418  char *sql,
5419  *p;
5420 
5421  cmd->argc = 1;
5422 
5423  p = sql = pg_strdup(cmd->lines.data);
5424  while ((p = strchr(p, ':')) != NULL)
5425  {
5426  char var[13];
5427  char *name;
5428  int eaten;
5429 
5430  name = parseVariable(p, &eaten);
5431  if (name == NULL)
5432  {
5433  while (*p == ':')
5434  {
5435  p++;
5436  }
5437  continue;
5438  }
5439 
5440  /*
5441  * cmd->argv[0] is the SQL statement itself, so the max number of
5442  * arguments is one less than MAX_ARGS
5443  */
5444  if (cmd->argc >= MAX_ARGS)
5445  {
5446  pg_log_error("statement has too many arguments (maximum is %d): %s",
5447  MAX_ARGS - 1, cmd->lines.data);
5448  pg_free(name);
5449  return false;
5450  }
5451 
5452  sprintf(var, "$%d", cmd->argc);
5453  p = replaceVariable(&sql, p, eaten, var);
5454 
5455  cmd->argv[cmd->argc] = name;
5456  cmd->argc++;
5457  }
5458 
5459  Assert(cmd->argv[0] == NULL);
5460  cmd->argv[0] = sql;
5461  return true;
5462 }
5463 
5464 /*
5465  * syntax error while parsing a script (in practice, while parsing a
5466  * backslash command, because we don't detect syntax errors in SQL)
5467  *
5468  * source: source of script (filename or builtin-script ID)
5469  * lineno: line number within script (count from 1)
5470  * line: whole line of backslash command, if available
5471  * command: backslash command name, if available
5472  * msg: the actual error message
5473  * more: optional extra message
5474  * column: zero-based column number, or -1 if unknown
5475  */
5476 void
5477 syntax_error(const char *source, int lineno,
5478  const char *line, const char *command,
5479  const char *msg, const char *more, int column)
5480 {
5482 
5483  initPQExpBuffer(&buf);
5484 
5485  printfPQExpBuffer(&buf, "%s:%d: %s", source, lineno, msg);
5486  if (more != NULL)
5487  appendPQExpBuffer(&buf, " (%s)", more);
5488  if (column >= 0 && line == NULL)
5489  appendPQExpBuffer(&buf, " at column %d", column + 1);
5490  if (command != NULL)
5491  appendPQExpBuffer(&buf, " in command \"%s\"", command);
5492 
5493  pg_log_error("%s", buf.data);
5494 
5495  termPQExpBuffer(&buf);
5496 
5497  if (line != NULL)
5498  {
5499  fprintf(stderr, "%s\n", line);
5500  if (column >= 0)
5501  fprintf(stderr, "%*c error found here\n", column + 1, '^');
5502  }
5503 
5504  exit(1);
5505 }
5506 
5507 /*
5508  * Return a pointer to the start of the SQL command, after skipping over
5509  * whitespace and "--" comments.
5510  * If the end of the string is reached, return NULL.
5511  */
5512 static char *
5513 skip_sql_comments(char *sql_command)
5514 {
5515  char *p = sql_command;
5516 
5517  /* Skip any leading whitespace, as well as "--" style comments */
5518  for (;;)
5519  {
5520  if (isspace((unsigned char) *p))
5521  p++;
5522  else if (strncmp(p, "--", 2) == 0)
5523  {
5524  p = strchr(p, '\n');
5525  if (p == NULL)
5526  return NULL;
5527  p++;
5528  }
5529  else
5530  break;
5531  }
5532 
5533  /* NULL if there's nothing but whitespace and comments */
5534  if (*p == '\0')
5535  return NULL;
5536 
5537  return p;
5538 }
5539 
5540 /*
5541  * Parse a SQL command; return a Command struct, or NULL if it's a comment
5542  *
5543  * On entry, psqlscan.l has collected the command into "buf", so we don't
5544  * really need to do much here except check f