PostgreSQL Source Code  git master
pgbench.c
Go to the documentation of this file.
1 /*
2  * pgbench.c
3  *
4  * A simple benchmark program for PostgreSQL
5  * Originally written by Tatsuo Ishii and enhanced by many contributors.
6  *
7  * src/bin/pgbench/pgbench.c
8  * Copyright (c) 2000-2022, PostgreSQL Global Development Group
9  * ALL RIGHTS RESERVED;
10  *
11  * Permission to use, copy, modify, and distribute this software and its
12  * documentation for any purpose, without fee, and without a written agreement
13  * is hereby granted, provided that the above copyright notice and this
14  * paragraph and the following two paragraphs appear in all copies.
15  *
16  * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
17  * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
18  * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
19  * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
20  * POSSIBILITY OF SUCH DAMAGE.
21  *
22  * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIMS ANY WARRANTIES,
23  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
24  * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
25  * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
26  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
27  *
28  */
29 
30 #ifdef WIN32
31 #define FD_SETSIZE 1024 /* must set before winsock2.h is included */
32 #endif
33 
34 #include "postgres_fe.h"
35 
36 #include <ctype.h>
37 #include <float.h>
38 #include <limits.h>
39 #include <math.h>
40 #include <signal.h>
41 #include <time.h>
42 #include <sys/time.h>
43 #ifdef HAVE_SYS_RESOURCE_H
44 #include <sys/resource.h> /* for getrlimit */
45 #endif
46 
47 /* For testing, PGBENCH_USE_SELECT can be defined to force use of that code */
48 #if defined(HAVE_PPOLL) && !defined(PGBENCH_USE_SELECT)
49 #define POLL_USING_PPOLL
50 #ifdef HAVE_POLL_H
51 #include <poll.h>
52 #endif
53 #else /* no ppoll(), so use select() */
54 #define POLL_USING_SELECT
55 #ifdef HAVE_SYS_SELECT_H
56 #include <sys/select.h>
57 #endif
58 #endif
59 
60 #include "common/int.h"
61 #include "common/logging.h"
62 #include "common/pg_prng.h"
63 #include "common/string.h"
64 #include "common/username.h"
65 #include "fe_utils/cancel.h"
66 #include "fe_utils/conditional.h"
67 #include "fe_utils/option_utils.h"
68 #include "fe_utils/string_utils.h"
69 #include "getopt_long.h"
70 #include "libpq-fe.h"
71 #include "pgbench.h"
72 #include "port/pg_bitutils.h"
73 #include "portability/instr_time.h"
74 
75 #ifndef M_PI
76 #define M_PI 3.14159265358979323846
77 #endif
78 
79 #define ERRCODE_T_R_SERIALIZATION_FAILURE "40001"
80 #define ERRCODE_T_R_DEADLOCK_DETECTED "40P01"
81 #define ERRCODE_UNDEFINED_TABLE "42P01"
82 
83 /*
84  * Hashing constants
85  */
86 #define FNV_PRIME UINT64CONST(0x100000001b3)
87 #define FNV_OFFSET_BASIS UINT64CONST(0xcbf29ce484222325)
88 #define MM2_MUL UINT64CONST(0xc6a4a7935bd1e995)
89 #define MM2_MUL_TIMES_8 UINT64CONST(0x35253c9ade8f4ca8)
90 #define MM2_ROT 47
91 
92 /*
93  * Multi-platform socket set implementations
94  */
95 
96 #ifdef POLL_USING_PPOLL
97 #define SOCKET_WAIT_METHOD "ppoll"
98 
99 typedef struct socket_set
100 {
101  int maxfds; /* allocated length of pollfds[] array */
102  int curfds; /* number currently in use */
103  struct pollfd pollfds[FLEXIBLE_ARRAY_MEMBER];
104 } socket_set;
105 
106 #endif /* POLL_USING_PPOLL */
107 
108 #ifdef POLL_USING_SELECT
109 #define SOCKET_WAIT_METHOD "select"
110 
111 typedef struct socket_set
112 {
113  int maxfd; /* largest FD currently set in fds */
114  fd_set fds;
116 
117 #endif /* POLL_USING_SELECT */
118 
119 /*
120  * Multi-platform thread implementations
121  */
122 
123 #ifdef WIN32
124 /* Use Windows threads */
125 #include <windows.h>
126 #define GETERRNO() (_dosmaperr(GetLastError()), errno)
127 #define THREAD_T HANDLE
128 #define THREAD_FUNC_RETURN_TYPE unsigned
129 #define THREAD_FUNC_RETURN return 0
130 #define THREAD_FUNC_CC __stdcall
131 #define THREAD_CREATE(handle, function, arg) \
132  ((*(handle) = (HANDLE) _beginthreadex(NULL, 0, (function), (arg), 0, NULL)) == 0 ? errno : 0)
133 #define THREAD_JOIN(handle) \
134  (WaitForSingleObject(handle, INFINITE) != WAIT_OBJECT_0 ? \
135  GETERRNO() : CloseHandle(handle) ? 0 : GETERRNO())
136 #define THREAD_BARRIER_T SYNCHRONIZATION_BARRIER
137 #define THREAD_BARRIER_INIT(barrier, n) \
138  (InitializeSynchronizationBarrier((barrier), (n), 0) ? 0 : GETERRNO())
139 #define THREAD_BARRIER_WAIT(barrier) \
140  EnterSynchronizationBarrier((barrier), \
141  SYNCHRONIZATION_BARRIER_FLAGS_BLOCK_ONLY)
142 #define THREAD_BARRIER_DESTROY(barrier)
143 #elif defined(ENABLE_THREAD_SAFETY)
144 /* Use POSIX threads */
145 #include "port/pg_pthread.h"
146 #define THREAD_T pthread_t
147 #define THREAD_FUNC_RETURN_TYPE void *
148 #define THREAD_FUNC_RETURN return NULL
149 #define THREAD_FUNC_CC
150 #define THREAD_CREATE(handle, function, arg) \
151  pthread_create((handle), NULL, (function), (arg))
152 #define THREAD_JOIN(handle) \
153  pthread_join((handle), NULL)
154 #define THREAD_BARRIER_T pthread_barrier_t
155 #define THREAD_BARRIER_INIT(barrier, n) \
156  pthread_barrier_init((barrier), NULL, (n))
157 #define THREAD_BARRIER_WAIT(barrier) pthread_barrier_wait((barrier))
158 #define THREAD_BARRIER_DESTROY(barrier) pthread_barrier_destroy((barrier))
159 #else
160 /* No threads implementation, use none (-j 1) */
161 #define THREAD_T void *
162 #define THREAD_FUNC_RETURN_TYPE void *
163 #define THREAD_FUNC_RETURN return NULL
164 #define THREAD_FUNC_CC
165 #define THREAD_BARRIER_T int
166 #define THREAD_BARRIER_INIT(barrier, n) (*(barrier) = 0)
167 #define THREAD_BARRIER_WAIT(barrier)
168 #define THREAD_BARRIER_DESTROY(barrier)
169 #endif
170 
171 
172 /********************************************************************
173  * some configurable parameters */
174 
175 #define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
176 #define ALL_INIT_STEPS "dtgGvpf" /* all possible steps */
177 
178 #define LOG_STEP_SECONDS 5 /* seconds between log messages */
179 #define DEFAULT_NXACTS 10 /* default nxacts */
180 
181 #define MIN_GAUSSIAN_PARAM 2.0 /* minimum parameter for gauss */
182 
183 #define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
184 #define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
185 
186 int nxacts = 0; /* number of transactions per client */
187 int duration = 0; /* duration in seconds */
188 int64 end_time = 0; /* when to stop in micro seconds, under -T */
189 
190 /*
191  * scaling factor. for example, scale = 10 will make 1000000 tuples in
192  * pgbench_accounts table.
193  */
194 int scale = 1;
195 
196 /*
197  * fillfactor. for example, fillfactor = 90 will use only 90 percent
198  * space during inserts and leave 10 percent free.
199  */
200 int fillfactor = 100;
201 
202 /*
203  * use unlogged tables?
204  */
205 bool unlogged_tables = false;
206 
207 /*
208  * log sampling rate (1.0 = log everything, 0.0 = option not given)
209  */
210 double sample_rate = 0.0;
211 
212 /*
213  * When threads are throttled to a given rate limit, this is the target delay
214  * to reach that rate in usec. 0 is the default and means no throttling.
215  */
216 double throttle_delay = 0;
217 
218 /*
219  * Transactions which take longer than this limit (in usec) are counted as
220  * late, and reported as such, although they are completed anyway. When
221  * throttling is enabled, execution time slots that are more than this late
222  * are skipped altogether, and counted separately.
223  */
224 int64 latency_limit = 0;
225 
226 /*
227  * tablespace selection
228  */
229 char *tablespace = NULL;
230 char *index_tablespace = NULL;
231 
232 /*
233  * Number of "pgbench_accounts" partitions. 0 is the default and means no
234  * partitioning.
235  */
236 static int partitions = 0;
237 
238 /* partitioning strategy for "pgbench_accounts" */
239 typedef enum
240 {
241  PART_NONE, /* no partitioning */
242  PART_RANGE, /* range partitioning */
243  PART_HASH /* hash partitioning */
245 
247 static const char *PARTITION_METHOD[] = {"none", "range", "hash"};
248 
249 /* random seed used to initialize base_random_sequence */
250 int64 random_seed = -1;
251 
252 /*
253  * end of configurable parameters
254  *********************************************************************/
255 
256 #define nbranches 1 /* Makes little sense to change this. Change
257  * -s instead */
258 #define ntellers 10
259 #define naccounts 100000
260 
261 /*
262  * The scale factor at/beyond which 32bit integers are incapable of storing
263  * 64bit values.
264  *
265  * Although the actual threshold is 21474, we use 20000 because it is easier to
266  * document and remember, and isn't that far away from the real threshold.
267  */
268 #define SCALE_32BIT_THRESHOLD 20000
269 
270 bool use_log; /* log transaction latencies to a file */
271 bool use_quiet; /* quiet logging onto stderr */
272 int agg_interval; /* log aggregates instead of individual
273  * transactions */
274 bool per_script_stats = false; /* whether to collect stats per script */
275 int progress = 0; /* thread progress report every this seconds */
276 bool progress_timestamp = false; /* progress report with Unix time */
277 int nclients = 1; /* number of clients */
278 int nthreads = 1; /* number of threads */
279 bool is_connect; /* establish connection for each transaction */
280 bool report_per_command = false; /* report per-command latencies,
281  * retries after errors and failures
282  * (errors without retrying) */
283 int main_pid; /* main process id used in log filename */
284 
285 /*
286  * There are different types of restrictions for deciding that the current
287  * transaction with a serialization/deadlock error can no longer be retried and
288  * should be reported as failed:
289  * - max_tries (--max-tries) can be used to limit the number of tries;
290  * - latency_limit (-L) can be used to limit the total time of tries;
291  * - duration (-T) can be used to limit the total benchmark time.
292  *
293  * They can be combined together, and you need to use at least one of them to
294  * retry the transactions with serialization/deadlock errors. If none of them is
295  * used, the default value of max_tries is 1 and such transactions will not be
296  * retried.
297  */
298 
299 /*
300  * We cannot retry a transaction after the serialization/deadlock error if its
301  * number of tries reaches this maximum; if its value is zero, it is not used.
302  */
303 uint32 max_tries = 1;
304 
305 bool failures_detailed = false; /* whether to group failures in
306  * reports or logs by basic types */
307 
308 const char *pghost = NULL;
309 const char *pgport = NULL;
310 const char *username = NULL;
311 const char *dbName = NULL;
312 char *logfile_prefix = NULL;
313 const char *progname;
314 
315 #define WSEP '@' /* weight separator */
316 
317 volatile bool timer_exceeded = false; /* flag from signal handler */
318 
319 /*
320  * We don't want to allocate variables one by one; for efficiency, add a
321  * constant margin each time it overflows.
322  */
323 #define VARIABLES_ALLOC_MARGIN 8
324 
325 /*
326  * Variable definitions.
327  *
328  * If a variable only has a string value, "svalue" is that value, and value is
329  * "not set". If the value is known, "value" contains the value (in any
330  * variant).
331  *
332  * In this case "svalue" contains the string equivalent of the value, if we've
333  * had occasion to compute that, or NULL if we haven't.
334  */
335 typedef struct
336 {
337  char *name; /* variable's name */
338  char *svalue; /* its value in string form, if known */
339  PgBenchValue value; /* actual variable's value */
340 } Variable;
341 
342 /*
343  * Data structure for client variables.
344  */
345 typedef struct
346 {
347  Variable *vars; /* array of variable definitions */
348  int nvars; /* number of variables */
349 
350  /*
351  * The maximum number of variables that we can currently store in 'vars'
352  * without having to reallocate more space. We must always have max_vars
353  * >= nvars.
354  */
355  int max_vars;
356 
357  bool vars_sorted; /* are variables sorted by name? */
358 } Variables;
359 
360 #define MAX_SCRIPTS 128 /* max number of SQL scripts allowed */
361 #define SHELL_COMMAND_SIZE 256 /* maximum size allowed for shell command */
362 
363 /*
364  * Simple data structure to keep stats about something.
365  *
366  * XXX probably the first value should be kept and used as an offset for
367  * better numerical stability...
368  */
369 typedef struct SimpleStats
370 {
371  int64 count; /* how many values were encountered */
372  double min; /* the minimum seen */
373  double max; /* the maximum seen */
374  double sum; /* sum of values */
375  double sum2; /* sum of squared values */
376 } SimpleStats;
377 
378 /*
379  * The instr_time type is expensive when dealing with time arithmetic. Define
380  * a type to hold microseconds instead. Type int64 is good enough for about
381  * 584500 years.
382  */
383 typedef int64 pg_time_usec_t;
384 
385 /*
386  * Data structure to hold various statistics: per-thread and per-script stats
387  * are maintained and merged together.
388  */
389 typedef struct StatsData
390 {
391  pg_time_usec_t start_time; /* interval start time, for aggregates */
392 
393  /*----------
394  * Transactions are counted depending on their execution and outcome.
395  * First a transaction may have started or not: skipped transactions occur
396  * under --rate and --latency-limit when the client is too late to execute
397  * them. Secondly, a started transaction may ultimately succeed or fail,
398  * possibly after some retries when --max-tries is not one. Thus
399  *
400  * the number of all transactions =
401  * 'skipped' (it was too late to execute them) +
402  * 'cnt' (the number of successful transactions) +
403  * 'failed' (the number of failed transactions).
404  *
405  * A successful transaction can have several unsuccessful tries before a
406  * successful run. Thus
407  *
408  * 'cnt' (the number of successful transactions) =
409  * successfully retried transactions (they got a serialization or a
410  * deadlock error(s), but were
411  * successfully retried from the very
412  * beginning) +
413  * directly successful transactions (they were successfully completed on
414  * the first try).
415  *
416  * A failed transaction is defined as unsuccessfully retried transactions.
417  * It can be one of two types:
418  *
419  * failed (the number of failed transactions) =
420  * 'serialization_failures' (they got a serialization error and were not
421  * successfully retried) +
422  * 'deadlock_failures' (they got a deadlock error and were not
423  * successfully retried).
424  *
425  * If the transaction was retried after a serialization or a deadlock
426  * error this does not guarantee that this retry was successful. Thus
427  *
428  * 'retries' (number of retries) =
429  * number of retries in all retried transactions =
430  * number of retries in (successfully retried transactions +
431  * failed transactions);
432  *
433  * 'retried' (number of all retried transactions) =
434  * successfully retried transactions +
435  * failed transactions.
436  *----------
437  */
438  int64 cnt; /* number of successful transactions, not
439  * including 'skipped' */
440  int64 skipped; /* number of transactions skipped under --rate
441  * and --latency-limit */
442  int64 retries; /* number of retries after a serialization or
443  * a deadlock error in all the transactions */
444  int64 retried; /* number of all transactions that were
445  * retried after a serialization or a deadlock
446  * error (perhaps the last try was
447  * unsuccessful) */
448  int64 serialization_failures; /* number of transactions that were
449  * not successfully retried after a
450  * serialization error */
451  int64 deadlock_failures; /* number of transactions that were not
452  * successfully retried after a deadlock
453  * error */
456 } StatsData;
457 
458 /*
459  * For displaying Unix epoch timestamps, as some time functions may have
460  * another reference.
461  */
463 
464 /*
465  * Error status for errors during script execution.
466  */
467 typedef enum EStatus
468 {
471 
472  /* SQL errors */
476 } EStatus;
477 
478 /*
479  * Transaction status at the end of a command.
480  */
481 typedef enum TStatus
482 {
487 } TStatus;
488 
489 /* Various random sequences are initialized from this one. */
491 
492 /* Synchronization barrier for start and connection */
494 
495 /*
496  * Connection state machine states.
497  */
498 typedef enum
499 {
500  /*
501  * The client must first choose a script to execute. Once chosen, it can
502  * either be throttled (state CSTATE_PREPARE_THROTTLE under --rate), start
503  * right away (state CSTATE_START_TX) or not start at all if the timer was
504  * exceeded (state CSTATE_FINISHED).
505  */
507 
508  /*
509  * CSTATE_START_TX performs start-of-transaction processing. Establishes
510  * a new connection for the transaction in --connect mode, records the
511  * transaction start time, and proceed to the first command.
512  *
513  * Note: once a script is started, it will either error or run till its
514  * end, where it may be interrupted. It is not interrupted while running,
515  * so pgbench --time is to be understood as tx are allowed to start in
516  * that time, and will finish when their work is completed.
517  */
519 
520  /*
521  * In CSTATE_PREPARE_THROTTLE state, we calculate when to begin the next
522  * transaction, and advance to CSTATE_THROTTLE. CSTATE_THROTTLE state
523  * sleeps until that moment, then advances to CSTATE_START_TX, or
524  * CSTATE_FINISHED if the next transaction would start beyond the end of
525  * the run.
526  */
529 
530  /*
531  * We loop through these states, to process each command in the script:
532  *
533  * CSTATE_START_COMMAND starts the execution of a command. On a SQL
534  * command, the command is sent to the server, and we move to
535  * CSTATE_WAIT_RESULT state unless in pipeline mode. On a \sleep
536  * meta-command, the timer is set, and we enter the CSTATE_SLEEP state to
537  * wait for it to expire. Other meta-commands are executed immediately. If
538  * the command about to start is actually beyond the end of the script,
539  * advance to CSTATE_END_TX.
540  *
541  * CSTATE_WAIT_RESULT waits until we get a result set back from the server
542  * for the current command.
543  *
544  * CSTATE_SLEEP waits until the end of \sleep.
545  *
546  * CSTATE_END_COMMAND records the end-of-command timestamp, increments the
547  * command counter, and loops back to CSTATE_START_COMMAND state.
548  *
549  * CSTATE_SKIP_COMMAND is used by conditional branches which are not
550  * executed. It quickly skip commands that do not need any evaluation.
551  * This state can move forward several commands, till there is something
552  * to do or the end of the script.
553  */
559 
560  /*
561  * States for failed commands.
562  *
563  * If the SQL/meta command fails, in CSTATE_ERROR clean up after an error:
564  * (1) clear the conditional stack; (2) if we have an unterminated
565  * (possibly failed) transaction block, send the rollback command to the
566  * server and wait for the result in CSTATE_WAIT_ROLLBACK_RESULT. If
567  * something goes wrong with rolling back, go to CSTATE_ABORTED.
568  *
569  * But if everything is ok we are ready for future transactions: if this
570  * is a serialization or deadlock error and we can re-execute the
571  * transaction from the very beginning, go to CSTATE_RETRY; otherwise go
572  * to CSTATE_FAILURE.
573  *
574  * In CSTATE_RETRY report an error, set the same parameters for the
575  * transaction execution as in the previous tries and process the first
576  * transaction command in CSTATE_START_COMMAND.
577  *
578  * In CSTATE_FAILURE report a failure, set the parameters for the
579  * transaction execution as they were before the first run of this
580  * transaction (except for a random state) and go to CSTATE_END_TX to
581  * complete this transaction.
582  */
587 
588  /*
589  * CSTATE_END_TX performs end-of-transaction processing. It calculates
590  * latency, and logs the transaction. In --connect mode, it closes the
591  * current connection.
592  *
593  * Then either starts over in CSTATE_CHOOSE_SCRIPT, or enters
594  * CSTATE_FINISHED if we have no more work to do.
595  */
597 
598  /*
599  * Final states. CSTATE_ABORTED means that the script execution was
600  * aborted because a command failed, CSTATE_FINISHED means success.
601  */
605 
606 /*
607  * Connection state.
608  */
609 typedef struct
610 {
611  PGconn *con; /* connection handle to DB */
612  int id; /* client No. */
613  ConnectionStateEnum state; /* state machine's current state. */
614  ConditionalStack cstack; /* enclosing conditionals state */
615 
616  /*
617  * Separate randomness for each client. This is used for random functions
618  * PGBENCH_RANDOM_* during the execution of the script.
619  */
620  pg_prng_state cs_func_rs;
621 
622  int use_file; /* index in sql_script for this client */
623  int command; /* command number in script */
624 
625  /* client variables */
626  Variables variables;
627 
628  /* various times about current transaction in microseconds */
629  pg_time_usec_t txn_scheduled; /* scheduled start time of transaction */
630  pg_time_usec_t sleep_until; /* scheduled start time of next cmd */
631  pg_time_usec_t txn_begin; /* used for measuring schedule lag times */
632  pg_time_usec_t stmt_begin; /* used for measuring statement latencies */
633 
634  bool prepared[MAX_SCRIPTS]; /* whether client prepared the script */
635 
636  /*
637  * For processing failures and repeating transactions with serialization
638  * or deadlock errors:
639  */
640  EStatus estatus; /* the error status of the current transaction
641  * execution; this is ESTATUS_NO_ERROR if
642  * there were no errors */
643  pg_prng_state random_state; /* random state */
644  uint32 tries; /* how many times have we already tried the
645  * current transaction? */
646 
647  /* per client collected stats */
648  int64 cnt; /* client transaction count, for -t; skipped
649  * and failed transactions are also counted
650  * here */
651 } CState;
652 
653 /*
654  * Thread state
655  */
656 typedef struct
657 {
658  int tid; /* thread id */
659  THREAD_T thread; /* thread handle */
660  CState *state; /* array of CState */
661  int nstate; /* length of state[] */
662 
663  /*
664  * Separate randomness for each thread. Each thread option uses its own
665  * random state to make all of them independent of each other and
666  * therefore deterministic at the thread level.
667  */
668  pg_prng_state ts_choose_rs; /* random state for selecting a script */
669  pg_prng_state ts_throttle_rs; /* random state for transaction throttling */
670  pg_prng_state ts_sample_rs; /* random state for log sampling */
671 
672  int64 throttle_trigger; /* previous/next throttling (us) */
673  FILE *logfile; /* where to log, or NULL */
674 
675  /* per thread collected stats in microseconds */
676  pg_time_usec_t create_time; /* thread creation time */
677  pg_time_usec_t started_time; /* thread is running */
678  pg_time_usec_t bench_start; /* thread is benchmarking */
679  pg_time_usec_t conn_duration; /* cumulated connection and disconnection
680  * delays */
681 
682  StatsData stats;
683  int64 latency_late; /* count executed but late transactions */
684 } TState;
685 
686 /*
687  * queries read from files
688  */
689 #define SQL_COMMAND 1
690 #define META_COMMAND 2
691 
692 /*
693  * max number of backslash command arguments or SQL variables,
694  * including the command or SQL statement itself
695  */
696 #define MAX_ARGS 256
697 
698 typedef enum MetaCommand
699 {
700  META_NONE, /* not a known meta-command */
701  META_SET, /* \set */
702  META_SETSHELL, /* \setshell */
703  META_SHELL, /* \shell */
704  META_SLEEP, /* \sleep */
705  META_GSET, /* \gset */
706  META_ASET, /* \aset */
707  META_IF, /* \if */
708  META_ELIF, /* \elif */
709  META_ELSE, /* \else */
710  META_ENDIF, /* \endif */
711  META_STARTPIPELINE, /* \startpipeline */
712  META_ENDPIPELINE /* \endpipeline */
713 } MetaCommand;
714 
715 typedef enum QueryMode
716 {
717  QUERY_SIMPLE, /* simple query */
718  QUERY_EXTENDED, /* extended query */
719  QUERY_PREPARED, /* extended query with prepared statements */
721 } QueryMode;
722 
724 static const char *QUERYMODE[] = {"simple", "extended", "prepared"};
725 
726 /*
727  * struct Command represents one command in a script.
728  *
729  * lines The raw, possibly multi-line command text. Variable substitution
730  * not applied.
731  * first_line A short, single-line extract of 'lines', for error reporting.
732  * type SQL_COMMAND or META_COMMAND
733  * meta The type of meta-command, with META_NONE/GSET/ASET if command
734  * is SQL.
735  * argc Number of arguments of the command, 0 if not yet processed.
736  * argv Command arguments, the first of which is the command or SQL
737  * string itself. For SQL commands, after post-processing
738  * argv[0] is the same as 'lines' with variables substituted.
739  * varprefix SQL commands terminated with \gset or \aset have this set
740  * to a non NULL value. If nonempty, it's used to prefix the
741  * variable name that receives the value.
742  * aset do gset on all possible queries of a combined query (\;).
743  * expr Parsed expression, if needed.
744  * stats Time spent in this command.
745  * retries Number of retries after a serialization or deadlock error in the
746  * current command.
747  * failures Number of errors in the current command that were not retried.
748  */
749 typedef struct Command
750 {
752  char *first_line;
753  int type;
755  int argc;
756  char *argv[MAX_ARGS];
757  char *varprefix;
760  int64 retries;
761  int64 failures;
762 } Command;
763 
764 typedef struct ParsedScript
765 {
766  const char *desc; /* script descriptor (eg, file name) */
767  int weight; /* selection weight */
768  Command **commands; /* NULL-terminated array of Commands */
769  StatsData stats; /* total time spent in script */
770 } ParsedScript;
771 
772 static ParsedScript sql_script[MAX_SCRIPTS]; /* SQL script files */
773 static int num_scripts; /* number of scripts in sql_script[] */
774 static int64 total_weight = 0;
775 
776 static bool verbose_errors = false; /* print verbose messages of all errors */
777 
778 /* Builtin test scripts */
779 typedef struct BuiltinScript
780 {
781  const char *name; /* very short name for -b ... */
782  const char *desc; /* short description */
783  const char *script; /* actual pgbench script */
784 } BuiltinScript;
785 
786 static const BuiltinScript builtin_script[] =
787 {
788  {
789  "tpcb-like",
790  "<builtin: TPC-B (sort of)>",
791  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
792  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
793  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
794  "\\set delta random(-5000, 5000)\n"
795  "BEGIN;\n"
796  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
797  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
798  "UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;\n"
799  "UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;\n"
800  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
801  "END;\n"
802  },
803  {
804  "simple-update",
805  "<builtin: simple update>",
806  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
807  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
808  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
809  "\\set delta random(-5000, 5000)\n"
810  "BEGIN;\n"
811  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
812  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
813  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
814  "END;\n"
815  },
816  {
817  "select-only",
818  "<builtin: select only>",
819  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
820  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
821  }
822 };
823 
824 
825 /* Function prototypes */
826 static void setNullValue(PgBenchValue *pv);
827 static void setBoolValue(PgBenchValue *pv, bool bval);
828 static void setIntValue(PgBenchValue *pv, int64 ival);
829 static void setDoubleValue(PgBenchValue *pv, double dval);
830 static bool evaluateExpr(CState *st, PgBenchExpr *expr,
831  PgBenchValue *retval);
833 static void doLog(TState *thread, CState *st,
834  StatsData *agg, bool skipped, double latency, double lag);
835 static void processXactStats(TState *thread, CState *st, pg_time_usec_t *now,
836  bool skipped, StatsData *agg);
837 static void addScript(const ParsedScript *script);
839 static void finishCon(CState *st);
840 static void setalarm(int seconds);
841 static socket_set *alloc_socket_set(int count);
842 static void free_socket_set(socket_set *sa);
843 static void clear_socket_set(socket_set *sa);
844 static void add_socket_to_set(socket_set *sa, int fd, int idx);
845 static int wait_on_socket_set(socket_set *sa, int64 usecs);
846 static bool socket_has_input(socket_set *sa, int fd, int idx);
847 
848 
849 /* callback functions for our flex lexer */
850 static const PsqlScanCallbacks pgbench_callbacks = {
851  NULL, /* don't need get_variable functionality */
852 };
853 
854 static inline pg_time_usec_t
855 pg_time_now(void)
856 {
857  instr_time now;
858 
860 
862 }
863 
864 static inline void
866 {
867  if ((*now) == 0)
868  (*now) = pg_time_now();
869 }
870 
871 #define PG_TIME_GET_DOUBLE(t) (0.000001 * (t))
872 
873 static void
874 usage(void)
875 {
876  printf("%s is a benchmarking tool for PostgreSQL.\n\n"
877  "Usage:\n"
878  " %s [OPTION]... [DBNAME]\n"
879  "\nInitialization options:\n"
880  " -i, --initialize invokes initialization mode\n"
881  " -I, --init-steps=[" ALL_INIT_STEPS "]+ (default \"" DEFAULT_INIT_STEPS "\")\n"
882  " run selected initialization steps\n"
883  " -F, --fillfactor=NUM set fill factor\n"
884  " -n, --no-vacuum do not run VACUUM during initialization\n"
885  " -q, --quiet quiet logging (one message each 5 seconds)\n"
886  " -s, --scale=NUM scaling factor\n"
887  " --foreign-keys create foreign key constraints between tables\n"
888  " --index-tablespace=TABLESPACE\n"
889  " create indexes in the specified tablespace\n"
890  " --partition-method=(range|hash)\n"
891  " partition pgbench_accounts with this method (default: range)\n"
892  " --partitions=NUM partition pgbench_accounts into NUM parts (default: 0)\n"
893  " --tablespace=TABLESPACE create tables in the specified tablespace\n"
894  " --unlogged-tables create tables as unlogged tables\n"
895  "\nOptions to select what to run:\n"
896  " -b, --builtin=NAME[@W] add builtin script NAME weighted at W (default: 1)\n"
897  " (use \"-b list\" to list available scripts)\n"
898  " -f, --file=FILENAME[@W] add script FILENAME weighted at W (default: 1)\n"
899  " -N, --skip-some-updates skip updates of pgbench_tellers and pgbench_branches\n"
900  " (same as \"-b simple-update\")\n"
901  " -S, --select-only perform SELECT-only transactions\n"
902  " (same as \"-b select-only\")\n"
903  "\nBenchmarking options:\n"
904  " -c, --client=NUM number of concurrent database clients (default: 1)\n"
905  " -C, --connect establish new connection for each transaction\n"
906  " -D, --define=VARNAME=VALUE\n"
907  " define variable for use by custom script\n"
908  " -j, --jobs=NUM number of threads (default: 1)\n"
909  " -l, --log write transaction times to log file\n"
910  " -L, --latency-limit=NUM count transactions lasting more than NUM ms as late\n"
911  " -M, --protocol=simple|extended|prepared\n"
912  " protocol for submitting queries (default: simple)\n"
913  " -n, --no-vacuum do not run VACUUM before tests\n"
914  " -P, --progress=NUM show thread progress report every NUM seconds\n"
915  " -r, --report-per-command report latencies, failures and retries per command\n"
916  " -R, --rate=NUM target rate in transactions per second\n"
917  " -s, --scale=NUM report this scale factor in output\n"
918  " -t, --transactions=NUM number of transactions each client runs (default: 10)\n"
919  " -T, --time=NUM duration of benchmark test in seconds\n"
920  " -v, --vacuum-all vacuum all four standard tables before tests\n"
921  " --aggregate-interval=NUM aggregate data over NUM seconds\n"
922  " --failures-detailed report the failures grouped by basic types\n"
923  " --log-prefix=PREFIX prefix for transaction time log file\n"
924  " (default: \"pgbench_log\")\n"
925  " --max-tries=NUM max number of tries to run transaction (default: 1)\n"
926  " --progress-timestamp use Unix epoch timestamps for progress\n"
927  " --random-seed=SEED set random seed (\"time\", \"rand\", integer)\n"
928  " --sampling-rate=NUM fraction of transactions to log (e.g., 0.01 for 1%%)\n"
929  " --show-script=NAME show builtin script code, then exit\n"
930  " --verbose-errors print messages of all errors\n"
931  "\nCommon options:\n"
932  " -d, --debug print debugging output\n"
933  " -h, --host=HOSTNAME database server host or socket directory\n"
934  " -p, --port=PORT database server port number\n"
935  " -U, --username=USERNAME connect as specified database user\n"
936  " -V, --version output version information, then exit\n"
937  " -?, --help show this help, then exit\n"
938  "\n"
939  "Report bugs to <%s>.\n"
940  "%s home page: <%s>\n",
941  progname, progname, PACKAGE_BUGREPORT, PACKAGE_NAME, PACKAGE_URL);
942 }
943 
944 /* return whether str matches "^\s*[-+]?[0-9]+$" */
945 static bool
946 is_an_int(const char *str)
947 {
948  const char *ptr = str;
949 
950  /* skip leading spaces; cast is consistent with strtoint64 */
951  while (*ptr && isspace((unsigned char) *ptr))
952  ptr++;
953 
954  /* skip sign */
955  if (*ptr == '+' || *ptr == '-')
956  ptr++;
957 
958  /* at least one digit */
959  if (*ptr && !isdigit((unsigned char) *ptr))
960  return false;
961 
962  /* eat all digits */
963  while (*ptr && isdigit((unsigned char) *ptr))
964  ptr++;
965 
966  /* must have reached end of string */
967  return *ptr == '\0';
968 }
969 
970 
971 /*
972  * strtoint64 -- convert a string to 64-bit integer
973  *
974  * This function is a slightly modified version of pg_strtoint64() from
975  * src/backend/utils/adt/numutils.c.
976  *
977  * The function returns whether the conversion worked, and if so
978  * "*result" is set to the result.
979  *
980  * If not errorOK, an error message is also printed out on errors.
981  */
982 bool
983 strtoint64(const char *str, bool errorOK, int64 *result)
984 {
985  const char *ptr = str;
986  int64 tmp = 0;
987  bool neg = false;
988 
989  /*
990  * Do our own scan, rather than relying on sscanf which might be broken
991  * for long long.
992  *
993  * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
994  * value as a negative number.
995  */
996 
997  /* skip leading spaces */
998  while (*ptr && isspace((unsigned char) *ptr))
999  ptr++;
1000 
1001  /* handle sign */
1002  if (*ptr == '-')
1003  {
1004  ptr++;
1005  neg = true;
1006  }
1007  else if (*ptr == '+')
1008  ptr++;
1009 
1010  /* require at least one digit */
1011  if (unlikely(!isdigit((unsigned char) *ptr)))
1012  goto invalid_syntax;
1013 
1014  /* process digits */
1015  while (*ptr && isdigit((unsigned char) *ptr))
1016  {
1017  int8 digit = (*ptr++ - '0');
1018 
1019  if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
1020  unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
1021  goto out_of_range;
1022  }
1023 
1024  /* allow trailing whitespace, but not other trailing chars */
1025  while (*ptr != '\0' && isspace((unsigned char) *ptr))
1026  ptr++;
1027 
1028  if (unlikely(*ptr != '\0'))
1029  goto invalid_syntax;
1030 
1031  if (!neg)
1032  {
1033  if (unlikely(tmp == PG_INT64_MIN))
1034  goto out_of_range;
1035  tmp = -tmp;
1036  }
1037 
1038  *result = tmp;
1039  return true;
1040 
1041 out_of_range:
1042  if (!errorOK)
1043  pg_log_error("value \"%s\" is out of range for type bigint", str);
1044  return false;
1045 
1046 invalid_syntax:
1047  if (!errorOK)
1048  pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
1049  return false;
1050 }
1051 
1052 /* convert string to double, detecting overflows/underflows */
1053 bool
1054 strtodouble(const char *str, bool errorOK, double *dv)
1055 {
1056  char *end;
1057 
1058  errno = 0;
1059  *dv = strtod(str, &end);
1060 
1061  if (unlikely(errno != 0))
1062  {
1063  if (!errorOK)
1064  pg_log_error("value \"%s\" is out of range for type double", str);
1065  return false;
1066  }
1067 
1068  if (unlikely(end == str || *end != '\0'))
1069  {
1070  if (!errorOK)
1071  pg_log_error("invalid input syntax for type double: \"%s\"", str);
1072  return false;
1073  }
1074  return true;
1075 }
1076 
1077 /*
1078  * Initialize a prng state struct.
1079  *
1080  * We derive the seed from base_random_sequence, which must be set up already.
1081  */
1082 static void
1084 {
1086 }
1087 
1088 
1089 /*
1090  * random number generator: uniform distribution from min to max inclusive.
1091  *
1092  * Although the limits are expressed as int64, you can't generate the full
1093  * int64 range in one call, because the difference of the limits mustn't
1094  * overflow int64. This is not checked.
1095  */
1096 static int64
1097 getrand(pg_prng_state *state, int64 min, int64 max)
1098 {
1099  return min + (int64) pg_prng_uint64_range(state, 0, max - min);
1100 }
1101 
1102 /*
1103  * random number generator: exponential distribution from min to max inclusive.
1104  * the parameter is so that the density of probability for the last cut-off max
1105  * value is exp(-parameter).
1106  */
1107 static int64
1108 getExponentialRand(pg_prng_state *state, int64 min, int64 max,
1109  double parameter)
1110 {
1111  double cut,
1112  uniform,
1113  rand;
1114 
1115  /* abort if wrong parameter, but must really be checked beforehand */
1116  Assert(parameter > 0.0);
1117  cut = exp(-parameter);
1118  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1119  uniform = 1.0 - pg_prng_double(state);
1120 
1121  /*
1122  * inner expression in (cut, 1] (if parameter > 0), rand in [0, 1)
1123  */
1124  Assert((1.0 - cut) != 0.0);
1125  rand = -log(cut + (1.0 - cut) * uniform) / parameter;
1126  /* return int64 random number within between min and max */
1127  return min + (int64) ((max - min + 1) * rand);
1128 }
1129 
1130 /* random number generator: gaussian distribution from min to max inclusive */
1131 static int64
1132 getGaussianRand(pg_prng_state *state, int64 min, int64 max,
1133  double parameter)
1134 {
1135  double stdev;
1136  double rand;
1137 
1138  /* abort if parameter is too low, but must really be checked beforehand */
1139  Assert(parameter >= MIN_GAUSSIAN_PARAM);
1140 
1141  /*
1142  * Get user specified random number from this loop, with -parameter <
1143  * stdev <= parameter
1144  *
1145  * This loop is executed until the number is in the expected range.
1146  *
1147  * As the minimum parameter is 2.0, the probability of looping is low:
1148  * sqrt(-2 ln(r)) <= 2 => r >= e^{-2} ~ 0.135, then when taking the
1149  * average sinus multiplier as 2/pi, we have a 8.6% looping probability in
1150  * the worst case. For a parameter value of 5.0, the looping probability
1151  * is about e^{-5} * 2 / pi ~ 0.43%.
1152  */
1153  do
1154  {
1155  /*
1156  * pg_prng_double generates [0, 1), but for the basic version of the
1157  * Box-Muller transform the two uniformly distributed random numbers
1158  * are expected to be in (0, 1] (see
1159  * https://en.wikipedia.org/wiki/Box-Muller_transform)
1160  */
1161  double rand1 = 1.0 - pg_prng_double(state);
1162  double rand2 = 1.0 - pg_prng_double(state);
1163 
1164  /* Box-Muller basic form transform */
1165  double var_sqrt = sqrt(-2.0 * log(rand1));
1166 
1167  stdev = var_sqrt * sin(2.0 * M_PI * rand2);
1168 
1169  /*
1170  * we may try with cos, but there may be a bias induced if the
1171  * previous value fails the test. To be on the safe side, let us try
1172  * over.
1173  */
1174  }
1175  while (stdev < -parameter || stdev >= parameter);
1176 
1177  /* stdev is in [-parameter, parameter), normalization to [0,1) */
1178  rand = (stdev + parameter) / (parameter * 2.0);
1179 
1180  /* return int64 random number within between min and max */
1181  return min + (int64) ((max - min + 1) * rand);
1182 }
1183 
1184 /*
1185  * random number generator: generate a value, such that the series of values
1186  * will approximate a Poisson distribution centered on the given value.
1187  *
1188  * Individual results are rounded to integers, though the center value need
1189  * not be one.
1190  */
1191 static int64
1192 getPoissonRand(pg_prng_state *state, double center)
1193 {
1194  /*
1195  * Use inverse transform sampling to generate a value > 0, such that the
1196  * expected (i.e. average) value is the given argument.
1197  */
1198  double uniform;
1199 
1200  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1201  uniform = 1.0 - pg_prng_double(state);
1202 
1203  return (int64) (-log(uniform) * center + 0.5);
1204 }
1205 
1206 /*
1207  * Computing zipfian using rejection method, based on
1208  * "Non-Uniform Random Variate Generation",
1209  * Luc Devroye, p. 550-551, Springer 1986.
1210  *
1211  * This works for s > 1.0, but may perform badly for s very close to 1.0.
1212  */
1213 static int64
1214 computeIterativeZipfian(pg_prng_state *state, int64 n, double s)
1215 {
1216  double b = pow(2.0, s - 1.0);
1217  double x,
1218  t,
1219  u,
1220  v;
1221 
1222  /* Ensure n is sane */
1223  if (n <= 1)
1224  return 1;
1225 
1226  while (true)
1227  {
1228  /* random variates */
1229  u = pg_prng_double(state);
1230  v = pg_prng_double(state);
1231 
1232  x = floor(pow(u, -1.0 / (s - 1.0)));
1233 
1234  t = pow(1.0 + 1.0 / x, s - 1.0);
1235  /* reject if too large or out of bound */
1236  if (v * x * (t - 1.0) / (b - 1.0) <= t / b && x <= n)
1237  break;
1238  }
1239  return (int64) x;
1240 }
1241 
1242 /* random number generator: zipfian distribution from min to max inclusive */
1243 static int64
1244 getZipfianRand(pg_prng_state *state, int64 min, int64 max, double s)
1245 {
1246  int64 n = max - min + 1;
1247 
1248  /* abort if parameter is invalid */
1250 
1251  return min - 1 + computeIterativeZipfian(state, n, s);
1252 }
1253 
1254 /*
1255  * FNV-1a hash function
1256  */
1257 static int64
1258 getHashFnv1a(int64 val, uint64 seed)
1259 {
1260  int64 result;
1261  int i;
1262 
1263  result = FNV_OFFSET_BASIS ^ seed;
1264  for (i = 0; i < 8; ++i)
1265  {
1266  int32 octet = val & 0xff;
1267 
1268  val = val >> 8;
1269  result = result ^ octet;
1270  result = result * FNV_PRIME;
1271  }
1272 
1273  return result;
1274 }
1275 
1276 /*
1277  * Murmur2 hash function
1278  *
1279  * Based on original work of Austin Appleby
1280  * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash2.cpp
1281  */
1282 static int64
1283 getHashMurmur2(int64 val, uint64 seed)
1284 {
1285  uint64 result = seed ^ MM2_MUL_TIMES_8; /* sizeof(int64) */
1286  uint64 k = (uint64) val;
1287 
1288  k *= MM2_MUL;
1289  k ^= k >> MM2_ROT;
1290  k *= MM2_MUL;
1291 
1292  result ^= k;
1293  result *= MM2_MUL;
1294 
1295  result ^= result >> MM2_ROT;
1296  result *= MM2_MUL;
1297  result ^= result >> MM2_ROT;
1298 
1299  return (int64) result;
1300 }
1301 
1302 /*
1303  * Pseudorandom permutation function
1304  *
1305  * For small sizes, this generates each of the (size!) possible permutations
1306  * of integers in the range [0, size) with roughly equal probability. Once
1307  * the size is larger than 20, the number of possible permutations exceeds the
1308  * number of distinct states of the internal pseudorandom number generator,
1309  * and so not all possible permutations can be generated, but the permutations
1310  * chosen should continue to give the appearance of being random.
1311  *
1312  * THIS FUNCTION IS NOT CRYPTOGRAPHICALLY SECURE.
1313  * DO NOT USE FOR SUCH PURPOSE.
1314  */
1315 static int64
1316 permute(const int64 val, const int64 isize, const int64 seed)
1317 {
1318  /* using a high-end PRNG is probably overkill */
1320  uint64 size;
1321  uint64 v;
1322  int masklen;
1323  uint64 mask;
1324  int i;
1325 
1326  if (isize < 2)
1327  return 0; /* nothing to permute */
1328 
1329  /* Initialize prng state using the seed */
1330  pg_prng_seed(&state, (uint64) seed);
1331 
1332  /* Computations are performed on unsigned values */
1333  size = (uint64) isize;
1334  v = (uint64) val % size;
1335 
1336  /* Mask to work modulo largest power of 2 less than or equal to size */
1337  masklen = pg_leftmost_one_pos64(size);
1338  mask = (((uint64) 1) << masklen) - 1;
1339 
1340  /*
1341  * Permute the input value by applying several rounds of pseudorandom
1342  * bijective transformations. The intention here is to distribute each
1343  * input uniformly randomly across the range, and separate adjacent inputs
1344  * approximately uniformly randomly from each other, leading to a fairly
1345  * random overall choice of permutation.
1346  *
1347  * To separate adjacent inputs, we multiply by a random number modulo
1348  * (mask + 1), which is a power of 2. For this to be a bijection, the
1349  * multiplier must be odd. Since this is known to lead to less randomness
1350  * in the lower bits, we also apply a rotation that shifts the topmost bit
1351  * into the least significant bit. In the special cases where size <= 3,
1352  * mask = 1 and each of these operations is actually a no-op, so we also
1353  * XOR the value with a different random number to inject additional
1354  * randomness. Since the size is generally not a power of 2, we apply
1355  * this bijection on overlapping upper and lower halves of the input.
1356  *
1357  * To distribute the inputs uniformly across the range, we then also apply
1358  * a random offset modulo the full range.
1359  *
1360  * Taken together, these operations resemble a modified linear
1361  * congruential generator, as is commonly used in pseudorandom number
1362  * generators. The number of rounds is fairly arbitrary, but six has been
1363  * found empirically to give a fairly good tradeoff between performance
1364  * and uniform randomness. For small sizes it selects each of the (size!)
1365  * possible permutations with roughly equal probability. For larger
1366  * sizes, not all permutations can be generated, but the intended random
1367  * spread is still produced.
1368  */
1369  for (i = 0; i < 6; i++)
1370  {
1371  uint64 m,
1372  r,
1373  t;
1374 
1375  /* Random multiply (by an odd number), XOR and rotate of lower half */
1376  m = (pg_prng_uint64(&state) & mask) | 1;
1377  r = pg_prng_uint64(&state) & mask;
1378  if (v <= mask)
1379  {
1380  v = ((v * m) ^ r) & mask;
1381  v = ((v << 1) & mask) | (v >> (masklen - 1));
1382  }
1383 
1384  /* Random multiply (by an odd number), XOR and rotate of upper half */
1385  m = (pg_prng_uint64(&state) & mask) | 1;
1386  r = pg_prng_uint64(&state) & mask;
1387  t = size - 1 - v;
1388  if (t <= mask)
1389  {
1390  t = ((t * m) ^ r) & mask;
1391  t = ((t << 1) & mask) | (t >> (masklen - 1));
1392  v = size - 1 - t;
1393  }
1394 
1395  /* Random offset */
1396  r = pg_prng_uint64_range(&state, 0, size - 1);
1397  v = (v + r) % size;
1398  }
1399 
1400  return (int64) v;
1401 }
1402 
1403 /*
1404  * Initialize the given SimpleStats struct to all zeroes
1405  */
1406 static void
1408 {
1409  memset(ss, 0, sizeof(SimpleStats));
1410 }
1411 
1412 /*
1413  * Accumulate one value into a SimpleStats struct.
1414  */
1415 static void
1416 addToSimpleStats(SimpleStats *ss, double val)
1417 {
1418  if (ss->count == 0 || val < ss->min)
1419  ss->min = val;
1420  if (ss->count == 0 || val > ss->max)
1421  ss->max = val;
1422  ss->count++;
1423  ss->sum += val;
1424  ss->sum2 += val * val;
1425 }
1426 
1427 /*
1428  * Merge two SimpleStats objects
1429  */
1430 static void
1432 {
1433  if (acc->count == 0 || ss->min < acc->min)
1434  acc->min = ss->min;
1435  if (acc->count == 0 || ss->max > acc->max)
1436  acc->max = ss->max;
1437  acc->count += ss->count;
1438  acc->sum += ss->sum;
1439  acc->sum2 += ss->sum2;
1440 }
1441 
1442 /*
1443  * Initialize a StatsData struct to mostly zeroes, with its start time set to
1444  * the given value.
1445  */
1446 static void
1448 {
1449  sd->start_time = start;
1450  sd->cnt = 0;
1451  sd->skipped = 0;
1452  sd->retries = 0;
1453  sd->retried = 0;
1454  sd->serialization_failures = 0;
1455  sd->deadlock_failures = 0;
1456  initSimpleStats(&sd->latency);
1457  initSimpleStats(&sd->lag);
1458 }
1459 
1460 /*
1461  * Accumulate one additional item into the given stats object.
1462  */
1463 static void
1464 accumStats(StatsData *stats, bool skipped, double lat, double lag,
1465  EStatus estatus, int64 tries)
1466 {
1467  /* Record the skipped transaction */
1468  if (skipped)
1469  {
1470  /* no latency to record on skipped transactions */
1471  stats->skipped++;
1472  return;
1473  }
1474 
1475  /*
1476  * Record the number of retries regardless of whether the transaction was
1477  * successful or failed.
1478  */
1479  if (tries > 1)
1480  {
1481  stats->retries += (tries - 1);
1482  stats->retried++;
1483  }
1484 
1485  switch (estatus)
1486  {
1487  /* Record the successful transaction */
1488  case ESTATUS_NO_ERROR:
1489  stats->cnt++;
1490 
1491  addToSimpleStats(&stats->latency, lat);
1492 
1493  /* and possibly the same for schedule lag */
1494  if (throttle_delay)
1495  addToSimpleStats(&stats->lag, lag);
1496  break;
1497 
1498  /* Record the failed transaction */
1500  stats->serialization_failures++;
1501  break;
1503  stats->deadlock_failures++;
1504  break;
1505  default:
1506  /* internal error which should never occur */
1507  pg_fatal("unexpected error status: %d", estatus);
1508  }
1509 }
1510 
1511 /* call PQexec() and exit() on failure */
1512 static void
1513 executeStatement(PGconn *con, const char *sql)
1514 {
1515  PGresult *res;
1516 
1517  res = PQexec(con, sql);
1519  {
1520  pg_log_error("query failed: %s", PQerrorMessage(con));
1521  pg_log_error_detail("Query was: %s", sql);
1522  exit(1);
1523  }
1524  PQclear(res);
1525 }
1526 
1527 /* call PQexec() and complain, but without exiting, on failure */
1528 static void
1529 tryExecuteStatement(PGconn *con, const char *sql)
1530 {
1531  PGresult *res;
1532 
1533  res = PQexec(con, sql);
1535  {
1536  pg_log_error("%s", PQerrorMessage(con));
1537  pg_log_error_detail("(ignoring this error and continuing anyway)");
1538  }
1539  PQclear(res);
1540 }
1541 
1542 /* set up a connection to the backend */
1543 static PGconn *
1544 doConnect(void)
1545 {
1546  PGconn *conn;
1547  bool new_pass;
1548  static char *password = NULL;
1549 
1550  /*
1551  * Start the connection. Loop until we have a password if requested by
1552  * backend.
1553  */
1554  do
1555  {
1556 #define PARAMS_ARRAY_SIZE 7
1557 
1558  const char *keywords[PARAMS_ARRAY_SIZE];
1559  const char *values[PARAMS_ARRAY_SIZE];
1560 
1561  keywords[0] = "host";
1562  values[0] = pghost;
1563  keywords[1] = "port";
1564  values[1] = pgport;
1565  keywords[2] = "user";
1566  values[2] = username;
1567  keywords[3] = "password";
1568  values[3] = password;
1569  keywords[4] = "dbname";
1570  values[4] = dbName;
1571  keywords[5] = "fallback_application_name";
1572  values[5] = progname;
1573  keywords[6] = NULL;
1574  values[6] = NULL;
1575 
1576  new_pass = false;
1577 
1578  conn = PQconnectdbParams(keywords, values, true);
1579 
1580  if (!conn)
1581  {
1582  pg_log_error("connection to database \"%s\" failed", dbName);
1583  return NULL;
1584  }
1585 
1586  if (PQstatus(conn) == CONNECTION_BAD &&
1588  !password)
1589  {
1590  PQfinish(conn);
1591  password = simple_prompt("Password: ", false);
1592  new_pass = true;
1593  }
1594  } while (new_pass);
1595 
1596  /* check to see that the backend connection was successfully made */
1597  if (PQstatus(conn) == CONNECTION_BAD)
1598  {
1600  PQfinish(conn);
1601  return NULL;
1602  }
1603 
1604  return conn;
1605 }
1606 
1607 /* qsort comparator for Variable array */
1608 static int
1609 compareVariableNames(const void *v1, const void *v2)
1610 {
1611  return strcmp(((const Variable *) v1)->name,
1612  ((const Variable *) v2)->name);
1613 }
1614 
1615 /* Locate a variable by name; returns NULL if unknown */
1616 static Variable *
1617 lookupVariable(Variables *variables, char *name)
1618 {
1619  Variable key;
1620 
1621  /* On some versions of Solaris, bsearch of zero items dumps core */
1622  if (variables->nvars <= 0)
1623  return NULL;
1624 
1625  /* Sort if we have to */
1626  if (!variables->vars_sorted)
1627  {
1628  qsort((void *) variables->vars, variables->nvars, sizeof(Variable),
1630  variables->vars_sorted = true;
1631  }
1632 
1633  /* Now we can search */
1634  key.name = name;
1635  return (Variable *) bsearch((void *) &key,
1636  (void *) variables->vars,
1637  variables->nvars,
1638  sizeof(Variable),
1640 }
1641 
1642 /* Get the value of a variable, in string form; returns NULL if unknown */
1643 static char *
1644 getVariable(Variables *variables, char *name)
1645 {
1646  Variable *var;
1647  char stringform[64];
1648 
1649  var = lookupVariable(variables, name);
1650  if (var == NULL)
1651  return NULL; /* not found */
1652 
1653  if (var->svalue)
1654  return var->svalue; /* we have it in string form */
1655 
1656  /* We need to produce a string equivalent of the value */
1657  Assert(var->value.type != PGBT_NO_VALUE);
1658  if (var->value.type == PGBT_NULL)
1659  snprintf(stringform, sizeof(stringform), "NULL");
1660  else if (var->value.type == PGBT_BOOLEAN)
1661  snprintf(stringform, sizeof(stringform),
1662  "%s", var->value.u.bval ? "true" : "false");
1663  else if (var->value.type == PGBT_INT)
1664  snprintf(stringform, sizeof(stringform),
1665  INT64_FORMAT, var->value.u.ival);
1666  else if (var->value.type == PGBT_DOUBLE)
1667  snprintf(stringform, sizeof(stringform),
1668  "%.*g", DBL_DIG, var->value.u.dval);
1669  else /* internal error, unexpected type */
1670  Assert(0);
1671  var->svalue = pg_strdup(stringform);
1672  return var->svalue;
1673 }
1674 
1675 /* Try to convert variable to a value; return false on failure */
1676 static bool
1678 {
1679  size_t slen;
1680 
1681  if (var->value.type != PGBT_NO_VALUE)
1682  return true; /* no work */
1683 
1684  slen = strlen(var->svalue);
1685 
1686  if (slen == 0)
1687  /* what should it do on ""? */
1688  return false;
1689 
1690  if (pg_strcasecmp(var->svalue, "null") == 0)
1691  {
1692  setNullValue(&var->value);
1693  }
1694 
1695  /*
1696  * accept prefixes such as y, ye, n, no... but not for "o". 0/1 are
1697  * recognized later as an int, which is converted to bool if needed.
1698  */
1699  else if (pg_strncasecmp(var->svalue, "true", slen) == 0 ||
1700  pg_strncasecmp(var->svalue, "yes", slen) == 0 ||
1701  pg_strcasecmp(var->svalue, "on") == 0)
1702  {
1703  setBoolValue(&var->value, true);
1704  }
1705  else if (pg_strncasecmp(var->svalue, "false", slen) == 0 ||
1706  pg_strncasecmp(var->svalue, "no", slen) == 0 ||
1707  pg_strcasecmp(var->svalue, "off") == 0 ||
1708  pg_strcasecmp(var->svalue, "of") == 0)
1709  {
1710  setBoolValue(&var->value, false);
1711  }
1712  else if (is_an_int(var->svalue))
1713  {
1714  /* if it looks like an int, it must be an int without overflow */
1715  int64 iv;
1716 
1717  if (!strtoint64(var->svalue, false, &iv))
1718  return false;
1719 
1720  setIntValue(&var->value, iv);
1721  }
1722  else /* type should be double */
1723  {
1724  double dv;
1725 
1726  if (!strtodouble(var->svalue, true, &dv))
1727  {
1728  pg_log_error("malformed variable \"%s\" value: \"%s\"",
1729  var->name, var->svalue);
1730  return false;
1731  }
1732  setDoubleValue(&var->value, dv);
1733  }
1734  return true;
1735 }
1736 
1737 /*
1738  * Check whether a variable's name is allowed.
1739  *
1740  * We allow any non-ASCII character, as well as ASCII letters, digits, and
1741  * underscore.
1742  *
1743  * Keep this in sync with the definitions of variable name characters in
1744  * "src/fe_utils/psqlscan.l", "src/bin/psql/psqlscanslash.l" and
1745  * "src/bin/pgbench/exprscan.l". Also see parseVariable(), below.
1746  *
1747  * Note: this static function is copied from "src/bin/psql/variables.c"
1748  * but changed to disallow variable names starting with a digit.
1749  */
1750 static bool
1751 valid_variable_name(const char *name)
1752 {
1753  const unsigned char *ptr = (const unsigned char *) name;
1754 
1755  /* Mustn't be zero-length */
1756  if (*ptr == '\0')
1757  return false;
1758 
1759  /* must not start with [0-9] */
1760  if (IS_HIGHBIT_SET(*ptr) ||
1761  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1762  "_", *ptr) != NULL)
1763  ptr++;
1764  else
1765  return false;
1766 
1767  /* remaining characters can include [0-9] */
1768  while (*ptr)
1769  {
1770  if (IS_HIGHBIT_SET(*ptr) ||
1771  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1772  "_0123456789", *ptr) != NULL)
1773  ptr++;
1774  else
1775  return false;
1776  }
1777 
1778  return true;
1779 }
1780 
1781 /*
1782  * Make sure there is enough space for 'needed' more variable in the variables
1783  * array.
1784  */
1785 static void
1786 enlargeVariables(Variables *variables, int needed)
1787 {
1788  /* total number of variables required now */
1789  needed += variables->nvars;
1790 
1791  if (variables->max_vars < needed)
1792  {
1793  variables->max_vars = needed + VARIABLES_ALLOC_MARGIN;
1794  variables->vars = (Variable *)
1795  pg_realloc(variables->vars, variables->max_vars * sizeof(Variable));
1796  }
1797 }
1798 
1799 /*
1800  * Lookup a variable by name, creating it if need be.
1801  * Caller is expected to assign a value to the variable.
1802  * Returns NULL on failure (bad name).
1803  */
1804 static Variable *
1805 lookupCreateVariable(Variables *variables, const char *context, char *name)
1806 {
1807  Variable *var;
1808 
1809  var = lookupVariable(variables, name);
1810  if (var == NULL)
1811  {
1812  /*
1813  * Check for the name only when declaring a new variable to avoid
1814  * overhead.
1815  */
1816  if (!valid_variable_name(name))
1817  {
1818  pg_log_error("%s: invalid variable name: \"%s\"", context, name);
1819  return NULL;
1820  }
1821 
1822  /* Create variable at the end of the array */
1823  enlargeVariables(variables, 1);
1824 
1825  var = &(variables->vars[variables->nvars]);
1826 
1827  var->name = pg_strdup(name);
1828  var->svalue = NULL;
1829  /* caller is expected to initialize remaining fields */
1830 
1831  variables->nvars++;
1832  /* we don't re-sort the array till we have to */
1833  variables->vars_sorted = false;
1834  }
1835 
1836  return var;
1837 }
1838 
1839 /* Assign a string value to a variable, creating it if need be */
1840 /* Returns false on failure (bad name) */
1841 static bool
1842 putVariable(Variables *variables, const char *context, char *name,
1843  const char *value)
1844 {
1845  Variable *var;
1846  char *val;
1847 
1848  var = lookupCreateVariable(variables, context, name);
1849  if (!var)
1850  return false;
1851 
1852  /* dup then free, in case value is pointing at this variable */
1853  val = pg_strdup(value);
1854 
1855  free(var->svalue);
1856  var->svalue = val;
1857  var->value.type = PGBT_NO_VALUE;
1858 
1859  return true;
1860 }
1861 
1862 /* Assign a value to a variable, creating it if need be */
1863 /* Returns false on failure (bad name) */
1864 static bool
1865 putVariableValue(Variables *variables, const char *context, char *name,
1866  const PgBenchValue *value)
1867 {
1868  Variable *var;
1869 
1870  var = lookupCreateVariable(variables, context, name);
1871  if (!var)
1872  return false;
1873 
1874  free(var->svalue);
1875  var->svalue = NULL;
1876  var->value = *value;
1877 
1878  return true;
1879 }
1880 
1881 /* Assign an integer value to a variable, creating it if need be */
1882 /* Returns false on failure (bad name) */
1883 static bool
1884 putVariableInt(Variables *variables, const char *context, char *name,
1885  int64 value)
1886 {
1887  PgBenchValue val;
1888 
1889  setIntValue(&val, value);
1890  return putVariableValue(variables, context, name, &val);
1891 }
1892 
1893 /*
1894  * Parse a possible variable reference (:varname).
1895  *
1896  * "sql" points at a colon. If what follows it looks like a valid
1897  * variable name, return a malloc'd string containing the variable name,
1898  * and set *eaten to the number of characters consumed (including the colon).
1899  * Otherwise, return NULL.
1900  */
1901 static char *
1902 parseVariable(const char *sql, int *eaten)
1903 {
1904  int i = 1; /* starting at 1 skips the colon */
1905  char *name;
1906 
1907  /* keep this logic in sync with valid_variable_name() */
1908  if (IS_HIGHBIT_SET(sql[i]) ||
1909  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1910  "_", sql[i]) != NULL)
1911  i++;
1912  else
1913  return NULL;
1914 
1915  while (IS_HIGHBIT_SET(sql[i]) ||
1916  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1917  "_0123456789", sql[i]) != NULL)
1918  i++;
1919 
1920  name = pg_malloc(i);
1921  memcpy(name, &sql[1], i - 1);
1922  name[i - 1] = '\0';
1923 
1924  *eaten = i;
1925  return name;
1926 }
1927 
1928 static char *
1929 replaceVariable(char **sql, char *param, int len, char *value)
1930 {
1931  int valueln = strlen(value);
1932 
1933  if (valueln > len)
1934  {
1935  size_t offset = param - *sql;
1936 
1937  *sql = pg_realloc(*sql, strlen(*sql) - len + valueln + 1);
1938  param = *sql + offset;
1939  }
1940 
1941  if (valueln != len)
1942  memmove(param + valueln, param + len, strlen(param + len) + 1);
1943  memcpy(param, value, valueln);
1944 
1945  return param + valueln;
1946 }
1947 
1948 static char *
1949 assignVariables(Variables *variables, char *sql)
1950 {
1951  char *p,
1952  *name,
1953  *val;
1954 
1955  p = sql;
1956  while ((p = strchr(p, ':')) != NULL)
1957  {
1958  int eaten;
1959 
1960  name = parseVariable(p, &eaten);
1961  if (name == NULL)
1962  {
1963  while (*p == ':')
1964  {
1965  p++;
1966  }
1967  continue;
1968  }
1969 
1970  val = getVariable(variables, name);
1971  free(name);
1972  if (val == NULL)
1973  {
1974  p++;
1975  continue;
1976  }
1977 
1978  p = replaceVariable(&sql, p, eaten, val);
1979  }
1980 
1981  return sql;
1982 }
1983 
1984 static void
1985 getQueryParams(Variables *variables, const Command *command,
1986  const char **params)
1987 {
1988  int i;
1989 
1990  for (i = 0; i < command->argc - 1; i++)
1991  params[i] = getVariable(variables, command->argv[i + 1]);
1992 }
1993 
1994 static char *
1996 {
1997  if (pval->type == PGBT_NO_VALUE)
1998  return "none";
1999  else if (pval->type == PGBT_NULL)
2000  return "null";
2001  else if (pval->type == PGBT_INT)
2002  return "int";
2003  else if (pval->type == PGBT_DOUBLE)
2004  return "double";
2005  else if (pval->type == PGBT_BOOLEAN)
2006  return "boolean";
2007  else
2008  {
2009  /* internal error, should never get there */
2010  Assert(false);
2011  return NULL;
2012  }
2013 }
2014 
2015 /* get a value as a boolean, or tell if there is a problem */
2016 static bool
2017 coerceToBool(PgBenchValue *pval, bool *bval)
2018 {
2019  if (pval->type == PGBT_BOOLEAN)
2020  {
2021  *bval = pval->u.bval;
2022  return true;
2023  }
2024  else /* NULL, INT or DOUBLE */
2025  {
2026  pg_log_error("cannot coerce %s to boolean", valueTypeName(pval));
2027  *bval = false; /* suppress uninitialized-variable warnings */
2028  return false;
2029  }
2030 }
2031 
2032 /*
2033  * Return true or false from an expression for conditional purposes.
2034  * Non zero numerical values are true, zero and NULL are false.
2035  */
2036 static bool
2037 valueTruth(PgBenchValue *pval)
2038 {
2039  switch (pval->type)
2040  {
2041  case PGBT_NULL:
2042  return false;
2043  case PGBT_BOOLEAN:
2044  return pval->u.bval;
2045  case PGBT_INT:
2046  return pval->u.ival != 0;
2047  case PGBT_DOUBLE:
2048  return pval->u.dval != 0.0;
2049  default:
2050  /* internal error, unexpected type */
2051  Assert(0);
2052  return false;
2053  }
2054 }
2055 
2056 /* get a value as an int, tell if there is a problem */
2057 static bool
2058 coerceToInt(PgBenchValue *pval, int64 *ival)
2059 {
2060  if (pval->type == PGBT_INT)
2061  {
2062  *ival = pval->u.ival;
2063  return true;
2064  }
2065  else if (pval->type == PGBT_DOUBLE)
2066  {
2067  double dval = rint(pval->u.dval);
2068 
2069  if (isnan(dval) || !FLOAT8_FITS_IN_INT64(dval))
2070  {
2071  pg_log_error("double to int overflow for %f", dval);
2072  return false;
2073  }
2074  *ival = (int64) dval;
2075  return true;
2076  }
2077  else /* BOOLEAN or NULL */
2078  {
2079  pg_log_error("cannot coerce %s to int", valueTypeName(pval));
2080  return false;
2081  }
2082 }
2083 
2084 /* get a value as a double, or tell if there is a problem */
2085 static bool
2086 coerceToDouble(PgBenchValue *pval, double *dval)
2087 {
2088  if (pval->type == PGBT_DOUBLE)
2089  {
2090  *dval = pval->u.dval;
2091  return true;
2092  }
2093  else if (pval->type == PGBT_INT)
2094  {
2095  *dval = (double) pval->u.ival;
2096  return true;
2097  }
2098  else /* BOOLEAN or NULL */
2099  {
2100  pg_log_error("cannot coerce %s to double", valueTypeName(pval));
2101  return false;
2102  }
2103 }
2104 
2105 /* assign a null value */
2106 static void
2108 {
2109  pv->type = PGBT_NULL;
2110  pv->u.ival = 0;
2111 }
2112 
2113 /* assign a boolean value */
2114 static void
2115 setBoolValue(PgBenchValue *pv, bool bval)
2116 {
2117  pv->type = PGBT_BOOLEAN;
2118  pv->u.bval = bval;
2119 }
2120 
2121 /* assign an integer value */
2122 static void
2123 setIntValue(PgBenchValue *pv, int64 ival)
2124 {
2125  pv->type = PGBT_INT;
2126  pv->u.ival = ival;
2127 }
2128 
2129 /* assign a double value */
2130 static void
2131 setDoubleValue(PgBenchValue *pv, double dval)
2132 {
2133  pv->type = PGBT_DOUBLE;
2134  pv->u.dval = dval;
2135 }
2136 
2137 static bool
2139 {
2140  return func == PGBENCH_AND || func == PGBENCH_OR || func == PGBENCH_CASE;
2141 }
2142 
2143 /* lazy evaluation of some functions */
2144 static bool
2145 evalLazyFunc(CState *st,
2147 {
2148  PgBenchValue a1,
2149  a2;
2150  bool ba1,
2151  ba2;
2152 
2153  Assert(isLazyFunc(func) && args != NULL && args->next != NULL);
2154 
2155  /* args points to first condition */
2156  if (!evaluateExpr(st, args->expr, &a1))
2157  return false;
2158 
2159  /* second condition for AND/OR and corresponding branch for CASE */
2160  args = args->next;
2161 
2162  switch (func)
2163  {
2164  case PGBENCH_AND:
2165  if (a1.type == PGBT_NULL)
2166  {
2167  setNullValue(retval);
2168  return true;
2169  }
2170 
2171  if (!coerceToBool(&a1, &ba1))
2172  return false;
2173 
2174  if (!ba1)
2175  {
2176  setBoolValue(retval, false);
2177  return true;
2178  }
2179 
2180  if (!evaluateExpr(st, args->expr, &a2))
2181  return false;
2182 
2183  if (a2.type == PGBT_NULL)
2184  {
2185  setNullValue(retval);
2186  return true;
2187  }
2188  else if (!coerceToBool(&a2, &ba2))
2189  return false;
2190  else
2191  {
2192  setBoolValue(retval, ba2);
2193  return true;
2194  }
2195 
2196  return true;
2197 
2198  case PGBENCH_OR:
2199 
2200  if (a1.type == PGBT_NULL)
2201  {
2202  setNullValue(retval);
2203  return true;
2204  }
2205 
2206  if (!coerceToBool(&a1, &ba1))
2207  return false;
2208 
2209  if (ba1)
2210  {
2211  setBoolValue(retval, true);
2212  return true;
2213  }
2214 
2215  if (!evaluateExpr(st, args->expr, &a2))
2216  return false;
2217 
2218  if (a2.type == PGBT_NULL)
2219  {
2220  setNullValue(retval);
2221  return true;
2222  }
2223  else if (!coerceToBool(&a2, &ba2))
2224  return false;
2225  else
2226  {
2227  setBoolValue(retval, ba2);
2228  return true;
2229  }
2230 
2231  case PGBENCH_CASE:
2232  /* when true, execute branch */
2233  if (valueTruth(&a1))
2234  return evaluateExpr(st, args->expr, retval);
2235 
2236  /* now args contains next condition or final else expression */
2237  args = args->next;
2238 
2239  /* final else case? */
2240  if (args->next == NULL)
2241  return evaluateExpr(st, args->expr, retval);
2242 
2243  /* no, another when, proceed */
2244  return evalLazyFunc(st, PGBENCH_CASE, args, retval);
2245 
2246  default:
2247  /* internal error, cannot get here */
2248  Assert(0);
2249  break;
2250  }
2251  return false;
2252 }
2253 
2254 /* maximum number of function arguments */
2255 #define MAX_FARGS 16
2256 
2257 /*
2258  * Recursive evaluation of standard functions,
2259  * which do not require lazy evaluation.
2260  */
2261 static bool
2264  PgBenchValue *retval)
2265 {
2266  /* evaluate all function arguments */
2267  int nargs = 0;
2268  PgBenchValue vargs[MAX_FARGS];
2269  PgBenchExprLink *l = args;
2270  bool has_null = false;
2271 
2272  for (nargs = 0; nargs < MAX_FARGS && l != NULL; nargs++, l = l->next)
2273  {
2274  if (!evaluateExpr(st, l->expr, &vargs[nargs]))
2275  return false;
2276  has_null |= vargs[nargs].type == PGBT_NULL;
2277  }
2278 
2279  if (l != NULL)
2280  {
2281  pg_log_error("too many function arguments, maximum is %d", MAX_FARGS);
2282  return false;
2283  }
2284 
2285  /* NULL arguments */
2286  if (has_null && func != PGBENCH_IS && func != PGBENCH_DEBUG)
2287  {
2288  setNullValue(retval);
2289  return true;
2290  }
2291 
2292  /* then evaluate function */
2293  switch (func)
2294  {
2295  /* overloaded operators */
2296  case PGBENCH_ADD:
2297  case PGBENCH_SUB:
2298  case PGBENCH_MUL:
2299  case PGBENCH_DIV:
2300  case PGBENCH_MOD:
2301  case PGBENCH_EQ:
2302  case PGBENCH_NE:
2303  case PGBENCH_LE:
2304  case PGBENCH_LT:
2305  {
2306  PgBenchValue *lval = &vargs[0],
2307  *rval = &vargs[1];
2308 
2309  Assert(nargs == 2);
2310 
2311  /* overloaded type management, double if some double */
2312  if ((lval->type == PGBT_DOUBLE ||
2313  rval->type == PGBT_DOUBLE) && func != PGBENCH_MOD)
2314  {
2315  double ld,
2316  rd;
2317 
2318  if (!coerceToDouble(lval, &ld) ||
2319  !coerceToDouble(rval, &rd))
2320  return false;
2321 
2322  switch (func)
2323  {
2324  case PGBENCH_ADD:
2325  setDoubleValue(retval, ld + rd);
2326  return true;
2327 
2328  case PGBENCH_SUB:
2329  setDoubleValue(retval, ld - rd);
2330  return true;
2331 
2332  case PGBENCH_MUL:
2333  setDoubleValue(retval, ld * rd);
2334  return true;
2335 
2336  case PGBENCH_DIV:
2337  setDoubleValue(retval, ld / rd);
2338  return true;
2339 
2340  case PGBENCH_EQ:
2341  setBoolValue(retval, ld == rd);
2342  return true;
2343 
2344  case PGBENCH_NE:
2345  setBoolValue(retval, ld != rd);
2346  return true;
2347 
2348  case PGBENCH_LE:
2349  setBoolValue(retval, ld <= rd);
2350  return true;
2351 
2352  case PGBENCH_LT:
2353  setBoolValue(retval, ld < rd);
2354  return true;
2355 
2356  default:
2357  /* cannot get here */
2358  Assert(0);
2359  }
2360  }
2361  else /* we have integer operands, or % */
2362  {
2363  int64 li,
2364  ri,
2365  res;
2366 
2367  if (!coerceToInt(lval, &li) ||
2368  !coerceToInt(rval, &ri))
2369  return false;
2370 
2371  switch (func)
2372  {
2373  case PGBENCH_ADD:
2374  if (pg_add_s64_overflow(li, ri, &res))
2375  {
2376  pg_log_error("bigint add out of range");
2377  return false;
2378  }
2379  setIntValue(retval, res);
2380  return true;
2381 
2382  case PGBENCH_SUB:
2383  if (pg_sub_s64_overflow(li, ri, &res))
2384  {
2385  pg_log_error("bigint sub out of range");
2386  return false;
2387  }
2388  setIntValue(retval, res);
2389  return true;
2390 
2391  case PGBENCH_MUL:
2392  if (pg_mul_s64_overflow(li, ri, &res))
2393  {
2394  pg_log_error("bigint mul out of range");
2395  return false;
2396  }
2397  setIntValue(retval, res);
2398  return true;
2399 
2400  case PGBENCH_EQ:
2401  setBoolValue(retval, li == ri);
2402  return true;
2403 
2404  case PGBENCH_NE:
2405  setBoolValue(retval, li != ri);
2406  return true;
2407 
2408  case PGBENCH_LE:
2409  setBoolValue(retval, li <= ri);
2410  return true;
2411 
2412  case PGBENCH_LT:
2413  setBoolValue(retval, li < ri);
2414  return true;
2415 
2416  case PGBENCH_DIV:
2417  case PGBENCH_MOD:
2418  if (ri == 0)
2419  {
2420  pg_log_error("division by zero");
2421  return false;
2422  }
2423  /* special handling of -1 divisor */
2424  if (ri == -1)
2425  {
2426  if (func == PGBENCH_DIV)
2427  {
2428  /* overflow check (needed for INT64_MIN) */
2429  if (li == PG_INT64_MIN)
2430  {
2431  pg_log_error("bigint div out of range");
2432  return false;
2433  }
2434  else
2435  setIntValue(retval, -li);
2436  }
2437  else
2438  setIntValue(retval, 0);
2439  return true;
2440  }
2441  /* else divisor is not -1 */
2442  if (func == PGBENCH_DIV)
2443  setIntValue(retval, li / ri);
2444  else /* func == PGBENCH_MOD */
2445  setIntValue(retval, li % ri);
2446 
2447  return true;
2448 
2449  default:
2450  /* cannot get here */
2451  Assert(0);
2452  }
2453  }
2454 
2455  Assert(0);
2456  return false; /* NOTREACHED */
2457  }
2458 
2459  /* integer bitwise operators */
2460  case PGBENCH_BITAND:
2461  case PGBENCH_BITOR:
2462  case PGBENCH_BITXOR:
2463  case PGBENCH_LSHIFT:
2464  case PGBENCH_RSHIFT:
2465  {
2466  int64 li,
2467  ri;
2468 
2469  if (!coerceToInt(&vargs[0], &li) || !coerceToInt(&vargs[1], &ri))
2470  return false;
2471 
2472  if (func == PGBENCH_BITAND)
2473  setIntValue(retval, li & ri);
2474  else if (func == PGBENCH_BITOR)
2475  setIntValue(retval, li | ri);
2476  else if (func == PGBENCH_BITXOR)
2477  setIntValue(retval, li ^ ri);
2478  else if (func == PGBENCH_LSHIFT)
2479  setIntValue(retval, li << ri);
2480  else if (func == PGBENCH_RSHIFT)
2481  setIntValue(retval, li >> ri);
2482  else /* cannot get here */
2483  Assert(0);
2484 
2485  return true;
2486  }
2487 
2488  /* logical operators */
2489  case PGBENCH_NOT:
2490  {
2491  bool b;
2492 
2493  if (!coerceToBool(&vargs[0], &b))
2494  return false;
2495 
2496  setBoolValue(retval, !b);
2497  return true;
2498  }
2499 
2500  /* no arguments */
2501  case PGBENCH_PI:
2502  setDoubleValue(retval, M_PI);
2503  return true;
2504 
2505  /* 1 overloaded argument */
2506  case PGBENCH_ABS:
2507  {
2508  PgBenchValue *varg = &vargs[0];
2509 
2510  Assert(nargs == 1);
2511 
2512  if (varg->type == PGBT_INT)
2513  {
2514  int64 i = varg->u.ival;
2515 
2516  setIntValue(retval, i < 0 ? -i : i);
2517  }
2518  else
2519  {
2520  double d = varg->u.dval;
2521 
2522  Assert(varg->type == PGBT_DOUBLE);
2523  setDoubleValue(retval, d < 0.0 ? -d : d);
2524  }
2525 
2526  return true;
2527  }
2528 
2529  case PGBENCH_DEBUG:
2530  {
2531  PgBenchValue *varg = &vargs[0];
2532 
2533  Assert(nargs == 1);
2534 
2535  fprintf(stderr, "debug(script=%d,command=%d): ",
2536  st->use_file, st->command + 1);
2537 
2538  if (varg->type == PGBT_NULL)
2539  fprintf(stderr, "null\n");
2540  else if (varg->type == PGBT_BOOLEAN)
2541  fprintf(stderr, "boolean %s\n", varg->u.bval ? "true" : "false");
2542  else if (varg->type == PGBT_INT)
2543  fprintf(stderr, "int " INT64_FORMAT "\n", varg->u.ival);
2544  else if (varg->type == PGBT_DOUBLE)
2545  fprintf(stderr, "double %.*g\n", DBL_DIG, varg->u.dval);
2546  else /* internal error, unexpected type */
2547  Assert(0);
2548 
2549  *retval = *varg;
2550 
2551  return true;
2552  }
2553 
2554  /* 1 double argument */
2555  case PGBENCH_DOUBLE:
2556  case PGBENCH_SQRT:
2557  case PGBENCH_LN:
2558  case PGBENCH_EXP:
2559  {
2560  double dval;
2561 
2562  Assert(nargs == 1);
2563 
2564  if (!coerceToDouble(&vargs[0], &dval))
2565  return false;
2566 
2567  if (func == PGBENCH_SQRT)
2568  dval = sqrt(dval);
2569  else if (func == PGBENCH_LN)
2570  dval = log(dval);
2571  else if (func == PGBENCH_EXP)
2572  dval = exp(dval);
2573  /* else is cast: do nothing */
2574 
2575  setDoubleValue(retval, dval);
2576  return true;
2577  }
2578 
2579  /* 1 int argument */
2580  case PGBENCH_INT:
2581  {
2582  int64 ival;
2583 
2584  Assert(nargs == 1);
2585 
2586  if (!coerceToInt(&vargs[0], &ival))
2587  return false;
2588 
2589  setIntValue(retval, ival);
2590  return true;
2591  }
2592 
2593  /* variable number of arguments */
2594  case PGBENCH_LEAST:
2595  case PGBENCH_GREATEST:
2596  {
2597  bool havedouble;
2598  int i;
2599 
2600  Assert(nargs >= 1);
2601 
2602  /* need double result if any input is double */
2603  havedouble = false;
2604  for (i = 0; i < nargs; i++)
2605  {
2606  if (vargs[i].type == PGBT_DOUBLE)
2607  {
2608  havedouble = true;
2609  break;
2610  }
2611  }
2612  if (havedouble)
2613  {
2614  double extremum;
2615 
2616  if (!coerceToDouble(&vargs[0], &extremum))
2617  return false;
2618  for (i = 1; i < nargs; i++)
2619  {
2620  double dval;
2621 
2622  if (!coerceToDouble(&vargs[i], &dval))
2623  return false;
2624  if (func == PGBENCH_LEAST)
2625  extremum = Min(extremum, dval);
2626  else
2627  extremum = Max(extremum, dval);
2628  }
2629  setDoubleValue(retval, extremum);
2630  }
2631  else
2632  {
2633  int64 extremum;
2634 
2635  if (!coerceToInt(&vargs[0], &extremum))
2636  return false;
2637  for (i = 1; i < nargs; i++)
2638  {
2639  int64 ival;
2640 
2641  if (!coerceToInt(&vargs[i], &ival))
2642  return false;
2643  if (func == PGBENCH_LEAST)
2644  extremum = Min(extremum, ival);
2645  else
2646  extremum = Max(extremum, ival);
2647  }
2648  setIntValue(retval, extremum);
2649  }
2650  return true;
2651  }
2652 
2653  /* random functions */
2654  case PGBENCH_RANDOM:
2658  {
2659  int64 imin,
2660  imax,
2661  delta;
2662 
2663  Assert(nargs >= 2);
2664 
2665  if (!coerceToInt(&vargs[0], &imin) ||
2666  !coerceToInt(&vargs[1], &imax))
2667  return false;
2668 
2669  /* check random range */
2670  if (unlikely(imin > imax))
2671  {
2672  pg_log_error("empty range given to random");
2673  return false;
2674  }
2675  else if (unlikely(pg_sub_s64_overflow(imax, imin, &delta) ||
2676  pg_add_s64_overflow(delta, 1, &delta)))
2677  {
2678  /* prevent int overflows in random functions */
2679  pg_log_error("random range is too large");
2680  return false;
2681  }
2682 
2683  if (func == PGBENCH_RANDOM)
2684  {
2685  Assert(nargs == 2);
2686  setIntValue(retval, getrand(&st->cs_func_rs, imin, imax));
2687  }
2688  else /* gaussian & exponential */
2689  {
2690  double param;
2691 
2692  Assert(nargs == 3);
2693 
2694  if (!coerceToDouble(&vargs[2], &param))
2695  return false;
2696 
2697  if (func == PGBENCH_RANDOM_GAUSSIAN)
2698  {
2699  if (param < MIN_GAUSSIAN_PARAM)
2700  {
2701  pg_log_error("gaussian parameter must be at least %f (not %f)",
2702  MIN_GAUSSIAN_PARAM, param);
2703  return false;
2704  }
2705 
2706  setIntValue(retval,
2708  imin, imax, param));
2709  }
2710  else if (func == PGBENCH_RANDOM_ZIPFIAN)
2711  {
2712  if (param < MIN_ZIPFIAN_PARAM || param > MAX_ZIPFIAN_PARAM)
2713  {
2714  pg_log_error("zipfian parameter must be in range [%.3f, %.0f] (not %f)",
2716  return false;
2717  }
2718 
2719  setIntValue(retval,
2720  getZipfianRand(&st->cs_func_rs, imin, imax, param));
2721  }
2722  else /* exponential */
2723  {
2724  if (param <= 0.0)
2725  {
2726  pg_log_error("exponential parameter must be greater than zero (not %f)",
2727  param);
2728  return false;
2729  }
2730 
2731  setIntValue(retval,
2733  imin, imax, param));
2734  }
2735  }
2736 
2737  return true;
2738  }
2739 
2740  case PGBENCH_POW:
2741  {
2742  PgBenchValue *lval = &vargs[0];
2743  PgBenchValue *rval = &vargs[1];
2744  double ld,
2745  rd;
2746 
2747  Assert(nargs == 2);
2748 
2749  if (!coerceToDouble(lval, &ld) ||
2750  !coerceToDouble(rval, &rd))
2751  return false;
2752 
2753  setDoubleValue(retval, pow(ld, rd));
2754 
2755  return true;
2756  }
2757 
2758  case PGBENCH_IS:
2759  {
2760  Assert(nargs == 2);
2761 
2762  /*
2763  * note: this simple implementation is more permissive than
2764  * SQL
2765  */
2766  setBoolValue(retval,
2767  vargs[0].type == vargs[1].type &&
2768  vargs[0].u.bval == vargs[1].u.bval);
2769  return true;
2770  }
2771 
2772  /* hashing */
2773  case PGBENCH_HASH_FNV1A:
2774  case PGBENCH_HASH_MURMUR2:
2775  {
2776  int64 val,
2777  seed;
2778 
2779  Assert(nargs == 2);
2780 
2781  if (!coerceToInt(&vargs[0], &val) ||
2782  !coerceToInt(&vargs[1], &seed))
2783  return false;
2784 
2785  if (func == PGBENCH_HASH_MURMUR2)
2786  setIntValue(retval, getHashMurmur2(val, seed));
2787  else if (func == PGBENCH_HASH_FNV1A)
2788  setIntValue(retval, getHashFnv1a(val, seed));
2789  else
2790  /* cannot get here */
2791  Assert(0);
2792 
2793  return true;
2794  }
2795 
2796  case PGBENCH_PERMUTE:
2797  {
2798  int64 val,
2799  size,
2800  seed;
2801 
2802  Assert(nargs == 3);
2803 
2804  if (!coerceToInt(&vargs[0], &val) ||
2805  !coerceToInt(&vargs[1], &size) ||
2806  !coerceToInt(&vargs[2], &seed))
2807  return false;
2808 
2809  if (size <= 0)
2810  {
2811  pg_log_error("permute size parameter must be greater than zero");
2812  return false;
2813  }
2814 
2815  setIntValue(retval, permute(val, size, seed));
2816  return true;
2817  }
2818 
2819  default:
2820  /* cannot get here */
2821  Assert(0);
2822  /* dead code to avoid a compiler warning */
2823  return false;
2824  }
2825 }
2826 
2827 /* evaluate some function */
2828 static bool
2829 evalFunc(CState *st,
2831 {
2832  if (isLazyFunc(func))
2833  return evalLazyFunc(st, func, args, retval);
2834  else
2835  return evalStandardFunc(st, func, args, retval);
2836 }
2837 
2838 /*
2839  * Recursive evaluation of an expression in a pgbench script
2840  * using the current state of variables.
2841  * Returns whether the evaluation was ok,
2842  * the value itself is returned through the retval pointer.
2843  */
2844 static bool
2845 evaluateExpr(CState *st, PgBenchExpr *expr, PgBenchValue *retval)
2846 {
2847  switch (expr->etype)
2848  {
2849  case ENODE_CONSTANT:
2850  {
2851  *retval = expr->u.constant;
2852  return true;
2853  }
2854 
2855  case ENODE_VARIABLE:
2856  {
2857  Variable *var;
2858 
2859  if ((var = lookupVariable(&st->variables, expr->u.variable.varname)) == NULL)
2860  {
2861  pg_log_error("undefined variable \"%s\"", expr->u.variable.varname);
2862  return false;
2863  }
2864 
2865  if (!makeVariableValue(var))
2866  return false;
2867 
2868  *retval = var->value;
2869  return true;
2870  }
2871 
2872  case ENODE_FUNCTION:
2873  return evalFunc(st,
2874  expr->u.function.function,
2875  expr->u.function.args,
2876  retval);
2877 
2878  default:
2879  /* internal error which should never occur */
2880  pg_fatal("unexpected enode type in evaluation: %d", expr->etype);
2881  }
2882 }
2883 
2884 /*
2885  * Convert command name to meta-command enum identifier
2886  */
2888 getMetaCommand(const char *cmd)
2889 {
2890  MetaCommand mc;
2891 
2892  if (cmd == NULL)
2893  mc = META_NONE;
2894  else if (pg_strcasecmp(cmd, "set") == 0)
2895  mc = META_SET;
2896  else if (pg_strcasecmp(cmd, "setshell") == 0)
2897  mc = META_SETSHELL;
2898  else if (pg_strcasecmp(cmd, "shell") == 0)
2899  mc = META_SHELL;
2900  else if (pg_strcasecmp(cmd, "sleep") == 0)
2901  mc = META_SLEEP;
2902  else if (pg_strcasecmp(cmd, "if") == 0)
2903  mc = META_IF;
2904  else if (pg_strcasecmp(cmd, "elif") == 0)
2905  mc = META_ELIF;
2906  else if (pg_strcasecmp(cmd, "else") == 0)
2907  mc = META_ELSE;
2908  else if (pg_strcasecmp(cmd, "endif") == 0)
2909  mc = META_ENDIF;
2910  else if (pg_strcasecmp(cmd, "gset") == 0)
2911  mc = META_GSET;
2912  else if (pg_strcasecmp(cmd, "aset") == 0)
2913  mc = META_ASET;
2914  else if (pg_strcasecmp(cmd, "startpipeline") == 0)
2915  mc = META_STARTPIPELINE;
2916  else if (pg_strcasecmp(cmd, "endpipeline") == 0)
2917  mc = META_ENDPIPELINE;
2918  else
2919  mc = META_NONE;
2920  return mc;
2921 }
2922 
2923 /*
2924  * Run a shell command. The result is assigned to the variable if not NULL.
2925  * Return true if succeeded, or false on error.
2926  */
2927 static bool
2928 runShellCommand(Variables *variables, char *variable, char **argv, int argc)
2929 {
2930  char command[SHELL_COMMAND_SIZE];
2931  int i,
2932  len = 0;
2933  FILE *fp;
2934  char res[64];
2935  char *endptr;
2936  int retval;
2937 
2938  /*----------
2939  * Join arguments with whitespace separators. Arguments starting with
2940  * exactly one colon are treated as variables:
2941  * name - append a string "name"
2942  * :var - append a variable named 'var'
2943  * ::name - append a string ":name"
2944  *----------
2945  */
2946  for (i = 0; i < argc; i++)
2947  {
2948  char *arg;
2949  int arglen;
2950 
2951  if (argv[i][0] != ':')
2952  {
2953  arg = argv[i]; /* a string literal */
2954  }
2955  else if (argv[i][1] == ':')
2956  {
2957  arg = argv[i] + 1; /* a string literal starting with colons */
2958  }
2959  else if ((arg = getVariable(variables, argv[i] + 1)) == NULL)
2960  {
2961  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[i]);
2962  return false;
2963  }
2964 
2965  arglen = strlen(arg);
2966  if (len + arglen + (i > 0 ? 1 : 0) >= SHELL_COMMAND_SIZE - 1)
2967  {
2968  pg_log_error("%s: shell command is too long", argv[0]);
2969  return false;
2970  }
2971 
2972  if (i > 0)
2973  command[len++] = ' ';
2974  memcpy(command + len, arg, arglen);
2975  len += arglen;
2976  }
2977 
2978  command[len] = '\0';
2979 
2980  /* Fast path for non-assignment case */
2981  if (variable == NULL)
2982  {
2983  if (system(command))
2984  {
2985  if (!timer_exceeded)
2986  pg_log_error("%s: could not launch shell command", argv[0]);
2987  return false;
2988  }
2989  return true;
2990  }
2991 
2992  /* Execute the command with pipe and read the standard output. */
2993  if ((fp = popen(command, "r")) == NULL)
2994  {
2995  pg_log_error("%s: could not launch shell command", argv[0]);
2996  return false;
2997  }
2998  if (fgets(res, sizeof(res), fp) == NULL)
2999  {
3000  if (!timer_exceeded)
3001  pg_log_error("%s: could not read result of shell command", argv[0]);
3002  (void) pclose(fp);
3003  return false;
3004  }
3005  if (pclose(fp) < 0)
3006  {
3007  pg_log_error("%s: could not close shell command", argv[0]);
3008  return false;
3009  }
3010 
3011  /* Check whether the result is an integer and assign it to the variable */
3012  retval = (int) strtol(res, &endptr, 10);
3013  while (*endptr != '\0' && isspace((unsigned char) *endptr))
3014  endptr++;
3015  if (*res == '\0' || *endptr != '\0')
3016  {
3017  pg_log_error("%s: shell command must return an integer (not \"%s\")", argv[0], res);
3018  return false;
3019  }
3020  if (!putVariableInt(variables, "setshell", variable, retval))
3021  return false;
3022 
3023  pg_log_debug("%s: shell parameter name: \"%s\", value: \"%s\"", argv[0], argv[1], res);
3024 
3025  return true;
3026 }
3028 #define MAX_PREPARE_NAME 32
3029 static void
3030 preparedStatementName(char *buffer, int file, int state)
3031 {
3032  sprintf(buffer, "P%d_%d", file, state);
3033 }
3034 
3035 /*
3036  * Report the abortion of the client when processing SQL commands.
3037  */
3038 static void
3039 commandFailed(CState *st, const char *cmd, const char *message)
3040 {
3041  pg_log_error("client %d aborted in command %d (%s) of script %d; %s",
3042  st->id, st->command, cmd, st->use_file, message);
3043 }
3044 
3045 /*
3046  * Report the error in the command while the script is executing.
3047  */
3048 static void
3049 commandError(CState *st, const char *message)
3050 {
3052  pg_log_info("client %d got an error in command %d (SQL) of script %d; %s",
3053  st->id, st->command, st->use_file, message);
3054 }
3055 
3056 /* return a script number with a weighted choice. */
3057 static int
3058 chooseScript(TState *thread)
3059 {
3060  int i = 0;
3061  int64 w;
3062 
3063  if (num_scripts == 1)
3064  return 0;
3065 
3066  w = getrand(&thread->ts_choose_rs, 0, total_weight - 1);
3067  do
3068  {
3069  w -= sql_script[i++].weight;
3070  } while (w >= 0);
3071 
3072  return i - 1;
3073 }
3074 
3075 /* Send a SQL command, using the chosen querymode */
3076 static bool
3077 sendCommand(CState *st, Command *command)
3078 {
3079  int r;
3080 
3081  if (querymode == QUERY_SIMPLE)
3082  {
3083  char *sql;
3084 
3085  sql = pg_strdup(command->argv[0]);
3086  sql = assignVariables(&st->variables, sql);
3087 
3088  pg_log_debug("client %d sending %s", st->id, sql);
3089  r = PQsendQuery(st->con, sql);
3090  free(sql);
3091  }
3092  else if (querymode == QUERY_EXTENDED)
3093  {
3094  const char *sql = command->argv[0];
3095  const char *params[MAX_ARGS];
3096 
3097  getQueryParams(&st->variables, command, params);
3098 
3099  pg_log_debug("client %d sending %s", st->id, sql);
3100  r = PQsendQueryParams(st->con, sql, command->argc - 1,
3101  NULL, params, NULL, NULL, 0);
3102  }
3103  else if (querymode == QUERY_PREPARED)
3104  {
3105  char name[MAX_PREPARE_NAME];
3106  const char *params[MAX_ARGS];
3107 
3108  if (!st->prepared[st->use_file])
3109  {
3110  int j;
3111  Command **commands = sql_script[st->use_file].commands;
3112 
3113  for (j = 0; commands[j] != NULL; j++)
3114  {
3115  PGresult *res;
3116  char name[MAX_PREPARE_NAME];
3117 
3118  if (commands[j]->type != SQL_COMMAND)
3119  continue;
3121  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3122  {
3123  res = PQprepare(st->con, name,
3124  commands[j]->argv[0], commands[j]->argc - 1, NULL);
3126  pg_log_error("%s", PQerrorMessage(st->con));
3127  PQclear(res);
3128  }
3129  else
3130  {
3131  /*
3132  * In pipeline mode, we use asynchronous functions. If a
3133  * server-side error occurs, it will be processed later
3134  * among the other results.
3135  */
3136  if (!PQsendPrepare(st->con, name,
3137  commands[j]->argv[0], commands[j]->argc - 1, NULL))
3138  pg_log_error("%s", PQerrorMessage(st->con));
3139  }
3140  }
3141  st->prepared[st->use_file] = true;
3142  }
3143 
3144  getQueryParams(&st->variables, command, params);
3146 
3147  pg_log_debug("client %d sending %s", st->id, name);
3148  r = PQsendQueryPrepared(st->con, name, command->argc - 1,
3149  params, NULL, NULL, 0);
3150  }
3151  else /* unknown sql mode */
3152  r = 0;
3153 
3154  if (r == 0)
3155  {
3156  pg_log_debug("client %d could not send %s", st->id, command->argv[0]);
3157  return false;
3158  }
3159  else
3160  return true;
3161 }
3162 
3163 /*
3164  * Get the error status from the error code.
3165  */
3166 static EStatus
3167 getSQLErrorStatus(const char *sqlState)
3168 {
3169  if (sqlState != NULL)
3170  {
3171  if (strcmp(sqlState, ERRCODE_T_R_SERIALIZATION_FAILURE) == 0)
3173  else if (strcmp(sqlState, ERRCODE_T_R_DEADLOCK_DETECTED) == 0)
3174  return ESTATUS_DEADLOCK_ERROR;
3175  }
3176 
3177  return ESTATUS_OTHER_SQL_ERROR;
3178 }
3179 
3180 /*
3181  * Returns true if this type of error can be retried.
3182  */
3183 static bool
3184 canRetryError(EStatus estatus)
3185 {
3186  return (estatus == ESTATUS_SERIALIZATION_ERROR ||
3187  estatus == ESTATUS_DEADLOCK_ERROR);
3188 }
3189 
3190 /*
3191  * Process query response from the backend.
3192  *
3193  * If varprefix is not NULL, it's the variable name prefix where to store
3194  * the results of the *last* command (META_GSET) or *all* commands
3195  * (META_ASET).
3196  *
3197  * Returns true if everything is A-OK, false if any error occurs.
3198  */
3199 static bool
3200 readCommandResponse(CState *st, MetaCommand meta, char *varprefix)
3201 {
3202  PGresult *res;
3203  PGresult *next_res;
3204  int qrynum = 0;
3205 
3206  /*
3207  * varprefix should be set only with \gset or \aset, and \endpipeline and
3208  * SQL commands do not need it.
3209  */
3210  Assert((meta == META_NONE && varprefix == NULL) ||
3211  ((meta == META_ENDPIPELINE) && varprefix == NULL) ||
3212  ((meta == META_GSET || meta == META_ASET) && varprefix != NULL));
3213 
3214  res = PQgetResult(st->con);
3215 
3216  while (res != NULL)
3217  {
3218  bool is_last;
3219 
3220  /* peek at the next result to know whether the current is last */
3221  next_res = PQgetResult(st->con);
3222  is_last = (next_res == NULL);
3223 
3224  switch (PQresultStatus(res))
3225  {
3226  case PGRES_COMMAND_OK: /* non-SELECT commands */
3227  case PGRES_EMPTY_QUERY: /* may be used for testing no-op overhead */
3228  if (is_last && meta == META_GSET)
3229  {
3230  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3231  st->id, st->use_file, st->command, qrynum, 0);
3233  goto error;
3234  }
3235  break;
3236 
3237  case PGRES_TUPLES_OK:
3238  if ((is_last && meta == META_GSET) || meta == META_ASET)
3239  {
3240  int ntuples = PQntuples(res);
3241 
3242  if (meta == META_GSET && ntuples != 1)
3243  {
3244  /* under \gset, report the error */
3245  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3246  st->id, st->use_file, st->command, qrynum, PQntuples(res));
3248  goto error;
3249  }
3250  else if (meta == META_ASET && ntuples <= 0)
3251  {
3252  /* coldly skip empty result under \aset */
3253  break;
3254  }
3255 
3256  /* store results into variables */
3257  for (int fld = 0; fld < PQnfields(res); fld++)
3258  {
3259  char *varname = PQfname(res, fld);
3260 
3261  /* allocate varname only if necessary, freed below */
3262  if (*varprefix != '\0')
3263  varname = psprintf("%s%s", varprefix, varname);
3264 
3265  /* store last row result as a string */
3266  if (!putVariable(&st->variables, meta == META_ASET ? "aset" : "gset", varname,
3267  PQgetvalue(res, ntuples - 1, fld)))
3268  {
3269  /* internal error */
3270  pg_log_error("client %d script %d command %d query %d: error storing into variable %s",
3271  st->id, st->use_file, st->command, qrynum, varname);
3273  goto error;
3274  }
3275 
3276  if (*varprefix != '\0')
3277  pg_free(varname);
3278  }
3279  }
3280  /* otherwise the result is simply thrown away by PQclear below */
3281  break;
3282 
3283  case PGRES_PIPELINE_SYNC:
3284  pg_log_debug("client %d pipeline ending", st->id);
3285  if (PQexitPipelineMode(st->con) != 1)
3286  pg_log_error("client %d failed to exit pipeline mode: %s", st->id,
3287  PQerrorMessage(st->con));
3288  break;
3289 
3290  case PGRES_NONFATAL_ERROR:
3291  case PGRES_FATAL_ERROR:
3293  PG_DIAG_SQLSTATE));
3294  if (canRetryError(st->estatus))
3295  {
3296  if (verbose_errors)
3297  commandError(st, PQerrorMessage(st->con));
3298  goto error;
3299  }
3300  /* fall through */
3301 
3302  default:
3303  /* anything else is unexpected */
3304  pg_log_error("client %d script %d aborted in command %d query %d: %s",
3305  st->id, st->use_file, st->command, qrynum,
3306  PQerrorMessage(st->con));
3307  goto error;
3308  }
3309 
3310  PQclear(res);
3311  qrynum++;
3312  res = next_res;
3313  }
3314 
3315  if (qrynum == 0)
3316  {
3317  pg_log_error("client %d command %d: no results", st->id, st->command);
3318  return false;
3319  }
3320 
3321  return true;
3322 
3323 error:
3324  PQclear(res);
3325  PQclear(next_res);
3326  do
3327  {
3328  res = PQgetResult(st->con);
3329  PQclear(res);
3330  } while (res);
3331 
3332  return false;
3333 }
3334 
3335 /*
3336  * Parse the argument to a \sleep command, and return the requested amount
3337  * of delay, in microseconds. Returns true on success, false on error.
3338  */
3339 static bool
3340 evaluateSleep(Variables *variables, int argc, char **argv, int *usecs)
3341 {
3342  char *var;
3343  int usec;
3344 
3345  if (*argv[1] == ':')
3346  {
3347  if ((var = getVariable(variables, argv[1] + 1)) == NULL)
3348  {
3349  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[1] + 1);
3350  return false;
3351  }
3352 
3353  usec = atoi(var);
3354 
3355  /* Raise an error if the value of a variable is not a number */
3356  if (usec == 0 && !isdigit((unsigned char) *var))
3357  {
3358  pg_log_error("%s: invalid sleep time \"%s\" for variable \"%s\"",
3359  argv[0], var, argv[1] + 1);
3360  return false;
3361  }
3362  }
3363  else
3364  usec = atoi(argv[1]);
3365 
3366  if (argc > 2)
3367  {
3368  if (pg_strcasecmp(argv[2], "ms") == 0)
3369  usec *= 1000;
3370  else if (pg_strcasecmp(argv[2], "s") == 0)
3371  usec *= 1000000;
3372  }
3373  else
3374  usec *= 1000000;
3375 
3376  *usecs = usec;
3377  return true;
3378 }
3379 
3380 
3381 /*
3382  * Returns true if the error can be retried.
3383  */
3384 static bool
3386 {
3388 
3389  /* We can only retry serialization or deadlock errors. */
3390  if (!canRetryError(st->estatus))
3391  return false;
3392 
3393  /*
3394  * We must have at least one option to limit the retrying of transactions
3395  * that got an error.
3396  */
3398 
3399  /*
3400  * We cannot retry the error if we have reached the maximum number of
3401  * tries.
3402  */
3403  if (max_tries && st->tries >= max_tries)
3404  return false;
3405 
3406  /*
3407  * We cannot retry the error if we spent too much time on this
3408  * transaction.
3409  */
3410  if (latency_limit)
3411  {
3413  if (*now - st->txn_scheduled > latency_limit)
3414  return false;
3415  }
3416 
3417  /*
3418  * We cannot retry the error if the benchmark duration is over.
3419  */
3420  if (timer_exceeded)
3421  return false;
3422 
3423  /* OK */
3424  return true;
3425 }
3426 
3427 /*
3428  * Read results and discard it until a sync point.
3429  */
3430 static int
3432 {
3433  /* send a sync */
3434  if (!PQpipelineSync(st->con))
3435  {
3436  pg_log_error("client %d aborted: failed to send a pipeline sync",
3437  st->id);
3438  return 0;
3439  }
3440 
3441  /* receive PGRES_PIPELINE_SYNC and null following it */
3442  for (;;)
3443  {
3444  PGresult *res = PQgetResult(st->con);
3445 
3447  {
3448  PQclear(res);
3449  res = PQgetResult(st->con);
3450  Assert(res == NULL);
3451  break;
3452  }
3453  PQclear(res);
3454  }
3455 
3456  /* exit pipeline */
3457  if (PQexitPipelineMode(st->con) != 1)
3458  {
3459  pg_log_error("client %d aborted: failed to exit pipeline mode for rolling back the failed transaction",
3460  st->id);
3461  return 0;
3462  }
3463  return 1;
3464 }
3465 
3466 /*
3467  * Get the transaction status at the end of a command especially for
3468  * checking if we are in a (failed) transaction block.
3469  */
3470 static TStatus
3472 {
3473  PGTransactionStatusType tx_status;
3474 
3475  tx_status = PQtransactionStatus(con);
3476  switch (tx_status)
3477  {
3478  case PQTRANS_IDLE:
3479  return TSTATUS_IDLE;
3480  case PQTRANS_INTRANS:
3481  case PQTRANS_INERROR:
3482  return TSTATUS_IN_BLOCK;
3483  case PQTRANS_UNKNOWN:
3484  /* PQTRANS_UNKNOWN is expected given a broken connection */
3485  if (PQstatus(con) == CONNECTION_BAD)
3486  return TSTATUS_CONN_ERROR;
3487  /* fall through */
3488  case PQTRANS_ACTIVE:
3489  default:
3490 
3491  /*
3492  * We cannot find out whether we are in a transaction block or
3493  * not. Internal error which should never occur.
3494  */
3495  pg_log_error("unexpected transaction status %d", tx_status);
3496  return TSTATUS_OTHER_ERROR;
3497  }
3498 
3499  /* not reached */
3500  Assert(false);
3501  return TSTATUS_OTHER_ERROR;
3502 }
3503 
3504 /*
3505  * Print verbose messages of an error
3506  */
3507 static void
3508 printVerboseErrorMessages(CState *st, pg_time_usec_t *now, bool is_retry)
3509 {
3510  static PQExpBuffer buf = NULL;
3511 
3512  if (buf == NULL)
3513  buf = createPQExpBuffer();
3514  else
3516 
3517  printfPQExpBuffer(buf, "client %d ", st->id);
3518  appendPQExpBuffer(buf, "%s",
3519  (is_retry ?
3520  "repeats the transaction after the error" :
3521  "ends the failed transaction"));
3522  appendPQExpBuffer(buf, " (try %u", st->tries);
3523 
3524  /* Print max_tries if it is not unlimitted. */
3525  if (max_tries)
3526  appendPQExpBuffer(buf, "/%u", max_tries);
3527 
3528  /*
3529  * If the latency limit is used, print a percentage of the current
3530  * transaction latency from the latency limit.
3531  */
3532  if (latency_limit)
3533  {
3535  appendPQExpBuffer(buf, ", %.3f%% of the maximum time of tries was used",
3536  (100.0 * (*now - st->txn_scheduled) / latency_limit));
3537  }
3538  appendPQExpBuffer(buf, ")\n");
3539 
3540  pg_log_info("%s", buf->data);
3541 }
3542 
3543 /*
3544  * Advance the state machine of a connection.
3545  */
3546 static void
3547 advanceConnectionState(TState *thread, CState *st, StatsData *agg)
3548 {
3549 
3550  /*
3551  * gettimeofday() isn't free, so we get the current timestamp lazily the
3552  * first time it's needed, and reuse the same value throughout this
3553  * function after that. This also ensures that e.g. the calculated
3554  * latency reported in the log file and in the totals are the same. Zero
3555  * means "not set yet". Reset "now" when we execute shell commands or
3556  * expressions, which might take a non-negligible amount of time, though.
3557  */
3558  pg_time_usec_t now = 0;
3559 
3560  /*
3561  * Loop in the state machine, until we have to wait for a result from the
3562  * server or have to sleep for throttling or \sleep.
3563  *
3564  * Note: In the switch-statement below, 'break' will loop back here,
3565  * meaning "continue in the state machine". Return is used to return to
3566  * the caller, giving the thread the opportunity to advance another
3567  * client.
3568  */
3569  for (;;)
3570  {
3571  Command *command;
3572 
3573  switch (st->state)
3574  {
3575  /* Select transaction (script) to run. */
3576  case CSTATE_CHOOSE_SCRIPT:
3577  st->use_file = chooseScript(thread);
3579 
3580  /* reset transaction variables to default values */
3581  st->estatus = ESTATUS_NO_ERROR;
3582  st->tries = 1;
3583 
3584  pg_log_debug("client %d executing script \"%s\"",
3585  st->id, sql_script[st->use_file].desc);
3586 
3587  /*
3588  * If time is over, we're done; otherwise, get ready to start
3589  * a new transaction, or to get throttled if that's requested.
3590  */
3593  break;
3594 
3595  /* Start new transaction (script) */
3596  case CSTATE_START_TX:
3598 
3599  /* establish connection if needed, i.e. under --connect */
3600  if (st->con == NULL)
3601  {
3602  pg_time_usec_t start = now;
3603 
3604  if ((st->con = doConnect()) == NULL)
3605  {
3606  /*
3607  * as the bench is already running, we do not abort
3608  * the process
3609  */
3610  pg_log_error("client %d aborted while establishing connection", st->id);
3611  st->state = CSTATE_ABORTED;
3612  break;
3613  }
3614 
3615  /* reset now after connection */
3616  now = pg_time_now();
3617 
3618  thread->conn_duration += now - start;
3619 
3620  /* Reset session-local state */
3621  memset(st->prepared, 0, sizeof(st->prepared));
3622  }
3623 
3624  /*
3625  * It is the first try to run this transaction. Remember the
3626  * random state: maybe it will get an error and we will need
3627  * to run it again.
3628  */
3629  st->random_state = st->cs_func_rs;
3630 
3631  /* record transaction start time */
3632  st->txn_begin = now;
3633 
3634  /*
3635  * When not throttling, this is also the transaction's
3636  * scheduled start time.
3637  */
3638  if (!throttle_delay)
3639  st->txn_scheduled = now;
3640 
3641  /* Begin with the first command */
3643  st->command = 0;
3644  break;
3645 
3646  /*
3647  * Handle throttling once per transaction by sleeping.
3648  */
3650 
3651  /*
3652  * Generate a delay such that the series of delays will
3653  * approximate a Poisson distribution centered on the
3654  * throttle_delay time.
3655  *
3656  * If transactions are too slow or a given wait is shorter
3657  * than a transaction, the next transaction will start right
3658  * away.
3659  */
3660  Assert(throttle_delay > 0);
3661 
3662  thread->throttle_trigger +=
3664  st->txn_scheduled = thread->throttle_trigger;
3665 
3666  /*
3667  * If --latency-limit is used, and this slot is already late
3668  * so that the transaction will miss the latency limit even if
3669  * it completed immediately, skip this time slot and loop to
3670  * reschedule.
3671  */
3672  if (latency_limit)
3673  {
3675 
3676  if (thread->throttle_trigger < now - latency_limit)
3677  {
3678  processXactStats(thread, st, &now, true, agg);
3679 
3680  /*
3681  * Finish client if -T or -t was exceeded.
3682  *
3683  * Stop counting skipped transactions under -T as soon
3684  * as the timer is exceeded. Because otherwise it can
3685  * take a very long time to count all of them
3686  * especially when quite a lot of them happen with
3687  * unrealistically high rate setting in -R, which
3688  * would prevent pgbench from ending immediately.
3689  * Because of this behavior, note that there is no
3690  * guarantee that all skipped transactions are counted
3691  * under -T though there is under -t. This is OK in
3692  * practice because it's very unlikely to happen with
3693  * realistic setting.
3694  */
3695  if (timer_exceeded || (nxacts > 0 && st->cnt >= nxacts))
3696  st->state = CSTATE_FINISHED;
3697 
3698  /* Go back to top of loop with CSTATE_PREPARE_THROTTLE */
3699  break;
3700  }
3701  }
3702 
3703  /*
3704  * stop client if next transaction is beyond pgbench end of
3705  * execution; otherwise, throttle it.
3706  */
3707  st->state = end_time > 0 && st->txn_scheduled > end_time ?
3709  break;
3710 
3711  /*
3712  * Wait until it's time to start next transaction.
3713  */
3714  case CSTATE_THROTTLE:
3716 
3717  if (now < st->txn_scheduled)
3718  return; /* still sleeping, nothing to do here */
3719 
3720  /* done sleeping, but don't start transaction if we're done */
3722  break;
3723 
3724  /*
3725  * Send a command to server (or execute a meta-command)
3726  */
3727  case CSTATE_START_COMMAND:
3728  command = sql_script[st->use_file].commands[st->command];
3729 
3730  /* Transition to script end processing if done */
3731  if (command == NULL)
3732  {
3733  st->state = CSTATE_END_TX;
3734  break;
3735  }
3736 
3737  /* record begin time of next command, and initiate it */
3738  if (report_per_command)
3739  {
3741  st->stmt_begin = now;
3742  }
3743 
3744  /* Execute the command */
3745  if (command->type == SQL_COMMAND)
3746  {
3747  /* disallow \aset and \gset in pipeline mode */
3748  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3749  {
3750  if (command->meta == META_GSET)
3751  {
3752  commandFailed(st, "gset", "\\gset is not allowed in pipeline mode");
3753  st->state = CSTATE_ABORTED;
3754  break;
3755  }
3756  else if (command->meta == META_ASET)
3757  {
3758  commandFailed(st, "aset", "\\aset is not allowed in pipeline mode");
3759  st->state = CSTATE_ABORTED;
3760  break;
3761  }
3762  }
3763 
3764  if (!sendCommand(st, command))
3765  {
3766  commandFailed(st, "SQL", "SQL command send failed");
3767  st->state = CSTATE_ABORTED;
3768  }
3769  else
3770  {
3771  /* Wait for results, unless in pipeline mode */
3772  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3773  st->state = CSTATE_WAIT_RESULT;
3774  else
3775  st->state = CSTATE_END_COMMAND;
3776  }
3777  }
3778  else if (command->type == META_COMMAND)
3779  {
3780  /*-----
3781  * Possible state changes when executing meta commands:
3782  * - on errors CSTATE_ABORTED
3783  * - on sleep CSTATE_SLEEP
3784  * - else CSTATE_END_COMMAND
3785  */
3786  st->state = executeMetaCommand(st, &now);
3787  if (st->state == CSTATE_ABORTED)
3789  }
3790 
3791  /*
3792  * We're now waiting for an SQL command to complete, or
3793  * finished processing a metacommand, or need to sleep, or
3794  * something bad happened.
3795  */
3796  Assert(st->state == CSTATE_WAIT_RESULT ||
3797  st->state == CSTATE_END_COMMAND ||
3798  st->state == CSTATE_SLEEP ||
3799  st->state == CSTATE_ABORTED);
3800  break;
3801 
3802  /*
3803  * non executed conditional branch
3804  */
3805  case CSTATE_SKIP_COMMAND:
3807  /* quickly skip commands until something to do... */
3808  while (true)
3809  {
3810  Command *command;
3811 
3812  command = sql_script[st->use_file].commands[st->command];
3813 
3814  /* cannot reach end of script in that state */
3815  Assert(command != NULL);
3816 
3817  /*
3818  * if this is conditional related, update conditional
3819  * state
3820  */
3821  if (command->type == META_COMMAND &&
3822  (command->meta == META_IF ||
3823  command->meta == META_ELIF ||
3824  command->meta == META_ELSE ||
3825  command->meta == META_ENDIF))
3826  {
3827  switch (conditional_stack_peek(st->cstack))
3828  {
3829  case IFSTATE_FALSE:
3830  if (command->meta == META_IF ||
3831  command->meta == META_ELIF)
3832  {
3833  /* we must evaluate the condition */
3835  }
3836  else if (command->meta == META_ELSE)
3837  {
3838  /* we must execute next command */
3842  st->command++;
3843  }
3844  else if (command->meta == META_ENDIF)
3845  {
3848  if (conditional_active(st->cstack))
3850 
3851  /*
3852  * else state remains in
3853  * CSTATE_SKIP_COMMAND
3854  */
3855  st->command++;
3856  }
3857  break;
3858 
3859  case IFSTATE_IGNORED:
3860  case IFSTATE_ELSE_FALSE:
3861  if (command->meta == META_IF)
3863  IFSTATE_IGNORED);
3864  else if (command->meta == META_ENDIF)
3865  {
3868  if (conditional_active(st->cstack))
3870  }
3871  /* could detect "else" & "elif" after "else" */
3872  st->command++;
3873  break;
3874 
3875  case IFSTATE_NONE:
3876  case IFSTATE_TRUE:
3877  case IFSTATE_ELSE_TRUE:
3878  default:
3879 
3880  /*
3881  * inconsistent if inactive, unreachable dead
3882  * code
3883  */
3884  Assert(false);
3885  }
3886  }
3887  else
3888  {
3889  /* skip and consider next */
3890  st->command++;
3891  }
3892 
3893  if (st->state != CSTATE_SKIP_COMMAND)
3894  /* out of quick skip command loop */
3895  break;
3896  }
3897  break;
3898 
3899  /*
3900  * Wait for the current SQL command to complete
3901  */
3902  case CSTATE_WAIT_RESULT:
3903  pg_log_debug("client %d receiving", st->id);
3904 
3905  /*
3906  * Only check for new network data if we processed all data
3907  * fetched prior. Otherwise we end up doing a syscall for each
3908  * individual pipelined query, which has a measurable
3909  * performance impact.
3910  */
3911  if (PQisBusy(st->con) && !PQconsumeInput(st->con))
3912  {
3913  /* there's something wrong */
3914  commandFailed(st, "SQL", "perhaps the backend died while processing");
3915  st->state = CSTATE_ABORTED;
3916  break;
3917  }
3918  if (PQisBusy(st->con))
3919  return; /* don't have the whole result yet */
3920 
3921  /* store or discard the query results */
3922  if (readCommandResponse(st,
3925  {
3926  /*
3927  * outside of pipeline mode: stop reading results.
3928  * pipeline mode: continue reading results until an
3929  * end-of-pipeline response.
3930  */
3931  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
3932  st->state = CSTATE_END_COMMAND;
3933  }
3934  else if (canRetryError(st->estatus))
3935  st->state = CSTATE_ERROR;
3936  else
3937  st->state = CSTATE_ABORTED;
3938  break;
3939 
3940  /*
3941  * Wait until sleep is done. This state is entered after a
3942  * \sleep metacommand. The behavior is similar to
3943  * CSTATE_THROTTLE, but proceeds to CSTATE_START_COMMAND
3944  * instead of CSTATE_START_TX.
3945  */
3946  case CSTATE_SLEEP:
3948  if (now < st->sleep_until)
3949  return; /* still sleeping, nothing to do here */
3950  /* Else done sleeping. */
3951  st->state = CSTATE_END_COMMAND;
3952  break;
3953 
3954  /*
3955  * End of command: record stats and proceed to next command.
3956  */
3957  case CSTATE_END_COMMAND:
3958 
3959  /*
3960  * command completed: accumulate per-command execution times
3961  * in thread-local data structure, if per-command latencies
3962  * are requested.
3963  */
3964  if (report_per_command)
3965  {
3966  Command *command;
3967 
3969 
3970  command = sql_script[st->use_file].commands[st->command];
3971  /* XXX could use a mutex here, but we choose not to */
3972  addToSimpleStats(&command->stats,
3974  }
3975 
3976  /* Go ahead with next command, to be executed or skipped */
3977  st->command++;
3978  st->state = conditional_active(st->cstack) ?
3980  break;
3981 
3982  /*
3983  * Clean up after an error.
3984  */
3985  case CSTATE_ERROR:
3986  {
3987  TStatus tstatus;
3988 
3990 
3991  /* Clear the conditional stack */
3993 
3994  /* Read and discard until a sync point in pipeline mode */
3995  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3996  {
3997  if (!discardUntilSync(st))
3998  {
3999  st->state = CSTATE_ABORTED;
4000  break;
4001  }
4002  }
4003 
4004  /*
4005  * Check if we have a (failed) transaction block or not,
4006  * and roll it back if any.
4007  */
4008  tstatus = getTransactionStatus(st->con);
4009  if (tstatus == TSTATUS_IN_BLOCK)
4010  {
4011  /* Try to rollback a (failed) transaction block. */
4012  if (!PQsendQuery(st->con, "ROLLBACK"))
4013  {
4014  pg_log_error("client %d aborted: failed to send sql command for rolling back the failed transaction",
4015  st->id);
4016  st->state = CSTATE_ABORTED;
4017  }
4018  else
4020  }
4021  else if (tstatus == TSTATUS_IDLE)
4022  {
4023  /*
4024  * If time is over, we're done; otherwise, check if we
4025  * can retry the error.
4026  */
4029  }
4030  else
4031  {
4032  if (tstatus == TSTATUS_CONN_ERROR)
4033  pg_log_error("perhaps the backend died while processing");
4034 
4035  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4036  st->state = CSTATE_ABORTED;
4037  }
4038  break;
4039  }
4040 
4041  /*
4042  * Wait for the rollback command to complete
4043  */
4045  {
4046  PGresult *res;
4047 
4048  pg_log_debug("client %d receiving", st->id);
4049  if (!PQconsumeInput(st->con))
4050  {
4051  pg_log_error("client %d aborted while rolling back the transaction after an error; perhaps the backend died while processing",
4052  st->id);
4053  st->state = CSTATE_ABORTED;
4054  break;
4055  }
4056  if (PQisBusy(st->con))
4057  return; /* don't have the whole result yet */
4058 
4059  /*
4060  * Read and discard the query result;
4061  */
4062  res = PQgetResult(st->con);
4063  switch (PQresultStatus(res))
4064  {
4065  case PGRES_COMMAND_OK:
4066  /* OK */
4067  PQclear(res);
4068  /* null must be returned */
4069  res = PQgetResult(st->con);
4070  Assert(res == NULL);
4071 
4072  /*
4073  * If time is over, we're done; otherwise, check
4074  * if we can retry the error.
4075  */
4078  break;
4079  default:
4080  pg_log_error("client %d aborted while rolling back the transaction after an error; %s",
4081  st->id, PQerrorMessage(st->con));
4082  PQclear(res);
4083  st->state = CSTATE_ABORTED;
4084  break;
4085  }
4086  break;
4087  }
4088 
4089  /*
4090  * Retry the transaction after an error.
4091  */
4092  case CSTATE_RETRY:
4093  command = sql_script[st->use_file].commands[st->command];
4094 
4095  /*
4096  * Inform that the transaction will be retried after the
4097  * error.
4098  */
4099  if (verbose_errors)
4100  printVerboseErrorMessages(st, &now, true);
4101 
4102  /* Count tries and retries */
4103  st->tries++;
4104  command->retries++;
4105 
4106  /*
4107  * Reset the random state as they were at the beginning of the
4108  * transaction.
4109  */
4110  st->cs_func_rs = st->random_state;
4111 
4112  /* Process the first transaction command. */
4113  st->command = 0;
4114  st->estatus = ESTATUS_NO_ERROR;
4116  break;
4117 
4118  /*
4119  * Record a failed transaction.
4120  */
4121  case CSTATE_FAILURE:
4122  command = sql_script[st->use_file].commands[st->command];
4123 
4124  /* Accumulate the failure. */
4125  command->failures++;
4126 
4127  /*
4128  * Inform that the failed transaction will not be retried.
4129  */
4130  if (verbose_errors)
4131  printVerboseErrorMessages(st, &now, false);
4132 
4133  /* End the failed transaction. */
4134  st->state = CSTATE_END_TX;
4135  break;
4136 
4137  /*
4138  * End of transaction (end of script, really).
4139  */
4140  case CSTATE_END_TX:
4141  {
4142  TStatus tstatus;
4143 
4144  /* transaction finished: calculate latency and do log */
4145  processXactStats(thread, st, &now, false, agg);
4146 
4147  /*
4148  * missing \endif... cannot happen if CheckConditional was
4149  * okay
4150  */
4152 
4153  /*
4154  * We must complete all the transaction blocks that were
4155  * started in this script.
4156  */
4157  tstatus = getTransactionStatus(st->con);
4158  if (tstatus == TSTATUS_IN_BLOCK)
4159  {
4160  pg_log_error("client %d aborted: end of script reached without completing the last transaction",
4161  st->id);
4162  st->state = CSTATE_ABORTED;
4163  break;
4164  }
4165  else if (tstatus != TSTATUS_IDLE)
4166  {
4167  if (tstatus == TSTATUS_CONN_ERROR)
4168  pg_log_error("perhaps the backend died while processing");
4169 
4170  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4171  st->state = CSTATE_ABORTED;
4172  break;
4173  }
4174 
4175  if (is_connect)
4176  {
4177  pg_time_usec_t start = now;
4178 
4179  pg_time_now_lazy(&start);
4180  finishCon(st);
4181  now = pg_time_now();
4182  thread->conn_duration += now - start;
4183  }
4184 
4185  if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded)
4186  {
4187  /* script completed */
4188  st->state = CSTATE_FINISHED;
4189  break;
4190  }
4191 
4192  /* next transaction (script) */
4194 
4195  /*
4196  * Ensure that we always return on this point, so as to
4197  * avoid an infinite loop if the script only contains meta
4198  * commands.
4199  */
4200  return;
4201  }
4202 
4203  /*
4204  * Final states. Close the connection if it's still open.
4205  */
4206  case CSTATE_ABORTED:
4207  case CSTATE_FINISHED:
4208 
4209  /*
4210  * Don't measure the disconnection delays here even if in
4211  * CSTATE_FINISHED and -C/--connect option is specified.
4212  * Because in this case all the connections that this thread
4213  * established are closed at the end of transactions and the
4214  * disconnection delays should have already been measured at
4215  * that moment.
4216  *
4217  * In CSTATE_ABORTED state, the measurement is no longer
4218  * necessary because we cannot report complete results anyways
4219  * in this case.
4220  */
4221  finishCon(st);
4222  return;
4223  }
4224  }
4225 }
4226 
4227 /*
4228  * Subroutine for advanceConnectionState -- initiate or execute the current
4229  * meta command, and return the next state to set.
4230  *
4231  * *now is updated to the current time, unless the command is expected to
4232  * take no time to execute.
4233  */
4236 {
4237  Command *command = sql_script[st->use_file].commands[st->command];
4238  int argc;
4239  char **argv;
4240 
4241  Assert(command != NULL && command->type == META_COMMAND);
4242 
4243  argc = command->argc;
4244  argv = command->argv;
4245 
4247  {
4249 
4250  initPQExpBuffer(&buf);
4251 
4252  printfPQExpBuffer(&buf, "client %d executing \\%s", st->id, argv[0]);
4253  for (int i = 1; i < argc; i++)
4254  appendPQExpBuffer(&buf, " %s", argv[i]);
4255 
4256  pg_log_debug("%s", buf.data);
4257 
4258  termPQExpBuffer(&buf);
4259  }
4260 
4261  if (command->meta == META_SLEEP)
4262  {
4263  int usec;
4264 
4265  /*
4266  * A \sleep doesn't execute anything, we just get the delay from the
4267  * argument, and enter the CSTATE_SLEEP state. (The per-command
4268  * latency will be recorded in CSTATE_SLEEP state, not here, after the
4269  * delay has elapsed.)
4270  */
4271  if (!evaluateSleep(&st->variables, argc, argv, &usec))
4272  {
4273  commandFailed(st, "sleep", "execution of meta-command failed");
4274  return CSTATE_ABORTED;
4275  }
4276 
4278  st->sleep_until = (*now) + usec;
4279  return CSTATE_SLEEP;
4280  }
4281  else if (command->meta == META_SET)
4282  {
4283  PgBenchExpr *expr = command->expr;
4284  PgBenchValue result;
4285 
4286  if (!evaluateExpr(st, expr, &result))
4287  {
4288  commandFailed(st, argv[0], "evaluation of meta-command failed");
4289  return CSTATE_ABORTED;
4290  }
4291 
4292  if (!putVariableValue(&st->variables, argv[0], argv[1], &result))
4293  {
4294  commandFailed(st, "set", "assignment of meta-command failed");
4295  return CSTATE_ABORTED;
4296  }
4297  }
4298  else if (command->meta == META_IF)
4299  {
4300  /* backslash commands with an expression to evaluate */
4301  PgBenchExpr *expr = command->expr;
4302  PgBenchValue result;
4303  bool cond;
4304 
4305  if (!evaluateExpr(st, expr, &result))
4306  {
4307  commandFailed(st, argv[0], "evaluation of meta-command failed");
4308  return CSTATE_ABORTED;
4309  }
4310 
4311  cond = valueTruth(&result);
4313  }
4314  else if (command->meta == META_ELIF)
4315  {
4316  /* backslash commands with an expression to evaluate */
4317  PgBenchExpr *expr = command->expr;
4318  PgBenchValue result;
4319  bool cond;
4320 
4322  {
4323  /* elif after executed block, skip eval and wait for endif. */
4325  return CSTATE_END_COMMAND;
4326  }
4327 
4328  if (!evaluateExpr(st, expr, &result))
4329  {
4330  commandFailed(st, argv[0], "evaluation of meta-command failed");
4331  return CSTATE_ABORTED;
4332  }
4333 
4334  cond = valueTruth(&result);
4337  }
4338  else if (command->meta == META_ELSE)
4339  {
4340  switch (conditional_stack_peek(st->cstack))
4341  {
4342  case IFSTATE_TRUE:
4344  break;
4345  case IFSTATE_FALSE: /* inconsistent if active */
4346  case IFSTATE_IGNORED: /* inconsistent if active */
4347  case IFSTATE_NONE: /* else without if */
4348  case IFSTATE_ELSE_TRUE: /* else after else */
4349  case IFSTATE_ELSE_FALSE: /* else after else */
4350  default:
4351  /* dead code if conditional check is ok */
4352  Assert(false);
4353  }
4354  }
4355  else if (command->meta == META_ENDIF)
4356  {
4359  }
4360  else if (command->meta == META_SETSHELL)
4361  {
4362  if (!runShellCommand(&st->variables, argv[1], argv + 2, argc - 2))
4363  {
4364  commandFailed(st, "setshell", "execution of meta-command failed");
4365  return CSTATE_ABORTED;
4366  }
4367  }
4368  else if (command->meta == META_SHELL)
4369  {
4370  if (!runShellCommand(&st->variables, NULL, argv + 1, argc - 1))
4371  {
4372  commandFailed(st, "shell", "execution of meta-command failed");
4373  return CSTATE_ABORTED;
4374  }
4375  }
4376  else if (command->meta == META_STARTPIPELINE)
4377  {
4378  /*
4379  * In pipeline mode, we use a workflow based on libpq pipeline
4380  * functions.
4381  */
4382  if (querymode == QUERY_SIMPLE)
4383  {
4384  commandFailed(st, "startpipeline", "cannot use pipeline mode with the simple query protocol");
4385  return CSTATE_ABORTED;
4386  }
4387 
4388  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4389  {
4390  commandFailed(st, "startpipeline", "already in pipeline mode");
4391  return CSTATE_ABORTED;
4392  }
4393  if (PQenterPipelineMode(st->con) == 0)
4394  {
4395  commandFailed(st, "startpipeline", "failed to enter pipeline mode");
4396  return CSTATE_ABORTED;
4397  }
4398  }
4399  else if (command->meta == META_ENDPIPELINE)
4400  {
4401  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
4402  {
4403  commandFailed(st, "endpipeline", "not in pipeline mode");
4404  return CSTATE_ABORTED;
4405  }
4406  if (!PQpipelineSync(st->con))
4407  {
4408  commandFailed(st, "endpipeline", "failed to send a pipeline sync");
4409  return CSTATE_ABORTED;
4410  }
4411  /* Now wait for the PGRES_PIPELINE_SYNC and exit pipeline mode there */
4412  /* collect pending results before getting out of pipeline mode */
4413  return CSTATE_WAIT_RESULT;
4414  }
4415 
4416  /*
4417  * executing the expression or shell command might have taken a
4418  * non-negligible amount of time, so reset 'now'
4419  */
4420  *now = 0;
4421 
4422  return CSTATE_END_COMMAND;
4423 }
4424 
4425 /*
4426  * Return the number fo failed transactions.
4427  */
4428 static int64
4429 getFailures(const StatsData *stats)
4430 {
4431  return (stats->serialization_failures +
4432  stats->deadlock_failures);
4433 }
4434 
4435 /*
4436  * Return a string constant representing the result of a transaction
4437  * that is not successfully processed.
4438  */
4439 static const char *
4440 getResultString(bool skipped, EStatus estatus)
4441 {
4442  if (skipped)
4443  return "skipped";
4444  else if (failures_detailed)
4445  {
4446  switch (estatus)
4447  {
4449  return "serialization";
4451  return "deadlock";
4452  default:
4453  /* internal error which should never occur */
4454  pg_fatal("unexpected error status: %d", estatus);
4455  }
4456  }
4457  else
4458  return "failed";
4459 }
4460 
4461 /*
4462  * Print log entry after completing one transaction.
4463  *
4464  * We print Unix-epoch timestamps in the log, so that entries can be
4465  * correlated against other logs.
4466  *
4467  * XXX We could obtain the time from the caller and just shift it here, to
4468  * avoid the cost of an extra call to pg_time_now().
4469  */
4470 static void
4471 doLog(TState *thread, CState *st,
4472  StatsData *agg, bool skipped, double latency, double lag)
4473 {
4474  FILE *logfile = thread->logfile;
4476 
4477  Assert(use_log);
4478 
4479  /*
4480  * Skip the log entry if sampling is enabled and this row doesn't belong
4481  * to the random sample.
4482  */
4483  if (sample_rate != 0.0 &&
4485  return;
4486 
4487  /* should we aggregate the results or not? */
4488  if (agg_interval > 0)
4489  {
4491 
4492  /*
4493  * Loop until we reach the interval of the current moment, and print
4494  * any empty intervals in between (this may happen with very low tps,
4495  * e.g. --rate=0.1).
4496  */
4497 
4498  while ((next = agg->start_time + agg_interval * INT64CONST(1000000)) <= now)
4499  {
4500  double lag_sum = 0.0;
4501  double lag_sum2 = 0.0;
4502  double lag_min = 0.0;
4503  double lag_max = 0.0;
4504  int64 skipped = 0;
4505  int64 serialization_failures = 0;
4506  int64 deadlock_failures = 0;
4507  int64 retried = 0;
4508  int64 retries = 0;
4509 
4510  /* print aggregated report to logfile */
4511  fprintf(logfile, INT64_FORMAT " " INT64_FORMAT " %.0f %.0f %.0f %.0f",
4512  agg->start_time / 1000000, /* seconds since Unix epoch */
4513  agg->cnt,
4514  agg->latency.sum,
4515  agg->latency.sum2,
4516  agg->latency.min,
4517  agg->latency.max);
4518 
4519  if (throttle_delay)
4520  {
4521  lag_sum = agg->lag.sum;
4522  lag_sum2 = agg->lag.sum2;
4523  lag_min = agg->lag.min;
4524  lag_max = agg->lag.max;
4525  }
4526  fprintf(logfile, " %.0f %.0f %.0f %.0f",
4527  lag_sum,
4528  lag_sum2,
4529  lag_min,
4530  lag_max);
4531 
4532  if (latency_limit)
4533  skipped = agg->skipped;
4534  fprintf(logfile, " " INT64_FORMAT, skipped);
4535 
4536  if (max_tries != 1)
4537  {
4538  retried = agg->retried;
4539  retries = agg->retries;
4540  }
4541  fprintf(logfile, " " INT64_FORMAT " " INT64_FORMAT, retried, retries);
4542 
4543  if (failures_detailed)
4544  {
4545  serialization_failures = agg->serialization_failures;
4546  deadlock_failures = agg->deadlock_failures;
4547  }
4549  serialization_failures,
4550  deadlock_failures);
4551 
4552  fputc('\n', logfile);
4553 
4554  /* reset data and move to next interval */
4555  initStats(agg, next);
4556  }
4557 
4558  /* accumulate the current transaction */
4559  accumStats(agg, skipped, latency, lag, st->estatus, st->tries);
4560  }
4561  else
4562  {
4563  /* no, print raw transactions */
4564  if (!skipped && st->estatus == ESTATUS_NO_ERROR)
4565  fprintf(logfile, "%d " INT64_FORMAT " %.0f %d " INT64_FORMAT " "
4566  INT64_FORMAT,
4567  st->id, st->cnt, latency, st->use_file,
4568  now / 1000000, now % 1000000);
4569  else
4570  fprintf(logfile, "%d " INT64_FORMAT " %s %d " INT64_FORMAT " "
4571  INT64_FORMAT,
4572  st->id, st->cnt, getResultString(skipped, st->estatus),
4573  st->use_file, now / 1000000, now % 1000000);
4574 
4575  if (throttle_delay)
4576  fprintf(logfile, " %.0f", lag);
4577  if (max_tries != 1)
4578  fprintf(logfile, " %u", st->tries - 1);
4579  fputc('\n', logfile);
4580  }
4581 }
4582 
4583 /*
4584  * Accumulate and report statistics at end of a transaction.
4585  *
4586  * (This is also called when a transaction is late and thus skipped.
4587  * Note that even skipped and failed transactions are counted in the CState
4588  * "cnt" field.)
4589  */
4590 static void
4592  bool skipped, StatsData *agg)
4593 {
4594  double latency = 0.0,
4595  lag = 0.0;
4596  bool detailed = progress || throttle_delay || latency_limit ||
4598 
4599  if (detailed && !skipped && st->estatus == ESTATUS_NO_ERROR)
4600  {
4602 
4603  /* compute latency & lag */
4604  latency = (*now) - st->txn_scheduled;
4605  lag = st->txn_begin - st->txn_scheduled;
4606  }
4607 
4608  /* keep detailed thread stats */
4609  accumStats(&thread->stats, skipped, latency, lag, st->estatus, st->tries);
4610 
4611  /* count transactions over the latency limit, if needed */
4612  if (latency_limit && latency > latency_limit)
4613  thread->latency_late++;
4614 
4615  /* client stat is just counting */
4616  st->cnt++;
4617 
4618  if (use_log)
4619  doLog(thread, st, agg, skipped, latency, lag);
4620 
4621  /* XXX could use a mutex here, but we choose not to */
4622  if (per_script_stats)
4623  accumStats(&sql_script[st->use_file].stats, skipped, latency, lag,
4624  st->estatus, st->tries);
4625 }
4626 
4627 
4628 /* discard connections */
4629 static void
4630 disconnect_all(CState *state, int length)
4631 {
4632  int i;
4633 
4634  for (i = 0; i < length; i++)
4635  finishCon(&state[i]);
4636 }
4637 
4638 /*
4639  * Remove old pgbench tables, if any exist
4640  */
4641 static void
4642 initDropTables(PGconn *con)
4643 {
4644  fprintf(stderr, "dropping old tables...\n");
4645 
4646  /*
4647  * We drop all the tables in one command, so that whether there are
4648  * foreign key dependencies or not doesn't matter.
4649  */
4650  executeStatement(con, "drop table if exists "
4651  "pgbench_accounts, "
4652  "pgbench_branches, "
4653  "pgbench_history, "
4654  "pgbench_tellers");
4655 }
4656 
4657 /*
4658  * Create "pgbench_accounts" partitions if needed.
4659  *
4660  * This is the larger table of pgbench default tpc-b like schema
4661  * with a known size, so we choose to partition it.
4662  */
4663 static void
4665 {
4666  PQExpBufferData query;
4667 
4668  /* we must have to create some partitions */
4669  Assert(partitions > 0);
4670 
4671  fprintf(stderr, "creating %d partitions...\n", partitions);
4672 
4673  initPQExpBuffer(&query);
4674 
4675  for (int p = 1; p <= partitions; p++)
4676  {
4678  {
4679  int64 part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
4680 
4681  printfPQExpBuffer(&query,
4682  "create%s table pgbench_accounts_%d\n"
4683  " partition of pgbench_accounts\n"
4684  " for values from (",
4685  unlogged_tables ? " unlogged" : "", p);
4686 
4687  /*
4688  * For RANGE, we use open-ended partitions at the beginning and
4689  * end to allow any valid value for the primary key. Although the
4690  * actual minimum and maximum values can be derived from the
4691  * scale, it is more generic and the performance is better.
4692  */
4693  if (p == 1)
4694  appendPQExpBufferStr(&query, "minvalue");
4695  else
4696  appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
4697 
4698  appendPQExpBufferStr(&query, ") to (");
4699 
4700  if (p < partitions)
4701  appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
4702  else
4703  appendPQExpBufferStr(&query, "maxvalue");
4704 
4705  appendPQExpBufferChar(&query, ')');
4706  }
4707  else if (partition_method == PART_HASH)
4708  printfPQExpBuffer(&query,
4709  "create%s table pgbench_accounts_%d\n"
4710  " partition of pgbench_accounts\n"
4711  " for values with (modulus %d, remainder %d)",
4712  unlogged_tables ? " unlogged" : "", p,
4713  partitions, p - 1);
4714  else /* cannot get there */
4715  Assert(0);
4716 
4717  /*
4718  * Per ddlinfo in initCreateTables, fillfactor is needed on table
4719  * pgbench_accounts.
4720  */
4721  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4722 
4723  executeStatement(con, query.data);
4724  }
4725 
4726  termPQExpBuffer(&query);
4727 }
4728 
4729 /*
4730  * Create pgbench's standard tables
4731  */
4732 static void
4734 {
4735  /*
4736  * Note: TPC-B requires at least 100 bytes per row, and the "filler"
4737  * fields in these table declarations were intended to comply with that.
4738  * The pgbench_accounts table complies with that because the "filler"
4739  * column is set to blank-padded empty string. But for all other tables
4740  * the columns default to NULL and so don't actually take any space. We
4741  * could fix that by giving them non-null default values. However, that
4742  * would completely break comparability of pgbench results with prior
4743  * versions. Since pgbench has never pretended to be fully TPC-B compliant
4744  * anyway, we stick with the historical behavior.
4745  */
4746  struct ddlinfo
4747  {
4748  const char *table; /* table name */
4749  const char *smcols; /* column decls if accountIDs are 32 bits */
4750  const char *bigcols; /* column decls if accountIDs are 64 bits */
4751  int declare_fillfactor;
4752  };
4753  static const struct ddlinfo DDLs[] = {
4754  {
4755  "pgbench_history",
4756  "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)",
4757  "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)",
4758  0
4759  },
4760  {
4761  "pgbench_tellers",
4762  "tid int not null,bid int,tbalance int,filler char(84)",
4763  "tid int not null,bid int,tbalance int,filler char(84)",
4764  1
4765  },
4766  {
4767  "pgbench_accounts",
4768  "aid int not null,bid int,abalance int,filler char(84)",
4769  "aid bigint not null,bid int,abalance int,filler char(84)",
4770  1
4771  },
4772  {
4773  "pgbench_branches",
4774  "bid int not null,bbalance int,filler char(88)",
4775  "bid int not null,bbalance int,filler char(88)",
4776  1
4777  }
4778  };
4779  int i;
4780  PQExpBufferData query;
4781 
4782  fprintf(stderr, "creating tables...\n");
4783 
4784  initPQExpBuffer(&query);
4785 
4786  for (i = 0; i < lengthof(DDLs); i++)
4787  {
4788  const struct ddlinfo *ddl = &DDLs[i];
4789 
4790  /* Construct new create table statement. */
4791  printfPQExpBuffer(&query, "create%s table %s(%s)",
4792  unlogged_tables ? " unlogged" : "",
4793  ddl->table,
4794  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
4795 
4796  /* Partition pgbench_accounts table */
4797  if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
4798  appendPQExpBuffer(&query,
4799  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
4800  else if (ddl->declare_fillfactor)
4801  {
4802  /* fillfactor is only expected on actual tables */
4803  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4804  }
4805 
4806  if (tablespace != NULL)
4807  {
4808  char *escape_tablespace;
4809 
4810  escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
4811  appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
4812  PQfreemem(escape_tablespace);
4813  }
4814 
4815  executeStatement(con, query.data);
4816  }
4817 
4818  termPQExpBuffer(&query);
4819 
4820  if (partition_method != PART_NONE)
4821  createPartitions(con);
4822 }
4823 
4824 /*
4825  * Truncate away any old data, in one command in case there are foreign keys
4826  */
4827 static void
4829 {
4830  executeStatement(con, "truncate table "
4831  "pgbench_accounts, "
4832  "pgbench_branches, "
4833  "pgbench_history, "
4834  "pgbench_tellers");
4835 }
4836 
4837 /*
4838  * Fill the standard tables with some data generated and sent from the client
4839  */
4840 static void
4842 {
4843  PQExpBufferData sql;
4844  PGresult *res;
4845  int i;
4846  int64 k;
4847  char *copy_statement;
4848 
4849  /* used to track elapsed time and estimate of the remaining time */
4850  pg_time_usec_t start;
4851  int log_interval = 1;
4852 
4853  /* Stay on the same line if reporting to a terminal */
4854  char eol = isatty(fileno(stderr)) ? '\r' : '\n';
4855 
4856  fprintf(stderr, "generating data (client-side)...\n");
4857 
4858  /*
4859  * we do all of this in one transaction to enable the backend's
4860  * data-loading optimizations
4861  */
4862  executeStatement(con, "begin");
4863 
4864  /* truncate away any old data */
4865  initTruncateTables(con);
4866 
4867  initPQExpBuffer(&sql);
4868 
4869  /*
4870  * fill branches, tellers, accounts in that order in case foreign keys
4871  * already exist
4872  */
4873  for (i = 0; i < nbranches * scale; i++)
4874  {
4875  /* "filler" column defaults to NULL */
4876  printfPQExpBuffer(&sql,
4877  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
4878  i + 1);
4879  executeStatement(con, sql.data);
4880  }
4881 
4882  for (i = 0; i < ntellers * scale; i++)
4883  {
4884  /* "filler" column defaults to NULL */
4885  printfPQExpBuffer(&sql,
4886  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
4887  i + 1, i / ntellers + 1);
4888  executeStatement(con, sql.data);
4889  }
4890 
4891  /*
4892  * accounts is big enough to be worth using COPY and tracking runtime
4893  */
4894 
4895  /* use COPY with FREEZE on v14 and later without partitioning */
4896  if (partitions == 0 && PQserverVersion(con) >= 140000)
4897  copy_statement = "copy pgbench_accounts from stdin with (freeze on)";
4898  else
4899  copy_statement = "copy pgbench_accounts from stdin";
4900 
4901  res = PQexec(con, copy_statement);
4902 
4904  pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
4905  PQclear(res);
4906 
4907  start = pg_time_now();
4908 
4909  for (k = 0; k < (int64) naccounts * scale; k++)
4910  {
4911  int64 j = k + 1;
4912 
4913  /* "filler" column defaults to blank padded empty string */
4914  printfPQExpBuffer(&sql,
4915  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
4916  j, k / naccounts + 1, 0);
4917  if (PQputline(con, sql.data))
4918  pg_fatal("PQputline failed");
4919 
4920  if (CancelRequested)
4921  break;
4922 
4923  /*
4924  * If we want to stick with the original logging, print a message each
4925  * 100k inserted rows.
4926  */
4927  if ((!use_quiet) && (j % 100000 == 0))
4928  {
4929  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4930  double remaining_sec = ((double) scale * naccounts - j) * elapsed_sec / j;
4931 
4932  fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) done (elapsed %.2f s, remaining %.2f s)%c",
4933  j, (int64) naccounts * scale,
4934  (int) (((int64) j * 100) / (naccounts * (int64) scale)),
4935  elapsed_sec, remaining_sec, eol);
4936  }
4937  /* let's not call the timing for each row, but only each 100 rows */
4938  else if (use_quiet && (j % 100 == 0))
4939  {
4940  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4941  double remaining_sec = ((double) scale * naccounts - j) * elapsed_sec / j;
4942 
4943  /* have we reached the next interval (or end)? */
4944  if ((j == scale * naccounts) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
4945  {
4946  fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) done (elapsed %.2f s, remaining %.2f s)%c",
4947  j, (int64) naccounts * scale,
4948  (int) (((int64) j * 100) / (naccounts * (int64) scale)), elapsed_sec, remaining_sec, eol);
4949 
4950  /* skip to the next interval */
4951  log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
4952  }
4953  }
4954  }
4955 
4956  if (eol != '\n')
4957  fputc('\n', stderr); /* Need to move to next line */
4958 
4959  if (PQputline(con, "\\.\n"))
4960  pg_fatal("very last PQputline failed");
4961  if (PQendcopy(con))
4962  pg_fatal("PQendcopy failed");
4963 
4964  termPQExpBuffer(&sql);
4965 
4966  executeStatement(con, "commit");
4967 }
4968 
4969 /*
4970  * Fill the standard tables with some data generated on the server
4971  *
4972  * As already the case with the client-side data generation, the filler
4973  * column defaults to NULL in pgbench_branches and pgbench_tellers,
4974  * and is a blank-padded string in pgbench_accounts.
4975  */
4976 static void
4978 {
4979  PQExpBufferData sql;
4980 
4981  fprintf(stderr, "generating data (server-side)...\n");
4982 
4983  /*
4984  * we do all of this in one transaction to enable the backend's
4985  * data-loading optimizations
4986  */
4987  executeStatement(con, "begin");
4988 
4989  /* truncate away any old data */
4990  initTruncateTables(con);
4991 
4992  initPQExpBuffer(&sql);
4993 
4994  printfPQExpBuffer(&sql,
4995  "insert into pgbench_branches(bid,bbalance) "
4996  "select bid, 0 "
4997  "from generate_series(1, %d) as bid", nbranches * scale);
4998  executeStatement(con, sql.data);
4999 
5000  printfPQExpBuffer(&sql,
5001  "insert into pgbench_tellers(tid,bid,tbalance) "
5002  "select tid, (tid - 1) / %d + 1, 0 "
5003  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
5004  executeStatement(con, sql.data);
5005 
5006  printfPQExpBuffer(&sql,
5007  "insert into pgbench_accounts(aid,bid,abalance,filler) "
5008  "select aid, (aid - 1) / %d + 1, 0, '' "
5009  "from generate_series(1, " INT64_FORMAT ") as aid",
5010  naccounts, (int64) naccounts * scale);
5011  executeStatement(con, sql.data);
5012 
5013  termPQExpBuffer(&sql);
5014 
5015  executeStatement(con, "commit");
5016 }
5017 
5018 /*
5019  * Invoke vacuum on the standard tables
5020  */
5021 static void
5022 initVacuum(PGconn *con)
5023 {
5024  fprintf(stderr, "vacuuming...\n");
5025  executeStatement(con, "vacuum analyze pgbench_branches");
5026  executeStatement(con, "vacuum analyze pgbench_tellers");
5027  executeStatement(con, "vacuum analyze pgbench_accounts");
5028  executeStatement(con, "vacuum analyze pgbench_history");
5029 }
5030 
5031 /*
5032  * Create primary keys on the standard tables
5033  */
5034 static void
5035 initCreatePKeys(PGconn *con)
5036 {
5037  static const char *const DDLINDEXes[] = {
5038  "alter table pgbench_branches add primary key (bid)",
5039  "alter table pgbench_tellers add primary key (tid)",
5040  "alter table pgbench_accounts add primary key (aid)"
5041  };
5042  int i;
5043  PQExpBufferData query;
5044 
5045  fprintf(stderr, "creating primary keys...\n");
5046  initPQExpBuffer(&query);
5047 
5048  for (i = 0; i < lengthof(DDLINDEXes); i++)
5049  {
5050  resetPQExpBuffer(&query);
5051  appendPQExpBufferStr(&query, DDLINDEXes[i]);
5052 
5053  if (index_tablespace != NULL)
5054  {
5055  char *escape_tablespace;
5056 
5057  escape_tablespace = PQescapeIdentifier(con, index_tablespace,
5058  strlen(index_tablespace));
5059  appendPQExpBuffer(&query, " using index tablespace %s", escape_tablespace);
5060  PQfreemem(escape_tablespace);
5061  }
5062 
5063  executeStatement(con, query.data);
5064  }
5065 
5066  termPQExpBuffer(&query);
5067 }
5068 
5069 /*
5070  * Create foreign key constraints between the standard tables
5071  */
5072 static void
5073 initCreateFKeys(PGconn *con)
5074 {
5075  static const char *const DDLKEYs[] = {
5076  "alter table pgbench_tellers add constraint pgbench_tellers_bid_fkey foreign key (bid) references pgbench_branches",
5077  "alter table pgbench_accounts add constraint pgbench_accounts_bid_fkey foreign key (bid) references pgbench_branches",
5078  "alter table pgbench_history add constraint pgbench_history_bid_fkey foreign key (bid) references pgbench_branches",
5079  "alter table pgbench_history add constraint pgbench_history_tid_fkey foreign key (tid) references pgbench_tellers",
5080  "alter table pgbench_history add constraint pgbench_history_aid_fkey foreign key (aid) references pgbench_accounts"
5081  };
5082  int i;
5083 
5084  fprintf(stderr, "creating foreign keys...\n");
5085  for (i = 0; i < lengthof(DDLKEYs); i++)
5086  {
5087  executeStatement(con, DDLKEYs[i]);
5088  }
5089 }
5090 
5091 /*
5092  * Validate an initialization-steps string
5093  *
5094  * (We could just leave it to runInitSteps() to fail if there are wrong
5095  * characters, but since initialization can take awhile, it seems friendlier
5096  * to check during option parsing.)
5097  */
5098 static void
5099 checkInitSteps(const char *initialize_steps)
5100 {
5101  if (initialize_steps[0] == '\0')
5102  pg_fatal("no initialization steps specified");
5103 
5104  for (const char *step = initialize_steps; *step != '\0'; step++)
5105  {
5106  if (strchr(ALL_INIT_STEPS " ", *step) == NULL)
5107  {
5108  pg_log_error("unrecognized initialization step \"%c\"", *step);
5109  pg_log_error_detail("Allowed step characters are: \"" ALL_INIT_STEPS "\".");
5110  exit(1);
5111  }
5112  }
5113 }
5114 
5115 /*
5116  * Invoke each initialization step in the given string
5117  */
5118 static void
5119 runInitSteps(const char *initialize_steps)
5120 {
5121  PQExpBufferData stats;
5122  PGconn *con;
5123  const char *step;
5124  double run_time = 0.0;
5125  bool first = true;
5126 
5127  initPQExpBuffer(&stats);
5128 
5129  if ((con = doConnect()) == NULL)
5130  pg_fatal("could not create connection for initialization");
5131 
5132  setup_cancel_handler(NULL);
5133  SetCancelConn(con);
5134 
5135  for (step = initialize_steps; *step != '\0'; step++)
5136  {
5137  char *op = NULL;
5138  pg_time_usec_t start = pg_time_now();
5139 
5140  switch (*step)
5141  {
5142  case 'd':
5143  op = "drop tables";
5144  initDropTables(con);
5145  break;
5146  case 't':
5147  op = "create tables";
5148  initCreateTables(con);
5149  break;
5150  case 'g':
5151  op = "client-side generate";
5153  break;
5154  case 'G':
5155  op = "server-side generate";
5157  break;
5158  case 'v':
5159  op = "vacuum";
5160  initVacuum(con);
5161  break;
5162  case 'p':
5163  op = "primary keys";
5164  initCreatePKeys(con);
5165  break;
5166  case 'f':
5167  op = "foreign keys";
5168  initCreateFKeys(con);
5169  break;
5170  case ' ':
5171  break; /* ignore */
5172  default:
5173  pg_log_error("unrecognized initialization step \"%c\"", *step);
5174  PQfinish(con);
5175  exit(1);
5176  }
5177 
5178  if (op != NULL)
5179  {
5180  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5181 
5182  if (!first)
5183  appendPQExpBufferStr(&stats, ", ");
5184  else
5185  first = false;
5186 
5187  appendPQExpBuffer(&stats, "%s %.2f s", op, elapsed_sec);
5188 
5189  run_time += elapsed_sec;
5190  }
5191  }
5192 
5193  fprintf(stderr, "done in %.2f s (%s).\n", run_time, stats.data);
5194  ResetCancelConn();
5195  PQfinish(con);
5196  termPQExpBuffer(&stats);
5197 }
5198 
5199 /*
5200  * Extract pgbench table information into global variables scale,
5201  * partition_method and partitions.
5202  */
5203 static void
5204 GetTableInfo(PGconn *con, bool scale_given)
5205 {
5206  PGresult *res;
5207 
5208  /*
5209  * get the scaling factor that should be same as count(*) from
5210  * pgbench_branches if this is not a custom query
5211  */
5212  res = PQexec(con, "select count(*) from pgbench_branches");
5214  {
5215  char *sqlState = PQresultErrorField(res, PG_DIAG_SQLSTATE);
5216 
5217  pg_log_error("could not count number of branches: %s", PQerrorMessage(con));
5218 
5219  if (sqlState && strcmp(sqlState, ERRCODE_UNDEFINED_TABLE) == 0)
5220  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".",
5221  PQdb(con));
5222 
5223  exit(1);
5224  }
5225  scale = atoi(PQgetvalue(res, 0, 0));
5226  if (scale < 0)
5227  pg_fatal("invalid count(*) from pgbench_branches: \"%s\"",
5228  PQgetvalue(res, 0, 0));
5229  PQclear(res);
5230 
5231  /* warn if we override user-given -s switch */
5232  if (scale_given)
5233  pg_log_warning("scale option ignored, using count from pgbench_branches table (%d)",
5234  scale);
5235 
5236  /*
5237  * Get the partition information for the first "pgbench_accounts" table
5238  * found in search_path.
5239  *
5240  * The result is empty if no "pgbench_accounts" is found.
5241  *
5242  * Otherwise, it always returns one row even if the table is not
5243  * partitioned (in which case the partition strategy is NULL).
5244  *
5245  * The number of partitions can be 0 even for partitioned tables, if no
5246  * partition is attached.
5247  *
5248  * We assume no partitioning on any failure, so as to avoid failing on an
5249  * old version without "pg_partitioned_table".
5250  */
5251  res = PQexec(con,
5252  "select o.n, p.partstrat, pg_catalog.count(i.inhparent) "
5253  "from pg_catalog.pg_class as c "
5254  "join pg_catalog.pg_namespace as n on (n.oid = c.relnamespace) "
5255  "cross join lateral (select pg_catalog.array_position(pg_catalog.current_schemas(true), n.nspname)) as o(n) "
5256  "left join pg_catalog.pg_partitioned_table as p on (p.partrelid = c.oid) "
5257  "left join pg_catalog.pg_inherits as i on (c.oid = i.inhparent) "
5258  "where c.relname = 'pgbench_accounts' and o.n is not null "
5259  "group by 1, 2 "
5260  "order by 1 asc "
5261  "limit 1");
5262 
5264  {
5265  /* probably an older version, coldly assume no partitioning */
5267  partitions = 0;
5268  }
5269  else if (PQntuples(res) == 0)
5270  {
5271  /*
5272  * This case is unlikely as pgbench already found "pgbench_branches"
5273  * above to compute the scale.
5274  */
5275  pg_log_error("no pgbench_accounts table found in search_path");
5276  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".", PQdb(con));
5277  exit(1);
5278  }
5279  else /* PQntupes(res) == 1 */
5280  {
5281  /* normal case, extract partition information */
5282  if (PQgetisnull(res, 0, 1))
5284  else
5285  {
5286  char *ps = PQgetvalue(res, 0, 1);
5287 
5288  /* column must be there */
5289  Assert(ps != NULL);
5290 
5291  if (strcmp(ps, "r") == 0)
5293  else if (strcmp(ps, "h") == 0)
5295  else
5296  {
5297  /* possibly a newer version with new partition method */
5298  pg_fatal("unexpected partition method: \"%s\"", ps);
5299  }
5300  }
5301 
5302  partitions = atoi(PQgetvalue(res, 0, 2));
5303  }
5304 
5305  PQclear(res);
5306 }
5307 
5308 /*
5309  * Replace :param with $n throughout the command's SQL text, which
5310  * is a modifiable string in cmd->lines.
5311  */
5312 static bool
5313 parseQuery(Command *cmd)
5314 {
5315  char *sql,
5316  *p;
5317 
5318  cmd->argc = 1;
5319 
5320  p = sql = pg_strdup(cmd->lines.data);
5321  while ((p = strchr(p, ':')) != NULL)
5322  {
5323  char var[13];
5324  char *name;
5325  int eaten;
5326 
5327  name = parseVariable(p, &eaten);
5328  if (name == NULL)
5329  {
5330  while (*p == ':')
5331  {
5332  p++;
5333  }
5334  continue;
5335  }
5336 
5337  /*
5338  * cmd->argv[0] is the SQL statement itself, so the max number of
5339  * arguments is one less than MAX_ARGS
5340  */
5341  if (cmd->argc >= MAX_ARGS)
5342  {
5343  pg_log_error("statement has too many arguments (maximum is %d): %s",
5344  MAX_ARGS - 1, cmd->lines.data);
5345  pg_free(name);
5346  return false;
5347  }
5348 
5349  sprintf(var, "$%d", cmd->argc);
5350  p = replaceVariable(&sql, p, eaten, var);
5351 
5352  cmd->argv[cmd->argc] = name;
5353  cmd->argc++;
5354  }
5355 
5356  Assert(cmd->argv[0] == NULL);
5357  cmd->argv[0] = sql;
5358  return true;
5359 }
5360 
5361 /*
5362  * syntax error while parsing a script (in practice, while parsing a
5363  * backslash command, because we don't detect syntax errors in SQL)
5364  *
5365  * source: source of script (filename or builtin-script ID)
5366  * lineno: line number within script (count from 1)
5367  * line: whole line of backslash command, if available
5368  * command: backslash command name, if available
5369  * msg: the actual error message
5370  * more: optional extra message
5371  * column: zero-based column number, or -1 if unknown
5372  */
5373 void
5374 syntax_error(const char *source, int lineno,
5375  const char *line, const char *command,
5376  const char *msg, const char *more, int column)
5377 {
5379 
5380  initPQExpBuffer(&buf);
5381 
5382  printfPQExpBuffer(&buf, "%s:%d: %s", source, lineno, msg);
5383  if (more != NULL)
5384  appendPQExpBuffer(&buf, " (%s)", more);
5385  if (column >= 0 && line == NULL)
5386  appendPQExpBuffer(&buf, " at column %d", column + 1);
5387  if (command != NULL)
5388  appendPQExpBuffer(&buf, " in command \"%s\"", command);
5389 
5390  pg_log_error("%s", buf.data);
5391 
5392  termPQExpBuffer(&buf);
5393 
5394  if (line != NULL)
5395  {
5396  fprintf(stderr, "%s\n", line);
5397  if (column >= 0)
5398  fprintf(stderr, "%*c error found here\n", column + 1, '^');
5399  }
5400 
5401  exit(1);
5402 }
5403 
5404 /*
5405  * Return a pointer to the start of the SQL command, after skipping over
5406  * whitespace and "--" comments.
5407  * If the end of the string is reached, return NULL.
5408  */
5409 static char *
5410 skip_sql_comments(char *sql_command)
5411 {
5412  char *p = sql_command;
5413 
5414  /* Skip any leading whitespace, as well as "--" style comments */
5415  for (;;)
5416  {
5417  if (isspace((unsigned char) *p))
5418  p++;
5419  else if (strncmp(p, "--", 2) == 0)
5420  {
5421  p = strchr(p, '\n');
5422  if (p == NULL)
5423  return NULL;
5424  p++;
5425  }
5426  else
5427  break;
5428  }
5429 
5430  /* NULL if there's nothing but whitespace and comments */
5431  if (*p == '\0')
5432  return NULL;
5433 
5434  return p;
5435 }
5436 
5437 /*
5438  * Parse a SQL command; return a Command struct, or NULL if it's a comment
5439  *
5440  * On entry, psqlscan.l has collected the command into "buf", so we don't
5441  * really need to do much here except check for comments and set up a Command
5442  * struct.
5443  */
5444 static Command *
5446 {
5447  Command *my_command;
5448  char *p = skip_sql_comments(buf->data);
5449 
5450  if (p == NULL)
5451  return NULL;
5452 
5453  /* Allocate and initialize Command structure */
5454  my_command = (Command *) pg_malloc(sizeof(Command));
5455  initPQExpBuffer(&my_command->lines);
5456  appendPQExpBufferStr(&my_command->lines, p);
5457  my_command->first_line = NULL; /* this is set later */
5458  my_command->type = SQL_COMMAND;
5459  my_command->meta = META_NONE;
5460  my_command->argc = 0;
5461  my_command->retries = 0;
5462  my_command->failures = 0;
5463  memset(my_command->argv, 0, sizeof(my_command->argv));
5464  my_command->varprefix = NULL; /* allocated later, if needed */
5465  my_command->expr = NULL;
5466  initSimpleStats(&my_command->stats);
5467 
5468  return my_command;
5469 }
5470 
5471 /* Free a Command structure and associated data */
5472 static void
5473 free_command(Command *command)
5474 {
5475  termPQExpBuffer(&command->lines);
5476  pg_free(command->first_line);
5477  for (int i = 0; i < command->argc; i++)
5478  pg_free(command->argv[i]);
5479  pg_free(command->varprefix);
5480 
5481  /*
5482  * It should also free expr recursively, but this is currently not needed
5483  * as only gset commands (which do not have an expression) are freed.
5484  */
5485  pg_free(command);
5486 }
5487 
5488 /*
5489  * Once an SQL command is fully parsed, possibly by accumulating several
5490  * parts, complete other fields of the Command structure.
5491  */
5492 static void
5493 postprocess_sql_command(Command *my_command)
5494 {
5495  char buffer[128];
5496 
5497  Assert(my_command->type == SQL_COMMAND);
5498 
5499  /* Save the first line for error display. */
5500  strlcpy(buffer, my_command->lines.data, sizeof(buffer));
5501  buffer[strcspn(buffer, "\n\r")] = '\0';
5502  my_command->first_line = pg_strdup(buffer);
5503 
5504  /* parse query if necessary */
5505  switch (querymode)
5506  {
5507  case QUERY_SIMPLE:
5508  my_command->argv[0] = my_command->lines.data;
5509  my_command->argc++;
5510  break;
5511  case QUERY_EXTENDED:
5512  case QUERY_PREPARED:
5513  if (!parseQuery(my_command))
5514  exit(1);
5515  break;
5516  default:
5517  exit(1);
5518  }
5519 }
5520 
5521 /*
5522  * Parse a backslash command; return a Command struct, or NULL if comment
5523  *
5524  * At call, we have scanned only the initial backslash.
5525  */
5526 static Command *
5527 process_backslash_command(PsqlScanState sstate, const char *source)
5528 {
5529  Command *my_command;
5530  PQExpBufferData word_buf;
5531  int word_offset;
5532  int offsets[MAX_ARGS]; /* offsets of argument words */
5533  int start_offset;
5534  int lineno;
5535  int j;
5536 
5537  initPQExpBuffer(&word_buf);
5538 
5539  /* Remember location of the backslash */
5540  start_offset = expr_scanner_offset(sstate) - 1;
5541  lineno = expr_scanner_get_lineno(sstate, start_offset);
5542 
5543  /* Collect first word of command */
5544  if (!expr_lex_one_word(sstate, &word_buf, &word_offset))
5545  {
5546  termPQExpBuffer(&word_buf);
5547  return NULL;
5548  }
5549 
5550  /* Allocate and initialize Command structure */
5551  my_command = (Command *