PostgreSQL Source Code  git master
pgbench.c
Go to the documentation of this file.
1 /*
2  * pgbench.c
3  *
4  * A simple benchmark program for PostgreSQL
5  * Originally written by Tatsuo Ishii and enhanced by many contributors.
6  *
7  * src/bin/pgbench/pgbench.c
8  * Copyright (c) 2000-2022, PostgreSQL Global Development Group
9  * ALL RIGHTS RESERVED;
10  *
11  * Permission to use, copy, modify, and distribute this software and its
12  * documentation for any purpose, without fee, and without a written agreement
13  * is hereby granted, provided that the above copyright notice and this
14  * paragraph and the following two paragraphs appear in all copies.
15  *
16  * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
17  * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
18  * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
19  * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
20  * POSSIBILITY OF SUCH DAMAGE.
21  *
22  * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIMS ANY WARRANTIES,
23  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
24  * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
25  * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
26  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
27  *
28  */
29 
30 #if defined(WIN32) && FD_SETSIZE < 1024
31 #error FD_SETSIZE needs to have been increased
32 #endif
33 
34 #include "postgres_fe.h"
35 
36 #include <ctype.h>
37 #include <float.h>
38 #include <limits.h>
39 #include <math.h>
40 #include <signal.h>
41 #include <time.h>
42 #include <sys/time.h>
43 #include <sys/resource.h> /* for getrlimit */
44 
45 /* For testing, PGBENCH_USE_SELECT can be defined to force use of that code */
46 #if defined(HAVE_PPOLL) && !defined(PGBENCH_USE_SELECT)
47 #define POLL_USING_PPOLL
48 #ifdef HAVE_POLL_H
49 #include <poll.h>
50 #endif
51 #else /* no ppoll(), so use select() */
52 #define POLL_USING_SELECT
53 #include <sys/select.h>
54 #endif
55 
56 #include "common/int.h"
57 #include "common/logging.h"
58 #include "common/pg_prng.h"
59 #include "common/string.h"
60 #include "common/username.h"
61 #include "fe_utils/cancel.h"
62 #include "fe_utils/conditional.h"
63 #include "fe_utils/option_utils.h"
64 #include "fe_utils/string_utils.h"
65 #include "getopt_long.h"
66 #include "libpq-fe.h"
67 #include "pgbench.h"
68 #include "port/pg_bitutils.h"
69 #include "portability/instr_time.h"
70 
71 #ifndef M_PI
72 #define M_PI 3.14159265358979323846
73 #endif
74 
75 #define ERRCODE_T_R_SERIALIZATION_FAILURE "40001"
76 #define ERRCODE_T_R_DEADLOCK_DETECTED "40P01"
77 #define ERRCODE_UNDEFINED_TABLE "42P01"
78 
79 /*
80  * Hashing constants
81  */
82 #define FNV_PRIME UINT64CONST(0x100000001b3)
83 #define FNV_OFFSET_BASIS UINT64CONST(0xcbf29ce484222325)
84 #define MM2_MUL UINT64CONST(0xc6a4a7935bd1e995)
85 #define MM2_MUL_TIMES_8 UINT64CONST(0x35253c9ade8f4ca8)
86 #define MM2_ROT 47
87 
88 /*
89  * Multi-platform socket set implementations
90  */
91 
92 #ifdef POLL_USING_PPOLL
93 #define SOCKET_WAIT_METHOD "ppoll"
94 
95 typedef struct socket_set
96 {
97  int maxfds; /* allocated length of pollfds[] array */
98  int curfds; /* number currently in use */
99  struct pollfd pollfds[FLEXIBLE_ARRAY_MEMBER];
100 } socket_set;
101 
102 #endif /* POLL_USING_PPOLL */
103 
104 #ifdef POLL_USING_SELECT
105 #define SOCKET_WAIT_METHOD "select"
106 
107 typedef struct socket_set
108 {
109  int maxfd; /* largest FD currently set in fds */
110  fd_set fds;
112 
113 #endif /* POLL_USING_SELECT */
114 
115 /*
116  * Multi-platform thread implementations
117  */
118 
119 #ifdef WIN32
120 /* Use Windows threads */
121 #include <windows.h>
122 #define GETERRNO() (_dosmaperr(GetLastError()), errno)
123 #define THREAD_T HANDLE
124 #define THREAD_FUNC_RETURN_TYPE unsigned
125 #define THREAD_FUNC_RETURN return 0
126 #define THREAD_FUNC_CC __stdcall
127 #define THREAD_CREATE(handle, function, arg) \
128  ((*(handle) = (HANDLE) _beginthreadex(NULL, 0, (function), (arg), 0, NULL)) == 0 ? errno : 0)
129 #define THREAD_JOIN(handle) \
130  (WaitForSingleObject(handle, INFINITE) != WAIT_OBJECT_0 ? \
131  GETERRNO() : CloseHandle(handle) ? 0 : GETERRNO())
132 #define THREAD_BARRIER_T SYNCHRONIZATION_BARRIER
133 #define THREAD_BARRIER_INIT(barrier, n) \
134  (InitializeSynchronizationBarrier((barrier), (n), 0) ? 0 : GETERRNO())
135 #define THREAD_BARRIER_WAIT(barrier) \
136  EnterSynchronizationBarrier((barrier), \
137  SYNCHRONIZATION_BARRIER_FLAGS_BLOCK_ONLY)
138 #define THREAD_BARRIER_DESTROY(barrier)
139 #elif defined(ENABLE_THREAD_SAFETY)
140 /* Use POSIX threads */
141 #include "port/pg_pthread.h"
142 #define THREAD_T pthread_t
143 #define THREAD_FUNC_RETURN_TYPE void *
144 #define THREAD_FUNC_RETURN return NULL
145 #define THREAD_FUNC_CC
146 #define THREAD_CREATE(handle, function, arg) \
147  pthread_create((handle), NULL, (function), (arg))
148 #define THREAD_JOIN(handle) \
149  pthread_join((handle), NULL)
150 #define THREAD_BARRIER_T pthread_barrier_t
151 #define THREAD_BARRIER_INIT(barrier, n) \
152  pthread_barrier_init((barrier), NULL, (n))
153 #define THREAD_BARRIER_WAIT(barrier) pthread_barrier_wait((barrier))
154 #define THREAD_BARRIER_DESTROY(barrier) pthread_barrier_destroy((barrier))
155 #else
156 /* No threads implementation, use none (-j 1) */
157 #define THREAD_T void *
158 #define THREAD_FUNC_RETURN_TYPE void *
159 #define THREAD_FUNC_RETURN return NULL
160 #define THREAD_FUNC_CC
161 #define THREAD_BARRIER_T int
162 #define THREAD_BARRIER_INIT(barrier, n) (*(barrier) = 0)
163 #define THREAD_BARRIER_WAIT(barrier)
164 #define THREAD_BARRIER_DESTROY(barrier)
165 #endif
166 
167 
168 /********************************************************************
169  * some configurable parameters */
170 
171 #define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
172 #define ALL_INIT_STEPS "dtgGvpf" /* all possible steps */
173 
174 #define LOG_STEP_SECONDS 5 /* seconds between log messages */
175 #define DEFAULT_NXACTS 10 /* default nxacts */
176 
177 #define MIN_GAUSSIAN_PARAM 2.0 /* minimum parameter for gauss */
178 
179 #define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
180 #define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
181 
182 int nxacts = 0; /* number of transactions per client */
183 int duration = 0; /* duration in seconds */
184 int64 end_time = 0; /* when to stop in micro seconds, under -T */
185 
186 /*
187  * scaling factor. for example, scale = 10 will make 1000000 tuples in
188  * pgbench_accounts table.
189  */
190 int scale = 1;
191 
192 /*
193  * fillfactor. for example, fillfactor = 90 will use only 90 percent
194  * space during inserts and leave 10 percent free.
195  */
196 int fillfactor = 100;
197 
198 /*
199  * use unlogged tables?
200  */
201 bool unlogged_tables = false;
202 
203 /*
204  * log sampling rate (1.0 = log everything, 0.0 = option not given)
205  */
206 double sample_rate = 0.0;
207 
208 /*
209  * When threads are throttled to a given rate limit, this is the target delay
210  * to reach that rate in usec. 0 is the default and means no throttling.
211  */
212 double throttle_delay = 0;
213 
214 /*
215  * Transactions which take longer than this limit (in usec) are counted as
216  * late, and reported as such, although they are completed anyway. When
217  * throttling is enabled, execution time slots that are more than this late
218  * are skipped altogether, and counted separately.
219  */
220 int64 latency_limit = 0;
221 
222 /*
223  * tablespace selection
224  */
225 char *tablespace = NULL;
226 char *index_tablespace = NULL;
227 
228 /*
229  * Number of "pgbench_accounts" partitions. 0 is the default and means no
230  * partitioning.
231  */
232 static int partitions = 0;
233 
234 /* partitioning strategy for "pgbench_accounts" */
235 typedef enum
236 {
237  PART_NONE, /* no partitioning */
238  PART_RANGE, /* range partitioning */
239  PART_HASH /* hash partitioning */
241 
243 static const char *PARTITION_METHOD[] = {"none", "range", "hash"};
244 
245 /* random seed used to initialize base_random_sequence */
246 int64 random_seed = -1;
247 
248 /*
249  * end of configurable parameters
250  *********************************************************************/
251 
252 #define nbranches 1 /* Makes little sense to change this. Change
253  * -s instead */
254 #define ntellers 10
255 #define naccounts 100000
256 
257 /*
258  * The scale factor at/beyond which 32bit integers are incapable of storing
259  * 64bit values.
260  *
261  * Although the actual threshold is 21474, we use 20000 because it is easier to
262  * document and remember, and isn't that far away from the real threshold.
263  */
264 #define SCALE_32BIT_THRESHOLD 20000
265 
266 bool use_log; /* log transaction latencies to a file */
267 bool use_quiet; /* quiet logging onto stderr */
268 int agg_interval; /* log aggregates instead of individual
269  * transactions */
270 bool per_script_stats = false; /* whether to collect stats per script */
271 int progress = 0; /* thread progress report every this seconds */
272 bool progress_timestamp = false; /* progress report with Unix time */
273 int nclients = 1; /* number of clients */
274 int nthreads = 1; /* number of threads */
275 bool is_connect; /* establish connection for each transaction */
276 bool report_per_command = false; /* report per-command latencies,
277  * retries after errors and failures
278  * (errors without retrying) */
279 int main_pid; /* main process id used in log filename */
280 
281 /*
282  * There are different types of restrictions for deciding that the current
283  * transaction with a serialization/deadlock error can no longer be retried and
284  * should be reported as failed:
285  * - max_tries (--max-tries) can be used to limit the number of tries;
286  * - latency_limit (-L) can be used to limit the total time of tries;
287  * - duration (-T) can be used to limit the total benchmark time.
288  *
289  * They can be combined together, and you need to use at least one of them to
290  * retry the transactions with serialization/deadlock errors. If none of them is
291  * used, the default value of max_tries is 1 and such transactions will not be
292  * retried.
293  */
294 
295 /*
296  * We cannot retry a transaction after the serialization/deadlock error if its
297  * number of tries reaches this maximum; if its value is zero, it is not used.
298  */
299 uint32 max_tries = 1;
300 
301 bool failures_detailed = false; /* whether to group failures in
302  * reports or logs by basic types */
303 
304 const char *pghost = NULL;
305 const char *pgport = NULL;
306 const char *username = NULL;
307 const char *dbName = NULL;
308 char *logfile_prefix = NULL;
309 const char *progname;
310 
311 #define WSEP '@' /* weight separator */
312 
313 volatile sig_atomic_t timer_exceeded = false; /* flag from signal handler */
314 
315 /*
316  * We don't want to allocate variables one by one; for efficiency, add a
317  * constant margin each time it overflows.
318  */
319 #define VARIABLES_ALLOC_MARGIN 8
320 
321 /*
322  * Variable definitions.
323  *
324  * If a variable only has a string value, "svalue" is that value, and value is
325  * "not set". If the value is known, "value" contains the value (in any
326  * variant).
327  *
328  * In this case "svalue" contains the string equivalent of the value, if we've
329  * had occasion to compute that, or NULL if we haven't.
330  */
331 typedef struct
332 {
333  char *name; /* variable's name */
334  char *svalue; /* its value in string form, if known */
335  PgBenchValue value; /* actual variable's value */
336 } Variable;
337 
338 /*
339  * Data structure for client variables.
340  */
341 typedef struct
342 {
343  Variable *vars; /* array of variable definitions */
344  int nvars; /* number of variables */
345 
346  /*
347  * The maximum number of variables that we can currently store in 'vars'
348  * without having to reallocate more space. We must always have max_vars
349  * >= nvars.
350  */
351  int max_vars;
352 
353  bool vars_sorted; /* are variables sorted by name? */
354 } Variables;
355 
356 #define MAX_SCRIPTS 128 /* max number of SQL scripts allowed */
357 #define SHELL_COMMAND_SIZE 256 /* maximum size allowed for shell command */
358 
359 /*
360  * Simple data structure to keep stats about something.
361  *
362  * XXX probably the first value should be kept and used as an offset for
363  * better numerical stability...
364  */
365 typedef struct SimpleStats
366 {
367  int64 count; /* how many values were encountered */
368  double min; /* the minimum seen */
369  double max; /* the maximum seen */
370  double sum; /* sum of values */
371  double sum2; /* sum of squared values */
372 } SimpleStats;
373 
374 /*
375  * The instr_time type is expensive when dealing with time arithmetic. Define
376  * a type to hold microseconds instead. Type int64 is good enough for about
377  * 584500 years.
378  */
379 typedef int64 pg_time_usec_t;
380 
381 /*
382  * Data structure to hold various statistics: per-thread and per-script stats
383  * are maintained and merged together.
384  */
385 typedef struct StatsData
386 {
387  pg_time_usec_t start_time; /* interval start time, for aggregates */
388 
389  /*----------
390  * Transactions are counted depending on their execution and outcome.
391  * First a transaction may have started or not: skipped transactions occur
392  * under --rate and --latency-limit when the client is too late to execute
393  * them. Secondly, a started transaction may ultimately succeed or fail,
394  * possibly after some retries when --max-tries is not one. Thus
395  *
396  * the number of all transactions =
397  * 'skipped' (it was too late to execute them) +
398  * 'cnt' (the number of successful transactions) +
399  * 'failed' (the number of failed transactions).
400  *
401  * A successful transaction can have several unsuccessful tries before a
402  * successful run. Thus
403  *
404  * 'cnt' (the number of successful transactions) =
405  * successfully retried transactions (they got a serialization or a
406  * deadlock error(s), but were
407  * successfully retried from the very
408  * beginning) +
409  * directly successful transactions (they were successfully completed on
410  * the first try).
411  *
412  * A failed transaction is defined as unsuccessfully retried transactions.
413  * It can be one of two types:
414  *
415  * failed (the number of failed transactions) =
416  * 'serialization_failures' (they got a serialization error and were not
417  * successfully retried) +
418  * 'deadlock_failures' (they got a deadlock error and were not
419  * successfully retried).
420  *
421  * If the transaction was retried after a serialization or a deadlock
422  * error this does not guarantee that this retry was successful. Thus
423  *
424  * 'retries' (number of retries) =
425  * number of retries in all retried transactions =
426  * number of retries in (successfully retried transactions +
427  * failed transactions);
428  *
429  * 'retried' (number of all retried transactions) =
430  * successfully retried transactions +
431  * failed transactions.
432  *----------
433  */
434  int64 cnt; /* number of successful transactions, not
435  * including 'skipped' */
436  int64 skipped; /* number of transactions skipped under --rate
437  * and --latency-limit */
438  int64 retries; /* number of retries after a serialization or
439  * a deadlock error in all the transactions */
440  int64 retried; /* number of all transactions that were
441  * retried after a serialization or a deadlock
442  * error (perhaps the last try was
443  * unsuccessful) */
444  int64 serialization_failures; /* number of transactions that were
445  * not successfully retried after a
446  * serialization error */
447  int64 deadlock_failures; /* number of transactions that were not
448  * successfully retried after a deadlock
449  * error */
452 } StatsData;
453 
454 /*
455  * For displaying Unix epoch timestamps, as some time functions may have
456  * another reference.
457  */
459 
460 /*
461  * Error status for errors during script execution.
462  */
463 typedef enum EStatus
464 {
467 
468  /* SQL errors */
472 } EStatus;
473 
474 /*
475  * Transaction status at the end of a command.
476  */
477 typedef enum TStatus
478 {
483 } TStatus;
484 
485 /* Various random sequences are initialized from this one. */
487 
488 /* Synchronization barrier for start and connection */
490 
491 /*
492  * Connection state machine states.
493  */
494 typedef enum
495 {
496  /*
497  * The client must first choose a script to execute. Once chosen, it can
498  * either be throttled (state CSTATE_PREPARE_THROTTLE under --rate), start
499  * right away (state CSTATE_START_TX) or not start at all if the timer was
500  * exceeded (state CSTATE_FINISHED).
501  */
503 
504  /*
505  * CSTATE_START_TX performs start-of-transaction processing. Establishes
506  * a new connection for the transaction in --connect mode, records the
507  * transaction start time, and proceed to the first command.
508  *
509  * Note: once a script is started, it will either error or run till its
510  * end, where it may be interrupted. It is not interrupted while running,
511  * so pgbench --time is to be understood as tx are allowed to start in
512  * that time, and will finish when their work is completed.
513  */
515 
516  /*
517  * In CSTATE_PREPARE_THROTTLE state, we calculate when to begin the next
518  * transaction, and advance to CSTATE_THROTTLE. CSTATE_THROTTLE state
519  * sleeps until that moment, then advances to CSTATE_START_TX, or
520  * CSTATE_FINISHED if the next transaction would start beyond the end of
521  * the run.
522  */
525 
526  /*
527  * We loop through these states, to process each command in the script:
528  *
529  * CSTATE_START_COMMAND starts the execution of a command. On a SQL
530  * command, the command is sent to the server, and we move to
531  * CSTATE_WAIT_RESULT state unless in pipeline mode. On a \sleep
532  * meta-command, the timer is set, and we enter the CSTATE_SLEEP state to
533  * wait for it to expire. Other meta-commands are executed immediately. If
534  * the command about to start is actually beyond the end of the script,
535  * advance to CSTATE_END_TX.
536  *
537  * CSTATE_WAIT_RESULT waits until we get a result set back from the server
538  * for the current command.
539  *
540  * CSTATE_SLEEP waits until the end of \sleep.
541  *
542  * CSTATE_END_COMMAND records the end-of-command timestamp, increments the
543  * command counter, and loops back to CSTATE_START_COMMAND state.
544  *
545  * CSTATE_SKIP_COMMAND is used by conditional branches which are not
546  * executed. It quickly skip commands that do not need any evaluation.
547  * This state can move forward several commands, till there is something
548  * to do or the end of the script.
549  */
555 
556  /*
557  * States for failed commands.
558  *
559  * If the SQL/meta command fails, in CSTATE_ERROR clean up after an error:
560  * (1) clear the conditional stack; (2) if we have an unterminated
561  * (possibly failed) transaction block, send the rollback command to the
562  * server and wait for the result in CSTATE_WAIT_ROLLBACK_RESULT. If
563  * something goes wrong with rolling back, go to CSTATE_ABORTED.
564  *
565  * But if everything is ok we are ready for future transactions: if this
566  * is a serialization or deadlock error and we can re-execute the
567  * transaction from the very beginning, go to CSTATE_RETRY; otherwise go
568  * to CSTATE_FAILURE.
569  *
570  * In CSTATE_RETRY report an error, set the same parameters for the
571  * transaction execution as in the previous tries and process the first
572  * transaction command in CSTATE_START_COMMAND.
573  *
574  * In CSTATE_FAILURE report a failure, set the parameters for the
575  * transaction execution as they were before the first run of this
576  * transaction (except for a random state) and go to CSTATE_END_TX to
577  * complete this transaction.
578  */
583 
584  /*
585  * CSTATE_END_TX performs end-of-transaction processing. It calculates
586  * latency, and logs the transaction. In --connect mode, it closes the
587  * current connection.
588  *
589  * Then either starts over in CSTATE_CHOOSE_SCRIPT, or enters
590  * CSTATE_FINISHED if we have no more work to do.
591  */
593 
594  /*
595  * Final states. CSTATE_ABORTED means that the script execution was
596  * aborted because a command failed, CSTATE_FINISHED means success.
597  */
601 
602 /*
603  * Connection state.
604  */
605 typedef struct
606 {
607  PGconn *con; /* connection handle to DB */
608  int id; /* client No. */
609  ConnectionStateEnum state; /* state machine's current state. */
610  ConditionalStack cstack; /* enclosing conditionals state */
611 
612  /*
613  * Separate randomness for each client. This is used for random functions
614  * PGBENCH_RANDOM_* during the execution of the script.
615  */
616  pg_prng_state cs_func_rs;
617 
618  int use_file; /* index in sql_script for this client */
619  int command; /* command number in script */
620 
621  /* client variables */
622  Variables variables;
623 
624  /* various times about current transaction in microseconds */
625  pg_time_usec_t txn_scheduled; /* scheduled start time of transaction */
626  pg_time_usec_t sleep_until; /* scheduled start time of next cmd */
627  pg_time_usec_t txn_begin; /* used for measuring schedule lag times */
628  pg_time_usec_t stmt_begin; /* used for measuring statement latencies */
629 
630  bool prepared[MAX_SCRIPTS]; /* whether client prepared the script */
631 
632  /*
633  * For processing failures and repeating transactions with serialization
634  * or deadlock errors:
635  */
636  EStatus estatus; /* the error status of the current transaction
637  * execution; this is ESTATUS_NO_ERROR if
638  * there were no errors */
639  pg_prng_state random_state; /* random state */
640  uint32 tries; /* how many times have we already tried the
641  * current transaction? */
642 
643  /* per client collected stats */
644  int64 cnt; /* client transaction count, for -t; skipped
645  * and failed transactions are also counted
646  * here */
647 } CState;
648 
649 /*
650  * Thread state
651  */
652 typedef struct
653 {
654  int tid; /* thread id */
655  THREAD_T thread; /* thread handle */
656  CState *state; /* array of CState */
657  int nstate; /* length of state[] */
658 
659  /*
660  * Separate randomness for each thread. Each thread option uses its own
661  * random state to make all of them independent of each other and
662  * therefore deterministic at the thread level.
663  */
664  pg_prng_state ts_choose_rs; /* random state for selecting a script */
665  pg_prng_state ts_throttle_rs; /* random state for transaction throttling */
666  pg_prng_state ts_sample_rs; /* random state for log sampling */
667 
668  int64 throttle_trigger; /* previous/next throttling (us) */
669  FILE *logfile; /* where to log, or NULL */
670 
671  /* per thread collected stats in microseconds */
672  pg_time_usec_t create_time; /* thread creation time */
673  pg_time_usec_t started_time; /* thread is running */
674  pg_time_usec_t bench_start; /* thread is benchmarking */
675  pg_time_usec_t conn_duration; /* cumulated connection and disconnection
676  * delays */
677 
678  StatsData stats;
679  int64 latency_late; /* count executed but late transactions */
680 } TState;
681 
682 /*
683  * queries read from files
684  */
685 #define SQL_COMMAND 1
686 #define META_COMMAND 2
687 
688 /*
689  * max number of backslash command arguments or SQL variables,
690  * including the command or SQL statement itself
691  */
692 #define MAX_ARGS 256
693 
694 typedef enum MetaCommand
695 {
696  META_NONE, /* not a known meta-command */
697  META_SET, /* \set */
698  META_SETSHELL, /* \setshell */
699  META_SHELL, /* \shell */
700  META_SLEEP, /* \sleep */
701  META_GSET, /* \gset */
702  META_ASET, /* \aset */
703  META_IF, /* \if */
704  META_ELIF, /* \elif */
705  META_ELSE, /* \else */
706  META_ENDIF, /* \endif */
707  META_STARTPIPELINE, /* \startpipeline */
708  META_ENDPIPELINE /* \endpipeline */
709 } MetaCommand;
710 
711 typedef enum QueryMode
712 {
713  QUERY_SIMPLE, /* simple query */
714  QUERY_EXTENDED, /* extended query */
715  QUERY_PREPARED, /* extended query with prepared statements */
717 } QueryMode;
718 
720 static const char *QUERYMODE[] = {"simple", "extended", "prepared"};
721 
722 /*
723  * struct Command represents one command in a script.
724  *
725  * lines The raw, possibly multi-line command text. Variable substitution
726  * not applied.
727  * first_line A short, single-line extract of 'lines', for error reporting.
728  * type SQL_COMMAND or META_COMMAND
729  * meta The type of meta-command, with META_NONE/GSET/ASET if command
730  * is SQL.
731  * argc Number of arguments of the command, 0 if not yet processed.
732  * argv Command arguments, the first of which is the command or SQL
733  * string itself. For SQL commands, after post-processing
734  * argv[0] is the same as 'lines' with variables substituted.
735  * varprefix SQL commands terminated with \gset or \aset have this set
736  * to a non NULL value. If nonempty, it's used to prefix the
737  * variable name that receives the value.
738  * aset do gset on all possible queries of a combined query (\;).
739  * expr Parsed expression, if needed.
740  * stats Time spent in this command.
741  * retries Number of retries after a serialization or deadlock error in the
742  * current command.
743  * failures Number of errors in the current command that were not retried.
744  */
745 typedef struct Command
746 {
748  char *first_line;
749  int type;
751  int argc;
752  char *argv[MAX_ARGS];
753  char *varprefix;
756  int64 retries;
757  int64 failures;
758 } Command;
759 
760 typedef struct ParsedScript
761 {
762  const char *desc; /* script descriptor (eg, file name) */
763  int weight; /* selection weight */
764  Command **commands; /* NULL-terminated array of Commands */
765  StatsData stats; /* total time spent in script */
766 } ParsedScript;
767 
768 static ParsedScript sql_script[MAX_SCRIPTS]; /* SQL script files */
769 static int num_scripts; /* number of scripts in sql_script[] */
770 static int64 total_weight = 0;
771 
772 static bool verbose_errors = false; /* print verbose messages of all errors */
773 
774 /* Builtin test scripts */
775 typedef struct BuiltinScript
776 {
777  const char *name; /* very short name for -b ... */
778  const char *desc; /* short description */
779  const char *script; /* actual pgbench script */
780 } BuiltinScript;
781 
782 static const BuiltinScript builtin_script[] =
783 {
784  {
785  "tpcb-like",
786  "<builtin: TPC-B (sort of)>",
787  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
788  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
789  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
790  "\\set delta random(-5000, 5000)\n"
791  "BEGIN;\n"
792  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
793  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
794  "UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;\n"
795  "UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;\n"
796  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
797  "END;\n"
798  },
799  {
800  "simple-update",
801  "<builtin: simple update>",
802  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
803  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
804  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
805  "\\set delta random(-5000, 5000)\n"
806  "BEGIN;\n"
807  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
808  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
809  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
810  "END;\n"
811  },
812  {
813  "select-only",
814  "<builtin: select only>",
815  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
816  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
817  }
818 };
819 
820 
821 /* Function prototypes */
822 static void setNullValue(PgBenchValue *pv);
823 static void setBoolValue(PgBenchValue *pv, bool bval);
824 static void setIntValue(PgBenchValue *pv, int64 ival);
825 static void setDoubleValue(PgBenchValue *pv, double dval);
826 static bool evaluateExpr(CState *st, PgBenchExpr *expr,
827  PgBenchValue *retval);
829 static void doLog(TState *thread, CState *st,
830  StatsData *agg, bool skipped, double latency, double lag);
831 static void processXactStats(TState *thread, CState *st, pg_time_usec_t *now,
832  bool skipped, StatsData *agg);
833 static void addScript(const ParsedScript *script);
835 static void finishCon(CState *st);
836 static void setalarm(int seconds);
837 static socket_set *alloc_socket_set(int count);
838 static void free_socket_set(socket_set *sa);
839 static void clear_socket_set(socket_set *sa);
840 static void add_socket_to_set(socket_set *sa, int fd, int idx);
841 static int wait_on_socket_set(socket_set *sa, int64 usecs);
842 static bool socket_has_input(socket_set *sa, int fd, int idx);
843 
844 
845 /* callback functions for our flex lexer */
846 static const PsqlScanCallbacks pgbench_callbacks = {
847  NULL, /* don't need get_variable functionality */
848 };
849 
850 static inline pg_time_usec_t
851 pg_time_now(void)
852 {
853  instr_time now;
854 
856 
858 }
859 
860 static inline void
862 {
863  if ((*now) == 0)
864  (*now) = pg_time_now();
865 }
866 
867 #define PG_TIME_GET_DOUBLE(t) (0.000001 * (t))
868 
869 static void
870 usage(void)
871 {
872  printf("%s is a benchmarking tool for PostgreSQL.\n\n"
873  "Usage:\n"
874  " %s [OPTION]... [DBNAME]\n"
875  "\nInitialization options:\n"
876  " -i, --initialize invokes initialization mode\n"
877  " -I, --init-steps=[" ALL_INIT_STEPS "]+ (default \"" DEFAULT_INIT_STEPS "\")\n"
878  " run selected initialization steps\n"
879  " -F, --fillfactor=NUM set fill factor\n"
880  " -n, --no-vacuum do not run VACUUM during initialization\n"
881  " -q, --quiet quiet logging (one message each 5 seconds)\n"
882  " -s, --scale=NUM scaling factor\n"
883  " --foreign-keys create foreign key constraints between tables\n"
884  " --index-tablespace=TABLESPACE\n"
885  " create indexes in the specified tablespace\n"
886  " --partition-method=(range|hash)\n"
887  " partition pgbench_accounts with this method (default: range)\n"
888  " --partitions=NUM partition pgbench_accounts into NUM parts (default: 0)\n"
889  " --tablespace=TABLESPACE create tables in the specified tablespace\n"
890  " --unlogged-tables create tables as unlogged tables\n"
891  "\nOptions to select what to run:\n"
892  " -b, --builtin=NAME[@W] add builtin script NAME weighted at W (default: 1)\n"
893  " (use \"-b list\" to list available scripts)\n"
894  " -f, --file=FILENAME[@W] add script FILENAME weighted at W (default: 1)\n"
895  " -N, --skip-some-updates skip updates of pgbench_tellers and pgbench_branches\n"
896  " (same as \"-b simple-update\")\n"
897  " -S, --select-only perform SELECT-only transactions\n"
898  " (same as \"-b select-only\")\n"
899  "\nBenchmarking options:\n"
900  " -c, --client=NUM number of concurrent database clients (default: 1)\n"
901  " -C, --connect establish new connection for each transaction\n"
902  " -D, --define=VARNAME=VALUE\n"
903  " define variable for use by custom script\n"
904  " -j, --jobs=NUM number of threads (default: 1)\n"
905  " -l, --log write transaction times to log file\n"
906  " -L, --latency-limit=NUM count transactions lasting more than NUM ms as late\n"
907  " -M, --protocol=simple|extended|prepared\n"
908  " protocol for submitting queries (default: simple)\n"
909  " -n, --no-vacuum do not run VACUUM before tests\n"
910  " -P, --progress=NUM show thread progress report every NUM seconds\n"
911  " -r, --report-per-command report latencies, failures, and retries per command\n"
912  " -R, --rate=NUM target rate in transactions per second\n"
913  " -s, --scale=NUM report this scale factor in output\n"
914  " -t, --transactions=NUM number of transactions each client runs (default: 10)\n"
915  " -T, --time=NUM duration of benchmark test in seconds\n"
916  " -v, --vacuum-all vacuum all four standard tables before tests\n"
917  " --aggregate-interval=NUM aggregate data over NUM seconds\n"
918  " --failures-detailed report the failures grouped by basic types\n"
919  " --log-prefix=PREFIX prefix for transaction time log file\n"
920  " (default: \"pgbench_log\")\n"
921  " --max-tries=NUM max number of tries to run transaction (default: 1)\n"
922  " --progress-timestamp use Unix epoch timestamps for progress\n"
923  " --random-seed=SEED set random seed (\"time\", \"rand\", integer)\n"
924  " --sampling-rate=NUM fraction of transactions to log (e.g., 0.01 for 1%%)\n"
925  " --show-script=NAME show builtin script code, then exit\n"
926  " --verbose-errors print messages of all errors\n"
927  "\nCommon options:\n"
928  " -d, --debug print debugging output\n"
929  " -h, --host=HOSTNAME database server host or socket directory\n"
930  " -p, --port=PORT database server port number\n"
931  " -U, --username=USERNAME connect as specified database user\n"
932  " -V, --version output version information, then exit\n"
933  " -?, --help show this help, then exit\n"
934  "\n"
935  "Report bugs to <%s>.\n"
936  "%s home page: <%s>\n",
937  progname, progname, PACKAGE_BUGREPORT, PACKAGE_NAME, PACKAGE_URL);
938 }
939 
940 /* return whether str matches "^\s*[-+]?[0-9]+$" */
941 static bool
942 is_an_int(const char *str)
943 {
944  const char *ptr = str;
945 
946  /* skip leading spaces; cast is consistent with strtoint64 */
947  while (*ptr && isspace((unsigned char) *ptr))
948  ptr++;
949 
950  /* skip sign */
951  if (*ptr == '+' || *ptr == '-')
952  ptr++;
953 
954  /* at least one digit */
955  if (*ptr && !isdigit((unsigned char) *ptr))
956  return false;
957 
958  /* eat all digits */
959  while (*ptr && isdigit((unsigned char) *ptr))
960  ptr++;
961 
962  /* must have reached end of string */
963  return *ptr == '\0';
964 }
965 
966 
967 /*
968  * strtoint64 -- convert a string to 64-bit integer
969  *
970  * This function is a slightly modified version of pg_strtoint64() from
971  * src/backend/utils/adt/numutils.c.
972  *
973  * The function returns whether the conversion worked, and if so
974  * "*result" is set to the result.
975  *
976  * If not errorOK, an error message is also printed out on errors.
977  */
978 bool
979 strtoint64(const char *str, bool errorOK, int64 *result)
980 {
981  const char *ptr = str;
982  int64 tmp = 0;
983  bool neg = false;
984 
985  /*
986  * Do our own scan, rather than relying on sscanf which might be broken
987  * for long long.
988  *
989  * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
990  * value as a negative number.
991  */
992 
993  /* skip leading spaces */
994  while (*ptr && isspace((unsigned char) *ptr))
995  ptr++;
996 
997  /* handle sign */
998  if (*ptr == '-')
999  {
1000  ptr++;
1001  neg = true;
1002  }
1003  else if (*ptr == '+')
1004  ptr++;
1005 
1006  /* require at least one digit */
1007  if (unlikely(!isdigit((unsigned char) *ptr)))
1008  goto invalid_syntax;
1009 
1010  /* process digits */
1011  while (*ptr && isdigit((unsigned char) *ptr))
1012  {
1013  int8 digit = (*ptr++ - '0');
1014 
1015  if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
1016  unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
1017  goto out_of_range;
1018  }
1019 
1020  /* allow trailing whitespace, but not other trailing chars */
1021  while (*ptr != '\0' && isspace((unsigned char) *ptr))
1022  ptr++;
1023 
1024  if (unlikely(*ptr != '\0'))
1025  goto invalid_syntax;
1026 
1027  if (!neg)
1028  {
1029  if (unlikely(tmp == PG_INT64_MIN))
1030  goto out_of_range;
1031  tmp = -tmp;
1032  }
1033 
1034  *result = tmp;
1035  return true;
1036 
1037 out_of_range:
1038  if (!errorOK)
1039  pg_log_error("value \"%s\" is out of range for type bigint", str);
1040  return false;
1041 
1042 invalid_syntax:
1043  if (!errorOK)
1044  pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
1045  return false;
1046 }
1047 
1048 /* convert string to double, detecting overflows/underflows */
1049 bool
1050 strtodouble(const char *str, bool errorOK, double *dv)
1051 {
1052  char *end;
1053 
1054  errno = 0;
1055  *dv = strtod(str, &end);
1056 
1057  if (unlikely(errno != 0))
1058  {
1059  if (!errorOK)
1060  pg_log_error("value \"%s\" is out of range for type double", str);
1061  return false;
1062  }
1063 
1064  if (unlikely(end == str || *end != '\0'))
1065  {
1066  if (!errorOK)
1067  pg_log_error("invalid input syntax for type double: \"%s\"", str);
1068  return false;
1069  }
1070  return true;
1071 }
1072 
1073 /*
1074  * Initialize a prng state struct.
1075  *
1076  * We derive the seed from base_random_sequence, which must be set up already.
1077  */
1078 static void
1080 {
1082 }
1083 
1084 
1085 /*
1086  * random number generator: uniform distribution from min to max inclusive.
1087  *
1088  * Although the limits are expressed as int64, you can't generate the full
1089  * int64 range in one call, because the difference of the limits mustn't
1090  * overflow int64. This is not checked.
1091  */
1092 static int64
1093 getrand(pg_prng_state *state, int64 min, int64 max)
1094 {
1095  return min + (int64) pg_prng_uint64_range(state, 0, max - min);
1096 }
1097 
1098 /*
1099  * random number generator: exponential distribution from min to max inclusive.
1100  * the parameter is so that the density of probability for the last cut-off max
1101  * value is exp(-parameter).
1102  */
1103 static int64
1104 getExponentialRand(pg_prng_state *state, int64 min, int64 max,
1105  double parameter)
1106 {
1107  double cut,
1108  uniform,
1109  rand;
1110 
1111  /* abort if wrong parameter, but must really be checked beforehand */
1112  Assert(parameter > 0.0);
1113  cut = exp(-parameter);
1114  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1115  uniform = 1.0 - pg_prng_double(state);
1116 
1117  /*
1118  * inner expression in (cut, 1] (if parameter > 0), rand in [0, 1)
1119  */
1120  Assert((1.0 - cut) != 0.0);
1121  rand = -log(cut + (1.0 - cut) * uniform) / parameter;
1122  /* return int64 random number within between min and max */
1123  return min + (int64) ((max - min + 1) * rand);
1124 }
1125 
1126 /* random number generator: gaussian distribution from min to max inclusive */
1127 static int64
1128 getGaussianRand(pg_prng_state *state, int64 min, int64 max,
1129  double parameter)
1130 {
1131  double stdev;
1132  double rand;
1133 
1134  /* abort if parameter is too low, but must really be checked beforehand */
1135  Assert(parameter >= MIN_GAUSSIAN_PARAM);
1136 
1137  /*
1138  * Get user specified random number from this loop, with -parameter <
1139  * stdev <= parameter
1140  *
1141  * This loop is executed until the number is in the expected range.
1142  *
1143  * As the minimum parameter is 2.0, the probability of looping is low:
1144  * sqrt(-2 ln(r)) <= 2 => r >= e^{-2} ~ 0.135, then when taking the
1145  * average sinus multiplier as 2/pi, we have a 8.6% looping probability in
1146  * the worst case. For a parameter value of 5.0, the looping probability
1147  * is about e^{-5} * 2 / pi ~ 0.43%.
1148  */
1149  do
1150  {
1151  /*
1152  * pg_prng_double generates [0, 1), but for the basic version of the
1153  * Box-Muller transform the two uniformly distributed random numbers
1154  * are expected to be in (0, 1] (see
1155  * https://en.wikipedia.org/wiki/Box-Muller_transform)
1156  */
1157  double rand1 = 1.0 - pg_prng_double(state);
1158  double rand2 = 1.0 - pg_prng_double(state);
1159 
1160  /* Box-Muller basic form transform */
1161  double var_sqrt = sqrt(-2.0 * log(rand1));
1162 
1163  stdev = var_sqrt * sin(2.0 * M_PI * rand2);
1164 
1165  /*
1166  * we may try with cos, but there may be a bias induced if the
1167  * previous value fails the test. To be on the safe side, let us try
1168  * over.
1169  */
1170  }
1171  while (stdev < -parameter || stdev >= parameter);
1172 
1173  /* stdev is in [-parameter, parameter), normalization to [0,1) */
1174  rand = (stdev + parameter) / (parameter * 2.0);
1175 
1176  /* return int64 random number within between min and max */
1177  return min + (int64) ((max - min + 1) * rand);
1178 }
1179 
1180 /*
1181  * random number generator: generate a value, such that the series of values
1182  * will approximate a Poisson distribution centered on the given value.
1183  *
1184  * Individual results are rounded to integers, though the center value need
1185  * not be one.
1186  */
1187 static int64
1188 getPoissonRand(pg_prng_state *state, double center)
1189 {
1190  /*
1191  * Use inverse transform sampling to generate a value > 0, such that the
1192  * expected (i.e. average) value is the given argument.
1193  */
1194  double uniform;
1195 
1196  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1197  uniform = 1.0 - pg_prng_double(state);
1198 
1199  return (int64) (-log(uniform) * center + 0.5);
1200 }
1201 
1202 /*
1203  * Computing zipfian using rejection method, based on
1204  * "Non-Uniform Random Variate Generation",
1205  * Luc Devroye, p. 550-551, Springer 1986.
1206  *
1207  * This works for s > 1.0, but may perform badly for s very close to 1.0.
1208  */
1209 static int64
1210 computeIterativeZipfian(pg_prng_state *state, int64 n, double s)
1211 {
1212  double b = pow(2.0, s - 1.0);
1213  double x,
1214  t,
1215  u,
1216  v;
1217 
1218  /* Ensure n is sane */
1219  if (n <= 1)
1220  return 1;
1221 
1222  while (true)
1223  {
1224  /* random variates */
1225  u = pg_prng_double(state);
1226  v = pg_prng_double(state);
1227 
1228  x = floor(pow(u, -1.0 / (s - 1.0)));
1229 
1230  t = pow(1.0 + 1.0 / x, s - 1.0);
1231  /* reject if too large or out of bound */
1232  if (v * x * (t - 1.0) / (b - 1.0) <= t / b && x <= n)
1233  break;
1234  }
1235  return (int64) x;
1236 }
1237 
1238 /* random number generator: zipfian distribution from min to max inclusive */
1239 static int64
1240 getZipfianRand(pg_prng_state *state, int64 min, int64 max, double s)
1241 {
1242  int64 n = max - min + 1;
1243 
1244  /* abort if parameter is invalid */
1246 
1247  return min - 1 + computeIterativeZipfian(state, n, s);
1248 }
1249 
1250 /*
1251  * FNV-1a hash function
1252  */
1253 static int64
1254 getHashFnv1a(int64 val, uint64 seed)
1255 {
1256  int64 result;
1257  int i;
1258 
1259  result = FNV_OFFSET_BASIS ^ seed;
1260  for (i = 0; i < 8; ++i)
1261  {
1262  int32 octet = val & 0xff;
1263 
1264  val = val >> 8;
1265  result = result ^ octet;
1266  result = result * FNV_PRIME;
1267  }
1268 
1269  return result;
1270 }
1271 
1272 /*
1273  * Murmur2 hash function
1274  *
1275  * Based on original work of Austin Appleby
1276  * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash2.cpp
1277  */
1278 static int64
1279 getHashMurmur2(int64 val, uint64 seed)
1280 {
1281  uint64 result = seed ^ MM2_MUL_TIMES_8; /* sizeof(int64) */
1282  uint64 k = (uint64) val;
1283 
1284  k *= MM2_MUL;
1285  k ^= k >> MM2_ROT;
1286  k *= MM2_MUL;
1287 
1288  result ^= k;
1289  result *= MM2_MUL;
1290 
1291  result ^= result >> MM2_ROT;
1292  result *= MM2_MUL;
1293  result ^= result >> MM2_ROT;
1294 
1295  return (int64) result;
1296 }
1297 
1298 /*
1299  * Pseudorandom permutation function
1300  *
1301  * For small sizes, this generates each of the (size!) possible permutations
1302  * of integers in the range [0, size) with roughly equal probability. Once
1303  * the size is larger than 20, the number of possible permutations exceeds the
1304  * number of distinct states of the internal pseudorandom number generator,
1305  * and so not all possible permutations can be generated, but the permutations
1306  * chosen should continue to give the appearance of being random.
1307  *
1308  * THIS FUNCTION IS NOT CRYPTOGRAPHICALLY SECURE.
1309  * DO NOT USE FOR SUCH PURPOSE.
1310  */
1311 static int64
1312 permute(const int64 val, const int64 isize, const int64 seed)
1313 {
1314  /* using a high-end PRNG is probably overkill */
1316  uint64 size;
1317  uint64 v;
1318  int masklen;
1319  uint64 mask;
1320  int i;
1321 
1322  if (isize < 2)
1323  return 0; /* nothing to permute */
1324 
1325  /* Initialize prng state using the seed */
1326  pg_prng_seed(&state, (uint64) seed);
1327 
1328  /* Computations are performed on unsigned values */
1329  size = (uint64) isize;
1330  v = (uint64) val % size;
1331 
1332  /* Mask to work modulo largest power of 2 less than or equal to size */
1333  masklen = pg_leftmost_one_pos64(size);
1334  mask = (((uint64) 1) << masklen) - 1;
1335 
1336  /*
1337  * Permute the input value by applying several rounds of pseudorandom
1338  * bijective transformations. The intention here is to distribute each
1339  * input uniformly randomly across the range, and separate adjacent inputs
1340  * approximately uniformly randomly from each other, leading to a fairly
1341  * random overall choice of permutation.
1342  *
1343  * To separate adjacent inputs, we multiply by a random number modulo
1344  * (mask + 1), which is a power of 2. For this to be a bijection, the
1345  * multiplier must be odd. Since this is known to lead to less randomness
1346  * in the lower bits, we also apply a rotation that shifts the topmost bit
1347  * into the least significant bit. In the special cases where size <= 3,
1348  * mask = 1 and each of these operations is actually a no-op, so we also
1349  * XOR the value with a different random number to inject additional
1350  * randomness. Since the size is generally not a power of 2, we apply
1351  * this bijection on overlapping upper and lower halves of the input.
1352  *
1353  * To distribute the inputs uniformly across the range, we then also apply
1354  * a random offset modulo the full range.
1355  *
1356  * Taken together, these operations resemble a modified linear
1357  * congruential generator, as is commonly used in pseudorandom number
1358  * generators. The number of rounds is fairly arbitrary, but six has been
1359  * found empirically to give a fairly good tradeoff between performance
1360  * and uniform randomness. For small sizes it selects each of the (size!)
1361  * possible permutations with roughly equal probability. For larger
1362  * sizes, not all permutations can be generated, but the intended random
1363  * spread is still produced.
1364  */
1365  for (i = 0; i < 6; i++)
1366  {
1367  uint64 m,
1368  r,
1369  t;
1370 
1371  /* Random multiply (by an odd number), XOR and rotate of lower half */
1372  m = (pg_prng_uint64(&state) & mask) | 1;
1373  r = pg_prng_uint64(&state) & mask;
1374  if (v <= mask)
1375  {
1376  v = ((v * m) ^ r) & mask;
1377  v = ((v << 1) & mask) | (v >> (masklen - 1));
1378  }
1379 
1380  /* Random multiply (by an odd number), XOR and rotate of upper half */
1381  m = (pg_prng_uint64(&state) & mask) | 1;
1382  r = pg_prng_uint64(&state) & mask;
1383  t = size - 1 - v;
1384  if (t <= mask)
1385  {
1386  t = ((t * m) ^ r) & mask;
1387  t = ((t << 1) & mask) | (t >> (masklen - 1));
1388  v = size - 1 - t;
1389  }
1390 
1391  /* Random offset */
1392  r = pg_prng_uint64_range(&state, 0, size - 1);
1393  v = (v + r) % size;
1394  }
1395 
1396  return (int64) v;
1397 }
1398 
1399 /*
1400  * Initialize the given SimpleStats struct to all zeroes
1401  */
1402 static void
1404 {
1405  memset(ss, 0, sizeof(SimpleStats));
1406 }
1407 
1408 /*
1409  * Accumulate one value into a SimpleStats struct.
1410  */
1411 static void
1412 addToSimpleStats(SimpleStats *ss, double val)
1413 {
1414  if (ss->count == 0 || val < ss->min)
1415  ss->min = val;
1416  if (ss->count == 0 || val > ss->max)
1417  ss->max = val;
1418  ss->count++;
1419  ss->sum += val;
1420  ss->sum2 += val * val;
1421 }
1422 
1423 /*
1424  * Merge two SimpleStats objects
1425  */
1426 static void
1428 {
1429  if (acc->count == 0 || ss->min < acc->min)
1430  acc->min = ss->min;
1431  if (acc->count == 0 || ss->max > acc->max)
1432  acc->max = ss->max;
1433  acc->count += ss->count;
1434  acc->sum += ss->sum;
1435  acc->sum2 += ss->sum2;
1436 }
1437 
1438 /*
1439  * Initialize a StatsData struct to mostly zeroes, with its start time set to
1440  * the given value.
1441  */
1442 static void
1444 {
1445  sd->start_time = start;
1446  sd->cnt = 0;
1447  sd->skipped = 0;
1448  sd->retries = 0;
1449  sd->retried = 0;
1450  sd->serialization_failures = 0;
1451  sd->deadlock_failures = 0;
1452  initSimpleStats(&sd->latency);
1453  initSimpleStats(&sd->lag);
1454 }
1455 
1456 /*
1457  * Accumulate one additional item into the given stats object.
1458  */
1459 static void
1460 accumStats(StatsData *stats, bool skipped, double lat, double lag,
1461  EStatus estatus, int64 tries)
1462 {
1463  /* Record the skipped transaction */
1464  if (skipped)
1465  {
1466  /* no latency to record on skipped transactions */
1467  stats->skipped++;
1468  return;
1469  }
1470 
1471  /*
1472  * Record the number of retries regardless of whether the transaction was
1473  * successful or failed.
1474  */
1475  if (tries > 1)
1476  {
1477  stats->retries += (tries - 1);
1478  stats->retried++;
1479  }
1480 
1481  switch (estatus)
1482  {
1483  /* Record the successful transaction */
1484  case ESTATUS_NO_ERROR:
1485  stats->cnt++;
1486 
1487  addToSimpleStats(&stats->latency, lat);
1488 
1489  /* and possibly the same for schedule lag */
1490  if (throttle_delay)
1491  addToSimpleStats(&stats->lag, lag);
1492  break;
1493 
1494  /* Record the failed transaction */
1496  stats->serialization_failures++;
1497  break;
1499  stats->deadlock_failures++;
1500  break;
1501  default:
1502  /* internal error which should never occur */
1503  pg_fatal("unexpected error status: %d", estatus);
1504  }
1505 }
1506 
1507 /* call PQexec() and exit() on failure */
1508 static void
1509 executeStatement(PGconn *con, const char *sql)
1510 {
1511  PGresult *res;
1512 
1513  res = PQexec(con, sql);
1515  {
1516  pg_log_error("query failed: %s", PQerrorMessage(con));
1517  pg_log_error_detail("Query was: %s", sql);
1518  exit(1);
1519  }
1520  PQclear(res);
1521 }
1522 
1523 /* call PQexec() and complain, but without exiting, on failure */
1524 static void
1525 tryExecuteStatement(PGconn *con, const char *sql)
1526 {
1527  PGresult *res;
1528 
1529  res = PQexec(con, sql);
1531  {
1532  pg_log_error("%s", PQerrorMessage(con));
1533  pg_log_error_detail("(ignoring this error and continuing anyway)");
1534  }
1535  PQclear(res);
1536 }
1537 
1538 /* set up a connection to the backend */
1539 static PGconn *
1540 doConnect(void)
1541 {
1542  PGconn *conn;
1543  bool new_pass;
1544  static char *password = NULL;
1545 
1546  /*
1547  * Start the connection. Loop until we have a password if requested by
1548  * backend.
1549  */
1550  do
1551  {
1552 #define PARAMS_ARRAY_SIZE 7
1553 
1554  const char *keywords[PARAMS_ARRAY_SIZE];
1555  const char *values[PARAMS_ARRAY_SIZE];
1556 
1557  keywords[0] = "host";
1558  values[0] = pghost;
1559  keywords[1] = "port";
1560  values[1] = pgport;
1561  keywords[2] = "user";
1562  values[2] = username;
1563  keywords[3] = "password";
1564  values[3] = password;
1565  keywords[4] = "dbname";
1566  values[4] = dbName;
1567  keywords[5] = "fallback_application_name";
1568  values[5] = progname;
1569  keywords[6] = NULL;
1570  values[6] = NULL;
1571 
1572  new_pass = false;
1573 
1574  conn = PQconnectdbParams(keywords, values, true);
1575 
1576  if (!conn)
1577  {
1578  pg_log_error("connection to database \"%s\" failed", dbName);
1579  return NULL;
1580  }
1581 
1582  if (PQstatus(conn) == CONNECTION_BAD &&
1584  !password)
1585  {
1586  PQfinish(conn);
1587  password = simple_prompt("Password: ", false);
1588  new_pass = true;
1589  }
1590  } while (new_pass);
1591 
1592  /* check to see that the backend connection was successfully made */
1593  if (PQstatus(conn) == CONNECTION_BAD)
1594  {
1596  PQfinish(conn);
1597  return NULL;
1598  }
1599 
1600  return conn;
1601 }
1602 
1603 /* qsort comparator for Variable array */
1604 static int
1605 compareVariableNames(const void *v1, const void *v2)
1606 {
1607  return strcmp(((const Variable *) v1)->name,
1608  ((const Variable *) v2)->name);
1609 }
1610 
1611 /* Locate a variable by name; returns NULL if unknown */
1612 static Variable *
1613 lookupVariable(Variables *variables, char *name)
1614 {
1615  Variable key;
1616 
1617  /* On some versions of Solaris, bsearch of zero items dumps core */
1618  if (variables->nvars <= 0)
1619  return NULL;
1620 
1621  /* Sort if we have to */
1622  if (!variables->vars_sorted)
1623  {
1624  qsort((void *) variables->vars, variables->nvars, sizeof(Variable),
1626  variables->vars_sorted = true;
1627  }
1628 
1629  /* Now we can search */
1630  key.name = name;
1631  return (Variable *) bsearch((void *) &key,
1632  (void *) variables->vars,
1633  variables->nvars,
1634  sizeof(Variable),
1636 }
1637 
1638 /* Get the value of a variable, in string form; returns NULL if unknown */
1639 static char *
1640 getVariable(Variables *variables, char *name)
1641 {
1642  Variable *var;
1643  char stringform[64];
1644 
1645  var = lookupVariable(variables, name);
1646  if (var == NULL)
1647  return NULL; /* not found */
1648 
1649  if (var->svalue)
1650  return var->svalue; /* we have it in string form */
1651 
1652  /* We need to produce a string equivalent of the value */
1653  Assert(var->value.type != PGBT_NO_VALUE);
1654  if (var->value.type == PGBT_NULL)
1655  snprintf(stringform, sizeof(stringform), "NULL");
1656  else if (var->value.type == PGBT_BOOLEAN)
1657  snprintf(stringform, sizeof(stringform),
1658  "%s", var->value.u.bval ? "true" : "false");
1659  else if (var->value.type == PGBT_INT)
1660  snprintf(stringform, sizeof(stringform),
1661  INT64_FORMAT, var->value.u.ival);
1662  else if (var->value.type == PGBT_DOUBLE)
1663  snprintf(stringform, sizeof(stringform),
1664  "%.*g", DBL_DIG, var->value.u.dval);
1665  else /* internal error, unexpected type */
1666  Assert(0);
1667  var->svalue = pg_strdup(stringform);
1668  return var->svalue;
1669 }
1670 
1671 /* Try to convert variable to a value; return false on failure */
1672 static bool
1674 {
1675  size_t slen;
1676 
1677  if (var->value.type != PGBT_NO_VALUE)
1678  return true; /* no work */
1679 
1680  slen = strlen(var->svalue);
1681 
1682  if (slen == 0)
1683  /* what should it do on ""? */
1684  return false;
1685 
1686  if (pg_strcasecmp(var->svalue, "null") == 0)
1687  {
1688  setNullValue(&var->value);
1689  }
1690 
1691  /*
1692  * accept prefixes such as y, ye, n, no... but not for "o". 0/1 are
1693  * recognized later as an int, which is converted to bool if needed.
1694  */
1695  else if (pg_strncasecmp(var->svalue, "true", slen) == 0 ||
1696  pg_strncasecmp(var->svalue, "yes", slen) == 0 ||
1697  pg_strcasecmp(var->svalue, "on") == 0)
1698  {
1699  setBoolValue(&var->value, true);
1700  }
1701  else if (pg_strncasecmp(var->svalue, "false", slen) == 0 ||
1702  pg_strncasecmp(var->svalue, "no", slen) == 0 ||
1703  pg_strcasecmp(var->svalue, "off") == 0 ||
1704  pg_strcasecmp(var->svalue, "of") == 0)
1705  {
1706  setBoolValue(&var->value, false);
1707  }
1708  else if (is_an_int(var->svalue))
1709  {
1710  /* if it looks like an int, it must be an int without overflow */
1711  int64 iv;
1712 
1713  if (!strtoint64(var->svalue, false, &iv))
1714  return false;
1715 
1716  setIntValue(&var->value, iv);
1717  }
1718  else /* type should be double */
1719  {
1720  double dv;
1721 
1722  if (!strtodouble(var->svalue, true, &dv))
1723  {
1724  pg_log_error("malformed variable \"%s\" value: \"%s\"",
1725  var->name, var->svalue);
1726  return false;
1727  }
1728  setDoubleValue(&var->value, dv);
1729  }
1730  return true;
1731 }
1732 
1733 /*
1734  * Check whether a variable's name is allowed.
1735  *
1736  * We allow any non-ASCII character, as well as ASCII letters, digits, and
1737  * underscore.
1738  *
1739  * Keep this in sync with the definitions of variable name characters in
1740  * "src/fe_utils/psqlscan.l", "src/bin/psql/psqlscanslash.l" and
1741  * "src/bin/pgbench/exprscan.l". Also see parseVariable(), below.
1742  *
1743  * Note: this static function is copied from "src/bin/psql/variables.c"
1744  * but changed to disallow variable names starting with a digit.
1745  */
1746 static bool
1747 valid_variable_name(const char *name)
1748 {
1749  const unsigned char *ptr = (const unsigned char *) name;
1750 
1751  /* Mustn't be zero-length */
1752  if (*ptr == '\0')
1753  return false;
1754 
1755  /* must not start with [0-9] */
1756  if (IS_HIGHBIT_SET(*ptr) ||
1757  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1758  "_", *ptr) != NULL)
1759  ptr++;
1760  else
1761  return false;
1762 
1763  /* remaining characters can include [0-9] */
1764  while (*ptr)
1765  {
1766  if (IS_HIGHBIT_SET(*ptr) ||
1767  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1768  "_0123456789", *ptr) != NULL)
1769  ptr++;
1770  else
1771  return false;
1772  }
1773 
1774  return true;
1775 }
1776 
1777 /*
1778  * Make sure there is enough space for 'needed' more variable in the variables
1779  * array.
1780  */
1781 static void
1782 enlargeVariables(Variables *variables, int needed)
1783 {
1784  /* total number of variables required now */
1785  needed += variables->nvars;
1786 
1787  if (variables->max_vars < needed)
1788  {
1789  variables->max_vars = needed + VARIABLES_ALLOC_MARGIN;
1790  variables->vars = (Variable *)
1791  pg_realloc(variables->vars, variables->max_vars * sizeof(Variable));
1792  }
1793 }
1794 
1795 /*
1796  * Lookup a variable by name, creating it if need be.
1797  * Caller is expected to assign a value to the variable.
1798  * Returns NULL on failure (bad name).
1799  */
1800 static Variable *
1801 lookupCreateVariable(Variables *variables, const char *context, char *name)
1802 {
1803  Variable *var;
1804 
1805  var = lookupVariable(variables, name);
1806  if (var == NULL)
1807  {
1808  /*
1809  * Check for the name only when declaring a new variable to avoid
1810  * overhead.
1811  */
1812  if (!valid_variable_name(name))
1813  {
1814  pg_log_error("%s: invalid variable name: \"%s\"", context, name);
1815  return NULL;
1816  }
1817 
1818  /* Create variable at the end of the array */
1819  enlargeVariables(variables, 1);
1820 
1821  var = &(variables->vars[variables->nvars]);
1822 
1823  var->name = pg_strdup(name);
1824  var->svalue = NULL;
1825  /* caller is expected to initialize remaining fields */
1826 
1827  variables->nvars++;
1828  /* we don't re-sort the array till we have to */
1829  variables->vars_sorted = false;
1830  }
1831 
1832  return var;
1833 }
1834 
1835 /* Assign a string value to a variable, creating it if need be */
1836 /* Returns false on failure (bad name) */
1837 static bool
1838 putVariable(Variables *variables, const char *context, char *name,
1839  const char *value)
1840 {
1841  Variable *var;
1842  char *val;
1843 
1844  var = lookupCreateVariable(variables, context, name);
1845  if (!var)
1846  return false;
1847 
1848  /* dup then free, in case value is pointing at this variable */
1849  val = pg_strdup(value);
1850 
1851  free(var->svalue);
1852  var->svalue = val;
1853  var->value.type = PGBT_NO_VALUE;
1854 
1855  return true;
1856 }
1857 
1858 /* Assign a value to a variable, creating it if need be */
1859 /* Returns false on failure (bad name) */
1860 static bool
1861 putVariableValue(Variables *variables, const char *context, char *name,
1862  const PgBenchValue *value)
1863 {
1864  Variable *var;
1865 
1866  var = lookupCreateVariable(variables, context, name);
1867  if (!var)
1868  return false;
1869 
1870  free(var->svalue);
1871  var->svalue = NULL;
1872  var->value = *value;
1873 
1874  return true;
1875 }
1876 
1877 /* Assign an integer value to a variable, creating it if need be */
1878 /* Returns false on failure (bad name) */
1879 static bool
1880 putVariableInt(Variables *variables, const char *context, char *name,
1881  int64 value)
1882 {
1883  PgBenchValue val;
1884 
1885  setIntValue(&val, value);
1886  return putVariableValue(variables, context, name, &val);
1887 }
1888 
1889 /*
1890  * Parse a possible variable reference (:varname).
1891  *
1892  * "sql" points at a colon. If what follows it looks like a valid
1893  * variable name, return a malloc'd string containing the variable name,
1894  * and set *eaten to the number of characters consumed (including the colon).
1895  * Otherwise, return NULL.
1896  */
1897 static char *
1898 parseVariable(const char *sql, int *eaten)
1899 {
1900  int i = 1; /* starting at 1 skips the colon */
1901  char *name;
1902 
1903  /* keep this logic in sync with valid_variable_name() */
1904  if (IS_HIGHBIT_SET(sql[i]) ||
1905  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1906  "_", sql[i]) != NULL)
1907  i++;
1908  else
1909  return NULL;
1910 
1911  while (IS_HIGHBIT_SET(sql[i]) ||
1912  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1913  "_0123456789", sql[i]) != NULL)
1914  i++;
1915 
1916  name = pg_malloc(i);
1917  memcpy(name, &sql[1], i - 1);
1918  name[i - 1] = '\0';
1919 
1920  *eaten = i;
1921  return name;
1922 }
1923 
1924 static char *
1925 replaceVariable(char **sql, char *param, int len, char *value)
1926 {
1927  int valueln = strlen(value);
1928 
1929  if (valueln > len)
1930  {
1931  size_t offset = param - *sql;
1932 
1933  *sql = pg_realloc(*sql, strlen(*sql) - len + valueln + 1);
1934  param = *sql + offset;
1935  }
1936 
1937  if (valueln != len)
1938  memmove(param + valueln, param + len, strlen(param + len) + 1);
1939  memcpy(param, value, valueln);
1940 
1941  return param + valueln;
1942 }
1943 
1944 static char *
1945 assignVariables(Variables *variables, char *sql)
1946 {
1947  char *p,
1948  *name,
1949  *val;
1950 
1951  p = sql;
1952  while ((p = strchr(p, ':')) != NULL)
1953  {
1954  int eaten;
1955 
1956  name = parseVariable(p, &eaten);
1957  if (name == NULL)
1958  {
1959  while (*p == ':')
1960  {
1961  p++;
1962  }
1963  continue;
1964  }
1965 
1966  val = getVariable(variables, name);
1967  free(name);
1968  if (val == NULL)
1969  {
1970  p++;
1971  continue;
1972  }
1973 
1974  p = replaceVariable(&sql, p, eaten, val);
1975  }
1976 
1977  return sql;
1978 }
1979 
1980 static void
1981 getQueryParams(Variables *variables, const Command *command,
1982  const char **params)
1983 {
1984  int i;
1985 
1986  for (i = 0; i < command->argc - 1; i++)
1987  params[i] = getVariable(variables, command->argv[i + 1]);
1988 }
1989 
1990 static char *
1992 {
1993  if (pval->type == PGBT_NO_VALUE)
1994  return "none";
1995  else if (pval->type == PGBT_NULL)
1996  return "null";
1997  else if (pval->type == PGBT_INT)
1998  return "int";
1999  else if (pval->type == PGBT_DOUBLE)
2000  return "double";
2001  else if (pval->type == PGBT_BOOLEAN)
2002  return "boolean";
2003  else
2004  {
2005  /* internal error, should never get there */
2006  Assert(false);
2007  return NULL;
2008  }
2009 }
2010 
2011 /* get a value as a boolean, or tell if there is a problem */
2012 static bool
2013 coerceToBool(PgBenchValue *pval, bool *bval)
2014 {
2015  if (pval->type == PGBT_BOOLEAN)
2016  {
2017  *bval = pval->u.bval;
2018  return true;
2019  }
2020  else /* NULL, INT or DOUBLE */
2021  {
2022  pg_log_error("cannot coerce %s to boolean", valueTypeName(pval));
2023  *bval = false; /* suppress uninitialized-variable warnings */
2024  return false;
2025  }
2026 }
2027 
2028 /*
2029  * Return true or false from an expression for conditional purposes.
2030  * Non zero numerical values are true, zero and NULL are false.
2031  */
2032 static bool
2033 valueTruth(PgBenchValue *pval)
2034 {
2035  switch (pval->type)
2036  {
2037  case PGBT_NULL:
2038  return false;
2039  case PGBT_BOOLEAN:
2040  return pval->u.bval;
2041  case PGBT_INT:
2042  return pval->u.ival != 0;
2043  case PGBT_DOUBLE:
2044  return pval->u.dval != 0.0;
2045  default:
2046  /* internal error, unexpected type */
2047  Assert(0);
2048  return false;
2049  }
2050 }
2051 
2052 /* get a value as an int, tell if there is a problem */
2053 static bool
2054 coerceToInt(PgBenchValue *pval, int64 *ival)
2055 {
2056  if (pval->type == PGBT_INT)
2057  {
2058  *ival = pval->u.ival;
2059  return true;
2060  }
2061  else if (pval->type == PGBT_DOUBLE)
2062  {
2063  double dval = rint(pval->u.dval);
2064 
2065  if (isnan(dval) || !FLOAT8_FITS_IN_INT64(dval))
2066  {
2067  pg_log_error("double to int overflow for %f", dval);
2068  return false;
2069  }
2070  *ival = (int64) dval;
2071  return true;
2072  }
2073  else /* BOOLEAN or NULL */
2074  {
2075  pg_log_error("cannot coerce %s to int", valueTypeName(pval));
2076  return false;
2077  }
2078 }
2079 
2080 /* get a value as a double, or tell if there is a problem */
2081 static bool
2082 coerceToDouble(PgBenchValue *pval, double *dval)
2083 {
2084  if (pval->type == PGBT_DOUBLE)
2085  {
2086  *dval = pval->u.dval;
2087  return true;
2088  }
2089  else if (pval->type == PGBT_INT)
2090  {
2091  *dval = (double) pval->u.ival;
2092  return true;
2093  }
2094  else /* BOOLEAN or NULL */
2095  {
2096  pg_log_error("cannot coerce %s to double", valueTypeName(pval));
2097  return false;
2098  }
2099 }
2100 
2101 /* assign a null value */
2102 static void
2104 {
2105  pv->type = PGBT_NULL;
2106  pv->u.ival = 0;
2107 }
2108 
2109 /* assign a boolean value */
2110 static void
2111 setBoolValue(PgBenchValue *pv, bool bval)
2112 {
2113  pv->type = PGBT_BOOLEAN;
2114  pv->u.bval = bval;
2115 }
2116 
2117 /* assign an integer value */
2118 static void
2119 setIntValue(PgBenchValue *pv, int64 ival)
2120 {
2121  pv->type = PGBT_INT;
2122  pv->u.ival = ival;
2123 }
2124 
2125 /* assign a double value */
2126 static void
2127 setDoubleValue(PgBenchValue *pv, double dval)
2128 {
2129  pv->type = PGBT_DOUBLE;
2130  pv->u.dval = dval;
2131 }
2132 
2133 static bool
2135 {
2136  return func == PGBENCH_AND || func == PGBENCH_OR || func == PGBENCH_CASE;
2137 }
2138 
2139 /* lazy evaluation of some functions */
2140 static bool
2141 evalLazyFunc(CState *st,
2143 {
2144  PgBenchValue a1,
2145  a2;
2146  bool ba1,
2147  ba2;
2148 
2149  Assert(isLazyFunc(func) && args != NULL && args->next != NULL);
2150 
2151  /* args points to first condition */
2152  if (!evaluateExpr(st, args->expr, &a1))
2153  return false;
2154 
2155  /* second condition for AND/OR and corresponding branch for CASE */
2156  args = args->next;
2157 
2158  switch (func)
2159  {
2160  case PGBENCH_AND:
2161  if (a1.type == PGBT_NULL)
2162  {
2163  setNullValue(retval);
2164  return true;
2165  }
2166 
2167  if (!coerceToBool(&a1, &ba1))
2168  return false;
2169 
2170  if (!ba1)
2171  {
2172  setBoolValue(retval, false);
2173  return true;
2174  }
2175 
2176  if (!evaluateExpr(st, args->expr, &a2))
2177  return false;
2178 
2179  if (a2.type == PGBT_NULL)
2180  {
2181  setNullValue(retval);
2182  return true;
2183  }
2184  else if (!coerceToBool(&a2, &ba2))
2185  return false;
2186  else
2187  {
2188  setBoolValue(retval, ba2);
2189  return true;
2190  }
2191 
2192  return true;
2193 
2194  case PGBENCH_OR:
2195 
2196  if (a1.type == PGBT_NULL)
2197  {
2198  setNullValue(retval);
2199  return true;
2200  }
2201 
2202  if (!coerceToBool(&a1, &ba1))
2203  return false;
2204 
2205  if (ba1)
2206  {
2207  setBoolValue(retval, true);
2208  return true;
2209  }
2210 
2211  if (!evaluateExpr(st, args->expr, &a2))
2212  return false;
2213 
2214  if (a2.type == PGBT_NULL)
2215  {
2216  setNullValue(retval);
2217  return true;
2218  }
2219  else if (!coerceToBool(&a2, &ba2))
2220  return false;
2221  else
2222  {
2223  setBoolValue(retval, ba2);
2224  return true;
2225  }
2226 
2227  case PGBENCH_CASE:
2228  /* when true, execute branch */
2229  if (valueTruth(&a1))
2230  return evaluateExpr(st, args->expr, retval);
2231 
2232  /* now args contains next condition or final else expression */
2233  args = args->next;
2234 
2235  /* final else case? */
2236  if (args->next == NULL)
2237  return evaluateExpr(st, args->expr, retval);
2238 
2239  /* no, another when, proceed */
2240  return evalLazyFunc(st, PGBENCH_CASE, args, retval);
2241 
2242  default:
2243  /* internal error, cannot get here */
2244  Assert(0);
2245  break;
2246  }
2247  return false;
2248 }
2249 
2250 /* maximum number of function arguments */
2251 #define MAX_FARGS 16
2252 
2253 /*
2254  * Recursive evaluation of standard functions,
2255  * which do not require lazy evaluation.
2256  */
2257 static bool
2260  PgBenchValue *retval)
2261 {
2262  /* evaluate all function arguments */
2263  int nargs = 0;
2264  PgBenchValue vargs[MAX_FARGS];
2265  PgBenchExprLink *l = args;
2266  bool has_null = false;
2267 
2268  for (nargs = 0; nargs < MAX_FARGS && l != NULL; nargs++, l = l->next)
2269  {
2270  if (!evaluateExpr(st, l->expr, &vargs[nargs]))
2271  return false;
2272  has_null |= vargs[nargs].type == PGBT_NULL;
2273  }
2274 
2275  if (l != NULL)
2276  {
2277  pg_log_error("too many function arguments, maximum is %d", MAX_FARGS);
2278  return false;
2279  }
2280 
2281  /* NULL arguments */
2282  if (has_null && func != PGBENCH_IS && func != PGBENCH_DEBUG)
2283  {
2284  setNullValue(retval);
2285  return true;
2286  }
2287 
2288  /* then evaluate function */
2289  switch (func)
2290  {
2291  /* overloaded operators */
2292  case PGBENCH_ADD:
2293  case PGBENCH_SUB:
2294  case PGBENCH_MUL:
2295  case PGBENCH_DIV:
2296  case PGBENCH_MOD:
2297  case PGBENCH_EQ:
2298  case PGBENCH_NE:
2299  case PGBENCH_LE:
2300  case PGBENCH_LT:
2301  {
2302  PgBenchValue *lval = &vargs[0],
2303  *rval = &vargs[1];
2304 
2305  Assert(nargs == 2);
2306 
2307  /* overloaded type management, double if some double */
2308  if ((lval->type == PGBT_DOUBLE ||
2309  rval->type == PGBT_DOUBLE) && func != PGBENCH_MOD)
2310  {
2311  double ld,
2312  rd;
2313 
2314  if (!coerceToDouble(lval, &ld) ||
2315  !coerceToDouble(rval, &rd))
2316  return false;
2317 
2318  switch (func)
2319  {
2320  case PGBENCH_ADD:
2321  setDoubleValue(retval, ld + rd);
2322  return true;
2323 
2324  case PGBENCH_SUB:
2325  setDoubleValue(retval, ld - rd);
2326  return true;
2327 
2328  case PGBENCH_MUL:
2329  setDoubleValue(retval, ld * rd);
2330  return true;
2331 
2332  case PGBENCH_DIV:
2333  setDoubleValue(retval, ld / rd);
2334  return true;
2335 
2336  case PGBENCH_EQ:
2337  setBoolValue(retval, ld == rd);
2338  return true;
2339 
2340  case PGBENCH_NE:
2341  setBoolValue(retval, ld != rd);
2342  return true;
2343 
2344  case PGBENCH_LE:
2345  setBoolValue(retval, ld <= rd);
2346  return true;
2347 
2348  case PGBENCH_LT:
2349  setBoolValue(retval, ld < rd);
2350  return true;
2351 
2352  default:
2353  /* cannot get here */
2354  Assert(0);
2355  }
2356  }
2357  else /* we have integer operands, or % */
2358  {
2359  int64 li,
2360  ri,
2361  res;
2362 
2363  if (!coerceToInt(lval, &li) ||
2364  !coerceToInt(rval, &ri))
2365  return false;
2366 
2367  switch (func)
2368  {
2369  case PGBENCH_ADD:
2370  if (pg_add_s64_overflow(li, ri, &res))
2371  {
2372  pg_log_error("bigint add out of range");
2373  return false;
2374  }
2375  setIntValue(retval, res);
2376  return true;
2377 
2378  case PGBENCH_SUB:
2379  if (pg_sub_s64_overflow(li, ri, &res))
2380  {
2381  pg_log_error("bigint sub out of range");
2382  return false;
2383  }
2384  setIntValue(retval, res);
2385  return true;
2386 
2387  case PGBENCH_MUL:
2388  if (pg_mul_s64_overflow(li, ri, &res))
2389  {
2390  pg_log_error("bigint mul out of range");
2391  return false;
2392  }
2393  setIntValue(retval, res);
2394  return true;
2395 
2396  case PGBENCH_EQ:
2397  setBoolValue(retval, li == ri);
2398  return true;
2399 
2400  case PGBENCH_NE:
2401  setBoolValue(retval, li != ri);
2402  return true;
2403 
2404  case PGBENCH_LE:
2405  setBoolValue(retval, li <= ri);
2406  return true;
2407 
2408  case PGBENCH_LT:
2409  setBoolValue(retval, li < ri);
2410  return true;
2411 
2412  case PGBENCH_DIV:
2413  case PGBENCH_MOD:
2414  if (ri == 0)
2415  {
2416  pg_log_error("division by zero");
2417  return false;
2418  }
2419  /* special handling of -1 divisor */
2420  if (ri == -1)
2421  {
2422  if (func == PGBENCH_DIV)
2423  {
2424  /* overflow check (needed for INT64_MIN) */
2425  if (li == PG_INT64_MIN)
2426  {
2427  pg_log_error("bigint div out of range");
2428  return false;
2429  }
2430  else
2431  setIntValue(retval, -li);
2432  }
2433  else
2434  setIntValue(retval, 0);
2435  return true;
2436  }
2437  /* else divisor is not -1 */
2438  if (func == PGBENCH_DIV)
2439  setIntValue(retval, li / ri);
2440  else /* func == PGBENCH_MOD */
2441  setIntValue(retval, li % ri);
2442 
2443  return true;
2444 
2445  default:
2446  /* cannot get here */
2447  Assert(0);
2448  }
2449  }
2450 
2451  Assert(0);
2452  return false; /* NOTREACHED */
2453  }
2454 
2455  /* integer bitwise operators */
2456  case PGBENCH_BITAND:
2457  case PGBENCH_BITOR:
2458  case PGBENCH_BITXOR:
2459  case PGBENCH_LSHIFT:
2460  case PGBENCH_RSHIFT:
2461  {
2462  int64 li,
2463  ri;
2464 
2465  if (!coerceToInt(&vargs[0], &li) || !coerceToInt(&vargs[1], &ri))
2466  return false;
2467 
2468  if (func == PGBENCH_BITAND)
2469  setIntValue(retval, li & ri);
2470  else if (func == PGBENCH_BITOR)
2471  setIntValue(retval, li | ri);
2472  else if (func == PGBENCH_BITXOR)
2473  setIntValue(retval, li ^ ri);
2474  else if (func == PGBENCH_LSHIFT)
2475  setIntValue(retval, li << ri);
2476  else if (func == PGBENCH_RSHIFT)
2477  setIntValue(retval, li >> ri);
2478  else /* cannot get here */
2479  Assert(0);
2480 
2481  return true;
2482  }
2483 
2484  /* logical operators */
2485  case PGBENCH_NOT:
2486  {
2487  bool b;
2488 
2489  if (!coerceToBool(&vargs[0], &b))
2490  return false;
2491 
2492  setBoolValue(retval, !b);
2493  return true;
2494  }
2495 
2496  /* no arguments */
2497  case PGBENCH_PI:
2498  setDoubleValue(retval, M_PI);
2499  return true;
2500 
2501  /* 1 overloaded argument */
2502  case PGBENCH_ABS:
2503  {
2504  PgBenchValue *varg = &vargs[0];
2505 
2506  Assert(nargs == 1);
2507 
2508  if (varg->type == PGBT_INT)
2509  {
2510  int64 i = varg->u.ival;
2511 
2512  setIntValue(retval, i < 0 ? -i : i);
2513  }
2514  else
2515  {
2516  double d = varg->u.dval;
2517 
2518  Assert(varg->type == PGBT_DOUBLE);
2519  setDoubleValue(retval, d < 0.0 ? -d : d);
2520  }
2521 
2522  return true;
2523  }
2524 
2525  case PGBENCH_DEBUG:
2526  {
2527  PgBenchValue *varg = &vargs[0];
2528 
2529  Assert(nargs == 1);
2530 
2531  fprintf(stderr, "debug(script=%d,command=%d): ",
2532  st->use_file, st->command + 1);
2533 
2534  if (varg->type == PGBT_NULL)
2535  fprintf(stderr, "null\n");
2536  else if (varg->type == PGBT_BOOLEAN)
2537  fprintf(stderr, "boolean %s\n", varg->u.bval ? "true" : "false");
2538  else if (varg->type == PGBT_INT)
2539  fprintf(stderr, "int " INT64_FORMAT "\n", varg->u.ival);
2540  else if (varg->type == PGBT_DOUBLE)
2541  fprintf(stderr, "double %.*g\n", DBL_DIG, varg->u.dval);
2542  else /* internal error, unexpected type */
2543  Assert(0);
2544 
2545  *retval = *varg;
2546 
2547  return true;
2548  }
2549 
2550  /* 1 double argument */
2551  case PGBENCH_DOUBLE:
2552  case PGBENCH_SQRT:
2553  case PGBENCH_LN:
2554  case PGBENCH_EXP:
2555  {
2556  double dval;
2557 
2558  Assert(nargs == 1);
2559 
2560  if (!coerceToDouble(&vargs[0], &dval))
2561  return false;
2562 
2563  if (func == PGBENCH_SQRT)
2564  dval = sqrt(dval);
2565  else if (func == PGBENCH_LN)
2566  dval = log(dval);
2567  else if (func == PGBENCH_EXP)
2568  dval = exp(dval);
2569  /* else is cast: do nothing */
2570 
2571  setDoubleValue(retval, dval);
2572  return true;
2573  }
2574 
2575  /* 1 int argument */
2576  case PGBENCH_INT:
2577  {
2578  int64 ival;
2579 
2580  Assert(nargs == 1);
2581 
2582  if (!coerceToInt(&vargs[0], &ival))
2583  return false;
2584 
2585  setIntValue(retval, ival);
2586  return true;
2587  }
2588 
2589  /* variable number of arguments */
2590  case PGBENCH_LEAST:
2591  case PGBENCH_GREATEST:
2592  {
2593  bool havedouble;
2594  int i;
2595 
2596  Assert(nargs >= 1);
2597 
2598  /* need double result if any input is double */
2599  havedouble = false;
2600  for (i = 0; i < nargs; i++)
2601  {
2602  if (vargs[i].type == PGBT_DOUBLE)
2603  {
2604  havedouble = true;
2605  break;
2606  }
2607  }
2608  if (havedouble)
2609  {
2610  double extremum;
2611 
2612  if (!coerceToDouble(&vargs[0], &extremum))
2613  return false;
2614  for (i = 1; i < nargs; i++)
2615  {
2616  double dval;
2617 
2618  if (!coerceToDouble(&vargs[i], &dval))
2619  return false;
2620  if (func == PGBENCH_LEAST)
2621  extremum = Min(extremum, dval);
2622  else
2623  extremum = Max(extremum, dval);
2624  }
2625  setDoubleValue(retval, extremum);
2626  }
2627  else
2628  {
2629  int64 extremum;
2630 
2631  if (!coerceToInt(&vargs[0], &extremum))
2632  return false;
2633  for (i = 1; i < nargs; i++)
2634  {
2635  int64 ival;
2636 
2637  if (!coerceToInt(&vargs[i], &ival))
2638  return false;
2639  if (func == PGBENCH_LEAST)
2640  extremum = Min(extremum, ival);
2641  else
2642  extremum = Max(extremum, ival);
2643  }
2644  setIntValue(retval, extremum);
2645  }
2646  return true;
2647  }
2648 
2649  /* random functions */
2650  case PGBENCH_RANDOM:
2654  {
2655  int64 imin,
2656  imax,
2657  delta;
2658 
2659  Assert(nargs >= 2);
2660 
2661  if (!coerceToInt(&vargs[0], &imin) ||
2662  !coerceToInt(&vargs[1], &imax))
2663  return false;
2664 
2665  /* check random range */
2666  if (unlikely(imin > imax))
2667  {
2668  pg_log_error("empty range given to random");
2669  return false;
2670  }
2671  else if (unlikely(pg_sub_s64_overflow(imax, imin, &delta) ||
2672  pg_add_s64_overflow(delta, 1, &delta)))
2673  {
2674  /* prevent int overflows in random functions */
2675  pg_log_error("random range is too large");
2676  return false;
2677  }
2678 
2679  if (func == PGBENCH_RANDOM)
2680  {
2681  Assert(nargs == 2);
2682  setIntValue(retval, getrand(&st->cs_func_rs, imin, imax));
2683  }
2684  else /* gaussian & exponential */
2685  {
2686  double param;
2687 
2688  Assert(nargs == 3);
2689 
2690  if (!coerceToDouble(&vargs[2], &param))
2691  return false;
2692 
2693  if (func == PGBENCH_RANDOM_GAUSSIAN)
2694  {
2695  if (param < MIN_GAUSSIAN_PARAM)
2696  {
2697  pg_log_error("gaussian parameter must be at least %f (not %f)",
2698  MIN_GAUSSIAN_PARAM, param);
2699  return false;
2700  }
2701 
2702  setIntValue(retval,
2704  imin, imax, param));
2705  }
2706  else if (func == PGBENCH_RANDOM_ZIPFIAN)
2707  {
2708  if (param < MIN_ZIPFIAN_PARAM || param > MAX_ZIPFIAN_PARAM)
2709  {
2710  pg_log_error("zipfian parameter must be in range [%.3f, %.0f] (not %f)",
2712  return false;
2713  }
2714 
2715  setIntValue(retval,
2716  getZipfianRand(&st->cs_func_rs, imin, imax, param));
2717  }
2718  else /* exponential */
2719  {
2720  if (param <= 0.0)
2721  {
2722  pg_log_error("exponential parameter must be greater than zero (not %f)",
2723  param);
2724  return false;
2725  }
2726 
2727  setIntValue(retval,
2729  imin, imax, param));
2730  }
2731  }
2732 
2733  return true;
2734  }
2735 
2736  case PGBENCH_POW:
2737  {
2738  PgBenchValue *lval = &vargs[0];
2739  PgBenchValue *rval = &vargs[1];
2740  double ld,
2741  rd;
2742 
2743  Assert(nargs == 2);
2744 
2745  if (!coerceToDouble(lval, &ld) ||
2746  !coerceToDouble(rval, &rd))
2747  return false;
2748 
2749  setDoubleValue(retval, pow(ld, rd));
2750 
2751  return true;
2752  }
2753 
2754  case PGBENCH_IS:
2755  {
2756  Assert(nargs == 2);
2757 
2758  /*
2759  * note: this simple implementation is more permissive than
2760  * SQL
2761  */
2762  setBoolValue(retval,
2763  vargs[0].type == vargs[1].type &&
2764  vargs[0].u.bval == vargs[1].u.bval);
2765  return true;
2766  }
2767 
2768  /* hashing */
2769  case PGBENCH_HASH_FNV1A:
2770  case PGBENCH_HASH_MURMUR2:
2771  {
2772  int64 val,
2773  seed;
2774 
2775  Assert(nargs == 2);
2776 
2777  if (!coerceToInt(&vargs[0], &val) ||
2778  !coerceToInt(&vargs[1], &seed))
2779  return false;
2780 
2781  if (func == PGBENCH_HASH_MURMUR2)
2782  setIntValue(retval, getHashMurmur2(val, seed));
2783  else if (func == PGBENCH_HASH_FNV1A)
2784  setIntValue(retval, getHashFnv1a(val, seed));
2785  else
2786  /* cannot get here */
2787  Assert(0);
2788 
2789  return true;
2790  }
2791 
2792  case PGBENCH_PERMUTE:
2793  {
2794  int64 val,
2795  size,
2796  seed;
2797 
2798  Assert(nargs == 3);
2799 
2800  if (!coerceToInt(&vargs[0], &val) ||
2801  !coerceToInt(&vargs[1], &size) ||
2802  !coerceToInt(&vargs[2], &seed))
2803  return false;
2804 
2805  if (size <= 0)
2806  {
2807  pg_log_error("permute size parameter must be greater than zero");
2808  return false;
2809  }
2810 
2811  setIntValue(retval, permute(val, size, seed));
2812  return true;
2813  }
2814 
2815  default:
2816  /* cannot get here */
2817  Assert(0);
2818  /* dead code to avoid a compiler warning */
2819  return false;
2820  }
2821 }
2822 
2823 /* evaluate some function */
2824 static bool
2825 evalFunc(CState *st,
2827 {
2828  if (isLazyFunc(func))
2829  return evalLazyFunc(st, func, args, retval);
2830  else
2831  return evalStandardFunc(st, func, args, retval);
2832 }
2833 
2834 /*
2835  * Recursive evaluation of an expression in a pgbench script
2836  * using the current state of variables.
2837  * Returns whether the evaluation was ok,
2838  * the value itself is returned through the retval pointer.
2839  */
2840 static bool
2841 evaluateExpr(CState *st, PgBenchExpr *expr, PgBenchValue *retval)
2842 {
2843  switch (expr->etype)
2844  {
2845  case ENODE_CONSTANT:
2846  {
2847  *retval = expr->u.constant;
2848  return true;
2849  }
2850 
2851  case ENODE_VARIABLE:
2852  {
2853  Variable *var;
2854 
2855  if ((var = lookupVariable(&st->variables, expr->u.variable.varname)) == NULL)
2856  {
2857  pg_log_error("undefined variable \"%s\"", expr->u.variable.varname);
2858  return false;
2859  }
2860 
2861  if (!makeVariableValue(var))
2862  return false;
2863 
2864  *retval = var->value;
2865  return true;
2866  }
2867 
2868  case ENODE_FUNCTION:
2869  return evalFunc(st,
2870  expr->u.function.function,
2871  expr->u.function.args,
2872  retval);
2873 
2874  default:
2875  /* internal error which should never occur */
2876  pg_fatal("unexpected enode type in evaluation: %d", expr->etype);
2877  }
2878 }
2879 
2880 /*
2881  * Convert command name to meta-command enum identifier
2882  */
2884 getMetaCommand(const char *cmd)
2885 {
2886  MetaCommand mc;
2887 
2888  if (cmd == NULL)
2889  mc = META_NONE;
2890  else if (pg_strcasecmp(cmd, "set") == 0)
2891  mc = META_SET;
2892  else if (pg_strcasecmp(cmd, "setshell") == 0)
2893  mc = META_SETSHELL;
2894  else if (pg_strcasecmp(cmd, "shell") == 0)
2895  mc = META_SHELL;
2896  else if (pg_strcasecmp(cmd, "sleep") == 0)
2897  mc = META_SLEEP;
2898  else if (pg_strcasecmp(cmd, "if") == 0)
2899  mc = META_IF;
2900  else if (pg_strcasecmp(cmd, "elif") == 0)
2901  mc = META_ELIF;
2902  else if (pg_strcasecmp(cmd, "else") == 0)
2903  mc = META_ELSE;
2904  else if (pg_strcasecmp(cmd, "endif") == 0)
2905  mc = META_ENDIF;
2906  else if (pg_strcasecmp(cmd, "gset") == 0)
2907  mc = META_GSET;
2908  else if (pg_strcasecmp(cmd, "aset") == 0)
2909  mc = META_ASET;
2910  else if (pg_strcasecmp(cmd, "startpipeline") == 0)
2911  mc = META_STARTPIPELINE;
2912  else if (pg_strcasecmp(cmd, "endpipeline") == 0)
2913  mc = META_ENDPIPELINE;
2914  else
2915  mc = META_NONE;
2916  return mc;
2917 }
2918 
2919 /*
2920  * Run a shell command. The result is assigned to the variable if not NULL.
2921  * Return true if succeeded, or false on error.
2922  */
2923 static bool
2924 runShellCommand(Variables *variables, char *variable, char **argv, int argc)
2925 {
2926  char command[SHELL_COMMAND_SIZE];
2927  int i,
2928  len = 0;
2929  FILE *fp;
2930  char res[64];
2931  char *endptr;
2932  int retval;
2933 
2934  /*----------
2935  * Join arguments with whitespace separators. Arguments starting with
2936  * exactly one colon are treated as variables:
2937  * name - append a string "name"
2938  * :var - append a variable named 'var'
2939  * ::name - append a string ":name"
2940  *----------
2941  */
2942  for (i = 0; i < argc; i++)
2943  {
2944  char *arg;
2945  int arglen;
2946 
2947  if (argv[i][0] != ':')
2948  {
2949  arg = argv[i]; /* a string literal */
2950  }
2951  else if (argv[i][1] == ':')
2952  {
2953  arg = argv[i] + 1; /* a string literal starting with colons */
2954  }
2955  else if ((arg = getVariable(variables, argv[i] + 1)) == NULL)
2956  {
2957  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[i]);
2958  return false;
2959  }
2960 
2961  arglen = strlen(arg);
2962  if (len + arglen + (i > 0 ? 1 : 0) >= SHELL_COMMAND_SIZE - 1)
2963  {
2964  pg_log_error("%s: shell command is too long", argv[0]);
2965  return false;
2966  }
2967 
2968  if (i > 0)
2969  command[len++] = ' ';
2970  memcpy(command + len, arg, arglen);
2971  len += arglen;
2972  }
2973 
2974  command[len] = '\0';
2975 
2976  fflush(NULL); /* needed before either system() or popen() */
2977 
2978  /* Fast path for non-assignment case */
2979  if (variable == NULL)
2980  {
2981  if (system(command))
2982  {
2983  if (!timer_exceeded)
2984  pg_log_error("%s: could not launch shell command", argv[0]);
2985  return false;
2986  }
2987  return true;
2988  }
2989 
2990  /* Execute the command with pipe and read the standard output. */
2991  if ((fp = popen(command, "r")) == NULL)
2992  {
2993  pg_log_error("%s: could not launch shell command", argv[0]);
2994  return false;
2995  }
2996  if (fgets(res, sizeof(res), fp) == NULL)
2997  {
2998  if (!timer_exceeded)
2999  pg_log_error("%s: could not read result of shell command", argv[0]);
3000  (void) pclose(fp);
3001  return false;
3002  }
3003  if (pclose(fp) < 0)
3004  {
3005  pg_log_error("%s: could not run shell command: %m", argv[0]);
3006  return false;
3007  }
3008 
3009  /* Check whether the result is an integer and assign it to the variable */
3010  retval = (int) strtol(res, &endptr, 10);
3011  while (*endptr != '\0' && isspace((unsigned char) *endptr))
3012  endptr++;
3013  if (*res == '\0' || *endptr != '\0')
3014  {
3015  pg_log_error("%s: shell command must return an integer (not \"%s\")", argv[0], res);
3016  return false;
3017  }
3018  if (!putVariableInt(variables, "setshell", variable, retval))
3019  return false;
3020 
3021  pg_log_debug("%s: shell parameter name: \"%s\", value: \"%s\"", argv[0], argv[1], res);
3022 
3023  return true;
3024 }
3026 #define MAX_PREPARE_NAME 32
3027 static void
3028 preparedStatementName(char *buffer, int file, int state)
3029 {
3030  sprintf(buffer, "P%d_%d", file, state);
3031 }
3032 
3033 /*
3034  * Report the abortion of the client when processing SQL commands.
3035  */
3036 static void
3037 commandFailed(CState *st, const char *cmd, const char *message)
3038 {
3039  pg_log_error("client %d aborted in command %d (%s) of script %d; %s",
3040  st->id, st->command, cmd, st->use_file, message);
3041 }
3042 
3043 /*
3044  * Report the error in the command while the script is executing.
3045  */
3046 static void
3047 commandError(CState *st, const char *message)
3048 {
3050  pg_log_info("client %d got an error in command %d (SQL) of script %d; %s",
3051  st->id, st->command, st->use_file, message);
3052 }
3053 
3054 /* return a script number with a weighted choice. */
3055 static int
3056 chooseScript(TState *thread)
3057 {
3058  int i = 0;
3059  int64 w;
3060 
3061  if (num_scripts == 1)
3062  return 0;
3063 
3064  w = getrand(&thread->ts_choose_rs, 0, total_weight - 1);
3065  do
3066  {
3067  w -= sql_script[i++].weight;
3068  } while (w >= 0);
3069 
3070  return i - 1;
3071 }
3072 
3073 /* Send a SQL command, using the chosen querymode */
3074 static bool
3075 sendCommand(CState *st, Command *command)
3076 {
3077  int r;
3078 
3079  if (querymode == QUERY_SIMPLE)
3080  {
3081  char *sql;
3082 
3083  sql = pg_strdup(command->argv[0]);
3084  sql = assignVariables(&st->variables, sql);
3085 
3086  pg_log_debug("client %d sending %s", st->id, sql);
3087  r = PQsendQuery(st->con, sql);
3088  free(sql);
3089  }
3090  else if (querymode == QUERY_EXTENDED)
3091  {
3092  const char *sql = command->argv[0];
3093  const char *params[MAX_ARGS];
3094 
3095  getQueryParams(&st->variables, command, params);
3096 
3097  pg_log_debug("client %d sending %s", st->id, sql);
3098  r = PQsendQueryParams(st->con, sql, command->argc - 1,
3099  NULL, params, NULL, NULL, 0);
3100  }
3101  else if (querymode == QUERY_PREPARED)
3102  {
3103  char name[MAX_PREPARE_NAME];
3104  const char *params[MAX_ARGS];
3105 
3106  if (!st->prepared[st->use_file])
3107  {
3108  int j;
3109  Command **commands = sql_script[st->use_file].commands;
3110 
3111  for (j = 0; commands[j] != NULL; j++)
3112  {
3113  PGresult *res;
3114 
3115  if (commands[j]->type != SQL_COMMAND)
3116  continue;
3118  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3119  {
3120  res = PQprepare(st->con, name,
3121  commands[j]->argv[0], commands[j]->argc - 1, NULL);
3123  pg_log_error("%s", PQerrorMessage(st->con));
3124  PQclear(res);
3125  }
3126  else
3127  {
3128  /*
3129  * In pipeline mode, we use asynchronous functions. If a
3130  * server-side error occurs, it will be processed later
3131  * among the other results.
3132  */
3133  if (!PQsendPrepare(st->con, name,
3134  commands[j]->argv[0], commands[j]->argc - 1, NULL))
3135  pg_log_error("%s", PQerrorMessage(st->con));
3136  }
3137  }
3138  st->prepared[st->use_file] = true;
3139  }
3140 
3141  getQueryParams(&st->variables, command, params);
3143 
3144  pg_log_debug("client %d sending %s", st->id, name);
3145  r = PQsendQueryPrepared(st->con, name, command->argc - 1,
3146  params, NULL, NULL, 0);
3147  }
3148  else /* unknown sql mode */
3149  r = 0;
3150 
3151  if (r == 0)
3152  {
3153  pg_log_debug("client %d could not send %s", st->id, command->argv[0]);
3154  return false;
3155  }
3156  else
3157  return true;
3158 }
3159 
3160 /*
3161  * Get the error status from the error code.
3162  */
3163 static EStatus
3164 getSQLErrorStatus(const char *sqlState)
3165 {
3166  if (sqlState != NULL)
3167  {
3168  if (strcmp(sqlState, ERRCODE_T_R_SERIALIZATION_FAILURE) == 0)
3170  else if (strcmp(sqlState, ERRCODE_T_R_DEADLOCK_DETECTED) == 0)
3171  return ESTATUS_DEADLOCK_ERROR;
3172  }
3173 
3174  return ESTATUS_OTHER_SQL_ERROR;
3175 }
3176 
3177 /*
3178  * Returns true if this type of error can be retried.
3179  */
3180 static bool
3181 canRetryError(EStatus estatus)
3182 {
3183  return (estatus == ESTATUS_SERIALIZATION_ERROR ||
3184  estatus == ESTATUS_DEADLOCK_ERROR);
3185 }
3186 
3187 /*
3188  * Process query response from the backend.
3189  *
3190  * If varprefix is not NULL, it's the variable name prefix where to store
3191  * the results of the *last* command (META_GSET) or *all* commands
3192  * (META_ASET).
3193  *
3194  * Returns true if everything is A-OK, false if any error occurs.
3195  */
3196 static bool
3197 readCommandResponse(CState *st, MetaCommand meta, char *varprefix)
3198 {
3199  PGresult *res;
3200  PGresult *next_res;
3201  int qrynum = 0;
3202 
3203  /*
3204  * varprefix should be set only with \gset or \aset, and \endpipeline and
3205  * SQL commands do not need it.
3206  */
3207  Assert((meta == META_NONE && varprefix == NULL) ||
3208  ((meta == META_ENDPIPELINE) && varprefix == NULL) ||
3209  ((meta == META_GSET || meta == META_ASET) && varprefix != NULL));
3210 
3211  res = PQgetResult(st->con);
3212 
3213  while (res != NULL)
3214  {
3215  bool is_last;
3216 
3217  /* peek at the next result to know whether the current is last */
3218  next_res = PQgetResult(st->con);
3219  is_last = (next_res == NULL);
3220 
3221  switch (PQresultStatus(res))
3222  {
3223  case PGRES_COMMAND_OK: /* non-SELECT commands */
3224  case PGRES_EMPTY_QUERY: /* may be used for testing no-op overhead */
3225  if (is_last && meta == META_GSET)
3226  {
3227  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3228  st->id, st->use_file, st->command, qrynum, 0);
3230  goto error;
3231  }
3232  break;
3233 
3234  case PGRES_TUPLES_OK:
3235  if ((is_last && meta == META_GSET) || meta == META_ASET)
3236  {
3237  int ntuples = PQntuples(res);
3238 
3239  if (meta == META_GSET && ntuples != 1)
3240  {
3241  /* under \gset, report the error */
3242  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3243  st->id, st->use_file, st->command, qrynum, PQntuples(res));
3245  goto error;
3246  }
3247  else if (meta == META_ASET && ntuples <= 0)
3248  {
3249  /* coldly skip empty result under \aset */
3250  break;
3251  }
3252 
3253  /* store results into variables */
3254  for (int fld = 0; fld < PQnfields(res); fld++)
3255  {
3256  char *varname = PQfname(res, fld);
3257 
3258  /* allocate varname only if necessary, freed below */
3259  if (*varprefix != '\0')
3260  varname = psprintf("%s%s", varprefix, varname);
3261 
3262  /* store last row result as a string */
3263  if (!putVariable(&st->variables, meta == META_ASET ? "aset" : "gset", varname,
3264  PQgetvalue(res, ntuples - 1, fld)))
3265  {
3266  /* internal error */
3267  pg_log_error("client %d script %d command %d query %d: error storing into variable %s",
3268  st->id, st->use_file, st->command, qrynum, varname);
3270  goto error;
3271  }
3272 
3273  if (*varprefix != '\0')
3274  pg_free(varname);
3275  }
3276  }
3277  /* otherwise the result is simply thrown away by PQclear below */
3278  break;
3279 
3280  case PGRES_PIPELINE_SYNC:
3281  pg_log_debug("client %d pipeline ending", st->id);
3282  if (PQexitPipelineMode(st->con) != 1)
3283  pg_log_error("client %d failed to exit pipeline mode: %s", st->id,
3284  PQerrorMessage(st->con));
3285  break;
3286 
3287  case PGRES_NONFATAL_ERROR:
3288  case PGRES_FATAL_ERROR:
3290  PG_DIAG_SQLSTATE));
3291  if (canRetryError(st->estatus))
3292  {
3293  if (verbose_errors)
3294  commandError(st, PQerrorMessage(st->con));
3295  goto error;
3296  }
3297  /* fall through */
3298 
3299  default:
3300  /* anything else is unexpected */
3301  pg_log_error("client %d script %d aborted in command %d query %d: %s",
3302  st->id, st->use_file, st->command, qrynum,
3303  PQerrorMessage(st->con));
3304  goto error;
3305  }
3306 
3307  PQclear(res);
3308  qrynum++;
3309  res = next_res;
3310  }
3311 
3312  if (qrynum == 0)
3313  {
3314  pg_log_error("client %d command %d: no results", st->id, st->command);
3315  return false;
3316  }
3317 
3318  return true;
3319 
3320 error:
3321  PQclear(res);
3322  PQclear(next_res);
3323  do
3324  {
3325  res = PQgetResult(st->con);
3326  PQclear(res);
3327  } while (res);
3328 
3329  return false;
3330 }
3331 
3332 /*
3333  * Parse the argument to a \sleep command, and return the requested amount
3334  * of delay, in microseconds. Returns true on success, false on error.
3335  */
3336 static bool
3337 evaluateSleep(Variables *variables, int argc, char **argv, int *usecs)
3338 {
3339  char *var;
3340  int usec;
3341 
3342  if (*argv[1] == ':')
3343  {
3344  if ((var = getVariable(variables, argv[1] + 1)) == NULL)
3345  {
3346  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[1] + 1);
3347  return false;
3348  }
3349 
3350  usec = atoi(var);
3351 
3352  /* Raise an error if the value of a variable is not a number */
3353  if (usec == 0 && !isdigit((unsigned char) *var))
3354  {
3355  pg_log_error("%s: invalid sleep time \"%s\" for variable \"%s\"",
3356  argv[0], var, argv[1] + 1);
3357  return false;
3358  }
3359  }
3360  else
3361  usec = atoi(argv[1]);
3362 
3363  if (argc > 2)
3364  {
3365  if (pg_strcasecmp(argv[2], "ms") == 0)
3366  usec *= 1000;
3367  else if (pg_strcasecmp(argv[2], "s") == 0)
3368  usec *= 1000000;
3369  }
3370  else
3371  usec *= 1000000;
3372 
3373  *usecs = usec;
3374  return true;
3375 }
3376 
3377 
3378 /*
3379  * Returns true if the error can be retried.
3380  */
3381 static bool
3383 {
3385 
3386  /* We can only retry serialization or deadlock errors. */
3387  if (!canRetryError(st->estatus))
3388  return false;
3389 
3390  /*
3391  * We must have at least one option to limit the retrying of transactions
3392  * that got an error.
3393  */
3395 
3396  /*
3397  * We cannot retry the error if we have reached the maximum number of
3398  * tries.
3399  */
3400  if (max_tries && st->tries >= max_tries)
3401  return false;
3402 
3403  /*
3404  * We cannot retry the error if we spent too much time on this
3405  * transaction.
3406  */
3407  if (latency_limit)
3408  {
3410  if (*now - st->txn_scheduled > latency_limit)
3411  return false;
3412  }
3413 
3414  /*
3415  * We cannot retry the error if the benchmark duration is over.
3416  */
3417  if (timer_exceeded)
3418  return false;
3419 
3420  /* OK */
3421  return true;
3422 }
3423 
3424 /*
3425  * Read results and discard it until a sync point.
3426  */
3427 static int
3429 {
3430  /* send a sync */
3431  if (!PQpipelineSync(st->con))
3432  {
3433  pg_log_error("client %d aborted: failed to send a pipeline sync",
3434  st->id);
3435  return 0;
3436  }
3437 
3438  /* receive PGRES_PIPELINE_SYNC and null following it */
3439  for (;;)
3440  {
3441  PGresult *res = PQgetResult(st->con);
3442 
3444  {
3445  PQclear(res);
3446  res = PQgetResult(st->con);
3447  Assert(res == NULL);
3448  break;
3449  }
3450  PQclear(res);
3451  }
3452 
3453  /* exit pipeline */
3454  if (PQexitPipelineMode(st->con) != 1)
3455  {
3456  pg_log_error("client %d aborted: failed to exit pipeline mode for rolling back the failed transaction",
3457  st->id);
3458  return 0;
3459  }
3460  return 1;
3461 }
3462 
3463 /*
3464  * Get the transaction status at the end of a command especially for
3465  * checking if we are in a (failed) transaction block.
3466  */
3467 static TStatus
3469 {
3470  PGTransactionStatusType tx_status;
3471 
3472  tx_status = PQtransactionStatus(con);
3473  switch (tx_status)
3474  {
3475  case PQTRANS_IDLE:
3476  return TSTATUS_IDLE;
3477  case PQTRANS_INTRANS:
3478  case PQTRANS_INERROR:
3479  return TSTATUS_IN_BLOCK;
3480  case PQTRANS_UNKNOWN:
3481  /* PQTRANS_UNKNOWN is expected given a broken connection */
3482  if (PQstatus(con) == CONNECTION_BAD)
3483  return TSTATUS_CONN_ERROR;
3484  /* fall through */
3485  case PQTRANS_ACTIVE:
3486  default:
3487 
3488  /*
3489  * We cannot find out whether we are in a transaction block or
3490  * not. Internal error which should never occur.
3491  */
3492  pg_log_error("unexpected transaction status %d", tx_status);
3493  return TSTATUS_OTHER_ERROR;
3494  }
3495 
3496  /* not reached */
3497  Assert(false);
3498  return TSTATUS_OTHER_ERROR;
3499 }
3500 
3501 /*
3502  * Print verbose messages of an error
3503  */
3504 static void
3505 printVerboseErrorMessages(CState *st, pg_time_usec_t *now, bool is_retry)
3506 {
3507  static PQExpBuffer buf = NULL;
3508 
3509  if (buf == NULL)
3510  buf = createPQExpBuffer();
3511  else
3513 
3514  printfPQExpBuffer(buf, "client %d ", st->id);
3515  appendPQExpBufferStr(buf, (is_retry ?
3516  "repeats the transaction after the error" :
3517  "ends the failed transaction"));
3518  appendPQExpBuffer(buf, " (try %u", st->tries);
3519 
3520  /* Print max_tries if it is not unlimitted. */
3521  if (max_tries)
3522  appendPQExpBuffer(buf, "/%u", max_tries);
3523 
3524  /*
3525  * If the latency limit is used, print a percentage of the current
3526  * transaction latency from the latency limit.
3527  */
3528  if (latency_limit)
3529  {
3531  appendPQExpBuffer(buf, ", %.3f%% of the maximum time of tries was used",
3532  (100.0 * (*now - st->txn_scheduled) / latency_limit));
3533  }
3534  appendPQExpBufferStr(buf, ")\n");
3535 
3536  pg_log_info("%s", buf->data);
3537 }
3538 
3539 /*
3540  * Advance the state machine of a connection.
3541  */
3542 static void
3543 advanceConnectionState(TState *thread, CState *st, StatsData *agg)
3544 {
3545 
3546  /*
3547  * gettimeofday() isn't free, so we get the current timestamp lazily the
3548  * first time it's needed, and reuse the same value throughout this
3549  * function after that. This also ensures that e.g. the calculated
3550  * latency reported in the log file and in the totals are the same. Zero
3551  * means "not set yet". Reset "now" when we execute shell commands or
3552  * expressions, which might take a non-negligible amount of time, though.
3553  */
3554  pg_time_usec_t now = 0;
3555 
3556  /*
3557  * Loop in the state machine, until we have to wait for a result from the
3558  * server or have to sleep for throttling or \sleep.
3559  *
3560  * Note: In the switch-statement below, 'break' will loop back here,
3561  * meaning "continue in the state machine". Return is used to return to
3562  * the caller, giving the thread the opportunity to advance another
3563  * client.
3564  */
3565  for (;;)
3566  {
3567  Command *command;
3568 
3569  switch (st->state)
3570  {
3571  /* Select transaction (script) to run. */
3572  case CSTATE_CHOOSE_SCRIPT:
3573  st->use_file = chooseScript(thread);
3575 
3576  /* reset transaction variables to default values */
3577  st->estatus = ESTATUS_NO_ERROR;
3578  st->tries = 1;
3579 
3580  pg_log_debug("client %d executing script \"%s\"",
3581  st->id, sql_script[st->use_file].desc);
3582 
3583  /*
3584  * If time is over, we're done; otherwise, get ready to start
3585  * a new transaction, or to get throttled if that's requested.
3586  */
3589  break;
3590 
3591  /* Start new transaction (script) */
3592  case CSTATE_START_TX:
3594 
3595  /* establish connection if needed, i.e. under --connect */
3596  if (st->con == NULL)
3597  {
3598  pg_time_usec_t start = now;
3599 
3600  if ((st->con = doConnect()) == NULL)
3601  {
3602  /*
3603  * as the bench is already running, we do not abort
3604  * the process
3605  */
3606  pg_log_error("client %d aborted while establishing connection", st->id);
3607  st->state = CSTATE_ABORTED;
3608  break;
3609  }
3610 
3611  /* reset now after connection */
3612  now = pg_time_now();
3613 
3614  thread->conn_duration += now - start;
3615 
3616  /* Reset session-local state */
3617  memset(st->prepared, 0, sizeof(st->prepared));
3618  }
3619 
3620  /*
3621  * It is the first try to run this transaction. Remember the
3622  * random state: maybe it will get an error and we will need
3623  * to run it again.
3624  */
3625  st->random_state = st->cs_func_rs;
3626 
3627  /* record transaction start time */
3628  st->txn_begin = now;
3629 
3630  /*
3631  * When not throttling, this is also the transaction's
3632  * scheduled start time.
3633  */
3634  if (!throttle_delay)
3635  st->txn_scheduled = now;
3636 
3637  /* Begin with the first command */
3639  st->command = 0;
3640  break;
3641 
3642  /*
3643  * Handle throttling once per transaction by sleeping.
3644  */
3646 
3647  /*
3648  * Generate a delay such that the series of delays will
3649  * approximate a Poisson distribution centered on the
3650  * throttle_delay time.
3651  *
3652  * If transactions are too slow or a given wait is shorter
3653  * than a transaction, the next transaction will start right
3654  * away.
3655  */
3656  Assert(throttle_delay > 0);
3657 
3658  thread->throttle_trigger +=
3660  st->txn_scheduled = thread->throttle_trigger;
3661 
3662  /*
3663  * If --latency-limit is used, and this slot is already late
3664  * so that the transaction will miss the latency limit even if
3665  * it completed immediately, skip this time slot and loop to
3666  * reschedule.
3667  */
3668  if (latency_limit)
3669  {
3671 
3672  if (thread->throttle_trigger < now - latency_limit)
3673  {
3674  processXactStats(thread, st, &now, true, agg);
3675 
3676  /*
3677  * Finish client if -T or -t was exceeded.
3678  *
3679  * Stop counting skipped transactions under -T as soon
3680  * as the timer is exceeded. Because otherwise it can
3681  * take a very long time to count all of them
3682  * especially when quite a lot of them happen with
3683  * unrealistically high rate setting in -R, which
3684  * would prevent pgbench from ending immediately.
3685  * Because of this behavior, note that there is no
3686  * guarantee that all skipped transactions are counted
3687  * under -T though there is under -t. This is OK in
3688  * practice because it's very unlikely to happen with
3689  * realistic setting.
3690  */
3691  if (timer_exceeded || (nxacts > 0 && st->cnt >= nxacts))
3692  st->state = CSTATE_FINISHED;
3693 
3694  /* Go back to top of loop with CSTATE_PREPARE_THROTTLE */
3695  break;
3696  }
3697  }
3698 
3699  /*
3700  * stop client if next transaction is beyond pgbench end of
3701  * execution; otherwise, throttle it.
3702  */
3703  st->state = end_time > 0 && st->txn_scheduled > end_time ?
3705  break;
3706 
3707  /*
3708  * Wait until it's time to start next transaction.
3709  */
3710  case CSTATE_THROTTLE:
3712 
3713  if (now < st->txn_scheduled)
3714  return; /* still sleeping, nothing to do here */
3715 
3716  /* done sleeping, but don't start transaction if we're done */
3718  break;
3719 
3720  /*
3721  * Send a command to server (or execute a meta-command)
3722  */
3723  case CSTATE_START_COMMAND:
3724  command = sql_script[st->use_file].commands[st->command];
3725 
3726  /* Transition to script end processing if done */
3727  if (command == NULL)
3728  {
3729  st->state = CSTATE_END_TX;
3730  break;
3731  }
3732 
3733  /* record begin time of next command, and initiate it */
3734  if (report_per_command)
3735  {
3737  st->stmt_begin = now;
3738  }
3739 
3740  /* Execute the command */
3741  if (command->type == SQL_COMMAND)
3742  {
3743  /* disallow \aset and \gset in pipeline mode */
3744  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3745  {
3746  if (command->meta == META_GSET)
3747  {
3748  commandFailed(st, "gset", "\\gset is not allowed in pipeline mode");
3749  st->state = CSTATE_ABORTED;
3750  break;
3751  }
3752  else if (command->meta == META_ASET)
3753  {
3754  commandFailed(st, "aset", "\\aset is not allowed in pipeline mode");
3755  st->state = CSTATE_ABORTED;
3756  break;
3757  }
3758  }
3759 
3760  if (!sendCommand(st, command))
3761  {
3762  commandFailed(st, "SQL", "SQL command send failed");
3763  st->state = CSTATE_ABORTED;
3764  }
3765  else
3766  {
3767  /* Wait for results, unless in pipeline mode */
3768  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3769  st->state = CSTATE_WAIT_RESULT;
3770  else
3771  st->state = CSTATE_END_COMMAND;
3772  }
3773  }
3774  else if (command->type == META_COMMAND)
3775  {
3776  /*-----
3777  * Possible state changes when executing meta commands:
3778  * - on errors CSTATE_ABORTED
3779  * - on sleep CSTATE_SLEEP
3780  * - else CSTATE_END_COMMAND
3781  */
3782  st->state = executeMetaCommand(st, &now);
3783  if (st->state == CSTATE_ABORTED)
3785  }
3786 
3787  /*
3788  * We're now waiting for an SQL command to complete, or
3789  * finished processing a metacommand, or need to sleep, or
3790  * something bad happened.
3791  */
3792  Assert(st->state == CSTATE_WAIT_RESULT ||
3793  st->state == CSTATE_END_COMMAND ||
3794  st->state == CSTATE_SLEEP ||
3795  st->state == CSTATE_ABORTED);
3796  break;
3797 
3798  /*
3799  * non executed conditional branch
3800  */
3801  case CSTATE_SKIP_COMMAND:
3803  /* quickly skip commands until something to do... */
3804  while (true)
3805  {
3806  command = sql_script[st->use_file].commands[st->command];
3807 
3808  /* cannot reach end of script in that state */
3809  Assert(command != NULL);
3810 
3811  /*
3812  * if this is conditional related, update conditional
3813  * state
3814  */
3815  if (command->type == META_COMMAND &&
3816  (command->meta == META_IF ||
3817  command->meta == META_ELIF ||
3818  command->meta == META_ELSE ||
3819  command->meta == META_ENDIF))
3820  {
3821  switch (conditional_stack_peek(st->cstack))
3822  {
3823  case IFSTATE_FALSE:
3824  if (command->meta == META_IF ||
3825  command->meta == META_ELIF)
3826  {
3827  /* we must evaluate the condition */
3829  }
3830  else if (command->meta == META_ELSE)
3831  {
3832  /* we must execute next command */
3836  st->command++;
3837  }
3838  else if (command->meta == META_ENDIF)
3839  {
3842  if (conditional_active(st->cstack))
3844 
3845  /*
3846  * else state remains in
3847  * CSTATE_SKIP_COMMAND
3848  */
3849  st->command++;
3850  }
3851  break;
3852 
3853  case IFSTATE_IGNORED:
3854  case IFSTATE_ELSE_FALSE:
3855  if (command->meta == META_IF)
3857  IFSTATE_IGNORED);
3858  else if (command->meta == META_ENDIF)
3859  {
3862  if (conditional_active(st->cstack))
3864  }
3865  /* could detect "else" & "elif" after "else" */
3866  st->command++;
3867  break;
3868 
3869  case IFSTATE_NONE:
3870  case IFSTATE_TRUE:
3871  case IFSTATE_ELSE_TRUE:
3872  default:
3873 
3874  /*
3875  * inconsistent if inactive, unreachable dead
3876  * code
3877  */
3878  Assert(false);
3879  }
3880  }
3881  else
3882  {
3883  /* skip and consider next */
3884  st->command++;
3885  }
3886 
3887  if (st->state != CSTATE_SKIP_COMMAND)
3888  /* out of quick skip command loop */
3889  break;
3890  }
3891  break;
3892 
3893  /*
3894  * Wait for the current SQL command to complete
3895  */
3896  case CSTATE_WAIT_RESULT:
3897  pg_log_debug("client %d receiving", st->id);
3898 
3899  /*
3900  * Only check for new network data if we processed all data
3901  * fetched prior. Otherwise we end up doing a syscall for each
3902  * individual pipelined query, which has a measurable
3903  * performance impact.
3904  */
3905  if (PQisBusy(st->con) && !PQconsumeInput(st->con))
3906  {
3907  /* there's something wrong */
3908  commandFailed(st, "SQL", "perhaps the backend died while processing");
3909  st->state = CSTATE_ABORTED;
3910  break;
3911  }
3912  if (PQisBusy(st->con))
3913  return; /* don't have the whole result yet */
3914 
3915  /* store or discard the query results */
3916  if (readCommandResponse(st,
3919  {
3920  /*
3921  * outside of pipeline mode: stop reading results.
3922  * pipeline mode: continue reading results until an
3923  * end-of-pipeline response.
3924  */
3925  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
3926  st->state = CSTATE_END_COMMAND;
3927  }
3928  else if (canRetryError(st->estatus))
3929  st->state = CSTATE_ERROR;
3930  else
3931  st->state = CSTATE_ABORTED;
3932  break;
3933 
3934  /*
3935  * Wait until sleep is done. This state is entered after a
3936  * \sleep metacommand. The behavior is similar to
3937  * CSTATE_THROTTLE, but proceeds to CSTATE_START_COMMAND
3938  * instead of CSTATE_START_TX.
3939  */
3940  case CSTATE_SLEEP:
3942  if (now < st->sleep_until)
3943  return; /* still sleeping, nothing to do here */
3944  /* Else done sleeping. */
3945  st->state = CSTATE_END_COMMAND;
3946  break;
3947 
3948  /*
3949  * End of command: record stats and proceed to next command.
3950  */
3951  case CSTATE_END_COMMAND:
3952 
3953  /*
3954  * command completed: accumulate per-command execution times
3955  * in thread-local data structure, if per-command latencies
3956  * are requested.
3957  */
3958  if (report_per_command)
3959  {
3961 
3962  command = sql_script[st->use_file].commands[st->command];
3963  /* XXX could use a mutex here, but we choose not to */
3964  addToSimpleStats(&command->stats,
3966  }
3967 
3968  /* Go ahead with next command, to be executed or skipped */
3969  st->command++;
3970  st->state = conditional_active(st->cstack) ?
3972  break;
3973 
3974  /*
3975  * Clean up after an error.
3976  */
3977  case CSTATE_ERROR:
3978  {
3979  TStatus tstatus;
3980 
3982 
3983  /* Clear the conditional stack */
3985 
3986  /* Read and discard until a sync point in pipeline mode */
3987  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3988  {
3989  if (!discardUntilSync(st))
3990  {
3991  st->state = CSTATE_ABORTED;
3992  break;
3993  }
3994  }
3995 
3996  /*
3997  * Check if we have a (failed) transaction block or not,
3998  * and roll it back if any.
3999  */
4000  tstatus = getTransactionStatus(st->con);
4001  if (tstatus == TSTATUS_IN_BLOCK)
4002  {
4003  /* Try to rollback a (failed) transaction block. */
4004  if (!PQsendQuery(st->con, "ROLLBACK"))
4005  {
4006  pg_log_error("client %d aborted: failed to send sql command for rolling back the failed transaction",
4007  st->id);
4008  st->state = CSTATE_ABORTED;
4009  }
4010  else
4012  }
4013  else if (tstatus == TSTATUS_IDLE)
4014  {
4015  /*
4016  * If time is over, we're done; otherwise, check if we
4017  * can retry the error.
4018  */
4021  }
4022  else
4023  {
4024  if (tstatus == TSTATUS_CONN_ERROR)
4025  pg_log_error("perhaps the backend died while processing");
4026 
4027  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4028  st->state = CSTATE_ABORTED;
4029  }
4030  break;
4031  }
4032 
4033  /*
4034  * Wait for the rollback command to complete
4035  */
4037  {
4038  PGresult *res;
4039 
4040  pg_log_debug("client %d receiving", st->id);
4041  if (!PQconsumeInput(st->con))
4042  {
4043  pg_log_error("client %d aborted while rolling back the transaction after an error; perhaps the backend died while processing",
4044  st->id);
4045  st->state = CSTATE_ABORTED;
4046  break;
4047  }
4048  if (PQisBusy(st->con))
4049  return; /* don't have the whole result yet */
4050 
4051  /*
4052  * Read and discard the query result;
4053  */
4054  res = PQgetResult(st->con);
4055  switch (PQresultStatus(res))
4056  {
4057  case PGRES_COMMAND_OK:
4058  /* OK */
4059  PQclear(res);
4060  /* null must be returned */
4061  res = PQgetResult(st->con);
4062  Assert(res == NULL);
4063 
4064  /*
4065  * If time is over, we're done; otherwise, check
4066  * if we can retry the error.
4067  */
4070  break;
4071  default:
4072  pg_log_error("client %d aborted while rolling back the transaction after an error; %s",
4073  st->id, PQerrorMessage(st->con));
4074  PQclear(res);
4075  st->state = CSTATE_ABORTED;
4076  break;
4077  }
4078  break;
4079  }
4080 
4081  /*
4082  * Retry the transaction after an error.
4083  */
4084  case CSTATE_RETRY:
4085  command = sql_script[st->use_file].commands[st->command];
4086 
4087  /*
4088  * Inform that the transaction will be retried after the
4089  * error.
4090  */
4091  if (verbose_errors)
4092  printVerboseErrorMessages(st, &now, true);
4093 
4094  /* Count tries and retries */
4095  st->tries++;
4096  command->retries++;
4097 
4098  /*
4099  * Reset the random state as they were at the beginning of the
4100  * transaction.
4101  */
4102  st->cs_func_rs = st->random_state;
4103 
4104  /* Process the first transaction command. */
4105  st->command = 0;
4106  st->estatus = ESTATUS_NO_ERROR;
4108  break;
4109 
4110  /*
4111  * Record a failed transaction.
4112  */
4113  case CSTATE_FAILURE:
4114  command = sql_script[st->use_file].commands[st->command];
4115 
4116  /* Accumulate the failure. */
4117  command->failures++;
4118 
4119  /*
4120  * Inform that the failed transaction will not be retried.
4121  */
4122  if (verbose_errors)
4123  printVerboseErrorMessages(st, &now, false);
4124 
4125  /* End the failed transaction. */
4126  st->state = CSTATE_END_TX;
4127  break;
4128 
4129  /*
4130  * End of transaction (end of script, really).
4131  */
4132  case CSTATE_END_TX:
4133  {
4134  TStatus tstatus;
4135 
4136  /* transaction finished: calculate latency and do log */
4137  processXactStats(thread, st, &now, false, agg);
4138 
4139  /*
4140  * missing \endif... cannot happen if CheckConditional was
4141  * okay
4142  */
4144 
4145  /*
4146  * We must complete all the transaction blocks that were
4147  * started in this script.
4148  */
4149  tstatus = getTransactionStatus(st->con);
4150  if (tstatus == TSTATUS_IN_BLOCK)
4151  {
4152  pg_log_error("client %d aborted: end of script reached without completing the last transaction",
4153  st->id);
4154  st->state = CSTATE_ABORTED;
4155  break;
4156  }
4157  else if (tstatus != TSTATUS_IDLE)
4158  {
4159  if (tstatus == TSTATUS_CONN_ERROR)
4160  pg_log_error("perhaps the backend died while processing");
4161 
4162  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4163  st->state = CSTATE_ABORTED;
4164  break;
4165  }
4166 
4167  if (is_connect)
4168  {
4169  pg_time_usec_t start = now;
4170 
4171  pg_time_now_lazy(&start);
4172  finishCon(st);
4173  now = pg_time_now();
4174  thread->conn_duration += now - start;
4175  }
4176 
4177  if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded)
4178  {
4179  /* script completed */
4180  st->state = CSTATE_FINISHED;
4181  break;
4182  }
4183 
4184  /* next transaction (script) */
4186 
4187  /*
4188  * Ensure that we always return on this point, so as to
4189  * avoid an infinite loop if the script only contains meta
4190  * commands.
4191  */
4192  return;
4193  }
4194 
4195  /*
4196  * Final states. Close the connection if it's still open.
4197  */
4198  case CSTATE_ABORTED:
4199  case CSTATE_FINISHED:
4200 
4201  /*
4202  * Don't measure the disconnection delays here even if in
4203  * CSTATE_FINISHED and -C/--connect option is specified.
4204  * Because in this case all the connections that this thread
4205  * established are closed at the end of transactions and the
4206  * disconnection delays should have already been measured at
4207  * that moment.
4208  *
4209  * In CSTATE_ABORTED state, the measurement is no longer
4210  * necessary because we cannot report complete results anyways
4211  * in this case.
4212  */
4213  finishCon(st);
4214  return;
4215  }
4216  }
4217 }
4218 
4219 /*
4220  * Subroutine for advanceConnectionState -- initiate or execute the current
4221  * meta command, and return the next state to set.
4222  *
4223  * *now is updated to the current time, unless the command is expected to
4224  * take no time to execute.
4225  */
4228 {
4229  Command *command = sql_script[st->use_file].commands[st->command];
4230  int argc;
4231  char **argv;
4232 
4233  Assert(command != NULL && command->type == META_COMMAND);
4234 
4235  argc = command->argc;
4236  argv = command->argv;
4237 
4239  {
4241 
4242  initPQExpBuffer(&buf);
4243 
4244  printfPQExpBuffer(&buf, "client %d executing \\%s", st->id, argv[0]);
4245  for (int i = 1; i < argc; i++)
4246  appendPQExpBuffer(&buf, " %s", argv[i]);
4247 
4248  pg_log_debug("%s", buf.data);
4249 
4250  termPQExpBuffer(&buf);
4251  }
4252 
4253  if (command->meta == META_SLEEP)
4254  {
4255  int usec;
4256 
4257  /*
4258  * A \sleep doesn't execute anything, we just get the delay from the
4259  * argument, and enter the CSTATE_SLEEP state. (The per-command
4260  * latency will be recorded in CSTATE_SLEEP state, not here, after the
4261  * delay has elapsed.)
4262  */
4263  if (!evaluateSleep(&st->variables, argc, argv, &usec))
4264  {
4265  commandFailed(st, "sleep", "execution of meta-command failed");
4266  return CSTATE_ABORTED;
4267  }
4268 
4270  st->sleep_until = (*now) + usec;
4271  return CSTATE_SLEEP;
4272  }
4273  else if (command->meta == META_SET)
4274  {
4275  PgBenchExpr *expr = command->expr;
4276  PgBenchValue result;
4277 
4278  if (!evaluateExpr(st, expr, &result))
4279  {
4280  commandFailed(st, argv[0], "evaluation of meta-command failed");
4281  return CSTATE_ABORTED;
4282  }
4283 
4284  if (!putVariableValue(&st->variables, argv[0], argv[1], &result))
4285  {
4286  commandFailed(st, "set", "assignment of meta-command failed");
4287  return CSTATE_ABORTED;
4288  }
4289  }
4290  else if (command->meta == META_IF)
4291  {
4292  /* backslash commands with an expression to evaluate */
4293  PgBenchExpr *expr = command->expr;
4294  PgBenchValue result;
4295  bool cond;
4296 
4297  if (!evaluateExpr(st, expr, &result))
4298  {
4299  commandFailed(st, argv[0], "evaluation of meta-command failed");
4300  return CSTATE_ABORTED;
4301  }
4302 
4303  cond = valueTruth(&result);
4305  }
4306  else if (command->meta == META_ELIF)
4307  {
4308  /* backslash commands with an expression to evaluate */
4309  PgBenchExpr *expr = command->expr;
4310  PgBenchValue result;
4311  bool cond;
4312 
4314  {
4315  /* elif after executed block, skip eval and wait for endif. */
4317  return CSTATE_END_COMMAND;
4318  }
4319 
4320  if (!evaluateExpr(st, expr, &result))
4321  {
4322  commandFailed(st, argv[0], "evaluation of meta-command failed");
4323  return CSTATE_ABORTED;
4324  }
4325 
4326  cond = valueTruth(&result);
4329  }
4330  else if (command->meta == META_ELSE)
4331  {
4332  switch (conditional_stack_peek(st->cstack))
4333  {
4334  case IFSTATE_TRUE:
4336  break;
4337  case IFSTATE_FALSE: /* inconsistent if active */
4338  case IFSTATE_IGNORED: /* inconsistent if active */
4339  case IFSTATE_NONE: /* else without if */
4340  case IFSTATE_ELSE_TRUE: /* else after else */
4341  case IFSTATE_ELSE_FALSE: /* else after else */
4342  default:
4343  /* dead code if conditional check is ok */
4344  Assert(false);
4345  }
4346  }
4347  else if (command->meta == META_ENDIF)
4348  {
4351  }
4352  else if (command->meta == META_SETSHELL)
4353  {
4354  if (!runShellCommand(&st->variables, argv[1], argv + 2, argc - 2))
4355  {
4356  commandFailed(st, "setshell", "execution of meta-command failed");
4357  return CSTATE_ABORTED;
4358  }
4359  }
4360  else if (command->meta == META_SHELL)
4361  {
4362  if (!runShellCommand(&st->variables, NULL, argv + 1, argc - 1))
4363  {
4364  commandFailed(st, "shell", "execution of meta-command failed");
4365  return CSTATE_ABORTED;
4366  }
4367  }
4368  else if (command->meta == META_STARTPIPELINE)
4369  {
4370  /*
4371  * In pipeline mode, we use a workflow based on libpq pipeline
4372  * functions.
4373  */
4374  if (querymode == QUERY_SIMPLE)
4375  {
4376  commandFailed(st, "startpipeline", "cannot use pipeline mode with the simple query protocol");
4377  return CSTATE_ABORTED;
4378  }
4379 
4380  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4381  {
4382  commandFailed(st, "startpipeline", "already in pipeline mode");
4383  return CSTATE_ABORTED;
4384  }
4385  if (PQenterPipelineMode(st->con) == 0)
4386  {
4387  commandFailed(st, "startpipeline", "failed to enter pipeline mode");
4388  return CSTATE_ABORTED;
4389  }
4390  }
4391  else if (command->meta == META_ENDPIPELINE)
4392  {
4393  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
4394  {
4395  commandFailed(st, "endpipeline", "not in pipeline mode");
4396  return CSTATE_ABORTED;
4397  }
4398  if (!PQpipelineSync(st->con))
4399  {
4400  commandFailed(st, "endpipeline", "failed to send a pipeline sync");
4401  return CSTATE_ABORTED;
4402  }
4403  /* Now wait for the PGRES_PIPELINE_SYNC and exit pipeline mode there */
4404  /* collect pending results before getting out of pipeline mode */
4405  return CSTATE_WAIT_RESULT;
4406  }
4407 
4408  /*
4409  * executing the expression or shell command might have taken a
4410  * non-negligible amount of time, so reset 'now'
4411  */
4412  *now = 0;
4413 
4414  return CSTATE_END_COMMAND;
4415 }
4416 
4417 /*
4418  * Return the number of failed transactions.
4419  */
4420 static int64
4421 getFailures(const StatsData *stats)
4422 {
4423  return (stats->serialization_failures +
4424  stats->deadlock_failures);
4425 }
4426 
4427 /*
4428  * Return a string constant representing the result of a transaction
4429  * that is not successfully processed.
4430  */
4431 static const char *
4432 getResultString(bool skipped, EStatus estatus)
4433 {
4434  if (skipped)
4435  return "skipped";
4436  else if (failures_detailed)
4437  {
4438  switch (estatus)
4439  {
4441  return "serialization";
4443  return "deadlock";
4444  default:
4445  /* internal error which should never occur */
4446  pg_fatal("unexpected error status: %d", estatus);
4447  }
4448  }
4449  else
4450  return "failed";
4451 }
4452 
4453 /*
4454  * Print log entry after completing one transaction.
4455  *
4456  * We print Unix-epoch timestamps in the log, so that entries can be
4457  * correlated against other logs.
4458  *
4459  * XXX We could obtain the time from the caller and just shift it here, to
4460  * avoid the cost of an extra call to pg_time_now().
4461  */
4462 static void
4463 doLog(TState *thread, CState *st,
4464  StatsData *agg, bool skipped, double latency, double lag)
4465 {
4466  FILE *logfile = thread->logfile;
4468 
4469  Assert(use_log);
4470 
4471  /*
4472  * Skip the log entry if sampling is enabled and this row doesn't belong
4473  * to the random sample.
4474  */
4475  if (sample_rate != 0.0 &&
4477  return;
4478 
4479  /* should we aggregate the results or not? */
4480  if (agg_interval > 0)
4481  {
4483 
4484  /*
4485  * Loop until we reach the interval of the current moment, and print
4486  * any empty intervals in between (this may happen with very low tps,
4487  * e.g. --rate=0.1).
4488  */
4489 
4490  while ((next = agg->start_time + agg_interval * INT64CONST(1000000)) <= now)
4491  {
4492  double lag_sum = 0.0;
4493  double lag_sum2 = 0.0;
4494  double lag_min = 0.0;
4495  double lag_max = 0.0;
4496  int64 skipped = 0;
4497  int64 serialization_failures = 0;
4498  int64 deadlock_failures = 0;
4499  int64 retried = 0;
4500  int64 retries = 0;
4501 
4502  /* print aggregated report to logfile */
4503  fprintf(logfile, INT64_FORMAT " " INT64_FORMAT " %.0f %.0f %.0f %.0f",
4504  agg->start_time / 1000000, /* seconds since Unix epoch */
4505  agg->cnt,
4506  agg->latency.sum,
4507  agg->latency.sum2,
4508  agg->latency.min,
4509  agg->latency.max);
4510 
4511  if (throttle_delay)
4512  {
4513  lag_sum = agg->lag.sum;
4514  lag_sum2 = agg->lag.sum2;
4515  lag_min = agg->lag.min;
4516  lag_max = agg->lag.max;
4517  }
4518  fprintf(logfile, " %.0f %.0f %.0f %.0f",
4519  lag_sum,
4520  lag_sum2,
4521  lag_min,
4522  lag_max);
4523 
4524  if (latency_limit)
4525  skipped = agg->skipped;
4526  fprintf(logfile, " " INT64_FORMAT, skipped);
4527 
4528  if (max_tries != 1)
4529  {
4530  retried = agg->retried;
4531  retries = agg->retries;
4532  }
4533  fprintf(logfile, " " INT64_FORMAT " " INT64_FORMAT, retried, retries);
4534 
4535  if (failures_detailed)
4536  {
4537  serialization_failures = agg->serialization_failures;
4538  deadlock_failures = agg->deadlock_failures;
4539  }
4541  serialization_failures,
4542  deadlock_failures);
4543 
4544  fputc('\n', logfile);
4545 
4546  /* reset data and move to next interval */
4547  initStats(agg, next);
4548  }
4549 
4550  /* accumulate the current transaction */
4551  accumStats(agg, skipped, latency, lag, st->estatus, st->tries);
4552  }
4553  else
4554  {
4555  /* no, print raw transactions */
4556  if (!skipped && st->estatus == ESTATUS_NO_ERROR)
4557  fprintf(logfile, "%d " INT64_FORMAT " %.0f %d " INT64_FORMAT " "
4558  INT64_FORMAT,
4559  st->id, st->cnt, latency, st->use_file,
4560  now / 1000000, now % 1000000);
4561  else
4562  fprintf(logfile, "%d " INT64_FORMAT " %s %d " INT64_FORMAT " "
4563  INT64_FORMAT,
4564  st->id, st->cnt, getResultString(skipped, st->estatus),
4565  st->use_file, now / 1000000, now % 1000000);
4566 
4567  if (throttle_delay)
4568  fprintf(logfile, " %.0f", lag);
4569  if (max_tries != 1)
4570  fprintf(logfile, " %u", st->tries - 1);
4571  fputc('\n', logfile);
4572  }
4573 }
4574 
4575 /*
4576  * Accumulate and report statistics at end of a transaction.
4577  *
4578  * (This is also called when a transaction is late and thus skipped.
4579  * Note that even skipped and failed transactions are counted in the CState
4580  * "cnt" field.)
4581  */
4582 static void
4584  bool skipped, StatsData *agg)
4585 {
4586  double latency = 0.0,
4587  lag = 0.0;
4588  bool detailed = progress || throttle_delay || latency_limit ||
4590 
4591  if (detailed && !skipped && st->estatus == ESTATUS_NO_ERROR)
4592  {
4594 
4595  /* compute latency & lag */
4596  latency = (*now) - st->txn_scheduled;
4597  lag = st->txn_begin - st->txn_scheduled;
4598  }
4599 
4600  /* keep detailed thread stats */
4601  accumStats(&thread->stats, skipped, latency, lag, st->estatus, st->tries);
4602 
4603  /* count transactions over the latency limit, if needed */
4604  if (latency_limit && latency > latency_limit)
4605  thread->latency_late++;
4606 
4607  /* client stat is just counting */
4608  st->cnt++;
4609 
4610  if (use_log)
4611  doLog(thread, st, agg, skipped, latency, lag);
4612 
4613  /* XXX could use a mutex here, but we choose not to */
4614  if (per_script_stats)
4615  accumStats(&sql_script[st->use_file].stats, skipped, latency, lag,
4616  st->estatus, st->tries);
4617 }
4618 
4619 
4620 /* discard connections */
4621 static void
4622 disconnect_all(CState *state, int length)
4623 {
4624  int i;
4625 
4626  for (i = 0; i < length; i++)
4627  finishCon(&state[i]);
4628 }
4629 
4630 /*
4631  * Remove old pgbench tables, if any exist
4632  */
4633 static void
4634 initDropTables(PGconn *con)
4635 {
4636  fprintf(stderr, "dropping old tables...\n");
4637 
4638  /*
4639  * We drop all the tables in one command, so that whether there are
4640  * foreign key dependencies or not doesn't matter.
4641  */
4642  executeStatement(con, "drop table if exists "
4643  "pgbench_accounts, "
4644  "pgbench_branches, "
4645  "pgbench_history, "
4646  "pgbench_tellers");
4647 }
4648 
4649 /*
4650  * Create "pgbench_accounts" partitions if needed.
4651  *
4652  * This is the larger table of pgbench default tpc-b like schema
4653  * with a known size, so we choose to partition it.
4654  */
4655 static void
4657 {
4658  PQExpBufferData query;
4659 
4660  /* we must have to create some partitions */
4661  Assert(partitions > 0);
4662 
4663  fprintf(stderr, "creating %d partitions...\n", partitions);
4664 
4665  initPQExpBuffer(&query);
4666 
4667  for (int p = 1; p <= partitions; p++)
4668  {
4670  {
4671  int64 part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
4672 
4673  printfPQExpBuffer(&query,
4674  "create%s table pgbench_accounts_%d\n"
4675  " partition of pgbench_accounts\n"
4676  " for values from (",
4677  unlogged_tables ? " unlogged" : "", p);
4678 
4679  /*
4680  * For RANGE, we use open-ended partitions at the beginning and
4681  * end to allow any valid value for the primary key. Although the
4682  * actual minimum and maximum values can be derived from the
4683  * scale, it is more generic and the performance is better.
4684  */
4685  if (p == 1)
4686  appendPQExpBufferStr(&query, "minvalue");
4687  else
4688  appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
4689 
4690  appendPQExpBufferStr(&query, ") to (");
4691 
4692  if (p < partitions)
4693  appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
4694  else
4695  appendPQExpBufferStr(&query, "maxvalue");
4696 
4697  appendPQExpBufferChar(&query, ')');
4698  }
4699  else if (partition_method == PART_HASH)
4700  printfPQExpBuffer(&query,
4701  "create%s table pgbench_accounts_%d\n"
4702  " partition of pgbench_accounts\n"
4703  " for values with (modulus %d, remainder %d)",
4704  unlogged_tables ? " unlogged" : "", p,
4705  partitions, p - 1);
4706  else /* cannot get there */
4707  Assert(0);
4708 
4709  /*
4710  * Per ddlinfo in initCreateTables, fillfactor is needed on table
4711  * pgbench_accounts.
4712  */
4713  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4714 
4715  executeStatement(con, query.data);
4716  }
4717 
4718  termPQExpBuffer(&query);
4719 }
4720 
4721 /*
4722  * Create pgbench's standard tables
4723  */
4724 static void
4726 {
4727  /*
4728  * Note: TPC-B requires at least 100 bytes per row, and the "filler"
4729  * fields in these table declarations were intended to comply with that.
4730  * The pgbench_accounts table complies with that because the "filler"
4731  * column is set to blank-padded empty string. But for all other tables
4732  * the columns default to NULL and so don't actually take any space. We
4733  * could fix that by giving them non-null default values. However, that
4734  * would completely break comparability of pgbench results with prior
4735  * versions. Since pgbench has never pretended to be fully TPC-B compliant
4736  * anyway, we stick with the historical behavior.
4737  */
4738  struct ddlinfo
4739  {
4740  const char *table; /* table name */
4741  const char *smcols; /* column decls if accountIDs are 32 bits */
4742  const char *bigcols; /* column decls if accountIDs are 64 bits */
4743  int declare_fillfactor;
4744  };
4745  static const struct ddlinfo DDLs[] = {
4746  {
4747  "pgbench_history",
4748  "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)",
4749  "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)",
4750  0
4751  },
4752  {
4753  "pgbench_tellers",
4754  "tid int not null,bid int,tbalance int,filler char(84)",
4755  "tid int not null,bid int,tbalance int,filler char(84)",
4756  1
4757  },
4758  {
4759  "pgbench_accounts",
4760  "aid int not null,bid int,abalance int,filler char(84)",
4761  "aid bigint not null,bid int,abalance int,filler char(84)",
4762  1
4763  },
4764  {
4765  "pgbench_branches",
4766  "bid int not null,bbalance int,filler char(88)",
4767  "bid int not null,bbalance int,filler char(88)",
4768  1
4769  }
4770  };
4771  int i;
4772  PQExpBufferData query;
4773 
4774  fprintf(stderr, "creating tables...\n");
4775 
4776  initPQExpBuffer(&query);
4777 
4778  for (i = 0; i < lengthof(DDLs); i++)
4779  {
4780  const struct ddlinfo *ddl = &DDLs[i];
4781 
4782  /* Construct new create table statement. */
4783  printfPQExpBuffer(&query, "create%s table %s(%s)",
4784  unlogged_tables ? " unlogged" : "",
4785  ddl->table,
4786  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
4787 
4788  /* Partition pgbench_accounts table */
4789  if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
4790  appendPQExpBuffer(&query,
4791  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
4792  else if (ddl->declare_fillfactor)
4793  {
4794  /* fillfactor is only expected on actual tables */
4795  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4796  }
4797 
4798  if (tablespace != NULL)
4799  {
4800  char *escape_tablespace;
4801 
4802  escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
4803  appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
4804  PQfreemem(escape_tablespace);
4805  }
4806 
4807  executeStatement(con, query.data);
4808  }
4809 
4810  termPQExpBuffer(&query);
4811 
4812  if (partition_method != PART_NONE)
4813  createPartitions(con);
4814 }
4815 
4816 /*
4817  * Truncate away any old data, in one command in case there are foreign keys
4818  */
4819 static void
4821 {
4822  executeStatement(con, "truncate table "
4823  "pgbench_accounts, "
4824  "pgbench_branches, "
4825  "pgbench_history, "
4826  "pgbench_tellers");
4827 }
4828 
4829 /*
4830  * Fill the standard tables with some data generated and sent from the client
4831  */
4832 static void
4834 {
4835  PQExpBufferData sql;
4836  PGresult *res;
4837  int i;
4838  int64 k;
4839  char *copy_statement;
4840 
4841  /* used to track elapsed time and estimate of the remaining time */
4842  pg_time_usec_t start;
4843  int log_interval = 1;
4844 
4845  /* Stay on the same line if reporting to a terminal */
4846  char eol = isatty(fileno(stderr)) ? '\r' : '\n';
4847 
4848  fprintf(stderr, "generating data (client-side)...\n");
4849 
4850  /*
4851  * we do all of this in one transaction to enable the backend's
4852  * data-loading optimizations
4853  */
4854  executeStatement(con, "begin");
4855 
4856  /* truncate away any old data */
4857  initTruncateTables(con);
4858 
4859  initPQExpBuffer(&sql);
4860 
4861  /*
4862  * fill branches, tellers, accounts in that order in case foreign keys
4863  * already exist
4864  */
4865  for (i = 0; i < nbranches * scale; i++)
4866  {
4867  /* "filler" column defaults to NULL */
4868  printfPQExpBuffer(&sql,
4869  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
4870  i + 1);
4871  executeStatement(con, sql.data);
4872  }
4873 
4874  for (i = 0; i < ntellers * scale; i++)
4875  {
4876  /* "filler" column defaults to NULL */
4877  printfPQExpBuffer(&sql,
4878  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
4879  i + 1, i / ntellers + 1);
4880  executeStatement(con, sql.data);
4881  }
4882 
4883  /*
4884  * accounts is big enough to be worth using COPY and tracking runtime
4885  */
4886 
4887  /* use COPY with FREEZE on v14 and later without partitioning */
4888  if (partitions == 0 && PQserverVersion(con) >= 140000)
4889  copy_statement = "copy pgbench_accounts from stdin with (freeze on)";
4890  else
4891  copy_statement = "copy pgbench_accounts from stdin";
4892 
4893  res = PQexec(con, copy_statement);
4894 
4896  pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
4897  PQclear(res);
4898 
4899  start = pg_time_now();
4900 
4901  for (k = 0; k < (int64) naccounts * scale; k++)
4902  {
4903  int64 j = k + 1;
4904 
4905  /* "filler" column defaults to blank padded empty string */
4906  printfPQExpBuffer(&sql,
4907  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
4908  j, k / naccounts + 1, 0);
4909  if (PQputline(con, sql.data))
4910  pg_fatal("PQputline failed");
4911 
4912  if (CancelRequested)
4913  break;
4914 
4915  /*
4916  * If we want to stick with the original logging, print a message each
4917  * 100k inserted rows.
4918  */
4919  if ((!use_quiet) && (j % 100000 == 0))
4920  {
4921  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4922  double remaining_sec = ((double) scale * naccounts - j) * elapsed_sec / j;
4923 
4924  fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) done (elapsed %.2f s, remaining %.2f s)%c",
4925  j, (int64) naccounts * scale,
4926  (int) (((int64) j * 100) / (naccounts * (int64) scale)),
4927  elapsed_sec, remaining_sec, eol);
4928  }
4929  /* let's not call the timing for each row, but only each 100 rows */
4930  else if (use_quiet && (j % 100 == 0))
4931  {
4932  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4933  double remaining_sec = ((double) scale * naccounts - j) * elapsed_sec / j;
4934 
4935  /* have we reached the next interval (or end)? */
4936  if ((j == scale * naccounts) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
4937  {
4938  fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) done (elapsed %.2f s, remaining %.2f s)%c",
4939  j, (int64) naccounts * scale,
4940  (int) (((int64) j * 100) / (naccounts * (int64) scale)), elapsed_sec, remaining_sec, eol);
4941 
4942  /* skip to the next interval */
4943  log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
4944  }
4945  }
4946  }
4947 
4948  if (eol != '\n')
4949  fputc('\n', stderr); /* Need to move to next line */
4950 
4951  if (PQputline(con, "\\.\n"))
4952  pg_fatal("very last PQputline failed");
4953  if (PQendcopy(con))
4954  pg_fatal("PQendcopy failed");
4955 
4956  termPQExpBuffer(&sql);
4957 
4958  executeStatement(con, "commit");
4959 }
4960 
4961 /*
4962  * Fill the standard tables with some data generated on the server
4963  *
4964  * As already the case with the client-side data generation, the filler
4965  * column defaults to NULL in pgbench_branches and pgbench_tellers,
4966  * and is a blank-padded string in pgbench_accounts.
4967  */
4968 static void
4970 {
4971  PQExpBufferData sql;
4972 
4973  fprintf(stderr, "generating data (server-side)...\n");
4974 
4975  /*
4976  * we do all of this in one transaction to enable the backend's
4977  * data-loading optimizations
4978  */
4979  executeStatement(con, "begin");
4980 
4981  /* truncate away any old data */
4982  initTruncateTables(con);
4983 
4984  initPQExpBuffer(&sql);
4985 
4986  printfPQExpBuffer(&sql,
4987  "insert into pgbench_branches(bid,bbalance) "
4988  "select bid, 0 "
4989  "from generate_series(1, %d) as bid", nbranches * scale);
4990  executeStatement(con, sql.data);
4991 
4992  printfPQExpBuffer(&sql,
4993  "insert into pgbench_tellers(tid,bid,tbalance) "
4994  "select tid, (tid - 1) / %d + 1, 0 "
4995  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
4996  executeStatement(con, sql.data);
4997 
4998  printfPQExpBuffer(&sql,
4999  "insert into pgbench_accounts(aid,bid,abalance,filler) "
5000  "select aid, (aid - 1) / %d + 1, 0, '' "
5001  "from generate_series(1, " INT64_FORMAT ") as aid",
5002  naccounts, (int64) naccounts * scale);
5003  executeStatement(con, sql.data);
5004 
5005  termPQExpBuffer(&sql);
5006 
5007  executeStatement(con, "commit");
5008 }
5009 
5010 /*
5011  * Invoke vacuum on the standard tables
5012  */
5013 static void
5014 initVacuum(PGconn *con)
5015 {
5016  fprintf(stderr, "vacuuming...\n");
5017  executeStatement(con, "vacuum analyze pgbench_branches");
5018  executeStatement(con, "vacuum analyze pgbench_tellers");
5019  executeStatement(con, "vacuum analyze pgbench_accounts");
5020  executeStatement(con, "vacuum analyze pgbench_history");
5021 }
5022 
5023 /*
5024  * Create primary keys on the standard tables
5025  */
5026 static void
5027 initCreatePKeys(PGconn *con)
5028 {
5029  static const char *const DDLINDEXes[] = {
5030  "alter table pgbench_branches add primary key (bid)",
5031  "alter table pgbench_tellers add primary key (tid)",
5032  "alter table pgbench_accounts add primary key (aid)"
5033  };
5034  int i;
5035  PQExpBufferData query;
5036 
5037  fprintf(stderr, "creating primary keys...\n");
5038  initPQExpBuffer(&query);
5039 
5040  for (i = 0; i < lengthof(DDLINDEXes); i++)
5041  {
5042  resetPQExpBuffer(&query);
5043  appendPQExpBufferStr(&query, DDLINDEXes[i]);
5044 
5045  if (index_tablespace != NULL)
5046  {
5047  char *escape_tablespace;
5048 
5049  escape_tablespace = PQescapeIdentifier(con, index_tablespace,
5050  strlen(index_tablespace));
5051  appendPQExpBuffer(&query, " using index tablespace %s", escape_tablespace);
5052  PQfreemem(escape_tablespace);
5053  }
5054 
5055  executeStatement(con, query.data);
5056  }
5057 
5058  termPQExpBuffer(&query);
5059 }
5060 
5061 /*
5062  * Create foreign key constraints between the standard tables
5063  */
5064 static void
5065 initCreateFKeys(PGconn *con)
5066 {
5067  static const char *const DDLKEYs[] = {
5068  "alter table pgbench_tellers add constraint pgbench_tellers_bid_fkey foreign key (bid) references pgbench_branches",
5069  "alter table pgbench_accounts add constraint pgbench_accounts_bid_fkey foreign key (bid) references pgbench_branches",
5070  "alter table pgbench_history add constraint pgbench_history_bid_fkey foreign key (bid) references pgbench_branches",
5071  "alter table pgbench_history add constraint pgbench_history_tid_fkey foreign key (tid) references pgbench_tellers",
5072  "alter table pgbench_history add constraint pgbench_history_aid_fkey foreign key (aid) references pgbench_accounts"
5073  };
5074  int i;
5075 
5076  fprintf(stderr, "creating foreign keys...\n");
5077  for (i = 0; i < lengthof(DDLKEYs); i++)
5078  {
5079  executeStatement(con, DDLKEYs[i]);
5080  }
5081 }
5082 
5083 /*
5084  * Validate an initialization-steps string
5085  *
5086  * (We could just leave it to runInitSteps() to fail if there are wrong
5087  * characters, but since initialization can take awhile, it seems friendlier
5088  * to check during option parsing.)
5089  */
5090 static void
5091 checkInitSteps(const char *initialize_steps)
5092 {
5093  if (initialize_steps[0] == '\0')
5094  pg_fatal("no initialization steps specified");
5095 
5096  for (const char *step = initialize_steps; *step != '\0'; step++)
5097  {
5098  if (strchr(ALL_INIT_STEPS " ", *step) == NULL)
5099  {
5100  pg_log_error("unrecognized initialization step \"%c\"", *step);
5101  pg_log_error_detail("Allowed step characters are: \"" ALL_INIT_STEPS "\".");
5102  exit(1);
5103  }
5104  }
5105 }
5106 
5107 /*
5108  * Invoke each initialization step in the given string
5109  */
5110 static void
5111 runInitSteps(const char *initialize_steps)
5112 {
5113  PQExpBufferData stats;
5114  PGconn *con;
5115  const char *step;
5116  double run_time = 0.0;
5117  bool first = true;
5118 
5119  initPQExpBuffer(&stats);
5120 
5121  if ((con = doConnect()) == NULL)
5122  pg_fatal("could not create connection for initialization");
5123 
5124  setup_cancel_handler(NULL);
5125  SetCancelConn(con);
5126 
5127  for (step = initialize_steps; *step != '\0'; step++)
5128  {
5129  char *op = NULL;
5130  pg_time_usec_t start = pg_time_now();
5131 
5132  switch (*step)
5133  {
5134  case 'd':
5135  op = "drop tables";
5136  initDropTables(con);
5137  break;
5138  case 't':
5139  op = "create tables";
5140  initCreateTables(con);
5141  break;
5142  case 'g':
5143  op = "client-side generate";
5145  break;
5146  case 'G':
5147  op = "server-side generate";
5149  break;
5150  case 'v':
5151  op = "vacuum";
5152  initVacuum(con);
5153  break;
5154  case 'p':
5155  op = "primary keys";
5156  initCreatePKeys(con);
5157  break;
5158  case 'f':
5159  op = "foreign keys";
5160  initCreateFKeys(con);
5161  break;
5162  case ' ':
5163  break; /* ignore */
5164  default:
5165  pg_log_error("unrecognized initialization step \"%c\"", *step);
5166  PQfinish(con);
5167  exit(1);
5168  }
5169 
5170  if (op != NULL)
5171  {
5172  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5173 
5174  if (!first)
5175  appendPQExpBufferStr(&stats, ", ");
5176  else
5177  first = false;
5178 
5179  appendPQExpBuffer(&stats, "%s %.2f s", op, elapsed_sec);
5180 
5181  run_time += elapsed_sec;
5182  }
5183  }
5184 
5185  fprintf(stderr, "done in %.2f s (%s).\n", run_time, stats.data);
5186  ResetCancelConn();
5187  PQfinish(con);
5188  termPQExpBuffer(&stats);
5189 }
5190 
5191 /*
5192  * Extract pgbench table information into global variables scale,
5193  * partition_method and partitions.
5194  */
5195 static void
5196 GetTableInfo(PGconn *con, bool scale_given)
5197 {
5198  PGresult *res;
5199 
5200  /*
5201  * get the scaling factor that should be same as count(*) from
5202  * pgbench_branches if this is not a custom query
5203  */
5204  res = PQexec(con, "select count(*) from pgbench_branches");
5206  {
5207  char *sqlState = PQresultErrorField(res, PG_DIAG_SQLSTATE);
5208 
5209  pg_log_error("could not count number of branches: %s", PQerrorMessage(con));
5210 
5211  if (sqlState && strcmp(sqlState, ERRCODE_UNDEFINED_TABLE) == 0)
5212  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".",
5213  PQdb(con));
5214 
5215  exit(1);
5216  }
5217  scale = atoi(PQgetvalue(res, 0, 0));
5218  if (scale < 0)
5219  pg_fatal("invalid count(*) from pgbench_branches: \"%s\"",
5220  PQgetvalue(res, 0, 0));
5221  PQclear(res);
5222 
5223  /* warn if we override user-given -s switch */
5224  if (scale_given)
5225  pg_log_warning("scale option ignored, using count from pgbench_branches table (%d)",
5226  scale);
5227 
5228  /*
5229  * Get the partition information for the first "pgbench_accounts" table
5230  * found in search_path.
5231  *
5232  * The result is empty if no "pgbench_accounts" is found.
5233  *
5234  * Otherwise, it always returns one row even if the table is not
5235  * partitioned (in which case the partition strategy is NULL).
5236  *
5237  * The number of partitions can be 0 even for partitioned tables, if no
5238  * partition is attached.
5239  *
5240  * We assume no partitioning on any failure, so as to avoid failing on an
5241  * old version without "pg_partitioned_table".
5242  */
5243  res = PQexec(con,
5244  "select o.n, p.partstrat, pg_catalog.count(i.inhparent) "
5245  "from pg_catalog.pg_class as c "
5246  "join pg_catalog.pg_namespace as n on (n.oid = c.relnamespace) "
5247  "cross join lateral (select pg_catalog.array_position(pg_catalog.current_schemas(true), n.nspname)) as o(n) "
5248  "left join pg_catalog.pg_partitioned_table as p on (p.partrelid = c.oid) "
5249  "left join pg_catalog.pg_inherits as i on (c.oid = i.inhparent) "
5250  "where c.relname = 'pgbench_accounts' and o.n is not null "
5251  "group by 1, 2 "
5252  "order by 1 asc "
5253  "limit 1");
5254 
5256  {
5257  /* probably an older version, coldly assume no partitioning */
5259  partitions = 0;
5260  }
5261  else if (PQntuples(res) == 0)
5262  {
5263  /*
5264  * This case is unlikely as pgbench already found "pgbench_branches"
5265  * above to compute the scale.
5266  */
5267  pg_log_error("no pgbench_accounts table found in search_path");
5268  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".", PQdb(con));
5269  exit(1);
5270  }
5271  else /* PQntupes(res) == 1 */
5272  {
5273  /* normal case, extract partition information */
5274  if (PQgetisnull(res, 0, 1))
5276  else
5277  {
5278  char *ps = PQgetvalue(res, 0, 1);
5279 
5280  /* column must be there */
5281  Assert(ps != NULL);
5282 
5283  if (strcmp(ps, "r") == 0)
5285  else if (strcmp(ps, "h") == 0)
5287  else
5288  {
5289  /* possibly a newer version with new partition method */
5290  pg_fatal("unexpected partition method: \"%s\"", ps);
5291  }
5292  }
5293 
5294  partitions = atoi(PQgetvalue(res, 0, 2));
5295  }
5296 
5297  PQclear(res);
5298 }
5299 
5300 /*
5301  * Replace :param with $n throughout the command's SQL text, which
5302  * is a modifiable string in cmd->lines.
5303  */
5304 static bool
5305 parseQuery(Command *cmd)
5306 {
5307  char *sql,
5308  *p;
5309 
5310  cmd->argc = 1;
5311 
5312  p = sql = pg_strdup(cmd->lines.data);
5313  while ((p = strchr(p, ':')) != NULL)
5314  {
5315  char var[13];
5316  char *name;
5317  int eaten;
5318 
5319  name = parseVariable(p, &eaten);
5320  if (name == NULL)
5321  {
5322  while (*p == ':')
5323  {
5324  p++;
5325  }
5326  continue;
5327  }
5328 
5329  /*
5330  * cmd->argv[0] is the SQL statement itself, so the max number of
5331  * arguments is one less than MAX_ARGS
5332  */
5333  if (cmd->argc >= MAX_ARGS)
5334  {
5335  pg_log_error("statement has too many arguments (maximum is %d): %s",
5336  MAX_ARGS - 1, cmd->lines.data);
5337  pg_free(name);
5338  return false;
5339  }
5340 
5341  sprintf(var, "$%d", cmd->argc);
5342  p = replaceVariable(&sql, p, eaten, var);
5343 
5344  cmd->argv[cmd->argc] = name;
5345  cmd->argc++;
5346  }
5347 
5348  Assert(cmd->argv[0] == NULL);
5349  cmd->argv[0] = sql;
5350  return true;
5351 }
5352 
5353 /*
5354  * syntax error while parsing a script (in practice, while parsing a
5355  * backslash command, because we don't detect syntax errors in SQL)
5356  *
5357  * source: source of script (filename or builtin-script ID)
5358  * lineno: line number within script (count from 1)
5359  * line: whole line of backslash command, if available
5360  * command: backslash command name, if available
5361  * msg: the actual error message
5362  * more: optional extra message
5363  * column: zero-based column number, or -1 if unknown
5364  */
5365 void
5366 syntax_error(const char *source, int lineno,
5367  const char *line, const char *command,
5368  const char *msg, const char *more, int column)
5369 {
5371 
5372  initPQExpBuffer(&buf);
5373 
5374  printfPQExpBuffer(&buf, "%s:%d: %s", source, lineno, msg);
5375  if (more != NULL)
5376  appendPQExpBuffer(&buf, " (%s)", more);
5377  if (column >= 0 && line == NULL)
5378  appendPQExpBuffer(&buf, " at column %d", column + 1);
5379  if (command != NULL)
5380  appendPQExpBuffer(&buf, " in command \"%s\"", command);
5381 
5382  pg_log_error("%s", buf.data);
5383 
5384  termPQExpBuffer(&buf);
5385 
5386  if (line != NULL)
5387  {
5388  fprintf(stderr, "%s\n", line);
5389  if (column >= 0)
5390  fprintf(stderr, "%*c error found here\n", column + 1, '^');
5391  }
5392 
5393  exit(1);
5394 }
5395 
5396 /*
5397  * Return a pointer to the start of the SQL command, after skipping over
5398  * whitespace and "--" comments.
5399  * If the end of the string is reached, return NULL.
5400  */
5401 static char *
5402 skip_sql_comments(char *sql_command)
5403 {
5404  char *p = sql_command;
5405 
5406  /* Skip any leading whitespace, as well as "--" style comments */
5407  for (;;)
5408  {
5409  if (isspace((unsigned char) *p))
5410  p++;
5411  else if (strncmp(p, "--", 2) == 0)
5412  {
5413  p = strchr(p, '\n');
5414  if (p == NULL)
5415  return NULL;
5416  p++;
5417  }
5418  else
5419  break;
5420  }
5421 
5422  /* NULL if there's nothing but whitespace and comments */
5423  if (*p == '\0')
5424  return NULL;
5425 
5426  return p;
5427 }
5428 
5429 /*
5430  * Parse a SQL command; return a Command struct, or NULL if it's a comment
5431  *
5432  * On entry, psqlscan.l has collected the command into "buf", so we don't
5433  * really need to do much here except check for comments and set up a Command
5434  * struct.
5435  */
5436 static Command *
5438 {
5439  Command *my_command;
5440  char *p = skip_sql_comments(buf->data);
5441 
5442  if (p == NULL)
5443  return NULL;
5444 
5445  /* Allocate and initialize Command structure */
5446  my_command = (Command *) pg_malloc(sizeof(Command));
5447  initPQExpBuffer(&my_command->lines);
5448  appendPQExpBufferStr(&my_command->lines, p);
5449  my_command->first_line = NULL; /* this is set later */
5450  my_command->type = SQL_COMMAND;
5451  my_command->meta = META_NONE;
5452  my_command->argc = 0;
5453  my_command->retries = 0;
5454  my_command->failures = 0;
5455  memset(my_command->argv, 0, sizeof(my_command->argv));
5456  my_command->varprefix = NULL; /* allocated later, if needed */
5457  my_command->expr = NULL;
5458  initSimpleStats(&my_command->stats);
5459 
5460  return my_command;
5461 }
5462 
5463 /* Free a Command structure and associated data */
5464 static void
5465 free_command(Command *command)
5466 {
5467  termPQExpBuffer(&command->lines);
5468  pg_free(command->first_line);
5469  for (int i = 0; i < command->argc; i++)
5470  pg_free(command->argv[i]);
5471  pg_free(command->varprefix);
5472 
5473  /*
5474  * It should also free expr recursively, but this is currently not needed
5475  * as only gset commands (which do not have an expression) are freed.
5476  */
5477  pg_free(command);
5478 }
5479 
5480 /*
5481  * Once an SQL command is fully parsed, possibly by accumulating several
5482  * parts, complete other fields of the Command structure.
5483  */
5484 static void
5485 postprocess_sql_command(Command *my_command)
5486 {
5487  char buffer[128];
5488 
5489  Assert(my_command->type == SQL_COMMAND);
5490 
5491  /* Save the first line for error display. */
5492  strlcpy(buffer, my_command->lines.data, sizeof(buffer));
5493  buffer[strcspn(buffer, "\n\r")] = '\0';
5494  my_command->first_line = pg_strdup(buffer);
5495 
5496  /* parse query if necessary */
5497  switch (querymode)
5498  {
5499  case QUERY_SIMPLE:
5500  my_command->argv[0] = my_command->lines.data;
5501  my_command->argc++;
5502  break;
5503  case QUERY_EXTENDED:
5504  case QUERY_PREPARED:
5505  if (!parseQuery(my_command))
5506  exit(1);
5507  break;
5508  default:
5509  exit(1);
5510  }
5511 }
5512 
5513 /*
5514  * Parse a backslash command; return a Command struct, or NULL if comment
5515  *
5516  * At call, we have scanned only the initial backslash.
5517  */
5518 static Command *
5519 process_backslash_command(PsqlScanState sstate, const char *source)
5520 {
5521  Command *my_command;
5522  PQExpBufferData word_buf;
5523  int word_offset;
5524  int offsets[MAX_ARGS]; /* offsets of argument words */
5525  int start_offset;
5526  int lineno;
5527  int j;
5528 
5529  initPQExpBuffer(&word_buf);
5530 
5531  /* Remember location of the backslash */
5532  start_offset = expr_scanner_offset(sstate) - 1;
5533  lineno = expr_scanner_get_lineno(sstate, start_offset);
5534 
5535  /* Collect first word of command */
5536  if (!expr_lex_one_word(sstate, &word_buf, &word_offset))
5537  {
5538  termPQExpBuffer(&word_buf);
5539  return NULL;
5540  }
5541 
5542  /* Allocate and initialize Command structure */
5543  my_command = (Command *) pg_malloc0(sizeof(Command));
5544  my_command->type = META_COMMAND;
5545  my_command->argc = 0;
5546  initSimpleStats(&my_command->stats);
5547 
5548