PostgreSQL Source Code  git master
pgbench.c
Go to the documentation of this file.
1 /*
2  * pgbench.c
3  *
4  * A simple benchmark program for PostgreSQL
5  * Originally written by Tatsuo Ishii and enhanced by many contributors.
6  *
7  * src/bin/pgbench/pgbench.c
8  * Copyright (c) 2000-2023, PostgreSQL Global Development Group
9  * ALL RIGHTS RESERVED;
10  *
11  * Permission to use, copy, modify, and distribute this software and its
12  * documentation for any purpose, without fee, and without a written agreement
13  * is hereby granted, provided that the above copyright notice and this
14  * paragraph and the following two paragraphs appear in all copies.
15  *
16  * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
17  * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
18  * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
19  * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
20  * POSSIBILITY OF SUCH DAMAGE.
21  *
22  * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIMS ANY WARRANTIES,
23  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
24  * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
25  * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
26  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
27  *
28  */
29 
30 #if defined(WIN32) && FD_SETSIZE < 1024
31 #error FD_SETSIZE needs to have been increased
32 #endif
33 
34 #include "postgres_fe.h"
35 
36 #include <ctype.h>
37 #include <float.h>
38 #include <limits.h>
39 #include <math.h>
40 #include <signal.h>
41 #include <time.h>
42 #include <sys/time.h>
43 #include <sys/resource.h> /* for getrlimit */
44 
45 /* For testing, PGBENCH_USE_SELECT can be defined to force use of that code */
46 #if defined(HAVE_PPOLL) && !defined(PGBENCH_USE_SELECT)
47 #define POLL_USING_PPOLL
48 #ifdef HAVE_POLL_H
49 #include <poll.h>
50 #endif
51 #else /* no ppoll(), so use select() */
52 #define POLL_USING_SELECT
53 #include <sys/select.h>
54 #endif
55 
56 #include "common/int.h"
57 #include "common/logging.h"
58 #include "common/pg_prng.h"
59 #include "common/string.h"
60 #include "common/username.h"
61 #include "fe_utils/cancel.h"
62 #include "fe_utils/conditional.h"
63 #include "fe_utils/option_utils.h"
64 #include "fe_utils/string_utils.h"
65 #include "getopt_long.h"
66 #include "libpq-fe.h"
67 #include "pgbench.h"
68 #include "port/pg_bitutils.h"
69 #include "portability/instr_time.h"
70 
71 /* X/Open (XSI) requires <math.h> to provide M_PI, but core POSIX does not */
72 #ifndef M_PI
73 #define M_PI 3.14159265358979323846
74 #endif
75 
76 #define ERRCODE_T_R_SERIALIZATION_FAILURE "40001"
77 #define ERRCODE_T_R_DEADLOCK_DETECTED "40P01"
78 #define ERRCODE_UNDEFINED_TABLE "42P01"
79 
80 /*
81  * Hashing constants
82  */
83 #define FNV_PRIME UINT64CONST(0x100000001b3)
84 #define FNV_OFFSET_BASIS UINT64CONST(0xcbf29ce484222325)
85 #define MM2_MUL UINT64CONST(0xc6a4a7935bd1e995)
86 #define MM2_MUL_TIMES_8 UINT64CONST(0x35253c9ade8f4ca8)
87 #define MM2_ROT 47
88 
89 /*
90  * Multi-platform socket set implementations
91  */
92 
93 #ifdef POLL_USING_PPOLL
94 #define SOCKET_WAIT_METHOD "ppoll"
95 
96 typedef struct socket_set
97 {
98  int maxfds; /* allocated length of pollfds[] array */
99  int curfds; /* number currently in use */
100  struct pollfd pollfds[FLEXIBLE_ARRAY_MEMBER];
101 } socket_set;
102 
103 #endif /* POLL_USING_PPOLL */
104 
105 #ifdef POLL_USING_SELECT
106 #define SOCKET_WAIT_METHOD "select"
107 
108 typedef struct socket_set
109 {
110  int maxfd; /* largest FD currently set in fds */
111  fd_set fds;
113 
114 #endif /* POLL_USING_SELECT */
115 
116 /*
117  * Multi-platform thread implementations
118  */
119 
120 #ifdef WIN32
121 /* Use Windows threads */
122 #include <windows.h>
123 #define GETERRNO() (_dosmaperr(GetLastError()), errno)
124 #define THREAD_T HANDLE
125 #define THREAD_FUNC_RETURN_TYPE unsigned
126 #define THREAD_FUNC_RETURN return 0
127 #define THREAD_FUNC_CC __stdcall
128 #define THREAD_CREATE(handle, function, arg) \
129  ((*(handle) = (HANDLE) _beginthreadex(NULL, 0, (function), (arg), 0, NULL)) == 0 ? errno : 0)
130 #define THREAD_JOIN(handle) \
131  (WaitForSingleObject(handle, INFINITE) != WAIT_OBJECT_0 ? \
132  GETERRNO() : CloseHandle(handle) ? 0 : GETERRNO())
133 #define THREAD_BARRIER_T SYNCHRONIZATION_BARRIER
134 #define THREAD_BARRIER_INIT(barrier, n) \
135  (InitializeSynchronizationBarrier((barrier), (n), 0) ? 0 : GETERRNO())
136 #define THREAD_BARRIER_WAIT(barrier) \
137  EnterSynchronizationBarrier((barrier), \
138  SYNCHRONIZATION_BARRIER_FLAGS_BLOCK_ONLY)
139 #define THREAD_BARRIER_DESTROY(barrier)
140 #else
141 /* Use POSIX threads */
142 #include "port/pg_pthread.h"
143 #define THREAD_T pthread_t
144 #define THREAD_FUNC_RETURN_TYPE void *
145 #define THREAD_FUNC_RETURN return NULL
146 #define THREAD_FUNC_CC
147 #define THREAD_CREATE(handle, function, arg) \
148  pthread_create((handle), NULL, (function), (arg))
149 #define THREAD_JOIN(handle) \
150  pthread_join((handle), NULL)
151 #define THREAD_BARRIER_T pthread_barrier_t
152 #define THREAD_BARRIER_INIT(barrier, n) \
153  pthread_barrier_init((barrier), NULL, (n))
154 #define THREAD_BARRIER_WAIT(barrier) pthread_barrier_wait((barrier))
155 #define THREAD_BARRIER_DESTROY(barrier) pthread_barrier_destroy((barrier))
156 #endif
157 
158 
159 /********************************************************************
160  * some configurable parameters */
161 
162 #define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
163 #define ALL_INIT_STEPS "dtgGvpf" /* all possible steps */
164 
165 #define LOG_STEP_SECONDS 5 /* seconds between log messages */
166 #define DEFAULT_NXACTS 10 /* default nxacts */
167 
168 #define MIN_GAUSSIAN_PARAM 2.0 /* minimum parameter for gauss */
169 
170 #define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
171 #define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
172 
173 int nxacts = 0; /* number of transactions per client */
174 int duration = 0; /* duration in seconds */
175 int64 end_time = 0; /* when to stop in micro seconds, under -T */
176 
177 /*
178  * scaling factor. for example, scale = 10 will make 1000000 tuples in
179  * pgbench_accounts table.
180  */
181 int scale = 1;
182 
183 /*
184  * fillfactor. for example, fillfactor = 90 will use only 90 percent
185  * space during inserts and leave 10 percent free.
186  */
187 int fillfactor = 100;
188 
189 /*
190  * use unlogged tables?
191  */
192 bool unlogged_tables = false;
193 
194 /*
195  * log sampling rate (1.0 = log everything, 0.0 = option not given)
196  */
197 double sample_rate = 0.0;
198 
199 /*
200  * When threads are throttled to a given rate limit, this is the target delay
201  * to reach that rate in usec. 0 is the default and means no throttling.
202  */
203 double throttle_delay = 0;
204 
205 /*
206  * Transactions which take longer than this limit (in usec) are counted as
207  * late, and reported as such, although they are completed anyway. When
208  * throttling is enabled, execution time slots that are more than this late
209  * are skipped altogether, and counted separately.
210  */
211 int64 latency_limit = 0;
212 
213 /*
214  * tablespace selection
215  */
216 char *tablespace = NULL;
217 char *index_tablespace = NULL;
218 
219 /*
220  * Number of "pgbench_accounts" partitions. 0 is the default and means no
221  * partitioning.
222  */
223 static int partitions = 0;
224 
225 /* partitioning strategy for "pgbench_accounts" */
226 typedef enum
227 {
228  PART_NONE, /* no partitioning */
229  PART_RANGE, /* range partitioning */
230  PART_HASH /* hash partitioning */
232 
234 static const char *PARTITION_METHOD[] = {"none", "range", "hash"};
235 
236 /* random seed used to initialize base_random_sequence */
237 int64 random_seed = -1;
238 
239 /*
240  * end of configurable parameters
241  *********************************************************************/
242 
243 #define nbranches 1 /* Makes little sense to change this. Change
244  * -s instead */
245 #define ntellers 10
246 #define naccounts 100000
247 
248 /*
249  * The scale factor at/beyond which 32bit integers are incapable of storing
250  * 64bit values.
251  *
252  * Although the actual threshold is 21474, we use 20000 because it is easier to
253  * document and remember, and isn't that far away from the real threshold.
254  */
255 #define SCALE_32BIT_THRESHOLD 20000
256 
257 bool use_log; /* log transaction latencies to a file */
258 bool use_quiet; /* quiet logging onto stderr */
259 int agg_interval; /* log aggregates instead of individual
260  * transactions */
261 bool per_script_stats = false; /* whether to collect stats per script */
262 int progress = 0; /* thread progress report every this seconds */
263 bool progress_timestamp = false; /* progress report with Unix time */
264 int nclients = 1; /* number of clients */
265 int nthreads = 1; /* number of threads */
266 bool is_connect; /* establish connection for each transaction */
267 bool report_per_command = false; /* report per-command latencies,
268  * retries after errors and failures
269  * (errors without retrying) */
270 int main_pid; /* main process id used in log filename */
271 
272 /*
273  * There are different types of restrictions for deciding that the current
274  * transaction with a serialization/deadlock error can no longer be retried and
275  * should be reported as failed:
276  * - max_tries (--max-tries) can be used to limit the number of tries;
277  * - latency_limit (-L) can be used to limit the total time of tries;
278  * - duration (-T) can be used to limit the total benchmark time.
279  *
280  * They can be combined together, and you need to use at least one of them to
281  * retry the transactions with serialization/deadlock errors. If none of them is
282  * used, the default value of max_tries is 1 and such transactions will not be
283  * retried.
284  */
285 
286 /*
287  * We cannot retry a transaction after the serialization/deadlock error if its
288  * number of tries reaches this maximum; if its value is zero, it is not used.
289  */
290 uint32 max_tries = 1;
291 
292 bool failures_detailed = false; /* whether to group failures in
293  * reports or logs by basic types */
294 
295 const char *pghost = NULL;
296 const char *pgport = NULL;
297 const char *username = NULL;
298 const char *dbName = NULL;
299 char *logfile_prefix = NULL;
300 const char *progname;
301 
302 #define WSEP '@' /* weight separator */
303 
304 volatile sig_atomic_t timer_exceeded = false; /* flag from signal handler */
305 
306 /*
307  * We don't want to allocate variables one by one; for efficiency, add a
308  * constant margin each time it overflows.
309  */
310 #define VARIABLES_ALLOC_MARGIN 8
311 
312 /*
313  * Variable definitions.
314  *
315  * If a variable only has a string value, "svalue" is that value, and value is
316  * "not set". If the value is known, "value" contains the value (in any
317  * variant).
318  *
319  * In this case "svalue" contains the string equivalent of the value, if we've
320  * had occasion to compute that, or NULL if we haven't.
321  */
322 typedef struct
323 {
324  char *name; /* variable's name */
325  char *svalue; /* its value in string form, if known */
326  PgBenchValue value; /* actual variable's value */
327 } Variable;
328 
329 /*
330  * Data structure for client variables.
331  */
332 typedef struct
333 {
334  Variable *vars; /* array of variable definitions */
335  int nvars; /* number of variables */
336 
337  /*
338  * The maximum number of variables that we can currently store in 'vars'
339  * without having to reallocate more space. We must always have max_vars
340  * >= nvars.
341  */
342  int max_vars;
343 
344  bool vars_sorted; /* are variables sorted by name? */
345 } Variables;
346 
347 #define MAX_SCRIPTS 128 /* max number of SQL scripts allowed */
348 #define SHELL_COMMAND_SIZE 256 /* maximum size allowed for shell command */
349 
350 /*
351  * Simple data structure to keep stats about something.
352  *
353  * XXX probably the first value should be kept and used as an offset for
354  * better numerical stability...
355  */
356 typedef struct SimpleStats
357 {
358  int64 count; /* how many values were encountered */
359  double min; /* the minimum seen */
360  double max; /* the maximum seen */
361  double sum; /* sum of values */
362  double sum2; /* sum of squared values */
363 } SimpleStats;
364 
365 /*
366  * The instr_time type is expensive when dealing with time arithmetic. Define
367  * a type to hold microseconds instead. Type int64 is good enough for about
368  * 584500 years.
369  */
370 typedef int64 pg_time_usec_t;
371 
372 /*
373  * Data structure to hold various statistics: per-thread and per-script stats
374  * are maintained and merged together.
375  */
376 typedef struct StatsData
377 {
378  pg_time_usec_t start_time; /* interval start time, for aggregates */
379 
380  /*----------
381  * Transactions are counted depending on their execution and outcome.
382  * First a transaction may have started or not: skipped transactions occur
383  * under --rate and --latency-limit when the client is too late to execute
384  * them. Secondly, a started transaction may ultimately succeed or fail,
385  * possibly after some retries when --max-tries is not one. Thus
386  *
387  * the number of all transactions =
388  * 'skipped' (it was too late to execute them) +
389  * 'cnt' (the number of successful transactions) +
390  * 'failed' (the number of failed transactions).
391  *
392  * A successful transaction can have several unsuccessful tries before a
393  * successful run. Thus
394  *
395  * 'cnt' (the number of successful transactions) =
396  * successfully retried transactions (they got a serialization or a
397  * deadlock error(s), but were
398  * successfully retried from the very
399  * beginning) +
400  * directly successful transactions (they were successfully completed on
401  * the first try).
402  *
403  * A failed transaction is defined as unsuccessfully retried transactions.
404  * It can be one of two types:
405  *
406  * failed (the number of failed transactions) =
407  * 'serialization_failures' (they got a serialization error and were not
408  * successfully retried) +
409  * 'deadlock_failures' (they got a deadlock error and were not
410  * successfully retried).
411  *
412  * If the transaction was retried after a serialization or a deadlock
413  * error this does not guarantee that this retry was successful. Thus
414  *
415  * 'retries' (number of retries) =
416  * number of retries in all retried transactions =
417  * number of retries in (successfully retried transactions +
418  * failed transactions);
419  *
420  * 'retried' (number of all retried transactions) =
421  * successfully retried transactions +
422  * failed transactions.
423  *----------
424  */
425  int64 cnt; /* number of successful transactions, not
426  * including 'skipped' */
427  int64 skipped; /* number of transactions skipped under --rate
428  * and --latency-limit */
429  int64 retries; /* number of retries after a serialization or
430  * a deadlock error in all the transactions */
431  int64 retried; /* number of all transactions that were
432  * retried after a serialization or a deadlock
433  * error (perhaps the last try was
434  * unsuccessful) */
435  int64 serialization_failures; /* number of transactions that were
436  * not successfully retried after a
437  * serialization error */
438  int64 deadlock_failures; /* number of transactions that were not
439  * successfully retried after a deadlock
440  * error */
443 } StatsData;
444 
445 /*
446  * For displaying Unix epoch timestamps, as some time functions may have
447  * another reference.
448  */
450 
451 /*
452  * Error status for errors during script execution.
453  */
454 typedef enum EStatus
455 {
458 
459  /* SQL errors */
463 } EStatus;
464 
465 /*
466  * Transaction status at the end of a command.
467  */
468 typedef enum TStatus
469 {
474 } TStatus;
475 
476 /* Various random sequences are initialized from this one. */
478 
479 /* Synchronization barrier for start and connection */
481 
482 /*
483  * Connection state machine states.
484  */
485 typedef enum
486 {
487  /*
488  * The client must first choose a script to execute. Once chosen, it can
489  * either be throttled (state CSTATE_PREPARE_THROTTLE under --rate), start
490  * right away (state CSTATE_START_TX) or not start at all if the timer was
491  * exceeded (state CSTATE_FINISHED).
492  */
494 
495  /*
496  * CSTATE_START_TX performs start-of-transaction processing. Establishes
497  * a new connection for the transaction in --connect mode, records the
498  * transaction start time, and proceed to the first command.
499  *
500  * Note: once a script is started, it will either error or run till its
501  * end, where it may be interrupted. It is not interrupted while running,
502  * so pgbench --time is to be understood as tx are allowed to start in
503  * that time, and will finish when their work is completed.
504  */
506 
507  /*
508  * In CSTATE_PREPARE_THROTTLE state, we calculate when to begin the next
509  * transaction, and advance to CSTATE_THROTTLE. CSTATE_THROTTLE state
510  * sleeps until that moment, then advances to CSTATE_START_TX, or
511  * CSTATE_FINISHED if the next transaction would start beyond the end of
512  * the run.
513  */
516 
517  /*
518  * We loop through these states, to process each command in the script:
519  *
520  * CSTATE_START_COMMAND starts the execution of a command. On a SQL
521  * command, the command is sent to the server, and we move to
522  * CSTATE_WAIT_RESULT state unless in pipeline mode. On a \sleep
523  * meta-command, the timer is set, and we enter the CSTATE_SLEEP state to
524  * wait for it to expire. Other meta-commands are executed immediately. If
525  * the command about to start is actually beyond the end of the script,
526  * advance to CSTATE_END_TX.
527  *
528  * CSTATE_WAIT_RESULT waits until we get a result set back from the server
529  * for the current command.
530  *
531  * CSTATE_SLEEP waits until the end of \sleep.
532  *
533  * CSTATE_END_COMMAND records the end-of-command timestamp, increments the
534  * command counter, and loops back to CSTATE_START_COMMAND state.
535  *
536  * CSTATE_SKIP_COMMAND is used by conditional branches which are not
537  * executed. It quickly skip commands that do not need any evaluation.
538  * This state can move forward several commands, till there is something
539  * to do or the end of the script.
540  */
546 
547  /*
548  * States for failed commands.
549  *
550  * If the SQL/meta command fails, in CSTATE_ERROR clean up after an error:
551  * (1) clear the conditional stack; (2) if we have an unterminated
552  * (possibly failed) transaction block, send the rollback command to the
553  * server and wait for the result in CSTATE_WAIT_ROLLBACK_RESULT. If
554  * something goes wrong with rolling back, go to CSTATE_ABORTED.
555  *
556  * But if everything is ok we are ready for future transactions: if this
557  * is a serialization or deadlock error and we can re-execute the
558  * transaction from the very beginning, go to CSTATE_RETRY; otherwise go
559  * to CSTATE_FAILURE.
560  *
561  * In CSTATE_RETRY report an error, set the same parameters for the
562  * transaction execution as in the previous tries and process the first
563  * transaction command in CSTATE_START_COMMAND.
564  *
565  * In CSTATE_FAILURE report a failure, set the parameters for the
566  * transaction execution as they were before the first run of this
567  * transaction (except for a random state) and go to CSTATE_END_TX to
568  * complete this transaction.
569  */
574 
575  /*
576  * CSTATE_END_TX performs end-of-transaction processing. It calculates
577  * latency, and logs the transaction. In --connect mode, it closes the
578  * current connection.
579  *
580  * Then either starts over in CSTATE_CHOOSE_SCRIPT, or enters
581  * CSTATE_FINISHED if we have no more work to do.
582  */
584 
585  /*
586  * Final states. CSTATE_ABORTED means that the script execution was
587  * aborted because a command failed, CSTATE_FINISHED means success.
588  */
592 
593 /*
594  * Connection state.
595  */
596 typedef struct
597 {
598  PGconn *con; /* connection handle to DB */
599  int id; /* client No. */
600  ConnectionStateEnum state; /* state machine's current state. */
601  ConditionalStack cstack; /* enclosing conditionals state */
602 
603  /*
604  * Separate randomness for each client. This is used for random functions
605  * PGBENCH_RANDOM_* during the execution of the script.
606  */
607  pg_prng_state cs_func_rs;
608 
609  int use_file; /* index in sql_script for this client */
610  int command; /* command number in script */
611 
612  /* client variables */
613  Variables variables;
614 
615  /* various times about current transaction in microseconds */
616  pg_time_usec_t txn_scheduled; /* scheduled start time of transaction */
617  pg_time_usec_t sleep_until; /* scheduled start time of next cmd */
618  pg_time_usec_t txn_begin; /* used for measuring schedule lag times */
619  pg_time_usec_t stmt_begin; /* used for measuring statement latencies */
620 
621  /* whether client prepared each command of each script */
622  bool **prepared;
623 
624  /*
625  * For processing failures and repeating transactions with serialization
626  * or deadlock errors:
627  */
628  EStatus estatus; /* the error status of the current transaction
629  * execution; this is ESTATUS_NO_ERROR if
630  * there were no errors */
631  pg_prng_state random_state; /* random state */
632  uint32 tries; /* how many times have we already tried the
633  * current transaction? */
634 
635  /* per client collected stats */
636  int64 cnt; /* client transaction count, for -t; skipped
637  * and failed transactions are also counted
638  * here */
639 } CState;
640 
641 /*
642  * Thread state
643  */
644 typedef struct
645 {
646  int tid; /* thread id */
647  THREAD_T thread; /* thread handle */
648  CState *state; /* array of CState */
649  int nstate; /* length of state[] */
650 
651  /*
652  * Separate randomness for each thread. Each thread option uses its own
653  * random state to make all of them independent of each other and
654  * therefore deterministic at the thread level.
655  */
656  pg_prng_state ts_choose_rs; /* random state for selecting a script */
657  pg_prng_state ts_throttle_rs; /* random state for transaction throttling */
658  pg_prng_state ts_sample_rs; /* random state for log sampling */
659 
660  int64 throttle_trigger; /* previous/next throttling (us) */
661  FILE *logfile; /* where to log, or NULL */
662 
663  /* per thread collected stats in microseconds */
664  pg_time_usec_t create_time; /* thread creation time */
665  pg_time_usec_t started_time; /* thread is running */
666  pg_time_usec_t bench_start; /* thread is benchmarking */
667  pg_time_usec_t conn_duration; /* cumulated connection and disconnection
668  * delays */
669 
670  StatsData stats;
671  int64 latency_late; /* count executed but late transactions */
672 } TState;
673 
674 /*
675  * queries read from files
676  */
677 #define SQL_COMMAND 1
678 #define META_COMMAND 2
679 
680 /*
681  * max number of backslash command arguments or SQL variables,
682  * including the command or SQL statement itself
683  */
684 #define MAX_ARGS 256
685 
686 typedef enum MetaCommand
687 {
688  META_NONE, /* not a known meta-command */
689  META_SET, /* \set */
690  META_SETSHELL, /* \setshell */
691  META_SHELL, /* \shell */
692  META_SLEEP, /* \sleep */
693  META_GSET, /* \gset */
694  META_ASET, /* \aset */
695  META_IF, /* \if */
696  META_ELIF, /* \elif */
697  META_ELSE, /* \else */
698  META_ENDIF, /* \endif */
699  META_STARTPIPELINE, /* \startpipeline */
700  META_ENDPIPELINE /* \endpipeline */
701 } MetaCommand;
702 
703 typedef enum QueryMode
704 {
705  QUERY_SIMPLE, /* simple query */
706  QUERY_EXTENDED, /* extended query */
707  QUERY_PREPARED, /* extended query with prepared statements */
709 } QueryMode;
710 
712 static const char *QUERYMODE[] = {"simple", "extended", "prepared"};
713 
714 /*
715  * struct Command represents one command in a script.
716  *
717  * lines The raw, possibly multi-line command text. Variable substitution
718  * not applied.
719  * first_line A short, single-line extract of 'lines', for error reporting.
720  * type SQL_COMMAND or META_COMMAND
721  * meta The type of meta-command, with META_NONE/GSET/ASET if command
722  * is SQL.
723  * argc Number of arguments of the command, 0 if not yet processed.
724  * argv Command arguments, the first of which is the command or SQL
725  * string itself. For SQL commands, after post-processing
726  * argv[0] is the same as 'lines' with variables substituted.
727  * prepname The name that this command is prepared under, in prepare mode
728  * varprefix SQL commands terminated with \gset or \aset have this set
729  * to a non NULL value. If nonempty, it's used to prefix the
730  * variable name that receives the value.
731  * aset do gset on all possible queries of a combined query (\;).
732  * expr Parsed expression, if needed.
733  * stats Time spent in this command.
734  * retries Number of retries after a serialization or deadlock error in the
735  * current command.
736  * failures Number of errors in the current command that were not retried.
737  */
738 typedef struct Command
739 {
741  char *first_line;
742  int type;
744  int argc;
745  char *argv[MAX_ARGS];
746  char *prepname;
747  char *varprefix;
750  int64 retries;
751  int64 failures;
752 } Command;
753 
754 typedef struct ParsedScript
755 {
756  const char *desc; /* script descriptor (eg, file name) */
757  int weight; /* selection weight */
758  Command **commands; /* NULL-terminated array of Commands */
759  StatsData stats; /* total time spent in script */
760 } ParsedScript;
761 
762 static ParsedScript sql_script[MAX_SCRIPTS]; /* SQL script files */
763 static int num_scripts; /* number of scripts in sql_script[] */
764 static int64 total_weight = 0;
765 
766 static bool verbose_errors = false; /* print verbose messages of all errors */
767 
768 static bool exit_on_abort = false; /* exit when any client is aborted */
769 
770 /* Builtin test scripts */
771 typedef struct BuiltinScript
772 {
773  const char *name; /* very short name for -b ... */
774  const char *desc; /* short description */
775  const char *script; /* actual pgbench script */
776 } BuiltinScript;
777 
778 static const BuiltinScript builtin_script[] =
779 {
780  {
781  "tpcb-like",
782  "<builtin: TPC-B (sort of)>",
783  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
784  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
785  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
786  "\\set delta random(-5000, 5000)\n"
787  "BEGIN;\n"
788  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
789  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
790  "UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;\n"
791  "UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;\n"
792  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
793  "END;\n"
794  },
795  {
796  "simple-update",
797  "<builtin: simple update>",
798  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
799  "\\set bid random(1, " CppAsString2(nbranches) " * :scale)\n"
800  "\\set tid random(1, " CppAsString2(ntellers) " * :scale)\n"
801  "\\set delta random(-5000, 5000)\n"
802  "BEGIN;\n"
803  "UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;\n"
804  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
805  "INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);\n"
806  "END;\n"
807  },
808  {
809  "select-only",
810  "<builtin: select only>",
811  "\\set aid random(1, " CppAsString2(naccounts) " * :scale)\n"
812  "SELECT abalance FROM pgbench_accounts WHERE aid = :aid;\n"
813  }
814 };
815 
816 
817 /* Function prototypes */
818 static void setNullValue(PgBenchValue *pv);
819 static void setBoolValue(PgBenchValue *pv, bool bval);
820 static void setIntValue(PgBenchValue *pv, int64 ival);
821 static void setDoubleValue(PgBenchValue *pv, double dval);
822 static bool evaluateExpr(CState *st, PgBenchExpr *expr,
823  PgBenchValue *retval);
825 static void doLog(TState *thread, CState *st,
826  StatsData *agg, bool skipped, double latency, double lag);
827 static void processXactStats(TState *thread, CState *st, pg_time_usec_t *now,
828  bool skipped, StatsData *agg);
829 static void addScript(const ParsedScript *script);
831 static void finishCon(CState *st);
832 static void setalarm(int seconds);
833 static socket_set *alloc_socket_set(int count);
834 static void free_socket_set(socket_set *sa);
835 static void clear_socket_set(socket_set *sa);
836 static void add_socket_to_set(socket_set *sa, int fd, int idx);
837 static int wait_on_socket_set(socket_set *sa, int64 usecs);
838 static bool socket_has_input(socket_set *sa, int fd, int idx);
839 
840 /* callback used to build rows for COPY during data loading */
841 typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
842 
843 /* callback functions for our flex lexer */
844 static const PsqlScanCallbacks pgbench_callbacks = {
845  NULL, /* don't need get_variable functionality */
846 };
847 
848 static inline pg_time_usec_t
849 pg_time_now(void)
850 {
851  instr_time now;
852 
854 
856 }
857 
858 static inline void
860 {
861  if ((*now) == 0)
862  (*now) = pg_time_now();
863 }
864 
865 #define PG_TIME_GET_DOUBLE(t) (0.000001 * (t))
866 
867 static void
868 usage(void)
869 {
870  printf("%s is a benchmarking tool for PostgreSQL.\n\n"
871  "Usage:\n"
872  " %s [OPTION]... [DBNAME]\n"
873  "\nInitialization options:\n"
874  " -i, --initialize invokes initialization mode\n"
875  " -I, --init-steps=[" ALL_INIT_STEPS "]+ (default \"" DEFAULT_INIT_STEPS "\")\n"
876  " run selected initialization steps\n"
877  " -F, --fillfactor=NUM set fill factor\n"
878  " -n, --no-vacuum do not run VACUUM during initialization\n"
879  " -q, --quiet quiet logging (one message each 5 seconds)\n"
880  " -s, --scale=NUM scaling factor\n"
881  " --foreign-keys create foreign key constraints between tables\n"
882  " --index-tablespace=TABLESPACE\n"
883  " create indexes in the specified tablespace\n"
884  " --partition-method=(range|hash)\n"
885  " partition pgbench_accounts with this method (default: range)\n"
886  " --partitions=NUM partition pgbench_accounts into NUM parts (default: 0)\n"
887  " --tablespace=TABLESPACE create tables in the specified tablespace\n"
888  " --unlogged-tables create tables as unlogged tables\n"
889  "\nOptions to select what to run:\n"
890  " -b, --builtin=NAME[@W] add builtin script NAME weighted at W (default: 1)\n"
891  " (use \"-b list\" to list available scripts)\n"
892  " -f, --file=FILENAME[@W] add script FILENAME weighted at W (default: 1)\n"
893  " -N, --skip-some-updates skip updates of pgbench_tellers and pgbench_branches\n"
894  " (same as \"-b simple-update\")\n"
895  " -S, --select-only perform SELECT-only transactions\n"
896  " (same as \"-b select-only\")\n"
897  "\nBenchmarking options:\n"
898  " -c, --client=NUM number of concurrent database clients (default: 1)\n"
899  " -C, --connect establish new connection for each transaction\n"
900  " -D, --define=VARNAME=VALUE\n"
901  " define variable for use by custom script\n"
902  " -j, --jobs=NUM number of threads (default: 1)\n"
903  " -l, --log write transaction times to log file\n"
904  " -L, --latency-limit=NUM count transactions lasting more than NUM ms as late\n"
905  " -M, --protocol=simple|extended|prepared\n"
906  " protocol for submitting queries (default: simple)\n"
907  " -n, --no-vacuum do not run VACUUM before tests\n"
908  " -P, --progress=NUM show thread progress report every NUM seconds\n"
909  " -r, --report-per-command report latencies, failures, and retries per command\n"
910  " -R, --rate=NUM target rate in transactions per second\n"
911  " -s, --scale=NUM report this scale factor in output\n"
912  " -t, --transactions=NUM number of transactions each client runs (default: 10)\n"
913  " -T, --time=NUM duration of benchmark test in seconds\n"
914  " -v, --vacuum-all vacuum all four standard tables before tests\n"
915  " --aggregate-interval=NUM aggregate data over NUM seconds\n"
916  " --exit-on-abort exit when any client is aborted\n"
917  " --failures-detailed report the failures grouped by basic types\n"
918  " --log-prefix=PREFIX prefix for transaction time log file\n"
919  " (default: \"pgbench_log\")\n"
920  " --max-tries=NUM max number of tries to run transaction (default: 1)\n"
921  " --progress-timestamp use Unix epoch timestamps for progress\n"
922  " --random-seed=SEED set random seed (\"time\", \"rand\", integer)\n"
923  " --sampling-rate=NUM fraction of transactions to log (e.g., 0.01 for 1%%)\n"
924  " --show-script=NAME show builtin script code, then exit\n"
925  " --verbose-errors print messages of all errors\n"
926  "\nCommon options:\n"
927  " -d, --debug print debugging output\n"
928  " -h, --host=HOSTNAME database server host or socket directory\n"
929  " -p, --port=PORT database server port number\n"
930  " -U, --username=USERNAME connect as specified database user\n"
931  " -V, --version output version information, then exit\n"
932  " -?, --help show this help, then exit\n"
933  "\n"
934  "Report bugs to <%s>.\n"
935  "%s home page: <%s>\n",
936  progname, progname, PACKAGE_BUGREPORT, PACKAGE_NAME, PACKAGE_URL);
937 }
938 
939 /* return whether str matches "^\s*[-+]?[0-9]+$" */
940 static bool
941 is_an_int(const char *str)
942 {
943  const char *ptr = str;
944 
945  /* skip leading spaces; cast is consistent with strtoint64 */
946  while (*ptr && isspace((unsigned char) *ptr))
947  ptr++;
948 
949  /* skip sign */
950  if (*ptr == '+' || *ptr == '-')
951  ptr++;
952 
953  /* at least one digit */
954  if (*ptr && !isdigit((unsigned char) *ptr))
955  return false;
956 
957  /* eat all digits */
958  while (*ptr && isdigit((unsigned char) *ptr))
959  ptr++;
960 
961  /* must have reached end of string */
962  return *ptr == '\0';
963 }
964 
965 
966 /*
967  * strtoint64 -- convert a string to 64-bit integer
968  *
969  * This function is a slightly modified version of pg_strtoint64() from
970  * src/backend/utils/adt/numutils.c.
971  *
972  * The function returns whether the conversion worked, and if so
973  * "*result" is set to the result.
974  *
975  * If not errorOK, an error message is also printed out on errors.
976  */
977 bool
978 strtoint64(const char *str, bool errorOK, int64 *result)
979 {
980  const char *ptr = str;
981  int64 tmp = 0;
982  bool neg = false;
983 
984  /*
985  * Do our own scan, rather than relying on sscanf which might be broken
986  * for long long.
987  *
988  * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
989  * value as a negative number.
990  */
991 
992  /* skip leading spaces */
993  while (*ptr && isspace((unsigned char) *ptr))
994  ptr++;
995 
996  /* handle sign */
997  if (*ptr == '-')
998  {
999  ptr++;
1000  neg = true;
1001  }
1002  else if (*ptr == '+')
1003  ptr++;
1004 
1005  /* require at least one digit */
1006  if (unlikely(!isdigit((unsigned char) *ptr)))
1007  goto invalid_syntax;
1008 
1009  /* process digits */
1010  while (*ptr && isdigit((unsigned char) *ptr))
1011  {
1012  int8 digit = (*ptr++ - '0');
1013 
1014  if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
1015  unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
1016  goto out_of_range;
1017  }
1018 
1019  /* allow trailing whitespace, but not other trailing chars */
1020  while (*ptr != '\0' && isspace((unsigned char) *ptr))
1021  ptr++;
1022 
1023  if (unlikely(*ptr != '\0'))
1024  goto invalid_syntax;
1025 
1026  if (!neg)
1027  {
1028  if (unlikely(tmp == PG_INT64_MIN))
1029  goto out_of_range;
1030  tmp = -tmp;
1031  }
1032 
1033  *result = tmp;
1034  return true;
1035 
1036 out_of_range:
1037  if (!errorOK)
1038  pg_log_error("value \"%s\" is out of range for type bigint", str);
1039  return false;
1040 
1041 invalid_syntax:
1042  if (!errorOK)
1043  pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
1044  return false;
1045 }
1046 
1047 /* convert string to double, detecting overflows/underflows */
1048 bool
1049 strtodouble(const char *str, bool errorOK, double *dv)
1050 {
1051  char *end;
1052 
1053  errno = 0;
1054  *dv = strtod(str, &end);
1055 
1056  if (unlikely(errno != 0))
1057  {
1058  if (!errorOK)
1059  pg_log_error("value \"%s\" is out of range for type double", str);
1060  return false;
1061  }
1062 
1063  if (unlikely(end == str || *end != '\0'))
1064  {
1065  if (!errorOK)
1066  pg_log_error("invalid input syntax for type double: \"%s\"", str);
1067  return false;
1068  }
1069  return true;
1070 }
1071 
1072 /*
1073  * Initialize a prng state struct.
1074  *
1075  * We derive the seed from base_random_sequence, which must be set up already.
1076  */
1077 static void
1079 {
1081 }
1082 
1083 
1084 /*
1085  * random number generator: uniform distribution from min to max inclusive.
1086  *
1087  * Although the limits are expressed as int64, you can't generate the full
1088  * int64 range in one call, because the difference of the limits mustn't
1089  * overflow int64. This is not checked.
1090  */
1091 static int64
1092 getrand(pg_prng_state *state, int64 min, int64 max)
1093 {
1094  return min + (int64) pg_prng_uint64_range(state, 0, max - min);
1095 }
1096 
1097 /*
1098  * random number generator: exponential distribution from min to max inclusive.
1099  * the parameter is so that the density of probability for the last cut-off max
1100  * value is exp(-parameter).
1101  */
1102 static int64
1103 getExponentialRand(pg_prng_state *state, int64 min, int64 max,
1104  double parameter)
1105 {
1106  double cut,
1107  uniform,
1108  rand;
1109 
1110  /* abort if wrong parameter, but must really be checked beforehand */
1111  Assert(parameter > 0.0);
1112  cut = exp(-parameter);
1113  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1114  uniform = 1.0 - pg_prng_double(state);
1115 
1116  /*
1117  * inner expression in (cut, 1] (if parameter > 0), rand in [0, 1)
1118  */
1119  Assert((1.0 - cut) != 0.0);
1120  rand = -log(cut + (1.0 - cut) * uniform) / parameter;
1121  /* return int64 random number within between min and max */
1122  return min + (int64) ((max - min + 1) * rand);
1123 }
1124 
1125 /* random number generator: gaussian distribution from min to max inclusive */
1126 static int64
1127 getGaussianRand(pg_prng_state *state, int64 min, int64 max,
1128  double parameter)
1129 {
1130  double stdev;
1131  double rand;
1132 
1133  /* abort if parameter is too low, but must really be checked beforehand */
1134  Assert(parameter >= MIN_GAUSSIAN_PARAM);
1135 
1136  /*
1137  * Get normally-distributed random number in the range -parameter <= stdev
1138  * < parameter.
1139  *
1140  * This loop is executed until the number is in the expected range.
1141  *
1142  * As the minimum parameter is 2.0, the probability of looping is low:
1143  * sqrt(-2 ln(r)) <= 2 => r >= e^{-2} ~ 0.135, then when taking the
1144  * average sinus multiplier as 2/pi, we have a 8.6% looping probability in
1145  * the worst case. For a parameter value of 5.0, the looping probability
1146  * is about e^{-5} * 2 / pi ~ 0.43%.
1147  */
1148  do
1149  {
1150  stdev = pg_prng_double_normal(state);
1151  }
1152  while (stdev < -parameter || stdev >= parameter);
1153 
1154  /* stdev is in [-parameter, parameter), normalization to [0,1) */
1155  rand = (stdev + parameter) / (parameter * 2.0);
1156 
1157  /* return int64 random number within between min and max */
1158  return min + (int64) ((max - min + 1) * rand);
1159 }
1160 
1161 /*
1162  * random number generator: generate a value, such that the series of values
1163  * will approximate a Poisson distribution centered on the given value.
1164  *
1165  * Individual results are rounded to integers, though the center value need
1166  * not be one.
1167  */
1168 static int64
1169 getPoissonRand(pg_prng_state *state, double center)
1170 {
1171  /*
1172  * Use inverse transform sampling to generate a value > 0, such that the
1173  * expected (i.e. average) value is the given argument.
1174  */
1175  double uniform;
1176 
1177  /* pg_prng_double value in [0, 1), uniform in (0, 1] */
1178  uniform = 1.0 - pg_prng_double(state);
1179 
1180  return (int64) (-log(uniform) * center + 0.5);
1181 }
1182 
1183 /*
1184  * Computing zipfian using rejection method, based on
1185  * "Non-Uniform Random Variate Generation",
1186  * Luc Devroye, p. 550-551, Springer 1986.
1187  *
1188  * This works for s > 1.0, but may perform badly for s very close to 1.0.
1189  */
1190 static int64
1191 computeIterativeZipfian(pg_prng_state *state, int64 n, double s)
1192 {
1193  double b = pow(2.0, s - 1.0);
1194  double x,
1195  t,
1196  u,
1197  v;
1198 
1199  /* Ensure n is sane */
1200  if (n <= 1)
1201  return 1;
1202 
1203  while (true)
1204  {
1205  /* random variates */
1206  u = pg_prng_double(state);
1207  v = pg_prng_double(state);
1208 
1209  x = floor(pow(u, -1.0 / (s - 1.0)));
1210 
1211  t = pow(1.0 + 1.0 / x, s - 1.0);
1212  /* reject if too large or out of bound */
1213  if (v * x * (t - 1.0) / (b - 1.0) <= t / b && x <= n)
1214  break;
1215  }
1216  return (int64) x;
1217 }
1218 
1219 /* random number generator: zipfian distribution from min to max inclusive */
1220 static int64
1221 getZipfianRand(pg_prng_state *state, int64 min, int64 max, double s)
1222 {
1223  int64 n = max - min + 1;
1224 
1225  /* abort if parameter is invalid */
1227 
1228  return min - 1 + computeIterativeZipfian(state, n, s);
1229 }
1230 
1231 /*
1232  * FNV-1a hash function
1233  */
1234 static int64
1235 getHashFnv1a(int64 val, uint64 seed)
1236 {
1237  int64 result;
1238  int i;
1239 
1240  result = FNV_OFFSET_BASIS ^ seed;
1241  for (i = 0; i < 8; ++i)
1242  {
1243  int32 octet = val & 0xff;
1244 
1245  val = val >> 8;
1246  result = result ^ octet;
1247  result = result * FNV_PRIME;
1248  }
1249 
1250  return result;
1251 }
1252 
1253 /*
1254  * Murmur2 hash function
1255  *
1256  * Based on original work of Austin Appleby
1257  * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash2.cpp
1258  */
1259 static int64
1260 getHashMurmur2(int64 val, uint64 seed)
1261 {
1262  uint64 result = seed ^ MM2_MUL_TIMES_8; /* sizeof(int64) */
1263  uint64 k = (uint64) val;
1264 
1265  k *= MM2_MUL;
1266  k ^= k >> MM2_ROT;
1267  k *= MM2_MUL;
1268 
1269  result ^= k;
1270  result *= MM2_MUL;
1271 
1272  result ^= result >> MM2_ROT;
1273  result *= MM2_MUL;
1274  result ^= result >> MM2_ROT;
1275 
1276  return (int64) result;
1277 }
1278 
1279 /*
1280  * Pseudorandom permutation function
1281  *
1282  * For small sizes, this generates each of the (size!) possible permutations
1283  * of integers in the range [0, size) with roughly equal probability. Once
1284  * the size is larger than 20, the number of possible permutations exceeds the
1285  * number of distinct states of the internal pseudorandom number generator,
1286  * and so not all possible permutations can be generated, but the permutations
1287  * chosen should continue to give the appearance of being random.
1288  *
1289  * THIS FUNCTION IS NOT CRYPTOGRAPHICALLY SECURE.
1290  * DO NOT USE FOR SUCH PURPOSE.
1291  */
1292 static int64
1293 permute(const int64 val, const int64 isize, const int64 seed)
1294 {
1295  /* using a high-end PRNG is probably overkill */
1297  uint64 size;
1298  uint64 v;
1299  int masklen;
1300  uint64 mask;
1301  int i;
1302 
1303  if (isize < 2)
1304  return 0; /* nothing to permute */
1305 
1306  /* Initialize prng state using the seed */
1307  pg_prng_seed(&state, (uint64) seed);
1308 
1309  /* Computations are performed on unsigned values */
1310  size = (uint64) isize;
1311  v = (uint64) val % size;
1312 
1313  /* Mask to work modulo largest power of 2 less than or equal to size */
1314  masklen = pg_leftmost_one_pos64(size);
1315  mask = (((uint64) 1) << masklen) - 1;
1316 
1317  /*
1318  * Permute the input value by applying several rounds of pseudorandom
1319  * bijective transformations. The intention here is to distribute each
1320  * input uniformly randomly across the range, and separate adjacent inputs
1321  * approximately uniformly randomly from each other, leading to a fairly
1322  * random overall choice of permutation.
1323  *
1324  * To separate adjacent inputs, we multiply by a random number modulo
1325  * (mask + 1), which is a power of 2. For this to be a bijection, the
1326  * multiplier must be odd. Since this is known to lead to less randomness
1327  * in the lower bits, we also apply a rotation that shifts the topmost bit
1328  * into the least significant bit. In the special cases where size <= 3,
1329  * mask = 1 and each of these operations is actually a no-op, so we also
1330  * XOR the value with a different random number to inject additional
1331  * randomness. Since the size is generally not a power of 2, we apply
1332  * this bijection on overlapping upper and lower halves of the input.
1333  *
1334  * To distribute the inputs uniformly across the range, we then also apply
1335  * a random offset modulo the full range.
1336  *
1337  * Taken together, these operations resemble a modified linear
1338  * congruential generator, as is commonly used in pseudorandom number
1339  * generators. The number of rounds is fairly arbitrary, but six has been
1340  * found empirically to give a fairly good tradeoff between performance
1341  * and uniform randomness. For small sizes it selects each of the (size!)
1342  * possible permutations with roughly equal probability. For larger
1343  * sizes, not all permutations can be generated, but the intended random
1344  * spread is still produced.
1345  */
1346  for (i = 0; i < 6; i++)
1347  {
1348  uint64 m,
1349  r,
1350  t;
1351 
1352  /* Random multiply (by an odd number), XOR and rotate of lower half */
1353  m = (pg_prng_uint64(&state) & mask) | 1;
1354  r = pg_prng_uint64(&state) & mask;
1355  if (v <= mask)
1356  {
1357  v = ((v * m) ^ r) & mask;
1358  v = ((v << 1) & mask) | (v >> (masklen - 1));
1359  }
1360 
1361  /* Random multiply (by an odd number), XOR and rotate of upper half */
1362  m = (pg_prng_uint64(&state) & mask) | 1;
1363  r = pg_prng_uint64(&state) & mask;
1364  t = size - 1 - v;
1365  if (t <= mask)
1366  {
1367  t = ((t * m) ^ r) & mask;
1368  t = ((t << 1) & mask) | (t >> (masklen - 1));
1369  v = size - 1 - t;
1370  }
1371 
1372  /* Random offset */
1373  r = pg_prng_uint64_range(&state, 0, size - 1);
1374  v = (v + r) % size;
1375  }
1376 
1377  return (int64) v;
1378 }
1379 
1380 /*
1381  * Initialize the given SimpleStats struct to all zeroes
1382  */
1383 static void
1385 {
1386  memset(ss, 0, sizeof(SimpleStats));
1387 }
1388 
1389 /*
1390  * Accumulate one value into a SimpleStats struct.
1391  */
1392 static void
1393 addToSimpleStats(SimpleStats *ss, double val)
1394 {
1395  if (ss->count == 0 || val < ss->min)
1396  ss->min = val;
1397  if (ss->count == 0 || val > ss->max)
1398  ss->max = val;
1399  ss->count++;
1400  ss->sum += val;
1401  ss->sum2 += val * val;
1402 }
1403 
1404 /*
1405  * Merge two SimpleStats objects
1406  */
1407 static void
1409 {
1410  if (acc->count == 0 || ss->min < acc->min)
1411  acc->min = ss->min;
1412  if (acc->count == 0 || ss->max > acc->max)
1413  acc->max = ss->max;
1414  acc->count += ss->count;
1415  acc->sum += ss->sum;
1416  acc->sum2 += ss->sum2;
1417 }
1418 
1419 /*
1420  * Initialize a StatsData struct to mostly zeroes, with its start time set to
1421  * the given value.
1422  */
1423 static void
1425 {
1426  sd->start_time = start;
1427  sd->cnt = 0;
1428  sd->skipped = 0;
1429  sd->retries = 0;
1430  sd->retried = 0;
1431  sd->serialization_failures = 0;
1432  sd->deadlock_failures = 0;
1433  initSimpleStats(&sd->latency);
1434  initSimpleStats(&sd->lag);
1435 }
1436 
1437 /*
1438  * Accumulate one additional item into the given stats object.
1439  */
1440 static void
1441 accumStats(StatsData *stats, bool skipped, double lat, double lag,
1442  EStatus estatus, int64 tries)
1443 {
1444  /* Record the skipped transaction */
1445  if (skipped)
1446  {
1447  /* no latency to record on skipped transactions */
1448  stats->skipped++;
1449  return;
1450  }
1451 
1452  /*
1453  * Record the number of retries regardless of whether the transaction was
1454  * successful or failed.
1455  */
1456  if (tries > 1)
1457  {
1458  stats->retries += (tries - 1);
1459  stats->retried++;
1460  }
1461 
1462  switch (estatus)
1463  {
1464  /* Record the successful transaction */
1465  case ESTATUS_NO_ERROR:
1466  stats->cnt++;
1467 
1468  addToSimpleStats(&stats->latency, lat);
1469 
1470  /* and possibly the same for schedule lag */
1471  if (throttle_delay)
1472  addToSimpleStats(&stats->lag, lag);
1473  break;
1474 
1475  /* Record the failed transaction */
1477  stats->serialization_failures++;
1478  break;
1480  stats->deadlock_failures++;
1481  break;
1482  default:
1483  /* internal error which should never occur */
1484  pg_fatal("unexpected error status: %d", estatus);
1485  }
1486 }
1487 
1488 /* call PQexec() and exit() on failure */
1489 static void
1490 executeStatement(PGconn *con, const char *sql)
1491 {
1492  PGresult *res;
1493 
1494  res = PQexec(con, sql);
1496  {
1497  pg_log_error("query failed: %s", PQerrorMessage(con));
1498  pg_log_error_detail("Query was: %s", sql);
1499  exit(1);
1500  }
1501  PQclear(res);
1502 }
1503 
1504 /* call PQexec() and complain, but without exiting, on failure */
1505 static void
1506 tryExecuteStatement(PGconn *con, const char *sql)
1507 {
1508  PGresult *res;
1509 
1510  res = PQexec(con, sql);
1512  {
1513  pg_log_error("%s", PQerrorMessage(con));
1514  pg_log_error_detail("(ignoring this error and continuing anyway)");
1515  }
1516  PQclear(res);
1517 }
1518 
1519 /* set up a connection to the backend */
1520 static PGconn *
1521 doConnect(void)
1522 {
1523  PGconn *conn;
1524  bool new_pass;
1525  static char *password = NULL;
1526 
1527  /*
1528  * Start the connection. Loop until we have a password if requested by
1529  * backend.
1530  */
1531  do
1532  {
1533 #define PARAMS_ARRAY_SIZE 7
1534 
1535  const char *keywords[PARAMS_ARRAY_SIZE];
1536  const char *values[PARAMS_ARRAY_SIZE];
1537 
1538  keywords[0] = "host";
1539  values[0] = pghost;
1540  keywords[1] = "port";
1541  values[1] = pgport;
1542  keywords[2] = "user";
1543  values[2] = username;
1544  keywords[3] = "password";
1545  values[3] = password;
1546  keywords[4] = "dbname";
1547  values[4] = dbName;
1548  keywords[5] = "fallback_application_name";
1549  values[5] = progname;
1550  keywords[6] = NULL;
1551  values[6] = NULL;
1552 
1553  new_pass = false;
1554 
1555  conn = PQconnectdbParams(keywords, values, true);
1556 
1557  if (!conn)
1558  {
1559  pg_log_error("connection to database \"%s\" failed", dbName);
1560  return NULL;
1561  }
1562 
1563  if (PQstatus(conn) == CONNECTION_BAD &&
1565  !password)
1566  {
1567  PQfinish(conn);
1568  password = simple_prompt("Password: ", false);
1569  new_pass = true;
1570  }
1571  } while (new_pass);
1572 
1573  /* check to see that the backend connection was successfully made */
1574  if (PQstatus(conn) == CONNECTION_BAD)
1575  {
1577  PQfinish(conn);
1578  return NULL;
1579  }
1580 
1581  return conn;
1582 }
1583 
1584 /* qsort comparator for Variable array */
1585 static int
1586 compareVariableNames(const void *v1, const void *v2)
1587 {
1588  return strcmp(((const Variable *) v1)->name,
1589  ((const Variable *) v2)->name);
1590 }
1591 
1592 /* Locate a variable by name; returns NULL if unknown */
1593 static Variable *
1594 lookupVariable(Variables *variables, char *name)
1595 {
1596  Variable key;
1597 
1598  /* On some versions of Solaris, bsearch of zero items dumps core */
1599  if (variables->nvars <= 0)
1600  return NULL;
1601 
1602  /* Sort if we have to */
1603  if (!variables->vars_sorted)
1604  {
1605  qsort(variables->vars, variables->nvars, sizeof(Variable),
1607  variables->vars_sorted = true;
1608  }
1609 
1610  /* Now we can search */
1611  key.name = name;
1612  return (Variable *) bsearch(&key,
1613  variables->vars,
1614  variables->nvars,
1615  sizeof(Variable),
1617 }
1618 
1619 /* Get the value of a variable, in string form; returns NULL if unknown */
1620 static char *
1621 getVariable(Variables *variables, char *name)
1622 {
1623  Variable *var;
1624  char stringform[64];
1625 
1626  var = lookupVariable(variables, name);
1627  if (var == NULL)
1628  return NULL; /* not found */
1629 
1630  if (var->svalue)
1631  return var->svalue; /* we have it in string form */
1632 
1633  /* We need to produce a string equivalent of the value */
1634  Assert(var->value.type != PGBT_NO_VALUE);
1635  if (var->value.type == PGBT_NULL)
1636  snprintf(stringform, sizeof(stringform), "NULL");
1637  else if (var->value.type == PGBT_BOOLEAN)
1638  snprintf(stringform, sizeof(stringform),
1639  "%s", var->value.u.bval ? "true" : "false");
1640  else if (var->value.type == PGBT_INT)
1641  snprintf(stringform, sizeof(stringform),
1642  INT64_FORMAT, var->value.u.ival);
1643  else if (var->value.type == PGBT_DOUBLE)
1644  snprintf(stringform, sizeof(stringform),
1645  "%.*g", DBL_DIG, var->value.u.dval);
1646  else /* internal error, unexpected type */
1647  Assert(0);
1648  var->svalue = pg_strdup(stringform);
1649  return var->svalue;
1650 }
1651 
1652 /* Try to convert variable to a value; return false on failure */
1653 static bool
1655 {
1656  size_t slen;
1657 
1658  if (var->value.type != PGBT_NO_VALUE)
1659  return true; /* no work */
1660 
1661  slen = strlen(var->svalue);
1662 
1663  if (slen == 0)
1664  /* what should it do on ""? */
1665  return false;
1666 
1667  if (pg_strcasecmp(var->svalue, "null") == 0)
1668  {
1669  setNullValue(&var->value);
1670  }
1671 
1672  /*
1673  * accept prefixes such as y, ye, n, no... but not for "o". 0/1 are
1674  * recognized later as an int, which is converted to bool if needed.
1675  */
1676  else if (pg_strncasecmp(var->svalue, "true", slen) == 0 ||
1677  pg_strncasecmp(var->svalue, "yes", slen) == 0 ||
1678  pg_strcasecmp(var->svalue, "on") == 0)
1679  {
1680  setBoolValue(&var->value, true);
1681  }
1682  else if (pg_strncasecmp(var->svalue, "false", slen) == 0 ||
1683  pg_strncasecmp(var->svalue, "no", slen) == 0 ||
1684  pg_strcasecmp(var->svalue, "off") == 0 ||
1685  pg_strcasecmp(var->svalue, "of") == 0)
1686  {
1687  setBoolValue(&var->value, false);
1688  }
1689  else if (is_an_int(var->svalue))
1690  {
1691  /* if it looks like an int, it must be an int without overflow */
1692  int64 iv;
1693 
1694  if (!strtoint64(var->svalue, false, &iv))
1695  return false;
1696 
1697  setIntValue(&var->value, iv);
1698  }
1699  else /* type should be double */
1700  {
1701  double dv;
1702 
1703  if (!strtodouble(var->svalue, true, &dv))
1704  {
1705  pg_log_error("malformed variable \"%s\" value: \"%s\"",
1706  var->name, var->svalue);
1707  return false;
1708  }
1709  setDoubleValue(&var->value, dv);
1710  }
1711  return true;
1712 }
1713 
1714 /*
1715  * Check whether a variable's name is allowed.
1716  *
1717  * We allow any non-ASCII character, as well as ASCII letters, digits, and
1718  * underscore.
1719  *
1720  * Keep this in sync with the definitions of variable name characters in
1721  * "src/fe_utils/psqlscan.l", "src/bin/psql/psqlscanslash.l" and
1722  * "src/bin/pgbench/exprscan.l". Also see parseVariable(), below.
1723  *
1724  * Note: this static function is copied from "src/bin/psql/variables.c"
1725  * but changed to disallow variable names starting with a digit.
1726  */
1727 static bool
1728 valid_variable_name(const char *name)
1729 {
1730  const unsigned char *ptr = (const unsigned char *) name;
1731 
1732  /* Mustn't be zero-length */
1733  if (*ptr == '\0')
1734  return false;
1735 
1736  /* must not start with [0-9] */
1737  if (IS_HIGHBIT_SET(*ptr) ||
1738  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1739  "_", *ptr) != NULL)
1740  ptr++;
1741  else
1742  return false;
1743 
1744  /* remaining characters can include [0-9] */
1745  while (*ptr)
1746  {
1747  if (IS_HIGHBIT_SET(*ptr) ||
1748  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1749  "_0123456789", *ptr) != NULL)
1750  ptr++;
1751  else
1752  return false;
1753  }
1754 
1755  return true;
1756 }
1757 
1758 /*
1759  * Make sure there is enough space for 'needed' more variable in the variables
1760  * array.
1761  */
1762 static void
1763 enlargeVariables(Variables *variables, int needed)
1764 {
1765  /* total number of variables required now */
1766  needed += variables->nvars;
1767 
1768  if (variables->max_vars < needed)
1769  {
1770  variables->max_vars = needed + VARIABLES_ALLOC_MARGIN;
1771  variables->vars = (Variable *)
1772  pg_realloc(variables->vars, variables->max_vars * sizeof(Variable));
1773  }
1774 }
1775 
1776 /*
1777  * Lookup a variable by name, creating it if need be.
1778  * Caller is expected to assign a value to the variable.
1779  * Returns NULL on failure (bad name).
1780  */
1781 static Variable *
1782 lookupCreateVariable(Variables *variables, const char *context, char *name)
1783 {
1784  Variable *var;
1785 
1786  var = lookupVariable(variables, name);
1787  if (var == NULL)
1788  {
1789  /*
1790  * Check for the name only when declaring a new variable to avoid
1791  * overhead.
1792  */
1793  if (!valid_variable_name(name))
1794  {
1795  pg_log_error("%s: invalid variable name: \"%s\"", context, name);
1796  return NULL;
1797  }
1798 
1799  /* Create variable at the end of the array */
1800  enlargeVariables(variables, 1);
1801 
1802  var = &(variables->vars[variables->nvars]);
1803 
1804  var->name = pg_strdup(name);
1805  var->svalue = NULL;
1806  /* caller is expected to initialize remaining fields */
1807 
1808  variables->nvars++;
1809  /* we don't re-sort the array till we have to */
1810  variables->vars_sorted = false;
1811  }
1812 
1813  return var;
1814 }
1815 
1816 /* Assign a string value to a variable, creating it if need be */
1817 /* Returns false on failure (bad name) */
1818 static bool
1819 putVariable(Variables *variables, const char *context, char *name,
1820  const char *value)
1821 {
1822  Variable *var;
1823  char *val;
1824 
1825  var = lookupCreateVariable(variables, context, name);
1826  if (!var)
1827  return false;
1828 
1829  /* dup then free, in case value is pointing at this variable */
1830  val = pg_strdup(value);
1831 
1832  free(var->svalue);
1833  var->svalue = val;
1834  var->value.type = PGBT_NO_VALUE;
1835 
1836  return true;
1837 }
1838 
1839 /* Assign a value to a variable, creating it if need be */
1840 /* Returns false on failure (bad name) */
1841 static bool
1842 putVariableValue(Variables *variables, const char *context, char *name,
1843  const PgBenchValue *value)
1844 {
1845  Variable *var;
1846 
1847  var = lookupCreateVariable(variables, context, name);
1848  if (!var)
1849  return false;
1850 
1851  free(var->svalue);
1852  var->svalue = NULL;
1853  var->value = *value;
1854 
1855  return true;
1856 }
1857 
1858 /* Assign an integer value to a variable, creating it if need be */
1859 /* Returns false on failure (bad name) */
1860 static bool
1861 putVariableInt(Variables *variables, const char *context, char *name,
1862  int64 value)
1863 {
1864  PgBenchValue val;
1865 
1866  setIntValue(&val, value);
1867  return putVariableValue(variables, context, name, &val);
1868 }
1869 
1870 /*
1871  * Parse a possible variable reference (:varname).
1872  *
1873  * "sql" points at a colon. If what follows it looks like a valid
1874  * variable name, return a malloc'd string containing the variable name,
1875  * and set *eaten to the number of characters consumed (including the colon).
1876  * Otherwise, return NULL.
1877  */
1878 static char *
1879 parseVariable(const char *sql, int *eaten)
1880 {
1881  int i = 1; /* starting at 1 skips the colon */
1882  char *name;
1883 
1884  /* keep this logic in sync with valid_variable_name() */
1885  if (IS_HIGHBIT_SET(sql[i]) ||
1886  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1887  "_", sql[i]) != NULL)
1888  i++;
1889  else
1890  return NULL;
1891 
1892  while (IS_HIGHBIT_SET(sql[i]) ||
1893  strchr("ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"
1894  "_0123456789", sql[i]) != NULL)
1895  i++;
1896 
1897  name = pg_malloc(i);
1898  memcpy(name, &sql[1], i - 1);
1899  name[i - 1] = '\0';
1900 
1901  *eaten = i;
1902  return name;
1903 }
1904 
1905 static char *
1906 replaceVariable(char **sql, char *param, int len, char *value)
1907 {
1908  int valueln = strlen(value);
1909 
1910  if (valueln > len)
1911  {
1912  size_t offset = param - *sql;
1913 
1914  *sql = pg_realloc(*sql, strlen(*sql) - len + valueln + 1);
1915  param = *sql + offset;
1916  }
1917 
1918  if (valueln != len)
1919  memmove(param + valueln, param + len, strlen(param + len) + 1);
1920  memcpy(param, value, valueln);
1921 
1922  return param + valueln;
1923 }
1924 
1925 static char *
1926 assignVariables(Variables *variables, char *sql)
1927 {
1928  char *p,
1929  *name,
1930  *val;
1931 
1932  p = sql;
1933  while ((p = strchr(p, ':')) != NULL)
1934  {
1935  int eaten;
1936 
1937  name = parseVariable(p, &eaten);
1938  if (name == NULL)
1939  {
1940  while (*p == ':')
1941  {
1942  p++;
1943  }
1944  continue;
1945  }
1946 
1947  val = getVariable(variables, name);
1948  free(name);
1949  if (val == NULL)
1950  {
1951  p++;
1952  continue;
1953  }
1954 
1955  p = replaceVariable(&sql, p, eaten, val);
1956  }
1957 
1958  return sql;
1959 }
1960 
1961 static void
1962 getQueryParams(Variables *variables, const Command *command,
1963  const char **params)
1964 {
1965  int i;
1966 
1967  for (i = 0; i < command->argc - 1; i++)
1968  params[i] = getVariable(variables, command->argv[i + 1]);
1969 }
1970 
1971 static char *
1973 {
1974  if (pval->type == PGBT_NO_VALUE)
1975  return "none";
1976  else if (pval->type == PGBT_NULL)
1977  return "null";
1978  else if (pval->type == PGBT_INT)
1979  return "int";
1980  else if (pval->type == PGBT_DOUBLE)
1981  return "double";
1982  else if (pval->type == PGBT_BOOLEAN)
1983  return "boolean";
1984  else
1985  {
1986  /* internal error, should never get there */
1987  Assert(false);
1988  return NULL;
1989  }
1990 }
1991 
1992 /* get a value as a boolean, or tell if there is a problem */
1993 static bool
1994 coerceToBool(PgBenchValue *pval, bool *bval)
1995 {
1996  if (pval->type == PGBT_BOOLEAN)
1997  {
1998  *bval = pval->u.bval;
1999  return true;
2000  }
2001  else /* NULL, INT or DOUBLE */
2002  {
2003  pg_log_error("cannot coerce %s to boolean", valueTypeName(pval));
2004  *bval = false; /* suppress uninitialized-variable warnings */
2005  return false;
2006  }
2007 }
2008 
2009 /*
2010  * Return true or false from an expression for conditional purposes.
2011  * Non zero numerical values are true, zero and NULL are false.
2012  */
2013 static bool
2014 valueTruth(PgBenchValue *pval)
2015 {
2016  switch (pval->type)
2017  {
2018  case PGBT_NULL:
2019  return false;
2020  case PGBT_BOOLEAN:
2021  return pval->u.bval;
2022  case PGBT_INT:
2023  return pval->u.ival != 0;
2024  case PGBT_DOUBLE:
2025  return pval->u.dval != 0.0;
2026  default:
2027  /* internal error, unexpected type */
2028  Assert(0);
2029  return false;
2030  }
2031 }
2032 
2033 /* get a value as an int, tell if there is a problem */
2034 static bool
2035 coerceToInt(PgBenchValue *pval, int64 *ival)
2036 {
2037  if (pval->type == PGBT_INT)
2038  {
2039  *ival = pval->u.ival;
2040  return true;
2041  }
2042  else if (pval->type == PGBT_DOUBLE)
2043  {
2044  double dval = rint(pval->u.dval);
2045 
2046  if (isnan(dval) || !FLOAT8_FITS_IN_INT64(dval))
2047  {
2048  pg_log_error("double to int overflow for %f", dval);
2049  return false;
2050  }
2051  *ival = (int64) dval;
2052  return true;
2053  }
2054  else /* BOOLEAN or NULL */
2055  {
2056  pg_log_error("cannot coerce %s to int", valueTypeName(pval));
2057  return false;
2058  }
2059 }
2060 
2061 /* get a value as a double, or tell if there is a problem */
2062 static bool
2063 coerceToDouble(PgBenchValue *pval, double *dval)
2064 {
2065  if (pval->type == PGBT_DOUBLE)
2066  {
2067  *dval = pval->u.dval;
2068  return true;
2069  }
2070  else if (pval->type == PGBT_INT)
2071  {
2072  *dval = (double) pval->u.ival;
2073  return true;
2074  }
2075  else /* BOOLEAN or NULL */
2076  {
2077  pg_log_error("cannot coerce %s to double", valueTypeName(pval));
2078  return false;
2079  }
2080 }
2081 
2082 /* assign a null value */
2083 static void
2085 {
2086  pv->type = PGBT_NULL;
2087  pv->u.ival = 0;
2088 }
2089 
2090 /* assign a boolean value */
2091 static void
2092 setBoolValue(PgBenchValue *pv, bool bval)
2093 {
2094  pv->type = PGBT_BOOLEAN;
2095  pv->u.bval = bval;
2096 }
2097 
2098 /* assign an integer value */
2099 static void
2100 setIntValue(PgBenchValue *pv, int64 ival)
2101 {
2102  pv->type = PGBT_INT;
2103  pv->u.ival = ival;
2104 }
2105 
2106 /* assign a double value */
2107 static void
2108 setDoubleValue(PgBenchValue *pv, double dval)
2109 {
2110  pv->type = PGBT_DOUBLE;
2111  pv->u.dval = dval;
2112 }
2113 
2114 static bool
2116 {
2117  return func == PGBENCH_AND || func == PGBENCH_OR || func == PGBENCH_CASE;
2118 }
2119 
2120 /* lazy evaluation of some functions */
2121 static bool
2122 evalLazyFunc(CState *st,
2124 {
2125  PgBenchValue a1,
2126  a2;
2127  bool ba1,
2128  ba2;
2129 
2130  Assert(isLazyFunc(func) && args != NULL && args->next != NULL);
2131 
2132  /* args points to first condition */
2133  if (!evaluateExpr(st, args->expr, &a1))
2134  return false;
2135 
2136  /* second condition for AND/OR and corresponding branch for CASE */
2137  args = args->next;
2138 
2139  switch (func)
2140  {
2141  case PGBENCH_AND:
2142  if (a1.type == PGBT_NULL)
2143  {
2144  setNullValue(retval);
2145  return true;
2146  }
2147 
2148  if (!coerceToBool(&a1, &ba1))
2149  return false;
2150 
2151  if (!ba1)
2152  {
2153  setBoolValue(retval, false);
2154  return true;
2155  }
2156 
2157  if (!evaluateExpr(st, args->expr, &a2))
2158  return false;
2159 
2160  if (a2.type == PGBT_NULL)
2161  {
2162  setNullValue(retval);
2163  return true;
2164  }
2165  else if (!coerceToBool(&a2, &ba2))
2166  return false;
2167  else
2168  {
2169  setBoolValue(retval, ba2);
2170  return true;
2171  }
2172 
2173  return true;
2174 
2175  case PGBENCH_OR:
2176 
2177  if (a1.type == PGBT_NULL)
2178  {
2179  setNullValue(retval);
2180  return true;
2181  }
2182 
2183  if (!coerceToBool(&a1, &ba1))
2184  return false;
2185 
2186  if (ba1)
2187  {
2188  setBoolValue(retval, true);
2189  return true;
2190  }
2191 
2192  if (!evaluateExpr(st, args->expr, &a2))
2193  return false;
2194 
2195  if (a2.type == PGBT_NULL)
2196  {
2197  setNullValue(retval);
2198  return true;
2199  }
2200  else if (!coerceToBool(&a2, &ba2))
2201  return false;
2202  else
2203  {
2204  setBoolValue(retval, ba2);
2205  return true;
2206  }
2207 
2208  case PGBENCH_CASE:
2209  /* when true, execute branch */
2210  if (valueTruth(&a1))
2211  return evaluateExpr(st, args->expr, retval);
2212 
2213  /* now args contains next condition or final else expression */
2214  args = args->next;
2215 
2216  /* final else case? */
2217  if (args->next == NULL)
2218  return evaluateExpr(st, args->expr, retval);
2219 
2220  /* no, another when, proceed */
2221  return evalLazyFunc(st, PGBENCH_CASE, args, retval);
2222 
2223  default:
2224  /* internal error, cannot get here */
2225  Assert(0);
2226  break;
2227  }
2228  return false;
2229 }
2230 
2231 /* maximum number of function arguments */
2232 #define MAX_FARGS 16
2233 
2234 /*
2235  * Recursive evaluation of standard functions,
2236  * which do not require lazy evaluation.
2237  */
2238 static bool
2241  PgBenchValue *retval)
2242 {
2243  /* evaluate all function arguments */
2244  int nargs = 0;
2245  PgBenchExprLink *l = args;
2246  bool has_null = false;
2247 
2248  /*
2249  * This value is double braced to workaround GCC bug 53119, which seems to
2250  * exist at least on gcc (Debian 4.7.2-5) 4.7.2, 32-bit.
2251  */
2252  PgBenchValue vargs[MAX_FARGS] = {{0}};
2253 
2254  for (nargs = 0; nargs < MAX_FARGS && l != NULL; nargs++, l = l->next)
2255  {
2256  if (!evaluateExpr(st, l->expr, &vargs[nargs]))
2257  return false;
2258  has_null |= vargs[nargs].type == PGBT_NULL;
2259  }
2260 
2261  if (l != NULL)
2262  {
2263  pg_log_error("too many function arguments, maximum is %d", MAX_FARGS);
2264  return false;
2265  }
2266 
2267  /* NULL arguments */
2268  if (has_null && func != PGBENCH_IS && func != PGBENCH_DEBUG)
2269  {
2270  setNullValue(retval);
2271  return true;
2272  }
2273 
2274  /* then evaluate function */
2275  switch (func)
2276  {
2277  /* overloaded operators */
2278  case PGBENCH_ADD:
2279  case PGBENCH_SUB:
2280  case PGBENCH_MUL:
2281  case PGBENCH_DIV:
2282  case PGBENCH_MOD:
2283  case PGBENCH_EQ:
2284  case PGBENCH_NE:
2285  case PGBENCH_LE:
2286  case PGBENCH_LT:
2287  {
2288  PgBenchValue *lval = &vargs[0],
2289  *rval = &vargs[1];
2290 
2291  Assert(nargs == 2);
2292 
2293  /* overloaded type management, double if some double */
2294  if ((lval->type == PGBT_DOUBLE ||
2295  rval->type == PGBT_DOUBLE) && func != PGBENCH_MOD)
2296  {
2297  double ld,
2298  rd;
2299 
2300  if (!coerceToDouble(lval, &ld) ||
2301  !coerceToDouble(rval, &rd))
2302  return false;
2303 
2304  switch (func)
2305  {
2306  case PGBENCH_ADD:
2307  setDoubleValue(retval, ld + rd);
2308  return true;
2309 
2310  case PGBENCH_SUB:
2311  setDoubleValue(retval, ld - rd);
2312  return true;
2313 
2314  case PGBENCH_MUL:
2315  setDoubleValue(retval, ld * rd);
2316  return true;
2317 
2318  case PGBENCH_DIV:
2319  setDoubleValue(retval, ld / rd);
2320  return true;
2321 
2322  case PGBENCH_EQ:
2323  setBoolValue(retval, ld == rd);
2324  return true;
2325 
2326  case PGBENCH_NE:
2327  setBoolValue(retval, ld != rd);
2328  return true;
2329 
2330  case PGBENCH_LE:
2331  setBoolValue(retval, ld <= rd);
2332  return true;
2333 
2334  case PGBENCH_LT:
2335  setBoolValue(retval, ld < rd);
2336  return true;
2337 
2338  default:
2339  /* cannot get here */
2340  Assert(0);
2341  }
2342  }
2343  else /* we have integer operands, or % */
2344  {
2345  int64 li,
2346  ri,
2347  res;
2348 
2349  if (!coerceToInt(lval, &li) ||
2350  !coerceToInt(rval, &ri))
2351  return false;
2352 
2353  switch (func)
2354  {
2355  case PGBENCH_ADD:
2356  if (pg_add_s64_overflow(li, ri, &res))
2357  {
2358  pg_log_error("bigint add out of range");
2359  return false;
2360  }
2361  setIntValue(retval, res);
2362  return true;
2363 
2364  case PGBENCH_SUB:
2365  if (pg_sub_s64_overflow(li, ri, &res))
2366  {
2367  pg_log_error("bigint sub out of range");
2368  return false;
2369  }
2370  setIntValue(retval, res);
2371  return true;
2372 
2373  case PGBENCH_MUL:
2374  if (pg_mul_s64_overflow(li, ri, &res))
2375  {
2376  pg_log_error("bigint mul out of range");
2377  return false;
2378  }
2379  setIntValue(retval, res);
2380  return true;
2381 
2382  case PGBENCH_EQ:
2383  setBoolValue(retval, li == ri);
2384  return true;
2385 
2386  case PGBENCH_NE:
2387  setBoolValue(retval, li != ri);
2388  return true;
2389 
2390  case PGBENCH_LE:
2391  setBoolValue(retval, li <= ri);
2392  return true;
2393 
2394  case PGBENCH_LT:
2395  setBoolValue(retval, li < ri);
2396  return true;
2397 
2398  case PGBENCH_DIV:
2399  case PGBENCH_MOD:
2400  if (ri == 0)
2401  {
2402  pg_log_error("division by zero");
2403  return false;
2404  }
2405  /* special handling of -1 divisor */
2406  if (ri == -1)
2407  {
2408  if (func == PGBENCH_DIV)
2409  {
2410  /* overflow check (needed for INT64_MIN) */
2411  if (li == PG_INT64_MIN)
2412  {
2413  pg_log_error("bigint div out of range");
2414  return false;
2415  }
2416  else
2417  setIntValue(retval, -li);
2418  }
2419  else
2420  setIntValue(retval, 0);
2421  return true;
2422  }
2423  /* else divisor is not -1 */
2424  if (func == PGBENCH_DIV)
2425  setIntValue(retval, li / ri);
2426  else /* func == PGBENCH_MOD */
2427  setIntValue(retval, li % ri);
2428 
2429  return true;
2430 
2431  default:
2432  /* cannot get here */
2433  Assert(0);
2434  }
2435  }
2436 
2437  Assert(0);
2438  return false; /* NOTREACHED */
2439  }
2440 
2441  /* integer bitwise operators */
2442  case PGBENCH_BITAND:
2443  case PGBENCH_BITOR:
2444  case PGBENCH_BITXOR:
2445  case PGBENCH_LSHIFT:
2446  case PGBENCH_RSHIFT:
2447  {
2448  int64 li,
2449  ri;
2450 
2451  if (!coerceToInt(&vargs[0], &li) || !coerceToInt(&vargs[1], &ri))
2452  return false;
2453 
2454  if (func == PGBENCH_BITAND)
2455  setIntValue(retval, li & ri);
2456  else if (func == PGBENCH_BITOR)
2457  setIntValue(retval, li | ri);
2458  else if (func == PGBENCH_BITXOR)
2459  setIntValue(retval, li ^ ri);
2460  else if (func == PGBENCH_LSHIFT)
2461  setIntValue(retval, li << ri);
2462  else if (func == PGBENCH_RSHIFT)
2463  setIntValue(retval, li >> ri);
2464  else /* cannot get here */
2465  Assert(0);
2466 
2467  return true;
2468  }
2469 
2470  /* logical operators */
2471  case PGBENCH_NOT:
2472  {
2473  bool b;
2474 
2475  if (!coerceToBool(&vargs[0], &b))
2476  return false;
2477 
2478  setBoolValue(retval, !b);
2479  return true;
2480  }
2481 
2482  /* no arguments */
2483  case PGBENCH_PI:
2484  setDoubleValue(retval, M_PI);
2485  return true;
2486 
2487  /* 1 overloaded argument */
2488  case PGBENCH_ABS:
2489  {
2490  PgBenchValue *varg = &vargs[0];
2491 
2492  Assert(nargs == 1);
2493 
2494  if (varg->type == PGBT_INT)
2495  {
2496  int64 i = varg->u.ival;
2497 
2498  setIntValue(retval, i < 0 ? -i : i);
2499  }
2500  else
2501  {
2502  double d = varg->u.dval;
2503 
2504  Assert(varg->type == PGBT_DOUBLE);
2505  setDoubleValue(retval, d < 0.0 ? -d : d);
2506  }
2507 
2508  return true;
2509  }
2510 
2511  case PGBENCH_DEBUG:
2512  {
2513  PgBenchValue *varg = &vargs[0];
2514 
2515  Assert(nargs == 1);
2516 
2517  fprintf(stderr, "debug(script=%d,command=%d): ",
2518  st->use_file, st->command + 1);
2519 
2520  if (varg->type == PGBT_NULL)
2521  fprintf(stderr, "null\n");
2522  else if (varg->type == PGBT_BOOLEAN)
2523  fprintf(stderr, "boolean %s\n", varg->u.bval ? "true" : "false");
2524  else if (varg->type == PGBT_INT)
2525  fprintf(stderr, "int " INT64_FORMAT "\n", varg->u.ival);
2526  else if (varg->type == PGBT_DOUBLE)
2527  fprintf(stderr, "double %.*g\n", DBL_DIG, varg->u.dval);
2528  else /* internal error, unexpected type */
2529  Assert(0);
2530 
2531  *retval = *varg;
2532 
2533  return true;
2534  }
2535 
2536  /* 1 double argument */
2537  case PGBENCH_DOUBLE:
2538  case PGBENCH_SQRT:
2539  case PGBENCH_LN:
2540  case PGBENCH_EXP:
2541  {
2542  double dval;
2543 
2544  Assert(nargs == 1);
2545 
2546  if (!coerceToDouble(&vargs[0], &dval))
2547  return false;
2548 
2549  if (func == PGBENCH_SQRT)
2550  dval = sqrt(dval);
2551  else if (func == PGBENCH_LN)
2552  dval = log(dval);
2553  else if (func == PGBENCH_EXP)
2554  dval = exp(dval);
2555  /* else is cast: do nothing */
2556 
2557  setDoubleValue(retval, dval);
2558  return true;
2559  }
2560 
2561  /* 1 int argument */
2562  case PGBENCH_INT:
2563  {
2564  int64 ival;
2565 
2566  Assert(nargs == 1);
2567 
2568  if (!coerceToInt(&vargs[0], &ival))
2569  return false;
2570 
2571  setIntValue(retval, ival);
2572  return true;
2573  }
2574 
2575  /* variable number of arguments */
2576  case PGBENCH_LEAST:
2577  case PGBENCH_GREATEST:
2578  {
2579  bool havedouble;
2580  int i;
2581 
2582  Assert(nargs >= 1);
2583 
2584  /* need double result if any input is double */
2585  havedouble = false;
2586  for (i = 0; i < nargs; i++)
2587  {
2588  if (vargs[i].type == PGBT_DOUBLE)
2589  {
2590  havedouble = true;
2591  break;
2592  }
2593  }
2594  if (havedouble)
2595  {
2596  double extremum;
2597 
2598  if (!coerceToDouble(&vargs[0], &extremum))
2599  return false;
2600  for (i = 1; i < nargs; i++)
2601  {
2602  double dval;
2603 
2604  if (!coerceToDouble(&vargs[i], &dval))
2605  return false;
2606  if (func == PGBENCH_LEAST)
2607  extremum = Min(extremum, dval);
2608  else
2609  extremum = Max(extremum, dval);
2610  }
2611  setDoubleValue(retval, extremum);
2612  }
2613  else
2614  {
2615  int64 extremum;
2616 
2617  if (!coerceToInt(&vargs[0], &extremum))
2618  return false;
2619  for (i = 1; i < nargs; i++)
2620  {
2621  int64 ival;
2622 
2623  if (!coerceToInt(&vargs[i], &ival))
2624  return false;
2625  if (func == PGBENCH_LEAST)
2626  extremum = Min(extremum, ival);
2627  else
2628  extremum = Max(extremum, ival);
2629  }
2630  setIntValue(retval, extremum);
2631  }
2632  return true;
2633  }
2634 
2635  /* random functions */
2636  case PGBENCH_RANDOM:
2640  {
2641  int64 imin,
2642  imax,
2643  delta;
2644 
2645  Assert(nargs >= 2);
2646 
2647  if (!coerceToInt(&vargs[0], &imin) ||
2648  !coerceToInt(&vargs[1], &imax))
2649  return false;
2650 
2651  /* check random range */
2652  if (unlikely(imin > imax))
2653  {
2654  pg_log_error("empty range given to random");
2655  return false;
2656  }
2657  else if (unlikely(pg_sub_s64_overflow(imax, imin, &delta) ||
2658  pg_add_s64_overflow(delta, 1, &delta)))
2659  {
2660  /* prevent int overflows in random functions */
2661  pg_log_error("random range is too large");
2662  return false;
2663  }
2664 
2665  if (func == PGBENCH_RANDOM)
2666  {
2667  Assert(nargs == 2);
2668  setIntValue(retval, getrand(&st->cs_func_rs, imin, imax));
2669  }
2670  else /* gaussian & exponential */
2671  {
2672  double param;
2673 
2674  Assert(nargs == 3);
2675 
2676  if (!coerceToDouble(&vargs[2], &param))
2677  return false;
2678 
2679  if (func == PGBENCH_RANDOM_GAUSSIAN)
2680  {
2681  if (param < MIN_GAUSSIAN_PARAM)
2682  {
2683  pg_log_error("gaussian parameter must be at least %f (not %f)",
2684  MIN_GAUSSIAN_PARAM, param);
2685  return false;
2686  }
2687 
2688  setIntValue(retval,
2690  imin, imax, param));
2691  }
2692  else if (func == PGBENCH_RANDOM_ZIPFIAN)
2693  {
2694  if (param < MIN_ZIPFIAN_PARAM || param > MAX_ZIPFIAN_PARAM)
2695  {
2696  pg_log_error("zipfian parameter must be in range [%.3f, %.0f] (not %f)",
2698  return false;
2699  }
2700 
2701  setIntValue(retval,
2702  getZipfianRand(&st->cs_func_rs, imin, imax, param));
2703  }
2704  else /* exponential */
2705  {
2706  if (param <= 0.0)
2707  {
2708  pg_log_error("exponential parameter must be greater than zero (not %f)",
2709  param);
2710  return false;
2711  }
2712 
2713  setIntValue(retval,
2715  imin, imax, param));
2716  }
2717  }
2718 
2719  return true;
2720  }
2721 
2722  case PGBENCH_POW:
2723  {
2724  PgBenchValue *lval = &vargs[0];
2725  PgBenchValue *rval = &vargs[1];
2726  double ld,
2727  rd;
2728 
2729  Assert(nargs == 2);
2730 
2731  if (!coerceToDouble(lval, &ld) ||
2732  !coerceToDouble(rval, &rd))
2733  return false;
2734 
2735  setDoubleValue(retval, pow(ld, rd));
2736 
2737  return true;
2738  }
2739 
2740  case PGBENCH_IS:
2741  {
2742  Assert(nargs == 2);
2743 
2744  /*
2745  * note: this simple implementation is more permissive than
2746  * SQL
2747  */
2748  setBoolValue(retval,
2749  vargs[0].type == vargs[1].type &&
2750  vargs[0].u.bval == vargs[1].u.bval);
2751  return true;
2752  }
2753 
2754  /* hashing */
2755  case PGBENCH_HASH_FNV1A:
2756  case PGBENCH_HASH_MURMUR2:
2757  {
2758  int64 val,
2759  seed;
2760 
2761  Assert(nargs == 2);
2762 
2763  if (!coerceToInt(&vargs[0], &val) ||
2764  !coerceToInt(&vargs[1], &seed))
2765  return false;
2766 
2767  if (func == PGBENCH_HASH_MURMUR2)
2768  setIntValue(retval, getHashMurmur2(val, seed));
2769  else if (func == PGBENCH_HASH_FNV1A)
2770  setIntValue(retval, getHashFnv1a(val, seed));
2771  else
2772  /* cannot get here */
2773  Assert(0);
2774 
2775  return true;
2776  }
2777 
2778  case PGBENCH_PERMUTE:
2779  {
2780  int64 val,
2781  size,
2782  seed;
2783 
2784  Assert(nargs == 3);
2785 
2786  if (!coerceToInt(&vargs[0], &val) ||
2787  !coerceToInt(&vargs[1], &size) ||
2788  !coerceToInt(&vargs[2], &seed))
2789  return false;
2790 
2791  if (size <= 0)
2792  {
2793  pg_log_error("permute size parameter must be greater than zero");
2794  return false;
2795  }
2796 
2797  setIntValue(retval, permute(val, size, seed));
2798  return true;
2799  }
2800 
2801  default:
2802  /* cannot get here */
2803  Assert(0);
2804  /* dead code to avoid a compiler warning */
2805  return false;
2806  }
2807 }
2808 
2809 /* evaluate some function */
2810 static bool
2811 evalFunc(CState *st,
2813 {
2814  if (isLazyFunc(func))
2815  return evalLazyFunc(st, func, args, retval);
2816  else
2817  return evalStandardFunc(st, func, args, retval);
2818 }
2819 
2820 /*
2821  * Recursive evaluation of an expression in a pgbench script
2822  * using the current state of variables.
2823  * Returns whether the evaluation was ok,
2824  * the value itself is returned through the retval pointer.
2825  */
2826 static bool
2827 evaluateExpr(CState *st, PgBenchExpr *expr, PgBenchValue *retval)
2828 {
2829  switch (expr->etype)
2830  {
2831  case ENODE_CONSTANT:
2832  {
2833  *retval = expr->u.constant;
2834  return true;
2835  }
2836 
2837  case ENODE_VARIABLE:
2838  {
2839  Variable *var;
2840 
2841  if ((var = lookupVariable(&st->variables, expr->u.variable.varname)) == NULL)
2842  {
2843  pg_log_error("undefined variable \"%s\"", expr->u.variable.varname);
2844  return false;
2845  }
2846 
2847  if (!makeVariableValue(var))
2848  return false;
2849 
2850  *retval = var->value;
2851  return true;
2852  }
2853 
2854  case ENODE_FUNCTION:
2855  return evalFunc(st,
2856  expr->u.function.function,
2857  expr->u.function.args,
2858  retval);
2859 
2860  default:
2861  /* internal error which should never occur */
2862  pg_fatal("unexpected enode type in evaluation: %d", expr->etype);
2863  }
2864 }
2865 
2866 /*
2867  * Convert command name to meta-command enum identifier
2868  */
2870 getMetaCommand(const char *cmd)
2871 {
2872  MetaCommand mc;
2873 
2874  if (cmd == NULL)
2875  mc = META_NONE;
2876  else if (pg_strcasecmp(cmd, "set") == 0)
2877  mc = META_SET;
2878  else if (pg_strcasecmp(cmd, "setshell") == 0)
2879  mc = META_SETSHELL;
2880  else if (pg_strcasecmp(cmd, "shell") == 0)
2881  mc = META_SHELL;
2882  else if (pg_strcasecmp(cmd, "sleep") == 0)
2883  mc = META_SLEEP;
2884  else if (pg_strcasecmp(cmd, "if") == 0)
2885  mc = META_IF;
2886  else if (pg_strcasecmp(cmd, "elif") == 0)
2887  mc = META_ELIF;
2888  else if (pg_strcasecmp(cmd, "else") == 0)
2889  mc = META_ELSE;
2890  else if (pg_strcasecmp(cmd, "endif") == 0)
2891  mc = META_ENDIF;
2892  else if (pg_strcasecmp(cmd, "gset") == 0)
2893  mc = META_GSET;
2894  else if (pg_strcasecmp(cmd, "aset") == 0)
2895  mc = META_ASET;
2896  else if (pg_strcasecmp(cmd, "startpipeline") == 0)
2897  mc = META_STARTPIPELINE;
2898  else if (pg_strcasecmp(cmd, "endpipeline") == 0)
2899  mc = META_ENDPIPELINE;
2900  else
2901  mc = META_NONE;
2902  return mc;
2903 }
2904 
2905 /*
2906  * Run a shell command. The result is assigned to the variable if not NULL.
2907  * Return true if succeeded, or false on error.
2908  */
2909 static bool
2910 runShellCommand(Variables *variables, char *variable, char **argv, int argc)
2911 {
2912  char command[SHELL_COMMAND_SIZE];
2913  int i,
2914  len = 0;
2915  FILE *fp;
2916  char res[64];
2917  char *endptr;
2918  int retval;
2919 
2920  /*----------
2921  * Join arguments with whitespace separators. Arguments starting with
2922  * exactly one colon are treated as variables:
2923  * name - append a string "name"
2924  * :var - append a variable named 'var'
2925  * ::name - append a string ":name"
2926  *----------
2927  */
2928  for (i = 0; i < argc; i++)
2929  {
2930  char *arg;
2931  int arglen;
2932 
2933  if (argv[i][0] != ':')
2934  {
2935  arg = argv[i]; /* a string literal */
2936  }
2937  else if (argv[i][1] == ':')
2938  {
2939  arg = argv[i] + 1; /* a string literal starting with colons */
2940  }
2941  else if ((arg = getVariable(variables, argv[i] + 1)) == NULL)
2942  {
2943  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[i]);
2944  return false;
2945  }
2946 
2947  arglen = strlen(arg);
2948  if (len + arglen + (i > 0 ? 1 : 0) >= SHELL_COMMAND_SIZE - 1)
2949  {
2950  pg_log_error("%s: shell command is too long", argv[0]);
2951  return false;
2952  }
2953 
2954  if (i > 0)
2955  command[len++] = ' ';
2956  memcpy(command + len, arg, arglen);
2957  len += arglen;
2958  }
2959 
2960  command[len] = '\0';
2961 
2962  fflush(NULL); /* needed before either system() or popen() */
2963 
2964  /* Fast path for non-assignment case */
2965  if (variable == NULL)
2966  {
2967  if (system(command))
2968  {
2969  if (!timer_exceeded)
2970  pg_log_error("%s: could not launch shell command", argv[0]);
2971  return false;
2972  }
2973  return true;
2974  }
2975 
2976  /* Execute the command with pipe and read the standard output. */
2977  if ((fp = popen(command, "r")) == NULL)
2978  {
2979  pg_log_error("%s: could not launch shell command", argv[0]);
2980  return false;
2981  }
2982  if (fgets(res, sizeof(res), fp) == NULL)
2983  {
2984  if (!timer_exceeded)
2985  pg_log_error("%s: could not read result of shell command", argv[0]);
2986  (void) pclose(fp);
2987  return false;
2988  }
2989  if (pclose(fp) < 0)
2990  {
2991  pg_log_error("%s: could not run shell command: %m", argv[0]);
2992  return false;
2993  }
2994 
2995  /* Check whether the result is an integer and assign it to the variable */
2996  retval = (int) strtol(res, &endptr, 10);
2997  while (*endptr != '\0' && isspace((unsigned char) *endptr))
2998  endptr++;
2999  if (*res == '\0' || *endptr != '\0')
3000  {
3001  pg_log_error("%s: shell command must return an integer (not \"%s\")", argv[0], res);
3002  return false;
3003  }
3004  if (!putVariableInt(variables, "setshell", variable, retval))
3005  return false;
3006 
3007  pg_log_debug("%s: shell parameter name: \"%s\", value: \"%s\"", argv[0], argv[1], res);
3008 
3009  return true;
3010 }
3011 
3012 /*
3013  * Report the abortion of the client when processing SQL commands.
3014  */
3015 static void
3016 commandFailed(CState *st, const char *cmd, const char *message)
3017 {
3018  pg_log_error("client %d aborted in command %d (%s) of script %d; %s",
3019  st->id, st->command, cmd, st->use_file, message);
3020 }
3021 
3022 /*
3023  * Report the error in the command while the script is executing.
3024  */
3025 static void
3026 commandError(CState *st, const char *message)
3027 {
3029  pg_log_info("client %d got an error in command %d (SQL) of script %d; %s",
3030  st->id, st->command, st->use_file, message);
3031 }
3032 
3033 /* return a script number with a weighted choice. */
3034 static int
3035 chooseScript(TState *thread)
3036 {
3037  int i = 0;
3038  int64 w;
3039 
3040  if (num_scripts == 1)
3041  return 0;
3042 
3043  w = getrand(&thread->ts_choose_rs, 0, total_weight - 1);
3044  do
3045  {
3046  w -= sql_script[i++].weight;
3047  } while (w >= 0);
3048 
3049  return i - 1;
3050 }
3051 
3052 /*
3053  * Allocate space for CState->prepared: we need one boolean for each command
3054  * of each script.
3055  */
3056 static void
3058 {
3059  Assert(st->prepared == NULL);
3060 
3061  st->prepared = pg_malloc(sizeof(bool *) * num_scripts);
3062  for (int i = 0; i < num_scripts; i++)
3063  {
3064  ParsedScript *script = &sql_script[i];
3065  int numcmds;
3066 
3067  for (numcmds = 0; script->commands[numcmds] != NULL; numcmds++)
3068  ;
3069  st->prepared[i] = pg_malloc0(sizeof(bool) * numcmds);
3070  }
3071 }
3072 
3073 /*
3074  * Prepare the SQL command from st->use_file at command_num.
3075  */
3076 static void
3077 prepareCommand(CState *st, int command_num)
3078 {
3079  Command *command = sql_script[st->use_file].commands[command_num];
3080 
3081  /* No prepare for non-SQL commands */
3082  if (command->type != SQL_COMMAND)
3083  return;
3084 
3085  if (!st->prepared)
3086  allocCStatePrepared(st);
3087 
3088  if (!st->prepared[st->use_file][command_num])
3089  {
3090  PGresult *res;
3091 
3092  pg_log_debug("client %d preparing %s", st->id, command->prepname);
3093  res = PQprepare(st->con, command->prepname,
3094  command->argv[0], command->argc - 1, NULL);
3096  pg_log_error("%s", PQerrorMessage(st->con));
3097  PQclear(res);
3098  st->prepared[st->use_file][command_num] = true;
3099  }
3100 }
3101 
3102 /*
3103  * Prepare all the commands in the script that come after the \startpipeline
3104  * that's at position st->command, and the first \endpipeline we find.
3105  *
3106  * This sets the ->prepared flag for each relevant command as well as the
3107  * \startpipeline itself, but doesn't move the st->command counter.
3108  */
3109 static void
3111 {
3112  int j;
3113  Command **commands = sql_script[st->use_file].commands;
3114 
3115  Assert(commands[st->command]->type == META_COMMAND &&
3116  commands[st->command]->meta == META_STARTPIPELINE);
3117 
3118  if (!st->prepared)
3119  allocCStatePrepared(st);
3120 
3121  /*
3122  * We set the 'prepared' flag on the \startpipeline itself to flag that we
3123  * don't need to do this next time without calling prepareCommand(), even
3124  * though we don't actually prepare this command.
3125  */
3126  if (st->prepared[st->use_file][st->command])
3127  return;
3128 
3129  for (j = st->command + 1; commands[j] != NULL; j++)
3130  {
3131  if (commands[j]->type == META_COMMAND &&
3132  commands[j]->meta == META_ENDPIPELINE)
3133  break;
3134 
3135  prepareCommand(st, j);
3136  }
3137 
3138  st->prepared[st->use_file][st->command] = true;
3139 }
3140 
3141 /* Send a SQL command, using the chosen querymode */
3142 static bool
3143 sendCommand(CState *st, Command *command)
3144 {
3145  int r;
3146 
3147  if (querymode == QUERY_SIMPLE)
3148  {
3149  char *sql;
3150 
3151  sql = pg_strdup(command->argv[0]);
3152  sql = assignVariables(&st->variables, sql);
3153 
3154  pg_log_debug("client %d sending %s", st->id, sql);
3155  r = PQsendQuery(st->con, sql);
3156  free(sql);
3157  }
3158  else if (querymode == QUERY_EXTENDED)
3159  {
3160  const char *sql = command->argv[0];
3161  const char *params[MAX_ARGS];
3162 
3163  getQueryParams(&st->variables, command, params);
3164 
3165  pg_log_debug("client %d sending %s", st->id, sql);
3166  r = PQsendQueryParams(st->con, sql, command->argc - 1,
3167  NULL, params, NULL, NULL, 0);
3168  }
3169  else if (querymode == QUERY_PREPARED)
3170  {
3171  const char *params[MAX_ARGS];
3172 
3173  prepareCommand(st, st->command);
3174  getQueryParams(&st->variables, command, params);
3175 
3176  pg_log_debug("client %d sending %s", st->id, command->prepname);
3177  r = PQsendQueryPrepared(st->con, command->prepname, command->argc - 1,
3178  params, NULL, NULL, 0);
3179  }
3180  else /* unknown sql mode */
3181  r = 0;
3182 
3183  if (r == 0)
3184  {
3185  pg_log_debug("client %d could not send %s", st->id, command->argv[0]);
3186  return false;
3187  }
3188  else
3189  return true;
3190 }
3191 
3192 /*
3193  * Get the error status from the error code.
3194  */
3195 static EStatus
3196 getSQLErrorStatus(const char *sqlState)
3197 {
3198  if (sqlState != NULL)
3199  {
3200  if (strcmp(sqlState, ERRCODE_T_R_SERIALIZATION_FAILURE) == 0)
3202  else if (strcmp(sqlState, ERRCODE_T_R_DEADLOCK_DETECTED) == 0)
3203  return ESTATUS_DEADLOCK_ERROR;
3204  }
3205 
3206  return ESTATUS_OTHER_SQL_ERROR;
3207 }
3208 
3209 /*
3210  * Returns true if this type of error can be retried.
3211  */
3212 static bool
3213 canRetryError(EStatus estatus)
3214 {
3215  return (estatus == ESTATUS_SERIALIZATION_ERROR ||
3216  estatus == ESTATUS_DEADLOCK_ERROR);
3217 }
3218 
3219 /*
3220  * Process query response from the backend.
3221  *
3222  * If varprefix is not NULL, it's the variable name prefix where to store
3223  * the results of the *last* command (META_GSET) or *all* commands
3224  * (META_ASET).
3225  *
3226  * Returns true if everything is A-OK, false if any error occurs.
3227  */
3228 static bool
3229 readCommandResponse(CState *st, MetaCommand meta, char *varprefix)
3230 {
3231  PGresult *res;
3232  PGresult *next_res;
3233  int qrynum = 0;
3234 
3235  /*
3236  * varprefix should be set only with \gset or \aset, and \endpipeline and
3237  * SQL commands do not need it.
3238  */
3239  Assert((meta == META_NONE && varprefix == NULL) ||
3240  ((meta == META_ENDPIPELINE) && varprefix == NULL) ||
3241  ((meta == META_GSET || meta == META_ASET) && varprefix != NULL));
3242 
3243  res = PQgetResult(st->con);
3244 
3245  while (res != NULL)
3246  {
3247  bool is_last;
3248 
3249  /* peek at the next result to know whether the current is last */
3250  next_res = PQgetResult(st->con);
3251  is_last = (next_res == NULL);
3252 
3253  switch (PQresultStatus(res))
3254  {
3255  case PGRES_COMMAND_OK: /* non-SELECT commands */
3256  case PGRES_EMPTY_QUERY: /* may be used for testing no-op overhead */
3257  if (is_last && meta == META_GSET)
3258  {
3259  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3260  st->id, st->use_file, st->command, qrynum, 0);
3262  goto error;
3263  }
3264  break;
3265 
3266  case PGRES_TUPLES_OK:
3267  if ((is_last && meta == META_GSET) || meta == META_ASET)
3268  {
3269  int ntuples = PQntuples(res);
3270 
3271  if (meta == META_GSET && ntuples != 1)
3272  {
3273  /* under \gset, report the error */
3274  pg_log_error("client %d script %d command %d query %d: expected one row, got %d",
3275  st->id, st->use_file, st->command, qrynum, PQntuples(res));
3277  goto error;
3278  }
3279  else if (meta == META_ASET && ntuples <= 0)
3280  {
3281  /* coldly skip empty result under \aset */
3282  break;
3283  }
3284 
3285  /* store results into variables */
3286  for (int fld = 0; fld < PQnfields(res); fld++)
3287  {
3288  char *varname = PQfname(res, fld);
3289 
3290  /* allocate varname only if necessary, freed below */
3291  if (*varprefix != '\0')
3292  varname = psprintf("%s%s", varprefix, varname);
3293 
3294  /* store last row result as a string */
3295  if (!putVariable(&st->variables, meta == META_ASET ? "aset" : "gset", varname,
3296  PQgetvalue(res, ntuples - 1, fld)))
3297  {
3298  /* internal error */
3299  pg_log_error("client %d script %d command %d query %d: error storing into variable %s",
3300  st->id, st->use_file, st->command, qrynum, varname);
3302  goto error;
3303  }
3304 
3305  if (*varprefix != '\0')
3306  pg_free(varname);
3307  }
3308  }
3309  /* otherwise the result is simply thrown away by PQclear below */
3310  break;
3311 
3312  case PGRES_PIPELINE_SYNC:
3313  pg_log_debug("client %d pipeline ending", st->id);
3314  if (PQexitPipelineMode(st->con) != 1)
3315  pg_log_error("client %d failed to exit pipeline mode: %s", st->id,
3316  PQerrorMessage(st->con));
3317  break;
3318 
3319  case PGRES_NONFATAL_ERROR:
3320  case PGRES_FATAL_ERROR:
3322  PG_DIAG_SQLSTATE));
3323  if (canRetryError(st->estatus))
3324  {
3325  if (verbose_errors)
3326  commandError(st, PQerrorMessage(st->con));
3327  goto error;
3328  }
3329  /* fall through */
3330 
3331  default:
3332  /* anything else is unexpected */
3333  pg_log_error("client %d script %d aborted in command %d query %d: %s",
3334  st->id, st->use_file, st->command, qrynum,
3335  PQerrorMessage(st->con));
3336  goto error;
3337  }
3338 
3339  PQclear(res);
3340  qrynum++;
3341  res = next_res;
3342  }
3343 
3344  if (qrynum == 0)
3345  {
3346  pg_log_error("client %d command %d: no results", st->id, st->command);
3347  return false;
3348  }
3349 
3350  return true;
3351 
3352 error:
3353  PQclear(res);
3354  PQclear(next_res);
3355  do
3356  {
3357  res = PQgetResult(st->con);
3358  PQclear(res);
3359  } while (res);
3360 
3361  return false;
3362 }
3363 
3364 /*
3365  * Parse the argument to a \sleep command, and return the requested amount
3366  * of delay, in microseconds. Returns true on success, false on error.
3367  */
3368 static bool
3369 evaluateSleep(Variables *variables, int argc, char **argv, int *usecs)
3370 {
3371  char *var;
3372  int usec;
3373 
3374  if (*argv[1] == ':')
3375  {
3376  if ((var = getVariable(variables, argv[1] + 1)) == NULL)
3377  {
3378  pg_log_error("%s: undefined variable \"%s\"", argv[0], argv[1] + 1);
3379  return false;
3380  }
3381 
3382  usec = atoi(var);
3383 
3384  /* Raise an error if the value of a variable is not a number */
3385  if (usec == 0 && !isdigit((unsigned char) *var))
3386  {
3387  pg_log_error("%s: invalid sleep time \"%s\" for variable \"%s\"",
3388  argv[0], var, argv[1] + 1);
3389  return false;
3390  }
3391  }
3392  else
3393  usec = atoi(argv[1]);
3394 
3395  if (argc > 2)
3396  {
3397  if (pg_strcasecmp(argv[2], "ms") == 0)
3398  usec *= 1000;
3399  else if (pg_strcasecmp(argv[2], "s") == 0)
3400  usec *= 1000000;
3401  }
3402  else
3403  usec *= 1000000;
3404 
3405  *usecs = usec;
3406  return true;
3407 }
3408 
3409 
3410 /*
3411  * Returns true if the error can be retried.
3412  */
3413 static bool
3415 {
3417 
3418  /* We can only retry serialization or deadlock errors. */
3419  if (!canRetryError(st->estatus))
3420  return false;
3421 
3422  /*
3423  * We must have at least one option to limit the retrying of transactions
3424  * that got an error.
3425  */
3427 
3428  /*
3429  * We cannot retry the error if we have reached the maximum number of
3430  * tries.
3431  */
3432  if (max_tries && st->tries >= max_tries)
3433  return false;
3434 
3435  /*
3436  * We cannot retry the error if we spent too much time on this
3437  * transaction.
3438  */
3439  if (latency_limit)
3440  {
3442  if (*now - st->txn_scheduled > latency_limit)
3443  return false;
3444  }
3445 
3446  /*
3447  * We cannot retry the error if the benchmark duration is over.
3448  */
3449  if (timer_exceeded)
3450  return false;
3451 
3452  /* OK */
3453  return true;
3454 }
3455 
3456 /*
3457  * Read results and discard it until a sync point.
3458  */
3459 static int
3461 {
3462  /* send a sync */
3463  if (!PQpipelineSync(st->con))
3464  {
3465  pg_log_error("client %d aborted: failed to send a pipeline sync",
3466  st->id);
3467  return 0;
3468  }
3469 
3470  /* receive PGRES_PIPELINE_SYNC and null following it */
3471  for (;;)
3472  {
3473  PGresult *res = PQgetResult(st->con);
3474 
3476  {
3477  PQclear(res);
3478  res = PQgetResult(st->con);
3479  Assert(res == NULL);
3480  break;
3481  }
3482  PQclear(res);
3483  }
3484 
3485  /* exit pipeline */
3486  if (PQexitPipelineMode(st->con) != 1)
3487  {
3488  pg_log_error("client %d aborted: failed to exit pipeline mode for rolling back the failed transaction",
3489  st->id);
3490  return 0;
3491  }
3492  return 1;
3493 }
3494 
3495 /*
3496  * Get the transaction status at the end of a command especially for
3497  * checking if we are in a (failed) transaction block.
3498  */
3499 static TStatus
3501 {
3502  PGTransactionStatusType tx_status;
3503 
3504  tx_status = PQtransactionStatus(con);
3505  switch (tx_status)
3506  {
3507  case PQTRANS_IDLE:
3508  return TSTATUS_IDLE;
3509  case PQTRANS_INTRANS:
3510  case PQTRANS_INERROR:
3511  return TSTATUS_IN_BLOCK;
3512  case PQTRANS_UNKNOWN:
3513  /* PQTRANS_UNKNOWN is expected given a broken connection */
3514  if (PQstatus(con) == CONNECTION_BAD)
3515  return TSTATUS_CONN_ERROR;
3516  /* fall through */
3517  case PQTRANS_ACTIVE:
3518  default:
3519 
3520  /*
3521  * We cannot find out whether we are in a transaction block or
3522  * not. Internal error which should never occur.
3523  */
3524  pg_log_error("unexpected transaction status %d", tx_status);
3525  return TSTATUS_OTHER_ERROR;
3526  }
3527 
3528  /* not reached */
3529  Assert(false);
3530  return TSTATUS_OTHER_ERROR;
3531 }
3532 
3533 /*
3534  * Print verbose messages of an error
3535  */
3536 static void
3537 printVerboseErrorMessages(CState *st, pg_time_usec_t *now, bool is_retry)
3538 {
3539  static PQExpBuffer buf = NULL;
3540 
3541  if (buf == NULL)
3542  buf = createPQExpBuffer();
3543  else
3545 
3546  printfPQExpBuffer(buf, "client %d ", st->id);
3547  appendPQExpBufferStr(buf, (is_retry ?
3548  "repeats the transaction after the error" :
3549  "ends the failed transaction"));
3550  appendPQExpBuffer(buf, " (try %u", st->tries);
3551 
3552  /* Print max_tries if it is not unlimited. */
3553  if (max_tries)
3554  appendPQExpBuffer(buf, "/%u", max_tries);
3555 
3556  /*
3557  * If the latency limit is used, print a percentage of the current
3558  * transaction latency from the latency limit.
3559  */
3560  if (latency_limit)
3561  {
3563  appendPQExpBuffer(buf, ", %.3f%% of the maximum time of tries was used",
3564  (100.0 * (*now - st->txn_scheduled) / latency_limit));
3565  }
3566  appendPQExpBufferStr(buf, ")\n");
3567 
3568  pg_log_info("%s", buf->data);
3569 }
3570 
3571 /*
3572  * Advance the state machine of a connection.
3573  */
3574 static void
3575 advanceConnectionState(TState *thread, CState *st, StatsData *agg)
3576 {
3577 
3578  /*
3579  * gettimeofday() isn't free, so we get the current timestamp lazily the
3580  * first time it's needed, and reuse the same value throughout this
3581  * function after that. This also ensures that e.g. the calculated
3582  * latency reported in the log file and in the totals are the same. Zero
3583  * means "not set yet". Reset "now" when we execute shell commands or
3584  * expressions, which might take a non-negligible amount of time, though.
3585  */
3586  pg_time_usec_t now = 0;
3587 
3588  /*
3589  * Loop in the state machine, until we have to wait for a result from the
3590  * server or have to sleep for throttling or \sleep.
3591  *
3592  * Note: In the switch-statement below, 'break' will loop back here,
3593  * meaning "continue in the state machine". Return is used to return to
3594  * the caller, giving the thread the opportunity to advance another
3595  * client.
3596  */
3597  for (;;)
3598  {
3599  Command *command;
3600 
3601  switch (st->state)
3602  {
3603  /* Select transaction (script) to run. */
3604  case CSTATE_CHOOSE_SCRIPT:
3605  st->use_file = chooseScript(thread);
3607 
3608  /* reset transaction variables to default values */
3609  st->estatus = ESTATUS_NO_ERROR;
3610  st->tries = 1;
3611 
3612  pg_log_debug("client %d executing script \"%s\"",
3613  st->id, sql_script[st->use_file].desc);
3614 
3615  /*
3616  * If time is over, we're done; otherwise, get ready to start
3617  * a new transaction, or to get throttled if that's requested.
3618  */
3621  break;
3622 
3623  /* Start new transaction (script) */
3624  case CSTATE_START_TX:
3626 
3627  /* establish connection if needed, i.e. under --connect */
3628  if (st->con == NULL)
3629  {
3630  pg_time_usec_t start = now;
3631 
3632  if ((st->con = doConnect()) == NULL)
3633  {
3634  /*
3635  * as the bench is already running, we do not abort
3636  * the process
3637  */
3638  pg_log_error("client %d aborted while establishing connection", st->id);
3639  st->state = CSTATE_ABORTED;
3640  break;
3641  }
3642 
3643  /* reset now after connection */
3644  now = pg_time_now();
3645 
3646  thread->conn_duration += now - start;
3647 
3648  /* Reset session-local state */
3649  pg_free(st->prepared);
3650  st->prepared = NULL;
3651  }
3652 
3653  /*
3654  * It is the first try to run this transaction. Remember the
3655  * random state: maybe it will get an error and we will need
3656  * to run it again.
3657  */
3658  st->random_state = st->cs_func_rs;
3659 
3660  /* record transaction start time */
3661  st->txn_begin = now;
3662 
3663  /*
3664  * When not throttling, this is also the transaction's
3665  * scheduled start time.
3666  */
3667  if (!throttle_delay)
3668  st->txn_scheduled = now;
3669 
3670  /* Begin with the first command */
3672  st->command = 0;
3673  break;
3674 
3675  /*
3676  * Handle throttling once per transaction by sleeping.
3677  */
3679 
3680  /*
3681  * Generate a delay such that the series of delays will
3682  * approximate a Poisson distribution centered on the
3683  * throttle_delay time.
3684  *
3685  * If transactions are too slow or a given wait is shorter
3686  * than a transaction, the next transaction will start right
3687  * away.
3688  */
3689  Assert(throttle_delay > 0);
3690 
3691  thread->throttle_trigger +=
3693  st->txn_scheduled = thread->throttle_trigger;
3694 
3695  /*
3696  * If --latency-limit is used, and this slot is already late
3697  * so that the transaction will miss the latency limit even if
3698  * it completed immediately, skip this time slot and loop to
3699  * reschedule.
3700  */
3701  if (latency_limit)
3702  {
3704 
3705  if (thread->throttle_trigger < now - latency_limit)
3706  {
3707  processXactStats(thread, st, &now, true, agg);
3708 
3709  /*
3710  * Finish client if -T or -t was exceeded.
3711  *
3712  * Stop counting skipped transactions under -T as soon
3713  * as the timer is exceeded. Because otherwise it can
3714  * take a very long time to count all of them
3715  * especially when quite a lot of them happen with
3716  * unrealistically high rate setting in -R, which
3717  * would prevent pgbench from ending immediately.
3718  * Because of this behavior, note that there is no
3719  * guarantee that all skipped transactions are counted
3720  * under -T though there is under -t. This is OK in
3721  * practice because it's very unlikely to happen with
3722  * realistic setting.
3723  */
3724  if (timer_exceeded || (nxacts > 0 && st->cnt >= nxacts))
3725  st->state = CSTATE_FINISHED;
3726 
3727  /* Go back to top of loop with CSTATE_PREPARE_THROTTLE */
3728  break;
3729  }
3730  }
3731 
3732  /*
3733  * stop client if next transaction is beyond pgbench end of
3734  * execution; otherwise, throttle it.
3735  */
3736  st->state = end_time > 0 && st->txn_scheduled > end_time ?
3738  break;
3739 
3740  /*
3741  * Wait until it's time to start next transaction.
3742  */
3743  case CSTATE_THROTTLE:
3745 
3746  if (now < st->txn_scheduled)
3747  return; /* still sleeping, nothing to do here */
3748 
3749  /* done sleeping, but don't start transaction if we're done */
3751  break;
3752 
3753  /*
3754  * Send a command to server (or execute a meta-command)
3755  */
3756  case CSTATE_START_COMMAND:
3757  command = sql_script[st->use_file].commands[st->command];
3758 
3759  /* Transition to script end processing if done */
3760  if (command == NULL)
3761  {
3762  st->state = CSTATE_END_TX;
3763  break;
3764  }
3765 
3766  /* record begin time of next command, and initiate it */
3767  if (report_per_command)
3768  {
3770  st->stmt_begin = now;
3771  }
3772 
3773  /* Execute the command */
3774  if (command->type == SQL_COMMAND)
3775  {
3776  /* disallow \aset and \gset in pipeline mode */
3777  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
3778  {
3779  if (command->meta == META_GSET)
3780  {
3781  commandFailed(st, "gset", "\\gset is not allowed in pipeline mode");
3782  st->state = CSTATE_ABORTED;
3783  break;
3784  }
3785  else if (command->meta == META_ASET)
3786  {
3787  commandFailed(st, "aset", "\\aset is not allowed in pipeline mode");
3788  st->state = CSTATE_ABORTED;
3789  break;
3790  }
3791  }
3792 
3793  if (!sendCommand(st, command))
3794  {
3795  commandFailed(st, "SQL", "SQL command send failed");
3796  st->state = CSTATE_ABORTED;
3797  }
3798  else
3799  {
3800  /* Wait for results, unless in pipeline mode */
3801  if (PQpipelineStatus(st->con) == PQ_PIPELINE_OFF)
3802  st->state = CSTATE_WAIT_RESULT;
3803  else
3804  st->state = CSTATE_END_COMMAND;
3805  }
3806  }
3807  else if (command->type == META_COMMAND)
3808  {
3809  /*-----
3810  * Possible state changes when executing meta commands:
3811  * - on errors CSTATE_ABORTED
3812  * - on sleep CSTATE_SLEEP
3813  * - else CSTATE_END_COMMAND
3814  */
3815  st->state = executeMetaCommand(st, &now);
3816  if (st->state == CSTATE_ABORTED)
3818  }
3819 
3820  /*
3821  * We're now waiting for an SQL command to complete, or
3822  * finished processing a metacommand, or need to sleep, or
3823  * something bad happened.
3824  */
3825  Assert(st->state == CSTATE_WAIT_RESULT ||
3826  st->state == CSTATE_END_COMMAND ||
3827  st->state == CSTATE_SLEEP ||
3828  st->state == CSTATE_ABORTED);
3829  break;
3830 
3831  /*
3832  * non executed conditional branch
3833  */
3834  case CSTATE_SKIP_COMMAND:
3836  /* quickly skip commands until something to do... */
3837  while (true)
3838  {
3839  command = sql_script[st->use_file].commands[st->command];
3840 
3841  /* cannot reach end of script in that state */
3842  Assert(command != NULL);
3843 
3844  /*
3845  * if this is conditional related, update conditional
3846  * state
3847  */
3848  if (command->type == META_COMMAND &&
3849  (command->meta == META_IF ||
3850  command->meta == META_ELIF ||
3851  command->meta == META_ELSE ||
3852  command->meta == META_ENDIF))
3853  {
3854  switch (conditional_stack_peek(st->cstack))
3855  {
3856  case IFSTATE_FALSE:
3857  if (command->meta == META_IF ||
3858  command->meta == META_ELIF)
3859  {
3860  /* we must evaluate the condition */
3862  }
3863  else if (command->meta == META_ELSE)
3864  {
3865  /* we must execute next command */
3869  st->command++;
3870  }
3871  else if (command->meta == META_ENDIF)
3872  {
3875  if (conditional_active(st->cstack))
3877 
3878  /*
3879  * else state remains in
3880  * CSTATE_SKIP_COMMAND
3881  */
3882  st->command++;
3883  }
3884  break;
3885 
3886  case IFSTATE_IGNORED:
3887  case IFSTATE_ELSE_FALSE:
3888  if (command->meta == META_IF)
3890  IFSTATE_IGNORED);
3891  else if (command->meta == META_ENDIF)
3892  {
3895  if (conditional_active(st->cstack))
3897  }
3898  /* could detect "else" & "elif" after "else" */
3899  st->command++;
3900  break;
3901 
3902  case IFSTATE_NONE:
3903  case IFSTATE_TRUE:
3904  case IFSTATE_ELSE_TRUE:
3905  default:
3906 
3907  /*
3908  * inconsistent if inactive, unreachable dead
3909  * code
3910  */
3911  Assert(false);
3912  }
3913  }
3914  else
3915  {
3916  /* skip and consider next */
3917  st->command++;
3918  }
3919 
3920  if (st->state != CSTATE_SKIP_COMMAND)
3921  /* out of quick skip command loop */
3922  break;
3923  }
3924  break;
3925 
3926  /*
3927  * Wait for the current SQL command to complete
3928  */
3929  case CSTATE_WAIT_RESULT:
3930  pg_log_debug("client %d receiving", st->id);
3931 
3932  /*
3933  * Only check for new network data if we processed all data
3934  * fetched prior. Otherwise we end up doing a syscall for each
3935  * individual pipelined query, which has a measurable
3936  * performance impact.
3937  */
3938  if (PQisBusy(st->con) && !PQconsumeInput(st->con))
3939  {
3940  /* there's something wrong */
3941  commandFailed(st, "SQL", "perhaps the backend died while processing");
3942  st->state = CSTATE_ABORTED;
3943  break;
3944  }
3945  if (PQisBusy(st->con))
3946  return; /* don't have the whole result yet */
3947 
3948  /* store or discard the query results */
3949  if (readCommandResponse(st,
3952  {
3953  /*
3954  * outside of pipeline mode: stop reading results.
3955  * pipeline mode: continue reading results until an
3956  * end-of-pipeline response.
3957  */
3958  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
3959  st->state = CSTATE_END_COMMAND;
3960  }
3961  else if (canRetryError(st->estatus))
3962  st->state = CSTATE_ERROR;
3963  else
3964  st->state = CSTATE_ABORTED;
3965  break;
3966 
3967  /*
3968  * Wait until sleep is done. This state is entered after a
3969  * \sleep metacommand. The behavior is similar to
3970  * CSTATE_THROTTLE, but proceeds to CSTATE_START_COMMAND
3971  * instead of CSTATE_START_TX.
3972  */
3973  case CSTATE_SLEEP:
3975  if (now < st->sleep_until)
3976  return; /* still sleeping, nothing to do here */
3977  /* Else done sleeping. */
3978  st->state = CSTATE_END_COMMAND;
3979  break;
3980 
3981  /*
3982  * End of command: record stats and proceed to next command.
3983  */
3984  case CSTATE_END_COMMAND:
3985 
3986  /*
3987  * command completed: accumulate per-command execution times
3988  * in thread-local data structure, if per-command latencies
3989  * are requested.
3990  */
3991  if (report_per_command)
3992  {
3994 
3995  command = sql_script[st->use_file].commands[st->command];
3996  /* XXX could use a mutex here, but we choose not to */
3997  addToSimpleStats(&command->stats,
3999  }
4000 
4001  /* Go ahead with next command, to be executed or skipped */
4002  st->command++;
4003  st->state = conditional_active(st->cstack) ?
4005  break;
4006 
4007  /*
4008  * Clean up after an error.
4009  */
4010  case CSTATE_ERROR:
4011  {
4012  TStatus tstatus;
4013 
4015 
4016  /* Clear the conditional stack */
4018 
4019  /* Read and discard until a sync point in pipeline mode */
4020  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4021  {
4022  if (!discardUntilSync(st))
4023  {
4024  st->state = CSTATE_ABORTED;
4025  break;
4026  }
4027  }
4028 
4029  /*
4030  * Check if we have a (failed) transaction block or not,
4031  * and roll it back if any.
4032  */
4033  tstatus = getTransactionStatus(st->con);
4034  if (tstatus == TSTATUS_IN_BLOCK)
4035  {
4036  /* Try to rollback a (failed) transaction block. */
4037  if (!PQsendQuery(st->con, "ROLLBACK"))
4038  {
4039  pg_log_error("client %d aborted: failed to send sql command for rolling back the failed transaction",
4040  st->id);
4041  st->state = CSTATE_ABORTED;
4042  }
4043  else
4045  }
4046  else if (tstatus == TSTATUS_IDLE)
4047  {
4048  /*
4049  * If time is over, we're done; otherwise, check if we
4050  * can retry the error.
4051  */
4054  }
4055  else
4056  {
4057  if (tstatus == TSTATUS_CONN_ERROR)
4058  pg_log_error("perhaps the backend died while processing");
4059 
4060  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4061  st->state = CSTATE_ABORTED;
4062  }
4063  break;
4064  }
4065 
4066  /*
4067  * Wait for the rollback command to complete
4068  */
4070  {
4071  PGresult *res;
4072 
4073  pg_log_debug("client %d receiving", st->id);
4074  if (!PQconsumeInput(st->con))
4075  {
4076  pg_log_error("client %d aborted while rolling back the transaction after an error; perhaps the backend died while processing",
4077  st->id);
4078  st->state = CSTATE_ABORTED;
4079  break;
4080  }
4081  if (PQisBusy(st->con))
4082  return; /* don't have the whole result yet */
4083 
4084  /*
4085  * Read and discard the query result;
4086  */
4087  res = PQgetResult(st->con);
4088  switch (PQresultStatus(res))
4089  {
4090  case PGRES_COMMAND_OK:
4091  /* OK */
4092  PQclear(res);
4093  /* null must be returned */
4094  res = PQgetResult(st->con);
4095  Assert(res == NULL);
4096 
4097  /*
4098  * If time is over, we're done; otherwise, check
4099  * if we can retry the error.
4100  */
4103  break;
4104  default:
4105  pg_log_error("client %d aborted while rolling back the transaction after an error; %s",
4106  st->id, PQerrorMessage(st->con));
4107  PQclear(res);
4108  st->state = CSTATE_ABORTED;
4109  break;
4110  }
4111  break;
4112  }
4113 
4114  /*
4115  * Retry the transaction after an error.
4116  */
4117  case CSTATE_RETRY:
4118  command = sql_script[st->use_file].commands[st->command];
4119 
4120  /*
4121  * Inform that the transaction will be retried after the
4122  * error.
4123  */
4124  if (verbose_errors)
4125  printVerboseErrorMessages(st, &now, true);
4126 
4127  /* Count tries and retries */
4128  st->tries++;
4129  command->retries++;
4130 
4131  /*
4132  * Reset the random state as they were at the beginning of the
4133  * transaction.
4134  */
4135  st->cs_func_rs = st->random_state;
4136 
4137  /* Process the first transaction command. */
4138  st->command = 0;
4139  st->estatus = ESTATUS_NO_ERROR;
4141  break;
4142 
4143  /*
4144  * Record a failed transaction.
4145  */
4146  case CSTATE_FAILURE:
4147  command = sql_script[st->use_file].commands[st->command];
4148 
4149  /* Accumulate the failure. */
4150  command->failures++;
4151 
4152  /*
4153  * Inform that the failed transaction will not be retried.
4154  */
4155  if (verbose_errors)
4156  printVerboseErrorMessages(st, &now, false);
4157 
4158  /* End the failed transaction. */
4159  st->state = CSTATE_END_TX;
4160  break;
4161 
4162  /*
4163  * End of transaction (end of script, really).
4164  */
4165  case CSTATE_END_TX:
4166  {
4167  TStatus tstatus;
4168 
4169  /* transaction finished: calculate latency and do log */
4170  processXactStats(thread, st, &now, false, agg);
4171 
4172  /*
4173  * missing \endif... cannot happen if CheckConditional was
4174  * okay
4175  */
4177 
4178  /*
4179  * We must complete all the transaction blocks that were
4180  * started in this script.
4181  */
4182  tstatus = getTransactionStatus(st->con);
4183  if (tstatus == TSTATUS_IN_BLOCK)
4184  {
4185  pg_log_error("client %d aborted: end of script reached without completing the last transaction",
4186  st->id);
4187  st->state = CSTATE_ABORTED;
4188  break;
4189  }
4190  else if (tstatus != TSTATUS_IDLE)
4191  {
4192  if (tstatus == TSTATUS_CONN_ERROR)
4193  pg_log_error("perhaps the backend died while processing");
4194 
4195  pg_log_error("client %d aborted while receiving the transaction status", st->id);
4196  st->state = CSTATE_ABORTED;
4197  break;
4198  }
4199 
4200  if (is_connect)
4201  {
4202  pg_time_usec_t start = now;
4203 
4204  pg_time_now_lazy(&start);
4205  finishCon(st);
4206  now = pg_time_now();
4207  thread->conn_duration += now - start;
4208  }
4209 
4210  if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded)
4211  {
4212  /* script completed */
4213  st->state = CSTATE_FINISHED;
4214  break;
4215  }
4216 
4217  /* next transaction (script) */
4219 
4220  /*
4221  * Ensure that we always return on this point, so as to
4222  * avoid an infinite loop if the script only contains meta
4223  * commands.
4224  */
4225  return;
4226  }
4227 
4228  /*
4229  * Final states. Close the connection if it's still open.
4230  */
4231  case CSTATE_ABORTED:
4232  case CSTATE_FINISHED:
4233 
4234  /*
4235  * Don't measure the disconnection delays here even if in
4236  * CSTATE_FINISHED and -C/--connect option is specified.
4237  * Because in this case all the connections that this thread
4238  * established are closed at the end of transactions and the
4239  * disconnection delays should have already been measured at
4240  * that moment.
4241  *
4242  * In CSTATE_ABORTED state, the measurement is no longer
4243  * necessary because we cannot report complete results anyways
4244  * in this case.
4245  */
4246  finishCon(st);
4247  return;
4248  }
4249  }
4250 }
4251 
4252 /*
4253  * Subroutine for advanceConnectionState -- initiate or execute the current
4254  * meta command, and return the next state to set.
4255  *
4256  * *now is updated to the current time, unless the command is expected to
4257  * take no time to execute.
4258  */
4261 {
4262  Command *command = sql_script[st->use_file].commands[st->command];
4263  int argc;
4264  char **argv;
4265 
4266  Assert(command != NULL && command->type == META_COMMAND);
4267 
4268  argc = command->argc;
4269  argv = command->argv;
4270 
4272  {
4274 
4275  initPQExpBuffer(&buf);
4276 
4277  printfPQExpBuffer(&buf, "client %d executing \\%s", st->id, argv[0]);
4278  for (int i = 1; i < argc; i++)
4279  appendPQExpBuffer(&buf, " %s", argv[i]);
4280 
4281  pg_log_debug("%s", buf.data);
4282 
4283  termPQExpBuffer(&buf);
4284  }
4285 
4286  if (command->meta == META_SLEEP)
4287  {
4288  int usec;
4289 
4290  /*
4291  * A \sleep doesn't execute anything, we just get the delay from the
4292  * argument, and enter the CSTATE_SLEEP state. (The per-command
4293  * latency will be recorded in CSTATE_SLEEP state, not here, after the
4294  * delay has elapsed.)
4295  */
4296  if (!evaluateSleep(&st->variables, argc, argv, &usec))
4297  {
4298  commandFailed(st, "sleep", "execution of meta-command failed");
4299  return CSTATE_ABORTED;
4300  }
4301 
4303  st->sleep_until = (*now) + usec;
4304  return CSTATE_SLEEP;
4305  }
4306  else if (command->meta == META_SET)
4307  {
4308  PgBenchExpr *expr = command->expr;
4309  PgBenchValue result;
4310 
4311  if (!evaluateExpr(st, expr, &result))
4312  {
4313  commandFailed(st, argv[0], "evaluation of meta-command failed");
4314  return CSTATE_ABORTED;
4315  }
4316 
4317  if (!putVariableValue(&st->variables, argv[0], argv[1], &result))
4318  {
4319  commandFailed(st, "set", "assignment of meta-command failed");
4320  return CSTATE_ABORTED;
4321  }
4322  }
4323  else if (command->meta == META_IF)
4324  {
4325  /* backslash commands with an expression to evaluate */
4326  PgBenchExpr *expr = command->expr;
4327  PgBenchValue result;
4328  bool cond;
4329 
4330  if (!evaluateExpr(st, expr, &result))
4331  {
4332  commandFailed(st, argv[0], "evaluation of meta-command failed");
4333  return CSTATE_ABORTED;
4334  }
4335 
4336  cond = valueTruth(&result);
4338  }
4339  else if (command->meta == META_ELIF)
4340  {
4341  /* backslash commands with an expression to evaluate */
4342  PgBenchExpr *expr = command->expr;
4343  PgBenchValue result;
4344  bool cond;
4345 
4347  {
4348  /* elif after executed block, skip eval and wait for endif. */
4350  return CSTATE_END_COMMAND;
4351  }
4352 
4353  if (!evaluateExpr(st, expr, &result))
4354  {
4355  commandFailed(st, argv[0], "evaluation of meta-command failed");
4356  return CSTATE_ABORTED;
4357  }
4358 
4359  cond = valueTruth(&result);
4362  }
4363  else if (command->meta == META_ELSE)
4364  {
4365  switch (conditional_stack_peek(st->cstack))
4366  {
4367  case IFSTATE_TRUE:
4369  break;
4370  case IFSTATE_FALSE: /* inconsistent if active */
4371  case IFSTATE_IGNORED: /* inconsistent if active */
4372  case IFSTATE_NONE: /* else without if */
4373  case IFSTATE_ELSE_TRUE: /* else after else */
4374  case IFSTATE_ELSE_FALSE: /* else after else */
4375  default:
4376  /* dead code if conditional check is ok */
4377  Assert(false);
4378  }
4379  }
4380  else if (command->meta == META_ENDIF)
4381  {
4384  }
4385  else if (command->meta == META_SETSHELL)
4386  {
4387  if (!runShellCommand(&st->variables, argv[1], argv + 2, argc - 2))
4388  {
4389  commandFailed(st, "setshell", "execution of meta-command failed");
4390  return CSTATE_ABORTED;
4391  }
4392  }
4393  else if (command->meta == META_SHELL)
4394  {
4395  if (!runShellCommand(&st->variables, NULL, argv + 1, argc - 1))
4396  {
4397  commandFailed(st, "shell", "execution of meta-command failed");
4398  return CSTATE_ABORTED;
4399  }
4400  }
4401  else if (command->meta == META_STARTPIPELINE)
4402  {
4403  /*
4404  * In pipeline mode, we use a workflow based on libpq pipeline
4405  * functions.
4406  */
4407  if (querymode == QUERY_SIMPLE)
4408  {
4409  commandFailed(st, "startpipeline", "cannot use pipeline mode with the simple query protocol");
4410  return CSTATE_ABORTED;
4411  }
4412 
4413  /*
4414  * If we're in prepared-query mode, we need to prepare all the
4415  * commands that are inside the pipeline before we actually start the
4416  * pipeline itself. This solves the problem that running BEGIN
4417  * ISOLATION LEVEL SERIALIZABLE in a pipeline would fail due to a
4418  * snapshot having been acquired by the prepare within the pipeline.
4419  */
4420  if (querymode == QUERY_PREPARED)
4422 
4423  if (PQpipelineStatus(st->con) != PQ_PIPELINE_OFF)
4424  {
4425  commandFailed(st, "startpipeline", "already in pipeline mode");
4426  return CSTATE_ABORTED;
4427  }
4428  if (PQenterPipelineMode(st->con) == 0)
4429  {
4430  commandFailed(st, "startpipeline", "failed to enter pipeline mode");
4431  return CSTATE_ABORTED;
4432  }
4433  }
4434  else if (command->meta == META_ENDPIPELINE)
4435  {
4436  if (PQpipelineStatus(st->con) != PQ_PIPELINE_ON)
4437  {
4438  commandFailed(st, "endpipeline", "not in pipeline mode");
4439  return CSTATE_ABORTED;
4440  }
4441  if (!PQpipelineSync(st->con))
4442  {
4443  commandFailed(st, "endpipeline", "failed to send a pipeline sync");
4444  return CSTATE_ABORTED;
4445  }
4446  /* Now wait for the PGRES_PIPELINE_SYNC and exit pipeline mode there */
4447  /* collect pending results before getting out of pipeline mode */
4448  return CSTATE_WAIT_RESULT;
4449  }
4450 
4451  /*
4452  * executing the expression or shell command might have taken a
4453  * non-negligible amount of time, so reset 'now'
4454  */
4455  *now = 0;
4456 
4457  return CSTATE_END_COMMAND;
4458 }
4459 
4460 /*
4461  * Return the number of failed transactions.
4462  */
4463 static int64
4464 getFailures(const StatsData *stats)
4465 {
4466  return (stats->serialization_failures +
4467  stats->deadlock_failures);
4468 }
4469 
4470 /*
4471  * Return a string constant representing the result of a transaction
4472  * that is not successfully processed.
4473  */
4474 static const char *
4475 getResultString(bool skipped, EStatus estatus)
4476 {
4477  if (skipped)
4478  return "skipped";
4479  else if (failures_detailed)
4480  {
4481  switch (estatus)
4482  {
4484  return "serialization";
4486  return "deadlock";
4487  default:
4488  /* internal error which should never occur */
4489  pg_fatal("unexpected error status: %d", estatus);
4490  }
4491  }
4492  else
4493  return "failed";
4494 }
4495 
4496 /*
4497  * Print log entry after completing one transaction.
4498  *
4499  * We print Unix-epoch timestamps in the log, so that entries can be
4500  * correlated against other logs.
4501  *
4502  * XXX We could obtain the time from the caller and just shift it here, to
4503  * avoid the cost of an extra call to pg_time_now().
4504  */
4505 static void
4506 doLog(TState *thread, CState *st,
4507  StatsData *agg, bool skipped, double latency, double lag)
4508 {
4509  FILE *logfile = thread->logfile;
4511 
4512  Assert(use_log);
4513 
4514  /*
4515  * Skip the log entry if sampling is enabled and this row doesn't belong
4516  * to the random sample.
4517  */
4518  if (sample_rate != 0.0 &&
4520  return;
4521 
4522  /* should we aggregate the results or not? */
4523  if (agg_interval > 0)
4524  {
4526 
4527  /*
4528  * Loop until we reach the interval of the current moment, and print
4529  * any empty intervals in between (this may happen with very low tps,
4530  * e.g. --rate=0.1).
4531  */
4532 
4533  while ((next = agg->start_time + agg_interval * INT64CONST(1000000)) <= now)
4534  {
4535  double lag_sum = 0.0;
4536  double lag_sum2 = 0.0;
4537  double lag_min = 0.0;
4538  double lag_max = 0.0;
4539  int64 skipped = 0;
4540  int64 serialization_failures = 0;
4541  int64 deadlock_failures = 0;
4542  int64 retried = 0;
4543  int64 retries = 0;
4544 
4545  /* print aggregated report to logfile */
4546  fprintf(logfile, INT64_FORMAT " " INT64_FORMAT " %.0f %.0f %.0f %.0f",
4547  agg->start_time / 1000000, /* seconds since Unix epoch */
4548  agg->cnt,
4549  agg->latency.sum,
4550  agg->latency.sum2,
4551  agg->latency.min,
4552  agg->latency.max);
4553 
4554  if (throttle_delay)
4555  {
4556  lag_sum = agg->lag.sum;
4557  lag_sum2 = agg->lag.sum2;
4558  lag_min = agg->lag.min;
4559  lag_max = agg->lag.max;
4560  }
4561  fprintf(logfile, " %.0f %.0f %.0f %.0f",
4562  lag_sum,
4563  lag_sum2,
4564  lag_min,
4565  lag_max);
4566 
4567  if (latency_limit)
4568  skipped = agg->skipped;
4569  fprintf(logfile, " " INT64_FORMAT, skipped);
4570 
4571  if (max_tries != 1)
4572  {
4573  retried = agg->retried;
4574  retries = agg->retries;
4575  }
4576  fprintf(logfile, " " INT64_FORMAT " " INT64_FORMAT, retried, retries);
4577 
4578  if (failures_detailed)
4579  {
4580  serialization_failures = agg->serialization_failures;
4581  deadlock_failures = agg->deadlock_failures;
4582  }
4584  serialization_failures,
4585  deadlock_failures);
4586 
4587  fputc('\n', logfile);
4588 
4589  /* reset data and move to next interval */
4590  initStats(agg, next);
4591  }
4592 
4593  /* accumulate the current transaction */
4594  accumStats(agg, skipped, latency, lag, st->estatus, st->tries);
4595  }
4596  else
4597  {
4598  /* no, print raw transactions */
4599  if (!skipped && st->estatus == ESTATUS_NO_ERROR)
4600  fprintf(logfile, "%d " INT64_FORMAT " %.0f %d " INT64_FORMAT " "
4601  INT64_FORMAT,
4602  st->id, st->cnt, latency, st->use_file,
4603  now / 1000000, now % 1000000);
4604  else
4605  fprintf(logfile, "%d " INT64_FORMAT " %s %d " INT64_FORMAT " "
4606  INT64_FORMAT,
4607  st->id, st->cnt, getResultString(skipped, st->estatus),
4608  st->use_file, now / 1000000, now % 1000000);
4609 
4610  if (throttle_delay)
4611  fprintf(logfile, " %.0f", lag);
4612  if (max_tries != 1)
4613  fprintf(logfile, " %u", st->tries - 1);
4614  fputc('\n', logfile);
4615  }
4616 }
4617 
4618 /*
4619  * Accumulate and report statistics at end of a transaction.
4620  *
4621  * (This is also called when a transaction is late and thus skipped.
4622  * Note that even skipped and failed transactions are counted in the CState
4623  * "cnt" field.)
4624  */
4625 static void
4627  bool skipped, StatsData *agg)
4628 {
4629  double latency = 0.0,
4630  lag = 0.0;
4631  bool detailed = progress || throttle_delay || latency_limit ||
4633 
4634  if (detailed && !skipped && st->estatus == ESTATUS_NO_ERROR)
4635  {
4637 
4638  /* compute latency & lag */
4639  latency = (*now) - st->txn_scheduled;
4640  lag = st->txn_begin - st->txn_scheduled;
4641  }
4642 
4643  /* keep detailed thread stats */
4644  accumStats(&thread->stats, skipped, latency, lag, st->estatus, st->tries);
4645 
4646  /* count transactions over the latency limit, if needed */
4647  if (latency_limit && latency > latency_limit)
4648  thread->latency_late++;
4649 
4650  /* client stat is just counting */
4651  st->cnt++;
4652 
4653  if (use_log)
4654  doLog(thread, st, agg, skipped, latency, lag);
4655 
4656  /* XXX could use a mutex here, but we choose not to */
4657  if (per_script_stats)
4658  accumStats(&sql_script[st->use_file].stats, skipped, latency, lag,
4659  st->estatus, st->tries);
4660 }
4661 
4662 
4663 /* discard connections */
4664 static void
4665 disconnect_all(CState *state, int length)
4666 {
4667  int i;
4668 
4669  for (i = 0; i < length; i++)
4670  finishCon(&state[i]);
4671 }
4672 
4673 /*
4674  * Remove old pgbench tables, if any exist
4675  */
4676 static void
4677 initDropTables(PGconn *con)
4678 {
4679  fprintf(stderr, "dropping old tables...\n");
4680 
4681  /*
4682  * We drop all the tables in one command, so that whether there are
4683  * foreign key dependencies or not doesn't matter.
4684  */
4685  executeStatement(con, "drop table if exists "
4686  "pgbench_accounts, "
4687  "pgbench_branches, "
4688  "pgbench_history, "
4689  "pgbench_tellers");
4690 }
4691 
4692 /*
4693  * Create "pgbench_accounts" partitions if needed.
4694  *
4695  * This is the larger table of pgbench default tpc-b like schema
4696  * with a known size, so we choose to partition it.
4697  */
4698 static void
4700 {
4701  PQExpBufferData query;
4702 
4703  /* we must have to create some partitions */
4704  Assert(partitions > 0);
4705 
4706  fprintf(stderr, "creating %d partitions...\n", partitions);
4707 
4708  initPQExpBuffer(&query);
4709 
4710  for (int p = 1; p <= partitions; p++)
4711  {
4713  {
4714  int64 part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
4715 
4716  printfPQExpBuffer(&query,
4717  "create%s table pgbench_accounts_%d\n"
4718  " partition of pgbench_accounts\n"
4719  " for values from (",
4720  unlogged_tables ? " unlogged" : "", p);
4721 
4722  /*
4723  * For RANGE, we use open-ended partitions at the beginning and
4724  * end to allow any valid value for the primary key. Although the
4725  * actual minimum and maximum values can be derived from the
4726  * scale, it is more generic and the performance is better.
4727  */
4728  if (p == 1)
4729  appendPQExpBufferStr(&query, "minvalue");
4730  else
4731  appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
4732 
4733  appendPQExpBufferStr(&query, ") to (");
4734 
4735  if (p < partitions)
4736  appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
4737  else
4738  appendPQExpBufferStr(&query, "maxvalue");
4739 
4740  appendPQExpBufferChar(&query, ')');
4741  }
4742  else if (partition_method == PART_HASH)
4743  printfPQExpBuffer(&query,
4744  "create%s table pgbench_accounts_%d\n"
4745  " partition of pgbench_accounts\n"
4746  " for values with (modulus %d, remainder %d)",
4747  unlogged_tables ? " unlogged" : "", p,
4748  partitions, p - 1);
4749  else /* cannot get there */
4750  Assert(0);
4751 
4752  /*
4753  * Per ddlinfo in initCreateTables, fillfactor is needed on table
4754  * pgbench_accounts.
4755  */
4756  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4757 
4758  executeStatement(con, query.data);
4759  }
4760 
4761  termPQExpBuffer(&query);
4762 }
4763 
4764 /*
4765  * Create pgbench's standard tables
4766  */
4767 static void
4769 {
4770  /*
4771  * Note: TPC-B requires at least 100 bytes per row, and the "filler"
4772  * fields in these table declarations were intended to comply with that.
4773  * The pgbench_accounts table complies with that because the "filler"
4774  * column is set to blank-padded empty string. But for all other tables
4775  * the columns default to NULL and so don't actually take any space. We
4776  * could fix that by giving them non-null default values. However, that
4777  * would completely break comparability of pgbench results with prior
4778  * versions. Since pgbench has never pretended to be fully TPC-B compliant
4779  * anyway, we stick with the historical behavior.
4780  */
4781  struct ddlinfo
4782  {
4783  const char *table; /* table name */
4784  const char *smcols; /* column decls if accountIDs are 32 bits */
4785  const char *bigcols; /* column decls if accountIDs are 64 bits */
4786  int declare_fillfactor;
4787  };
4788  static const struct ddlinfo DDLs[] = {
4789  {
4790  "pgbench_history",
4791  "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)",
4792  "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)",
4793  0
4794  },
4795  {
4796  "pgbench_tellers",
4797  "tid int not null,bid int,tbalance int,filler char(84)",
4798  "tid int not null,bid int,tbalance int,filler char(84)",
4799  1
4800  },
4801  {
4802  "pgbench_accounts",
4803  "aid int not null,bid int,abalance int,filler char(84)",
4804  "aid bigint not null,bid int,abalance int,filler char(84)",
4805  1
4806  },
4807  {
4808  "pgbench_branches",
4809  "bid int not null,bbalance int,filler char(88)",
4810  "bid int not null,bbalance int,filler char(88)",
4811  1
4812  }
4813  };
4814  int i;
4815  PQExpBufferData query;
4816 
4817  fprintf(stderr, "creating tables...\n");
4818 
4819  initPQExpBuffer(&query);
4820 
4821  for (i = 0; i < lengthof(DDLs); i++)
4822  {
4823  const struct ddlinfo *ddl = &DDLs[i];
4824 
4825  /* Construct new create table statement. */
4826  printfPQExpBuffer(&query, "create%s table %s(%s)",
4827  unlogged_tables ? " unlogged" : "",
4828  ddl->table,
4829  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
4830 
4831  /* Partition pgbench_accounts table */
4832  if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
4833  appendPQExpBuffer(&query,
4834  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
4835  else if (ddl->declare_fillfactor)
4836  {
4837  /* fillfactor is only expected on actual tables */
4838  appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor);
4839  }
4840 
4841  if (tablespace != NULL)
4842  {
4843  char *escape_tablespace;
4844 
4845  escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
4846  appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
4847  PQfreemem(escape_tablespace);
4848  }
4849 
4850  executeStatement(con, query.data);
4851  }
4852 
4853  termPQExpBuffer(&query);
4854 
4855  if (partition_method != PART_NONE)
4856  createPartitions(con);
4857 }
4858 
4859 /*
4860  * Truncate away any old data, in one command in case there are foreign keys
4861  */
4862 static void
4864 {
4865  executeStatement(con, "truncate table "
4866  "pgbench_accounts, "
4867  "pgbench_branches, "
4868  "pgbench_history, "
4869  "pgbench_tellers");
4870 }
4871 
4872 static void
4873 initBranch(PQExpBufferData *sql, int64 curr)
4874 {
4875  /* "filler" column uses NULL */
4876  printfPQExpBuffer(sql,
4877  INT64_FORMAT "\t0\t\\N\n",
4878  curr + 1);
4879 }
4880 
4881 static void
4882 initTeller(PQExpBufferData *sql, int64 curr)
4883 {
4884  /* "filler" column uses NULL */
4885  printfPQExpBuffer(sql,
4886  INT64_FORMAT "\t" INT64_FORMAT "\t0\t\\N\n",
4887  curr + 1, curr / ntellers + 1);
4888 }
4889 
4890 static void
4891 initAccount(PQExpBufferData *sql, int64 curr)
4892 {
4893  /* "filler" column defaults to blank padded empty string */
4894  printfPQExpBuffer(sql,
4895  INT64_FORMAT "\t" INT64_FORMAT "\t0\t\n",
4896  curr + 1, curr / naccounts + 1);
4897 }
4898 
4899 static void
4900 initPopulateTable(PGconn *con, const char *table, int64 base,
4901  initRowMethod init_row)
4902 {
4903  int n;
4904  int k;
4905  int chars = 0;
4906  PGresult *res;
4907  PQExpBufferData sql;
4908  char copy_statement[256];
4909  const char *copy_statement_fmt = "copy %s from stdin";
4910  int64 total = base * scale;
4911 
4912  /* used to track elapsed time and estimate of the remaining time */
4913  pg_time_usec_t start;
4914  int log_interval = 1;
4915 
4916  /* Stay on the same line if reporting to a terminal */
4917  char eol = isatty(fileno(stderr)) ? '\r' : '\n';
4918 
4919  initPQExpBuffer(&sql);
4920 
4921  /*
4922  * Use COPY with FREEZE on v14 and later for all the tables except
4923  * pgbench_accounts when it is partitioned.
4924  */
4925  if (PQserverVersion(con) >= 140000)
4926  {
4927  if (strcmp(table, "pgbench_accounts") != 0 ||
4928  partitions == 0)
4929  copy_statement_fmt = "copy %s from stdin with (freeze on)";
4930  }
4931 
4932  n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table);
4933  if (n >= sizeof(copy_statement))
4934  pg_fatal("invalid buffer size: must be at least %d characters long", n);
4935  else if (n == -1)
4936  pg_fatal("invalid format string");
4937 
4938  res = PQexec(con, copy_statement);
4939 
4941  pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
4942  PQclear(res);
4943 
4944  start = pg_time_now();
4945 
4946  for (k = 0; k < total; k++)
4947  {
4948  int64 j = k + 1;
4949 
4950  init_row(&sql, k);
4951  if (PQputline(con, sql.data))
4952  pg_fatal("PQputline failed");
4953 
4954  if (CancelRequested)
4955  break;
4956 
4957  /*
4958  * If we want to stick with the original logging, print a message each
4959  * 100k inserted rows.
4960  */
4961  if ((!use_quiet) && (j % 100000 == 0))
4962  {
4963  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4964  double remaining_sec = ((double) total - j) * elapsed_sec / j;
4965 
4966  chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)%c",
4967  j, total,
4968  (int) ((j * 100) / total),
4969  table, elapsed_sec, remaining_sec, eol);
4970  }
4971  /* let's not call the timing for each row, but only each 100 rows */
4972  else if (use_quiet && (j % 100 == 0))
4973  {
4974  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
4975  double remaining_sec = ((double) total - j) * elapsed_sec / j;
4976 
4977  /* have we reached the next interval (or end)? */
4978  if ((j == total) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
4979  {
4980  chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)%c",
4981  j, total,
4982  (int) ((j * 100) / total),
4983  table, elapsed_sec, remaining_sec, eol);
4984 
4985  /* skip to the next interval */
4986  log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
4987  }
4988  }
4989  }
4990 
4991  if (chars != 0 && eol != '\n')
4992  fprintf(stderr, "%*c\r", chars - 1, ' '); /* Clear the current line */
4993 
4994  if (PQputline(con, "\\.\n"))
4995  pg_fatal("very last PQputline failed");
4996  if (PQendcopy(con))
4997  pg_fatal("PQendcopy failed");
4998 
4999  termPQExpBuffer(&sql);
5000 }
5001 
5002 /*
5003  * Fill the standard tables with some data generated and sent from the client.
5004  *
5005  * The filler column is NULL in pgbench_branches and pgbench_tellers, and is
5006  * a blank-padded string in pgbench_accounts.
5007  */
5008 static void
5010 {
5011  fprintf(stderr, "generating data (client-side)...\n");
5012 
5013  /*
5014  * we do all of this in one transaction to enable the backend's
5015  * data-loading optimizations
5016  */
5017  executeStatement(con, "begin");
5018 
5019  /* truncate away any old data */
5020  initTruncateTables(con);
5021 
5022  /*
5023  * fill branches, tellers, accounts in that order in case foreign keys
5024  * already exist
5025  */
5026  initPopulateTable(con, "pgbench_branches", nbranches, initBranch);
5027  initPopulateTable(con, "pgbench_tellers", ntellers, initTeller);
5028  initPopulateTable(con, "pgbench_accounts", naccounts, initAccount);
5029 
5030  executeStatement(con, "commit");
5031 }
5032 
5033 /*
5034  * Fill the standard tables with some data generated on the server
5035  *
5036  * As already the case with the client-side data generation, the filler
5037  * column defaults to NULL in pgbench_branches and pgbench_tellers,
5038  * and is a blank-padded string in pgbench_accounts.
5039  */
5040 static void
5042 {
5043  PQExpBufferData sql;
5044 
5045  fprintf(stderr, "generating data (server-side)...\n");
5046 
5047  /*
5048  * we do all of this in one transaction to enable the backend's
5049  * data-loading optimizations
5050  */
5051  executeStatement(con, "begin");
5052 
5053  /* truncate away any old data */
5054  initTruncateTables(con);
5055 
5056  initPQExpBuffer(&sql);
5057 
5058  printfPQExpBuffer(&sql,
5059  "insert into pgbench_branches(bid,bbalance) "
5060  "select bid, 0 "
5061  "from generate_series(1, %d) as bid", nbranches * scale);
5062  executeStatement(con, sql.data);
5063 
5064  printfPQExpBuffer(&sql,
5065  "insert into pgbench_tellers(tid,bid,tbalance) "
5066  "select tid, (tid - 1) / %d + 1, 0 "
5067  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
5068  executeStatement(con, sql.data);
5069 
5070  printfPQExpBuffer(&sql,
5071  "insert into pgbench_accounts(aid,bid,abalance,filler) "
5072  "select aid, (aid - 1) / %d + 1, 0, '' "
5073  "from generate_series(1, " INT64_FORMAT ") as aid",
5074  naccounts, (int64) naccounts * scale);
5075  executeStatement(con, sql.data);
5076 
5077  termPQExpBuffer(&sql);
5078 
5079  executeStatement(con, "commit");
5080 }
5081 
5082 /*
5083  * Invoke vacuum on the standard tables
5084  */
5085 static void
5086 initVacuum(PGconn *con)
5087 {
5088  fprintf(stderr, "vacuuming...\n");
5089  executeStatement(con, "vacuum analyze pgbench_branches");
5090  executeStatement(con, "vacuum analyze pgbench_tellers");
5091  executeStatement(con, "vacuum analyze pgbench_accounts");
5092  executeStatement(con, "vacuum analyze pgbench_history");
5093 }
5094 
5095 /*
5096  * Create primary keys on the standard tables
5097  */
5098 static void
5099 initCreatePKeys(PGconn *con)
5100 {
5101  static const char *const DDLINDEXes[] = {
5102  "alter table pgbench_branches add primary key (bid)",
5103  "alter table pgbench_tellers add primary key (tid)",
5104  "alter table pgbench_accounts add primary key (aid)"
5105  };
5106  int i;
5107  PQExpBufferData query;
5108 
5109  fprintf(stderr, "creating primary keys...\n");
5110  initPQExpBuffer(&query);
5111 
5112  for (i = 0; i < lengthof(DDLINDEXes); i++)
5113  {
5114  resetPQExpBuffer(&query);
5115  appendPQExpBufferStr(&query, DDLINDEXes[i]);
5116 
5117  if (index_tablespace != NULL)
5118  {
5119  char *escape_tablespace;
5120 
5121  escape_tablespace = PQescapeIdentifier(con, index_tablespace,
5122  strlen(index_tablespace));
5123  appendPQExpBuffer(&query, " using index tablespace %s", escape_tablespace);
5124  PQfreemem(escape_tablespace);
5125  }
5126 
5127  executeStatement(con, query.data);
5128  }
5129 
5130  termPQExpBuffer(&query);
5131 }
5132 
5133 /*
5134  * Create foreign key constraints between the standard tables
5135  */
5136 static void
5137 initCreateFKeys(PGconn *con)
5138 {
5139  static const char *const DDLKEYs[] = {
5140  "alter table pgbench_tellers add constraint pgbench_tellers_bid_fkey foreign key (bid) references pgbench_branches",
5141  "alter table pgbench_accounts add constraint pgbench_accounts_bid_fkey foreign key (bid) references pgbench_branches",
5142  "alter table pgbench_history add constraint pgbench_history_bid_fkey foreign key (bid) references pgbench_branches",
5143  "alter table pgbench_history add constraint pgbench_history_tid_fkey foreign key (tid) references pgbench_tellers",
5144  "alter table pgbench_history add constraint pgbench_history_aid_fkey foreign key (aid) references pgbench_accounts"
5145  };
5146  int i;
5147 
5148  fprintf(stderr, "creating foreign keys...\n");
5149  for (i = 0; i < lengthof(DDLKEYs); i++)
5150  {
5151  executeStatement(con, DDLKEYs[i]);
5152  }
5153 }
5154 
5155 /*
5156  * Validate an initialization-steps string
5157  *
5158  * (We could just leave it to runInitSteps() to fail if there are wrong
5159  * characters, but since initialization can take awhile, it seems friendlier
5160  * to check during option parsing.)
5161  */
5162 static void
5163 checkInitSteps(const char *initialize_steps)
5164 {
5165  if (initialize_steps[0] == '\0')
5166  pg_fatal("no initialization steps specified");
5167 
5168  for (const char *step = initialize_steps; *step != '\0'; step++)
5169  {
5170  if (strchr(ALL_INIT_STEPS " ", *step) == NULL)
5171  {
5172  pg_log_error("unrecognized initialization step \"%c\"", *step);
5173  pg_log_error_detail("Allowed step characters are: \"" ALL_INIT_STEPS "\".");
5174  exit(1);
5175  }
5176  }
5177 }
5178 
5179 /*
5180  * Invoke each initialization step in the given string
5181  */
5182 static void
5183 runInitSteps(const char *initialize_steps)
5184 {
5185  PQExpBufferData stats;
5186  PGconn *con;
5187  const char *step;
5188  double run_time = 0.0;
5189  bool first = true;
5190 
5191  initPQExpBuffer(&stats);
5192 
5193  if ((con = doConnect()) == NULL)
5194  pg_fatal("could not create connection for initialization");
5195 
5196  setup_cancel_handler(NULL);
5197  SetCancelConn(con);
5198 
5199  for (step = initialize_steps; *step != '\0'; step++)
5200  {
5201  char *op = NULL;
5202  pg_time_usec_t start = pg_time_now();
5203 
5204  switch (*step)
5205  {
5206  case 'd':
5207  op = "drop tables";
5208  initDropTables(con);
5209  break;
5210  case 't':
5211  op = "create tables";
5212  initCreateTables(con);
5213  break;
5214  case 'g':
5215  op = "client-side generate";
5217  break;
5218  case 'G':
5219  op = "server-side generate";
5221  break;
5222  case 'v':
5223  op = "vacuum";
5224  initVacuum(con);
5225  break;
5226  case 'p':
5227  op = "primary keys";
5228  initCreatePKeys(con);
5229  break;
5230  case 'f':
5231  op = "foreign keys";
5232  initCreateFKeys(con);
5233  break;
5234  case ' ':
5235  break; /* ignore */
5236  default:
5237  pg_log_error("unrecognized initialization step \"%c\"", *step);
5238  PQfinish(con);
5239  exit(1);
5240  }
5241 
5242  if (op != NULL)
5243  {
5244  double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
5245 
5246  if (!first)
5247  appendPQExpBufferStr(&stats, ", ");
5248  else
5249  first = false;
5250 
5251  appendPQExpBuffer(&stats, "%s %.2f s", op, elapsed_sec);
5252 
5253  run_time += elapsed_sec;
5254  }
5255  }
5256 
5257  fprintf(stderr, "done in %.2f s (%s).\n", run_time, stats.data);
5258  ResetCancelConn();
5259  PQfinish(con);
5260  termPQExpBuffer(&stats);
5261 }
5262 
5263 /*
5264  * Extract pgbench table information into global variables scale,
5265  * partition_method and partitions.
5266  */
5267 static void
5268 GetTableInfo(PGconn *con, bool scale_given)
5269 {
5270  PGresult *res;
5271 
5272  /*
5273  * get the scaling factor that should be same as count(*) from
5274  * pgbench_branches if this is not a custom query
5275  */
5276  res = PQexec(con, "select count(*) from pgbench_branches");
5278  {
5279  char *sqlState = PQresultErrorField(res, PG_DIAG_SQLSTATE);
5280 
5281  pg_log_error("could not count number of branches: %s", PQerrorMessage(con));
5282 
5283  if (sqlState && strcmp(sqlState, ERRCODE_UNDEFINED_TABLE) == 0)
5284  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".",
5285  PQdb(con));
5286 
5287  exit(1);
5288  }
5289  scale = atoi(PQgetvalue(res, 0, 0));
5290  if (scale < 0)
5291  pg_fatal("invalid count(*) from pgbench_branches: \"%s\"",
5292  PQgetvalue(res, 0, 0));
5293  PQclear(res);
5294 
5295  /* warn if we override user-given -s switch */
5296  if (scale_given)
5297  pg_log_warning("scale option ignored, using count from pgbench_branches table (%d)",
5298  scale);
5299 
5300  /*
5301  * Get the partition information for the first "pgbench_accounts" table
5302  * found in search_path.
5303  *
5304  * The result is empty if no "pgbench_accounts" is found.
5305  *
5306  * Otherwise, it always returns one row even if the table is not
5307  * partitioned (in which case the partition strategy is NULL).
5308  *
5309  * The number of partitions can be 0 even for partitioned tables, if no
5310  * partition is attached.
5311  *
5312  * We assume no partitioning on any failure, so as to avoid failing on an
5313  * old version without "pg_partitioned_table".
5314  */
5315  res = PQexec(con,
5316  "select o.n, p.partstrat, pg_catalog.count(i.inhparent) "
5317  "from pg_catalog.pg_class as c "
5318  "join pg_catalog.pg_namespace as n on (n.oid = c.relnamespace) "
5319  "cross join lateral (select pg_catalog.array_position(pg_catalog.current_schemas(true), n.nspname)) as o(n) "
5320  "left join pg_catalog.pg_partitioned_table as p on (p.partrelid = c.oid) "
5321  "left join pg_catalog.pg_inherits as i on (c.oid = i.inhparent) "
5322  "where c.relname = 'pgbench_accounts' and o.n is not null "
5323  "group by 1, 2 "
5324  "order by 1 asc "
5325  "limit 1");
5326 
5328  {
5329  /* probably an older version, coldly assume no partitioning */
5331  partitions = 0;
5332  }
5333  else if (PQntuples(res) == 0)
5334  {
5335  /*
5336  * This case is unlikely as pgbench already found "pgbench_branches"
5337  * above to compute the scale.
5338  */
5339  pg_log_error("no pgbench_accounts table found in search_path");
5340  pg_log_error_hint("Perhaps you need to do initialization (\"pgbench -i\") in database \"%s\".", PQdb(con));
5341  exit(1);
5342  }
5343  else /* PQntuples(res) == 1 */
5344  {
5345  /* normal case, extract partition information */
5346  if (PQgetisnull(res, 0, 1))
5348  else
5349  {
5350  char *ps = PQgetvalue(res, 0, 1);
5351 
5352  /* column must be there */
5353  Assert(ps != NULL);
5354 
5355  if (strcmp(ps, "r") == 0)
5357  else if (strcmp(ps, "h") == 0)
5359  else
5360  {
5361  /* possibly a newer version with new partition method */
5362  pg_fatal("unexpected partition method: \"%s\"", ps);
5363  }
5364  }
5365 
5366  partitions = atoi(PQgetvalue(res, 0, 2));
5367  }
5368 
5369  PQclear(res);
5370 }
5371 
5372 /*
5373  * Replace :param with $n throughout the command's SQL text, which
5374  * is a modifiable string in cmd->lines.
5375  */
5376 static bool
5377 parseQuery(Command *cmd)
5378 {
5379  char *sql,
5380  *p;
5381 
5382  cmd->argc = 1;
5383 
5384  p = sql = pg_strdup(cmd->lines.data);
5385  while ((p = strchr(p, ':')) != NULL)
5386  {
5387  char var[13];
5388  char *name;
5389  int eaten;
5390 
5391  name = parseVariable(p, &eaten);
5392  if (name == NULL)
5393  {
5394  while (*p == ':')
5395  {
5396  p++;
5397  }
5398  continue;
5399  }
5400 
5401  /*
5402  * cmd->argv[0] is the SQL statement itself, so the max number of
5403  * arguments is one less than MAX_ARGS
5404  */
5405  if (cmd->argc >= MAX_ARGS)
5406  {
5407  pg_log_error("statement has too many arguments (maximum is %d): %s",
5408  MAX_ARGS - 1, cmd->lines.data);
5409  pg_free(name);
5410  return false;
5411  }
5412 
5413  sprintf(var, "$%d", cmd->argc);
5414  p = replaceVariable(&sql, p, eaten, var);
5415 
5416  cmd->argv[cmd->argc] = name;
5417  cmd->argc++;
5418  }
5419 
5420  Assert(cmd->argv[0] == NULL);
5421  cmd->argv[0] = sql;
5422  return true;
5423 }
5424 
5425 /*
5426  * syntax error while parsing a script (in practice, while parsing a
5427  * backslash command, because we don't detect syntax errors in SQL)
5428  *
5429  * source: source of script (filename or builtin-script ID)
5430  * lineno: line number within script (count from 1)
5431  * line: whole line of backslash command, if available
5432  * command: backslash command name, if available
5433  * msg: the actual error message
5434  * more: optional extra message
5435  * column: zero-based column number, or -1 if unknown
5436  */
5437 void
5438 syntax_error(const char *source, int lineno,
5439  const char *line, const char *command,
5440  const char *msg, const char *more, int column)
5441 {
5443 
5444  initPQExpBuffer(&buf);
5445 
5446  printfPQExpBuffer(&buf, "%s:%d: %s", source, lineno, msg);
5447  if (more != NULL)
5448  appendPQExpBuffer(&buf, " (%s)", more);
5449  if (column >= 0 && line == NULL)
5450  appendPQExpBuffer(&buf, " at column %d", column + 1);
5451  if (command != NULL)
5452  appendPQExpBuffer(&buf, " in command \"%s\"", command);
5453 
5454  pg_log_error("%s", buf.data);
5455 
5456  termPQExpBuffer(&buf);
5457 
5458  if (line != NULL)
5459  {
5460  fprintf(stderr, "%s\n", line);
5461  if (column >= 0)
5462  fprintf(stderr, "%*c error found here\n", column + 1, '^');
5463  }
5464 
5465  exit(1);
5466 }
5467 
5468 /*
5469  * Return a pointer to the start of the SQL command, after skipping over
5470  * whitespace and "--" comments.
5471  * If the end of the string is reached, return NULL.
5472  */
5473 static char *
5474 skip_sql_comments(char *sql_command)
5475 {
5476  char *p = sql_command;
5477 
5478  /* Skip any leading whitespace, as well as "--" style comments */
5479  for (;;)
5480  {
5481  if (isspace((unsigned char) *p))
5482  p++;
5483  else if (strncmp(p, "--", 2) == 0)
5484  {
5485  p = strchr(p, '\n');
5486  if (p == NULL)
5487  return NULL;
5488  p++;
5489  }
5490  else
5491  break;
5492  }
5493 
5494  /* NULL if there's nothing but whitespace and comments */
5495  if (*p == '\0')
5496  return NULL;
5497 
5498  return p;
5499 }
5500 
5501 /*
5502  * Parse a SQL command; return a Command struct, or NULL if it's a comment
5503  *
5504  * On entry, psqlscan.l has collected the command into "buf", so we don't
5505  * really need to do much here except check for comments and set up a Command
5506  * struct.
5507  */
5508 static Command *
5510 {
5511  Command *my_command;
5512  char *p = skip_sql_comments(buf->data);
5513 
5514  if (p == NULL)
5515  return NULL;
5516 
5517  /* Allocate and initialize Command structure */
5518  my_command = (Command *) pg_malloc(sizeof(Command));
5519  initPQExpBuffer(&my_command->lines);
5520  appendPQExpBufferStr(&my_command->lines, p);
5521  my_command->first_line = NULL; /* this is set later */
5522  my_command->type = SQL_COMMAND;
5523  my_command->meta = META_NONE;
5524  my_command->argc = 0;
5525  my_command->retries = 0;
5526  my_command->failures = 0;
5527  memset(my_command->argv, 0, sizeof(my_command->argv));
5528  my_command->varprefix = NULL; /* allocated later, if needed */
5529  my_command->expr = NULL;
5530  initSimpleStats(&my_command->stats);
5531  my_command->prepname = NULL; /* set later, if needed */
5532 
5533  return my_command;
5534 }
5535 
5536 /* Free a Command structure and associated data */
5537 static void
5538 free_command(Command *command)
5539 {
5540  termPQExpBuffer(&command->lines);
5541  pg_free(command->first_line);
5542  for (int i = 0; i < command->argc; i++)
5543  pg_free(command->argv[i]);
5544  pg_free(command->varprefix);
5545 
5546  /*
5547