PostgreSQL Source Code  git master
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 
21 #include "access/heapam.h"
22 #include "access/htup_details.h"
23 #include "access/sysattr.h"
24 #include "access/tableam.h"
25 #include "access/xact.h"
26 #include "access/xlog.h"
27 #include "catalog/dependency.h"
28 #include "catalog/pg_authid.h"
29 #include "catalog/pg_type.h"
30 #include "commands/copy.h"
31 #include "commands/defrem.h"
32 #include "commands/trigger.h"
33 #include "executor/execPartition.h"
34 #include "executor/executor.h"
35 #include "executor/nodeModifyTable.h"
36 #include "executor/tuptable.h"
37 #include "foreign/fdwapi.h"
38 #include "libpq/libpq.h"
39 #include "libpq/pqformat.h"
40 #include "mb/pg_wchar.h"
41 #include "miscadmin.h"
42 #include "nodes/makefuncs.h"
43 #include "optimizer/optimizer.h"
44 #include "parser/parse_coerce.h"
45 #include "parser/parse_collate.h"
46 #include "parser/parse_expr.h"
47 #include "parser/parse_relation.h"
48 #include "port/pg_bswap.h"
49 #include "rewrite/rewriteHandler.h"
50 #include "storage/fd.h"
51 #include "tcop/tcopprot.h"
52 #include "utils/builtins.h"
53 #include "utils/lsyscache.h"
54 #include "utils/memutils.h"
55 #include "utils/partcache.h"
56 #include "utils/portal.h"
57 #include "utils/rel.h"
58 #include "utils/rls.h"
59 #include "utils/snapmgr.h"
60 
61 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
62 #define OCTVALUE(c) ((c) - '0')
63 
64 /*
65  * Represents the different source/dest cases we need to worry about at
66  * the bottom level
67  */
68 typedef enum CopyDest
69 {
70  COPY_FILE, /* to/from file (or a piped program) */
71  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
72  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
73  COPY_CALLBACK /* to/from callback function */
74 } CopyDest;
75 
76 /*
77  * Represents the end-of-line terminator type of the input
78  */
79 typedef enum EolType
80 {
81  EOL_UNKNOWN,
82  EOL_NL,
83  EOL_CR,
84  EOL_CRNL
85 } EolType;
86 
87 /*
88  * Represents the heap insert method to be used during COPY FROM.
89  */
90 typedef enum CopyInsertMethod
91 {
92  CIM_SINGLE, /* use table_tuple_insert or fdw routine */
93  CIM_MULTI, /* always use table_multi_insert */
94  CIM_MULTI_CONDITIONAL /* use table_multi_insert only if valid */
95 } CopyInsertMethod;
96 
97 /*
98  * This struct contains all the state variables used throughout a COPY
99  * operation. For simplicity, we use the same struct for all variants of COPY,
100  * even though some fields are used in only some cases.
101  *
102  * Multi-byte encodings: all supported client-side encodings encode multi-byte
103  * characters by having the first byte's high bit set. Subsequent bytes of the
104  * character can have the high bit not set. When scanning data in such an
105  * encoding to look for a match to a single-byte (ie ASCII) character, we must
106  * use the full pg_encoding_mblen() machinery to skip over multibyte
107  * characters, else we might find a false match to a trailing byte. In
108  * supported server encodings, there is no possibility of a false match, and
109  * it's faster to make useless comparisons to trailing bytes than it is to
110  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
111  * when we have to do it the hard way.
112  */
113 typedef struct CopyStateData
114 {
115  /* low-level state data */
116  CopyDest copy_dest; /* type of copy source/destination */
117  FILE *copy_file; /* used if copy_dest == COPY_FILE */
118  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
119  * dest == COPY_NEW_FE in COPY FROM */
120  bool is_copy_from; /* COPY TO, or COPY FROM? */
121  bool reached_eof; /* true if we read to end of copy data (not
122  * all copy_dest types maintain this) */
123  EolType eol_type; /* EOL type of input */
124  int file_encoding; /* file or remote side's character encoding */
125  bool need_transcoding; /* file encoding diff from server? */
126  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
127 
128  /* parameters from the COPY command */
129  Relation rel; /* relation to copy to or from */
130  QueryDesc *queryDesc; /* executable query to copy from */
131  List *attnumlist; /* integer list of attnums to copy */
132  char *filename; /* filename, or NULL for STDIN/STDOUT */
133  bool is_program; /* is 'filename' a program to popen? */
134  copy_data_source_cb data_source_cb; /* function for reading data */
135  bool binary; /* binary format? */
136  bool freeze; /* freeze rows on loading? */
137  bool csv_mode; /* Comma Separated Value format? */
138  bool header_line; /* CSV header line? */
139  char *null_print; /* NULL marker string (server encoding!) */
140  int null_print_len; /* length of same */
141  char *null_print_client; /* same converted to file encoding */
142  char *delim; /* column delimiter (must be 1 byte) */
143  char *quote; /* CSV quote char (must be 1 byte) */
144  char *escape; /* CSV escape char (must be 1 byte) */
145  List *force_quote; /* list of column names */
146  bool force_quote_all; /* FORCE_QUOTE *? */
147  bool *force_quote_flags; /* per-column CSV FQ flags */
148  List *force_notnull; /* list of column names */
149  bool *force_notnull_flags; /* per-column CSV FNN flags */
150  List *force_null; /* list of column names */
151  bool *force_null_flags; /* per-column CSV FN flags */
152  bool convert_selectively; /* do selective binary conversion? */
153  List *convert_select; /* list of column names (can be NIL) */
154  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
155  Node *whereClause; /* WHERE condition (or NULL) */
156 
157  /* these are just for error messages, see CopyFromErrorCallback */
158  const char *cur_relname; /* table name for error messages */
159  uint64 cur_lineno; /* line number for error messages */
160  const char *cur_attname; /* current att for error messages */
161  const char *cur_attval; /* current att value for error messages */
162 
163  /*
164  * Working state for COPY TO/FROM
165  */
166  MemoryContext copycontext; /* per-copy execution context */
167 
168  /*
169  * Working state for COPY TO
170  */
171  FmgrInfo *out_functions; /* lookup info for output functions */
172  MemoryContext rowcontext; /* per-row evaluation context */
173 
174  /*
175  * Working state for COPY FROM
176  */
177  AttrNumber num_defaults;
178  FmgrInfo *in_functions; /* array of input functions for each attrs */
179  Oid *typioparams; /* array of element types for in_functions */
180  int *defmap; /* array of default att numbers */
181  ExprState **defexprs; /* array of default att expressions */
182  bool volatile_defexprs; /* is any of defexprs volatile? */
183  List *range_table;
184  ExprState *qualexpr;
185 
186  TransitionCaptureState *transition_capture;
187 
188  /*
189  * These variables are used to reduce overhead in textual COPY FROM.
190  *
191  * attribute_buf holds the separated, de-escaped text for each field of
192  * the current line. The CopyReadAttributes functions return arrays of
193  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
194  * the buffer on each cycle.
195  */
196  StringInfoData attribute_buf;
197 
198  /* field raw data pointers found by COPY FROM */
199 
200  int max_fields;
201  char **raw_fields;
202 
203  /*
204  * Similarly, line_buf holds the whole input line being processed. The
205  * input cycle is first to read the whole line into line_buf, convert it
206  * to server encoding there, and then extract the individual attribute
207  * fields into attribute_buf. line_buf is preserved unmodified so that we
208  * can display it in error messages if appropriate.
209  */
210  StringInfoData line_buf;
211  bool line_buf_converted; /* converted to server encoding? */
212  bool line_buf_valid; /* contains the row being processed? */
213 
214  /*
215  * Finally, raw_buf holds raw data read from the data source (file or
216  * client connection). CopyReadLine parses this data sufficiently to
217  * locate line boundaries, then transfers the data to line_buf and
218  * converts it. Note: we guarantee that there is a \0 at
219  * raw_buf[raw_buf_len].
220  */
221 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
222  char *raw_buf;
223  int raw_buf_index; /* next byte to process */
224  int raw_buf_len; /* total # of bytes stored */
225 } CopyStateData;
226 
227 /* DestReceiver for COPY (query) TO */
228 typedef struct
229 {
230  DestReceiver pub; /* publicly-known function pointers */
231  CopyState cstate; /* CopyStateData for the command */
232  uint64 processed; /* # of tuples processed */
233 } DR_copy;
234 
235 
236 /*
237  * No more than this many tuples per CopyMultiInsertBuffer
238  *
239  * Caution: Don't make this too big, as we could end up with this many
240  * CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's
241  * multiInsertBuffers list. Increasing this can cause quadratic growth in
242  * memory requirements during copies into partitioned tables with a large
243  * number of partitions.
244  */
245 #define MAX_BUFFERED_TUPLES 1000
246 
247 /*
248  * Flush buffers if there are >= this many bytes, as counted by the input
249  * size, of tuples stored.
250  */
251 #define MAX_BUFFERED_BYTES 65535
252 
253 /* Trim the list of buffers back down to this number after flushing */
254 #define MAX_PARTITION_BUFFERS 32
255 
256 /* Stores multi-insert data related to a single relation in CopyFrom. */
257 typedef struct CopyMultiInsertBuffer
258 {
259  TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */
260  ResultRelInfo *resultRelInfo; /* ResultRelInfo for 'relid' */
261  BulkInsertState bistate; /* BulkInsertState for this rel */
262  int nused; /* number of 'slots' containing tuples */
263  uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy
264  * stream */
265 } CopyMultiInsertBuffer;
266 
267 /*
268  * Stores one or many CopyMultiInsertBuffers and details about the size and
269  * number of tuples which are stored in them. This allows multiple buffers to
270  * exist at once when COPYing into a partitioned table.
271  */
272 typedef struct CopyMultiInsertInfo
273 {
274  List *multiInsertBuffers; /* List of tracked CopyMultiInsertBuffers */
275  int bufferedTuples; /* number of tuples buffered over all buffers */
276  int bufferedBytes; /* number of bytes from all buffered tuples */
277  CopyState cstate; /* Copy state for this CopyMultiInsertInfo */
278  EState *estate; /* Executor state used for COPY */
279  CommandId mycid; /* Command Id used for COPY */
280  int ti_options; /* table insert options */
281 } CopyMultiInsertInfo;
282 
283 
284 /*
285  * These macros centralize code used to process line_buf and raw_buf buffers.
286  * They are macros because they often do continue/break control and to avoid
287  * function call overhead in tight COPY loops.
288  *
289  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
290  * prevent the continue/break processing from working. We end the "if (1)"
291  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
292  * any "else" in the calling code, and to avoid any compiler warnings about
293  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
294  */
295 
296 /*
297  * This keeps the character read at the top of the loop in the buffer
298  * even if there is more than one read-ahead.
299  */
300 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
301 if (1) \
302 { \
303  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
304  { \
305  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
306  need_data = true; \
307  continue; \
308  } \
309 } else ((void) 0)
310 
311 /* This consumes the remainder of the buffer and breaks */
312 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
313 if (1) \
314 { \
315  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
316  { \
317  if (extralen) \
318  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
319  /* backslash just before EOF, treat as data char */ \
320  result = true; \
321  break; \
322  } \
323 } else ((void) 0)
324 
325 /*
326  * Transfer any approved data to line_buf; must do this to be sure
327  * there is some room in raw_buf.
328  */
329 #define REFILL_LINEBUF \
330 if (1) \
331 { \
332  if (raw_buf_ptr > cstate->raw_buf_index) \
333  { \
334  appendBinaryStringInfo(&cstate->line_buf, \
335  cstate->raw_buf + cstate->raw_buf_index, \
336  raw_buf_ptr - cstate->raw_buf_index); \
337  cstate->raw_buf_index = raw_buf_ptr; \
338  } \
339 } else ((void) 0)
340 
341 /* Undo any read-ahead and jump out of the block. */
342 #define NO_END_OF_COPY_GOTO \
343 if (1) \
344 { \
345  raw_buf_ptr = prev_raw_ptr + 1; \
346  goto not_end_of_copy; \
347 } else ((void) 0)
348 
349 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
350 
351 
352 /* non-export function prototypes */
353 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
354  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
355  List *options);
356 static void EndCopy(CopyState cstate);
357 static void ClosePipeToProgram(CopyState cstate);
358 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
359  Oid queryRelId, const char *filename, bool is_program,
360  List *attnamelist, List *options);
361 static void EndCopyTo(CopyState cstate);
362 static uint64 DoCopyTo(CopyState cstate);
363 static uint64 CopyTo(CopyState cstate);
364 static void CopyOneRowTo(CopyState cstate, TupleTableSlot *slot);
365 static bool CopyReadLine(CopyState cstate);
366 static bool CopyReadLineText(CopyState cstate);
367 static int CopyReadAttributesText(CopyState cstate);
368 static int CopyReadAttributesCSV(CopyState cstate);
369 static Datum CopyReadBinaryAttribute(CopyState cstate,
370  int column_no, FmgrInfo *flinfo,
371  Oid typioparam, int32 typmod,
372  bool *isnull);
373 static void CopyAttributeOutText(CopyState cstate, char *string);
374 static void CopyAttributeOutCSV(CopyState cstate, char *string,
375  bool use_quote, bool single_attr);
376 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
377  List *attnamelist);
378 static char *limit_printout_length(const char *str);
379 
380 /* Low-level communications functions */
381 static void SendCopyBegin(CopyState cstate);
382 static void ReceiveCopyBegin(CopyState cstate);
383 static void SendCopyEnd(CopyState cstate);
384 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
385 static void CopySendString(CopyState cstate, const char *str);
386 static void CopySendChar(CopyState cstate, char c);
387 static void CopySendEndOfRow(CopyState cstate);
388 static int CopyGetData(CopyState cstate, void *databuf,
389  int minread, int maxread);
390 static void CopySendInt32(CopyState cstate, int32 val);
391 static bool CopyGetInt32(CopyState cstate, int32 *val);
392 static void CopySendInt16(CopyState cstate, int16 val);
393 static bool CopyGetInt16(CopyState cstate, int16 *val);
394 
395 
396 /*
397  * Send copy start/stop messages for frontend copies. These have changed
398  * in past protocol redesigns.
399  */
400 static void
401 SendCopyBegin(CopyState cstate)
402 {
403  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
404  {
405  /* new way */
406  StringInfoData buf;
407  int natts = list_length(cstate->attnumlist);
408  int16 format = (cstate->binary ? 1 : 0);
409  int i;
410 
411  pq_beginmessage(&buf, 'H');
412  pq_sendbyte(&buf, format); /* overall format */
413  pq_sendint16(&buf, natts);
414  for (i = 0; i < natts; i++)
415  pq_sendint16(&buf, format); /* per-column formats */
416  pq_endmessage(&buf);
417  cstate->copy_dest = COPY_NEW_FE;
418  }
419  else
420  {
421  /* old way */
422  if (cstate->binary)
423  ereport(ERROR,
424  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
425  errmsg("COPY BINARY is not supported to stdout or from stdin")));
426  pq_putemptymessage('H');
427  /* grottiness needed for old COPY OUT protocol */
428  pq_startcopyout();
429  cstate->copy_dest = COPY_OLD_FE;
430  }
431 }
432 
433 static void
434 ReceiveCopyBegin(CopyState cstate)
435 {
436  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
437  {
438  /* new way */
439  StringInfoData buf;
440  int natts = list_length(cstate->attnumlist);
441  int16 format = (cstate->binary ? 1 : 0);
442  int i;
443 
444  pq_beginmessage(&buf, 'G');
445  pq_sendbyte(&buf, format); /* overall format */
446  pq_sendint16(&buf, natts);
447  for (i = 0; i < natts; i++)
448  pq_sendint16(&buf, format); /* per-column formats */
449  pq_endmessage(&buf);
450  cstate->copy_dest = COPY_NEW_FE;
451  cstate->fe_msgbuf = makeStringInfo();
452  }
453  else
454  {
455  /* old way */
456  if (cstate->binary)
457  ereport(ERROR,
458  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
459  errmsg("COPY BINARY is not supported to stdout or from stdin")));
460  pq_putemptymessage('G');
461  /* any error in old protocol will make us lose sync */
462  pq_startmsgread();
463  cstate->copy_dest = COPY_OLD_FE;
464  }
465  /* We *must* flush here to ensure FE knows it can send. */
466  pq_flush();
467 }
468 
469 static void
470 SendCopyEnd(CopyState cstate)
471 {
472  if (cstate->copy_dest == COPY_NEW_FE)
473  {
474  /* Shouldn't have any unsent data */
475  Assert(cstate->fe_msgbuf->len == 0);
476  /* Send Copy Done message */
477  pq_putemptymessage('c');
478  }
479  else
480  {
481  CopySendData(cstate, "\\.", 2);
482  /* Need to flush out the trailer (this also appends a newline) */
483  CopySendEndOfRow(cstate);
484  pq_endcopyout(false);
485  }
486 }
487 
488 /*----------
489  * CopySendData sends output data to the destination (file or frontend)
490  * CopySendString does the same for null-terminated strings
491  * CopySendChar does the same for single characters
492  * CopySendEndOfRow does the appropriate thing at end of each data row
493  * (data is not actually flushed except by CopySendEndOfRow)
494  *
495  * NB: no data conversion is applied by these functions
496  *----------
497  */
498 static void
499 CopySendData(CopyState cstate, const void *databuf, int datasize)
500 {
501  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
502 }
503 
504 static void
505 CopySendString(CopyState cstate, const char *str)
506 {
507  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
508 }
509 
510 static void
511 CopySendChar(CopyState cstate, char c)
512 {
513  appendStringInfoCharMacro(cstate->fe_msgbuf, c);
514 }
515 
516 static void
517 CopySendEndOfRow(CopyState cstate)
518 {
519  StringInfo fe_msgbuf = cstate->fe_msgbuf;
520 
521  switch (cstate->copy_dest)
522  {
523  case COPY_FILE:
524  if (!cstate->binary)
525  {
526  /* Default line termination depends on platform */
527 #ifndef WIN32
528  CopySendChar(cstate, '\n');
529 #else
530  CopySendString(cstate, "\r\n");
531 #endif
532  }
533 
534  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
535  cstate->copy_file) != 1 ||
536  ferror(cstate->copy_file))
537  {
538  if (cstate->is_program)
539  {
540  if (errno == EPIPE)
541  {
542  /*
543  * The pipe will be closed automatically on error at
544  * the end of transaction, but we might get a better
545  * error message from the subprocess' exit code than
546  * just "Broken Pipe"
547  */
548  ClosePipeToProgram(cstate);
549 
550  /*
551  * If ClosePipeToProgram() didn't throw an error, the
552  * program terminated normally, but closed the pipe
553  * first. Restore errno, and throw an error.
554  */
555  errno = EPIPE;
556  }
557  ereport(ERROR,
558  (errcode_for_file_access(),
559  errmsg("could not write to COPY program: %m")));
560  }
561  else
562  ereport(ERROR,
563  (errcode_for_file_access(),
564  errmsg("could not write to COPY file: %m")));
565  }
566  break;
567  case COPY_OLD_FE:
568  /* The FE/BE protocol uses \n as newline for all platforms */
569  if (!cstate->binary)
570  CopySendChar(cstate, '\n');
571 
572  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
573  {
574  /* no hope of recovering connection sync, so FATAL */
575  ereport(FATAL,
576  (errcode(ERRCODE_CONNECTION_FAILURE),
577  errmsg("connection lost during COPY to stdout")));
578  }
579  break;
580  case COPY_NEW_FE:
581  /* The FE/BE protocol uses \n as newline for all platforms */
582  if (!cstate->binary)
583  CopySendChar(cstate, '\n');
584 
585  /* Dump the accumulated row as one CopyData message */
586  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
587  break;
588  case COPY_CALLBACK:
589  Assert(false); /* Not yet supported. */
590  break;
591  }
592 
593  resetStringInfo(fe_msgbuf);
594 }
595 
596 /*
597  * CopyGetData reads data from the source (file or frontend)
598  *
599  * We attempt to read at least minread, and at most maxread, bytes from
600  * the source. The actual number of bytes read is returned; if this is
601  * less than minread, EOF was detected.
602  *
603  * Note: when copying from the frontend, we expect a proper EOF mark per
604  * protocol; if the frontend simply drops the connection, we raise error.
605  * It seems unwise to allow the COPY IN to complete normally in that case.
606  *
607  * NB: no data conversion is applied here.
608  */
609 static int
610 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
611 {
612  int bytesread = 0;
613 
614  switch (cstate->copy_dest)
615  {
616  case COPY_FILE:
617  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
618  if (ferror(cstate->copy_file))
619  ereport(ERROR,
620  (errcode_for_file_access(),
621  errmsg("could not read from COPY file: %m")));
622  if (bytesread == 0)
623  cstate->reached_eof = true;
624  break;
625  case COPY_OLD_FE:
626 
627  /*
628  * We cannot read more than minread bytes (which in practice is 1)
629  * because old protocol doesn't have any clear way of separating
630  * the COPY stream from following data. This is slow, but not any
631  * slower than the code path was originally, and we don't care
632  * much anymore about the performance of old protocol.
633  */
634  if (pq_getbytes((char *) databuf, minread))
635  {
636  /* Only a \. terminator is legal EOF in old protocol */
637  ereport(ERROR,
638  (errcode(ERRCODE_CONNECTION_FAILURE),
639  errmsg("unexpected EOF on client connection with an open transaction")));
640  }
641  bytesread = minread;
642  break;
643  case COPY_NEW_FE:
644  while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
645  {
646  int avail;
647 
648  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
649  {
650  /* Try to receive another message */
651  int mtype;
652 
653  readmessage:
654  HOLD_CANCEL_INTERRUPTS();
655  pq_startmsgread();
656  mtype = pq_getbyte();
657  if (mtype == EOF)
658  ereport(ERROR,
659  (errcode(ERRCODE_CONNECTION_FAILURE),
660  errmsg("unexpected EOF on client connection with an open transaction")));
661  if (pq_getmessage(cstate->fe_msgbuf, 0))
662  ereport(ERROR,
663  (errcode(ERRCODE_CONNECTION_FAILURE),
664  errmsg("unexpected EOF on client connection with an open transaction")));
665  RESUME_CANCEL_INTERRUPTS();
666  switch (mtype)
667  {
668  case 'd': /* CopyData */
669  break;
670  case 'c': /* CopyDone */
671  /* COPY IN correctly terminated by frontend */
672  cstate->reached_eof = true;
673  return bytesread;
674  case 'f': /* CopyFail */
675  ereport(ERROR,
676  (errcode(ERRCODE_QUERY_CANCELED),
677  errmsg("COPY from stdin failed: %s",
678  pq_getmsgstring(cstate->fe_msgbuf))));
679  break;
680  case 'H': /* Flush */
681  case 'S': /* Sync */
682 
683  /*
684  * Ignore Flush/Sync for the convenience of client
685  * libraries (such as libpq) that may send those
686  * without noticing that the command they just
687  * sent was COPY.
688  */
689  goto readmessage;
690  default:
691  ereport(ERROR,
692  (errcode(ERRCODE_PROTOCOL_VIOLATION),
693  errmsg("unexpected message type 0x%02X during COPY from stdin",
694  mtype)));
695  break;
696  }
697  }
698  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
699  if (avail > maxread)
700  avail = maxread;
701  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
702  databuf = (void *) ((char *) databuf + avail);
703  maxread -= avail;
704  bytesread += avail;
705  }
706  break;
707  case COPY_CALLBACK:
708  bytesread = cstate->data_source_cb(databuf, minread, maxread);
709  break;
710  }
711 
712  return bytesread;
713 }
714 
715 
716 /*
717  * These functions do apply some data conversion
718  */
719 
720 /*
721  * CopySendInt32 sends an int32 in network byte order
722  */
723 static void
724 CopySendInt32(CopyState cstate, int32 val)
725 {
726  uint32 buf;
727 
728  buf = pg_hton32((uint32) val);
729  CopySendData(cstate, &buf, sizeof(buf));
730 }
731 
732 /*
733  * CopyGetInt32 reads an int32 that appears in network byte order
734  *
735  * Returns true if OK, false if EOF
736  */
737 static bool
738 CopyGetInt32(CopyState cstate, int32 *val)
739 {
740  uint32 buf;
741 
742  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
743  {
744  *val = 0; /* suppress compiler warning */
745  return false;
746  }
747  *val = (int32) pg_ntoh32(buf);
748  return true;
749 }
750 
751 /*
752  * CopySendInt16 sends an int16 in network byte order
753  */
754 static void
755 CopySendInt16(CopyState cstate, int16 val)
756 {
757  uint16 buf;
758 
759  buf = pg_hton16((uint16) val);
760  CopySendData(cstate, &buf, sizeof(buf));
761 }
762 
763 /*
764  * CopyGetInt16 reads an int16 that appears in network byte order
765  */
766 static bool
767 CopyGetInt16(CopyState cstate, int16 *val)
768 {
769  uint16 buf;
770 
771  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
772  {
773  *val = 0; /* suppress compiler warning */
774  return false;
775  }
776  *val = (int16) pg_ntoh16(buf);
777  return true;
778 }
779 
780 
781 /*
782  * CopyLoadRawBuf loads some more data into raw_buf
783  *
784  * Returns true if able to obtain at least one more byte, else false.
785  *
786  * If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
787  * down to the start of the buffer and then we load more data after that.
788  * This case is used only when a frontend multibyte character crosses a
789  * bufferload boundary.
790  */
791 static bool
792 CopyLoadRawBuf(CopyState cstate)
793 {
794  int nbytes;
795  int inbytes;
796 
797  if (cstate->raw_buf_index < cstate->raw_buf_len)
798  {
799  /* Copy down the unprocessed data */
800  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
801  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
802  nbytes);
803  }
804  else
805  nbytes = 0; /* no data need be saved */
806 
807  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
808  1, RAW_BUF_SIZE - nbytes);
809  nbytes += inbytes;
810  cstate->raw_buf[nbytes] = '\0';
811  cstate->raw_buf_index = 0;
812  cstate->raw_buf_len = nbytes;
813  return (inbytes > 0);
814 }
815 
816 
817 /*
818  * DoCopy executes the SQL COPY statement
819  *
820  * Either unload or reload contents of table <relation>, depending on <from>.
821  * (<from> = true means we are inserting into the table.) In the "TO" case
822  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
823  * or DELETE query.
824  *
825  * If <pipe> is false, transfer is between the table and the file named
826  * <filename>. Otherwise, transfer is between the table and our regular
827  * input/output stream. The latter could be either stdin/stdout or a
828  * socket, depending on whether we're running under Postmaster control.
829  *
830  * Do not allow a Postgres user without the 'pg_read_server_files' or
831  * 'pg_write_server_files' role to read from or write to a file.
832  *
833  * Do not allow the copy if user doesn't have proper permission to access
834  * the table or the specifically requested columns.
835  */
836 void
837 DoCopy(ParseState *pstate, const CopyStmt *stmt,
838  int stmt_location, int stmt_len,
839  uint64 *processed)
840 {
841  CopyState cstate;
842  bool is_from = stmt->is_from;
843  bool pipe = (stmt->filename == NULL);
844  Relation rel;
845  Oid relid;
846  RawStmt *query = NULL;
847  Node *whereClause = NULL;
848 
849  /*
850  * Disallow COPY to/from file or program except to users with the
851  * appropriate role.
852  */
853  if (!pipe)
854  {
855  if (stmt->is_program)
856  {
857  if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_EXECUTE_SERVER_PROGRAM))
858  ereport(ERROR,
859  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
860  errmsg("must be superuser or a member of the pg_execute_server_program role to COPY to or from an external program"),
861  errhint("Anyone can COPY to stdout or from stdin. "
862  "psql's \\copy command also works for anyone.")));
863  }
864  else
865  {
866  if (is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_SERVER_FILES))
867  ereport(ERROR,
868  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
869  errmsg("must be superuser or a member of the pg_read_server_files role to COPY from a file"),
870  errhint("Anyone can COPY to stdout or from stdin. "
871  "psql's \\copy command also works for anyone.")));
872 
873  if (!is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_WRITE_SERVER_FILES))
874  ereport(ERROR,
875  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
876  errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
877  errhint("Anyone can COPY to stdout or from stdin. "
878  "psql's \\copy command also works for anyone.")));
879  }
880  }
881 
882  if (stmt->relation)
883  {
884  LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock;
885  RangeTblEntry *rte;
886  TupleDesc tupDesc;
887  List *attnums;
888  ListCell *cur;
889 
890  Assert(!stmt->query);
891 
892  /* Open and lock the relation, using the appropriate lock type. */
893  rel = table_openrv(stmt->relation, lockmode);
894 
895  relid = RelationGetRelid(rel);
896 
897  rte = addRangeTableEntryForRelation(pstate, rel, lockmode,
898  NULL, false, false);
899  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
900 
901  if (stmt->whereClause)
902  {
903  /* add rte to column namespace */
904  addRTEtoQuery(pstate, rte, false, true, true);
905 
906  /* Transform the raw expression tree */
907  whereClause = transformExpr(pstate, stmt->whereClause, EXPR_KIND_COPY_WHERE);
908 
909  /* Make sure it yields a boolean result. */
910  whereClause = coerce_to_boolean(pstate, whereClause, "WHERE");
911 
912  /* we have to fix its collations too */
913  assign_expr_collations(pstate, whereClause);
914 
915  whereClause = eval_const_expressions(NULL, whereClause);
916 
917  whereClause = (Node *) canonicalize_qual((Expr *) whereClause, false);
918  whereClause = (Node *) make_ands_implicit((Expr *) whereClause);
919  }
920 
921  tupDesc = RelationGetDescr(rel);
922  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
923  foreach(cur, attnums)
924  {
925  int attno = lfirst_int(cur) -
926  FirstLowInvalidHeapAttributeNumber;
927 
928  if (is_from)
929  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
930  else
931  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
932  }
933  ExecCheckRTPerms(pstate->p_rtable, true);
934 
935  /*
936  * Permission check for row security policies.
937  *
938  * check_enable_rls will ereport(ERROR) if the user has requested
939  * something invalid and will otherwise indicate if we should enable
940  * RLS (returns RLS_ENABLED) or not for this COPY statement.
941  *
942  * If the relation has a row security policy and we are to apply it
943  * then perform a "query" copy and allow the normal query processing
944  * to handle the policies.
945  *
946  * If RLS is not enabled for this, then just fall through to the
947  * normal non-filtering relation handling.
948  */
949  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
950  {
951  SelectStmt *select;
952  ColumnRef *cr;
953  ResTarget *target;
954  RangeVar *from;
955  List *targetList = NIL;
956 
957  if (is_from)
958  ereport(ERROR,
959  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
960  errmsg("COPY FROM not supported with row-level security"),
961  errhint("Use INSERT statements instead.")));
962 
963  /*
964  * Build target list
965  *
966  * If no columns are specified in the attribute list of the COPY
967  * command, then the target list is 'all' columns. Therefore, '*'
968  * should be used as the target list for the resulting SELECT
969  * statement.
970  *
971  * In the case that columns are specified in the attribute list,
972  * create a ColumnRef and ResTarget for each column and add them
973  * to the target list for the resulting SELECT statement.
974  */
975  if (!stmt->attlist)
976  {
977  cr = makeNode(ColumnRef);
978  cr->fields = list_make1(makeNode(A_Star));
979  cr->location = -1;
980 
981  target = makeNode(ResTarget);
982  target->name = NULL;
983  target->indirection = NIL;
984  target->val = (Node *) cr;
985  target->location = -1;
986 
987  targetList = list_make1(target);
988  }
989  else
990  {
991  ListCell *lc;
992 
993  foreach(lc, stmt->attlist)
994  {
995  /*
996  * Build the ColumnRef for each column. The ColumnRef
997  * 'fields' property is a String 'Value' node (see
998  * nodes/value.h) that corresponds to the column name
999  * respectively.
1000  */
1001  cr = makeNode(ColumnRef);
1002  cr->fields = list_make1(lfirst(lc));
1003  cr->location = -1;
1004 
1005  /* Build the ResTarget and add the ColumnRef to it. */
1006  target = makeNode(ResTarget);
1007  target->name = NULL;
1008  target->indirection = NIL;
1009  target->val = (Node *) cr;
1010  target->location = -1;
1011 
1012  /* Add each column to the SELECT statement's target list */
1013  targetList = lappend(targetList, target);
1014  }
1015  }
1016 
1017  /*
1018  * Build RangeVar for from clause, fully qualified based on the
1019  * relation which we have opened and locked.
1020  */
1021  from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
1022  pstrdup(RelationGetRelationName(rel)),
1023  -1);
1024 
1025  /* Build query */
1026  select = makeNode(SelectStmt);
1027  select->targetList = targetList;
1028  select->fromClause = list_make1(from);
1029 
1030  query = makeNode(RawStmt);
1031  query->stmt = (Node *) select;
1032  query->stmt_location = stmt_location;
1033  query->stmt_len = stmt_len;
1034 
1035  /*
1036  * Close the relation for now, but keep the lock on it to prevent
1037  * changes between now and when we start the query-based COPY.
1038  *
1039  * We'll reopen it later as part of the query-based COPY.
1040  */
1041  table_close(rel, NoLock);
1042  rel = NULL;
1043  }
1044  }
1045  else
1046  {
1047  Assert(stmt->query);
1048 
1049  query = makeNode(RawStmt);
1050  query->stmt = stmt->query;
1051  query->stmt_location = stmt_location;
1052  query->stmt_len = stmt_len;
1053 
1054  relid = InvalidOid;
1055  rel = NULL;
1056  }
1057 
1058  if (is_from)
1059  {
1060  Assert(rel);
1061 
1062  /* check read-only transaction and parallel mode */
1063  if (XactReadOnly && !rel->rd_islocaltemp)
1064  PreventCommandIfReadOnly("COPY FROM");
1065  PreventCommandIfParallelMode("COPY FROM");
1066 
1067  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
1068  NULL, stmt->attlist, stmt->options);
1069  cstate->whereClause = whereClause;
1070  *processed = CopyFrom(cstate); /* copy from file to database */
1071  EndCopyFrom(cstate);
1072  }
1073  else
1074  {
1075  cstate = BeginCopyTo(pstate, rel, query, relid,
1076  stmt->filename, stmt->is_program,
1077  stmt->attlist, stmt->options);
1078  *processed = DoCopyTo(cstate); /* copy from database to file */
1079  EndCopyTo(cstate);
1080  }
1081 
1082  /*
1083  * Close the relation. If reading, we can release the AccessShareLock we
1084  * got; if writing, we should hold the lock until end of transaction to
1085  * ensure that updates will be committed before lock is released.
1086  */
1087  if (rel != NULL)
1088  table_close(rel, (is_from ? NoLock : AccessShareLock));
1089 }
1090 
1091 /*
1092  * Process the statement option list for COPY.
1093  *
1094  * Scan the options list (a list of DefElem) and transpose the information
1095  * into cstate, applying appropriate error checking.
1096  *
1097  * cstate is assumed to be filled with zeroes initially.
1098  *
1099  * This is exported so that external users of the COPY API can sanity-check
1100  * a list of options. In that usage, cstate should be passed as NULL
1101  * (since external users don't know sizeof(CopyStateData)) and the collected
1102  * data is just leaked until CurrentMemoryContext is reset.
1103  *
1104  * Note that additional checking, such as whether column names listed in FORCE
1105  * QUOTE actually exist, has to be applied later. This just checks for
1106  * self-consistency of the options list.
1107  */
1108 void
1109 ProcessCopyOptions(ParseState *pstate,
1110  CopyState cstate,
1111  bool is_from,
1112  List *options)
1113 {
1114  bool format_specified = false;
1115  ListCell *option;
1116 
1117  /* Support external use for option sanity checking */
1118  if (cstate == NULL)
1119  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1120 
1121  cstate->is_copy_from = is_from;
1122 
1123  cstate->file_encoding = -1;
1124 
1125  /* Extract options from the statement node tree */
1126  foreach(option, options)
1127  {
1128  DefElem *defel = lfirst_node(DefElem, option);
1129 
1130  if (strcmp(defel->defname, "format") == 0)
1131  {
1132  char *fmt = defGetString(defel);
1133 
1134  if (format_specified)
1135  ereport(ERROR,
1136  (errcode(ERRCODE_SYNTAX_ERROR),
1137  errmsg("conflicting or redundant options"),
1138  parser_errposition(pstate, defel->location)));
1139  format_specified = true;
1140  if (strcmp(fmt, "text") == 0)
1141  /* default format */ ;
1142  else if (strcmp(fmt, "csv") == 0)
1143  cstate->csv_mode = true;
1144  else if (strcmp(fmt, "binary") == 0)
1145  cstate->binary = true;
1146  else
1147  ereport(ERROR,
1148  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1149  errmsg("COPY format \"%s\" not recognized", fmt),
1150  parser_errposition(pstate, defel->location)));
1151  }
1152  else if (strcmp(defel->defname, "freeze") == 0)
1153  {
1154  if (cstate->freeze)
1155  ereport(ERROR,
1156  (errcode(ERRCODE_SYNTAX_ERROR),
1157  errmsg("conflicting or redundant options"),
1158  parser_errposition(pstate, defel->location)));
1159  cstate->freeze = defGetBoolean(defel);
1160  }
1161  else if (strcmp(defel->defname, "delimiter") == 0)
1162  {
1163  if (cstate->delim)
1164  ereport(ERROR,
1165  (errcode(ERRCODE_SYNTAX_ERROR),
1166  errmsg("conflicting or redundant options"),
1167  parser_errposition(pstate, defel->location)));
1168  cstate->delim = defGetString(defel);
1169  }
1170  else if (strcmp(defel->defname, "null") == 0)
1171  {
1172  if (cstate->null_print)
1173  ereport(ERROR,
1174  (errcode(ERRCODE_SYNTAX_ERROR),
1175  errmsg("conflicting or redundant options"),
1176  parser_errposition(pstate, defel->location)));
1177  cstate->null_print = defGetString(defel);
1178  }
1179  else if (strcmp(defel->defname, "header") == 0)
1180  {
1181  if (cstate->header_line)
1182  ereport(ERROR,
1183  (errcode(ERRCODE_SYNTAX_ERROR),
1184  errmsg("conflicting or redundant options"),
1185  parser_errposition(pstate, defel->location)));
1186  cstate->header_line = defGetBoolean(defel);
1187  }
1188  else if (strcmp(defel->defname, "quote") == 0)
1189  {
1190  if (cstate->quote)
1191  ereport(ERROR,
1192  (errcode(ERRCODE_SYNTAX_ERROR),
1193  errmsg("conflicting or redundant options"),
1194  parser_errposition(pstate, defel->location)));
1195  cstate->quote = defGetString(defel);
1196  }
1197  else if (strcmp(defel->defname, "escape") == 0)
1198  {
1199  if (cstate->escape)
1200  ereport(ERROR,
1201  (errcode(ERRCODE_SYNTAX_ERROR),
1202  errmsg("conflicting or redundant options"),
1203  parser_errposition(pstate, defel->location)));
1204  cstate->escape = defGetString(defel);
1205  }
1206  else if (strcmp(defel->defname, "force_quote") == 0)
1207  {
1208  if (cstate->force_quote || cstate->force_quote_all)
1209  ereport(ERROR,
1210  (errcode(ERRCODE_SYNTAX_ERROR),
1211  errmsg("conflicting or redundant options"),
1212  parser_errposition(pstate, defel->location)));
1213  if (defel->arg && IsA(defel->arg, A_Star))
1214  cstate->force_quote_all = true;
1215  else if (defel->arg && IsA(defel->arg, List))
1216  cstate->force_quote = castNode(List, defel->arg);
1217  else
1218  ereport(ERROR,
1219  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1220  errmsg("argument to option \"%s\" must be a list of column names",
1221  defel->defname),
1222  parser_errposition(pstate, defel->location)));
1223  }
1224  else if (strcmp(defel->defname, "force_not_null") == 0)
1225  {
1226  if (cstate->force_notnull)
1227  ereport(ERROR,
1228  (errcode(ERRCODE_SYNTAX_ERROR),
1229  errmsg("conflicting or redundant options"),
1230  parser_errposition(pstate, defel->location)));
1231  if (defel->arg && IsA(defel->arg, List))
1232  cstate->force_notnull = castNode(List, defel->arg);
1233  else
1234  ereport(ERROR,
1235  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1236  errmsg("argument to option \"%s\" must be a list of column names",
1237  defel->defname),
1238  parser_errposition(pstate, defel->location)));
1239  }
1240  else if (strcmp(defel->defname, "force_null") == 0)
1241  {
1242  if (cstate->force_null)
1243  ereport(ERROR,
1244  (errcode(ERRCODE_SYNTAX_ERROR),
1245  errmsg("conflicting or redundant options")));
1246  if (defel->arg && IsA(defel->arg, List))
1247  cstate->force_null = castNode(List, defel->arg);
1248  else
1249  ereport(ERROR,
1250  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1251  errmsg("argument to option \"%s\" must be a list of column names",
1252  defel->defname),
1253  parser_errposition(pstate, defel->location)));
1254  }
1255  else if (strcmp(defel->defname, "convert_selectively") == 0)
1256  {
1257  /*
1258  * Undocumented, not-accessible-from-SQL option: convert only the
1259  * named columns to binary form, storing the rest as NULLs. It's
1260  * allowed for the column list to be NIL.
1261  */
1262  if (cstate->convert_selectively)
1263  ereport(ERROR,
1264  (errcode(ERRCODE_SYNTAX_ERROR),
1265  errmsg("conflicting or redundant options"),
1266  parser_errposition(pstate, defel->location)));
1267  cstate->convert_selectively = true;
1268  if (defel->arg == NULL || IsA(defel->arg, List))
1269  cstate->convert_select = castNode(List, defel->arg);
1270  else
1271  ereport(ERROR,
1272  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1273  errmsg("argument to option \"%s\" must be a list of column names",
1274  defel->defname),
1275  parser_errposition(pstate, defel->location)));
1276  }
1277  else if (strcmp(defel->defname, "encoding") == 0)
1278  {
1279  if (cstate->file_encoding >= 0)
1280  ereport(ERROR,
1281  (errcode(ERRCODE_SYNTAX_ERROR),
1282  errmsg("conflicting or redundant options"),
1283  parser_errposition(pstate, defel->location)));
1284  cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
1285  if (cstate->file_encoding < 0)
1286  ereport(ERROR,
1287  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1288  errmsg("argument to option \"%s\" must be a valid encoding name",
1289  defel->defname),
1290  parser_errposition(pstate, defel->location)));
1291  }
1292  else
1293  ereport(ERROR,
1294  (errcode(ERRCODE_SYNTAX_ERROR),
1295  errmsg("option \"%s\" not recognized",
1296  defel->defname),
1297  parser_errposition(pstate, defel->location)));
1298  }
1299 
1300  /*
1301  * Check for incompatible options (must do these two before inserting
1302  * defaults)
1303  */
1304  if (cstate->binary && cstate->delim)
1305  ereport(ERROR,
1306  (errcode(ERRCODE_SYNTAX_ERROR),
1307  errmsg("cannot specify DELIMITER in BINARY mode")));
1308 
1309  if (cstate->binary && cstate->null_print)
1310  ereport(ERROR,
1311  (errcode(ERRCODE_SYNTAX_ERROR),
1312  errmsg("cannot specify NULL in BINARY mode")));
1313 
1314  /* Set defaults for omitted options */
1315  if (!cstate->delim)
1316  cstate->delim = cstate->csv_mode ? "," : "\t";
1317 
1318  if (!cstate->null_print)
1319  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1320  cstate->null_print_len = strlen(cstate->null_print);
1321 
1322  if (cstate->csv_mode)
1323  {
1324  if (!cstate->quote)
1325  cstate->quote = "\"";
1326  if (!cstate->escape)
1327  cstate->escape = cstate->quote;
1328  }
1329 
1330  /* Only single-byte delimiter strings are supported. */
1331  if (strlen(cstate->delim) != 1)
1332  ereport(ERROR,
1333  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1334  errmsg("COPY delimiter must be a single one-byte character")));
1335 
1336  /* Disallow end-of-line characters */
1337  if (strchr(cstate->delim, '\r') != NULL ||
1338  strchr(cstate->delim, '\n') != NULL)
1339  ereport(ERROR,
1340  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1341  errmsg("COPY delimiter cannot be newline or carriage return")));
1342 
1343  if (strchr(cstate->null_print, '\r') != NULL ||
1344  strchr(cstate->null_print, '\n') != NULL)
1345  ereport(ERROR,
1346  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1347  errmsg("COPY null representation cannot use newline or carriage return")));
1348 
1349  /*
1350  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1351  * backslash because it would be ambiguous. We can't allow the other
1352  * cases because data characters matching the delimiter must be
1353  * backslashed, and certain backslash combinations are interpreted
1354  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1355  * more than strictly necessary, but seems best for consistency and
1356  * future-proofing. Likewise we disallow all digits though only octal
1357  * digits are actually dangerous.
1358  */
1359  if (!cstate->csv_mode &&
1360  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1361  cstate->delim[0]) != NULL)
1362  ereport(ERROR,
1363  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1364  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1365 
1366  /* Check header */
1367  if (!cstate->csv_mode && cstate->header_line)
1368  ereport(ERROR,
1369  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1370  errmsg("COPY HEADER available only in CSV mode")));
1371 
1372  /* Check quote */
1373  if (!cstate->csv_mode && cstate->quote != NULL)
1374  ereport(ERROR,
1375  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1376  errmsg("COPY quote available only in CSV mode")));
1377 
1378  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1379  ereport(ERROR,
1380  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1381  errmsg("COPY quote must be a single one-byte character")));
1382 
1383  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1384  ereport(ERROR,
1385  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1386  errmsg("COPY delimiter and quote must be different")));
1387 
1388  /* Check escape */
1389  if (!cstate->csv_mode && cstate->escape != NULL)
1390  ereport(ERROR,
1391  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1392  errmsg("COPY escape available only in CSV mode")));
1393 
1394  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1395  ereport(ERROR,
1396  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1397  errmsg("COPY escape must be a single one-byte character")));
1398 
1399  /* Check force_quote */
1400  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1401  ereport(ERROR,
1402  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1403  errmsg("COPY force quote available only in CSV mode")));
1404  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1405  ereport(ERROR,
1406  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1407  errmsg("COPY force quote only available using COPY TO")));
1408 
1409  /* Check force_notnull */
1410  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1411  ereport(ERROR,
1412  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1413  errmsg("COPY force not null available only in CSV mode")));
1414  if (cstate->force_notnull != NIL && !is_from)
1415  ereport(ERROR,
1416  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1417  errmsg("COPY force not null only available using COPY FROM")));
1418 
1419  /* Check force_null */
1420  if (!cstate->csv_mode && cstate->force_null != NIL)
1421  ereport(ERROR,
1422  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1423  errmsg("COPY force null available only in CSV mode")));
1424 
1425  if (cstate->force_null != NIL && !is_from)
1426  ereport(ERROR,
1427  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1428  errmsg("COPY force null only available using COPY FROM")));
1429 
1430  /* Don't allow the delimiter to appear in the null string. */
1431  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1432  ereport(ERROR,
1433  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1434  errmsg("COPY delimiter must not appear in the NULL specification")));
1435 
1436  /* Don't allow the CSV quote char to appear in the null string. */
1437  if (cstate->csv_mode &&
1438  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1439  ereport(ERROR,
1440  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1441  errmsg("CSV quote character must not appear in the NULL specification")));
1442 }
1443 
1444 /*
1445  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1446  *
1447  * Iff <binary>, unload or reload in the binary format, as opposed to the
1448  * more wasteful but more robust and portable text format.
1449  *
1450  * Iff <oids>, unload or reload the format that includes OID information.
1451  * On input, we accept OIDs whether or not the table has an OID column,
1452  * but silently drop them if it does not. On output, we report an error
1453  * if the user asks for OIDs in a table that has none (not providing an
1454  * OID column might seem friendlier, but could seriously confuse programs).
1455  *
1456  * If in the text format, delimit columns with delimiter <delim> and print
1457  * NULL values as <null_print>.
1458  */
1459 static CopyState
1460 BeginCopy(ParseState *pstate,
1461  bool is_from,
1462  Relation rel,
1463  RawStmt *raw_query,
1464  Oid queryRelId,
1465  List *attnamelist,
1466  List *options)
1467 {
1468  CopyState cstate;
1469  TupleDesc tupDesc;
1470  int num_phys_attrs;
1471  MemoryContext oldcontext;
1472 
1473  /* Allocate workspace and zero all fields */
1474  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1475 
1476  /*
1477  * We allocate everything used by a cstate in a new memory context. This
1478  * avoids memory leaks during repeated use of COPY in a query.
1479  */
1480  cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
1481  "COPY",
1482  ALLOCSET_DEFAULT_SIZES);
1483 
1484  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1485 
1486  /* Extract options from the statement node tree */
1487  ProcessCopyOptions(pstate, cstate, is_from, options);
1488 
1489  /* Process the source/target relation or query */
1490  if (rel)
1491  {
1492  Assert(!raw_query);
1493 
1494  cstate->rel = rel;
1495 
1496  tupDesc = RelationGetDescr(cstate->rel);
1497  }
1498  else
1499  {
1500  List *rewritten;
1501  Query *query;
1502  PlannedStmt *plan;
1503  DestReceiver *dest;
1504 
1505  Assert(!is_from);
1506  cstate->rel = NULL;
1507 
1508  /*
1509  * Run parse analysis and rewrite. Note this also acquires sufficient
1510  * locks on the source table(s).
1511  *
1512  * Because the parser and planner tend to scribble on their input, we
1513  * make a preliminary copy of the source querytree. This prevents
1514  * problems in the case that the COPY is in a portal or plpgsql
1515  * function and is executed repeatedly. (See also the same hack in
1516  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1517  */
1518  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1519  pstate->p_sourcetext, NULL, 0,
1520  NULL);
1521 
1522  /* check that we got back something we can work with */
1523  if (rewritten == NIL)
1524  {
1525  ereport(ERROR,
1526  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1527  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1528  }
1529  else if (list_length(rewritten) > 1)
1530  {
1531  ListCell *lc;
1532 
1533  /* examine queries to determine which error message to issue */
1534  foreach(lc, rewritten)
1535  {
1536  Query *q = lfirst_node(Query, lc);
1537 
1538  if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
1539  ereport(ERROR,
1540  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1541  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1542  if (q->querySource == QSRC_NON_INSTEAD_RULE)
1543  ereport(ERROR,
1544  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1545  errmsg("DO ALSO rules are not supported for the COPY")));
1546  }
1547 
1548  ereport(ERROR,
1549  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1550  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1551  }
1552 
1553  query = linitial_node(Query, rewritten);
1554 
1555  /* The grammar allows SELECT INTO, but we don't support that */
1556  if (query->utilityStmt != NULL &&
1557  IsA(query->utilityStmt, CreateTableAsStmt))
1558  ereport(ERROR,
1559  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1560  errmsg("COPY (SELECT INTO) is not supported")));
1561 
1562  Assert(query->utilityStmt == NULL);
1563 
1564  /*
1565  * Similarly the grammar doesn't enforce the presence of a RETURNING
1566  * clause, but this is required here.
1567  */
1568  if (query->commandType != CMD_SELECT &&
1569  query->returningList == NIL)
1570  {
1571  Assert(query->commandType == CMD_INSERT ||
1572  query->commandType == CMD_UPDATE ||
1573  query->commandType == CMD_DELETE);
1574 
1575  ereport(ERROR,
1576  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1577  errmsg("COPY query must have a RETURNING clause")));
1578  }
1579 
1580  /* plan the query */
1581  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1582 
1583  /*
1584  * With row level security and a user using "COPY relation TO", we
1585  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1586  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1587  * in any RLS clauses.
1588  *
1589  * When this happens, we are passed in the relid of the originally
1590  * found relation (which we have locked). As the planner will look up
1591  * the relation again, we double-check here to make sure it found the
1592  * same one that we have locked.
1593  */
1594  if (queryRelId != InvalidOid)
1595  {
1596  /*
1597  * Note that with RLS involved there may be multiple relations,
1598  * and while the one we need is almost certainly first, we don't
1599  * make any guarantees of that in the planner, so check the whole
1600  * list and make sure we find the original relation.
1601  */
1602  if (!list_member_oid(plan->relationOids, queryRelId))
1603  ereport(ERROR,
1604  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1605  errmsg("relation referenced by COPY statement has changed")));
1606  }
1607 
1608  /*
1609  * Use a snapshot with an updated command ID to ensure this query sees
1610  * results of any previously executed queries.
1611  */
1612  PushCopiedSnapshot(GetActiveSnapshot());
1613  UpdateActiveSnapshotCommandId();
1614 
1615  /* Create dest receiver for COPY OUT */
1616  dest = CreateDestReceiver(DestCopyOut);
1617  ((DR_copy *) dest)->cstate = cstate;
1618 
1619  /* Create a QueryDesc requesting no output */
1620  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1621  GetActiveSnapshot(),
1622  InvalidSnapshot,
1623  dest, NULL, NULL, 0);
1624 
1625  /*
1626  * Call ExecutorStart to prepare the plan for execution.
1627  *
1628  * ExecutorStart computes a result tupdesc for us
1629  */
1630  ExecutorStart(cstate->queryDesc, 0);
1631 
1632  tupDesc = cstate->queryDesc->tupDesc;
1633  }
1634 
1635  /* Generate or convert list of attributes to process */
1636  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1637 
1638  num_phys_attrs = tupDesc->natts;
1639 
1640  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1641  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1642  if (cstate->force_quote_all)
1643  {
1644  int i;
1645 
1646  for (i = 0; i < num_phys_attrs; i++)
1647  cstate->force_quote_flags[i] = true;
1648  }
1649  else if (cstate->force_quote)
1650  {
1651  List *attnums;
1652  ListCell *cur;
1653 
1654  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1655 
1656  foreach(cur, attnums)
1657  {
1658  int attnum = lfirst_int(cur);
1659  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1660 
1661  if (!list_member_int(cstate->attnumlist, attnum))
1662  ereport(ERROR,
1663  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1664  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1665  NameStr(attr->attname))));
1666  cstate->force_quote_flags[attnum - 1] = true;
1667  }
1668  }
1669 
1670  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1671  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1672  if (cstate->force_notnull)
1673  {
1674  List *attnums;
1675  ListCell *cur;
1676 
1677  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1678 
1679  foreach(cur, attnums)
1680  {
1681  int attnum = lfirst_int(cur);
1682  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1683 
1684  if (!list_member_int(cstate->attnumlist, attnum))
1685  ereport(ERROR,
1686  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1687  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1688  NameStr(attr->attname))));
1689  cstate->force_notnull_flags[attnum - 1] = true;
1690  }
1691  }
1692 
1693  /* Convert FORCE_NULL name list to per-column flags, check validity */
1694  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1695  if (cstate->force_null)
1696  {
1697  List *attnums;
1698  ListCell *cur;
1699 
1700  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1701 
1702  foreach(cur, attnums)
1703  {
1704  int attnum = lfirst_int(cur);
1705  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1706 
1707  if (!list_member_int(cstate->attnumlist, attnum))
1708  ereport(ERROR,
1709  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1710  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1711  NameStr(attr->attname))));
1712  cstate->force_null_flags[attnum - 1] = true;
1713  }
1714  }
1715 
1716  /* Convert convert_selectively name list to per-column flags */
1717  if (cstate->convert_selectively)
1718  {
1719  List *attnums;
1720  ListCell *cur;
1721 
1722  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1723 
1724  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1725 
1726  foreach(cur, attnums)
1727  {
1728  int attnum = lfirst_int(cur);
1729  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1730 
1731  if (!list_member_int(cstate->attnumlist, attnum))
1732  ereport(ERROR,
1733  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1734  errmsg_internal("selected column \"%s\" not referenced by COPY",
1735  NameStr(attr->attname))));
1736  cstate->convert_select_flags[attnum - 1] = true;
1737  }
1738  }
1739 
1740  /* Use client encoding when ENCODING option is not specified. */
1741  if (cstate->file_encoding < 0)
1742  cstate->file_encoding = pg_get_client_encoding();
1743 
1744  /*
1745  * Set up encoding conversion info. Even if the file and server encodings
1746  * are the same, we must apply pg_any_to_server() to validate data in
1747  * multibyte encodings.
1748  */
1749  cstate->need_transcoding =
1750  (cstate->file_encoding != GetDatabaseEncoding() ||
1751  pg_database_encoding_max_length() > 1);
1752  /* See Multibyte encoding comment above */
1753  cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
1754 
1755  cstate->copy_dest = COPY_FILE; /* default */
1756 
1757  MemoryContextSwitchTo(oldcontext);
1758 
1759  return cstate;
1760 }
1761 
1762 /*
1763  * Closes the pipe to an external program, checking the pclose() return code.
1764  */
1765 static void
1766 ClosePipeToProgram(CopyState cstate)
1767 {
1768  int pclose_rc;
1769 
1770  Assert(cstate->is_program);
1771 
1772  pclose_rc = ClosePipeStream(cstate->copy_file);
1773  if (pclose_rc == -1)
1774  ereport(ERROR,
1775  (errcode_for_file_access(),
1776  errmsg("could not close pipe to external command: %m")));
1777  else if (pclose_rc != 0)
1778  {
1779  /*
1780  * If we ended a COPY FROM PROGRAM before reaching EOF, then it's
1781  * expectable for the called program to fail with SIGPIPE, and we
1782  * should not report that as an error. Otherwise, SIGPIPE indicates a
1783  * problem.
1784  */
1785  if (cstate->is_copy_from && !cstate->reached_eof &&
1786  wait_result_is_signal(pclose_rc, SIGPIPE))
1787  return;
1788 
1789  ereport(ERROR,
1790  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1791  errmsg("program \"%s\" failed",
1792  cstate->filename),
1793  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1794  }
1795 }
1796 
1797 /*
1798  * Release resources allocated in a cstate for COPY TO/FROM.
1799  */
1800 static void
1801 EndCopy(CopyState cstate)
1802 {
1803  if (cstate->is_program)
1804  {
1805  ClosePipeToProgram(cstate);
1806  }
1807  else
1808  {
1809  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1810  ereport(ERROR,
1811  (errcode_for_file_access(),
1812  errmsg("could not close file \"%s\": %m",
1813  cstate->filename)));
1814  }
1815 
1816  MemoryContextDelete(cstate->copycontext);
1817  pfree(cstate);
1818 }
1819 
1820 /*
1821  * Setup CopyState to read tuples from a table or a query for COPY TO.
1822  */
1823 static CopyState
1824 BeginCopyTo(ParseState *pstate,
1825  Relation rel,
1826  RawStmt *query,
1827  Oid queryRelId,
1828  const char *filename,
1829  bool is_program,
1830  List *attnamelist,
1831  List *options)
1832 {
1833  CopyState cstate;
1834  bool pipe = (filename == NULL);
1835  MemoryContext oldcontext;
1836 
1837  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1838  {
1839  if (rel->rd_rel->relkind == RELKIND_VIEW)
1840  ereport(ERROR,
1841  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1842  errmsg("cannot copy from view \"%s\"",
1843  RelationGetRelationName(rel)),
1844  errhint("Try the COPY (SELECT ...) TO variant.")));
1845  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1846  ereport(ERROR,
1847  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1848  errmsg("cannot copy from materialized view \"%s\"",
1849  RelationGetRelationName(rel)),
1850  errhint("Try the COPY (SELECT ...) TO variant.")));
1851  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1852  ereport(ERROR,
1853  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1854  errmsg("cannot copy from foreign table \"%s\"",
1855  RelationGetRelationName(rel)),
1856  errhint("Try the COPY (SELECT ...) TO variant.")));
1857  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1858  ereport(ERROR,
1859  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1860  errmsg("cannot copy from sequence \"%s\"",
1861  RelationGetRelationName(rel))));
1862  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1863  ereport(ERROR,
1864  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1865  errmsg("cannot copy from partitioned table \"%s\"",
1866  RelationGetRelationName(rel)),
1867  errhint("Try the COPY (SELECT ...) TO variant.")));
1868  else
1869  ereport(ERROR,
1870  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1871  errmsg("cannot copy from non-table relation \"%s\"",
1872  RelationGetRelationName(rel))));
1873  }
1874 
1875  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1876  options);
1877  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1878 
1879  if (pipe)
1880  {
1881  Assert(!is_program); /* the grammar does not allow this */
1882  if (whereToSendOutput != DestRemote)
1883  cstate->copy_file = stdout;
1884  }
1885  else
1886  {
1887  cstate->filename = pstrdup(filename);
1888  cstate->is_program = is_program;
1889 
1890  if (is_program)
1891  {
1892  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1893  if (cstate->copy_file == NULL)
1894  ereport(ERROR,
1895  (errcode_for_file_access(),
1896  errmsg("could not execute command \"%s\": %m",
1897  cstate->filename)));
1898  }
1899  else
1900  {
1901  mode_t oumask; /* Pre-existing umask value */
1902  struct stat st;
1903 
1904  /*
1905  * Prevent write to relative path ... too easy to shoot oneself in
1906  * the foot by overwriting a database file ...
1907  */
1908  if (!is_absolute_path(filename))
1909  ereport(ERROR,
1910  (errcode(ERRCODE_INVALID_NAME),
1911  errmsg("relative path not allowed for COPY to file")));
1912 
1913  oumask = umask(S_IWGRP | S_IWOTH);
1914  PG_TRY();
1915  {
1916  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1917  }
1918  PG_FINALLY();
1919  {
1920  umask(oumask);
1921  }
1922  PG_END_TRY();
1923  if (cstate->copy_file == NULL)
1924  {
1925  /* copy errno because ereport subfunctions might change it */
1926  int save_errno = errno;
1927 
1928  ereport(ERROR,
1929  (errcode_for_file_access(),
1930  errmsg("could not open file \"%s\" for writing: %m",
1931  cstate->filename),
1932  (save_errno == ENOENT || save_errno == EACCES) ?
1933  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1934  "You may want a client-side facility such as psql's \\copy.") : 0));
1935  }
1936 
1937  if (fstat(fileno(cstate->copy_file), &st))
1938  ereport(ERROR,
1939  (errcode_for_file_access(),
1940  errmsg("could not stat file \"%s\": %m",
1941  cstate->filename)));
1942 
1943  if (S_ISDIR(st.st_mode))
1944  ereport(ERROR,
1945  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1946  errmsg("\"%s\" is a directory", cstate->filename)));
1947  }
1948  }
1949 
1950  MemoryContextSwitchTo(oldcontext);
1951 
1952  return cstate;
1953 }
1954 
1955 /*
1956  * This intermediate routine exists mainly to localize the effects of setjmp
1957  * so we don't need to plaster a lot of variables with "volatile".
1958  */
1959 static uint64
1960 DoCopyTo(CopyState cstate)
1961 {
1962  bool pipe = (cstate->filename == NULL);
1963  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
1964  uint64 processed;
1965 
1966  PG_TRY();
1967  {
1968  if (fe_copy)
1969  SendCopyBegin(cstate);
1970 
1971  processed = CopyTo(cstate);
1972 
1973  if (fe_copy)
1974  SendCopyEnd(cstate);
1975  }
1976  PG_CATCH();
1977  {
1978  /*
1979  * Make sure we turn off old-style COPY OUT mode upon error. It is
1980  * okay to do this in all cases, since it does nothing if the mode is
1981  * not on.
1982  */
1983  pq_endcopyout(true);
1984  PG_RE_THROW();
1985  }
1986  PG_END_TRY();
1987 
1988  return processed;
1989 }
1990 
1991 /*
1992  * Clean up storage and release resources for COPY TO.
1993  */
1994 static void
1995 EndCopyTo(CopyState cstate)
1996 {
1997  if (cstate->queryDesc != NULL)
1998  {
1999  /* Close down the query and free resources. */
2000  ExecutorFinish(cstate->queryDesc);
2001  ExecutorEnd(cstate->queryDesc);
2002  FreeQueryDesc(cstate->queryDesc);
2003  PopActiveSnapshot();
2004  }
2005 
2006  /* Clean up storage */
2007  EndCopy(cstate);
2008 }
2009 
2010 /*
2011  * Copy from relation or query TO file.
2012  */
2013 static uint64
2014 CopyTo(CopyState cstate)
2015 {
2016  TupleDesc tupDesc;
2017  int num_phys_attrs;
2018  ListCell *cur;
2019  uint64 processed;
2020 
2021  if (cstate->rel)
2022  tupDesc = RelationGetDescr(cstate->rel);
2023  else
2024  tupDesc = cstate->queryDesc->tupDesc;
2025  num_phys_attrs = tupDesc->natts;
2026  cstate->null_print_client = cstate->null_print; /* default */
2027 
2028  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
2029  cstate->fe_msgbuf = makeStringInfo();
2030 
2031  /* Get info about the columns we need to process. */
2032  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2033  foreach(cur, cstate->attnumlist)
2034  {
2035  int attnum = lfirst_int(cur);
2036  Oid out_func_oid;
2037  bool isvarlena;
2038  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
2039 
2040  if (cstate->binary)
2041  getTypeBinaryOutputInfo(attr->atttypid,
2042  &out_func_oid,
2043  &isvarlena);
2044  else
2045  getTypeOutputInfo(attr->atttypid,
2046  &out_func_oid,
2047  &isvarlena);
2048  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
2049  }
2050 
2051  /*
2052  * Create a temporary memory context that we can reset once per row to
2053  * recover palloc'd memory. This avoids any problems with leaks inside
2054  * datatype output routines, and should be faster than retail pfree's
2055  * anyway. (We don't need a whole econtext as CopyFrom does.)
2056  */
2057  cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
2058  "COPY TO",
2059  ALLOCSET_DEFAULT_SIZES);
2060 
2061  if (cstate->binary)
2062  {
2063  /* Generate header for a binary copy */
2064  int32 tmp;
2065 
2066  /* Signature */
2067  CopySendData(cstate, BinarySignature, 11);
2068  /* Flags field */
2069  tmp = 0;
2070  CopySendInt32(cstate, tmp);
2071  /* No header extension */
2072  tmp = 0;
2073  CopySendInt32(cstate, tmp);
2074  }
2075  else
2076  {
2077  /*
2078  * For non-binary copy, we need to convert null_print to file
2079  * encoding, because it will be sent directly with CopySendString.
2080  */
2081  if (cstate->need_transcoding)
2082  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2083  cstate->null_print_len,
2084  cstate->file_encoding);
2085 
2086  /* if a header has been requested send the line */
2087  if (cstate->header_line)
2088  {
2089  bool hdr_delim = false;
2090 
2091  foreach(cur, cstate->attnumlist)
2092  {
2093  int attnum = lfirst_int(cur);
2094  char *colname;
2095 
2096  if (hdr_delim)
2097  CopySendChar(cstate, cstate->delim[0]);
2098  hdr_delim = true;
2099 
2100  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
2101 
2102  CopyAttributeOutCSV(cstate, colname, false,
2103  list_length(cstate->attnumlist) == 1);
2104  }
2105 
2106  CopySendEndOfRow(cstate);
2107  }
2108  }
2109 
2110  if (cstate->rel)
2111  {
2112  TupleTableSlot *slot;
2113  TableScanDesc scandesc;
2114 
2115  scandesc = table_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2116  slot = table_slot_create(cstate->rel, NULL);
2117 
2118  processed = 0;
2119  while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot))
2120  {
2121  CHECK_FOR_INTERRUPTS();
2122 
2123  /* Deconstruct the tuple ... */
2124  slot_getallattrs(slot);
2125 
2126  /* Format and send the data */
2127  CopyOneRowTo(cstate, slot);
2128  processed++;
2129  }
2130 
2131  ExecDropSingleTupleTableSlot(slot);
2132  table_endscan(scandesc);
2133  }
2134  else
2135  {
2136  /* run the plan --- the dest receiver will send tuples */
2137  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2138  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2139  }
2140 
2141  if (cstate->binary)
2142  {
2143  /* Generate trailer for a binary copy */
2144  CopySendInt16(cstate, -1);
2145  /* Need to flush out the trailer */
2146  CopySendEndOfRow(cstate);
2147  }
2148 
2149  MemoryContextDelete(cstate->rowcontext);
2150 
2151  return processed;
2152 }
2153 
2154 /*
2155  * Emit one row during CopyTo().
2156  */
2157 static void
2158 CopyOneRowTo(CopyState cstate, TupleTableSlot *slot)
2159 {
2160  bool need_delim = false;
2161  FmgrInfo *out_functions = cstate->out_functions;
2162  MemoryContext oldcontext;
2163  ListCell *cur;
2164  char *string;
2165 
2166  MemoryContextReset(cstate->rowcontext);
2167  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2168 
2169  if (cstate->binary)
2170  {
2171  /* Binary per-tuple header */
2172  CopySendInt16(cstate, list_length(cstate->attnumlist));
2173  }
2174 
2175  /* Make sure the tuple is fully deconstructed */
2176  slot_getallattrs(slot);
2177 
2178  foreach(cur, cstate->attnumlist)
2179  {
2180  int attnum = lfirst_int(cur);
2181  Datum value = slot->tts_values[attnum - 1];
2182  bool isnull = slot->tts_isnull[attnum - 1];
2183 
2184  if (!cstate->binary)
2185  {
2186  if (need_delim)
2187  CopySendChar(cstate, cstate->delim[0]);
2188  need_delim = true;
2189  }
2190 
2191  if (isnull)
2192  {
2193  if (!cstate->binary)
2194  CopySendString(cstate, cstate->null_print_client);
2195  else
2196  CopySendInt32(cstate, -1);
2197  }
2198  else
2199  {
2200  if (!cstate->binary)
2201  {
2202  string = OutputFunctionCall(&out_functions[attnum - 1],
2203  value);
2204  if (cstate->csv_mode)
2205  CopyAttributeOutCSV(cstate, string,
2206  cstate->force_quote_flags[attnum - 1],
2207  list_length(cstate->attnumlist) == 1);
2208  else
2209  CopyAttributeOutText(cstate, string);
2210  }
2211  else
2212  {
2213  bytea *outputbytes;
2214 
2215  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2216  value);
2217  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2218  CopySendData(cstate, VARDATA(outputbytes),
2219  VARSIZE(outputbytes) - VARHDRSZ);
2220  }
2221  }
2222  }
2223 
2224  CopySendEndOfRow(cstate);
2225 
2226  MemoryContextSwitchTo(oldcontext);
2227 }
2228 
2229 
2230 /*
2231  * error context callback for COPY FROM
2232  *
2233  * The argument for the error context must be CopyState.
2234  */
2235 void
2236 CopyFromErrorCallback(void *arg)
2237 {
2238  CopyState cstate = (CopyState) arg;
2239  char curlineno_str[32];
2240 
2241  snprintf(curlineno_str, sizeof(curlineno_str), UINT64_FORMAT,
2242  cstate->cur_lineno);
2243 
2244  if (cstate->binary)
2245  {
2246  /* can't usefully display the data */
2247  if (cstate->cur_attname)
2248  errcontext("COPY %s, line %s, column %s",
2249  cstate->cur_relname, curlineno_str,
2250  cstate->cur_attname);
2251  else
2252  errcontext("COPY %s, line %s",
2253  cstate->cur_relname, curlineno_str);
2254  }
2255  else
2256  {
2257  if (cstate->cur_attname && cstate->cur_attval)
2258  {
2259  /* error is relevant to a particular column */
2260  char *attval;
2261 
2262  attval = limit_printout_length(cstate->cur_attval);
2263  errcontext("COPY %s, line %s, column %s: \"%s\"",
2264  cstate->cur_relname, curlineno_str,
2265  cstate->cur_attname, attval);
2266  pfree(attval);
2267  }
2268  else if (cstate->cur_attname)
2269  {
2270  /* error is relevant to a particular column, value is NULL */
2271  errcontext("COPY %s, line %s, column %s: null input",
2272  cstate->cur_relname, curlineno_str,
2273  cstate->cur_attname);
2274  }
2275  else
2276  {
2277  /*
2278  * Error is relevant to a particular line.
2279  *
2280  * If line_buf still contains the correct line, and it's already
2281  * transcoded, print it. If it's still in a foreign encoding, it's
2282  * quite likely that the error is precisely a failure to do
2283  * encoding conversion (ie, bad data). We dare not try to convert
2284  * it, and at present there's no way to regurgitate it without
2285  * conversion. So we have to punt and just report the line number.
2286  */
2287  if (cstate->line_buf_valid &&
2288  (cstate->line_buf_converted || !cstate->need_transcoding))
2289  {
2290  char *lineval;
2291 
2292  lineval = limit_printout_length(cstate->line_buf.data);
2293  errcontext("COPY %s, line %s: \"%s\"",
2294  cstate->cur_relname, curlineno_str, lineval);
2295  pfree(lineval);
2296  }
2297  else
2298  {
2299  errcontext("COPY %s, line %s",
2300  cstate->cur_relname, curlineno_str);
2301  }
2302  }
2303  }
2304 }
2305 
2306 /*
2307  * Make sure we don't print an unreasonable amount of COPY data in a message.
2308  *
2309  * It would seem a lot easier to just use the sprintf "precision" limit to
2310  * truncate the string. However, some versions of glibc have a bug/misfeature
2311  * that vsnprintf will always fail (return -1) if it is asked to truncate
2312  * a string that contains invalid byte sequences for the current encoding.
2313  * So, do our own truncation. We return a pstrdup'd copy of the input.
2314  */
2315 static char *
2316 limit_printout_length(const char *str)
2317 {
2318 #define MAX_COPY_DATA_DISPLAY 100
2319 
2320  int slen = strlen(str);
2321  int len;
2322  char *res;
2323 
2324  /* Fast path if definitely okay */
2325  if (slen <= MAX_COPY_DATA_DISPLAY)
2326  return pstrdup(str);
2327 
2328  /* Apply encoding-dependent truncation */
2329  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2330 
2331  /*
2332  * Truncate, and add "..." to show we truncated the input.
2333  */
2334  res = (char *) palloc(len + 4);
2335  memcpy(res, str, len);
2336  strcpy(res + len, "...");
2337 
2338  return res;
2339 }
2340 
2341 /*
2342  * Allocate memory and initialize a new CopyMultiInsertBuffer for this
2343  * ResultRelInfo.
2344  */
2345 static CopyMultiInsertBuffer *
2346 CopyMultiInsertBufferInit(ResultRelInfo *rri)
2347 {
2348  CopyMultiInsertBuffer *buffer;
2349 
2350  buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer));
2351  memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES);
2352  buffer->resultRelInfo = rri;
2353  buffer->bistate = GetBulkInsertState();
2354  buffer->nused = 0;
2355 
2356  return buffer;
2357 }
2358 
2359 /*
2360  * Make a new buffer for this ResultRelInfo.
2361  */
2362 static inline void
2363 CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo,
2364  ResultRelInfo *rri)
2365 {
2366  CopyMultiInsertBuffer *buffer;
2367 
2368  buffer = CopyMultiInsertBufferInit(rri);
2369 
2370  /* Setup back-link so we can easily find this buffer again */
2371  rri->ri_CopyMultiInsertBuffer = buffer;
2372  /* Record that we're tracking this buffer */
2373  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2374 }
2375 
2376 /*
2377  * Initialize an already allocated CopyMultiInsertInfo.
2378  *
2379  * If rri is a non-partitioned table then a CopyMultiInsertBuffer is set up
2380  * for that table.
2381  */
2382 static void
2383 CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2384  CopyState cstate, EState *estate, CommandId mycid,
2385  int ti_options)
2386 {
2387  miinfo->multiInsertBuffers = NIL;
2388  miinfo->bufferedTuples = 0;
2389  miinfo->bufferedBytes = 0;
2390  miinfo->cstate = cstate;
2391  miinfo->estate = estate;
2392  miinfo->mycid = mycid;
2393  miinfo->ti_options = ti_options;
2394 
2395  /*
2396  * Only setup the buffer when not dealing with a partitioned table.
2397  * Buffers for partitioned tables will just be setup when we need to send
2398  * tuples their way for the first time.
2399  */
2400  if (rri->ri_RelationDesc->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
2401  CopyMultiInsertInfoSetupBuffer(miinfo, rri);
2402 }
2403 
2404 /*
2405  * Returns true if the buffers are full
2406  */
2407 static inline bool
2408 CopyMultiInsertInfoIsFull(CopyMultiInsertInfo *miinfo)
2409 {
2410  if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES ||
2411  miinfo->bufferedBytes >= MAX_BUFFERED_BYTES)
2412  return true;
2413  return false;
2414 }
2415 
2416 /*
2417  * Returns true if we have no buffered tuples
2418  */
2419 static inline bool
2420 CopyMultiInsertInfoIsEmpty(CopyMultiInsertInfo *miinfo)
2421 {
2422  return miinfo->bufferedTuples == 0;
2423 }
2424 
2425 /*
2426  * Write the tuples stored in 'buffer' out to the table.
2427  */
2428 static inline void
2429 CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
2430  CopyMultiInsertBuffer *buffer)
2431 {
2432  MemoryContext oldcontext;
2433  int i;
2434  uint64 save_cur_lineno;
2435  CopyState cstate = miinfo->cstate;
2436  EState *estate = miinfo->estate;
2437  CommandId mycid = miinfo->mycid;
2438  int ti_options = miinfo->ti_options;
2439  bool line_buf_valid = cstate->line_buf_valid;
2440  int nused = buffer->nused;
2441  ResultRelInfo *resultRelInfo = buffer->resultRelInfo;
2442  TupleTableSlot **slots = buffer->slots;
2443 
2444  /* Set es_result_relation_info to the ResultRelInfo we're flushing. */
2445  estate->es_result_relation_info = resultRelInfo;
2446 
2447  /*
2448  * Print error context information correctly, if one of the operations
2449  * below fail.
2450  */
2451  cstate->line_buf_valid = false;
2452  save_cur_lineno = cstate->cur_lineno;
2453 
2454  /*
2455  * table_multi_insert may leak memory, so switch to short-lived memory
2456  * context before calling it.
2457  */
2458  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2459  table_multi_insert(resultRelInfo->ri_RelationDesc,
2460  slots,
2461  nused,
2462  mycid,
2463  ti_options,
2464  buffer->bistate);
2465  MemoryContextSwitchTo(oldcontext);
2466 
2467  for (i = 0; i < nused; i++)
2468  {
2469  /*
2470  * If there are any indexes, update them for all the inserted tuples,
2471  * and run AFTER ROW INSERT triggers.
2472  */
2473  if (resultRelInfo->ri_NumIndices > 0)
2474  {
2475  List *recheckIndexes;
2476 
2477  cstate->cur_lineno = buffer->linenos[i];
2478  recheckIndexes =
2479  ExecInsertIndexTuples(buffer->slots[i], estate, false, NULL,
2480  NIL);
2481  ExecARInsertTriggers(estate, resultRelInfo,
2482  slots[i], recheckIndexes,
2483  cstate->transition_capture);
2484  list_free(recheckIndexes);
2485  }
2486 
2487  /*
2488  * There's no indexes, but see if we need to run AFTER ROW INSERT
2489  * triggers anyway.
2490  */
2491  else if (resultRelInfo->ri_TrigDesc != NULL &&
2492  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2493  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2494  {
2495  cstate->cur_lineno = buffer->linenos[i];
2496  ExecARInsertTriggers(estate, resultRelInfo,
2497  slots[i], NIL, cstate->transition_capture);
2498  }
2499 
2500  ExecClearTuple(slots[i]);
2501  }
2502 
2503  /* Mark that all slots are free */
2504  buffer->nused = 0;
2505 
2506  /* reset cur_lineno and line_buf_valid to what they were */
2507  cstate->line_buf_valid = line_buf_valid;
2508  cstate->cur_lineno = save_cur_lineno;
2509 }
2510 
2511 /*
2512  * Drop used slots and free member for this buffer.
2513  *
2514  * The buffer must be flushed before cleanup.
2515  */
2516 static inline void
2517 CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo,
2518  CopyMultiInsertBuffer *buffer)
2519 {
2520  int i;
2521 
2522  /* Ensure buffer was flushed */
2523  Assert(buffer->nused == 0);
2524 
2525  /* Remove back-link to ourself */
2526  buffer->resultRelInfo->ri_CopyMultiInsertBuffer = NULL;
2527 
2528  FreeBulkInsertState(buffer->bistate);
2529 
2530  /* Since we only create slots on demand, just drop the non-null ones. */
2531  for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++)
2532  ExecDropSingleTupleTableSlot(buffer->slots[i]);
2533 
2534  table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc,
2535  miinfo->ti_options);
2536 
2537  pfree(buffer);
2538 }
2539 
2540 /*
2541  * Write out all stored tuples in all buffers out to the tables.
2542  *
2543  * Once flushed we also trim the tracked buffers list down to size by removing
2544  * the buffers created earliest first.
2545  *
2546  * Callers should pass 'curr_rri' is the ResultRelInfo that's currently being
2547  * used. When cleaning up old buffers we'll never remove the one for
2548  * 'curr_rri'.
2549  */
2550 static inline void
2551 CopyMultiInsertInfoFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *curr_rri)
2552 {
2553  ListCell *lc;
2554 
2555  foreach(lc, miinfo->multiInsertBuffers)
2556  {
2557  CopyMultiInsertBuffer *buffer = (CopyMultiInsertBuffer *) lfirst(lc);
2558 
2559  CopyMultiInsertBufferFlush(miinfo, buffer);
2560  }
2561 
2562  miinfo->bufferedTuples = 0;
2563  miinfo->bufferedBytes = 0;
2564 
2565  /*
2566  * Trim the list of tracked buffers down if it exceeds the limit. Here we
2567  * remove buffers starting with the ones we created first. It seems more
2568  * likely that these older ones are less likely to be needed than ones
2569  * that were just created.
2570  */
2571  while (list_length(miinfo->multiInsertBuffers) > MAX_PARTITION_BUFFERS)
2572  {
2573  CopyMultiInsertBuffer *buffer;
2574 
2575  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2576 
2577  /*
2578  * We never want to remove the buffer that's currently being used, so
2579  * if we happen to find that then move it to the end of the list.
2580  */
2581  if (buffer->resultRelInfo == curr_rri)
2582  {
2583  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2584  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2585  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2586  }
2587 
2588  CopyMultiInsertBufferCleanup(miinfo, buffer);
2589  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2590  }
2591 }
2592 
2593 /*
2594  * Cleanup allocated buffers and free memory
2595  */
2596 static inline void
2597 CopyMultiInsertInfoCleanup(CopyMultiInsertInfo *miinfo)
2598 {
2599  ListCell *lc;
2600 
2601  foreach(lc, miinfo->multiInsertBuffers)
2602  CopyMultiInsertBufferCleanup(miinfo, lfirst(lc));
2603 
2604  list_free(miinfo->multiInsertBuffers);
2605 }
2606 
2607 /*
2608  * Get the next TupleTableSlot that the next tuple should be stored in.
2609  *
2610  * Callers must ensure that the buffer is not full.
2611  */
2612 static inline TupleTableSlot *
2613 CopyMultiInsertInfoNextFreeSlot(CopyMultiInsertInfo *miinfo,
2614  ResultRelInfo *rri)
2615 {
2616  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2617  int nused = buffer->nused;
2618 
2619  Assert(buffer != NULL);
2620  Assert(nused < MAX_BUFFERED_TUPLES);
2621 
2622  if (buffer->slots[nused] == NULL)
2623  buffer->slots[nused] = table_slot_create(rri->ri_RelationDesc, NULL);
2624  return buffer->slots[nused];
2625 }
2626 
2627 /*
2628  * Record the previously reserved TupleTableSlot that was reserved by
2629  * CopyMultiInsertInfoNextFreeSlot as being consumed.
2630  */
2631 static inline void
2632 CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2633  TupleTableSlot *slot, int tuplen, uint64 lineno)
2634 {
2635  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2636 
2637  Assert(buffer != NULL);
2638  Assert(slot == buffer->slots[buffer->nused]);
2639 
2640  /* Store the line number so we can properly report any errors later */
2641  buffer->linenos[buffer->nused] = lineno;
2642 
2643  /* Record this slot as being used */
2644  buffer->nused++;
2645 
2646  /* Update how many tuples are stored and their size */
2647  miinfo->bufferedTuples++;
2648  miinfo->bufferedBytes += tuplen;
2649 }
2650 
2651 /*
2652  * Copy FROM file to relation.
2653  */
2654 uint64
2655 CopyFrom(CopyState cstate)
2656 {
2657  ResultRelInfo *resultRelInfo;
2658  ResultRelInfo *target_resultRelInfo;
2659  ResultRelInfo *prevResultRelInfo = NULL;
2660  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2661  ModifyTableState *mtstate;
2662  ExprContext *econtext;
2663  TupleTableSlot *singleslot = NULL;
2664  MemoryContext oldcontext = CurrentMemoryContext;
2665 
2666  PartitionTupleRouting *proute = NULL;
2667  ErrorContextCallback errcallback;
2668  CommandId mycid = GetCurrentCommandId(true);
2669  int ti_options = 0; /* start with default options for insert */
2670  BulkInsertState bistate = NULL;
2671  CopyInsertMethod insertMethod;
2672  CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */
2673  uint64 processed = 0;
2674  bool has_before_insert_row_trig;
2675  bool has_instead_insert_row_trig;
2676  bool leafpart_use_multi_insert = false;
2677 
2678  Assert(cstate->rel);
2679 
2680  /*
2681  * The target must be a plain, foreign, or partitioned relation, or have
2682  * an INSTEAD OF INSERT row trigger. (Currently, such triggers are only
2683  * allowed on views, so we only hint about them in the view case.)
2684  */
2685  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2686  cstate->rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE &&
2687  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2688  !(cstate->rel->trigdesc &&
2689  cstate->rel->trigdesc->trig_insert_instead_row))
2690  {
2691  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2692  ereport(ERROR,
2693  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2694  errmsg("cannot copy to view \"%s\"",
2695  RelationGetRelationName(cstate->rel)),
2696  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2697  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2698  ereport(ERROR,
2699  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2700  errmsg("cannot copy to materialized view \"%s\"",
2701  RelationGetRelationName(cstate->rel))));
2702  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2703  ereport(ERROR,
2704  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2705  errmsg("cannot copy to sequence \"%s\"",
2706  RelationGetRelationName(cstate->rel))));
2707  else
2708  ereport(ERROR,
2709  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2710  errmsg("cannot copy to non-table relation \"%s\"",
2711  RelationGetRelationName(cstate->rel))));
2712  }
2713 
2714  /*----------
2715  * Check to see if we can avoid writing WAL
2716  *
2717  * If archive logging/streaming is not enabled *and* either
2718  * - table was created in same transaction as this COPY
2719  * - data is being written to relfilenode created in this transaction
2720  * then we can skip writing WAL. It's safe because if the transaction
2721  * doesn't commit, we'll discard the table (or the new relfilenode file).
2722  * If it does commit, we'll have done the table_finish_bulk_insert() at
2723  * the bottom of this routine first.
2724  *
2725  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2726  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2727  * can be cleared before the end of the transaction. The exact case is
2728  * when a relation sets a new relfilenode twice in same transaction, yet
2729  * the second one fails in an aborted subtransaction, e.g.
2730  *
2731  * BEGIN;
2732  * TRUNCATE t;
2733  * SAVEPOINT save;
2734  * TRUNCATE t;
2735  * ROLLBACK TO save;
2736  * COPY ...
2737  *
2738  * Also, if the target file is new-in-transaction, we assume that checking
2739  * FSM for free space is a waste of time, even if we must use WAL because
2740  * of archiving. This could possibly be wrong, but it's unlikely.
2741  *
2742  * The comments for table_tuple_insert and RelationGetBufferForTuple
2743  * specify that skipping WAL logging is only safe if we ensure that our
2744  * tuples do not go into pages containing tuples from any other
2745  * transactions --- but this must be the case if we have a new table or
2746  * new relfilenode, so we need no additional work to enforce that.
2747  *
2748  * We currently don't support this optimization if the COPY target is a
2749  * partitioned table as we currently only lazily initialize partition
2750  * information when routing the first tuple to the partition. We cannot
2751  * know at this stage if we can perform this optimization. It should be
2752  * possible to improve on this, but it does mean maintaining heap insert
2753  * option flags per partition and setting them when we first open the
2754  * partition.
2755  *
2756  * This optimization is not supported for relation types which do not
2757  * have any physical storage, with foreign tables and views using
2758  * INSTEAD OF triggers entering in this category. Partitioned tables
2759  * are not supported as per the description above.
2760  *----------
2761  */
2762  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2763  if (RELKIND_HAS_STORAGE(cstate->rel->rd_rel->relkind) &&
2764  (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2765  cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId))
2766  {
2767  ti_options |= TABLE_INSERT_SKIP_FSM;
2768  if (!XLogIsNeeded())
2769  ti_options |= TABLE_INSERT_SKIP_WAL;
2770  }
2771 
2772  /*
2773  * Optimize if new relfilenode was created in this subxact or one of its
2774  * committed children and we won't see those rows later as part of an
2775  * earlier scan or command. The subxact test ensures that if this subxact
2776  * aborts then the frozen rows won't be visible after xact cleanup. Note
2777  * that the stronger test of exactly which subtransaction created it is
2778  * crucial for correctness of this optimization. The test for an earlier
2779  * scan or command tolerates false negatives. FREEZE causes other sessions
2780  * to see rows they would not see under MVCC, and a false negative merely
2781  * spreads that anomaly to the current session.
2782  */
2783  if (cstate->freeze)
2784  {
2785  /*
2786  * We currently disallow COPY FREEZE on partitioned tables. The
2787  * reason for this is that we've simply not yet opened the partitions
2788  * to determine if the optimization can be applied to them. We could
2789  * go and open them all here, but doing so may be quite a costly
2790  * overhead for small copies. In any case, we may just end up routing
2791  * tuples to a small number of partitions. It seems better just to
2792  * raise an ERROR for partitioned tables.
2793  */
2794  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2795  {
2796  ereport(ERROR,
2797  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2798  errmsg("cannot perform COPY FREEZE on a partitioned table")));
2799  }
2800 
2801  /*
2802  * Tolerate one registration for the benefit of FirstXactSnapshot.
2803  * Scan-bearing queries generally create at least two registrations,
2804  * though relying on that is fragile, as is ignoring ActiveSnapshot.
2805  * Clear CatalogSnapshot to avoid counting its registration. We'll
2806  * still detect ongoing catalog scans, each of which separately
2807  * registers the snapshot it uses.
2808  */
2809  InvalidateCatalogSnapshot();
2810  if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
2811  ereport(ERROR,
2812  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2813  errmsg("cannot perform COPY FREEZE because of prior transaction activity")));
2814 
2815  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2816  cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
2817  ereport(ERROR,
2818  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2819  errmsg("cannot perform COPY FREEZE because the table was not created or truncated in the current subtransaction")));
2820 
2821  ti_options |= TABLE_INSERT_FROZEN;
2822  }
2823 
2824  /*
2825  * We need a ResultRelInfo so we can use the regular executor's
2826  * index-entry-making machinery. (There used to be a huge amount of code
2827  * here that basically duplicated execUtils.c ...)
2828  */
2829  resultRelInfo = makeNode(ResultRelInfo);
2830  InitResultRelInfo(resultRelInfo,
2831  cstate->rel,
2832  1, /* must match rel's position in range_table */
2833  NULL,
2834  0);
2835  target_resultRelInfo = resultRelInfo;
2836 
2837  /* Verify the named relation is a valid target for INSERT */
2838  CheckValidResultRel(resultRelInfo, CMD_INSERT);
2839 
2840  ExecOpenIndices(resultRelInfo, false);
2841 
2842  estate->es_result_relations = resultRelInfo;
2843  estate->es_num_result_relations = 1;
2844  estate->es_result_relation_info = resultRelInfo;
2845 
2846  ExecInitRangeTable(estate, cstate->range_table);
2847 
2848  /*
2849  * Set up a ModifyTableState so we can let FDW(s) init themselves for
2850  * foreign-table result relation(s).
2851  */
2852  mtstate = makeNode(ModifyTableState);
2853  mtstate->ps.plan = NULL;
2854  mtstate->ps.state = estate;
2855  mtstate->operation = CMD_INSERT;
2856  mtstate->resultRelInfo = estate->es_result_relations;
2857 
2858  if (resultRelInfo->ri_FdwRoutine != NULL &&
2859  resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
2860  resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
2861  resultRelInfo);
2862 
2863  /* Prepare to catch AFTER triggers. */
2864  AfterTriggerBeginQuery();
2865 
2866  /*
2867  * If there are any triggers with transition tables on the named relation,
2868  * we need to be prepared to capture transition tuples.
2869  *
2870  * Because partition tuple routing would like to know about whether
2871  * transition capture is active, we also set it in mtstate, which is
2872  * passed to ExecFindPartition() below.
2873  */
2874  cstate->transition_capture = mtstate->mt_transition_capture =
2875  MakeTransitionCaptureState(cstate->rel->trigdesc,
2876  RelationGetRelid(cstate->rel),
2877  CMD_INSERT);
2878 
2879  /*
2880  * If the named relation is a partitioned table, initialize state for
2881  * CopyFrom tuple routing.
2882  */
2883  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2884  proute = ExecSetupPartitionTupleRouting(estate, NULL, cstate->rel);
2885 
2886  if (cstate->whereClause)
2887  cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
2888  &mtstate->ps);
2889 
2890  /*
2891  * It's generally more efficient to prepare a bunch of tuples for
2892  * insertion, and insert them in one table_multi_insert() call, than call
2893  * table_tuple_insert() separately for every tuple. However, there are a
2894  * number of reasons why we might not be able to do this. These are
2895  * explained below.
2896  */
2897  if (resultRelInfo->ri_TrigDesc != NULL &&
2898  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2899  resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
2900  {
2901  /*
2902  * Can't support multi-inserts when there are any BEFORE/INSTEAD OF
2903  * triggers on the table. Such triggers might query the table we're
2904  * inserting into and act differently if the tuples that have already
2905  * been processed and prepared for insertion are not there.
2906  */
2907  insertMethod = CIM_SINGLE;
2908  }
2909  else if (proute != NULL && resultRelInfo->ri_TrigDesc != NULL &&
2910  resultRelInfo->ri_TrigDesc->trig_insert_new_table)
2911  {
2912  /*
2913  * For partitioned tables we can't support multi-inserts when there
2914  * are any statement level insert triggers. It might be possible to
2915  * allow partitioned tables with such triggers in the future, but for
2916  * now, CopyMultiInsertInfoFlush expects that any before row insert
2917  * and statement level insert triggers are on the same relation.
2918  */
2919  insertMethod = CIM_SINGLE;
2920  }
2921  else if (resultRelInfo->ri_FdwRoutine != NULL ||
2922  cstate->volatile_defexprs)
2923  {
2924  /*
2925  * Can't support multi-inserts to foreign tables or if there are any
2926  * volatile default expressions in the table. Similarly to the
2927  * trigger case above, such expressions may query the table we're
2928  * inserting into.
2929  *
2930  * Note: It does not matter if any partitions have any volatile
2931  * default expressions as we use the defaults from the target of the
2932  * COPY command.
2933  */
2934  insertMethod = CIM_SINGLE;
2935  }
2936  else if (contain_volatile_functions(cstate->whereClause))
2937  {
2938  /*
2939  * Can't support multi-inserts if there are any volatile function
2940  * expressions in WHERE clause. Similarly to the trigger case above,
2941  * such expressions may query the table we're inserting into.
2942  */
2943  insertMethod = CIM_SINGLE;
2944  }
2945  else
2946  {
2947  /*
2948  * For partitioned tables, we may still be able to perform bulk
2949  * inserts. However, the possibility of this depends on which types
2950  * of triggers exist on the partition. We must disable bulk inserts
2951  * if the partition is a foreign table or it has any before row insert
2952  * or insert instead triggers (same as we checked above for the parent
2953  * table). Since the partition's resultRelInfos are initialized only
2954  * when we actually need to insert the first tuple into them, we must
2955  * have the intermediate insert method of CIM_MULTI_CONDITIONAL to
2956  * flag that we must later determine if we can use bulk-inserts for
2957  * the partition being inserted into.
2958  */
2959  if (proute)
2960  insertMethod = CIM_MULTI_CONDITIONAL;
2961  else
2962  insertMethod = CIM_MULTI;
2963 
2964  CopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, cstate,
2965  estate, mycid, ti_options);
2966  }
2967 
2968  /*
2969  * If not using batch mode (which allocates slots as needed) set up a
2970  * tuple slot too. When inserting into a partitioned table, we also need
2971  * one, even if we might batch insert, to read the tuple in the root
2972  * partition's form.
2973  */
2974  if (insertMethod == CIM_SINGLE || insertMethod == CIM_MULTI_CONDITIONAL)
2975  {
2976  singleslot = table_slot_create(resultRelInfo->ri_RelationDesc,
2977  &estate->es_tupleTable);
2978  bistate = GetBulkInsertState();
2979  }
2980 
2981  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2982  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
2983 
2984  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2985  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
2986 
2987  /*
2988  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2989  * should do this for COPY, since it's not really an "INSERT" statement as
2990  * such. However, executing these triggers maintains consistency with the
2991  * EACH ROW triggers that we already fire on COPY.
2992  */
2993  ExecBSInsertTriggers(estate, resultRelInfo);
2994 
2995  econtext = GetPerTupleExprContext(estate);
2996 
2997  /* Set up callback to identify error line number */
2998  errcallback.callback = CopyFromErrorCallback;
2999  errcallback.arg = (void *) cstate;
3000  errcallback.previous = error_context_stack;
3001  error_context_stack = &errcallback;
3002 
3003  for (;;)
3004  {
3005  TupleTableSlot *myslot;
3006  bool skip_tuple;
3007 
3008  CHECK_FOR_INTERRUPTS();
3009 
3010  /*
3011  * Reset the per-tuple exprcontext. We do this after every tuple, to
3012  * clean-up after expression evaluations etc.
3013  */
3014  ResetPerTupleExprContext(estate);
3015 
3016  /* select slot to (initially) load row into */
3017  if (insertMethod == CIM_SINGLE || proute)
3018  {
3019  myslot = singleslot;
3020  Assert(myslot != NULL);
3021  }
3022  else
3023  {
3024  Assert(resultRelInfo == target_resultRelInfo);
3025  Assert(insertMethod == CIM_MULTI);
3026 
3027  myslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3028  resultRelInfo);
3029  }
3030 
3031  /*
3032  * Switch to per-tuple context before calling NextCopyFrom, which does
3033  * evaluate default expressions etc. and requires per-tuple context.
3034  */
3035  MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3036 
3037  ExecClearTuple(myslot);
3038 
3039  /* Directly store the values/nulls array in the slot */
3040  if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
3041  break;
3042 
3043  ExecStoreVirtualTuple(myslot);
3044 
3045  /*
3046  * Constraints and where clause might reference the tableoid column,
3047  * so (re-)initialize tts_tableOid before evaluating them.
3048  */
3049  myslot->tts_tableOid = RelationGetRelid(target_resultRelInfo->ri_RelationDesc);
3050 
3051  /* Triggers and stuff need to be invoked in query context. */
3052  MemoryContextSwitchTo(oldcontext);
3053 
3054  if (cstate->whereClause)
3055  {
3056  econtext->ecxt_scantuple = myslot;
3057  /* Skip items that don't match COPY's WHERE clause */
3058  if (!ExecQual(cstate->qualexpr, econtext))
3059  continue;
3060  }
3061 
3062  /* Determine the partition to insert the tuple into */
3063  if (proute)
3064  {
3065  TupleConversionMap *map;
3066 
3067  /*
3068  * Attempt to find a partition suitable for this tuple.
3069  * ExecFindPartition() will raise an error if none can be found or
3070  * if the found partition is not suitable for INSERTs.
3071  */
3072  resultRelInfo = ExecFindPartition(mtstate, target_resultRelInfo,
3073  proute, myslot, estate);
3074 
3075  if (prevResultRelInfo != resultRelInfo)
3076  {
3077  /* Determine which triggers exist on this partition */
3078  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3079  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
3080 
3081  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3082  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
3083 
3084  /*
3085  * Disable multi-inserts when the partition has BEFORE/INSTEAD
3086  * OF triggers, or if the partition is a foreign partition.
3087  */
3088  leafpart_use_multi_insert = insertMethod == CIM_MULTI_CONDITIONAL &&
3089  !has_before_insert_row_trig &&
3090  !has_instead_insert_row_trig &&
3091  resultRelInfo->ri_FdwRoutine == NULL;
3092 
3093  /* Set the multi-insert buffer to use for this partition. */
3094  if (leafpart_use_multi_insert)
3095  {
3096  if (resultRelInfo->ri_CopyMultiInsertBuffer == NULL)
3097  CopyMultiInsertInfoSetupBuffer(&multiInsertInfo,
3098  resultRelInfo);
3099  }
3100  else if (insertMethod == CIM_MULTI_CONDITIONAL &&
3101  !CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3102  {
3103  /*
3104  * Flush pending inserts if this partition can't use
3105  * batching, so rows are visible to triggers etc.
3106  */
3107  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3108  }
3109 
3110  if (bistate != NULL)
3111  ReleaseBulkInsertStatePin(bistate);
3112  prevResultRelInfo = resultRelInfo;
3113  }
3114 
3115  /*
3116  * For ExecInsertIndexTuples() to work on the partition's indexes
3117  */
3118  estate->es_result_relation_info = resultRelInfo;
3119 
3120  /*
3121  * If we're capturing transition tuples, we might need to convert
3122  * from the partition rowtype to root rowtype.
3123  */
3124  if (cstate->transition_capture != NULL)
3125  {
3126  if (has_before_insert_row_trig)
3127  {
3128  /*
3129  * If there are any BEFORE triggers on the partition,
3130  * we'll have to be ready to convert their result back to
3131  * tuplestore format.
3132  */
3133  cstate->transition_capture->tcs_original_insert_tuple = NULL;
3134  cstate->transition_capture->tcs_map =
3135  resultRelInfo->ri_PartitionInfo->pi_PartitionToRootMap;
3136  }
3137  else
3138  {
3139  /*
3140  * Otherwise, just remember the original unconverted
3141  * tuple, to avoid a needless round trip conversion.
3142  */
3143  cstate->transition_capture->tcs_original_insert_tuple = myslot;
3144  cstate->transition_capture->tcs_map = NULL;
3145  }
3146  }
3147 
3148  /*
3149  * We might need to convert from the root rowtype to the partition
3150  * rowtype.
3151  */
3152  map = resultRelInfo->ri_PartitionInfo->pi_RootToPartitionMap;
3153  if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert)
3154  {
3155  /* non batch insert */
3156  if (map != NULL)
3157  {
3158  TupleTableSlot *new_slot;
3159 
3160  new_slot = resultRelInfo->ri_PartitionInfo->pi_PartitionTupleSlot;
3161  myslot = execute_attr_map_slot(map->attrMap, myslot, new_slot);
3162  }
3163  }
3164  else
3165  {
3166  /*
3167  * Prepare to queue up tuple for later batch insert into
3168  * current partition.
3169  */
3170  TupleTableSlot *batchslot;
3171 
3172  /* no other path available for partitioned table */
3173  Assert(insertMethod == CIM_MULTI_CONDITIONAL);
3174 
3175  batchslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3176  resultRelInfo);
3177 
3178  if (map != NULL)
3179  myslot = execute_attr_map_slot(map->attrMap, myslot,
3180  batchslot);
3181  else
3182  {
3183  /*
3184  * This looks more expensive than it is (Believe me, I
3185  * optimized it away. Twice.). The input is in virtual
3186  * form, and we'll materialize the slot below - for most
3187  * slot types the copy performs the work materialization
3188  * would later require anyway.
3189  */
3190  ExecCopySlot(batchslot, myslot);
3191  myslot = batchslot;
3192  }
3193  }
3194 
3195  /* ensure that triggers etc see the right relation */
3196  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3197  }
3198 
3199  skip_tuple = false;
3200 
3201  /* BEFORE ROW INSERT Triggers */
3202  if (has_before_insert_row_trig)
3203  {
3204  if (!ExecBRInsertTriggers(estate, resultRelInfo, myslot))
3205  skip_tuple = true; /* "do nothing" */
3206  }
3207 
3208  if (!skip_tuple)
3209  {
3210  /*
3211  * If there is an INSTEAD OF INSERT ROW trigger, let it handle the
3212  * tuple. Otherwise, proceed with inserting the tuple into the
3213  * table or foreign table.
3214  */
3215  if (has_instead_insert_row_trig)
3216  {
3217  ExecIRInsertTriggers(estate, resultRelInfo, myslot);
3218  }
3219  else
3220  {
3221  /* Compute stored generated columns */
3222  if (resultRelInfo->ri_RelationDesc->rd_att->constr &&
3223  resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored)
3224  ExecComputeStoredGenerated(estate, myslot);
3225 
3226  /*
3227  * If the target is a plain table, check the constraints of
3228  * the tuple.
3229  */
3230  if (resultRelInfo->ri_FdwRoutine == NULL &&
3231  resultRelInfo->ri_RelationDesc->rd_att->constr)
3232  ExecConstraints(resultRelInfo, myslot, estate);
3233 
3234  /*
3235  * Also check the tuple against the partition constraint, if
3236  * there is one; except that if we got here via tuple-routing,
3237  * we don't need to if there's no BR trigger defined on the
3238  * partition.
3239  */
3240  if (resultRelInfo->ri_PartitionCheck &&
3241  (proute == NULL || has_before_insert_row_trig))
3242  ExecPartitionCheck(resultRelInfo, myslot, estate, true);
3243 
3244  /* Store the slot in the multi-insert buffer, when enabled. */
3245  if (insertMethod == CIM_MULTI || leafpart_use_multi_insert)
3246  {
3247  /*
3248  * The slot previously might point into the per-tuple
3249  * context. For batching it needs to be longer lived.
3250  */
3251  ExecMaterializeSlot(myslot);
3252 
3253  /* Add this tuple to the tuple buffer */
3254  CopyMultiInsertInfoStore(&multiInsertInfo,
3255  resultRelInfo, myslot,
3256  cstate->line_buf.len,
3257  cstate->cur_lineno);
3258 
3259  /*
3260  * If enough inserts have queued up, then flush all
3261  * buffers out to their tables.
3262  */
3263  if (CopyMultiInsertInfoIsFull(&multiInsertInfo))
3264  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3265  }
3266  else
3267  {
3268  List *recheckIndexes = NIL;
3269 
3270  /* OK, store the tuple */
3271  if (resultRelInfo->ri_FdwRoutine != NULL)
3272  {
3273  myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
3274  resultRelInfo,
3275  myslot,
3276  NULL);
3277 
3278  if (myslot == NULL) /* "do nothing" */
3279  continue; /* next tuple please */
3280 
3281  /*
3282  * AFTER ROW Triggers might reference the tableoid
3283  * column, so (re-)initialize tts_tableOid before
3284  * evaluating them.
3285  */
3286  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3287  }
3288  else
3289  {
3290  /* OK, store the tuple and create index entries for it */
3291  table_tuple_insert(resultRelInfo->ri_RelationDesc,
3292  myslot, mycid, ti_options, bistate);
3293 
3294  if (resultRelInfo->ri_NumIndices > 0)
3295  recheckIndexes = ExecInsertIndexTuples(myslot,
3296  estate,
3297  false,
3298  NULL,
3299  NIL);
3300  }
3301 
3302  /* AFTER ROW INSERT Triggers */
3303  ExecARInsertTriggers(estate, resultRelInfo, myslot,
3304  recheckIndexes, cstate->transition_capture);
3305 
3306  list_free(recheckIndexes);
3307  }
3308  }
3309 
3310  /*
3311  * We count only tuples not suppressed by a BEFORE INSERT trigger
3312  * or FDW; this is the same definition used by nodeModifyTable.c
3313  * for counting tuples inserted by an INSERT command.
3314  */
3315  processed++;
3316  }
3317  }
3318 
3319  /* Flush any remaining buffered tuples */
3320  if (insertMethod != CIM_SINGLE)
3321  {
3322  if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3323  CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);
3324  }
3325 
3326  /* Done, clean up */
3327  error_context_stack = errcallback.previous;
3328 
3329  if (bistate != NULL)
3330  FreeBulkInsertState(bistate);
3331 
3332  MemoryContextSwitchTo(oldcontext);
3333 
3334  /*
3335  * In the old protocol, tell pqcomm that we can process normal protocol
3336  * messages again.
3337  */
3338  if (cstate->copy_dest == COPY_OLD_FE)
3339  pq_endmsgread();
3340 
3341  /* Execute AFTER STATEMENT insertion triggers */
3342  ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
3343 
3344  /* Handle queued AFTER triggers */
3345  AfterTriggerEndQuery(estate);
3346 
3347  ExecResetTupleTable(estate->es_tupleTable, false);
3348 
3349  /* Allow the FDW to shut down */
3350  if (target_resultRelInfo->ri_FdwRoutine != NULL &&
3351  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
3352  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,
3353  target_resultRelInfo);
3354 
3355  /* Tear down the multi-insert buffer data */
3356  if (insertMethod != CIM_SINGLE)
3357  CopyMultiInsertInfoCleanup(&multiInsertInfo);
3358 
3359  ExecCloseIndices(target_resultRelInfo);
3360 
3361  /* Close all the partitioned tables, leaf partitions, and their indices */
3362  if (proute)
3363  ExecCleanupTupleRouting(mtstate, proute);
3364 
3365  /* Close any trigger target relations */
3366  ExecCleanUpTriggerState(estate);
3367 
3368  FreeExecutorState(estate);
3369 
3370  return processed;
3371 }
3372 
3373 /*
3374  * Setup to read tuples from a file for COPY FROM.
3375  *
3376  * 'rel': Used as a template for the tuples
3377  * 'filename': Name of server-local file to read
3378  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
3379  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
3380  *
3381  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
3382  */
3383 CopyState
3384 BeginCopyFrom(ParseState *pstate,
3385  Relation rel,
3386  const char *filename,
3387  bool is_program,
3388  copy_data_source_cb data_source_cb,
3389  List *attnamelist,
3390  List *options)
3391 {
3392  CopyState cstate;
3393  bool pipe = (filename == NULL);
3394  TupleDesc tupDesc;
3395  AttrNumber num_phys_attrs,
3396  num_defaults;
3397  FmgrInfo *in_functions;
3398  Oid *typioparams;
3399  int attnum;
3400  Oid in_func_oid;
3401  int *defmap;
3402  ExprState **defexprs;
3403  MemoryContext oldcontext;
3404  bool volatile_defexprs;
3405 
3406  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
3407  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
3408 
3409  /* Initialize state variables */
3410  cstate->reached_eof = false;
3411  cstate->eol_type = EOL_UNKNOWN;
3412  cstate->cur_relname = RelationGetRelationName(cstate->rel);
3413  cstate->cur_lineno = 0;
3414  cstate->cur_attname = NULL;
3415  cstate->cur_attval = NULL;
3416 
3417  /* Set up variables to avoid per-attribute overhead. */
3418  initStringInfo(&cstate->attribute_buf);
3419  initStringInfo(&cstate->line_buf);
3420  cstate->line_buf_converted = false;
3421  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
3422  cstate->raw_buf_index = cstate->raw_buf_len = 0;
3423 
3424  /* Assign range table, we'll need it in CopyFrom. */
3425  if (pstate)
3426  cstate->range_table = pstate->p_rtable;
3427 
3428  tupDesc = RelationGetDescr(cstate->rel);
3429  num_phys_attrs = tupDesc->natts;
3430  num_defaults = 0;
3431  volatile_defexprs = false;
3432 
3433  /*
3434  * Pick up the required catalog information for each attribute in the
3435  * relation, including the input function, the element type (to pass to
3436  * the input function), and info about defaults and constraints. (Which
3437  * input function we use depends on text/binary format choice.)
3438  */
3439  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
3440  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3441  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3442  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3443 
3444  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3445  {
3446  Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
3447 
3448  /* We don't need info for dropped attributes */
3449  if (att->attisdropped)
3450  continue;
3451 
3452  /* Fetch the input function and typioparam info */
3453  if (cstate->binary)
3454  getTypeBinaryInputInfo(att->atttypid,
3455  &in_func_oid, &typioparams[attnum - 1]);
3456  else
3457  getTypeInputInfo(att->atttypid,
3458  &in_func_oid, &typioparams[attnum - 1]);
3459  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3460 
3461  /* Get default info if needed */
3462  if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated)
3463  {
3464  /* attribute is NOT to be copied from input */
3465  /* use default value if one exists */
3466  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
3467  attnum);
3468 
3469  if (defexpr != NULL)
3470  {
3471  /* Run the expression through planner */
3472  defexpr = expression_planner(defexpr);
3473 
3474  /* Initialize executable expression in copycontext */
3475  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3476  defmap[num_defaults] = attnum - 1;
3477  num_defaults++;
3478 
3479  /*
3480  * If a default expression looks at the table being loaded,
3481  * then it could give the wrong answer when using
3482  * multi-insert. Since database access can be dynamic this is
3483  * hard to test for exactly, so we use the much wider test of
3484  * whether the default expression is volatile. We allow for
3485  * the special case of when the default expression is the
3486  * nextval() of a sequence which in this specific case is
3487  * known to be safe for use with the multi-insert
3488  * optimization. Hence we use this special case function
3489  * checker rather than the standard check for
3490  * contain_volatile_functions().
3491  */
3492  if (!volatile_defexprs)
3493  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3494  }
3495  }
3496  }
3497 
3498  /* We keep those variables in cstate. */
3499  cstate->in_functions = in_functions;
3500  cstate->typioparams = typioparams;
3501  cstate->defmap = defmap;
3502  cstate->defexprs = defexprs;
3503  cstate->volatile_defexprs = volatile_defexprs;
3504  cstate->num_defaults = num_defaults;
3505  cstate->is_program = is_program;
3506 
3507  if (data_source_cb)
3508  {
3509  cstate->copy_dest = COPY_CALLBACK;
3510  cstate->data_source_cb = data_source_cb;
3511  }
3512  else if (pipe)
3513  {
3514  Assert(!is_program); /* the grammar does not allow this */
3515  if (whereToSendOutput == DestRemote)
3516  ReceiveCopyBegin(cstate);
3517  else
3518  cstate->copy_file = stdin;
3519  }
3520  else
3521  {
3522  cstate->filename = pstrdup(filename);
3523 
3524  if (cstate->is_program)
3525  {
3526  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3527  if (cstate->copy_file == NULL)
3528  ereport(ERROR,
3529  (errcode_for_file_access(),
3530  errmsg("could not execute command \"%s\": %m",
3531  cstate->filename)));
3532  }
3533  else
3534  {
3535  struct stat st;
3536 
3537  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3538  if (cstate->copy_file == NULL)
3539  {
3540  /* copy errno because ereport subfunctions might change it */
3541  int save_errno = errno;
3542 
3543  ereport(ERROR,
3544  (errcode_for_file_access(),
3545  errmsg("could not open file \"%s\" for reading: %m",
3546  cstate->filename),
3547  (save_errno == ENOENT || save_errno == EACCES) ?
3548  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3549  "You may want a client-side facility such as psql's \\copy.") : 0));
3550  }
3551 
3552  if (fstat(fileno(cstate->copy_file), &st))
3553  ereport(ERROR,
3554  (errcode_for_file_access(),
3555  errmsg("could not stat file \"%s\": %m",
3556  cstate->filename)));
3557 
3558  if (S_ISDIR(st.st_mode))
3559  ereport(ERROR,
3560  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3561  errmsg("\"%s\" is a directory", cstate->filename)));
3562  }
3563  }
3564 
3565  if (cstate->binary)
3566  {
3567  /* Read and verify binary header */
3568  char readSig[11];
3569  int32 tmp;
3570 
3571  /* Signature */
3572  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3573  memcmp(readSig, BinarySignature, 11) != 0)
3574  ereport(ERROR,
3575  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3576  errmsg("COPY file signature not recognized")));
3577  /* Flags field */
3578  if (!CopyGetInt32(cstate, &tmp))
3579  ereport(ERROR,
3580  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3581  errmsg("invalid COPY file header (missing flags)")));
3582  if ((tmp & (1 << 16)) != 0)
3583  ereport(ERROR,
3584  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3585  errmsg("invalid COPY file header (WITH OIDS)")));
3586  tmp &= ~(1 << 16);
3587  if ((tmp >> 16) != 0)
3588  ereport(ERROR,
3589  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3590  errmsg("unrecognized critical flags in COPY file header")));
3591  /* Header extension length */
3592  if (!CopyGetInt32(cstate, &tmp) ||
3593  tmp < 0)
3594  ereport(ERROR,
3595  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3596  errmsg("invalid COPY file header (missing length)")));
3597  /* Skip extension header, if present */
3598  while (tmp-- > 0)
3599  {
3600  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3601  ereport(ERROR,
3602  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3603  errmsg("invalid COPY file header (wrong length)")));
3604  }
3605  }
3606 
3607  /* create workspace for CopyReadAttributes results */
3608  if (!cstate->binary)
3609  {
3610  AttrNumber attr_count = list_length(cstate->attnumlist);
3611 
3612  cstate->max_fields = attr_count;
3613  cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
3614  }
3615 
3616  MemoryContextSwitchTo(oldcontext);
3617 
3618  return cstate;
3619 }
3620 
3621 /*
3622  * Read raw fields in the next line for COPY FROM in text or csv mode.
3623  * Return false if no more lines.
3624  *
3625  * An internal temporary buffer is returned via 'fields'. It is valid until
3626  * the next call of the function. Since the function returns all raw fields
3627  * in the input file, 'nfields' could be different from the number of columns
3628  * in the relation.
3629  *
3630  * NOTE: force_not_null option are not applied to the returned fields.
3631  */
3632 bool
3633 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3634 {
3635  int fldct;
3636  bool done;
3637 
3638  /* only available for text or csv input */
3639  Assert(!cstate->binary);
3640 
3641  /* on input just throw the header line away */
3642  if (cstate->cur_lineno == 0 && cstate->header_line)
3643  {
3644  cstate->cur_lineno++;
3645  if (CopyReadLine(cstate))
3646  return false; /* done */
3647  }
3648 
3649  cstate->cur_lineno++;
3650 
3651  /* Actually read the line into memory here */
3652  done = CopyReadLine(cstate);
3653 
3654  /*
3655  * EOF at start of line means we're done. If we see EOF after some
3656  * characters, we act as though it was newline followed by EOF, ie,
3657  * process the line and then exit loop on next iteration.
3658  */
3659  if (done && cstate->line_buf.len == 0)
3660  return false;
3661 
3662  /* Parse the line into de-escaped field values */
3663  if (cstate->csv_mode)
3664  fldct = CopyReadAttributesCSV(cstate);
3665  else
3666  fldct = CopyReadAttributesText(cstate);
3667 
3668  *fields = cstate->raw_fields;
3669  *nfields = fldct;
3670  return true;
3671 }
3672 
3673 /*
3674  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3675  *
3676  * 'econtext' is used to evaluate default expression for each columns not
3677  * read from the file. It can be NULL when no default values are used, i.e.
3678  * when all columns are read from the file.
3679  *
3680  * 'values' and 'nulls' arrays must be the same length as columns of the
3681  * relation passed to BeginCopyFrom. This function fills the arrays.
3682  * Oid of the tuple is returned with 'tupleOid' separately.
3683  */
3684 bool
3685 NextCopyFrom(CopyState cstate, ExprContext *econtext,
3686  Datum *values, bool *nulls)
3687 {
3688  TupleDesc tupDesc;
3689  AttrNumber num_phys_attrs,
3690  attr_count,
3691  num_defaults = cstate->num_defaults;
3692  FmgrInfo *in_functions = cstate->in_functions;
3693  Oid *typioparams = cstate->typioparams;
3694  int i;
3695  int *defmap = cstate->defmap;
3696  ExprState **defexprs = cstate->defexprs;
3697 
3698  tupDesc = RelationGetDescr(cstate->rel);
3699  num_phys_attrs = tupDesc->natts;
3700  attr_count = list_length(cstate->attnumlist);
3701 
3702  /* Initialize all values for row to NULL */
3703  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3704  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3705 
3706  if (!cstate->binary)
3707  {
3708  char **field_strings;
3709  ListCell *cur;
3710  int fldct;
3711  int fieldno;
3712  char *string;
3713 
3714  /* read raw fields in the next line */
3715  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3716  return false;
3717 
3718  /* check for overflowing fields */
3719  if (attr_count > 0 && fldct > attr_count)
3720  ereport(ERROR,
3721  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3722  errmsg("extra data after last expected column")));
3723 
3724  fieldno = 0;
3725 
3726  /* Loop to read the user attributes on the line. */
3727  foreach(cur, cstate->attnumlist)
3728  {
3729  int attnum = lfirst_int(cur);
3730  int m = attnum - 1;
3731  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3732 
3733  if (fieldno >= fldct)
3734  ereport(ERROR,
3735  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3736  errmsg("missing data for column \"%s\"",
3737  NameStr(att->attname))));
3738  string = field_strings[fieldno++];
3739 
3740  if (cstate->convert_select_flags &&
3741  !cstate->convert_select_flags[m])
3742  {
3743  /* ignore input field, leaving column as NULL */
3744  continue;
3745  }
3746 
3747  if (cstate->csv_mode)
3748  {
3749  if (string == NULL &&
3750  cstate->force_notnull_flags[m])
3751  {
3752  /*
3753  * FORCE_NOT_NULL option is set and column is NULL -
3754  * convert it to the NULL string.
3755  */
3756  string = cstate->null_print;
3757  }
3758  else if (string != NULL && cstate->force_null_flags[m]
3759  && strcmp(string, cstate->null_print) == 0)
3760  {
3761  /*
3762  * FORCE_NULL option is set and column matches the NULL
3763  * string. It must have been quoted, or otherwise the
3764  * string would already have been set to NULL. Convert it
3765  * to NULL as specified.
3766  */
3767  string = NULL;
3768  }
3769  }
3770 
3771  cstate->cur_attname = NameStr(att->attname);
3772  cstate->cur_attval = string;
3773  values[m] = InputFunctionCall(&in_functions[m],
3774  string,
3775  typioparams[m],
3776  att->atttypmod);
3777  if (string != NULL)
3778  nulls[m] = false;
3779  cstate->cur_attname = NULL;
3780  cstate->cur_attval = NULL;
3781  }
3782 
3783  Assert(fieldno == attr_count);
3784  }
3785  else
3786  {
3787  /* binary */
3788  int16 fld_count;
3789  ListCell *cur;
3790 
3791  cstate->cur_lineno++;
3792 
3793  if (!CopyGetInt16(cstate, &fld_count))
3794  {
3795  /* EOF detected (end of file, or protocol-level EOF) */
3796  return false;
3797  }
3798 
3799  if (fld_count == -1)
3800  {
3801  /*
3802  * Received EOF marker. In a V3-protocol copy, wait for the
3803  * protocol-level EOF, and complain if it doesn't come
3804  * immediately. This ensures that we correctly handle CopyFail,
3805  * if client chooses to send that now.
3806  *
3807  * Note that we MUST NOT try to read more data in an old-protocol
3808  * copy, since there is no protocol-level EOF marker then. We
3809  * could go either way for copy from file, but choose to throw
3810  * error if there's data after the EOF marker, for consistency
3811  * with the new-protocol case.
3812  */
3813  char dummy;
3814 
3815  if (cstate->copy_dest != COPY_OLD_FE &&
3816  CopyGetData(cstate, &dummy, 1, 1) > 0)
3817  ereport(ERROR,
3818  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3819  errmsg("received copy data after EOF marker")));
3820  return false;
3821  }
3822 
3823  if (fld_count != attr_count)
3824  ereport(ERROR,
3825  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3826  errmsg("row field count is %d, expected %d",
3827  (int) fld_count, attr_count)));
3828 
3829  i = 0;
3830  foreach(cur, cstate->attnumlist)
3831  {
3832  int attnum = lfirst_int(cur);
3833  int m = attnum - 1;
3834  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3835 
3836  cstate->cur_attname = NameStr(att->attname);
3837  i++;
3838  values[m] = CopyReadBinaryAttribute(cstate,
3839  i,
3840  &in_functions[m],
3841  typioparams[m],
3842  att->atttypmod,
3843  &nulls[m]);
3844  cstate->cur_attname = NULL;
3845  }
3846  }
3847 
3848  /*
3849  * Now compute and insert any defaults available for the columns not
3850  * provided by the input data. Anything not processed here or above will
3851  * remain NULL.
3852  */
3853  for (i = 0; i < num_defaults; i++)
3854  {
3855  /*
3856  * The caller must supply econtext and have switched into the
3857  * per-tuple memory context in it.
3858  */
3859  Assert(econtext != NULL);
3860  Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
3861 
3862  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3863  &nulls[defmap[i]]);
3864  }
3865 
3866  return true;
3867 }
3868 
3869 /*
3870  * Clean up storage and release resources for COPY FROM.
3871  */
3872 void
3873 EndCopyFrom(CopyState cstate)
3874 {
3875  /* No COPY FROM related resources except memory. */
3876 
3877  EndCopy(cstate);
3878 }
3879 
3880 /*
3881  * Read the next input line and stash it in line_buf, with conversion to
3882  * server encoding.
3883  *
3884  * Result is true if read was terminated by EOF, false if terminated
3885  * by newline. The terminating newline or EOF marker is not included
3886  * in the final value of line_buf.
3887  */
3888 static bool
3889 CopyReadLine(CopyState cstate)
3890 {
3891  bool result;
3892 
3893  resetStringInfo(&cstate->line_buf);
3894  cstate->line_buf_valid = true;
3895 
3896  /* Mark that encoding conversion hasn't occurred yet */
3897  cstate->line_buf_converted = false;
3898 
3899  /* Parse data and transfer into line_buf */
3900  result = CopyReadLineText(cstate);
3901 
3902  if (result)
3903  {
3904  /*
3905  * Reached EOF. In protocol version 3, we should ignore anything
3906  * after \. up to the protocol end of copy data. (XXX maybe better
3907  * not to treat \. as special?)
3908  */
3909  if (cstate->copy_dest == COPY_NEW_FE)
3910  {
3911  do
3912  {
3913  cstate->raw_buf_index = cstate->raw_buf_len;
3914  } while (CopyLoadRawBuf(cstate));
3915  }
3916  }
3917  else
3918  {
3919  /*
3920  * If we didn't hit EOF, then we must have transferred the EOL marker
3921  * to line_buf along with the data. Get rid of it.
3922  */
3923  switch (cstate->eol_type)
3924  {
3925  case EOL_NL:
3926  Assert(cstate->line_buf.len >= 1);
3927  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3928  cstate->line_buf.len--;
3929  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3930  break;
3931  case EOL_CR:
3932  Assert(cstate->line_buf.len >= 1);
3933  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3934  cstate->line_buf.len--;
3935  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3936  break;
3937  case EOL_CRNL:
3938  Assert(cstate->line_buf.len >= 2);
3939  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3940  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3941  cstate->line_buf.len -= 2;
3942  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3943  break;
3944  case EOL_UNKNOWN:
3945  /* shouldn't get here */
3946  Assert(false);
3947  break;
3948  }
3949  }
3950 
3951  /* Done reading the line. Convert it to server encoding. */
3952  if (cstate->need_transcoding)
3953  {
3954  char *cvt;
3955 
3956  cvt = pg_any_to_server(cstate->line_buf.data,
3957  cstate->line_buf.len,
3958  cstate->file_encoding);
3959  if (cvt != cstate->line_buf.data)
3960  {
3961  /* transfer converted data back to line_buf */
3962  resetStringInfo(&cstate->line_buf);
3963  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3964  pfree(cvt);
3965  }
3966  }
3967 
3968  /* Now it's safe to use the buffer in error messages */
3969  cstate->line_buf_converted = true;
3970 
3971  return result;
3972 }
3973 
3974 /*
3975  * CopyReadLineText - inner loop of CopyReadLine for text mode
3976  */
3977 static bool
3978 CopyReadLineText(CopyState cstate)
3979 {
3980  char *copy_raw_buf;
3981  int raw_buf_ptr;
3982  int copy_buf_len;
3983  bool need_data = false;
3984  bool hit_eof = false;
3985  bool result = false;
3986  char mblen_str[2];
3987 
3988  /* CSV variables */
3989  bool first_char_in_line = true;
3990  bool in_quote = false,
3991  last_was_esc = false;
3992  char quotec = '\0';
3993  char escapec = '\0';
3994 
3995  if (cstate->csv_mode)
3996  {
3997  quotec = cstate->quote[0];
3998  escapec = cstate->escape[0];
3999  /* ignore special escape processing if it's the same as quotec */
4000  if (quotec == escapec)
4001  escapec = '\0';
4002  }
4003 
4004  mblen_str[1] = '\0';
4005 
4006  /*
4007  * The objective of this loop is to transfer the entire next input line
4008  * into line_buf. Hence, we only care for detecting newlines (\r and/or
4009  * \n) and the end-of-copy marker (\.).
4010  *
4011  * In CSV mode, \r and \n inside a quoted field are just part of the data
4012  * value and are put in line_buf. We keep just enough state to know if we
4013  * are currently in a quoted field or not.
4014  *
4015  * These four characters, and the CSV escape and quote characters, are
4016  * assumed the same in frontend and backend encodings.
4017  *
4018  * For speed, we try to move data from raw_buf to line_buf in chunks
4019  * rather than one character at a time. raw_buf_ptr points to the next
4020  * character to examine; any characters from raw_buf_index to raw_buf_ptr
4021  * have been determined to be part of the line, but not yet transferred to
4022  * line_buf.
4023  *
4024  * For a little extra speed within the loop, we copy raw_buf and
4025  * raw_buf_len into local variables.
4026  */
4027  copy_raw_buf = cstate->raw_buf;
4028  raw_buf_ptr = cstate->raw_buf_index;
4029  copy_buf_len = cstate->raw_buf_len;
4030 
4031  for (;;)
4032  {
4033  int prev_raw_ptr;
4034  char c;
4035 
4036  /*
4037  * Load more data if needed. Ideally we would just force four bytes
4038  * of read-ahead and avoid the many calls to
4039  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
4040  * does not allow us to read too far ahead or we might read into the
4041  * next data, so we read-ahead only as far we know we can. One
4042  * optimization would be to read-ahead four byte here if
4043  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
4044  * considering the size of the buffer.
4045  */
4046  if (raw_buf_ptr >= copy_buf_len || need_data)
4047  {
4048  REFILL_LINEBUF;
4049 
4050  /*
4051  * Try to read some more data. This will certainly reset
4052  * raw_buf_index to zero, and raw_buf_ptr must go with it.
4053  */
4054  if (!CopyLoadRawBuf(cstate))
4055  hit_eof = true;
4056  raw_buf_ptr = 0;
4057  copy_buf_len = cstate->raw_buf_len;
4058 
4059  /*
4060  * If we are completely out of data, break out of the loop,
4061  * reporting EOF.
4062  */
4063  if (copy_buf_len <= 0)
4064  {
4065  result = true;
4066  break;
4067  }
4068  need_data = false;
4069  }
4070 
4071  /* OK to fetch a character */
4072  prev_raw_ptr = raw_buf_ptr;
4073  c = copy_raw_buf[raw_buf_ptr++];
4074 
4075  if (cstate->csv_mode)
4076  {
4077  /*
4078  * If character is '\\' or '\r', we may need to look ahead below.
4079  * Force fetch of the next character if we don't already have it.
4080  * We need to do this before changing CSV state, in case one of
4081  * these characters is also the quote or escape character.
4082  *
4083  * Note: old-protocol does not like forced prefetch, but it's OK
4084  * here since we cannot validly be at EOF.
4085  */
4086  if (c == '\\' || c == '\r')
4087  {
4088  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4089  }
4090 
4091  /*
4092  * Dealing with quotes and escapes here is mildly tricky. If the
4093  * quote char is also the escape char, there's no problem - we
4094  * just use the char as a toggle. If they are different, we need
4095  * to ensure that we only take account of an escape inside a
4096  * quoted field and immediately preceding a quote char, and not
4097  * the second in an escape-escape sequence.
4098  */
4099  if (in_quote && c == escapec)
4100  last_was_esc = !last_was_esc;
4101  if (c == quotec && !last_was_esc)
4102  in_quote = !in_quote;
4103  if (c != escapec)
4104  last_was_esc = false;
4105 
4106  /*
4107  * Updating the line count for embedded CR and/or LF chars is
4108  * necessarily a little fragile - this test is probably about the
4109  * best we can do. (XXX it's arguable whether we should do this
4110  * at all --- is cur_lineno a physical or logical count?)
4111  */
4112  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
4113  cstate->cur_lineno++;
4114  }
4115 
4116  /* Process \r */
4117  if (c == '\r' && (!cstate->csv_mode || !in_quote))
4118  {
4119  /* Check for \r\n on first line, _and_ handle \r\n. */
4120  if (cstate->eol_type == EOL_UNKNOWN ||
4121  cstate->eol_type == EOL_CRNL)
4122  {
4123  /*
4124  * If need more data, go back to loop top to load it.
4125  *
4126  * Note that if we are at EOF, c will wind up as '\0' because
4127  * of the guaranteed pad of raw_buf.
4128  */
4129  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4130 
4131  /* get next char */
4132  c = copy_raw_buf[raw_buf_ptr];
4133 
4134  if (c == '\n')
4135  {
4136  raw_buf_ptr++; /* eat newline */
4137  cstate->eol_type = EOL_CRNL; /* in case not set yet */
4138  }
4139  else
4140  {
4141  /* found \r, but no \n */
4142  if (cstate->eol_type == EOL_CRNL)
4143  ereport(ERROR,
4144  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4145  !cstate->csv_mode ?
4146  errmsg("literal carriage return found in data") :
4147  errmsg("unquoted carriage return found in data"),
4148  !cstate->csv_mode ?
4149  errhint("Use \"\\r\" to represent carriage return.") :
4150  errhint("Use quoted CSV field to represent carriage return.")));
4151 
4152  /*
4153  * if we got here, it is the first line and we didn't find
4154  * \n, so don't consume the peeked character
4155  */
4156  cstate->eol_type = EOL_CR;
4157  }
4158  }
4159  else if (cstate->eol_type == EOL_NL)
4160  ereport(ERROR,
4161  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4162  !cstate->csv_mode ?
4163  errmsg("literal carriage return found in data") :
4164  errmsg("unquoted carriage return found in data"),
4165  !cstate->csv_mode ?
4166  errhint("Use \"\\r\" to represent carriage return.") :
4167  errhint("Use quoted CSV field to represent carriage return.")));
4168  /* If reach here, we have found the line terminator */
4169  break;
4170  }
4171 
4172  /* Process \n */
4173  if (c == '\n' && (!cstate->csv_mode || !in_quote))
4174  {
4175  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
4176  ereport(ERROR,
4177  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4178  !cstate->csv_mode ?
4179  errmsg("literal newline found in data") :
4180  errmsg("unquoted newline found in data"),
4181  !cstate->csv_mode ?
4182  errhint("Use \"\\n\" to represent newline.") :
4183  errhint("Use quoted CSV field to represent newline.")));
4184  cstate->eol_type = EOL_NL; /* in case not set yet */
4185  /* If reach here, we have found the line terminator */
4186  break;
4187  }
4188 
4189  /*
4190  * In CSV mode, we only recognize \. alone on a line. This is because
4191  * \. is a valid CSV data value.
4192  */
4193  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
4194  {
4195  char c2;
4196 
4197  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4198  IF_NEED_REFILL_AND_EOF_BREAK(0);
4199 
4200  /* -----
4201  * get next character
4202  * Note: we do not change c so if it isn't \., we can fall
4203  * through and continue processing for file encoding.
4204  * -----
4205  */
4206  c2 = copy_raw_buf[raw_buf_ptr];
4207 
4208  if (c2 == '.')
4209  {
4210  raw_buf_ptr++; /* consume the '.' */
4211 
4212  /*
4213  * Note: if we loop back for more data here, it does not
4214  * matter that the CSV state change checks are re-executed; we
4215  * will come back here with no important state changed.
4216  */
4217  if (cstate->eol_type == EOL_CRNL)
4218  {
4219  /* Get the next character */
4220  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4221  /* if hit_eof, c2 will become '\0' */
4222  c2 = copy_raw_buf[raw_buf_ptr++];
4223 
4224  if (c2 == '\n')
4225  {
4226  if (!cstate->csv_mode)
4227  ereport(ERROR,
4228  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4229  errmsg("end-of-copy marker does not match previous newline style")));
4230  else
4231  NO_END_OF_COPY_GOTO;
4232  }
4233  else if (c2 != '\r')
4234  {
4235  if (!cstate->csv_mode)
4236  ereport(ERROR,
4237  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4238  errmsg("end-of-copy marker corrupt")));
4239  else
4240  NO_END_OF_COPY_GOTO;
4241  }
4242  }
4243 
4244  /* Get the next character */
4245  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4246  /* if hit_eof, c2 will become '\0' */
4247  c2 = copy_raw_buf[raw_buf_ptr++];
4248 
4249  if (c2 != '\r' && c2 != '\n')
4250  {
4251  if (!cstate->csv_mode)
4252  ereport(ERROR,
4253  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4254  errmsg("end-of-copy marker corrupt")));
4255  else
4256  NO_END_OF_COPY_GOTO;
4257  }
4258 
4259  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
4260  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
4261  (cstate->eol_type == EOL_CR && c2 != '\r'))
4262  {
4263  ereport(ERROR,
4264  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4265  errmsg("end-of-copy marker does not match previous newline style")));
4266  }
4267 
4268  /*
4269  * Transfer only the data before the \. into line_buf, then
4270  * discard the data and the \. sequence.
4271  */
4272  if (prev_raw_ptr > cstate->raw_buf_index)
4273  appendBinaryStringInfo(&cstate->line_buf,
4274  cstate->raw_buf + cstate->raw_buf_index,
4275  prev_raw_ptr - cstate->raw_buf_index);
4276  cstate->raw_buf_index = raw_buf_ptr;
4277  result = true; /* report EOF */
4278  break;
4279  }
4280  else if (!cstate->csv_mode)
4281 
4282  /*
4283  * If we are here, it means we found a backslash followed by
4284  * something other than a period. In non-CSV mode, anything
4285  * after a backslash is special, so we skip over that second
4286  * character too. If we didn't do that \\. would be
4287  * considered an eof-of copy, while in non-CSV mode it is a
4288  * literal backslash followed by a period. In CSV mode,
4289  * backslashes are not special, so we want to process the
4290  * character after the backslash just like a normal character,
4291  * so we don't increment in those cases.
4292  */
4293  raw_buf_ptr++;
4294  }
4295 
4296  /*
4297  * This label is for CSV cases where \. appears at the start of a
4298  * line, but there is more text after it, meaning it was a data value.
4299  * We are more strict for \. in CSV mode because \. could be a data
4300  * value, while in non-CSV mode, \. cannot be a data value.
4301  */
4302 not_end_of_copy:
4303 
4304  /*
4305  * Process all bytes of a multi-byte character as a group.
4306  *
4307  * We only support multi-byte sequences where the first byte has the
4308  * high-bit set, so as an optimization we can avoid this block
4309  * entirely if it is not set.
4310  */
4311  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
4312  {
4313  int mblen;
4314 
4315  /*
4316  * It is enough to look at the first byte in all our encodings, to
4317  * get the length. (GB18030 is a bit special, but still works for
4318  * our purposes; see comment in pg_gb18030_mblen())
4319  */
4320  mblen_str[0] = c;
4321  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
4322 
4323  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
4324  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
4325  raw_buf_ptr += mblen - 1;
4326  }
4327  first_char_in_line = false;
4328  } /* end of outer loop */
4329 
4330  /*
4331  * Transfer any still-uncopied data to line_buf.
4332  */
4333  REFILL_LINEBUF;
4334 
4335  return result;
4336 }
4337 
4338 /*
4339  * Return decimal value for a hexadecimal digit
4340  */
4341 static int
4342 GetDecimalFromHex(char hex)
4343 {
4344  if (isdigit((unsigned char) hex))
4345  return hex - '0';
4346  else
4347  return tolower((unsigned char) hex) - 'a' + 10;
4348 }
4349 
4350 /*
4351  * Parse the current line into separate attributes (fields),
4352  * performing de-escaping as needed.
4353  *
4354  * The input is in line_buf. We use attribute_buf to hold the result
4355  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
4356  * string, or NULL when the input matches the null marker string.
4357  * This array is expanded as necessary.
4358  *
4359  * (Note that the caller cannot check for nulls since the returned
4360  * string would be the post-de-escaping equivalent, which may look
4361  * the same as some valid data string.)
4362  *
4363  * delim is the column delimiter string (must be just one byte for now).
4364  * null_print is the null marker string. Note that this is compared to
4365  * the pre-de-escaped input string.
4366  *
4367  * The return value is the number of fields actually read.
4368  */
4369 static int
4370 CopyReadAttributesText(CopyState cstate)
4371 {
4372  char delimc = cstate->delim[0];
4373  int fieldno;
4374  char *output_ptr;
4375  char *cur_ptr;
4376  char *line_end_ptr;
4377 
4378  /*
4379  * We need a special case for zero-column tables: check that the input
4380  * line is empty, and return.
4381  */
4382  if (cstate->max_fields <= 0)
4383  {
4384  if (cstate->line_buf.len != 0)
4385  ereport(ERROR,
4386  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4387  errmsg("extra data after last expected column")));
4388  return 0;
4389  }
4390 
4391  resetStringInfo(&cstate->attribute_buf);
4392 
4393  /*
4394  * The de-escaped attributes will certainly not be longer than the input
4395  * data line, so we can just force attribute_buf to be large enough and
4396  * then transfer data without any checks for enough space. We need to do
4397  * it this way because enlarging attribute_buf mid-stream would invalidate
4398  * pointers already stored into cstate->raw_fields[].
4399  */
4400  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4401  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4402  output_ptr = cstate->attribute_buf.data;
4403 
4404  /* set pointer variables for loop */
4405  cur_ptr = cstate->line_buf.data;
4406  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4407 
4408  /* Outer loop iterates over fields */
4409  fieldno = 0;
4410  for (;;)
4411  {
4412  bool found_delim = false;
4413  char *start_ptr;
4414  char *end_ptr;
4415  int input_len;
4416  bool saw_non_ascii = false;
4417 
4418  /* Make sure there is enough space for the next value */
4419  if (fieldno >= cstate->max_fields)
4420  {
4421  cstate->max_fields *= 2;
4422  cstate->raw_fields =
4423  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4424  }
4425 
4426  /* Remember start of field on both input and output sides */
4427  start_ptr = cur_ptr;
4428  cstate->raw_fields[fieldno] = output_ptr;
4429 
4430  /*
4431  * Scan data for field.
4432  *
4433  * Note that in this loop, we are scanning to locate the end of field
4434  * and also speculatively performing de-escaping. Once we find the
4435  * end-of-field, we can match the raw field contents against the null
4436  * marker string. Only after that comparison fails do we know that
4437  * de-escaping is actually the right thing to do; therefore we *must
4438  * not* throw any syntax errors before we've done the null-marker
4439  * check.
4440  */
4441  for (;;)
4442  {
4443  char c;
4444 
4445  end_ptr = cur_ptr;
4446  if (cur_ptr >= line_end_ptr)
4447  break;
4448  c = *cur_ptr++;
4449  if (c == delimc)
4450  {
4451  found_delim = true;
4452  break;
4453  }
4454  if (c == '\\')
4455  {
4456  if (cur_ptr >= line_end_ptr)
4457  break;
4458  c = *cur_ptr++;
4459  switch (c)
4460  {
4461  case '0':
4462  case '1':
4463  case '2':
4464  case '3':
4465  case '4':
4466  case '5':
4467  case '6':
4468  case '7':
4469  {
4470  /* handle \013 */
4471  int val;
4472 
4473  val = OCTVALUE(c);
4474  if (cur_ptr < line_end_ptr)
4475  {
4476  c = *cur_ptr;
4477  if (ISOCTAL(c))
4478  {
4479  cur_ptr++;
4480  val = (val << 3) + OCTVALUE(c);
4481  if (cur_ptr < line_end_ptr)
4482  {
4483  c = *cur_ptr;
4484  if (ISOCTAL(c))
4485  {
4486  cur_ptr++;
4487  val = (val << 3) + OCTVALUE(c);
4488  }
4489  }
4490  }
4491  }
4492  c = val & 0377;
4493  if (c == '\0' || IS_HIGHBIT_SET(c))
4494  saw_non_ascii = true;
4495  }
4496  break;
4497  case 'x':
4498  /* Handle \x3F */
4499  if (cur_ptr < line_end_ptr)
4500  {
4501  char hexchar = *cur_ptr;
4502 
4503  if (isxdigit((unsigned char) hexchar))
4504  {
4505  int val = GetDecimalFromHex(hexchar);
4506 
4507  cur_ptr++;
4508  if (cur_ptr < line_end_ptr)
4509  {
4510  hexchar = *cur_ptr;
4511  if (isxdigit((unsigned char) hexchar))
4512  {
4513  cur_ptr++;
4514  val = (val << 4) + GetDecimalFromHex(hexchar);
4515  }
4516  }
4517  c = val & 0xff;
4518  if (c == '\0' || IS_HIGHBIT_SET(c))
4519  saw_non_ascii = true;
4520  }
4521  }
4522  break;
4523  case 'b':
4524  c = '\b';
4525  break;
4526  case 'f':
4527  c = '\f';
4528  break;
4529  case 'n':
4530  c = '\n';
4531  break;
4532  case 'r':
4533  c = '\r';
4534  break;
4535  case 't':
4536  c = '\t';
4537  break;
4538  case 'v':
4539  c = '\v';
4540  break;
4541 
4542  /*
4543  * in all other cases, take the char after '\'
4544  * literally
4545  */
4546  }
4547  }
4548 
4549  /* Add c to output string */
4550  *output_ptr++ = c;
4551  }
4552 
4553  /* Check whether raw input matched null marker */
4554  input_len = end_ptr - start_ptr;
4555  if (input_len == cstate->null_print_len &&
4556  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4557  cstate->raw_fields[fieldno] = NULL;
4558  else
4559  {
4560  /*
4561  * At this point we know the field is supposed to contain data.
4562  *
4563  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4564  * resulting string is valid data for the db encoding.
4565  */
4566  if (saw_non_ascii)
4567  {
4568  char *fld = cstate->raw_fields[fieldno];
4569 
4570  pg_verifymbstr(fld, output_ptr - fld, false);
4571  }
4572  }
4573 
4574  /* Terminate attribute value in output area */
4575  *output_ptr++ = '\0';
4576 
4577  fieldno++;
4578  /* Done if we hit EOL instead of a delim */
4579  if (!found_delim)
4580  break;
4581  }
4582 
4583  /* Clean up state of attribute_buf */
4584  output_ptr--;
4585  Assert(*output_ptr == '\0');
4586  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4587 
4588  return fieldno;
4589 }
4590 
4591 /*
4592  * Parse the current line into separate attributes (fields),
4593  * performing de-escaping as needed. This has exactly the same API as
4594  * CopyReadAttributesText, except we parse the fields according to
4595  * "standard" (i.e. common) CSV usage.
4596  */
4597 static int
4598 CopyReadAttributesCSV(CopyState cstate)
4599 {
4600  char delimc = cstate->delim[0];
4601  char quotec = cstate->quote[0];
4602  char escapec = cstate->escape[0];
4603  int fieldno;
4604  char *output_ptr;
4605  char *cur_ptr;
4606  char *line_end_ptr;
4607 
4608  /*
4609  * We need a special case for zero-column tables: check that the input
4610  * line is empty, and return.
4611  */
4612  if (cstate->max_fields <= 0)
4613  {
4614  if (cstate->line_buf.len != 0)
4615  ereport(ERROR,
4616  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4617  errmsg("extra data after last expected column")));
4618  return 0;
4619  }
4620 
4621  resetStringInfo(&cstate->attribute_buf);
4622 
4623  /*
4624  * The de-escaped attributes will certainly not be longer than the input
4625  * data line, so we can just force attribute_buf to be large enough and
4626  * then transfer data without any checks for enough space. We need to do
4627  * it this way because enlarging attribute_buf mid-stream would invalidate
4628  * pointers already stored into cstate->raw_fields[].
4629  */
4630  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4631  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4632  output_ptr = cstate->attribute_buf.data;
4633 
4634  /* set pointer variables for loop */
4635  cur_ptr = cstate->line_buf.data;
4636  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4637 
4638  /* Outer loop iterates over fields */
4639  fieldno = 0;
4640  for (;;)
4641  {
4642  bool found_delim = false;
4643  bool saw_quote = false;
4644  char *start_ptr;
4645  char *end_ptr;
4646  int input_len;
4647 
4648  /* Make sure there is enough space for the next value */
4649  if (fieldno >= cstate->max_fields)
4650  {
4651  cstate->max_fields *= 2;
4652  cstate->raw_fields =
4653  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4654  }
4655 
4656  /* Remember start of field on both input and output sides */
4657  start_ptr = cur_ptr;
4658  cstate->raw_fields[fieldno] = output_ptr;
4659 
4660  /*
4661  * Scan data for field,
4662  *
4663  * The loop starts in "not quote" mode and then toggles between that
4664  * and "in quote" mode. The loop exits normally if it is in "not
4665  * quote" mode and a delimiter or line end is seen.
4666  */
4667  for (;;)
4668  {
4669  char c;
4670 
4671  /* Not in quote */
4672  for (;;)
4673  {
4674  end_ptr = cur_ptr;
4675  if (cur_ptr >= line_end_ptr)
4676  goto endfield;
4677  c = *cur_ptr++;
4678  /* unquoted field delimiter */
4679  if (c == delimc)
4680  {
4681  found_delim = true;
4682  goto endfield;
4683  }
4684  /* start of quoted field (or part of field) */
4685  if (c == quotec)
4686  {
4687  saw_quote = true;
4688  break;
4689  }
4690  /* Add c to output string */
4691  *output_ptr++ = c;
4692  }
4693 
4694  /* In quote */
4695  for (;;)
4696  {
4697  end_ptr = cur_ptr;
4698  if (cur_ptr >= line_end_ptr)
4699  ereport(ERROR,
4700  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4701  errmsg("unterminated CSV quoted field")));
4702 
4703  c = *cur_ptr++;
4704 
4705  /* escape within a quoted field */
4706  if (c == escapec)
4707  {
4708  /*
4709  * peek at the next char if available, and escape it if it
4710  * is an escape char or a quote char
4711  */
4712  if (cur_ptr < line_end_ptr)
4713  {
4714  char nextc = *cur_ptr;
4715 
4716  if (nextc == escapec || nextc == quotec)
4717  {
4718  *output_ptr++ = nextc;
4719  cur_ptr++;
4720  continue;
4721  }
4722  }
4723  }
4724 
4725  /*
4726  * end of quoted field. Must do this test after testing for
4727  * escape in case quote char and escape char are the same
4728  * (which is the common case).
4729  */
4730  if (c == quotec)
4731  break;
4732 
4733  /* Add c to output string */
4734  *output_ptr++ = c;
4735  }
4736  }
4737 endfield:
4738 
4739  /* Terminate attribute value in output area */
4740  *output_ptr++ = '\0';
4741 
4742  /* Check whether raw input matched null marker */
4743  input_len = end_ptr - start_ptr;
4744  if (!saw_quote && input_len == cstate->null_print_len &&
4745  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4746  cstate->raw_fields[fieldno] = NULL;
4747 
4748  fieldno++;
4749  /* Done if we hit EOL instead of a delim */
4750  if (!found_delim)
4751  break;
4752  }
4753 
4754  /* Clean up state of attribute_buf */
4755  output_ptr--;
4756  Assert(*output_ptr == '\0');
4757  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4758 
4759  return fieldno;
4760 }
4761 
4762 
4763 /*
4764  * Read a binary attribute
4765  */
4766 static Datum
4767 CopyReadBinaryAttribute(CopyState cstate,
4768  int column_no, FmgrInfo *flinfo,
4769  Oid typioparam, int32 typmod,
4770  bool *isnull)
4771 {
4772  int32 fld_size;
4773  Datum result;
4774 
4775  if (!CopyGetInt32(cstate, &fld_size))
4776  ereport(ERROR,
4777  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4778  errmsg("unexpected EOF in COPY data")));
4779  if (fld_size == -1)
4780  {
4781  *isnull = true;
4782  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4783  }
4784  if (fld_size < 0)
4785  ereport(ERROR,
4786  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4787  errmsg("invalid field size")));
4788 
4789  /* reset attribute_buf to empty, and load raw data in it */
4790  resetStringInfo(&cstate->attribute_buf);
4791 
4792  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4793  if (CopyGetData(cstate, cstate->attribute_buf.data,
4794  fld_size, fld_size) != fld_size)
4795  ereport(ERROR,
4796  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4797  errmsg("unexpected EOF in COPY data")));
4798 
4799  cstate->attribute_buf.len = fld_size;
4800  cstate->attribute_buf.data[fld_size] = '\0';
4801 
4802  /* Call the column type's binary input converter */
4803  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4804  typioparam, typmod);
4805 
4806  /* Trouble if it didn't eat the whole buffer */
4807  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4808  ereport(ERROR,
4809  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4810  errmsg("incorrect binary data format")));
4811 
4812  *isnull = false;
4813  return result;
4814 }
4815 
4816 /*
4817  * Send text representation of one attribute, with conversion and escaping
4818  */
4819 #define DUMPSOFAR() \
4820  do { \
4821  if (ptr > start) \
4822  CopySendData(cstate, start, ptr - start); \
4823  } while (0)
4824 
4825 static void
4826 CopyAttributeOutText(CopyState cstate, char *string)
4827 {
4828  char *ptr;
4829  char *start;
4830  char c;
4831  char delimc = cstate->delim[0];
4832 
4833  if (cstate->need_transcoding)
4834  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4835  else
4836  ptr = string;
4837 
4838  /*
4839  * We have to grovel through the string searching for control characters
4840  * and instances of the delimiter character. In most cases, though, these
4841  * are infrequent. To avoid overhead from calling CopySendData once per
4842  * character, we dump out all characters between escaped characters in a
4843  * single call. The loop invariant is that the data from "start" to "ptr"
4844  * can be sent literally, but hasn't yet been.
4845  *
4846  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4847  * in valid backend encodings, extra bytes of a multibyte character never
4848  * look like ASCII. This loop is sufficiently performance-critical that
4849  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4850  * of the normal safe-encoding path.
4851  */
4852  if (cstate->encoding_embeds_ascii)
4853  {
4854  start = ptr;
4855  while ((c = *ptr) != '\0')
4856  {
4857  if ((unsigned char) c < (unsigned char) 0x20)
4858  {
4859  /*
4860  * \r and \n must be escaped, the others are traditional. We
4861  * prefer to dump these using the C-like notation, rather than
4862  * a backslash and the literal character, because it makes the
4863  * dump file a bit more proof against Microsoftish data
4864  * mangling.
4865  */
4866  switch (c)
4867  {
4868  case '\b':
4869  c = 'b';
4870  break;
4871  case '\f':
4872  c = 'f';
4873  break;
4874  case '\n':
4875  c = 'n';
4876  break;
4877  case '\r':
4878  c = 'r';
4879  break;
4880  case '\t':
4881  c = 't';
4882  break;
4883  case '\v':
4884  c = 'v';
4885  break;
4886  default:
4887  /* If it's the delimiter, must backslash it */
4888  if (c == delimc)
4889  break;
4890  /* All ASCII control chars are length 1 */
4891  ptr++;
4892  continue; /* fall to end of loop */
4893  }
4894  /* if we get here, we need to convert the control char */
4895  DUMPSOFAR();
4896  CopySendChar(cstate, '\\');
4897  CopySendChar(cstate, c);
4898  start = ++ptr; /* do not include char in next run */
4899  }
4900  else if (c == '\\' || c == delimc)
4901  {
4902  DUMPSOFAR();
4903  CopySendChar(cstate, '\\');
4904  start = ptr++; /* we include char in next run */
4905  }
4906  else if (IS_HIGHBIT_SET(c))
4907  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4908  else
4909  ptr++;
4910  }
4911  }
4912  else
4913  {
4914  start = ptr;
4915  while ((c = *ptr) != '\0')
4916  {
4917  if ((unsigned char) c < (unsigned char) 0x20)
4918  {
4919  /*
4920  * \r and \n must be escaped, the others are traditional. We
4921  * prefer to dump these using the C-like notation, rather than
4922  * a backslash and the literal character, because it makes the
4923  * dump file a bit more proof against Microsoftish data
4924  * mangling.
4925  */
4926  switch (c)
4927  {
4928  case '\b':
4929  c = 'b';
4930  break;
4931  case '\f':
4932  c = 'f';
4933  break;
4934  case '\n':
4935  c = 'n';
4936  break;
4937  case '\r':
4938  c = 'r';
4939  break;
4940  case '\t':
4941  c = 't';
4942  break;
4943  case '\v':
4944  c = 'v';
4945  break;
4946  default:
4947  /* If it's the delimiter, must backslash it */
4948  if (c == delimc)
4949  break;
4950  /* All ASCII control chars are length 1 */
4951  ptr++;
4952  continue; /* fall to end of loop */
4953  }
4954  /* if we get here, we need to convert the control char */
4955  DUMPSOFAR();
4956  CopySendChar(cstate, '\\');
4957  CopySendChar(cstate, c);
4958  start = ++ptr; /* do not include char in next run */
4959  }
4960  else if (c == '\\' || c == delimc)
4961  {
4962  DUMPSOFAR();
4963  CopySendChar(cstate, '\\');
4964  start = ptr++; /* we include char in next run */
4965  }
4966  else
4967  ptr++;
4968  }
4969  }
4970 
4971  DUMPSOFAR();
4972 }
4973 
4974 /*
4975  * Send text representation of one attribute, with conversion and
4976  * CSV-style escaping
4977  */
4978 static void
4979 CopyAttributeOutCSV(CopyState cstate, char *string,
4980  bool use_quote, bool single_attr)
4981 {
4982  char *ptr;
4983  char *start;
4984  char c;
4985  char delimc = cstate->delim[0];
4986  char quotec = cstate->quote[0];
4987  char escapec = cstate->escape[0];
4988 
4989  /* force quoting if it matches null_print (before conversion!) */
4990  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4991  use_quote = true;
4992 
4993  if (cstate->need_transcoding)
4994  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4995  else
4996  ptr = string;
4997 
4998  /*
4999  * Make a preliminary pass to discover if it needs quoting
5000  */
5001  if (!use_quote)
5002  {
5003  /*
5004  * Because '\.' can be a data value, quote it if it appears alone on a
5005  * line so it is not interpreted as the end-of-data marker.
5006  */
5007  if (single_attr && strcmp(ptr, "\\.") == 0)
5008  use_quote = true;
5009  else
5010  {
5011  char *tptr = ptr;
5012 
5013  while ((c = *tptr) != '\0')
5014  {
5015  if (c == delimc || c == quotec || c == '\n' || c == '\r')
5016  {
5017  use_quote = true;
5018  break;
5019  }
5020  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5021  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
5022  else
5023  tptr++;
5024  }
5025  }
5026  }
5027 
5028  if (use_quote)
5029  {
5030  CopySendChar(cstate, quotec);
5031 
5032  /*
5033  * We adopt the same optimization strategy as in CopyAttributeOutText
5034  */
5035  start = ptr;
5036  while ((c = *ptr) != '\0')
5037  {
5038  if (c == quotec || c == escapec)
5039  {
5040  DUMPSOFAR();
5041  CopySendChar(cstate, escapec);
5042  start = ptr; /* we include char in next run */
5043  }
5044  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5045  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
5046  else
5047  ptr++;
5048  }
5049  DUMPSOFAR();
5050 
5051  CopySendChar(cstate, quotec);
5052  }
5053  else
5054  {
5055  /* If it doesn't need quoting, we can just dump it as-is */
5056  CopySendString(cstate, ptr);
5057  }
5058 }
5059 
5060 /*
5061  * CopyGetAttnums - build an integer list of attnums to be copied
5062  *
5063  * The input attnamelist is either the user-specified column list,
5064  * or NIL if there was none (in which case we want all the non-dropped
5065  * columns).
5066  *
5067  * We don't include generated columns in the generated full list and we don't
5068  * allow them to be specified explicitly. They don't make sense for COPY
5069  * FROM, but we could possibly allow them for COPY TO. But this way it's at
5070  * least ensured that whatever we copy out can be copied back in.
5071  *
5072  * rel can be NULL ... it's only used for error reports.
5073  */
5074 static List *
5075 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
5076 {
5077  List *attnums = NIL;
5078 
5079  if (attnamelist == NIL)
5080  {
5081  /* Generate default column list */
5082  int attr_count = tupDesc->natts;
5083  int i;
5084 
5085  for (i = 0; i < attr_count; i++)
5086  {
5087  if (TupleDescAttr(tupDesc, i)->attisdropped)
5088  continue;
5089  if (TupleDescAttr(tupDesc, i)->attgenerated)
5090  continue;
5091  attnums = lappend_int(attnums, i + 1);
5092  }
5093  }
5094  else
5095  {
5096  /* Validate the user-supplied list and extract attnums */
5097  ListCell *l;
5098 
5099  foreach(l, attnamelist)
5100  {
5101  char *name = strVal(lfirst(l));
5102  int attnum;
5103  int i;
5104 
5105  /* Lookup column name */
5106  attnum = InvalidAttrNumber;
5107  for (i = 0; i < tupDesc->natts; i++)
5108  {
5109  Form_pg_attribute att = TupleDescAttr(tupDesc, i);
5110 
5111  if (att->attisdropped)
5112  continue;
5113  if (namestrcmp(&(att->attname), name) == 0)
5114  {
5115  if (att->attgenerated)
5116  ereport(ERROR,
5117  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
5118  errmsg("column \"%s\" is a generated column",
5119  name),
5120  errdetail("Generated columns cannot be used in COPY.")));
5121  attnum = att->attnum;
5122  break;
5123  }
5124  }
5125  if (attnum == InvalidAttrNumber)
5126  {
5127  if (rel != NULL)
5128  ereport(ERROR,
5129  (errcode(ERRCODE_UNDEFINED_COLUMN),
5130  errmsg("column \"%s\" of relation \"%s\" does not exist",
5131  name, RelationGetRelationName(rel))));
5132  else
5133  ereport(ERROR,
5134  (errcode(ERRCODE_UNDEFINED_COLUMN),
5135  errmsg("column \"%s\" does not exist",
5136  name)));
5137  }
5138  /* Check for duplicates */
5139  if (list_member_int(attnums, attnum))
5140  ereport(ERROR,
5141  (errcode(ERRCODE_DUPLICATE_COLUMN),
5142  errmsg("column \"%s\" specified more than once",
5143  name)));
5144  attnums = lappend_int(attnums, attnum);
5145  }
5146  }
5147 
5148  return attnums;
5149 }
5150 
5151 
5152 /*
5153  * copy_dest_startup --- executor startup
5154  */
5155 static void
5156 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
5157 {
5158  /* no-op */
5159 }
5160 
5161 /*
5162  * copy_dest_receive --- receive one tuple
5163  */
5164 static bool
5165 copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
5166 {
5167  DR_copy *myState = (DR_copy *) self;
5168  CopyState cstate = myState->cstate;
5169 
5170  /* Send the data */
5171  CopyOneRowTo(cstate, slot);
5172  myState->processed++;
5173 
5174  return true;
5175 }
5176 
5177 /*
5178  * copy_dest_shutdown --- executor end
5179  */
5180 static void
5181 copy_dest_shutdown(DestReceiver *self)
5182 {
5183  /* no-op */
5184 }
5185 
5186 /*
5187  * copy_dest_destroy --- release DestReceiver object
5188  */
5189 static void
5190 copy_dest_destroy(DestReceiver *self)
5191 {
5192  pfree(self);
5193 }
5194 
5195 /*
5196  * CreateCopyDestReceiver -- create a suitable DestReceiver object
5197  */
5198 DestReceiver *
5199 CreateCopyDestReceiver(void)
5200 {
5201  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
5202 
5203  self->pub.receiveSlot = copy_dest_receive;
5204  self->pub.rStartup = copy_dest_startup;
5205  self->pub.rShutdown = copy_dest_shutdown;
5206  self->pub.rDestroy = copy_dest_destroy;
5207  self->pub.mydest = DestCopyOut;
5208 
5209  self->cstate = NULL; /* will be set later */
5210  self->processed = 0;
5211 
5212  return (DestReceiver *) self;
5213 }
table_slot_create
TupleTableSlot * table_slot_create(Relation relation, List **reglist)
Definition: tableam.c:77
CopyMultiInsertBuffer
struct CopyMultiInsertBuffer CopyMultiInsertBuffer
TupleConversionMap::attrMap
AttrNumber * attrMap
Definition: tupconvert.h:26
int16
signed short int16
Definition: c.h:346
ResTarget::indirection
List * indirection
Definition: parsenodes.h:441
NextCopyFromRawFields
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3633
ResultRelInfo::ri_NumIndices
int ri_NumIndices
Definition: execnodes.h:413
CopyStateData::whereClause
Node * whereClause
Definition: copy.c:155
ExecCopySlot
static TupleTableSlot * ExecCopySlot(TupleTableSlot *dstslot, TupleTableSlot *srcslot)
Definition: tuptable.h:475
NIL
#define NIL
Definition: pg_list.h:65
CopyStmt::whereClause
Node * whereClause
Definition: parsenodes.h:1996
TupleTableSlot::tts_tableOid
Oid tts_tableOid
Definition: tuptable.h:131
pq_sendint16
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
CommandId
uint32 CommandId
Definition: c.h:528
ExecInitRangeTable
void ExecInitRangeTable(EState *estate, List *rangeTable)
Definition: execUtils.c:724
CopyReadBinaryAttribute
static Datum CopyReadBinaryAttribute(CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4767
GetDecimalFromHex
static int GetDecimalFromHex(char hex)
Definition: copy.c:4342
FmgrInfo
Definition: fmgr.h:56
MAX_COPY_DATA_DISPLAY
#define MAX_COPY_DATA_DISPLAY
CopyStateData::csv_mode
bool csv_mode
Definition: copy.c:137
SendCopyEnd
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:470
InitResultRelInfo
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1277
ResultRelInfo::ri_RelationDesc
Relation ri_RelationDesc
Definition: execnodes.h:410
CopyStateData::range_table
List * range_table
Definition: copy.c:183
BeginCopy
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1460
UpdateActiveSnapshotCommandId
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:783
IsA
#define IsA(nodeptr, _type_)
Definition: nodes.h:576
CopyReadLineText
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3978
MemoryContextDelete
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:211
AllocSetContextCreate
#define AllocSetContextCreate
Definition: memutils.h:170
HOLD_CANCEL_INTERRUPTS
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:124
CopyMultiInsertInfoStore
static void CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, TupleTableSlot *slot, int tuplen, uint64 lineno)
Definition: copy.c:2632
contain_volatile_functions_not_nextval
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:774
table_close
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:133
ResTarget::val
Node * val
Definition: parsenodes.h:442
CopyGetInt32
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:738
errhint
int errhint(const char *fmt,...)
Definition: elog.c:1069
ExecARInsertTriggers
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2599
CopyMultiInsertInfo::estate
EState * estate
Definition: copy.c:278
pg_char_to_encoding
int pg_char_to_encoding(const char *name)
Definition: encnames.c:551
CopyStateData::raw_fields
char ** raw_fields
Definition: copy.c:201
EOL_NL
Definition: copy.c:82
getTypeOutputInfo
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2674
VARDATA
#define VARDATA(PTR)
Definition: postgres.h:302
EndCopy
static void EndCopy(CopyState cstate)
Definition: copy.c:1801
CopyStateData::binary
bool binary
Definition: copy.c:135
ResultRelInfo::ri_CopyMultiInsertBuffer
struct CopyMultiInsertBuffer * ri_CopyMultiInsertBuffer
Definition: execnodes.h:487
pq_flush
#define pq_flush()
Definition: libpq.h:39
CopyFromErrorCallback
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2236
PreventCommandIfParallelMode
void PreventCommandIfParallelMode(const char *cmdname)
Definition: utility.c:256
CopyStmt::attlist
List * attlist
Definition: parsenodes.h:1990
SelectStmt::fromClause
List * fromClause
Definition: parsenodes.h:1575
ISOCTAL
#define ISOCTAL(c)
Definition: copy.c:61
ExecClearTuple
static TupleTableSlot * ExecClearTuple(TupleTableSlot *slot)
Definition: tuptable.h:425
OCTVALUE
#define OCTVALUE(c)
Definition: copy.c:62
ResetPerTupleExprContext
#define ResetPerTupleExprContext(estate)
Definition: executor.h:510
RelationGetDescr
#define RelationGetDescr(relation)
Definition: rel.h:448
LOCKMODE
int LOCKMODE
Definition: lockdefs.h:26
ResTarget::name
char * name
Definition: parsenodes.h:440
GetUserId
Oid GetUserId(void)
Definition: miscinit.c:380
TABLE_INSERT_FROZEN
#define TABLE_INSERT_FROZEN
Definition: tableam.h:132
CopyStateData::need_transcoding
bool need_transcoding
Definition: copy.c:125
castNode
#define castNode(_type_, nodeptr)
Definition: nodes.h:594
FreeQueryDesc
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
CopyStateData::in_functions
FmgrInfo * in_functions
Definition: copy.c:178
CopyStateData::num_defaults
AttrNumber num_defaults
Definition: copy.c:177
CopyStateData::attnumlist
List * attnumlist
Definition: copy.c:131
VARSIZE
#define VARSIZE(PTR)
Definition: postgres.h:303
pq_getmsgstring
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:581
CopyStateData::filename
char * filename
Definition: copy.c:132
FdwRoutine::BeginForeignInsert
BeginForeignInsert_function BeginForeignInsert
Definition: fdwapi.h:215
ModifyTableState::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: execnodes.h:1168
TupleDescAttr
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
CopyMultiInsertInfo
struct CopyMultiInsertInfo CopyMultiInsertInfo
BeginCopyFrom
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:3384
VARHDRSZ
#define VARHDRSZ
Definition: c.h:562
FdwRoutine::ExecForeignInsert
ExecForeignInsert_function ExecForeignInsert
Definition: fdwapi.h:211
ResultRelInfo::ri_PartitionInfo
struct PartitionRoutingInfo * ri_PartitionInfo
Definition: execnodes.h:484
PlannedStmt::relationOids
List * relationOids
Definition: plannodes.h:86
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1186
pg_hton16
#define pg_hton16(x)
Definition: pg_bswap.h:120
ExprContext::ecxt_per_tuple_memory
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:231
ReceiveCopyBegin
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:434
XLogIsNeeded
#define XLogIsNeeded()
Definition: xlog.h:181
pg_ntoh16
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
DR_copy
Definition: copy.c:228
makeStringInfo
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
CopyMultiInsertInfoSetupBuffer
static void CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri)
Definition: copy.c:2363
MAX_BUFFERED_TUPLES
#define MAX_BUFFERED_TUPLES
Definition: copy.c:245
RelationData::rd_islocaltemp
bool rd_islocaltemp
Definition: rel.h:59
CopyMultiInsertInfo::mycid
CommandId mycid
Definition: copy.c:279
EXPR_KIND_COPY_WHERE
Definition: parse_node.h:74
S_IWOTH
#define S_IWOTH
Definition: win32_port.h:287
expression_planner
Expr * expression_planner(Expr *expr)
Definition: planner.c:6051
ExecutorStart
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:143
CopyMultiInsertBuffer::linenos
uint64 linenos[MAX_BUFFERED_TUPLES]
Definition: copy.c:263
transformExpr
Node * transformExpr(ParseState *pstate, Node *expr, ParseExprKind exprKind)
Definition: parse_expr.c:144
DR_copy::pub
DestReceiver pub
Definition: copy.c:230
ProcessCopyOptions
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1109
ExecConstraints
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1896
CopyStateData::line_buf
StringInfoData line_buf
Definition: copy.c:210
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
IF_NEED_REFILL_AND_EOF_BREAK
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:312
GetActiveSnapshot
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:841
AccessShareLock
#define AccessShareLock
Definition: lockdefs.h:36
value
static struct @145 value
CopyStateData
struct CopyStateData CopyStateData
Node
Definition: nodes.h:525
strVal
#define strVal(v)
Definition: value.h:54
cur
struct cursor * cur
Definition: ecpg.c:28
CopyStateData::raw_buf_index
int raw_buf_index
Definition: copy.c:223
CopyAttributeOutText
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4826
CopyStateData::line_buf_valid
bool line_buf_valid
Definition: copy.c:212
ThereAreNoPriorRegisteredSnapshots
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1689
DR_copy::cstate
CopyState cstate
Definition: copy.c:231
errcode
int errcode(int sqlerrcode)
Definition: elog.c:608
PG_BINARY_W
#define PG_BINARY_W
Definition: c.h:1225
namestrcmp
int namestrcmp(Name name, const char *str)
Definition: name.c:287
parse_relation.h
MemSet
#define MemSet(start, val, len)
Definition: c.h:962
generate_unaccent_rules.dest
dest
Definition: generate_unaccent_rules.py:284
execute_attr_map_slot
TupleTableSlot * execute_attr_map_slot(AttrNumber *attrMap, TupleTableSlot *in_slot, TupleTableSlot *out_slot)
Definition: tupconvert.c:425
eval_const_expressions
Node * eval_const_expressions(PlannerInfo *root, Node *node)
Definition: clauses.c:2253
CopyFrom
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2655
ModifyTableState::operation
CmdType operation
Definition: execnodes.h:1160
parse_expr.h
RelationData::rd_newRelfilenodeSubid
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:80
pq_putemptymessage
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
table_finish_bulk_insert
static void table_finish_bulk_insert(Relation rel, int options)
Definition: tableam.h:1318
TupleTableSlot::tts_values
Datum * tts_values
Definition: tuptable.h:126
FirstLowInvalidHeapAttributeNumber
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:27
ClosePipeToProgram
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1766
MemoryContextReset
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:136
CopyReadAttributesCSV
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4598
PopActiveSnapshot
void PopActiveSnapshot(void)
Definition: snapmgr.c:814
RangeTblEntry::requiredPerms
AclMode requiredPerms
Definition: parsenodes.h:1096
contain_volatile_functions
bool contain_volatile_functions(Node *clause)
Definition: clauses.c:724
SIGPIPE
#define SIGPIPE
Definition: win32_port.h:159
PlanState::state
EState * state
Definition: execnodes.h:941
CopyStateData::force_quote_flags
bool * force_quote_flags
Definition: copy.c:147
table_scan_getnextslot
static bool table_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot)
Definition: tableam.h:872
RelationData::rd_rel
Form_pg_class rd_rel
Definition: rel.h:83
Oid
unsigned int Oid
Definition: postgres_ext.h:31
pg_analyze_and_rewrite
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:675
ExecQual
static bool ExecQual(ExprState *state, ExprContext *econtext)
Definition: executor.h:365
CopyStateData::delim
char * delim
Definition: copy.c:142
pg_type.h
PG_PROTOCOL_MAJOR
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Query::utilityStmt
Node * utilityStmt
Definition: parsenodes.h:120
CopyStmt::is_program
bool is_program
Definition: parsenodes.h:1993
linitial_node
#define linitial_node(type, l)
Definition: pg_list.h:198
CopyStateData::volatile_defexprs
bool volatile_defexprs
Definition: copy.c:182
pg_wchar.h
ErrorContextCallback::callback
void(* callback)(void *arg)
Definition: elog.h:256
ErrorContextCallback::previous
struct ErrorContextCallback * previous
Definition: elog.h:255
CopyMultiInsertInfoInit
static void CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, CopyState cstate, EState *estate, CommandId mycid, int ti_options)
Definition: copy.c:2383
PG_BINARY_R
#define PG_BINARY_R
Definition: c.h:1224
CopyMultiInsertBuffer::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: copy.c:260
copy_dest_destroy
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:5190
CopyStateData::force_null_flags
bool * force_null_flags
Definition: copy.c:151
wait_result_to_str
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
CopyStateData::rowcontext
MemoryContext rowcontext
Definition: copy.c:172
parse_coerce.h
_DestReceiver
Definition: dest.h:115
CopyStateData::line_buf_converted
bool line_buf_converted
Definition: copy.c:211
table_tuple_insert
static void table_tuple_insert(Relation rel, TupleTableSlot *slot, CommandId cid, int options, struct BulkInsertStateData *bistate)
Definition: tableam.h:1123
pq_beginmessage
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
ExecInitQual
ExprState * ExecInitQual(List *qual, PlanState *parent)
Definition: execExpr.c:207
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:654
int32
signed int int32
Definition: c.h:347
ClosePipeStream
int ClosePipeStream(FILE *file)
Definition: fd.c:2651
errdetail_internal
int errdetail_internal(const char *fmt,...)
Definition: elog.c:982
CopyStateData::convert_select_flags
bool * convert_select_flags
Definition: copy.c:154
CopySendChar
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:511
OutputFunctionCall
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1575
ExecOpenIndices
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:151
ExecComputeStoredGenerated
void ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot)
Definition: nodeModifyTable.c:249
ColumnRef::location
int location
Definition: parsenodes.h:236
CopyStateData::copy_dest
CopyDest copy_dest
Definition: copy.c:116
ResTarget::location
int location
Definition: parsenodes.h:443
pq_sendbyte
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
REFILL_LINEBUF
#define REFILL_LINEBUF
Definition: copy.c:329
error_context_stack
ErrorContextCallback * error_context_stack
Definition: elog.c:91
CopyStateData::cur_lineno
uint64 cur_lineno
Definition: copy.c:159
list_make1
#define list_make1(x1)
Definition: pg_list.h:227
CopyStateData::null_print
char * null_print
Definition: copy.c:139
CopyStateData::cur_attname
const char * cur_attname
Definition: copy.c:160
assign_expr_collations
void assign_expr_collations(ParseState *pstate, Node *expr)
Definition: parse_collate.c:177
ExecutorEnd
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:462
COPY_FILE
Definition: copy.c:70