PostgreSQL Source Code  git master
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 
21 #include "access/heapam.h"
22 #include "access/htup_details.h"
23 #include "access/sysattr.h"
24 #include "access/tableam.h"
25 #include "access/xact.h"
26 #include "access/xlog.h"
27 #include "catalog/dependency.h"
28 #include "catalog/pg_authid.h"
29 #include "catalog/pg_type.h"
30 #include "commands/copy.h"
31 #include "commands/defrem.h"
32 #include "commands/trigger.h"
33 #include "executor/execPartition.h"
34 #include "executor/executor.h"
35 #include "executor/nodeModifyTable.h"
36 #include "executor/tuptable.h"
37 #include "foreign/fdwapi.h"
38 #include "libpq/libpq.h"
39 #include "libpq/pqformat.h"
40 #include "mb/pg_wchar.h"
41 #include "miscadmin.h"
42 #include "nodes/makefuncs.h"
43 #include "optimizer/optimizer.h"
44 #include "parser/parse_coerce.h"
45 #include "parser/parse_collate.h"
46 #include "parser/parse_expr.h"
47 #include "parser/parse_relation.h"
48 #include "port/pg_bswap.h"
49 #include "rewrite/rewriteHandler.h"
50 #include "storage/fd.h"
51 #include "tcop/tcopprot.h"
52 #include "utils/acl.h"
53 #include "utils/builtins.h"
54 #include "utils/lsyscache.h"
55 #include "utils/memutils.h"
56 #include "utils/partcache.h"
57 #include "utils/portal.h"
58 #include "utils/rel.h"
59 #include "utils/rls.h"
60 #include "utils/snapmgr.h"
61 
62 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
63 #define OCTVALUE(c) ((c) - '0')
64 
65 /*
66  * Represents the different source/dest cases we need to worry about at
67  * the bottom level
68  */
69 typedef enum CopyDest
70 {
71  COPY_FILE, /* to/from file (or a piped program) */
72  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
73  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
74  COPY_CALLBACK /* to/from callback function */
75 } CopyDest;
76 
77 /*
78  * Represents the end-of-line terminator type of the input
79  */
80 typedef enum EolType
81 {
82  EOL_UNKNOWN,
83  EOL_NL,
84  EOL_CR,
85  EOL_CRNL
86 } EolType;
87 
88 /*
89  * Represents the heap insert method to be used during COPY FROM.
90  */
91 typedef enum CopyInsertMethod
92 {
93  CIM_SINGLE, /* use table_tuple_insert or fdw routine */
94  CIM_MULTI, /* always use table_multi_insert */
95  CIM_MULTI_CONDITIONAL /* use table_multi_insert only if valid */
96 } CopyInsertMethod;
97 
98 /*
99  * This struct contains all the state variables used throughout a COPY
100  * operation. For simplicity, we use the same struct for all variants of COPY,
101  * even though some fields are used in only some cases.
102  *
103  * Multi-byte encodings: all supported client-side encodings encode multi-byte
104  * characters by having the first byte's high bit set. Subsequent bytes of the
105  * character can have the high bit not set. When scanning data in such an
106  * encoding to look for a match to a single-byte (ie ASCII) character, we must
107  * use the full pg_encoding_mblen() machinery to skip over multibyte
108  * characters, else we might find a false match to a trailing byte. In
109  * supported server encodings, there is no possibility of a false match, and
110  * it's faster to make useless comparisons to trailing bytes than it is to
111  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
112  * when we have to do it the hard way.
113  */
114 typedef struct CopyStateData
115 {
116  /* low-level state data */
117  CopyDest copy_dest; /* type of copy source/destination */
118  FILE *copy_file; /* used if copy_dest == COPY_FILE */
119  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
120  * dest == COPY_NEW_FE in COPY FROM */
121  bool is_copy_from; /* COPY TO, or COPY FROM? */
122  bool reached_eof; /* true if we read to end of copy data (not
123  * all copy_dest types maintain this) */
124  EolType eol_type; /* EOL type of input */
125  int file_encoding; /* file or remote side's character encoding */
126  bool need_transcoding; /* file encoding diff from server? */
127  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
128 
129  /* parameters from the COPY command */
130  Relation rel; /* relation to copy to or from */
131  QueryDesc *queryDesc; /* executable query to copy from */
132  List *attnumlist; /* integer list of attnums to copy */
133  char *filename; /* filename, or NULL for STDIN/STDOUT */
134  bool is_program; /* is 'filename' a program to popen? */
135  copy_data_source_cb data_source_cb; /* function for reading data */
136  bool binary; /* binary format? */
137  bool freeze; /* freeze rows on loading? */
138  bool csv_mode; /* Comma Separated Value format? */
139  bool header_line; /* CSV header line? */
140  char *null_print; /* NULL marker string (server encoding!) */
141  int null_print_len; /* length of same */
142  char *null_print_client; /* same converted to file encoding */
143  char *delim; /* column delimiter (must be 1 byte) */
144  char *quote; /* CSV quote char (must be 1 byte) */
145  char *escape; /* CSV escape char (must be 1 byte) */
146  List *force_quote; /* list of column names */
147  bool force_quote_all; /* FORCE_QUOTE *? */
148  bool *force_quote_flags; /* per-column CSV FQ flags */
149  List *force_notnull; /* list of column names */
150  bool *force_notnull_flags; /* per-column CSV FNN flags */
151  List *force_null; /* list of column names */
152  bool *force_null_flags; /* per-column CSV FN flags */
153  bool convert_selectively; /* do selective binary conversion? */
154  List *convert_select; /* list of column names (can be NIL) */
155  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
156  Node *whereClause; /* WHERE condition (or NULL) */
157 
158  /* these are just for error messages, see CopyFromErrorCallback */
159  const char *cur_relname; /* table name for error messages */
160  uint64 cur_lineno; /* line number for error messages */
161  const char *cur_attname; /* current att for error messages */
162  const char *cur_attval; /* current att value for error messages */
163 
164  /*
165  * Working state for COPY TO/FROM
166  */
167  MemoryContext copycontext; /* per-copy execution context */
168 
169  /*
170  * Working state for COPY TO
171  */
172  FmgrInfo *out_functions; /* lookup info for output functions */
173  MemoryContext rowcontext; /* per-row evaluation context */
174 
175  /*
176  * Working state for COPY FROM
177  */
178  AttrNumber num_defaults;
179  FmgrInfo *in_functions; /* array of input functions for each attrs */
180  Oid *typioparams; /* array of element types for in_functions */
181  int *defmap; /* array of default att numbers */
182  ExprState **defexprs; /* array of default att expressions */
183  bool volatile_defexprs; /* is any of defexprs volatile? */
184  List *range_table;
185  ExprState *qualexpr;
186 
187  TransitionCaptureState *transition_capture;
188 
189  /*
190  * These variables are used to reduce overhead in textual COPY FROM.
191  *
192  * attribute_buf holds the separated, de-escaped text for each field of
193  * the current line. The CopyReadAttributes functions return arrays of
194  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
195  * the buffer on each cycle.
196  */
197  StringInfoData attribute_buf;
198 
199  /* field raw data pointers found by COPY FROM */
200 
201  int max_fields;
202  char **raw_fields;
203 
204  /*
205  * Similarly, line_buf holds the whole input line being processed. The
206  * input cycle is first to read the whole line into line_buf, convert it
207  * to server encoding there, and then extract the individual attribute
208  * fields into attribute_buf. line_buf is preserved unmodified so that we
209  * can display it in error messages if appropriate.
210  */
211  StringInfoData line_buf;
212  bool line_buf_converted; /* converted to server encoding? */
213  bool line_buf_valid; /* contains the row being processed? */
214 
215  /*
216  * Finally, raw_buf holds raw data read from the data source (file or
217  * client connection). CopyReadLine parses this data sufficiently to
218  * locate line boundaries, then transfers the data to line_buf and
219  * converts it. Note: we guarantee that there is a \0 at
220  * raw_buf[raw_buf_len].
221  */
222 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
223  char *raw_buf;
224  int raw_buf_index; /* next byte to process */
225  int raw_buf_len; /* total # of bytes stored */
226 } CopyStateData;
227 
228 /* DestReceiver for COPY (query) TO */
229 typedef struct
230 {
231  DestReceiver pub; /* publicly-known function pointers */
232  CopyState cstate; /* CopyStateData for the command */
233  uint64 processed; /* # of tuples processed */
234 } DR_copy;
235 
236 
237 /*
238  * No more than this many tuples per CopyMultiInsertBuffer
239  *
240  * Caution: Don't make this too big, as we could end up with this many
241  * CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's
242  * multiInsertBuffers list. Increasing this can cause quadratic growth in
243  * memory requirements during copies into partitioned tables with a large
244  * number of partitions.
245  */
246 #define MAX_BUFFERED_TUPLES 1000
247 
248 /*
249  * Flush buffers if there are >= this many bytes, as counted by the input
250  * size, of tuples stored.
251  */
252 #define MAX_BUFFERED_BYTES 65535
253 
254 /* Trim the list of buffers back down to this number after flushing */
255 #define MAX_PARTITION_BUFFERS 32
256 
257 /* Stores multi-insert data related to a single relation in CopyFrom. */
258 typedef struct CopyMultiInsertBuffer
259 {
260  TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */
261  ResultRelInfo *resultRelInfo; /* ResultRelInfo for 'relid' */
262  BulkInsertState bistate; /* BulkInsertState for this rel */
263  int nused; /* number of 'slots' containing tuples */
264  uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy
265  * stream */
266 } CopyMultiInsertBuffer;
267 
268 /*
269  * Stores one or many CopyMultiInsertBuffers and details about the size and
270  * number of tuples which are stored in them. This allows multiple buffers to
271  * exist at once when COPYing into a partitioned table.
272  */
273 typedef struct CopyMultiInsertInfo
274 {
275  List *multiInsertBuffers; /* List of tracked CopyMultiInsertBuffers */
276  int bufferedTuples; /* number of tuples buffered over all buffers */
277  int bufferedBytes; /* number of bytes from all buffered tuples */
278  CopyState cstate; /* Copy state for this CopyMultiInsertInfo */
279  EState *estate; /* Executor state used for COPY */
280  CommandId mycid; /* Command Id used for COPY */
281  int ti_options; /* table insert options */
282 } CopyMultiInsertInfo;
283 
284 
285 /*
286  * These macros centralize code used to process line_buf and raw_buf buffers.
287  * They are macros because they often do continue/break control and to avoid
288  * function call overhead in tight COPY loops.
289  *
290  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
291  * prevent the continue/break processing from working. We end the "if (1)"
292  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
293  * any "else" in the calling code, and to avoid any compiler warnings about
294  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
295  */
296 
297 /*
298  * This keeps the character read at the top of the loop in the buffer
299  * even if there is more than one read-ahead.
300  */
301 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
302 if (1) \
303 { \
304  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
305  { \
306  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
307  need_data = true; \
308  continue; \
309  } \
310 } else ((void) 0)
311 
312 /* This consumes the remainder of the buffer and breaks */
313 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
314 if (1) \
315 { \
316  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
317  { \
318  if (extralen) \
319  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
320  /* backslash just before EOF, treat as data char */ \
321  result = true; \
322  break; \
323  } \
324 } else ((void) 0)
325 
326 /*
327  * Transfer any approved data to line_buf; must do this to be sure
328  * there is some room in raw_buf.
329  */
330 #define REFILL_LINEBUF \
331 if (1) \
332 { \
333  if (raw_buf_ptr > cstate->raw_buf_index) \
334  { \
335  appendBinaryStringInfo(&cstate->line_buf, \
336  cstate->raw_buf + cstate->raw_buf_index, \
337  raw_buf_ptr - cstate->raw_buf_index); \
338  cstate->raw_buf_index = raw_buf_ptr; \
339  } \
340 } else ((void) 0)
341 
342 /* Undo any read-ahead and jump out of the block. */
343 #define NO_END_OF_COPY_GOTO \
344 if (1) \
345 { \
346  raw_buf_ptr = prev_raw_ptr + 1; \
347  goto not_end_of_copy; \
348 } else ((void) 0)
349 
350 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
351 
352 
353 /* non-export function prototypes */
354 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
355  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
356  List *options);
357 static void EndCopy(CopyState cstate);
358 static void ClosePipeToProgram(CopyState cstate);
359 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
360  Oid queryRelId, const char *filename, bool is_program,
361  List *attnamelist, List *options);
362 static void EndCopyTo(CopyState cstate);
363 static uint64 DoCopyTo(CopyState cstate);
364 static uint64 CopyTo(CopyState cstate);
365 static void CopyOneRowTo(CopyState cstate, TupleTableSlot *slot);
366 static bool CopyReadLine(CopyState cstate);
367 static bool CopyReadLineText(CopyState cstate);
368 static int CopyReadAttributesText(CopyState cstate);
369 static int CopyReadAttributesCSV(CopyState cstate);
370 static Datum CopyReadBinaryAttribute(CopyState cstate,
371  int column_no, FmgrInfo *flinfo,
372  Oid typioparam, int32 typmod,
373  bool *isnull);
374 static void CopyAttributeOutText(CopyState cstate, char *string);
375 static void CopyAttributeOutCSV(CopyState cstate, char *string,
376  bool use_quote, bool single_attr);
377 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
378  List *attnamelist);
379 static char *limit_printout_length(const char *str);
380 
381 /* Low-level communications functions */
382 static void SendCopyBegin(CopyState cstate);
383 static void ReceiveCopyBegin(CopyState cstate);
384 static void SendCopyEnd(CopyState cstate);
385 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
386 static void CopySendString(CopyState cstate, const char *str);
387 static void CopySendChar(CopyState cstate, char c);
388 static void CopySendEndOfRow(CopyState cstate);
389 static int CopyGetData(CopyState cstate, void *databuf,
390  int minread, int maxread);
391 static void CopySendInt32(CopyState cstate, int32 val);
392 static bool CopyGetInt32(CopyState cstate, int32 *val);
393 static void CopySendInt16(CopyState cstate, int16 val);
394 static bool CopyGetInt16(CopyState cstate, int16 *val);
395 
396 
397 /*
398  * Send copy start/stop messages for frontend copies. These have changed
399  * in past protocol redesigns.
400  */
401 static void
402 SendCopyBegin(CopyState cstate)
403 {
404  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
405  {
406  /* new way */
407  StringInfoData buf;
408  int natts = list_length(cstate->attnumlist);
409  int16 format = (cstate->binary ? 1 : 0);
410  int i;
411 
412  pq_beginmessage(&buf, 'H');
413  pq_sendbyte(&buf, format); /* overall format */
414  pq_sendint16(&buf, natts);
415  for (i = 0; i < natts; i++)
416  pq_sendint16(&buf, format); /* per-column formats */
417  pq_endmessage(&buf);
418  cstate->copy_dest = COPY_NEW_FE;
419  }
420  else
421  {
422  /* old way */
423  if (cstate->binary)
424  ereport(ERROR,
425  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
426  errmsg("COPY BINARY is not supported to stdout or from stdin")));
427  pq_putemptymessage('H');
428  /* grottiness needed for old COPY OUT protocol */
429  pq_startcopyout();
430  cstate->copy_dest = COPY_OLD_FE;
431  }
432 }
433 
434 static void
435 ReceiveCopyBegin(CopyState cstate)
436 {
437  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
438  {
439  /* new way */
440  StringInfoData buf;
441  int natts = list_length(cstate->attnumlist);
442  int16 format = (cstate->binary ? 1 : 0);
443  int i;
444 
445  pq_beginmessage(&buf, 'G');
446  pq_sendbyte(&buf, format); /* overall format */
447  pq_sendint16(&buf, natts);
448  for (i = 0; i < natts; i++)
449  pq_sendint16(&buf, format); /* per-column formats */
450  pq_endmessage(&buf);
451  cstate->copy_dest = COPY_NEW_FE;
452  cstate->fe_msgbuf = makeStringInfo();
453  }
454  else
455  {
456  /* old way */
457  if (cstate->binary)
458  ereport(ERROR,
459  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
460  errmsg("COPY BINARY is not supported to stdout or from stdin")));
461  pq_putemptymessage('G');
462  /* any error in old protocol will make us lose sync */
463  pq_startmsgread();
464  cstate->copy_dest = COPY_OLD_FE;
465  }
466  /* We *must* flush here to ensure FE knows it can send. */
467  pq_flush();
468 }
469 
470 static void
471 SendCopyEnd(CopyState cstate)
472 {
473  if (cstate->copy_dest == COPY_NEW_FE)
474  {
475  /* Shouldn't have any unsent data */
476  Assert(cstate->fe_msgbuf->len == 0);
477  /* Send Copy Done message */
478  pq_putemptymessage('c');
479  }
480  else
481  {
482  CopySendData(cstate, "\\.", 2);
483  /* Need to flush out the trailer (this also appends a newline) */
484  CopySendEndOfRow(cstate);
485  pq_endcopyout(false);
486  }
487 }
488 
489 /*----------
490  * CopySendData sends output data to the destination (file or frontend)
491  * CopySendString does the same for null-terminated strings
492  * CopySendChar does the same for single characters
493  * CopySendEndOfRow does the appropriate thing at end of each data row
494  * (data is not actually flushed except by CopySendEndOfRow)
495  *
496  * NB: no data conversion is applied by these functions
497  *----------
498  */
499 static void
500 CopySendData(CopyState cstate, const void *databuf, int datasize)
501 {
502  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
503 }
504 
505 static void
506 CopySendString(CopyState cstate, const char *str)
507 {
508  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
509 }
510 
511 static void
512 CopySendChar(CopyState cstate, char c)
513 {
514  appendStringInfoCharMacro(cstate->fe_msgbuf, c);
515 }
516 
517 static void
518 CopySendEndOfRow(CopyState cstate)
519 {
520  StringInfo fe_msgbuf = cstate->fe_msgbuf;
521 
522  switch (cstate->copy_dest)
523  {
524  case COPY_FILE:
525  if (!cstate->binary)
526  {
527  /* Default line termination depends on platform */
528 #ifndef WIN32
529  CopySendChar(cstate, '\n');
530 #else
531  CopySendString(cstate, "\r\n");
532 #endif
533  }
534 
535  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
536  cstate->copy_file) != 1 ||
537  ferror(cstate->copy_file))
538  {
539  if (cstate->is_program)
540  {
541  if (errno == EPIPE)
542  {
543  /*
544  * The pipe will be closed automatically on error at
545  * the end of transaction, but we might get a better
546  * error message from the subprocess' exit code than
547  * just "Broken Pipe"
548  */
549  ClosePipeToProgram(cstate);
550 
551  /*
552  * If ClosePipeToProgram() didn't throw an error, the
553  * program terminated normally, but closed the pipe
554  * first. Restore errno, and throw an error.
555  */
556  errno = EPIPE;
557  }
558  ereport(ERROR,
559  (errcode_for_file_access(),
560  errmsg("could not write to COPY program: %m")));
561  }
562  else
563  ereport(ERROR,
564  (errcode_for_file_access(),
565  errmsg("could not write to COPY file: %m")));
566  }
567  break;
568  case COPY_OLD_FE:
569  /* The FE/BE protocol uses \n as newline for all platforms */
570  if (!cstate->binary)
571  CopySendChar(cstate, '\n');
572 
573  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
574  {
575  /* no hope of recovering connection sync, so FATAL */
576  ereport(FATAL,
577  (errcode(ERRCODE_CONNECTION_FAILURE),
578  errmsg("connection lost during COPY to stdout")));
579  }
580  break;
581  case COPY_NEW_FE:
582  /* The FE/BE protocol uses \n as newline for all platforms */
583  if (!cstate->binary)
584  CopySendChar(cstate, '\n');
585 
586  /* Dump the accumulated row as one CopyData message */
587  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
588  break;
589  case COPY_CALLBACK:
590  Assert(false); /* Not yet supported. */
591  break;
592  }
593 
594  resetStringInfo(fe_msgbuf);
595 }
596 
597 /*
598  * CopyGetData reads data from the source (file or frontend)
599  *
600  * We attempt to read at least minread, and at most maxread, bytes from
601  * the source. The actual number of bytes read is returned; if this is
602  * less than minread, EOF was detected.
603  *
604  * Note: when copying from the frontend, we expect a proper EOF mark per
605  * protocol; if the frontend simply drops the connection, we raise error.
606  * It seems unwise to allow the COPY IN to complete normally in that case.
607  *
608  * NB: no data conversion is applied here.
609  */
610 static int
611 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
612 {
613  int bytesread = 0;
614 
615  switch (cstate->copy_dest)
616  {
617  case COPY_FILE:
618  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
619  if (ferror(cstate->copy_file))
620  ereport(ERROR,
621  (errcode_for_file_access(),
622  errmsg("could not read from COPY file: %m")));
623  if (bytesread == 0)
624  cstate->reached_eof = true;
625  break;
626  case COPY_OLD_FE:
627 
628  /*
629  * We cannot read more than minread bytes (which in practice is 1)
630  * because old protocol doesn't have any clear way of separating
631  * the COPY stream from following data. This is slow, but not any
632  * slower than the code path was originally, and we don't care
633  * much anymore about the performance of old protocol.
634  */
635  if (pq_getbytes((char *) databuf, minread))
636  {
637  /* Only a \. terminator is legal EOF in old protocol */
638  ereport(ERROR,
639  (errcode(ERRCODE_CONNECTION_FAILURE),
640  errmsg("unexpected EOF on client connection with an open transaction")));
641  }
642  bytesread = minread;
643  break;
644  case COPY_NEW_FE:
645  while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
646  {
647  int avail;
648 
649  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
650  {
651  /* Try to receive another message */
652  int mtype;
653 
654  readmessage:
655  HOLD_CANCEL_INTERRUPTS();
656  pq_startmsgread();
657  mtype = pq_getbyte();
658  if (mtype == EOF)
659  ereport(ERROR,
660  (errcode(ERRCODE_CONNECTION_FAILURE),
661  errmsg("unexpected EOF on client connection with an open transaction")));
662  if (pq_getmessage(cstate->fe_msgbuf, 0))
663  ereport(ERROR,
664  (errcode(ERRCODE_CONNECTION_FAILURE),
665  errmsg("unexpected EOF on client connection with an open transaction")));
666  RESUME_CANCEL_INTERRUPTS();
667  switch (mtype)
668  {
669  case 'd': /* CopyData */
670  break;
671  case 'c': /* CopyDone */
672  /* COPY IN correctly terminated by frontend */
673  cstate->reached_eof = true;
674  return bytesread;
675  case 'f': /* CopyFail */
676  ereport(ERROR,
677  (errcode(ERRCODE_QUERY_CANCELED),
678  errmsg("COPY from stdin failed: %s",
679  pq_getmsgstring(cstate->fe_msgbuf))));
680  break;
681  case 'H': /* Flush */
682  case 'S': /* Sync */
683 
684  /*
685  * Ignore Flush/Sync for the convenience of client
686  * libraries (such as libpq) that may send those
687  * without noticing that the command they just
688  * sent was COPY.
689  */
690  goto readmessage;
691  default:
692  ereport(ERROR,
693  (errcode(ERRCODE_PROTOCOL_VIOLATION),
694  errmsg("unexpected message type 0x%02X during COPY from stdin",
695  mtype)));
696  break;
697  }
698  }
699  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
700  if (avail > maxread)
701  avail = maxread;
702  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
703  databuf = (void *) ((char *) databuf + avail);
704  maxread -= avail;
705  bytesread += avail;
706  }
707  break;
708  case COPY_CALLBACK:
709  bytesread = cstate->data_source_cb(databuf, minread, maxread);
710  break;
711  }
712 
713  return bytesread;
714 }
715 
716 
717 /*
718  * These functions do apply some data conversion
719  */
720 
721 /*
722  * CopySendInt32 sends an int32 in network byte order
723  */
724 static void
725 CopySendInt32(CopyState cstate, int32 val)
726 {
727  uint32 buf;
728 
729  buf = pg_hton32((uint32) val);
730  CopySendData(cstate, &buf, sizeof(buf));
731 }
732 
733 /*
734  * CopyGetInt32 reads an int32 that appears in network byte order
735  *
736  * Returns true if OK, false if EOF
737  */
738 static bool
739 CopyGetInt32(CopyState cstate, int32 *val)
740 {
741  uint32 buf;
742 
743  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
744  {
745  *val = 0; /* suppress compiler warning */
746  return false;
747  }
748  *val = (int32) pg_ntoh32(buf);
749  return true;
750 }
751 
752 /*
753  * CopySendInt16 sends an int16 in network byte order
754  */
755 static void
756 CopySendInt16(CopyState cstate, int16 val)
757 {
758  uint16 buf;
759 
760  buf = pg_hton16((uint16) val);
761  CopySendData(cstate, &buf, sizeof(buf));
762 }
763 
764 /*
765  * CopyGetInt16 reads an int16 that appears in network byte order
766  */
767 static bool
768 CopyGetInt16(CopyState cstate, int16 *val)
769 {
770  uint16 buf;
771 
772  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
773  {
774  *val = 0; /* suppress compiler warning */
775  return false;
776  }
777  *val = (int16) pg_ntoh16(buf);
778  return true;
779 }
780 
781 
782 /*
783  * CopyLoadRawBuf loads some more data into raw_buf
784  *
785  * Returns true if able to obtain at least one more byte, else false.
786  *
787  * If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
788  * down to the start of the buffer and then we load more data after that.
789  * This case is used only when a frontend multibyte character crosses a
790  * bufferload boundary.
791  */
792 static bool
793 CopyLoadRawBuf(CopyState cstate)
794 {
795  int nbytes;
796  int inbytes;
797 
798  if (cstate->raw_buf_index < cstate->raw_buf_len)
799  {
800  /* Copy down the unprocessed data */
801  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
802  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
803  nbytes);
804  }
805  else
806  nbytes = 0; /* no data need be saved */
807 
808  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
809  1, RAW_BUF_SIZE - nbytes);
810  nbytes += inbytes;
811  cstate->raw_buf[nbytes] = '\0';
812  cstate->raw_buf_index = 0;
813  cstate->raw_buf_len = nbytes;
814  return (inbytes > 0);
815 }
816 
817 
818 /*
819  * DoCopy executes the SQL COPY statement
820  *
821  * Either unload or reload contents of table <relation>, depending on <from>.
822  * (<from> = true means we are inserting into the table.) In the "TO" case
823  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
824  * or DELETE query.
825  *
826  * If <pipe> is false, transfer is between the table and the file named
827  * <filename>. Otherwise, transfer is between the table and our regular
828  * input/output stream. The latter could be either stdin/stdout or a
829  * socket, depending on whether we're running under Postmaster control.
830  *
831  * Do not allow a Postgres user without the 'pg_read_server_files' or
832  * 'pg_write_server_files' role to read from or write to a file.
833  *
834  * Do not allow the copy if user doesn't have proper permission to access
835  * the table or the specifically requested columns.
836  */
837 void
838 DoCopy(ParseState *pstate, const CopyStmt *stmt,
839  int stmt_location, int stmt_len,
840  uint64 *processed)
841 {
842  CopyState cstate;
843  bool is_from = stmt->is_from;
844  bool pipe = (stmt->filename == NULL);
845  Relation rel;
846  Oid relid;
847  RawStmt *query = NULL;
848  Node *whereClause = NULL;
849 
850  /*
851  * Disallow COPY to/from file or program except to users with the
852  * appropriate role.
853  */
854  if (!pipe)
855  {
856  if (stmt->is_program)
857  {
858  if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_EXECUTE_SERVER_PROGRAM))
859  ereport(ERROR,
860  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
861  errmsg("must be superuser or a member of the pg_execute_server_program role to COPY to or from an external program"),
862  errhint("Anyone can COPY to stdout or from stdin. "
863  "psql's \\copy command also works for anyone.")));
864  }
865  else
866  {
867  if (is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_SERVER_FILES))
868  ereport(ERROR,
869  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
870  errmsg("must be superuser or a member of the pg_read_server_files role to COPY from a file"),
871  errhint("Anyone can COPY to stdout or from stdin. "
872  "psql's \\copy command also works for anyone.")));
873 
874  if (!is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_WRITE_SERVER_FILES))
875  ereport(ERROR,
876  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
877  errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
878  errhint("Anyone can COPY to stdout or from stdin. "
879  "psql's \\copy command also works for anyone.")));
880  }
881  }
882 
883  if (stmt->relation)
884  {
885  LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock;
886  ParseNamespaceItem *nsitem;
887  RangeTblEntry *rte;
888  TupleDesc tupDesc;
889  List *attnums;
890  ListCell *cur;
891 
892  Assert(!stmt->query);
893 
894  /* Open and lock the relation, using the appropriate lock type. */
895  rel = table_openrv(stmt->relation, lockmode);
896 
897  relid = RelationGetRelid(rel);
898 
899  nsitem = addRangeTableEntryForRelation(pstate, rel, lockmode,
900  NULL, false, false);
901  rte = nsitem->p_rte;
902  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
903 
904  if (stmt->whereClause)
905  {
906  /* add nsitem to query namespace */
907  addNSItemToQuery(pstate, nsitem, false, true, true);
908 
909  /* Transform the raw expression tree */
910  whereClause = transformExpr(pstate, stmt->whereClause, EXPR_KIND_COPY_WHERE);
911 
912  /* Make sure it yields a boolean result. */
913  whereClause = coerce_to_boolean(pstate, whereClause, "WHERE");
914 
915  /* we have to fix its collations too */
916  assign_expr_collations(pstate, whereClause);
917 
918  whereClause = eval_const_expressions(NULL, whereClause);
919 
920  whereClause = (Node *) canonicalize_qual((Expr *) whereClause, false);
921  whereClause = (Node *) make_ands_implicit((Expr *) whereClause);
922  }
923 
924  tupDesc = RelationGetDescr(rel);
925  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
926  foreach(cur, attnums)
927  {
928  int attno = lfirst_int(cur) -
929  FirstLowInvalidHeapAttributeNumber;
930 
931  if (is_from)
932  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
933  else
934  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
935  }
936  ExecCheckRTPerms(pstate->p_rtable, true);
937 
938  /*
939  * Permission check for row security policies.
940  *
941  * check_enable_rls will ereport(ERROR) if the user has requested
942  * something invalid and will otherwise indicate if we should enable
943  * RLS (returns RLS_ENABLED) or not for this COPY statement.
944  *
945  * If the relation has a row security policy and we are to apply it
946  * then perform a "query" copy and allow the normal query processing
947  * to handle the policies.
948  *
949  * If RLS is not enabled for this, then just fall through to the
950  * normal non-filtering relation handling.
951  */
952  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
953  {
954  SelectStmt *select;
955  ColumnRef *cr;
956  ResTarget *target;
957  RangeVar *from;
958  List *targetList = NIL;
959 
960  if (is_from)
961  ereport(ERROR,
962  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
963  errmsg("COPY FROM not supported with row-level security"),
964  errhint("Use INSERT statements instead.")));
965 
966  /*
967  * Build target list
968  *
969  * If no columns are specified in the attribute list of the COPY
970  * command, then the target list is 'all' columns. Therefore, '*'
971  * should be used as the target list for the resulting SELECT
972  * statement.
973  *
974  * In the case that columns are specified in the attribute list,
975  * create a ColumnRef and ResTarget for each column and add them
976  * to the target list for the resulting SELECT statement.
977  */
978  if (!stmt->attlist)
979  {
980  cr = makeNode(ColumnRef);
981  cr->fields = list_make1(makeNode(A_Star));
982  cr->location = -1;
983 
984  target = makeNode(ResTarget);
985  target->name = NULL;
986  target->indirection = NIL;
987  target->val = (Node *) cr;
988  target->location = -1;
989 
990  targetList = list_make1(target);
991  }
992  else
993  {
994  ListCell *lc;
995 
996  foreach(lc, stmt->attlist)
997  {
998  /*
999  * Build the ColumnRef for each column. The ColumnRef
1000  * 'fields' property is a String 'Value' node (see
1001  * nodes/value.h) that corresponds to the column name
1002  * respectively.
1003  */
1004  cr = makeNode(ColumnRef);
1005  cr->fields = list_make1(lfirst(lc));
1006  cr->location = -1;
1007 
1008  /* Build the ResTarget and add the ColumnRef to it. */
1009  target = makeNode(ResTarget);
1010  target->name = NULL;
1011  target->indirection = NIL;
1012  target->val = (Node *) cr;
1013  target->location = -1;
1014 
1015  /* Add each column to the SELECT statement's target list */
1016  targetList = lappend(targetList, target);
1017  }
1018  }
1019 
1020  /*
1021  * Build RangeVar for from clause, fully qualified based on the
1022  * relation which we have opened and locked.
1023  */
1024  from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
1025  pstrdup(RelationGetRelationName(rel)),
1026  -1);
1027 
1028  /* Build query */
1029  select = makeNode(SelectStmt);
1030  select->targetList = targetList;
1031  select->fromClause = list_make1(from);
1032 
1033  query = makeNode(RawStmt);
1034  query->stmt = (Node *) select;
1035  query->stmt_location = stmt_location;
1036  query->stmt_len = stmt_len;
1037 
1038  /*
1039  * Close the relation for now, but keep the lock on it to prevent
1040  * changes between now and when we start the query-based COPY.
1041  *
1042  * We'll reopen it later as part of the query-based COPY.
1043  */
1044  table_close(rel, NoLock);
1045  rel = NULL;
1046  }
1047  }
1048  else
1049  {
1050  Assert(stmt->query);
1051 
1052  query = makeNode(RawStmt);
1053  query->stmt = stmt->query;
1054  query->stmt_location = stmt_location;
1055  query->stmt_len = stmt_len;
1056 
1057  relid = InvalidOid;
1058  rel = NULL;
1059  }
1060 
1061  if (is_from)
1062  {
1063  Assert(rel);
1064 
1065  /* check read-only transaction and parallel mode */
1066  if (XactReadOnly && !rel->rd_islocaltemp)
1067  PreventCommandIfReadOnly("COPY FROM");
1068 
1069  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
1070  NULL, stmt->attlist, stmt->options);
1071  cstate->whereClause = whereClause;
1072  *processed = CopyFrom(cstate); /* copy from file to database */
1073  EndCopyFrom(cstate);
1074  }
1075  else
1076  {
1077  cstate = BeginCopyTo(pstate, rel, query, relid,
1078  stmt->filename, stmt->is_program,
1079  stmt->attlist, stmt->options);
1080  *processed = DoCopyTo(cstate); /* copy from database to file */
1081  EndCopyTo(cstate);
1082  }
1083 
1084  /*
1085  * Close the relation. If reading, we can release the AccessShareLock we
1086  * got; if writing, we should hold the lock until end of transaction to
1087  * ensure that updates will be committed before lock is released.
1088  */
1089  if (rel != NULL)
1090  table_close(rel, (is_from ? NoLock : AccessShareLock));
1091 }
1092 
1093 /*
1094  * Process the statement option list for COPY.
1095  *
1096  * Scan the options list (a list of DefElem) and transpose the information
1097  * into cstate, applying appropriate error checking.
1098  *
1099  * cstate is assumed to be filled with zeroes initially.
1100  *
1101  * This is exported so that external users of the COPY API can sanity-check
1102  * a list of options. In that usage, cstate should be passed as NULL
1103  * (since external users don't know sizeof(CopyStateData)) and the collected
1104  * data is just leaked until CurrentMemoryContext is reset.
1105  *
1106  * Note that additional checking, such as whether column names listed in FORCE
1107  * QUOTE actually exist, has to be applied later. This just checks for
1108  * self-consistency of the options list.
1109  */
1110 void
1111 ProcessCopyOptions(ParseState *pstate,
1112  CopyState cstate,
1113  bool is_from,
1114  List *options)
1115 {
1116  bool format_specified = false;
1117  ListCell *option;
1118 
1119  /* Support external use for option sanity checking */
1120  if (cstate == NULL)
1121  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1122 
1123  cstate->is_copy_from = is_from;
1124 
1125  cstate->file_encoding = -1;
1126 
1127  /* Extract options from the statement node tree */
1128  foreach(option, options)
1129  {
1130  DefElem *defel = lfirst_node(DefElem, option);
1131 
1132  if (strcmp(defel->defname, "format") == 0)
1133  {
1134  char *fmt = defGetString(defel);
1135 
1136  if (format_specified)
1137  ereport(ERROR,
1138  (errcode(ERRCODE_SYNTAX_ERROR),
1139  errmsg("conflicting or redundant options"),
1140  parser_errposition(pstate, defel->location)));
1141  format_specified = true;
1142  if (strcmp(fmt, "text") == 0)
1143  /* default format */ ;
1144  else if (strcmp(fmt, "csv") == 0)
1145  cstate->csv_mode = true;
1146  else if (strcmp(fmt, "binary") == 0)
1147  cstate->binary = true;
1148  else
1149  ereport(ERROR,
1150  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1151  errmsg("COPY format \"%s\" not recognized", fmt),
1152  parser_errposition(pstate, defel->location)));
1153  }
1154  else if (strcmp(defel->defname, "freeze") == 0)
1155  {
1156  if (cstate->freeze)
1157  ereport(ERROR,
1158  (errcode(ERRCODE_SYNTAX_ERROR),
1159  errmsg("conflicting or redundant options"),
1160  parser_errposition(pstate, defel->location)));
1161  cstate->freeze = defGetBoolean(defel);
1162  }
1163  else if (strcmp(defel->defname, "delimiter") == 0)
1164  {
1165  if (cstate->delim)
1166  ereport(ERROR,
1167  (errcode(ERRCODE_SYNTAX_ERROR),
1168  errmsg("conflicting or redundant options"),
1169  parser_errposition(pstate, defel->location)));
1170  cstate->delim = defGetString(defel);
1171  }
1172  else if (strcmp(defel->defname, "null") == 0)
1173  {
1174  if (cstate->null_print)
1175  ereport(ERROR,
1176  (errcode(ERRCODE_SYNTAX_ERROR),
1177  errmsg("conflicting or redundant options"),
1178  parser_errposition(pstate, defel->location)));
1179  cstate->null_print = defGetString(defel);
1180  }
1181  else if (strcmp(defel->defname, "header") == 0)
1182  {
1183  if (cstate->header_line)
1184  ereport(ERROR,
1185  (errcode(ERRCODE_SYNTAX_ERROR),
1186  errmsg("conflicting or redundant options"),
1187  parser_errposition(pstate, defel->location)));
1188  cstate->header_line = defGetBoolean(defel);
1189  }
1190  else if (strcmp(defel->defname, "quote") == 0)
1191  {
1192  if (cstate->quote)
1193  ereport(ERROR,
1194  (errcode(ERRCODE_SYNTAX_ERROR),
1195  errmsg("conflicting or redundant options"),
1196  parser_errposition(pstate, defel->location)));
1197  cstate->quote = defGetString(defel);
1198  }
1199  else if (strcmp(defel->defname, "escape") == 0)
1200  {
1201  if (cstate->escape)
1202  ereport(ERROR,
1203  (errcode(ERRCODE_SYNTAX_ERROR),
1204  errmsg("conflicting or redundant options"),
1205  parser_errposition(pstate, defel->location)));
1206  cstate->escape = defGetString(defel);
1207  }
1208  else if (strcmp(defel->defname, "force_quote") == 0)
1209  {
1210  if (cstate->force_quote || cstate->force_quote_all)
1211  ereport(ERROR,
1212  (errcode(ERRCODE_SYNTAX_ERROR),
1213  errmsg("conflicting or redundant options"),
1214  parser_errposition(pstate, defel->location)));
1215  if (defel->arg && IsA(defel->arg, A_Star))
1216  cstate->force_quote_all = true;
1217  else if (defel->arg && IsA(defel->arg, List))
1218  cstate->force_quote = castNode(List, defel->arg);
1219  else
1220  ereport(ERROR,
1221  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1222  errmsg("argument to option \"%s\" must be a list of column names",
1223  defel->defname),
1224  parser_errposition(pstate, defel->location)));
1225  }
1226  else if (strcmp(defel->defname, "force_not_null") == 0)
1227  {
1228  if (cstate->force_notnull)
1229  ereport(ERROR,
1230  (errcode(ERRCODE_SYNTAX_ERROR),
1231  errmsg("conflicting or redundant options"),
1232  parser_errposition(pstate, defel->location)));
1233  if (defel->arg && IsA(defel->arg, List))
1234  cstate->force_notnull = castNode(List, defel->arg);
1235  else
1236  ereport(ERROR,
1237  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1238  errmsg("argument to option \"%s\" must be a list of column names",
1239  defel->defname),
1240  parser_errposition(pstate, defel->location)));
1241  }
1242  else if (strcmp(defel->defname, "force_null") == 0)
1243  {
1244  if (cstate->force_null)
1245  ereport(ERROR,
1246  (errcode(ERRCODE_SYNTAX_ERROR),
1247  errmsg("conflicting or redundant options")));
1248  if (defel->arg && IsA(defel->arg, List))
1249  cstate->force_null = castNode(List, defel->arg);
1250  else
1251  ereport(ERROR,
1252  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1253  errmsg("argument to option \"%s\" must be a list of column names",
1254  defel->defname),
1255  parser_errposition(pstate, defel->location)));
1256  }
1257  else if (strcmp(defel->defname, "convert_selectively") == 0)
1258  {
1259  /*
1260  * Undocumented, not-accessible-from-SQL option: convert only the
1261  * named columns to binary form, storing the rest as NULLs. It's
1262  * allowed for the column list to be NIL.
1263  */
1264  if (cstate->convert_selectively)
1265  ereport(ERROR,
1266  (errcode(ERRCODE_SYNTAX_ERROR),
1267  errmsg("conflicting or redundant options"),
1268  parser_errposition(pstate, defel->location)));
1269  cstate->convert_selectively = true;
1270  if (defel->arg == NULL || IsA(defel->arg, List))
1271  cstate->convert_select = castNode(List, defel->arg);
1272  else
1273  ereport(ERROR,
1274  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1275  errmsg("argument to option \"%s\" must be a list of column names",
1276  defel->defname),
1277  parser_errposition(pstate, defel->location)));
1278  }
1279  else if (strcmp(defel->defname, "encoding") == 0)
1280  {
1281  if (cstate->file_encoding >= 0)
1282  ereport(ERROR,
1283  (errcode(ERRCODE_SYNTAX_ERROR),
1284  errmsg("conflicting or redundant options"),
1285  parser_errposition(pstate, defel->location)));
1286  cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
1287  if (cstate->file_encoding < 0)
1288  ereport(ERROR,
1289  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1290  errmsg("argument to option \"%s\" must be a valid encoding name",
1291  defel->defname),
1292  parser_errposition(pstate, defel->location)));
1293  }
1294  else
1295  ereport(ERROR,
1296  (errcode(ERRCODE_SYNTAX_ERROR),
1297  errmsg("option \"%s\" not recognized",
1298  defel->defname),
1299  parser_errposition(pstate, defel->location)));
1300  }
1301 
1302  /*
1303  * Check for incompatible options (must do these two before inserting
1304  * defaults)
1305  */
1306  if (cstate->binary && cstate->delim)
1307  ereport(ERROR,
1308  (errcode(ERRCODE_SYNTAX_ERROR),
1309  errmsg("cannot specify DELIMITER in BINARY mode")));
1310 
1311  if (cstate->binary && cstate->null_print)
1312  ereport(ERROR,
1313  (errcode(ERRCODE_SYNTAX_ERROR),
1314  errmsg("cannot specify NULL in BINARY mode")));
1315 
1316  /* Set defaults for omitted options */
1317  if (!cstate->delim)
1318  cstate->delim = cstate->csv_mode ? "," : "\t";
1319 
1320  if (!cstate->null_print)
1321  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1322  cstate->null_print_len = strlen(cstate->null_print);
1323 
1324  if (cstate->csv_mode)
1325  {
1326  if (!cstate->quote)
1327  cstate->quote = "\"";
1328  if (!cstate->escape)
1329  cstate->escape = cstate->quote;
1330  }
1331 
1332  /* Only single-byte delimiter strings are supported. */
1333  if (strlen(cstate->delim) != 1)
1334  ereport(ERROR,
1335  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1336  errmsg("COPY delimiter must be a single one-byte character")));
1337 
1338  /* Disallow end-of-line characters */
1339  if (strchr(cstate->delim, '\r') != NULL ||
1340  strchr(cstate->delim, '\n') != NULL)
1341  ereport(ERROR,
1342  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1343  errmsg("COPY delimiter cannot be newline or carriage return")));
1344 
1345  if (strchr(cstate->null_print, '\r') != NULL ||
1346  strchr(cstate->null_print, '\n') != NULL)
1347  ereport(ERROR,
1348  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1349  errmsg("COPY null representation cannot use newline or carriage return")));
1350 
1351  /*
1352  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1353  * backslash because it would be ambiguous. We can't allow the other
1354  * cases because data characters matching the delimiter must be
1355  * backslashed, and certain backslash combinations are interpreted
1356  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1357  * more than strictly necessary, but seems best for consistency and
1358  * future-proofing. Likewise we disallow all digits though only octal
1359  * digits are actually dangerous.
1360  */
1361  if (!cstate->csv_mode &&
1362  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1363  cstate->delim[0]) != NULL)
1364  ereport(ERROR,
1365  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1366  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1367 
1368  /* Check header */
1369  if (!cstate->csv_mode && cstate->header_line)
1370  ereport(ERROR,
1371  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1372  errmsg("COPY HEADER available only in CSV mode")));
1373 
1374  /* Check quote */
1375  if (!cstate->csv_mode && cstate->quote != NULL)
1376  ereport(ERROR,
1377  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1378  errmsg("COPY quote available only in CSV mode")));
1379 
1380  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1381  ereport(ERROR,
1382  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1383  errmsg("COPY quote must be a single one-byte character")));
1384 
1385  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1386  ereport(ERROR,
1387  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1388  errmsg("COPY delimiter and quote must be different")));
1389 
1390  /* Check escape */
1391  if (!cstate->csv_mode && cstate->escape != NULL)
1392  ereport(ERROR,
1393  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1394  errmsg("COPY escape available only in CSV mode")));
1395 
1396  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1397  ereport(ERROR,
1398  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1399  errmsg("COPY escape must be a single one-byte character")));
1400 
1401  /* Check force_quote */
1402  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1403  ereport(ERROR,
1404  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1405  errmsg("COPY force quote available only in CSV mode")));
1406  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1407  ereport(ERROR,
1408  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1409  errmsg("COPY force quote only available using COPY TO")));
1410 
1411  /* Check force_notnull */
1412  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1413  ereport(ERROR,
1414  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1415  errmsg("COPY force not null available only in CSV mode")));
1416  if (cstate->force_notnull != NIL && !is_from)
1417  ereport(ERROR,
1418  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1419  errmsg("COPY force not null only available using COPY FROM")));
1420 
1421  /* Check force_null */
1422  if (!cstate->csv_mode && cstate->force_null != NIL)
1423  ereport(ERROR,
1424  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1425  errmsg("COPY force null available only in CSV mode")));
1426 
1427  if (cstate->force_null != NIL && !is_from)
1428  ereport(ERROR,
1429  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1430  errmsg("COPY force null only available using COPY FROM")));
1431 
1432  /* Don't allow the delimiter to appear in the null string. */
1433  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1434  ereport(ERROR,
1435  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1436  errmsg("COPY delimiter must not appear in the NULL specification")));
1437 
1438  /* Don't allow the CSV quote char to appear in the null string. */
1439  if (cstate->csv_mode &&
1440  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1441  ereport(ERROR,
1442  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1443  errmsg("CSV quote character must not appear in the NULL specification")));
1444 }
1445 
1446 /*
1447  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1448  *
1449  * Iff <binary>, unload or reload in the binary format, as opposed to the
1450  * more wasteful but more robust and portable text format.
1451  *
1452  * Iff <oids>, unload or reload the format that includes OID information.
1453  * On input, we accept OIDs whether or not the table has an OID column,
1454  * but silently drop them if it does not. On output, we report an error
1455  * if the user asks for OIDs in a table that has none (not providing an
1456  * OID column might seem friendlier, but could seriously confuse programs).
1457  *
1458  * If in the text format, delimit columns with delimiter <delim> and print
1459  * NULL values as <null_print>.
1460  */
1461 static CopyState
1462 BeginCopy(ParseState *pstate,
1463  bool is_from,
1464  Relation rel,
1465  RawStmt *raw_query,
1466  Oid queryRelId,
1467  List *attnamelist,
1468  List *options)
1469 {
1470  CopyState cstate;
1471  TupleDesc tupDesc;
1472  int num_phys_attrs;
1473  MemoryContext oldcontext;
1474 
1475  /* Allocate workspace and zero all fields */
1476  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1477 
1478  /*
1479  * We allocate everything used by a cstate in a new memory context. This
1480  * avoids memory leaks during repeated use of COPY in a query.
1481  */
1482  cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
1483  "COPY",
1484  ALLOCSET_DEFAULT_SIZES);
1485 
1486  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1487 
1488  /* Extract options from the statement node tree */
1489  ProcessCopyOptions(pstate, cstate, is_from, options);
1490 
1491  /* Process the source/target relation or query */
1492  if (rel)
1493  {
1494  Assert(!raw_query);
1495 
1496  cstate->rel = rel;
1497 
1498  tupDesc = RelationGetDescr(cstate->rel);
1499  }
1500  else
1501  {
1502  List *rewritten;
1503  Query *query;
1504  PlannedStmt *plan;
1505  DestReceiver *dest;
1506 
1507  Assert(!is_from);
1508  cstate->rel = NULL;
1509 
1510  /*
1511  * Run parse analysis and rewrite. Note this also acquires sufficient
1512  * locks on the source table(s).
1513  *
1514  * Because the parser and planner tend to scribble on their input, we
1515  * make a preliminary copy of the source querytree. This prevents
1516  * problems in the case that the COPY is in a portal or plpgsql
1517  * function and is executed repeatedly. (See also the same hack in
1518  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1519  */
1520  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1521  pstate->p_sourcetext, NULL, 0,
1522  NULL);
1523 
1524  /* check that we got back something we can work with */
1525  if (rewritten == NIL)
1526  {
1527  ereport(ERROR,
1528  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1529  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1530  }
1531  else if (list_length(rewritten) > 1)
1532  {
1533  ListCell *lc;
1534 
1535  /* examine queries to determine which error message to issue */
1536  foreach(lc, rewritten)
1537  {
1538  Query *q = lfirst_node(Query, lc);
1539 
1540  if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
1541  ereport(ERROR,
1542  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1543  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1544  if (q->querySource == QSRC_NON_INSTEAD_RULE)
1545  ereport(ERROR,
1546  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1547  errmsg("DO ALSO rules are not supported for the COPY")));
1548  }
1549 
1550  ereport(ERROR,
1551  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1552  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1553  }
1554 
1555  query = linitial_node(Query, rewritten);
1556 
1557  /* The grammar allows SELECT INTO, but we don't support that */
1558  if (query->utilityStmt != NULL &&
1559  IsA(query->utilityStmt, CreateTableAsStmt))
1560  ereport(ERROR,
1561  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1562  errmsg("COPY (SELECT INTO) is not supported")));
1563 
1564  Assert(query->utilityStmt == NULL);
1565 
1566  /*
1567  * Similarly the grammar doesn't enforce the presence of a RETURNING
1568  * clause, but this is required here.
1569  */
1570  if (query->commandType != CMD_SELECT &&
1571  query->returningList == NIL)
1572  {
1573  Assert(query->commandType == CMD_INSERT ||
1574  query->commandType == CMD_UPDATE ||
1575  query->commandType == CMD_DELETE);
1576 
1577  ereport(ERROR,
1578  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1579  errmsg("COPY query must have a RETURNING clause")));
1580  }
1581 
1582  /* plan the query */
1583  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1584 
1585  /*
1586  * With row level security and a user using "COPY relation TO", we
1587  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1588  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1589  * in any RLS clauses.
1590  *
1591  * When this happens, we are passed in the relid of the originally
1592  * found relation (which we have locked). As the planner will look up
1593  * the relation again, we double-check here to make sure it found the
1594  * same one that we have locked.
1595  */
1596  if (queryRelId != InvalidOid)
1597  {
1598  /*
1599  * Note that with RLS involved there may be multiple relations,
1600  * and while the one we need is almost certainly first, we don't
1601  * make any guarantees of that in the planner, so check the whole
1602  * list and make sure we find the original relation.
1603  */
1604  if (!list_member_oid(plan->relationOids, queryRelId))
1605  ereport(ERROR,
1606  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1607  errmsg("relation referenced by COPY statement has changed")));
1608  }
1609 
1610  /*
1611  * Use a snapshot with an updated command ID to ensure this query sees
1612  * results of any previously executed queries.
1613  */
1614  PushCopiedSnapshot(GetActiveSnapshot());
1615  UpdateActiveSnapshotCommandId();
1616 
1617  /* Create dest receiver for COPY OUT */
1618  dest = CreateDestReceiver(DestCopyOut);
1619  ((DR_copy *) dest)->cstate = cstate;
1620 
1621  /* Create a QueryDesc requesting no output */
1622  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1623  GetActiveSnapshot(),
1624  InvalidSnapshot,
1625  dest, NULL, NULL, 0);
1626 
1627  /*
1628  * Call ExecutorStart to prepare the plan for execution.
1629  *
1630  * ExecutorStart computes a result tupdesc for us
1631  */
1632  ExecutorStart(cstate->queryDesc, 0);
1633 
1634  tupDesc = cstate->queryDesc->tupDesc;
1635  }
1636 
1637  /* Generate or convert list of attributes to process */
1638  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1639 
1640  num_phys_attrs = tupDesc->natts;
1641 
1642  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1643  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1644  if (cstate->force_quote_all)
1645  {
1646  int i;
1647 
1648  for (i = 0; i < num_phys_attrs; i++)
1649  cstate->force_quote_flags[i] = true;
1650  }
1651  else if (cstate->force_quote)
1652  {
1653  List *attnums;
1654  ListCell *cur;
1655 
1656  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1657 
1658  foreach(cur, attnums)
1659  {
1660  int attnum = lfirst_int(cur);
1661  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1662 
1663  if (!list_member_int(cstate->attnumlist, attnum))
1664  ereport(ERROR,
1665  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1666  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1667  NameStr(attr->attname))));
1668  cstate->force_quote_flags[attnum - 1] = true;
1669  }
1670  }
1671 
1672  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1673  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1674  if (cstate->force_notnull)
1675  {
1676  List *attnums;
1677  ListCell *cur;
1678 
1679  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1680 
1681  foreach(cur, attnums)
1682  {
1683  int attnum = lfirst_int(cur);
1684  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1685 
1686  if (!list_member_int(cstate->attnumlist, attnum))
1687  ereport(ERROR,
1688  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1689  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1690  NameStr(attr->attname))));
1691  cstate->force_notnull_flags[attnum - 1] = true;
1692  }
1693  }
1694 
1695  /* Convert FORCE_NULL name list to per-column flags, check validity */
1696  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1697  if (cstate->force_null)
1698  {
1699  List *attnums;
1700  ListCell *cur;
1701 
1702  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1703 
1704  foreach(cur, attnums)
1705  {
1706  int attnum = lfirst_int(cur);
1707  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1708 
1709  if (!list_member_int(cstate->attnumlist, attnum))
1710  ereport(ERROR,
1711  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1712  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1713  NameStr(attr->attname))));
1714  cstate->force_null_flags[attnum - 1] = true;
1715  }
1716  }
1717 
1718  /* Convert convert_selectively name list to per-column flags */
1719  if (cstate->convert_selectively)
1720  {
1721  List *attnums;
1722  ListCell *cur;
1723 
1724  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1725 
1726  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1727 
1728  foreach(cur, attnums)
1729  {
1730  int attnum = lfirst_int(cur);
1731  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1732 
1733  if (!list_member_int(cstate->attnumlist, attnum))
1734  ereport(ERROR,
1735  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1736  errmsg_internal("selected column \"%s\" not referenced by COPY",
1737  NameStr(attr->attname))));
1738  cstate->convert_select_flags[attnum - 1] = true;
1739  }
1740  }
1741 
1742  /* Use client encoding when ENCODING option is not specified. */
1743  if (cstate->file_encoding < 0)
1744  cstate->file_encoding = pg_get_client_encoding();
1745 
1746  /*
1747  * Set up encoding conversion info. Even if the file and server encodings
1748  * are the same, we must apply pg_any_to_server() to validate data in
1749  * multibyte encodings.
1750  */
1751  cstate->need_transcoding =
1752  (cstate->file_encoding != GetDatabaseEncoding() ||
1753  pg_database_encoding_max_length() > 1);
1754  /* See Multibyte encoding comment above */
1755  cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
1756 
1757  cstate->copy_dest = COPY_FILE; /* default */
1758 
1759  MemoryContextSwitchTo(oldcontext);
1760 
1761  return cstate;
1762 }
1763 
1764 /*
1765  * Closes the pipe to an external program, checking the pclose() return code.
1766  */
1767 static void
1768 ClosePipeToProgram(CopyState cstate)
1769 {
1770  int pclose_rc;
1771 
1772  Assert(cstate->is_program);
1773 
1774  pclose_rc = ClosePipeStream(cstate->copy_file);
1775  if (pclose_rc == -1)
1776  ereport(ERROR,
1777  (errcode_for_file_access(),
1778  errmsg("could not close pipe to external command: %m")));
1779  else if (pclose_rc != 0)
1780  {
1781  /*
1782  * If we ended a COPY FROM PROGRAM before reaching EOF, then it's
1783  * expectable for the called program to fail with SIGPIPE, and we
1784  * should not report that as an error. Otherwise, SIGPIPE indicates a
1785  * problem.
1786  */
1787  if (cstate->is_copy_from && !cstate->reached_eof &&
1788  wait_result_is_signal(pclose_rc, SIGPIPE))
1789  return;
1790 
1791  ereport(ERROR,
1792  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1793  errmsg("program \"%s\" failed",
1794  cstate->filename),
1795  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1796  }
1797 }
1798 
1799 /*
1800  * Release resources allocated in a cstate for COPY TO/FROM.
1801  */
1802 static void
1803 EndCopy(CopyState cstate)
1804 {
1805  if (cstate->is_program)
1806  {
1807  ClosePipeToProgram(cstate);
1808  }
1809  else
1810  {
1811  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1812  ereport(ERROR,
1813  (errcode_for_file_access(),
1814  errmsg("could not close file \"%s\": %m",
1815  cstate->filename)));
1816  }
1817 
1818  MemoryContextDelete(cstate->copycontext);
1819  pfree(cstate);
1820 }
1821 
1822 /*
1823  * Setup CopyState to read tuples from a table or a query for COPY TO.
1824  */
1825 static CopyState
1826 BeginCopyTo(ParseState *pstate,
1827  Relation rel,
1828  RawStmt *query,
1829  Oid queryRelId,
1830  const char *filename,
1831  bool is_program,
1832  List *attnamelist,
1833  List *options)
1834 {
1835  CopyState cstate;
1836  bool pipe = (filename == NULL);
1837  MemoryContext oldcontext;
1838 
1839  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1840  {
1841  if (rel->rd_rel->relkind == RELKIND_VIEW)
1842  ereport(ERROR,
1843  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1844  errmsg("cannot copy from view \"%s\"",
1845  RelationGetRelationName(rel)),
1846  errhint("Try the COPY (SELECT ...) TO variant.")));
1847  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1848  ereport(ERROR,
1849  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1850  errmsg("cannot copy from materialized view \"%s\"",
1851  RelationGetRelationName(rel)),
1852  errhint("Try the COPY (SELECT ...) TO variant.")));
1853  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1854  ereport(ERROR,
1855  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1856  errmsg("cannot copy from foreign table \"%s\"",
1857  RelationGetRelationName(rel)),
1858  errhint("Try the COPY (SELECT ...) TO variant.")));
1859  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1860  ereport(ERROR,
1861  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1862  errmsg("cannot copy from sequence \"%s\"",
1863  RelationGetRelationName(rel))));
1864  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1865  ereport(ERROR,
1866  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1867  errmsg("cannot copy from partitioned table \"%s\"",
1868  RelationGetRelationName(rel)),
1869  errhint("Try the COPY (SELECT ...) TO variant.")));
1870  else
1871  ereport(ERROR,
1872  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1873  errmsg("cannot copy from non-table relation \"%s\"",
1874  RelationGetRelationName(rel))));
1875  }
1876 
1877  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1878  options);
1879  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1880 
1881  if (pipe)
1882  {
1883  Assert(!is_program); /* the grammar does not allow this */
1884  if (whereToSendOutput != DestRemote)
1885  cstate->copy_file = stdout;
1886  }
1887  else
1888  {
1889  cstate->filename = pstrdup(filename);
1890  cstate->is_program = is_program;
1891 
1892  if (is_program)
1893  {
1894  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1895  if (cstate->copy_file == NULL)
1896  ereport(ERROR,
1897  (errcode_for_file_access(),
1898  errmsg("could not execute command \"%s\": %m",
1899  cstate->filename)));
1900  }
1901  else
1902  {
1903  mode_t oumask; /* Pre-existing umask value */
1904  struct stat st;
1905 
1906  /*
1907  * Prevent write to relative path ... too easy to shoot oneself in
1908  * the foot by overwriting a database file ...
1909  */
1910  if (!is_absolute_path(filename))
1911  ereport(ERROR,
1912  (errcode(ERRCODE_INVALID_NAME),
1913  errmsg("relative path not allowed for COPY to file")));
1914 
1915  oumask = umask(S_IWGRP | S_IWOTH);
1916  PG_TRY();
1917  {
1918  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1919  }
1920  PG_FINALLY();
1921  {
1922  umask(oumask);
1923  }
1924  PG_END_TRY();
1925  if (cstate->copy_file == NULL)
1926  {
1927  /* copy errno because ereport subfunctions might change it */
1928  int save_errno = errno;
1929 
1930  ereport(ERROR,
1931  (errcode_for_file_access(),
1932  errmsg("could not open file \"%s\" for writing: %m",
1933  cstate->filename),
1934  (save_errno == ENOENT || save_errno == EACCES) ?
1935  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1936  "You may want a client-side facility such as psql's \\copy.") : 0));
1937  }
1938 
1939  if (fstat(fileno(cstate->copy_file), &st))
1940  ereport(ERROR,
1941  (errcode_for_file_access(),
1942  errmsg("could not stat file \"%s\": %m",
1943  cstate->filename)));
1944 
1945  if (S_ISDIR(st.st_mode))
1946  ereport(ERROR,
1947  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1948  errmsg("\"%s\" is a directory", cstate->filename)));
1949  }
1950  }
1951 
1952  MemoryContextSwitchTo(oldcontext);
1953 
1954  return cstate;
1955 }
1956 
1957 /*
1958  * This intermediate routine exists mainly to localize the effects of setjmp
1959  * so we don't need to plaster a lot of variables with "volatile".
1960  */
1961 static uint64
1962 DoCopyTo(CopyState cstate)
1963 {
1964  bool pipe = (cstate->filename == NULL);
1965  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
1966  uint64 processed;
1967 
1968  PG_TRY();
1969  {
1970  if (fe_copy)
1971  SendCopyBegin(cstate);
1972 
1973  processed = CopyTo(cstate);
1974 
1975  if (fe_copy)
1976  SendCopyEnd(cstate);
1977  }
1978  PG_CATCH();
1979  {
1980  /*
1981  * Make sure we turn off old-style COPY OUT mode upon error. It is
1982  * okay to do this in all cases, since it does nothing if the mode is
1983  * not on.
1984  */
1985  pq_endcopyout(true);
1986  PG_RE_THROW();
1987  }
1988  PG_END_TRY();
1989 
1990  return processed;
1991 }
1992 
1993 /*
1994  * Clean up storage and release resources for COPY TO.
1995  */
1996 static void
1997 EndCopyTo(CopyState cstate)
1998 {
1999  if (cstate->queryDesc != NULL)
2000  {
2001  /* Close down the query and free resources. */
2002  ExecutorFinish(cstate->queryDesc);
2003  ExecutorEnd(cstate->queryDesc);
2004  FreeQueryDesc(cstate->queryDesc);
2005  PopActiveSnapshot();
2006  }
2007 
2008  /* Clean up storage */
2009  EndCopy(cstate);
2010 }
2011 
2012 /*
2013  * Copy from relation or query TO file.
2014  */
2015 static uint64
2016 CopyTo(CopyState cstate)
2017 {
2018  TupleDesc tupDesc;
2019  int num_phys_attrs;
2020  ListCell *cur;
2021  uint64 processed;
2022 
2023  if (cstate->rel)
2024  tupDesc = RelationGetDescr(cstate->rel);
2025  else
2026  tupDesc = cstate->queryDesc->tupDesc;
2027  num_phys_attrs = tupDesc->natts;
2028  cstate->null_print_client = cstate->null_print; /* default */
2029 
2030  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
2031  cstate->fe_msgbuf = makeStringInfo();
2032 
2033  /* Get info about the columns we need to process. */
2034  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2035  foreach(cur, cstate->attnumlist)
2036  {
2037  int attnum = lfirst_int(cur);
2038  Oid out_func_oid;
2039  bool isvarlena;
2040  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
2041 
2042  if (cstate->binary)
2043  getTypeBinaryOutputInfo(attr->atttypid,
2044  &out_func_oid,
2045  &isvarlena);
2046  else
2047  getTypeOutputInfo(attr->atttypid,
2048  &out_func_oid,
2049  &isvarlena);
2050  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
2051  }
2052 
2053  /*
2054  * Create a temporary memory context that we can reset once per row to
2055  * recover palloc'd memory. This avoids any problems with leaks inside
2056  * datatype output routines, and should be faster than retail pfree's
2057  * anyway. (We don't need a whole econtext as CopyFrom does.)
2058  */
2059  cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
2060  "COPY TO",
2061  ALLOCSET_DEFAULT_SIZES);
2062 
2063  if (cstate->binary)
2064  {
2065  /* Generate header for a binary copy */
2066  int32 tmp;
2067 
2068  /* Signature */
2069  CopySendData(cstate, BinarySignature, 11);
2070  /* Flags field */
2071  tmp = 0;
2072  CopySendInt32(cstate, tmp);
2073  /* No header extension */
2074  tmp = 0;
2075  CopySendInt32(cstate, tmp);
2076  }
2077  else
2078  {
2079  /*
2080  * For non-binary copy, we need to convert null_print to file
2081  * encoding, because it will be sent directly with CopySendString.
2082  */
2083  if (cstate->need_transcoding)
2084  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2085  cstate->null_print_len,
2086  cstate->file_encoding);
2087 
2088  /* if a header has been requested send the line */
2089  if (cstate->header_line)
2090  {
2091  bool hdr_delim = false;
2092 
2093  foreach(cur, cstate->attnumlist)
2094  {
2095  int attnum = lfirst_int(cur);
2096  char *colname;
2097 
2098  if (hdr_delim)
2099  CopySendChar(cstate, cstate->delim[0]);
2100  hdr_delim = true;
2101 
2102  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
2103 
2104  CopyAttributeOutCSV(cstate, colname, false,
2105  list_length(cstate->attnumlist) == 1);
2106  }
2107 
2108  CopySendEndOfRow(cstate);
2109  }
2110  }
2111 
2112  if (cstate->rel)
2113  {
2114  TupleTableSlot *slot;
2115  TableScanDesc scandesc;
2116 
2117  scandesc = table_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2118  slot = table_slot_create(cstate->rel, NULL);
2119 
2120  processed = 0;
2121  while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot))
2122  {
2123  CHECK_FOR_INTERRUPTS();
2124 
2125  /* Deconstruct the tuple ... */
2126  slot_getallattrs(slot);
2127 
2128  /* Format and send the data */
2129  CopyOneRowTo(cstate, slot);
2130  processed++;
2131  }
2132 
2133  ExecDropSingleTupleTableSlot(slot);
2134  table_endscan(scandesc);
2135  }
2136  else
2137  {
2138  /* run the plan --- the dest receiver will send tuples */
2139  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2140  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2141  }
2142 
2143  if (cstate->binary)
2144  {
2145  /* Generate trailer for a binary copy */
2146  CopySendInt16(cstate, -1);
2147  /* Need to flush out the trailer */
2148  CopySendEndOfRow(cstate);
2149  }
2150 
2151  MemoryContextDelete(cstate->rowcontext);
2152 
2153  return processed;
2154 }
2155 
2156 /*
2157  * Emit one row during CopyTo().
2158  */
2159 static void
2160 CopyOneRowTo(CopyState cstate, TupleTableSlot *slot)
2161 {
2162  bool need_delim = false;
2163  FmgrInfo *out_functions = cstate->out_functions;
2164  MemoryContext oldcontext;
2165  ListCell *cur;
2166  char *string;
2167 
2168  MemoryContextReset(cstate->rowcontext);
2169  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2170 
2171  if (cstate->binary)
2172  {
2173  /* Binary per-tuple header */
2174  CopySendInt16(cstate, list_length(cstate->attnumlist));
2175  }
2176 
2177  /* Make sure the tuple is fully deconstructed */
2178  slot_getallattrs(slot);
2179 
2180  foreach(cur, cstate->attnumlist)
2181  {
2182  int attnum = lfirst_int(cur);
2183  Datum value = slot->tts_values[attnum - 1];
2184  bool isnull = slot->tts_isnull[attnum - 1];
2185 
2186  if (!cstate->binary)
2187  {
2188  if (need_delim)
2189  CopySendChar(cstate, cstate->delim[0]);
2190  need_delim = true;
2191  }
2192 
2193  if (isnull)
2194  {
2195  if (!cstate->binary)
2196  CopySendString(cstate, cstate->null_print_client);
2197  else
2198  CopySendInt32(cstate, -1);
2199  }
2200  else
2201  {
2202  if (!cstate->binary)
2203  {
2204  string = OutputFunctionCall(&out_functions[attnum - 1],
2205  value);
2206  if (cstate->csv_mode)
2207  CopyAttributeOutCSV(cstate, string,
2208  cstate->force_quote_flags[attnum - 1],
2209  list_length(cstate->attnumlist) == 1);
2210  else
2211  CopyAttributeOutText(cstate, string);
2212  }
2213  else
2214  {
2215  bytea *outputbytes;
2216 
2217  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2218  value);
2219  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2220  CopySendData(cstate, VARDATA(outputbytes),
2221  VARSIZE(outputbytes) - VARHDRSZ);
2222  }
2223  }
2224  }
2225 
2226  CopySendEndOfRow(cstate);
2227 
2228  MemoryContextSwitchTo(oldcontext);
2229 }
2230 
2231 
2232 /*
2233  * error context callback for COPY FROM
2234  *
2235  * The argument for the error context must be CopyState.
2236  */
2237 void
2238 CopyFromErrorCallback(void *arg)
2239 {
2240  CopyState cstate = (CopyState) arg;
2241  char curlineno_str[32];
2242 
2243  snprintf(curlineno_str, sizeof(curlineno_str), UINT64_FORMAT,
2244  cstate->cur_lineno);
2245 
2246  if (cstate->binary)
2247  {
2248  /* can't usefully display the data */
2249  if (cstate->cur_attname)
2250  errcontext("COPY %s, line %s, column %s",
2251  cstate->cur_relname, curlineno_str,
2252  cstate->cur_attname);
2253  else
2254  errcontext("COPY %s, line %s",
2255  cstate->cur_relname, curlineno_str);
2256  }
2257  else
2258  {
2259  if (cstate->cur_attname && cstate->cur_attval)
2260  {
2261  /* error is relevant to a particular column */
2262  char *attval;
2263 
2264  attval = limit_printout_length(cstate->cur_attval);
2265  errcontext("COPY %s, line %s, column %s: \"%s\"",
2266  cstate->cur_relname, curlineno_str,
2267  cstate->cur_attname, attval);
2268  pfree(attval);
2269  }
2270  else if (cstate->cur_attname)
2271  {
2272  /* error is relevant to a particular column, value is NULL */
2273  errcontext("COPY %s, line %s, column %s: null input",
2274  cstate->cur_relname, curlineno_str,
2275  cstate->cur_attname);
2276  }
2277  else
2278  {
2279  /*
2280  * Error is relevant to a particular line.
2281  *
2282  * If line_buf still contains the correct line, and it's already
2283  * transcoded, print it. If it's still in a foreign encoding, it's
2284  * quite likely that the error is precisely a failure to do
2285  * encoding conversion (ie, bad data). We dare not try to convert
2286  * it, and at present there's no way to regurgitate it without
2287  * conversion. So we have to punt and just report the line number.
2288  */
2289  if (cstate->line_buf_valid &&
2290  (cstate->line_buf_converted || !cstate->need_transcoding))
2291  {
2292  char *lineval;
2293 
2294  lineval = limit_printout_length(cstate->line_buf.data);
2295  errcontext("COPY %s, line %s: \"%s\"",
2296  cstate->cur_relname, curlineno_str, lineval);
2297  pfree(lineval);
2298  }
2299  else
2300  {
2301  errcontext("COPY %s, line %s",
2302  cstate->cur_relname, curlineno_str);
2303  }
2304  }
2305  }
2306 }
2307 
2308 /*
2309  * Make sure we don't print an unreasonable amount of COPY data in a message.
2310  *
2311  * It would seem a lot easier to just use the sprintf "precision" limit to
2312  * truncate the string. However, some versions of glibc have a bug/misfeature
2313  * that vsnprintf will always fail (return -1) if it is asked to truncate
2314  * a string that contains invalid byte sequences for the current encoding.
2315  * So, do our own truncation. We return a pstrdup'd copy of the input.
2316  */
2317 static char *
2318 limit_printout_length(const char *str)
2319 {
2320 #define MAX_COPY_DATA_DISPLAY 100
2321 
2322  int slen = strlen(str);
2323  int len;
2324  char *res;
2325 
2326  /* Fast path if definitely okay */
2327  if (slen <= MAX_COPY_DATA_DISPLAY)
2328  return pstrdup(str);
2329 
2330  /* Apply encoding-dependent truncation */
2331  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2332 
2333  /*
2334  * Truncate, and add "..." to show we truncated the input.
2335  */
2336  res = (char *) palloc(len + 4);
2337  memcpy(res, str, len);
2338  strcpy(res + len, "...");
2339 
2340  return res;
2341 }
2342 
2343 /*
2344  * Allocate memory and initialize a new CopyMultiInsertBuffer for this
2345  * ResultRelInfo.
2346  */
2347 static CopyMultiInsertBuffer *
2348 CopyMultiInsertBufferInit(ResultRelInfo *rri)
2349 {
2350  CopyMultiInsertBuffer *buffer;
2351 
2352  buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer));
2353  memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES);
2354  buffer->resultRelInfo = rri;
2355  buffer->bistate = GetBulkInsertState();
2356  buffer->nused = 0;
2357 
2358  return buffer;
2359 }
2360 
2361 /*
2362  * Make a new buffer for this ResultRelInfo.
2363  */
2364 static inline void
2365 CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo,
2366  ResultRelInfo *rri)
2367 {
2368  CopyMultiInsertBuffer *buffer;
2369 
2370  buffer = CopyMultiInsertBufferInit(rri);
2371 
2372  /* Setup back-link so we can easily find this buffer again */
2373  rri->ri_CopyMultiInsertBuffer = buffer;
2374  /* Record that we're tracking this buffer */
2375  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2376 }
2377 
2378 /*
2379  * Initialize an already allocated CopyMultiInsertInfo.
2380  *
2381  * If rri is a non-partitioned table then a CopyMultiInsertBuffer is set up
2382  * for that table.
2383  */
2384 static void
2385 CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2386  CopyState cstate, EState *estate, CommandId mycid,
2387  int ti_options)
2388 {
2389  miinfo->multiInsertBuffers = NIL;
2390  miinfo->bufferedTuples = 0;
2391  miinfo->bufferedBytes = 0;
2392  miinfo->cstate = cstate;
2393  miinfo->estate = estate;
2394  miinfo->mycid = mycid;
2395  miinfo->ti_options = ti_options;
2396 
2397  /*
2398  * Only setup the buffer when not dealing with a partitioned table.
2399  * Buffers for partitioned tables will just be setup when we need to send
2400  * tuples their way for the first time.
2401  */
2402  if (rri->ri_RelationDesc->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
2403  CopyMultiInsertInfoSetupBuffer(miinfo, rri);
2404 }
2405 
2406 /*
2407  * Returns true if the buffers are full
2408  */
2409 static inline bool
2410 CopyMultiInsertInfoIsFull(CopyMultiInsertInfo *miinfo)
2411 {
2412  if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES ||
2413  miinfo->bufferedBytes >= MAX_BUFFERED_BYTES)
2414  return true;
2415  return false;
2416 }
2417 
2418 /*
2419  * Returns true if we have no buffered tuples
2420  */
2421 static inline bool
2422 CopyMultiInsertInfoIsEmpty(CopyMultiInsertInfo *miinfo)
2423 {
2424  return miinfo->bufferedTuples == 0;
2425 }
2426 
2427 /*
2428  * Write the tuples stored in 'buffer' out to the table.
2429  */
2430 static inline void
2431 CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
2432  CopyMultiInsertBuffer *buffer)
2433 {
2434  MemoryContext oldcontext;
2435  int i;
2436  uint64 save_cur_lineno;
2437  CopyState cstate = miinfo->cstate;
2438  EState *estate = miinfo->estate;
2439  CommandId mycid = miinfo->mycid;
2440  int ti_options = miinfo->ti_options;
2441  bool line_buf_valid = cstate->line_buf_valid;
2442  int nused = buffer->nused;
2443  ResultRelInfo *resultRelInfo = buffer->resultRelInfo;
2444  TupleTableSlot **slots = buffer->slots;
2445 
2446  /* Set es_result_relation_info to the ResultRelInfo we're flushing. */
2447  estate->es_result_relation_info = resultRelInfo;
2448 
2449  /*
2450  * Print error context information correctly, if one of the operations
2451  * below fail.
2452  */
2453  cstate->line_buf_valid = false;
2454  save_cur_lineno = cstate->cur_lineno;
2455 
2456  /*
2457  * table_multi_insert may leak memory, so switch to short-lived memory
2458  * context before calling it.
2459  */
2460  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2461  table_multi_insert(resultRelInfo->ri_RelationDesc,
2462  slots,
2463  nused,
2464  mycid,
2465  ti_options,
2466  buffer->bistate);
2467  MemoryContextSwitchTo(oldcontext);
2468 
2469  for (i = 0; i < nused; i++)
2470  {
2471  /*
2472  * If there are any indexes, update them for all the inserted tuples,
2473  * and run AFTER ROW INSERT triggers.
2474  */
2475  if (resultRelInfo->ri_NumIndices > 0)
2476  {
2477  List *recheckIndexes;
2478 
2479  cstate->cur_lineno = buffer->linenos[i];
2480  recheckIndexes =
2481  ExecInsertIndexTuples(buffer->slots[i], estate, false, NULL,
2482  NIL);
2483  ExecARInsertTriggers(estate, resultRelInfo,
2484  slots[i], recheckIndexes,
2485  cstate->transition_capture);
2486  list_free(recheckIndexes);
2487  }
2488 
2489  /*
2490  * There's no indexes, but see if we need to run AFTER ROW INSERT
2491  * triggers anyway.
2492  */
2493  else if (resultRelInfo->ri_TrigDesc != NULL &&
2494  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2495  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2496  {
2497  cstate->cur_lineno = buffer->linenos[i];
2498  ExecARInsertTriggers(estate, resultRelInfo,
2499  slots[i], NIL, cstate->transition_capture);
2500  }
2501 
2502  ExecClearTuple(slots[i]);
2503  }
2504 
2505  /* Mark that all slots are free */
2506  buffer->nused = 0;
2507 
2508  /* reset cur_lineno and line_buf_valid to what they were */
2509  cstate->line_buf_valid = line_buf_valid;
2510  cstate->cur_lineno = save_cur_lineno;
2511 }
2512 
2513 /*
2514  * Drop used slots and free member for this buffer.
2515  *
2516  * The buffer must be flushed before cleanup.
2517  */
2518 static inline void
2519 CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo,
2520  CopyMultiInsertBuffer *buffer)
2521 {
2522  int i;
2523 
2524  /* Ensure buffer was flushed */
2525  Assert(buffer->nused == 0);
2526 
2527  /* Remove back-link to ourself */
2528  buffer->resultRelInfo->ri_CopyMultiInsertBuffer = NULL;
2529 
2530  FreeBulkInsertState(buffer->bistate);
2531 
2532  /* Since we only create slots on demand, just drop the non-null ones. */
2533  for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++)
2534  ExecDropSingleTupleTableSlot(buffer->slots[i]);
2535 
2536  table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc,
2537  miinfo->ti_options);
2538 
2539  pfree(buffer);
2540 }
2541 
2542 /*
2543  * Write out all stored tuples in all buffers out to the tables.
2544  *
2545  * Once flushed we also trim the tracked buffers list down to size by removing
2546  * the buffers created earliest first.
2547  *
2548  * Callers should pass 'curr_rri' is the ResultRelInfo that's currently being
2549  * used. When cleaning up old buffers we'll never remove the one for
2550  * 'curr_rri'.
2551  */
2552 static inline void
2553 CopyMultiInsertInfoFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *curr_rri)
2554 {
2555  ListCell *lc;
2556 
2557  foreach(lc, miinfo->multiInsertBuffers)
2558  {
2559  CopyMultiInsertBuffer *buffer = (CopyMultiInsertBuffer *) lfirst(lc);
2560 
2561  CopyMultiInsertBufferFlush(miinfo, buffer);
2562  }
2563 
2564  miinfo->bufferedTuples = 0;
2565  miinfo->bufferedBytes = 0;
2566 
2567  /*
2568  * Trim the list of tracked buffers down if it exceeds the limit. Here we
2569  * remove buffers starting with the ones we created first. It seems more
2570  * likely that these older ones are less likely to be needed than ones
2571  * that were just created.
2572  */
2573  while (list_length(miinfo->multiInsertBuffers) > MAX_PARTITION_BUFFERS)
2574  {
2575  CopyMultiInsertBuffer *buffer;
2576 
2577  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2578 
2579  /*
2580  * We never want to remove the buffer that's currently being used, so
2581  * if we happen to find that then move it to the end of the list.
2582  */
2583  if (buffer->resultRelInfo == curr_rri)
2584  {
2585  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2586  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2587  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2588  }
2589 
2590  CopyMultiInsertBufferCleanup(miinfo, buffer);
2591  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2592  }
2593 }
2594 
2595 /*
2596  * Cleanup allocated buffers and free memory
2597  */
2598 static inline void
2599 CopyMultiInsertInfoCleanup(CopyMultiInsertInfo *miinfo)
2600 {
2601  ListCell *lc;
2602 
2603  foreach(lc, miinfo->multiInsertBuffers)
2604  CopyMultiInsertBufferCleanup(miinfo, lfirst(lc));
2605 
2606  list_free(miinfo->multiInsertBuffers);
2607 }
2608 
2609 /*
2610  * Get the next TupleTableSlot that the next tuple should be stored in.
2611  *
2612  * Callers must ensure that the buffer is not full.
2613  */
2614 static inline TupleTableSlot *
2615 CopyMultiInsertInfoNextFreeSlot(CopyMultiInsertInfo *miinfo,
2616  ResultRelInfo *rri)
2617 {
2618  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2619  int nused = buffer->nused;
2620 
2621  Assert(buffer != NULL);
2622  Assert(nused < MAX_BUFFERED_TUPLES);
2623 
2624  if (buffer->slots[nused] == NULL)
2625  buffer->slots[nused] = table_slot_create(rri->ri_RelationDesc, NULL);
2626  return buffer->slots[nused];
2627 }
2628 
2629 /*
2630  * Record the previously reserved TupleTableSlot that was reserved by
2631  * CopyMultiInsertInfoNextFreeSlot as being consumed.
2632  */
2633 static inline void
2634 CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2635  TupleTableSlot *slot, int tuplen, uint64 lineno)
2636 {
2637  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2638 
2639  Assert(buffer != NULL);
2640  Assert(slot == buffer->slots[buffer->nused]);
2641 
2642  /* Store the line number so we can properly report any errors later */
2643  buffer->linenos[buffer->nused] = lineno;
2644 
2645  /* Record this slot as being used */
2646  buffer->nused++;
2647 
2648  /* Update how many tuples are stored and their size */
2649  miinfo->bufferedTuples++;
2650  miinfo->bufferedBytes += tuplen;
2651 }
2652 
2653 /*
2654  * Copy FROM file to relation.
2655  */
2656 uint64
2657 CopyFrom(CopyState cstate)
2658 {
2659  ResultRelInfo *resultRelInfo;
2660  ResultRelInfo *target_resultRelInfo;
2661  ResultRelInfo *prevResultRelInfo = NULL;
2662  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2663  ModifyTableState *mtstate;
2664  ExprContext *econtext;
2665  TupleTableSlot *singleslot = NULL;
2666  MemoryContext oldcontext = CurrentMemoryContext;
2667 
2668  PartitionTupleRouting *proute = NULL;
2669  ErrorContextCallback errcallback;
2670  CommandId mycid = GetCurrentCommandId(true);
2671  int ti_options = 0; /* start with default options for insert */
2672  BulkInsertState bistate = NULL;
2673  CopyInsertMethod insertMethod;
2674  CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */
2675  uint64 processed = 0;
2676  bool has_before_insert_row_trig;
2677  bool has_instead_insert_row_trig;
2678  bool leafpart_use_multi_insert = false;
2679 
2680  Assert(cstate->rel);
2681 
2682  /*
2683  * The target must be a plain, foreign, or partitioned relation, or have
2684  * an INSTEAD OF INSERT row trigger. (Currently, such triggers are only
2685  * allowed on views, so we only hint about them in the view case.)
2686  */
2687  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2688  cstate->rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE &&
2689  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2690  !(cstate->rel->trigdesc &&
2691  cstate->rel->trigdesc->trig_insert_instead_row))
2692  {
2693  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2694  ereport(ERROR,
2695  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2696  errmsg("cannot copy to view \"%s\"",
2697  RelationGetRelationName(cstate->rel)),
2698  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2699  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2700  ereport(ERROR,
2701  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2702  errmsg("cannot copy to materialized view \"%s\"",
2703  RelationGetRelationName(cstate->rel))));
2704  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2705  ereport(ERROR,
2706  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2707  errmsg("cannot copy to sequence \"%s\"",
2708  RelationGetRelationName(cstate->rel))));
2709  else
2710  ereport(ERROR,
2711  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2712  errmsg("cannot copy to non-table relation \"%s\"",
2713  RelationGetRelationName(cstate->rel))));
2714  }
2715 
2716  /*----------
2717  * Check to see if we can avoid writing WAL
2718  *
2719  * If archive logging/streaming is not enabled *and* either
2720  * - table was created in same transaction as this COPY
2721  * - data is being written to relfilenode created in this transaction
2722  * then we can skip writing WAL. It's safe because if the transaction
2723  * doesn't commit, we'll discard the table (or the new relfilenode file).
2724  * If it does commit, we'll have done the table_finish_bulk_insert() at
2725  * the bottom of this routine first.
2726  *
2727  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2728  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2729  * can be cleared before the end of the transaction. The exact case is
2730  * when a relation sets a new relfilenode twice in same transaction, yet
2731  * the second one fails in an aborted subtransaction, e.g.
2732  *
2733  * BEGIN;
2734  * TRUNCATE t;
2735  * SAVEPOINT save;
2736  * TRUNCATE t;
2737  * ROLLBACK TO save;
2738  * COPY ...
2739  *
2740  * Also, if the target file is new-in-transaction, we assume that checking
2741  * FSM for free space is a waste of time, even if we must use WAL because
2742  * of archiving. This could possibly be wrong, but it's unlikely.
2743  *
2744  * The comments for table_tuple_insert and RelationGetBufferForTuple
2745  * specify that skipping WAL logging is only safe if we ensure that our
2746  * tuples do not go into pages containing tuples from any other
2747  * transactions --- but this must be the case if we have a new table or
2748  * new relfilenode, so we need no additional work to enforce that.
2749  *
2750  * We currently don't support this optimization if the COPY target is a
2751  * partitioned table as we currently only lazily initialize partition
2752  * information when routing the first tuple to the partition. We cannot
2753  * know at this stage if we can perform this optimization. It should be
2754  * possible to improve on this, but it does mean maintaining heap insert
2755  * option flags per partition and setting them when we first open the
2756  * partition.
2757  *
2758  * This optimization is not supported for relation types which do not
2759  * have any physical storage, with foreign tables and views using
2760  * INSTEAD OF triggers entering in this category. Partitioned tables
2761  * are not supported as per the description above.
2762  *----------
2763  */
2764  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2765  if (RELKIND_HAS_STORAGE(cstate->rel->rd_rel->relkind) &&
2766  (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2767  cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId))
2768  {
2769  ti_options |= TABLE_INSERT_SKIP_FSM;
2770  if (!XLogIsNeeded())
2771  ti_options |= TABLE_INSERT_SKIP_WAL;
2772  }
2773 
2774  /*
2775  * Optimize if new relfilenode was created in this subxact or one of its
2776  * committed children and we won't see those rows later as part of an
2777  * earlier scan or command. The subxact test ensures that if this subxact
2778  * aborts then the frozen rows won't be visible after xact cleanup. Note
2779  * that the stronger test of exactly which subtransaction created it is
2780  * crucial for correctness of this optimization. The test for an earlier
2781  * scan or command tolerates false negatives. FREEZE causes other sessions
2782  * to see rows they would not see under MVCC, and a false negative merely
2783  * spreads that anomaly to the current session.
2784  */
2785  if (cstate->freeze)
2786  {
2787  /*
2788  * We currently disallow COPY FREEZE on partitioned tables. The
2789  * reason for this is that we've simply not yet opened the partitions
2790  * to determine if the optimization can be applied to them. We could
2791  * go and open them all here, but doing so may be quite a costly
2792  * overhead for small copies. In any case, we may just end up routing
2793  * tuples to a small number of partitions. It seems better just to
2794  * raise an ERROR for partitioned tables.
2795  */
2796  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2797  {
2798  ereport(ERROR,
2799  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2800  errmsg("cannot perform COPY FREEZE on a partitioned table")));
2801  }
2802 
2803  /*
2804  * Tolerate one registration for the benefit of FirstXactSnapshot.
2805  * Scan-bearing queries generally create at least two registrations,
2806  * though relying on that is fragile, as is ignoring ActiveSnapshot.
2807  * Clear CatalogSnapshot to avoid counting its registration. We'll
2808  * still detect ongoing catalog scans, each of which separately
2809  * registers the snapshot it uses.
2810  */
2811  InvalidateCatalogSnapshot();
2812  if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
2813  ereport(ERROR,
2814  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2815  errmsg("cannot perform COPY FREEZE because of prior transaction activity")));
2816 
2817  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2818  cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
2819  ereport(ERROR,
2820  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2821  errmsg("cannot perform COPY FREEZE because the table was not created or truncated in the current subtransaction")));
2822 
2823  ti_options |= TABLE_INSERT_FROZEN;
2824  }
2825 
2826  /*
2827  * We need a ResultRelInfo so we can use the regular executor's
2828  * index-entry-making machinery. (There used to be a huge amount of code
2829  * here that basically duplicated execUtils.c ...)
2830  */
2831  resultRelInfo = makeNode(ResultRelInfo);
2832  InitResultRelInfo(resultRelInfo,
2833  cstate->rel,
2834  1, /* must match rel's position in range_table */
2835  NULL,
2836  0);
2837  target_resultRelInfo = resultRelInfo;
2838 
2839  /* Verify the named relation is a valid target for INSERT */
2840  CheckValidResultRel(resultRelInfo, CMD_INSERT);
2841 
2842  ExecOpenIndices(resultRelInfo, false);
2843 
2844  estate->es_result_relations = resultRelInfo;
2845  estate->es_num_result_relations = 1;
2846  estate->es_result_relation_info = resultRelInfo;
2847 
2848  ExecInitRangeTable(estate, cstate->range_table);
2849 
2850  /*
2851  * Set up a ModifyTableState so we can let FDW(s) init themselves for
2852  * foreign-table result relation(s).
2853  */
2854  mtstate = makeNode(ModifyTableState);
2855  mtstate->ps.plan = NULL;
2856  mtstate->ps.state = estate;
2857  mtstate->operation = CMD_INSERT;
2858  mtstate->resultRelInfo = estate->es_result_relations;
2859 
2860  if (resultRelInfo->ri_FdwRoutine != NULL &&
2861  resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
2862  resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
2863  resultRelInfo);
2864 
2865  /* Prepare to catch AFTER triggers. */
2866  AfterTriggerBeginQuery();
2867 
2868  /*
2869  * If there are any triggers with transition tables on the named relation,
2870  * we need to be prepared to capture transition tuples.
2871  *
2872  * Because partition tuple routing would like to know about whether
2873  * transition capture is active, we also set it in mtstate, which is
2874  * passed to ExecFindPartition() below.
2875  */
2876  cstate->transition_capture = mtstate->mt_transition_capture =
2877  MakeTransitionCaptureState(cstate->rel->trigdesc,
2878  RelationGetRelid(cstate->rel),
2879  CMD_INSERT);
2880 
2881  /*
2882  * If the named relation is a partitioned table, initialize state for
2883  * CopyFrom tuple routing.
2884  */
2885  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2886  proute = ExecSetupPartitionTupleRouting(estate, NULL, cstate->rel);
2887 
2888  if (cstate->whereClause)
2889  cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
2890  &mtstate->ps);
2891 
2892  /*
2893  * It's generally more efficient to prepare a bunch of tuples for
2894  * insertion, and insert them in one table_multi_insert() call, than call
2895  * table_tuple_insert() separately for every tuple. However, there are a
2896  * number of reasons why we might not be able to do this. These are
2897  * explained below.
2898  */
2899  if (resultRelInfo->ri_TrigDesc != NULL &&
2900  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2901  resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
2902  {
2903  /*
2904  * Can't support multi-inserts when there are any BEFORE/INSTEAD OF
2905  * triggers on the table. Such triggers might query the table we're
2906  * inserting into and act differently if the tuples that have already
2907  * been processed and prepared for insertion are not there.
2908  */
2909  insertMethod = CIM_SINGLE;
2910  }
2911  else if (proute != NULL && resultRelInfo->ri_TrigDesc != NULL &&
2912  resultRelInfo->ri_TrigDesc->trig_insert_new_table)
2913  {
2914  /*
2915  * For partitioned tables we can't support multi-inserts when there
2916  * are any statement level insert triggers. It might be possible to
2917  * allow partitioned tables with such triggers in the future, but for
2918  * now, CopyMultiInsertInfoFlush expects that any before row insert
2919  * and statement level insert triggers are on the same relation.
2920  */
2921  insertMethod = CIM_SINGLE;
2922  }
2923  else if (resultRelInfo->ri_FdwRoutine != NULL ||
2924  cstate->volatile_defexprs)
2925  {
2926  /*
2927  * Can't support multi-inserts to foreign tables or if there are any
2928  * volatile default expressions in the table. Similarly to the
2929  * trigger case above, such expressions may query the table we're
2930  * inserting into.
2931  *
2932  * Note: It does not matter if any partitions have any volatile
2933  * default expressions as we use the defaults from the target of the
2934  * COPY command.
2935  */
2936  insertMethod = CIM_SINGLE;
2937  }
2938  else if (contain_volatile_functions(cstate->whereClause))
2939  {
2940  /*
2941  * Can't support multi-inserts if there are any volatile function
2942  * expressions in WHERE clause. Similarly to the trigger case above,
2943  * such expressions may query the table we're inserting into.
2944  */
2945  insertMethod = CIM_SINGLE;
2946  }
2947  else
2948  {
2949  /*
2950  * For partitioned tables, we may still be able to perform bulk
2951  * inserts. However, the possibility of this depends on which types
2952  * of triggers exist on the partition. We must disable bulk inserts
2953  * if the partition is a foreign table or it has any before row insert
2954  * or insert instead triggers (same as we checked above for the parent
2955  * table). Since the partition's resultRelInfos are initialized only
2956  * when we actually need to insert the first tuple into them, we must
2957  * have the intermediate insert method of CIM_MULTI_CONDITIONAL to
2958  * flag that we must later determine if we can use bulk-inserts for
2959  * the partition being inserted into.
2960  */
2961  if (proute)
2962  insertMethod = CIM_MULTI_CONDITIONAL;
2963  else
2964  insertMethod = CIM_MULTI;
2965 
2966  CopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, cstate,
2967  estate, mycid, ti_options);
2968  }
2969 
2970  /*
2971  * If not using batch mode (which allocates slots as needed) set up a
2972  * tuple slot too. When inserting into a partitioned table, we also need
2973  * one, even if we might batch insert, to read the tuple in the root
2974  * partition's form.
2975  */
2976  if (insertMethod == CIM_SINGLE || insertMethod == CIM_MULTI_CONDITIONAL)
2977  {
2978  singleslot = table_slot_create(resultRelInfo->ri_RelationDesc,
2979  &estate->es_tupleTable);
2980  bistate = GetBulkInsertState();
2981  }
2982 
2983  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2984  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
2985 
2986  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2987  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
2988 
2989  /*
2990  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2991  * should do this for COPY, since it's not really an "INSERT" statement as
2992  * such. However, executing these triggers maintains consistency with the
2993  * EACH ROW triggers that we already fire on COPY.
2994  */
2995  ExecBSInsertTriggers(estate, resultRelInfo);
2996 
2997  econtext = GetPerTupleExprContext(estate);
2998 
2999  /* Set up callback to identify error line number */
3000  errcallback.callback = CopyFromErrorCallback;
3001  errcallback.arg = (void *) cstate;
3002  errcallback.previous = error_context_stack;
3003  error_context_stack = &errcallback;
3004 
3005  for (;;)
3006  {
3007  TupleTableSlot *myslot;
3008  bool skip_tuple;
3009 
3010  CHECK_FOR_INTERRUPTS();
3011 
3012  /*
3013  * Reset the per-tuple exprcontext. We do this after every tuple, to
3014  * clean-up after expression evaluations etc.
3015  */
3016  ResetPerTupleExprContext(estate);
3017 
3018  /* select slot to (initially) load row into */
3019  if (insertMethod == CIM_SINGLE || proute)
3020  {
3021  myslot = singleslot;
3022  Assert(myslot != NULL);
3023  }
3024  else
3025  {
3026  Assert(resultRelInfo == target_resultRelInfo);
3027  Assert(insertMethod == CIM_MULTI);
3028 
3029  myslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3030  resultRelInfo);
3031  }
3032 
3033  /*
3034  * Switch to per-tuple context before calling NextCopyFrom, which does
3035  * evaluate default expressions etc. and requires per-tuple context.
3036  */
3037  MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3038 
3039  ExecClearTuple(myslot);
3040 
3041  /* Directly store the values/nulls array in the slot */
3042  if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
3043  break;
3044 
3045  ExecStoreVirtualTuple(myslot);
3046 
3047  /*
3048  * Constraints and where clause might reference the tableoid column,
3049  * so (re-)initialize tts_tableOid before evaluating them.
3050  */
3051  myslot->tts_tableOid = RelationGetRelid(target_resultRelInfo->ri_RelationDesc);
3052 
3053  /* Triggers and stuff need to be invoked in query context. */
3054  MemoryContextSwitchTo(oldcontext);
3055 
3056  if (cstate->whereClause)
3057  {
3058  econtext->ecxt_scantuple = myslot;
3059  /* Skip items that don't match COPY's WHERE clause */
3060  if (!ExecQual(cstate->qualexpr, econtext))
3061  continue;
3062  }
3063 
3064  /* Determine the partition to insert the tuple into */
3065  if (proute)
3066  {
3067  TupleConversionMap *map;
3068 
3069  /*
3070  * Attempt to find a partition suitable for this tuple.
3071  * ExecFindPartition() will raise an error if none can be found or
3072  * if the found partition is not suitable for INSERTs.
3073  */
3074  resultRelInfo = ExecFindPartition(mtstate, target_resultRelInfo,
3075  proute, myslot, estate);
3076 
3077  if (prevResultRelInfo != resultRelInfo)
3078  {
3079  /* Determine which triggers exist on this partition */
3080  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3081  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
3082 
3083  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3084  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
3085 
3086  /*
3087  * Disable multi-inserts when the partition has BEFORE/INSTEAD
3088  * OF triggers, or if the partition is a foreign partition.
3089  */
3090  leafpart_use_multi_insert = insertMethod == CIM_MULTI_CONDITIONAL &&
3091  !has_before_insert_row_trig &&
3092  !has_instead_insert_row_trig &&
3093  resultRelInfo->ri_FdwRoutine == NULL;
3094 
3095  /* Set the multi-insert buffer to use for this partition. */
3096  if (leafpart_use_multi_insert)
3097  {
3098  if (resultRelInfo->ri_CopyMultiInsertBuffer == NULL)
3099  CopyMultiInsertInfoSetupBuffer(&multiInsertInfo,
3100  resultRelInfo);
3101  }
3102  else if (insertMethod == CIM_MULTI_CONDITIONAL &&
3103  !CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3104  {
3105  /*
3106  * Flush pending inserts if this partition can't use
3107  * batching, so rows are visible to triggers etc.
3108  */
3109  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3110  }
3111 
3112  if (bistate != NULL)
3113  ReleaseBulkInsertStatePin(bistate);
3114  prevResultRelInfo = resultRelInfo;
3115  }
3116 
3117  /*
3118  * For ExecInsertIndexTuples() to work on the partition's indexes
3119  */
3120  estate->es_result_relation_info = resultRelInfo;
3121 
3122  /*
3123  * If we're capturing transition tuples, we might need to convert
3124  * from the partition rowtype to root rowtype.
3125  */
3126  if (cstate->transition_capture != NULL)
3127  {
3128  if (has_before_insert_row_trig)
3129  {
3130  /*
3131  * If there are any BEFORE triggers on the partition,
3132  * we'll have to be ready to convert their result back to
3133  * tuplestore format.
3134  */
3135  cstate->transition_capture->tcs_original_insert_tuple = NULL;
3136  cstate->transition_capture->tcs_map =
3137  resultRelInfo->ri_PartitionInfo->pi_PartitionToRootMap;
3138  }
3139  else
3140  {
3141  /*
3142  * Otherwise, just remember the original unconverted
3143  * tuple, to avoid a needless round trip conversion.
3144  */
3145  cstate->transition_capture->tcs_original_insert_tuple = myslot;
3146  cstate->transition_capture->tcs_map = NULL;
3147  }
3148  }
3149 
3150  /*
3151  * We might need to convert from the root rowtype to the partition
3152  * rowtype.
3153  */
3154  map = resultRelInfo->ri_PartitionInfo->pi_RootToPartitionMap;
3155  if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert)
3156  {
3157  /* non batch insert */
3158  if (map != NULL)
3159  {
3160  TupleTableSlot *new_slot;
3161 
3162  new_slot = resultRelInfo->ri_PartitionInfo->pi_PartitionTupleSlot;
3163  myslot = execute_attr_map_slot(map->attrMap, myslot, new_slot);
3164  }
3165  }
3166  else
3167  {
3168  /*
3169  * Prepare to queue up tuple for later batch insert into
3170  * current partition.
3171  */
3172  TupleTableSlot *batchslot;
3173 
3174  /* no other path available for partitioned table */
3175  Assert(insertMethod == CIM_MULTI_CONDITIONAL);
3176 
3177  batchslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3178  resultRelInfo);
3179 
3180  if (map != NULL)
3181  myslot = execute_attr_map_slot(map->attrMap, myslot,
3182  batchslot);
3183  else
3184  {
3185  /*
3186  * This looks more expensive than it is (Believe me, I
3187  * optimized it away. Twice.). The input is in virtual
3188  * form, and we'll materialize the slot below - for most
3189  * slot types the copy performs the work materialization
3190  * would later require anyway.
3191  */
3192  ExecCopySlot(batchslot, myslot);
3193  myslot = batchslot;
3194  }
3195  }
3196 
3197  /* ensure that triggers etc see the right relation */
3198  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3199  }
3200 
3201  skip_tuple = false;
3202 
3203  /* BEFORE ROW INSERT Triggers */
3204  if (has_before_insert_row_trig)
3205  {
3206  if (!ExecBRInsertTriggers(estate, resultRelInfo, myslot))
3207  skip_tuple = true; /* "do nothing" */
3208  }
3209 
3210  if (!skip_tuple)
3211  {
3212  /*
3213  * If there is an INSTEAD OF INSERT ROW trigger, let it handle the
3214  * tuple. Otherwise, proceed with inserting the tuple into the
3215  * table or foreign table.
3216  */
3217  if (has_instead_insert_row_trig)
3218  {
3219  ExecIRInsertTriggers(estate, resultRelInfo, myslot);
3220  }
3221  else
3222  {
3223  /* Compute stored generated columns */
3224  if (resultRelInfo->ri_RelationDesc->rd_att->constr &&
3225  resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored)
3226  ExecComputeStoredGenerated(estate, myslot, CMD_INSERT);
3227 
3228  /*
3229  * If the target is a plain table, check the constraints of
3230  * the tuple.
3231  */
3232  if (resultRelInfo->ri_FdwRoutine == NULL &&
3233  resultRelInfo->ri_RelationDesc->rd_att->constr)
3234  ExecConstraints(resultRelInfo, myslot, estate);
3235 
3236  /*
3237  * Also check the tuple against the partition constraint, if
3238  * there is one; except that if we got here via tuple-routing,
3239  * we don't need to if there's no BR trigger defined on the
3240  * partition.
3241  */
3242  if (resultRelInfo->ri_PartitionCheck &&
3243  (proute == NULL || has_before_insert_row_trig))
3244  ExecPartitionCheck(resultRelInfo, myslot, estate, true);
3245 
3246  /* Store the slot in the multi-insert buffer, when enabled. */
3247  if (insertMethod == CIM_MULTI || leafpart_use_multi_insert)
3248  {
3249  /*
3250  * The slot previously might point into the per-tuple
3251  * context. For batching it needs to be longer lived.
3252  */
3253  ExecMaterializeSlot(myslot);
3254 
3255  /* Add this tuple to the tuple buffer */
3256  CopyMultiInsertInfoStore(&multiInsertInfo,
3257  resultRelInfo, myslot,
3258  cstate->line_buf.len,
3259  cstate->cur_lineno);
3260 
3261  /*
3262  * If enough inserts have queued up, then flush all
3263  * buffers out to their tables.
3264  */
3265  if (CopyMultiInsertInfoIsFull(&multiInsertInfo))
3266  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3267  }
3268  else
3269  {
3270  List *recheckIndexes = NIL;
3271 
3272  /* OK, store the tuple */
3273  if (resultRelInfo->ri_FdwRoutine != NULL)
3274  {
3275  myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
3276  resultRelInfo,
3277  myslot,
3278  NULL);
3279 
3280  if (myslot == NULL) /* "do nothing" */
3281  continue; /* next tuple please */
3282 
3283  /*
3284  * AFTER ROW Triggers might reference the tableoid
3285  * column, so (re-)initialize tts_tableOid before
3286  * evaluating them.
3287  */
3288  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3289  }
3290  else
3291  {
3292  /* OK, store the tuple and create index entries for it */
3293  table_tuple_insert(resultRelInfo->ri_RelationDesc,
3294  myslot, mycid, ti_options, bistate);
3295 
3296  if (resultRelInfo->ri_NumIndices > 0)
3297  recheckIndexes = ExecInsertIndexTuples(myslot,
3298  estate,
3299  false,
3300  NULL,
3301  NIL);
3302  }
3303 
3304  /* AFTER ROW INSERT Triggers */
3305  ExecARInsertTriggers(estate, resultRelInfo, myslot,
3306  recheckIndexes, cstate->transition_capture);
3307 
3308  list_free(recheckIndexes);
3309  }
3310  }
3311 
3312  /*
3313  * We count only tuples not suppressed by a BEFORE INSERT trigger
3314  * or FDW; this is the same definition used by nodeModifyTable.c
3315  * for counting tuples inserted by an INSERT command.
3316  */
3317  processed++;
3318  }
3319  }
3320 
3321  /* Flush any remaining buffered tuples */
3322  if (insertMethod != CIM_SINGLE)
3323  {
3324  if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3325  CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);
3326  }
3327 
3328  /* Done, clean up */
3329  error_context_stack = errcallback.previous;
3330 
3331  if (bistate != NULL)
3332  FreeBulkInsertState(bistate);
3333 
3334  MemoryContextSwitchTo(oldcontext);
3335 
3336  /*
3337  * In the old protocol, tell pqcomm that we can process normal protocol
3338  * messages again.
3339  */
3340  if (cstate->copy_dest == COPY_OLD_FE)
3341  pq_endmsgread();
3342 
3343  /* Execute AFTER STATEMENT insertion triggers */
3344  ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
3345 
3346  /* Handle queued AFTER triggers */
3347  AfterTriggerEndQuery(estate);
3348 
3349  ExecResetTupleTable(estate->es_tupleTable, false);
3350 
3351  /* Allow the FDW to shut down */
3352  if (target_resultRelInfo->ri_FdwRoutine != NULL &&
3353  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
3354  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,
3355  target_resultRelInfo);
3356 
3357  /* Tear down the multi-insert buffer data */
3358  if (insertMethod != CIM_SINGLE)
3359  CopyMultiInsertInfoCleanup(&multiInsertInfo);
3360 
3361  ExecCloseIndices(target_resultRelInfo);
3362 
3363  /* Close all the partitioned tables, leaf partitions, and their indices */
3364  if (proute)
3365  ExecCleanupTupleRouting(mtstate, proute);
3366 
3367  /* Close any trigger target relations */
3368  ExecCleanUpTriggerState(estate);
3369 
3370  FreeExecutorState(estate);
3371 
3372  return processed;
3373 }
3374 
3375 /*
3376  * Setup to read tuples from a file for COPY FROM.
3377  *
3378  * 'rel': Used as a template for the tuples
3379  * 'filename': Name of server-local file to read
3380  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
3381  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
3382  *
3383  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
3384  */
3385 CopyState
3386 BeginCopyFrom(ParseState *pstate,
3387  Relation rel,
3388  const char *filename,
3389  bool is_program,
3390  copy_data_source_cb data_source_cb,
3391  List *attnamelist,
3392  List *options)
3393 {
3394  CopyState cstate;
3395  bool pipe = (filename == NULL);
3396  TupleDesc tupDesc;
3397  AttrNumber num_phys_attrs,
3398  num_defaults;
3399  FmgrInfo *in_functions;
3400  Oid *typioparams;
3401  int attnum;
3402  Oid in_func_oid;
3403  int *defmap;
3404  ExprState **defexprs;
3405  MemoryContext oldcontext;
3406  bool volatile_defexprs;
3407 
3408  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
3409  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
3410 
3411  /* Initialize state variables */
3412  cstate->reached_eof = false;
3413  cstate->eol_type = EOL_UNKNOWN;
3414  cstate->cur_relname = RelationGetRelationName(cstate->rel);
3415  cstate->cur_lineno = 0;
3416  cstate->cur_attname = NULL;
3417  cstate->cur_attval = NULL;
3418 
3419  /* Set up variables to avoid per-attribute overhead. */
3420  initStringInfo(&cstate->attribute_buf);
3421  initStringInfo(&cstate->line_buf);
3422  cstate->line_buf_converted = false;
3423  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
3424  cstate->raw_buf_index = cstate->raw_buf_len = 0;
3425 
3426  /* Assign range table, we'll need it in CopyFrom. */
3427  if (pstate)
3428  cstate->range_table = pstate->p_rtable;
3429 
3430  tupDesc = RelationGetDescr(cstate->rel);
3431  num_phys_attrs = tupDesc->natts;
3432  num_defaults = 0;
3433  volatile_defexprs = false;
3434 
3435  /*
3436  * Pick up the required catalog information for each attribute in the
3437  * relation, including the input function, the element type (to pass to
3438  * the input function), and info about defaults and constraints. (Which
3439  * input function we use depends on text/binary format choice.)
3440  */
3441  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
3442  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3443  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3444  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3445 
3446  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3447  {
3448  Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
3449 
3450  /* We don't need info for dropped attributes */
3451  if (att->attisdropped)
3452  continue;
3453 
3454  /* Fetch the input function and typioparam info */
3455  if (cstate->binary)
3456  getTypeBinaryInputInfo(att->atttypid,
3457  &in_func_oid, &typioparams[attnum - 1]);
3458  else
3459  getTypeInputInfo(att->atttypid,
3460  &in_func_oid, &typioparams[attnum - 1]);
3461  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3462 
3463  /* Get default info if needed */
3464  if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated)
3465  {
3466  /* attribute is NOT to be copied from input */
3467  /* use default value if one exists */
3468  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
3469  attnum);
3470 
3471  if (defexpr != NULL)
3472  {
3473  /* Run the expression through planner */
3474  defexpr = expression_planner(defexpr);
3475 
3476  /* Initialize executable expression in copycontext */
3477  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3478  defmap[num_defaults] = attnum - 1;
3479  num_defaults++;
3480 
3481  /*
3482  * If a default expression looks at the table being loaded,
3483  * then it could give the wrong answer when using
3484  * multi-insert. Since database access can be dynamic this is
3485  * hard to test for exactly, so we use the much wider test of
3486  * whether the default expression is volatile. We allow for
3487  * the special case of when the default expression is the
3488  * nextval() of a sequence which in this specific case is
3489  * known to be safe for use with the multi-insert
3490  * optimization. Hence we use this special case function
3491  * checker rather than the standard check for
3492  * contain_volatile_functions().
3493  */
3494  if (!volatile_defexprs)
3495  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3496  }
3497  }
3498  }
3499 
3500  /* We keep those variables in cstate. */
3501  cstate->in_functions = in_functions;
3502  cstate->typioparams = typioparams;
3503  cstate->defmap = defmap;
3504  cstate->defexprs = defexprs;
3505  cstate->volatile_defexprs = volatile_defexprs;
3506  cstate->num_defaults = num_defaults;
3507  cstate->is_program = is_program;
3508 
3509  if (data_source_cb)
3510  {
3511  cstate->copy_dest = COPY_CALLBACK;
3512  cstate->data_source_cb = data_source_cb;
3513  }
3514  else if (pipe)
3515  {
3516  Assert(!is_program); /* the grammar does not allow this */
3517  if (whereToSendOutput == DestRemote)
3518  ReceiveCopyBegin(cstate);
3519  else
3520  cstate->copy_file = stdin;
3521  }
3522  else
3523  {
3524  cstate->filename = pstrdup(filename);
3525 
3526  if (cstate->is_program)
3527  {
3528  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3529  if (cstate->copy_file == NULL)
3530  ereport(ERROR,
3531  (errcode_for_file_access(),
3532  errmsg("could not execute command \"%s\": %m",
3533  cstate->filename)));
3534  }
3535  else
3536  {
3537  struct stat st;
3538 
3539  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3540  if (cstate->copy_file == NULL)
3541  {
3542  /* copy errno because ereport subfunctions might change it */
3543  int save_errno = errno;
3544 
3545  ereport(ERROR,
3546  (errcode_for_file_access(),
3547  errmsg("could not open file \"%s\" for reading: %m",
3548  cstate->filename),
3549  (save_errno == ENOENT || save_errno == EACCES) ?
3550  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3551  "You may want a client-side facility such as psql's \\copy.") : 0));
3552  }
3553 
3554  if (fstat(fileno(cstate->copy_file), &st))
3555  ereport(ERROR,
3556  (errcode_for_file_access(),
3557  errmsg("could not stat file \"%s\": %m",
3558  cstate->filename)));
3559 
3560  if (S_ISDIR(st.st_mode))
3561  ereport(ERROR,
3562  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3563  errmsg("\"%s\" is a directory", cstate->filename)));
3564  }
3565  }
3566 
3567  if (cstate->binary)
3568  {
3569  /* Read and verify binary header */
3570  char readSig[11];
3571  int32 tmp;
3572 
3573  /* Signature */
3574  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3575  memcmp(readSig, BinarySignature, 11) != 0)
3576  ereport(ERROR,
3577  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3578  errmsg("COPY file signature not recognized")));
3579  /* Flags field */
3580  if (!CopyGetInt32(cstate, &tmp))
3581  ereport(ERROR,
3582  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3583  errmsg("invalid COPY file header (missing flags)")));
3584  if ((tmp & (1 << 16)) != 0)
3585  ereport(ERROR,
3586  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3587  errmsg("invalid COPY file header (WITH OIDS)")));
3588  tmp &= ~(1 << 16);
3589  if ((tmp >> 16) != 0)
3590  ereport(ERROR,
3591  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3592  errmsg("unrecognized critical flags in COPY file header")));
3593  /* Header extension length */
3594  if (!CopyGetInt32(cstate, &tmp) ||
3595  tmp < 0)
3596  ereport(ERROR,
3597  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3598  errmsg("invalid COPY file header (missing length)")));
3599  /* Skip extension header, if present */
3600  while (tmp-- > 0)
3601  {
3602  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3603  ereport(ERROR,
3604  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3605  errmsg("invalid COPY file header (wrong length)")));
3606  }
3607  }
3608 
3609  /* create workspace for CopyReadAttributes results */
3610  if (!cstate->binary)
3611  {
3612  AttrNumber attr_count = list_length(cstate->attnumlist);
3613 
3614  cstate->max_fields = attr_count;
3615  cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
3616  }
3617 
3618  MemoryContextSwitchTo(oldcontext);
3619 
3620  return cstate;
3621 }
3622 
3623 /*
3624  * Read raw fields in the next line for COPY FROM in text or csv mode.
3625  * Return false if no more lines.
3626  *
3627  * An internal temporary buffer is returned via 'fields'. It is valid until
3628  * the next call of the function. Since the function returns all raw fields
3629  * in the input file, 'nfields' could be different from the number of columns
3630  * in the relation.
3631  *
3632  * NOTE: force_not_null option are not applied to the returned fields.
3633  */
3634 bool
3635 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3636 {
3637  int fldct;
3638  bool done;
3639 
3640  /* only available for text or csv input */
3641  Assert(!cstate->binary);
3642 
3643  /* on input just throw the header line away */
3644  if (cstate->cur_lineno == 0 && cstate->header_line)
3645  {
3646  cstate->cur_lineno++;
3647  if (CopyReadLine(cstate))
3648  return false; /* done */
3649  }
3650 
3651  cstate->cur_lineno++;
3652 
3653  /* Actually read the line into memory here */
3654  done = CopyReadLine(cstate);
3655 
3656  /*
3657  * EOF at start of line means we're done. If we see EOF after some
3658  * characters, we act as though it was newline followed by EOF, ie,
3659  * process the line and then exit loop on next iteration.
3660  */
3661  if (done && cstate->line_buf.len == 0)
3662  return false;
3663 
3664  /* Parse the line into de-escaped field values */
3665  if (cstate->csv_mode)
3666  fldct = CopyReadAttributesCSV(cstate);
3667  else
3668  fldct = CopyReadAttributesText(cstate);
3669 
3670  *fields = cstate->raw_fields;
3671  *nfields = fldct;
3672  return true;
3673 }
3674 
3675 /*
3676  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3677  *
3678  * 'econtext' is used to evaluate default expression for each columns not
3679  * read from the file. It can be NULL when no default values are used, i.e.
3680  * when all columns are read from the file.
3681  *
3682  * 'values' and 'nulls' arrays must be the same length as columns of the
3683  * relation passed to BeginCopyFrom. This function fills the arrays.
3684  * Oid of the tuple is returned with 'tupleOid' separately.
3685  */
3686 bool
3687 NextCopyFrom(CopyState cstate, ExprContext *econtext,
3688  Datum *values, bool *nulls)
3689 {
3690  TupleDesc tupDesc;
3691  AttrNumber num_phys_attrs,
3692  attr_count,
3693  num_defaults = cstate->num_defaults;
3694  FmgrInfo *in_functions = cstate->in_functions;
3695  Oid *typioparams = cstate->typioparams;
3696  int i;
3697  int *defmap = cstate->defmap;
3698  ExprState **defexprs = cstate->defexprs;
3699 
3700  tupDesc = RelationGetDescr(cstate->rel);
3701  num_phys_attrs = tupDesc->natts;
3702  attr_count = list_length(cstate->attnumlist);
3703 
3704  /* Initialize all values for row to NULL */
3705  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3706  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3707 
3708  if (!cstate->binary)
3709  {
3710  char **field_strings;
3711  ListCell *cur;
3712  int fldct;
3713  int fieldno;
3714  char *string;
3715 
3716  /* read raw fields in the next line */
3717  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3718  return false;
3719 
3720  /* check for overflowing fields */
3721  if (attr_count > 0 && fldct > attr_count)
3722  ereport(ERROR,
3723  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3724  errmsg("extra data after last expected column")));
3725 
3726  fieldno = 0;
3727 
3728  /* Loop to read the user attributes on the line. */
3729  foreach(cur, cstate->attnumlist)
3730  {
3731  int attnum = lfirst_int(cur);
3732  int m = attnum - 1;
3733  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3734 
3735  if (fieldno >= fldct)
3736  ereport(ERROR,
3737  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3738  errmsg("missing data for column \"%s\"",
3739  NameStr(att->attname))));
3740  string = field_strings[fieldno++];
3741 
3742  if (cstate->convert_select_flags &&
3743  !cstate->convert_select_flags[m])
3744  {
3745  /* ignore input field, leaving column as NULL */
3746  continue;
3747  }
3748 
3749  if (cstate->csv_mode)
3750  {
3751  if (string == NULL &&
3752  cstate->force_notnull_flags[m])
3753  {
3754  /*
3755  * FORCE_NOT_NULL option is set and column is NULL -
3756  * convert it to the NULL string.
3757  */
3758  string = cstate->null_print;
3759  }
3760  else if (string != NULL && cstate->force_null_flags[m]
3761  && strcmp(string, cstate->null_print) == 0)
3762  {
3763  /*
3764  * FORCE_NULL option is set and column matches the NULL
3765  * string. It must have been quoted, or otherwise the
3766  * string would already have been set to NULL. Convert it
3767  * to NULL as specified.
3768  */
3769  string = NULL;
3770  }
3771  }
3772 
3773  cstate->cur_attname = NameStr(att->attname);
3774  cstate->cur_attval = string;
3775  values[m] = InputFunctionCall(&in_functions[m],
3776  string,
3777  typioparams[m],
3778  att->atttypmod);
3779  if (string != NULL)
3780  nulls[m] = false;
3781  cstate->cur_attname = NULL;
3782  cstate->cur_attval = NULL;
3783  }
3784 
3785  Assert(fieldno == attr_count);
3786  }
3787  else
3788  {
3789  /* binary */
3790  int16 fld_count;
3791  ListCell *cur;
3792 
3793  cstate->cur_lineno++;
3794 
3795  if (!CopyGetInt16(cstate, &fld_count))
3796  {
3797  /* EOF detected (end of file, or protocol-level EOF) */
3798  return false;
3799  }
3800 
3801  if (fld_count == -1)
3802  {
3803  /*
3804  * Received EOF marker. In a V3-protocol copy, wait for the
3805  * protocol-level EOF, and complain if it doesn't come
3806  * immediately. This ensures that we correctly handle CopyFail,
3807  * if client chooses to send that now.
3808  *
3809  * Note that we MUST NOT try to read more data in an old-protocol
3810  * copy, since there is no protocol-level EOF marker then. We
3811  * could go either way for copy from file, but choose to throw
3812  * error if there's data after the EOF marker, for consistency
3813  * with the new-protocol case.
3814  */
3815  char dummy;
3816 
3817  if (cstate->copy_dest != COPY_OLD_FE &&
3818  CopyGetData(cstate, &dummy, 1, 1) > 0)
3819  ereport(ERROR,
3820  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3821  errmsg("received copy data after EOF marker")));
3822  return false;
3823  }
3824 
3825  if (fld_count != attr_count)
3826  ereport(ERROR,
3827  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3828  errmsg("row field count is %d, expected %d",
3829  (int) fld_count, attr_count)));
3830 
3831  i = 0;
3832  foreach(cur, cstate->attnumlist)
3833  {
3834  int attnum = lfirst_int(cur);
3835  int m = attnum - 1;
3836  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3837 
3838  cstate->cur_attname = NameStr(att->attname);
3839  i++;
3840  values[m] = CopyReadBinaryAttribute(cstate,
3841  i,
3842  &in_functions[m],
3843  typioparams[m],
3844  att->atttypmod,
3845  &nulls[m]);
3846  cstate->cur_attname = NULL;
3847  }
3848  }
3849 
3850  /*
3851  * Now compute and insert any defaults available for the columns not
3852  * provided by the input data. Anything not processed here or above will
3853  * remain NULL.
3854  */
3855  for (i = 0; i < num_defaults; i++)
3856  {
3857  /*
3858  * The caller must supply econtext and have switched into the
3859  * per-tuple memory context in it.
3860  */
3861  Assert(econtext != NULL);
3862  Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
3863 
3864  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3865  &nulls[defmap[i]]);
3866  }
3867 
3868  return true;
3869 }
3870 
3871 /*
3872  * Clean up storage and release resources for COPY FROM.
3873  */
3874 void
3875 EndCopyFrom(CopyState cstate)
3876 {
3877  /* No COPY FROM related resources except memory. */
3878 
3879  EndCopy(cstate);
3880 }
3881 
3882 /*
3883  * Read the next input line and stash it in line_buf, with conversion to
3884  * server encoding.
3885  *
3886  * Result is true if read was terminated by EOF, false if terminated
3887  * by newline. The terminating newline or EOF marker is not included
3888  * in the final value of line_buf.
3889  */
3890 static bool
3891 CopyReadLine(CopyState cstate)
3892 {
3893  bool result;
3894 
3895  resetStringInfo(&cstate->line_buf);
3896  cstate->line_buf_valid = true;
3897 
3898  /* Mark that encoding conversion hasn't occurred yet */
3899  cstate->line_buf_converted = false;
3900 
3901  /* Parse data and transfer into line_buf */
3902  result = CopyReadLineText(cstate);
3903 
3904  if (result)
3905  {
3906  /*
3907  * Reached EOF. In protocol version 3, we should ignore anything
3908  * after \. up to the protocol end of copy data. (XXX maybe better
3909  * not to treat \. as special?)
3910  */
3911  if (cstate->copy_dest == COPY_NEW_FE)
3912  {
3913  do
3914  {
3915  cstate->raw_buf_index = cstate->raw_buf_len;
3916  } while (CopyLoadRawBuf(cstate));
3917  }
3918  }
3919  else
3920  {
3921  /*
3922  * If we didn't hit EOF, then we must have transferred the EOL marker
3923  * to line_buf along with the data. Get rid of it.
3924  */
3925  switch (cstate->eol_type)
3926  {
3927  case EOL_NL:
3928  Assert(cstate->line_buf.len >= 1);
3929  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3930  cstate->line_buf.len--;
3931  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3932  break;
3933  case EOL_CR:
3934  Assert(cstate->line_buf.len >= 1);
3935  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3936  cstate->line_buf.len--;
3937  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3938  break;
3939  case EOL_CRNL:
3940  Assert(cstate->line_buf.len >= 2);
3941  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3942  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3943  cstate->line_buf.len -= 2;
3944  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3945  break;
3946  case EOL_UNKNOWN:
3947  /* shouldn't get here */
3948  Assert(false);
3949  break;
3950  }
3951  }
3952 
3953  /* Done reading the line. Convert it to server encoding. */
3954  if (cstate->need_transcoding)
3955  {
3956  char *cvt;
3957 
3958  cvt = pg_any_to_server(cstate->line_buf.data,
3959  cstate->line_buf.len,
3960  cstate->file_encoding);
3961  if (cvt != cstate->line_buf.data)
3962  {
3963  /* transfer converted data back to line_buf */
3964  resetStringInfo(&cstate->line_buf);
3965  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3966  pfree(cvt);
3967  }
3968  }
3969 
3970  /* Now it's safe to use the buffer in error messages */
3971  cstate->line_buf_converted = true;
3972 
3973  return result;
3974 }
3975 
3976 /*
3977  * CopyReadLineText - inner loop of CopyReadLine for text mode
3978  */
3979 static bool
3980 CopyReadLineText(CopyState cstate)
3981 {
3982  char *copy_raw_buf;
3983  int raw_buf_ptr;
3984  int copy_buf_len;
3985  bool need_data = false;
3986  bool hit_eof = false;
3987  bool result = false;
3988  char mblen_str[2];
3989 
3990  /* CSV variables */
3991  bool first_char_in_line = true;
3992  bool in_quote = false,
3993  last_was_esc = false;
3994  char quotec = '\0';
3995  char escapec = '\0';
3996 
3997  if (cstate->csv_mode)
3998  {
3999  quotec = cstate->quote[0];
4000  escapec = cstate->escape[0];
4001  /* ignore special escape processing if it's the same as quotec */
4002  if (quotec == escapec)
4003  escapec = '\0';
4004  }
4005 
4006  mblen_str[1] = '\0';
4007 
4008  /*
4009  * The objective of this loop is to transfer the entire next input line
4010  * into line_buf. Hence, we only care for detecting newlines (\r and/or
4011  * \n) and the end-of-copy marker (\.).
4012  *
4013  * In CSV mode, \r and \n inside a quoted field are just part of the data
4014  * value and are put in line_buf. We keep just enough state to know if we
4015  * are currently in a quoted field or not.
4016  *
4017  * These four characters, and the CSV escape and quote characters, are
4018  * assumed the same in frontend and backend encodings.
4019  *
4020  * For speed, we try to move data from raw_buf to line_buf in chunks
4021  * rather than one character at a time. raw_buf_ptr points to the next
4022  * character to examine; any characters from raw_buf_index to raw_buf_ptr
4023  * have been determined to be part of the line, but not yet transferred to
4024  * line_buf.
4025  *
4026  * For a little extra speed within the loop, we copy raw_buf and
4027  * raw_buf_len into local variables.
4028  */
4029  copy_raw_buf = cstate->raw_buf;
4030  raw_buf_ptr = cstate->raw_buf_index;
4031  copy_buf_len = cstate->raw_buf_len;
4032 
4033  for (;;)
4034  {
4035  int prev_raw_ptr;
4036  char c;
4037 
4038  /*
4039  * Load more data if needed. Ideally we would just force four bytes
4040  * of read-ahead and avoid the many calls to
4041  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
4042  * does not allow us to read too far ahead or we might read into the
4043  * next data, so we read-ahead only as far we know we can. One
4044  * optimization would be to read-ahead four byte here if
4045  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
4046  * considering the size of the buffer.
4047  */
4048  if (raw_buf_ptr >= copy_buf_len || need_data)
4049  {
4050  REFILL_LINEBUF;
4051 
4052  /*
4053  * Try to read some more data. This will certainly reset
4054  * raw_buf_index to zero, and raw_buf_ptr must go with it.
4055  */
4056  if (!CopyLoadRawBuf(cstate))
4057  hit_eof = true;
4058  raw_buf_ptr = 0;
4059  copy_buf_len = cstate->raw_buf_len;
4060 
4061  /*
4062  * If we are completely out of data, break out of the loop,
4063  * reporting EOF.
4064  */
4065  if (copy_buf_len <= 0)
4066  {
4067  result = true;
4068  break;
4069  }
4070  need_data = false;
4071  }
4072 
4073  /* OK to fetch a character */
4074  prev_raw_ptr = raw_buf_ptr;
4075  c = copy_raw_buf[raw_buf_ptr++];
4076 
4077  if (cstate->csv_mode)
4078  {
4079  /*
4080  * If character is '\\' or '\r', we may need to look ahead below.
4081  * Force fetch of the next character if we don't already have it.
4082  * We need to do this before changing CSV state, in case one of
4083  * these characters is also the quote or escape character.
4084  *
4085  * Note: old-protocol does not like forced prefetch, but it's OK
4086  * here since we cannot validly be at EOF.
4087  */
4088  if (c == '\\' || c == '\r')
4089  {
4090  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4091  }
4092 
4093  /*
4094  * Dealing with quotes and escapes here is mildly tricky. If the
4095  * quote char is also the escape char, there's no problem - we
4096  * just use the char as a toggle. If they are different, we need
4097  * to ensure that we only take account of an escape inside a
4098  * quoted field and immediately preceding a quote char, and not
4099  * the second in an escape-escape sequence.
4100  */
4101  if (in_quote && c == escapec)
4102  last_was_esc = !last_was_esc;
4103  if (c == quotec && !last_was_esc)
4104  in_quote = !in_quote;
4105  if (c != escapec)
4106  last_was_esc = false;
4107 
4108  /*
4109  * Updating the line count for embedded CR and/or LF chars is
4110  * necessarily a little fragile - this test is probably about the
4111  * best we can do. (XXX it's arguable whether we should do this
4112  * at all --- is cur_lineno a physical or logical count?)
4113  */
4114  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
4115  cstate->cur_lineno++;
4116  }
4117 
4118  /* Process \r */
4119  if (c == '\r' && (!cstate->csv_mode || !in_quote))
4120  {
4121  /* Check for \r\n on first line, _and_ handle \r\n. */
4122  if (cstate->eol_type == EOL_UNKNOWN ||
4123  cstate->eol_type == EOL_CRNL)
4124  {
4125  /*
4126  * If need more data, go back to loop top to load it.
4127  *
4128  * Note that if we are at EOF, c will wind up as '\0' because
4129  * of the guaranteed pad of raw_buf.
4130  */
4131  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4132 
4133  /* get next char */
4134  c = copy_raw_buf[raw_buf_ptr];
4135 
4136  if (c == '\n')
4137  {
4138  raw_buf_ptr++; /* eat newline */
4139  cstate->eol_type = EOL_CRNL; /* in case not set yet */
4140  }
4141  else
4142  {
4143  /* found \r, but no \n */
4144  if (cstate->eol_type == EOL_CRNL)
4145  ereport(ERROR,
4146  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4147  !cstate->csv_mode ?
4148  errmsg("literal carriage return found in data") :
4149  errmsg("unquoted carriage return found in data"),
4150  !cstate->csv_mode ?
4151  errhint("Use \"\\r\" to represent carriage return.") :
4152  errhint("Use quoted CSV field to represent carriage return.")));
4153 
4154  /*
4155  * if we got here, it is the first line and we didn't find
4156  * \n, so don't consume the peeked character
4157  */
4158  cstate->eol_type = EOL_CR;
4159  }
4160  }
4161  else if (cstate->eol_type == EOL_NL)
4162  ereport(ERROR,
4163  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4164  !cstate->csv_mode ?
4165  errmsg("literal carriage return found in data") :
4166  errmsg("unquoted carriage return found in data"),
4167  !cstate->csv_mode ?
4168  errhint("Use \"\\r\" to represent carriage return.") :
4169  errhint("Use quoted CSV field to represent carriage return.")));
4170  /* If reach here, we have found the line terminator */
4171  break;
4172  }
4173 
4174  /* Process \n */
4175  if (c == '\n' && (!cstate->csv_mode || !in_quote))
4176  {
4177  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
4178  ereport(ERROR,
4179  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4180  !cstate->csv_mode ?
4181  errmsg("literal newline found in data") :
4182  errmsg("unquoted newline found in data"),
4183  !cstate->csv_mode ?
4184  errhint("Use \"\\n\" to represent newline.") :
4185  errhint("Use quoted CSV field to represent newline.")));
4186  cstate->eol_type = EOL_NL; /* in case not set yet */
4187  /* If reach here, we have found the line terminator */
4188  break;
4189  }
4190 
4191  /*
4192  * In CSV mode, we only recognize \. alone on a line. This is because
4193  * \. is a valid CSV data value.
4194  */
4195  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
4196  {
4197  char c2;
4198 
4199  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4200  IF_NEED_REFILL_AND_EOF_BREAK(0);
4201 
4202  /* -----
4203  * get next character
4204  * Note: we do not change c so if it isn't \., we can fall
4205  * through and continue processing for file encoding.
4206  * -----
4207  */
4208  c2 = copy_raw_buf[raw_buf_ptr];
4209 
4210  if (c2 == '.')
4211  {
4212  raw_buf_ptr++; /* consume the '.' */
4213 
4214  /*
4215  * Note: if we loop back for more data here, it does not
4216  * matter that the CSV state change checks are re-executed; we
4217  * will come back here with no important state changed.
4218  */
4219  if (cstate->eol_type == EOL_CRNL)
4220  {
4221  /* Get the next character */
4222  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4223  /* if hit_eof, c2 will become '\0' */
4224  c2 = copy_raw_buf[raw_buf_ptr++];
4225 
4226  if (c2 == '\n')
4227  {
4228  if (!cstate->csv_mode)
4229  ereport(ERROR,
4230  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4231  errmsg("end-of-copy marker does not match previous newline style")));
4232  else
4233  NO_END_OF_COPY_GOTO;
4234  }
4235  else if (c2 != '\r')
4236  {
4237  if (!cstate->csv_mode)
4238  ereport(ERROR,
4239  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4240  errmsg("end-of-copy marker corrupt")));
4241  else
4242  NO_END_OF_COPY_GOTO;
4243  }
4244  }
4245 
4246  /* Get the next character */
4247  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4248  /* if hit_eof, c2 will become '\0' */
4249  c2 = copy_raw_buf[raw_buf_ptr++];
4250 
4251  if (c2 != '\r' && c2 != '\n')
4252  {
4253  if (!cstate->csv_mode)
4254  ereport(ERROR,
4255  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4256  errmsg("end-of-copy marker corrupt")));
4257  else
4258  NO_END_OF_COPY_GOTO;
4259  }
4260 
4261  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
4262  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
4263  (cstate->eol_type == EOL_CR && c2 != '\r'))
4264  {
4265  ereport(ERROR,
4266  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4267  errmsg("end-of-copy marker does not match previous newline style")));
4268  }
4269 
4270  /*
4271  * Transfer only the data before the \. into line_buf, then
4272  * discard the data and the \. sequence.
4273  */
4274  if (prev_raw_ptr > cstate->raw_buf_index)
4275  appendBinaryStringInfo(&cstate->line_buf,
4276  cstate->raw_buf + cstate->raw_buf_index,
4277  prev_raw_ptr - cstate->raw_buf_index);
4278  cstate->raw_buf_index = raw_buf_ptr;
4279  result = true; /* report EOF */
4280  break;
4281  }
4282  else if (!cstate->csv_mode)
4283 
4284  /*
4285  * If we are here, it means we found a backslash followed by
4286  * something other than a period. In non-CSV mode, anything
4287  * after a backslash is special, so we skip over that second
4288  * character too. If we didn't do that \\. would be
4289  * considered an eof-of copy, while in non-CSV mode it is a
4290  * literal backslash followed by a period. In CSV mode,
4291  * backslashes are not special, so we want to process the
4292  * character after the backslash just like a normal character,
4293  * so we don't increment in those cases.
4294  */
4295  raw_buf_ptr++;
4296  }
4297 
4298  /*
4299  * This label is for CSV cases where \. appears at the start of a
4300  * line, but there is more text after it, meaning it was a data value.
4301  * We are more strict for \. in CSV mode because \. could be a data
4302  * value, while in non-CSV mode, \. cannot be a data value.
4303  */
4304 not_end_of_copy:
4305 
4306  /*
4307  * Process all bytes of a multi-byte character as a group.
4308  *
4309  * We only support multi-byte sequences where the first byte has the
4310  * high-bit set, so as an optimization we can avoid this block
4311  * entirely if it is not set.
4312  */
4313  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
4314  {
4315  int mblen;
4316 
4317  /*
4318  * It is enough to look at the first byte in all our encodings, to
4319  * get the length. (GB18030 is a bit special, but still works for
4320  * our purposes; see comment in pg_gb18030_mblen())
4321  */
4322  mblen_str[0] = c;
4323  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
4324 
4325  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
4326  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
4327  raw_buf_ptr += mblen - 1;
4328  }
4329  first_char_in_line = false;
4330  } /* end of outer loop */
4331 
4332  /*
4333  * Transfer any still-uncopied data to line_buf.
4334  */
4335  REFILL_LINEBUF;
4336 
4337  return result;
4338 }
4339 
4340 /*
4341  * Return decimal value for a hexadecimal digit
4342  */
4343 static int
4344 GetDecimalFromHex(char hex)
4345 {
4346  if (isdigit((unsigned char) hex))
4347  return hex - '0';
4348  else
4349  return tolower((unsigned char) hex) - 'a' + 10;
4350 }
4351 
4352 /*
4353  * Parse the current line into separate attributes (fields),
4354  * performing de-escaping as needed.
4355  *
4356  * The input is in line_buf. We use attribute_buf to hold the result
4357  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
4358  * string, or NULL when the input matches the null marker string.
4359  * This array is expanded as necessary.
4360  *
4361  * (Note that the caller cannot check for nulls since the returned
4362  * string would be the post-de-escaping equivalent, which may look
4363  * the same as some valid data string.)
4364  *
4365  * delim is the column delimiter string (must be just one byte for now).
4366  * null_print is the null marker string. Note that this is compared to
4367  * the pre-de-escaped input string.
4368  *
4369  * The return value is the number of fields actually read.
4370  */
4371 static int
4372 CopyReadAttributesText(CopyState cstate)
4373 {
4374  char delimc = cstate->delim[0];
4375  int fieldno;
4376  char *output_ptr;
4377  char *cur_ptr;
4378  char *line_end_ptr;
4379 
4380  /*
4381  * We need a special case for zero-column tables: check that the input
4382  * line is empty, and return.
4383  */
4384  if (cstate->max_fields <= 0)
4385  {
4386  if (cstate->line_buf.len != 0)
4387  ereport(ERROR,
4388  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4389  errmsg("extra data after last expected column")));
4390  return 0;
4391  }
4392 
4393  resetStringInfo(&cstate->attribute_buf);
4394 
4395  /*
4396  * The de-escaped attributes will certainly not be longer than the input
4397  * data line, so we can just force attribute_buf to be large enough and
4398  * then transfer data without any checks for enough space. We need to do
4399  * it this way because enlarging attribute_buf mid-stream would invalidate
4400  * pointers already stored into cstate->raw_fields[].
4401  */
4402  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4403  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4404  output_ptr = cstate->attribute_buf.data;
4405 
4406  /* set pointer variables for loop */
4407  cur_ptr = cstate->line_buf.data;
4408  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4409 
4410  /* Outer loop iterates over fields */
4411  fieldno = 0;
4412  for (;;)
4413  {
4414  bool found_delim = false;
4415  char *start_ptr;
4416  char *end_ptr;
4417  int input_len;
4418  bool saw_non_ascii = false;
4419 
4420  /* Make sure there is enough space for the next value */
4421  if (fieldno >= cstate->max_fields)
4422  {
4423  cstate->max_fields *= 2;
4424  cstate->raw_fields =
4425  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4426  }
4427 
4428  /* Remember start of field on both input and output sides */
4429  start_ptr = cur_ptr;
4430  cstate->raw_fields[fieldno] = output_ptr;
4431 
4432  /*
4433  * Scan data for field.
4434  *
4435  * Note that in this loop, we are scanning to locate the end of field
4436  * and also speculatively performing de-escaping. Once we find the
4437  * end-of-field, we can match the raw field contents against the null
4438  * marker string. Only after that comparison fails do we know that
4439  * de-escaping is actually the right thing to do; therefore we *must
4440  * not* throw any syntax errors before we've done the null-marker
4441  * check.
4442  */
4443  for (;;)
4444  {
4445  char c;
4446 
4447  end_ptr = cur_ptr;
4448  if (cur_ptr >= line_end_ptr)
4449  break;
4450  c = *cur_ptr++;
4451  if (c == delimc)
4452  {
4453  found_delim = true;
4454  break;
4455  }
4456  if (c == '\\')
4457  {
4458  if (cur_ptr >= line_end_ptr)
4459  break;
4460  c = *cur_ptr++;
4461  switch (c)
4462  {
4463  case '0':
4464  case '1':
4465  case '2':
4466  case '3':
4467  case '4':
4468  case '5':
4469  case '6':
4470  case '7':
4471  {
4472  /* handle \013 */
4473  int val;
4474 
4475  val = OCTVALUE(c);
4476  if (cur_ptr < line_end_ptr)
4477  {
4478  c = *cur_ptr;
4479  if (ISOCTAL(c))
4480  {
4481  cur_ptr++;
4482  val = (val << 3) + OCTVALUE(c);
4483  if (cur_ptr < line_end_ptr)
4484  {
4485  c = *cur_ptr;
4486  if (ISOCTAL(c))
4487  {
4488  cur_ptr++;
4489  val = (val << 3) + OCTVALUE(c);
4490  }
4491  }
4492  }
4493  }
4494  c = val & 0377;
4495  if (c == '\0' || IS_HIGHBIT_SET(c))
4496  saw_non_ascii = true;
4497  }
4498  break;
4499  case 'x':
4500  /* Handle \x3F */
4501  if (cur_ptr < line_end_ptr)
4502  {
4503  char hexchar = *cur_ptr;
4504 
4505  if (isxdigit((unsigned char) hexchar))
4506  {
4507  int val = GetDecimalFromHex(hexchar);
4508 
4509  cur_ptr++;
4510  if (cur_ptr < line_end_ptr)
4511  {
4512  hexchar = *cur_ptr;
4513  if (isxdigit((unsigned char) hexchar))
4514  {
4515  cur_ptr++;
4516  val = (val << 4) + GetDecimalFromHex(hexchar);
4517  }
4518  }
4519  c = val & 0xff;
4520  if (c == '\0' || IS_HIGHBIT_SET(c))
4521  saw_non_ascii = true;
4522  }
4523  }
4524  break;
4525  case 'b':
4526  c = '\b';
4527  break;
4528  case 'f':
4529  c = '\f';
4530  break;
4531  case 'n':
4532  c = '\n';
4533  break;
4534  case 'r':
4535  c = '\r';
4536  break;
4537  case 't':
4538  c = '\t';
4539  break;
4540  case 'v':
4541  c = '\v';
4542  break;
4543 
4544  /*
4545  * in all other cases, take the char after '\'
4546  * literally
4547  */
4548  }
4549  }
4550 
4551  /* Add c to output string */
4552  *output_ptr++ = c;
4553  }
4554 
4555  /* Check whether raw input matched null marker */
4556  input_len = end_ptr - start_ptr;
4557  if (input_len == cstate->null_print_len &&
4558  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4559  cstate->raw_fields[fieldno] = NULL;
4560  else
4561  {
4562  /*
4563  * At this point we know the field is supposed to contain data.
4564  *
4565  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4566  * resulting string is valid data for the db encoding.
4567  */
4568  if (saw_non_ascii)
4569  {
4570  char *fld = cstate->raw_fields[fieldno];
4571 
4572  pg_verifymbstr(fld, output_ptr - fld, false);
4573  }
4574  }
4575 
4576  /* Terminate attribute value in output area */
4577  *output_ptr++ = '\0';
4578 
4579  fieldno++;
4580  /* Done if we hit EOL instead of a delim */
4581  if (!found_delim)
4582  break;
4583  }
4584 
4585  /* Clean up state of attribute_buf */
4586  output_ptr--;
4587  Assert(*output_ptr == '\0');
4588  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4589 
4590  return fieldno;
4591 }
4592 
4593 /*
4594  * Parse the current line into separate attributes (fields),
4595  * performing de-escaping as needed. This has exactly the same API as
4596  * CopyReadAttributesText, except we parse the fields according to
4597  * "standard" (i.e. common) CSV usage.
4598  */
4599 static int
4600 CopyReadAttributesCSV(CopyState cstate)
4601 {
4602  char delimc = cstate->delim[0];
4603  char quotec = cstate->quote[0];
4604  char escapec = cstate->escape[0];
4605  int fieldno;
4606  char *output_ptr;
4607  char *cur_ptr;
4608  char *line_end_ptr;
4609 
4610  /*
4611  * We need a special case for zero-column tables: check that the input
4612  * line is empty, and return.
4613  */
4614  if (cstate->max_fields <= 0)
4615  {
4616  if (cstate->line_buf.len != 0)
4617  ereport(ERROR,
4618  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4619  errmsg("extra data after last expected column")));
4620  return 0;
4621  }
4622 
4623  resetStringInfo(&cstate->attribute_buf);
4624 
4625  /*
4626  * The de-escaped attributes will certainly not be longer than the input
4627  * data line, so we can just force attribute_buf to be large enough and
4628  * then transfer data without any checks for enough space. We need to do
4629  * it this way because enlarging attribute_buf mid-stream would invalidate
4630  * pointers already stored into cstate->raw_fields[].
4631  */
4632  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4633  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4634  output_ptr = cstate->attribute_buf.data;
4635 
4636  /* set pointer variables for loop */
4637  cur_ptr = cstate->line_buf.data;
4638  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4639 
4640  /* Outer loop iterates over fields */
4641  fieldno = 0;
4642  for (;;)
4643  {
4644  bool found_delim = false;
4645  bool saw_quote = false;
4646  char *start_ptr;
4647  char *end_ptr;
4648  int input_len;
4649 
4650  /* Make sure there is enough space for the next value */
4651  if (fieldno >= cstate->max_fields)
4652  {
4653  cstate->max_fields *= 2;
4654  cstate->raw_fields =
4655  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4656  }
4657 
4658  /* Remember start of field on both input and output sides */
4659  start_ptr = cur_ptr;
4660  cstate->raw_fields[fieldno] = output_ptr;
4661 
4662  /*
4663  * Scan data for field,
4664  *
4665  * The loop starts in "not quote" mode and then toggles between that
4666  * and "in quote" mode. The loop exits normally if it is in "not
4667  * quote" mode and a delimiter or line end is seen.
4668  */
4669  for (;;)
4670  {
4671  char c;
4672 
4673  /* Not in quote */
4674  for (;;)
4675  {
4676  end_ptr = cur_ptr;
4677  if (cur_ptr >= line_end_ptr)
4678  goto endfield;
4679  c = *cur_ptr++;
4680  /* unquoted field delimiter */
4681  if (c == delimc)
4682  {
4683  found_delim = true;
4684  goto endfield;
4685  }
4686  /* start of quoted field (or part of field) */
4687  if (c == quotec)
4688  {
4689  saw_quote = true;
4690  break;
4691  }
4692  /* Add c to output string */
4693  *output_ptr++ = c;
4694  }
4695 
4696  /* In quote */
4697  for (;;)
4698  {
4699  end_ptr = cur_ptr;
4700  if (cur_ptr >= line_end_ptr)
4701  ereport(ERROR,
4702  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4703  errmsg("unterminated CSV quoted field")));
4704 
4705  c = *cur_ptr++;
4706 
4707  /* escape within a quoted field */
4708  if (c == escapec)
4709  {
4710  /*
4711  * peek at the next char if available, and escape it if it
4712  * is an escape char or a quote char
4713  */
4714  if (cur_ptr < line_end_ptr)
4715  {
4716  char nextc = *cur_ptr;
4717 
4718  if (nextc == escapec || nextc == quotec)
4719  {
4720  *output_ptr++ = nextc;
4721  cur_ptr++;
4722  continue;
4723  }
4724  }
4725  }
4726 
4727  /*
4728  * end of quoted field. Must do this test after testing for
4729  * escape in case quote char and escape char are the same
4730  * (which is the common case).
4731  */
4732  if (c == quotec)
4733  break;
4734 
4735  /* Add c to output string */
4736  *output_ptr++ = c;
4737  }
4738  }
4739 endfield:
4740 
4741  /* Terminate attribute value in output area */
4742  *output_ptr++ = '\0';
4743 
4744  /* Check whether raw input matched null marker */
4745  input_len = end_ptr - start_ptr;
4746  if (!saw_quote && input_len == cstate->null_print_len &&
4747  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4748  cstate->raw_fields[fieldno] = NULL;
4749 
4750  fieldno++;
4751  /* Done if we hit EOL instead of a delim */
4752  if (!found_delim)
4753  break;
4754  }
4755 
4756  /* Clean up state of attribute_buf */
4757  output_ptr--;
4758  Assert(*output_ptr == '\0');
4759  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4760 
4761  return fieldno;
4762 }
4763 
4764 
4765 /*
4766  * Read a binary attribute
4767  */
4768 static Datum
4769 CopyReadBinaryAttribute(CopyState cstate,
4770  int column_no, FmgrInfo *flinfo,
4771  Oid typioparam, int32 typmod,
4772  bool *isnull)
4773 {
4774  int32 fld_size;
4775  Datum result;
4776 
4777  if (!CopyGetInt32(cstate, &fld_size))
4778  ereport(ERROR,
4779  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4780  errmsg("unexpected EOF in COPY data")));
4781  if (fld_size == -1)
4782  {
4783  *isnull = true;
4784  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4785  }
4786  if (fld_size < 0)
4787  ereport(ERROR,
4788  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4789  errmsg("invalid field size")));
4790 
4791  /* reset attribute_buf to empty, and load raw data in it */
4792  resetStringInfo(&cstate->attribute_buf);
4793 
4794  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4795  if (CopyGetData(cstate, cstate->attribute_buf.data,
4796  fld_size, fld_size) != fld_size)
4797  ereport(ERROR,
4798  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4799  errmsg("unexpected EOF in COPY data")));
4800 
4801  cstate->attribute_buf.len = fld_size;
4802  cstate->attribute_buf.data[fld_size] = '\0';
4803 
4804  /* Call the column type's binary input converter */
4805  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4806  typioparam, typmod);
4807 
4808  /* Trouble if it didn't eat the whole buffer */
4809  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4810  ereport(ERROR,
4811  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4812  errmsg("incorrect binary data format")));
4813 
4814  *isnull = false;
4815  return result;
4816 }
4817 
4818 /*
4819  * Send text representation of one attribute, with conversion and escaping
4820  */
4821 #define DUMPSOFAR() \
4822  do { \
4823  if (ptr > start) \
4824  CopySendData(cstate, start, ptr - start); \
4825  } while (0)
4826 
4827 static void
4828 CopyAttributeOutText(CopyState cstate, char *string)
4829 {
4830  char *ptr;
4831  char *start;
4832  char c;
4833  char delimc = cstate->delim[0];
4834 
4835  if (cstate->need_transcoding)
4836  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4837  else
4838  ptr = string;
4839 
4840  /*
4841  * We have to grovel through the string searching for control characters
4842  * and instances of the delimiter character. In most cases, though, these
4843  * are infrequent. To avoid overhead from calling CopySendData once per
4844  * character, we dump out all characters between escaped characters in a
4845  * single call. The loop invariant is that the data from "start" to "ptr"
4846  * can be sent literally, but hasn't yet been.
4847  *
4848  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4849  * in valid backend encodings, extra bytes of a multibyte character never
4850  * look like ASCII. This loop is sufficiently performance-critical that
4851  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4852  * of the normal safe-encoding path.
4853  */
4854  if (cstate->encoding_embeds_ascii)
4855  {
4856  start = ptr;
4857  while ((c = *ptr) != '\0')
4858  {
4859  if ((unsigned char) c < (unsigned char) 0x20)
4860  {
4861  /*
4862  * \r and \n must be escaped, the others are traditional. We
4863  * prefer to dump these using the C-like notation, rather than
4864  * a backslash and the literal character, because it makes the
4865  * dump file a bit more proof against Microsoftish data
4866  * mangling.
4867  */
4868  switch (c)
4869  {
4870  case '\b':
4871  c = 'b';
4872  break;
4873  case '\f':
4874  c = 'f';
4875  break;
4876  case '\n':
4877  c = 'n';
4878  break;
4879  case '\r':
4880  c = 'r';
4881  break;
4882  case '\t':
4883  c = 't';
4884  break;
4885  case '\v':
4886  c = 'v';
4887  break;
4888  default:
4889  /* If it's the delimiter, must backslash it */
4890  if (c == delimc)
4891  break;
4892  /* All ASCII control chars are length 1 */
4893  ptr++;
4894  continue; /* fall to end of loop */
4895  }
4896  /* if we get here, we need to convert the control char */
4897  DUMPSOFAR();
4898  CopySendChar(cstate, '\\');
4899  CopySendChar(cstate, c);
4900  start = ++ptr; /* do not include char in next run */
4901  }
4902  else if (c == '\\' || c == delimc)
4903  {
4904  DUMPSOFAR();
4905  CopySendChar(cstate, '\\');
4906  start = ptr++; /* we include char in next run */
4907  }
4908  else if (IS_HIGHBIT_SET(c))
4909  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4910  else
4911  ptr++;
4912  }
4913  }
4914  else
4915  {
4916  start = ptr;
4917  while ((c = *ptr) != '\0')
4918  {
4919  if ((unsigned char) c < (unsigned char) 0x20)
4920  {
4921  /*
4922  * \r and \n must be escaped, the others are traditional. We
4923  * prefer to dump these using the C-like notation, rather than
4924  * a backslash and the literal character, because it makes the
4925  * dump file a bit more proof against Microsoftish data
4926  * mangling.
4927  */
4928  switch (c)
4929  {
4930  case '\b':
4931  c = 'b';
4932  break;
4933  case '\f':
4934  c = 'f';
4935  break;
4936  case '\n':
4937  c = 'n';
4938  break;
4939  case '\r':
4940  c = 'r';
4941  break;
4942  case '\t':
4943  c = 't';
4944  break;
4945  case '\v':
4946  c = 'v';
4947  break;
4948  default:
4949  /* If it's the delimiter, must backslash it */
4950  if (c == delimc)
4951  break;
4952  /* All ASCII control chars are length 1 */
4953  ptr++;
4954  continue; /* fall to end of loop */
4955  }
4956  /* if we get here, we need to convert the control char */
4957  DUMPSOFAR();
4958  CopySendChar(cstate, '\\');
4959  CopySendChar(cstate, c);
4960  start = ++ptr; /* do not include char in next run */
4961  }
4962  else if (c == '\\' || c == delimc)
4963  {
4964  DUMPSOFAR();
4965  CopySendChar(cstate, '\\');
4966  start = ptr++; /* we include char in next run */
4967  }
4968  else
4969  ptr++;
4970  }
4971  }
4972 
4973  DUMPSOFAR();
4974 }
4975 
4976 /*
4977  * Send text representation of one attribute, with conversion and
4978  * CSV-style escaping
4979  */
4980 static void
4981 CopyAttributeOutCSV(CopyState cstate, char *string,
4982  bool use_quote, bool single_attr)
4983 {
4984  char *ptr;
4985  char *start;
4986  char c;
4987  char delimc = cstate->delim[0];
4988  char quotec = cstate->quote[0];
4989  char escapec = cstate->escape[0];
4990 
4991  /* force quoting if it matches null_print (before conversion!) */
4992  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4993  use_quote = true;
4994 
4995  if (cstate->need_transcoding)
4996  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4997  else
4998  ptr = string;
4999 
5000  /*
5001  * Make a preliminary pass to discover if it needs quoting
5002  */
5003  if (!use_quote)
5004  {
5005  /*
5006  * Because '\.' can be a data value, quote it if it appears alone on a
5007  * line so it is not interpreted as the end-of-data marker.
5008  */
5009  if (single_attr && strcmp(ptr, "\\.") == 0)
5010  use_quote = true;
5011  else
5012  {
5013  char *tptr = ptr;
5014 
5015  while ((c = *tptr) != '\0')
5016  {
5017  if (c == delimc || c == quotec || c == '\n' || c == '\r')
5018  {
5019  use_quote = true;
5020  break;
5021  }
5022  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5023  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
5024  else
5025  tptr++;
5026  }
5027  }
5028  }
5029 
5030  if (use_quote)
5031  {
5032  CopySendChar(cstate, quotec);
5033 
5034  /*
5035  * We adopt the same optimization strategy as in CopyAttributeOutText
5036  */
5037  start = ptr;
5038  while ((c = *ptr) != '\0')
5039  {
5040  if (c == quotec || c == escapec)
5041  {
5042  DUMPSOFAR();
5043  CopySendChar(cstate, escapec);
5044  start = ptr; /* we include char in next run */
5045  }
5046  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5047  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
5048  else
5049  ptr++;
5050  }
5051  DUMPSOFAR();
5052 
5053  CopySendChar(cstate, quotec);
5054  }
5055  else
5056  {
5057  /* If it doesn't need quoting, we can just dump it as-is */
5058  CopySendString(cstate, ptr);
5059  }
5060 }
5061 
5062 /*
5063  * CopyGetAttnums - build an integer list of attnums to be copied
5064  *
5065  * The input attnamelist is either the user-specified column list,
5066  * or NIL if there was none (in which case we want all the non-dropped
5067  * columns).
5068  *
5069  * We don't include generated columns in the generated full list and we don't
5070  * allow them to be specified explicitly. They don't make sense for COPY
5071  * FROM, but we could possibly allow them for COPY TO. But this way it's at
5072  * least ensured that whatever we copy out can be copied back in.
5073  *
5074  * rel can be NULL ... it's only used for error reports.
5075  */
5076 static List *
5077 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
5078 {
5079  List *attnums = NIL;
5080 
5081  if (attnamelist == NIL)
5082  {
5083  /* Generate default column list */
5084  int attr_count = tupDesc->natts;
5085  int i;
5086 
5087  for (i = 0; i < attr_count; i++)
5088  {
5089  if (TupleDescAttr(tupDesc, i)->attisdropped)
5090  continue;
5091  if (TupleDescAttr(tupDesc, i)->attgenerated)
5092  continue;
5093  attnums = lappend_int(attnums, i + 1);
5094  }
5095  }
5096  else
5097  {
5098  /* Validate the user-supplied list and extract attnums */
5099  ListCell *l;
5100 
5101  foreach(l, attnamelist)
5102  {
5103  char *name = strVal(lfirst(l));
5104  int attnum;
5105  int i;
5106 
5107  /* Lookup column name */
5108  attnum = InvalidAttrNumber;
5109  for (i = 0; i < tupDesc->natts; i++)
5110  {
5111  Form_pg_attribute att = TupleDescAttr(tupDesc, i);
5112 
5113  if (att->attisdropped)
5114  continue;
5115  if (namestrcmp(&(att->attname), name) == 0)
5116  {
5117  if (att->attgenerated)
5118  ereport(ERROR,
5119  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
5120  errmsg("column \"%s\" is a generated column",
5121  name),
5122  errdetail("Generated columns cannot be used in COPY.")));
5123  attnum = att->attnum;
5124  break;
5125  }
5126  }
5127  if (attnum == InvalidAttrNumber)
5128  {
5129  if (rel != NULL)
5130  ereport(ERROR,
5131  (errcode(ERRCODE_UNDEFINED_COLUMN),
5132  errmsg("column \"%s\" of relation \"%s\" does not exist",
5133  name, RelationGetRelationName(rel))));
5134  else
5135  ereport(ERROR,
5136  (errcode(ERRCODE_UNDEFINED_COLUMN),
5137  errmsg("column \"%s\" does not exist",
5138  name)));
5139  }
5140  /* Check for duplicates */
5141  if (list_member_int(attnums, attnum))
5142  ereport(ERROR,
5143  (errcode(ERRCODE_DUPLICATE_COLUMN),
5144  errmsg("column \"%s\" specified more than once",
5145  name)));
5146  attnums = lappend_int(attnums, attnum);
5147  }
5148  }
5149 
5150  return attnums;
5151 }
5152 
5153 
5154 /*
5155  * copy_dest_startup --- executor startup
5156  */
5157 static void
5158 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
5159 {
5160  /* no-op */
5161 }
5162 
5163 /*
5164  * copy_dest_receive --- receive one tuple
5165  */
5166 static bool
5167 copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
5168 {
5169  DR_copy *myState = (DR_copy *) self;
5170  CopyState cstate = myState->cstate;
5171 
5172  /* Send the data */
5173  CopyOneRowTo(cstate, slot);
5174  myState->processed++;
5175 
5176  return true;
5177 }
5178 
5179 /*
5180  * copy_dest_shutdown --- executor end
5181  */
5182 static void
5183 copy_dest_shutdown(DestReceiver *self)
5184 {
5185  /* no-op */
5186 }
5187 
5188 /*
5189  * copy_dest_destroy --- release DestReceiver object
5190  */
5191 static void
5192 copy_dest_destroy(DestReceiver *self)
5193 {
5194  pfree(self);
5195 }
5196 
5197 /*
5198  * CreateCopyDestReceiver -- create a suitable DestReceiver object
5199  */
5200 DestReceiver *
5201 CreateCopyDestReceiver(void)
5202 {
5203  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
5204 
5205  self->pub.receiveSlot = copy_dest_receive;
5206  self->pub.rStartup = copy_dest_startup;
5207  self->pub.rShutdown = copy_dest_shutdown;
5208  self->pub.rDestroy = copy_dest_destroy;
5209  self->pub.mydest = DestCopyOut;
5210 
5211  self->cstate = NULL; /* will be set later */
5212  self->processed = 0;
5213 
5214  return (DestReceiver *) self;
5215 }
table_slot_create
TupleTableSlot * table_slot_create(Relation relation, List **reglist)
Definition: tableam.c:77
CopyMultiInsertBuffer
struct CopyMultiInsertBuffer CopyMultiInsertBuffer
int16
signed short int16
Definition: c.h:354
ResTarget::indirection
List * indirection
Definition: parsenodes.h:441
NextCopyFromRawFields
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3635
ResultRelInfo::ri_NumIndices
int ri_NumIndices
Definition: execnodes.h:414
CopyStateData::whereClause
Node * whereClause
Definition: copy.c:156
ExecCopySlot
static TupleTableSlot * ExecCopySlot(TupleTableSlot *dstslot, TupleTableSlot *srcslot)
Definition: tuptable.h:475
NIL
#define NIL
Definition: pg_list.h:65
CopyStmt::whereClause
Node * whereClause
Definition: parsenodes.h:2016
TupleTableSlot::tts_tableOid
Oid tts_tableOid
Definition: tuptable.h:131
pq_sendint16
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
CommandId
uint32 CommandId
Definition: c.h:527
ExecInitRangeTable
void ExecInitRangeTable(EState *estate, List *rangeTable)
Definition: execUtils.c:724
CopyReadBinaryAttribute
static Datum CopyReadBinaryAttribute(CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4769
GetDecimalFromHex
static int GetDecimalFromHex(char hex)
Definition: copy.c:4344
FmgrInfo
Definition: fmgr.h:56
MAX_COPY_DATA_DISPLAY
#define MAX_COPY_DATA_DISPLAY
CopyStateData::csv_mode
bool csv_mode
Definition: copy.c:138
SendCopyEnd
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:471
InitResultRelInfo
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1277
ResultRelInfo::ri_RelationDesc
Relation ri_RelationDesc
Definition: execnodes.h:411
CopyStateData::range_table
List * range_table
Definition: copy.c:184
BeginCopy
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1462
UpdateActiveSnapshotCommandId
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:783
IsA
#define IsA(nodeptr, _type_)
Definition: nodes.h:577
CopyReadLineText
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3980
MemoryContextDelete
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:211
AllocSetContextCreate
#define AllocSetContextCreate
Definition: memutils.h:170
HOLD_CANCEL_INTERRUPTS
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:124
CopyMultiInsertInfoStore
static void CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, TupleTableSlot *slot, int tuplen, uint64 lineno)
Definition: copy.c:2634
contain_volatile_functions_not_nextval
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:776
table_close
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:133
ResTarget::val
Node * val
Definition: parsenodes.h:442
CopyGetInt32
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:739
errhint
int errhint(const char *fmt,...)
Definition: elog.c:1071
ExecARInsertTriggers
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2268
CopyMultiInsertInfo::estate
EState * estate
Definition: copy.c:279
pg_char_to_encoding
int pg_char_to_encoding(const char *name)
Definition: encnames.c:550
CopyStateData::raw_fields
char ** raw_fields
Definition: copy.c:202
EOL_NL
Definition: copy.c:83
getTypeOutputInfo
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2700
VARDATA
#define VARDATA(PTR)
Definition: postgres.h:302
EndCopy
static void EndCopy(CopyState cstate)
Definition: copy.c:1803
CopyStateData::binary
bool binary
Definition: copy.c:136
ResultRelInfo::ri_CopyMultiInsertBuffer
struct CopyMultiInsertBuffer * ri_CopyMultiInsertBuffer
Definition: execnodes.h:491
pq_flush
#define pq_flush()
Definition: libpq.h:39
CopyFromErrorCallback
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2238
CopyStmt::attlist
List * attlist
Definition: parsenodes.h:2010
SelectStmt::fromClause
List * fromClause
Definition: parsenodes.h:1596
ISOCTAL
#define ISOCTAL(c)
Definition: copy.c:62
ExecClearTuple
static TupleTableSlot * ExecClearTuple(TupleTableSlot *slot)
Definition: tuptable.h:425
OCTVALUE
#define OCTVALUE(c)
Definition: copy.c:63
ResetPerTupleExprContext
#define ResetPerTupleExprContext(estate)
Definition: executor.h:515
RelationGetDescr
#define RelationGetDescr(relation)
Definition: rel.h:461
LOCKMODE
int LOCKMODE
Definition: lockdefs.h:26
ResTarget::name
char * name
Definition: parsenodes.h:440
GetUserId
Oid GetUserId(void)
Definition: miscinit.c:439
TABLE_INSERT_FROZEN
#define TABLE_INSERT_FROZEN
Definition: tableam.h:133
CopyStateData::need_transcoding
bool need_transcoding
Definition: copy.c:126
castNode
#define castNode(_type_, nodeptr)
Definition: nodes.h:595
FreeQueryDesc
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
CopyStateData::in_functions
FmgrInfo * in_functions
Definition: copy.c:179
CopyStateData::num_defaults
AttrNumber num_defaults
Definition: copy.c:178
CopyStateData::attnumlist
List * attnumlist
Definition: copy.c:132
VARSIZE
#define VARSIZE(PTR)
Definition: postgres.h:303
pq_getmsgstring
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:581
CopyStateData::filename
char * filename
Definition: copy.c:133
FdwRoutine::BeginForeignInsert
BeginForeignInsert_function BeginForeignInsert
Definition: fdwapi.h:215
ModifyTableState::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: execnodes.h:1172
TupleDescAttr
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
CopyMultiInsertInfo
struct CopyMultiInsertInfo CopyMultiInsertInfo
BeginCopyFrom
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:3386
VARHDRSZ
#define VARHDRSZ
Definition: c.h:561
FdwRoutine::ExecForeignInsert
ExecForeignInsert_function ExecForeignInsert
Definition: fdwapi.h:211
ResultRelInfo::ri_PartitionInfo
struct PartitionRoutingInfo * ri_PartitionInfo
Definition: execnodes.h:488
PlannedStmt::relationOids
List * relationOids
Definition: plannodes.h:86
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1186
pg_hton16
#define pg_hton16(x)
Definition: pg_bswap.h:120
ExprContext::ecxt_per_tuple_memory
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:232
ReceiveCopyBegin
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:435
XLogIsNeeded
#define XLogIsNeeded()
Definition: xlog.h:182
pg_ntoh16
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
DR_copy
Definition: copy.c:229
ExecComputeStoredGenerated
void ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot, CmdType cmdtype)
Definition: nodeModifyTable.c:249
makeStringInfo
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
CopyMultiInsertInfoSetupBuffer
static void CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri)
Definition: copy.c:2365
MAX_BUFFERED_TUPLES
#define MAX_BUFFERED_TUPLES
Definition: copy.c:246
RelationData::rd_islocaltemp
bool rd_islocaltemp
Definition: rel.h:60
CopyMultiInsertInfo::mycid
CommandId mycid
Definition: copy.c:280
EXPR_KIND_COPY_WHERE
Definition: parse_node.h:79
S_IWOTH
#define S_IWOTH
Definition: win32_port.h:287
expression_planner
Expr * expression_planner(Expr *expr)
Definition: planner.c:6044
ExecutorStart
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:143
CopyMultiInsertBuffer::linenos
uint64 linenos[MAX_BUFFERED_TUPLES]
Definition: copy.c:264
transformExpr
Node * transformExpr(ParseState *pstate, Node *expr, ParseExprKind exprKind)
Definition: parse_expr.c:145
DR_copy::pub
DestReceiver pub
Definition: copy.c:231
ProcessCopyOptions
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1111
ExecConstraints
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1897
CopyStateData::line_buf
StringInfoData line_buf
Definition: copy.c:211
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
IF_NEED_REFILL_AND_EOF_BREAK
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:313
GetActiveSnapshot
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:841
AccessShareLock
#define AccessShareLock
Definition: lockdefs.h:36
addNSItemToQuery
void addNSItemToQuery(ParseState *pstate, ParseNamespaceItem *nsitem, bool addToJoinList, bool addToRelNameSpace, bool addToVarNameSpace)
Definition: parse_relation.c:2466
CopyStateData
struct CopyStateData CopyStateData
Node
Definition: nodes.h:526
strVal
#define strVal(v)
Definition: value.h:54
cur
struct cursor * cur
Definition: ecpg.c:28
CopyStateData::raw_buf_index
int raw_buf_index
Definition: copy.c:224
CopyAttributeOutText
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4828
CopyStateData::line_buf_valid
bool line_buf_valid
Definition: copy.c:213
ThereAreNoPriorRegisteredSnapshots
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1689
DR_copy::cstate
CopyState cstate
Definition: copy.c:232
errcode
int errcode(int sqlerrcode)
Definition: elog.c:610
PG_BINARY_W
#define PG_BINARY_W
Definition: c.h:1237
namestrcmp
int namestrcmp(Name name, const char *str)
Definition: name.c:287
parse_relation.h
MemSet
#define MemSet(start, val, len)
Definition: c.h:971
generate_unaccent_rules.dest
dest
Definition: generate_unaccent_rules.py:282
eval_const_expressions
Node * eval_const_expressions(PlannerInfo *root, Node *node)
Definition: clauses.c:2255
CopyFrom
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2657
ModifyTableState::operation
CmdType operation
Definition: execnodes.h:1164
parse_expr.h
RelationData::rd_newRelfilenodeSubid
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:86
pq_putemptymessage
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
table_finish_bulk_insert
static void table_finish_bulk_insert(Relation rel, int options)
Definition: tableam.h:1350
TupleTableSlot::tts_values
Datum * tts_values
Definition: tuptable.h:126
FirstLowInvalidHeapAttributeNumber
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:27
ClosePipeToProgram
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1768
MemoryContextReset
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:136
CopyReadAttributesCSV
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4600
PopActiveSnapshot
void PopActiveSnapshot(void)
Definition: snapmgr.c:814
RangeTblEntry::requiredPerms
AclMode requiredPerms
Definition: parsenodes.h:1117
contain_volatile_functions
bool contain_volatile_functions(Node *clause)
Definition: clauses.c:726
SIGPIPE
#define SIGPIPE
Definition: win32_port.h:158
PlanState::state
EState * state
Definition: execnodes.h:945
CopyStateData::force_quote_flags
bool * force_quote_flags
Definition: copy.c:148
table_scan_getnextslot
static bool table_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot)
Definition: tableam.h:904
RelationData::rd_rel
Form_pg_class rd_rel
Definition: rel.h:89
Oid
unsigned int Oid
Definition: postgres_ext.h:31
pg_analyze_and_rewrite
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:676
ExecQual
static bool ExecQual(ExprState *state, ExprContext *econtext)
Definition: executor.h:370
CopyStateData::delim
char * delim
Definition: copy.c:143
pg_type.h
PG_PROTOCOL_MAJOR
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Query::utilityStmt
Node * utilityStmt
Definition: parsenodes.h:120
CopyStmt::is_program
bool is_program
Definition: parsenodes.h:2013
linitial_node
#define linitial_node(type, l)
Definition: pg_list.h:198
CopyStateData::volatile_defexprs
bool volatile_defexprs
Definition: copy.c:183
pg_wchar.h
ErrorContextCallback::callback
void(* callback)(void *arg)
Definition: elog.h:229
ErrorContextCallback::previous
struct ErrorContextCallback * previous
Definition: elog.h:228
CopyMultiInsertInfoInit
static void CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, CopyState cstate, EState *estate, CommandId mycid, int ti_options)
Definition: copy.c:2385
PG_BINARY_R
#define PG_BINARY_R
Definition: c.h:1236
CopyMultiInsertBuffer::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: copy.c:261
copy_dest_destroy
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:5192
CopyStateData::force_null_flags
bool * force_null_flags
Definition: copy.c:152
wait_result_to_str
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
CopyStateData::rowcontext
MemoryContext rowcontext
Definition: copy.c:173
parse_coerce.h
_DestReceiver
Definition: dest.h:116
CopyStateData::line_buf_converted
bool line_buf_converted
Definition: copy.c:212
table_tuple_insert
static void table_tuple_insert(Relation rel, TupleTableSlot *slot, CommandId cid, int options, struct BulkInsertStateData *bistate)
Definition: tableam.h:1155
pq_beginmessage
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
ExecInitQual
ExprState * ExecInitQual(List *qual, PlanState *parent)
Definition: execExpr.c:209
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:692
int32
signed int int32
Definition: c.h:355
ClosePipeStream
int ClosePipeStream(FILE *file)
Definition: fd.c:2729
errdetail_internal
int errdetail_internal(const char *fmt,...)
Definition: elog.c:984
CopyStateData::convert_select_flags
bool * convert_select_flags
Definition: copy.c:155
CopySendChar
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:512
OutputFunctionCall
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1575
ExecOpenIndices
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:151
addRangeTableEntryForRelation
ParseNamespaceItem * addRangeTableEntryForRelation(ParseState *pstate, Relation rel, int lockmode, Alias *alias, bool inh, bool inFromCl)
Definition: parse_relation.c:1497
ColumnRef::location
int location
Definition: parsenodes.h:236
CopyStateData::copy_dest
CopyDest copy_dest
Definition: copy.c:117
ResTarget::location
int location
Definition: parsenodes.h:443
pq_sendbyte
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
REFILL_LINEBUF
#define REFILL_LINEBUF
Definition: copy.c:330
error_context_stack
ErrorContextCallback * error_context_stack
Definition: elog.c:92
CopyStateData::cur_lineno
uint64 cur_lineno
Definition: copy.c:160
list_make1
#define list_make1(x1)
Definition: pg_list.h:227
CopyStateData::null_print
char * null_print
Definition: copy.c:140
CopyStateData::cur_attname
const char * cur_attname
Definition: copy.c:161
assign_expr_collations
void assign_expr_collations(ParseState *pstate, Node *expr)
Definition: parse_collate.c:177
ExecutorEnd
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:462
COPY_FILE
Definition: copy.c:71