PostgreSQL Source Code  git master
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 
21 #include "access/heapam.h"
22 #include "access/htup_details.h"
23 #include "access/sysattr.h"
24 #include "access/tableam.h"
25 #include "access/xact.h"
26 #include "access/xlog.h"
27 #include "catalog/dependency.h"
28 #include "catalog/pg_authid.h"
29 #include "catalog/pg_type.h"
30 #include "commands/copy.h"
31 #include "commands/defrem.h"
32 #include "commands/trigger.h"
33 #include "executor/execPartition.h"
34 #include "executor/executor.h"
35 #include "executor/nodeModifyTable.h"
36 #include "executor/tuptable.h"
37 #include "foreign/fdwapi.h"
38 #include "libpq/libpq.h"
39 #include "libpq/pqformat.h"
40 #include "mb/pg_wchar.h"
41 #include "miscadmin.h"
42 #include "nodes/makefuncs.h"
43 #include "optimizer/optimizer.h"
44 #include "parser/parse_coerce.h"
45 #include "parser/parse_collate.h"
46 #include "parser/parse_expr.h"
47 #include "parser/parse_relation.h"
48 #include "port/pg_bswap.h"
49 #include "rewrite/rewriteHandler.h"
50 #include "storage/fd.h"
51 #include "tcop/tcopprot.h"
52 #include "utils/acl.h"
53 #include "utils/builtins.h"
54 #include "utils/lsyscache.h"
55 #include "utils/memutils.h"
56 #include "utils/partcache.h"
57 #include "utils/portal.h"
58 #include "utils/rel.h"
59 #include "utils/rls.h"
60 #include "utils/snapmgr.h"
61 
62 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
63 #define OCTVALUE(c) ((c) - '0')
64 
65 /*
66  * Represents the different source/dest cases we need to worry about at
67  * the bottom level
68  */
69 typedef enum CopyDest
70 {
71  COPY_FILE, /* to/from file (or a piped program) */
72  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
73  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
74  COPY_CALLBACK /* to/from callback function */
75 } CopyDest;
76 
77 /*
78  * Represents the end-of-line terminator type of the input
79  */
80 typedef enum EolType
81 {
82  EOL_UNKNOWN,
83  EOL_NL,
84  EOL_CR,
85  EOL_CRNL
86 } EolType;
87 
88 /*
89  * Represents the heap insert method to be used during COPY FROM.
90  */
91 typedef enum CopyInsertMethod
92 {
93  CIM_SINGLE, /* use table_tuple_insert or fdw routine */
94  CIM_MULTI, /* always use table_multi_insert */
95  CIM_MULTI_CONDITIONAL /* use table_multi_insert only if valid */
96 } CopyInsertMethod;
97 
98 /*
99  * This struct contains all the state variables used throughout a COPY
100  * operation. For simplicity, we use the same struct for all variants of COPY,
101  * even though some fields are used in only some cases.
102  *
103  * Multi-byte encodings: all supported client-side encodings encode multi-byte
104  * characters by having the first byte's high bit set. Subsequent bytes of the
105  * character can have the high bit not set. When scanning data in such an
106  * encoding to look for a match to a single-byte (ie ASCII) character, we must
107  * use the full pg_encoding_mblen() machinery to skip over multibyte
108  * characters, else we might find a false match to a trailing byte. In
109  * supported server encodings, there is no possibility of a false match, and
110  * it's faster to make useless comparisons to trailing bytes than it is to
111  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
112  * when we have to do it the hard way.
113  */
114 typedef struct CopyStateData
115 {
116  /* low-level state data */
117  CopyDest copy_dest; /* type of copy source/destination */
118  FILE *copy_file; /* used if copy_dest == COPY_FILE */
119  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
120  * dest == COPY_NEW_FE in COPY FROM */
121  bool is_copy_from; /* COPY TO, or COPY FROM? */
122  bool reached_eof; /* true if we read to end of copy data (not
123  * all copy_dest types maintain this) */
124  EolType eol_type; /* EOL type of input */
125  int file_encoding; /* file or remote side's character encoding */
126  bool need_transcoding; /* file encoding diff from server? */
127  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
128 
129  /* parameters from the COPY command */
130  Relation rel; /* relation to copy to or from */
131  QueryDesc *queryDesc; /* executable query to copy from */
132  List *attnumlist; /* integer list of attnums to copy */
133  char *filename; /* filename, or NULL for STDIN/STDOUT */
134  bool is_program; /* is 'filename' a program to popen? */
135  copy_data_source_cb data_source_cb; /* function for reading data */
136  bool binary; /* binary format? */
137  bool freeze; /* freeze rows on loading? */
138  bool csv_mode; /* Comma Separated Value format? */
139  bool header_line; /* CSV header line? */
140  char *null_print; /* NULL marker string (server encoding!) */
141  int null_print_len; /* length of same */
142  char *null_print_client; /* same converted to file encoding */
143  char *delim; /* column delimiter (must be 1 byte) */
144  char *quote; /* CSV quote char (must be 1 byte) */
145  char *escape; /* CSV escape char (must be 1 byte) */
146  List *force_quote; /* list of column names */
147  bool force_quote_all; /* FORCE_QUOTE *? */
148  bool *force_quote_flags; /* per-column CSV FQ flags */
149  List *force_notnull; /* list of column names */
150  bool *force_notnull_flags; /* per-column CSV FNN flags */
151  List *force_null; /* list of column names */
152  bool *force_null_flags; /* per-column CSV FN flags */
153  bool convert_selectively; /* do selective binary conversion? */
154  List *convert_select; /* list of column names (can be NIL) */
155  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
156  Node *whereClause; /* WHERE condition (or NULL) */
157 
158  /* these are just for error messages, see CopyFromErrorCallback */
159  const char *cur_relname; /* table name for error messages */
160  uint64 cur_lineno; /* line number for error messages */
161  const char *cur_attname; /* current att for error messages */
162  const char *cur_attval; /* current att value for error messages */
163 
164  /*
165  * Working state for COPY TO/FROM
166  */
167  MemoryContext copycontext; /* per-copy execution context */
168 
169  /*
170  * Working state for COPY TO
171  */
172  FmgrInfo *out_functions; /* lookup info for output functions */
173  MemoryContext rowcontext; /* per-row evaluation context */
174 
175  /*
176  * Working state for COPY FROM
177  */
178  AttrNumber num_defaults;
179  FmgrInfo *in_functions; /* array of input functions for each attrs */
180  Oid *typioparams; /* array of element types for in_functions */
181  int *defmap; /* array of default att numbers */
182  ExprState **defexprs; /* array of default att expressions */
183  bool volatile_defexprs; /* is any of defexprs volatile? */
184  List *range_table;
185  ExprState *qualexpr;
186 
187  TransitionCaptureState *transition_capture;
188 
189  /*
190  * These variables are used to reduce overhead in COPY FROM.
191  *
192  * attribute_buf holds the separated, de-escaped text for each field of
193  * the current line. The CopyReadAttributes functions return arrays of
194  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
195  * the buffer on each cycle.
196  *
197  * In binary COPY FROM, attribute_buf holds the binary data for the
198  * current field, but the usage is otherwise similar.
199  */
200  StringInfoData attribute_buf;
201 
202  /* field raw data pointers found by COPY FROM */
203 
204  int max_fields;
205  char **raw_fields;
206 
207  /*
208  * Similarly, line_buf holds the whole input line being processed. The
209  * input cycle is first to read the whole line into line_buf, convert it
210  * to server encoding there, and then extract the individual attribute
211  * fields into attribute_buf. line_buf is preserved unmodified so that we
212  * can display it in error messages if appropriate. (In binary mode,
213  * line_buf is not used.)
214  */
215  StringInfoData line_buf;
216  bool line_buf_converted; /* converted to server encoding? */
217  bool line_buf_valid; /* contains the row being processed? */
218 
219  /*
220  * Finally, raw_buf holds raw data read from the data source (file or
221  * client connection). In text mode, CopyReadLine parses this data
222  * sufficiently to locate line boundaries, then transfers the data to
223  * line_buf and converts it. In binary mode, CopyReadBinaryData fetches
224  * appropriate amounts of data from this buffer. In both modes, we
225  * guarantee that there is a \0 at raw_buf[raw_buf_len].
226  */
227 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
228  char *raw_buf;
229  int raw_buf_index; /* next byte to process */
230  int raw_buf_len; /* total # of bytes stored */
231  /* Shorthand for number of unconsumed bytes available in raw_buf */
232 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
233 } CopyStateData;
234 
235 /* DestReceiver for COPY (query) TO */
236 typedef struct
237 {
238  DestReceiver pub; /* publicly-known function pointers */
239  CopyState cstate; /* CopyStateData for the command */
240  uint64 processed; /* # of tuples processed */
241 } DR_copy;
242 
243 
244 /*
245  * No more than this many tuples per CopyMultiInsertBuffer
246  *
247  * Caution: Don't make this too big, as we could end up with this many
248  * CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's
249  * multiInsertBuffers list. Increasing this can cause quadratic growth in
250  * memory requirements during copies into partitioned tables with a large
251  * number of partitions.
252  */
253 #define MAX_BUFFERED_TUPLES 1000
254 
255 /*
256  * Flush buffers if there are >= this many bytes, as counted by the input
257  * size, of tuples stored.
258  */
259 #define MAX_BUFFERED_BYTES 65535
260 
261 /* Trim the list of buffers back down to this number after flushing */
262 #define MAX_PARTITION_BUFFERS 32
263 
264 /* Stores multi-insert data related to a single relation in CopyFrom. */
265 typedef struct CopyMultiInsertBuffer
266 {
267  TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */
268  ResultRelInfo *resultRelInfo; /* ResultRelInfo for 'relid' */
269  BulkInsertState bistate; /* BulkInsertState for this rel */
270  int nused; /* number of 'slots' containing tuples */
271  uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy
272  * stream */
273 } CopyMultiInsertBuffer;
274 
275 /*
276  * Stores one or many CopyMultiInsertBuffers and details about the size and
277  * number of tuples which are stored in them. This allows multiple buffers to
278  * exist at once when COPYing into a partitioned table.
279  */
280 typedef struct CopyMultiInsertInfo
281 {
282  List *multiInsertBuffers; /* List of tracked CopyMultiInsertBuffers */
283  int bufferedTuples; /* number of tuples buffered over all buffers */
284  int bufferedBytes; /* number of bytes from all buffered tuples */
285  CopyState cstate; /* Copy state for this CopyMultiInsertInfo */
286  EState *estate; /* Executor state used for COPY */
287  CommandId mycid; /* Command Id used for COPY */
288  int ti_options; /* table insert options */
289 } CopyMultiInsertInfo;
290 
291 
292 /*
293  * These macros centralize code used to process line_buf and raw_buf buffers.
294  * They are macros because they often do continue/break control and to avoid
295  * function call overhead in tight COPY loops.
296  *
297  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
298  * prevent the continue/break processing from working. We end the "if (1)"
299  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
300  * any "else" in the calling code, and to avoid any compiler warnings about
301  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
302  */
303 
304 /*
305  * This keeps the character read at the top of the loop in the buffer
306  * even if there is more than one read-ahead.
307  */
308 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
309 if (1) \
310 { \
311  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
312  { \
313  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
314  need_data = true; \
315  continue; \
316  } \
317 } else ((void) 0)
318 
319 /* This consumes the remainder of the buffer and breaks */
320 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
321 if (1) \
322 { \
323  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
324  { \
325  if (extralen) \
326  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
327  /* backslash just before EOF, treat as data char */ \
328  result = true; \
329  break; \
330  } \
331 } else ((void) 0)
332 
333 /*
334  * Transfer any approved data to line_buf; must do this to be sure
335  * there is some room in raw_buf.
336  */
337 #define REFILL_LINEBUF \
338 if (1) \
339 { \
340  if (raw_buf_ptr > cstate->raw_buf_index) \
341  { \
342  appendBinaryStringInfo(&cstate->line_buf, \
343  cstate->raw_buf + cstate->raw_buf_index, \
344  raw_buf_ptr - cstate->raw_buf_index); \
345  cstate->raw_buf_index = raw_buf_ptr; \
346  } \
347 } else ((void) 0)
348 
349 /* Undo any read-ahead and jump out of the block. */
350 #define NO_END_OF_COPY_GOTO \
351 if (1) \
352 { \
353  raw_buf_ptr = prev_raw_ptr + 1; \
354  goto not_end_of_copy; \
355 } else ((void) 0)
356 
357 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
358 
359 
360 /* non-export function prototypes */
361 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
362  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
363  List *options);
364 static void EndCopy(CopyState cstate);
365 static void ClosePipeToProgram(CopyState cstate);
366 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
367  Oid queryRelId, const char *filename, bool is_program,
368  List *attnamelist, List *options);
369 static void EndCopyTo(CopyState cstate);
370 static uint64 DoCopyTo(CopyState cstate);
371 static uint64 CopyTo(CopyState cstate);
372 static void CopyOneRowTo(CopyState cstate, TupleTableSlot *slot);
373 static bool CopyReadLine(CopyState cstate);
374 static bool CopyReadLineText(CopyState cstate);
375 static int CopyReadAttributesText(CopyState cstate);
376 static int CopyReadAttributesCSV(CopyState cstate);
377 static Datum CopyReadBinaryAttribute(CopyState cstate, FmgrInfo *flinfo,
378  Oid typioparam, int32 typmod,
379  bool *isnull);
380 static void CopyAttributeOutText(CopyState cstate, char *string);
381 static void CopyAttributeOutCSV(CopyState cstate, char *string,
382  bool use_quote, bool single_attr);
383 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
384  List *attnamelist);
385 static char *limit_printout_length(const char *str);
386 
387 /* Low-level communications functions */
388 static void SendCopyBegin(CopyState cstate);
389 static void ReceiveCopyBegin(CopyState cstate);
390 static void SendCopyEnd(CopyState cstate);
391 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
392 static void CopySendString(CopyState cstate, const char *str);
393 static void CopySendChar(CopyState cstate, char c);
394 static void CopySendEndOfRow(CopyState cstate);
395 static int CopyGetData(CopyState cstate, void *databuf,
396  int minread, int maxread);
397 static void CopySendInt32(CopyState cstate, int32 val);
398 static bool CopyGetInt32(CopyState cstate, int32 *val);
399 static void CopySendInt16(CopyState cstate, int16 val);
400 static bool CopyGetInt16(CopyState cstate, int16 *val);
401 static bool CopyLoadRawBuf(CopyState cstate);
402 static int CopyReadBinaryData(CopyState cstate, char *dest, int nbytes);
403 
404 
405 /*
406  * Send copy start/stop messages for frontend copies. These have changed
407  * in past protocol redesigns.
408  */
409 static void
410 SendCopyBegin(CopyState cstate)
411 {
412  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
413  {
414  /* new way */
415  StringInfoData buf;
416  int natts = list_length(cstate->attnumlist);
417  int16 format = (cstate->binary ? 1 : 0);
418  int i;
419 
420  pq_beginmessage(&buf, 'H');
421  pq_sendbyte(&buf, format); /* overall format */
422  pq_sendint16(&buf, natts);
423  for (i = 0; i < natts; i++)
424  pq_sendint16(&buf, format); /* per-column formats */
425  pq_endmessage(&buf);
426  cstate->copy_dest = COPY_NEW_FE;
427  }
428  else
429  {
430  /* old way */
431  if (cstate->binary)
432  ereport(ERROR,
433  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
434  errmsg("COPY BINARY is not supported to stdout or from stdin")));
435  pq_putemptymessage('H');
436  /* grottiness needed for old COPY OUT protocol */
437  pq_startcopyout();
438  cstate->copy_dest = COPY_OLD_FE;
439  }
440 }
441 
442 static void
443 ReceiveCopyBegin(CopyState cstate)
444 {
445  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
446  {
447  /* new way */
448  StringInfoData buf;
449  int natts = list_length(cstate->attnumlist);
450  int16 format = (cstate->binary ? 1 : 0);
451  int i;
452 
453  pq_beginmessage(&buf, 'G');
454  pq_sendbyte(&buf, format); /* overall format */
455  pq_sendint16(&buf, natts);
456  for (i = 0; i < natts; i++)
457  pq_sendint16(&buf, format); /* per-column formats */
458  pq_endmessage(&buf);
459  cstate->copy_dest = COPY_NEW_FE;
460  cstate->fe_msgbuf = makeStringInfo();
461  }
462  else
463  {
464  /* old way */
465  if (cstate->binary)
466  ereport(ERROR,
467  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
468  errmsg("COPY BINARY is not supported to stdout or from stdin")));
469  pq_putemptymessage('G');
470  /* any error in old protocol will make us lose sync */
471  pq_startmsgread();
472  cstate->copy_dest = COPY_OLD_FE;
473  }
474  /* We *must* flush here to ensure FE knows it can send. */
475  pq_flush();
476 }
477 
478 static void
479 SendCopyEnd(CopyState cstate)
480 {
481  if (cstate->copy_dest == COPY_NEW_FE)
482  {
483  /* Shouldn't have any unsent data */
484  Assert(cstate->fe_msgbuf->len == 0);
485  /* Send Copy Done message */
486  pq_putemptymessage('c');
487  }
488  else
489  {
490  CopySendData(cstate, "\\.", 2);
491  /* Need to flush out the trailer (this also appends a newline) */
492  CopySendEndOfRow(cstate);
493  pq_endcopyout(false);
494  }
495 }
496 
497 /*----------
498  * CopySendData sends output data to the destination (file or frontend)
499  * CopySendString does the same for null-terminated strings
500  * CopySendChar does the same for single characters
501  * CopySendEndOfRow does the appropriate thing at end of each data row
502  * (data is not actually flushed except by CopySendEndOfRow)
503  *
504  * NB: no data conversion is applied by these functions
505  *----------
506  */
507 static void
508 CopySendData(CopyState cstate, const void *databuf, int datasize)
509 {
510  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
511 }
512 
513 static void
514 CopySendString(CopyState cstate, const char *str)
515 {
516  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
517 }
518 
519 static void
520 CopySendChar(CopyState cstate, char c)
521 {
522  appendStringInfoCharMacro(cstate->fe_msgbuf, c);
523 }
524 
525 static void
526 CopySendEndOfRow(CopyState cstate)
527 {
528  StringInfo fe_msgbuf = cstate->fe_msgbuf;
529 
530  switch (cstate->copy_dest)
531  {
532  case COPY_FILE:
533  if (!cstate->binary)
534  {
535  /* Default line termination depends on platform */
536 #ifndef WIN32
537  CopySendChar(cstate, '\n');
538 #else
539  CopySendString(cstate, "\r\n");
540 #endif
541  }
542 
543  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
544  cstate->copy_file) != 1 ||
545  ferror(cstate->copy_file))
546  {
547  if (cstate->is_program)
548  {
549  if (errno == EPIPE)
550  {
551  /*
552  * The pipe will be closed automatically on error at
553  * the end of transaction, but we might get a better
554  * error message from the subprocess' exit code than
555  * just "Broken Pipe"
556  */
557  ClosePipeToProgram(cstate);
558 
559  /*
560  * If ClosePipeToProgram() didn't throw an error, the
561  * program terminated normally, but closed the pipe
562  * first. Restore errno, and throw an error.
563  */
564  errno = EPIPE;
565  }
566  ereport(ERROR,
567  (errcode_for_file_access(),
568  errmsg("could not write to COPY program: %m")));
569  }
570  else
571  ereport(ERROR,
572  (errcode_for_file_access(),
573  errmsg("could not write to COPY file: %m")));
574  }
575  break;
576  case COPY_OLD_FE:
577  /* The FE/BE protocol uses \n as newline for all platforms */
578  if (!cstate->binary)
579  CopySendChar(cstate, '\n');
580 
581  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
582  {
583  /* no hope of recovering connection sync, so FATAL */
584  ereport(FATAL,
585  (errcode(ERRCODE_CONNECTION_FAILURE),
586  errmsg("connection lost during COPY to stdout")));
587  }
588  break;
589  case COPY_NEW_FE:
590  /* The FE/BE protocol uses \n as newline for all platforms */
591  if (!cstate->binary)
592  CopySendChar(cstate, '\n');
593 
594  /* Dump the accumulated row as one CopyData message */
595  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
596  break;
597  case COPY_CALLBACK:
598  Assert(false); /* Not yet supported. */
599  break;
600  }
601 
602  resetStringInfo(fe_msgbuf);
603 }
604 
605 /*
606  * CopyGetData reads data from the source (file or frontend)
607  *
608  * We attempt to read at least minread, and at most maxread, bytes from
609  * the source. The actual number of bytes read is returned; if this is
610  * less than minread, EOF was detected.
611  *
612  * Note: when copying from the frontend, we expect a proper EOF mark per
613  * protocol; if the frontend simply drops the connection, we raise error.
614  * It seems unwise to allow the COPY IN to complete normally in that case.
615  *
616  * NB: no data conversion is applied here.
617  */
618 static int
619 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
620 {
621  int bytesread = 0;
622 
623  switch (cstate->copy_dest)
624  {
625  case COPY_FILE:
626  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
627  if (ferror(cstate->copy_file))
628  ereport(ERROR,
629  (errcode_for_file_access(),
630  errmsg("could not read from COPY file: %m")));
631  if (bytesread == 0)
632  cstate->reached_eof = true;
633  break;
634  case COPY_OLD_FE:
635 
636  /*
637  * We cannot read more than minread bytes (which in practice is 1)
638  * because old protocol doesn't have any clear way of separating
639  * the COPY stream from following data. This is slow, but not any
640  * slower than the code path was originally, and we don't care
641  * much anymore about the performance of old protocol.
642  */
643  if (pq_getbytes((char *) databuf, minread))
644  {
645  /* Only a \. terminator is legal EOF in old protocol */
646  ereport(ERROR,
647  (errcode(ERRCODE_CONNECTION_FAILURE),
648  errmsg("unexpected EOF on client connection with an open transaction")));
649  }
650  bytesread = minread;
651  break;
652  case COPY_NEW_FE:
653  while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
654  {
655  int avail;
656 
657  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
658  {
659  /* Try to receive another message */
660  int mtype;
661 
662  readmessage:
663  HOLD_CANCEL_INTERRUPTS();
664  pq_startmsgread();
665  mtype = pq_getbyte();
666  if (mtype == EOF)
667  ereport(ERROR,
668  (errcode(ERRCODE_CONNECTION_FAILURE),
669  errmsg("unexpected EOF on client connection with an open transaction")));
670  if (pq_getmessage(cstate->fe_msgbuf, 0))
671  ereport(ERROR,
672  (errcode(ERRCODE_CONNECTION_FAILURE),
673  errmsg("unexpected EOF on client connection with an open transaction")));
674  RESUME_CANCEL_INTERRUPTS();
675  switch (mtype)
676  {
677  case 'd': /* CopyData */
678  break;
679  case 'c': /* CopyDone */
680  /* COPY IN correctly terminated by frontend */
681  cstate->reached_eof = true;
682  return bytesread;
683  case 'f': /* CopyFail */
684  ereport(ERROR,
685  (errcode(ERRCODE_QUERY_CANCELED),
686  errmsg("COPY from stdin failed: %s",
687  pq_getmsgstring(cstate->fe_msgbuf))));
688  break;
689  case 'H': /* Flush */
690  case 'S': /* Sync */
691 
692  /*
693  * Ignore Flush/Sync for the convenience of client
694  * libraries (such as libpq) that may send those
695  * without noticing that the command they just
696  * sent was COPY.
697  */
698  goto readmessage;
699  default:
700  ereport(ERROR,
701  (errcode(ERRCODE_PROTOCOL_VIOLATION),
702  errmsg("unexpected message type 0x%02X during COPY from stdin",
703  mtype)));
704  break;
705  }
706  }
707  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
708  if (avail > maxread)
709  avail = maxread;
710  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
711  databuf = (void *) ((char *) databuf + avail);
712  maxread -= avail;
713  bytesread += avail;
714  }
715  break;
716  case COPY_CALLBACK:
717  bytesread = cstate->data_source_cb(databuf, minread, maxread);
718  break;
719  }
720 
721  return bytesread;
722 }
723 
724 
725 /*
726  * These functions do apply some data conversion
727  */
728 
729 /*
730  * CopySendInt32 sends an int32 in network byte order
731  */
732 static inline void
733 CopySendInt32(CopyState cstate, int32 val)
734 {
735  uint32 buf;
736 
737  buf = pg_hton32((uint32) val);
738  CopySendData(cstate, &buf, sizeof(buf));
739 }
740 
741 /*
742  * CopyGetInt32 reads an int32 that appears in network byte order
743  *
744  * Returns true if OK, false if EOF
745  */
746 static inline bool
747 CopyGetInt32(CopyState cstate, int32 *val)
748 {
749  uint32 buf;
750 
751  if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
752  {
753  *val = 0; /* suppress compiler warning */
754  return false;
755  }
756  *val = (int32) pg_ntoh32(buf);
757  return true;
758 }
759 
760 /*
761  * CopySendInt16 sends an int16 in network byte order
762  */
763 static inline void
764 CopySendInt16(CopyState cstate, int16 val)
765 {
766  uint16 buf;
767 
768  buf = pg_hton16((uint16) val);
769  CopySendData(cstate, &buf, sizeof(buf));
770 }
771 
772 /*
773  * CopyGetInt16 reads an int16 that appears in network byte order
774  */
775 static inline bool
776 CopyGetInt16(CopyState cstate, int16 *val)
777 {
778  uint16 buf;
779 
780  if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
781  {
782  *val = 0; /* suppress compiler warning */
783  return false;
784  }
785  *val = (int16) pg_ntoh16(buf);
786  return true;
787 }
788 
789 
790 /*
791  * CopyLoadRawBuf loads some more data into raw_buf
792  *
793  * Returns true if able to obtain at least one more byte, else false.
794  *
795  * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
796  * of the buffer and then we load more data after that. This case occurs only
797  * when a multibyte character crosses a bufferload boundary.
798  */
799 static bool
800 CopyLoadRawBuf(CopyState cstate)
801 {
802  int nbytes = RAW_BUF_BYTES(cstate);
803  int inbytes;
804 
805  /* Copy down the unprocessed data if any. */
806  if (nbytes > 0)
807  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
808  nbytes);
809 
810  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
811  1, RAW_BUF_SIZE - nbytes);
812  nbytes += inbytes;
813  cstate->raw_buf[nbytes] = '\0';
814  cstate->raw_buf_index = 0;
815  cstate->raw_buf_len = nbytes;
816  return (inbytes > 0);
817 }
818 
819 /*
820  * CopyReadBinaryData
821  *
822  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
823  * and writes them to 'dest'. Returns the number of bytes read (which
824  * would be less than 'nbytes' only if we reach EOF).
825  */
826 static int
827 CopyReadBinaryData(CopyState cstate, char *dest, int nbytes)
828 {
829  int copied_bytes = 0;
830 
831  if (RAW_BUF_BYTES(cstate) >= nbytes)
832  {
833  /* Enough bytes are present in the buffer. */
834  memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
835  cstate->raw_buf_index += nbytes;
836  copied_bytes = nbytes;
837  }
838  else
839  {
840  /*
841  * Not enough bytes in the buffer, so must read from the file. Need
842  * to loop since 'nbytes' could be larger than the buffer size.
843  */
844  do
845  {
846  int copy_bytes;
847 
848  /* Load more data if buffer is empty. */
849  if (RAW_BUF_BYTES(cstate) == 0)
850  {
851  if (!CopyLoadRawBuf(cstate))
852  break; /* EOF */
853  }
854 
855  /* Transfer some bytes. */
856  copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
857  memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
858  cstate->raw_buf_index += copy_bytes;
859  dest += copy_bytes;
860  copied_bytes += copy_bytes;
861  } while (copied_bytes < nbytes);
862  }
863 
864  return copied_bytes;
865 }
866 
867 
868 /*
869  * DoCopy executes the SQL COPY statement
870  *
871  * Either unload or reload contents of table <relation>, depending on <from>.
872  * (<from> = true means we are inserting into the table.) In the "TO" case
873  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
874  * or DELETE query.
875  *
876  * If <pipe> is false, transfer is between the table and the file named
877  * <filename>. Otherwise, transfer is between the table and our regular
878  * input/output stream. The latter could be either stdin/stdout or a
879  * socket, depending on whether we're running under Postmaster control.
880  *
881  * Do not allow a Postgres user without the 'pg_read_server_files' or
882  * 'pg_write_server_files' role to read from or write to a file.
883  *
884  * Do not allow the copy if user doesn't have proper permission to access
885  * the table or the specifically requested columns.
886  */
887 void
888 DoCopy(ParseState *pstate, const CopyStmt *stmt,
889  int stmt_location, int stmt_len,
890  uint64 *processed)
891 {
892  CopyState cstate;
893  bool is_from = stmt->is_from;
894  bool pipe = (stmt->filename == NULL);
895  Relation rel;
896  Oid relid;
897  RawStmt *query = NULL;
898  Node *whereClause = NULL;
899 
900  /*
901  * Disallow COPY to/from file or program except to users with the
902  * appropriate role.
903  */
904  if (!pipe)
905  {
906  if (stmt->is_program)
907  {
908  if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_EXECUTE_SERVER_PROGRAM))
909  ereport(ERROR,
910  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
911  errmsg("must be superuser or a member of the pg_execute_server_program role to COPY to or from an external program"),
912  errhint("Anyone can COPY to stdout or from stdin. "
913  "psql's \\copy command also works for anyone.")));
914  }
915  else
916  {
917  if (is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_SERVER_FILES))
918  ereport(ERROR,
919  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
920  errmsg("must be superuser or a member of the pg_read_server_files role to COPY from a file"),
921  errhint("Anyone can COPY to stdout or from stdin. "
922  "psql's \\copy command also works for anyone.")));
923 
924  if (!is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_WRITE_SERVER_FILES))
925  ereport(ERROR,
926  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
927  errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
928  errhint("Anyone can COPY to stdout or from stdin. "
929  "psql's \\copy command also works for anyone.")));
930  }
931  }
932 
933  if (stmt->relation)
934  {
935  LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock;
936  ParseNamespaceItem *nsitem;
937  RangeTblEntry *rte;
938  TupleDesc tupDesc;
939  List *attnums;
940  ListCell *cur;
941 
942  Assert(!stmt->query);
943 
944  /* Open and lock the relation, using the appropriate lock type. */
945  rel = table_openrv(stmt->relation, lockmode);
946 
947  relid = RelationGetRelid(rel);
948 
949  nsitem = addRangeTableEntryForRelation(pstate, rel, lockmode,
950  NULL, false, false);
951  rte = nsitem->p_rte;
952  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
953 
954  if (stmt->whereClause)
955  {
956  /* add nsitem to query namespace */
957  addNSItemToQuery(pstate, nsitem, false, true, true);
958 
959  /* Transform the raw expression tree */
960  whereClause = transformExpr(pstate, stmt->whereClause, EXPR_KIND_COPY_WHERE);
961 
962  /* Make sure it yields a boolean result. */
963  whereClause = coerce_to_boolean(pstate, whereClause, "WHERE");
964 
965  /* we have to fix its collations too */
966  assign_expr_collations(pstate, whereClause);
967 
968  whereClause = eval_const_expressions(NULL, whereClause);
969 
970  whereClause = (Node *) canonicalize_qual((Expr *) whereClause, false);
971  whereClause = (Node *) make_ands_implicit((Expr *) whereClause);
972  }
973 
974  tupDesc = RelationGetDescr(rel);
975  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
976  foreach(cur, attnums)
977  {
978  int attno = lfirst_int(cur) -
979  FirstLowInvalidHeapAttributeNumber;
980 
981  if (is_from)
982  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
983  else
984  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
985  }
986  ExecCheckRTPerms(pstate->p_rtable, true);
987 
988  /*
989  * Permission check for row security policies.
990  *
991  * check_enable_rls will ereport(ERROR) if the user has requested
992  * something invalid and will otherwise indicate if we should enable
993  * RLS (returns RLS_ENABLED) or not for this COPY statement.
994  *
995  * If the relation has a row security policy and we are to apply it
996  * then perform a "query" copy and allow the normal query processing
997  * to handle the policies.
998  *
999  * If RLS is not enabled for this, then just fall through to the
1000  * normal non-filtering relation handling.
1001  */
1002  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
1003  {
1004  SelectStmt *select;
1005  ColumnRef *cr;
1006  ResTarget *target;
1007  RangeVar *from;
1008  List *targetList = NIL;
1009 
1010  if (is_from)
1011  ereport(ERROR,
1012  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1013  errmsg("COPY FROM not supported with row-level security"),
1014  errhint("Use INSERT statements instead.")));
1015 
1016  /*
1017  * Build target list
1018  *
1019  * If no columns are specified in the attribute list of the COPY
1020  * command, then the target list is 'all' columns. Therefore, '*'
1021  * should be used as the target list for the resulting SELECT
1022  * statement.
1023  *
1024  * In the case that columns are specified in the attribute list,
1025  * create a ColumnRef and ResTarget for each column and add them
1026  * to the target list for the resulting SELECT statement.
1027  */
1028  if (!stmt->attlist)
1029  {
1030  cr = makeNode(ColumnRef);
1031  cr->fields = list_make1(makeNode(A_Star));
1032  cr->location = -1;
1033 
1034  target = makeNode(ResTarget);
1035  target->name = NULL;
1036  target->indirection = NIL;
1037  target->val = (Node *) cr;
1038  target->location = -1;
1039 
1040  targetList = list_make1(target);
1041  }
1042  else
1043  {
1044  ListCell *lc;
1045 
1046  foreach(lc, stmt->attlist)
1047  {
1048  /*
1049  * Build the ColumnRef for each column. The ColumnRef
1050  * 'fields' property is a String 'Value' node (see
1051  * nodes/value.h) that corresponds to the column name
1052  * respectively.
1053  */
1054  cr = makeNode(ColumnRef);
1055  cr->fields = list_make1(lfirst(lc));
1056  cr->location = -1;
1057 
1058  /* Build the ResTarget and add the ColumnRef to it. */
1059  target = makeNode(ResTarget);
1060  target->name = NULL;
1061  target->indirection = NIL;
1062  target->val = (Node *) cr;
1063  target->location = -1;
1064 
1065  /* Add each column to the SELECT statement's target list */
1066  targetList = lappend(targetList, target);
1067  }
1068  }
1069 
1070  /*
1071  * Build RangeVar for from clause, fully qualified based on the
1072  * relation which we have opened and locked.
1073  */
1074  from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
1075  pstrdup(RelationGetRelationName(rel)),
1076  -1);
1077 
1078  /* Build query */
1079  select = makeNode(SelectStmt);
1080  select->targetList = targetList;
1081  select->fromClause = list_make1(from);
1082 
1083  query = makeNode(RawStmt);
1084  query->stmt = (Node *) select;
1085  query->stmt_location = stmt_location;
1086  query->stmt_len = stmt_len;
1087 
1088  /*
1089  * Close the relation for now, but keep the lock on it to prevent
1090  * changes between now and when we start the query-based COPY.
1091  *
1092  * We'll reopen it later as part of the query-based COPY.
1093  */
1094  table_close(rel, NoLock);
1095  rel = NULL;
1096  }
1097  }
1098  else
1099  {
1100  Assert(stmt->query);
1101 
1102  query = makeNode(RawStmt);
1103  query->stmt = stmt->query;
1104  query->stmt_location = stmt_location;
1105  query->stmt_len = stmt_len;
1106 
1107  relid = InvalidOid;
1108  rel = NULL;
1109  }
1110 
1111  if (is_from)
1112  {
1113  Assert(rel);
1114 
1115  /* check read-only transaction and parallel mode */
1116  if (XactReadOnly && !rel->rd_islocaltemp)
1117  PreventCommandIfReadOnly("COPY FROM");
1118 
1119  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
1120  NULL, stmt->attlist, stmt->options);
1121  cstate->whereClause = whereClause;
1122  *processed = CopyFrom(cstate); /* copy from file to database */
1123  EndCopyFrom(cstate);
1124  }
1125  else
1126  {
1127  cstate = BeginCopyTo(pstate, rel, query, relid,
1128  stmt->filename, stmt->is_program,
1129  stmt->attlist, stmt->options);
1130  *processed = DoCopyTo(cstate); /* copy from database to file */
1131  EndCopyTo(cstate);
1132  }
1133 
1134  if (rel != NULL)
1135  table_close(rel, NoLock);
1136 }
1137 
1138 /*
1139  * Process the statement option list for COPY.
1140  *
1141  * Scan the options list (a list of DefElem) and transpose the information
1142  * into cstate, applying appropriate error checking.
1143  *
1144  * cstate is assumed to be filled with zeroes initially.
1145  *
1146  * This is exported so that external users of the COPY API can sanity-check
1147  * a list of options. In that usage, cstate should be passed as NULL
1148  * (since external users don't know sizeof(CopyStateData)) and the collected
1149  * data is just leaked until CurrentMemoryContext is reset.
1150  *
1151  * Note that additional checking, such as whether column names listed in FORCE
1152  * QUOTE actually exist, has to be applied later. This just checks for
1153  * self-consistency of the options list.
1154  */
1155 void
1156 ProcessCopyOptions(ParseState *pstate,
1157  CopyState cstate,
1158  bool is_from,
1159  List *options)
1160 {
1161  bool format_specified = false;
1162  ListCell *option;
1163 
1164  /* Support external use for option sanity checking */
1165  if (cstate == NULL)
1166  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1167 
1168  cstate->is_copy_from = is_from;
1169 
1170  cstate->file_encoding = -1;
1171 
1172  /* Extract options from the statement node tree */
1173  foreach(option, options)
1174  {
1175  DefElem *defel = lfirst_node(DefElem, option);
1176 
1177  if (strcmp(defel->defname, "format") == 0)
1178  {
1179  char *fmt = defGetString(defel);
1180 
1181  if (format_specified)
1182  ereport(ERROR,
1183  (errcode(ERRCODE_SYNTAX_ERROR),
1184  errmsg("conflicting or redundant options"),
1185  parser_errposition(pstate, defel->location)));
1186  format_specified = true;
1187  if (strcmp(fmt, "text") == 0)
1188  /* default format */ ;
1189  else if (strcmp(fmt, "csv") == 0)
1190  cstate->csv_mode = true;
1191  else if (strcmp(fmt, "binary") == 0)
1192  cstate->binary = true;
1193  else
1194  ereport(ERROR,
1195  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1196  errmsg("COPY format \"%s\" not recognized", fmt),
1197  parser_errposition(pstate, defel->location)));
1198  }
1199  else if (strcmp(defel->defname, "freeze") == 0)
1200  {
1201  if (cstate->freeze)
1202  ereport(ERROR,
1203  (errcode(ERRCODE_SYNTAX_ERROR),
1204  errmsg("conflicting or redundant options"),
1205  parser_errposition(pstate, defel->location)));
1206  cstate->freeze = defGetBoolean(defel);
1207  }
1208  else if (strcmp(defel->defname, "delimiter") == 0)
1209  {
1210  if (cstate->delim)
1211  ereport(ERROR,
1212  (errcode(ERRCODE_SYNTAX_ERROR),
1213  errmsg("conflicting or redundant options"),
1214  parser_errposition(pstate, defel->location)));
1215  cstate->delim = defGetString(defel);
1216  }
1217  else if (strcmp(defel->defname, "null") == 0)
1218  {
1219  if (cstate->null_print)
1220  ereport(ERROR,
1221  (errcode(ERRCODE_SYNTAX_ERROR),
1222  errmsg("conflicting or redundant options"),
1223  parser_errposition(pstate, defel->location)));
1224  cstate->null_print = defGetString(defel);
1225  }
1226  else if (strcmp(defel->defname, "header") == 0)
1227  {
1228  if (cstate->header_line)
1229  ereport(ERROR,
1230  (errcode(ERRCODE_SYNTAX_ERROR),
1231  errmsg("conflicting or redundant options"),
1232  parser_errposition(pstate, defel->location)));
1233  cstate->header_line = defGetBoolean(defel);
1234  }
1235  else if (strcmp(defel->defname, "quote") == 0)
1236  {
1237  if (cstate->quote)
1238  ereport(ERROR,
1239  (errcode(ERRCODE_SYNTAX_ERROR),
1240  errmsg("conflicting or redundant options"),
1241  parser_errposition(pstate, defel->location)));
1242  cstate->quote = defGetString(defel);
1243  }
1244  else if (strcmp(defel->defname, "escape") == 0)
1245  {
1246  if (cstate->escape)
1247  ereport(ERROR,
1248  (errcode(ERRCODE_SYNTAX_ERROR),
1249  errmsg("conflicting or redundant options"),
1250  parser_errposition(pstate, defel->location)));
1251  cstate->escape = defGetString(defel);
1252  }
1253  else if (strcmp(defel->defname, "force_quote") == 0)
1254  {
1255  if (cstate->force_quote || cstate->force_quote_all)
1256  ereport(ERROR,
1257  (errcode(ERRCODE_SYNTAX_ERROR),
1258  errmsg("conflicting or redundant options"),
1259  parser_errposition(pstate, defel->location)));
1260  if (defel->arg && IsA(defel->arg, A_Star))
1261  cstate->force_quote_all = true;
1262  else if (defel->arg && IsA(defel->arg, List))
1263  cstate->force_quote = castNode(List, defel->arg);
1264  else
1265  ereport(ERROR,
1266  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1267  errmsg("argument to option \"%s\" must be a list of column names",
1268  defel->defname),
1269  parser_errposition(pstate, defel->location)));
1270  }
1271  else if (strcmp(defel->defname, "force_not_null") == 0)
1272  {
1273  if (cstate->force_notnull)
1274  ereport(ERROR,
1275  (errcode(ERRCODE_SYNTAX_ERROR),
1276  errmsg("conflicting or redundant options"),
1277  parser_errposition(pstate, defel->location)));
1278  if (defel->arg && IsA(defel->arg, List))
1279  cstate->force_notnull = castNode(List, defel->arg);
1280  else
1281  ereport(ERROR,
1282  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1283  errmsg("argument to option \"%s\" must be a list of column names",
1284  defel->defname),
1285  parser_errposition(pstate, defel->location)));
1286  }
1287  else if (strcmp(defel->defname, "force_null") == 0)
1288  {
1289  if (cstate->force_null)
1290  ereport(ERROR,
1291  (errcode(ERRCODE_SYNTAX_ERROR),
1292  errmsg("conflicting or redundant options")));
1293  if (defel->arg && IsA(defel->arg, List))
1294  cstate->force_null = castNode(List, defel->arg);
1295  else
1296  ereport(ERROR,
1297  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1298  errmsg("argument to option \"%s\" must be a list of column names",
1299  defel->defname),
1300  parser_errposition(pstate, defel->location)));
1301  }
1302  else if (strcmp(defel->defname, "convert_selectively") == 0)
1303  {
1304  /*
1305  * Undocumented, not-accessible-from-SQL option: convert only the
1306  * named columns to binary form, storing the rest as NULLs. It's
1307  * allowed for the column list to be NIL.
1308  */
1309  if (cstate->convert_selectively)
1310  ereport(ERROR,
1311  (errcode(ERRCODE_SYNTAX_ERROR),
1312  errmsg("conflicting or redundant options"),
1313  parser_errposition(pstate, defel->location)));
1314  cstate->convert_selectively = true;
1315  if (defel->arg == NULL || IsA(defel->arg, List))
1316  cstate->convert_select = castNode(List, defel->arg);
1317  else
1318  ereport(ERROR,
1319  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1320  errmsg("argument to option \"%s\" must be a list of column names",
1321  defel->defname),
1322  parser_errposition(pstate, defel->location)));
1323  }
1324  else if (strcmp(defel->defname, "encoding") == 0)
1325  {
1326  if (cstate->file_encoding >= 0)
1327  ereport(ERROR,
1328  (errcode(ERRCODE_SYNTAX_ERROR),
1329  errmsg("conflicting or redundant options"),
1330  parser_errposition(pstate, defel->location)));
1331  cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
1332  if (cstate->file_encoding < 0)
1333  ereport(ERROR,
1334  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1335  errmsg("argument to option \"%s\" must be a valid encoding name",
1336  defel->defname),
1337  parser_errposition(pstate, defel->location)));
1338  }
1339  else
1340  ereport(ERROR,
1341  (errcode(ERRCODE_SYNTAX_ERROR),
1342  errmsg("option \"%s\" not recognized",
1343  defel->defname),
1344  parser_errposition(pstate, defel->location)));
1345  }
1346 
1347  /*
1348  * Check for incompatible options (must do these two before inserting
1349  * defaults)
1350  */
1351  if (cstate->binary && cstate->delim)
1352  ereport(ERROR,
1353  (errcode(ERRCODE_SYNTAX_ERROR),
1354  errmsg("cannot specify DELIMITER in BINARY mode")));
1355 
1356  if (cstate->binary && cstate->null_print)
1357  ereport(ERROR,
1358  (errcode(ERRCODE_SYNTAX_ERROR),
1359  errmsg("cannot specify NULL in BINARY mode")));
1360 
1361  /* Set defaults for omitted options */
1362  if (!cstate->delim)
1363  cstate->delim = cstate->csv_mode ? "," : "\t";
1364 
1365  if (!cstate->null_print)
1366  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1367  cstate->null_print_len = strlen(cstate->null_print);
1368 
1369  if (cstate->csv_mode)
1370  {
1371  if (!cstate->quote)
1372  cstate->quote = "\"";
1373  if (!cstate->escape)
1374  cstate->escape = cstate->quote;
1375  }
1376 
1377  /* Only single-byte delimiter strings are supported. */
1378  if (strlen(cstate->delim) != 1)
1379  ereport(ERROR,
1380  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1381  errmsg("COPY delimiter must be a single one-byte character")));
1382 
1383  /* Disallow end-of-line characters */
1384  if (strchr(cstate->delim, '\r') != NULL ||
1385  strchr(cstate->delim, '\n') != NULL)
1386  ereport(ERROR,
1387  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1388  errmsg("COPY delimiter cannot be newline or carriage return")));
1389 
1390  if (strchr(cstate->null_print, '\r') != NULL ||
1391  strchr(cstate->null_print, '\n') != NULL)
1392  ereport(ERROR,
1393  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1394  errmsg("COPY null representation cannot use newline or carriage return")));
1395 
1396  /*
1397  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1398  * backslash because it would be ambiguous. We can't allow the other
1399  * cases because data characters matching the delimiter must be
1400  * backslashed, and certain backslash combinations are interpreted
1401  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1402  * more than strictly necessary, but seems best for consistency and
1403  * future-proofing. Likewise we disallow all digits though only octal
1404  * digits are actually dangerous.
1405  */
1406  if (!cstate->csv_mode &&
1407  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1408  cstate->delim[0]) != NULL)
1409  ereport(ERROR,
1410  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1411  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1412 
1413  /* Check header */
1414  if (!cstate->csv_mode && cstate->header_line)
1415  ereport(ERROR,
1416  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1417  errmsg("COPY HEADER available only in CSV mode")));
1418 
1419  /* Check quote */
1420  if (!cstate->csv_mode && cstate->quote != NULL)
1421  ereport(ERROR,
1422  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1423  errmsg("COPY quote available only in CSV mode")));
1424 
1425  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1426  ereport(ERROR,
1427  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1428  errmsg("COPY quote must be a single one-byte character")));
1429 
1430  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1431  ereport(ERROR,
1432  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1433  errmsg("COPY delimiter and quote must be different")));
1434 
1435  /* Check escape */
1436  if (!cstate->csv_mode && cstate->escape != NULL)
1437  ereport(ERROR,
1438  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1439  errmsg("COPY escape available only in CSV mode")));
1440 
1441  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1442  ereport(ERROR,
1443  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1444  errmsg("COPY escape must be a single one-byte character")));
1445 
1446  /* Check force_quote */
1447  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1448  ereport(ERROR,
1449  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1450  errmsg("COPY force quote available only in CSV mode")));
1451  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1452  ereport(ERROR,
1453  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1454  errmsg("COPY force quote only available using COPY TO")));
1455 
1456  /* Check force_notnull */
1457  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1458  ereport(ERROR,
1459  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1460  errmsg("COPY force not null available only in CSV mode")));
1461  if (cstate->force_notnull != NIL && !is_from)
1462  ereport(ERROR,
1463  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1464  errmsg("COPY force not null only available using COPY FROM")));
1465 
1466  /* Check force_null */
1467  if (!cstate->csv_mode && cstate->force_null != NIL)
1468  ereport(ERROR,
1469  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1470  errmsg("COPY force null available only in CSV mode")));
1471 
1472  if (cstate->force_null != NIL && !is_from)
1473  ereport(ERROR,
1474  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1475  errmsg("COPY force null only available using COPY FROM")));
1476 
1477  /* Don't allow the delimiter to appear in the null string. */
1478  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1479  ereport(ERROR,
1480  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1481  errmsg("COPY delimiter must not appear in the NULL specification")));
1482 
1483  /* Don't allow the CSV quote char to appear in the null string. */
1484  if (cstate->csv_mode &&
1485  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1486  ereport(ERROR,
1487  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1488  errmsg("CSV quote character must not appear in the NULL specification")));
1489 }
1490 
1491 /*
1492  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1493  *
1494  * Iff <binary>, unload or reload in the binary format, as opposed to the
1495  * more wasteful but more robust and portable text format.
1496  *
1497  * Iff <oids>, unload or reload the format that includes OID information.
1498  * On input, we accept OIDs whether or not the table has an OID column,
1499  * but silently drop them if it does not. On output, we report an error
1500  * if the user asks for OIDs in a table that has none (not providing an
1501  * OID column might seem friendlier, but could seriously confuse programs).
1502  *
1503  * If in the text format, delimit columns with delimiter <delim> and print
1504  * NULL values as <null_print>.
1505  */
1506 static CopyState
1507 BeginCopy(ParseState *pstate,
1508  bool is_from,
1509  Relation rel,
1510  RawStmt *raw_query,
1511  Oid queryRelId,
1512  List *attnamelist,
1513  List *options)
1514 {
1515  CopyState cstate;
1516  TupleDesc tupDesc;
1517  int num_phys_attrs;
1518  MemoryContext oldcontext;
1519 
1520  /* Allocate workspace and zero all fields */
1521  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1522 
1523  /*
1524  * We allocate everything used by a cstate in a new memory context. This
1525  * avoids memory leaks during repeated use of COPY in a query.
1526  */
1527  cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
1528  "COPY",
1529  ALLOCSET_DEFAULT_SIZES);
1530 
1531  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1532 
1533  /* Extract options from the statement node tree */
1534  ProcessCopyOptions(pstate, cstate, is_from, options);
1535 
1536  /* Process the source/target relation or query */
1537  if (rel)
1538  {
1539  Assert(!raw_query);
1540 
1541  cstate->rel = rel;
1542 
1543  tupDesc = RelationGetDescr(cstate->rel);
1544  }
1545  else
1546  {
1547  List *rewritten;
1548  Query *query;
1549  PlannedStmt *plan;
1550  DestReceiver *dest;
1551 
1552  Assert(!is_from);
1553  cstate->rel = NULL;
1554 
1555  /*
1556  * Run parse analysis and rewrite. Note this also acquires sufficient
1557  * locks on the source table(s).
1558  *
1559  * Because the parser and planner tend to scribble on their input, we
1560  * make a preliminary copy of the source querytree. This prevents
1561  * problems in the case that the COPY is in a portal or plpgsql
1562  * function and is executed repeatedly. (See also the same hack in
1563  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1564  */
1565  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1566  pstate->p_sourcetext, NULL, 0,
1567  NULL);
1568 
1569  /* check that we got back something we can work with */
1570  if (rewritten == NIL)
1571  {
1572  ereport(ERROR,
1573  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1574  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1575  }
1576  else if (list_length(rewritten) > 1)
1577  {
1578  ListCell *lc;
1579 
1580  /* examine queries to determine which error message to issue */
1581  foreach(lc, rewritten)
1582  {
1583  Query *q = lfirst_node(Query, lc);
1584 
1585  if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
1586  ereport(ERROR,
1587  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1588  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1589  if (q->querySource == QSRC_NON_INSTEAD_RULE)
1590  ereport(ERROR,
1591  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1592  errmsg("DO ALSO rules are not supported for the COPY")));
1593  }
1594 
1595  ereport(ERROR,
1596  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1597  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1598  }
1599 
1600  query = linitial_node(Query, rewritten);
1601 
1602  /* The grammar allows SELECT INTO, but we don't support that */
1603  if (query->utilityStmt != NULL &&
1604  IsA(query->utilityStmt, CreateTableAsStmt))
1605  ereport(ERROR,
1606  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1607  errmsg("COPY (SELECT INTO) is not supported")));
1608 
1609  Assert(query->utilityStmt == NULL);
1610 
1611  /*
1612  * Similarly the grammar doesn't enforce the presence of a RETURNING
1613  * clause, but this is required here.
1614  */
1615  if (query->commandType != CMD_SELECT &&
1616  query->returningList == NIL)
1617  {
1618  Assert(query->commandType == CMD_INSERT ||
1619  query->commandType == CMD_UPDATE ||
1620  query->commandType == CMD_DELETE);
1621 
1622  ereport(ERROR,
1623  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1624  errmsg("COPY query must have a RETURNING clause")));
1625  }
1626 
1627  /* plan the query */
1628  plan = pg_plan_query(query, pstate->p_sourcetext,
1629  CURSOR_OPT_PARALLEL_OK, NULL);
1630 
1631  /*
1632  * With row level security and a user using "COPY relation TO", we
1633  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1634  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1635  * in any RLS clauses.
1636  *
1637  * When this happens, we are passed in the relid of the originally
1638  * found relation (which we have locked). As the planner will look up
1639  * the relation again, we double-check here to make sure it found the
1640  * same one that we have locked.
1641  */
1642  if (queryRelId != InvalidOid)
1643  {
1644  /*
1645  * Note that with RLS involved there may be multiple relations,
1646  * and while the one we need is almost certainly first, we don't
1647  * make any guarantees of that in the planner, so check the whole
1648  * list and make sure we find the original relation.
1649  */
1650  if (!list_member_oid(plan->relationOids, queryRelId))
1651  ereport(ERROR,
1652  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1653  errmsg("relation referenced by COPY statement has changed")));
1654  }
1655 
1656  /*
1657  * Use a snapshot with an updated command ID to ensure this query sees
1658  * results of any previously executed queries.
1659  */
1660  PushCopiedSnapshot(GetActiveSnapshot());
1661  UpdateActiveSnapshotCommandId();
1662 
1663  /* Create dest receiver for COPY OUT */
1664  dest = CreateDestReceiver(DestCopyOut);
1665  ((DR_copy *) dest)->cstate = cstate;
1666 
1667  /* Create a QueryDesc requesting no output */
1668  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1669  GetActiveSnapshot(),
1670  InvalidSnapshot,
1671  dest, NULL, NULL, 0);
1672 
1673  /*
1674  * Call ExecutorStart to prepare the plan for execution.
1675  *
1676  * ExecutorStart computes a result tupdesc for us
1677  */
1678  ExecutorStart(cstate->queryDesc, 0);
1679 
1680  tupDesc = cstate->queryDesc->tupDesc;
1681  }
1682 
1683  /* Generate or convert list of attributes to process */
1684  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1685 
1686  num_phys_attrs = tupDesc->natts;
1687 
1688  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1689  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1690  if (cstate->force_quote_all)
1691  {
1692  int i;
1693 
1694  for (i = 0; i < num_phys_attrs; i++)
1695  cstate->force_quote_flags[i] = true;
1696  }
1697  else if (cstate->force_quote)
1698  {
1699  List *attnums;
1700  ListCell *cur;
1701 
1702  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1703 
1704  foreach(cur, attnums)
1705  {
1706  int attnum = lfirst_int(cur);
1707  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1708 
1709  if (!list_member_int(cstate->attnumlist, attnum))
1710  ereport(ERROR,
1711  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1712  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1713  NameStr(attr->attname))));
1714  cstate->force_quote_flags[attnum - 1] = true;
1715  }
1716  }
1717 
1718  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1719  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1720  if (cstate->force_notnull)
1721  {
1722  List *attnums;
1723  ListCell *cur;
1724 
1725  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1726 
1727  foreach(cur, attnums)
1728  {
1729  int attnum = lfirst_int(cur);
1730  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1731 
1732  if (!list_member_int(cstate->attnumlist, attnum))
1733  ereport(ERROR,
1734  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1735  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1736  NameStr(attr->attname))));
1737  cstate->force_notnull_flags[attnum - 1] = true;
1738  }
1739  }
1740 
1741  /* Convert FORCE_NULL name list to per-column flags, check validity */
1742  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1743  if (cstate->force_null)
1744  {
1745  List *attnums;
1746  ListCell *cur;
1747 
1748  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1749 
1750  foreach(cur, attnums)
1751  {
1752  int attnum = lfirst_int(cur);
1753  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1754 
1755  if (!list_member_int(cstate->attnumlist, attnum))
1756  ereport(ERROR,
1757  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1758  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1759  NameStr(attr->attname))));
1760  cstate->force_null_flags[attnum - 1] = true;
1761  }
1762  }
1763 
1764  /* Convert convert_selectively name list to per-column flags */
1765  if (cstate->convert_selectively)
1766  {
1767  List *attnums;
1768  ListCell *cur;
1769 
1770  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1771 
1772  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1773 
1774  foreach(cur, attnums)
1775  {
1776  int attnum = lfirst_int(cur);
1777  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1778 
1779  if (!list_member_int(cstate->attnumlist, attnum))
1780  ereport(ERROR,
1781  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1782  errmsg_internal("selected column \"%s\" not referenced by COPY",
1783  NameStr(attr->attname))));
1784  cstate->convert_select_flags[attnum - 1] = true;
1785  }
1786  }
1787 
1788  /* Use client encoding when ENCODING option is not specified. */
1789  if (cstate->file_encoding < 0)
1790  cstate->file_encoding = pg_get_client_encoding();
1791 
1792  /*
1793  * Set up encoding conversion info. Even if the file and server encodings
1794  * are the same, we must apply pg_any_to_server() to validate data in
1795  * multibyte encodings.
1796  */
1797  cstate->need_transcoding =
1798  (cstate->file_encoding != GetDatabaseEncoding() ||
1799  pg_database_encoding_max_length() > 1);
1800  /* See Multibyte encoding comment above */
1801  cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
1802 
1803  cstate->copy_dest = COPY_FILE; /* default */
1804 
1805  MemoryContextSwitchTo(oldcontext);
1806 
1807  return cstate;
1808 }
1809 
1810 /*
1811  * Closes the pipe to an external program, checking the pclose() return code.
1812  */
1813 static void
1814 ClosePipeToProgram(CopyState cstate)
1815 {
1816  int pclose_rc;
1817 
1818  Assert(cstate->is_program);
1819 
1820  pclose_rc = ClosePipeStream(cstate->copy_file);
1821  if (pclose_rc == -1)
1822  ereport(ERROR,
1823  (errcode_for_file_access(),
1824  errmsg("could not close pipe to external command: %m")));
1825  else if (pclose_rc != 0)
1826  {
1827  /*
1828  * If we ended a COPY FROM PROGRAM before reaching EOF, then it's
1829  * expectable for the called program to fail with SIGPIPE, and we
1830  * should not report that as an error. Otherwise, SIGPIPE indicates a
1831  * problem.
1832  */
1833  if (cstate->is_copy_from && !cstate->reached_eof &&
1834  wait_result_is_signal(pclose_rc, SIGPIPE))
1835  return;
1836 
1837  ereport(ERROR,
1838  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1839  errmsg("program \"%s\" failed",
1840  cstate->filename),
1841  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1842  }
1843 }
1844 
1845 /*
1846  * Release resources allocated in a cstate for COPY TO/FROM.
1847  */
1848 static void
1849 EndCopy(CopyState cstate)
1850 {
1851  if (cstate->is_program)
1852  {
1853  ClosePipeToProgram(cstate);
1854  }
1855  else
1856  {
1857  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1858  ereport(ERROR,
1859  (errcode_for_file_access(),
1860  errmsg("could not close file \"%s\": %m",
1861  cstate->filename)));
1862  }
1863 
1864  MemoryContextDelete(cstate->copycontext);
1865  pfree(cstate);
1866 }
1867 
1868 /*
1869  * Setup CopyState to read tuples from a table or a query for COPY TO.
1870  */
1871 static CopyState
1872 BeginCopyTo(ParseState *pstate,
1873  Relation rel,
1874  RawStmt *query,
1875  Oid queryRelId,
1876  const char *filename,
1877  bool is_program,
1878  List *attnamelist,
1879  List *options)
1880 {
1881  CopyState cstate;
1882  bool pipe = (filename == NULL);
1883  MemoryContext oldcontext;
1884 
1885  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1886  {
1887  if (rel->rd_rel->relkind == RELKIND_VIEW)
1888  ereport(ERROR,
1889  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1890  errmsg("cannot copy from view \"%s\"",
1891  RelationGetRelationName(rel)),
1892  errhint("Try the COPY (SELECT ...) TO variant.")));
1893  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1894  ereport(ERROR,
1895  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1896  errmsg("cannot copy from materialized view \"%s\"",
1897  RelationGetRelationName(rel)),
1898  errhint("Try the COPY (SELECT ...) TO variant.")));
1899  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1900  ereport(ERROR,
1901  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1902  errmsg("cannot copy from foreign table \"%s\"",
1903  RelationGetRelationName(rel)),
1904  errhint("Try the COPY (SELECT ...) TO variant.")));
1905  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1906  ereport(ERROR,
1907  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1908  errmsg("cannot copy from sequence \"%s\"",
1909  RelationGetRelationName(rel))));
1910  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1911  ereport(ERROR,
1912  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1913  errmsg("cannot copy from partitioned table \"%s\"",
1914  RelationGetRelationName(rel)),
1915  errhint("Try the COPY (SELECT ...) TO variant.")));
1916  else
1917  ereport(ERROR,
1918  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1919  errmsg("cannot copy from non-table relation \"%s\"",
1920  RelationGetRelationName(rel))));
1921  }
1922 
1923  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1924  options);
1925  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1926 
1927  if (pipe)
1928  {
1929  Assert(!is_program); /* the grammar does not allow this */
1930  if (whereToSendOutput != DestRemote)
1931  cstate->copy_file = stdout;
1932  }
1933  else
1934  {
1935  cstate->filename = pstrdup(filename);
1936  cstate->is_program = is_program;
1937 
1938  if (is_program)
1939  {
1940  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1941  if (cstate->copy_file == NULL)
1942  ereport(ERROR,
1943  (errcode_for_file_access(),
1944  errmsg("could not execute command \"%s\": %m",
1945  cstate->filename)));
1946  }
1947  else
1948  {
1949  mode_t oumask; /* Pre-existing umask value */
1950  struct stat st;
1951 
1952  /*
1953  * Prevent write to relative path ... too easy to shoot oneself in
1954  * the foot by overwriting a database file ...
1955  */
1956  if (!is_absolute_path(filename))
1957  ereport(ERROR,
1958  (errcode(ERRCODE_INVALID_NAME),
1959  errmsg("relative path not allowed for COPY to file")));
1960 
1961  oumask = umask(S_IWGRP | S_IWOTH);
1962  PG_TRY();
1963  {
1964  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1965  }
1966  PG_FINALLY();
1967  {
1968  umask(oumask);
1969  }
1970  PG_END_TRY();
1971  if (cstate->copy_file == NULL)
1972  {
1973  /* copy errno because ereport subfunctions might change it */
1974  int save_errno = errno;
1975 
1976  ereport(ERROR,
1977  (errcode_for_file_access(),
1978  errmsg("could not open file \"%s\" for writing: %m",
1979  cstate->filename),
1980  (save_errno == ENOENT || save_errno == EACCES) ?
1981  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1982  "You may want a client-side facility such as psql's \\copy.") : 0));
1983  }
1984 
1985  if (fstat(fileno(cstate->copy_file), &st))
1986  ereport(ERROR,
1987  (errcode_for_file_access(),
1988  errmsg("could not stat file \"%s\": %m",
1989  cstate->filename)));
1990 
1991  if (S_ISDIR(st.st_mode))
1992  ereport(ERROR,
1993  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1994  errmsg("\"%s\" is a directory", cstate->filename)));
1995  }
1996  }
1997 
1998  MemoryContextSwitchTo(oldcontext);
1999 
2000  return cstate;
2001 }
2002 
2003 /*
2004  * This intermediate routine exists mainly to localize the effects of setjmp
2005  * so we don't need to plaster a lot of variables with "volatile".
2006  */
2007 static uint64
2008 DoCopyTo(CopyState cstate)
2009 {
2010  bool pipe = (cstate->filename == NULL);
2011  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
2012  uint64 processed;
2013 
2014  PG_TRY();
2015  {
2016  if (fe_copy)
2017  SendCopyBegin(cstate);
2018 
2019  processed = CopyTo(cstate);
2020 
2021  if (fe_copy)
2022  SendCopyEnd(cstate);
2023  }
2024  PG_CATCH();
2025  {
2026  /*
2027  * Make sure we turn off old-style COPY OUT mode upon error. It is
2028  * okay to do this in all cases, since it does nothing if the mode is
2029  * not on.
2030  */
2031  pq_endcopyout(true);
2032  PG_RE_THROW();
2033  }
2034  PG_END_TRY();
2035 
2036  return processed;
2037 }
2038 
2039 /*
2040  * Clean up storage and release resources for COPY TO.
2041  */
2042 static void
2043 EndCopyTo(CopyState cstate)
2044 {
2045  if (cstate->queryDesc != NULL)
2046  {
2047  /* Close down the query and free resources. */
2048  ExecutorFinish(cstate->queryDesc);
2049  ExecutorEnd(cstate->queryDesc);
2050  FreeQueryDesc(cstate->queryDesc);
2051  PopActiveSnapshot();
2052  }
2053 
2054  /* Clean up storage */
2055  EndCopy(cstate);
2056 }
2057 
2058 /*
2059  * Copy from relation or query TO file.
2060  */
2061 static uint64
2062 CopyTo(CopyState cstate)
2063 {
2064  TupleDesc tupDesc;
2065  int num_phys_attrs;
2066  ListCell *cur;
2067  uint64 processed;
2068 
2069  if (cstate->rel)
2070  tupDesc = RelationGetDescr(cstate->rel);
2071  else
2072  tupDesc = cstate->queryDesc->tupDesc;
2073  num_phys_attrs = tupDesc->natts;
2074  cstate->null_print_client = cstate->null_print; /* default */
2075 
2076  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
2077  cstate->fe_msgbuf = makeStringInfo();
2078 
2079  /* Get info about the columns we need to process. */
2080  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2081  foreach(cur, cstate->attnumlist)
2082  {
2083  int attnum = lfirst_int(cur);
2084  Oid out_func_oid;
2085  bool isvarlena;
2086  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
2087 
2088  if (cstate->binary)
2089  getTypeBinaryOutputInfo(attr->atttypid,
2090  &out_func_oid,
2091  &isvarlena);
2092  else
2093  getTypeOutputInfo(attr->atttypid,
2094  &out_func_oid,
2095  &isvarlena);
2096  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
2097  }
2098 
2099  /*
2100  * Create a temporary memory context that we can reset once per row to
2101  * recover palloc'd memory. This avoids any problems with leaks inside
2102  * datatype output routines, and should be faster than retail pfree's
2103  * anyway. (We don't need a whole econtext as CopyFrom does.)
2104  */
2105  cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
2106  "COPY TO",
2107  ALLOCSET_DEFAULT_SIZES);
2108 
2109  if (cstate->binary)
2110  {
2111  /* Generate header for a binary copy */
2112  int32 tmp;
2113 
2114  /* Signature */
2115  CopySendData(cstate, BinarySignature, 11);
2116  /* Flags field */
2117  tmp = 0;
2118  CopySendInt32(cstate, tmp);
2119  /* No header extension */
2120  tmp = 0;
2121  CopySendInt32(cstate, tmp);
2122  }
2123  else
2124  {
2125  /*
2126  * For non-binary copy, we need to convert null_print to file
2127  * encoding, because it will be sent directly with CopySendString.
2128  */
2129  if (cstate->need_transcoding)
2130  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2131  cstate->null_print_len,
2132  cstate->file_encoding);
2133 
2134  /* if a header has been requested send the line */
2135  if (cstate->header_line)
2136  {
2137  bool hdr_delim = false;
2138 
2139  foreach(cur, cstate->attnumlist)
2140  {
2141  int attnum = lfirst_int(cur);
2142  char *colname;
2143 
2144  if (hdr_delim)
2145  CopySendChar(cstate, cstate->delim[0]);
2146  hdr_delim = true;
2147 
2148  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
2149 
2150  CopyAttributeOutCSV(cstate, colname, false,
2151  list_length(cstate->attnumlist) == 1);
2152  }
2153 
2154  CopySendEndOfRow(cstate);
2155  }
2156  }
2157 
2158  if (cstate->rel)
2159  {
2160  TupleTableSlot *slot;
2161  TableScanDesc scandesc;
2162 
2163  scandesc = table_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2164  slot = table_slot_create(cstate->rel, NULL);
2165 
2166  processed = 0;
2167  while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot))
2168  {
2169  CHECK_FOR_INTERRUPTS();
2170 
2171  /* Deconstruct the tuple ... */
2172  slot_getallattrs(slot);
2173 
2174  /* Format and send the data */
2175  CopyOneRowTo(cstate, slot);
2176  processed++;
2177  }
2178 
2179  ExecDropSingleTupleTableSlot(slot);
2180  table_endscan(scandesc);
2181  }
2182  else
2183  {
2184  /* run the plan --- the dest receiver will send tuples */
2185  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2186  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2187  }
2188 
2189  if (cstate->binary)
2190  {
2191  /* Generate trailer for a binary copy */
2192  CopySendInt16(cstate, -1);
2193  /* Need to flush out the trailer */
2194  CopySendEndOfRow(cstate);
2195  }
2196 
2197  MemoryContextDelete(cstate->rowcontext);
2198 
2199  return processed;
2200 }
2201 
2202 /*
2203  * Emit one row during CopyTo().
2204  */
2205 static void
2206 CopyOneRowTo(CopyState cstate, TupleTableSlot *slot)
2207 {
2208  bool need_delim = false;
2209  FmgrInfo *out_functions = cstate->out_functions;
2210  MemoryContext oldcontext;
2211  ListCell *cur;
2212  char *string;
2213 
2214  MemoryContextReset(cstate->rowcontext);
2215  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2216 
2217  if (cstate->binary)
2218  {
2219  /* Binary per-tuple header */
2220  CopySendInt16(cstate, list_length(cstate->attnumlist));
2221  }
2222 
2223  /* Make sure the tuple is fully deconstructed */
2224  slot_getallattrs(slot);
2225 
2226  foreach(cur, cstate->attnumlist)
2227  {
2228  int attnum = lfirst_int(cur);
2229  Datum value = slot->tts_values[attnum - 1];
2230  bool isnull = slot->tts_isnull[attnum - 1];
2231 
2232  if (!cstate->binary)
2233  {
2234  if (need_delim)
2235  CopySendChar(cstate, cstate->delim[0]);
2236  need_delim = true;
2237  }
2238 
2239  if (isnull)
2240  {
2241  if (!cstate->binary)
2242  CopySendString(cstate, cstate->null_print_client);
2243  else
2244  CopySendInt32(cstate, -1);
2245  }
2246  else
2247  {
2248  if (!cstate->binary)
2249  {
2250  string = OutputFunctionCall(&out_functions[attnum - 1],
2251  value);
2252  if (cstate->csv_mode)
2253  CopyAttributeOutCSV(cstate, string,
2254  cstate->force_quote_flags[attnum - 1],
2255  list_length(cstate->attnumlist) == 1);
2256  else
2257  CopyAttributeOutText(cstate, string);
2258  }
2259  else
2260  {
2261  bytea *outputbytes;
2262 
2263  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2264  value);
2265  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2266  CopySendData(cstate, VARDATA(outputbytes),
2267  VARSIZE(outputbytes) - VARHDRSZ);
2268  }
2269  }
2270  }
2271 
2272  CopySendEndOfRow(cstate);
2273 
2274  MemoryContextSwitchTo(oldcontext);
2275 }
2276 
2277 
2278 /*
2279  * error context callback for COPY FROM
2280  *
2281  * The argument for the error context must be CopyState.
2282  */
2283 void
2284 CopyFromErrorCallback(void *arg)
2285 {
2286  CopyState cstate = (CopyState) arg;
2287  char curlineno_str[32];
2288 
2289  snprintf(curlineno_str, sizeof(curlineno_str), UINT64_FORMAT,
2290  cstate->cur_lineno);
2291 
2292  if (cstate->binary)
2293  {
2294  /* can't usefully display the data */
2295  if (cstate->cur_attname)
2296  errcontext("COPY %s, line %s, column %s",
2297  cstate->cur_relname, curlineno_str,
2298  cstate->cur_attname);
2299  else
2300  errcontext("COPY %s, line %s",
2301  cstate->cur_relname, curlineno_str);
2302  }
2303  else
2304  {
2305  if (cstate->cur_attname && cstate->cur_attval)
2306  {
2307  /* error is relevant to a particular column */
2308  char *attval;
2309 
2310  attval = limit_printout_length(cstate->cur_attval);
2311  errcontext("COPY %s, line %s, column %s: \"%s\"",
2312  cstate->cur_relname, curlineno_str,
2313  cstate->cur_attname, attval);
2314  pfree(attval);
2315  }
2316  else if (cstate->cur_attname)
2317  {
2318  /* error is relevant to a particular column, value is NULL */
2319  errcontext("COPY %s, line %s, column %s: null input",
2320  cstate->cur_relname, curlineno_str,
2321  cstate->cur_attname);
2322  }
2323  else
2324  {
2325  /*
2326  * Error is relevant to a particular line.
2327  *
2328  * If line_buf still contains the correct line, and it's already
2329  * transcoded, print it. If it's still in a foreign encoding, it's
2330  * quite likely that the error is precisely a failure to do
2331  * encoding conversion (ie, bad data). We dare not try to convert
2332  * it, and at present there's no way to regurgitate it without
2333  * conversion. So we have to punt and just report the line number.
2334  */
2335  if (cstate->line_buf_valid &&
2336  (cstate->line_buf_converted || !cstate->need_transcoding))
2337  {
2338  char *lineval;
2339 
2340  lineval = limit_printout_length(cstate->line_buf.data);
2341  errcontext("COPY %s, line %s: \"%s\"",
2342  cstate->cur_relname, curlineno_str, lineval);
2343  pfree(lineval);
2344  }
2345  else
2346  {
2347  errcontext("COPY %s, line %s",
2348  cstate->cur_relname, curlineno_str);
2349  }
2350  }
2351  }
2352 }
2353 
2354 /*
2355  * Make sure we don't print an unreasonable amount of COPY data in a message.
2356  *
2357  * Returns a pstrdup'd copy of the input.
2358  */
2359 static char *
2360 limit_printout_length(const char *str)
2361 {
2362 #define MAX_COPY_DATA_DISPLAY 100
2363 
2364  int slen = strlen(str);
2365  int len;
2366  char *res;
2367 
2368  /* Fast path if definitely okay */
2369  if (slen <= MAX_COPY_DATA_DISPLAY)
2370  return pstrdup(str);
2371 
2372  /* Apply encoding-dependent truncation */
2373  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2374 
2375  /*
2376  * Truncate, and add "..." to show we truncated the input.
2377  */
2378  res = (char *) palloc(len + 4);
2379  memcpy(res, str, len);
2380  strcpy(res + len, "...");
2381 
2382  return res;
2383 }
2384 
2385 /*
2386  * Allocate memory and initialize a new CopyMultiInsertBuffer for this
2387  * ResultRelInfo.
2388  */
2389 static CopyMultiInsertBuffer *
2390 CopyMultiInsertBufferInit(ResultRelInfo *rri)
2391 {
2392  CopyMultiInsertBuffer *buffer;
2393 
2394  buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer));
2395  memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES);
2396  buffer->resultRelInfo = rri;
2397  buffer->bistate = GetBulkInsertState();
2398  buffer->nused = 0;
2399 
2400  return buffer;
2401 }
2402 
2403 /*
2404  * Make a new buffer for this ResultRelInfo.
2405  */
2406 static inline void
2407 CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo,
2408  ResultRelInfo *rri)
2409 {
2410  CopyMultiInsertBuffer *buffer;
2411 
2412  buffer = CopyMultiInsertBufferInit(rri);
2413 
2414  /* Setup back-link so we can easily find this buffer again */
2415  rri->ri_CopyMultiInsertBuffer = buffer;
2416  /* Record that we're tracking this buffer */
2417  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2418 }
2419 
2420 /*
2421  * Initialize an already allocated CopyMultiInsertInfo.
2422  *
2423  * If rri is a non-partitioned table then a CopyMultiInsertBuffer is set up
2424  * for that table.
2425  */
2426 static void
2427 CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2428  CopyState cstate, EState *estate, CommandId mycid,
2429  int ti_options)
2430 {
2431  miinfo->multiInsertBuffers = NIL;
2432  miinfo->bufferedTuples = 0;
2433  miinfo->bufferedBytes = 0;
2434  miinfo->cstate = cstate;
2435  miinfo->estate = estate;
2436  miinfo->mycid = mycid;
2437  miinfo->ti_options = ti_options;
2438 
2439  /*
2440  * Only setup the buffer when not dealing with a partitioned table.
2441  * Buffers for partitioned tables will just be setup when we need to send
2442  * tuples their way for the first time.
2443  */
2444  if (rri->ri_RelationDesc->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
2445  CopyMultiInsertInfoSetupBuffer(miinfo, rri);
2446 }
2447 
2448 /*
2449  * Returns true if the buffers are full
2450  */
2451 static inline bool
2452 CopyMultiInsertInfoIsFull(CopyMultiInsertInfo *miinfo)
2453 {
2454  if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES ||
2455  miinfo->bufferedBytes >= MAX_BUFFERED_BYTES)
2456  return true;
2457  return false;
2458 }
2459 
2460 /*
2461  * Returns true if we have no buffered tuples
2462  */
2463 static inline bool
2464 CopyMultiInsertInfoIsEmpty(CopyMultiInsertInfo *miinfo)
2465 {
2466  return miinfo->bufferedTuples == 0;
2467 }
2468 
2469 /*
2470  * Write the tuples stored in 'buffer' out to the table.
2471  */
2472 static inline void
2473 CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
2474  CopyMultiInsertBuffer *buffer)
2475 {
2476  MemoryContext oldcontext;
2477  int i;
2478  uint64 save_cur_lineno;
2479  CopyState cstate = miinfo->cstate;
2480  EState *estate = miinfo->estate;
2481  CommandId mycid = miinfo->mycid;
2482  int ti_options = miinfo->ti_options;
2483  bool line_buf_valid = cstate->line_buf_valid;
2484  int nused = buffer->nused;
2485  ResultRelInfo *resultRelInfo = buffer->resultRelInfo;
2486  TupleTableSlot **slots = buffer->slots;
2487 
2488  /* Set es_result_relation_info to the ResultRelInfo we're flushing. */
2489  estate->es_result_relation_info = resultRelInfo;
2490 
2491  /*
2492  * Print error context information correctly, if one of the operations
2493  * below fail.
2494  */
2495  cstate->line_buf_valid = false;
2496  save_cur_lineno = cstate->cur_lineno;
2497 
2498  /*
2499  * table_multi_insert may leak memory, so switch to short-lived memory
2500  * context before calling it.
2501  */
2502  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2503  table_multi_insert(resultRelInfo->ri_RelationDesc,
2504  slots,
2505  nused,
2506  mycid,
2507  ti_options,
2508  buffer->bistate);
2509  MemoryContextSwitchTo(oldcontext);
2510 
2511  for (i = 0; i < nused; i++)
2512  {
2513  /*
2514  * If there are any indexes, update them for all the inserted tuples,
2515  * and run AFTER ROW INSERT triggers.
2516  */
2517  if (resultRelInfo->ri_NumIndices > 0)
2518  {
2519  List *recheckIndexes;
2520 
2521  cstate->cur_lineno = buffer->linenos[i];
2522  recheckIndexes =
2523  ExecInsertIndexTuples(buffer->slots[i], estate, false, NULL,
2524  NIL);
2525  ExecARInsertTriggers(estate, resultRelInfo,
2526  slots[i], recheckIndexes,
2527  cstate->transition_capture);
2528  list_free(recheckIndexes);
2529  }
2530 
2531  /*
2532  * There's no indexes, but see if we need to run AFTER ROW INSERT
2533  * triggers anyway.
2534  */
2535  else if (resultRelInfo->ri_TrigDesc != NULL &&
2536  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2537  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2538  {
2539  cstate->cur_lineno = buffer->linenos[i];
2540  ExecARInsertTriggers(estate, resultRelInfo,
2541  slots[i], NIL, cstate->transition_capture);
2542  }
2543 
2544  ExecClearTuple(slots[i]);
2545  }
2546 
2547  /* Mark that all slots are free */
2548  buffer->nused = 0;
2549 
2550  /* reset cur_lineno and line_buf_valid to what they were */
2551  cstate->line_buf_valid = line_buf_valid;
2552  cstate->cur_lineno = save_cur_lineno;
2553 }
2554 
2555 /*
2556  * Drop used slots and free member for this buffer.
2557  *
2558  * The buffer must be flushed before cleanup.
2559  */
2560 static inline void
2561 CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo,
2562  CopyMultiInsertBuffer *buffer)
2563 {
2564  int i;
2565 
2566  /* Ensure buffer was flushed */
2567  Assert(buffer->nused == 0);
2568 
2569  /* Remove back-link to ourself */
2570  buffer->resultRelInfo->ri_CopyMultiInsertBuffer = NULL;
2571 
2572  FreeBulkInsertState(buffer->bistate);
2573 
2574  /* Since we only create slots on demand, just drop the non-null ones. */
2575  for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++)
2576  ExecDropSingleTupleTableSlot(buffer->slots[i]);
2577 
2578  table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc,
2579  miinfo->ti_options);
2580 
2581  pfree(buffer);
2582 }
2583 
2584 /*
2585  * Write out all stored tuples in all buffers out to the tables.
2586  *
2587  * Once flushed we also trim the tracked buffers list down to size by removing
2588  * the buffers created earliest first.
2589  *
2590  * Callers should pass 'curr_rri' is the ResultRelInfo that's currently being
2591  * used. When cleaning up old buffers we'll never remove the one for
2592  * 'curr_rri'.
2593  */
2594 static inline void
2595 CopyMultiInsertInfoFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *curr_rri)
2596 {
2597  ListCell *lc;
2598 
2599  foreach(lc, miinfo->multiInsertBuffers)
2600  {
2601  CopyMultiInsertBuffer *buffer = (CopyMultiInsertBuffer *) lfirst(lc);
2602 
2603  CopyMultiInsertBufferFlush(miinfo, buffer);
2604  }
2605 
2606  miinfo->bufferedTuples = 0;
2607  miinfo->bufferedBytes = 0;
2608 
2609  /*
2610  * Trim the list of tracked buffers down if it exceeds the limit. Here we
2611  * remove buffers starting with the ones we created first. It seems more
2612  * likely that these older ones are less likely to be needed than ones
2613  * that were just created.
2614  */
2615  while (list_length(miinfo->multiInsertBuffers) > MAX_PARTITION_BUFFERS)
2616  {
2617  CopyMultiInsertBuffer *buffer;
2618 
2619  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2620 
2621  /*
2622  * We never want to remove the buffer that's currently being used, so
2623  * if we happen to find that then move it to the end of the list.
2624  */
2625  if (buffer->resultRelInfo == curr_rri)
2626  {
2627  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2628  miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
2629  buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
2630  }
2631 
2632  CopyMultiInsertBufferCleanup(miinfo, buffer);
2633  miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
2634  }
2635 }
2636 
2637 /*
2638  * Cleanup allocated buffers and free memory
2639  */
2640 static inline void
2641 CopyMultiInsertInfoCleanup(CopyMultiInsertInfo *miinfo)
2642 {
2643  ListCell *lc;
2644 
2645  foreach(lc, miinfo->multiInsertBuffers)
2646  CopyMultiInsertBufferCleanup(miinfo, lfirst(lc));
2647 
2648  list_free(miinfo->multiInsertBuffers);
2649 }
2650 
2651 /*
2652  * Get the next TupleTableSlot that the next tuple should be stored in.
2653  *
2654  * Callers must ensure that the buffer is not full.
2655  *
2656  * Note: 'miinfo' is unused but has been included for consistency with the
2657  * other functions in this area.
2658  */
2659 static inline TupleTableSlot *
2660 CopyMultiInsertInfoNextFreeSlot(CopyMultiInsertInfo *miinfo,
2661  ResultRelInfo *rri)
2662 {
2663  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2664  int nused = buffer->nused;
2665 
2666  Assert(buffer != NULL);
2667  Assert(nused < MAX_BUFFERED_TUPLES);
2668 
2669  if (buffer->slots[nused] == NULL)
2670  buffer->slots[nused] = table_slot_create(rri->ri_RelationDesc, NULL);
2671  return buffer->slots[nused];
2672 }
2673 
2674 /*
2675  * Record the previously reserved TupleTableSlot that was reserved by
2676  * CopyMultiInsertInfoNextFreeSlot as being consumed.
2677  */
2678 static inline void
2679 CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
2680  TupleTableSlot *slot, int tuplen, uint64 lineno)
2681 {
2682  CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
2683 
2684  Assert(buffer != NULL);
2685  Assert(slot == buffer->slots[buffer->nused]);
2686 
2687  /* Store the line number so we can properly report any errors later */
2688  buffer->linenos[buffer->nused] = lineno;
2689 
2690  /* Record this slot as being used */
2691  buffer->nused++;
2692 
2693  /* Update how many tuples are stored and their size */
2694  miinfo->bufferedTuples++;
2695  miinfo->bufferedBytes += tuplen;
2696 }
2697 
2698 /*
2699  * Copy FROM file to relation.
2700  */
2701 uint64
2702 CopyFrom(CopyState cstate)
2703 {
2704  ResultRelInfo *resultRelInfo;
2705  ResultRelInfo *target_resultRelInfo;
2706  ResultRelInfo *prevResultRelInfo = NULL;
2707  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2708  ModifyTableState *mtstate;
2709  ExprContext *econtext;
2710  TupleTableSlot *singleslot = NULL;
2711  MemoryContext oldcontext = CurrentMemoryContext;
2712 
2713  PartitionTupleRouting *proute = NULL;
2714  ErrorContextCallback errcallback;
2715  CommandId mycid = GetCurrentCommandId(true);
2716  int ti_options = 0; /* start with default options for insert */
2717  BulkInsertState bistate = NULL;
2718  CopyInsertMethod insertMethod;
2719  CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */
2720  uint64 processed = 0;
2721  bool has_before_insert_row_trig;
2722  bool has_instead_insert_row_trig;
2723  bool leafpart_use_multi_insert = false;
2724 
2725  Assert(cstate->rel);
2726 
2727  /*
2728  * The target must be a plain, foreign, or partitioned relation, or have
2729  * an INSTEAD OF INSERT row trigger. (Currently, such triggers are only
2730  * allowed on views, so we only hint about them in the view case.)
2731  */
2732  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2733  cstate->rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE &&
2734  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2735  !(cstate->rel->trigdesc &&
2736  cstate->rel->trigdesc->trig_insert_instead_row))
2737  {
2738  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2739  ereport(ERROR,
2740  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2741  errmsg("cannot copy to view \"%s\"",
2742  RelationGetRelationName(cstate->rel)),
2743  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2744  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2745  ereport(ERROR,
2746  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2747  errmsg("cannot copy to materialized view \"%s\"",
2748  RelationGetRelationName(cstate->rel))));
2749  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2750  ereport(ERROR,
2751  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2752  errmsg("cannot copy to sequence \"%s\"",
2753  RelationGetRelationName(cstate->rel))));
2754  else
2755  ereport(ERROR,
2756  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2757  errmsg("cannot copy to non-table relation \"%s\"",
2758  RelationGetRelationName(cstate->rel))));
2759  }
2760 
2761  /*
2762  * If the target file is new-in-transaction, we assume that checking FSM
2763  * for free space is a waste of time. This could possibly be wrong, but
2764  * it's unlikely.
2765  */
2766  if (RELKIND_HAS_STORAGE(cstate->rel->rd_rel->relkind) &&
2767  (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2768  cstate->rel->rd_firstRelfilenodeSubid != InvalidSubTransactionId))
2769  ti_options |= TABLE_INSERT_SKIP_FSM;
2770 
2771  /*
2772  * Optimize if new relfilenode was created in this subxact or one of its
2773  * committed children and we won't see those rows later as part of an
2774  * earlier scan or command. The subxact test ensures that if this subxact
2775  * aborts then the frozen rows won't be visible after xact cleanup. Note
2776  * that the stronger test of exactly which subtransaction created it is
2777  * crucial for correctness of this optimization. The test for an earlier
2778  * scan or command tolerates false negatives. FREEZE causes other sessions
2779  * to see rows they would not see under MVCC, and a false negative merely
2780  * spreads that anomaly to the current session.
2781  */
2782  if (cstate->freeze)
2783  {
2784  /*
2785  * We currently disallow COPY FREEZE on partitioned tables. The
2786  * reason for this is that we've simply not yet opened the partitions
2787  * to determine if the optimization can be applied to them. We could
2788  * go and open them all here, but doing so may be quite a costly
2789  * overhead for small copies. In any case, we may just end up routing
2790  * tuples to a small number of partitions. It seems better just to
2791  * raise an ERROR for partitioned tables.
2792  */
2793  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2794  {
2795  ereport(ERROR,
2796  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2797  errmsg("cannot perform COPY FREEZE on a partitioned table")));
2798  }
2799 
2800  /*
2801  * Tolerate one registration for the benefit of FirstXactSnapshot.
2802  * Scan-bearing queries generally create at least two registrations,
2803  * though relying on that is fragile, as is ignoring ActiveSnapshot.
2804  * Clear CatalogSnapshot to avoid counting its registration. We'll
2805  * still detect ongoing catalog scans, each of which separately
2806  * registers the snapshot it uses.
2807  */
2808  InvalidateCatalogSnapshot();
2809  if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
2810  ereport(ERROR,
2811  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2812  errmsg("cannot perform COPY FREEZE because of prior transaction activity")));
2813 
2814  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2815  cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
2816  ereport(ERROR,
2817  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2818  errmsg("cannot perform COPY FREEZE because the table was not created or truncated in the current subtransaction")));
2819 
2820  ti_options |= TABLE_INSERT_FROZEN;
2821  }
2822 
2823  /*
2824  * We need a ResultRelInfo so we can use the regular executor's
2825  * index-entry-making machinery. (There used to be a huge amount of code
2826  * here that basically duplicated execUtils.c ...)
2827  */
2828  resultRelInfo = makeNode(ResultRelInfo);
2829  InitResultRelInfo(resultRelInfo,
2830  cstate->rel,
2831  1, /* must match rel's position in range_table */
2832  NULL,
2833  0);
2834  target_resultRelInfo = resultRelInfo;
2835 
2836  /* Verify the named relation is a valid target for INSERT */
2837  CheckValidResultRel(resultRelInfo, CMD_INSERT);
2838 
2839  ExecOpenIndices(resultRelInfo, false);
2840 
2841  estate->es_result_relations = resultRelInfo;
2842  estate->es_num_result_relations = 1;
2843  estate->es_result_relation_info = resultRelInfo;
2844 
2845  ExecInitRangeTable(estate, cstate->range_table);
2846 
2847  /*
2848  * Set up a ModifyTableState so we can let FDW(s) init themselves for
2849  * foreign-table result relation(s).
2850  */
2851  mtstate = makeNode(ModifyTableState);
2852  mtstate->ps.plan = NULL;
2853  mtstate->ps.state = estate;
2854  mtstate->operation = CMD_INSERT;
2855  mtstate->resultRelInfo = estate->es_result_relations;
2856 
2857  if (resultRelInfo->ri_FdwRoutine != NULL &&
2858  resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
2859  resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
2860  resultRelInfo);
2861 
2862  /* Prepare to catch AFTER triggers. */
2863  AfterTriggerBeginQuery();
2864 
2865  /*
2866  * If there are any triggers with transition tables on the named relation,
2867  * we need to be prepared to capture transition tuples.
2868  *
2869  * Because partition tuple routing would like to know about whether
2870  * transition capture is active, we also set it in mtstate, which is
2871  * passed to ExecFindPartition() below.
2872  */
2873  cstate->transition_capture = mtstate->mt_transition_capture =
2874  MakeTransitionCaptureState(cstate->rel->trigdesc,
2875  RelationGetRelid(cstate->rel),
2876  CMD_INSERT);
2877 
2878  /*
2879  * If the named relation is a partitioned table, initialize state for
2880  * CopyFrom tuple routing.
2881  */
2882  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2883  proute = ExecSetupPartitionTupleRouting(estate, NULL, cstate->rel);
2884 
2885  if (cstate->whereClause)
2886  cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
2887  &mtstate->ps);
2888 
2889  /*
2890  * It's generally more efficient to prepare a bunch of tuples for
2891  * insertion, and insert them in one table_multi_insert() call, than call
2892  * table_tuple_insert() separately for every tuple. However, there are a
2893  * number of reasons why we might not be able to do this. These are
2894  * explained below.
2895  */
2896  if (resultRelInfo->ri_TrigDesc != NULL &&
2897  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2898  resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
2899  {
2900  /*
2901  * Can't support multi-inserts when there are any BEFORE/INSTEAD OF
2902  * triggers on the table. Such triggers might query the table we're
2903  * inserting into and act differently if the tuples that have already
2904  * been processed and prepared for insertion are not there.
2905  */
2906  insertMethod = CIM_SINGLE;
2907  }
2908  else if (proute != NULL && resultRelInfo->ri_TrigDesc != NULL &&
2909  resultRelInfo->ri_TrigDesc->trig_insert_new_table)
2910  {
2911  /*
2912  * For partitioned tables we can't support multi-inserts when there
2913  * are any statement level insert triggers. It might be possible to
2914  * allow partitioned tables with such triggers in the future, but for
2915  * now, CopyMultiInsertInfoFlush expects that any before row insert
2916  * and statement level insert triggers are on the same relation.
2917  */
2918  insertMethod = CIM_SINGLE;
2919  }
2920  else if (resultRelInfo->ri_FdwRoutine != NULL ||
2921  cstate->volatile_defexprs)
2922  {
2923  /*
2924  * Can't support multi-inserts to foreign tables or if there are any
2925  * volatile default expressions in the table. Similarly to the
2926  * trigger case above, such expressions may query the table we're
2927  * inserting into.
2928  *
2929  * Note: It does not matter if any partitions have any volatile
2930  * default expressions as we use the defaults from the target of the
2931  * COPY command.
2932  */
2933  insertMethod = CIM_SINGLE;
2934  }
2935  else if (contain_volatile_functions(cstate->whereClause))
2936  {
2937  /*
2938  * Can't support multi-inserts if there are any volatile function
2939  * expressions in WHERE clause. Similarly to the trigger case above,
2940  * such expressions may query the table we're inserting into.
2941  */
2942  insertMethod = CIM_SINGLE;
2943  }
2944  else
2945  {
2946  /*
2947  * For partitioned tables, we may still be able to perform bulk
2948  * inserts. However, the possibility of this depends on which types
2949  * of triggers exist on the partition. We must disable bulk inserts
2950  * if the partition is a foreign table or it has any before row insert
2951  * or insert instead triggers (same as we checked above for the parent
2952  * table). Since the partition's resultRelInfos are initialized only
2953  * when we actually need to insert the first tuple into them, we must
2954  * have the intermediate insert method of CIM_MULTI_CONDITIONAL to
2955  * flag that we must later determine if we can use bulk-inserts for
2956  * the partition being inserted into.
2957  */
2958  if (proute)
2959  insertMethod = CIM_MULTI_CONDITIONAL;
2960  else
2961  insertMethod = CIM_MULTI;
2962 
2963  CopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, cstate,
2964  estate, mycid, ti_options);
2965  }
2966 
2967  /*
2968  * If not using batch mode (which allocates slots as needed) set up a
2969  * tuple slot too. When inserting into a partitioned table, we also need
2970  * one, even if we might batch insert, to read the tuple in the root
2971  * partition's form.
2972  */
2973  if (insertMethod == CIM_SINGLE || insertMethod == CIM_MULTI_CONDITIONAL)
2974  {
2975  singleslot = table_slot_create(resultRelInfo->ri_RelationDesc,
2976  &estate->es_tupleTable);
2977  bistate = GetBulkInsertState();
2978  }
2979 
2980  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2981  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
2982 
2983  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
2984  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
2985 
2986  /*
2987  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2988  * should do this for COPY, since it's not really an "INSERT" statement as
2989  * such. However, executing these triggers maintains consistency with the
2990  * EACH ROW triggers that we already fire on COPY.
2991  */
2992  ExecBSInsertTriggers(estate, resultRelInfo);
2993 
2994  econtext = GetPerTupleExprContext(estate);
2995 
2996  /* Set up callback to identify error line number */
2997  errcallback.callback = CopyFromErrorCallback;
2998  errcallback.arg = (void *) cstate;
2999  errcallback.previous = error_context_stack;
3000  error_context_stack = &errcallback;
3001 
3002  for (;;)
3003  {
3004  TupleTableSlot *myslot;
3005  bool skip_tuple;
3006 
3007  CHECK_FOR_INTERRUPTS();
3008 
3009  /*
3010  * Reset the per-tuple exprcontext. We do this after every tuple, to
3011  * clean-up after expression evaluations etc.
3012  */
3013  ResetPerTupleExprContext(estate);
3014 
3015  /* select slot to (initially) load row into */
3016  if (insertMethod == CIM_SINGLE || proute)
3017  {
3018  myslot = singleslot;
3019  Assert(myslot != NULL);
3020  }
3021  else
3022  {
3023  Assert(resultRelInfo == target_resultRelInfo);
3024  Assert(insertMethod == CIM_MULTI);
3025 
3026  myslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3027  resultRelInfo);
3028  }
3029 
3030  /*
3031  * Switch to per-tuple context before calling NextCopyFrom, which does
3032  * evaluate default expressions etc. and requires per-tuple context.
3033  */
3034  MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3035 
3036  ExecClearTuple(myslot);
3037 
3038  /* Directly store the values/nulls array in the slot */
3039  if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
3040  break;
3041 
3042  ExecStoreVirtualTuple(myslot);
3043 
3044  /*
3045  * Constraints and where clause might reference the tableoid column,
3046  * so (re-)initialize tts_tableOid before evaluating them.
3047  */
3048  myslot->tts_tableOid = RelationGetRelid(target_resultRelInfo->ri_RelationDesc);
3049 
3050  /* Triggers and stuff need to be invoked in query context. */
3051  MemoryContextSwitchTo(oldcontext);
3052 
3053  if (cstate->whereClause)
3054  {
3055  econtext->ecxt_scantuple = myslot;
3056  /* Skip items that don't match COPY's WHERE clause */
3057  if (!ExecQual(cstate->qualexpr, econtext))
3058  continue;
3059  }
3060 
3061  /* Determine the partition to insert the tuple into */
3062  if (proute)
3063  {
3064  TupleConversionMap *map;
3065 
3066  /*
3067  * Attempt to find a partition suitable for this tuple.
3068  * ExecFindPartition() will raise an error if none can be found or
3069  * if the found partition is not suitable for INSERTs.
3070  */
3071  resultRelInfo = ExecFindPartition(mtstate, target_resultRelInfo,
3072  proute, myslot, estate);
3073 
3074  if (prevResultRelInfo != resultRelInfo)
3075  {
3076  /* Determine which triggers exist on this partition */
3077  has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3078  resultRelInfo->ri_TrigDesc->trig_insert_before_row);
3079 
3080  has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
3081  resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
3082 
3083  /*
3084  * Disable multi-inserts when the partition has BEFORE/INSTEAD
3085  * OF triggers, or if the partition is a foreign partition.
3086  */
3087  leafpart_use_multi_insert = insertMethod == CIM_MULTI_CONDITIONAL &&
3088  !has_before_insert_row_trig &&
3089  !has_instead_insert_row_trig &&
3090  resultRelInfo->ri_FdwRoutine == NULL;
3091 
3092  /* Set the multi-insert buffer to use for this partition. */
3093  if (leafpart_use_multi_insert)
3094  {
3095  if (resultRelInfo->ri_CopyMultiInsertBuffer == NULL)
3096  CopyMultiInsertInfoSetupBuffer(&multiInsertInfo,
3097  resultRelInfo);
3098  }
3099  else if (insertMethod == CIM_MULTI_CONDITIONAL &&
3100  !CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3101  {
3102  /*
3103  * Flush pending inserts if this partition can't use
3104  * batching, so rows are visible to triggers etc.
3105  */
3106  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3107  }
3108 
3109  if (bistate != NULL)
3110  ReleaseBulkInsertStatePin(bistate);
3111  prevResultRelInfo = resultRelInfo;
3112  }
3113 
3114  /*
3115  * For ExecInsertIndexTuples() to work on the partition's indexes
3116  */
3117  estate->es_result_relation_info = resultRelInfo;
3118 
3119  /*
3120  * If we're capturing transition tuples, we might need to convert
3121  * from the partition rowtype to root rowtype.
3122  */
3123  if (cstate->transition_capture != NULL)
3124  {
3125  if (has_before_insert_row_trig)
3126  {
3127  /*
3128  * If there are any BEFORE triggers on the partition,
3129  * we'll have to be ready to convert their result back to
3130  * tuplestore format.
3131  */
3132  cstate->transition_capture->tcs_original_insert_tuple = NULL;
3133  cstate->transition_capture->tcs_map =
3134  resultRelInfo->ri_PartitionInfo->pi_PartitionToRootMap;
3135  }
3136  else
3137  {
3138  /*
3139  * Otherwise, just remember the original unconverted
3140  * tuple, to avoid a needless round trip conversion.
3141  */
3142  cstate->transition_capture->tcs_original_insert_tuple = myslot;
3143  cstate->transition_capture->tcs_map = NULL;
3144  }
3145  }
3146 
3147  /*
3148  * We might need to convert from the root rowtype to the partition
3149  * rowtype.
3150  */
3151  map = resultRelInfo->ri_PartitionInfo->pi_RootToPartitionMap;
3152  if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert)
3153  {
3154  /* non batch insert */
3155  if (map != NULL)
3156  {
3157  TupleTableSlot *new_slot;
3158 
3159  new_slot = resultRelInfo->ri_PartitionInfo->pi_PartitionTupleSlot;
3160  myslot = execute_attr_map_slot(map->attrMap, myslot, new_slot);
3161  }
3162  }
3163  else
3164  {
3165  /*
3166  * Prepare to queue up tuple for later batch insert into
3167  * current partition.
3168  */
3169  TupleTableSlot *batchslot;
3170 
3171  /* no other path available for partitioned table */
3172  Assert(insertMethod == CIM_MULTI_CONDITIONAL);
3173 
3174  batchslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
3175  resultRelInfo);
3176 
3177  if (map != NULL)
3178  myslot = execute_attr_map_slot(map->attrMap, myslot,
3179  batchslot);
3180  else
3181  {
3182  /*
3183  * This looks more expensive than it is (Believe me, I
3184  * optimized it away. Twice.). The input is in virtual
3185  * form, and we'll materialize the slot below - for most
3186  * slot types the copy performs the work materialization
3187  * would later require anyway.
3188  */
3189  ExecCopySlot(batchslot, myslot);
3190  myslot = batchslot;
3191  }
3192  }
3193 
3194  /* ensure that triggers etc see the right relation */
3195  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3196  }
3197 
3198  skip_tuple = false;
3199 
3200  /* BEFORE ROW INSERT Triggers */
3201  if (has_before_insert_row_trig)
3202  {
3203  if (!ExecBRInsertTriggers(estate, resultRelInfo, myslot))
3204  skip_tuple = true; /* "do nothing" */
3205  }
3206 
3207  if (!skip_tuple)
3208  {
3209  /*
3210  * If there is an INSTEAD OF INSERT ROW trigger, let it handle the
3211  * tuple. Otherwise, proceed with inserting the tuple into the
3212  * table or foreign table.
3213  */
3214  if (has_instead_insert_row_trig)
3215  {
3216  ExecIRInsertTriggers(estate, resultRelInfo, myslot);
3217  }
3218  else
3219  {
3220  /* Compute stored generated columns */
3221  if (resultRelInfo->ri_RelationDesc->rd_att->constr &&
3222  resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored)
3223  ExecComputeStoredGenerated(estate, myslot, CMD_INSERT);
3224 
3225  /*
3226  * If the target is a plain table, check the constraints of
3227  * the tuple.
3228  */
3229  if (resultRelInfo->ri_FdwRoutine == NULL &&
3230  resultRelInfo->ri_RelationDesc->rd_att->constr)
3231  ExecConstraints(resultRelInfo, myslot, estate);
3232 
3233  /*
3234  * Also check the tuple against the partition constraint, if
3235  * there is one; except that if we got here via tuple-routing,
3236  * we don't need to if there's no BR trigger defined on the
3237  * partition.
3238  */
3239  if (resultRelInfo->ri_RelationDesc->rd_rel->relispartition &&
3240  (proute == NULL || has_before_insert_row_trig))
3241  ExecPartitionCheck(resultRelInfo, myslot, estate, true);
3242 
3243  /* Store the slot in the multi-insert buffer, when enabled. */
3244  if (insertMethod == CIM_MULTI || leafpart_use_multi_insert)
3245  {
3246  /*
3247  * The slot previously might point into the per-tuple
3248  * context. For batching it needs to be longer lived.
3249  */
3250  ExecMaterializeSlot(myslot);
3251 
3252  /* Add this tuple to the tuple buffer */
3253  CopyMultiInsertInfoStore(&multiInsertInfo,
3254  resultRelInfo, myslot,
3255  cstate->line_buf.len,
3256  cstate->cur_lineno);
3257 
3258  /*
3259  * If enough inserts have queued up, then flush all
3260  * buffers out to their tables.
3261  */
3262  if (CopyMultiInsertInfoIsFull(&multiInsertInfo))
3263  CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
3264  }
3265  else
3266  {
3267  List *recheckIndexes = NIL;
3268 
3269  /* OK, store the tuple */
3270  if (resultRelInfo->ri_FdwRoutine != NULL)
3271  {
3272  myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
3273  resultRelInfo,
3274  myslot,
3275  NULL);
3276 
3277  if (myslot == NULL) /* "do nothing" */
3278  continue; /* next tuple please */
3279 
3280  /*
3281  * AFTER ROW Triggers might reference the tableoid
3282  * column, so (re-)initialize tts_tableOid before
3283  * evaluating them.
3284  */
3285  myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
3286  }
3287  else
3288  {
3289  /* OK, store the tuple and create index entries for it */
3290  table_tuple_insert(resultRelInfo->ri_RelationDesc,
3291  myslot, mycid, ti_options, bistate);
3292 
3293  if (resultRelInfo->ri_NumIndices > 0)
3294  recheckIndexes = ExecInsertIndexTuples(myslot,
3295  estate,
3296  false,
3297  NULL,
3298  NIL);
3299  }
3300 
3301  /* AFTER ROW INSERT Triggers */
3302  ExecARInsertTriggers(estate, resultRelInfo, myslot,
3303  recheckIndexes, cstate->transition_capture);
3304 
3305  list_free(recheckIndexes);
3306  }
3307  }
3308 
3309  /*
3310  * We count only tuples not suppressed by a BEFORE INSERT trigger
3311  * or FDW; this is the same definition used by nodeModifyTable.c
3312  * for counting tuples inserted by an INSERT command.
3313  */
3314  processed++;
3315  }
3316  }
3317 
3318  /* Flush any remaining buffered tuples */
3319  if (insertMethod != CIM_SINGLE)
3320  {
3321  if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
3322  CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);
3323  }
3324 
3325  /* Done, clean up */
3326  error_context_stack = errcallback.previous;
3327 
3328  if (bistate != NULL)
3329  FreeBulkInsertState(bistate);
3330 
3331  MemoryContextSwitchTo(oldcontext);
3332 
3333  /*
3334  * In the old protocol, tell pqcomm that we can process normal protocol
3335  * messages again.
3336  */
3337  if (cstate->copy_dest == COPY_OLD_FE)
3338  pq_endmsgread();
3339 
3340  /* Execute AFTER STATEMENT insertion triggers */
3341  ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
3342 
3343  /* Handle queued AFTER triggers */
3344  AfterTriggerEndQuery(estate);
3345 
3346  ExecResetTupleTable(estate->es_tupleTable, false);
3347 
3348  /* Allow the FDW to shut down */
3349  if (target_resultRelInfo->ri_FdwRoutine != NULL &&
3350  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
3351  target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,
3352  target_resultRelInfo);
3353 
3354  /* Tear down the multi-insert buffer data */
3355  if (insertMethod != CIM_SINGLE)
3356  CopyMultiInsertInfoCleanup(&multiInsertInfo);
3357 
3358  ExecCloseIndices(target_resultRelInfo);
3359 
3360  /* Close all the partitioned tables, leaf partitions, and their indices */
3361  if (proute)
3362  ExecCleanupTupleRouting(mtstate, proute);
3363 
3364  /* Close any trigger target relations */
3365  ExecCleanUpTriggerState(estate);
3366 
3367  FreeExecutorState(estate);
3368 
3369  return processed;
3370 }
3371 
3372 /*
3373  * Setup to read tuples from a file for COPY FROM.
3374  *
3375  * 'rel': Used as a template for the tuples
3376  * 'filename': Name of server-local file to read
3377  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
3378  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
3379  *
3380  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
3381  */
3382 CopyState
3383 BeginCopyFrom(ParseState *pstate,
3384  Relation rel,
3385  const char *filename,
3386  bool is_program,
3387  copy_data_source_cb data_source_cb,
3388  List *attnamelist,
3389  List *options)
3390 {
3391  CopyState cstate;
3392  bool pipe = (filename == NULL);
3393  TupleDesc tupDesc;
3394  AttrNumber num_phys_attrs,
3395  num_defaults;
3396  FmgrInfo *in_functions;
3397  Oid *typioparams;
3398  int attnum;
3399  Oid in_func_oid;
3400  int *defmap;
3401  ExprState **defexprs;
3402  MemoryContext oldcontext;
3403  bool volatile_defexprs;
3404 
3405  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
3406  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
3407 
3408  /* Initialize state variables */
3409  cstate->reached_eof = false;
3410  cstate->eol_type = EOL_UNKNOWN;
3411  cstate->cur_relname = RelationGetRelationName(cstate->rel);
3412  cstate->cur_lineno = 0;
3413  cstate->cur_attname = NULL;
3414  cstate->cur_attval = NULL;
3415 
3416  /*
3417  * Set up variables to avoid per-attribute overhead. attribute_buf and
3418  * raw_buf are used in both text and binary modes, but we use line_buf
3419  * only in text mode.
3420  */
3421  initStringInfo(&cstate->attribute_buf);
3422  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
3423  cstate->raw_buf_index = cstate->raw_buf_len = 0;
3424  if (!cstate->binary)
3425  {
3426  initStringInfo(&cstate->line_buf);
3427  cstate->line_buf_converted = false;
3428  }
3429 
3430  /* Assign range table, we'll need it in CopyFrom. */
3431  if (pstate)
3432  cstate->range_table = pstate->p_rtable;
3433 
3434  tupDesc = RelationGetDescr(cstate->rel);
3435  num_phys_attrs = tupDesc->natts;
3436  num_defaults = 0;
3437  volatile_defexprs = false;
3438 
3439  /*
3440  * Pick up the required catalog information for each attribute in the
3441  * relation, including the input function, the element type (to pass to
3442  * the input function), and info about defaults and constraints. (Which
3443  * input function we use depends on text/binary format choice.)
3444  */
3445  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
3446  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3447  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3448  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3449 
3450  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3451  {
3452  Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
3453 
3454  /* We don't need info for dropped attributes */
3455  if (att->attisdropped)
3456  continue;
3457 
3458  /* Fetch the input function and typioparam info */
3459  if (cstate->binary)
3460  getTypeBinaryInputInfo(att->atttypid,
3461  &in_func_oid, &typioparams[attnum - 1]);
3462  else
3463  getTypeInputInfo(att->atttypid,
3464  &in_func_oid, &typioparams[attnum - 1]);
3465  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3466 
3467  /* Get default info if needed */
3468  if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated)
3469  {
3470  /* attribute is NOT to be copied from input */
3471  /* use default value if one exists */
3472  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
3473  attnum);
3474 
3475  if (defexpr != NULL)
3476  {
3477  /* Run the expression through planner */
3478  defexpr = expression_planner(defexpr);
3479 
3480  /* Initialize executable expression in copycontext */
3481  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3482  defmap[num_defaults] = attnum - 1;
3483  num_defaults++;
3484 
3485  /*
3486  * If a default expression looks at the table being loaded,
3487  * then it could give the wrong answer when using
3488  * multi-insert. Since database access can be dynamic this is
3489  * hard to test for exactly, so we use the much wider test of
3490  * whether the default expression is volatile. We allow for
3491  * the special case of when the default expression is the
3492  * nextval() of a sequence which in this specific case is
3493  * known to be safe for use with the multi-insert
3494  * optimization. Hence we use this special case function
3495  * checker rather than the standard check for
3496  * contain_volatile_functions().
3497  */
3498  if (!volatile_defexprs)
3499  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3500  }
3501  }
3502  }
3503 
3504  /* We keep those variables in cstate. */
3505  cstate->in_functions = in_functions;
3506  cstate->typioparams = typioparams;
3507  cstate->defmap = defmap;
3508  cstate->defexprs = defexprs;
3509  cstate->volatile_defexprs = volatile_defexprs;
3510  cstate->num_defaults = num_defaults;
3511  cstate->is_program = is_program;
3512 
3513  if (data_source_cb)
3514  {
3515  cstate->copy_dest = COPY_CALLBACK;
3516  cstate->data_source_cb = data_source_cb;
3517  }
3518  else if (pipe)
3519  {
3520  Assert(!is_program); /* the grammar does not allow this */
3521  if (whereToSendOutput == DestRemote)
3522  ReceiveCopyBegin(cstate);
3523  else
3524  cstate->copy_file = stdin;
3525  }
3526  else
3527  {
3528  cstate->filename = pstrdup(filename);
3529 
3530  if (cstate->is_program)
3531  {
3532  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3533  if (cstate->copy_file == NULL)
3534  ereport(ERROR,
3535  (errcode_for_file_access(),
3536  errmsg("could not execute command \"%s\": %m",
3537  cstate->filename)));
3538  }
3539  else
3540  {
3541  struct stat st;
3542 
3543  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3544  if (cstate->copy_file == NULL)
3545  {
3546  /* copy errno because ereport subfunctions might change it */
3547  int save_errno = errno;
3548 
3549  ereport(ERROR,
3550  (errcode_for_file_access(),
3551  errmsg("could not open file \"%s\" for reading: %m",
3552  cstate->filename),
3553  (save_errno == ENOENT || save_errno == EACCES) ?
3554  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3555  "You may want a client-side facility such as psql's \\copy.") : 0));
3556  }
3557 
3558  if (fstat(fileno(cstate->copy_file), &st))
3559  ereport(ERROR,
3560  (errcode_for_file_access(),
3561  errmsg("could not stat file \"%s\": %m",
3562  cstate->filename)));
3563 
3564  if (S_ISDIR(st.st_mode))
3565  ereport(ERROR,
3566  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3567  errmsg("\"%s\" is a directory", cstate->filename)));
3568  }
3569  }
3570 
3571  if (cstate->binary)
3572  {
3573  /* Read and verify binary header */
3574  char readSig[11];
3575  int32 tmp;
3576 
3577  /* Signature */
3578  if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
3579  memcmp(readSig, BinarySignature, 11) != 0)
3580  ereport(ERROR,
3581  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3582  errmsg("COPY file signature not recognized")));
3583  /* Flags field */
3584  if (!CopyGetInt32(cstate, &tmp))
3585  ereport(ERROR,
3586  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3587  errmsg("invalid COPY file header (missing flags)")));
3588  if ((tmp & (1 << 16)) != 0)
3589  ereport(ERROR,
3590  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3591  errmsg("invalid COPY file header (WITH OIDS)")));
3592  tmp &= ~(1 << 16);
3593  if ((tmp >> 16) != 0)
3594  ereport(ERROR,
3595  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3596  errmsg("unrecognized critical flags in COPY file header")));
3597  /* Header extension length */
3598  if (!CopyGetInt32(cstate, &tmp) ||
3599  tmp < 0)
3600  ereport(ERROR,
3601  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3602  errmsg("invalid COPY file header (missing length)")));
3603  /* Skip extension header, if present */
3604  while (tmp-- > 0)
3605  {
3606  if (CopyReadBinaryData(cstate, readSig, 1) != 1)
3607  ereport(ERROR,
3608  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3609  errmsg("invalid COPY file header (wrong length)")));
3610  }
3611  }
3612 
3613  /* create workspace for CopyReadAttributes results */
3614  if (!cstate->binary)
3615  {
3616  AttrNumber attr_count = list_length(cstate->attnumlist);
3617 
3618  cstate->max_fields = attr_count;
3619  cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
3620  }
3621 
3622  MemoryContextSwitchTo(oldcontext);
3623 
3624  return cstate;
3625 }
3626 
3627 /*
3628  * Read raw fields in the next line for COPY FROM in text or csv mode.
3629  * Return false if no more lines.
3630  *
3631  * An internal temporary buffer is returned via 'fields'. It is valid until
3632  * the next call of the function. Since the function returns all raw fields
3633  * in the input file, 'nfields' could be different from the number of columns
3634  * in the relation.
3635  *
3636  * NOTE: force_not_null option are not applied to the returned fields.
3637  */
3638 bool
3639 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3640 {
3641  int fldct;
3642  bool done;
3643 
3644  /* only available for text or csv input */
3645  Assert(!cstate->binary);
3646 
3647  /* on input just throw the header line away */
3648  if (cstate->cur_lineno == 0 && cstate->header_line)
3649  {
3650  cstate->cur_lineno++;
3651  if (CopyReadLine(cstate))
3652  return false; /* done */
3653  }
3654 
3655  cstate->cur_lineno++;
3656 
3657  /* Actually read the line into memory here */
3658  done = CopyReadLine(cstate);
3659 
3660  /*
3661  * EOF at start of line means we're done. If we see EOF after some
3662  * characters, we act as though it was newline followed by EOF, ie,
3663  * process the line and then exit loop on next iteration.
3664  */
3665  if (done && cstate->line_buf.len == 0)
3666  return false;
3667 
3668  /* Parse the line into de-escaped field values */
3669  if (cstate->csv_mode)
3670  fldct = CopyReadAttributesCSV(cstate);
3671  else
3672  fldct = CopyReadAttributesText(cstate);
3673 
3674  *fields = cstate->raw_fields;
3675  *nfields = fldct;
3676  return true;
3677 }
3678 
3679 /*
3680  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3681  *
3682  * 'econtext' is used to evaluate default expression for each columns not
3683  * read from the file. It can be NULL when no default values are used, i.e.
3684  * when all columns are read from the file.
3685  *
3686  * 'values' and 'nulls' arrays must be the same length as columns of the
3687  * relation passed to BeginCopyFrom. This function fills the arrays.
3688  * Oid of the tuple is returned with 'tupleOid' separately.
3689  */
3690 bool
3691 NextCopyFrom(CopyState cstate, ExprContext *econtext,
3692  Datum *values, bool *nulls)
3693 {
3694  TupleDesc tupDesc;
3695  AttrNumber num_phys_attrs,
3696  attr_count,
3697  num_defaults = cstate->num_defaults;
3698  FmgrInfo *in_functions = cstate->in_functions;
3699  Oid *typioparams = cstate->typioparams;
3700  int i;
3701  int *defmap = cstate->defmap;
3702  ExprState **defexprs = cstate->defexprs;
3703 
3704  tupDesc = RelationGetDescr(cstate->rel);
3705  num_phys_attrs = tupDesc->natts;
3706  attr_count = list_length(cstate->attnumlist);
3707 
3708  /* Initialize all values for row to NULL */
3709  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3710  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3711 
3712  if (!cstate->binary)
3713  {
3714  char **field_strings;
3715  ListCell *cur;
3716  int fldct;
3717  int fieldno;
3718  char *string;
3719 
3720  /* read raw fields in the next line */
3721  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3722  return false;
3723 
3724  /* check for overflowing fields */
3725  if (attr_count > 0 && fldct > attr_count)
3726  ereport(ERROR,
3727  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3728  errmsg("extra data after last expected column")));
3729 
3730  fieldno = 0;
3731 
3732  /* Loop to read the user attributes on the line. */
3733  foreach(cur, cstate->attnumlist)
3734  {
3735  int attnum = lfirst_int(cur);
3736  int m = attnum - 1;
3737  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3738 
3739  if (fieldno >= fldct)
3740  ereport(ERROR,
3741  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3742  errmsg("missing data for column \"%s\"",
3743  NameStr(att->attname))));
3744  string = field_strings[fieldno++];
3745 
3746  if (cstate->convert_select_flags &&
3747  !cstate->convert_select_flags[m])
3748  {
3749  /* ignore input field, leaving column as NULL */
3750  continue;
3751  }
3752 
3753  if (cstate->csv_mode)
3754  {
3755  if (string == NULL &&
3756  cstate->force_notnull_flags[m])
3757  {
3758  /*
3759  * FORCE_NOT_NULL option is set and column is NULL -
3760  * convert it to the NULL string.
3761  */
3762  string = cstate->null_print;
3763  }
3764  else if (string != NULL && cstate->force_null_flags[m]
3765  && strcmp(string, cstate->null_print) == 0)
3766  {
3767  /*
3768  * FORCE_NULL option is set and column matches the NULL
3769  * string. It must have been quoted, or otherwise the
3770  * string would already have been set to NULL. Convert it
3771  * to NULL as specified.
3772  */
3773  string = NULL;
3774  }
3775  }
3776 
3777  cstate->cur_attname = NameStr(att->attname);
3778  cstate->cur_attval = string;
3779  values[m] = InputFunctionCall(&in_functions[m],
3780  string,
3781  typioparams[m],
3782  att->atttypmod);
3783  if (string != NULL)
3784  nulls[m] = false;
3785  cstate->cur_attname = NULL;
3786  cstate->cur_attval = NULL;
3787  }
3788 
3789  Assert(fieldno == attr_count);
3790  }
3791  else
3792  {
3793  /* binary */
3794  int16 fld_count;
3795  ListCell *cur;
3796 
3797  cstate->cur_lineno++;
3798 
3799  if (!CopyGetInt16(cstate, &fld_count))
3800  {
3801  /* EOF detected (end of file, or protocol-level EOF) */
3802  return false;
3803  }
3804 
3805  if (fld_count == -1)
3806  {
3807  /*
3808  * Received EOF marker. In a V3-protocol copy, wait for the
3809  * protocol-level EOF, and complain if it doesn't come
3810  * immediately. This ensures that we correctly handle CopyFail,
3811  * if client chooses to send that now.
3812  *
3813  * Note that we MUST NOT try to read more data in an old-protocol
3814  * copy, since there is no protocol-level EOF marker then. We
3815  * could go either way for copy from file, but choose to throw
3816  * error if there's data after the EOF marker, for consistency
3817  * with the new-protocol case.
3818  */
3819  char dummy;
3820 
3821  if (cstate->copy_dest != COPY_OLD_FE &&
3822  CopyReadBinaryData(cstate, &dummy, 1) > 0)
3823  ereport(ERROR,
3824  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3825  errmsg("received copy data after EOF marker")));
3826  return false;
3827  }
3828 
3829  if (fld_count != attr_count)
3830  ereport(ERROR,
3831  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3832  errmsg("row field count is %d, expected %d",
3833  (int) fld_count, attr_count)));
3834 
3835  foreach(cur, cstate->attnumlist)
3836  {
3837  int attnum = lfirst_int(cur);
3838  int m = attnum - 1;
3839  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3840 
3841  cstate->cur_attname = NameStr(att->attname);
3842  values[m] = CopyReadBinaryAttribute(cstate,
3843  &in_functions[m],
3844  typioparams[m],
3845  att->atttypmod,
3846  &nulls[m]);
3847  cstate->cur_attname = NULL;
3848  }
3849  }
3850 
3851  /*
3852  * Now compute and insert any defaults available for the columns not
3853  * provided by the input data. Anything not processed here or above will
3854  * remain NULL.
3855  */
3856  for (i = 0; i < num_defaults; i++)
3857  {
3858  /*
3859  * The caller must supply econtext and have switched into the
3860  * per-tuple memory context in it.
3861  */
3862  Assert(econtext != NULL);
3863  Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
3864 
3865  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3866  &nulls[defmap[i]]);
3867  }
3868 
3869  return true;
3870 }
3871 
3872 /*
3873  * Clean up storage and release resources for COPY FROM.
3874  */
3875 void
3876 EndCopyFrom(CopyState cstate)
3877 {
3878  /* No COPY FROM related resources except memory. */
3879 
3880  EndCopy(cstate);
3881 }
3882 
3883 /*
3884  * Read the next input line and stash it in line_buf, with conversion to
3885  * server encoding.
3886  *
3887  * Result is true if read was terminated by EOF, false if terminated
3888  * by newline. The terminating newline or EOF marker is not included
3889  * in the final value of line_buf.
3890  */
3891 static bool
3892 CopyReadLine(CopyState cstate)
3893 {
3894  bool result;
3895 
3896  resetStringInfo(&cstate->line_buf);
3897  cstate->line_buf_valid = true;
3898 
3899  /* Mark that encoding conversion hasn't occurred yet */
3900  cstate->line_buf_converted = false;
3901 
3902  /* Parse data and transfer into line_buf */
3903  result = CopyReadLineText(cstate);
3904 
3905  if (result)
3906  {
3907  /*
3908  * Reached EOF. In protocol version 3, we should ignore anything
3909  * after \. up to the protocol end of copy data. (XXX maybe better
3910  * not to treat \. as special?)
3911  */
3912  if (cstate->copy_dest == COPY_NEW_FE)
3913  {
3914  do
3915  {
3916  cstate->raw_buf_index = cstate->raw_buf_len;
3917  } while (CopyLoadRawBuf(cstate));
3918  }
3919  }
3920  else
3921  {
3922  /*
3923  * If we didn't hit EOF, then we must have transferred the EOL marker
3924  * to line_buf along with the data. Get rid of it.
3925  */
3926  switch (cstate->eol_type)
3927  {
3928  case EOL_NL:
3929  Assert(cstate->line_buf.len >= 1);
3930  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3931  cstate->line_buf.len--;
3932  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3933  break;
3934  case EOL_CR:
3935  Assert(cstate->line_buf.len >= 1);
3936  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3937  cstate->line_buf.len--;
3938  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3939  break;
3940  case EOL_CRNL:
3941  Assert(cstate->line_buf.len >= 2);
3942  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3943  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3944  cstate->line_buf.len -= 2;
3945  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3946  break;
3947  case EOL_UNKNOWN:
3948  /* shouldn't get here */
3949  Assert(false);
3950  break;
3951  }
3952  }
3953 
3954  /* Done reading the line. Convert it to server encoding. */
3955  if (cstate->need_transcoding)
3956  {
3957  char *cvt;
3958 
3959  cvt = pg_any_to_server(cstate->line_buf.data,
3960  cstate->line_buf.len,
3961  cstate->file_encoding);
3962  if (cvt != cstate->line_buf.data)
3963  {
3964  /* transfer converted data back to line_buf */
3965  resetStringInfo(&cstate->line_buf);
3966  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3967  pfree(cvt);
3968  }
3969  }
3970 
3971  /* Now it's safe to use the buffer in error messages */
3972  cstate->line_buf_converted = true;
3973 
3974  return result;
3975 }
3976 
3977 /*
3978  * CopyReadLineText - inner loop of CopyReadLine for text mode
3979  */
3980 static bool
3981 CopyReadLineText(CopyState cstate)
3982 {
3983  char *copy_raw_buf;
3984  int raw_buf_ptr;
3985  int copy_buf_len;
3986  bool need_data = false;
3987  bool hit_eof = false;
3988  bool result = false;
3989  char mblen_str[2];
3990 
3991  /* CSV variables */
3992  bool first_char_in_line = true;
3993  bool in_quote = false,
3994  last_was_esc = false;
3995  char quotec = '\0';
3996  char escapec = '\0';
3997 
3998  if (cstate->csv_mode)
3999  {
4000  quotec = cstate->quote[0];
4001  escapec = cstate->escape[0];
4002  /* ignore special escape processing if it's the same as quotec */
4003  if (quotec == escapec)
4004  escapec = '\0';
4005  }
4006 
4007  mblen_str[1] = '\0';
4008 
4009  /*
4010  * The objective of this loop is to transfer the entire next input line
4011  * into line_buf. Hence, we only care for detecting newlines (\r and/or
4012  * \n) and the end-of-copy marker (\.).
4013  *
4014  * In CSV mode, \r and \n inside a quoted field are just part of the data
4015  * value and are put in line_buf. We keep just enough state to know if we
4016  * are currently in a quoted field or not.
4017  *
4018  * These four characters, and the CSV escape and quote characters, are
4019  * assumed the same in frontend and backend encodings.
4020  *
4021  * For speed, we try to move data from raw_buf to line_buf in chunks
4022  * rather than one character at a time. raw_buf_ptr points to the next
4023  * character to examine; any characters from raw_buf_index to raw_buf_ptr
4024  * have been determined to be part of the line, but not yet transferred to
4025  * line_buf.
4026  *
4027  * For a little extra speed within the loop, we copy raw_buf and
4028  * raw_buf_len into local variables.
4029  */
4030  copy_raw_buf = cstate->raw_buf;
4031  raw_buf_ptr = cstate->raw_buf_index;
4032  copy_buf_len = cstate->raw_buf_len;
4033 
4034  for (;;)
4035  {
4036  int prev_raw_ptr;
4037  char c;
4038 
4039  /*
4040  * Load more data if needed. Ideally we would just force four bytes
4041  * of read-ahead and avoid the many calls to
4042  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
4043  * does not allow us to read too far ahead or we might read into the
4044  * next data, so we read-ahead only as far we know we can. One
4045  * optimization would be to read-ahead four byte here if
4046  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
4047  * considering the size of the buffer.
4048  */
4049  if (raw_buf_ptr >= copy_buf_len || need_data)
4050  {
4051  REFILL_LINEBUF;
4052 
4053  /*
4054  * Try to read some more data. This will certainly reset
4055  * raw_buf_index to zero, and raw_buf_ptr must go with it.
4056  */
4057  if (!CopyLoadRawBuf(cstate))
4058  hit_eof = true;
4059  raw_buf_ptr = 0;
4060  copy_buf_len = cstate->raw_buf_len;
4061 
4062  /*
4063  * If we are completely out of data, break out of the loop,
4064  * reporting EOF.
4065  */
4066  if (copy_buf_len <= 0)
4067  {
4068  result = true;
4069  break;
4070  }
4071  need_data = false;
4072  }
4073 
4074  /* OK to fetch a character */
4075  prev_raw_ptr = raw_buf_ptr;
4076  c = copy_raw_buf[raw_buf_ptr++];
4077 
4078  if (cstate->csv_mode)
4079  {
4080  /*
4081  * If character is '\\' or '\r', we may need to look ahead below.
4082  * Force fetch of the next character if we don't already have it.
4083  * We need to do this before changing CSV state, in case one of
4084  * these characters is also the quote or escape character.
4085  *
4086  * Note: old-protocol does not like forced prefetch, but it's OK
4087  * here since we cannot validly be at EOF.
4088  */
4089  if (c == '\\' || c == '\r')
4090  {
4091  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4092  }
4093 
4094  /*
4095  * Dealing with quotes and escapes here is mildly tricky. If the
4096  * quote char is also the escape char, there's no problem - we
4097  * just use the char as a toggle. If they are different, we need
4098  * to ensure that we only take account of an escape inside a
4099  * quoted field and immediately preceding a quote char, and not
4100  * the second in an escape-escape sequence.
4101  */
4102  if (in_quote && c == escapec)
4103  last_was_esc = !last_was_esc;
4104  if (c == quotec && !last_was_esc)
4105  in_quote = !in_quote;
4106  if (c != escapec)
4107  last_was_esc = false;
4108 
4109  /*
4110  * Updating the line count for embedded CR and/or LF chars is
4111  * necessarily a little fragile - this test is probably about the
4112  * best we can do. (XXX it's arguable whether we should do this
4113  * at all --- is cur_lineno a physical or logical count?)
4114  */
4115  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
4116  cstate->cur_lineno++;
4117  }
4118 
4119  /* Process \r */
4120  if (c == '\r' && (!cstate->csv_mode || !in_quote))
4121  {
4122  /* Check for \r\n on first line, _and_ handle \r\n. */
4123  if (cstate->eol_type == EOL_UNKNOWN ||
4124  cstate->eol_type == EOL_CRNL)
4125  {
4126  /*
4127  * If need more data, go back to loop top to load it.
4128  *
4129  * Note that if we are at EOF, c will wind up as '\0' because
4130  * of the guaranteed pad of raw_buf.
4131  */
4132  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4133 
4134  /* get next char */
4135  c = copy_raw_buf[raw_buf_ptr];
4136 
4137  if (c == '\n')
4138  {
4139  raw_buf_ptr++; /* eat newline */
4140  cstate->eol_type = EOL_CRNL; /* in case not set yet */
4141  }
4142  else
4143  {
4144  /* found \r, but no \n */
4145  if (cstate->eol_type == EOL_CRNL)
4146  ereport(ERROR,
4147  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4148  !cstate->csv_mode ?
4149  errmsg("literal carriage return found in data") :
4150  errmsg("unquoted carriage return found in data"),
4151  !cstate->csv_mode ?
4152  errhint("Use \"\\r\" to represent carriage return.") :
4153  errhint("Use quoted CSV field to represent carriage return.")));
4154 
4155  /*
4156  * if we got here, it is the first line and we didn't find
4157  * \n, so don't consume the peeked character
4158  */
4159  cstate->eol_type = EOL_CR;
4160  }
4161  }
4162  else if (cstate->eol_type == EOL_NL)
4163  ereport(ERROR,
4164  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4165  !cstate->csv_mode ?
4166  errmsg("literal carriage return found in data") :
4167  errmsg("unquoted carriage return found in data"),
4168  !cstate->csv_mode ?
4169  errhint("Use \"\\r\" to represent carriage return.") :
4170  errhint("Use quoted CSV field to represent carriage return.")));
4171  /* If reach here, we have found the line terminator */
4172  break;
4173  }
4174 
4175  /* Process \n */
4176  if (c == '\n' && (!cstate->csv_mode || !in_quote))
4177  {
4178  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
4179  ereport(ERROR,
4180  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4181  !cstate->csv_mode ?
4182  errmsg("literal newline found in data") :
4183  errmsg("unquoted newline found in data"),
4184  !cstate->csv_mode ?
4185  errhint("Use \"\\n\" to represent newline.") :
4186  errhint("Use quoted CSV field to represent newline.")));
4187  cstate->eol_type = EOL_NL; /* in case not set yet */
4188  /* If reach here, we have found the line terminator */
4189  break;
4190  }
4191 
4192  /*
4193  * In CSV mode, we only recognize \. alone on a line. This is because
4194  * \. is a valid CSV data value.
4195  */
4196  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
4197  {
4198  char c2;
4199 
4200  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4201  IF_NEED_REFILL_AND_EOF_BREAK(0);
4202 
4203  /* -----
4204  * get next character
4205  * Note: we do not change c so if it isn't \., we can fall
4206  * through and continue processing for file encoding.
4207  * -----
4208  */
4209  c2 = copy_raw_buf[raw_buf_ptr];
4210 
4211  if (c2 == '.')
4212  {
4213  raw_buf_ptr++; /* consume the '.' */
4214 
4215  /*
4216  * Note: if we loop back for more data here, it does not
4217  * matter that the CSV state change checks are re-executed; we
4218  * will come back here with no important state changed.
4219  */
4220  if (cstate->eol_type == EOL_CRNL)
4221  {
4222  /* Get the next character */
4223  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4224  /* if hit_eof, c2 will become '\0' */
4225  c2 = copy_raw_buf[raw_buf_ptr++];
4226 
4227  if (c2 == '\n')
4228  {
4229  if (!cstate->csv_mode)
4230  ereport(ERROR,
4231  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4232  errmsg("end-of-copy marker does not match previous newline style")));
4233  else
4234  NO_END_OF_COPY_GOTO;
4235  }
4236  else if (c2 != '\r')
4237  {
4238  if (!cstate->csv_mode)
4239  ereport(ERROR,
4240  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4241  errmsg("end-of-copy marker corrupt")));
4242  else
4243  NO_END_OF_COPY_GOTO;
4244  }
4245  }
4246 
4247  /* Get the next character */
4248  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
4249  /* if hit_eof, c2 will become '\0' */
4250  c2 = copy_raw_buf[raw_buf_ptr++];
4251 
4252  if (c2 != '\r' && c2 != '\n')
4253  {
4254  if (!cstate->csv_mode)
4255  ereport(ERROR,
4256  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4257  errmsg("end-of-copy marker corrupt")));
4258  else
4259  NO_END_OF_COPY_GOTO;
4260  }
4261 
4262  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
4263  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
4264  (cstate->eol_type == EOL_CR && c2 != '\r'))
4265  {
4266  ereport(ERROR,
4267  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4268  errmsg("end-of-copy marker does not match previous newline style")));
4269  }
4270 
4271  /*
4272  * Transfer only the data before the \. into line_buf, then
4273  * discard the data and the \. sequence.
4274  */
4275  if (prev_raw_ptr > cstate->raw_buf_index)
4276  appendBinaryStringInfo(&cstate->line_buf,
4277  cstate->raw_buf + cstate->raw_buf_index,
4278  prev_raw_ptr - cstate->raw_buf_index);
4279  cstate->raw_buf_index = raw_buf_ptr;
4280  result = true; /* report EOF */
4281  break;
4282  }
4283  else if (!cstate->csv_mode)
4284 
4285  /*
4286  * If we are here, it means we found a backslash followed by
4287  * something other than a period. In non-CSV mode, anything
4288  * after a backslash is special, so we skip over that second
4289  * character too. If we didn't do that \\. would be
4290  * considered an eof-of copy, while in non-CSV mode it is a
4291  * literal backslash followed by a period. In CSV mode,
4292  * backslashes are not special, so we want to process the
4293  * character after the backslash just like a normal character,
4294  * so we don't increment in those cases.
4295  */
4296  raw_buf_ptr++;
4297  }
4298 
4299  /*
4300  * This label is for CSV cases where \. appears at the start of a
4301  * line, but there is more text after it, meaning it was a data value.
4302  * We are more strict for \. in CSV mode because \. could be a data
4303  * value, while in non-CSV mode, \. cannot be a data value.
4304  */
4305 not_end_of_copy:
4306 
4307  /*
4308  * Process all bytes of a multi-byte character as a group.
4309  *
4310  * We only support multi-byte sequences where the first byte has the
4311  * high-bit set, so as an optimization we can avoid this block
4312  * entirely if it is not set.
4313  */
4314  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
4315  {
4316  int mblen;
4317 
4318  /*
4319  * It is enough to look at the first byte in all our encodings, to
4320  * get the length. (GB18030 is a bit special, but still works for
4321  * our purposes; see comment in pg_gb18030_mblen())
4322  */
4323  mblen_str[0] = c;
4324  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
4325 
4326  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
4327  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
4328  raw_buf_ptr += mblen - 1;
4329  }
4330  first_char_in_line = false;
4331  } /* end of outer loop */
4332 
4333  /*
4334  * Transfer any still-uncopied data to line_buf.
4335  */
4336  REFILL_LINEBUF;
4337 
4338  return result;
4339 }
4340 
4341 /*
4342  * Return decimal value for a hexadecimal digit
4343  */
4344 static int
4345 GetDecimalFromHex(char hex)
4346 {
4347  if (isdigit((unsigned char) hex))
4348  return hex - '0';
4349  else
4350  return tolower((unsigned char) hex) - 'a' + 10;
4351 }
4352 
4353 /*
4354  * Parse the current line into separate attributes (fields),
4355  * performing de-escaping as needed.
4356  *
4357  * The input is in line_buf. We use attribute_buf to hold the result
4358  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
4359  * string, or NULL when the input matches the null marker string.
4360  * This array is expanded as necessary.
4361  *
4362  * (Note that the caller cannot check for nulls since the returned
4363  * string would be the post-de-escaping equivalent, which may look
4364  * the same as some valid data string.)
4365  *
4366  * delim is the column delimiter string (must be just one byte for now).
4367  * null_print is the null marker string. Note that this is compared to
4368  * the pre-de-escaped input string.
4369  *
4370  * The return value is the number of fields actually read.
4371  */
4372 static int
4373 CopyReadAttributesText(CopyState cstate)
4374 {
4375  char delimc = cstate->delim[0];
4376  int fieldno;
4377  char *output_ptr;
4378  char *cur_ptr;
4379  char *line_end_ptr;
4380 
4381  /*
4382  * We need a special case for zero-column tables: check that the input
4383  * line is empty, and return.
4384  */
4385  if (cstate->max_fields <= 0)
4386  {
4387  if (cstate->line_buf.len != 0)
4388  ereport(ERROR,
4389  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4390  errmsg("extra data after last expected column")));
4391  return 0;
4392  }
4393 
4394  resetStringInfo(&cstate->attribute_buf);
4395 
4396  /*
4397  * The de-escaped attributes will certainly not be longer than the input
4398  * data line, so we can just force attribute_buf to be large enough and
4399  * then transfer data without any checks for enough space. We need to do
4400  * it this way because enlarging attribute_buf mid-stream would invalidate
4401  * pointers already stored into cstate->raw_fields[].
4402  */
4403  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4404  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4405  output_ptr = cstate->attribute_buf.data;
4406 
4407  /* set pointer variables for loop */
4408  cur_ptr = cstate->line_buf.data;
4409  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4410 
4411  /* Outer loop iterates over fields */
4412  fieldno = 0;
4413  for (;;)
4414  {
4415  bool found_delim = false;
4416  char *start_ptr;
4417  char *end_ptr;
4418  int input_len;
4419  bool saw_non_ascii = false;
4420 
4421  /* Make sure there is enough space for the next value */
4422  if (fieldno >= cstate->max_fields)
4423  {
4424  cstate->max_fields *= 2;
4425  cstate->raw_fields =
4426  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4427  }
4428 
4429  /* Remember start of field on both input and output sides */
4430  start_ptr = cur_ptr;
4431  cstate->raw_fields[fieldno] = output_ptr;
4432 
4433  /*
4434  * Scan data for field.
4435  *
4436  * Note that in this loop, we are scanning to locate the end of field
4437  * and also speculatively performing de-escaping. Once we find the
4438  * end-of-field, we can match the raw field contents against the null
4439  * marker string. Only after that comparison fails do we know that
4440  * de-escaping is actually the right thing to do; therefore we *must
4441  * not* throw any syntax errors before we've done the null-marker
4442  * check.
4443  */
4444  for (;;)
4445  {
4446  char c;
4447 
4448  end_ptr = cur_ptr;
4449  if (cur_ptr >= line_end_ptr)
4450  break;
4451  c = *cur_ptr++;
4452  if (c == delimc)
4453  {
4454  found_delim = true;
4455  break;
4456  }
4457  if (c == '\\')
4458  {
4459  if (cur_ptr >= line_end_ptr)
4460  break;
4461  c = *cur_ptr++;
4462  switch (c)
4463  {
4464  case '0':
4465  case '1':
4466  case '2':
4467  case '3':
4468  case '4':
4469  case '5':
4470  case '6':
4471  case '7':
4472  {
4473  /* handle \013 */
4474  int val;
4475 
4476  val = OCTVALUE(c);
4477  if (cur_ptr < line_end_ptr)
4478  {
4479  c = *cur_ptr;
4480  if (ISOCTAL(c))
4481  {
4482  cur_ptr++;
4483  val = (val << 3) + OCTVALUE(c);
4484  if (cur_ptr < line_end_ptr)
4485  {
4486  c = *cur_ptr;
4487  if (ISOCTAL(c))
4488  {
4489  cur_ptr++;
4490  val = (val << 3) + OCTVALUE(c);
4491  }
4492  }
4493  }
4494  }
4495  c = val & 0377;
4496  if (c == '\0' || IS_HIGHBIT_SET(c))
4497  saw_non_ascii = true;
4498  }
4499  break;
4500  case 'x':
4501  /* Handle \x3F */
4502  if (cur_ptr < line_end_ptr)
4503  {
4504  char hexchar = *cur_ptr;
4505 
4506  if (isxdigit((unsigned char) hexchar))
4507  {
4508  int val = GetDecimalFromHex(hexchar);
4509 
4510  cur_ptr++;
4511  if (cur_ptr < line_end_ptr)
4512  {
4513  hexchar = *cur_ptr;
4514  if (isxdigit((unsigned char) hexchar))
4515  {
4516  cur_ptr++;
4517  val = (val << 4) + GetDecimalFromHex(hexchar);
4518  }
4519  }
4520  c = val & 0xff;
4521  if (c == '\0' || IS_HIGHBIT_SET(c))
4522  saw_non_ascii = true;
4523  }
4524  }
4525  break;
4526  case 'b':
4527  c = '\b';
4528  break;
4529  case 'f':
4530  c = '\f';
4531  break;
4532  case 'n':
4533  c = '\n';
4534  break;
4535  case 'r':
4536  c = '\r';
4537  break;
4538  case 't':
4539  c = '\t';
4540  break;
4541  case 'v':
4542  c = '\v';
4543  break;
4544 
4545  /*
4546  * in all other cases, take the char after '\'
4547  * literally
4548  */
4549  }
4550  }
4551 
4552  /* Add c to output string */
4553  *output_ptr++ = c;
4554  }
4555 
4556  /* Check whether raw input matched null marker */
4557  input_len = end_ptr - start_ptr;
4558  if (input_len == cstate->null_print_len &&
4559  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4560  cstate->raw_fields[fieldno] = NULL;
4561  else
4562  {
4563  /*
4564  * At this point we know the field is supposed to contain data.
4565  *
4566  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4567  * resulting string is valid data for the db encoding.
4568  */
4569  if (saw_non_ascii)
4570  {
4571  char *fld = cstate->raw_fields[fieldno];
4572 
4573  pg_verifymbstr(fld, output_ptr - fld, false);
4574  }
4575  }
4576 
4577  /* Terminate attribute value in output area */
4578  *output_ptr++ = '\0';
4579 
4580  fieldno++;
4581  /* Done if we hit EOL instead of a delim */
4582  if (!found_delim)
4583  break;
4584  }
4585 
4586  /* Clean up state of attribute_buf */
4587  output_ptr--;
4588  Assert(*output_ptr == '\0');
4589  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4590 
4591  return fieldno;
4592 }
4593 
4594 /*
4595  * Parse the current line into separate attributes (fields),
4596  * performing de-escaping as needed. This has exactly the same API as
4597  * CopyReadAttributesText, except we parse the fields according to
4598  * "standard" (i.e. common) CSV usage.
4599  */
4600 static int
4601 CopyReadAttributesCSV(CopyState cstate)
4602 {
4603  char delimc = cstate->delim[0];
4604  char quotec = cstate->quote[0];
4605  char escapec = cstate->escape[0];
4606  int fieldno;
4607  char *output_ptr;
4608  char *cur_ptr;
4609  char *line_end_ptr;
4610 
4611  /*
4612  * We need a special case for zero-column tables: check that the input
4613  * line is empty, and return.
4614  */
4615  if (cstate->max_fields <= 0)
4616  {
4617  if (cstate->line_buf.len != 0)
4618  ereport(ERROR,
4619  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4620  errmsg("extra data after last expected column")));
4621  return 0;
4622  }
4623 
4624  resetStringInfo(&cstate->attribute_buf);
4625 
4626  /*
4627  * The de-escaped attributes will certainly not be longer than the input
4628  * data line, so we can just force attribute_buf to be large enough and
4629  * then transfer data without any checks for enough space. We need to do
4630  * it this way because enlarging attribute_buf mid-stream would invalidate
4631  * pointers already stored into cstate->raw_fields[].
4632  */
4633  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4634  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4635  output_ptr = cstate->attribute_buf.data;
4636 
4637  /* set pointer variables for loop */
4638  cur_ptr = cstate->line_buf.data;
4639  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4640 
4641  /* Outer loop iterates over fields */
4642  fieldno = 0;
4643  for (;;)
4644  {
4645  bool found_delim = false;
4646  bool saw_quote = false;
4647  char *start_ptr;
4648  char *end_ptr;
4649  int input_len;
4650 
4651  /* Make sure there is enough space for the next value */
4652  if (fieldno >= cstate->max_fields)
4653  {
4654  cstate->max_fields *= 2;
4655  cstate->raw_fields =
4656  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4657  }
4658 
4659  /* Remember start of field on both input and output sides */
4660  start_ptr = cur_ptr;
4661  cstate->raw_fields[fieldno] = output_ptr;
4662 
4663  /*
4664  * Scan data for field,
4665  *
4666  * The loop starts in "not quote" mode and then toggles between that
4667  * and "in quote" mode. The loop exits normally if it is in "not
4668  * quote" mode and a delimiter or line end is seen.
4669  */
4670  for (;;)
4671  {
4672  char c;
4673 
4674  /* Not in quote */
4675  for (;;)
4676  {
4677  end_ptr = cur_ptr;
4678  if (cur_ptr >= line_end_ptr)
4679  goto endfield;
4680  c = *cur_ptr++;
4681  /* unquoted field delimiter */
4682  if (c == delimc)
4683  {
4684  found_delim = true;
4685  goto endfield;
4686  }
4687  /* start of quoted field (or part of field) */
4688  if (c == quotec)
4689  {
4690  saw_quote = true;
4691  break;
4692  }
4693  /* Add c to output string */
4694  *output_ptr++ = c;
4695  }
4696 
4697  /* In quote */
4698  for (;;)
4699  {
4700  end_ptr = cur_ptr;
4701  if (cur_ptr >= line_end_ptr)
4702  ereport(ERROR,
4703  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4704  errmsg("unterminated CSV quoted field")));
4705 
4706  c = *cur_ptr++;
4707 
4708  /* escape within a quoted field */
4709  if (c == escapec)
4710  {
4711  /*
4712  * peek at the next char if available, and escape it if it
4713  * is an escape char or a quote char
4714  */
4715  if (cur_ptr < line_end_ptr)
4716  {
4717  char nextc = *cur_ptr;
4718 
4719  if (nextc == escapec || nextc == quotec)
4720  {
4721  *output_ptr++ = nextc;
4722  cur_ptr++;
4723  continue;
4724  }
4725  }
4726  }
4727 
4728  /*
4729  * end of quoted field. Must do this test after testing for
4730  * escape in case quote char and escape char are the same
4731  * (which is the common case).
4732  */
4733  if (c == quotec)
4734  break;
4735 
4736  /* Add c to output string */
4737  *output_ptr++ = c;
4738  }
4739  }
4740 endfield:
4741 
4742  /* Terminate attribute value in output area */
4743  *output_ptr++ = '\0';
4744 
4745  /* Check whether raw input matched null marker */
4746  input_len = end_ptr - start_ptr;
4747  if (!saw_quote && input_len == cstate->null_print_len &&
4748  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4749  cstate->raw_fields[fieldno] = NULL;
4750 
4751  fieldno++;
4752  /* Done if we hit EOL instead of a delim */
4753  if (!found_delim)
4754  break;
4755  }
4756 
4757  /* Clean up state of attribute_buf */
4758  output_ptr--;
4759  Assert(*output_ptr == '\0');
4760  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4761 
4762  return fieldno;
4763 }
4764 
4765 
4766 /*
4767  * Read a binary attribute
4768  */
4769 static Datum
4770 CopyReadBinaryAttribute(CopyState cstate, FmgrInfo *flinfo,
4771  Oid typioparam, int32 typmod,
4772  bool *isnull)
4773 {
4774  int32 fld_size;
4775  Datum result;
4776 
4777  if (!CopyGetInt32(cstate, &fld_size))
4778  ereport(ERROR,
4779  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4780  errmsg("unexpected EOF in COPY data")));
4781  if (fld_size == -1)
4782  {
4783  *isnull = true;
4784  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4785  }
4786  if (fld_size < 0)
4787  ereport(ERROR,
4788  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4789  errmsg("invalid field size")));
4790 
4791  /* reset attribute_buf to empty, and load raw data in it */
4792  resetStringInfo(&cstate->attribute_buf);
4793 
4794  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4795  if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
4796  fld_size) != fld_size)
4797  ereport(ERROR,
4798  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4799  errmsg("unexpected EOF in COPY data")));
4800 
4801  cstate->attribute_buf.len = fld_size;
4802  cstate->attribute_buf.data[fld_size] = '\0';
4803 
4804  /* Call the column type's binary input converter */
4805  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4806  typioparam, typmod);
4807 
4808  /* Trouble if it didn't eat the whole buffer */
4809  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4810  ereport(ERROR,
4811  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4812  errmsg("incorrect binary data format")));
4813 
4814  *isnull = false;
4815  return result;
4816 }
4817 
4818 /*
4819  * Send text representation of one attribute, with conversion and escaping
4820  */
4821 #define DUMPSOFAR() \
4822  do { \
4823  if (ptr > start) \
4824  CopySendData(cstate, start, ptr - start); \
4825  } while (0)
4826 
4827 static void
4828 CopyAttributeOutText(CopyState cstate, char *string)
4829 {
4830  char *ptr;
4831  char *start;
4832  char c;
4833  char delimc = cstate->delim[0];
4834 
4835  if (cstate->need_transcoding)
4836  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4837  else
4838  ptr = string;
4839 
4840  /*
4841  * We have to grovel through the string searching for control characters
4842  * and instances of the delimiter character. In most cases, though, these
4843  * are infrequent. To avoid overhead from calling CopySendData once per
4844  * character, we dump out all characters between escaped characters in a
4845  * single call. The loop invariant is that the data from "start" to "ptr"
4846  * can be sent literally, but hasn't yet been.
4847  *
4848  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4849  * in valid backend encodings, extra bytes of a multibyte character never
4850  * look like ASCII. This loop is sufficiently performance-critical that
4851  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4852  * of the normal safe-encoding path.
4853  */
4854  if (cstate->encoding_embeds_ascii)
4855  {
4856  start = ptr;
4857  while ((c = *ptr) != '\0')
4858  {
4859  if ((unsigned char) c < (unsigned char) 0x20)
4860  {
4861  /*
4862  * \r and \n must be escaped, the others are traditional. We
4863  * prefer to dump these using the C-like notation, rather than
4864  * a backslash and the literal character, because it makes the
4865  * dump file a bit more proof against Microsoftish data
4866  * mangling.
4867  */
4868  switch (c)
4869  {
4870  case '\b':
4871  c = 'b';
4872  break;
4873  case '\f':
4874  c = 'f';
4875  break;
4876  case '\n':
4877  c = 'n';
4878  break;
4879  case '\r':
4880  c = 'r';
4881  break;
4882  case '\t':
4883  c = 't';
4884  break;
4885  case '\v':
4886  c = 'v';
4887  break;
4888  default:
4889  /* If it's the delimiter, must backslash it */
4890  if (c == delimc)
4891  break;
4892  /* All ASCII control chars are length 1 */
4893  ptr++;
4894  continue; /* fall to end of loop */
4895  }
4896  /* if we get here, we need to convert the control char */
4897  DUMPSOFAR();
4898  CopySendChar(cstate, '\\');
4899  CopySendChar(cstate, c);
4900  start = ++ptr; /* do not include char in next run */
4901  }
4902  else if (c == '\\' || c == delimc)
4903  {
4904  DUMPSOFAR();
4905  CopySendChar(cstate, '\\');
4906  start = ptr++; /* we include char in next run */
4907  }
4908  else if (IS_HIGHBIT_SET(c))
4909  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4910  else
4911  ptr++;
4912  }
4913  }
4914  else
4915  {
4916  start = ptr;
4917  while ((c = *ptr) != '\0')
4918  {
4919  if ((unsigned char) c < (unsigned char) 0x20)
4920  {
4921  /*
4922  * \r and \n must be escaped, the others are traditional. We
4923  * prefer to dump these using the C-like notation, rather than
4924  * a backslash and the literal character, because it makes the
4925  * dump file a bit more proof against Microsoftish data
4926  * mangling.
4927  */
4928  switch (c)
4929  {
4930  case '\b':
4931  c = 'b';
4932  break;
4933  case '\f':
4934  c = 'f';
4935  break;
4936  case '\n':
4937  c = 'n';
4938  break;
4939  case '\r':
4940  c = 'r';
4941  break;
4942  case '\t':
4943  c = 't';
4944  break;
4945  case '\v':
4946  c = 'v';
4947  break;
4948  default:
4949  /* If it's the delimiter, must backslash it */
4950  if (c == delimc)
4951  break;
4952  /* All ASCII control chars are length 1 */
4953  ptr++;
4954  continue; /* fall to end of loop */
4955  }
4956  /* if we get here, we need to convert the control char */
4957  DUMPSOFAR();
4958  CopySendChar(cstate, '\\');
4959  CopySendChar(cstate, c);
4960  start = ++ptr; /* do not include char in next run */
4961  }
4962  else if (c == '\\' || c == delimc)
4963  {
4964  DUMPSOFAR();
4965  CopySendChar(cstate, '\\');
4966  start = ptr++; /* we include char in next run */
4967  }
4968  else
4969  ptr++;
4970  }
4971  }
4972 
4973  DUMPSOFAR();
4974 }
4975 
4976 /*
4977  * Send text representation of one attribute, with conversion and
4978  * CSV-style escaping
4979  */
4980 static void
4981 CopyAttributeOutCSV(CopyState cstate, char *string,
4982  bool use_quote, bool single_attr)
4983 {
4984  char *ptr;
4985  char *start;
4986  char c;
4987  char delimc = cstate->delim[0];
4988  char quotec = cstate->quote[0];
4989  char escapec = cstate->escape[0];
4990 
4991  /* force quoting if it matches null_print (before conversion!) */
4992  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4993  use_quote = true;
4994 
4995  if (cstate->need_transcoding)
4996  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4997  else
4998  ptr = string;
4999 
5000  /*
5001  * Make a preliminary pass to discover if it needs quoting
5002  */
5003  if (!use_quote)
5004  {
5005  /*
5006  * Because '\.' can be a data value, quote it if it appears alone on a
5007  * line so it is not interpreted as the end-of-data marker.
5008  */
5009  if (single_attr && strcmp(ptr, "\\.") == 0)
5010  use_quote = true;
5011  else
5012  {
5013  char *tptr = ptr;
5014 
5015  while ((c = *tptr) != '\0')
5016  {
5017  if (c == delimc || c == quotec || c == '\n' || c == '\r')
5018  {
5019  use_quote = true;
5020  break;
5021  }
5022  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5023  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
5024  else
5025  tptr++;
5026  }
5027  }
5028  }
5029 
5030  if (use_quote)
5031  {
5032  CopySendChar(cstate, quotec);
5033 
5034  /*
5035  * We adopt the same optimization strategy as in CopyAttributeOutText
5036  */
5037  start = ptr;
5038  while ((c = *ptr) != '\0')
5039  {
5040  if (c == quotec || c == escapec)
5041  {
5042  DUMPSOFAR();
5043  CopySendChar(cstate, escapec);
5044  start = ptr; /* we include char in next run */
5045  }
5046  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
5047  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
5048  else
5049  ptr++;
5050  }
5051  DUMPSOFAR();
5052 
5053  CopySendChar(cstate, quotec);
5054  }
5055  else
5056  {
5057  /* If it doesn't need quoting, we can just dump it as-is */
5058  CopySendString(cstate, ptr);
5059  }
5060 }
5061 
5062 /*
5063  * CopyGetAttnums - build an integer list of attnums to be copied
5064  *
5065  * The input attnamelist is either the user-specified column list,
5066  * or NIL if there was none (in which case we want all the non-dropped
5067  * columns).
5068  *
5069  * We don't include generated columns in the generated full list and we don't
5070  * allow them to be specified explicitly. They don't make sense for COPY
5071  * FROM, but we could possibly allow them for COPY TO. But this way it's at
5072  * least ensured that whatever we copy out can be copied back in.
5073  *
5074  * rel can be NULL ... it's only used for error reports.
5075  */
5076 static List *
5077 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
5078 {
5079  List *attnums = NIL;
5080 
5081  if (attnamelist == NIL)
5082  {
5083  /* Generate default column list */
5084  int attr_count = tupDesc->natts;
5085  int i;
5086 
5087  for (i = 0; i < attr_count; i++)
5088  {
5089  if (TupleDescAttr(tupDesc, i)->attisdropped)
5090  continue;
5091  if (TupleDescAttr(tupDesc, i)->attgenerated)
5092  continue;
5093  attnums = lappend_int(attnums, i + 1);
5094  }
5095  }
5096  else
5097  {
5098  /* Validate the user-supplied list and extract attnums */
5099  ListCell *l;
5100 
5101  foreach(l, attnamelist)
5102  {
5103  char *name = strVal(lfirst(l));
5104  int attnum;
5105  int i;
5106 
5107  /* Lookup column name */
5108  attnum = InvalidAttrNumber;
5109  for (i = 0; i < tupDesc->natts; i++)
5110  {
5111  Form_pg_attribute att = TupleDescAttr(tupDesc, i);
5112 
5113  if (att->attisdropped)
5114  continue;
5115  if (namestrcmp(&(att->attname), name) == 0)
5116  {
5117  if (att->attgenerated)
5118  ereport(ERROR,
5119  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
5120  errmsg("column \"%s\" is a generated column",
5121  name),
5122  errdetail("Generated columns cannot be used in COPY.")));
5123  attnum = att->attnum;
5124  break;
5125  }
5126  }
5127  if (attnum == InvalidAttrNumber)
5128  {
5129  if (rel != NULL)
5130  ereport(ERROR,
5131  (errcode(ERRCODE_UNDEFINED_COLUMN),
5132  errmsg("column \"%s\" of relation \"%s\" does not exist",
5133  name, RelationGetRelationName(rel))));
5134  else
5135  ereport(ERROR,
5136  (errcode(ERRCODE_UNDEFINED_COLUMN),
5137  errmsg("column \"%s\" does not exist",
5138  name)));
5139  }
5140  /* Check for duplicates */
5141  if (list_member_int(attnums, attnum))
5142  ereport(ERROR,
5143  (errcode(ERRCODE_DUPLICATE_COLUMN),
5144  errmsg("column \"%s\" specified more than once",
5145  name)));
5146  attnums = lappend_int(attnums, attnum);
5147  }
5148  }
5149 
5150  return attnums;
5151 }
5152 
5153 
5154 /*
5155  * copy_dest_startup --- executor startup
5156  */
5157 static void
5158 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
5159 {
5160  /* no-op */
5161 }
5162 
5163 /*
5164  * copy_dest_receive --- receive one tuple
5165  */
5166 static bool
5167 copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
5168 {
5169  DR_copy *myState = (DR_copy *) self;
5170  CopyState cstate = myState->cstate;
5171 
5172  /* Send the data */
5173  CopyOneRowTo(cstate, slot);
5174  myState->processed++;
5175 
5176  return true;
5177 }
5178 
5179 /*
5180  * copy_dest_shutdown --- executor end
5181  */
5182 static void
5183 copy_dest_shutdown(DestReceiver *self)
5184 {
5185  /* no-op */
5186 }
5187 
5188 /*
5189  * copy_dest_destroy --- release DestReceiver object
5190  */
5191 static void
5192 copy_dest_destroy(DestReceiver *self)
5193 {
5194  pfree(self);
5195 }
5196 
5197 /*
5198  * CreateCopyDestReceiver -- create a suitable DestReceiver object
5199  */
5200 DestReceiver *
5201 CreateCopyDestReceiver(void)
5202 {
5203  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
5204 
5205  self->pub.receiveSlot = copy_dest_receive;
5206  self->pub.rStartup = copy_dest_startup;
5207  self->pub.rShutdown = copy_dest_shutdown;
5208  self->pub.rDestroy = copy_dest_destroy;
5209  self->pub.mydest = DestCopyOut;
5210 
5211  self->cstate = NULL; /* will be set later */
5212  self->processed = 0;
5213 
5214  return (DestReceiver *) self;
5215 }
table_slot_create
TupleTableSlot * table_slot_create(Relation relation, List **reglist)
Definition: tableam.c:91
CopyMultiInsertBuffer
struct CopyMultiInsertBuffer CopyMultiInsertBuffer
int16
signed short int16
Definition: c.h:361
ResTarget::indirection
List * indirection
Definition: parsenodes.h:443
NextCopyFromRawFields
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3639
ResultRelInfo::ri_NumIndices
int ri_NumIndices
Definition: execnodes.h:416
CopyStateData::whereClause
Node * whereClause
Definition: copy.c:156
ExecCopySlot
static TupleTableSlot * ExecCopySlot(TupleTableSlot *dstslot, TupleTableSlot *srcslot)
Definition: tuptable.h:475
NIL
#define NIL
Definition: pg_list.h:65
CopyStmt::whereClause
Node * whereClause
Definition: parsenodes.h:2021
TupleTableSlot::tts_tableOid
Oid tts_tableOid
Definition: tuptable.h:131
pq_sendint16
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
CommandId
uint32 CommandId
Definition: c.h:534
ExecInitRangeTable
void ExecInitRangeTable(EState *estate, List *rangeTable)
Definition: execUtils.c:765
GetDecimalFromHex
static int GetDecimalFromHex(char hex)
Definition: copy.c:4345
FmgrInfo
Definition: fmgr.h:56
MAX_COPY_DATA_DISPLAY
#define MAX_COPY_DATA_DISPLAY
CopyStateData::csv_mode
bool csv_mode
Definition: copy.c:138
SendCopyEnd
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:479
InitResultRelInfo
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1277
ResultRelInfo::ri_RelationDesc
Relation ri_RelationDesc
Definition: execnodes.h:413
CopyStateData::range_table
List * range_table
Definition: copy.c:184
BeginCopy
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1507
UpdateActiveSnapshotCommandId
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:728
IsA
#define IsA(nodeptr, _type_)
Definition: nodes.h:579
CopyReadLineText
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3981
MemoryContextDelete
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:212
AllocSetContextCreate
#define AllocSetContextCreate
Definition: memutils.h:170
HOLD_CANCEL_INTERRUPTS
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:124
CopyMultiInsertInfoStore
static void CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, TupleTableSlot *slot, int tuplen, uint64 lineno)
Definition: copy.c:2679
contain_volatile_functions_not_nextval
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:774
table_close
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:167
ResTarget::val
Node * val
Definition: parsenodes.h:444
CopyGetInt32
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:747
errhint
int errhint(const char *fmt,...)
Definition: elog.c:1071
ExecARInsertTriggers
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2268
CopyMultiInsertInfo::estate
EState * estate
Definition: copy.c:286
pg_char_to_encoding
int pg_char_to_encoding(const char *name)
Definition: encnames.c:550
CopyStateData::raw_fields
char ** raw_fields
Definition: copy.c:205
EOL_NL
Definition: copy.c:83
getTypeOutputInfo
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2784
VARDATA
#define VARDATA(PTR)
Definition: postgres.h:302
EndCopy
static void EndCopy(CopyState cstate)
Definition: copy.c:1849
CopyStateData::binary
bool binary
Definition: copy.c:136
ResultRelInfo::ri_CopyMultiInsertBuffer
struct CopyMultiInsertBuffer * ri_CopyMultiInsertBuffer
Definition: execnodes.h:490
pq_flush
#define pq_flush()
Definition: libpq.h:39
CopyFromErrorCallback
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2284
CopyStmt::attlist
List * attlist
Definition: parsenodes.h:2015
SelectStmt::fromClause
List * fromClause
Definition: parsenodes.h:1599
value
static struct @142 value
ISOCTAL
#define ISOCTAL(c)
Definition: copy.c:62
ExecClearTuple
static TupleTableSlot * ExecClearTuple(TupleTableSlot *slot)
Definition: tuptable.h:425
OCTVALUE
#define OCTVALUE(c)
Definition: copy.c:63
ResetPerTupleExprContext
#define ResetPerTupleExprContext(estate)
Definition: executor.h:516
RelationGetDescr
#define RelationGetDescr(relation)
Definition: rel.h:482
LOCKMODE
int LOCKMODE
Definition: lockdefs.h:26
ResTarget::name
char * name
Definition: parsenodes.h:442
CopyReadBinaryAttribute
static Datum CopyReadBinaryAttribute(CopyState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4770
GetUserId
Oid GetUserId(void)
Definition: miscinit.c:476
TABLE_INSERT_FROZEN
#define TABLE_INSERT_FROZEN
Definition: tableam.h:134
CopyStateData::need_transcoding
bool need_transcoding
Definition: copy.c:126
castNode
#define castNode(_type_, nodeptr)
Definition: nodes.h:597
FreeQueryDesc
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
CopyStateData::in_functions
FmgrInfo * in_functions
Definition: copy.c:179
CopyStateData::num_defaults
AttrNumber num_defaults
Definition: copy.c:178
CopyStateData::attnumlist
List * attnumlist
Definition: copy.c:132
VARSIZE
#define VARSIZE(PTR)
Definition: postgres.h:303
pq_getmsgstring
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:581
CopyStateData::filename
char * filename
Definition: copy.c:133
FdwRoutine::BeginForeignInsert
BeginForeignInsert_function BeginForeignInsert
Definition: fdwapi.h:215
ModifyTableState::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: execnodes.h:1161
TupleDescAttr
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
CopyMultiInsertInfo
struct CopyMultiInsertInfo CopyMultiInsertInfo
BeginCopyFrom
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:3383
VARHDRSZ
#define VARHDRSZ
Definition: c.h:568
FdwRoutine::ExecForeignInsert
ExecForeignInsert_function ExecForeignInsert
Definition: fdwapi.h:211
ResultRelInfo::ri_PartitionInfo
struct PartitionRoutingInfo * ri_PartitionInfo
Definition: execnodes.h:487
PlannedStmt::relationOids
List * relationOids
Definition: plannodes.h:86
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1187
pg_hton16
#define pg_hton16(x)
Definition: pg_bswap.h:120
ExprContext::ecxt_per_tuple_memory
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:234
ReceiveCopyBegin
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:443
pg_ntoh16
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
DR_copy
Definition: copy.c:236
ExecComputeStoredGenerated
void ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot, CmdType cmdtype)
Definition: nodeModifyTable.c:249
makeStringInfo
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
CopyMultiInsertInfoSetupBuffer
static void CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri)
Definition: copy.c:2407
MAX_BUFFERED_TUPLES
#define MAX_BUFFERED_TUPLES
Definition: copy.c:253
Min
#define Min(x, y)
Definition: c.h:927
RelationData::rd_islocaltemp
bool rd_islocaltemp
Definition: rel.h:60
CopyMultiInsertInfo::mycid
CommandId mycid
Definition: copy.c:287
EXPR_KIND_COPY_WHERE
Definition: parse_node.h:79
S_IWOTH
#define S_IWOTH
Definition: win32_port.h:287
expression_planner
Expr * expression_planner(Expr *expr)
Definition: planner.c:6177
ExecutorStart
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:143
CopyMultiInsertBuffer::linenos
uint64 linenos[MAX_BUFFERED_TUPLES]
Definition: copy.c:271
transformExpr
Node * transformExpr(ParseState *pstate, Node *expr, ParseExprKind exprKind)
Definition: parse_expr.c:145
DR_copy::pub
DestReceiver pub
Definition: copy.c:238
ProcessCopyOptions
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1156
ExecConstraints
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1886
CopyStateData::line_buf
StringInfoData line_buf
Definition: copy.c:215
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
IF_NEED_REFILL_AND_EOF_BREAK
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:320
GetActiveSnapshot
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:786
AccessShareLock
#define AccessShareLock
Definition: lockdefs.h:36
addNSItemToQuery
void addNSItemToQuery(ParseState *pstate, ParseNamespaceItem *nsitem, bool addToJoinList, bool addToRelNameSpace, bool addToVarNameSpace)
Definition: parse_relation.c:2496
CopyStateData
struct CopyStateData CopyStateData
Node
Definition: nodes.h:528
strVal
#define strVal(v)
Definition: value.h:54
cur
struct cursor * cur
Definition: ecpg.c:28
CopyStateData::raw_buf_index
int raw_buf_index
Definition: copy.c:229
CopyAttributeOutText
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4828
CopyStateData::line_buf_valid
bool line_buf_valid
Definition: copy.c:217
ThereAreNoPriorRegisteredSnapshots
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1604
DR_copy::cstate
CopyState cstate
Definition: copy.c:239
errcode
int errcode(int sqlerrcode)
Definition: elog.c:610
PG_BINARY_W
#define PG_BINARY_W
Definition: c.h:1214
namestrcmp
int namestrcmp(Name name, const char *str)
Definition: name.c:247
parse_relation.h
MemSet
#define MemSet(start, val, len)
Definition: c.h:949
generate_unaccent_rules.dest
dest
Definition: generate_unaccent_rules.py:282
eval_const_expressions
Node * eval_const_expressions(PlannerInfo *root, Node *node)
Definition: clauses.c:2287
CopyFrom
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2702
ModifyTableState::operation
CmdType operation
Definition: execnodes.h:1153
parse_expr.h
RelationData::rd_newRelfilenodeSubid
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:103
pq_putemptymessage
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
table_finish_bulk_insert
static void table_finish_bulk_insert(Relation rel, int options)
Definition: tableam.h:1371
TupleTableSlot::tts_values
Datum * tts_values
Definition: tuptable.h:126
FirstLowInvalidHeapAttributeNumber
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:27
ClosePipeToProgram
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1814
MemoryContextReset
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:137
CopyReadAttributesCSV
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4601
PopActiveSnapshot
void PopActiveSnapshot(void)
Definition: snapmgr.c:759
RangeTblEntry::requiredPerms
AclMode requiredPerms
Definition: parsenodes.h:1120
contain_volatile_functions
bool contain_volatile_functions(Node *clause)
Definition: clauses.c:724
SIGPIPE
#define SIGPIPE
Definition: win32_port.h:158
PlanState::state
EState * state
Definition: execnodes.h:934
CopyStateData::force_quote_flags
bool * force_quote_flags
Definition: copy.c:148
table_scan_getnextslot
static bool table_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot)
Definition: tableam.h:904
RelationData::rd_rel
Form_pg_class rd_rel
Definition: rel.h:109
Oid
unsigned int Oid
Definition: postgres_ext.h:31
pg_analyze_and_rewrite
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:676
ExecQual
static bool ExecQual(ExprState *state, ExprContext *econtext)
Definition: executor.h:370
CopyStateData::delim
char * delim
Definition: copy.c:143
pg_type.h
PG_PROTOCOL_MAJOR
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Query::utilityStmt
Node * utilityStmt
Definition: parsenodes.h:120
CopyStmt::is_program
bool is_program
Definition: parsenodes.h:2018
linitial_node
#define linitial_node(type, l)
Definition: pg_list.h:177
CopyStateData::volatile_defexprs
bool volatile_defexprs
Definition: copy.c:183
pg_wchar.h
ErrorContextCallback::callback
void(* callback)(void *arg)
Definition: elog.h:229
ErrorContextCallback::previous
struct ErrorContextCallback * previous
Definition: elog.h:228
CopyMultiInsertInfoInit
static void CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, CopyState cstate, EState *estate, CommandId mycid, int ti_options)
Definition: copy.c:2427
PG_BINARY_R
#define PG_BINARY_R
Definition: c.h:1213
CopyMultiInsertBuffer::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: copy.c:268
copy_dest_destroy
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:5192
CopyStateData::force_null_flags
bool * force_null_flags
Definition: copy.c:152
wait_result_to_str
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
CopyStateData::rowcontext
MemoryContext rowcontext
Definition: copy.c:173
parse_coerce.h
_DestReceiver
Definition: dest.h:116
CopyStateData::line_buf_converted
bool line_buf_converted
Definition: copy.c:216
table_tuple_insert
static void table_tuple_insert(Relation rel, TupleTableSlot *slot, CommandId cid, int options, struct BulkInsertStateData *bistate)
Definition: tableam.h:1178
pq_beginmessage
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
ExecInitQual
ExprState * ExecInitQual(List *qual, PlanState *parent)
Definition: execExpr.c:209
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:692
int32
signed int int32
Definition: c.h:362
ClosePipeStream
int ClosePipeStream(FILE *file)
Definition: fd.c:2731
errdetail_internal
int errdetail_internal(const char *fmt,...)
Definition: elog.c:984
CopyStateData::convert_select_flags
bool * convert_select_flags
Definition: copy.c:155
CopySendChar
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:520
OutputFunctionCall
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1577
ExecOpenIndices
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:151
addRangeTableEntryForRelation
ParseNamespaceItem * addRangeTableEntryForRelation(ParseState *pstate, Relation rel, int lockmode, Alias *alias, bool inh, bool inFromCl)
Definition: parse_relation.c:1497
ColumnRef::location
int location
Definition: parsenodes.h:237
CopyStateData::copy_dest
CopyDest copy_dest
Definition: copy.c:117
ResTarget::location
int location
Definition: parsenodes.h:445
pq_sendbyte
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
REFILL_LINEBUF
#define REFILL_LINEBUF
Definition: copy.c:337
error_context_stack
ErrorContextCallback * error_context_stack
Definition: elog.c:92
CopyStateData::cur_lineno
uint64 cur_lineno
Definition: copy.c:160
list_make1
#define list_make1(x1)
Definition: pg_list.h:206
CopyStateData::null_print
char * null_print
Definition: copy.c:140
CopyStateData::cur_attname
const char * cur_attname
Definition: copy.c:161
assign_expr_collations
void assign_expr_collations(ParseState *pstate, Node *expr)
Definition: parse_collate.c:177