PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 #include <netinet/in.h>
21 #include <arpa/inet.h>
22 
23 #include "access/heapam.h"
24 #include "access/htup_details.h"
25 #include "access/sysattr.h"
26 #include "access/xact.h"
27 #include "access/xlog.h"
28 #include "catalog/pg_type.h"
29 #include "commands/copy.h"
30 #include "commands/defrem.h"
31 #include "commands/trigger.h"
32 #include "executor/executor.h"
33 #include "libpq/libpq.h"
34 #include "libpq/pqformat.h"
35 #include "mb/pg_wchar.h"
36 #include "miscadmin.h"
37 #include "optimizer/clauses.h"
38 #include "optimizer/planner.h"
39 #include "nodes/makefuncs.h"
40 #include "parser/parse_relation.h"
41 #include "rewrite/rewriteHandler.h"
42 #include "storage/fd.h"
43 #include "tcop/tcopprot.h"
44 #include "utils/builtins.h"
45 #include "utils/lsyscache.h"
46 #include "utils/memutils.h"
47 #include "utils/portal.h"
48 #include "utils/rel.h"
49 #include "utils/rls.h"
50 #include "utils/snapmgr.h"
51 
52 
53 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
54 #define OCTVALUE(c) ((c) - '0')
55 
56 /*
57  * Represents the different source/dest cases we need to worry about at
58  * the bottom level
59  */
60 typedef enum CopyDest
61 {
62  COPY_FILE, /* to/from file (or a piped program) */
63  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
64  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
65  COPY_CALLBACK /* to/from callback function */
66 } CopyDest;
67 
68 /*
69  * Represents the end-of-line terminator type of the input
70  */
71 typedef enum EolType
72 {
73  EOL_UNKNOWN,
74  EOL_NL,
75  EOL_CR,
76  EOL_CRNL
77 } EolType;
78 
79 /*
80  * This struct contains all the state variables used throughout a COPY
81  * operation. For simplicity, we use the same struct for all variants of COPY,
82  * even though some fields are used in only some cases.
83  *
84  * Multi-byte encodings: all supported client-side encodings encode multi-byte
85  * characters by having the first byte's high bit set. Subsequent bytes of the
86  * character can have the high bit not set. When scanning data in such an
87  * encoding to look for a match to a single-byte (ie ASCII) character, we must
88  * use the full pg_encoding_mblen() machinery to skip over multibyte
89  * characters, else we might find a false match to a trailing byte. In
90  * supported server encodings, there is no possibility of a false match, and
91  * it's faster to make useless comparisons to trailing bytes than it is to
92  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is TRUE
93  * when we have to do it the hard way.
94  */
95 typedef struct CopyStateData
96 {
97  /* low-level state data */
98  CopyDest copy_dest; /* type of copy source/destination */
99  FILE *copy_file; /* used if copy_dest == COPY_FILE */
100  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
101  * dest == COPY_NEW_FE in COPY FROM */
102  bool fe_eof; /* true if detected end of copy data */
103  EolType eol_type; /* EOL type of input */
104  int file_encoding; /* file or remote side's character encoding */
105  bool need_transcoding; /* file encoding diff from server? */
106  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
107 
108  /* parameters from the COPY command */
109  Relation rel; /* relation to copy to or from */
110  QueryDesc *queryDesc; /* executable query to copy from */
111  List *attnumlist; /* integer list of attnums to copy */
112  char *filename; /* filename, or NULL for STDIN/STDOUT */
113  bool is_program; /* is 'filename' a program to popen? */
114  copy_data_source_cb data_source_cb; /* function for reading data */
115  bool binary; /* binary format? */
116  bool oids; /* include OIDs? */
117  bool freeze; /* freeze rows on loading? */
118  bool csv_mode; /* Comma Separated Value format? */
119  bool header_line; /* CSV header line? */
120  char *null_print; /* NULL marker string (server encoding!) */
121  int null_print_len; /* length of same */
122  char *null_print_client; /* same converted to file encoding */
123  char *delim; /* column delimiter (must be 1 byte) */
124  char *quote; /* CSV quote char (must be 1 byte) */
125  char *escape; /* CSV escape char (must be 1 byte) */
126  List *force_quote; /* list of column names */
127  bool force_quote_all; /* FORCE_QUOTE *? */
128  bool *force_quote_flags; /* per-column CSV FQ flags */
129  List *force_notnull; /* list of column names */
130  bool *force_notnull_flags; /* per-column CSV FNN flags */
131  List *force_null; /* list of column names */
132  bool *force_null_flags; /* per-column CSV FN flags */
133  bool convert_selectively; /* do selective binary conversion? */
134  List *convert_select; /* list of column names (can be NIL) */
135  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
136 
137  /* these are just for error messages, see CopyFromErrorCallback */
138  const char *cur_relname; /* table name for error messages */
139  int cur_lineno; /* line number for error messages */
140  const char *cur_attname; /* current att for error messages */
141  const char *cur_attval; /* current att value for error messages */
142 
143  /*
144  * Working state for COPY TO/FROM
145  */
146  MemoryContext copycontext; /* per-copy execution context */
147 
148  /*
149  * Working state for COPY TO
150  */
151  FmgrInfo *out_functions; /* lookup info for output functions */
152  MemoryContext rowcontext; /* per-row evaluation context */
153 
154  /*
155  * Working state for COPY FROM
156  */
157  AttrNumber num_defaults;
158  bool file_has_oids;
159  FmgrInfo oid_in_function;
160  Oid oid_typioparam;
161  FmgrInfo *in_functions; /* array of input functions for each attrs */
162  Oid *typioparams; /* array of element types for in_functions */
163  int *defmap; /* array of default att numbers */
164  ExprState **defexprs; /* array of default att expressions */
165  bool volatile_defexprs; /* is any of defexprs volatile? */
166  List *range_table;
167 
168  PartitionDispatch *partition_dispatch_info;
169  int num_dispatch; /* Number of entries in the above array */
170  int num_partitions; /* Number of members in the following arrays */
171  ResultRelInfo *partitions; /* Per partition result relation */
172  TupleConversionMap **partition_tupconv_maps;
173  TupleTableSlot *partition_tuple_slot;
174 
175  /*
176  * These variables are used to reduce overhead in textual COPY FROM.
177  *
178  * attribute_buf holds the separated, de-escaped text for each field of
179  * the current line. The CopyReadAttributes functions return arrays of
180  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
181  * the buffer on each cycle.
182  */
183  StringInfoData attribute_buf;
184 
185  /* field raw data pointers found by COPY FROM */
186 
187  int max_fields;
188  char **raw_fields;
189 
190  /*
191  * Similarly, line_buf holds the whole input line being processed. The
192  * input cycle is first to read the whole line into line_buf, convert it
193  * to server encoding there, and then extract the individual attribute
194  * fields into attribute_buf. line_buf is preserved unmodified so that we
195  * can display it in error messages if appropriate.
196  */
197  StringInfoData line_buf;
198  bool line_buf_converted; /* converted to server encoding? */
199  bool line_buf_valid; /* contains the row being processed? */
200 
201  /*
202  * Finally, raw_buf holds raw data read from the data source (file or
203  * client connection). CopyReadLine parses this data sufficiently to
204  * locate line boundaries, then transfers the data to line_buf and
205  * converts it. Note: we guarantee that there is a \0 at
206  * raw_buf[raw_buf_len].
207  */
208 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
209  char *raw_buf;
210  int raw_buf_index; /* next byte to process */
211  int raw_buf_len; /* total # of bytes stored */
212 } CopyStateData;
213 
214 /* DestReceiver for COPY (query) TO */
215 typedef struct
216 {
217  DestReceiver pub; /* publicly-known function pointers */
218  CopyState cstate; /* CopyStateData for the command */
219  uint64 processed; /* # of tuples processed */
220 } DR_copy;
221 
222 
223 /*
224  * These macros centralize code used to process line_buf and raw_buf buffers.
225  * They are macros because they often do continue/break control and to avoid
226  * function call overhead in tight COPY loops.
227  *
228  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
229  * prevent the continue/break processing from working. We end the "if (1)"
230  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
231  * any "else" in the calling code, and to avoid any compiler warnings about
232  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
233  */
234 
235 /*
236  * This keeps the character read at the top of the loop in the buffer
237  * even if there is more than one read-ahead.
238  */
239 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
240 if (1) \
241 { \
242  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
243  { \
244  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
245  need_data = true; \
246  continue; \
247  } \
248 } else ((void) 0)
249 
250 /* This consumes the remainder of the buffer and breaks */
251 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
252 if (1) \
253 { \
254  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
255  { \
256  if (extralen) \
257  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
258  /* backslash just before EOF, treat as data char */ \
259  result = true; \
260  break; \
261  } \
262 } else ((void) 0)
263 
264 /*
265  * Transfer any approved data to line_buf; must do this to be sure
266  * there is some room in raw_buf.
267  */
268 #define REFILL_LINEBUF \
269 if (1) \
270 { \
271  if (raw_buf_ptr > cstate->raw_buf_index) \
272  { \
273  appendBinaryStringInfo(&cstate->line_buf, \
274  cstate->raw_buf + cstate->raw_buf_index, \
275  raw_buf_ptr - cstate->raw_buf_index); \
276  cstate->raw_buf_index = raw_buf_ptr; \
277  } \
278 } else ((void) 0)
279 
280 /* Undo any read-ahead and jump out of the block. */
281 #define NO_END_OF_COPY_GOTO \
282 if (1) \
283 { \
284  raw_buf_ptr = prev_raw_ptr + 1; \
285  goto not_end_of_copy; \
286 } else ((void) 0)
287 
288 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
289 
290 
291 /* non-export function prototypes */
292 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
293  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
294  List *options);
295 static void EndCopy(CopyState cstate);
296 static void ClosePipeToProgram(CopyState cstate);
297 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
298  Oid queryRelId, const char *filename, bool is_program,
299  List *attnamelist, List *options);
300 static void EndCopyTo(CopyState cstate);
301 static uint64 DoCopyTo(CopyState cstate);
302 static uint64 CopyTo(CopyState cstate);
303 static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
304  Datum *values, bool *nulls);
305 static void CopyFromInsertBatch(CopyState cstate, EState *estate,
306  CommandId mycid, int hi_options,
307  ResultRelInfo *resultRelInfo, TupleTableSlot *myslot,
308  BulkInsertState bistate,
309  int nBufferedTuples, HeapTuple *bufferedTuples,
310  int firstBufferedLineNo);
311 static bool CopyReadLine(CopyState cstate);
312 static bool CopyReadLineText(CopyState cstate);
313 static int CopyReadAttributesText(CopyState cstate);
314 static int CopyReadAttributesCSV(CopyState cstate);
315 static Datum CopyReadBinaryAttribute(CopyState cstate,
316  int column_no, FmgrInfo *flinfo,
317  Oid typioparam, int32 typmod,
318  bool *isnull);
319 static void CopyAttributeOutText(CopyState cstate, char *string);
320 static void CopyAttributeOutCSV(CopyState cstate, char *string,
321  bool use_quote, bool single_attr);
322 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
323  List *attnamelist);
324 static char *limit_printout_length(const char *str);
325 
326 /* Low-level communications functions */
327 static void SendCopyBegin(CopyState cstate);
328 static void ReceiveCopyBegin(CopyState cstate);
329 static void SendCopyEnd(CopyState cstate);
330 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
331 static void CopySendString(CopyState cstate, const char *str);
332 static void CopySendChar(CopyState cstate, char c);
333 static void CopySendEndOfRow(CopyState cstate);
334 static int CopyGetData(CopyState cstate, void *databuf,
335  int minread, int maxread);
336 static void CopySendInt32(CopyState cstate, int32 val);
337 static bool CopyGetInt32(CopyState cstate, int32 *val);
338 static void CopySendInt16(CopyState cstate, int16 val);
339 static bool CopyGetInt16(CopyState cstate, int16 *val);
340 
341 
342 /*
343  * Send copy start/stop messages for frontend copies. These have changed
344  * in past protocol redesigns.
345  */
346 static void
347 SendCopyBegin(CopyState cstate)
348 {
349  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
350  {
351  /* new way */
352  StringInfoData buf;
353  int natts = list_length(cstate->attnumlist);
354  int16 format = (cstate->binary ? 1 : 0);
355  int i;
356 
357  pq_beginmessage(&buf, 'H');
358  pq_sendbyte(&buf, format); /* overall format */
359  pq_sendint(&buf, natts, 2);
360  for (i = 0; i < natts; i++)
361  pq_sendint(&buf, format, 2); /* per-column formats */
362  pq_endmessage(&buf);
363  cstate->copy_dest = COPY_NEW_FE;
364  }
365  else
366  {
367  /* old way */
368  if (cstate->binary)
369  ereport(ERROR,
370  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
371  errmsg("COPY BINARY is not supported to stdout or from stdin")));
372  pq_putemptymessage('H');
373  /* grottiness needed for old COPY OUT protocol */
374  pq_startcopyout();
375  cstate->copy_dest = COPY_OLD_FE;
376  }
377 }
378 
379 static void
380 ReceiveCopyBegin(CopyState cstate)
381 {
382  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
383  {
384  /* new way */
385  StringInfoData buf;
386  int natts = list_length(cstate->attnumlist);
387  int16 format = (cstate->binary ? 1 : 0);
388  int i;
389 
390  pq_beginmessage(&buf, 'G');
391  pq_sendbyte(&buf, format); /* overall format */
392  pq_sendint(&buf, natts, 2);
393  for (i = 0; i < natts; i++)
394  pq_sendint(&buf, format, 2); /* per-column formats */
395  pq_endmessage(&buf);
396  cstate->copy_dest = COPY_NEW_FE;
397  cstate->fe_msgbuf = makeStringInfo();
398  }
399  else
400  {
401  /* old way */
402  if (cstate->binary)
403  ereport(ERROR,
404  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
405  errmsg("COPY BINARY is not supported to stdout or from stdin")));
406  pq_putemptymessage('G');
407  /* any error in old protocol will make us lose sync */
408  pq_startmsgread();
409  cstate->copy_dest = COPY_OLD_FE;
410  }
411  /* We *must* flush here to ensure FE knows it can send. */
412  pq_flush();
413 }
414 
415 static void
416 SendCopyEnd(CopyState cstate)
417 {
418  if (cstate->copy_dest == COPY_NEW_FE)
419  {
420  /* Shouldn't have any unsent data */
421  Assert(cstate->fe_msgbuf->len == 0);
422  /* Send Copy Done message */
423  pq_putemptymessage('c');
424  }
425  else
426  {
427  CopySendData(cstate, "\\.", 2);
428  /* Need to flush out the trailer (this also appends a newline) */
429  CopySendEndOfRow(cstate);
430  pq_endcopyout(false);
431  }
432 }
433 
434 /*----------
435  * CopySendData sends output data to the destination (file or frontend)
436  * CopySendString does the same for null-terminated strings
437  * CopySendChar does the same for single characters
438  * CopySendEndOfRow does the appropriate thing at end of each data row
439  * (data is not actually flushed except by CopySendEndOfRow)
440  *
441  * NB: no data conversion is applied by these functions
442  *----------
443  */
444 static void
445 CopySendData(CopyState cstate, const void *databuf, int datasize)
446 {
447  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
448 }
449 
450 static void
451 CopySendString(CopyState cstate, const char *str)
452 {
453  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
454 }
455 
456 static void
457 CopySendChar(CopyState cstate, char c)
458 {
459  appendStringInfoCharMacro(cstate->fe_msgbuf, c);
460 }
461 
462 static void
463 CopySendEndOfRow(CopyState cstate)
464 {
465  StringInfo fe_msgbuf = cstate->fe_msgbuf;
466 
467  switch (cstate->copy_dest)
468  {
469  case COPY_FILE:
470  if (!cstate->binary)
471  {
472  /* Default line termination depends on platform */
473 #ifndef WIN32
474  CopySendChar(cstate, '\n');
475 #else
476  CopySendString(cstate, "\r\n");
477 #endif
478  }
479 
480  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
481  cstate->copy_file) != 1 ||
482  ferror(cstate->copy_file))
483  {
484  if (cstate->is_program)
485  {
486  if (errno == EPIPE)
487  {
488  /*
489  * The pipe will be closed automatically on error at
490  * the end of transaction, but we might get a better
491  * error message from the subprocess' exit code than
492  * just "Broken Pipe"
493  */
494  ClosePipeToProgram(cstate);
495 
496  /*
497  * If ClosePipeToProgram() didn't throw an error, the
498  * program terminated normally, but closed the pipe
499  * first. Restore errno, and throw an error.
500  */
501  errno = EPIPE;
502  }
503  ereport(ERROR,
504  (errcode_for_file_access(),
505  errmsg("could not write to COPY program: %m")));
506  }
507  else
508  ereport(ERROR,
509  (errcode_for_file_access(),
510  errmsg("could not write to COPY file: %m")));
511  }
512  break;
513  case COPY_OLD_FE:
514  /* The FE/BE protocol uses \n as newline for all platforms */
515  if (!cstate->binary)
516  CopySendChar(cstate, '\n');
517 
518  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
519  {
520  /* no hope of recovering connection sync, so FATAL */
521  ereport(FATAL,
522  (errcode(ERRCODE_CONNECTION_FAILURE),
523  errmsg("connection lost during COPY to stdout")));
524  }
525  break;
526  case COPY_NEW_FE:
527  /* The FE/BE protocol uses \n as newline for all platforms */
528  if (!cstate->binary)
529  CopySendChar(cstate, '\n');
530 
531  /* Dump the accumulated row as one CopyData message */
532  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
533  break;
534  case COPY_CALLBACK:
535  Assert(false); /* Not yet supported. */
536  break;
537  }
538 
539  resetStringInfo(fe_msgbuf);
540 }
541 
542 /*
543  * CopyGetData reads data from the source (file or frontend)
544  *
545  * We attempt to read at least minread, and at most maxread, bytes from
546  * the source. The actual number of bytes read is returned; if this is
547  * less than minread, EOF was detected.
548  *
549  * Note: when copying from the frontend, we expect a proper EOF mark per
550  * protocol; if the frontend simply drops the connection, we raise error.
551  * It seems unwise to allow the COPY IN to complete normally in that case.
552  *
553  * NB: no data conversion is applied here.
554  */
555 static int
556 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
557 {
558  int bytesread = 0;
559 
560  switch (cstate->copy_dest)
561  {
562  case COPY_FILE:
563  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
564  if (ferror(cstate->copy_file))
565  ereport(ERROR,
566  (errcode_for_file_access(),
567  errmsg("could not read from COPY file: %m")));
568  break;
569  case COPY_OLD_FE:
570 
571  /*
572  * We cannot read more than minread bytes (which in practice is 1)
573  * because old protocol doesn't have any clear way of separating
574  * the COPY stream from following data. This is slow, but not any
575  * slower than the code path was originally, and we don't care
576  * much anymore about the performance of old protocol.
577  */
578  if (pq_getbytes((char *) databuf, minread))
579  {
580  /* Only a \. terminator is legal EOF in old protocol */
581  ereport(ERROR,
582  (errcode(ERRCODE_CONNECTION_FAILURE),
583  errmsg("unexpected EOF on client connection with an open transaction")));
584  }
585  bytesread = minread;
586  break;
587  case COPY_NEW_FE:
588  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
589  {
590  int avail;
591 
592  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
593  {
594  /* Try to receive another message */
595  int mtype;
596 
597  readmessage:
598  HOLD_CANCEL_INTERRUPTS();
599  pq_startmsgread();
600  mtype = pq_getbyte();
601  if (mtype == EOF)
602  ereport(ERROR,
603  (errcode(ERRCODE_CONNECTION_FAILURE),
604  errmsg("unexpected EOF on client connection with an open transaction")));
605  if (pq_getmessage(cstate->fe_msgbuf, 0))
606  ereport(ERROR,
607  (errcode(ERRCODE_CONNECTION_FAILURE),
608  errmsg("unexpected EOF on client connection with an open transaction")));
609  RESUME_CANCEL_INTERRUPTS();
610  switch (mtype)
611  {
612  case 'd': /* CopyData */
613  break;
614  case 'c': /* CopyDone */
615  /* COPY IN correctly terminated by frontend */
616  cstate->fe_eof = true;
617  return bytesread;
618  case 'f': /* CopyFail */
619  ereport(ERROR,
620  (errcode(ERRCODE_QUERY_CANCELED),
621  errmsg("COPY from stdin failed: %s",
622  pq_getmsgstring(cstate->fe_msgbuf))));
623  break;
624  case 'H': /* Flush */
625  case 'S': /* Sync */
626 
627  /*
628  * Ignore Flush/Sync for the convenience of client
629  * libraries (such as libpq) that may send those
630  * without noticing that the command they just
631  * sent was COPY.
632  */
633  goto readmessage;
634  default:
635  ereport(ERROR,
636  (errcode(ERRCODE_PROTOCOL_VIOLATION),
637  errmsg("unexpected message type 0x%02X during COPY from stdin",
638  mtype)));
639  break;
640  }
641  }
642  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
643  if (avail > maxread)
644  avail = maxread;
645  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
646  databuf = (void *) ((char *) databuf + avail);
647  maxread -= avail;
648  bytesread += avail;
649  }
650  break;
651  case COPY_CALLBACK:
652  bytesread = cstate->data_source_cb(databuf, minread, maxread);
653  break;
654  }
655 
656  return bytesread;
657 }
658 
659 
660 /*
661  * These functions do apply some data conversion
662  */
663 
664 /*
665  * CopySendInt32 sends an int32 in network byte order
666  */
667 static void
668 CopySendInt32(CopyState cstate, int32 val)
669 {
670  uint32 buf;
671 
672  buf = htonl((uint32) val);
673  CopySendData(cstate, &buf, sizeof(buf));
674 }
675 
676 /*
677  * CopyGetInt32 reads an int32 that appears in network byte order
678  *
679  * Returns true if OK, false if EOF
680  */
681 static bool
682 CopyGetInt32(CopyState cstate, int32 *val)
683 {
684  uint32 buf;
685 
686  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
687  {
688  *val = 0; /* suppress compiler warning */
689  return false;
690  }
691  *val = (int32) ntohl(buf);
692  return true;
693 }
694 
695 /*
696  * CopySendInt16 sends an int16 in network byte order
697  */
698 static void
699 CopySendInt16(CopyState cstate, int16 val)
700 {
701  uint16 buf;
702 
703  buf = htons((uint16) val);
704  CopySendData(cstate, &buf, sizeof(buf));
705 }
706 
707 /*
708  * CopyGetInt16 reads an int16 that appears in network byte order
709  */
710 static bool
711 CopyGetInt16(CopyState cstate, int16 *val)
712 {
713  uint16 buf;
714 
715  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
716  {
717  *val = 0; /* suppress compiler warning */
718  return false;
719  }
720  *val = (int16) ntohs(buf);
721  return true;
722 }
723 
724 
725 /*
726  * CopyLoadRawBuf loads some more data into raw_buf
727  *
728  * Returns TRUE if able to obtain at least one more byte, else FALSE.
729  *
730  * If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
731  * down to the start of the buffer and then we load more data after that.
732  * This case is used only when a frontend multibyte character crosses a
733  * bufferload boundary.
734  */
735 static bool
736 CopyLoadRawBuf(CopyState cstate)
737 {
738  int nbytes;
739  int inbytes;
740 
741  if (cstate->raw_buf_index < cstate->raw_buf_len)
742  {
743  /* Copy down the unprocessed data */
744  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
745  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
746  nbytes);
747  }
748  else
749  nbytes = 0; /* no data need be saved */
750 
751  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
752  1, RAW_BUF_SIZE - nbytes);
753  nbytes += inbytes;
754  cstate->raw_buf[nbytes] = '\0';
755  cstate->raw_buf_index = 0;
756  cstate->raw_buf_len = nbytes;
757  return (inbytes > 0);
758 }
759 
760 
761 /*
762  * DoCopy executes the SQL COPY statement
763  *
764  * Either unload or reload contents of table <relation>, depending on <from>.
765  * (<from> = TRUE means we are inserting into the table.) In the "TO" case
766  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
767  * or DELETE query.
768  *
769  * If <pipe> is false, transfer is between the table and the file named
770  * <filename>. Otherwise, transfer is between the table and our regular
771  * input/output stream. The latter could be either stdin/stdout or a
772  * socket, depending on whether we're running under Postmaster control.
773  *
774  * Do not allow a Postgres user without superuser privilege to read from
775  * or write to a file.
776  *
777  * Do not allow the copy if user doesn't have proper permission to access
778  * the table or the specifically requested columns.
779  */
780 void
781 DoCopy(ParseState *pstate, const CopyStmt *stmt,
782  int stmt_location, int stmt_len,
783  uint64 *processed)
784 {
785  CopyState cstate;
786  bool is_from = stmt->is_from;
787  bool pipe = (stmt->filename == NULL);
788  Relation rel;
789  Oid relid;
790  RawStmt *query = NULL;
791 
792  /* Disallow COPY to/from file or program except to superusers. */
793  if (!pipe && !superuser())
794  {
795  if (stmt->is_program)
796  ereport(ERROR,
797  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
798  errmsg("must be superuser to COPY to or from an external program"),
799  errhint("Anyone can COPY to stdout or from stdin. "
800  "psql's \\copy command also works for anyone.")));
801  else
802  ereport(ERROR,
803  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
804  errmsg("must be superuser to COPY to or from a file"),
805  errhint("Anyone can COPY to stdout or from stdin. "
806  "psql's \\copy command also works for anyone.")));
807  }
808 
809  if (stmt->relation)
810  {
811  TupleDesc tupDesc;
812  List *attnums;
813  ListCell *cur;
814  RangeTblEntry *rte;
815 
816  Assert(!stmt->query);
817 
818  /* Open and lock the relation, using the appropriate lock type. */
819  rel = heap_openrv(stmt->relation,
820  (is_from ? RowExclusiveLock : AccessShareLock));
821 
822  relid = RelationGetRelid(rel);
823 
824  rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
825  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
826 
827  tupDesc = RelationGetDescr(rel);
828  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
829  foreach(cur, attnums)
830  {
831  int attno = lfirst_int(cur) -
832  FirstLowInvalidHeapAttributeNumber;
833 
834  if (is_from)
835  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
836  else
837  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
838  }
839  ExecCheckRTPerms(pstate->p_rtable, true);
840 
841  /*
842  * Permission check for row security policies.
843  *
844  * check_enable_rls will ereport(ERROR) if the user has requested
845  * something invalid and will otherwise indicate if we should enable
846  * RLS (returns RLS_ENABLED) or not for this COPY statement.
847  *
848  * If the relation has a row security policy and we are to apply it
849  * then perform a "query" copy and allow the normal query processing
850  * to handle the policies.
851  *
852  * If RLS is not enabled for this, then just fall through to the
853  * normal non-filtering relation handling.
854  */
855  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
856  {
857  SelectStmt *select;
858  ColumnRef *cr;
859  ResTarget *target;
860  RangeVar *from;
861  List *targetList = NIL;
862 
863  if (is_from)
864  ereport(ERROR,
865  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
866  errmsg("COPY FROM not supported with row-level security"),
867  errhint("Use INSERT statements instead.")));
868 
869  /*
870  * Build target list
871  *
872  * If no columns are specified in the attribute list of the COPY
873  * command, then the target list is 'all' columns. Therefore, '*'
874  * should be used as the target list for the resulting SELECT
875  * statement.
876  *
877  * In the case that columns are specified in the attribute list,
878  * create a ColumnRef and ResTarget for each column and add them
879  * to the target list for the resulting SELECT statement.
880  */
881  if (!stmt->attlist)
882  {
883  cr = makeNode(ColumnRef);
884  cr->fields = list_make1(makeNode(A_Star));
885  cr->location = -1;
886 
887  target = makeNode(ResTarget);
888  target->name = NULL;
889  target->indirection = NIL;
890  target->val = (Node *) cr;
891  target->location = -1;
892 
893  targetList = list_make1(target);
894  }
895  else
896  {
897  ListCell *lc;
898 
899  foreach(lc, stmt->attlist)
900  {
901  /*
902  * Build the ColumnRef for each column. The ColumnRef
903  * 'fields' property is a String 'Value' node (see
904  * nodes/value.h) that corresponds to the column name
905  * respectively.
906  */
907  cr = makeNode(ColumnRef);
908  cr->fields = list_make1(lfirst(lc));
909  cr->location = -1;
910 
911  /* Build the ResTarget and add the ColumnRef to it. */
912  target = makeNode(ResTarget);
913  target->name = NULL;
914  target->indirection = NIL;
915  target->val = (Node *) cr;
916  target->location = -1;
917 
918  /* Add each column to the SELECT statement's target list */
919  targetList = lappend(targetList, target);
920  }
921  }
922 
923  /*
924  * Build RangeVar for from clause, fully qualified based on the
925  * relation which we have opened and locked.
926  */
927  from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
928  pstrdup(RelationGetRelationName(rel)),
929  -1);
930 
931  /* Build query */
932  select = makeNode(SelectStmt);
933  select->targetList = targetList;
934  select->fromClause = list_make1(from);
935 
936  query = makeNode(RawStmt);
937  query->stmt = (Node *) select;
938  query->stmt_location = stmt_location;
939  query->stmt_len = stmt_len;
940 
941  /*
942  * Close the relation for now, but keep the lock on it to prevent
943  * changes between now and when we start the query-based COPY.
944  *
945  * We'll reopen it later as part of the query-based COPY.
946  */
947  heap_close(rel, NoLock);
948  rel = NULL;
949  }
950  }
951  else
952  {
953  Assert(stmt->query);
954 
955  query = makeNode(RawStmt);
956  query->stmt = stmt->query;
957  query->stmt_location = stmt_location;
958  query->stmt_len = stmt_len;
959 
960  relid = InvalidOid;
961  rel = NULL;
962  }
963 
964  if (is_from)
965  {
966  Assert(rel);
967 
968  /* check read-only transaction and parallel mode */
969  if (XactReadOnly && !rel->rd_islocaltemp)
970  PreventCommandIfReadOnly("COPY FROM");
971  PreventCommandIfParallelMode("COPY FROM");
972 
973  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
974  NULL, stmt->attlist, stmt->options);
975  *processed = CopyFrom(cstate); /* copy from file to database */
976  EndCopyFrom(cstate);
977  }
978  else
979  {
980  cstate = BeginCopyTo(pstate, rel, query, relid,
981  stmt->filename, stmt->is_program,
982  stmt->attlist, stmt->options);
983  *processed = DoCopyTo(cstate); /* copy from database to file */
984  EndCopyTo(cstate);
985  }
986 
987  /*
988  * Close the relation. If reading, we can release the AccessShareLock we
989  * got; if writing, we should hold the lock until end of transaction to
990  * ensure that updates will be committed before lock is released.
991  */
992  if (rel != NULL)
993  heap_close(rel, (is_from ? NoLock : AccessShareLock));
994 }
995 
996 /*
997  * Process the statement option list for COPY.
998  *
999  * Scan the options list (a list of DefElem) and transpose the information
1000  * into cstate, applying appropriate error checking.
1001  *
1002  * cstate is assumed to be filled with zeroes initially.
1003  *
1004  * This is exported so that external users of the COPY API can sanity-check
1005  * a list of options. In that usage, cstate should be passed as NULL
1006  * (since external users don't know sizeof(CopyStateData)) and the collected
1007  * data is just leaked until CurrentMemoryContext is reset.
1008  *
1009  * Note that additional checking, such as whether column names listed in FORCE
1010  * QUOTE actually exist, has to be applied later. This just checks for
1011  * self-consistency of the options list.
1012  */
1013 void
1014 ProcessCopyOptions(ParseState *pstate,
1015  CopyState cstate,
1016  bool is_from,
1017  List *options)
1018 {
1019  bool format_specified = false;
1020  ListCell *option;
1021 
1022  /* Support external use for option sanity checking */
1023  if (cstate == NULL)
1024  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1025 
1026  cstate->file_encoding = -1;
1027 
1028  /* Extract options from the statement node tree */
1029  foreach(option, options)
1030  {
1031  DefElem *defel = lfirst_node(DefElem, option);
1032 
1033  if (strcmp(defel->defname, "format") == 0)
1034  {
1035  char *fmt = defGetString(defel);
1036 
1037  if (format_specified)
1038  ereport(ERROR,
1039  (errcode(ERRCODE_SYNTAX_ERROR),
1040  errmsg("conflicting or redundant options"),
1041  parser_errposition(pstate, defel->location)));
1042  format_specified = true;
1043  if (strcmp(fmt, "text") == 0)
1044  /* default format */ ;
1045  else if (strcmp(fmt, "csv") == 0)
1046  cstate->csv_mode = true;
1047  else if (strcmp(fmt, "binary") == 0)
1048  cstate->binary = true;
1049  else
1050  ereport(ERROR,
1051  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1052  errmsg("COPY format \"%s\" not recognized", fmt),
1053  parser_errposition(pstate, defel->location)));
1054  }
1055  else if (strcmp(defel->defname, "oids") == 0)
1056  {
1057  if (cstate->oids)
1058  ereport(ERROR,
1059  (errcode(ERRCODE_SYNTAX_ERROR),
1060  errmsg("conflicting or redundant options"),
1061  parser_errposition(pstate, defel->location)));
1062  cstate->oids = defGetBoolean(defel);
1063  }
1064  else if (strcmp(defel->defname, "freeze") == 0)
1065  {
1066  if (cstate->freeze)
1067  ereport(ERROR,
1068  (errcode(ERRCODE_SYNTAX_ERROR),
1069  errmsg("conflicting or redundant options"),
1070  parser_errposition(pstate, defel->location)));
1071  cstate->freeze = defGetBoolean(defel);
1072  }
1073  else if (strcmp(defel->defname, "delimiter") == 0)
1074  {
1075  if (cstate->delim)
1076  ereport(ERROR,
1077  (errcode(ERRCODE_SYNTAX_ERROR),
1078  errmsg("conflicting or redundant options"),
1079  parser_errposition(pstate, defel->location)));
1080  cstate->delim = defGetString(defel);
1081  }
1082  else if (strcmp(defel->defname, "null") == 0)
1083  {
1084  if (cstate->null_print)
1085  ereport(ERROR,
1086  (errcode(ERRCODE_SYNTAX_ERROR),
1087  errmsg("conflicting or redundant options"),
1088  parser_errposition(pstate, defel->location)));
1089  cstate->null_print = defGetString(defel);
1090  }
1091  else if (strcmp(defel->defname, "header") == 0)
1092  {
1093  if (cstate->header_line)
1094  ereport(ERROR,
1095  (errcode(ERRCODE_SYNTAX_ERROR),
1096  errmsg("conflicting or redundant options"),
1097  parser_errposition(pstate, defel->location)));
1098  cstate->header_line = defGetBoolean(defel);
1099  }
1100  else if (strcmp(defel->defname, "quote") == 0)
1101  {
1102  if (cstate->quote)
1103  ereport(ERROR,
1104  (errcode(ERRCODE_SYNTAX_ERROR),
1105  errmsg("conflicting or redundant options"),
1106  parser_errposition(pstate, defel->location)));
1107  cstate->quote = defGetString(defel);
1108  }
1109  else if (strcmp(defel->defname, "escape") == 0)
1110  {
1111  if (cstate->escape)
1112  ereport(ERROR,
1113  (errcode(ERRCODE_SYNTAX_ERROR),
1114  errmsg("conflicting or redundant options"),
1115  parser_errposition(pstate, defel->location)));
1116  cstate->escape = defGetString(defel);
1117  }
1118  else if (strcmp(defel->defname, "force_quote") == 0)
1119  {
1120  if (cstate->force_quote || cstate->force_quote_all)
1121  ereport(ERROR,
1122  (errcode(ERRCODE_SYNTAX_ERROR),
1123  errmsg("conflicting or redundant options"),
1124  parser_errposition(pstate, defel->location)));
1125  if (defel->arg && IsA(defel->arg, A_Star))
1126  cstate->force_quote_all = true;
1127  else if (defel->arg && IsA(defel->arg, List))
1128  cstate->force_quote = castNode(List, defel->arg);
1129  else
1130  ereport(ERROR,
1131  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1132  errmsg("argument to option \"%s\" must be a list of column names",
1133  defel->defname),
1134  parser_errposition(pstate, defel->location)));
1135  }
1136  else if (strcmp(defel->defname, "force_not_null") == 0)
1137  {
1138  if (cstate->force_notnull)
1139  ereport(ERROR,
1140  (errcode(ERRCODE_SYNTAX_ERROR),
1141  errmsg("conflicting or redundant options"),
1142  parser_errposition(pstate, defel->location)));
1143  if (defel->arg && IsA(defel->arg, List))
1144  cstate->force_notnull = castNode(List, defel->arg);
1145  else
1146  ereport(ERROR,
1147  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1148  errmsg("argument to option \"%s\" must be a list of column names",
1149  defel->defname),
1150  parser_errposition(pstate, defel->location)));
1151  }
1152  else if (strcmp(defel->defname, "force_null") == 0)
1153  {
1154  if (cstate->force_null)
1155  ereport(ERROR,
1156  (errcode(ERRCODE_SYNTAX_ERROR),
1157  errmsg("conflicting or redundant options")));
1158  if (defel->arg && IsA(defel->arg, List))
1159  cstate->force_null = castNode(List, defel->arg);
1160  else
1161  ereport(ERROR,
1162  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1163  errmsg("argument to option \"%s\" must be a list of column names",
1164  defel->defname),
1165  parser_errposition(pstate, defel->location)));
1166  }
1167  else if (strcmp(defel->defname, "convert_selectively") == 0)
1168  {
1169  /*
1170  * Undocumented, not-accessible-from-SQL option: convert only the
1171  * named columns to binary form, storing the rest as NULLs. It's
1172  * allowed for the column list to be NIL.
1173  */
1174  if (cstate->convert_selectively)
1175  ereport(ERROR,
1176  (errcode(ERRCODE_SYNTAX_ERROR),
1177  errmsg("conflicting or redundant options"),
1178  parser_errposition(pstate, defel->location)));
1179  cstate->convert_selectively = true;
1180  if (defel->arg == NULL || IsA(defel->arg, List))
1181  cstate->convert_select = castNode(List, defel->arg);
1182  else
1183  ereport(ERROR,
1184  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1185  errmsg("argument to option \"%s\" must be a list of column names",
1186  defel->defname),
1187  parser_errposition(pstate, defel->location)));
1188  }
1189  else if (strcmp(defel->defname, "encoding") == 0)
1190  {
1191  if (cstate->file_encoding >= 0)
1192  ereport(ERROR,
1193  (errcode(ERRCODE_SYNTAX_ERROR),
1194  errmsg("conflicting or redundant options"),
1195  parser_errposition(pstate, defel->location)));
1196  cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
1197  if (cstate->file_encoding < 0)
1198  ereport(ERROR,
1199  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1200  errmsg("argument to option \"%s\" must be a valid encoding name",
1201  defel->defname),
1202  parser_errposition(pstate, defel->location)));
1203  }
1204  else
1205  ereport(ERROR,
1206  (errcode(ERRCODE_SYNTAX_ERROR),
1207  errmsg("option \"%s\" not recognized",
1208  defel->defname),
1209  parser_errposition(pstate, defel->location)));
1210  }
1211 
1212  /*
1213  * Check for incompatible options (must do these two before inserting
1214  * defaults)
1215  */
1216  if (cstate->binary && cstate->delim)
1217  ereport(ERROR,
1218  (errcode(ERRCODE_SYNTAX_ERROR),
1219  errmsg("cannot specify DELIMITER in BINARY mode")));
1220 
1221  if (cstate->binary && cstate->null_print)
1222  ereport(ERROR,
1223  (errcode(ERRCODE_SYNTAX_ERROR),
1224  errmsg("cannot specify NULL in BINARY mode")));
1225 
1226  /* Set defaults for omitted options */
1227  if (!cstate->delim)
1228  cstate->delim = cstate->csv_mode ? "," : "\t";
1229 
1230  if (!cstate->null_print)
1231  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1232  cstate->null_print_len = strlen(cstate->null_print);
1233 
1234  if (cstate->csv_mode)
1235  {
1236  if (!cstate->quote)
1237  cstate->quote = "\"";
1238  if (!cstate->escape)
1239  cstate->escape = cstate->quote;
1240  }
1241 
1242  /* Only single-byte delimiter strings are supported. */
1243  if (strlen(cstate->delim) != 1)
1244  ereport(ERROR,
1245  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1246  errmsg("COPY delimiter must be a single one-byte character")));
1247 
1248  /* Disallow end-of-line characters */
1249  if (strchr(cstate->delim, '\r') != NULL ||
1250  strchr(cstate->delim, '\n') != NULL)
1251  ereport(ERROR,
1252  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1253  errmsg("COPY delimiter cannot be newline or carriage return")));
1254 
1255  if (strchr(cstate->null_print, '\r') != NULL ||
1256  strchr(cstate->null_print, '\n') != NULL)
1257  ereport(ERROR,
1258  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1259  errmsg("COPY null representation cannot use newline or carriage return")));
1260 
1261  /*
1262  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1263  * backslash because it would be ambiguous. We can't allow the other
1264  * cases because data characters matching the delimiter must be
1265  * backslashed, and certain backslash combinations are interpreted
1266  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1267  * more than strictly necessary, but seems best for consistency and
1268  * future-proofing. Likewise we disallow all digits though only octal
1269  * digits are actually dangerous.
1270  */
1271  if (!cstate->csv_mode &&
1272  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1273  cstate->delim[0]) != NULL)
1274  ereport(ERROR,
1275  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1276  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1277 
1278  /* Check header */
1279  if (!cstate->csv_mode && cstate->header_line)
1280  ereport(ERROR,
1281  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1282  errmsg("COPY HEADER available only in CSV mode")));
1283 
1284  /* Check quote */
1285  if (!cstate->csv_mode && cstate->quote != NULL)
1286  ereport(ERROR,
1287  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1288  errmsg("COPY quote available only in CSV mode")));
1289 
1290  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1291  ereport(ERROR,
1292  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1293  errmsg("COPY quote must be a single one-byte character")));
1294 
1295  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1296  ereport(ERROR,
1297  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1298  errmsg("COPY delimiter and quote must be different")));
1299 
1300  /* Check escape */
1301  if (!cstate->csv_mode && cstate->escape != NULL)
1302  ereport(ERROR,
1303  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1304  errmsg("COPY escape available only in CSV mode")));
1305 
1306  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1307  ereport(ERROR,
1308  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1309  errmsg("COPY escape must be a single one-byte character")));
1310 
1311  /* Check force_quote */
1312  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1313  ereport(ERROR,
1314  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1315  errmsg("COPY force quote available only in CSV mode")));
1316  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1317  ereport(ERROR,
1318  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1319  errmsg("COPY force quote only available using COPY TO")));
1320 
1321  /* Check force_notnull */
1322  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1323  ereport(ERROR,
1324  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1325  errmsg("COPY force not null available only in CSV mode")));
1326  if (cstate->force_notnull != NIL && !is_from)
1327  ereport(ERROR,
1328  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1329  errmsg("COPY force not null only available using COPY FROM")));
1330 
1331  /* Check force_null */
1332  if (!cstate->csv_mode && cstate->force_null != NIL)
1333  ereport(ERROR,
1334  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1335  errmsg("COPY force null available only in CSV mode")));
1336 
1337  if (cstate->force_null != NIL && !is_from)
1338  ereport(ERROR,
1339  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1340  errmsg("COPY force null only available using COPY FROM")));
1341 
1342  /* Don't allow the delimiter to appear in the null string. */
1343  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1344  ereport(ERROR,
1345  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1346  errmsg("COPY delimiter must not appear in the NULL specification")));
1347 
1348  /* Don't allow the CSV quote char to appear in the null string. */
1349  if (cstate->csv_mode &&
1350  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1351  ereport(ERROR,
1352  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1353  errmsg("CSV quote character must not appear in the NULL specification")));
1354 }
1355 
1356 /*
1357  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1358  *
1359  * Iff <binary>, unload or reload in the binary format, as opposed to the
1360  * more wasteful but more robust and portable text format.
1361  *
1362  * Iff <oids>, unload or reload the format that includes OID information.
1363  * On input, we accept OIDs whether or not the table has an OID column,
1364  * but silently drop them if it does not. On output, we report an error
1365  * if the user asks for OIDs in a table that has none (not providing an
1366  * OID column might seem friendlier, but could seriously confuse programs).
1367  *
1368  * If in the text format, delimit columns with delimiter <delim> and print
1369  * NULL values as <null_print>.
1370  */
1371 static CopyState
1372 BeginCopy(ParseState *pstate,
1373  bool is_from,
1374  Relation rel,
1375  RawStmt *raw_query,
1376  Oid queryRelId,
1377  List *attnamelist,
1378  List *options)
1379 {
1380  CopyState cstate;
1381  TupleDesc tupDesc;
1382  int num_phys_attrs;
1383  MemoryContext oldcontext;
1384 
1385  /* Allocate workspace and zero all fields */
1386  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1387 
1388  /*
1389  * We allocate everything used by a cstate in a new memory context. This
1390  * avoids memory leaks during repeated use of COPY in a query.
1391  */
1392  cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
1393  "COPY",
1394  ALLOCSET_DEFAULT_SIZES);
1395 
1396  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1397 
1398  /* Extract options from the statement node tree */
1399  ProcessCopyOptions(pstate, cstate, is_from, options);
1400 
1401  /* Process the source/target relation or query */
1402  if (rel)
1403  {
1404  Assert(!raw_query);
1405 
1406  cstate->rel = rel;
1407 
1408  tupDesc = RelationGetDescr(cstate->rel);
1409 
1410  /* Don't allow COPY w/ OIDs to or from a table without them */
1411  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1412  ereport(ERROR,
1413  (errcode(ERRCODE_UNDEFINED_COLUMN),
1414  errmsg("table \"%s\" does not have OIDs",
1415  RelationGetRelationName(cstate->rel))));
1416 
1417  /* Initialize state for CopyFrom tuple routing. */
1418  if (is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1419  {
1420  PartitionDispatch *partition_dispatch_info;
1421  ResultRelInfo *partitions;
1422  TupleConversionMap **partition_tupconv_maps;
1423  TupleTableSlot *partition_tuple_slot;
1424  int num_parted,
1425  num_partitions;
1426 
1427  ExecSetupPartitionTupleRouting(rel,
1428  &partition_dispatch_info,
1429  &partitions,
1430  &partition_tupconv_maps,
1431  &partition_tuple_slot,
1432  &num_parted, &num_partitions);
1433  cstate->partition_dispatch_info = partition_dispatch_info;
1434  cstate->num_dispatch = num_parted;
1435  cstate->partitions = partitions;
1436  cstate->num_partitions = num_partitions;
1437  cstate->partition_tupconv_maps = partition_tupconv_maps;
1438  cstate->partition_tuple_slot = partition_tuple_slot;
1439  }
1440  }
1441  else
1442  {
1443  List *rewritten;
1444  Query *query;
1445  PlannedStmt *plan;
1446  DestReceiver *dest;
1447 
1448  Assert(!is_from);
1449  cstate->rel = NULL;
1450 
1451  /* Don't allow COPY w/ OIDs from a query */
1452  if (cstate->oids)
1453  ereport(ERROR,
1454  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1455  errmsg("COPY (query) WITH OIDS is not supported")));
1456 
1457  /*
1458  * Run parse analysis and rewrite. Note this also acquires sufficient
1459  * locks on the source table(s).
1460  *
1461  * Because the parser and planner tend to scribble on their input, we
1462  * make a preliminary copy of the source querytree. This prevents
1463  * problems in the case that the COPY is in a portal or plpgsql
1464  * function and is executed repeatedly. (See also the same hack in
1465  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1466  */
1467  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1468  pstate->p_sourcetext, NULL, 0,
1469  NULL);
1470 
1471  /* check that we got back something we can work with */
1472  if (rewritten == NIL)
1473  {
1474  ereport(ERROR,
1475  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1476  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1477  }
1478  else if (list_length(rewritten) > 1)
1479  {
1480  ListCell *lc;
1481 
1482  /* examine queries to determine which error message to issue */
1483  foreach(lc, rewritten)
1484  {
1485  Query *q = lfirst_node(Query, lc);
1486 
1487  if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
1488  ereport(ERROR,
1489  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1490  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1491  if (q->querySource == QSRC_NON_INSTEAD_RULE)
1492  ereport(ERROR,
1493  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1494  errmsg("DO ALSO rules are not supported for the COPY")));
1495  }
1496 
1497  ereport(ERROR,
1498  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1499  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1500  }
1501 
1502  query = linitial_node(Query, rewritten);
1503 
1504  /* The grammar allows SELECT INTO, but we don't support that */
1505  if (query->utilityStmt != NULL &&
1506  IsA(query->utilityStmt, CreateTableAsStmt))
1507  ereport(ERROR,
1508  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1509  errmsg("COPY (SELECT INTO) is not supported")));
1510 
1511  Assert(query->utilityStmt == NULL);
1512 
1513  /*
1514  * Similarly the grammar doesn't enforce the presence of a RETURNING
1515  * clause, but this is required here.
1516  */
1517  if (query->commandType != CMD_SELECT &&
1518  query->returningList == NIL)
1519  {
1520  Assert(query->commandType == CMD_INSERT ||
1521  query->commandType == CMD_UPDATE ||
1522  query->commandType == CMD_DELETE);
1523 
1524  ereport(ERROR,
1525  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1526  errmsg("COPY query must have a RETURNING clause")));
1527  }
1528 
1529  /* plan the query */
1530  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1531 
1532  /*
1533  * With row level security and a user using "COPY relation TO", we
1534  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1535  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1536  * in any RLS clauses.
1537  *
1538  * When this happens, we are passed in the relid of the originally
1539  * found relation (which we have locked). As the planner will look up
1540  * the relation again, we double-check here to make sure it found the
1541  * same one that we have locked.
1542  */
1543  if (queryRelId != InvalidOid)
1544  {
1545  /*
1546  * Note that with RLS involved there may be multiple relations,
1547  * and while the one we need is almost certainly first, we don't
1548  * make any guarantees of that in the planner, so check the whole
1549  * list and make sure we find the original relation.
1550  */
1551  if (!list_member_oid(plan->relationOids, queryRelId))
1552  ereport(ERROR,
1553  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1554  errmsg("relation referenced by COPY statement has changed")));
1555  }
1556 
1557  /*
1558  * Use a snapshot with an updated command ID to ensure this query sees
1559  * results of any previously executed queries.
1560  */
1561  PushCopiedSnapshot(GetActiveSnapshot());
1562  UpdateActiveSnapshotCommandId();
1563 
1564  /* Create dest receiver for COPY OUT */
1565  dest = CreateDestReceiver(DestCopyOut);
1566  ((DR_copy *) dest)->cstate = cstate;
1567 
1568  /* Create a QueryDesc requesting no output */
1569  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1570  GetActiveSnapshot(),
1571  InvalidSnapshot,
1572  dest, NULL, NULL, 0);
1573 
1574  /*
1575  * Call ExecutorStart to prepare the plan for execution.
1576  *
1577  * ExecutorStart computes a result tupdesc for us
1578  */
1579  ExecutorStart(cstate->queryDesc, 0);
1580 
1581  tupDesc = cstate->queryDesc->tupDesc;
1582  }
1583 
1584  /* Generate or convert list of attributes to process */
1585  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1586 
1587  num_phys_attrs = tupDesc->natts;
1588 
1589  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1590  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1591  if (cstate->force_quote_all)
1592  {
1593  int i;
1594 
1595  for (i = 0; i < num_phys_attrs; i++)
1596  cstate->force_quote_flags[i] = true;
1597  }
1598  else if (cstate->force_quote)
1599  {
1600  List *attnums;
1601  ListCell *cur;
1602 
1603  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1604 
1605  foreach(cur, attnums)
1606  {
1607  int attnum = lfirst_int(cur);
1608 
1609  if (!list_member_int(cstate->attnumlist, attnum))
1610  ereport(ERROR,
1611  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1612  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1613  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1614  cstate->force_quote_flags[attnum - 1] = true;
1615  }
1616  }
1617 
1618  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1619  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1620  if (cstate->force_notnull)
1621  {
1622  List *attnums;
1623  ListCell *cur;
1624 
1625  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1626 
1627  foreach(cur, attnums)
1628  {
1629  int attnum = lfirst_int(cur);
1630 
1631  if (!list_member_int(cstate->attnumlist, attnum))
1632  ereport(ERROR,
1633  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1634  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1635  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1636  cstate->force_notnull_flags[attnum - 1] = true;
1637  }
1638  }
1639 
1640  /* Convert FORCE_NULL name list to per-column flags, check validity */
1641  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1642  if (cstate->force_null)
1643  {
1644  List *attnums;
1645  ListCell *cur;
1646 
1647  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1648 
1649  foreach(cur, attnums)
1650  {
1651  int attnum = lfirst_int(cur);
1652 
1653  if (!list_member_int(cstate->attnumlist, attnum))
1654  ereport(ERROR,
1655  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1656  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1657  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1658  cstate->force_null_flags[attnum - 1] = true;
1659  }
1660  }
1661 
1662  /* Convert convert_selectively name list to per-column flags */
1663  if (cstate->convert_selectively)
1664  {
1665  List *attnums;
1666  ListCell *cur;
1667 
1668  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1669 
1670  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1671 
1672  foreach(cur, attnums)
1673  {
1674  int attnum = lfirst_int(cur);
1675 
1676  if (!list_member_int(cstate->attnumlist, attnum))
1677  ereport(ERROR,
1678  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1679  errmsg_internal("selected column \"%s\" not referenced by COPY",
1680  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1681  cstate->convert_select_flags[attnum - 1] = true;
1682  }
1683  }
1684 
1685  /* Use client encoding when ENCODING option is not specified. */
1686  if (cstate->file_encoding < 0)
1687  cstate->file_encoding = pg_get_client_encoding();
1688 
1689  /*
1690  * Set up encoding conversion info. Even if the file and server encodings
1691  * are the same, we must apply pg_any_to_server() to validate data in
1692  * multibyte encodings.
1693  */
1694  cstate->need_transcoding =
1695  (cstate->file_encoding != GetDatabaseEncoding() ||
1696  pg_database_encoding_max_length() > 1);
1697  /* See Multibyte encoding comment above */
1698  cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
1699 
1700  cstate->copy_dest = COPY_FILE; /* default */
1701 
1702  MemoryContextSwitchTo(oldcontext);
1703 
1704  return cstate;
1705 }
1706 
1707 /*
1708  * Closes the pipe to an external program, checking the pclose() return code.
1709  */
1710 static void
1711 ClosePipeToProgram(CopyState cstate)
1712 {
1713  int pclose_rc;
1714 
1715  Assert(cstate->is_program);
1716 
1717  pclose_rc = ClosePipeStream(cstate->copy_file);
1718  if (pclose_rc == -1)
1719  ereport(ERROR,
1720  (errcode_for_file_access(),
1721  errmsg("could not close pipe to external command: %m")));
1722  else if (pclose_rc != 0)
1723  ereport(ERROR,
1724  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1725  errmsg("program \"%s\" failed",
1726  cstate->filename),
1727  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1728 }
1729 
1730 /*
1731  * Release resources allocated in a cstate for COPY TO/FROM.
1732  */
1733 static void
1734 EndCopy(CopyState cstate)
1735 {
1736  if (cstate->is_program)
1737  {
1738  ClosePipeToProgram(cstate);
1739  }
1740  else
1741  {
1742  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1743  ereport(ERROR,
1744  (errcode_for_file_access(),
1745  errmsg("could not close file \"%s\": %m",
1746  cstate->filename)));
1747  }
1748 
1749  MemoryContextDelete(cstate->copycontext);
1750  pfree(cstate);
1751 }
1752 
1753 /*
1754  * Setup CopyState to read tuples from a table or a query for COPY TO.
1755  */
1756 static CopyState
1757 BeginCopyTo(ParseState *pstate,
1758  Relation rel,
1759  RawStmt *query,
1760  Oid queryRelId,
1761  const char *filename,
1762  bool is_program,
1763  List *attnamelist,
1764  List *options)
1765 {
1766  CopyState cstate;
1767  bool pipe = (filename == NULL);
1768  MemoryContext oldcontext;
1769 
1770  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1771  {
1772  if (rel->rd_rel->relkind == RELKIND_VIEW)
1773  ereport(ERROR,
1774  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1775  errmsg("cannot copy from view \"%s\"",
1776  RelationGetRelationName(rel)),
1777  errhint("Try the COPY (SELECT ...) TO variant.")));
1778  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1779  ereport(ERROR,
1780  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1781  errmsg("cannot copy from materialized view \"%s\"",
1782  RelationGetRelationName(rel)),
1783  errhint("Try the COPY (SELECT ...) TO variant.")));
1784  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1785  ereport(ERROR,
1786  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1787  errmsg("cannot copy from foreign table \"%s\"",
1788  RelationGetRelationName(rel)),
1789  errhint("Try the COPY (SELECT ...) TO variant.")));
1790  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1791  ereport(ERROR,
1792  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1793  errmsg("cannot copy from sequence \"%s\"",
1794  RelationGetRelationName(rel))));
1795  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1796  ereport(ERROR,
1797  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1798  errmsg("cannot copy from partitioned table \"%s\"",
1799  RelationGetRelationName(rel)),
1800  errhint("Try the COPY (SELECT ...) TO variant.")));
1801  else
1802  ereport(ERROR,
1803  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1804  errmsg("cannot copy from non-table relation \"%s\"",
1805  RelationGetRelationName(rel))));
1806  }
1807 
1808  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1809  options);
1810  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1811 
1812  if (pipe)
1813  {
1814  Assert(!is_program); /* the grammar does not allow this */
1815  if (whereToSendOutput != DestRemote)
1816  cstate->copy_file = stdout;
1817  }
1818  else
1819  {
1820  cstate->filename = pstrdup(filename);
1821  cstate->is_program = is_program;
1822 
1823  if (is_program)
1824  {
1825  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1826  if (cstate->copy_file == NULL)
1827  ereport(ERROR,
1828  (errcode_for_file_access(),
1829  errmsg("could not execute command \"%s\": %m",
1830  cstate->filename)));
1831  }
1832  else
1833  {
1834  mode_t oumask; /* Pre-existing umask value */
1835  struct stat st;
1836 
1837  /*
1838  * Prevent write to relative path ... too easy to shoot oneself in
1839  * the foot by overwriting a database file ...
1840  */
1841  if (!is_absolute_path(filename))
1842  ereport(ERROR,
1843  (errcode(ERRCODE_INVALID_NAME),
1844  errmsg("relative path not allowed for COPY to file")));
1845 
1846  oumask = umask(S_IWGRP | S_IWOTH);
1847  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1848  umask(oumask);
1849  if (cstate->copy_file == NULL)
1850  {
1851  /* copy errno because ereport subfunctions might change it */
1852  int save_errno = errno;
1853 
1854  ereport(ERROR,
1855  (errcode_for_file_access(),
1856  errmsg("could not open file \"%s\" for writing: %m",
1857  cstate->filename),
1858  (save_errno == ENOENT || save_errno == EACCES) ?
1859  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1860  "You may want a client-side facility such as psql's \\copy.") : 0));
1861  }
1862 
1863  if (fstat(fileno(cstate->copy_file), &st))
1864  ereport(ERROR,
1865  (errcode_for_file_access(),
1866  errmsg("could not stat file \"%s\": %m",
1867  cstate->filename)));
1868 
1869  if (S_ISDIR(st.st_mode))
1870  ereport(ERROR,
1871  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1872  errmsg("\"%s\" is a directory", cstate->filename)));
1873  }
1874  }
1875 
1876  MemoryContextSwitchTo(oldcontext);
1877 
1878  return cstate;
1879 }
1880 
1881 /*
1882  * This intermediate routine exists mainly to localize the effects of setjmp
1883  * so we don't need to plaster a lot of variables with "volatile".
1884  */
1885 static uint64
1886 DoCopyTo(CopyState cstate)
1887 {
1888  bool pipe = (cstate->filename == NULL);
1889  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
1890  uint64 processed;
1891 
1892  PG_TRY();
1893  {
1894  if (fe_copy)
1895  SendCopyBegin(cstate);
1896 
1897  processed = CopyTo(cstate);
1898 
1899  if (fe_copy)
1900  SendCopyEnd(cstate);
1901  }
1902  PG_CATCH();
1903  {
1904  /*
1905  * Make sure we turn off old-style COPY OUT mode upon error. It is
1906  * okay to do this in all cases, since it does nothing if the mode is
1907  * not on.
1908  */
1909  pq_endcopyout(true);
1910  PG_RE_THROW();
1911  }
1912  PG_END_TRY();
1913 
1914  return processed;
1915 }
1916 
1917 /*
1918  * Clean up storage and release resources for COPY TO.
1919  */
1920 static void
1921 EndCopyTo(CopyState cstate)
1922 {
1923  if (cstate->queryDesc != NULL)
1924  {
1925  /* Close down the query and free resources. */
1926  ExecutorFinish(cstate->queryDesc);
1927  ExecutorEnd(cstate->queryDesc);
1928  FreeQueryDesc(cstate->queryDesc);
1929  PopActiveSnapshot();
1930  }
1931 
1932  /* Clean up storage */
1933  EndCopy(cstate);
1934 }
1935 
1936 /*
1937  * Copy from relation or query TO file.
1938  */
1939 static uint64
1940 CopyTo(CopyState cstate)
1941 {
1942  TupleDesc tupDesc;
1943  int num_phys_attrs;
1944  Form_pg_attribute *attr;
1945  ListCell *cur;
1946  uint64 processed;
1947 
1948  if (cstate->rel)
1949  tupDesc = RelationGetDescr(cstate->rel);
1950  else
1951  tupDesc = cstate->queryDesc->tupDesc;
1952  attr = tupDesc->attrs;
1953  num_phys_attrs = tupDesc->natts;
1954  cstate->null_print_client = cstate->null_print; /* default */
1955 
1956  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1957  cstate->fe_msgbuf = makeStringInfo();
1958 
1959  /* Get info about the columns we need to process. */
1960  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1961  foreach(cur, cstate->attnumlist)
1962  {
1963  int attnum = lfirst_int(cur);
1964  Oid out_func_oid;
1965  bool isvarlena;
1966 
1967  if (cstate->binary)
1968  getTypeBinaryOutputInfo(attr[attnum - 1]->atttypid,
1969  &out_func_oid,
1970  &isvarlena);
1971  else
1972  getTypeOutputInfo(attr[attnum - 1]->atttypid,
1973  &out_func_oid,
1974  &isvarlena);
1975  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
1976  }
1977 
1978  /*
1979  * Create a temporary memory context that we can reset once per row to
1980  * recover palloc'd memory. This avoids any problems with leaks inside
1981  * datatype output routines, and should be faster than retail pfree's
1982  * anyway. (We don't need a whole econtext as CopyFrom does.)
1983  */
1984  cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
1985  "COPY TO",
1986  ALLOCSET_DEFAULT_SIZES);
1987 
1988  if (cstate->binary)
1989  {
1990  /* Generate header for a binary copy */
1991  int32 tmp;
1992 
1993  /* Signature */
1994  CopySendData(cstate, BinarySignature, 11);
1995  /* Flags field */
1996  tmp = 0;
1997  if (cstate->oids)
1998  tmp |= (1 << 16);
1999  CopySendInt32(cstate, tmp);
2000  /* No header extension */
2001  tmp = 0;
2002  CopySendInt32(cstate, tmp);
2003  }
2004  else
2005  {
2006  /*
2007  * For non-binary copy, we need to convert null_print to file
2008  * encoding, because it will be sent directly with CopySendString.
2009  */
2010  if (cstate->need_transcoding)
2011  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2012  cstate->null_print_len,
2013  cstate->file_encoding);
2014 
2015  /* if a header has been requested send the line */
2016  if (cstate->header_line)
2017  {
2018  bool hdr_delim = false;
2019 
2020  foreach(cur, cstate->attnumlist)
2021  {
2022  int attnum = lfirst_int(cur);
2023  char *colname;
2024 
2025  if (hdr_delim)
2026  CopySendChar(cstate, cstate->delim[0]);
2027  hdr_delim = true;
2028 
2029  colname = NameStr(attr[attnum - 1]->attname);
2030 
2031  CopyAttributeOutCSV(cstate, colname, false,
2032  list_length(cstate->attnumlist) == 1);
2033  }
2034 
2035  CopySendEndOfRow(cstate);
2036  }
2037  }
2038 
2039  if (cstate->rel)
2040  {
2041  Datum *values;
2042  bool *nulls;
2043  HeapScanDesc scandesc;
2044  HeapTuple tuple;
2045 
2046  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2047  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2048 
2049  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2050 
2051  processed = 0;
2052  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2053  {
2054  CHECK_FOR_INTERRUPTS();
2055 
2056  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2057  heap_deform_tuple(tuple, tupDesc, values, nulls);
2058 
2059  /* Format and send the data */
2060  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2061  processed++;
2062  }
2063 
2064  heap_endscan(scandesc);
2065 
2066  pfree(values);
2067  pfree(nulls);
2068  }
2069  else
2070  {
2071  /* run the plan --- the dest receiver will send tuples */
2072  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2073  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2074  }
2075 
2076  if (cstate->binary)
2077  {
2078  /* Generate trailer for a binary copy */
2079  CopySendInt16(cstate, -1);
2080  /* Need to flush out the trailer */
2081  CopySendEndOfRow(cstate);
2082  }
2083 
2084  MemoryContextDelete(cstate->rowcontext);
2085 
2086  return processed;
2087 }
2088 
2089 /*
2090  * Emit one row during CopyTo().
2091  */
2092 static void
2093 CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
2094 {
2095  bool need_delim = false;
2096  FmgrInfo *out_functions = cstate->out_functions;
2097  MemoryContext oldcontext;
2098  ListCell *cur;
2099  char *string;
2100 
2101  MemoryContextReset(cstate->rowcontext);
2102  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2103 
2104  if (cstate->binary)
2105  {
2106  /* Binary per-tuple header */
2107  CopySendInt16(cstate, list_length(cstate->attnumlist));
2108  /* Send OID if wanted --- note attnumlist doesn't include it */
2109  if (cstate->oids)
2110  {
2111  /* Hack --- assume Oid is same size as int32 */
2112  CopySendInt32(cstate, sizeof(int32));
2113  CopySendInt32(cstate, tupleOid);
2114  }
2115  }
2116  else
2117  {
2118  /* Text format has no per-tuple header, but send OID if wanted */
2119  /* Assume digits don't need any quoting or encoding conversion */
2120  if (cstate->oids)
2121  {
2122  string = DatumGetCString(DirectFunctionCall1(oidout,
2123  ObjectIdGetDatum(tupleOid)));
2124  CopySendString(cstate, string);
2125  need_delim = true;
2126  }
2127  }
2128 
2129  foreach(cur, cstate->attnumlist)
2130  {
2131  int attnum = lfirst_int(cur);
2132  Datum value = values[attnum - 1];
2133  bool isnull = nulls[attnum - 1];
2134 
2135  if (!cstate->binary)
2136  {
2137  if (need_delim)
2138  CopySendChar(cstate, cstate->delim[0]);
2139  need_delim = true;
2140  }
2141 
2142  if (isnull)
2143  {
2144  if (!cstate->binary)
2145  CopySendString(cstate, cstate->null_print_client);
2146  else
2147  CopySendInt32(cstate, -1);
2148  }
2149  else
2150  {
2151  if (!cstate->binary)
2152  {
2153  string = OutputFunctionCall(&out_functions[attnum - 1],
2154  value);
2155  if (cstate->csv_mode)
2156  CopyAttributeOutCSV(cstate, string,
2157  cstate->force_quote_flags[attnum - 1],
2158  list_length(cstate->attnumlist) == 1);
2159  else
2160  CopyAttributeOutText(cstate, string);
2161  }
2162  else
2163  {
2164  bytea *outputbytes;
2165 
2166  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2167  value);
2168  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2169  CopySendData(cstate, VARDATA(outputbytes),
2170  VARSIZE(outputbytes) - VARHDRSZ);
2171  }
2172  }
2173  }
2174 
2175  CopySendEndOfRow(cstate);
2176 
2177  MemoryContextSwitchTo(oldcontext);
2178 }
2179 
2180 
2181 /*
2182  * error context callback for COPY FROM
2183  *
2184  * The argument for the error context must be CopyState.
2185  */
2186 void
2187 CopyFromErrorCallback(void *arg)
2188 {
2189  CopyState cstate = (CopyState) arg;
2190 
2191  if (cstate->binary)
2192  {
2193  /* can't usefully display the data */
2194  if (cstate->cur_attname)
2195  errcontext("COPY %s, line %d, column %s",
2196  cstate->cur_relname, cstate->cur_lineno,
2197  cstate->cur_attname);
2198  else
2199  errcontext("COPY %s, line %d",
2200  cstate->cur_relname, cstate->cur_lineno);
2201  }
2202  else
2203  {
2204  if (cstate->cur_attname && cstate->cur_attval)
2205  {
2206  /* error is relevant to a particular column */
2207  char *attval;
2208 
2209  attval = limit_printout_length(cstate->cur_attval);
2210  errcontext("COPY %s, line %d, column %s: \"%s\"",
2211  cstate->cur_relname, cstate->cur_lineno,
2212  cstate->cur_attname, attval);
2213  pfree(attval);
2214  }
2215  else if (cstate->cur_attname)
2216  {
2217  /* error is relevant to a particular column, value is NULL */
2218  errcontext("COPY %s, line %d, column %s: null input",
2219  cstate->cur_relname, cstate->cur_lineno,
2220  cstate->cur_attname);
2221  }
2222  else
2223  {
2224  /*
2225  * Error is relevant to a particular line.
2226  *
2227  * If line_buf still contains the correct line, and it's already
2228  * transcoded, print it. If it's still in a foreign encoding, it's
2229  * quite likely that the error is precisely a failure to do
2230  * encoding conversion (ie, bad data). We dare not try to convert
2231  * it, and at present there's no way to regurgitate it without
2232  * conversion. So we have to punt and just report the line number.
2233  */
2234  if (cstate->line_buf_valid &&
2235  (cstate->line_buf_converted || !cstate->need_transcoding))
2236  {
2237  char *lineval;
2238 
2239  lineval = limit_printout_length(cstate->line_buf.data);
2240  errcontext("COPY %s, line %d: \"%s\"",
2241  cstate->cur_relname, cstate->cur_lineno, lineval);
2242  pfree(lineval);
2243  }
2244  else
2245  {
2246  errcontext("COPY %s, line %d",
2247  cstate->cur_relname, cstate->cur_lineno);
2248  }
2249  }
2250  }
2251 }
2252 
2253 /*
2254  * Make sure we don't print an unreasonable amount of COPY data in a message.
2255  *
2256  * It would seem a lot easier to just use the sprintf "precision" limit to
2257  * truncate the string. However, some versions of glibc have a bug/misfeature
2258  * that vsnprintf will always fail (return -1) if it is asked to truncate
2259  * a string that contains invalid byte sequences for the current encoding.
2260  * So, do our own truncation. We return a pstrdup'd copy of the input.
2261  */
2262 static char *
2263 limit_printout_length(const char *str)
2264 {
2265 #define MAX_COPY_DATA_DISPLAY 100
2266 
2267  int slen = strlen(str);
2268  int len;
2269  char *res;
2270 
2271  /* Fast path if definitely okay */
2272  if (slen <= MAX_COPY_DATA_DISPLAY)
2273  return pstrdup(str);
2274 
2275  /* Apply encoding-dependent truncation */
2276  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2277 
2278  /*
2279  * Truncate, and add "..." to show we truncated the input.
2280  */
2281  res = (char *) palloc(len + 4);
2282  memcpy(res, str, len);
2283  strcpy(res + len, "...");
2284 
2285  return res;
2286 }
2287 
2288 /*
2289  * Copy FROM file to relation.
2290  */
2291 uint64
2292 CopyFrom(CopyState cstate)
2293 {
2294  HeapTuple tuple;
2295  TupleDesc tupDesc;
2296  Datum *values;
2297  bool *nulls;
2298  ResultRelInfo *resultRelInfo;
2299  ResultRelInfo *saved_resultRelInfo = NULL;
2300  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2301  ExprContext *econtext;
2302  TupleTableSlot *myslot;
2303  MemoryContext oldcontext = CurrentMemoryContext;
2304 
2305  ErrorContextCallback errcallback;
2306  CommandId mycid = GetCurrentCommandId(true);
2307  int hi_options = 0; /* start with default heap_insert options */
2308  BulkInsertState bistate;
2309  uint64 processed = 0;
2310  bool useHeapMultiInsert;
2311  int nBufferedTuples = 0;
2312  int prev_leaf_part_index = -1;
2313 
2314 #define MAX_BUFFERED_TUPLES 1000
2315  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2316  Size bufferedTuplesSize = 0;
2317  int firstBufferedLineNo = 0;
2318 
2319  Assert(cstate->rel);
2320 
2321  /*
2322  * The target must be a plain relation or have an INSTEAD OF INSERT row
2323  * trigger. (Currently, such triggers are only allowed on views, so we
2324  * only hint about them in the view case.)
2325  */
2326  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2327  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2328  !(cstate->rel->trigdesc &&
2329  cstate->rel->trigdesc->trig_insert_instead_row))
2330  {
2331  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2332  ereport(ERROR,
2333  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2334  errmsg("cannot copy to view \"%s\"",
2335  RelationGetRelationName(cstate->rel)),
2336  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2337  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2338  ereport(ERROR,
2339  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2340  errmsg("cannot copy to materialized view \"%s\"",
2341  RelationGetRelationName(cstate->rel))));
2342  else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
2343  ereport(ERROR,
2344  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2345  errmsg("cannot copy to foreign table \"%s\"",
2346  RelationGetRelationName(cstate->rel))));
2347  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2348  ereport(ERROR,
2349  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2350  errmsg("cannot copy to sequence \"%s\"",
2351  RelationGetRelationName(cstate->rel))));
2352  else
2353  ereport(ERROR,
2354  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2355  errmsg("cannot copy to non-table relation \"%s\"",
2356  RelationGetRelationName(cstate->rel))));
2357  }
2358 
2359  tupDesc = RelationGetDescr(cstate->rel);
2360 
2361  /*----------
2362  * Check to see if we can avoid writing WAL
2363  *
2364  * If archive logging/streaming is not enabled *and* either
2365  * - table was created in same transaction as this COPY
2366  * - data is being written to relfilenode created in this transaction
2367  * then we can skip writing WAL. It's safe because if the transaction
2368  * doesn't commit, we'll discard the table (or the new relfilenode file).
2369  * If it does commit, we'll have done the heap_sync at the bottom of this
2370  * routine first.
2371  *
2372  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2373  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2374  * can be cleared before the end of the transaction. The exact case is
2375  * when a relation sets a new relfilenode twice in same transaction, yet
2376  * the second one fails in an aborted subtransaction, e.g.
2377  *
2378  * BEGIN;
2379  * TRUNCATE t;
2380  * SAVEPOINT save;
2381  * TRUNCATE t;
2382  * ROLLBACK TO save;
2383  * COPY ...
2384  *
2385  * Also, if the target file is new-in-transaction, we assume that checking
2386  * FSM for free space is a waste of time, even if we must use WAL because
2387  * of archiving. This could possibly be wrong, but it's unlikely.
2388  *
2389  * The comments for heap_insert and RelationGetBufferForTuple specify that
2390  * skipping WAL logging is only safe if we ensure that our tuples do not
2391  * go into pages containing tuples from any other transactions --- but this
2392  * must be the case if we have a new table or new relfilenode, so we need
2393  * no additional work to enforce that.
2394  *----------
2395  */
2396  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2397  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2398  cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId)
2399  {
2400  hi_options |= HEAP_INSERT_SKIP_FSM;
2401  if (!XLogIsNeeded())
2402  hi_options |= HEAP_INSERT_SKIP_WAL;
2403  }
2404 
2405  /*
2406  * Optimize if new relfilenode was created in this subxact or one of its
2407  * committed children and we won't see those rows later as part of an
2408  * earlier scan or command. This ensures that if this subtransaction
2409  * aborts then the frozen rows won't be visible after xact cleanup. Note
2410  * that the stronger test of exactly which subtransaction created it is
2411  * crucial for correctness of this optimization.
2412  */
2413  if (cstate->freeze)
2414  {
2415  if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
2416  ereport(ERROR,
2417  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2418  errmsg("cannot perform FREEZE because of prior transaction activity")));
2419 
2420  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2421  cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
2422  ereport(ERROR,
2423  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2424  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2425 
2426  hi_options |= HEAP_INSERT_FROZEN;
2427  }
2428 
2429  /*
2430  * We need a ResultRelInfo so we can use the regular executor's
2431  * index-entry-making machinery. (There used to be a huge amount of code
2432  * here that basically duplicated execUtils.c ...)
2433  */
2434  resultRelInfo = makeNode(ResultRelInfo);
2435  InitResultRelInfo(resultRelInfo,
2436  cstate->rel,
2437  1, /* dummy rangetable index */
2438  NULL,
2439  0);
2440 
2441  ExecOpenIndices(resultRelInfo, false);
2442 
2443  estate->es_result_relations = resultRelInfo;
2444  estate->es_num_result_relations = 1;
2445  estate->es_result_relation_info = resultRelInfo;
2446  estate->es_range_table = cstate->range_table;
2447 
2448  /* Set up a tuple slot too */
2449  myslot = ExecInitExtraTupleSlot(estate);
2450  ExecSetSlotDescriptor(myslot, tupDesc);
2451  /* Triggers might need a slot as well */
2452  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);
2453 
2454  /*
2455  * It's more efficient to prepare a bunch of tuples for insertion, and
2456  * insert them in one heap_multi_insert() call, than call heap_insert()
2457  * separately for every tuple. However, we can't do that if there are
2458  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2459  * expressions. Such triggers or expressions might query the table we're
2460  * inserting to, and act differently if the tuples that have already been
2461  * processed and prepared for insertion are not there. We also can't do
2462  * it if the table is partitioned.
2463  */
2464  if ((resultRelInfo->ri_TrigDesc != NULL &&
2465  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2466  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2467  cstate->partition_dispatch_info != NULL ||
2468  cstate->volatile_defexprs)
2469  {
2470  useHeapMultiInsert = false;
2471  }
2472  else
2473  {
2474  useHeapMultiInsert = true;
2475  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2476  }
2477 
2478  /* Prepare to catch AFTER triggers. */
2479  AfterTriggerBeginQuery();
2480 
2481  /*
2482  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2483  * should do this for COPY, since it's not really an "INSERT" statement as
2484  * such. However, executing these triggers maintains consistency with the
2485  * EACH ROW triggers that we already fire on COPY.
2486  */
2487  ExecBSInsertTriggers(estate, resultRelInfo);
2488 
2489  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2490  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2491 
2492  bistate = GetBulkInsertState();
2493  econtext = GetPerTupleExprContext(estate);
2494 
2495  /* Set up callback to identify error line number */
2496  errcallback.callback = CopyFromErrorCallback;
2497  errcallback.arg = (void *) cstate;
2498  errcallback.previous = error_context_stack;
2499  error_context_stack = &errcallback;
2500 
2501  for (;;)
2502  {
2503  TupleTableSlot *slot;
2504  bool skip_tuple;
2505  Oid loaded_oid = InvalidOid;
2506 
2507  CHECK_FOR_INTERRUPTS();
2508 
2509  if (nBufferedTuples == 0)
2510  {
2511  /*
2512  * Reset the per-tuple exprcontext. We can only do this if the
2513  * tuple buffer is empty. (Calling the context the per-tuple
2514  * memory context is a bit of a misnomer now.)
2515  */
2516  ResetPerTupleExprContext(estate);
2517  }
2518 
2519  /* Switch into its memory context */
2520  MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2521 
2522  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2523  break;
2524 
2525  /* And now we can form the input tuple. */
2526  tuple = heap_form_tuple(tupDesc, values, nulls);
2527 
2528  if (loaded_oid != InvalidOid)
2529  HeapTupleSetOid(tuple, loaded_oid);
2530 
2531  /*
2532  * Constraints might reference the tableoid column, so initialize
2533  * t_tableOid before evaluating them.
2534  */
2535  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2536 
2537  /* Triggers and stuff need to be invoked in query context. */
2538  MemoryContextSwitchTo(oldcontext);
2539 
2540  /* Place tuple in tuple slot --- but slot shouldn't free it */
2541  slot = myslot;
2542  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2543 
2544  /* Determine the partition to heap_insert the tuple into */
2545  if (cstate->partition_dispatch_info)
2546  {
2547  int leaf_part_index;
2548  TupleConversionMap *map;
2549 
2550  /*
2551  * Away we go ... If we end up not finding a partition after all,
2552  * ExecFindPartition() does not return and errors out instead.
2553  * Otherwise, the returned value is to be used as an index into
2554  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2555  * will get us the ResultRelInfo and TupleConversionMap for the
2556  * partition, respectively.
2557  */
2558  leaf_part_index = ExecFindPartition(resultRelInfo,
2559  cstate->partition_dispatch_info,
2560  slot,
2561  estate);
2562  Assert(leaf_part_index >= 0 &&
2563  leaf_part_index < cstate->num_partitions);
2564 
2565  /*
2566  * If this tuple is mapped to a partition that is not same as the
2567  * previous one, we'd better make the bulk insert mechanism gets a
2568  * new buffer.
2569  */
2570  if (prev_leaf_part_index != leaf_part_index)
2571  {
2572  ReleaseBulkInsertStatePin(bistate);
2573  prev_leaf_part_index = leaf_part_index;
2574  }
2575 
2576  /*
2577  * Save the old ResultRelInfo and switch to the one corresponding
2578  * to the selected partition.
2579  */
2580  saved_resultRelInfo = resultRelInfo;
2581  resultRelInfo = cstate->partitions + leaf_part_index;
2582 
2583  /* We do not yet have a way to insert into a foreign partition */
2584  if (resultRelInfo->ri_FdwRoutine)
2585  ereport(ERROR,
2586  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2587  errmsg("cannot route inserted tuples to a foreign table")));
2588 
2589  /*
2590  * For ExecInsertIndexTuples() to work on the partition's indexes
2591  */
2592  estate->es_result_relation_info = resultRelInfo;
2593 
2594  /*
2595  * We might need to convert from the parent rowtype to the
2596  * partition rowtype.
2597  */
2598  map = cstate->partition_tupconv_maps[leaf_part_index];
2599  if (map)
2600  {
2601  Relation partrel = resultRelInfo->ri_RelationDesc;
2602 
2603  tuple = do_convert_tuple(tuple, map);
2604 
2605  /*
2606  * We must use the partition's tuple descriptor from this
2607  * point on. Use a dedicated slot from this point on until
2608  * we're finished dealing with the partition.
2609  */
2610  slot = cstate->partition_tuple_slot;
2611  Assert(slot != NULL);
2612  ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
2613  ExecStoreTuple(tuple, slot, InvalidBuffer, true);
2614  }
2615 
2616  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2617  }
2618 
2619  skip_tuple = false;
2620 
2621  /* BEFORE ROW INSERT Triggers */
2622  if (resultRelInfo->ri_TrigDesc &&
2623  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2624  {
2625  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2626 
2627  if (slot == NULL) /* "do nothing" */
2628  skip_tuple = true;
2629  else /* trigger might have changed tuple */
2630  tuple = ExecMaterializeSlot(slot);
2631  }
2632 
2633  if (!skip_tuple)
2634  {
2635  if (resultRelInfo->ri_TrigDesc &&
2636  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2637  {
2638  /* Pass the data to the INSTEAD ROW INSERT trigger */
2639  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2640  }
2641  else
2642  {
2643  /*
2644  * We always check the partition constraint, including when
2645  * the tuple got here via tuple-routing. However we don't
2646  * need to in the latter case if no BR trigger is defined on
2647  * the partition. Note that a BR trigger might modify the
2648  * tuple such that the partition constraint is no longer
2649  * satisfied, so we need to check in that case.
2650  */
2651  bool check_partition_constr =
2652  (resultRelInfo->ri_PartitionCheck != NIL);
2653 
2654  if (saved_resultRelInfo != NULL &&
2655  !(resultRelInfo->ri_TrigDesc &&
2656  resultRelInfo->ri_TrigDesc->trig_insert_before_row))
2657  check_partition_constr = false;
2658 
2659  /* Check the constraints of the tuple */
2660  if (cstate->rel->rd_att->constr || check_partition_constr)
2661  ExecConstraints(resultRelInfo, slot, estate);
2662 
2663  if (useHeapMultiInsert)
2664  {
2665  /* Add this tuple to the tuple buffer */
2666  if (nBufferedTuples == 0)
2667  firstBufferedLineNo = cstate->cur_lineno;
2668  bufferedTuples[nBufferedTuples++] = tuple;
2669  bufferedTuplesSize += tuple->t_len;
2670 
2671  /*
2672  * If the buffer filled up, flush it. Also flush if the
2673  * total size of all the tuples in the buffer becomes
2674  * large, to avoid using large amounts of memory for the
2675  * buffer when the tuples are exceptionally wide.
2676  */
2677  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2678  bufferedTuplesSize > 65535)
2679  {
2680  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2681  resultRelInfo, myslot, bistate,
2682  nBufferedTuples, bufferedTuples,
2683  firstBufferedLineNo);
2684  nBufferedTuples = 0;
2685  bufferedTuplesSize = 0;
2686  }
2687  }
2688  else
2689  {
2690  List *recheckIndexes = NIL;
2691 
2692  /* OK, store the tuple and create index entries for it */
2693  heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
2694  hi_options, bistate);
2695 
2696  if (resultRelInfo->ri_NumIndices > 0)
2697  recheckIndexes = ExecInsertIndexTuples(slot,
2698  &(tuple->t_self),
2699  estate,
2700  false,
2701  NULL,
2702  NIL);
2703 
2704  /* AFTER ROW INSERT Triggers */
2705  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2706  recheckIndexes);
2707 
2708  list_free(recheckIndexes);
2709  }
2710  }
2711 
2712  /*
2713  * We count only tuples not suppressed by a BEFORE INSERT trigger;
2714  * this is the same definition used by execMain.c for counting
2715  * tuples inserted by an INSERT command.
2716  */
2717  processed++;
2718 
2719  if (saved_resultRelInfo)
2720  {
2721  resultRelInfo = saved_resultRelInfo;
2722  estate->es_result_relation_info = resultRelInfo;
2723  }
2724  }
2725  }
2726 
2727  /* Flush any remaining buffered tuples */
2728  if (nBufferedTuples > 0)
2729  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2730  resultRelInfo, myslot, bistate,
2731  nBufferedTuples, bufferedTuples,
2732  firstBufferedLineNo);
2733 
2734  /* Done, clean up */
2735  error_context_stack = errcallback.previous;
2736 
2737  FreeBulkInsertState(bistate);
2738 
2739  MemoryContextSwitchTo(oldcontext);
2740 
2741  /*
2742  * In the old protocol, tell pqcomm that we can process normal protocol
2743  * messages again.
2744  */
2745  if (cstate->copy_dest == COPY_OLD_FE)
2746  pq_endmsgread();
2747 
2748  /* Execute AFTER STATEMENT insertion triggers */
2749  ExecASInsertTriggers(estate, resultRelInfo);
2750 
2751  /* Handle queued AFTER triggers */
2752  AfterTriggerEndQuery(estate);
2753 
2754  pfree(values);
2755  pfree(nulls);
2756 
2757  ExecResetTupleTable(estate->es_tupleTable, false);
2758 
2759  ExecCloseIndices(resultRelInfo);
2760 
2761  /* Close all the partitioned tables, leaf partitions, and their indices */
2762  if (cstate->partition_dispatch_info)
2763  {
2764  int i;
2765 
2766  /*
2767  * Remember cstate->partition_dispatch_info[0] corresponds to the root
2768  * partitioned table, which we must not try to close, because it is
2769  * the main target table of COPY that will be closed eventually by
2770  * DoCopy(). Also, tupslot is NULL for the root partitioned table.
2771  */
2772  for (i = 1; i < cstate->num_dispatch; i++)
2773  {
2774  PartitionDispatch pd = cstate->partition_dispatch_info[i];
2775 
2776  heap_close(pd->reldesc, NoLock);
2777  ExecDropSingleTupleTableSlot(pd->tupslot);
2778  }
2779  for (i = 0; i < cstate->num_partitions; i++)
2780  {
2781  ResultRelInfo *resultRelInfo = cstate->partitions + i;
2782 
2783  ExecCloseIndices(resultRelInfo);
2784  heap_close(resultRelInfo->ri_RelationDesc, NoLock);
2785  }
2786 
2787  /* Release the standalone partition tuple descriptor */
2788  ExecDropSingleTupleTableSlot(cstate->partition_tuple_slot);
2789  }
2790 
2791  /* Close any trigger target relations */
2792  ExecCleanUpTriggerState(estate);
2793 
2794  FreeExecutorState(estate);
2795 
2796  /*
2797  * If we skipped writing WAL, then we need to sync the heap (but not
2798  * indexes since those use WAL anyway)
2799  */
2800  if (hi_options & HEAP_INSERT_SKIP_WAL)
2801  heap_sync(cstate->rel);
2802 
2803  return processed;
2804 }
2805 
2806 /*
2807  * A subroutine of CopyFrom, to write the current batch of buffered heap
2808  * tuples to the heap. Also updates indexes and runs AFTER ROW INSERT
2809  * triggers.
2810  */
2811 static void
2812 CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid,
2813  int hi_options, ResultRelInfo *resultRelInfo,
2814  TupleTableSlot *myslot, BulkInsertState bistate,
2815  int nBufferedTuples, HeapTuple *bufferedTuples,
2816  int firstBufferedLineNo)
2817 {
2818  MemoryContext oldcontext;
2819  int i;
2820  int save_cur_lineno;
2821 
2822  /*
2823  * Print error context information correctly, if one of the operations
2824  * below fail.
2825  */
2826  cstate->line_buf_valid = false;
2827  save_cur_lineno = cstate->cur_lineno;
2828 
2829  /*
2830  * heap_multi_insert leaks memory, so switch to short-lived memory context
2831  * before calling it.
2832  */
2833  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2834  heap_multi_insert(cstate->rel,
2835  bufferedTuples,
2836  nBufferedTuples,
2837  mycid,
2838  hi_options,
2839  bistate);
2840  MemoryContextSwitchTo(oldcontext);
2841 
2842  /*
2843  * If there are any indexes, update them for all the inserted tuples, and
2844  * run AFTER ROW INSERT triggers.
2845  */
2846  if (resultRelInfo->ri_NumIndices > 0)
2847  {
2848  for (i = 0; i < nBufferedTuples; i++)
2849  {
2850  List *recheckIndexes;
2851 
2852  cstate->cur_lineno = firstBufferedLineNo + i;
2853  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2854  recheckIndexes =
2855  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2856  estate, false, NULL, NIL);
2857  ExecARInsertTriggers(estate, resultRelInfo,
2858  bufferedTuples[i],
2859  recheckIndexes);
2860  list_free(recheckIndexes);
2861  }
2862  }
2863 
2864  /*
2865  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2866  * anyway.
2867  */
2868  else if (resultRelInfo->ri_TrigDesc != NULL &&
2869  resultRelInfo->ri_TrigDesc->trig_insert_after_row)
2870  {
2871  for (i = 0; i < nBufferedTuples; i++)
2872  {
2873  cstate->cur_lineno = firstBufferedLineNo + i;
2874  ExecARInsertTriggers(estate, resultRelInfo,
2875  bufferedTuples[i],
2876  NIL);
2877  }
2878  }
2879 
2880  /* reset cur_lineno to where we were */
2881  cstate->cur_lineno = save_cur_lineno;
2882 }
2883 
2884 /*
2885  * Setup to read tuples from a file for COPY FROM.
2886  *
2887  * 'rel': Used as a template for the tuples
2888  * 'filename': Name of server-local file to read
2889  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
2890  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
2891  *
2892  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
2893  */
2894 CopyState
2895 BeginCopyFrom(ParseState *pstate,
2896  Relation rel,
2897  const char *filename,
2898  bool is_program,
2899  copy_data_source_cb data_source_cb,
2900  List *attnamelist,
2901  List *options)
2902 {
2903  CopyState cstate;
2904  bool pipe = (filename == NULL);
2905  TupleDesc tupDesc;
2906  Form_pg_attribute *attr;
2907  AttrNumber num_phys_attrs,
2908  num_defaults;
2909  FmgrInfo *in_functions;
2910  Oid *typioparams;
2911  int attnum;
2912  Oid in_func_oid;
2913  int *defmap;
2914  ExprState **defexprs;
2915  MemoryContext oldcontext;
2916  bool volatile_defexprs;
2917 
2918  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
2919  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
2920 
2921  /* Initialize state variables */
2922  cstate->fe_eof = false;
2923  cstate->eol_type = EOL_UNKNOWN;
2924  cstate->cur_relname = RelationGetRelationName(cstate->rel);
2925  cstate->cur_lineno = 0;
2926  cstate->cur_attname = NULL;
2927  cstate->cur_attval = NULL;
2928 
2929  /* Set up variables to avoid per-attribute overhead. */
2930  initStringInfo(&cstate->attribute_buf);
2931  initStringInfo(&cstate->line_buf);
2932  cstate->line_buf_converted = false;
2933  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
2934  cstate->raw_buf_index = cstate->raw_buf_len = 0;
2935 
2936  /* Assign range table, we'll need it in CopyFrom. */
2937  if (pstate)
2938  cstate->range_table = pstate->p_rtable;
2939 
2940  tupDesc = RelationGetDescr(cstate->rel);
2941  attr = tupDesc->attrs;
2942  num_phys_attrs = tupDesc->natts;
2943  num_defaults = 0;
2944  volatile_defexprs = false;
2945 
2946  /*
2947  * Pick up the required catalog information for each attribute in the
2948  * relation, including the input function, the element type (to pass to
2949  * the input function), and info about defaults and constraints. (Which
2950  * input function we use depends on text/binary format choice.)
2951  */
2952  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2953  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
2954  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
2955  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
2956 
2957  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
2958  {
2959  /* We don't need info for dropped attributes */
2960  if (attr[attnum - 1]->attisdropped)
2961  continue;
2962 
2963  /* Fetch the input function and typioparam info */
2964  if (cstate->binary)
2965  getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
2966  &in_func_oid, &typioparams[attnum - 1]);
2967  else
2968  getTypeInputInfo(attr[attnum - 1]->atttypid,
2969  &in_func_oid, &typioparams[attnum - 1]);
2970  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
2971 
2972  /* Get default info if needed */
2973  if (!list_member_int(cstate->attnumlist, attnum))
2974  {
2975  /* attribute is NOT to be copied from input */
2976  /* use default value if one exists */
2977  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
2978  attnum);
2979 
2980  if (defexpr != NULL)
2981  {
2982  /* Run the expression through planner */
2983  defexpr = expression_planner(defexpr);
2984 
2985  /* Initialize executable expression in copycontext */
2986  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
2987  defmap[num_defaults] = attnum - 1;
2988  num_defaults++;
2989 
2990  /*
2991  * If a default expression looks at the table being loaded,
2992  * then it could give the wrong answer when using
2993  * multi-insert. Since database access can be dynamic this is
2994  * hard to test for exactly, so we use the much wider test of
2995  * whether the default expression is volatile. We allow for
2996  * the special case of when the default expression is the
2997  * nextval() of a sequence which in this specific case is
2998  * known to be safe for use with the multi-insert
2999  * optimization. Hence we use this special case function
3000  * checker rather than the standard check for
3001  * contain_volatile_functions().
3002  */
3003  if (!volatile_defexprs)
3004  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3005  }
3006  }
3007  }
3008 
3009  /* We keep those variables in cstate. */
3010  cstate->in_functions = in_functions;
3011  cstate->typioparams = typioparams;
3012  cstate->defmap = defmap;
3013  cstate->defexprs = defexprs;
3014  cstate->volatile_defexprs = volatile_defexprs;
3015  cstate->num_defaults = num_defaults;
3016  cstate->is_program = is_program;
3017 
3018  if (data_source_cb)
3019  {
3020  cstate->copy_dest = COPY_CALLBACK;
3021  cstate->data_source_cb = data_source_cb;
3022  }
3023  else if (pipe)
3024  {
3025  Assert(!is_program); /* the grammar does not allow this */
3026  if (whereToSendOutput == DestRemote)
3027  ReceiveCopyBegin(cstate);
3028  else
3029  cstate->copy_file = stdin;
3030  }
3031  else
3032  {
3033  cstate->filename = pstrdup(filename);
3034 
3035  if (cstate->is_program)
3036  {
3037  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3038  if (cstate->copy_file == NULL)
3039  ereport(ERROR,
3040  (errcode_for_file_access(),
3041  errmsg("could not execute command \"%s\": %m",
3042  cstate->filename)));
3043  }
3044  else
3045  {
3046  struct stat st;
3047 
3048  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3049  if (cstate->copy_file == NULL)
3050  {
3051  /* copy errno because ereport subfunctions might change it */
3052  int save_errno = errno;
3053 
3054  ereport(ERROR,
3055  (errcode_for_file_access(),
3056  errmsg("could not open file \"%s\" for reading: %m",
3057  cstate->filename),
3058  (save_errno == ENOENT || save_errno == EACCES) ?
3059  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3060  "You may want a client-side facility such as psql's \\copy.") : 0));
3061  }
3062 
3063  if (fstat(fileno(cstate->copy_file), &st))
3064  ereport(ERROR,
3065  (errcode_for_file_access(),
3066  errmsg("could not stat file \"%s\": %m",
3067  cstate->filename)));
3068 
3069  if (S_ISDIR(st.st_mode))
3070  ereport(ERROR,
3071  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3072  errmsg("\"%s\" is a directory", cstate->filename)));
3073  }
3074  }
3075 
3076  if (!cstate->binary)
3077  {
3078  /* must rely on user to tell us... */
3079  cstate->file_has_oids = cstate->oids;
3080  }
3081  else
3082  {
3083  /* Read and verify binary header */
3084  char readSig[11];
3085  int32 tmp;
3086 
3087  /* Signature */
3088  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3089  memcmp(readSig, BinarySignature, 11) != 0)
3090  ereport(ERROR,
3091  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3092  errmsg("COPY file signature not recognized")));
3093  /* Flags field */
3094  if (!CopyGetInt32(cstate, &tmp))
3095  ereport(ERROR,
3096  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3097  errmsg("invalid COPY file header (missing flags)")));
3098  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3099  tmp &= ~(1 << 16);
3100  if ((tmp >> 16) != 0)
3101  ereport(ERROR,
3102  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3103  errmsg("unrecognized critical flags in COPY file header")));
3104  /* Header extension length */
3105  if (!CopyGetInt32(cstate, &tmp) ||
3106  tmp < 0)
3107  ereport(ERROR,
3108  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3109  errmsg("invalid COPY file header (missing length)")));
3110  /* Skip extension header, if present */
3111  while (tmp-- > 0)
3112  {
3113  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3114  ereport(ERROR,
3115  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3116  errmsg("invalid COPY file header (wrong length)")));
3117  }
3118  }
3119 
3120  if (cstate->file_has_oids && cstate->binary)
3121  {
3122  getTypeBinaryInputInfo(OIDOID,
3123  &in_func_oid, &cstate->oid_typioparam);
3124  fmgr_info(in_func_oid, &cstate->oid_in_function);
3125  }
3126 
3127  /* create workspace for CopyReadAttributes results */
3128  if (!cstate->binary)
3129  {
3130  AttrNumber attr_count = list_length(cstate->attnumlist);
3131  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3132 
3133  cstate->max_fields = nfields;
3134  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3135  }
3136 
3137  MemoryContextSwitchTo(oldcontext);
3138 
3139  return cstate;
3140 }
3141 
3142 /*
3143  * Read raw fields in the next line for COPY FROM in text or csv mode.
3144  * Return false if no more lines.
3145  *
3146  * An internal temporary buffer is returned via 'fields'. It is valid until
3147  * the next call of the function. Since the function returns all raw fields
3148  * in the input file, 'nfields' could be different from the number of columns
3149  * in the relation.
3150  *
3151  * NOTE: force_not_null option are not applied to the returned fields.
3152  */
3153 bool
3154 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3155 {
3156  int fldct;
3157  bool done;
3158 
3159  /* only available for text or csv input */
3160  Assert(!cstate->binary);
3161 
3162  /* on input just throw the header line away */
3163  if (cstate->cur_lineno == 0 && cstate->header_line)
3164  {
3165  cstate->cur_lineno++;
3166  if (CopyReadLine(cstate))
3167  return false; /* done */
3168  }
3169 
3170  cstate->cur_lineno++;
3171 
3172  /* Actually read the line into memory here */
3173  done = CopyReadLine(cstate);
3174 
3175  /*
3176  * EOF at start of line means we're done. If we see EOF after some
3177  * characters, we act as though it was newline followed by EOF, ie,
3178  * process the line and then exit loop on next iteration.
3179  */
3180  if (done && cstate->line_buf.len == 0)
3181  return false;
3182 
3183  /* Parse the line into de-escaped field values */
3184  if (cstate->csv_mode)
3185  fldct = CopyReadAttributesCSV(cstate);
3186  else
3187  fldct = CopyReadAttributesText(cstate);
3188 
3189  *fields = cstate->raw_fields;
3190  *nfields = fldct;
3191  return true;
3192 }
3193 
3194 /*
3195  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3196  *
3197  * 'econtext' is used to evaluate default expression for each columns not
3198  * read from the file. It can be NULL when no default values are used, i.e.
3199  * when all columns are read from the file.
3200  *
3201  * 'values' and 'nulls' arrays must be the same length as columns of the
3202  * relation passed to BeginCopyFrom. This function fills the arrays.
3203  * Oid of the tuple is returned with 'tupleOid' separately.
3204  */
3205 bool
3206 NextCopyFrom(CopyState cstate, ExprContext *econtext,
3207  Datum *values, bool *nulls, Oid *tupleOid)
3208 {
3209  TupleDesc tupDesc;
3210  Form_pg_attribute *attr;
3211  AttrNumber num_phys_attrs,
3212  attr_count,
3213  num_defaults = cstate->num_defaults;
3214  FmgrInfo *in_functions = cstate->in_functions;
3215  Oid *typioparams = cstate->typioparams;
3216  int i;
3217  int nfields;
3218  bool isnull;
3219  bool file_has_oids = cstate->file_has_oids;
3220  int *defmap = cstate->defmap;
3221  ExprState **defexprs = cstate->defexprs;
3222 
3223  tupDesc = RelationGetDescr(cstate->rel);
3224  attr = tupDesc->attrs;
3225  num_phys_attrs = tupDesc->natts;
3226  attr_count = list_length(cstate->attnumlist);
3227  nfields = file_has_oids ? (attr_count + 1) : attr_count;
3228 
3229  /* Initialize all values for row to NULL */
3230  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3231  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3232 
3233  if (!cstate->binary)
3234  {
3235  char **field_strings;
3236  ListCell *cur;
3237  int fldct;
3238  int fieldno;
3239  char *string;
3240 
3241  /* read raw fields in the next line */
3242  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3243  return false;
3244 
3245  /* check for overflowing fields */
3246  if (nfields > 0 && fldct > nfields)
3247  ereport(ERROR,
3248  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3249  errmsg("extra data after last expected column")));
3250 
3251  fieldno = 0;
3252 
3253  /* Read the OID field if present */
3254  if (file_has_oids)
3255  {
3256  if (fieldno >= fldct)
3257  ereport(ERROR,
3258  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3259  errmsg("missing data for OID column")));
3260  string = field_strings[fieldno++];
3261 
3262  if (string == NULL)
3263  ereport(ERROR,
3264  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3265  errmsg("null OID in COPY data")));
3266  else if (cstate->oids && tupleOid != NULL)
3267  {
3268  cstate->cur_attname = "oid";
3269  cstate->cur_attval = string;
3270  *tupleOid = DatumGetObjectId(DirectFunctionCall1(oidin,
3271  CStringGetDatum(string)));
3272  if (*tupleOid == InvalidOid)
3273  ereport(ERROR,
3274  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3275  errmsg("invalid OID in COPY data")));
3276  cstate->cur_attname = NULL;
3277  cstate->cur_attval = NULL;
3278  }
3279  }
3280 
3281  /* Loop to read the user attributes on the line. */
3282  foreach(cur, cstate->attnumlist)
3283  {
3284  int attnum = lfirst_int(cur);
3285  int m = attnum - 1;
3286 
3287  if (fieldno >= fldct)
3288  ereport(ERROR,
3289  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3290  errmsg("missing data for column \"%s\"",
3291  NameStr(attr[m]->attname))));
3292  string = field_strings[fieldno++];
3293 
3294  if (cstate->convert_select_flags &&
3295  !cstate->convert_select_flags[m])
3296  {
3297  /* ignore input field, leaving column as NULL */
3298  continue;
3299  }
3300 
3301  if (cstate->csv_mode)
3302  {
3303  if (string == NULL &&
3304  cstate->force_notnull_flags[m])
3305  {
3306  /*
3307  * FORCE_NOT_NULL option is set and column is NULL -
3308  * convert it to the NULL string.
3309  */
3310  string = cstate->null_print;
3311  }
3312  else if (string != NULL && cstate->force_null_flags[m]
3313  && strcmp(string, cstate->null_print) == 0)
3314  {
3315  /*
3316  * FORCE_NULL option is set and column matches the NULL
3317  * string. It must have been quoted, or otherwise the
3318  * string would already have been set to NULL. Convert it
3319  * to NULL as specified.
3320  */
3321  string = NULL;
3322  }
3323  }
3324 
3325  cstate->cur_attname = NameStr(attr[m]->attname);
3326  cstate->cur_attval = string;
3327  values[m] = InputFunctionCall(&in_functions[m],
3328  string,
3329  typioparams[m],
3330  attr[m]->atttypmod);
3331  if (string != NULL)
3332  nulls[m] = false;
3333  cstate->cur_attname = NULL;
3334  cstate->cur_attval = NULL;
3335  }
3336 
3337  Assert(fieldno == nfields);
3338  }
3339  else
3340  {
3341  /* binary */
3342  int16 fld_count;
3343  ListCell *cur;
3344 
3345  cstate->cur_lineno++;
3346 
3347  if (!CopyGetInt16(cstate, &fld_count))
3348  {
3349  /* EOF detected (end of file, or protocol-level EOF) */
3350  return false;
3351  }
3352 
3353  if (fld_count == -1)
3354  {
3355  /*
3356  * Received EOF marker. In a V3-protocol copy, wait for the
3357  * protocol-level EOF, and complain if it doesn't come
3358  * immediately. This ensures that we correctly handle CopyFail,
3359  * if client chooses to send that now.
3360  *
3361  * Note that we MUST NOT try to read more data in an old-protocol
3362  * copy, since there is no protocol-level EOF marker then. We
3363  * could go either way for copy from file, but choose to throw
3364  * error if there's data after the EOF marker, for consistency
3365  * with the new-protocol case.
3366  */
3367  char dummy;
3368 
3369  if (cstate->copy_dest != COPY_OLD_FE &&
3370  CopyGetData(cstate, &dummy, 1, 1) > 0)
3371  ereport(ERROR,
3372  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3373  errmsg("received copy data after EOF marker")));
3374  return false;
3375  }
3376 
3377  if (fld_count != attr_count)
3378  ereport(ERROR,
3379  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3380  errmsg("row field count is %d, expected %d",
3381  (int) fld_count, attr_count)));
3382 
3383  if (file_has_oids)
3384  {
3385  Oid loaded_oid;
3386 
3387  cstate->cur_attname = "oid";
3388  loaded_oid =
3389  DatumGetObjectId(CopyReadBinaryAttribute(cstate,
3390  0,
3391  &cstate->oid_in_function,
3392  cstate->oid_typioparam,
3393  -1,
3394  &isnull));
3395  if (isnull || loaded_oid == InvalidOid)
3396  ereport(ERROR,
3397  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3398  errmsg("invalid OID in COPY data")));
3399  cstate->cur_attname = NULL;
3400  if (cstate->oids && tupleOid != NULL)
3401  *tupleOid = loaded_oid;
3402  }
3403 
3404  i = 0;
3405  foreach(cur, cstate->attnumlist)
3406  {
3407  int attnum = lfirst_int(cur);
3408  int m = attnum - 1;
3409 
3410  cstate->cur_attname = NameStr(attr[m]->attname);
3411  i++;
3412  values[m] = CopyReadBinaryAttribute(cstate,
3413  i,
3414  &in_functions[m],
3415  typioparams[m],
3416  attr[m]->atttypmod,
3417  &nulls[m]);
3418  cstate->cur_attname = NULL;
3419  }
3420  }
3421 
3422  /*
3423  * Now compute and insert any defaults available for the columns not
3424  * provided by the input data. Anything not processed here or above will
3425  * remain NULL.
3426  */
3427  for (i = 0; i < num_defaults; i++)
3428  {
3429  /*
3430  * The caller must supply econtext and have switched into the
3431  * per-tuple memory context in it.
3432  */
3433  Assert(econtext != NULL);
3434  Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
3435 
3436  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3437  &nulls[defmap[i]]);
3438  }
3439 
3440  return true;
3441 }
3442 
3443 /*
3444  * Clean up storage and release resources for COPY FROM.
3445  */
3446 void
3447 EndCopyFrom(CopyState cstate)
3448 {
3449  /* No COPY FROM related resources except memory. */
3450 
3451  EndCopy(cstate);
3452 }
3453 
3454 /*
3455  * Read the next input line and stash it in line_buf, with conversion to
3456  * server encoding.
3457  *
3458  * Result is true if read was terminated by EOF, false if terminated
3459  * by newline. The terminating newline or EOF marker is not included
3460  * in the final value of line_buf.
3461  */
3462 static bool
3463 CopyReadLine(CopyState cstate)
3464 {
3465  bool result;
3466 
3467  resetStringInfo(&cstate->line_buf);
3468  cstate->line_buf_valid = true;
3469 
3470  /* Mark that encoding conversion hasn't occurred yet */
3471  cstate->line_buf_converted = false;
3472 
3473  /* Parse data and transfer into line_buf */
3474  result = CopyReadLineText(cstate);
3475 
3476  if (result)
3477  {
3478  /*
3479  * Reached EOF. In protocol version 3, we should ignore anything
3480  * after \. up to the protocol end of copy data. (XXX maybe better
3481  * not to treat \. as special?)
3482  */
3483  if (cstate->copy_dest == COPY_NEW_FE)
3484  {
3485  do
3486  {
3487  cstate->raw_buf_index = cstate->raw_buf_len;
3488  } while (CopyLoadRawBuf(cstate));
3489  }
3490  }
3491  else
3492  {
3493  /*
3494  * If we didn't hit EOF, then we must have transferred the EOL marker
3495  * to line_buf along with the data. Get rid of it.
3496  */
3497  switch (cstate->eol_type)
3498  {
3499  case EOL_NL:
3500  Assert(cstate->line_buf.len >= 1);
3501  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3502  cstate->line_buf.len--;
3503  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3504  break;
3505  case EOL_CR:
3506  Assert(cstate->line_buf.len >= 1);
3507  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3508  cstate->line_buf.len--;
3509  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3510  break;
3511  case EOL_CRNL:
3512  Assert(cstate->line_buf.len >= 2);
3513  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3514  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3515  cstate->line_buf.len -= 2;
3516  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3517  break;
3518  case EOL_UNKNOWN:
3519  /* shouldn't get here */
3520  Assert(false);
3521  break;
3522  }
3523  }
3524 
3525  /* Done reading the line. Convert it to server encoding. */
3526  if (cstate->need_transcoding)
3527  {
3528  char *cvt;
3529 
3530  cvt = pg_any_to_server(cstate->line_buf.data,
3531  cstate->line_buf.len,
3532  cstate->file_encoding);
3533  if (cvt != cstate->line_buf.data)
3534  {
3535  /* transfer converted data back to line_buf */
3536  resetStringInfo(&cstate->line_buf);
3537  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3538  pfree(cvt);
3539  }
3540  }
3541 
3542  /* Now it's safe to use the buffer in error messages */
3543  cstate->line_buf_converted = true;
3544 
3545  return result;
3546 }
3547 
3548 /*
3549  * CopyReadLineText - inner loop of CopyReadLine for text mode
3550  */
3551 static bool
3552 CopyReadLineText(CopyState cstate)
3553 {
3554  char *copy_raw_buf;
3555  int raw_buf_ptr;
3556  int copy_buf_len;
3557  bool need_data = false;
3558  bool hit_eof = false;
3559  bool result = false;
3560  char mblen_str[2];
3561 
3562  /* CSV variables */
3563  bool first_char_in_line = true;
3564  bool in_quote = false,
3565  last_was_esc = false;
3566  char quotec = '\0';
3567  char escapec = '\0';
3568 
3569  if (cstate->csv_mode)
3570  {
3571  quotec = cstate->quote[0];
3572  escapec = cstate->escape[0];
3573  /* ignore special escape processing if it's the same as quotec */
3574  if (quotec == escapec)
3575  escapec = '\0';
3576  }
3577 
3578  mblen_str[1] = '\0';
3579 
3580  /*
3581  * The objective of this loop is to transfer the entire next input line
3582  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3583  * \n) and the end-of-copy marker (\.).
3584  *
3585  * In CSV mode, \r and \n inside a quoted field are just part of the data
3586  * value and are put in line_buf. We keep just enough state to know if we
3587  * are currently in a quoted field or not.
3588  *
3589  * These four characters, and the CSV escape and quote characters, are
3590  * assumed the same in frontend and backend encodings.
3591  *
3592  * For speed, we try to move data from raw_buf to line_buf in chunks
3593  * rather than one character at a time. raw_buf_ptr points to the next
3594  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3595  * have been determined to be part of the line, but not yet transferred to
3596  * line_buf.
3597  *
3598  * For a little extra speed within the loop, we copy raw_buf and
3599  * raw_buf_len into local variables.
3600  */
3601  copy_raw_buf = cstate->raw_buf;
3602  raw_buf_ptr = cstate->raw_buf_index;
3603  copy_buf_len = cstate->raw_buf_len;
3604 
3605  for (;;)
3606  {
3607  int prev_raw_ptr;
3608  char c;
3609 
3610  /*
3611  * Load more data if needed. Ideally we would just force four bytes
3612  * of read-ahead and avoid the many calls to
3613  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3614  * does not allow us to read too far ahead or we might read into the
3615  * next data, so we read-ahead only as far we know we can. One
3616  * optimization would be to read-ahead four byte here if
3617  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3618  * considering the size of the buffer.
3619  */
3620  if (raw_buf_ptr >= copy_buf_len || need_data)
3621  {
3622  REFILL_LINEBUF;
3623 
3624  /*
3625  * Try to read some more data. This will certainly reset
3626  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3627  */
3628  if (!CopyLoadRawBuf(cstate))
3629  hit_eof = true;
3630  raw_buf_ptr = 0;
3631  copy_buf_len = cstate->raw_buf_len;
3632 
3633  /*
3634  * If we are completely out of data, break out of the loop,
3635  * reporting EOF.
3636  */
3637  if (copy_buf_len <= 0)
3638  {
3639  result = true;
3640  break;
3641  }
3642  need_data = false;
3643  }
3644 
3645  /* OK to fetch a character */
3646  prev_raw_ptr = raw_buf_ptr;
3647  c = copy_raw_buf[raw_buf_ptr++];
3648 
3649  if (cstate->csv_mode)
3650  {
3651  /*
3652  * If character is '\\' or '\r', we may need to look ahead below.
3653  * Force fetch of the next character if we don't already have it.
3654  * We need to do this before changing CSV state, in case one of
3655  * these characters is also the quote or escape character.
3656  *
3657  * Note: old-protocol does not like forced prefetch, but it's OK
3658  * here since we cannot validly be at EOF.
3659  */
3660  if (c == '\\' || c == '\r')
3661  {
3662  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3663  }
3664 
3665  /*
3666  * Dealing with quotes and escapes here is mildly tricky. If the
3667  * quote char is also the escape char, there's no problem - we
3668  * just use the char as a toggle. If they are different, we need
3669  * to ensure that we only take account of an escape inside a
3670  * quoted field and immediately preceding a quote char, and not
3671  * the second in an escape-escape sequence.
3672  */
3673  if (in_quote && c == escapec)
3674  last_was_esc = !last_was_esc;
3675  if (c == quotec && !last_was_esc)
3676  in_quote = !in_quote;
3677  if (c != escapec)
3678  last_was_esc = false;
3679 
3680  /*
3681  * Updating the line count for embedded CR and/or LF chars is
3682  * necessarily a little fragile - this test is probably about the
3683  * best we can do. (XXX it's arguable whether we should do this
3684  * at all --- is cur_lineno a physical or logical count?)
3685  */
3686  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3687  cstate->cur_lineno++;
3688  }
3689 
3690  /* Process \r */
3691  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3692  {
3693  /* Check for \r\n on first line, _and_ handle \r\n. */
3694  if (cstate->eol_type == EOL_UNKNOWN ||
3695  cstate->eol_type == EOL_CRNL)
3696  {
3697  /*
3698  * If need more data, go back to loop top to load it.
3699  *
3700  * Note that if we are at EOF, c will wind up as '\0' because
3701  * of the guaranteed pad of raw_buf.
3702  */
3703  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3704 
3705  /* get next char */
3706  c = copy_raw_buf[raw_buf_ptr];
3707 
3708  if (c == '\n')
3709  {
3710  raw_buf_ptr++; /* eat newline */
3711  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3712  }
3713  else
3714  {
3715  /* found \r, but no \n */
3716  if (cstate->eol_type == EOL_CRNL)
3717  ereport(ERROR,
3718  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3719  !cstate->csv_mode ?
3720  errmsg("literal carriage return found in data") :
3721  errmsg("unquoted carriage return found in data"),
3722  !cstate->csv_mode ?
3723  errhint("Use \"\\r\" to represent carriage return.") :
3724  errhint("Use quoted CSV field to represent carriage return.")));
3725 
3726  /*
3727  * if we got here, it is the first line and we didn't find
3728  * \n, so don't consume the peeked character
3729  */
3730  cstate->eol_type = EOL_CR;
3731  }
3732  }
3733  else if (cstate->eol_type == EOL_NL)
3734  ereport(ERROR,
3735  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3736  !cstate->csv_mode ?
3737  errmsg("literal carriage return found in data") :
3738  errmsg("unquoted carriage return found in data"),
3739  !cstate->csv_mode ?
3740  errhint("Use \"\\r\" to represent carriage return.") :
3741  errhint("Use quoted CSV field to represent carriage return.")));
3742  /* If reach here, we have found the line terminator */
3743  break;
3744  }
3745 
3746  /* Process \n */
3747  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3748  {
3749  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3750  ereport(ERROR,
3751  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3752  !cstate->csv_mode ?
3753  errmsg("literal newline found in data") :
3754  errmsg("unquoted newline found in data"),
3755  !cstate->csv_mode ?
3756  errhint("Use \"\\n\" to represent newline.") :
3757  errhint("Use quoted CSV field to represent newline.")));
3758  cstate->eol_type = EOL_NL; /* in case not set yet */
3759  /* If reach here, we have found the line terminator */
3760  break;
3761  }
3762 
3763  /*
3764  * In CSV mode, we only recognize \. alone on a line. This is because
3765  * \. is a valid CSV data value.
3766  */
3767  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3768  {
3769  char c2;
3770 
3771  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3772  IF_NEED_REFILL_AND_EOF_BREAK(0);
3773 
3774  /* -----
3775  * get next character
3776  * Note: we do not change c so if it isn't \., we can fall
3777  * through and continue processing for file encoding.
3778  * -----
3779  */
3780  c2 = copy_raw_buf[raw_buf_ptr];
3781 
3782  if (c2 == '.')
3783  {
3784  raw_buf_ptr++; /* consume the '.' */
3785 
3786  /*
3787  * Note: if we loop back for more data here, it does not
3788  * matter that the CSV state change checks are re-executed; we
3789  * will come back here with no important state changed.
3790  */
3791  if (cstate->eol_type == EOL_CRNL)
3792  {
3793  /* Get the next character */
3794  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3795  /* if hit_eof, c2 will become '\0' */
3796  c2 = copy_raw_buf[raw_buf_ptr++];
3797 
3798  if (c2 == '\n')
3799  {
3800  if (!cstate->csv_mode)
3801  ereport(ERROR,
3802  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3803  errmsg("end-of-copy marker does not match previous newline style")));
3804  else
3805  NO_END_OF_COPY_GOTO;
3806  }
3807  else if (c2 != '\r')
3808  {
3809  if (!cstate->csv_mode)
3810  ereport(ERROR,
3811  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3812  errmsg("end-of-copy marker corrupt")));
3813  else
3814  NO_END_OF_COPY_GOTO;
3815  }
3816  }
3817 
3818  /* Get the next character */
3819  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3820  /* if hit_eof, c2 will become '\0' */
3821  c2 = copy_raw_buf[raw_buf_ptr++];
3822 
3823  if (c2 != '\r' && c2 != '\n')
3824  {
3825  if (!cstate->csv_mode)
3826  ereport(ERROR,
3827  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3828  errmsg("end-of-copy marker corrupt")));
3829  else
3830  NO_END_OF_COPY_GOTO;
3831  }
3832 
3833  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3834  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3835  (cstate->eol_type == EOL_CR && c2 != '\r'))
3836  {
3837  ereport(ERROR,
3838  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3839  errmsg("end-of-copy marker does not match previous newline style")));
3840  }
3841 
3842  /*
3843  * Transfer only the data before the \. into line_buf, then
3844  * discard the data and the \. sequence.
3845  */
3846  if (prev_raw_ptr > cstate->raw_buf_index)
3847  appendBinaryStringInfo(&cstate->line_buf,
3848  cstate->raw_buf + cstate->raw_buf_index,
3849  prev_raw_ptr - cstate->raw_buf_index);
3850  cstate->raw_buf_index = raw_buf_ptr;
3851  result = true; /* report EOF */
3852  break;
3853  }
3854  else if (!cstate->csv_mode)
3855 
3856  /*
3857  * If we are here, it means we found a backslash followed by
3858  * something other than a period. In non-CSV mode, anything
3859  * after a backslash is special, so we skip over that second
3860  * character too. If we didn't do that \\. would be
3861  * considered an eof-of copy, while in non-CSV mode it is a
3862  * literal backslash followed by a period. In CSV mode,
3863  * backslashes are not special, so we want to process the
3864  * character after the backslash just like a normal character,
3865  * so we don't increment in those cases.
3866  */
3867  raw_buf_ptr++;
3868  }
3869 
3870  /*
3871  * This label is for CSV cases where \. appears at the start of a
3872  * line, but there is more text after it, meaning it was a data value.
3873  * We are more strict for \. in CSV mode because \. could be a data
3874  * value, while in non-CSV mode, \. cannot be a data value.
3875  */
3876 not_end_of_copy:
3877 
3878  /*
3879  * Process all bytes of a multi-byte character as a group.
3880  *
3881  * We only support multi-byte sequences where the first byte has the
3882  * high-bit set, so as an optimization we can avoid this block
3883  * entirely if it is not set.
3884  */
3885  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3886  {
3887  int mblen;
3888 
3889  mblen_str[0] = c;
3890  /* All our encodings only read the first byte to get the length */
3891  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3892  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
3893  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3894  raw_buf_ptr += mblen - 1;
3895  }
3896  first_char_in_line = false;
3897  } /* end of outer loop */
3898 
3899  /*
3900  * Transfer any still-uncopied data to line_buf.
3901  */
3902  REFILL_LINEBUF;
3903 
3904  return result;
3905 }
3906 
3907 /*
3908  * Return decimal value for a hexadecimal digit
3909  */
3910 static int
3911 GetDecimalFromHex(char hex)
3912 {
3913  if (isdigit((unsigned char) hex))
3914  return hex - '0';
3915  else
3916  return tolower((unsigned char) hex) - 'a' + 10;
3917 }
3918 
3919 /*
3920  * Parse the current line into separate attributes (fields),
3921  * performing de-escaping as needed.
3922  *
3923  * The input is in line_buf. We use attribute_buf to hold the result
3924  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
3925  * string, or NULL when the input matches the null marker string.
3926  * This array is expanded as necessary.
3927  *
3928  * (Note that the caller cannot check for nulls since the returned
3929  * string would be the post-de-escaping equivalent, which may look
3930  * the same as some valid data string.)
3931  *
3932  * delim is the column delimiter string (must be just one byte for now).
3933  * null_print is the null marker string. Note that this is compared to
3934  * the pre-de-escaped input string.
3935  *
3936  * The return value is the number of fields actually read.
3937  */
3938 static int
3939 CopyReadAttributesText(CopyState cstate)
3940 {
3941  char delimc = cstate->delim[0];
3942  int fieldno;
3943  char *output_ptr;
3944  char *cur_ptr;
3945  char *line_end_ptr;
3946 
3947  /*
3948  * We need a special case for zero-column tables: check that the input
3949  * line is empty, and return.
3950  */
3951  if (cstate->max_fields <= 0)
3952  {
3953  if (cstate->line_buf.len != 0)
3954  ereport(ERROR,
3955  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3956  errmsg("extra data after last expected column")));
3957  return 0;
3958  }
3959 
3960  resetStringInfo(&cstate->attribute_buf);
3961 
3962  /*
3963  * The de-escaped attributes will certainly not be longer than the input
3964  * data line, so we can just force attribute_buf to be large enough and
3965  * then transfer data without any checks for enough space. We need to do
3966  * it this way because enlarging attribute_buf mid-stream would invalidate
3967  * pointers already stored into cstate->raw_fields[].
3968  */
3969  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
3970  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
3971  output_ptr = cstate->attribute_buf.data;
3972 
3973  /* set pointer variables for loop */
3974  cur_ptr = cstate->line_buf.data;
3975  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
3976 
3977  /* Outer loop iterates over fields */
3978  fieldno = 0;
3979  for (;;)
3980  {
3981  bool found_delim = false;
3982  char *start_ptr;
3983  char *end_ptr;
3984  int input_len;
3985  bool saw_non_ascii = false;
3986 
3987  /* Make sure there is enough space for the next value */
3988  if (fieldno >= cstate->max_fields)
3989  {
3990  cstate->max_fields *= 2;
3991  cstate->raw_fields =
3992  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
3993  }
3994 
3995  /* Remember start of field on both input and output sides */
3996  start_ptr = cur_ptr;
3997  cstate->raw_fields[fieldno] = output_ptr;
3998 
3999  /*
4000  * Scan data for field.
4001  *
4002  * Note that in this loop, we are scanning to locate the end of field
4003  * and also speculatively performing de-escaping. Once we find the
4004  * end-of-field, we can match the raw field contents against the null
4005  * marker string. Only after that comparison fails do we know that
4006  * de-escaping is actually the right thing to do; therefore we *must
4007  * not* throw any syntax errors before we've done the null-marker
4008  * check.
4009  */
4010  for (;;)
4011  {
4012  char c;
4013 
4014  end_ptr = cur_ptr;
4015  if (cur_ptr >= line_end_ptr)
4016  break;
4017  c = *cur_ptr++;
4018  if (c == delimc)
4019  {
4020  found_delim = true;
4021  break;
4022  }
4023  if (c == '\\')
4024  {
4025  if (cur_ptr >= line_end_ptr)
4026  break;
4027  c = *cur_ptr++;
4028  switch (c)
4029  {
4030  case '0':
4031  case '1':
4032  case '2':
4033  case '3':
4034  case '4':
4035  case '5':
4036  case '6':
4037  case '7':
4038  {
4039  /* handle \013 */
4040  int val;
4041 
4042  val = OCTVALUE(c);
4043  if (cur_ptr < line_end_ptr)
4044  {
4045  c = *cur_ptr;
4046  if (ISOCTAL(c))
4047  {
4048  cur_ptr++;
4049  val = (val << 3) + OCTVALUE(c);
4050  if (cur_ptr < line_end_ptr)
4051  {
4052  c = *cur_ptr;
4053  if (ISOCTAL(c))
4054  {
4055  cur_ptr++;
4056  val = (val << 3) + OCTVALUE(c);
4057  }
4058  }
4059  }
4060  }
4061  c = val & 0377;
4062  if (c == '\0' || IS_HIGHBIT_SET(c))
4063  saw_non_ascii = true;
4064  }
4065  break;
4066  case 'x':
4067  /* Handle \x3F */
4068  if (cur_ptr < line_end_ptr)
4069  {
4070  char hexchar = *cur_ptr;
4071 
4072  if (isxdigit((unsigned char) hexchar))
4073  {
4074  int val = GetDecimalFromHex(hexchar);
4075 
4076  cur_ptr++;
4077  if (cur_ptr < line_end_ptr)
4078  {
4079  hexchar = *cur_ptr;
4080  if (isxdigit((unsigned char) hexchar))
4081  {
4082  cur_ptr++;
4083  val = (val << 4) + GetDecimalFromHex(hexchar);
4084  }
4085  }
4086  c = val & 0xff;
4087  if (c == '\0' || IS_HIGHBIT_SET(c))
4088  saw_non_ascii = true;
4089  }
4090  }
4091  break;
4092  case 'b':
4093  c = '\b';
4094  break;
4095  case 'f':
4096  c = '\f';
4097  break;
4098  case 'n':
4099  c = '\n';
4100  break;
4101  case 'r':
4102  c = '\r';
4103  break;
4104  case 't':
4105  c = '\t';
4106  break;
4107  case 'v':
4108  c = '\v';
4109  break;
4110 
4111  /*
4112  * in all other cases, take the char after '\'
4113  * literally
4114  */
4115  }
4116  }
4117 
4118  /* Add c to output string */
4119  *output_ptr++ = c;
4120  }
4121 
4122  /* Check whether raw input matched null marker */
4123  input_len = end_ptr - start_ptr;
4124  if (input_len == cstate->null_print_len &&
4125  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4126  cstate->raw_fields[fieldno] = NULL;
4127  else
4128  {
4129  /*
4130  * At this point we know the field is supposed to contain data.
4131  *
4132  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4133  * resulting string is valid data for the db encoding.
4134  */
4135  if (saw_non_ascii)
4136  {
4137  char *fld = cstate->raw_fields[fieldno];
4138 
4139  pg_verifymbstr(fld, output_ptr - fld, false);
4140  }
4141  }
4142 
4143  /* Terminate attribute value in output area */
4144  *output_ptr++ = '\0';
4145 
4146  fieldno++;
4147  /* Done if we hit EOL instead of a delim */
4148  if (!found_delim)
4149  break;
4150  }
4151 
4152  /* Clean up state of attribute_buf */
4153  output_ptr--;
4154  Assert(*output_ptr == '\0');
4155  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4156 
4157  return fieldno;
4158 }
4159 
4160 /*
4161  * Parse the current line into separate attributes (fields),
4162  * performing de-escaping as needed. This has exactly the same API as
4163  * CopyReadAttributesText, except we parse the fields according to
4164  * "standard" (i.e. common) CSV usage.
4165  */
4166 static int
4167 CopyReadAttributesCSV(CopyState cstate)
4168 {
4169  char delimc = cstate->delim[0];
4170  char quotec = cstate->quote[0];
4171  char escapec = cstate->escape[0];
4172  int fieldno;
4173  char *output_ptr;
4174  char *cur_ptr;
4175  char *line_end_ptr;
4176 
4177  /*
4178  * We need a special case for zero-column tables: check that the input
4179  * line is empty, and return.
4180  */
4181  if (cstate->max_fields <= 0)
4182  {
4183  if (cstate->line_buf.len != 0)
4184  ereport(ERROR,
4185  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4186  errmsg("extra data after last expected column")));
4187  return 0;
4188  }
4189 
4190  resetStringInfo(&cstate->attribute_buf);
4191 
4192  /*
4193  * The de-escaped attributes will certainly not be longer than the input
4194  * data line, so we can just force attribute_buf to be large enough and
4195  * then transfer data without any checks for enough space. We need to do
4196  * it this way because enlarging attribute_buf mid-stream would invalidate
4197  * pointers already stored into cstate->raw_fields[].
4198  */
4199  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4200  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4201  output_ptr = cstate->attribute_buf.data;
4202 
4203  /* set pointer variables for loop */
4204  cur_ptr = cstate->line_buf.data;
4205  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4206 
4207  /* Outer loop iterates over fields */
4208  fieldno = 0;
4209  for (;;)
4210  {
4211  bool found_delim = false;
4212  bool saw_quote = false;
4213  char *start_ptr;
4214  char *end_ptr;
4215  int input_len;
4216 
4217  /* Make sure there is enough space for the next value */
4218  if (fieldno >= cstate->max_fields)
4219  {
4220  cstate->max_fields *= 2;
4221  cstate->raw_fields =
4222  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4223  }
4224 
4225  /* Remember start of field on both input and output sides */
4226  start_ptr = cur_ptr;
4227  cstate->raw_fields[fieldno] = output_ptr;
4228 
4229  /*
4230  * Scan data for field,
4231  *
4232  * The loop starts in "not quote" mode and then toggles between that
4233  * and "in quote" mode. The loop exits normally if it is in "not
4234  * quote" mode and a delimiter or line end is seen.
4235  */
4236  for (;;)
4237  {
4238  char c;
4239 
4240  /* Not in quote */
4241  for (;;)
4242  {
4243  end_ptr = cur_ptr;
4244  if (cur_ptr >= line_end_ptr)
4245  goto endfield;
4246  c = *cur_ptr++;
4247  /* unquoted field delimiter */
4248  if (c == delimc)
4249  {
4250  found_delim = true;
4251  goto endfield;
4252  }
4253  /* start of quoted field (or part of field) */
4254  if (c == quotec)
4255  {
4256  saw_quote = true;
4257  break;
4258  }
4259  /* Add c to output string */
4260  *output_ptr++ = c;
4261  }
4262 
4263  /* In quote */
4264  for (;;)
4265  {
4266  end_ptr = cur_ptr;
4267  if (cur_ptr >= line_end_ptr)
4268  ereport(ERROR,
4269  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4270  errmsg("unterminated CSV quoted field")));
4271 
4272  c = *cur_ptr++;
4273 
4274  /* escape within a quoted field */
4275  if (c == escapec)
4276  {
4277  /*
4278  * peek at the next char if available, and escape it if it
4279  * is an escape char or a quote char
4280  */
4281  if (cur_ptr < line_end_ptr)
4282  {
4283  char nextc = *cur_ptr;
4284 
4285  if (nextc == escapec || nextc == quotec)
4286  {
4287  *output_ptr++ = nextc;
4288  cur_ptr++;
4289  continue;
4290  }
4291  }
4292  }
4293 
4294  /*
4295  * end of quoted field. Must do this test after testing for
4296  * escape in case quote char and escape char are the same
4297  * (which is the common case).
4298  */
4299  if (c == quotec)
4300  break;
4301 
4302  /* Add c to output string */
4303  *output_ptr++ = c;
4304  }
4305  }
4306 endfield:
4307 
4308  /* Terminate attribute value in output area */
4309  *output_ptr++ = '\0';
4310 
4311  /* Check whether raw input matched null marker */
4312  input_len = end_ptr - start_ptr;
4313  if (!saw_quote && input_len == cstate->null_print_len &&
4314  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4315  cstate->raw_fields[fieldno] = NULL;
4316 
4317  fieldno++;
4318  /* Done if we hit EOL instead of a delim */
4319  if (!found_delim)
4320  break;
4321  }
4322 
4323  /* Clean up state of attribute_buf */
4324  output_ptr--;
4325  Assert(*output_ptr == '\0');
4326  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4327 
4328  return fieldno;
4329 }
4330 
4331 
4332 /*
4333  * Read a binary attribute
4334  */
4335 static Datum
4336 CopyReadBinaryAttribute(CopyState cstate,
4337  int column_no, FmgrInfo *flinfo,
4338  Oid typioparam, int32 typmod,
4339  bool *isnull)
4340 {
4341  int32 fld_size;
4342  Datum result;
4343 
4344  if (!CopyGetInt32(cstate, &fld_size))
4345  ereport(ERROR,
4346  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4347  errmsg("unexpected EOF in COPY data")));
4348  if (fld_size == -1)
4349  {
4350  *isnull = true;
4351  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4352  }
4353  if (fld_size < 0)
4354  ereport(ERROR,
4355  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4356  errmsg("invalid field size")));
4357 
4358  /* reset attribute_buf to empty, and load raw data in it */
4359  resetStringInfo(&cstate->attribute_buf);
4360 
4361  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4362  if (CopyGetData(cstate, cstate->attribute_buf.data,
4363  fld_size, fld_size) != fld_size)
4364  ereport(ERROR,
4365  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4366  errmsg("unexpected EOF in COPY data")));
4367 
4368  cstate->attribute_buf.len = fld_size;
4369  cstate->attribute_buf.data[fld_size] = '\0';
4370 
4371  /* Call the column type's binary input converter */
4372  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4373  typioparam, typmod);
4374 
4375  /* Trouble if it didn't eat the whole buffer */
4376  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4377  ereport(ERROR,
4378  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4379  errmsg("incorrect binary data format")));
4380 
4381  *isnull = false;
4382  return result;
4383 }
4384 
4385 /*
4386  * Send text representation of one attribute, with conversion and escaping
4387  */
4388 #define DUMPSOFAR() \
4389  do { \
4390  if (ptr > start) \
4391  CopySendData(cstate, start, ptr - start); \
4392  } while (0)
4393 
4394 static void
4395 CopyAttributeOutText(CopyState cstate, char *string)
4396 {
4397  char *ptr;
4398  char *start;
4399  char c;
4400  char delimc = cstate->delim[0];
4401 
4402  if (cstate->need_transcoding)
4403  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4404  else
4405  ptr = string;
4406 
4407  /*
4408  * We have to grovel through the string searching for control characters
4409  * and instances of the delimiter character. In most cases, though, these
4410  * are infrequent. To avoid overhead from calling CopySendData once per
4411  * character, we dump out all characters between escaped characters in a
4412  * single call. The loop invariant is that the data from "start" to "ptr"
4413  * can be sent literally, but hasn't yet been.
4414  *
4415  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4416  * in valid backend encodings, extra bytes of a multibyte character never
4417  * look like ASCII. This loop is sufficiently performance-critical that
4418  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4419  * of the normal safe-encoding path.
4420  */
4421  if (cstate->encoding_embeds_ascii)
4422  {
4423  start = ptr;
4424  while ((c = *ptr) != '\0')
4425  {
4426  if ((unsigned char) c < (unsigned char) 0x20)
4427  {
4428  /*
4429  * \r and \n must be escaped, the others are traditional. We
4430  * prefer to dump these using the C-like notation, rather than
4431  * a backslash and the literal character, because it makes the
4432  * dump file a bit more proof against Microsoftish data
4433  * mangling.
4434  */
4435  switch (c)
4436  {
4437  case '\b':
4438  c = 'b';
4439  break;
4440  case '\f':
4441  c = 'f';
4442  break;
4443  case '\n':
4444  c = 'n';
4445  break;
4446  case '\r':
4447  c = 'r';
4448  break;
4449  case '\t':
4450  c = 't';
4451  break;
4452  case '\v':
4453  c = 'v';
4454  break;
4455  default:
4456  /* If it's the delimiter, must backslash it */
4457  if (c == delimc)
4458  break;
4459  /* All ASCII control chars are length 1 */
4460  ptr++;
4461  continue; /* fall to end of loop */
4462  }
4463  /* if we get here, we need to convert the control char */
4464  DUMPSOFAR();
4465  CopySendChar(cstate, '\\');
4466  CopySendChar(cstate, c);
4467  start = ++ptr; /* do not include char in next run */
4468  }
4469  else if (c == '\\' || c == delimc)
4470  {
4471  DUMPSOFAR();
4472  CopySendChar(cstate, '\\');
4473  start = ptr++; /* we include char in next run */
4474  }
4475  else if (IS_HIGHBIT_SET(c))
4476  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4477  else
4478  ptr++;
4479  }
4480  }
4481  else
4482  {
4483  start = ptr;
4484  while ((c = *ptr) != '\0')
4485  {
4486  if ((unsigned char) c < (unsigned char) 0x20)
4487  {
4488  /*
4489  * \r and \n must be escaped, the others are traditional. We
4490  * prefer to dump these using the C-like notation, rather than
4491  * a backslash and the literal character, because it makes the
4492  * dump file a bit more proof against Microsoftish data
4493  * mangling.
4494  */
4495  switch (c)
4496  {
4497  case '\b':
4498  c = 'b';
4499  break;
4500  case '\f':
4501  c = 'f';
4502  break;
4503  case '\n':
4504  c = 'n';
4505  break;
4506  case '\r':
4507  c = 'r';
4508  break;
4509  case '\t':
4510  c = 't';
4511  break;
4512  case '\v':
4513  c = 'v';
4514  break;
4515  default:
4516  /* If it's the delimiter, must backslash it */
4517  if (c == delimc)
4518  break;
4519  /* All ASCII control chars are length 1 */
4520  ptr++;
4521  continue; /* fall to end of loop */
4522  }
4523  /* if we get here, we need to convert the control char */
4524  DUMPSOFAR();
4525  CopySendChar(cstate, '\\');
4526  CopySendChar(cstate, c);
4527  start = ++ptr; /* do not include char in next run */
4528  }
4529  else if (c == '\\' || c == delimc)
4530  {
4531  DUMPSOFAR();
4532  CopySendChar(cstate, '\\');
4533  start = ptr++; /* we include char in next run */
4534  }
4535  else
4536  ptr++;
4537  }
4538  }
4539 
4540  DUMPSOFAR();
4541 }
4542 
4543 /*
4544  * Send text representation of one attribute, with conversion and
4545  * CSV-style escaping
4546  */
4547 static void
4548 CopyAttributeOutCSV(CopyState cstate, char *string,
4549  bool use_quote, bool single_attr)
4550 {
4551  char *ptr;
4552  char *start;
4553  char c;
4554  char delimc = cstate->delim[0];
4555  char quotec = cstate->quote[0];
4556  char escapec = cstate->escape[0];
4557 
4558  /* force quoting if it matches null_print (before conversion!) */
4559  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4560  use_quote = true;
4561 
4562  if (cstate->need_transcoding)
4563  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4564  else
4565  ptr = string;
4566 
4567  /*
4568  * Make a preliminary pass to discover if it needs quoting
4569  */
4570  if (!use_quote)
4571  {
4572  /*
4573  * Because '\.' can be a data value, quote it if it appears alone on a
4574  * line so it is not interpreted as the end-of-data marker.
4575  */
4576  if (single_attr && strcmp(ptr, "\\.") == 0)
4577  use_quote = true;
4578  else
4579  {
4580  char *tptr = ptr;
4581 
4582  while ((c = *tptr) != '\0')
4583  {
4584  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4585  {
4586  use_quote = true;
4587  break;
4588  }
4589  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4590  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4591  else
4592  tptr++;
4593  }
4594  }
4595  }
4596 
4597  if (use_quote)
4598  {
4599  CopySendChar(cstate, quotec);
4600 
4601  /*
4602  * We adopt the same optimization strategy as in CopyAttributeOutText
4603  */
4604  start = ptr;
4605  while ((c = *ptr) != '\0')
4606  {
4607  if (c == quotec || c == escapec)
4608  {
4609  DUMPSOFAR();
4610  CopySendChar(cstate, escapec);
4611  start = ptr; /* we include char in next run */
4612  }
4613  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4614  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4615  else
4616  ptr++;
4617  }
4618  DUMPSOFAR();
4619 
4620  CopySendChar(cstate, quotec);
4621  }
4622  else
4623  {
4624  /* If it doesn't need quoting, we can just dump it as-is */
4625  CopySendString(cstate, ptr);
4626  }
4627 }
4628 
4629 /*
4630  * CopyGetAttnums - build an integer list of attnums to be copied
4631  *
4632  * The input attnamelist is either the user-specified column list,
4633  * or NIL if there was none (in which case we want all the non-dropped
4634  * columns).
4635  *
4636  * rel can be NULL ... it's only used for error reports.
4637  */
4638 static List *
4639 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
4640 {
4641  List *attnums = NIL;
4642 
4643  if (attnamelist == NIL)
4644  {
4645  /* Generate default column list */
4646  Form_pg_attribute *attr = tupDesc->attrs;
4647  int attr_count = tupDesc->natts;
4648  int i;
4649 
4650  for (i = 0; i < attr_count; i++)
4651  {
4652  if (attr[i]->attisdropped)
4653  continue;
4654  attnums = lappend_int(attnums, i + 1);
4655  }
4656  }
4657  else
4658  {
4659  /* Validate the user-supplied list and extract attnums */
4660  ListCell *l;
4661 
4662  foreach(l, attnamelist)
4663  {
4664  char *name = strVal(lfirst(l));
4665  int attnum;
4666  int i;
4667 
4668  /* Lookup column name */
4669  attnum = InvalidAttrNumber;
4670  for (i = 0; i < tupDesc->natts; i++)
4671  {
4672  if (tupDesc->attrs[i]->attisdropped)
4673  continue;
4674  if (namestrcmp(&(tupDesc->attrs[i]->attname), name) == 0)
4675  {
4676  attnum = tupDesc->attrs[i]->attnum;
4677  break;
4678  }
4679  }
4680  if (attnum == InvalidAttrNumber)
4681  {
4682  if (rel != NULL)
4683  ereport(ERROR,
4684  (errcode(ERRCODE_UNDEFINED_COLUMN),
4685  errmsg("column \"%s\" of relation \"%s\" does not exist",
4686  name, RelationGetRelationName(rel))));
4687  else
4688  ereport(ERROR,
4689  (errcode(ERRCODE_UNDEFINED_COLUMN),
4690  errmsg("column \"%s\" does not exist",
4691  name)));
4692  }
4693  /* Check for duplicates */
4694  if (list_member_int(attnums, attnum))
4695  ereport(ERROR,
4696  (errcode(ERRCODE_DUPLICATE_COLUMN),
4697  errmsg("column \"%s\" specified more than once",
4698  name)));
4699  attnums = lappend_int(attnums, attnum);
4700  }
4701  }
4702 
4703  return attnums;
4704 }
4705 
4706 
4707 /*
4708  * copy_dest_startup --- executor startup
4709  */
4710 static void
4711 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
4712 {
4713  /* no-op */
4714 }
4715 
4716 /*
4717  * copy_dest_receive --- receive one tuple
4718  */
4719 static bool
4720 copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
4721 {
4722  DR_copy *myState = (DR_copy *) self;
4723  CopyState cstate = myState->cstate;
4724 
4725  /* Make sure the tuple is fully deconstructed */
4726  slot_getallattrs(slot);
4727 
4728  /* And send the data */
4729  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4730  myState->processed++;
4731 
4732  return true;
4733 }
4734 
4735 /*
4736  * copy_dest_shutdown --- executor end
4737  */
4738 static void
4739 copy_dest_shutdown(DestReceiver *self)
4740 {
4741  /* no-op */
4742 }
4743 
4744 /*
4745  * copy_dest_destroy --- release DestReceiver object
4746  */
4747 static void
4748 copy_dest_destroy(DestReceiver *self)
4749 {
4750  pfree(self);
4751 }
4752 
4753 /*
4754  * CreateCopyDestReceiver -- create a suitable DestReceiver object
4755  */
4756 DestReceiver *
4757 CreateCopyDestReceiver(void)
4758 {
4759  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4760 
4761  self->pub.receiveSlot = copy_dest_receive;
4762  self->pub.rStartup = copy_dest_startup;
4763  self->pub.rShutdown = copy_dest_shutdown;
4764  self->pub.rDestroy = copy_dest_destroy;
4765  self->pub.mydest = DestCopyOut;
4766 
4767  self->cstate = NULL; /* will be set later */
4768  self->processed = 0;
4769 
4770  return (DestReceiver *) self;
4771 }
int16
signed short int16
Definition: c.h:255
ResTarget::indirection
List * indirection
Definition: parsenodes.h:440
NextCopyFromRawFields
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3154
ResultRelInfo::ri_NumIndices
int ri_NumIndices
Definition: execnodes.h:357
NIL
#define NIL
Definition: pg_list.h:69
CommandId
uint32 CommandId
Definition: c.h:411
CopyReadBinaryAttribute
static Datum CopyReadBinaryAttribute(CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4336
ExecStoreTuple
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
GetDecimalFromHex
static int GetDecimalFromHex(char hex)
Definition: copy.c:3911
FmgrInfo
Definition: fmgr.h:56
MAX_COPY_DATA_DISPLAY
#define MAX_COPY_DATA_DISPLAY
CopyStateData::csv_mode
bool csv_mode
Definition: copy.c:118
SendCopyEnd
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:416
InitResultRelInfo
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1299
ResultRelInfo::ri_RelationDesc
Relation ri_RelationDesc
Definition: execnodes.h:354
CopyStateData::range_table
List * range_table
Definition: copy.c:166
BeginCopy
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1372
UpdateActiveSnapshotCommandId
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:781
IsA
#define IsA(nodeptr, _type_)
Definition: nodes.h:560
CopyReadLineText
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3552
MemoryContextDelete
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:200
HOLD_CANCEL_INTERRUPTS
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:125
ExecInitExtraTupleSlot
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate)
Definition: execTuples.c:852
contain_volatile_functions_not_nextval
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:995
ResTarget::val
Node * val
Definition: parsenodes.h:441
CopyGetInt32
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:682
errhint
int errhint(const char *fmt,...)
Definition: elog.c:987
pg_char_to_encoding
int pg_char_to_encoding(const char *name)
Definition: encnames.c:551
CopyStateData::raw_fields
char ** raw_fields
Definition: copy.c:188
EOL_NL
Definition: copy.c:74
getTypeOutputInfo
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2632
VARDATA
#define VARDATA(PTR)
Definition: postgres.h:303
EndCopy
static void EndCopy(CopyState cstate)
Definition: copy.c:1734
CopyStateData::binary
bool binary
Definition: copy.c:115
pq_flush
#define pq_flush()
Definition: libpq.h:39
CopyFromErrorCallback
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2187
heap_endscan
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1578
PreventCommandIfParallelMode
void PreventCommandIfParallelMode(const char *cmdname)
Definition: utility.c:254
ExecInsertIndexTuples
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
pq_sendbyte
void pq_sendbyte(StringInfo buf, int byt)
Definition: pqformat.c:105
CopyStmt::attlist
List * attlist
Definition: parsenodes.h:1929
SelectStmt::fromClause
List * fromClause
Definition: parsenodes.h:1501
ISOCTAL
#define ISOCTAL(c)
Definition: copy.c:53
OCTVALUE
#define OCTVALUE(c)
Definition: copy.c:54
ResetPerTupleExprContext
#define ResetPerTupleExprContext(estate)
Definition: executor.h:465
RelationGetDescr
#define RelationGetDescr(relation)
Definition: rel.h:428
HEAP_INSERT_FROZEN
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
ResTarget::name
char * name
Definition: parsenodes.h:439
CopyStateData::need_transcoding
bool need_transcoding
Definition: copy.c:105
CopyStateData::partitions
ResultRelInfo * partitions
Definition: copy.c:171
castNode
#define castNode(_type_, nodeptr)
Definition: nodes.h:578
FreeQueryDesc
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
CopyStateData::in_functions
FmgrInfo * in_functions
Definition: copy.c:161
CopyStateData::num_defaults
AttrNumber num_defaults
Definition: copy.c:157
CopyStateData::attnumlist
List * attnumlist
Definition: copy.c:111
OIDOID
#define OIDOID
Definition: pg_type.h:328
VARSIZE
#define VARSIZE(PTR)
Definition: postgres.h:304
pq_getmsgstring
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:621
CopyStateData::filename
char * filename
Definition: copy.c:112
BeginCopyFrom
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:2895
VARHDRSZ
#define VARHDRSZ
Definition: c.h:445
CopyStateData::file_has_oids
bool file_has_oids
Definition: copy.c:158
DatumGetObjectId
#define DatumGetObjectId(X)
Definition: postgres.h:506
PlannedStmt::relationOids
List * relationOids
Definition: plannodes.h:88
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1077
ExprContext::ecxt_per_tuple_memory
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:203
ReceiveCopyBegin
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:380
XLogIsNeeded
#define XLogIsNeeded()
Definition: xlog.h:145
DR_copy
Definition: copy.c:215
makeStringInfo
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
MAX_BUFFERED_TUPLES
#define MAX_BUFFERED_TUPLES
RelationData::rd_islocaltemp
bool rd_islocaltemp
Definition: rel.h:90
ExecIRInsertTriggers
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2277
expression_planner
Expr * expression_planner(Expr *expr)
Definition: planner.c:5931
ExecutorStart
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:146
tupleDesc::attrs
Form_pg_attribute * attrs
Definition: tupdesc.h:74
DR_copy::pub
DestReceiver pub
Definition: copy.c:217
ProcessCopyOptions
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1014
RELKIND_MATVIEW
#define RELKIND_MATVIEW
Definition: pg_class.h:165
ExecConstraints
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1913
CopyStateData::line_buf
StringInfoData line_buf
Definition: copy.c:197
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
IF_NEED_REFILL_AND_EOF_BREAK
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:251
GetActiveSnapshot
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
AccessShareLock
#define AccessShareLock
Definition: lockdefs.h:36
InvalidBuffer
#define InvalidBuffer
Definition: buf.h:25
CopyStateData
struct CopyStateData CopyStateData
Node
Definition: nodes.h:509
strVal
#define strVal(v)
Definition: value.h:54
cur
struct cursor * cur
Definition: ecpg.c:28
CopyStateData::raw_buf_index
int raw_buf_index
Definition: copy.c:210
CopyAttributeOutText
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4395
CopyStateData::line_buf_valid
bool line_buf_valid
Definition: copy.c:199
ThereAreNoPriorRegisteredSnapshots
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1649
DR_copy::cstate
CopyState cstate
Definition: copy.c:218
errcode
int errcode(int sqlerrcode)
Definition: elog.c:575
PG_BINARY_W
#define PG_BINARY_W
Definition: c.h:1041
superuser
bool superuser(void)
Definition: superuser.c:47
namestrcmp
int namestrcmp(Name name, const char *str)
Definition: name.c:247
parse_relation.h
MemSet
#define MemSet(start, val, len)
Definition: c.h:857
CopyStateData::fe_eof
bool fe_eof
Definition: copy.c:102
CopyFrom
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2292
RelationData::rd_newRelfilenodeSubid
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:111
pq_putemptymessage
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:421
select
#define select(n, r, w, e, timeout)
Definition: win32.h:374
heap_sync
void heap_sync(Relation rel)
Definition: heapam.c:9084
TupleTableSlot::tts_values
Datum * tts_values
Definition: tuptable.h:125
result
return result
Definition: formatting.c:1633
FirstLowInvalidHeapAttributeNumber
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:28
ClosePipeToProgram
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1711
HEAP_INSERT_SKIP_WAL
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
MemoryContextReset
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:135
CopyReadAttributesCSV
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4167
PopActiveSnapshot
void PopActiveSnapshot(void)
Definition: snapmgr.c:812
RangeTblEntry::requiredPerms
AclMode requiredPerms
Definition: parsenodes.h:1039
heap_form_tuple
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:692
heap_close
#define heap_close(r, l)
Definition: heapam.h:97
DirectFunctionCall1
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:584
CopyStateData::force_quote_flags
bool * force_quote_flags
Definition: copy.c:128
EState::es_range_table
List * es_range_table
Definition: execnodes.h:431
RelationData::rd_rel
Form_pg_class rd_rel
Definition: rel.h:114
Oid
unsigned int Oid
Definition: postgres_ext.h:31
pg_analyze_and_rewrite
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:638
CopyStateData::delim
char * delim
Definition: copy.c:123
PG_PROTOCOL_MAJOR
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Query::utilityStmt
Node * utilityStmt
Definition: parsenodes.h:118
CopyStmt::is_program
bool is_program
Definition: parsenodes.h:1932
linitial_node
#define linitial_node(type, l)
Definition: pg_list.h:114
CopyStateData::volatile_defexprs
bool volatile_defexprs
Definition: copy.c:165
pg_wchar.h
oidout
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
ErrorContextCallback::previous
struct ErrorContextCallback * previous
Definition: elog.h:238
PG_BINARY_R
#define PG_BINARY_R
Definition: c.h:1040
copy_dest_destroy
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4748
CopyStateData::force_null_flags
bool * force_null_flags
Definition: copy.c:132
wait_result_to_str
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
CopyStateData::rowcontext
MemoryContext rowcontext
Definition: copy.c:152
_DestReceiver
Definition: dest.h:115
tupleDesc::natts
int natts
Definition: tupdesc.h:73
CopyStateData::line_buf_converted
bool line_buf_converted
Definition: copy.c:198
pq_beginmessage
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
int32
signed int int32
Definition: c.h:256
ClosePipeStream
int ClosePipeStream(FILE *file)
Definition: fd.c:2470
errdetail_internal
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
CopyStateData::convert_select_flags
bool * convert_select_flags
Definition: copy.c:135
CopySendChar
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
OutputFunctionCall
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1667
ExecOpenIndices
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
ColumnRef::location
int location
Definition: parsenodes.h:235
CopyStateData::copy_dest
CopyDest copy_dest
Definition: copy.c:98
ResTarget::location
int location
Definition: parsenodes.h:442
REFILL_LINEBUF
#define REFILL_LINEBUF
Definition: copy.c:268
HeapTupleSetOid
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:698
error_context_stack
ErrorContextCallback * error_context_stack
Definition: elog.c:88
list_make1
#define list_make1(x1)
Definition: pg_list.h:139
CopyStateData::null_print
char * null_print
Definition: copy.c:120
CopyStateData::cur_attname
const char * cur_attname
Definition: copy.c:140
ExecutorEnd
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:460
COPY_FILE
Definition: copy.c:62
TriggerDesc::trig_insert_instead_row
bool trig_insert_instead_row
Definition: reltrigger.h:57
FreeExecutorState
void FreeExecutorState(EState *estate)
Definition: execUtils.c:178
CopyStateData::rel
Relation rel
Definition: copy.c:109
GetPerTupleExprContext
#define GetPerTupleExprContext(estate)
Definition: executor.h:456
GetBulkInsertState
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2319
CopyStateData::copycontext
MemoryContext copycontext
Definition: copy.c:146
pq_startcopyout
#define pq_startcopyout()
Definition: libpq.h:46
CopyStateData::data_source_cb
copy_data_source_cb data_source_cb
Definition: copy.c:114
defGetBoolean
bool defGetBoolean(DefElem *def)
Definition: define.c:111
appendStringInfoCharMacro
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:127
RangeTblEntry::selectedCols
Bitmapset * selectedCols
Definition: parsenodes.h:1041
uint16
unsigned short uint16
Definition: c.h:267
pfree
void pfree(void *pointer)
Definition: mcxt.c:950
ExecSetupPartitionTupleRouting
void ExecSetupPartitionTupleRouting(Relation rel, PartitionDispatch **pd, ResultRelInfo **partitions, TupleConversionMap ***tup_conv_maps, TupleTableSlot **partition_tuple_slot, int *num_parted, int *num_partitions)
Definition: execMain.c:3215
IS_HIGHBIT_SET
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
BeginCopyTo
static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
Definition: copy.c:1757
CopySendInt16
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:699
ThereAreNoReadyPortals
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1127
CopyStateData::partition_tuple_slot
TupleTableSlot * partition_tuple_slot
Definition: copy.c:173
ObjectIdGetDatum
#define ObjectIdGetDatum(X)
Definition: postgres.h:513
ERROR
#define ERROR
Definition: elog.h:43
pq_startmsgread
void pq_startmsgread(void)
Definition: pqcomm.c:1191
DatumGetCString
#define DatumGetCString(X)
Definition: postgres.h:572
CopyGetData
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
S_IWGRP
#define S_IWGRP
Definition: win32.h:449
lfirst_int
#define lfirst_int(lc)
Definition: pg_list.h:107
ExecutorRun
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:298
CopyAttributeOutCSV
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4548
ReceiveFunctionCall
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1681
CopyFromInsertBatch
static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
Definition: copy.c:2812
FATAL
#define FATAL
Definition: elog.h:52
fmgr_info
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:127
defGetString
char * defGetString(DefElem *def)
Definition: define.c:49
CopyStmt::relation
RangeVar * relation
Definition: parsenodes.h:1926
value
static struct @121 value
HeapTupleData::t_self
ItemPointerData t_self
Definition: htup.h:65
PushCopiedSnapshot
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:769
CreateQueryDesc
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
pg_mbcliplen
int pg_mbcliplen(const char *mbstr, int len, int limit)
Definition: mbutils.c:831
RelationData::trigdesc
TriggerDesc * trigdesc
Definition: rel.h:120
ALLOCSET_DEFAULT_SIZES
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
limit_printout_length
static char * limit_printout_length(const char *str)
Definition: copy.c:2263
lfirst_node
#define lfirst_node(type, lc)
Definition: pg_list.h:109
CopyStateData::queryDesc
QueryDesc * queryDesc
Definition: copy.c:110
EolType
EolType
Definition: copy.c:71
list_member_int
bool list_member_int(const List *list, int datum)
Definition: list.c:485
copy_dest_shutdown
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4739
CopyStateData::num_dispatch
int num_dispatch
Definition: copy.c:169
CopyStateData::encoding_embeds_ascii
bool encoding_embeds_ascii
Definition: copy.c:106
HeapTupleData::t_len
uint32 t_len
Definition: htup.h:64
get_namespace_name
char * get_namespace_name(Oid nspid)
Definition: lsyscache.c:3033
c
char * c
Definition: preproc-cursor.c:31
ExecBSInsertTriggers
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2137
CopyStateData::raw_buf
char * raw_buf
Definition: copy.c:209
CopyLoadRawBuf
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:736
RawStmt::stmt
Node * stmt
Definition: parsenodes.h:1411
NoLock
#define NoLock
Definition: lockdefs.h:34
pq_getbytes
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1075
CopyStateData::quote
char * quote
Definition: copy.c:124
buf
static char * buf
Definition: pg_test_fsync.c:66
memmove
#define memmove(d, s, c)
Definition: c.h:1058
TupleTableSlot::tts_isnull
bool * tts_isnull
Definition: tuptable.h:126
ExecDropSingleTupleTableSlot
void ExecDropSingleTupleTableSlot(TupleTableSlot *slot)
Definition: execTuples.c:216
ExecEvalExpr
static Datum ExecEvalExpr(ExprState *state, ExprContext *econtext, bool *isNull)
Definition: executor.h:266
pg_database_encoding_max_length
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
SelectStmt::targetList
List * targetList
Definition: parsenodes.h:1500
EState::es_result_relations
ResultRelInfo * es_result_relations
Definition: execnodes.h:441
DoCopyTo
static uint64 DoCopyTo(CopyState cstate)
Definition: copy.c:1886
CreateDestReceiver
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
copy_dest_startup
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4711
DefElem::location
int location
Definition: parsenodes.h:722
RowExclusiveLock
#define RowExclusiveLock
Definition: lockdefs.h:38
CopyStateData::defexprs
ExprState ** defexprs
Definition: copy.c:164
CopyStateData::cur_relname
const char * cur_relname
Definition: copy.c:138
errcode_for_file_access
int errcode_for_file_access(void)
Definition: elog.c:598
pg_encoding_mblen
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
is_absolute_path
#define is_absolute_path(filename)
Definition: port.h:77
CStringGetDatum
#define CStringGetDatum(X)
Definition: postgres.h:584
string
char string[11]
Definition: preproc-type.c:46
EOL_CR
Definition: copy.c:75
CopyStmt::options
List * options
Definition: parsenodes.h:1934
AllocateFile
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
CopyStateData::oid_in_function
FmgrInfo oid_in_function
Definition: copy.c:159
enlargeStringInfo
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
RelationGetRelationName
#define RelationGetRelationName(relation)
Definition: rel.h:436
BinarySignature
static const char BinarySignature[11]
Definition: copy.c:288
Form_pg_attribute
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
resetStringInfo
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
ResultRelInfo::ri_FdwRoutine
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:378
uint32
unsigned int uint32
Definition: c.h:268
SendFunctionCall
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1728
CopyStateData::raw_buf_len
int raw_buf_len
Definition: copy.c:211
HeapTupleData::t_tableOid
Oid t_tableOid
Definition: htup.h:66
RELKIND_FOREIGN_TABLE
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
TriggerDesc::trig_insert_after_row
bool trig_insert_after_row
Definition: reltrigger.h:56
CurrentMemoryContext
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
CopyStateData::max_fields
int max_fields
Definition: copy.c:187
CopyStateData::escape
char * escape
Definition: copy.c:125
ParseState::p_sourcetext
const char * p_sourcetext
Definition: parse_node.h:171
Query::returningList
List * returningList
Definition: parsenodes.h:144
EState::es_trig_tuple_slot
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:457
OpenPipeStream
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2184
getTypeBinaryInputInfo
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2665
htup_details.h
getTypeInputInfo
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2599
ereport
#define ereport(elevel, rest)
Definition: elog.h:122
heap_insert
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2396
CopyStateData::null_print_len
int null_print_len
Definition: copy.c:121
A_Star
Definition: parsenodes.h:367
slot_getallattrs
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1237
CopyStateData::force_null
List * force_null
Definition: copy.c:131
ResultRelInfo::ri_TrigDesc
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:366
ExecutorFinish
void ExecutorFinish(QueryDesc *queryDesc)
Definition: execMain.c:400
CreateExecutorState
EState * CreateExecutorState(void)
Definition: execUtils.c:80
lappend_int
List * lappend_int(List *list, int datum)
Definition: list.c:146
DefElem::arg
Node * arg
Definition: parsenodes.h:720
EOL_CRNL
Definition: copy.c:76
lappend
List * lappend(List *list, void *datum)
Definition: list.c:128
S_IWOTH
#define S_IWOTH
Definition: win32.h:453
CopyStateData::file_encoding
int file_encoding
Definition: copy.c:104
CopyGetAttnums
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4639
initStringInfo
void initStringInfo(StringInfo str)
Definition: stringinfo.c:46
QueryDesc::tupDesc
TupleDesc tupDesc
Definition: execdesc.h:47
pq_getmessage
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1253
InvalidSnapshot
#define InvalidSnapshot
Definition: snapshot.h:25
RelationData::rd_createSubid
SubTransactionId rd_createSubid
Definition: rel.h:110
CopyStateData::typioparams
Oid * typioparams
Definition: copy.c:162
CopyStateData::is_program
bool is_program
Definition: copy.c:113
RELKIND_PARTITIONED_TABLE
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
build_column_default
Node * build_column_default(Relation rel, int attrno)
Definition: rewriteHandler.c:1081
TriggerDesc::trig_insert_before_row
bool trig_insert_before_row
Definition: reltrigger.h:55
getTypeBinaryOutputInfo
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2698
EState::es_tupleTable
List * es_tupleTable
Definition: execnodes.h:470
AllocSetContextCreate
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
ExecResetTupleTable
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:156
palloc0
void * palloc0(Size size)
Definition: mcxt.c:878
CopyStateData::header_line
bool header_line
Definition: copy.c:119
Datum
uintptr_t Datum
Definition: postgres.h:372
GetDatabaseEncoding
int GetDatabaseEncoding(void)
Definition: mbutils.c:1015
pg_get_client_encoding
int pg_get_client_encoding(void)
Definition: mbutils.c:317
ACL_SELECT
#define ACL_SELECT
Definition: parsenodes.h:73
PartitionDispatchData::tupslot
TupleTableSlot * tupslot
Definition: partition.h:66
ExecSetSlotDescriptor
void ExecSetSlotDescriptor(TupleTableSlot *slot, TupleDesc tupdesc)
Definition: execTuples.c:247
ExecASInsertTriggers
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2189
ExecFindPartition
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:3304
heap_getnext
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1794
RawStmt::stmt_len
int stmt_len
Definition: parsenodes.h:1413
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:239
RawStmt::stmt_location
int stmt_location
Definition: parsenodes.h:1412
EState::es_num_result_relations
int es_num_result_relations
Definition: execnodes.h:442
ResultRelInfo::ri_PartitionCheck
List * ri_PartitionCheck
Definition: execnodes.h:408
RAW_BUF_SIZE
#define RAW_BUF_SIZE
Definition: copy.c:208
RelationData::rd_att
TupleDesc rd_att
Definition: rel.h:115
CopyStateData::freeze
bool freeze
Definition: copy.c:117
InputFunctionCall
Datum InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod)
Definition: fmgr.c:1623
pq_getbyte
int pq_getbyte(void)
Definition: pqcomm.c:981
pq_endmsgread
void pq_endmsgread(void)
Definition: pqcomm.c:1215
heap_openrv
Relation heap_openrv(const RangeVar *relation, LOCKMODE lockmode)
Definition: heapam.c:1312
InvalidOid
#define InvalidOid
Definition: postgres_ext.h:36
copy_dest_receive
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4720
XactReadOnly
bool XactReadOnly
Definition: xact.c:77
CopyStateData::force_notnull_flags
bool * force_notnull_flags
Definition: copy.c:130
Query::commandType
CmdType commandType
Definition: parsenodes.h:110
AfterTriggerBeginQuery
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4167
CopyStateData::force_notnull
List * force_notnull
Definition: copy.c:129
check_enable_rls
int check_enable_rls(Oid relid, Oid checkAsUser, bool noError)
Definition: rls.c:53
errmsg_internal
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
EndCopyFrom
void EndCopyFrom(CopyState cstate)
Definition: copy.c:3447
PG_CATCH
#define PG_CATCH()
Definition: elog.h:293
makeNode
#define makeNode(_type_)
Definition: nodes.h:557
list_member_oid
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:505
CopyStateData::eol_type
EolType eol_type
Definition: copy.c:103
NULL
#define NULL
Definition: c.h:229
Query::querySource
QuerySource querySource
Definition: parsenodes.h:112
DoCopy
void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
Definition: copy.c:781
heap_multi_insert
void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2658
Assert
#define Assert(condition)
Definition: c.h:675
lfirst
#define lfirst(lc)
Definition: pg_list.h:106
FreeBulkInsertState
void FreeBulkInsertState(BulkInsertState bistate)
Definition: heapam.c:2333
CopySendData
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
CopyStateData::convert_select
List * convert_select
Definition: copy.c:134
GetCurrentSubTransactionId
SubTransactionId GetCurrentSubTransactionId(void)
Definition: xact.c:649
CopyStateData::fe_msgbuf
StringInfo fe_msgbuf
Definition: copy.c:100
CopyDest
CopyDest
Definition: copy.c:60
pq_copymsgbytes
void pq_copymsgbytes(StringInfo msg, char *buf, int datalen)
Definition: pqformat.c:570
CopyReadLine
static bool CopyReadLine(CopyState cstate)
Definition: copy.c:3463
ACL_INSERT
#define ACL_INSERT
Definition: parsenodes.h:72
tupleDesc::constr
TupleConstr * constr
Definition: tupdesc.h:76
pq_endcopyout
#define pq_endcopyout(errorAbort)
Definition: libpq.h:47
Size
size_t Size
Definition: c.h:356
CopySendString
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
list_length
static int list_length(const List *l)
Definition: pg_list.h:89
parser_errposition
int parser_errposition(ParseState *pstate, int location)
Definition: parse_node.c:111
CopyTo
static uint64 CopyTo(CopyState cstate)
Definition: copy.c:1940
addRangeTableEntryForRelation
RangeTblEntry * addRangeTableEntryForRelation(ParseState *pstate, Relation rel, Alias *alias, bool inh, bool inFromCl)
Definition: parse_relation.c:1273
PG_RE_THROW
#define PG_RE_THROW()
Definition: elog.h:314
InvalidSubTransactionId
#define InvalidSubTransactionId
Definition: c.h:403
bms_add_member
Bitmapset * bms_add_member(Bitmapset *a, int x)
Definition: bitmapset.c:698
ExecMaterializeSlot
HeapTuple ExecMaterializeSlot(TupleTableSlot *slot)
Definition: execTuples.c:725
ReleaseBulkInsertStatePin
void ReleaseBulkInsertStatePin(BulkInsertState bistate)
Definition: heapam.c:2345
repalloc
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
ExecARInsertTriggers
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes)
Definition: trigger.c:2265
name
const char * name
Definition: encode.c:521
do_convert_tuple
HeapTuple do_convert_tuple(HeapTuple tuple, TupleConversionMap *map)
Definition: tupconvert.c:343
<