PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 #include <netinet/in.h>
21 #include <arpa/inet.h>
22 
23 #include "access/heapam.h"
24 #include "access/htup_details.h"
25 #include "access/sysattr.h"
26 #include "access/xact.h"
27 #include "access/xlog.h"
28 #include "catalog/pg_type.h"
29 #include "commands/copy.h"
30 #include "commands/defrem.h"
31 #include "commands/trigger.h"
32 #include "executor/executor.h"
33 #include "libpq/libpq.h"
34 #include "libpq/pqformat.h"
35 #include "mb/pg_wchar.h"
36 #include "miscadmin.h"
37 #include "optimizer/clauses.h"
38 #include "optimizer/planner.h"
39 #include "nodes/makefuncs.h"
40 #include "rewrite/rewriteHandler.h"
41 #include "storage/fd.h"
42 #include "tcop/tcopprot.h"
43 #include "utils/builtins.h"
44 #include "utils/lsyscache.h"
45 #include "utils/memutils.h"
46 #include "utils/portal.h"
47 #include "utils/rel.h"
48 #include "utils/rls.h"
49 #include "utils/snapmgr.h"
50 
51 
52 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
53 #define OCTVALUE(c) ((c) - '0')
54 
55 /*
56  * Represents the different source/dest cases we need to worry about at
57  * the bottom level
58  */
59 typedef enum CopyDest
60 {
61  COPY_FILE, /* to/from file (or a piped program) */
62  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
63  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
64  COPY_CALLBACK /* to/from callback function */
65 } CopyDest;
66 
67 /*
68  * Represents the end-of-line terminator type of the input
69  */
70 typedef enum EolType
71 {
76 } EolType;
77 
78 /*
79  * This struct contains all the state variables used throughout a COPY
80  * operation. For simplicity, we use the same struct for all variants of COPY,
81  * even though some fields are used in only some cases.
82  *
83  * Multi-byte encodings: all supported client-side encodings encode multi-byte
84  * characters by having the first byte's high bit set. Subsequent bytes of the
85  * character can have the high bit not set. When scanning data in such an
86  * encoding to look for a match to a single-byte (ie ASCII) character, we must
87  * use the full pg_encoding_mblen() machinery to skip over multibyte
88  * characters, else we might find a false match to a trailing byte. In
89  * supported server encodings, there is no possibility of a false match, and
90  * it's faster to make useless comparisons to trailing bytes than it is to
91  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is TRUE
92  * when we have to do it the hard way.
93  */
94 typedef struct CopyStateData
95 {
96  /* low-level state data */
97  CopyDest copy_dest; /* type of copy source/destination */
98  FILE *copy_file; /* used if copy_dest == COPY_FILE */
99  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
100  * dest == COPY_NEW_FE in COPY FROM */
101  bool fe_eof; /* true if detected end of copy data */
102  EolType eol_type; /* EOL type of input */
103  int file_encoding; /* file or remote side's character encoding */
104  bool need_transcoding; /* file encoding diff from server? */
105  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
106 
107  /* parameters from the COPY command */
108  Relation rel; /* relation to copy to or from */
109  QueryDesc *queryDesc; /* executable query to copy from */
110  List *attnumlist; /* integer list of attnums to copy */
111  char *filename; /* filename, or NULL for STDIN/STDOUT */
112  bool is_program; /* is 'filename' a program to popen? */
113  copy_data_source_cb data_source_cb; /* function for reading data*/
114  bool binary; /* binary format? */
115  bool oids; /* include OIDs? */
116  bool freeze; /* freeze rows on loading? */
117  bool csv_mode; /* Comma Separated Value format? */
118  bool header_line; /* CSV header line? */
119  char *null_print; /* NULL marker string (server encoding!) */
120  int null_print_len; /* length of same */
121  char *null_print_client; /* same converted to file encoding */
122  char *delim; /* column delimiter (must be 1 byte) */
123  char *quote; /* CSV quote char (must be 1 byte) */
124  char *escape; /* CSV escape char (must be 1 byte) */
125  List *force_quote; /* list of column names */
126  bool force_quote_all; /* FORCE_QUOTE *? */
127  bool *force_quote_flags; /* per-column CSV FQ flags */
128  List *force_notnull; /* list of column names */
129  bool *force_notnull_flags; /* per-column CSV FNN flags */
130  List *force_null; /* list of column names */
131  bool *force_null_flags; /* per-column CSV FN flags */
132  bool convert_selectively; /* do selective binary conversion? */
133  List *convert_select; /* list of column names (can be NIL) */
134  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
135 
136  /* these are just for error messages, see CopyFromErrorCallback */
137  const char *cur_relname; /* table name for error messages */
138  int cur_lineno; /* line number for error messages */
139  const char *cur_attname; /* current att for error messages */
140  const char *cur_attval; /* current att value for error messages */
141 
142  /*
143  * Working state for COPY TO/FROM
144  */
145  MemoryContext copycontext; /* per-copy execution context */
146 
147  /*
148  * Working state for COPY TO
149  */
150  FmgrInfo *out_functions; /* lookup info for output functions */
151  MemoryContext rowcontext; /* per-row evaluation context */
152 
153  /*
154  * Working state for COPY FROM
155  */
160  FmgrInfo *in_functions; /* array of input functions for each attrs */
161  Oid *typioparams; /* array of element types for in_functions */
162  int *defmap; /* array of default att numbers */
163  ExprState **defexprs; /* array of default att expressions */
164  bool volatile_defexprs; /* is any of defexprs volatile? */
166 
168  int num_dispatch; /* Number of entries in the above array */
169  int num_partitions; /* Number of members in the following arrays */
170  ResultRelInfo *partitions; /* Per partition result relation */
173 
174  /*
175  * These variables are used to reduce overhead in textual COPY FROM.
176  *
177  * attribute_buf holds the separated, de-escaped text for each field of
178  * the current line. The CopyReadAttributes functions return arrays of
179  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
180  * the buffer on each cycle.
181  */
183 
184  /* field raw data pointers found by COPY FROM */
185 
187  char **raw_fields;
188 
189  /*
190  * Similarly, line_buf holds the whole input line being processed. The
191  * input cycle is first to read the whole line into line_buf, convert it
192  * to server encoding there, and then extract the individual attribute
193  * fields into attribute_buf. line_buf is preserved unmodified so that we
194  * can display it in error messages if appropriate.
195  */
197  bool line_buf_converted; /* converted to server encoding? */
198  bool line_buf_valid; /* contains the row being processed? */
199 
200  /*
201  * Finally, raw_buf holds raw data read from the data source (file or
202  * client connection). CopyReadLine parses this data sufficiently to
203  * locate line boundaries, then transfers the data to line_buf and
204  * converts it. Note: we guarantee that there is a \0 at
205  * raw_buf[raw_buf_len].
206  */
207 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
208  char *raw_buf;
209  int raw_buf_index; /* next byte to process */
210  int raw_buf_len; /* total # of bytes stored */
211 } CopyStateData;
212 
213 /* DestReceiver for COPY (query) TO */
214 typedef struct
215 {
216  DestReceiver pub; /* publicly-known function pointers */
217  CopyState cstate; /* CopyStateData for the command */
218  uint64 processed; /* # of tuples processed */
219 } DR_copy;
220 
221 
222 /*
223  * These macros centralize code used to process line_buf and raw_buf buffers.
224  * They are macros because they often do continue/break control and to avoid
225  * function call overhead in tight COPY loops.
226  *
227  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
228  * prevent the continue/break processing from working. We end the "if (1)"
229  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
230  * any "else" in the calling code, and to avoid any compiler warnings about
231  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
232  */
233 
234 /*
235  * This keeps the character read at the top of the loop in the buffer
236  * even if there is more than one read-ahead.
237  */
238 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
239 if (1) \
240 { \
241  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
242  { \
243  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
244  need_data = true; \
245  continue; \
246  } \
247 } else ((void) 0)
248 
249 /* This consumes the remainder of the buffer and breaks */
250 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
251 if (1) \
252 { \
253  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
254  { \
255  if (extralen) \
256  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
257  /* backslash just before EOF, treat as data char */ \
258  result = true; \
259  break; \
260  } \
261 } else ((void) 0)
262 
263 /*
264  * Transfer any approved data to line_buf; must do this to be sure
265  * there is some room in raw_buf.
266  */
267 #define REFILL_LINEBUF \
268 if (1) \
269 { \
270  if (raw_buf_ptr > cstate->raw_buf_index) \
271  { \
272  appendBinaryStringInfo(&cstate->line_buf, \
273  cstate->raw_buf + cstate->raw_buf_index, \
274  raw_buf_ptr - cstate->raw_buf_index); \
275  cstate->raw_buf_index = raw_buf_ptr; \
276  } \
277 } else ((void) 0)
278 
279 /* Undo any read-ahead and jump out of the block. */
280 #define NO_END_OF_COPY_GOTO \
281 if (1) \
282 { \
283  raw_buf_ptr = prev_raw_ptr + 1; \
284  goto not_end_of_copy; \
285 } else ((void) 0)
286 
287 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
288 
289 
290 /* non-export function prototypes */
291 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
292  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
293  List *options);
294 static void EndCopy(CopyState cstate);
295 static void ClosePipeToProgram(CopyState cstate);
296 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
297  Oid queryRelId, const char *filename, bool is_program,
298  List *attnamelist, List *options);
299 static void EndCopyTo(CopyState cstate);
300 static uint64 DoCopyTo(CopyState cstate);
301 static uint64 CopyTo(CopyState cstate);
302 static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
303  Datum *values, bool *nulls);
304 static void CopyFromInsertBatch(CopyState cstate, EState *estate,
305  CommandId mycid, int hi_options,
306  ResultRelInfo *resultRelInfo, TupleTableSlot *myslot,
307  BulkInsertState bistate,
308  int nBufferedTuples, HeapTuple *bufferedTuples,
309  int firstBufferedLineNo);
310 static bool CopyReadLine(CopyState cstate);
311 static bool CopyReadLineText(CopyState cstate);
312 static int CopyReadAttributesText(CopyState cstate);
313 static int CopyReadAttributesCSV(CopyState cstate);
315  int column_no, FmgrInfo *flinfo,
316  Oid typioparam, int32 typmod,
317  bool *isnull);
318 static void CopyAttributeOutText(CopyState cstate, char *string);
319 static void CopyAttributeOutCSV(CopyState cstate, char *string,
320  bool use_quote, bool single_attr);
321 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
322  List *attnamelist);
323 static char *limit_printout_length(const char *str);
324 
325 /* Low-level communications functions */
326 static void SendCopyBegin(CopyState cstate);
327 static void ReceiveCopyBegin(CopyState cstate);
328 static void SendCopyEnd(CopyState cstate);
329 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
330 static void CopySendString(CopyState cstate, const char *str);
331 static void CopySendChar(CopyState cstate, char c);
332 static void CopySendEndOfRow(CopyState cstate);
333 static int CopyGetData(CopyState cstate, void *databuf,
334  int minread, int maxread);
335 static void CopySendInt32(CopyState cstate, int32 val);
336 static bool CopyGetInt32(CopyState cstate, int32 *val);
337 static void CopySendInt16(CopyState cstate, int16 val);
338 static bool CopyGetInt16(CopyState cstate, int16 *val);
339 
340 
341 /*
342  * Send copy start/stop messages for frontend copies. These have changed
343  * in past protocol redesigns.
344  */
345 static void
347 {
349  {
350  /* new way */
352  int natts = list_length(cstate->attnumlist);
353  int16 format = (cstate->binary ? 1 : 0);
354  int i;
355 
356  pq_beginmessage(&buf, 'H');
357  pq_sendbyte(&buf, format); /* overall format */
358  pq_sendint(&buf, natts, 2);
359  for (i = 0; i < natts; i++)
360  pq_sendint(&buf, format, 2); /* per-column formats */
361  pq_endmessage(&buf);
362  cstate->copy_dest = COPY_NEW_FE;
363  }
364  else
365  {
366  /* old way */
367  if (cstate->binary)
368  ereport(ERROR,
369  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
370  errmsg("COPY BINARY is not supported to stdout or from stdin")));
371  pq_putemptymessage('H');
372  /* grottiness needed for old COPY OUT protocol */
373  pq_startcopyout();
374  cstate->copy_dest = COPY_OLD_FE;
375  }
376 }
377 
378 static void
380 {
382  {
383  /* new way */
385  int natts = list_length(cstate->attnumlist);
386  int16 format = (cstate->binary ? 1 : 0);
387  int i;
388 
389  pq_beginmessage(&buf, 'G');
390  pq_sendbyte(&buf, format); /* overall format */
391  pq_sendint(&buf, natts, 2);
392  for (i = 0; i < natts; i++)
393  pq_sendint(&buf, format, 2); /* per-column formats */
394  pq_endmessage(&buf);
395  cstate->copy_dest = COPY_NEW_FE;
396  cstate->fe_msgbuf = makeLongStringInfo();
397  }
398  else
399  {
400  /* old way */
401  if (cstate->binary)
402  ereport(ERROR,
403  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
404  errmsg("COPY BINARY is not supported to stdout or from stdin")));
405  pq_putemptymessage('G');
406  /* any error in old protocol will make us lose sync */
407  pq_startmsgread();
408  cstate->copy_dest = COPY_OLD_FE;
409  }
410  /* We *must* flush here to ensure FE knows it can send. */
411  pq_flush();
412 }
413 
414 static void
416 {
417  if (cstate->copy_dest == COPY_NEW_FE)
418  {
419  /* Shouldn't have any unsent data */
420  Assert(cstate->fe_msgbuf->len == 0);
421  /* Send Copy Done message */
422  pq_putemptymessage('c');
423  }
424  else
425  {
426  CopySendData(cstate, "\\.", 2);
427  /* Need to flush out the trailer (this also appends a newline) */
428  CopySendEndOfRow(cstate);
429  pq_endcopyout(false);
430  }
431 }
432 
433 /*----------
434  * CopySendData sends output data to the destination (file or frontend)
435  * CopySendString does the same for null-terminated strings
436  * CopySendChar does the same for single characters
437  * CopySendEndOfRow does the appropriate thing at end of each data row
438  * (data is not actually flushed except by CopySendEndOfRow)
439  *
440  * NB: no data conversion is applied by these functions
441  *----------
442  */
443 static void
444 CopySendData(CopyState cstate, const void *databuf, int datasize)
445 {
446  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
447 }
448 
449 static void
450 CopySendString(CopyState cstate, const char *str)
451 {
452  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
453 }
454 
455 static void
456 CopySendChar(CopyState cstate, char c)
457 {
459 }
460 
461 static void
463 {
464  StringInfo fe_msgbuf = cstate->fe_msgbuf;
465 
466  switch (cstate->copy_dest)
467  {
468  case COPY_FILE:
469  if (!cstate->binary)
470  {
471  /* Default line termination depends on platform */
472 #ifndef WIN32
473  CopySendChar(cstate, '\n');
474 #else
475  CopySendString(cstate, "\r\n");
476 #endif
477  }
478 
479  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
480  cstate->copy_file) != 1 ||
481  ferror(cstate->copy_file))
482  {
483  if (cstate->is_program)
484  {
485  if (errno == EPIPE)
486  {
487  /*
488  * The pipe will be closed automatically on error at
489  * the end of transaction, but we might get a better
490  * error message from the subprocess' exit code than
491  * just "Broken Pipe"
492  */
493  ClosePipeToProgram(cstate);
494 
495  /*
496  * If ClosePipeToProgram() didn't throw an error, the
497  * program terminated normally, but closed the pipe
498  * first. Restore errno, and throw an error.
499  */
500  errno = EPIPE;
501  }
502  ereport(ERROR,
504  errmsg("could not write to COPY program: %m")));
505  }
506  else
507  ereport(ERROR,
509  errmsg("could not write to COPY file: %m")));
510  }
511  break;
512  case COPY_OLD_FE:
513  /* The FE/BE protocol uses \n as newline for all platforms */
514  if (!cstate->binary)
515  CopySendChar(cstate, '\n');
516 
517  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
518  {
519  /* no hope of recovering connection sync, so FATAL */
520  ereport(FATAL,
521  (errcode(ERRCODE_CONNECTION_FAILURE),
522  errmsg("connection lost during COPY to stdout")));
523  }
524  break;
525  case COPY_NEW_FE:
526  /* The FE/BE protocol uses \n as newline for all platforms */
527  if (!cstate->binary)
528  CopySendChar(cstate, '\n');
529 
530  /* Dump the accumulated row as one CopyData message */
531  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
532  break;
533  case COPY_CALLBACK:
534  Assert(false); /* Not yet supported. */
535  break;
536  }
537 
538  resetStringInfo(fe_msgbuf);
539 }
540 
541 /*
542  * CopyGetData reads data from the source (file or frontend)
543  *
544  * We attempt to read at least minread, and at most maxread, bytes from
545  * the source. The actual number of bytes read is returned; if this is
546  * less than minread, EOF was detected.
547  *
548  * Note: when copying from the frontend, we expect a proper EOF mark per
549  * protocol; if the frontend simply drops the connection, we raise error.
550  * It seems unwise to allow the COPY IN to complete normally in that case.
551  *
552  * NB: no data conversion is applied here.
553  */
554 static int
555 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
556 {
557  int bytesread = 0;
558 
559  switch (cstate->copy_dest)
560  {
561  case COPY_FILE:
562  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
563  if (ferror(cstate->copy_file))
564  ereport(ERROR,
566  errmsg("could not read from COPY file: %m")));
567  break;
568  case COPY_OLD_FE:
569 
570  /*
571  * We cannot read more than minread bytes (which in practice is 1)
572  * because old protocol doesn't have any clear way of separating
573  * the COPY stream from following data. This is slow, but not any
574  * slower than the code path was originally, and we don't care
575  * much anymore about the performance of old protocol.
576  */
577  if (pq_getbytes((char *) databuf, minread))
578  {
579  /* Only a \. terminator is legal EOF in old protocol */
580  ereport(ERROR,
581  (errcode(ERRCODE_CONNECTION_FAILURE),
582  errmsg("unexpected EOF on client connection with an open transaction")));
583  }
584  bytesread = minread;
585  break;
586  case COPY_NEW_FE:
587  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
588  {
589  int avail;
590 
591  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
592  {
593  /* Try to receive another message */
594  int mtype;
595 
596  readmessage:
598  pq_startmsgread();
599  mtype = pq_getbyte();
600  if (mtype == EOF)
601  ereport(ERROR,
602  (errcode(ERRCODE_CONNECTION_FAILURE),
603  errmsg("unexpected EOF on client connection with an open transaction")));
604  if (pq_getmessage(cstate->fe_msgbuf, 0))
605  ereport(ERROR,
606  (errcode(ERRCODE_CONNECTION_FAILURE),
607  errmsg("unexpected EOF on client connection with an open transaction")));
609  switch (mtype)
610  {
611  case 'd': /* CopyData */
612  break;
613  case 'c': /* CopyDone */
614  /* COPY IN correctly terminated by frontend */
615  cstate->fe_eof = true;
616  return bytesread;
617  case 'f': /* CopyFail */
618  ereport(ERROR,
619  (errcode(ERRCODE_QUERY_CANCELED),
620  errmsg("COPY from stdin failed: %s",
621  pq_getmsgstring(cstate->fe_msgbuf))));
622  break;
623  case 'H': /* Flush */
624  case 'S': /* Sync */
625 
626  /*
627  * Ignore Flush/Sync for the convenience of client
628  * libraries (such as libpq) that may send those
629  * without noticing that the command they just
630  * sent was COPY.
631  */
632  goto readmessage;
633  default:
634  ereport(ERROR,
635  (errcode(ERRCODE_PROTOCOL_VIOLATION),
636  errmsg("unexpected message type 0x%02X during COPY from stdin",
637  mtype)));
638  break;
639  }
640  }
641  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
642  if (avail > maxread)
643  avail = maxread;
644  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
645  databuf = (void *) ((char *) databuf + avail);
646  maxread -= avail;
647  bytesread += avail;
648  }
649  break;
650  case COPY_CALLBACK:
651  bytesread = cstate->data_source_cb(databuf, minread, maxread);
652  break;
653  }
654 
655  return bytesread;
656 }
657 
658 
659 /*
660  * These functions do apply some data conversion
661  */
662 
663 /*
664  * CopySendInt32 sends an int32 in network byte order
665  */
666 static void
668 {
669  uint32 buf;
670 
671  buf = htonl((uint32) val);
672  CopySendData(cstate, &buf, sizeof(buf));
673 }
674 
675 /*
676  * CopyGetInt32 reads an int32 that appears in network byte order
677  *
678  * Returns true if OK, false if EOF
679  */
680 static bool
682 {
683  uint32 buf;
684 
685  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
686  {
687  *val = 0; /* suppress compiler warning */
688  return false;
689  }
690  *val = (int32) ntohl(buf);
691  return true;
692 }
693 
694 /*
695  * CopySendInt16 sends an int16 in network byte order
696  */
697 static void
699 {
700  uint16 buf;
701 
702  buf = htons((uint16) val);
703  CopySendData(cstate, &buf, sizeof(buf));
704 }
705 
706 /*
707  * CopyGetInt16 reads an int16 that appears in network byte order
708  */
709 static bool
711 {
712  uint16 buf;
713 
714  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
715  {
716  *val = 0; /* suppress compiler warning */
717  return false;
718  }
719  *val = (int16) ntohs(buf);
720  return true;
721 }
722 
723 
724 /*
725  * CopyLoadRawBuf loads some more data into raw_buf
726  *
727  * Returns TRUE if able to obtain at least one more byte, else FALSE.
728  *
729  * If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
730  * down to the start of the buffer and then we load more data after that.
731  * This case is used only when a frontend multibyte character crosses a
732  * bufferload boundary.
733  */
734 static bool
736 {
737  int nbytes;
738  int inbytes;
739 
740  if (cstate->raw_buf_index < cstate->raw_buf_len)
741  {
742  /* Copy down the unprocessed data */
743  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
744  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
745  nbytes);
746  }
747  else
748  nbytes = 0; /* no data need be saved */
749 
750  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
751  1, RAW_BUF_SIZE - nbytes);
752  nbytes += inbytes;
753  cstate->raw_buf[nbytes] = '\0';
754  cstate->raw_buf_index = 0;
755  cstate->raw_buf_len = nbytes;
756  return (inbytes > 0);
757 }
758 
759 
760 /*
761  * DoCopy executes the SQL COPY statement
762  *
763  * Either unload or reload contents of table <relation>, depending on <from>.
764  * (<from> = TRUE means we are inserting into the table.) In the "TO" case
765  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
766  * or DELETE query.
767  *
768  * If <pipe> is false, transfer is between the table and the file named
769  * <filename>. Otherwise, transfer is between the table and our regular
770  * input/output stream. The latter could be either stdin/stdout or a
771  * socket, depending on whether we're running under Postmaster control.
772  *
773  * Do not allow a Postgres user without superuser privilege to read from
774  * or write to a file.
775  *
776  * Do not allow the copy if user doesn't have proper permission to access
777  * the table or the specifically requested columns.
778  */
779 void
780 DoCopy(ParseState *pstate, const CopyStmt *stmt,
781  int stmt_location, int stmt_len,
782  uint64 *processed)
783 {
784  CopyState cstate;
785  bool is_from = stmt->is_from;
786  bool pipe = (stmt->filename == NULL);
787  Relation rel;
788  Oid relid;
789  RawStmt *query = NULL;
790  List *range_table = NIL;
791 
792  /* Disallow COPY to/from file or program except to superusers. */
793  if (!pipe && !superuser())
794  {
795  if (stmt->is_program)
796  ereport(ERROR,
797  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
798  errmsg("must be superuser to COPY to or from an external program"),
799  errhint("Anyone can COPY to stdout or from stdin. "
800  "psql's \\copy command also works for anyone.")));
801  else
802  ereport(ERROR,
803  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
804  errmsg("must be superuser to COPY to or from a file"),
805  errhint("Anyone can COPY to stdout or from stdin. "
806  "psql's \\copy command also works for anyone.")));
807  }
808 
809  if (stmt->relation)
810  {
811  TupleDesc tupDesc;
812  AclMode required_access = (is_from ? ACL_INSERT : ACL_SELECT);
813  List *attnums;
814  ListCell *cur;
815  RangeTblEntry *rte;
816 
817  Assert(!stmt->query);
818 
819  /* Open and lock the relation, using the appropriate lock type. */
820  rel = heap_openrv(stmt->relation,
821  (is_from ? RowExclusiveLock : AccessShareLock));
822 
823  relid = RelationGetRelid(rel);
824 
825  rte = makeNode(RangeTblEntry);
826  rte->rtekind = RTE_RELATION;
827  rte->relid = RelationGetRelid(rel);
828  rte->relkind = rel->rd_rel->relkind;
829  rte->requiredPerms = required_access;
830  range_table = list_make1(rte);
831 
832  tupDesc = RelationGetDescr(rel);
833  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
834  foreach(cur, attnums)
835  {
836  int attno = lfirst_int(cur) -
838 
839  if (is_from)
840  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
841  else
842  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
843  }
844  ExecCheckRTPerms(range_table, true);
845 
846  /*
847  * Permission check for row security policies.
848  *
849  * check_enable_rls will ereport(ERROR) if the user has requested
850  * something invalid and will otherwise indicate if we should enable
851  * RLS (returns RLS_ENABLED) or not for this COPY statement.
852  *
853  * If the relation has a row security policy and we are to apply it
854  * then perform a "query" copy and allow the normal query processing
855  * to handle the policies.
856  *
857  * If RLS is not enabled for this, then just fall through to the
858  * normal non-filtering relation handling.
859  */
860  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
861  {
863  ColumnRef *cr;
864  ResTarget *target;
865  RangeVar *from;
866  List *targetList = NIL;
867 
868  if (is_from)
869  ereport(ERROR,
870  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
871  errmsg("COPY FROM not supported with row-level security"),
872  errhint("Use INSERT statements instead.")));
873 
874  /*
875  * Build target list
876  *
877  * If no columns are specified in the attribute list of the COPY
878  * command, then the target list is 'all' columns. Therefore, '*'
879  * should be used as the target list for the resulting SELECT
880  * statement.
881  *
882  * In the case that columns are specified in the attribute list,
883  * create a ColumnRef and ResTarget for each column and add them
884  * to the target list for the resulting SELECT statement.
885  */
886  if (!stmt->attlist)
887  {
888  cr = makeNode(ColumnRef);
890  cr->location = -1;
891 
892  target = makeNode(ResTarget);
893  target->name = NULL;
894  target->indirection = NIL;
895  target->val = (Node *) cr;
896  target->location = -1;
897 
898  targetList = list_make1(target);
899  }
900  else
901  {
902  ListCell *lc;
903 
904  foreach(lc, stmt->attlist)
905  {
906  /*
907  * Build the ColumnRef for each column. The ColumnRef
908  * 'fields' property is a String 'Value' node (see
909  * nodes/value.h) that corresponds to the column name
910  * respectively.
911  */
912  cr = makeNode(ColumnRef);
913  cr->fields = list_make1(lfirst(lc));
914  cr->location = -1;
915 
916  /* Build the ResTarget and add the ColumnRef to it. */
917  target = makeNode(ResTarget);
918  target->name = NULL;
919  target->indirection = NIL;
920  target->val = (Node *) cr;
921  target->location = -1;
922 
923  /* Add each column to the SELECT statement's target list */
924  targetList = lappend(targetList, target);
925  }
926  }
927 
928  /*
929  * Build RangeVar for from clause, fully qualified based on the
930  * relation which we have opened and locked.
931  */
934  -1);
935 
936  /* Build query */
937  select = makeNode(SelectStmt);
938  select->targetList = targetList;
939  select->fromClause = list_make1(from);
940 
941  query = makeNode(RawStmt);
942  query->stmt = (Node *) select;
943  query->stmt_location = stmt_location;
944  query->stmt_len = stmt_len;
945 
946  /*
947  * Close the relation for now, but keep the lock on it to prevent
948  * changes between now and when we start the query-based COPY.
949  *
950  * We'll reopen it later as part of the query-based COPY.
951  */
952  heap_close(rel, NoLock);
953  rel = NULL;
954  }
955  }
956  else
957  {
958  Assert(stmt->query);
959 
960  query = makeNode(RawStmt);
961  query->stmt = stmt->query;
962  query->stmt_location = stmt_location;
963  query->stmt_len = stmt_len;
964 
965  relid = InvalidOid;
966  rel = NULL;
967  }
968 
969  if (is_from)
970  {
971  Assert(rel);
972 
973  /* check read-only transaction and parallel mode */
974  if (XactReadOnly && !rel->rd_islocaltemp)
975  PreventCommandIfReadOnly("COPY FROM");
976  PreventCommandIfParallelMode("COPY FROM");
977 
978  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
979  NULL, stmt->attlist, stmt->options);
980  cstate->range_table = range_table;
981  *processed = CopyFrom(cstate); /* copy from file to database */
982  EndCopyFrom(cstate);
983  }
984  else
985  {
986  cstate = BeginCopyTo(pstate, rel, query, relid,
987  stmt->filename, stmt->is_program,
988  stmt->attlist, stmt->options);
989  *processed = DoCopyTo(cstate); /* copy from database to file */
990  EndCopyTo(cstate);
991  }
992 
993  /*
994  * Close the relation. If reading, we can release the AccessShareLock we
995  * got; if writing, we should hold the lock until end of transaction to
996  * ensure that updates will be committed before lock is released.
997  */
998  if (rel != NULL)
999  heap_close(rel, (is_from ? NoLock : AccessShareLock));
1000 }
1001 
1002 /*
1003  * Process the statement option list for COPY.
1004  *
1005  * Scan the options list (a list of DefElem) and transpose the information
1006  * into cstate, applying appropriate error checking.
1007  *
1008  * cstate is assumed to be filled with zeroes initially.
1009  *
1010  * This is exported so that external users of the COPY API can sanity-check
1011  * a list of options. In that usage, cstate should be passed as NULL
1012  * (since external users don't know sizeof(CopyStateData)) and the collected
1013  * data is just leaked until CurrentMemoryContext is reset.
1014  *
1015  * Note that additional checking, such as whether column names listed in FORCE
1016  * QUOTE actually exist, has to be applied later. This just checks for
1017  * self-consistency of the options list.
1018  */
1019 void
1021  CopyState cstate,
1022  bool is_from,
1023  List *options)
1024 {
1025  bool format_specified = false;
1026  ListCell *option;
1027 
1028  /* Support external use for option sanity checking */
1029  if (cstate == NULL)
1030  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1031 
1032  cstate->file_encoding = -1;
1033 
1034  /* Extract options from the statement node tree */
1035  foreach(option, options)
1036  {
1037  DefElem *defel = castNode(DefElem, lfirst(option));
1038 
1039  if (strcmp(defel->defname, "format") == 0)
1040  {
1041  char *fmt = defGetString(defel);
1042 
1043  if (format_specified)
1044  ereport(ERROR,
1045  (errcode(ERRCODE_SYNTAX_ERROR),
1046  errmsg("conflicting or redundant options"),
1047  parser_errposition(pstate, defel->location)));
1048  format_specified = true;
1049  if (strcmp(fmt, "text") == 0)
1050  /* default format */ ;
1051  else if (strcmp(fmt, "csv") == 0)
1052  cstate->csv_mode = true;
1053  else if (strcmp(fmt, "binary") == 0)
1054  cstate->binary = true;
1055  else
1056  ereport(ERROR,
1057  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1058  errmsg("COPY format \"%s\" not recognized", fmt),
1059  parser_errposition(pstate, defel->location)));
1060  }
1061  else if (strcmp(defel->defname, "oids") == 0)
1062  {
1063  if (cstate->oids)
1064  ereport(ERROR,
1065  (errcode(ERRCODE_SYNTAX_ERROR),
1066  errmsg("conflicting or redundant options"),
1067  parser_errposition(pstate, defel->location)));
1068  cstate->oids = defGetBoolean(defel);
1069  }
1070  else if (strcmp(defel->defname, "freeze") == 0)
1071  {
1072  if (cstate->freeze)
1073  ereport(ERROR,
1074  (errcode(ERRCODE_SYNTAX_ERROR),
1075  errmsg("conflicting or redundant options"),
1076  parser_errposition(pstate, defel->location)));
1077  cstate->freeze = defGetBoolean(defel);
1078  }
1079  else if (strcmp(defel->defname, "delimiter") == 0)
1080  {
1081  if (cstate->delim)
1082  ereport(ERROR,
1083  (errcode(ERRCODE_SYNTAX_ERROR),
1084  errmsg("conflicting or redundant options"),
1085  parser_errposition(pstate, defel->location)));
1086  cstate->delim = defGetString(defel);
1087  }
1088  else if (strcmp(defel->defname, "null") == 0)
1089  {
1090  if (cstate->null_print)
1091  ereport(ERROR,
1092  (errcode(ERRCODE_SYNTAX_ERROR),
1093  errmsg("conflicting or redundant options"),
1094  parser_errposition(pstate, defel->location)));
1095  cstate->null_print = defGetString(defel);
1096  }
1097  else if (strcmp(defel->defname, "header") == 0)
1098  {
1099  if (cstate->header_line)
1100  ereport(ERROR,
1101  (errcode(ERRCODE_SYNTAX_ERROR),
1102  errmsg("conflicting or redundant options"),
1103  parser_errposition(pstate, defel->location)));
1104  cstate->header_line = defGetBoolean(defel);
1105  }
1106  else if (strcmp(defel->defname, "quote") == 0)
1107  {
1108  if (cstate->quote)
1109  ereport(ERROR,
1110  (errcode(ERRCODE_SYNTAX_ERROR),
1111  errmsg("conflicting or redundant options"),
1112  parser_errposition(pstate, defel->location)));
1113  cstate->quote = defGetString(defel);
1114  }
1115  else if (strcmp(defel->defname, "escape") == 0)
1116  {
1117  if (cstate->escape)
1118  ereport(ERROR,
1119  (errcode(ERRCODE_SYNTAX_ERROR),
1120  errmsg("conflicting or redundant options"),
1121  parser_errposition(pstate, defel->location)));
1122  cstate->escape = defGetString(defel);
1123  }
1124  else if (strcmp(defel->defname, "force_quote") == 0)
1125  {
1126  if (cstate->force_quote || cstate->force_quote_all)
1127  ereport(ERROR,
1128  (errcode(ERRCODE_SYNTAX_ERROR),
1129  errmsg("conflicting or redundant options"),
1130  parser_errposition(pstate, defel->location)));
1131  if (defel->arg && IsA(defel->arg, A_Star))
1132  cstate->force_quote_all = true;
1133  else if (defel->arg && IsA(defel->arg, List))
1134  cstate->force_quote = castNode(List, defel->arg);
1135  else
1136  ereport(ERROR,
1137  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1138  errmsg("argument to option \"%s\" must be a list of column names",
1139  defel->defname),
1140  parser_errposition(pstate, defel->location)));
1141  }
1142  else if (strcmp(defel->defname, "force_not_null") == 0)
1143  {
1144  if (cstate->force_notnull)
1145  ereport(ERROR,
1146  (errcode(ERRCODE_SYNTAX_ERROR),
1147  errmsg("conflicting or redundant options"),
1148  parser_errposition(pstate, defel->location)));
1149  if (defel->arg && IsA(defel->arg, List))
1150  cstate->force_notnull = castNode(List, defel->arg);
1151  else
1152  ereport(ERROR,
1153  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1154  errmsg("argument to option \"%s\" must be a list of column names",
1155  defel->defname),
1156  parser_errposition(pstate, defel->location)));
1157  }
1158  else if (strcmp(defel->defname, "force_null") == 0)
1159  {
1160  if (cstate->force_null)
1161  ereport(ERROR,
1162  (errcode(ERRCODE_SYNTAX_ERROR),
1163  errmsg("conflicting or redundant options")));
1164  if (defel->arg && IsA(defel->arg, List))
1165  cstate->force_null = castNode(List, defel->arg);
1166  else
1167  ereport(ERROR,
1168  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1169  errmsg("argument to option \"%s\" must be a list of column names",
1170  defel->defname),
1171  parser_errposition(pstate, defel->location)));
1172  }
1173  else if (strcmp(defel->defname, "convert_selectively") == 0)
1174  {
1175  /*
1176  * Undocumented, not-accessible-from-SQL option: convert only the
1177  * named columns to binary form, storing the rest as NULLs. It's
1178  * allowed for the column list to be NIL.
1179  */
1180  if (cstate->convert_selectively)
1181  ereport(ERROR,
1182  (errcode(ERRCODE_SYNTAX_ERROR),
1183  errmsg("conflicting or redundant options"),
1184  parser_errposition(pstate, defel->location)));
1185  cstate->convert_selectively = true;
1186  if (defel->arg == NULL || IsA(defel->arg, List))
1187  cstate->convert_select = castNode(List, defel->arg);
1188  else
1189  ereport(ERROR,
1190  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1191  errmsg("argument to option \"%s\" must be a list of column names",
1192  defel->defname),
1193  parser_errposition(pstate, defel->location)));
1194  }
1195  else if (strcmp(defel->defname, "encoding") == 0)
1196  {
1197  if (cstate->file_encoding >= 0)
1198  ereport(ERROR,
1199  (errcode(ERRCODE_SYNTAX_ERROR),
1200  errmsg("conflicting or redundant options"),
1201  parser_errposition(pstate, defel->location)));
1203  if (cstate->file_encoding < 0)
1204  ereport(ERROR,
1205  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1206  errmsg("argument to option \"%s\" must be a valid encoding name",
1207  defel->defname),
1208  parser_errposition(pstate, defel->location)));
1209  }
1210  else
1211  ereport(ERROR,
1212  (errcode(ERRCODE_SYNTAX_ERROR),
1213  errmsg("option \"%s\" not recognized",
1214  defel->defname),
1215  parser_errposition(pstate, defel->location)));
1216  }
1217 
1218  /*
1219  * Check for incompatible options (must do these two before inserting
1220  * defaults)
1221  */
1222  if (cstate->binary && cstate->delim)
1223  ereport(ERROR,
1224  (errcode(ERRCODE_SYNTAX_ERROR),
1225  errmsg("cannot specify DELIMITER in BINARY mode")));
1226 
1227  if (cstate->binary && cstate->null_print)
1228  ereport(ERROR,
1229  (errcode(ERRCODE_SYNTAX_ERROR),
1230  errmsg("cannot specify NULL in BINARY mode")));
1231 
1232  /* Set defaults for omitted options */
1233  if (!cstate->delim)
1234  cstate->delim = cstate->csv_mode ? "," : "\t";
1235 
1236  if (!cstate->null_print)
1237  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1238  cstate->null_print_len = strlen(cstate->null_print);
1239 
1240  if (cstate->csv_mode)
1241  {
1242  if (!cstate->quote)
1243  cstate->quote = "\"";
1244  if (!cstate->escape)
1245  cstate->escape = cstate->quote;
1246  }
1247 
1248  /* Only single-byte delimiter strings are supported. */
1249  if (strlen(cstate->delim) != 1)
1250  ereport(ERROR,
1251  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1252  errmsg("COPY delimiter must be a single one-byte character")));
1253 
1254  /* Disallow end-of-line characters */
1255  if (strchr(cstate->delim, '\r') != NULL ||
1256  strchr(cstate->delim, '\n') != NULL)
1257  ereport(ERROR,
1258  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1259  errmsg("COPY delimiter cannot be newline or carriage return")));
1260 
1261  if (strchr(cstate->null_print, '\r') != NULL ||
1262  strchr(cstate->null_print, '\n') != NULL)
1263  ereport(ERROR,
1264  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1265  errmsg("COPY null representation cannot use newline or carriage return")));
1266 
1267  /*
1268  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1269  * backslash because it would be ambiguous. We can't allow the other
1270  * cases because data characters matching the delimiter must be
1271  * backslashed, and certain backslash combinations are interpreted
1272  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1273  * more than strictly necessary, but seems best for consistency and
1274  * future-proofing. Likewise we disallow all digits though only octal
1275  * digits are actually dangerous.
1276  */
1277  if (!cstate->csv_mode &&
1278  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1279  cstate->delim[0]) != NULL)
1280  ereport(ERROR,
1281  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1282  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1283 
1284  /* Check header */
1285  if (!cstate->csv_mode && cstate->header_line)
1286  ereport(ERROR,
1287  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1288  errmsg("COPY HEADER available only in CSV mode")));
1289 
1290  /* Check quote */
1291  if (!cstate->csv_mode && cstate->quote != NULL)
1292  ereport(ERROR,
1293  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1294  errmsg("COPY quote available only in CSV mode")));
1295 
1296  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1297  ereport(ERROR,
1298  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1299  errmsg("COPY quote must be a single one-byte character")));
1300 
1301  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1302  ereport(ERROR,
1303  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1304  errmsg("COPY delimiter and quote must be different")));
1305 
1306  /* Check escape */
1307  if (!cstate->csv_mode && cstate->escape != NULL)
1308  ereport(ERROR,
1309  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1310  errmsg("COPY escape available only in CSV mode")));
1311 
1312  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1313  ereport(ERROR,
1314  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1315  errmsg("COPY escape must be a single one-byte character")));
1316 
1317  /* Check force_quote */
1318  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1319  ereport(ERROR,
1320  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1321  errmsg("COPY force quote available only in CSV mode")));
1322  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1323  ereport(ERROR,
1324  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1325  errmsg("COPY force quote only available using COPY TO")));
1326 
1327  /* Check force_notnull */
1328  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1329  ereport(ERROR,
1330  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1331  errmsg("COPY force not null available only in CSV mode")));
1332  if (cstate->force_notnull != NIL && !is_from)
1333  ereport(ERROR,
1334  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1335  errmsg("COPY force not null only available using COPY FROM")));
1336 
1337  /* Check force_null */
1338  if (!cstate->csv_mode && cstate->force_null != NIL)
1339  ereport(ERROR,
1340  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1341  errmsg("COPY force null available only in CSV mode")));
1342 
1343  if (cstate->force_null != NIL && !is_from)
1344  ereport(ERROR,
1345  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1346  errmsg("COPY force null only available using COPY FROM")));
1347 
1348  /* Don't allow the delimiter to appear in the null string. */
1349  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1350  ereport(ERROR,
1351  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1352  errmsg("COPY delimiter must not appear in the NULL specification")));
1353 
1354  /* Don't allow the CSV quote char to appear in the null string. */
1355  if (cstate->csv_mode &&
1356  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1357  ereport(ERROR,
1358  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1359  errmsg("CSV quote character must not appear in the NULL specification")));
1360 }
1361 
1362 /*
1363  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1364  *
1365  * Iff <binary>, unload or reload in the binary format, as opposed to the
1366  * more wasteful but more robust and portable text format.
1367  *
1368  * Iff <oids>, unload or reload the format that includes OID information.
1369  * On input, we accept OIDs whether or not the table has an OID column,
1370  * but silently drop them if it does not. On output, we report an error
1371  * if the user asks for OIDs in a table that has none (not providing an
1372  * OID column might seem friendlier, but could seriously confuse programs).
1373  *
1374  * If in the text format, delimit columns with delimiter <delim> and print
1375  * NULL values as <null_print>.
1376  */
1377 static CopyState
1379  bool is_from,
1380  Relation rel,
1381  RawStmt *raw_query,
1382  Oid queryRelId,
1383  List *attnamelist,
1384  List *options)
1385 {
1386  CopyState cstate;
1387  TupleDesc tupDesc;
1388  int num_phys_attrs;
1389  MemoryContext oldcontext;
1390 
1391  /* Allocate workspace and zero all fields */
1392  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1393 
1394  /*
1395  * We allocate everything used by a cstate in a new memory context. This
1396  * avoids memory leaks during repeated use of COPY in a query.
1397  */
1399  "COPY",
1401 
1402  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1403 
1404  /* Extract options from the statement node tree */
1405  ProcessCopyOptions(pstate, cstate, is_from, options);
1406 
1407  /* Process the source/target relation or query */
1408  if (rel)
1409  {
1410  Assert(!raw_query);
1411 
1412  cstate->rel = rel;
1413 
1414  tupDesc = RelationGetDescr(cstate->rel);
1415 
1416  /* Don't allow COPY w/ OIDs to or from a table without them */
1417  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1418  ereport(ERROR,
1419  (errcode(ERRCODE_UNDEFINED_COLUMN),
1420  errmsg("table \"%s\" does not have OIDs",
1421  RelationGetRelationName(cstate->rel))));
1422 
1423  /* Initialize state for CopyFrom tuple routing. */
1424  if (is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1425  {
1426  PartitionDispatch *partition_dispatch_info;
1427  ResultRelInfo *partitions;
1428  TupleConversionMap **partition_tupconv_maps;
1429  TupleTableSlot *partition_tuple_slot;
1430  int num_parted,
1431  num_partitions;
1432 
1434  &partition_dispatch_info,
1435  &partitions,
1436  &partition_tupconv_maps,
1437  &partition_tuple_slot,
1438  &num_parted, &num_partitions);
1439  cstate->partition_dispatch_info = partition_dispatch_info;
1440  cstate->num_dispatch = num_parted;
1441  cstate->partitions = partitions;
1442  cstate->num_partitions = num_partitions;
1443  cstate->partition_tupconv_maps = partition_tupconv_maps;
1444  cstate->partition_tuple_slot = partition_tuple_slot;
1445  }
1446  }
1447  else
1448  {
1449  List *rewritten;
1450  Query *query;
1451  PlannedStmt *plan;
1452  DestReceiver *dest;
1453 
1454  Assert(!is_from);
1455  cstate->rel = NULL;
1456 
1457  /* Don't allow COPY w/ OIDs from a query */
1458  if (cstate->oids)
1459  ereport(ERROR,
1460  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1461  errmsg("COPY (query) WITH OIDS is not supported")));
1462 
1463  /*
1464  * Run parse analysis and rewrite. Note this also acquires sufficient
1465  * locks on the source table(s).
1466  *
1467  * Because the parser and planner tend to scribble on their input, we
1468  * make a preliminary copy of the source querytree. This prevents
1469  * problems in the case that the COPY is in a portal or plpgsql
1470  * function and is executed repeatedly. (See also the same hack in
1471  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1472  */
1473  rewritten = pg_analyze_and_rewrite((RawStmt *) copyObject(raw_query),
1474  pstate->p_sourcetext, NULL, 0);
1475 
1476  /* check that we got back something we can work with */
1477  if (rewritten == NIL)
1478  {
1479  ereport(ERROR,
1480  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1481  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1482  }
1483  else if (list_length(rewritten) > 1)
1484  {
1485  ListCell *lc;
1486 
1487  /* examine queries to determine which error message to issue */
1488  foreach(lc, rewritten)
1489  {
1490  Query *q = castNode(Query, lfirst(lc));
1491 
1493  ereport(ERROR,
1494  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1495  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1497  ereport(ERROR,
1498  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1499  errmsg("DO ALSO rules are not supported for the COPY")));
1500  }
1501 
1502  ereport(ERROR,
1503  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1504  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1505  }
1506 
1507  query = castNode(Query, linitial(rewritten));
1508 
1509  /* The grammar allows SELECT INTO, but we don't support that */
1510  if (query->utilityStmt != NULL &&
1512  ereport(ERROR,
1513  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1514  errmsg("COPY (SELECT INTO) is not supported")));
1515 
1516  Assert(query->utilityStmt == NULL);
1517 
1518  /*
1519  * Similarly the grammar doesn't enforce the presence of a RETURNING
1520  * clause, but this is required here.
1521  */
1522  if (query->commandType != CMD_SELECT &&
1523  query->returningList == NIL)
1524  {
1525  Assert(query->commandType == CMD_INSERT ||
1526  query->commandType == CMD_UPDATE ||
1527  query->commandType == CMD_DELETE);
1528 
1529  ereport(ERROR,
1530  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1531  errmsg("COPY query must have a RETURNING clause")));
1532  }
1533 
1534  /* plan the query */
1535  plan = pg_plan_query(query, 0, NULL);
1536 
1537  /*
1538  * With row level security and a user using "COPY relation TO", we
1539  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1540  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1541  * in any RLS clauses.
1542  *
1543  * When this happens, we are passed in the relid of the originally
1544  * found relation (which we have locked). As the planner will look up
1545  * the relation again, we double-check here to make sure it found the
1546  * same one that we have locked.
1547  */
1548  if (queryRelId != InvalidOid)
1549  {
1550  /*
1551  * Note that with RLS involved there may be multiple relations,
1552  * and while the one we need is almost certainly first, we don't
1553  * make any guarantees of that in the planner, so check the whole
1554  * list and make sure we find the original relation.
1555  */
1556  if (!list_member_oid(plan->relationOids, queryRelId))
1557  ereport(ERROR,
1558  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1559  errmsg("relation referenced by COPY statement has changed")));
1560  }
1561 
1562  /*
1563  * Use a snapshot with an updated command ID to ensure this query sees
1564  * results of any previously executed queries.
1565  */
1568 
1569  /* Create dest receiver for COPY OUT */
1571  ((DR_copy *) dest)->cstate = cstate;
1572 
1573  /* Create a QueryDesc requesting no output */
1574  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1577  dest, NULL, 0);
1578 
1579  /*
1580  * Call ExecutorStart to prepare the plan for execution.
1581  *
1582  * ExecutorStart computes a result tupdesc for us
1583  */
1584  ExecutorStart(cstate->queryDesc, 0);
1585 
1586  tupDesc = cstate->queryDesc->tupDesc;
1587  }
1588 
1589  /* Generate or convert list of attributes to process */
1590  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1591 
1592  num_phys_attrs = tupDesc->natts;
1593 
1594  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1595  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1596  if (cstate->force_quote_all)
1597  {
1598  int i;
1599 
1600  for (i = 0; i < num_phys_attrs; i++)
1601  cstate->force_quote_flags[i] = true;
1602  }
1603  else if (cstate->force_quote)
1604  {
1605  List *attnums;
1606  ListCell *cur;
1607 
1608  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1609 
1610  foreach(cur, attnums)
1611  {
1612  int attnum = lfirst_int(cur);
1613 
1614  if (!list_member_int(cstate->attnumlist, attnum))
1615  ereport(ERROR,
1616  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1617  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1618  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1619  cstate->force_quote_flags[attnum - 1] = true;
1620  }
1621  }
1622 
1623  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1624  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1625  if (cstate->force_notnull)
1626  {
1627  List *attnums;
1628  ListCell *cur;
1629 
1630  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1631 
1632  foreach(cur, attnums)
1633  {
1634  int attnum = lfirst_int(cur);
1635 
1636  if (!list_member_int(cstate->attnumlist, attnum))
1637  ereport(ERROR,
1638  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1639  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1640  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1641  cstate->force_notnull_flags[attnum - 1] = true;
1642  }
1643  }
1644 
1645  /* Convert FORCE_NULL name list to per-column flags, check validity */
1646  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1647  if (cstate->force_null)
1648  {
1649  List *attnums;
1650  ListCell *cur;
1651 
1652  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1653 
1654  foreach(cur, attnums)
1655  {
1656  int attnum = lfirst_int(cur);
1657 
1658  if (!list_member_int(cstate->attnumlist, attnum))
1659  ereport(ERROR,
1660  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1661  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1662  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1663  cstate->force_null_flags[attnum - 1] = true;
1664  }
1665  }
1666 
1667  /* Convert convert_selectively name list to per-column flags */
1668  if (cstate->convert_selectively)
1669  {
1670  List *attnums;
1671  ListCell *cur;
1672 
1673  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1674 
1675  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1676 
1677  foreach(cur, attnums)
1678  {
1679  int attnum = lfirst_int(cur);
1680 
1681  if (!list_member_int(cstate->attnumlist, attnum))
1682  ereport(ERROR,
1683  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1684  errmsg_internal("selected column \"%s\" not referenced by COPY",
1685  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1686  cstate->convert_select_flags[attnum - 1] = true;
1687  }
1688  }
1689 
1690  /* Use client encoding when ENCODING option is not specified. */
1691  if (cstate->file_encoding < 0)
1693 
1694  /*
1695  * Set up encoding conversion info. Even if the file and server encodings
1696  * are the same, we must apply pg_any_to_server() to validate data in
1697  * multibyte encodings.
1698  */
1699  cstate->need_transcoding =
1700  (cstate->file_encoding != GetDatabaseEncoding() ||
1702  /* See Multibyte encoding comment above */
1704 
1705  cstate->copy_dest = COPY_FILE; /* default */
1706 
1707  MemoryContextSwitchTo(oldcontext);
1708 
1709  return cstate;
1710 }
1711 
1712 /*
1713  * Closes the pipe to an external program, checking the pclose() return code.
1714  */
1715 static void
1717 {
1718  int pclose_rc;
1719 
1720  Assert(cstate->is_program);
1721 
1722  pclose_rc = ClosePipeStream(cstate->copy_file);
1723  if (pclose_rc == -1)
1724  ereport(ERROR,
1726  errmsg("could not close pipe to external command: %m")));
1727  else if (pclose_rc != 0)
1728  ereport(ERROR,
1729  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1730  errmsg("program \"%s\" failed",
1731  cstate->filename),
1732  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1733 }
1734 
1735 /*
1736  * Release resources allocated in a cstate for COPY TO/FROM.
1737  */
1738 static void
1740 {
1741  if (cstate->is_program)
1742  {
1743  ClosePipeToProgram(cstate);
1744  }
1745  else
1746  {
1747  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1748  ereport(ERROR,
1750  errmsg("could not close file \"%s\": %m",
1751  cstate->filename)));
1752  }
1753 
1755  pfree(cstate);
1756 }
1757 
1758 /*
1759  * Setup CopyState to read tuples from a table or a query for COPY TO.
1760  */
1761 static CopyState
1763  Relation rel,
1764  RawStmt *query,
1765  Oid queryRelId,
1766  const char *filename,
1767  bool is_program,
1768  List *attnamelist,
1769  List *options)
1770 {
1771  CopyState cstate;
1772  bool pipe = (filename == NULL);
1773  MemoryContext oldcontext;
1774 
1775  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1776  {
1777  if (rel->rd_rel->relkind == RELKIND_VIEW)
1778  ereport(ERROR,
1779  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1780  errmsg("cannot copy from view \"%s\"",
1782  errhint("Try the COPY (SELECT ...) TO variant.")));
1783  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1784  ereport(ERROR,
1785  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1786  errmsg("cannot copy from materialized view \"%s\"",
1788  errhint("Try the COPY (SELECT ...) TO variant.")));
1789  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1790  ereport(ERROR,
1791  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1792  errmsg("cannot copy from foreign table \"%s\"",
1794  errhint("Try the COPY (SELECT ...) TO variant.")));
1795  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1796  ereport(ERROR,
1797  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1798  errmsg("cannot copy from sequence \"%s\"",
1799  RelationGetRelationName(rel))));
1800  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1801  ereport(ERROR,
1802  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1803  errmsg("cannot copy from partitioned table \"%s\"",
1805  errhint("Try the COPY (SELECT ...) TO variant.")));
1806  else
1807  ereport(ERROR,
1808  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1809  errmsg("cannot copy from non-table relation \"%s\"",
1810  RelationGetRelationName(rel))));
1811  }
1812 
1813  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1814  options);
1815  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1816 
1817  if (pipe)
1818  {
1819  Assert(!is_program); /* the grammar does not allow this */
1821  cstate->copy_file = stdout;
1822  }
1823  else
1824  {
1825  cstate->filename = pstrdup(filename);
1826  cstate->is_program = is_program;
1827 
1828  if (is_program)
1829  {
1830  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1831  if (cstate->copy_file == NULL)
1832  ereport(ERROR,
1834  errmsg("could not execute command \"%s\": %m",
1835  cstate->filename)));
1836  }
1837  else
1838  {
1839  mode_t oumask; /* Pre-existing umask value */
1840  struct stat st;
1841 
1842  /*
1843  * Prevent write to relative path ... too easy to shoot oneself in
1844  * the foot by overwriting a database file ...
1845  */
1846  if (!is_absolute_path(filename))
1847  ereport(ERROR,
1848  (errcode(ERRCODE_INVALID_NAME),
1849  errmsg("relative path not allowed for COPY to file")));
1850 
1851  oumask = umask(S_IWGRP | S_IWOTH);
1852  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1853  umask(oumask);
1854  if (cstate->copy_file == NULL)
1855  {
1856  /* copy errno because ereport subfunctions might change it */
1857  int save_errno = errno;
1858 
1859  ereport(ERROR,
1861  errmsg("could not open file \"%s\" for writing: %m",
1862  cstate->filename),
1863  (save_errno == ENOENT || save_errno == EACCES) ?
1864  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1865  "You may want a client-side facility such as psql's \\copy.") : 0));
1866  }
1867 
1868  if (fstat(fileno(cstate->copy_file), &st))
1869  ereport(ERROR,
1871  errmsg("could not stat file \"%s\": %m",
1872  cstate->filename)));
1873 
1874  if (S_ISDIR(st.st_mode))
1875  ereport(ERROR,
1876  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1877  errmsg("\"%s\" is a directory", cstate->filename)));
1878  }
1879  }
1880 
1881  MemoryContextSwitchTo(oldcontext);
1882 
1883  return cstate;
1884 }
1885 
1886 /*
1887  * This intermediate routine exists mainly to localize the effects of setjmp
1888  * so we don't need to plaster a lot of variables with "volatile".
1889  */
1890 static uint64
1892 {
1893  bool pipe = (cstate->filename == NULL);
1894  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
1895  uint64 processed;
1896 
1897  PG_TRY();
1898  {
1899  if (fe_copy)
1900  SendCopyBegin(cstate);
1901 
1902  processed = CopyTo(cstate);
1903 
1904  if (fe_copy)
1905  SendCopyEnd(cstate);
1906  }
1907  PG_CATCH();
1908  {
1909  /*
1910  * Make sure we turn off old-style COPY OUT mode upon error. It is
1911  * okay to do this in all cases, since it does nothing if the mode is
1912  * not on.
1913  */
1914  pq_endcopyout(true);
1915  PG_RE_THROW();
1916  }
1917  PG_END_TRY();
1918 
1919  return processed;
1920 }
1921 
1922 /*
1923  * Clean up storage and release resources for COPY TO.
1924  */
1925 static void
1927 {
1928  if (cstate->queryDesc != NULL)
1929  {
1930  /* Close down the query and free resources. */
1931  ExecutorFinish(cstate->queryDesc);
1932  ExecutorEnd(cstate->queryDesc);
1933  FreeQueryDesc(cstate->queryDesc);
1935  }
1936 
1937  /* Clean up storage */
1938  EndCopy(cstate);
1939 }
1940 
1941 /*
1942  * Copy from relation or query TO file.
1943  */
1944 static uint64
1946 {
1947  TupleDesc tupDesc;
1948  int num_phys_attrs;
1949  Form_pg_attribute *attr;
1950  ListCell *cur;
1951  uint64 processed;
1952 
1953  if (cstate->rel)
1954  tupDesc = RelationGetDescr(cstate->rel);
1955  else
1956  tupDesc = cstate->queryDesc->tupDesc;
1957  attr = tupDesc->attrs;
1958  num_phys_attrs = tupDesc->natts;
1959  cstate->null_print_client = cstate->null_print; /* default */
1960 
1961  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1962  cstate->fe_msgbuf = makeLongStringInfo();
1963 
1964  /* Get info about the columns we need to process. */
1965  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1966  foreach(cur, cstate->attnumlist)
1967  {
1968  int attnum = lfirst_int(cur);
1969  Oid out_func_oid;
1970  bool isvarlena;
1971 
1972  if (cstate->binary)
1973  getTypeBinaryOutputInfo(attr[attnum - 1]->atttypid,
1974  &out_func_oid,
1975  &isvarlena);
1976  else
1977  getTypeOutputInfo(attr[attnum - 1]->atttypid,
1978  &out_func_oid,
1979  &isvarlena);
1980  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
1981  }
1982 
1983  /*
1984  * Create a temporary memory context that we can reset once per row to
1985  * recover palloc'd memory. This avoids any problems with leaks inside
1986  * datatype output routines, and should be faster than retail pfree's
1987  * anyway. (We don't need a whole econtext as CopyFrom does.)
1988  */
1990  "COPY TO",
1992 
1993  if (cstate->binary)
1994  {
1995  /* Generate header for a binary copy */
1996  int32 tmp;
1997 
1998  /* Signature */
1999  CopySendData(cstate, BinarySignature, 11);
2000  /* Flags field */
2001  tmp = 0;
2002  if (cstate->oids)
2003  tmp |= (1 << 16);
2004  CopySendInt32(cstate, tmp);
2005  /* No header extension */
2006  tmp = 0;
2007  CopySendInt32(cstate, tmp);
2008  }
2009  else
2010  {
2011  /*
2012  * For non-binary copy, we need to convert null_print to file
2013  * encoding, because it will be sent directly with CopySendString.
2014  */
2015  if (cstate->need_transcoding)
2016  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2017  cstate->null_print_len,
2018  cstate->file_encoding);
2019 
2020  /* if a header has been requested send the line */
2021  if (cstate->header_line)
2022  {
2023  bool hdr_delim = false;
2024 
2025  foreach(cur, cstate->attnumlist)
2026  {
2027  int attnum = lfirst_int(cur);
2028  char *colname;
2029 
2030  if (hdr_delim)
2031  CopySendChar(cstate, cstate->delim[0]);
2032  hdr_delim = true;
2033 
2034  colname = NameStr(attr[attnum - 1]->attname);
2035 
2036  CopyAttributeOutCSV(cstate, colname, false,
2037  list_length(cstate->attnumlist) == 1);
2038  }
2039 
2040  CopySendEndOfRow(cstate);
2041  }
2042  }
2043 
2044  if (cstate->rel)
2045  {
2046  Datum *values;
2047  bool *nulls;
2048  HeapScanDesc scandesc;
2049  HeapTuple tuple;
2050 
2051  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2052  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2053 
2054  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2055 
2056  processed = 0;
2057  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2058  {
2060 
2061  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2062  heap_deform_tuple(tuple, tupDesc, values, nulls);
2063 
2064  /* Format and send the data */
2065  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2066  processed++;
2067  }
2068 
2069  heap_endscan(scandesc);
2070 
2071  pfree(values);
2072  pfree(nulls);
2073  }
2074  else
2075  {
2076  /* run the plan --- the dest receiver will send tuples */
2077  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2078  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2079  }
2080 
2081  if (cstate->binary)
2082  {
2083  /* Generate trailer for a binary copy */
2084  CopySendInt16(cstate, -1);
2085  /* Need to flush out the trailer */
2086  CopySendEndOfRow(cstate);
2087  }
2088 
2090 
2091  return processed;
2092 }
2093 
2094 /*
2095  * Emit one row during CopyTo().
2096  */
2097 static void
2098 CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
2099 {
2100  bool need_delim = false;
2101  FmgrInfo *out_functions = cstate->out_functions;
2102  MemoryContext oldcontext;
2103  ListCell *cur;
2104  char *string;
2105 
2106  MemoryContextReset(cstate->rowcontext);
2107  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2108 
2109  if (cstate->binary)
2110  {
2111  /* Binary per-tuple header */
2112  CopySendInt16(cstate, list_length(cstate->attnumlist));
2113  /* Send OID if wanted --- note attnumlist doesn't include it */
2114  if (cstate->oids)
2115  {
2116  /* Hack --- assume Oid is same size as int32 */
2117  CopySendInt32(cstate, sizeof(int32));
2118  CopySendInt32(cstate, tupleOid);
2119  }
2120  }
2121  else
2122  {
2123  /* Text format has no per-tuple header, but send OID if wanted */
2124  /* Assume digits don't need any quoting or encoding conversion */
2125  if (cstate->oids)
2126  {
2128  ObjectIdGetDatum(tupleOid)));
2129  CopySendString(cstate, string);
2130  need_delim = true;
2131  }
2132  }
2133 
2134  foreach(cur, cstate->attnumlist)
2135  {
2136  int attnum = lfirst_int(cur);
2137  Datum value = values[attnum - 1];
2138  bool isnull = nulls[attnum - 1];
2139 
2140  if (!cstate->binary)
2141  {
2142  if (need_delim)
2143  CopySendChar(cstate, cstate->delim[0]);
2144  need_delim = true;
2145  }
2146 
2147  if (isnull)
2148  {
2149  if (!cstate->binary)
2150  CopySendString(cstate, cstate->null_print_client);
2151  else
2152  CopySendInt32(cstate, -1);
2153  }
2154  else
2155  {
2156  if (!cstate->binary)
2157  {
2158  string = OutputFunctionCall(&out_functions[attnum - 1],
2159  value);
2160  if (cstate->csv_mode)
2161  CopyAttributeOutCSV(cstate, string,
2162  cstate->force_quote_flags[attnum - 1],
2163  list_length(cstate->attnumlist) == 1);
2164  else
2165  CopyAttributeOutText(cstate, string);
2166  }
2167  else
2168  {
2169  bytea *outputbytes;
2170 
2171  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2172  value);
2173  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2174  CopySendData(cstate, VARDATA(outputbytes),
2175  VARSIZE(outputbytes) - VARHDRSZ);
2176  }
2177  }
2178  }
2179 
2180  CopySendEndOfRow(cstate);
2181 
2182  MemoryContextSwitchTo(oldcontext);
2183 }
2184 
2185 
2186 /*
2187  * error context callback for COPY FROM
2188  *
2189  * The argument for the error context must be CopyState.
2190  */
2191 void
2193 {
2194  CopyState cstate = (CopyState) arg;
2195 
2196  if (cstate->binary)
2197  {
2198  /* can't usefully display the data */
2199  if (cstate->cur_attname)
2200  errcontext("COPY %s, line %d, column %s",
2201  cstate->cur_relname, cstate->cur_lineno,
2202  cstate->cur_attname);
2203  else
2204  errcontext("COPY %s, line %d",
2205  cstate->cur_relname, cstate->cur_lineno);
2206  }
2207  else
2208  {
2209  if (cstate->cur_attname && cstate->cur_attval)
2210  {
2211  /* error is relevant to a particular column */
2212  char *attval;
2213 
2214  attval = limit_printout_length(cstate->cur_attval);
2215  errcontext("COPY %s, line %d, column %s: \"%s\"",
2216  cstate->cur_relname, cstate->cur_lineno,
2217  cstate->cur_attname, attval);
2218  pfree(attval);
2219  }
2220  else if (cstate->cur_attname)
2221  {
2222  /* error is relevant to a particular column, value is NULL */
2223  errcontext("COPY %s, line %d, column %s: null input",
2224  cstate->cur_relname, cstate->cur_lineno,
2225  cstate->cur_attname);
2226  }
2227  else
2228  {
2229  /*
2230  * Error is relevant to a particular line.
2231  *
2232  * If line_buf still contains the correct line, and it's already
2233  * transcoded, print it. If it's still in a foreign encoding, it's
2234  * quite likely that the error is precisely a failure to do
2235  * encoding conversion (ie, bad data). We dare not try to convert
2236  * it, and at present there's no way to regurgitate it without
2237  * conversion. So we have to punt and just report the line number.
2238  */
2239  if (cstate->line_buf_valid &&
2240  (cstate->line_buf_converted || !cstate->need_transcoding))
2241  {
2242  char *lineval;
2243 
2244  lineval = limit_printout_length(cstate->line_buf.data);
2245  errcontext("COPY %s, line %d: \"%s\"",
2246  cstate->cur_relname, cstate->cur_lineno, lineval);
2247  pfree(lineval);
2248  }
2249  else
2250  {
2251  errcontext("COPY %s, line %d",
2252  cstate->cur_relname, cstate->cur_lineno);
2253  }
2254  }
2255  }
2256 }
2257 
2258 /*
2259  * Make sure we don't print an unreasonable amount of COPY data in a message.
2260  *
2261  * It would seem a lot easier to just use the sprintf "precision" limit to
2262  * truncate the string. However, some versions of glibc have a bug/misfeature
2263  * that vsnprintf will always fail (return -1) if it is asked to truncate
2264  * a string that contains invalid byte sequences for the current encoding.
2265  * So, do our own truncation. We return a pstrdup'd copy of the input.
2266  */
2267 static char *
2268 limit_printout_length(const char *str)
2269 {
2270 #define MAX_COPY_DATA_DISPLAY 100
2271 
2272  int slen = strlen(str);
2273  int len;
2274  char *res;
2275 
2276  /* Fast path if definitely okay */
2277  if (slen <= MAX_COPY_DATA_DISPLAY)
2278  return pstrdup(str);
2279 
2280  /* Apply encoding-dependent truncation */
2281  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2282 
2283  /*
2284  * Truncate, and add "..." to show we truncated the input.
2285  */
2286  res = (char *) palloc(len + 4);
2287  memcpy(res, str, len);
2288  strcpy(res + len, "...");
2289 
2290  return res;
2291 }
2292 
2293 /*
2294  * Copy FROM file to relation.
2295  */
2296 uint64
2298 {
2299  HeapTuple tuple;
2300  TupleDesc tupDesc;
2301  Datum *values;
2302  bool *nulls;
2303  ResultRelInfo *resultRelInfo;
2304  ResultRelInfo *saved_resultRelInfo = NULL;
2305  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2306  ExprContext *econtext;
2307  TupleTableSlot *myslot;
2308  MemoryContext oldcontext = CurrentMemoryContext;
2309 
2310  ErrorContextCallback errcallback;
2311  CommandId mycid = GetCurrentCommandId(true);
2312  int hi_options = 0; /* start with default heap_insert options */
2313  BulkInsertState bistate;
2314  uint64 processed = 0;
2315  bool useHeapMultiInsert;
2316  int nBufferedTuples = 0;
2317  int prev_leaf_part_index = -1;
2318 
2319 #define MAX_BUFFERED_TUPLES 1000
2320  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2321  Size bufferedTuplesSize = 0;
2322  int firstBufferedLineNo = 0;
2323 
2324  Assert(cstate->rel);
2325 
2326  /*
2327  * The target must be a plain relation or have an INSTEAD OF INSERT row
2328  * trigger. (Currently, such triggers are only allowed on views, so we
2329  * only hint about them in the view case.)
2330  */
2331  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2332  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2333  !(cstate->rel->trigdesc &&
2335  {
2336  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2337  ereport(ERROR,
2338  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2339  errmsg("cannot copy to view \"%s\"",
2340  RelationGetRelationName(cstate->rel)),
2341  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2342  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2343  ereport(ERROR,
2344  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2345  errmsg("cannot copy to materialized view \"%s\"",
2346  RelationGetRelationName(cstate->rel))));
2347  else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
2348  ereport(ERROR,
2349  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2350  errmsg("cannot copy to foreign table \"%s\"",
2351  RelationGetRelationName(cstate->rel))));
2352  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2353  ereport(ERROR,
2354  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2355  errmsg("cannot copy to sequence \"%s\"",
2356  RelationGetRelationName(cstate->rel))));
2357  else
2358  ereport(ERROR,
2359  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2360  errmsg("cannot copy to non-table relation \"%s\"",
2361  RelationGetRelationName(cstate->rel))));
2362  }
2363 
2364  tupDesc = RelationGetDescr(cstate->rel);
2365 
2366  /*----------
2367  * Check to see if we can avoid writing WAL
2368  *
2369  * If archive logging/streaming is not enabled *and* either
2370  * - table was created in same transaction as this COPY
2371  * - data is being written to relfilenode created in this transaction
2372  * then we can skip writing WAL. It's safe because if the transaction
2373  * doesn't commit, we'll discard the table (or the new relfilenode file).
2374  * If it does commit, we'll have done the heap_sync at the bottom of this
2375  * routine first.
2376  *
2377  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2378  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2379  * can be cleared before the end of the transaction. The exact case is
2380  * when a relation sets a new relfilenode twice in same transaction, yet
2381  * the second one fails in an aborted subtransaction, e.g.
2382  *
2383  * BEGIN;
2384  * TRUNCATE t;
2385  * SAVEPOINT save;
2386  * TRUNCATE t;
2387  * ROLLBACK TO save;
2388  * COPY ...
2389  *
2390  * Also, if the target file is new-in-transaction, we assume that checking
2391  * FSM for free space is a waste of time, even if we must use WAL because
2392  * of archiving. This could possibly be wrong, but it's unlikely.
2393  *
2394  * The comments for heap_insert and RelationGetBufferForTuple specify that
2395  * skipping WAL logging is only safe if we ensure that our tuples do not
2396  * go into pages containing tuples from any other transactions --- but this
2397  * must be the case if we have a new table or new relfilenode, so we need
2398  * no additional work to enforce that.
2399  *----------
2400  */
2401  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2402  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2404  {
2405  hi_options |= HEAP_INSERT_SKIP_FSM;
2406  if (!XLogIsNeeded())
2407  hi_options |= HEAP_INSERT_SKIP_WAL;
2408  }
2409 
2410  /*
2411  * Optimize if new relfilenode was created in this subxact or one of its
2412  * committed children and we won't see those rows later as part of an
2413  * earlier scan or command. This ensures that if this subtransaction
2414  * aborts then the frozen rows won't be visible after xact cleanup. Note
2415  * that the stronger test of exactly which subtransaction created it is
2416  * crucial for correctness of this optimization.
2417  */
2418  if (cstate->freeze)
2419  {
2421  ereport(ERROR,
2422  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2423  errmsg("cannot perform FREEZE because of prior transaction activity")));
2424 
2425  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2427  ereport(ERROR,
2428  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2429  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2430 
2431  hi_options |= HEAP_INSERT_FROZEN;
2432  }
2433 
2434  /*
2435  * We need a ResultRelInfo so we can use the regular executor's
2436  * index-entry-making machinery. (There used to be a huge amount of code
2437  * here that basically duplicated execUtils.c ...)
2438  */
2439  resultRelInfo = makeNode(ResultRelInfo);
2440  InitResultRelInfo(resultRelInfo,
2441  cstate->rel,
2442  1, /* dummy rangetable index */
2443  NULL,
2444  0);
2445 
2446  ExecOpenIndices(resultRelInfo, false);
2447 
2448  estate->es_result_relations = resultRelInfo;
2449  estate->es_num_result_relations = 1;
2450  estate->es_result_relation_info = resultRelInfo;
2451  estate->es_range_table = cstate->range_table;
2452 
2453  /* Set up a tuple slot too */
2454  myslot = ExecInitExtraTupleSlot(estate);
2455  ExecSetSlotDescriptor(myslot, tupDesc);
2456  /* Triggers might need a slot as well */
2457  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);
2458 
2459  /*
2460  * It's more efficient to prepare a bunch of tuples for insertion, and
2461  * insert them in one heap_multi_insert() call, than call heap_insert()
2462  * separately for every tuple. However, we can't do that if there are
2463  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2464  * expressions. Such triggers or expressions might query the table we're
2465  * inserting to, and act differently if the tuples that have already been
2466  * processed and prepared for insertion are not there. We also can't do
2467  * it if the table is partitioned.
2468  */
2469  if ((resultRelInfo->ri_TrigDesc != NULL &&
2470  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2471  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2472  cstate->partition_dispatch_info != NULL ||
2473  cstate->volatile_defexprs)
2474  {
2475  useHeapMultiInsert = false;
2476  }
2477  else
2478  {
2479  useHeapMultiInsert = true;
2480  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2481  }
2482 
2483  /* Prepare to catch AFTER triggers. */
2485 
2486  /*
2487  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2488  * should do this for COPY, since it's not really an "INSERT" statement as
2489  * such. However, executing these triggers maintains consistency with the
2490  * EACH ROW triggers that we already fire on COPY.
2491  */
2492  ExecBSInsertTriggers(estate, resultRelInfo);
2493 
2494  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2495  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2496 
2497  bistate = GetBulkInsertState();
2498  econtext = GetPerTupleExprContext(estate);
2499 
2500  /* Set up callback to identify error line number */
2501  errcallback.callback = CopyFromErrorCallback;
2502  errcallback.arg = (void *) cstate;
2503  errcallback.previous = error_context_stack;
2504  error_context_stack = &errcallback;
2505 
2506  for (;;)
2507  {
2508  TupleTableSlot *slot,
2509  *oldslot;
2510  bool skip_tuple;
2511  Oid loaded_oid = InvalidOid;
2512 
2514 
2515  if (nBufferedTuples == 0)
2516  {
2517  /*
2518  * Reset the per-tuple exprcontext. We can only do this if the
2519  * tuple buffer is empty. (Calling the context the per-tuple
2520  * memory context is a bit of a misnomer now.)
2521  */
2522  ResetPerTupleExprContext(estate);
2523  }
2524 
2525  /* Switch into its memory context */
2527 
2528  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2529  break;
2530 
2531  /* And now we can form the input tuple. */
2532  tuple = heap_form_tuple(tupDesc, values, nulls);
2533 
2534  if (loaded_oid != InvalidOid)
2535  HeapTupleSetOid(tuple, loaded_oid);
2536 
2537  /*
2538  * Constraints might reference the tableoid column, so initialize
2539  * t_tableOid before evaluating them.
2540  */
2541  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2542 
2543  /* Triggers and stuff need to be invoked in query context. */
2544  MemoryContextSwitchTo(oldcontext);
2545 
2546  /* Place tuple in tuple slot --- but slot shouldn't free it */
2547  slot = myslot;
2548  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2549 
2550  /* Determine the partition to heap_insert the tuple into */
2551  oldslot = slot;
2552  if (cstate->partition_dispatch_info)
2553  {
2554  int leaf_part_index;
2555  TupleConversionMap *map;
2556 
2557  /*
2558  * Away we go ... If we end up not finding a partition after all,
2559  * ExecFindPartition() does not return and errors out instead.
2560  * Otherwise, the returned value is to be used as an index into
2561  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2562  * will get us the ResultRelInfo and TupleConversionMap for the
2563  * partition, respectively.
2564  */
2565  leaf_part_index = ExecFindPartition(resultRelInfo,
2566  cstate->partition_dispatch_info,
2567  slot,
2568  estate);
2569  Assert(leaf_part_index >= 0 &&
2570  leaf_part_index < cstate->num_partitions);
2571 
2572  /*
2573  * If this tuple is mapped to a partition that is not same as the
2574  * previous one, we'd better make the bulk insert mechanism gets a
2575  * new buffer.
2576  */
2577  if (prev_leaf_part_index != leaf_part_index)
2578  {
2579  ReleaseBulkInsertStatePin(bistate);
2580  prev_leaf_part_index = leaf_part_index;
2581  }
2582 
2583  /*
2584  * Save the old ResultRelInfo and switch to the one corresponding
2585  * to the selected partition.
2586  */
2587  saved_resultRelInfo = resultRelInfo;
2588  resultRelInfo = cstate->partitions + leaf_part_index;
2589 
2590  /* We do not yet have a way to insert into a foreign partition */
2591  if (resultRelInfo->ri_FdwRoutine)
2592  ereport(ERROR,
2593  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2594  errmsg("cannot route inserted tuples to a foreign table")));
2595 
2596  /*
2597  * For ExecInsertIndexTuples() to work on the partition's indexes
2598  */
2599  estate->es_result_relation_info = resultRelInfo;
2600 
2601  /*
2602  * We might need to convert from the parent rowtype to the
2603  * partition rowtype.
2604  */
2605  map = cstate->partition_tupconv_maps[leaf_part_index];
2606  if (map)
2607  {
2608  Relation partrel = resultRelInfo->ri_RelationDesc;
2609 
2610  tuple = do_convert_tuple(tuple, map);
2611 
2612  /*
2613  * We must use the partition's tuple descriptor from this
2614  * point on. Use a dedicated slot from this point on until
2615  * we're finished dealing with the partition.
2616  */
2617  slot = cstate->partition_tuple_slot;
2618  Assert(slot != NULL);
2619  ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
2620  ExecStoreTuple(tuple, slot, InvalidBuffer, true);
2621  }
2622 
2623  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2624  }
2625 
2626  skip_tuple = false;
2627 
2628  /* BEFORE ROW INSERT Triggers */
2629  if (resultRelInfo->ri_TrigDesc &&
2630  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2631  {
2632  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2633 
2634  if (slot == NULL) /* "do nothing" */
2635  skip_tuple = true;
2636  else /* trigger might have changed tuple */
2637  tuple = ExecMaterializeSlot(slot);
2638  }
2639 
2640  if (!skip_tuple)
2641  {
2642  if (resultRelInfo->ri_TrigDesc &&
2643  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2644  {
2645  /* Pass the data to the INSTEAD ROW INSERT trigger */
2646  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2647  }
2648  else
2649  {
2650  /* Check the constraints of the tuple */
2651  if (cstate->rel->rd_att->constr ||
2652  resultRelInfo->ri_PartitionCheck)
2653  ExecConstraints(resultRelInfo, slot, oldslot, estate);
2654 
2655  if (useHeapMultiInsert)
2656  {
2657  /* Add this tuple to the tuple buffer */
2658  if (nBufferedTuples == 0)
2659  firstBufferedLineNo = cstate->cur_lineno;
2660  bufferedTuples[nBufferedTuples++] = tuple;
2661  bufferedTuplesSize += tuple->t_len;
2662 
2663  /*
2664  * If the buffer filled up, flush it. Also flush if the
2665  * total size of all the tuples in the buffer becomes
2666  * large, to avoid using large amounts of memory for the
2667  * buffer when the tuples are exceptionally wide.
2668  */
2669  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2670  bufferedTuplesSize > 65535)
2671  {
2672  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2673  resultRelInfo, myslot, bistate,
2674  nBufferedTuples, bufferedTuples,
2675  firstBufferedLineNo);
2676  nBufferedTuples = 0;
2677  bufferedTuplesSize = 0;
2678  }
2679  }
2680  else
2681  {
2682  List *recheckIndexes = NIL;
2683 
2684  /* OK, store the tuple and create index entries for it */
2685  heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
2686  hi_options, bistate);
2687 
2688  if (resultRelInfo->ri_NumIndices > 0)
2689  recheckIndexes = ExecInsertIndexTuples(slot,
2690  &(tuple->t_self),
2691  estate,
2692  false,
2693  NULL,
2694  NIL);
2695 
2696  /* AFTER ROW INSERT Triggers */
2697  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2698  recheckIndexes);
2699 
2700  list_free(recheckIndexes);
2701  }
2702  }
2703 
2704  /*
2705  * We count only tuples not suppressed by a BEFORE INSERT trigger;
2706  * this is the same definition used by execMain.c for counting
2707  * tuples inserted by an INSERT command.
2708  */
2709  processed++;
2710 
2711  if (saved_resultRelInfo)
2712  {
2713  resultRelInfo = saved_resultRelInfo;
2714  estate->es_result_relation_info = resultRelInfo;
2715  }
2716  }
2717  }
2718 
2719  /* Flush any remaining buffered tuples */
2720  if (nBufferedTuples > 0)
2721  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2722  resultRelInfo, myslot, bistate,
2723  nBufferedTuples, bufferedTuples,
2724  firstBufferedLineNo);
2725 
2726  /* Done, clean up */
2727  error_context_stack = errcallback.previous;
2728 
2729  FreeBulkInsertState(bistate);
2730 
2731  MemoryContextSwitchTo(oldcontext);
2732 
2733  /*
2734  * In the old protocol, tell pqcomm that we can process normal protocol
2735  * messages again.
2736  */
2737  if (cstate->copy_dest == COPY_OLD_FE)
2738  pq_endmsgread();
2739 
2740  /* Execute AFTER STATEMENT insertion triggers */
2741  ExecASInsertTriggers(estate, resultRelInfo);
2742 
2743  /* Handle queued AFTER triggers */
2744  AfterTriggerEndQuery(estate);
2745 
2746  pfree(values);
2747  pfree(nulls);
2748 
2749  ExecResetTupleTable(estate->es_tupleTable, false);
2750 
2751  ExecCloseIndices(resultRelInfo);
2752 
2753  /* Close all the partitioned tables, leaf partitions, and their indices */
2754  if (cstate->partition_dispatch_info)
2755  {
2756  int i;
2757 
2758  /*
2759  * Remember cstate->partition_dispatch_info[0] corresponds to the root
2760  * partitioned table, which we must not try to close, because it is
2761  * the main target table of COPY that will be closed eventually by
2762  * DoCopy(). Also, tupslot is NULL for the root partitioned table.
2763  */
2764  for (i = 1; i < cstate->num_dispatch; i++)
2765  {
2767 
2768  heap_close(pd->reldesc, NoLock);
2770  }
2771  for (i = 0; i < cstate->num_partitions; i++)
2772  {
2773  ResultRelInfo *resultRelInfo = cstate->partitions + i;
2774 
2775  ExecCloseIndices(resultRelInfo);
2776  heap_close(resultRelInfo->ri_RelationDesc, NoLock);
2777  }
2778 
2779  /* Release the standalone partition tuple descriptor */
2781  }
2782 
2783  FreeExecutorState(estate);
2784 
2785  /*
2786  * If we skipped writing WAL, then we need to sync the heap (but not
2787  * indexes since those use WAL anyway)
2788  */
2789  if (hi_options & HEAP_INSERT_SKIP_WAL)
2790  heap_sync(cstate->rel);
2791 
2792  return processed;
2793 }
2794 
2795 /*
2796  * A subroutine of CopyFrom, to write the current batch of buffered heap
2797  * tuples to the heap. Also updates indexes and runs AFTER ROW INSERT
2798  * triggers.
2799  */
2800 static void
2802  int hi_options, ResultRelInfo *resultRelInfo,
2803  TupleTableSlot *myslot, BulkInsertState bistate,
2804  int nBufferedTuples, HeapTuple *bufferedTuples,
2805  int firstBufferedLineNo)
2806 {
2807  MemoryContext oldcontext;
2808  int i;
2809  int save_cur_lineno;
2810 
2811  /*
2812  * Print error context information correctly, if one of the operations
2813  * below fail.
2814  */
2815  cstate->line_buf_valid = false;
2816  save_cur_lineno = cstate->cur_lineno;
2817 
2818  /*
2819  * heap_multi_insert leaks memory, so switch to short-lived memory context
2820  * before calling it.
2821  */
2822  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2823  heap_multi_insert(cstate->rel,
2824  bufferedTuples,
2825  nBufferedTuples,
2826  mycid,
2827  hi_options,
2828  bistate);
2829  MemoryContextSwitchTo(oldcontext);
2830 
2831  /*
2832  * If there are any indexes, update them for all the inserted tuples, and
2833  * run AFTER ROW INSERT triggers.
2834  */
2835  if (resultRelInfo->ri_NumIndices > 0)
2836  {
2837  for (i = 0; i < nBufferedTuples; i++)
2838  {
2839  List *recheckIndexes;
2840 
2841  cstate->cur_lineno = firstBufferedLineNo + i;
2842  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2843  recheckIndexes =
2844  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2845  estate, false, NULL, NIL);
2846  ExecARInsertTriggers(estate, resultRelInfo,
2847  bufferedTuples[i],
2848  recheckIndexes);
2849  list_free(recheckIndexes);
2850  }
2851  }
2852 
2853  /*
2854  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2855  * anyway.
2856  */
2857  else if (resultRelInfo->ri_TrigDesc != NULL &&
2858  resultRelInfo->ri_TrigDesc->trig_insert_after_row)
2859  {
2860  for (i = 0; i < nBufferedTuples; i++)
2861  {
2862  cstate->cur_lineno = firstBufferedLineNo + i;
2863  ExecARInsertTriggers(estate, resultRelInfo,
2864  bufferedTuples[i],
2865  NIL);
2866  }
2867  }
2868 
2869  /* reset cur_lineno to where we were */
2870  cstate->cur_lineno = save_cur_lineno;
2871 }
2872 
2873 /*
2874  * Setup to read tuples from a file for COPY FROM.
2875  *
2876  * 'rel': Used as a template for the tuples
2877  * 'filename': Name of server-local file to read
2878  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
2879  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
2880  *
2881  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
2882  */
2883 CopyState
2885  Relation rel,
2886  const char *filename,
2887  bool is_program,
2888  copy_data_source_cb data_source_cb,
2889  List *attnamelist,
2890  List *options)
2891 {
2892  CopyState cstate;
2893  bool pipe = (filename == NULL);
2894  TupleDesc tupDesc;
2895  Form_pg_attribute *attr;
2896  AttrNumber num_phys_attrs,
2897  num_defaults;
2898  FmgrInfo *in_functions;
2899  Oid *typioparams;
2900  int attnum;
2901  Oid in_func_oid;
2902  int *defmap;
2903  ExprState **defexprs;
2904  MemoryContext oldcontext;
2905  bool volatile_defexprs;
2906 
2907  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
2908  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
2909 
2910  /* Initialize state variables */
2911  cstate->fe_eof = false;
2912  cstate->eol_type = EOL_UNKNOWN;
2913  cstate->cur_relname = RelationGetRelationName(cstate->rel);
2914  cstate->cur_lineno = 0;
2915  cstate->cur_attname = NULL;
2916  cstate->cur_attval = NULL;
2917 
2918  /* Set up variables to avoid per-attribute overhead. */
2920  initLongStringInfo(&cstate->line_buf);
2921  cstate->line_buf_converted = false;
2922  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
2923  cstate->raw_buf_index = cstate->raw_buf_len = 0;
2924 
2925  tupDesc = RelationGetDescr(cstate->rel);
2926  attr = tupDesc->attrs;
2927  num_phys_attrs = tupDesc->natts;
2928  num_defaults = 0;
2929  volatile_defexprs = false;
2930 
2931  /*
2932  * Pick up the required catalog information for each attribute in the
2933  * relation, including the input function, the element type (to pass to
2934  * the input function), and info about defaults and constraints. (Which
2935  * input function we use depends on text/binary format choice.)
2936  */
2937  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2938  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
2939  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
2940  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
2941 
2942  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
2943  {
2944  /* We don't need info for dropped attributes */
2945  if (attr[attnum - 1]->attisdropped)
2946  continue;
2947 
2948  /* Fetch the input function and typioparam info */
2949  if (cstate->binary)
2950  getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
2951  &in_func_oid, &typioparams[attnum - 1]);
2952  else
2953  getTypeInputInfo(attr[attnum - 1]->atttypid,
2954  &in_func_oid, &typioparams[attnum - 1]);
2955  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
2956 
2957  /* Get default info if needed */
2958  if (!list_member_int(cstate->attnumlist, attnum))
2959  {
2960  /* attribute is NOT to be copied from input */
2961  /* use default value if one exists */
2962  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
2963  attnum);
2964 
2965  if (defexpr != NULL)
2966  {
2967  /* Run the expression through planner */
2968  defexpr = expression_planner(defexpr);
2969 
2970  /* Initialize executable expression in copycontext */
2971  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
2972  defmap[num_defaults] = attnum - 1;
2973  num_defaults++;
2974 
2975  /*
2976  * If a default expression looks at the table being loaded,
2977  * then it could give the wrong answer when using
2978  * multi-insert. Since database access can be dynamic this is
2979  * hard to test for exactly, so we use the much wider test of
2980  * whether the default expression is volatile. We allow for
2981  * the special case of when the default expression is the
2982  * nextval() of a sequence which in this specific case is
2983  * known to be safe for use with the multi-insert
2984  * optimization. Hence we use this special case function
2985  * checker rather than the standard check for
2986  * contain_volatile_functions().
2987  */
2988  if (!volatile_defexprs)
2989  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
2990  }
2991  }
2992  }
2993 
2994  /* We keep those variables in cstate. */
2995  cstate->in_functions = in_functions;
2996  cstate->typioparams = typioparams;
2997  cstate->defmap = defmap;
2998  cstate->defexprs = defexprs;
2999  cstate->volatile_defexprs = volatile_defexprs;
3000  cstate->num_defaults = num_defaults;
3001  cstate->is_program = is_program;
3002 
3003  if (data_source_cb)
3004  {
3005  cstate->copy_dest = COPY_CALLBACK;
3006  cstate->data_source_cb = data_source_cb;
3007  }
3008  else if (pipe)
3009  {
3010  Assert(!is_program); /* the grammar does not allow this */
3012  ReceiveCopyBegin(cstate);
3013  else
3014  cstate->copy_file = stdin;
3015  }
3016  else
3017  {
3018  cstate->filename = pstrdup(filename);
3019 
3020  if (cstate->is_program)
3021  {
3022  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3023  if (cstate->copy_file == NULL)
3024  ereport(ERROR,
3026  errmsg("could not execute command \"%s\": %m",
3027  cstate->filename)));
3028  }
3029  else
3030  {
3031  struct stat st;
3032 
3033  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3034  if (cstate->copy_file == NULL)
3035  {
3036  /* copy errno because ereport subfunctions might change it */
3037  int save_errno = errno;
3038 
3039  ereport(ERROR,
3041  errmsg("could not open file \"%s\" for reading: %m",
3042  cstate->filename),
3043  (save_errno == ENOENT || save_errno == EACCES) ?
3044  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3045  "You may want a client-side facility such as psql's \\copy.") : 0));
3046  }
3047 
3048  if (fstat(fileno(cstate->copy_file), &st))
3049  ereport(ERROR,
3051  errmsg("could not stat file \"%s\": %m",
3052  cstate->filename)));
3053 
3054  if (S_ISDIR(st.st_mode))
3055  ereport(ERROR,
3056  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3057  errmsg("\"%s\" is a directory", cstate->filename)));
3058  }
3059  }
3060 
3061  if (!cstate->binary)
3062  {
3063  /* must rely on user to tell us... */
3064  cstate->file_has_oids = cstate->oids;
3065  }
3066  else
3067  {
3068  /* Read and verify binary header */
3069  char readSig[11];
3070  int32 tmp;
3071 
3072  /* Signature */
3073  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3074  memcmp(readSig, BinarySignature, 11) != 0)
3075  ereport(ERROR,
3076  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3077  errmsg("COPY file signature not recognized")));
3078  /* Flags field */
3079  if (!CopyGetInt32(cstate, &tmp))
3080  ereport(ERROR,
3081  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3082  errmsg("invalid COPY file header (missing flags)")));
3083  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3084  tmp &= ~(1 << 16);
3085  if ((tmp >> 16) != 0)
3086  ereport(ERROR,
3087  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3088  errmsg("unrecognized critical flags in COPY file header")));
3089  /* Header extension length */
3090  if (!CopyGetInt32(cstate, &tmp) ||
3091  tmp < 0)
3092  ereport(ERROR,
3093  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3094  errmsg("invalid COPY file header (missing length)")));
3095  /* Skip extension header, if present */
3096  while (tmp-- > 0)
3097  {
3098  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3099  ereport(ERROR,
3100  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3101  errmsg("invalid COPY file header (wrong length)")));
3102  }
3103  }
3104 
3105  if (cstate->file_has_oids && cstate->binary)
3106  {
3108  &in_func_oid, &cstate->oid_typioparam);
3109  fmgr_info(in_func_oid, &cstate->oid_in_function);
3110  }
3111 
3112  /* create workspace for CopyReadAttributes results */
3113  if (!cstate->binary)
3114  {
3115  AttrNumber attr_count = list_length(cstate->attnumlist);
3116  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3117 
3118  cstate->max_fields = nfields;
3119  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3120  }
3121 
3122  MemoryContextSwitchTo(oldcontext);
3123 
3124  return cstate;
3125 }
3126 
3127 /*
3128  * Read raw fields in the next line for COPY FROM in text or csv mode.
3129  * Return false if no more lines.
3130  *
3131  * An internal temporary buffer is returned via 'fields'. It is valid until
3132  * the next call of the function. Since the function returns all raw fields
3133  * in the input file, 'nfields' could be different from the number of columns
3134  * in the relation.
3135  *
3136  * NOTE: force_not_null option are not applied to the returned fields.
3137  */
3138 bool
3139 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3140 {
3141  int fldct;
3142  bool done;
3143 
3144  /* only available for text or csv input */
3145  Assert(!cstate->binary);
3146 
3147  /* on input just throw the header line away */
3148  if (cstate->cur_lineno == 0 && cstate->header_line)
3149  {
3150  cstate->cur_lineno++;
3151  if (CopyReadLine(cstate))
3152  return false; /* done */
3153  }
3154 
3155  cstate->cur_lineno++;
3156 
3157  /* Actually read the line into memory here */
3158  done = CopyReadLine(cstate);
3159 
3160  /*
3161  * EOF at start of line means we're done. If we see EOF after some
3162  * characters, we act as though it was newline followed by EOF, ie,
3163  * process the line and then exit loop on next iteration.
3164  */
3165  if (done && cstate->line_buf.len == 0)
3166  return false;
3167 
3168  /* Parse the line into de-escaped field values */
3169  if (cstate->csv_mode)
3170  fldct = CopyReadAttributesCSV(cstate);
3171  else
3172  fldct = CopyReadAttributesText(cstate);
3173 
3174  *fields = cstate->raw_fields;
3175  *nfields = fldct;
3176  return true;
3177 }
3178 
3179 /*
3180  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3181  *
3182  * 'econtext' is used to evaluate default expression for each columns not
3183  * read from the file. It can be NULL when no default values are used, i.e.
3184  * when all columns are read from the file.
3185  *
3186  * 'values' and 'nulls' arrays must be the same length as columns of the
3187  * relation passed to BeginCopyFrom. This function fills the arrays.
3188  * Oid of the tuple is returned with 'tupleOid' separately.
3189  */
3190 bool
3192  Datum *values, bool *nulls, Oid *tupleOid)
3193 {
3194  TupleDesc tupDesc;
3195  Form_pg_attribute *attr;
3196  AttrNumber num_phys_attrs,
3197  attr_count,
3198  num_defaults = cstate->num_defaults;
3199  FmgrInfo *in_functions = cstate->in_functions;
3200  Oid *typioparams = cstate->typioparams;
3201  int i;
3202  int nfields;
3203  bool isnull;
3204  bool file_has_oids = cstate->file_has_oids;
3205  int *defmap = cstate->defmap;
3206  ExprState **defexprs = cstate->defexprs;
3207 
3208  tupDesc = RelationGetDescr(cstate->rel);
3209  attr = tupDesc->attrs;
3210  num_phys_attrs = tupDesc->natts;
3211  attr_count = list_length(cstate->attnumlist);
3212  nfields = file_has_oids ? (attr_count + 1) : attr_count;
3213 
3214  /* Initialize all values for row to NULL */
3215  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3216  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3217 
3218  if (!cstate->binary)
3219  {
3220  char **field_strings;
3221  ListCell *cur;
3222  int fldct;
3223  int fieldno;
3224  char *string;
3225 
3226  /* read raw fields in the next line */
3227  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3228  return false;
3229 
3230  /* check for overflowing fields */
3231  if (nfields > 0 && fldct > nfields)
3232  ereport(ERROR,
3233  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3234  errmsg("extra data after last expected column")));
3235 
3236  fieldno = 0;
3237 
3238  /* Read the OID field if present */
3239  if (file_has_oids)
3240  {
3241  if (fieldno >= fldct)
3242  ereport(ERROR,
3243  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3244  errmsg("missing data for OID column")));
3245  string = field_strings[fieldno++];
3246 
3247  if (string == NULL)
3248  ereport(ERROR,
3249  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3250  errmsg("null OID in COPY data")));
3251  else if (cstate->oids && tupleOid != NULL)
3252  {
3253  cstate->cur_attname = "oid";
3254  cstate->cur_attval = string;
3256  CStringGetDatum(string)));
3257  if (*tupleOid == InvalidOid)
3258  ereport(ERROR,
3259  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3260  errmsg("invalid OID in COPY data")));
3261  cstate->cur_attname = NULL;
3262  cstate->cur_attval = NULL;
3263  }
3264  }
3265 
3266  /* Loop to read the user attributes on the line. */
3267  foreach(cur, cstate->attnumlist)
3268  {
3269  int attnum = lfirst_int(cur);
3270  int m = attnum - 1;
3271 
3272  if (fieldno >= fldct)
3273  ereport(ERROR,
3274  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3275  errmsg("missing data for column \"%s\"",
3276  NameStr(attr[m]->attname))));
3277  string = field_strings[fieldno++];
3278 
3279  if (cstate->convert_select_flags &&
3280  !cstate->convert_select_flags[m])
3281  {
3282  /* ignore input field, leaving column as NULL */
3283  continue;
3284  }
3285 
3286  if (cstate->csv_mode)
3287  {
3288  if (string == NULL &&
3289  cstate->force_notnull_flags[m])
3290  {
3291  /*
3292  * FORCE_NOT_NULL option is set and column is NULL -
3293  * convert it to the NULL string.
3294  */
3295  string = cstate->null_print;
3296  }
3297  else if (string != NULL && cstate->force_null_flags[m]
3298  && strcmp(string, cstate->null_print) == 0)
3299  {
3300  /*
3301  * FORCE_NULL option is set and column matches the NULL
3302  * string. It must have been quoted, or otherwise the
3303  * string would already have been set to NULL. Convert it
3304  * to NULL as specified.
3305  */
3306  string = NULL;
3307  }
3308  }
3309 
3310  cstate->cur_attname = NameStr(attr[m]->attname);
3311  cstate->cur_attval = string;
3312  values[m] = InputFunctionCall(&in_functions[m],
3313  string,
3314  typioparams[m],
3315  attr[m]->atttypmod);
3316  if (string != NULL)
3317  nulls[m] = false;
3318  cstate->cur_attname = NULL;
3319  cstate->cur_attval = NULL;
3320  }
3321 
3322  Assert(fieldno == nfields);
3323  }
3324  else
3325  {
3326  /* binary */
3327  int16 fld_count;
3328  ListCell *cur;
3329 
3330  cstate->cur_lineno++;
3331 
3332  if (!CopyGetInt16(cstate, &fld_count))
3333  {
3334  /* EOF detected (end of file, or protocol-level EOF) */
3335  return false;
3336  }
3337 
3338  if (fld_count == -1)
3339  {
3340  /*
3341  * Received EOF marker. In a V3-protocol copy, wait for the
3342  * protocol-level EOF, and complain if it doesn't come
3343  * immediately. This ensures that we correctly handle CopyFail,
3344  * if client chooses to send that now.
3345  *
3346  * Note that we MUST NOT try to read more data in an old-protocol
3347  * copy, since there is no protocol-level EOF marker then. We
3348  * could go either way for copy from file, but choose to throw
3349  * error if there's data after the EOF marker, for consistency
3350  * with the new-protocol case.
3351  */
3352  char dummy;
3353 
3354  if (cstate->copy_dest != COPY_OLD_FE &&
3355  CopyGetData(cstate, &dummy, 1, 1) > 0)
3356  ereport(ERROR,
3357  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3358  errmsg("received copy data after EOF marker")));
3359  return false;
3360  }
3361 
3362  if (fld_count != attr_count)
3363  ereport(ERROR,
3364  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3365  errmsg("row field count is %d, expected %d",
3366  (int) fld_count, attr_count)));
3367 
3368  if (file_has_oids)
3369  {
3370  Oid loaded_oid;
3371 
3372  cstate->cur_attname = "oid";
3373  loaded_oid =
3375  0,
3376  &cstate->oid_in_function,
3377  cstate->oid_typioparam,
3378  -1,
3379  &isnull));
3380  if (isnull || loaded_oid == InvalidOid)
3381  ereport(ERROR,
3382  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3383  errmsg("invalid OID in COPY data")));
3384  cstate->cur_attname = NULL;
3385  if (cstate->oids && tupleOid != NULL)
3386  *tupleOid = loaded_oid;
3387  }
3388 
3389  i = 0;
3390  foreach(cur, cstate->attnumlist)
3391  {
3392  int attnum = lfirst_int(cur);
3393  int m = attnum - 1;
3394 
3395  cstate->cur_attname = NameStr(attr[m]->attname);
3396  i++;
3397  values[m] = CopyReadBinaryAttribute(cstate,
3398  i,
3399  &in_functions[m],
3400  typioparams[m],
3401  attr[m]->atttypmod,
3402  &nulls[m]);
3403  cstate->cur_attname = NULL;
3404  }
3405  }
3406 
3407  /*
3408  * Now compute and insert any defaults available for the columns not
3409  * provided by the input data. Anything not processed here or above will
3410  * remain NULL.
3411  */
3412  for (i = 0; i < num_defaults; i++)
3413  {
3414  /*
3415  * The caller must supply econtext and have switched into the
3416  * per-tuple memory context in it.
3417  */
3418  Assert(econtext != NULL);
3420 
3421  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3422  &nulls[defmap[i]]);
3423  }
3424 
3425  return true;
3426 }
3427 
3428 /*
3429  * Clean up storage and release resources for COPY FROM.
3430  */
3431 void
3433 {
3434  /* No COPY FROM related resources except memory. */
3435 
3436  EndCopy(cstate);
3437 }
3438 
3439 /*
3440  * Read the next input line and stash it in line_buf, with conversion to
3441  * server encoding.
3442  *
3443  * Result is true if read was terminated by EOF, false if terminated
3444  * by newline. The terminating newline or EOF marker is not included
3445  * in the final value of line_buf.
3446  */
3447 static bool
3449 {
3450  bool result;
3451 
3452  resetStringInfo(&cstate->line_buf);
3453  cstate->line_buf_valid = true;
3454 
3455  /* Mark that encoding conversion hasn't occurred yet */
3456  cstate->line_buf_converted = false;
3457 
3458  /* Parse data and transfer into line_buf */
3459  result = CopyReadLineText(cstate);
3460 
3461  if (result)
3462  {
3463  /*
3464  * Reached EOF. In protocol version 3, we should ignore anything
3465  * after \. up to the protocol end of copy data. (XXX maybe better
3466  * not to treat \. as special?)
3467  */
3468  if (cstate->copy_dest == COPY_NEW_FE)
3469  {
3470  do
3471  {
3472  cstate->raw_buf_index = cstate->raw_buf_len;
3473  } while (CopyLoadRawBuf(cstate));
3474  }
3475  }
3476  else
3477  {
3478  /*
3479  * If we didn't hit EOF, then we must have transferred the EOL marker
3480  * to line_buf along with the data. Get rid of it.
3481  */
3482  switch (cstate->eol_type)
3483  {
3484  case EOL_NL:
3485  Assert(cstate->line_buf.len >= 1);
3486  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3487  cstate->line_buf.len--;
3488  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3489  break;
3490  case EOL_CR:
3491  Assert(cstate->line_buf.len >= 1);
3492  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3493  cstate->line_buf.len--;
3494  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3495  break;
3496  case EOL_CRNL:
3497  Assert(cstate->line_buf.len >= 2);
3498  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3499  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3500  cstate->line_buf.len -= 2;
3501  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3502  break;
3503  case EOL_UNKNOWN:
3504  /* shouldn't get here */
3505  Assert(false);
3506  break;
3507  }
3508  }
3509 
3510  /* Done reading the line. Convert it to server encoding. */
3511  if (cstate->need_transcoding)
3512  {
3513  char *cvt;
3514 
3515  cvt = pg_any_to_server(cstate->line_buf.data,
3516  cstate->line_buf.len,
3517  cstate->file_encoding);
3518  if (cvt != cstate->line_buf.data)
3519  {
3520  /* transfer converted data back to line_buf */
3521  resetStringInfo(&cstate->line_buf);
3522  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3523  pfree(cvt);
3524  }
3525  }
3526 
3527  /* Now it's safe to use the buffer in error messages */
3528  cstate->line_buf_converted = true;
3529 
3530  return result;
3531 }
3532 
3533 /*
3534  * CopyReadLineText - inner loop of CopyReadLine for text mode
3535  */
3536 static bool
3538 {
3539  char *copy_raw_buf;
3540  int raw_buf_ptr;
3541  int copy_buf_len;
3542  bool need_data = false;
3543  bool hit_eof = false;
3544  bool result = false;
3545  char mblen_str[2];
3546 
3547  /* CSV variables */
3548  bool first_char_in_line = true;
3549  bool in_quote = false,
3550  last_was_esc = false;
3551  char quotec = '\0';
3552  char escapec = '\0';
3553 
3554  if (cstate->csv_mode)
3555  {
3556  quotec = cstate->quote[0];
3557  escapec = cstate->escape[0];
3558  /* ignore special escape processing if it's the same as quotec */
3559  if (quotec == escapec)
3560  escapec = '\0';
3561  }
3562 
3563  mblen_str[1] = '\0';
3564 
3565  /*
3566  * The objective of this loop is to transfer the entire next input line
3567  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3568  * \n) and the end-of-copy marker (\.).
3569  *
3570  * In CSV mode, \r and \n inside a quoted field are just part of the data
3571  * value and are put in line_buf. We keep just enough state to know if we
3572  * are currently in a quoted field or not.
3573  *
3574  * These four characters, and the CSV escape and quote characters, are
3575  * assumed the same in frontend and backend encodings.
3576  *
3577  * For speed, we try to move data from raw_buf to line_buf in chunks
3578  * rather than one character at a time. raw_buf_ptr points to the next
3579  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3580  * have been determined to be part of the line, but not yet transferred to
3581  * line_buf.
3582  *
3583  * For a little extra speed within the loop, we copy raw_buf and
3584  * raw_buf_len into local variables.
3585  */
3586  copy_raw_buf = cstate->raw_buf;
3587  raw_buf_ptr = cstate->raw_buf_index;
3588  copy_buf_len = cstate->raw_buf_len;
3589 
3590  for (;;)
3591  {
3592  int prev_raw_ptr;
3593  char c;
3594 
3595  /*
3596  * Load more data if needed. Ideally we would just force four bytes
3597  * of read-ahead and avoid the many calls to
3598  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3599  * does not allow us to read too far ahead or we might read into the
3600  * next data, so we read-ahead only as far we know we can. One
3601  * optimization would be to read-ahead four byte here if
3602  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3603  * considering the size of the buffer.
3604  */
3605  if (raw_buf_ptr >= copy_buf_len || need_data)
3606  {
3608 
3609  /*
3610  * Try to read some more data. This will certainly reset
3611  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3612  */
3613  if (!CopyLoadRawBuf(cstate))
3614  hit_eof = true;
3615  raw_buf_ptr = 0;
3616  copy_buf_len = cstate->raw_buf_len;
3617 
3618  /*
3619  * If we are completely out of data, break out of the loop,
3620  * reporting EOF.
3621  */
3622  if (copy_buf_len <= 0)
3623  {
3624  result = true;
3625  break;
3626  }
3627  need_data = false;
3628  }
3629 
3630  /* OK to fetch a character */
3631  prev_raw_ptr = raw_buf_ptr;
3632  c = copy_raw_buf[raw_buf_ptr++];
3633 
3634  if (cstate->csv_mode)
3635  {
3636  /*
3637  * If character is '\\' or '\r', we may need to look ahead below.
3638  * Force fetch of the next character if we don't already have it.
3639  * We need to do this before changing CSV state, in case one of
3640  * these characters is also the quote or escape character.
3641  *
3642  * Note: old-protocol does not like forced prefetch, but it's OK
3643  * here since we cannot validly be at EOF.
3644  */
3645  if (c == '\\' || c == '\r')
3646  {
3648  }
3649 
3650  /*
3651  * Dealing with quotes and escapes here is mildly tricky. If the
3652  * quote char is also the escape char, there's no problem - we
3653  * just use the char as a toggle. If they are different, we need
3654  * to ensure that we only take account of an escape inside a
3655  * quoted field and immediately preceding a quote char, and not
3656  * the second in an escape-escape sequence.
3657  */
3658  if (in_quote && c == escapec)
3659  last_was_esc = !last_was_esc;
3660  if (c == quotec && !last_was_esc)
3661  in_quote = !in_quote;
3662  if (c != escapec)
3663  last_was_esc = false;
3664 
3665  /*
3666  * Updating the line count for embedded CR and/or LF chars is
3667  * necessarily a little fragile - this test is probably about the
3668  * best we can do. (XXX it's arguable whether we should do this
3669  * at all --- is cur_lineno a physical or logical count?)
3670  */
3671  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3672  cstate->cur_lineno++;
3673  }
3674 
3675  /* Process \r */
3676  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3677  {
3678  /* Check for \r\n on first line, _and_ handle \r\n. */
3679  if (cstate->eol_type == EOL_UNKNOWN ||
3680  cstate->eol_type == EOL_CRNL)
3681  {
3682  /*
3683  * If need more data, go back to loop top to load it.
3684  *
3685  * Note that if we are at EOF, c will wind up as '\0' because
3686  * of the guaranteed pad of raw_buf.
3687  */
3689 
3690  /* get next char */
3691  c = copy_raw_buf[raw_buf_ptr];
3692 
3693  if (c == '\n')
3694  {
3695  raw_buf_ptr++; /* eat newline */
3696  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3697  }
3698  else
3699  {
3700  /* found \r, but no \n */
3701  if (cstate->eol_type == EOL_CRNL)
3702  ereport(ERROR,
3703  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3704  !cstate->csv_mode ?
3705  errmsg("literal carriage return found in data") :
3706  errmsg("unquoted carriage return found in data"),
3707  !cstate->csv_mode ?
3708  errhint("Use \"\\r\" to represent carriage return.") :
3709  errhint("Use quoted CSV field to represent carriage return.")));
3710 
3711  /*
3712  * if we got here, it is the first line and we didn't find
3713  * \n, so don't consume the peeked character
3714  */
3715  cstate->eol_type = EOL_CR;
3716  }
3717  }
3718  else if (cstate->eol_type == EOL_NL)
3719  ereport(ERROR,
3720  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3721  !cstate->csv_mode ?
3722  errmsg("literal carriage return found in data") :
3723  errmsg("unquoted carriage return found in data"),
3724  !cstate->csv_mode ?
3725  errhint("Use \"\\r\" to represent carriage return.") :
3726  errhint("Use quoted CSV field to represent carriage return.")));
3727  /* If reach here, we have found the line terminator */
3728  break;
3729  }
3730 
3731  /* Process \n */
3732  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3733  {
3734  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3735  ereport(ERROR,
3736  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3737  !cstate->csv_mode ?
3738  errmsg("literal newline found in data") :
3739  errmsg("unquoted newline found in data"),
3740  !cstate->csv_mode ?
3741  errhint("Use \"\\n\" to represent newline.") :
3742  errhint("Use quoted CSV field to represent newline.")));
3743  cstate->eol_type = EOL_NL; /* in case not set yet */
3744  /* If reach here, we have found the line terminator */
3745  break;
3746  }
3747 
3748  /*
3749  * In CSV mode, we only recognize \. alone on a line. This is because
3750  * \. is a valid CSV data value.
3751  */
3752  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3753  {
3754  char c2;
3755 
3758 
3759  /* -----
3760  * get next character
3761  * Note: we do not change c so if it isn't \., we can fall
3762  * through and continue processing for file encoding.
3763  * -----
3764  */
3765  c2 = copy_raw_buf[raw_buf_ptr];
3766 
3767  if (c2 == '.')
3768  {
3769  raw_buf_ptr++; /* consume the '.' */
3770 
3771  /*
3772  * Note: if we loop back for more data here, it does not
3773  * matter that the CSV state change checks are re-executed; we
3774  * will come back here with no important state changed.
3775  */
3776  if (cstate->eol_type == EOL_CRNL)
3777  {
3778  /* Get the next character */
3780  /* if hit_eof, c2 will become '\0' */
3781  c2 = copy_raw_buf[raw_buf_ptr++];
3782 
3783  if (c2 == '\n')
3784  {
3785  if (!cstate->csv_mode)
3786  ereport(ERROR,
3787  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3788  errmsg("end-of-copy marker does not match previous newline style")));
3789  else
3791  }
3792  else if (c2 != '\r')
3793  {
3794  if (!cstate->csv_mode)
3795  ereport(ERROR,
3796  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3797  errmsg("end-of-copy marker corrupt")));
3798  else
3800  }
3801  }
3802 
3803  /* Get the next character */
3805  /* if hit_eof, c2 will become '\0' */
3806  c2 = copy_raw_buf[raw_buf_ptr++];
3807 
3808  if (c2 != '\r' && c2 != '\n')
3809  {
3810  if (!cstate->csv_mode)
3811  ereport(ERROR,
3812  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3813  errmsg("end-of-copy marker corrupt")));
3814  else
3816  }
3817 
3818  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3819  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3820  (cstate->eol_type == EOL_CR && c2 != '\r'))
3821  {
3822  ereport(ERROR,
3823  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3824  errmsg("end-of-copy marker does not match previous newline style")));
3825  }
3826 
3827  /*
3828  * Transfer only the data before the \. into line_buf, then
3829  * discard the data and the \. sequence.
3830  */
3831  if (prev_raw_ptr > cstate->raw_buf_index)
3833  cstate->raw_buf + cstate->raw_buf_index,
3834  prev_raw_ptr - cstate->raw_buf_index);
3835  cstate->raw_buf_index = raw_buf_ptr;
3836  result = true; /* report EOF */
3837  break;
3838  }
3839  else if (!cstate->csv_mode)
3840 
3841  /*
3842  * If we are here, it means we found a backslash followed by
3843  * something other than a period. In non-CSV mode, anything
3844  * after a backslash is special, so we skip over that second
3845  * character too. If we didn't do that \\. would be
3846  * considered an eof-of copy, while in non-CSV mode it is a
3847  * literal backslash followed by a period. In CSV mode,
3848  * backslashes are not special, so we want to process the
3849  * character after the backslash just like a normal character,
3850  * so we don't increment in those cases.
3851  */
3852  raw_buf_ptr++;
3853  }
3854 
3855  /*
3856  * This label is for CSV cases where \. appears at the start of a
3857  * line, but there is more text after it, meaning it was a data value.
3858  * We are more strict for \. in CSV mode because \. could be a data
3859  * value, while in non-CSV mode, \. cannot be a data value.
3860  */
3861 not_end_of_copy:
3862 
3863  /*
3864  * Process all bytes of a multi-byte character as a group.
3865  *
3866  * We only support multi-byte sequences where the first byte has the
3867  * high-bit set, so as an optimization we can avoid this block
3868  * entirely if it is not set.
3869  */
3870  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3871  {
3872  int mblen;
3873 
3874  mblen_str[0] = c;
3875  /* All our encodings only read the first byte to get the length */
3876  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3878  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3879  raw_buf_ptr += mblen - 1;
3880  }
3881  first_char_in_line = false;
3882  } /* end of outer loop */
3883 
3884  /*
3885  * Transfer any still-uncopied data to line_buf.
3886  */
3888 
3889  return result;
3890 }
3891 
3892 /*
3893  * Return decimal value for a hexadecimal digit
3894  */
3895 static int
3897 {
3898  if (isdigit((unsigned char) hex))
3899  return hex - '0';
3900  else
3901  return tolower((unsigned char) hex) - 'a' + 10;
3902 }
3903 
3904 /*
3905  * Parse the current line into separate attributes (fields),
3906  * performing de-escaping as needed.
3907  *
3908  * The input is in line_buf. We use attribute_buf to hold the result
3909  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
3910  * string, or NULL when the input matches the null marker string.
3911  * This array is expanded as necessary.
3912  *
3913  * (Note that the caller cannot check for nulls since the returned
3914  * string would be the post-de-escaping equivalent, which may look
3915  * the same as some valid data string.)
3916  *
3917  * delim is the column delimiter string (must be just one byte for now).
3918  * null_print is the null marker string. Note that this is compared to
3919  * the pre-de-escaped input string.
3920  *
3921  * The return value is the number of fields actually read.
3922  */
3923 static int
3925 {
3926  char delimc = cstate->delim[0];
3927  int fieldno;
3928  char *output_ptr;
3929  char *cur_ptr;
3930  char *line_end_ptr;
3931 
3932  /*
3933  * We need a special case for zero-column tables: check that the input
3934  * line is empty, and return.
3935  */
3936  if (cstate->max_fields <= 0)
3937  {
3938  if (cstate->line_buf.len != 0)
3939  ereport(ERROR,
3940  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3941  errmsg("extra data after last expected column")));
3942  return 0;
3943  }
3944 
3945  resetStringInfo(&cstate->attribute_buf);
3946 
3947  /*
3948  * The de-escaped attributes will certainly not be longer than the input
3949  * data line, so we can just force attribute_buf to be large enough and
3950  * then transfer data without any checks for enough space. We need to do
3951  * it this way because enlarging attribute_buf mid-stream would invalidate
3952  * pointers already stored into cstate->raw_fields[].
3953  */
3954  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
3955  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
3956  output_ptr = cstate->attribute_buf.data;
3957 
3958  /* set pointer variables for loop */
3959  cur_ptr = cstate->line_buf.data;
3960  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
3961 
3962  /* Outer loop iterates over fields */
3963  fieldno = 0;
3964  for (;;)
3965  {
3966  bool found_delim = false;
3967  char *start_ptr;
3968  char *end_ptr;
3969  int input_len;
3970  bool saw_non_ascii = false;
3971 
3972  /* Make sure there is enough space for the next value */
3973  if (fieldno >= cstate->max_fields)
3974  {
3975  cstate->max_fields *= 2;
3976  cstate->raw_fields =
3977  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
3978  }
3979 
3980  /* Remember start of field on both input and output sides */
3981  start_ptr = cur_ptr;
3982  cstate->raw_fields[fieldno] = output_ptr;
3983 
3984  /*
3985  * Scan data for field.
3986  *
3987  * Note that in this loop, we are scanning to locate the end of field
3988  * and also speculatively performing de-escaping. Once we find the
3989  * end-of-field, we can match the raw field contents against the null
3990  * marker string. Only after that comparison fails do we know that
3991  * de-escaping is actually the right thing to do; therefore we *must
3992  * not* throw any syntax errors before we've done the null-marker
3993  * check.
3994  */
3995  for (;;)
3996  {
3997  char c;
3998 
3999  end_ptr = cur_ptr;
4000  if (cur_ptr >= line_end_ptr)
4001  break;
4002  c = *cur_ptr++;
4003  if (c == delimc)
4004  {
4005  found_delim = true;
4006  break;
4007  }
4008  if (c == '\\')
4009  {
4010  if (cur_ptr >= line_end_ptr)
4011  break;
4012  c = *cur_ptr++;
4013  switch (c)
4014  {
4015  case '0':
4016  case '1':
4017  case '2':
4018  case '3':
4019  case '4':
4020  case '5':
4021  case '6':
4022  case '7':
4023  {
4024  /* handle \013 */
4025  int val;
4026 
4027  val = OCTVALUE(c);
4028  if (cur_ptr < line_end_ptr)
4029  {
4030  c = *cur_ptr;
4031  if (ISOCTAL(c))
4032  {
4033  cur_ptr++;
4034  val = (val << 3) + OCTVALUE(c);
4035  if (cur_ptr < line_end_ptr)
4036  {
4037  c = *cur_ptr;
4038  if (ISOCTAL(c))
4039  {
4040  cur_ptr++;
4041  val = (val << 3) + OCTVALUE(c);
4042  }
4043  }
4044  }
4045  }
4046  c = val & 0377;
4047  if (c == '\0' || IS_HIGHBIT_SET(c))
4048  saw_non_ascii = true;
4049  }
4050  break;
4051  case 'x':
4052  /* Handle \x3F */
4053  if (cur_ptr < line_end_ptr)
4054  {
4055  char hexchar = *cur_ptr;
4056 
4057  if (isxdigit((unsigned char) hexchar))
4058  {
4059  int val = GetDecimalFromHex(hexchar);
4060 
4061  cur_ptr++;
4062  if (cur_ptr < line_end_ptr)
4063  {
4064  hexchar = *cur_ptr;
4065  if (isxdigit((unsigned char) hexchar))
4066  {
4067  cur_ptr++;
4068  val = (val << 4) + GetDecimalFromHex(hexchar);
4069  }
4070  }
4071  c = val & 0xff;
4072  if (c == '\0' || IS_HIGHBIT_SET(c))
4073  saw_non_ascii = true;
4074  }
4075  }
4076  break;
4077  case 'b':
4078  c = '\b';
4079  break;
4080  case 'f':
4081  c = '\f';
4082  break;
4083  case 'n':
4084  c = '\n';
4085  break;
4086  case 'r':
4087  c = '\r';
4088  break;
4089  case 't':
4090  c = '\t';
4091  break;
4092  case 'v':
4093  c = '\v';
4094  break;
4095 
4096  /*
4097  * in all other cases, take the char after '\'
4098  * literally
4099  */
4100  }
4101  }
4102 
4103  /* Add c to output string */
4104  *output_ptr++ = c;
4105  }
4106 
4107  /* Check whether raw input matched null marker */
4108  input_len = end_ptr - start_ptr;
4109  if (input_len == cstate->null_print_len &&
4110  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4111  cstate->raw_fields[fieldno] = NULL;
4112  else
4113  {
4114  /*
4115  * At this point we know the field is supposed to contain data.
4116  *
4117  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4118  * resulting string is valid data for the db encoding.
4119  */
4120  if (saw_non_ascii)
4121  {
4122  char *fld = cstate->raw_fields[fieldno];
4123 
4124  pg_verifymbstr(fld, output_ptr - fld, false);
4125  }
4126  }
4127 
4128  /* Terminate attribute value in output area */
4129  *output_ptr++ = '\0';
4130 
4131  fieldno++;
4132  /* Done if we hit EOL instead of a delim */
4133  if (!found_delim)
4134  break;
4135  }
4136 
4137  /* Clean up state of attribute_buf */
4138  output_ptr--;
4139  Assert(*output_ptr == '\0');
4140  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4141 
4142  return fieldno;
4143 }
4144 
4145 /*
4146  * Parse the current line into separate attributes (fields),
4147  * performing de-escaping as needed. This has exactly the same API as
4148  * CopyReadAttributesText, except we parse the fields according to
4149  * "standard" (i.e. common) CSV usage.
4150  */
4151 static int
4153 {
4154  char delimc = cstate->delim[0];
4155  char quotec = cstate->quote[0];
4156  char escapec = cstate->escape[0];
4157  int fieldno;
4158  char *output_ptr;
4159  char *cur_ptr;
4160  char *line_end_ptr;
4161 
4162  /*
4163  * We need a special case for zero-column tables: check that the input
4164  * line is empty, and return.
4165  */
4166  if (cstate->max_fields <= 0)
4167  {
4168  if (cstate->line_buf.len != 0)
4169  ereport(ERROR,
4170  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4171  errmsg("extra data after last expected column")));
4172  return 0;
4173  }
4174 
4175  resetStringInfo(&cstate->attribute_buf);
4176 
4177  /*
4178  * The de-escaped attributes will certainly not be longer than the input
4179  * data line, so we can just force attribute_buf to be large enough and
4180  * then transfer data without any checks for enough space. We need to do
4181  * it this way because enlarging attribute_buf mid-stream would invalidate
4182  * pointers already stored into cstate->raw_fields[].
4183  */
4184  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4185  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4186  output_ptr = cstate->attribute_buf.data;
4187 
4188  /* set pointer variables for loop */
4189  cur_ptr = cstate->line_buf.data;
4190  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4191 
4192  /* Outer loop iterates over fields */
4193  fieldno = 0;
4194  for (;;)
4195  {
4196  bool found_delim = false;
4197  bool saw_quote = false;
4198  char *start_ptr;
4199  char *end_ptr;
4200  int input_len;
4201 
4202  /* Make sure there is enough space for the next value */
4203  if (fieldno >= cstate->max_fields)
4204  {
4205  cstate->max_fields *= 2;
4206  cstate->raw_fields =
4207  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4208  }
4209 
4210  /* Remember start of field on both input and output sides */
4211  start_ptr = cur_ptr;
4212  cstate->raw_fields[fieldno] = output_ptr;
4213 
4214  /*
4215  * Scan data for field,
4216  *
4217  * The loop starts in "not quote" mode and then toggles between that
4218  * and "in quote" mode. The loop exits normally if it is in "not
4219  * quote" mode and a delimiter or line end is seen.
4220  */
4221  for (;;)
4222  {
4223  char c;
4224 
4225  /* Not in quote */
4226  for (;;)
4227  {
4228  end_ptr = cur_ptr;
4229  if (cur_ptr >= line_end_ptr)
4230  goto endfield;
4231  c = *cur_ptr++;
4232  /* unquoted field delimiter */
4233  if (c == delimc)
4234  {
4235  found_delim = true;
4236  goto endfield;
4237  }
4238  /* start of quoted field (or part of field) */
4239  if (c == quotec)
4240  {
4241  saw_quote = true;
4242  break;
4243  }
4244  /* Add c to output string */
4245  *output_ptr++ = c;
4246  }
4247 
4248  /* In quote */
4249  for (;;)
4250  {
4251  end_ptr = cur_ptr;
4252  if (cur_ptr >= line_end_ptr)
4253  ereport(ERROR,
4254  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4255  errmsg("unterminated CSV quoted field")));
4256 
4257  c = *cur_ptr++;
4258 
4259  /* escape within a quoted field */
4260  if (c == escapec)
4261  {
4262  /*
4263  * peek at the next char if available, and escape it if it
4264  * is an escape char or a quote char
4265  */
4266  if (cur_ptr < line_end_ptr)
4267  {
4268  char nextc = *cur_ptr;
4269 
4270  if (nextc == escapec || nextc == quotec)
4271  {
4272  *output_ptr++ = nextc;
4273  cur_ptr++;
4274  continue;
4275  }
4276  }
4277  }
4278 
4279  /*
4280  * end of quoted field. Must do this test after testing for
4281  * escape in case quote char and escape char are the same
4282  * (which is the common case).
4283  */
4284  if (c == quotec)
4285  break;
4286 
4287  /* Add c to output string */
4288  *output_ptr++ = c;
4289  }
4290  }
4291 endfield:
4292 
4293  /* Terminate attribute value in output area */
4294  *output_ptr++ = '\0';
4295 
4296  /* Check whether raw input matched null marker */
4297  input_len = end_ptr - start_ptr;
4298  if (!saw_quote && input_len == cstate->null_print_len &&
4299  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4300  cstate->raw_fields[fieldno] = NULL;
4301 
4302  fieldno++;
4303  /* Done if we hit EOL instead of a delim */
4304  if (!found_delim)
4305  break;
4306  }
4307 
4308  /* Clean up state of attribute_buf */
4309  output_ptr--;
4310  Assert(*output_ptr == '\0');
4311  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4312 
4313  return fieldno;
4314 }
4315 
4316 
4317 /*
4318  * Read a binary attribute
4319  */
4320 static Datum
4322  int column_no, FmgrInfo *flinfo,
4323  Oid typioparam, int32 typmod,
4324  bool *isnull)
4325 {
4326  int32 fld_size;
4327  Datum result;
4328 
4329  if (!CopyGetInt32(cstate, &fld_size))
4330  ereport(ERROR,
4331  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4332  errmsg("unexpected EOF in COPY data")));
4333  if (fld_size == -1)
4334  {
4335  *isnull = true;
4336  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4337  }
4338  if (fld_size < 0)
4339  ereport(ERROR,
4340  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4341  errmsg("invalid field size")));
4342 
4343  /* reset attribute_buf to empty, and load raw data in it */
4344  resetStringInfo(&cstate->attribute_buf);
4345 
4346  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4347  if (CopyGetData(cstate, cstate->attribute_buf.data,
4348  fld_size, fld_size) != fld_size)
4349  ereport(ERROR,
4350  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4351  errmsg("unexpected EOF in COPY data")));
4352 
4353  cstate->attribute_buf.len = fld_size;
4354  cstate->attribute_buf.data[fld_size] = '\0';
4355 
4356  /* Call the column type's binary input converter */
4357  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4358  typioparam, typmod);
4359 
4360  /* Trouble if it didn't eat the whole buffer */
4361  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4362  ereport(ERROR,
4363  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4364  errmsg("incorrect binary data format")));
4365 
4366  *isnull = false;
4367  return result;
4368 }
4369 
4370 /*
4371  * Send text representation of one attribute, with conversion and escaping
4372  */
4373 #define DUMPSOFAR() \
4374  do { \
4375  if (ptr > start) \
4376  CopySendData(cstate, start, ptr - start); \
4377  } while (0)
4378 
4379 static void
4380 CopyAttributeOutText(CopyState cstate, char *string)
4381 {
4382  char *ptr;
4383  char *start;
4384  char c;
4385  char delimc = cstate->delim[0];
4386 
4387  if (cstate->need_transcoding)
4388  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4389  else
4390  ptr = string;
4391 
4392  /*
4393  * We have to grovel through the string searching for control characters
4394  * and instances of the delimiter character. In most cases, though, these
4395  * are infrequent. To avoid overhead from calling CopySendData once per
4396  * character, we dump out all characters between escaped characters in a
4397  * single call. The loop invariant is that the data from "start" to "ptr"
4398  * can be sent literally, but hasn't yet been.
4399  *
4400  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4401  * in valid backend encodings, extra bytes of a multibyte character never
4402  * look like ASCII. This loop is sufficiently performance-critical that
4403  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4404  * of the normal safe-encoding path.
4405  */
4406  if (cstate->encoding_embeds_ascii)
4407  {
4408  start = ptr;
4409  while ((c = *ptr) != '\0')
4410  {
4411  if ((unsigned char) c < (unsigned char) 0x20)
4412  {
4413  /*
4414  * \r and \n must be escaped, the others are traditional. We
4415  * prefer to dump these using the C-like notation, rather than
4416  * a backslash and the literal character, because it makes the
4417  * dump file a bit more proof against Microsoftish data
4418  * mangling.
4419  */
4420  switch (c)
4421  {
4422  case '\b':
4423  c = 'b';
4424  break;
4425  case '\f':
4426  c = 'f';
4427  break;
4428  case '\n':
4429  c = 'n';
4430  break;
4431  case '\r':
4432  c = 'r';
4433  break;
4434  case '\t':
4435  c = 't';
4436  break;
4437  case '\v':
4438  c = 'v';
4439  break;
4440  default:
4441  /* If it's the delimiter, must backslash it */
4442  if (c == delimc)
4443  break;
4444  /* All ASCII control chars are length 1 */
4445  ptr++;
4446  continue; /* fall to end of loop */
4447  }
4448  /* if we get here, we need to convert the control char */
4449  DUMPSOFAR();
4450  CopySendChar(cstate, '\\');
4451  CopySendChar(cstate, c);
4452  start = ++ptr; /* do not include char in next run */
4453  }
4454  else if (c == '\\' || c == delimc)
4455  {
4456  DUMPSOFAR();
4457  CopySendChar(cstate, '\\');
4458  start = ptr++; /* we include char in next run */
4459  }
4460  else if (IS_HIGHBIT_SET(c))
4461  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4462  else
4463  ptr++;
4464  }
4465  }
4466  else
4467  {
4468  start = ptr;
4469  while ((c = *ptr) != '\0')
4470  {
4471  if ((unsigned char) c < (unsigned char) 0x20)
4472  {
4473  /*
4474  * \r and \n must be escaped, the others are traditional. We
4475  * prefer to dump these using the C-like notation, rather than
4476  * a backslash and the literal character, because it makes the
4477  * dump file a bit more proof against Microsoftish data
4478  * mangling.
4479  */
4480  switch (c)
4481  {
4482  case '\b':
4483  c = 'b';
4484  break;
4485  case '\f':
4486  c = 'f';
4487  break;
4488  case '\n':
4489  c = 'n';
4490  break;
4491  case '\r':
4492  c = 'r';
4493  break;
4494  case '\t':
4495  c = 't';
4496  break;
4497  case '\v':
4498  c = 'v';
4499  break;
4500  default:
4501  /* If it's the delimiter, must backslash it */
4502  if (c == delimc)
4503  break;
4504  /* All ASCII control chars are length 1 */
4505  ptr++;
4506  continue; /* fall to end of loop */
4507  }
4508  /* if we get here, we need to convert the control char */
4509  DUMPSOFAR();
4510  CopySendChar(cstate, '\\');
4511  CopySendChar(cstate, c);
4512  start = ++ptr; /* do not include char in next run */
4513  }
4514  else if (c == '\\' || c == delimc)
4515  {
4516  DUMPSOFAR();
4517  CopySendChar(cstate, '\\');
4518  start = ptr++; /* we include char in next run */
4519  }
4520  else
4521  ptr++;
4522  }
4523  }
4524 
4525  DUMPSOFAR();
4526 }
4527 
4528 /*
4529  * Send text representation of one attribute, with conversion and
4530  * CSV-style escaping
4531  */
4532 static void
4533 CopyAttributeOutCSV(CopyState cstate, char *string,
4534  bool use_quote, bool single_attr)
4535 {
4536  char *ptr;
4537  char *start;
4538  char c;
4539  char delimc = cstate->delim[0];
4540  char quotec = cstate->quote[0];
4541  char escapec = cstate->escape[0];
4542 
4543  /* force quoting if it matches null_print (before conversion!) */
4544  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4545  use_quote = true;
4546 
4547  if (cstate->need_transcoding)
4548  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4549  else
4550  ptr = string;
4551 
4552  /*
4553  * Make a preliminary pass to discover if it needs quoting
4554  */
4555  if (!use_quote)
4556  {
4557  /*
4558  * Because '\.' can be a data value, quote it if it appears alone on a
4559  * line so it is not interpreted as the end-of-data marker.
4560  */
4561  if (single_attr && strcmp(ptr, "\\.") == 0)
4562  use_quote = true;
4563  else
4564  {
4565  char *tptr = ptr;
4566 
4567  while ((c = *tptr) != '\0')
4568  {
4569  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4570  {
4571  use_quote = true;
4572  break;
4573  }
4574  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4575  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4576  else
4577  tptr++;
4578  }
4579  }
4580  }
4581 
4582  if (use_quote)
4583  {
4584  CopySendChar(cstate, quotec);
4585 
4586  /*
4587  * We adopt the same optimization strategy as in CopyAttributeOutText
4588  */
4589  start = ptr;
4590  while ((c = *ptr) != '\0')
4591  {
4592  if (c == quotec || c == escapec)
4593  {
4594  DUMPSOFAR();
4595  CopySendChar(cstate, escapec);
4596  start = ptr; /* we include char in next run */
4597  }
4598  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4599  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4600  else
4601  ptr++;
4602  }
4603  DUMPSOFAR();
4604 
4605  CopySendChar(cstate, quotec);
4606  }
4607  else
4608  {
4609  /* If it doesn't need quoting, we can just dump it as-is */
4610  CopySendString(cstate, ptr);
4611  }
4612 }
4613 
4614 /*
4615  * CopyGetAttnums - build an integer list of attnums to be copied
4616  *
4617  * The input attnamelist is either the user-specified column list,
4618  * or NIL if there was none (in which case we want all the non-dropped
4619  * columns).
4620  *
4621  * rel can be NULL ... it's only used for error reports.
4622  */
4623 static List *
4624 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
4625 {
4626  List *attnums = NIL;
4627 
4628  if (attnamelist == NIL)
4629  {
4630  /* Generate default column list */
4631  Form_pg_attribute *attr = tupDesc->attrs;
4632  int attr_count = tupDesc->natts;
4633  int i;
4634 
4635  for (i = 0; i < attr_count; i++)
4636  {
4637  if (attr[i]->attisdropped)
4638  continue;
4639  attnums = lappend_int(attnums, i + 1);
4640  }
4641  }
4642  else
4643  {
4644  /* Validate the user-supplied list and extract attnums */
4645  ListCell *l;
4646 
4647  foreach(l, attnamelist)
4648  {
4649  char *name = strVal(lfirst(l));
4650  int attnum;
4651  int i;
4652 
4653  /* Lookup column name */
4654  attnum = InvalidAttrNumber;
4655  for (i = 0; i < tupDesc->natts; i++)
4656  {
4657  if (tupDesc->attrs[i]->attisdropped)
4658  continue;
4659  if (namestrcmp(&(tupDesc->attrs[i]->attname), name) == 0)
4660  {
4661  attnum = tupDesc->attrs[i]->attnum;
4662  break;
4663  }
4664  }
4665  if (attnum == InvalidAttrNumber)
4666  {
4667  if (rel != NULL)
4668  ereport(ERROR,
4669  (errcode(ERRCODE_UNDEFINED_COLUMN),
4670  errmsg("column \"%s\" of relation \"%s\" does not exist",
4671  name, RelationGetRelationName(rel))));
4672  else
4673  ereport(ERROR,
4674  (errcode(ERRCODE_UNDEFINED_COLUMN),
4675  errmsg("column \"%s\" does not exist",
4676  name)));
4677  }
4678  /* Check for duplicates */
4679  if (list_member_int(attnums, attnum))
4680  ereport(ERROR,
4681  (errcode(ERRCODE_DUPLICATE_COLUMN),
4682  errmsg("column \"%s\" specified more than once",
4683  name)));
4684  attnums = lappend_int(attnums, attnum);
4685  }
4686  }
4687 
4688  return attnums;
4689 }
4690 
4691 
4692 /*
4693  * copy_dest_startup --- executor startup
4694  */
4695 static void
4696 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
4697 {
4698  /* no-op */
4699 }
4700 
4701 /*
4702  * copy_dest_receive --- receive one tuple
4703  */
4704 static bool
4706 {
4707  DR_copy *myState = (DR_copy *) self;
4708  CopyState cstate = myState->cstate;
4709 
4710  /* Make sure the tuple is fully deconstructed */
4711  slot_getallattrs(slot);
4712 
4713  /* And send the data */
4714  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4715  myState->processed++;
4716 
4717  return true;
4718 }
4719 
4720 /*
4721  * copy_dest_shutdown --- executor end
4722  */
4723 static void
4725 {
4726  /* no-op */
4727 }
4728 
4729 /*
4730  * copy_dest_destroy --- release DestReceiver object
4731  */
4732 static void
4734 {
4735  pfree(self);
4736 }
4737 
4738 /*
4739  * CreateCopyDestReceiver -- create a suitable DestReceiver object
4740  */
4741 DestReceiver *
4743 {
4744  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4745 
4746  self->pub.receiveSlot = copy_dest_receive;
4747  self->pub.rStartup = copy_dest_startup;
4748  self->pub.rShutdown = copy_dest_shutdown;
4749  self->pub.rDestroy = copy_dest_destroy;
4750  self->pub.mydest = DestCopyOut;
4751 
4752  self->cstate = NULL; /* will be set later */
4753  self->processed = 0;
4754 
4755  return (DestReceiver *) self;
4756 }
signed short int16
Definition: c.h:255
List * indirection
Definition: parsenodes.h:432
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3139
int ri_NumIndices
Definition: execnodes.h:338
#define NIL
Definition: pg_list.h:69
uint32 CommandId
Definition: c.h:411
static Datum CopyReadBinaryAttribute(CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4321
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
static int GetDecimalFromHex(char hex)
Definition: copy.c:3896
Definition: fmgr.h:53
#define MAX_COPY_DATA_DISPLAY
bool csv_mode
Definition: copy.c:117
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:415
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1255
Relation ri_RelationDesc
Definition: execnodes.h:337
List * range_table
Definition: copy.c:165
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1378
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:776
#define IsA(nodeptr, _type_)
Definition: nodes.h:571
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3537
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:200
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:122
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate)
Definition: execTuples.c:852
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:994
Node * val
Definition: parsenodes.h:433
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:681
int errhint(const char *fmt,...)
Definition: elog.c:987
int pg_char_to_encoding(const char *name)
Definition: encnames.c:551
char ** raw_fields
Definition: copy.c:187
Definition: copy.c:73
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2600
#define VARDATA(PTR)
Definition: postgres.h:303
static void EndCopy(CopyState cstate)
Definition: copy.c:1739
bool binary
Definition: copy.c:114
static struct @78 value
#define pq_flush()
Definition: libpq.h:39
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2192
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1581
void PreventCommandIfParallelMode(const char *cmdname)
Definition: utility.c:253
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
void pq_sendbyte(StringInfo buf, int byt)
Definition: pqformat.c:105
List * attlist
Definition: parsenodes.h:1889
List * fromClause
Definition: parsenodes.h:1465
#define ISOCTAL(c)
Definition: copy.c:52
#define OCTVALUE(c)
Definition: copy.c:53
#define ResetPerTupleExprContext(estate)
Definition: executor.h:348
#define RelationGetDescr(relation)
Definition: rel.h:425
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
char * name
Definition: parsenodes.h:431
bool need_transcoding
Definition: copy.c:104
ResultRelInfo * partitions
Definition: copy.c:170
#define castNode(_type_, nodeptr)
Definition: nodes.h:589
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:103
FmgrInfo * in_functions
Definition: copy.c:160
AttrNumber num_defaults
Definition: copy.c:156
List * attnumlist
Definition: copy.c:110
#define OIDOID
Definition: pg_type.h:328
#define VARSIZE(PTR)
Definition: postgres.h:304
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:621
char * filename
Definition: copy.c:111
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:2884
#define VARHDRSZ
Definition: c.h:445
bool file_has_oids
Definition: copy.c:157
#define DatumGetObjectId(X)
Definition: postgres.h:506
List * relationOids
Definition: plannodes.h:77
char * pstrdup(const char *in)
Definition: mcxt.c:1077
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:136
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:379
void initLongStringInfo(StringInfo str)
Definition: stringinfo.c:81
#define XLogIsNeeded()
Definition: xlog.h:145
Definition: copy.c:214
#define MAX_BUFFERED_TUPLES
bool rd_islocaltemp
Definition: rel.h:90
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2251
Expr * expression_planner(Expr *expr)
Definition: planner.c:5382
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:144
Form_pg_attribute * attrs
Definition: tupdesc.h:74
DestReceiver pub
Definition: copy.c:216
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1020
#define RELKIND_MATVIEW
Definition: pg_class.h:165
StringInfoData line_buf
Definition: copy.c:196
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:250
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:834
#define AccessShareLock
Definition: lockdefs.h:36
#define InvalidBuffer
Definition: buf.h:25
struct CopyStateData CopyStateData
Definition: nodes.h:520
#define strVal(v)
Definition: value.h:54
struct cursor * cur
Definition: ecpg.c:28
int raw_buf_index
Definition: copy.c:209
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4380
bool line_buf_valid
Definition: copy.c:198
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1603
CopyState cstate
Definition: copy.c:217
int errcode(int sqlerrcode)
Definition: elog.c:575
#define PG_BINARY_W
Definition: c.h:1041
bool superuser(void)
Definition: superuser.c:47
int namestrcmp(Name name, const char *str)
Definition: name.c:248
#define MemSet(start, val, len)
Definition: c.h:857
bool fe_eof
Definition: copy.c:101
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2297
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:110
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:421
#define select(n, r, w, e, timeout)
Definition: win32.h:384
void heap_sync(Relation rel)
Definition: heapam.c:9138
Datum * tts_values
Definition: tuptable.h:125
return result
Definition: formatting.c:1618
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:28
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1716
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:135
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4152
void PopActiveSnapshot(void)
Definition: snapmgr.c:807
AclMode requiredPerms
Definition: parsenodes.h:1004
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:692
#define heap_close(r, l)
Definition: heapam.h:97
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, int instrument_options)
Definition: pquery.c:66
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:576
bool * force_quote_flags
Definition: copy.c:127
List * es_range_table
Definition: execnodes.h:374
Form_pg_class rd_rel
Definition: rel.h:113
unsigned int Oid
Definition: postgres_ext.h:31
char * delim
Definition: copy.c:122
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Node * utilityStmt
Definition: parsenodes.h:111
bool is_program
Definition: parsenodes.h:1892
bool volatile_defexprs
Definition: copy.c:164
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
struct ErrorContextCallback * previous
Definition: elog.h:238
#define PG_BINARY_R
Definition: c.h:1040
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4733
bool * force_null_flags
Definition: copy.c:131
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
MemoryContext rowcontext
Definition: copy.c:151
int natts
Definition: tupdesc.h:73
bool line_buf_converted
Definition: copy.c:197
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
signed int int32
Definition: c.h:256
int ClosePipeStream(FILE *file)
Definition: fd.c:2433
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
bool * convert_select_flags
Definition: copy.c:134
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:456
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1976
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
int location
Definition: parsenodes.h:227
void * copyObject(const void *from)
Definition: copyfuncs.c:4605
CopyDest copy_dest
Definition: copy.c:97
int location
Definition: parsenodes.h:434
#define REFILL_LINEBUF
Definition: copy.c:267
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:698
ErrorContextCallback * error_context_stack
Definition: elog.c:88
#define list_make1(x1)
Definition: pg_list.h:133
char * null_print
Definition: copy.c:119
const char * cur_attname
Definition: copy.c:139
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:453
Definition: copy.c:61
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams)
Definition: postgres.c:644
bool trig_insert_instead_row
Definition: reltrigger.h:57
void FreeExecutorState(EState *estate)
Definition: execUtils.c:169
Relation rel
Definition: copy.c:108
#define GetPerTupleExprContext(estate)
Definition: executor.h:339
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2322
MemoryContext copycontext
Definition: copy.c:145
uint32 AclMode
Definition: parsenodes.h:63
#define pq_startcopyout()
Definition: libpq.h:46
copy_data_source_cb data_source_cb
Definition: copy.c:113
bool defGetBoolean(DefElem *def)
Definition: define.c:111
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:135
ExprState * ExecInitExpr(Expr *node, PlanState *parent)
Definition: execQual.c:4267
Bitmapset * selectedCols
Definition: parsenodes.h:1006
unsigned short uint16
Definition: c.h:267
void pfree(void *pointer)
Definition: mcxt.c:950
void ExecSetupPartitionTupleRouting(Relation rel, PartitionDispatch **pd, ResultRelInfo **partitions, TupleConversionMap ***tup_conv_maps, TupleTableSlot **partition_tuple_slot, int *num_parted, int *num_partitions)
Definition: execMain.c:3138
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
#define linitial(l)
Definition: pg_list.h:110
static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
Definition: copy.c:1762
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:698
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1127
#define ExecEvalExpr(expr, econtext, isNull)
Definition: executor.h:73
TupleTableSlot * partition_tuple_slot
Definition: copy.c:172
#define ObjectIdGetDatum(X)
Definition: postgres.h:513
#define ERROR
Definition: elog.h:43
void pq_startmsgread(void)
Definition: pqcomm.c:1191
#define DatumGetCString(X)
Definition: postgres.h:572
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:555
#define S_IWGRP
Definition: win32.h:478
#define lfirst_int(lc)
Definition: pg_list.h:107
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:291
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4533
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1990
static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
Definition: copy.c:2801
#define FATAL
Definition: elog.h:52
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:159
char * defGetString(DefElem *def)
Definition: define.c:49
RangeVar * relation
Definition: parsenodes.h:1886
ItemPointerData t_self
Definition: htup.h:65
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:764
int pg_mbcliplen(const char *mbstr, int len, int limit)
Definition: mbutils.c:831
TriggerDesc * trigdesc
Definition: rel.h:119
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
static char * limit_printout_length(const char *str)
Definition: copy.c:2268
QueryDesc * queryDesc
Definition: copy.c:109
EolType
Definition: copy.c:70
bool list_member_int(const List *list, int datum)
Definition: list.c:485
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4724
int num_dispatch
Definition: copy.c:168
bool encoding_embeds_ascii
Definition: copy.c:105
uint32 t_len
Definition: htup.h:64
char * get_namespace_name(Oid nspid)
Definition: lsyscache.c:3006
char * c
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2111
char * raw_buf
Definition: copy.c:208
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:735
Node * stmt
Definition: parsenodes.h:1376
#define NoLock
Definition: lockdefs.h:34
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1075
char * quote
Definition: copy.c:123
static char * buf
Definition: pg_test_fsync.c:65
#define memmove(d, s, c)
Definition: c.h:1058
bool * tts_isnull
Definition: tuptable.h:126
void ExecDropSingleTupleTableSlot(TupleTableSlot *slot)
Definition: execTuples.c:216
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
List * targetList
Definition: parsenodes.h:1464
ResultRelInfo * es_result_relations
Definition: execnodes.h:384
static uint64 DoCopyTo(CopyState cstate)
Definition: copy.c:1891
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4696
int location
Definition: parsenodes.h:711
#define RowExclusiveLock
Definition: lockdefs.h:38
ExprState ** defexprs
Definition: copy.c:163
const char * cur_relname
Definition: copy.c:137
int errcode_for_file_access(void)
Definition: elog.c:598
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
#define is_absolute_path(filename)
Definition: port.h:77
#define CStringGetDatum(X)
Definition: postgres.h:584
char string[11]
Definition: preproc-type.c:46
Definition: copy.c:74
List * options
Definition: parsenodes.h:1894
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2057
FmgrInfo oid_in_function
Definition: copy.c:158
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:277
#define RelationGetRelationName(relation)
Definition: rel.h:433
static const char BinarySignature[11]
Definition: copy.c:287
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:184
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:94
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:345
unsigned int uint32
Definition: c.h:268
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:2037
int raw_buf_len
Definition: copy.c:210
Oid t_tableOid
Definition: htup.h:66
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
bool trig_insert_after_row
Definition: reltrigger.h:56
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
int max_fields
Definition: copy.c:186
char * escape
Definition: copy.c:124
const char * p_sourcetext
Definition: parse_node.h:167
List * returningList
Definition: parsenodes.h:135
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:390
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2147
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2633
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2567
#define ereport(elevel, rest)
Definition: elog.h:122
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2399
int null_print_len
Definition: copy.c:120
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1239
List * force_null
Definition: copy.c:130
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:341
void ExecutorFinish(QueryDesc *queryDesc)
Definition: execMain.c:393
EState * CreateExecutorState(void)
Definition: execUtils.c:73
List * lappend_int(List *list, int datum)
Definition: list.c:146
Node * arg
Definition: parsenodes.h:709
Definition: copy.c:75
List * lappend(List *list, void *datum)
Definition: list.c:128
#define S_IWOTH
Definition: win32.h:482
int file_encoding
Definition: copy.c:103
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4624
TupleDesc tupDesc
Definition: execdesc.h:46
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1253
#define InvalidSnapshot
Definition: snapshot.h:25
SubTransactionId rd_createSubid
Definition: rel.h:109
Oid * typioparams
Definition: copy.c:161
bool is_program
Definition: copy.c:112
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
Node * build_column_default(Relation rel, int attrno)
bool trig_insert_before_row
Definition: reltrigger.h:55
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2666
List * es_tupleTable
Definition: execnodes.h:401
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:156
void * palloc0(Size size)
Definition: mcxt.c:878
bool header_line
Definition: copy.c:118
uintptr_t Datum
Definition: postgres.h:372
int GetDatabaseEncoding(void)
Definition: mbutils.c:1015
int pg_get_client_encoding(void)
Definition: mbutils.c:317
#define ACL_SELECT
Definition: parsenodes.h:66
TupleTableSlot * tupslot
Definition: partition.h:66
void ExecSetSlotDescriptor(TupleTableSlot *slot, TupleDesc tupdesc)
Definition: execTuples.c:247
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2163
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:3227
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1797
int stmt_len
Definition: parsenodes.h:1378
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:238
int stmt_location
Definition: parsenodes.h:1377
int es_num_result_relations
Definition: execnodes.h:385
List * ri_PartitionCheck
Definition: execnodes.h:355
#define RAW_BUF_SIZE
Definition: copy.c:207
TupleDesc rd_att
Definition: rel.h:114
bool freeze
Definition: copy.c:116
Datum InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod)
Definition: fmgr.c:1932
int pq_getbyte(void)
Definition: pqcomm.c:981
void pq_endmsgread(void)
Definition: pqcomm.c:1215
Relation heap_openrv(const RangeVar *relation, LOCKMODE lockmode)
Definition: heapam.c:1315
#define InvalidOid
Definition: postgres_ext.h:36
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4705
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *orig_slot, EState *estate)
Definition: execMain.c:1828
bool XactReadOnly
Definition: xact.c:77
bool * force_notnull_flags
Definition: copy.c:129
CmdType commandType
Definition: parsenodes.h:103
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4150
List * force_notnull
Definition: copy.c:128
int check_enable_rls(Oid relid, Oid checkAsUser, bool noError)
Definition: rls.c:53
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
void EndCopyFrom(CopyState cstate)
Definition: copy.c:3432
#define PG_CATCH()
Definition: elog.h:293
#define makeNode(_type_)
Definition: nodes.h:568
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:505
EolType eol_type
Definition: copy.c:102
#define NULL
Definition: c.h:229
QuerySource querySource
Definition: parsenodes.h:105
void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
Definition: copy.c:780
void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2661
#define Assert(condition)
Definition: c.h:675
#define lfirst(lc)
Definition: pg_list.h:106
void FreeBulkInsertState(BulkInsertState bistate)
Definition: heapam.c:2336
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:444
List * convert_select
Definition: copy.c:133
SubTransactionId GetCurrentSubTransactionId(void)
Definition: xact.c:649
StringInfo fe_msgbuf
Definition: copy.c:99
CopyDest
Definition: copy.c:59
void pq_copymsgbytes(StringInfo msg, char *buf, int datalen)
Definition: pqformat.c:570
static bool CopyReadLine(CopyState cstate)
Definition: copy.c:3448
#define ACL_INSERT
Definition: parsenodes.h:65
TupleConstr * constr
Definition: tupdesc.h:76
#define pq_endcopyout(errorAbort)
Definition: libpq.h:47
size_t Size
Definition: c.h:356
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:450
static int list_length(const List *l)
Definition: pg_list.h:89
int parser_errposition(ParseState *pstate, int location)
Definition: parse_node.c:109
static uint64 CopyTo(CopyState cstate)
Definition: copy.c:1945
#define PG_RE_THROW()
Definition: elog.h:314
#define InvalidSubTransactionId
Definition: c.h:403
Bitmapset * bms_add_member(Bitmapset *a, int x)
Definition: bitmapset.c:668
HeapTuple ExecMaterializeSlot(TupleTableSlot *slot)
Definition: execTuples.c:725
void ReleaseBulkInsertStatePin(BulkInsertState bistate)
Definition: heapam.c:2348
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes)
Definition: trigger.c:2239
const char * name
Definition: encode.c:521
HeapTuple do_convert_tuple(HeapTuple tuple, TupleConversionMap *map)
Definition: tupconvert.c:341
void PreventCommandIfReadOnly(const char *cmdname)
Definition: utility.c:235
#define InvalidAttrNumber
Definition: attnum.h:23
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:344
#define DUMPSOFAR()
Definition: copy.c:4373
#define HEAP_INSERT_SKIP_FSM
Definition: heapam.h:29
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:667