PostgreSQL Source Code  git master
copy.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copy.c
4  * Implements the COPY utility command
5  *
6  * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/commands/copy.c
12  *
13  *-------------------------------------------------------------------------
14  */
15 #include "postgres.h"
16 
17 #include <ctype.h>
18 #include <unistd.h>
19 #include <sys/stat.h>
20 
21 #include "access/heapam.h"
22 #include "access/htup_details.h"
23 #include "access/sysattr.h"
24 #include "access/xact.h"
25 #include "access/xlog.h"
26 #include "catalog/dependency.h"
27 #include "catalog/pg_authid.h"
28 #include "catalog/pg_type.h"
29 #include "commands/copy.h"
30 #include "commands/defrem.h"
31 #include "commands/trigger.h"
32 #include "executor/execPartition.h"
33 #include "executor/executor.h"
34 #include "foreign/fdwapi.h"
35 #include "libpq/libpq.h"
36 #include "libpq/pqformat.h"
37 #include "mb/pg_wchar.h"
38 #include "miscadmin.h"
39 #include "optimizer/clauses.h"
40 #include "optimizer/planner.h"
41 #include "nodes/makefuncs.h"
42 #include "parser/parse_relation.h"
43 #include "port/pg_bswap.h"
44 #include "rewrite/rewriteHandler.h"
45 #include "storage/fd.h"
46 #include "tcop/tcopprot.h"
47 #include "utils/builtins.h"
48 #include "utils/lsyscache.h"
49 #include "utils/memutils.h"
50 #include "utils/portal.h"
51 #include "utils/rel.h"
52 #include "utils/rls.h"
53 #include "utils/snapmgr.h"
54 
55 
56 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
57 #define OCTVALUE(c) ((c) - '0')
58 
59 /*
60  * Represents the different source/dest cases we need to worry about at
61  * the bottom level
62  */
63 typedef enum CopyDest
64 {
65  COPY_FILE, /* to/from file (or a piped program) */
66  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
67  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
68  COPY_CALLBACK /* to/from callback function */
69 } CopyDest;
70 
71 /*
72  * Represents the end-of-line terminator type of the input
73  */
74 typedef enum EolType
75 {
76  EOL_UNKNOWN,
77  EOL_NL,
78  EOL_CR,
79  EOL_CRNL
80 } EolType;
81 
82 /*
83  * This struct contains all the state variables used throughout a COPY
84  * operation. For simplicity, we use the same struct for all variants of COPY,
85  * even though some fields are used in only some cases.
86  *
87  * Multi-byte encodings: all supported client-side encodings encode multi-byte
88  * characters by having the first byte's high bit set. Subsequent bytes of the
89  * character can have the high bit not set. When scanning data in such an
90  * encoding to look for a match to a single-byte (ie ASCII) character, we must
91  * use the full pg_encoding_mblen() machinery to skip over multibyte
92  * characters, else we might find a false match to a trailing byte. In
93  * supported server encodings, there is no possibility of a false match, and
94  * it's faster to make useless comparisons to trailing bytes than it is to
95  * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
96  * when we have to do it the hard way.
97  */
98 typedef struct CopyStateData
99 {
100  /* low-level state data */
101  CopyDest copy_dest; /* type of copy source/destination */
102  FILE *copy_file; /* used if copy_dest == COPY_FILE */
103  StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
104  * dest == COPY_NEW_FE in COPY FROM */
105  bool fe_eof; /* true if detected end of copy data */
106  EolType eol_type; /* EOL type of input */
107  int file_encoding; /* file or remote side's character encoding */
108  bool need_transcoding; /* file encoding diff from server? */
109  bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
110 
111  /* parameters from the COPY command */
112  Relation rel; /* relation to copy to or from */
113  QueryDesc *queryDesc; /* executable query to copy from */
114  List *attnumlist; /* integer list of attnums to copy */
115  char *filename; /* filename, or NULL for STDIN/STDOUT */
116  bool is_program; /* is 'filename' a program to popen? */
117  copy_data_source_cb data_source_cb; /* function for reading data */
118  bool binary; /* binary format? */
119  bool oids; /* include OIDs? */
120  bool freeze; /* freeze rows on loading? */
121  bool csv_mode; /* Comma Separated Value format? */
122  bool header_line; /* CSV header line? */
123  char *null_print; /* NULL marker string (server encoding!) */
124  int null_print_len; /* length of same */
125  char *null_print_client; /* same converted to file encoding */
126  char *delim; /* column delimiter (must be 1 byte) */
127  char *quote; /* CSV quote char (must be 1 byte) */
128  char *escape; /* CSV escape char (must be 1 byte) */
129  List *force_quote; /* list of column names */
130  bool force_quote_all; /* FORCE_QUOTE *? */
131  bool *force_quote_flags; /* per-column CSV FQ flags */
132  List *force_notnull; /* list of column names */
133  bool *force_notnull_flags; /* per-column CSV FNN flags */
134  List *force_null; /* list of column names */
135  bool *force_null_flags; /* per-column CSV FN flags */
136  bool convert_selectively; /* do selective binary conversion? */
137  List *convert_select; /* list of column names (can be NIL) */
138  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
139 
140  /* these are just for error messages, see CopyFromErrorCallback */
141  const char *cur_relname; /* table name for error messages */
142  uint64 cur_lineno; /* line number for error messages */
143  const char *cur_attname; /* current att for error messages */
144  const char *cur_attval; /* current att value for error messages */
145 
146  /*
147  * Working state for COPY TO/FROM
148  */
149  MemoryContext copycontext; /* per-copy execution context */
150 
151  /*
152  * Working state for COPY TO
153  */
154  FmgrInfo *out_functions; /* lookup info for output functions */
155  MemoryContext rowcontext; /* per-row evaluation context */
156 
157  /*
158  * Working state for COPY FROM
159  */
160  AttrNumber num_defaults;
161  bool file_has_oids;
162  FmgrInfo oid_in_function;
163  Oid oid_typioparam;
164  FmgrInfo *in_functions; /* array of input functions for each attrs */
165  Oid *typioparams; /* array of element types for in_functions */
166  int *defmap; /* array of default att numbers */
167  ExprState **defexprs; /* array of default att expressions */
168  bool volatile_defexprs; /* is any of defexprs volatile? */
169  List *range_table;
170 
171  /* Tuple-routing support info */
172  PartitionTupleRouting *partition_tuple_routing;
173 
174  TransitionCaptureState *transition_capture;
175 
176  /*
177  * These variables are used to reduce overhead in textual COPY FROM.
178  *
179  * attribute_buf holds the separated, de-escaped text for each field of
180  * the current line. The CopyReadAttributes functions return arrays of
181  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
182  * the buffer on each cycle.
183  */
184  StringInfoData attribute_buf;
185 
186  /* field raw data pointers found by COPY FROM */
187 
188  int max_fields;
189  char **raw_fields;
190 
191  /*
192  * Similarly, line_buf holds the whole input line being processed. The
193  * input cycle is first to read the whole line into line_buf, convert it
194  * to server encoding there, and then extract the individual attribute
195  * fields into attribute_buf. line_buf is preserved unmodified so that we
196  * can display it in error messages if appropriate.
197  */
198  StringInfoData line_buf;
199  bool line_buf_converted; /* converted to server encoding? */
200  bool line_buf_valid; /* contains the row being processed? */
201 
202  /*
203  * Finally, raw_buf holds raw data read from the data source (file or
204  * client connection). CopyReadLine parses this data sufficiently to
205  * locate line boundaries, then transfers the data to line_buf and
206  * converts it. Note: we guarantee that there is a \0 at
207  * raw_buf[raw_buf_len].
208  */
209 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
210  char *raw_buf;
211  int raw_buf_index; /* next byte to process */
212  int raw_buf_len; /* total # of bytes stored */
213 } CopyStateData;
214 
215 /* DestReceiver for COPY (query) TO */
216 typedef struct
217 {
218  DestReceiver pub; /* publicly-known function pointers */
219  CopyState cstate; /* CopyStateData for the command */
220  uint64 processed; /* # of tuples processed */
221 } DR_copy;
222 
223 
224 /*
225  * These macros centralize code used to process line_buf and raw_buf buffers.
226  * They are macros because they often do continue/break control and to avoid
227  * function call overhead in tight COPY loops.
228  *
229  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
230  * prevent the continue/break processing from working. We end the "if (1)"
231  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
232  * any "else" in the calling code, and to avoid any compiler warnings about
233  * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
234  */
235 
236 /*
237  * This keeps the character read at the top of the loop in the buffer
238  * even if there is more than one read-ahead.
239  */
240 #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
241 if (1) \
242 { \
243  if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
244  { \
245  raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
246  need_data = true; \
247  continue; \
248  } \
249 } else ((void) 0)
250 
251 /* This consumes the remainder of the buffer and breaks */
252 #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
253 if (1) \
254 { \
255  if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
256  { \
257  if (extralen) \
258  raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
259  /* backslash just before EOF, treat as data char */ \
260  result = true; \
261  break; \
262  } \
263 } else ((void) 0)
264 
265 /*
266  * Transfer any approved data to line_buf; must do this to be sure
267  * there is some room in raw_buf.
268  */
269 #define REFILL_LINEBUF \
270 if (1) \
271 { \
272  if (raw_buf_ptr > cstate->raw_buf_index) \
273  { \
274  appendBinaryStringInfo(&cstate->line_buf, \
275  cstate->raw_buf + cstate->raw_buf_index, \
276  raw_buf_ptr - cstate->raw_buf_index); \
277  cstate->raw_buf_index = raw_buf_ptr; \
278  } \
279 } else ((void) 0)
280 
281 /* Undo any read-ahead and jump out of the block. */
282 #define NO_END_OF_COPY_GOTO \
283 if (1) \
284 { \
285  raw_buf_ptr = prev_raw_ptr + 1; \
286  goto not_end_of_copy; \
287 } else ((void) 0)
288 
289 static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
290 
291 
292 /* non-export function prototypes */
293 static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
294  RawStmt *raw_query, Oid queryRelId, List *attnamelist,
295  List *options);
296 static void EndCopy(CopyState cstate);
297 static void ClosePipeToProgram(CopyState cstate);
298 static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
299  Oid queryRelId, const char *filename, bool is_program,
300  List *attnamelist, List *options);
301 static void EndCopyTo(CopyState cstate);
302 static uint64 DoCopyTo(CopyState cstate);
303 static uint64 CopyTo(CopyState cstate);
304 static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
305  Datum *values, bool *nulls);
306 static void CopyFromInsertBatch(CopyState cstate, EState *estate,
307  CommandId mycid, int hi_options,
308  ResultRelInfo *resultRelInfo, TupleTableSlot *myslot,
309  BulkInsertState bistate,
310  int nBufferedTuples, HeapTuple *bufferedTuples,
311  uint64 firstBufferedLineNo);
312 static bool CopyReadLine(CopyState cstate);
313 static bool CopyReadLineText(CopyState cstate);
314 static int CopyReadAttributesText(CopyState cstate);
315 static int CopyReadAttributesCSV(CopyState cstate);
316 static Datum CopyReadBinaryAttribute(CopyState cstate,
317  int column_no, FmgrInfo *flinfo,
318  Oid typioparam, int32 typmod,
319  bool *isnull);
320 static void CopyAttributeOutText(CopyState cstate, char *string);
321 static void CopyAttributeOutCSV(CopyState cstate, char *string,
322  bool use_quote, bool single_attr);
323 static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
324  List *attnamelist);
325 static char *limit_printout_length(const char *str);
326 
327 /* Low-level communications functions */
328 static void SendCopyBegin(CopyState cstate);
329 static void ReceiveCopyBegin(CopyState cstate);
330 static void SendCopyEnd(CopyState cstate);
331 static void CopySendData(CopyState cstate, const void *databuf, int datasize);
332 static void CopySendString(CopyState cstate, const char *str);
333 static void CopySendChar(CopyState cstate, char c);
334 static void CopySendEndOfRow(CopyState cstate);
335 static int CopyGetData(CopyState cstate, void *databuf,
336  int minread, int maxread);
337 static void CopySendInt32(CopyState cstate, int32 val);
338 static bool CopyGetInt32(CopyState cstate, int32 *val);
339 static void CopySendInt16(CopyState cstate, int16 val);
340 static bool CopyGetInt16(CopyState cstate, int16 *val);
341 
342 
343 /*
344  * Send copy start/stop messages for frontend copies. These have changed
345  * in past protocol redesigns.
346  */
347 static void
348 SendCopyBegin(CopyState cstate)
349 {
350  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
351  {
352  /* new way */
353  StringInfoData buf;
354  int natts = list_length(cstate->attnumlist);
355  int16 format = (cstate->binary ? 1 : 0);
356  int i;
357 
358  pq_beginmessage(&buf, 'H');
359  pq_sendbyte(&buf, format); /* overall format */
360  pq_sendint16(&buf, natts);
361  for (i = 0; i < natts; i++)
362  pq_sendint16(&buf, format); /* per-column formats */
363  pq_endmessage(&buf);
364  cstate->copy_dest = COPY_NEW_FE;
365  }
366  else
367  {
368  /* old way */
369  if (cstate->binary)
370  ereport(ERROR,
371  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
372  errmsg("COPY BINARY is not supported to stdout or from stdin")));
373  pq_putemptymessage('H');
374  /* grottiness needed for old COPY OUT protocol */
375  pq_startcopyout();
376  cstate->copy_dest = COPY_OLD_FE;
377  }
378 }
379 
380 static void
381 ReceiveCopyBegin(CopyState cstate)
382 {
383  if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
384  {
385  /* new way */
386  StringInfoData buf;
387  int natts = list_length(cstate->attnumlist);
388  int16 format = (cstate->binary ? 1 : 0);
389  int i;
390 
391  pq_beginmessage(&buf, 'G');
392  pq_sendbyte(&buf, format); /* overall format */
393  pq_sendint16(&buf, natts);
394  for (i = 0; i < natts; i++)
395  pq_sendint16(&buf, format); /* per-column formats */
396  pq_endmessage(&buf);
397  cstate->copy_dest = COPY_NEW_FE;
398  cstate->fe_msgbuf = makeStringInfo();
399  }
400  else
401  {
402  /* old way */
403  if (cstate->binary)
404  ereport(ERROR,
405  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
406  errmsg("COPY BINARY is not supported to stdout or from stdin")));
407  pq_putemptymessage('G');
408  /* any error in old protocol will make us lose sync */
409  pq_startmsgread();
410  cstate->copy_dest = COPY_OLD_FE;
411  }
412  /* We *must* flush here to ensure FE knows it can send. */
413  pq_flush();
414 }
415 
416 static void
417 SendCopyEnd(CopyState cstate)
418 {
419  if (cstate->copy_dest == COPY_NEW_FE)
420  {
421  /* Shouldn't have any unsent data */
422  Assert(cstate->fe_msgbuf->len == 0);
423  /* Send Copy Done message */
424  pq_putemptymessage('c');
425  }
426  else
427  {
428  CopySendData(cstate, "\\.", 2);
429  /* Need to flush out the trailer (this also appends a newline) */
430  CopySendEndOfRow(cstate);
431  pq_endcopyout(false);
432  }
433 }
434 
435 /*----------
436  * CopySendData sends output data to the destination (file or frontend)
437  * CopySendString does the same for null-terminated strings
438  * CopySendChar does the same for single characters
439  * CopySendEndOfRow does the appropriate thing at end of each data row
440  * (data is not actually flushed except by CopySendEndOfRow)
441  *
442  * NB: no data conversion is applied by these functions
443  *----------
444  */
445 static void
446 CopySendData(CopyState cstate, const void *databuf, int datasize)
447 {
448  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
449 }
450 
451 static void
452 CopySendString(CopyState cstate, const char *str)
453 {
454  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
455 }
456 
457 static void
458 CopySendChar(CopyState cstate, char c)
459 {
460  appendStringInfoCharMacro(cstate->fe_msgbuf, c);
461 }
462 
463 static void
464 CopySendEndOfRow(CopyState cstate)
465 {
466  StringInfo fe_msgbuf = cstate->fe_msgbuf;
467 
468  switch (cstate->copy_dest)
469  {
470  case COPY_FILE:
471  if (!cstate->binary)
472  {
473  /* Default line termination depends on platform */
474 #ifndef WIN32
475  CopySendChar(cstate, '\n');
476 #else
477  CopySendString(cstate, "\r\n");
478 #endif
479  }
480 
481  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
482  cstate->copy_file) != 1 ||
483  ferror(cstate->copy_file))
484  {
485  if (cstate->is_program)
486  {
487  if (errno == EPIPE)
488  {
489  /*
490  * The pipe will be closed automatically on error at
491  * the end of transaction, but we might get a better
492  * error message from the subprocess' exit code than
493  * just "Broken Pipe"
494  */
495  ClosePipeToProgram(cstate);
496 
497  /*
498  * If ClosePipeToProgram() didn't throw an error, the
499  * program terminated normally, but closed the pipe
500  * first. Restore errno, and throw an error.
501  */
502  errno = EPIPE;
503  }
504  ereport(ERROR,
505  (errcode_for_file_access(),
506  errmsg("could not write to COPY program: %m")));
507  }
508  else
509  ereport(ERROR,
510  (errcode_for_file_access(),
511  errmsg("could not write to COPY file: %m")));
512  }
513  break;
514  case COPY_OLD_FE:
515  /* The FE/BE protocol uses \n as newline for all platforms */
516  if (!cstate->binary)
517  CopySendChar(cstate, '\n');
518 
519  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
520  {
521  /* no hope of recovering connection sync, so FATAL */
522  ereport(FATAL,
523  (errcode(ERRCODE_CONNECTION_FAILURE),
524  errmsg("connection lost during COPY to stdout")));
525  }
526  break;
527  case COPY_NEW_FE:
528  /* The FE/BE protocol uses \n as newline for all platforms */
529  if (!cstate->binary)
530  CopySendChar(cstate, '\n');
531 
532  /* Dump the accumulated row as one CopyData message */
533  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
534  break;
535  case COPY_CALLBACK:
536  Assert(false); /* Not yet supported. */
537  break;
538  }
539 
540  resetStringInfo(fe_msgbuf);
541 }
542 
543 /*
544  * CopyGetData reads data from the source (file or frontend)
545  *
546  * We attempt to read at least minread, and at most maxread, bytes from
547  * the source. The actual number of bytes read is returned; if this is
548  * less than minread, EOF was detected.
549  *
550  * Note: when copying from the frontend, we expect a proper EOF mark per
551  * protocol; if the frontend simply drops the connection, we raise error.
552  * It seems unwise to allow the COPY IN to complete normally in that case.
553  *
554  * NB: no data conversion is applied here.
555  */
556 static int
557 CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
558 {
559  int bytesread = 0;
560 
561  switch (cstate->copy_dest)
562  {
563  case COPY_FILE:
564  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
565  if (ferror(cstate->copy_file))
566  ereport(ERROR,
567  (errcode_for_file_access(),
568  errmsg("could not read from COPY file: %m")));
569  break;
570  case COPY_OLD_FE:
571 
572  /*
573  * We cannot read more than minread bytes (which in practice is 1)
574  * because old protocol doesn't have any clear way of separating
575  * the COPY stream from following data. This is slow, but not any
576  * slower than the code path was originally, and we don't care
577  * much anymore about the performance of old protocol.
578  */
579  if (pq_getbytes((char *) databuf, minread))
580  {
581  /* Only a \. terminator is legal EOF in old protocol */
582  ereport(ERROR,
583  (errcode(ERRCODE_CONNECTION_FAILURE),
584  errmsg("unexpected EOF on client connection with an open transaction")));
585  }
586  bytesread = minread;
587  break;
588  case COPY_NEW_FE:
589  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
590  {
591  int avail;
592 
593  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
594  {
595  /* Try to receive another message */
596  int mtype;
597 
598  readmessage:
599  HOLD_CANCEL_INTERRUPTS();
600  pq_startmsgread();
601  mtype = pq_getbyte();
602  if (mtype == EOF)
603  ereport(ERROR,
604  (errcode(ERRCODE_CONNECTION_FAILURE),
605  errmsg("unexpected EOF on client connection with an open transaction")));
606  if (pq_getmessage(cstate->fe_msgbuf, 0))
607  ereport(ERROR,
608  (errcode(ERRCODE_CONNECTION_FAILURE),
609  errmsg("unexpected EOF on client connection with an open transaction")));
610  RESUME_CANCEL_INTERRUPTS();
611  switch (mtype)
612  {
613  case 'd': /* CopyData */
614  break;
615  case 'c': /* CopyDone */
616  /* COPY IN correctly terminated by frontend */
617  cstate->fe_eof = true;
618  return bytesread;
619  case 'f': /* CopyFail */
620  ereport(ERROR,
621  (errcode(ERRCODE_QUERY_CANCELED),
622  errmsg("COPY from stdin failed: %s",
623  pq_getmsgstring(cstate->fe_msgbuf))));
624  break;
625  case 'H': /* Flush */
626  case 'S': /* Sync */
627 
628  /*
629  * Ignore Flush/Sync for the convenience of client
630  * libraries (such as libpq) that may send those
631  * without noticing that the command they just
632  * sent was COPY.
633  */
634  goto readmessage;
635  default:
636  ereport(ERROR,
637  (errcode(ERRCODE_PROTOCOL_VIOLATION),
638  errmsg("unexpected message type 0x%02X during COPY from stdin",
639  mtype)));
640  break;
641  }
642  }
643  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
644  if (avail > maxread)
645  avail = maxread;
646  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
647  databuf = (void *) ((char *) databuf + avail);
648  maxread -= avail;
649  bytesread += avail;
650  }
651  break;
652  case COPY_CALLBACK:
653  bytesread = cstate->data_source_cb(databuf, minread, maxread);
654  break;
655  }
656 
657  return bytesread;
658 }
659 
660 
661 /*
662  * These functions do apply some data conversion
663  */
664 
665 /*
666  * CopySendInt32 sends an int32 in network byte order
667  */
668 static void
669 CopySendInt32(CopyState cstate, int32 val)
670 {
671  uint32 buf;
672 
673  buf = pg_hton32((uint32) val);
674  CopySendData(cstate, &buf, sizeof(buf));
675 }
676 
677 /*
678  * CopyGetInt32 reads an int32 that appears in network byte order
679  *
680  * Returns true if OK, false if EOF
681  */
682 static bool
683 CopyGetInt32(CopyState cstate, int32 *val)
684 {
685  uint32 buf;
686 
687  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
688  {
689  *val = 0; /* suppress compiler warning */
690  return false;
691  }
692  *val = (int32) pg_ntoh32(buf);
693  return true;
694 }
695 
696 /*
697  * CopySendInt16 sends an int16 in network byte order
698  */
699 static void
700 CopySendInt16(CopyState cstate, int16 val)
701 {
702  uint16 buf;
703 
704  buf = pg_hton16((uint16) val);
705  CopySendData(cstate, &buf, sizeof(buf));
706 }
707 
708 /*
709  * CopyGetInt16 reads an int16 that appears in network byte order
710  */
711 static bool
712 CopyGetInt16(CopyState cstate, int16 *val)
713 {
714  uint16 buf;
715 
716  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
717  {
718  *val = 0; /* suppress compiler warning */
719  return false;
720  }
721  *val = (int16) pg_ntoh16(buf);
722  return true;
723 }
724 
725 
726 /*
727  * CopyLoadRawBuf loads some more data into raw_buf
728  *
729  * Returns true if able to obtain at least one more byte, else false.
730  *
731  * If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
732  * down to the start of the buffer and then we load more data after that.
733  * This case is used only when a frontend multibyte character crosses a
734  * bufferload boundary.
735  */
736 static bool
737 CopyLoadRawBuf(CopyState cstate)
738 {
739  int nbytes;
740  int inbytes;
741 
742  if (cstate->raw_buf_index < cstate->raw_buf_len)
743  {
744  /* Copy down the unprocessed data */
745  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
746  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
747  nbytes);
748  }
749  else
750  nbytes = 0; /* no data need be saved */
751 
752  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
753  1, RAW_BUF_SIZE - nbytes);
754  nbytes += inbytes;
755  cstate->raw_buf[nbytes] = '\0';
756  cstate->raw_buf_index = 0;
757  cstate->raw_buf_len = nbytes;
758  return (inbytes > 0);
759 }
760 
761 
762 /*
763  * DoCopy executes the SQL COPY statement
764  *
765  * Either unload or reload contents of table <relation>, depending on <from>.
766  * (<from> = true means we are inserting into the table.) In the "TO" case
767  * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
768  * or DELETE query.
769  *
770  * If <pipe> is false, transfer is between the table and the file named
771  * <filename>. Otherwise, transfer is between the table and our regular
772  * input/output stream. The latter could be either stdin/stdout or a
773  * socket, depending on whether we're running under Postmaster control.
774  *
775  * Do not allow a Postgres user without the 'pg_access_server_files' role to
776  * read from or write to a file.
777  *
778  * Do not allow the copy if user doesn't have proper permission to access
779  * the table or the specifically requested columns.
780  */
781 void
782 DoCopy(ParseState *pstate, const CopyStmt *stmt,
783  int stmt_location, int stmt_len,
784  uint64 *processed)
785 {
786  CopyState cstate;
787  bool is_from = stmt->is_from;
788  bool pipe = (stmt->filename == NULL);
789  Relation rel;
790  Oid relid;
791  RawStmt *query = NULL;
792 
793  /*
794  * Disallow COPY to/from file or program except to users with the
795  * appropriate role.
796  */
797  if (!pipe)
798  {
799  if (stmt->is_program)
800  {
801  if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_EXECUTE_SERVER_PROGRAM))
802  ereport(ERROR,
803  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
804  errmsg("must be superuser or a member of the pg_execute_server_program role to COPY to or from an external program"),
805  errhint("Anyone can COPY to stdout or from stdin. "
806  "psql's \\copy command also works for anyone.")));
807  }
808  else
809  {
810  if (is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_SERVER_FILES))
811  ereport(ERROR,
812  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
813  errmsg("must be superuser or a member of the pg_read_server_files role to COPY from a file"),
814  errhint("Anyone can COPY to stdout or from stdin. "
815  "psql's \\copy command also works for anyone.")));
816 
817  if (!is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_WRITE_SERVER_FILES))
818  ereport(ERROR,
819  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
820  errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
821  errhint("Anyone can COPY to stdout or from stdin. "
822  "psql's \\copy command also works for anyone.")));
823  }
824  }
825 
826  if (stmt->relation)
827  {
828  TupleDesc tupDesc;
829  List *attnums;
830  ListCell *cur;
831  RangeTblEntry *rte;
832 
833  Assert(!stmt->query);
834 
835  /* Open and lock the relation, using the appropriate lock type. */
836  rel = heap_openrv(stmt->relation,
837  (is_from ? RowExclusiveLock : AccessShareLock));
838 
839  relid = RelationGetRelid(rel);
840 
841  rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
842  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
843 
844  tupDesc = RelationGetDescr(rel);
845  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
846  foreach(cur, attnums)
847  {
848  int attno = lfirst_int(cur) -
849  FirstLowInvalidHeapAttributeNumber;
850 
851  if (is_from)
852  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
853  else
854  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
855  }
856  ExecCheckRTPerms(pstate->p_rtable, true);
857 
858  /*
859  * Permission check for row security policies.
860  *
861  * check_enable_rls will ereport(ERROR) if the user has requested
862  * something invalid and will otherwise indicate if we should enable
863  * RLS (returns RLS_ENABLED) or not for this COPY statement.
864  *
865  * If the relation has a row security policy and we are to apply it
866  * then perform a "query" copy and allow the normal query processing
867  * to handle the policies.
868  *
869  * If RLS is not enabled for this, then just fall through to the
870  * normal non-filtering relation handling.
871  */
872  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
873  {
874  SelectStmt *select;
875  ColumnRef *cr;
876  ResTarget *target;
877  RangeVar *from;
878  List *targetList = NIL;
879 
880  if (is_from)
881  ereport(ERROR,
882  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
883  errmsg("COPY FROM not supported with row-level security"),
884  errhint("Use INSERT statements instead.")));
885 
886  /*
887  * Build target list
888  *
889  * If no columns are specified in the attribute list of the COPY
890  * command, then the target list is 'all' columns. Therefore, '*'
891  * should be used as the target list for the resulting SELECT
892  * statement.
893  *
894  * In the case that columns are specified in the attribute list,
895  * create a ColumnRef and ResTarget for each column and add them
896  * to the target list for the resulting SELECT statement.
897  */
898  if (!stmt->attlist)
899  {
900  cr = makeNode(ColumnRef);
901  cr->fields = list_make1(makeNode(A_Star));
902  cr->location = -1;
903 
904  target = makeNode(ResTarget);
905  target->name = NULL;
906  target->indirection = NIL;
907  target->val = (Node *) cr;
908  target->location = -1;
909 
910  targetList = list_make1(target);
911  }
912  else
913  {
914  ListCell *lc;
915 
916  foreach(lc, stmt->attlist)
917  {
918  /*
919  * Build the ColumnRef for each column. The ColumnRef
920  * 'fields' property is a String 'Value' node (see
921  * nodes/value.h) that corresponds to the column name
922  * respectively.
923  */
924  cr = makeNode(ColumnRef);
925  cr->fields = list_make1(lfirst(lc));
926  cr->location = -1;
927 
928  /* Build the ResTarget and add the ColumnRef to it. */
929  target = makeNode(ResTarget);
930  target->name = NULL;
931  target->indirection = NIL;
932  target->val = (Node *) cr;
933  target->location = -1;
934 
935  /* Add each column to the SELECT statement's target list */
936  targetList = lappend(targetList, target);
937  }
938  }
939 
940  /*
941  * Build RangeVar for from clause, fully qualified based on the
942  * relation which we have opened and locked.
943  */
944  from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
945  pstrdup(RelationGetRelationName(rel)),
946  -1);
947 
948  /* Build query */
949  select = makeNode(SelectStmt);
950  select->targetList = targetList;
951  select->fromClause = list_make1(from);
952 
953  query = makeNode(RawStmt);
954  query->stmt = (Node *) select;
955  query->stmt_location = stmt_location;
956  query->stmt_len = stmt_len;
957 
958  /*
959  * Close the relation for now, but keep the lock on it to prevent
960  * changes between now and when we start the query-based COPY.
961  *
962  * We'll reopen it later as part of the query-based COPY.
963  */
964  heap_close(rel, NoLock);
965  rel = NULL;
966  }
967  }
968  else
969  {
970  Assert(stmt->query);
971 
972  query = makeNode(RawStmt);
973  query->stmt = stmt->query;
974  query->stmt_location = stmt_location;
975  query->stmt_len = stmt_len;
976 
977  relid = InvalidOid;
978  rel = NULL;
979  }
980 
981  if (is_from)
982  {
983  Assert(rel);
984 
985  /* check read-only transaction and parallel mode */
986  if (XactReadOnly && !rel->rd_islocaltemp)
987  PreventCommandIfReadOnly("COPY FROM");
988  PreventCommandIfParallelMode("COPY FROM");
989 
990  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
991  NULL, stmt->attlist, stmt->options);
992  *processed = CopyFrom(cstate); /* copy from file to database */
993  EndCopyFrom(cstate);
994  }
995  else
996  {
997  cstate = BeginCopyTo(pstate, rel, query, relid,
998  stmt->filename, stmt->is_program,
999  stmt->attlist, stmt->options);
1000  *processed = DoCopyTo(cstate); /* copy from database to file */
1001  EndCopyTo(cstate);
1002  }
1003 
1004  /*
1005  * Close the relation. If reading, we can release the AccessShareLock we
1006  * got; if writing, we should hold the lock until end of transaction to
1007  * ensure that updates will be committed before lock is released.
1008  */
1009  if (rel != NULL)
1010  heap_close(rel, (is_from ? NoLock : AccessShareLock));
1011 }
1012 
1013 /*
1014  * Process the statement option list for COPY.
1015  *
1016  * Scan the options list (a list of DefElem) and transpose the information
1017  * into cstate, applying appropriate error checking.
1018  *
1019  * cstate is assumed to be filled with zeroes initially.
1020  *
1021  * This is exported so that external users of the COPY API can sanity-check
1022  * a list of options. In that usage, cstate should be passed as NULL
1023  * (since external users don't know sizeof(CopyStateData)) and the collected
1024  * data is just leaked until CurrentMemoryContext is reset.
1025  *
1026  * Note that additional checking, such as whether column names listed in FORCE
1027  * QUOTE actually exist, has to be applied later. This just checks for
1028  * self-consistency of the options list.
1029  */
1030 void
1031 ProcessCopyOptions(ParseState *pstate,
1032  CopyState cstate,
1033  bool is_from,
1034  List *options)
1035 {
1036  bool format_specified = false;
1037  ListCell *option;
1038 
1039  /* Support external use for option sanity checking */
1040  if (cstate == NULL)
1041  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1042 
1043  cstate->file_encoding = -1;
1044 
1045  /* Extract options from the statement node tree */
1046  foreach(option, options)
1047  {
1048  DefElem *defel = lfirst_node(DefElem, option);
1049 
1050  if (strcmp(defel->defname, "format") == 0)
1051  {
1052  char *fmt = defGetString(defel);
1053 
1054  if (format_specified)
1055  ereport(ERROR,
1056  (errcode(ERRCODE_SYNTAX_ERROR),
1057  errmsg("conflicting or redundant options"),
1058  parser_errposition(pstate, defel->location)));
1059  format_specified = true;
1060  if (strcmp(fmt, "text") == 0)
1061  /* default format */ ;
1062  else if (strcmp(fmt, "csv") == 0)
1063  cstate->csv_mode = true;
1064  else if (strcmp(fmt, "binary") == 0)
1065  cstate->binary = true;
1066  else
1067  ereport(ERROR,
1068  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1069  errmsg("COPY format \"%s\" not recognized", fmt),
1070  parser_errposition(pstate, defel->location)));
1071  }
1072  else if (strcmp(defel->defname, "oids") == 0)
1073  {
1074  if (cstate->oids)
1075  ereport(ERROR,
1076  (errcode(ERRCODE_SYNTAX_ERROR),
1077  errmsg("conflicting or redundant options"),
1078  parser_errposition(pstate, defel->location)));
1079  cstate->oids = defGetBoolean(defel);
1080  }
1081  else if (strcmp(defel->defname, "freeze") == 0)
1082  {
1083  if (cstate->freeze)
1084  ereport(ERROR,
1085  (errcode(ERRCODE_SYNTAX_ERROR),
1086  errmsg("conflicting or redundant options"),
1087  parser_errposition(pstate, defel->location)));
1088  cstate->freeze = defGetBoolean(defel);
1089  }
1090  else if (strcmp(defel->defname, "delimiter") == 0)
1091  {
1092  if (cstate->delim)
1093  ereport(ERROR,
1094  (errcode(ERRCODE_SYNTAX_ERROR),
1095  errmsg("conflicting or redundant options"),
1096  parser_errposition(pstate, defel->location)));
1097  cstate->delim = defGetString(defel);
1098  }
1099  else if (strcmp(defel->defname, "null") == 0)
1100  {
1101  if (cstate->null_print)
1102  ereport(ERROR,
1103  (errcode(ERRCODE_SYNTAX_ERROR),
1104  errmsg("conflicting or redundant options"),
1105  parser_errposition(pstate, defel->location)));
1106  cstate->null_print = defGetString(defel);
1107  }
1108  else if (strcmp(defel->defname, "header") == 0)
1109  {
1110  if (cstate->header_line)
1111  ereport(ERROR,
1112  (errcode(ERRCODE_SYNTAX_ERROR),
1113  errmsg("conflicting or redundant options"),
1114  parser_errposition(pstate, defel->location)));
1115  cstate->header_line = defGetBoolean(defel);
1116  }
1117  else if (strcmp(defel->defname, "quote") == 0)
1118  {
1119  if (cstate->quote)
1120  ereport(ERROR,
1121  (errcode(ERRCODE_SYNTAX_ERROR),
1122  errmsg("conflicting or redundant options"),
1123  parser_errposition(pstate, defel->location)));
1124  cstate->quote = defGetString(defel);
1125  }
1126  else if (strcmp(defel->defname, "escape") == 0)
1127  {
1128  if (cstate->escape)
1129  ereport(ERROR,
1130  (errcode(ERRCODE_SYNTAX_ERROR),
1131  errmsg("conflicting or redundant options"),
1132  parser_errposition(pstate, defel->location)));
1133  cstate->escape = defGetString(defel);
1134  }
1135  else if (strcmp(defel->defname, "force_quote") == 0)
1136  {
1137  if (cstate->force_quote || cstate->force_quote_all)
1138  ereport(ERROR,
1139  (errcode(ERRCODE_SYNTAX_ERROR),
1140  errmsg("conflicting or redundant options"),
1141  parser_errposition(pstate, defel->location)));
1142  if (defel->arg && IsA(defel->arg, A_Star))
1143  cstate->force_quote_all = true;
1144  else if (defel->arg && IsA(defel->arg, List))
1145  cstate->force_quote = castNode(List, defel->arg);
1146  else
1147  ereport(ERROR,
1148  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1149  errmsg("argument to option \"%s\" must be a list of column names",
1150  defel->defname),
1151  parser_errposition(pstate, defel->location)));
1152  }
1153  else if (strcmp(defel->defname, "force_not_null") == 0)
1154  {
1155  if (cstate->force_notnull)
1156  ereport(ERROR,
1157  (errcode(ERRCODE_SYNTAX_ERROR),
1158  errmsg("conflicting or redundant options"),
1159  parser_errposition(pstate, defel->location)));
1160  if (defel->arg && IsA(defel->arg, List))
1161  cstate->force_notnull = castNode(List, defel->arg);
1162  else
1163  ereport(ERROR,
1164  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1165  errmsg("argument to option \"%s\" must be a list of column names",
1166  defel->defname),
1167  parser_errposition(pstate, defel->location)));
1168  }
1169  else if (strcmp(defel->defname, "force_null") == 0)
1170  {
1171  if (cstate->force_null)
1172  ereport(ERROR,
1173  (errcode(ERRCODE_SYNTAX_ERROR),
1174  errmsg("conflicting or redundant options")));
1175  if (defel->arg && IsA(defel->arg, List))
1176  cstate->force_null = castNode(List, defel->arg);
1177  else
1178  ereport(ERROR,
1179  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1180  errmsg("argument to option \"%s\" must be a list of column names",
1181  defel->defname),
1182  parser_errposition(pstate, defel->location)));
1183  }
1184  else if (strcmp(defel->defname, "convert_selectively") == 0)
1185  {
1186  /*
1187  * Undocumented, not-accessible-from-SQL option: convert only the
1188  * named columns to binary form, storing the rest as NULLs. It's
1189  * allowed for the column list to be NIL.
1190  */
1191  if (cstate->convert_selectively)
1192  ereport(ERROR,
1193  (errcode(ERRCODE_SYNTAX_ERROR),
1194  errmsg("conflicting or redundant options"),
1195  parser_errposition(pstate, defel->location)));
1196  cstate->convert_selectively = true;
1197  if (defel->arg == NULL || IsA(defel->arg, List))
1198  cstate->convert_select = castNode(List, defel->arg);
1199  else
1200  ereport(ERROR,
1201  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1202  errmsg("argument to option \"%s\" must be a list of column names",
1203  defel->defname),
1204  parser_errposition(pstate, defel->location)));
1205  }
1206  else if (strcmp(defel->defname, "encoding") == 0)
1207  {
1208  if (cstate->file_encoding >= 0)
1209  ereport(ERROR,
1210  (errcode(ERRCODE_SYNTAX_ERROR),
1211  errmsg("conflicting or redundant options"),
1212  parser_errposition(pstate, defel->location)));
1213  cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
1214  if (cstate->file_encoding < 0)
1215  ereport(ERROR,
1216  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1217  errmsg("argument to option \"%s\" must be a valid encoding name",
1218  defel->defname),
1219  parser_errposition(pstate, defel->location)));
1220  }
1221  else
1222  ereport(ERROR,
1223  (errcode(ERRCODE_SYNTAX_ERROR),
1224  errmsg("option \"%s\" not recognized",
1225  defel->defname),
1226  parser_errposition(pstate, defel->location)));
1227  }
1228 
1229  /*
1230  * Check for incompatible options (must do these two before inserting
1231  * defaults)
1232  */
1233  if (cstate->binary && cstate->delim)
1234  ereport(ERROR,
1235  (errcode(ERRCODE_SYNTAX_ERROR),
1236  errmsg("cannot specify DELIMITER in BINARY mode")));
1237 
1238  if (cstate->binary && cstate->null_print)
1239  ereport(ERROR,
1240  (errcode(ERRCODE_SYNTAX_ERROR),
1241  errmsg("cannot specify NULL in BINARY mode")));
1242 
1243  /* Set defaults for omitted options */
1244  if (!cstate->delim)
1245  cstate->delim = cstate->csv_mode ? "," : "\t";
1246 
1247  if (!cstate->null_print)
1248  cstate->null_print = cstate->csv_mode ? "" : "\\N";
1249  cstate->null_print_len = strlen(cstate->null_print);
1250 
1251  if (cstate->csv_mode)
1252  {
1253  if (!cstate->quote)
1254  cstate->quote = "\"";
1255  if (!cstate->escape)
1256  cstate->escape = cstate->quote;
1257  }
1258 
1259  /* Only single-byte delimiter strings are supported. */
1260  if (strlen(cstate->delim) != 1)
1261  ereport(ERROR,
1262  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1263  errmsg("COPY delimiter must be a single one-byte character")));
1264 
1265  /* Disallow end-of-line characters */
1266  if (strchr(cstate->delim, '\r') != NULL ||
1267  strchr(cstate->delim, '\n') != NULL)
1268  ereport(ERROR,
1269  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1270  errmsg("COPY delimiter cannot be newline or carriage return")));
1271 
1272  if (strchr(cstate->null_print, '\r') != NULL ||
1273  strchr(cstate->null_print, '\n') != NULL)
1274  ereport(ERROR,
1275  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1276  errmsg("COPY null representation cannot use newline or carriage return")));
1277 
1278  /*
1279  * Disallow unsafe delimiter characters in non-CSV mode. We can't allow
1280  * backslash because it would be ambiguous. We can't allow the other
1281  * cases because data characters matching the delimiter must be
1282  * backslashed, and certain backslash combinations are interpreted
1283  * non-literally by COPY IN. Disallowing all lower case ASCII letters is
1284  * more than strictly necessary, but seems best for consistency and
1285  * future-proofing. Likewise we disallow all digits though only octal
1286  * digits are actually dangerous.
1287  */
1288  if (!cstate->csv_mode &&
1289  strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
1290  cstate->delim[0]) != NULL)
1291  ereport(ERROR,
1292  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1293  errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
1294 
1295  /* Check header */
1296  if (!cstate->csv_mode && cstate->header_line)
1297  ereport(ERROR,
1298  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1299  errmsg("COPY HEADER available only in CSV mode")));
1300 
1301  /* Check quote */
1302  if (!cstate->csv_mode && cstate->quote != NULL)
1303  ereport(ERROR,
1304  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1305  errmsg("COPY quote available only in CSV mode")));
1306 
1307  if (cstate->csv_mode && strlen(cstate->quote) != 1)
1308  ereport(ERROR,
1309  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1310  errmsg("COPY quote must be a single one-byte character")));
1311 
1312  if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
1313  ereport(ERROR,
1314  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1315  errmsg("COPY delimiter and quote must be different")));
1316 
1317  /* Check escape */
1318  if (!cstate->csv_mode && cstate->escape != NULL)
1319  ereport(ERROR,
1320  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1321  errmsg("COPY escape available only in CSV mode")));
1322 
1323  if (cstate->csv_mode && strlen(cstate->escape) != 1)
1324  ereport(ERROR,
1325  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1326  errmsg("COPY escape must be a single one-byte character")));
1327 
1328  /* Check force_quote */
1329  if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
1330  ereport(ERROR,
1331  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1332  errmsg("COPY force quote available only in CSV mode")));
1333  if ((cstate->force_quote || cstate->force_quote_all) && is_from)
1334  ereport(ERROR,
1335  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1336  errmsg("COPY force quote only available using COPY TO")));
1337 
1338  /* Check force_notnull */
1339  if (!cstate->csv_mode && cstate->force_notnull != NIL)
1340  ereport(ERROR,
1341  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1342  errmsg("COPY force not null available only in CSV mode")));
1343  if (cstate->force_notnull != NIL && !is_from)
1344  ereport(ERROR,
1345  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1346  errmsg("COPY force not null only available using COPY FROM")));
1347 
1348  /* Check force_null */
1349  if (!cstate->csv_mode && cstate->force_null != NIL)
1350  ereport(ERROR,
1351  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1352  errmsg("COPY force null available only in CSV mode")));
1353 
1354  if (cstate->force_null != NIL && !is_from)
1355  ereport(ERROR,
1356  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1357  errmsg("COPY force null only available using COPY FROM")));
1358 
1359  /* Don't allow the delimiter to appear in the null string. */
1360  if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
1361  ereport(ERROR,
1362  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1363  errmsg("COPY delimiter must not appear in the NULL specification")));
1364 
1365  /* Don't allow the CSV quote char to appear in the null string. */
1366  if (cstate->csv_mode &&
1367  strchr(cstate->null_print, cstate->quote[0]) != NULL)
1368  ereport(ERROR,
1369  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1370  errmsg("CSV quote character must not appear in the NULL specification")));
1371 }
1372 
1373 /*
1374  * Common setup routines used by BeginCopyFrom and BeginCopyTo.
1375  *
1376  * Iff <binary>, unload or reload in the binary format, as opposed to the
1377  * more wasteful but more robust and portable text format.
1378  *
1379  * Iff <oids>, unload or reload the format that includes OID information.
1380  * On input, we accept OIDs whether or not the table has an OID column,
1381  * but silently drop them if it does not. On output, we report an error
1382  * if the user asks for OIDs in a table that has none (not providing an
1383  * OID column might seem friendlier, but could seriously confuse programs).
1384  *
1385  * If in the text format, delimit columns with delimiter <delim> and print
1386  * NULL values as <null_print>.
1387  */
1388 static CopyState
1389 BeginCopy(ParseState *pstate,
1390  bool is_from,
1391  Relation rel,
1392  RawStmt *raw_query,
1393  Oid queryRelId,
1394  List *attnamelist,
1395  List *options)
1396 {
1397  CopyState cstate;
1398  TupleDesc tupDesc;
1399  int num_phys_attrs;
1400  MemoryContext oldcontext;
1401 
1402  /* Allocate workspace and zero all fields */
1403  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1404 
1405  /*
1406  * We allocate everything used by a cstate in a new memory context. This
1407  * avoids memory leaks during repeated use of COPY in a query.
1408  */
1409  cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
1410  "COPY",
1411  ALLOCSET_DEFAULT_SIZES);
1412 
1413  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1414 
1415  /* Extract options from the statement node tree */
1416  ProcessCopyOptions(pstate, cstate, is_from, options);
1417 
1418  /* Process the source/target relation or query */
1419  if (rel)
1420  {
1421  Assert(!raw_query);
1422 
1423  cstate->rel = rel;
1424 
1425  tupDesc = RelationGetDescr(cstate->rel);
1426 
1427  /* Don't allow COPY w/ OIDs to or from a table without them */
1428  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1429  ereport(ERROR,
1430  (errcode(ERRCODE_UNDEFINED_COLUMN),
1431  errmsg("table \"%s\" does not have OIDs",
1432  RelationGetRelationName(cstate->rel))));
1433  }
1434  else
1435  {
1436  List *rewritten;
1437  Query *query;
1438  PlannedStmt *plan;
1439  DestReceiver *dest;
1440 
1441  Assert(!is_from);
1442  cstate->rel = NULL;
1443 
1444  /* Don't allow COPY w/ OIDs from a query */
1445  if (cstate->oids)
1446  ereport(ERROR,
1447  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1448  errmsg("COPY (query) WITH OIDS is not supported")));
1449 
1450  /*
1451  * Run parse analysis and rewrite. Note this also acquires sufficient
1452  * locks on the source table(s).
1453  *
1454  * Because the parser and planner tend to scribble on their input, we
1455  * make a preliminary copy of the source querytree. This prevents
1456  * problems in the case that the COPY is in a portal or plpgsql
1457  * function and is executed repeatedly. (See also the same hack in
1458  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1459  */
1460  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1461  pstate->p_sourcetext, NULL, 0,
1462  NULL);
1463 
1464  /* check that we got back something we can work with */
1465  if (rewritten == NIL)
1466  {
1467  ereport(ERROR,
1468  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1469  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1470  }
1471  else if (list_length(rewritten) > 1)
1472  {
1473  ListCell *lc;
1474 
1475  /* examine queries to determine which error message to issue */
1476  foreach(lc, rewritten)
1477  {
1478  Query *q = lfirst_node(Query, lc);
1479 
1480  if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
1481  ereport(ERROR,
1482  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1483  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1484  if (q->querySource == QSRC_NON_INSTEAD_RULE)
1485  ereport(ERROR,
1486  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1487  errmsg("DO ALSO rules are not supported for the COPY")));
1488  }
1489 
1490  ereport(ERROR,
1491  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1492  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1493  }
1494 
1495  query = linitial_node(Query, rewritten);
1496 
1497  /* The grammar allows SELECT INTO, but we don't support that */
1498  if (query->utilityStmt != NULL &&
1499  IsA(query->utilityStmt, CreateTableAsStmt))
1500  ereport(ERROR,
1501  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1502  errmsg("COPY (SELECT INTO) is not supported")));
1503 
1504  Assert(query->utilityStmt == NULL);
1505 
1506  /*
1507  * Similarly the grammar doesn't enforce the presence of a RETURNING
1508  * clause, but this is required here.
1509  */
1510  if (query->commandType != CMD_SELECT &&
1511  query->returningList == NIL)
1512  {
1513  Assert(query->commandType == CMD_INSERT ||
1514  query->commandType == CMD_UPDATE ||
1515  query->commandType == CMD_DELETE);
1516 
1517  ereport(ERROR,
1518  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1519  errmsg("COPY query must have a RETURNING clause")));
1520  }
1521 
1522  /* plan the query */
1523  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1524 
1525  /*
1526  * With row level security and a user using "COPY relation TO", we
1527  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1528  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1529  * in any RLS clauses.
1530  *
1531  * When this happens, we are passed in the relid of the originally
1532  * found relation (which we have locked). As the planner will look up
1533  * the relation again, we double-check here to make sure it found the
1534  * same one that we have locked.
1535  */
1536  if (queryRelId != InvalidOid)
1537  {
1538  /*
1539  * Note that with RLS involved there may be multiple relations,
1540  * and while the one we need is almost certainly first, we don't
1541  * make any guarantees of that in the planner, so check the whole
1542  * list and make sure we find the original relation.
1543  */
1544  if (!list_member_oid(plan->relationOids, queryRelId))
1545  ereport(ERROR,
1546  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1547  errmsg("relation referenced by COPY statement has changed")));
1548  }
1549 
1550  /*
1551  * Use a snapshot with an updated command ID to ensure this query sees
1552  * results of any previously executed queries.
1553  */
1554  PushCopiedSnapshot(GetActiveSnapshot());
1555  UpdateActiveSnapshotCommandId();
1556 
1557  /* Create dest receiver for COPY OUT */
1558  dest = CreateDestReceiver(DestCopyOut);
1559  ((DR_copy *) dest)->cstate = cstate;
1560 
1561  /* Create a QueryDesc requesting no output */
1562  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1563  GetActiveSnapshot(),
1564  InvalidSnapshot,
1565  dest, NULL, NULL, 0);
1566 
1567  /*
1568  * Call ExecutorStart to prepare the plan for execution.
1569  *
1570  * ExecutorStart computes a result tupdesc for us
1571  */
1572  ExecutorStart(cstate->queryDesc, 0);
1573 
1574  tupDesc = cstate->queryDesc->tupDesc;
1575  }
1576 
1577  /* Generate or convert list of attributes to process */
1578  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1579 
1580  num_phys_attrs = tupDesc->natts;
1581 
1582  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1583  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1584  if (cstate->force_quote_all)
1585  {
1586  int i;
1587 
1588  for (i = 0; i < num_phys_attrs; i++)
1589  cstate->force_quote_flags[i] = true;
1590  }
1591  else if (cstate->force_quote)
1592  {
1593  List *attnums;
1594  ListCell *cur;
1595 
1596  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1597 
1598  foreach(cur, attnums)
1599  {
1600  int attnum = lfirst_int(cur);
1601  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1602 
1603  if (!list_member_int(cstate->attnumlist, attnum))
1604  ereport(ERROR,
1605  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1606  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1607  NameStr(attr->attname))));
1608  cstate->force_quote_flags[attnum - 1] = true;
1609  }
1610  }
1611 
1612  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1613  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1614  if (cstate->force_notnull)
1615  {
1616  List *attnums;
1617  ListCell *cur;
1618 
1619  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1620 
1621  foreach(cur, attnums)
1622  {
1623  int attnum = lfirst_int(cur);
1624  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1625 
1626  if (!list_member_int(cstate->attnumlist, attnum))
1627  ereport(ERROR,
1628  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1629  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1630  NameStr(attr->attname))));
1631  cstate->force_notnull_flags[attnum - 1] = true;
1632  }
1633  }
1634 
1635  /* Convert FORCE_NULL name list to per-column flags, check validity */
1636  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1637  if (cstate->force_null)
1638  {
1639  List *attnums;
1640  ListCell *cur;
1641 
1642  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1643 
1644  foreach(cur, attnums)
1645  {
1646  int attnum = lfirst_int(cur);
1647  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1648 
1649  if (!list_member_int(cstate->attnumlist, attnum))
1650  ereport(ERROR,
1651  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1652  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1653  NameStr(attr->attname))));
1654  cstate->force_null_flags[attnum - 1] = true;
1655  }
1656  }
1657 
1658  /* Convert convert_selectively name list to per-column flags */
1659  if (cstate->convert_selectively)
1660  {
1661  List *attnums;
1662  ListCell *cur;
1663 
1664  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1665 
1666  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1667 
1668  foreach(cur, attnums)
1669  {
1670  int attnum = lfirst_int(cur);
1671  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1672 
1673  if (!list_member_int(cstate->attnumlist, attnum))
1674  ereport(ERROR,
1675  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1676  errmsg_internal("selected column \"%s\" not referenced by COPY",
1677  NameStr(attr->attname))));
1678  cstate->convert_select_flags[attnum - 1] = true;
1679  }
1680  }
1681 
1682  /* Use client encoding when ENCODING option is not specified. */
1683  if (cstate->file_encoding < 0)
1684  cstate->file_encoding = pg_get_client_encoding();
1685 
1686  /*
1687  * Set up encoding conversion info. Even if the file and server encodings
1688  * are the same, we must apply pg_any_to_server() to validate data in
1689  * multibyte encodings.
1690  */
1691  cstate->need_transcoding =
1692  (cstate->file_encoding != GetDatabaseEncoding() ||
1693  pg_database_encoding_max_length() > 1);
1694  /* See Multibyte encoding comment above */
1695  cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
1696 
1697  cstate->copy_dest = COPY_FILE; /* default */
1698 
1699  MemoryContextSwitchTo(oldcontext);
1700 
1701  return cstate;
1702 }
1703 
1704 /*
1705  * Closes the pipe to an external program, checking the pclose() return code.
1706  */
1707 static void
1708 ClosePipeToProgram(CopyState cstate)
1709 {
1710  int pclose_rc;
1711 
1712  Assert(cstate->is_program);
1713 
1714  pclose_rc = ClosePipeStream(cstate->copy_file);
1715  if (pclose_rc == -1)
1716  ereport(ERROR,
1717  (errcode_for_file_access(),
1718  errmsg("could not close pipe to external command: %m")));
1719  else if (pclose_rc != 0)
1720  ereport(ERROR,
1721  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1722  errmsg("program \"%s\" failed",
1723  cstate->filename),
1724  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1725 }
1726 
1727 /*
1728  * Release resources allocated in a cstate for COPY TO/FROM.
1729  */
1730 static void
1731 EndCopy(CopyState cstate)
1732 {
1733  if (cstate->is_program)
1734  {
1735  ClosePipeToProgram(cstate);
1736  }
1737  else
1738  {
1739  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
1740  ereport(ERROR,
1741  (errcode_for_file_access(),
1742  errmsg("could not close file \"%s\": %m",
1743  cstate->filename)));
1744  }
1745 
1746  MemoryContextDelete(cstate->copycontext);
1747  pfree(cstate);
1748 }
1749 
1750 /*
1751  * Setup CopyState to read tuples from a table or a query for COPY TO.
1752  */
1753 static CopyState
1754 BeginCopyTo(ParseState *pstate,
1755  Relation rel,
1756  RawStmt *query,
1757  Oid queryRelId,
1758  const char *filename,
1759  bool is_program,
1760  List *attnamelist,
1761  List *options)
1762 {
1763  CopyState cstate;
1764  bool pipe = (filename == NULL);
1765  MemoryContext oldcontext;
1766 
1767  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1768  {
1769  if (rel->rd_rel->relkind == RELKIND_VIEW)
1770  ereport(ERROR,
1771  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1772  errmsg("cannot copy from view \"%s\"",
1773  RelationGetRelationName(rel)),
1774  errhint("Try the COPY (SELECT ...) TO variant.")));
1775  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1776  ereport(ERROR,
1777  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1778  errmsg("cannot copy from materialized view \"%s\"",
1779  RelationGetRelationName(rel)),
1780  errhint("Try the COPY (SELECT ...) TO variant.")));
1781  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1782  ereport(ERROR,
1783  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1784  errmsg("cannot copy from foreign table \"%s\"",
1785  RelationGetRelationName(rel)),
1786  errhint("Try the COPY (SELECT ...) TO variant.")));
1787  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1788  ereport(ERROR,
1789  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1790  errmsg("cannot copy from sequence \"%s\"",
1791  RelationGetRelationName(rel))));
1792  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1793  ereport(ERROR,
1794  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1795  errmsg("cannot copy from partitioned table \"%s\"",
1796  RelationGetRelationName(rel)),
1797  errhint("Try the COPY (SELECT ...) TO variant.")));
1798  else
1799  ereport(ERROR,
1800  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1801  errmsg("cannot copy from non-table relation \"%s\"",
1802  RelationGetRelationName(rel))));
1803  }
1804 
1805  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1806  options);
1807  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1808 
1809  if (pipe)
1810  {
1811  Assert(!is_program); /* the grammar does not allow this */
1812  if (whereToSendOutput != DestRemote)
1813  cstate->copy_file = stdout;
1814  }
1815  else
1816  {
1817  cstate->filename = pstrdup(filename);
1818  cstate->is_program = is_program;
1819 
1820  if (is_program)
1821  {
1822  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1823  if (cstate->copy_file == NULL)
1824  ereport(ERROR,
1825  (errcode_for_file_access(),
1826  errmsg("could not execute command \"%s\": %m",
1827  cstate->filename)));
1828  }
1829  else
1830  {
1831  mode_t oumask; /* Pre-existing umask value */
1832  struct stat st;
1833 
1834  /*
1835  * Prevent write to relative path ... too easy to shoot oneself in
1836  * the foot by overwriting a database file ...
1837  */
1838  if (!is_absolute_path(filename))
1839  ereport(ERROR,
1840  (errcode(ERRCODE_INVALID_NAME),
1841  errmsg("relative path not allowed for COPY to file")));
1842 
1843  oumask = umask(S_IWGRP | S_IWOTH);
1844  PG_TRY();
1845  {
1846  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1847  }
1848  PG_CATCH();
1849  {
1850  umask(oumask);
1851  PG_RE_THROW();
1852  }
1853  PG_END_TRY();
1854  umask(oumask);
1855  if (cstate->copy_file == NULL)
1856  {
1857  /* copy errno because ereport subfunctions might change it */
1858  int save_errno = errno;
1859 
1860  ereport(ERROR,
1861  (errcode_for_file_access(),
1862  errmsg("could not open file \"%s\" for writing: %m",
1863  cstate->filename),
1864  (save_errno == ENOENT || save_errno == EACCES) ?
1865  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1866  "You may want a client-side facility such as psql's \\copy.") : 0));
1867  }
1868 
1869  if (fstat(fileno(cstate->copy_file), &st))
1870  ereport(ERROR,
1871  (errcode_for_file_access(),
1872  errmsg("could not stat file \"%s\": %m",
1873  cstate->filename)));
1874 
1875  if (S_ISDIR(st.st_mode))
1876  ereport(ERROR,
1877  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1878  errmsg("\"%s\" is a directory", cstate->filename)));
1879  }
1880  }
1881 
1882  MemoryContextSwitchTo(oldcontext);
1883 
1884  return cstate;
1885 }
1886 
1887 /*
1888  * This intermediate routine exists mainly to localize the effects of setjmp
1889  * so we don't need to plaster a lot of variables with "volatile".
1890  */
1891 static uint64
1892 DoCopyTo(CopyState cstate)
1893 {
1894  bool pipe = (cstate->filename == NULL);
1895  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
1896  uint64 processed;
1897 
1898  PG_TRY();
1899  {
1900  if (fe_copy)
1901  SendCopyBegin(cstate);
1902 
1903  processed = CopyTo(cstate);
1904 
1905  if (fe_copy)
1906  SendCopyEnd(cstate);
1907  }
1908  PG_CATCH();
1909  {
1910  /*
1911  * Make sure we turn off old-style COPY OUT mode upon error. It is
1912  * okay to do this in all cases, since it does nothing if the mode is
1913  * not on.
1914  */
1915  pq_endcopyout(true);
1916  PG_RE_THROW();
1917  }
1918  PG_END_TRY();
1919 
1920  return processed;
1921 }
1922 
1923 /*
1924  * Clean up storage and release resources for COPY TO.
1925  */
1926 static void
1927 EndCopyTo(CopyState cstate)
1928 {
1929  if (cstate->queryDesc != NULL)
1930  {
1931  /* Close down the query and free resources. */
1932  ExecutorFinish(cstate->queryDesc);
1933  ExecutorEnd(cstate->queryDesc);
1934  FreeQueryDesc(cstate->queryDesc);
1935  PopActiveSnapshot();
1936  }
1937 
1938  /* Clean up storage */
1939  EndCopy(cstate);
1940 }
1941 
1942 /*
1943  * Copy from relation or query TO file.
1944  */
1945 static uint64
1946 CopyTo(CopyState cstate)
1947 {
1948  TupleDesc tupDesc;
1949  int num_phys_attrs;
1950  ListCell *cur;
1951  uint64 processed;
1952 
1953  if (cstate->rel)
1954  tupDesc = RelationGetDescr(cstate->rel);
1955  else
1956  tupDesc = cstate->queryDesc->tupDesc;
1957  num_phys_attrs = tupDesc->natts;
1958  cstate->null_print_client = cstate->null_print; /* default */
1959 
1960  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1961  cstate->fe_msgbuf = makeStringInfo();
1962 
1963  /* Get info about the columns we need to process. */
1964  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1965  foreach(cur, cstate->attnumlist)
1966  {
1967  int attnum = lfirst_int(cur);
1968  Oid out_func_oid;
1969  bool isvarlena;
1970  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1971 
1972  if (cstate->binary)
1973  getTypeBinaryOutputInfo(attr->atttypid,
1974  &out_func_oid,
1975  &isvarlena);
1976  else
1977  getTypeOutputInfo(attr->atttypid,
1978  &out_func_oid,
1979  &isvarlena);
1980  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
1981  }
1982 
1983  /*
1984  * Create a temporary memory context that we can reset once per row to
1985  * recover palloc'd memory. This avoids any problems with leaks inside
1986  * datatype output routines, and should be faster than retail pfree's
1987  * anyway. (We don't need a whole econtext as CopyFrom does.)
1988  */
1989  cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
1990  "COPY TO",
1991  ALLOCSET_DEFAULT_SIZES);
1992 
1993  if (cstate->binary)
1994  {
1995  /* Generate header for a binary copy */
1996  int32 tmp;
1997 
1998  /* Signature */
1999  CopySendData(cstate, BinarySignature, 11);
2000  /* Flags field */
2001  tmp = 0;
2002  if (cstate->oids)
2003  tmp |= (1 << 16);
2004  CopySendInt32(cstate, tmp);
2005  /* No header extension */
2006  tmp = 0;
2007  CopySendInt32(cstate, tmp);
2008  }
2009  else
2010  {
2011  /*
2012  * For non-binary copy, we need to convert null_print to file
2013  * encoding, because it will be sent directly with CopySendString.
2014  */
2015  if (cstate->need_transcoding)
2016  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2017  cstate->null_print_len,
2018  cstate->file_encoding);
2019 
2020  /* if a header has been requested send the line */
2021  if (cstate->header_line)
2022  {
2023  bool hdr_delim = false;
2024 
2025  foreach(cur, cstate->attnumlist)
2026  {
2027  int attnum = lfirst_int(cur);
2028  char *colname;
2029 
2030  if (hdr_delim)
2031  CopySendChar(cstate, cstate->delim[0]);
2032  hdr_delim = true;
2033 
2034  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
2035 
2036  CopyAttributeOutCSV(cstate, colname, false,
2037  list_length(cstate->attnumlist) == 1);
2038  }
2039 
2040  CopySendEndOfRow(cstate);
2041  }
2042  }
2043 
2044  if (cstate->rel)
2045  {
2046  Datum *values;
2047  bool *nulls;
2048  HeapScanDesc scandesc;
2049  HeapTuple tuple;
2050 
2051  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2052  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2053 
2054  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2055 
2056  processed = 0;
2057  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2058  {
2059  CHECK_FOR_INTERRUPTS();
2060 
2061  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2062  heap_deform_tuple(tuple, tupDesc, values, nulls);
2063 
2064  /* Format and send the data */
2065  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2066  processed++;
2067  }
2068 
2069  heap_endscan(scandesc);
2070 
2071  pfree(values);
2072  pfree(nulls);
2073  }
2074  else
2075  {
2076  /* run the plan --- the dest receiver will send tuples */
2077  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2078  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2079  }
2080 
2081  if (cstate->binary)
2082  {
2083  /* Generate trailer for a binary copy */
2084  CopySendInt16(cstate, -1);
2085  /* Need to flush out the trailer */
2086  CopySendEndOfRow(cstate);
2087  }
2088 
2089  MemoryContextDelete(cstate->rowcontext);
2090 
2091  return processed;
2092 }
2093 
2094 /*
2095  * Emit one row during CopyTo().
2096  */
2097 static void
2098 CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
2099 {
2100  bool need_delim = false;
2101  FmgrInfo *out_functions = cstate->out_functions;
2102  MemoryContext oldcontext;
2103  ListCell *cur;
2104  char *string;
2105 
2106  MemoryContextReset(cstate->rowcontext);
2107  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2108 
2109  if (cstate->binary)
2110  {
2111  /* Binary per-tuple header */
2112  CopySendInt16(cstate, list_length(cstate->attnumlist));
2113  /* Send OID if wanted --- note attnumlist doesn't include it */
2114  if (cstate->oids)
2115  {
2116  /* Hack --- assume Oid is same size as int32 */
2117  CopySendInt32(cstate, sizeof(int32));
2118  CopySendInt32(cstate, tupleOid);
2119  }
2120  }
2121  else
2122  {
2123  /* Text format has no per-tuple header, but send OID if wanted */
2124  /* Assume digits don't need any quoting or encoding conversion */
2125  if (cstate->oids)
2126  {
2127  string = DatumGetCString(DirectFunctionCall1(oidout,
2128  ObjectIdGetDatum(tupleOid)));
2129  CopySendString(cstate, string);
2130  need_delim = true;
2131  }
2132  }
2133 
2134  foreach(cur, cstate->attnumlist)
2135  {
2136  int attnum = lfirst_int(cur);
2137  Datum value = values[attnum - 1];
2138  bool isnull = nulls[attnum - 1];
2139 
2140  if (!cstate->binary)
2141  {
2142  if (need_delim)
2143  CopySendChar(cstate, cstate->delim[0]);
2144  need_delim = true;
2145  }
2146 
2147  if (isnull)
2148  {
2149  if (!cstate->binary)
2150  CopySendString(cstate, cstate->null_print_client);
2151  else
2152  CopySendInt32(cstate, -1);
2153  }
2154  else
2155  {
2156  if (!cstate->binary)
2157  {
2158  string = OutputFunctionCall(&out_functions[attnum - 1],
2159  value);
2160  if (cstate->csv_mode)
2161  CopyAttributeOutCSV(cstate, string,
2162  cstate->force_quote_flags[attnum - 1],
2163  list_length(cstate->attnumlist) == 1);
2164  else
2165  CopyAttributeOutText(cstate, string);
2166  }
2167  else
2168  {
2169  bytea *outputbytes;
2170 
2171  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2172  value);
2173  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2174  CopySendData(cstate, VARDATA(outputbytes),
2175  VARSIZE(outputbytes) - VARHDRSZ);
2176  }
2177  }
2178  }
2179 
2180  CopySendEndOfRow(cstate);
2181 
2182  MemoryContextSwitchTo(oldcontext);
2183 }
2184 
2185 
2186 /*
2187  * error context callback for COPY FROM
2188  *
2189  * The argument for the error context must be CopyState.
2190  */
2191 void
2192 CopyFromErrorCallback(void *arg)
2193 {
2194  CopyState cstate = (CopyState) arg;
2195  char curlineno_str[32];
2196 
2197  snprintf(curlineno_str, sizeof(curlineno_str), UINT64_FORMAT,
2198  cstate->cur_lineno);
2199 
2200  if (cstate->binary)
2201  {
2202  /* can't usefully display the data */
2203  if (cstate->cur_attname)
2204  errcontext("COPY %s, line %s, column %s",
2205  cstate->cur_relname, curlineno_str,
2206  cstate->cur_attname);
2207  else
2208  errcontext("COPY %s, line %s",
2209  cstate->cur_relname, curlineno_str);
2210  }
2211  else
2212  {
2213  if (cstate->cur_attname && cstate->cur_attval)
2214  {
2215  /* error is relevant to a particular column */
2216  char *attval;
2217 
2218  attval = limit_printout_length(cstate->cur_attval);
2219  errcontext("COPY %s, line %s, column %s: \"%s\"",
2220  cstate->cur_relname, curlineno_str,
2221  cstate->cur_attname, attval);
2222  pfree(attval);
2223  }
2224  else if (cstate->cur_attname)
2225  {
2226  /* error is relevant to a particular column, value is NULL */
2227  errcontext("COPY %s, line %s, column %s: null input",
2228  cstate->cur_relname, curlineno_str,
2229  cstate->cur_attname);
2230  }
2231  else
2232  {
2233  /*
2234  * Error is relevant to a particular line.
2235  *
2236  * If line_buf still contains the correct line, and it's already
2237  * transcoded, print it. If it's still in a foreign encoding, it's
2238  * quite likely that the error is precisely a failure to do
2239  * encoding conversion (ie, bad data). We dare not try to convert
2240  * it, and at present there's no way to regurgitate it without
2241  * conversion. So we have to punt and just report the line number.
2242  */
2243  if (cstate->line_buf_valid &&
2244  (cstate->line_buf_converted || !cstate->need_transcoding))
2245  {
2246  char *lineval;
2247 
2248  lineval = limit_printout_length(cstate->line_buf.data);
2249  errcontext("COPY %s, line %s: \"%s\"",
2250  cstate->cur_relname, curlineno_str, lineval);
2251  pfree(lineval);
2252  }
2253  else
2254  {
2255  errcontext("COPY %s, line %s",
2256  cstate->cur_relname, curlineno_str);
2257  }
2258  }
2259  }
2260 }
2261 
2262 /*
2263  * Make sure we don't print an unreasonable amount of COPY data in a message.
2264  *
2265  * It would seem a lot easier to just use the sprintf "precision" limit to
2266  * truncate the string. However, some versions of glibc have a bug/misfeature
2267  * that vsnprintf will always fail (return -1) if it is asked to truncate
2268  * a string that contains invalid byte sequences for the current encoding.
2269  * So, do our own truncation. We return a pstrdup'd copy of the input.
2270  */
2271 static char *
2272 limit_printout_length(const char *str)
2273 {
2274 #define MAX_COPY_DATA_DISPLAY 100
2275 
2276  int slen = strlen(str);
2277  int len;
2278  char *res;
2279 
2280  /* Fast path if definitely okay */
2281  if (slen <= MAX_COPY_DATA_DISPLAY)
2282  return pstrdup(str);
2283 
2284  /* Apply encoding-dependent truncation */
2285  len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
2286 
2287  /*
2288  * Truncate, and add "..." to show we truncated the input.
2289  */
2290  res = (char *) palloc(len + 4);
2291  memcpy(res, str, len);
2292  strcpy(res + len, "...");
2293 
2294  return res;
2295 }
2296 
2297 /*
2298  * Copy FROM file to relation.
2299  */
2300 uint64
2301 CopyFrom(CopyState cstate)
2302 {
2303  HeapTuple tuple;
2304  TupleDesc tupDesc;
2305  Datum *values;
2306  bool *nulls;
2307  ResultRelInfo *resultRelInfo;
2308  ResultRelInfo *saved_resultRelInfo = NULL;
2309  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2310  ModifyTableState *mtstate;
2311  ExprContext *econtext;
2312  TupleTableSlot *myslot;
2313  MemoryContext oldcontext = CurrentMemoryContext;
2314 
2315  ErrorContextCallback errcallback;
2316  CommandId mycid = GetCurrentCommandId(true);
2317  int hi_options = 0; /* start with default heap_insert options */
2318  BulkInsertState bistate;
2319  uint64 processed = 0;
2320  bool useHeapMultiInsert;
2321  int nBufferedTuples = 0;
2322  int prev_leaf_part_index = -1;
2323 
2324 #define MAX_BUFFERED_TUPLES 1000
2325  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2326  Size bufferedTuplesSize = 0;
2327  uint64 firstBufferedLineNo = 0;
2328 
2329  Assert(cstate->rel);
2330 
2331  /*
2332  * The target must be a plain, foreign, or partitioned relation, or have
2333  * an INSTEAD OF INSERT row trigger. (Currently, such triggers are only
2334  * allowed on views, so we only hint about them in the view case.)
2335  */
2336  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2337  cstate->rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE &&
2338  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2339  !(cstate->rel->trigdesc &&
2340  cstate->rel->trigdesc->trig_insert_instead_row))
2341  {
2342  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2343  ereport(ERROR,
2344  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2345  errmsg("cannot copy to view \"%s\"",
2346  RelationGetRelationName(cstate->rel)),
2347  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2348  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2349  ereport(ERROR,
2350  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2351  errmsg("cannot copy to materialized view \"%s\"",
2352  RelationGetRelationName(cstate->rel))));
2353  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2354  ereport(ERROR,
2355  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2356  errmsg("cannot copy to sequence \"%s\"",
2357  RelationGetRelationName(cstate->rel))));
2358  else
2359  ereport(ERROR,
2360  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2361  errmsg("cannot copy to non-table relation \"%s\"",
2362  RelationGetRelationName(cstate->rel))));
2363  }
2364 
2365  tupDesc = RelationGetDescr(cstate->rel);
2366 
2367  /*----------
2368  * Check to see if we can avoid writing WAL
2369  *
2370  * If archive logging/streaming is not enabled *and* either
2371  * - table was created in same transaction as this COPY
2372  * - data is being written to relfilenode created in this transaction
2373  * then we can skip writing WAL. It's safe because if the transaction
2374  * doesn't commit, we'll discard the table (or the new relfilenode file).
2375  * If it does commit, we'll have done the heap_sync at the bottom of this
2376  * routine first.
2377  *
2378  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2379  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2380  * can be cleared before the end of the transaction. The exact case is
2381  * when a relation sets a new relfilenode twice in same transaction, yet
2382  * the second one fails in an aborted subtransaction, e.g.
2383  *
2384  * BEGIN;
2385  * TRUNCATE t;
2386  * SAVEPOINT save;
2387  * TRUNCATE t;
2388  * ROLLBACK TO save;
2389  * COPY ...
2390  *
2391  * Also, if the target file is new-in-transaction, we assume that checking
2392  * FSM for free space is a waste of time, even if we must use WAL because
2393  * of archiving. This could possibly be wrong, but it's unlikely.
2394  *
2395  * The comments for heap_insert and RelationGetBufferForTuple specify that
2396  * skipping WAL logging is only safe if we ensure that our tuples do not
2397  * go into pages containing tuples from any other transactions --- but this
2398  * must be the case if we have a new table or new relfilenode, so we need
2399  * no additional work to enforce that.
2400  *----------
2401  */
2402  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2403  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2404  cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId)
2405  {
2406  hi_options |= HEAP_INSERT_SKIP_FSM;
2407  if (!XLogIsNeeded())
2408  hi_options |= HEAP_INSERT_SKIP_WAL;
2409  }
2410 
2411  /*
2412  * Optimize if new relfilenode was created in this subxact or one of its
2413  * committed children and we won't see those rows later as part of an
2414  * earlier scan or command. The subxact test ensures that if this subxact
2415  * aborts then the frozen rows won't be visible after xact cleanup. Note
2416  * that the stronger test of exactly which subtransaction created it is
2417  * crucial for correctness of this optimization. The test for an earlier
2418  * scan or command tolerates false negatives. FREEZE causes other sessions
2419  * to see rows they would not see under MVCC, and a false negative merely
2420  * spreads that anomaly to the current session.
2421  */
2422  if (cstate->freeze)
2423  {
2424  /*
2425  * Tolerate one registration for the benefit of FirstXactSnapshot.
2426  * Scan-bearing queries generally create at least two registrations,
2427  * though relying on that is fragile, as is ignoring ActiveSnapshot.
2428  * Clear CatalogSnapshot to avoid counting its registration. We'll
2429  * still detect ongoing catalog scans, each of which separately
2430  * registers the snapshot it uses.
2431  */
2432  InvalidateCatalogSnapshot();
2433  if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
2434  ereport(ERROR,
2435  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2436  errmsg("cannot perform FREEZE because of prior transaction activity")));
2437 
2438  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2439  cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
2440  ereport(ERROR,
2441  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2442  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2443 
2444  hi_options |= HEAP_INSERT_FROZEN;
2445  }
2446 
2447  /*
2448  * We need a ResultRelInfo so we can use the regular executor's
2449  * index-entry-making machinery. (There used to be a huge amount of code
2450  * here that basically duplicated execUtils.c ...)
2451  */
2452  resultRelInfo = makeNode(ResultRelInfo);
2453  InitResultRelInfo(resultRelInfo,
2454  cstate->rel,
2455  1, /* dummy rangetable index */
2456  NULL,
2457  0);
2458 
2459  /* Verify the named relation is a valid target for INSERT */
2460  CheckValidResultRel(resultRelInfo, CMD_INSERT);
2461 
2462  ExecOpenIndices(resultRelInfo, false);
2463 
2464  estate->es_result_relations = resultRelInfo;
2465  estate->es_num_result_relations = 1;
2466  estate->es_result_relation_info = resultRelInfo;
2467  estate->es_range_table = cstate->range_table;
2468 
2469  /* Set up a tuple slot too */
2470  myslot = ExecInitExtraTupleSlot(estate, tupDesc);
2471  /* Triggers might need a slot as well */
2472  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate, NULL);
2473 
2474  /*
2475  * Set up a ModifyTableState so we can let FDW(s) init themselves for
2476  * foreign-table result relation(s).
2477  */
2478  mtstate = makeNode(ModifyTableState);
2479  mtstate->ps.plan = NULL;
2480  mtstate->ps.state = estate;
2481  mtstate->operation = CMD_INSERT;
2482  mtstate->resultRelInfo = estate->es_result_relations;
2483 
2484  if (resultRelInfo->ri_FdwRoutine != NULL &&
2485  resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
2486  resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
2487  resultRelInfo);
2488 
2489  /* Prepare to catch AFTER triggers. */
2490  AfterTriggerBeginQuery();
2491 
2492  /*
2493  * If there are any triggers with transition tables on the named relation,
2494  * we need to be prepared to capture transition tuples.
2495  */
2496  cstate->transition_capture =
2497  MakeTransitionCaptureState(cstate->rel->trigdesc,
2498  RelationGetRelid(cstate->rel),
2499  CMD_INSERT);
2500 
2501  /*
2502  * If the named relation is a partitioned table, initialize state for
2503  * CopyFrom tuple routing.
2504  */
2505  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2506  {
2507  PartitionTupleRouting *proute;
2508 
2509  proute = cstate->partition_tuple_routing =
2510  ExecSetupPartitionTupleRouting(NULL, cstate->rel);
2511 
2512  /*
2513  * If we are capturing transition tuples, they may need to be
2514  * converted from partition format back to partitioned table format
2515  * (this is only ever necessary if a BEFORE trigger modifies the
2516  * tuple).
2517  */
2518  if (cstate->transition_capture != NULL)
2519  ExecSetupChildParentMapForLeaf(proute);
2520  }
2521 
2522  /*
2523  * It's more efficient to prepare a bunch of tuples for insertion, and
2524  * insert them in one heap_multi_insert() call, than call heap_insert()
2525  * separately for every tuple. However, we can't do that if there are
2526  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2527  * expressions. Such triggers or expressions might query the table we're
2528  * inserting to, and act differently if the tuples that have already been
2529  * processed and prepared for insertion are not there. We also can't do
2530  * it if the table is foreign or partitioned.
2531  */
2532  if ((resultRelInfo->ri_TrigDesc != NULL &&
2533  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2534  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2535  resultRelInfo->ri_FdwRoutine != NULL ||
2536  cstate->partition_tuple_routing != NULL ||
2537  cstate->volatile_defexprs)
2538  {
2539  useHeapMultiInsert = false;
2540  }
2541  else
2542  {
2543  useHeapMultiInsert = true;
2544  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2545  }
2546 
2547  /*
2548  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2549  * should do this for COPY, since it's not really an "INSERT" statement as
2550  * such. However, executing these triggers maintains consistency with the
2551  * EACH ROW triggers that we already fire on COPY.
2552  */
2553  ExecBSInsertTriggers(estate, resultRelInfo);
2554 
2555  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2556  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2557 
2558  bistate = GetBulkInsertState();
2559  econtext = GetPerTupleExprContext(estate);
2560 
2561  /* Set up callback to identify error line number */
2562  errcallback.callback = CopyFromErrorCallback;
2563  errcallback.arg = (void *) cstate;
2564  errcallback.previous = error_context_stack;
2565  error_context_stack = &errcallback;
2566 
2567  for (;;)
2568  {
2569  TupleTableSlot *slot;
2570  bool skip_tuple;
2571  Oid loaded_oid = InvalidOid;
2572 
2573  CHECK_FOR_INTERRUPTS();
2574 
2575  if (nBufferedTuples == 0)
2576  {
2577  /*
2578  * Reset the per-tuple exprcontext. We can only do this if the
2579  * tuple buffer is empty. (Calling the context the per-tuple
2580  * memory context is a bit of a misnomer now.)
2581  */
2582  ResetPerTupleExprContext(estate);
2583  }
2584 
2585  /* Switch into its memory context */
2586  MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2587 
2588  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2589  break;
2590 
2591  /* And now we can form the input tuple. */
2592  tuple = heap_form_tuple(tupDesc, values, nulls);
2593 
2594  if (loaded_oid != InvalidOid)
2595  HeapTupleSetOid(tuple, loaded_oid);
2596 
2597  /*
2598  * Constraints might reference the tableoid column, so initialize
2599  * t_tableOid before evaluating them.
2600  */
2601  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2602 
2603  /* Triggers and stuff need to be invoked in query context. */
2604  MemoryContextSwitchTo(oldcontext);
2605 
2606  /* Place tuple in tuple slot --- but slot shouldn't free it */
2607  slot = myslot;
2608  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2609 
2610  /* Determine the partition to heap_insert the tuple into */
2611  if (cstate->partition_tuple_routing)
2612  {
2613  int leaf_part_index;
2614  PartitionTupleRouting *proute = cstate->partition_tuple_routing;
2615 
2616  /*
2617  * Away we go ... If we end up not finding a partition after all,
2618  * ExecFindPartition() does not return and errors out instead.
2619  * Otherwise, the returned value is to be used as an index into
2620  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2621  * will get us the ResultRelInfo and TupleConversionMap for the
2622  * partition, respectively.
2623  */
2624  leaf_part_index = ExecFindPartition(resultRelInfo,
2625  proute->partition_dispatch_info,
2626  slot,
2627  estate);
2628  Assert(leaf_part_index >= 0 &&
2629  leaf_part_index < proute->num_partitions);
2630 
2631  /*
2632  * If this tuple is mapped to a partition that is not same as the
2633  * previous one, we'd better make the bulk insert mechanism gets a
2634  * new buffer.
2635  */
2636  if (prev_leaf_part_index != leaf_part_index)
2637  {
2638  ReleaseBulkInsertStatePin(bistate);
2639  prev_leaf_part_index = leaf_part_index;
2640  }
2641 
2642  /*
2643  * Save the old ResultRelInfo and switch to the one corresponding
2644  * to the selected partition.
2645  */
2646  saved_resultRelInfo = resultRelInfo;
2647  resultRelInfo = proute->partitions[leaf_part_index];
2648  if (resultRelInfo == NULL)
2649  {
2650  resultRelInfo = ExecInitPartitionInfo(mtstate,
2651  saved_resultRelInfo,
2652  proute, estate,
2653  leaf_part_index);
2654  Assert(resultRelInfo != NULL);
2655  }
2656 
2657  /*
2658  * For ExecInsertIndexTuples() to work on the partition's indexes
2659  */
2660  estate->es_result_relation_info = resultRelInfo;
2661 
2662  /*
2663  * If we're capturing transition tuples, we might need to convert
2664  * from the partition rowtype to parent rowtype.
2665  */
2666  if (cstate->transition_capture != NULL)
2667  {
2668  if (resultRelInfo->ri_TrigDesc &&
2669  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2670  {
2671  /*
2672  * If there are any BEFORE triggers on the partition,
2673  * we'll have to be ready to convert their result back to
2674  * tuplestore format.
2675  */
2676  cstate->transition_capture->tcs_original_insert_tuple = NULL;
2677  cstate->transition_capture->tcs_map =
2678  TupConvMapForLeaf(proute, saved_resultRelInfo,
2679  leaf_part_index);
2680  }
2681  else
2682  {
2683  /*
2684  * Otherwise, just remember the original unconverted
2685  * tuple, to avoid a needless round trip conversion.
2686  */
2687  cstate->transition_capture->tcs_original_insert_tuple = tuple;
2688  cstate->transition_capture->tcs_map = NULL;
2689  }
2690  }
2691 
2692  /*
2693  * We might need to convert from the parent rowtype to the
2694  * partition rowtype.
2695  */
2696  tuple = ConvertPartitionTupleSlot(proute->parent_child_tupconv_maps[leaf_part_index],
2697  tuple,
2698  proute->partition_tuple_slot,
2699  &slot);
2700 
2701  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2702  }
2703 
2704  skip_tuple = false;
2705 
2706  /* BEFORE ROW INSERT Triggers */
2707  if (resultRelInfo->ri_TrigDesc &&
2708  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2709  {
2710  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2711 
2712  if (slot == NULL) /* "do nothing" */
2713  skip_tuple = true;
2714  else /* trigger might have changed tuple */
2715  tuple = ExecMaterializeSlot(slot);
2716  }
2717 
2718  if (!skip_tuple)
2719  {
2720  if (resultRelInfo->ri_TrigDesc &&
2721  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2722  {
2723  /* Pass the data to the INSTEAD ROW INSERT trigger */
2724  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2725  }
2726  else
2727  {
2728  /*
2729  * If the target is a plain table, check the constraints of
2730  * the tuple.
2731  */
2732  if (resultRelInfo->ri_FdwRoutine == NULL &&
2733  resultRelInfo->ri_RelationDesc->rd_att->constr)
2734  ExecConstraints(resultRelInfo, slot, estate);
2735 
2736  /*
2737  * Also check the tuple against the partition constraint, if
2738  * there is one; except that if we got here via tuple-routing,
2739  * we don't need to if there's no BR trigger defined on the
2740  * partition.
2741  */
2742  if (resultRelInfo->ri_PartitionCheck &&
2743  (saved_resultRelInfo == NULL ||
2744  (resultRelInfo->ri_TrigDesc &&
2745  resultRelInfo->ri_TrigDesc->trig_insert_before_row)))
2746  ExecPartitionCheck(resultRelInfo, slot, estate, true);
2747 
2748  if (useHeapMultiInsert)
2749  {
2750  /* Add this tuple to the tuple buffer */
2751  if (nBufferedTuples == 0)
2752  firstBufferedLineNo = cstate->cur_lineno;
2753  bufferedTuples[nBufferedTuples++] = tuple;
2754  bufferedTuplesSize += tuple->t_len;
2755 
2756  /*
2757  * If the buffer filled up, flush it. Also flush if the
2758  * total size of all the tuples in the buffer becomes
2759  * large, to avoid using large amounts of memory for the
2760  * buffer when the tuples are exceptionally wide.
2761  */
2762  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2763  bufferedTuplesSize > 65535)
2764  {
2765  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2766  resultRelInfo, myslot, bistate,
2767  nBufferedTuples, bufferedTuples,
2768  firstBufferedLineNo);
2769  nBufferedTuples = 0;
2770  bufferedTuplesSize = 0;
2771  }
2772  }
2773  else
2774  {
2775  List *recheckIndexes = NIL;
2776 
2777  /* OK, store the tuple */
2778  if (resultRelInfo->ri_FdwRoutine != NULL)
2779  {
2780  slot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
2781  resultRelInfo,
2782  slot,
2783  NULL);
2784 
2785  if (slot == NULL) /* "do nothing" */
2786  goto next_tuple;
2787 
2788  /* FDW might have changed tuple */
2789  tuple = ExecMaterializeSlot(slot);
2790 
2791  /*
2792  * AFTER ROW Triggers might reference the tableoid
2793  * column, so initialize t_tableOid before evaluating
2794  * them.
2795  */
2796  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2797  }
2798  else
2799  heap_insert(resultRelInfo->ri_RelationDesc, tuple,
2800  mycid, hi_options, bistate);
2801 
2802  /* And create index entries for it */
2803  if (resultRelInfo->ri_NumIndices > 0)
2804  recheckIndexes = ExecInsertIndexTuples(slot,
2805  &(tuple->t_self),
2806  estate,
2807  false,
2808  NULL,
2809  NIL);
2810 
2811  /* AFTER ROW INSERT Triggers */
2812  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2813  recheckIndexes, cstate->transition_capture);
2814 
2815  list_free(recheckIndexes);
2816  }
2817  }
2818 
2819  /*
2820  * We count only tuples not suppressed by a BEFORE INSERT trigger
2821  * or FDW; this is the same definition used by nodeModifyTable.c
2822  * for counting tuples inserted by an INSERT command.
2823  */
2824  processed++;
2825  }
2826 
2827 next_tuple:
2828  /* Restore the saved ResultRelInfo */
2829  if (saved_resultRelInfo)
2830  {
2831  resultRelInfo = saved_resultRelInfo;
2832  estate->es_result_relation_info = resultRelInfo;
2833  }
2834  }
2835 
2836  /* Flush any remaining buffered tuples */
2837  if (nBufferedTuples > 0)
2838  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2839  resultRelInfo, myslot, bistate,
2840  nBufferedTuples, bufferedTuples,
2841  firstBufferedLineNo);
2842 
2843  /* Done, clean up */
2844  error_context_stack = errcallback.previous;
2845 
2846  FreeBulkInsertState(bistate);
2847 
2848  MemoryContextSwitchTo(oldcontext);
2849 
2850  /*
2851  * In the old protocol, tell pqcomm that we can process normal protocol
2852  * messages again.
2853  */
2854  if (cstate->copy_dest == COPY_OLD_FE)
2855  pq_endmsgread();
2856 
2857  /* Execute AFTER STATEMENT insertion triggers */
2858  ExecASInsertTriggers(estate, resultRelInfo, cstate->transition_capture);
2859 
2860  /* Handle queued AFTER triggers */
2861  AfterTriggerEndQuery(estate);
2862 
2863  pfree(values);
2864  pfree(nulls);
2865 
2866  ExecResetTupleTable(estate->es_tupleTable, false);
2867 
2868  /* Allow the FDW to shut down */
2869  if (resultRelInfo->ri_FdwRoutine != NULL &&
2870  resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
2871  resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,
2872  resultRelInfo);
2873 
2874  ExecCloseIndices(resultRelInfo);
2875 
2876  /* Close all the partitioned tables, leaf partitions, and their indices */
2877  if (cstate->partition_tuple_routing)
2878  ExecCleanupTupleRouting(mtstate, cstate->partition_tuple_routing);
2879 
2880  /* Close any trigger target relations */
2881  ExecCleanUpTriggerState(estate);
2882 
2883  FreeExecutorState(estate);
2884 
2885  /*
2886  * If we skipped writing WAL, then we need to sync the heap (but not
2887  * indexes since those use WAL anyway)
2888  */
2889  if (hi_options & HEAP_INSERT_SKIP_WAL)
2890  heap_sync(cstate->rel);
2891 
2892  return processed;
2893 }
2894 
2895 /*
2896  * A subroutine of CopyFrom, to write the current batch of buffered heap
2897  * tuples to the heap. Also updates indexes and runs AFTER ROW INSERT
2898  * triggers.
2899  */
2900 static void
2901 CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid,
2902  int hi_options, ResultRelInfo *resultRelInfo,
2903  TupleTableSlot *myslot, BulkInsertState bistate,
2904  int nBufferedTuples, HeapTuple *bufferedTuples,
2905  uint64 firstBufferedLineNo)
2906 {
2907  MemoryContext oldcontext;
2908  int i;
2909  uint64 save_cur_lineno;
2910 
2911  /*
2912  * Print error context information correctly, if one of the operations
2913  * below fail.
2914  */
2915  cstate->line_buf_valid = false;
2916  save_cur_lineno = cstate->cur_lineno;
2917 
2918  /*
2919  * heap_multi_insert leaks memory, so switch to short-lived memory context
2920  * before calling it.
2921  */
2922  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2923  heap_multi_insert(cstate->rel,
2924  bufferedTuples,
2925  nBufferedTuples,
2926  mycid,
2927  hi_options,
2928  bistate);
2929  MemoryContextSwitchTo(oldcontext);
2930 
2931  /*
2932  * If there are any indexes, update them for all the inserted tuples, and
2933  * run AFTER ROW INSERT triggers.
2934  */
2935  if (resultRelInfo->ri_NumIndices > 0)
2936  {
2937  for (i = 0; i < nBufferedTuples; i++)
2938  {
2939  List *recheckIndexes;
2940 
2941  cstate->cur_lineno = firstBufferedLineNo + i;
2942  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2943  recheckIndexes =
2944  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2945  estate, false, NULL, NIL);
2946  ExecARInsertTriggers(estate, resultRelInfo,
2947  bufferedTuples[i],
2948  recheckIndexes, cstate->transition_capture);
2949  list_free(recheckIndexes);
2950  }
2951  }
2952 
2953  /*
2954  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2955  * anyway.
2956  */
2957  else if (resultRelInfo->ri_TrigDesc != NULL &&
2958  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2959  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2960  {
2961  for (i = 0; i < nBufferedTuples; i++)
2962  {
2963  cstate->cur_lineno = firstBufferedLineNo + i;
2964  ExecARInsertTriggers(estate, resultRelInfo,
2965  bufferedTuples[i],
2966  NIL, cstate->transition_capture);
2967  }
2968  }
2969 
2970  /* reset cur_lineno to where we were */
2971  cstate->cur_lineno = save_cur_lineno;
2972 }
2973 
2974 /*
2975  * Setup to read tuples from a file for COPY FROM.
2976  *
2977  * 'rel': Used as a template for the tuples
2978  * 'filename': Name of server-local file to read
2979  * 'attnamelist': List of char *, columns to include. NIL selects all cols.
2980  * 'options': List of DefElem. See copy_opt_item in gram.y for selections.
2981  *
2982  * Returns a CopyState, to be passed to NextCopyFrom and related functions.
2983  */
2984 CopyState
2985 BeginCopyFrom(ParseState *pstate,
2986  Relation rel,
2987  const char *filename,
2988  bool is_program,
2989  copy_data_source_cb data_source_cb,
2990  List *attnamelist,
2991  List *options)
2992 {
2993  CopyState cstate;
2994  bool pipe = (filename == NULL);
2995  TupleDesc tupDesc;
2996  AttrNumber num_phys_attrs,
2997  num_defaults;
2998  FmgrInfo *in_functions;
2999  Oid *typioparams;
3000  int attnum;
3001  Oid in_func_oid;
3002  int *defmap;
3003  ExprState **defexprs;
3004  MemoryContext oldcontext;
3005  bool volatile_defexprs;
3006 
3007  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
3008  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
3009 
3010  /* Initialize state variables */
3011  cstate->fe_eof = false;
3012  cstate->eol_type = EOL_UNKNOWN;
3013  cstate->cur_relname = RelationGetRelationName(cstate->rel);
3014  cstate->cur_lineno = 0;
3015  cstate->cur_attname = NULL;
3016  cstate->cur_attval = NULL;
3017 
3018  /* Set up variables to avoid per-attribute overhead. */
3019  initStringInfo(&cstate->attribute_buf);
3020  initStringInfo(&cstate->line_buf);
3021  cstate->line_buf_converted = false;
3022  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
3023  cstate->raw_buf_index = cstate->raw_buf_len = 0;
3024 
3025  /* Assign range table, we'll need it in CopyFrom. */
3026  if (pstate)
3027  cstate->range_table = pstate->p_rtable;
3028 
3029  tupDesc = RelationGetDescr(cstate->rel);
3030  num_phys_attrs = tupDesc->natts;
3031  num_defaults = 0;
3032  volatile_defexprs = false;
3033 
3034  /*
3035  * Pick up the required catalog information for each attribute in the
3036  * relation, including the input function, the element type (to pass to
3037  * the input function), and info about defaults and constraints. (Which
3038  * input function we use depends on text/binary format choice.)
3039  */
3040  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
3041  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3042  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3043  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3044 
3045  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3046  {
3047  Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
3048 
3049  /* We don't need info for dropped attributes */
3050  if (att->attisdropped)
3051  continue;
3052 
3053  /* Fetch the input function and typioparam info */
3054  if (cstate->binary)
3055  getTypeBinaryInputInfo(att->atttypid,
3056  &in_func_oid, &typioparams[attnum - 1]);
3057  else
3058  getTypeInputInfo(att->atttypid,
3059  &in_func_oid, &typioparams[attnum - 1]);
3060  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3061 
3062  /* Get default info if needed */
3063  if (!list_member_int(cstate->attnumlist, attnum))
3064  {
3065  /* attribute is NOT to be copied from input */
3066  /* use default value if one exists */
3067  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
3068  attnum);
3069 
3070  if (defexpr != NULL)
3071  {
3072  /* Run the expression through planner */
3073  defexpr = expression_planner(defexpr);
3074 
3075  /* Initialize executable expression in copycontext */
3076  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3077  defmap[num_defaults] = attnum - 1;
3078  num_defaults++;
3079 
3080  /*
3081  * If a default expression looks at the table being loaded,
3082  * then it could give the wrong answer when using
3083  * multi-insert. Since database access can be dynamic this is
3084  * hard to test for exactly, so we use the much wider test of
3085  * whether the default expression is volatile. We allow for
3086  * the special case of when the default expression is the
3087  * nextval() of a sequence which in this specific case is
3088  * known to be safe for use with the multi-insert
3089  * optimization. Hence we use this special case function
3090  * checker rather than the standard check for
3091  * contain_volatile_functions().
3092  */
3093  if (!volatile_defexprs)
3094  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3095  }
3096  }
3097  }
3098 
3099  /* We keep those variables in cstate. */
3100  cstate->in_functions = in_functions;
3101  cstate->typioparams = typioparams;
3102  cstate->defmap = defmap;
3103  cstate->defexprs = defexprs;
3104  cstate->volatile_defexprs = volatile_defexprs;
3105  cstate->num_defaults = num_defaults;
3106  cstate->is_program = is_program;
3107 
3108  if (data_source_cb)
3109  {
3110  cstate->copy_dest = COPY_CALLBACK;
3111  cstate->data_source_cb = data_source_cb;
3112  }
3113  else if (pipe)
3114  {
3115  Assert(!is_program); /* the grammar does not allow this */
3116  if (whereToSendOutput == DestRemote)
3117  ReceiveCopyBegin(cstate);
3118  else
3119  cstate->copy_file = stdin;
3120  }
3121  else
3122  {
3123  cstate->filename = pstrdup(filename);
3124 
3125  if (cstate->is_program)
3126  {
3127  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3128  if (cstate->copy_file == NULL)
3129  ereport(ERROR,
3130  (errcode_for_file_access(),
3131  errmsg("could not execute command \"%s\": %m",
3132  cstate->filename)));
3133  }
3134  else
3135  {
3136  struct stat st;
3137 
3138  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3139  if (cstate->copy_file == NULL)
3140  {
3141  /* copy errno because ereport subfunctions might change it */
3142  int save_errno = errno;
3143 
3144  ereport(ERROR,
3145  (errcode_for_file_access(),
3146  errmsg("could not open file \"%s\" for reading: %m",
3147  cstate->filename),
3148  (save_errno == ENOENT || save_errno == EACCES) ?
3149  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3150  "You may want a client-side facility such as psql's \\copy.") : 0));
3151  }
3152 
3153  if (fstat(fileno(cstate->copy_file), &st))
3154  ereport(ERROR,
3155  (errcode_for_file_access(),
3156  errmsg("could not stat file \"%s\": %m",
3157  cstate->filename)));
3158 
3159  if (S_ISDIR(st.st_mode))
3160  ereport(ERROR,
3161  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3162  errmsg("\"%s\" is a directory", cstate->filename)));
3163  }
3164  }
3165 
3166  if (!cstate->binary)
3167  {
3168  /* must rely on user to tell us... */
3169  cstate->file_has_oids = cstate->oids;
3170  }
3171  else
3172  {
3173  /* Read and verify binary header */
3174  char readSig[11];
3175  int32 tmp;
3176 
3177  /* Signature */
3178  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3179  memcmp(readSig, BinarySignature, 11) != 0)
3180  ereport(ERROR,
3181  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3182  errmsg("COPY file signature not recognized")));
3183  /* Flags field */
3184  if (!CopyGetInt32(cstate, &tmp))
3185  ereport(ERROR,
3186  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3187  errmsg("invalid COPY file header (missing flags)")));
3188  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3189  tmp &= ~(1 << 16);
3190  if ((tmp >> 16) != 0)
3191  ereport(ERROR,
3192  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3193  errmsg("unrecognized critical flags in COPY file header")));
3194  /* Header extension length */
3195  if (!CopyGetInt32(cstate, &tmp) ||
3196  tmp < 0)
3197  ereport(ERROR,
3198  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3199  errmsg("invalid COPY file header (missing length)")));
3200  /* Skip extension header, if present */
3201  while (tmp-- > 0)
3202  {
3203  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3204  ereport(ERROR,
3205  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3206  errmsg("invalid COPY file header (wrong length)")));
3207  }
3208  }
3209 
3210  if (cstate->file_has_oids && cstate->binary)
3211  {
3212  getTypeBinaryInputInfo(OIDOID,
3213  &in_func_oid, &cstate->oid_typioparam);
3214  fmgr_info(in_func_oid, &cstate->oid_in_function);
3215  }
3216 
3217  /* create workspace for CopyReadAttributes results */
3218  if (!cstate->binary)
3219  {
3220  AttrNumber attr_count = list_length(cstate->attnumlist);
3221  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3222 
3223  cstate->max_fields = nfields;
3224  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3225  }
3226 
3227  MemoryContextSwitchTo(oldcontext);
3228 
3229  return cstate;
3230 }
3231 
3232 /*
3233  * Read raw fields in the next line for COPY FROM in text or csv mode.
3234  * Return false if no more lines.
3235  *
3236  * An internal temporary buffer is returned via 'fields'. It is valid until
3237  * the next call of the function. Since the function returns all raw fields
3238  * in the input file, 'nfields' could be different from the number of columns
3239  * in the relation.
3240  *
3241  * NOTE: force_not_null option are not applied to the returned fields.
3242  */
3243 bool
3244 NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
3245 {
3246  int fldct;
3247  bool done;
3248 
3249  /* only available for text or csv input */
3250  Assert(!cstate->binary);
3251 
3252  /* on input just throw the header line away */
3253  if (cstate->cur_lineno == 0 && cstate->header_line)
3254  {
3255  cstate->cur_lineno++;
3256  if (CopyReadLine(cstate))
3257  return false; /* done */
3258  }
3259 
3260  cstate->cur_lineno++;
3261 
3262  /* Actually read the line into memory here */
3263  done = CopyReadLine(cstate);
3264 
3265  /*
3266  * EOF at start of line means we're done. If we see EOF after some
3267  * characters, we act as though it was newline followed by EOF, ie,
3268  * process the line and then exit loop on next iteration.
3269  */
3270  if (done && cstate->line_buf.len == 0)
3271  return false;
3272 
3273  /* Parse the line into de-escaped field values */
3274  if (cstate->csv_mode)
3275  fldct = CopyReadAttributesCSV(cstate);
3276  else
3277  fldct = CopyReadAttributesText(cstate);
3278 
3279  *fields = cstate->raw_fields;
3280  *nfields = fldct;
3281  return true;
3282 }
3283 
3284 /*
3285  * Read next tuple from file for COPY FROM. Return false if no more tuples.
3286  *
3287  * 'econtext' is used to evaluate default expression for each columns not
3288  * read from the file. It can be NULL when no default values are used, i.e.
3289  * when all columns are read from the file.
3290  *
3291  * 'values' and 'nulls' arrays must be the same length as columns of the
3292  * relation passed to BeginCopyFrom. This function fills the arrays.
3293  * Oid of the tuple is returned with 'tupleOid' separately.
3294  */
3295 bool
3296 NextCopyFrom(CopyState cstate, ExprContext *econtext,
3297  Datum *values, bool *nulls, Oid *tupleOid)
3298 {
3299  TupleDesc tupDesc;
3300  AttrNumber num_phys_attrs,
3301  attr_count,
3302  num_defaults = cstate->num_defaults;
3303  FmgrInfo *in_functions = cstate->in_functions;
3304  Oid *typioparams = cstate->typioparams;
3305  int i;
3306  int nfields;
3307  bool isnull;
3308  bool file_has_oids = cstate->file_has_oids;
3309  int *defmap = cstate->defmap;
3310  ExprState **defexprs = cstate->defexprs;
3311 
3312  tupDesc = RelationGetDescr(cstate->rel);
3313  num_phys_attrs = tupDesc->natts;
3314  attr_count = list_length(cstate->attnumlist);
3315  nfields = file_has_oids ? (attr_count + 1) : attr_count;
3316 
3317  /* Initialize all values for row to NULL */
3318  MemSet(values, 0, num_phys_attrs * sizeof(Datum));
3319  MemSet(nulls, true, num_phys_attrs * sizeof(bool));
3320 
3321  if (!cstate->binary)
3322  {
3323  char **field_strings;
3324  ListCell *cur;
3325  int fldct;
3326  int fieldno;
3327  char *string;
3328 
3329  /* read raw fields in the next line */
3330  if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
3331  return false;
3332 
3333  /* check for overflowing fields */
3334  if (nfields > 0 && fldct > nfields)
3335  ereport(ERROR,
3336  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3337  errmsg("extra data after last expected column")));
3338 
3339  fieldno = 0;
3340 
3341  /* Read the OID field if present */
3342  if (file_has_oids)
3343  {
3344  if (fieldno >= fldct)
3345  ereport(ERROR,
3346  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3347  errmsg("missing data for OID column")));
3348  string = field_strings[fieldno++];
3349 
3350  if (string == NULL)
3351  ereport(ERROR,
3352  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3353  errmsg("null OID in COPY data")));
3354  else if (cstate->oids && tupleOid != NULL)
3355  {
3356  cstate->cur_attname = "oid";
3357  cstate->cur_attval = string;
3358  *tupleOid = DatumGetObjectId(DirectFunctionCall1(oidin,
3359  CStringGetDatum(string)));
3360  if (*tupleOid == InvalidOid)
3361  ereport(ERROR,
3362  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3363  errmsg("invalid OID in COPY data")));
3364  cstate->cur_attname = NULL;
3365  cstate->cur_attval = NULL;
3366  }
3367  }
3368 
3369  /* Loop to read the user attributes on the line. */
3370  foreach(cur, cstate->attnumlist)
3371  {
3372  int attnum = lfirst_int(cur);
3373  int m = attnum - 1;
3374  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3375 
3376  if (fieldno >= fldct)
3377  ereport(ERROR,
3378  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3379  errmsg("missing data for column \"%s\"",
3380  NameStr(att->attname))));
3381  string = field_strings[fieldno++];
3382 
3383  if (cstate->convert_select_flags &&
3384  !cstate->convert_select_flags[m])
3385  {
3386  /* ignore input field, leaving column as NULL */
3387  continue;
3388  }
3389 
3390  if (cstate->csv_mode)
3391  {
3392  if (string == NULL &&
3393  cstate->force_notnull_flags[m])
3394  {
3395  /*
3396  * FORCE_NOT_NULL option is set and column is NULL -
3397  * convert it to the NULL string.
3398  */
3399  string = cstate->null_print;
3400  }
3401  else if (string != NULL && cstate->force_null_flags[m]
3402  && strcmp(string, cstate->null_print) == 0)
3403  {
3404  /*
3405  * FORCE_NULL option is set and column matches the NULL
3406  * string. It must have been quoted, or otherwise the
3407  * string would already have been set to NULL. Convert it
3408  * to NULL as specified.
3409  */
3410  string = NULL;
3411  }
3412  }
3413 
3414  cstate->cur_attname = NameStr(att->attname);
3415  cstate->cur_attval = string;
3416  values[m] = InputFunctionCall(&in_functions[m],
3417  string,
3418  typioparams[m],
3419  att->atttypmod);
3420  if (string != NULL)
3421  nulls[m] = false;
3422  cstate->cur_attname = NULL;
3423  cstate->cur_attval = NULL;
3424  }
3425 
3426  Assert(fieldno == nfields);
3427  }
3428  else
3429  {
3430  /* binary */
3431  int16 fld_count;
3432  ListCell *cur;
3433 
3434  cstate->cur_lineno++;
3435 
3436  if (!CopyGetInt16(cstate, &fld_count))
3437  {
3438  /* EOF detected (end of file, or protocol-level EOF) */
3439  return false;
3440  }
3441 
3442  if (fld_count == -1)
3443  {
3444  /*
3445  * Received EOF marker. In a V3-protocol copy, wait for the
3446  * protocol-level EOF, and complain if it doesn't come
3447  * immediately. This ensures that we correctly handle CopyFail,
3448  * if client chooses to send that now.
3449  *
3450  * Note that we MUST NOT try to read more data in an old-protocol
3451  * copy, since there is no protocol-level EOF marker then. We
3452  * could go either way for copy from file, but choose to throw
3453  * error if there's data after the EOF marker, for consistency
3454  * with the new-protocol case.
3455  */
3456  char dummy;
3457 
3458  if (cstate->copy_dest != COPY_OLD_FE &&
3459  CopyGetData(cstate, &dummy, 1, 1) > 0)
3460  ereport(ERROR,
3461  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3462  errmsg("received copy data after EOF marker")));
3463  return false;
3464  }
3465 
3466  if (fld_count != attr_count)
3467  ereport(ERROR,
3468  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3469  errmsg("row field count is %d, expected %d",
3470  (int) fld_count, attr_count)));
3471 
3472  if (file_has_oids)
3473  {
3474  Oid loaded_oid;
3475 
3476  cstate->cur_attname = "oid";
3477  loaded_oid =
3478  DatumGetObjectId(CopyReadBinaryAttribute(cstate,
3479  0,
3480  &cstate->oid_in_function,
3481  cstate->oid_typioparam,
3482  -1,
3483  &isnull));
3484  if (isnull || loaded_oid == InvalidOid)
3485  ereport(ERROR,
3486  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3487  errmsg("invalid OID in COPY data")));
3488  cstate->cur_attname = NULL;
3489  if (cstate->oids && tupleOid != NULL)
3490  *tupleOid = loaded_oid;
3491  }
3492 
3493  i = 0;
3494  foreach(cur, cstate->attnumlist)
3495  {
3496  int attnum = lfirst_int(cur);
3497  int m = attnum - 1;
3498  Form_pg_attribute att = TupleDescAttr(tupDesc, m);
3499 
3500  cstate->cur_attname = NameStr(att->attname);
3501  i++;
3502  values[m] = CopyReadBinaryAttribute(cstate,
3503  i,
3504  &in_functions[m],
3505  typioparams[m],
3506  att->atttypmod,
3507  &nulls[m]);
3508  cstate->cur_attname = NULL;
3509  }
3510  }
3511 
3512  /*
3513  * Now compute and insert any defaults available for the columns not
3514  * provided by the input data. Anything not processed here or above will
3515  * remain NULL.
3516  */
3517  for (i = 0; i < num_defaults; i++)
3518  {
3519  /*
3520  * The caller must supply econtext and have switched into the
3521  * per-tuple memory context in it.
3522  */
3523  Assert(econtext != NULL);
3524  Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
3525 
3526  values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
3527  &nulls[defmap[i]]);
3528  }
3529 
3530  return true;
3531 }
3532 
3533 /*
3534  * Clean up storage and release resources for COPY FROM.
3535  */
3536 void
3537 EndCopyFrom(CopyState cstate)
3538 {
3539  /* No COPY FROM related resources except memory. */
3540 
3541  EndCopy(cstate);
3542 }
3543 
3544 /*
3545  * Read the next input line and stash it in line_buf, with conversion to
3546  * server encoding.
3547  *
3548  * Result is true if read was terminated by EOF, false if terminated
3549  * by newline. The terminating newline or EOF marker is not included
3550  * in the final value of line_buf.
3551  */
3552 static bool
3553 CopyReadLine(CopyState cstate)
3554 {
3555  bool result;
3556 
3557  resetStringInfo(&cstate->line_buf);
3558  cstate->line_buf_valid = true;
3559 
3560  /* Mark that encoding conversion hasn't occurred yet */
3561  cstate->line_buf_converted = false;
3562 
3563  /* Parse data and transfer into line_buf */
3564  result = CopyReadLineText(cstate);
3565 
3566  if (result)
3567  {
3568  /*
3569  * Reached EOF. In protocol version 3, we should ignore anything
3570  * after \. up to the protocol end of copy data. (XXX maybe better
3571  * not to treat \. as special?)
3572  */
3573  if (cstate->copy_dest == COPY_NEW_FE)
3574  {
3575  do
3576  {
3577  cstate->raw_buf_index = cstate->raw_buf_len;
3578  } while (CopyLoadRawBuf(cstate));
3579  }
3580  }
3581  else
3582  {
3583  /*
3584  * If we didn't hit EOF, then we must have transferred the EOL marker
3585  * to line_buf along with the data. Get rid of it.
3586  */
3587  switch (cstate->eol_type)
3588  {
3589  case EOL_NL:
3590  Assert(cstate->line_buf.len >= 1);
3591  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3592  cstate->line_buf.len--;
3593  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3594  break;
3595  case EOL_CR:
3596  Assert(cstate->line_buf.len >= 1);
3597  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3598  cstate->line_buf.len--;
3599  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3600  break;
3601  case EOL_CRNL:
3602  Assert(cstate->line_buf.len >= 2);
3603  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3604  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3605  cstate->line_buf.len -= 2;
3606  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3607  break;
3608  case EOL_UNKNOWN:
3609  /* shouldn't get here */
3610  Assert(false);
3611  break;
3612  }
3613  }
3614 
3615  /* Done reading the line. Convert it to server encoding. */
3616  if (cstate->need_transcoding)
3617  {
3618  char *cvt;
3619 
3620  cvt = pg_any_to_server(cstate->line_buf.data,
3621  cstate->line_buf.len,
3622  cstate->file_encoding);
3623  if (cvt != cstate->line_buf.data)
3624  {
3625  /* transfer converted data back to line_buf */
3626  resetStringInfo(&cstate->line_buf);
3627  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3628  pfree(cvt);
3629  }
3630  }
3631 
3632  /* Now it's safe to use the buffer in error messages */
3633  cstate->line_buf_converted = true;
3634 
3635  return result;
3636 }
3637 
3638 /*
3639  * CopyReadLineText - inner loop of CopyReadLine for text mode
3640  */
3641 static bool
3642 CopyReadLineText(CopyState cstate)
3643 {
3644  char *copy_raw_buf;
3645  int raw_buf_ptr;
3646  int copy_buf_len;
3647  bool need_data = false;
3648  bool hit_eof = false;
3649  bool result = false;
3650  char mblen_str[2];
3651 
3652  /* CSV variables */
3653  bool first_char_in_line = true;
3654  bool in_quote = false,
3655  last_was_esc = false;
3656  char quotec = '\0';
3657  char escapec = '\0';
3658 
3659  if (cstate->csv_mode)
3660  {
3661  quotec = cstate->quote[0];
3662  escapec = cstate->escape[0];
3663  /* ignore special escape processing if it's the same as quotec */
3664  if (quotec == escapec)
3665  escapec = '\0';
3666  }
3667 
3668  mblen_str[1] = '\0';
3669 
3670  /*
3671  * The objective of this loop is to transfer the entire next input line
3672  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3673  * \n) and the end-of-copy marker (\.).
3674  *
3675  * In CSV mode, \r and \n inside a quoted field are just part of the data
3676  * value and are put in line_buf. We keep just enough state to know if we
3677  * are currently in a quoted field or not.
3678  *
3679  * These four characters, and the CSV escape and quote characters, are
3680  * assumed the same in frontend and backend encodings.
3681  *
3682  * For speed, we try to move data from raw_buf to line_buf in chunks
3683  * rather than one character at a time. raw_buf_ptr points to the next
3684  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3685  * have been determined to be part of the line, but not yet transferred to
3686  * line_buf.
3687  *
3688  * For a little extra speed within the loop, we copy raw_buf and
3689  * raw_buf_len into local variables.
3690  */
3691  copy_raw_buf = cstate->raw_buf;
3692  raw_buf_ptr = cstate->raw_buf_index;
3693  copy_buf_len = cstate->raw_buf_len;
3694 
3695  for (;;)
3696  {
3697  int prev_raw_ptr;
3698  char c;
3699 
3700  /*
3701  * Load more data if needed. Ideally we would just force four bytes
3702  * of read-ahead and avoid the many calls to
3703  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3704  * does not allow us to read too far ahead or we might read into the
3705  * next data, so we read-ahead only as far we know we can. One
3706  * optimization would be to read-ahead four byte here if
3707  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3708  * considering the size of the buffer.
3709  */
3710  if (raw_buf_ptr >= copy_buf_len || need_data)
3711  {
3712  REFILL_LINEBUF;
3713 
3714  /*
3715  * Try to read some more data. This will certainly reset
3716  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3717  */
3718  if (!CopyLoadRawBuf(cstate))
3719  hit_eof = true;
3720  raw_buf_ptr = 0;
3721  copy_buf_len = cstate->raw_buf_len;
3722 
3723  /*
3724  * If we are completely out of data, break out of the loop,
3725  * reporting EOF.
3726  */
3727  if (copy_buf_len <= 0)
3728  {
3729  result = true;
3730  break;
3731  }
3732  need_data = false;
3733  }
3734 
3735  /* OK to fetch a character */
3736  prev_raw_ptr = raw_buf_ptr;
3737  c = copy_raw_buf[raw_buf_ptr++];
3738 
3739  if (cstate->csv_mode)
3740  {
3741  /*
3742  * If character is '\\' or '\r', we may need to look ahead below.
3743  * Force fetch of the next character if we don't already have it.
3744  * We need to do this before changing CSV state, in case one of
3745  * these characters is also the quote or escape character.
3746  *
3747  * Note: old-protocol does not like forced prefetch, but it's OK
3748  * here since we cannot validly be at EOF.
3749  */
3750  if (c == '\\' || c == '\r')
3751  {
3752  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3753  }
3754 
3755  /*
3756  * Dealing with quotes and escapes here is mildly tricky. If the
3757  * quote char is also the escape char, there's no problem - we
3758  * just use the char as a toggle. If they are different, we need
3759  * to ensure that we only take account of an escape inside a
3760  * quoted field and immediately preceding a quote char, and not
3761  * the second in an escape-escape sequence.
3762  */
3763  if (in_quote && c == escapec)
3764  last_was_esc = !last_was_esc;
3765  if (c == quotec && !last_was_esc)
3766  in_quote = !in_quote;
3767  if (c != escapec)
3768  last_was_esc = false;
3769 
3770  /*
3771  * Updating the line count for embedded CR and/or LF chars is
3772  * necessarily a little fragile - this test is probably about the
3773  * best we can do. (XXX it's arguable whether we should do this
3774  * at all --- is cur_lineno a physical or logical count?)
3775  */
3776  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3777  cstate->cur_lineno++;
3778  }
3779 
3780  /* Process \r */
3781  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3782  {
3783  /* Check for \r\n on first line, _and_ handle \r\n. */
3784  if (cstate->eol_type == EOL_UNKNOWN ||
3785  cstate->eol_type == EOL_CRNL)
3786  {
3787  /*
3788  * If need more data, go back to loop top to load it.
3789  *
3790  * Note that if we are at EOF, c will wind up as '\0' because
3791  * of the guaranteed pad of raw_buf.
3792  */
3793  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3794 
3795  /* get next char */
3796  c = copy_raw_buf[raw_buf_ptr];
3797 
3798  if (c == '\n')
3799  {
3800  raw_buf_ptr++; /* eat newline */
3801  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3802  }
3803  else
3804  {
3805  /* found \r, but no \n */
3806  if (cstate->eol_type == EOL_CRNL)
3807  ereport(ERROR,
3808  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3809  !cstate->csv_mode ?
3810  errmsg("literal carriage return found in data") :
3811  errmsg("unquoted carriage return found in data"),
3812  !cstate->csv_mode ?
3813  errhint("Use \"\\r\" to represent carriage return.") :
3814  errhint("Use quoted CSV field to represent carriage return.")));
3815 
3816  /*
3817  * if we got here, it is the first line and we didn't find
3818  * \n, so don't consume the peeked character
3819  */
3820  cstate->eol_type = EOL_CR;
3821  }
3822  }
3823  else if (cstate->eol_type == EOL_NL)
3824  ereport(ERROR,
3825  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3826  !cstate->csv_mode ?
3827  errmsg("literal carriage return found in data") :
3828  errmsg("unquoted carriage return found in data"),
3829  !cstate->csv_mode ?
3830  errhint("Use \"\\r\" to represent carriage return.") :
3831  errhint("Use quoted CSV field to represent carriage return.")));
3832  /* If reach here, we have found the line terminator */
3833  break;
3834  }
3835 
3836  /* Process \n */
3837  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3838  {
3839  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3840  ereport(ERROR,
3841  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3842  !cstate->csv_mode ?
3843  errmsg("literal newline found in data") :
3844  errmsg("unquoted newline found in data"),
3845  !cstate->csv_mode ?
3846  errhint("Use \"\\n\" to represent newline.") :
3847  errhint("Use quoted CSV field to represent newline.")));
3848  cstate->eol_type = EOL_NL; /* in case not set yet */
3849  /* If reach here, we have found the line terminator */
3850  break;
3851  }
3852 
3853  /*
3854  * In CSV mode, we only recognize \. alone on a line. This is because
3855  * \. is a valid CSV data value.
3856  */
3857  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3858  {
3859  char c2;
3860 
3861  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3862  IF_NEED_REFILL_AND_EOF_BREAK(0);
3863 
3864  /* -----
3865  * get next character
3866  * Note: we do not change c so if it isn't \., we can fall
3867  * through and continue processing for file encoding.
3868  * -----
3869  */
3870  c2 = copy_raw_buf[raw_buf_ptr];
3871 
3872  if (c2 == '.')
3873  {
3874  raw_buf_ptr++; /* consume the '.' */
3875 
3876  /*
3877  * Note: if we loop back for more data here, it does not
3878  * matter that the CSV state change checks are re-executed; we
3879  * will come back here with no important state changed.
3880  */
3881  if (cstate->eol_type == EOL_CRNL)
3882  {
3883  /* Get the next character */
3884  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3885  /* if hit_eof, c2 will become '\0' */
3886  c2 = copy_raw_buf[raw_buf_ptr++];
3887 
3888  if (c2 == '\n')
3889  {
3890  if (!cstate->csv_mode)
3891  ereport(ERROR,
3892  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3893  errmsg("end-of-copy marker does not match previous newline style")));
3894  else
3895  NO_END_OF_COPY_GOTO;
3896  }
3897  else if (c2 != '\r')
3898  {
3899  if (!cstate->csv_mode)
3900  ereport(ERROR,
3901  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3902  errmsg("end-of-copy marker corrupt")));
3903  else
3904  NO_END_OF_COPY_GOTO;
3905  }
3906  }
3907 
3908  /* Get the next character */
3909  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
3910  /* if hit_eof, c2 will become '\0' */
3911  c2 = copy_raw_buf[raw_buf_ptr++];
3912 
3913  if (c2 != '\r' && c2 != '\n')
3914  {
3915  if (!cstate->csv_mode)
3916  ereport(ERROR,
3917  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3918  errmsg("end-of-copy marker corrupt")));
3919  else
3920  NO_END_OF_COPY_GOTO;
3921  }
3922 
3923  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3924  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3925  (cstate->eol_type == EOL_CR && c2 != '\r'))
3926  {
3927  ereport(ERROR,
3928  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3929  errmsg("end-of-copy marker does not match previous newline style")));
3930  }
3931 
3932  /*
3933  * Transfer only the data before the \. into line_buf, then
3934  * discard the data and the \. sequence.
3935  */
3936  if (prev_raw_ptr > cstate->raw_buf_index)
3937  appendBinaryStringInfo(&cstate->line_buf,
3938  cstate->raw_buf + cstate->raw_buf_index,
3939  prev_raw_ptr - cstate->raw_buf_index);
3940  cstate->raw_buf_index = raw_buf_ptr;
3941  result = true; /* report EOF */
3942  break;
3943  }
3944  else if (!cstate->csv_mode)
3945 
3946  /*
3947  * If we are here, it means we found a backslash followed by
3948  * something other than a period. In non-CSV mode, anything
3949  * after a backslash is special, so we skip over that second
3950  * character too. If we didn't do that \\. would be
3951  * considered an eof-of copy, while in non-CSV mode it is a
3952  * literal backslash followed by a period. In CSV mode,
3953  * backslashes are not special, so we want to process the
3954  * character after the backslash just like a normal character,
3955  * so we don't increment in those cases.
3956  */
3957  raw_buf_ptr++;
3958  }
3959 
3960  /*
3961  * This label is for CSV cases where \. appears at the start of a
3962  * line, but there is more text after it, meaning it was a data value.
3963  * We are more strict for \. in CSV mode because \. could be a data
3964  * value, while in non-CSV mode, \. cannot be a data value.
3965  */
3966 not_end_of_copy:
3967 
3968  /*
3969  * Process all bytes of a multi-byte character as a group.
3970  *
3971  * We only support multi-byte sequences where the first byte has the
3972  * high-bit set, so as an optimization we can avoid this block
3973  * entirely if it is not set.
3974  */
3975  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3976  {
3977  int mblen;
3978 
3979  mblen_str[0] = c;
3980  /* All our encodings only read the first byte to get the length */
3981  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3982  IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
3983  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3984  raw_buf_ptr += mblen - 1;
3985  }
3986  first_char_in_line = false;
3987  } /* end of outer loop */
3988 
3989  /*
3990  * Transfer any still-uncopied data to line_buf.
3991  */
3992  REFILL_LINEBUF;
3993 
3994  return result;
3995 }
3996 
3997 /*
3998  * Return decimal value for a hexadecimal digit
3999  */
4000 static int
4001 GetDecimalFromHex(char hex)
4002 {
4003  if (isdigit((unsigned char) hex))
4004  return hex - '0';
4005  else
4006  return tolower((unsigned char) hex) - 'a' + 10;
4007 }
4008 
4009 /*
4010  * Parse the current line into separate attributes (fields),
4011  * performing de-escaping as needed.
4012  *
4013  * The input is in line_buf. We use attribute_buf to hold the result
4014  * strings. cstate->raw_fields[k] is set to point to the k'th attribute
4015  * string, or NULL when the input matches the null marker string.
4016  * This array is expanded as necessary.
4017  *
4018  * (Note that the caller cannot check for nulls since the returned
4019  * string would be the post-de-escaping equivalent, which may look
4020  * the same as some valid data string.)
4021  *
4022  * delim is the column delimiter string (must be just one byte for now).
4023  * null_print is the null marker string. Note that this is compared to
4024  * the pre-de-escaped input string.
4025  *
4026  * The return value is the number of fields actually read.
4027  */
4028 static int
4029 CopyReadAttributesText(CopyState cstate)
4030 {
4031  char delimc = cstate->delim[0];
4032  int fieldno;
4033  char *output_ptr;
4034  char *cur_ptr;
4035  char *line_end_ptr;
4036 
4037  /*
4038  * We need a special case for zero-column tables: check that the input
4039  * line is empty, and return.
4040  */
4041  if (cstate->max_fields <= 0)
4042  {
4043  if (cstate->line_buf.len != 0)
4044  ereport(ERROR,
4045  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4046  errmsg("extra data after last expected column")));
4047  return 0;
4048  }
4049 
4050  resetStringInfo(&cstate->attribute_buf);
4051 
4052  /*
4053  * The de-escaped attributes will certainly not be longer than the input
4054  * data line, so we can just force attribute_buf to be large enough and
4055  * then transfer data without any checks for enough space. We need to do
4056  * it this way because enlarging attribute_buf mid-stream would invalidate
4057  * pointers already stored into cstate->raw_fields[].
4058  */
4059  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4060  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4061  output_ptr = cstate->attribute_buf.data;
4062 
4063  /* set pointer variables for loop */
4064  cur_ptr = cstate->line_buf.data;
4065  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4066 
4067  /* Outer loop iterates over fields */
4068  fieldno = 0;
4069  for (;;)
4070  {
4071  bool found_delim = false;
4072  char *start_ptr;
4073  char *end_ptr;
4074  int input_len;
4075  bool saw_non_ascii = false;
4076 
4077  /* Make sure there is enough space for the next value */
4078  if (fieldno >= cstate->max_fields)
4079  {
4080  cstate->max_fields *= 2;
4081  cstate->raw_fields =
4082  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4083  }
4084 
4085  /* Remember start of field on both input and output sides */
4086  start_ptr = cur_ptr;
4087  cstate->raw_fields[fieldno] = output_ptr;
4088 
4089  /*
4090  * Scan data for field.
4091  *
4092  * Note that in this loop, we are scanning to locate the end of field
4093  * and also speculatively performing de-escaping. Once we find the
4094  * end-of-field, we can match the raw field contents against the null
4095  * marker string. Only after that comparison fails do we know that
4096  * de-escaping is actually the right thing to do; therefore we *must
4097  * not* throw any syntax errors before we've done the null-marker
4098  * check.
4099  */
4100  for (;;)
4101  {
4102  char c;
4103 
4104  end_ptr = cur_ptr;
4105  if (cur_ptr >= line_end_ptr)
4106  break;
4107  c = *cur_ptr++;
4108  if (c == delimc)
4109  {
4110  found_delim = true;
4111  break;
4112  }
4113  if (c == '\\')
4114  {
4115  if (cur_ptr >= line_end_ptr)
4116  break;
4117  c = *cur_ptr++;
4118  switch (c)
4119  {
4120  case '0':
4121  case '1':
4122  case '2':
4123  case '3':
4124  case '4':
4125  case '5':
4126  case '6':
4127  case '7':
4128  {
4129  /* handle \013 */
4130  int val;
4131 
4132  val = OCTVALUE(c);
4133  if (cur_ptr < line_end_ptr)
4134  {
4135  c = *cur_ptr;
4136  if (ISOCTAL(c))
4137  {
4138  cur_ptr++;
4139  val = (val << 3) + OCTVALUE(c);
4140  if (cur_ptr < line_end_ptr)
4141  {
4142  c = *cur_ptr;
4143  if (ISOCTAL(c))
4144  {
4145  cur_ptr++;
4146  val = (val << 3) + OCTVALUE(c);
4147  }
4148  }
4149  }
4150  }
4151  c = val & 0377;
4152  if (c == '\0' || IS_HIGHBIT_SET(c))
4153  saw_non_ascii = true;
4154  }
4155  break;
4156  case 'x':
4157  /* Handle \x3F */
4158  if (cur_ptr < line_end_ptr)
4159  {
4160  char hexchar = *cur_ptr;
4161 
4162  if (isxdigit((unsigned char) hexchar))
4163  {
4164  int val = GetDecimalFromHex(hexchar);
4165 
4166  cur_ptr++;
4167  if (cur_ptr < line_end_ptr)
4168  {
4169  hexchar = *cur_ptr;
4170  if (isxdigit((unsigned char) hexchar))
4171  {
4172  cur_ptr++;
4173  val = (val << 4) + GetDecimalFromHex(hexchar);
4174  }
4175  }
4176  c = val & 0xff;
4177  if (c == '\0' || IS_HIGHBIT_SET(c))
4178  saw_non_ascii = true;
4179  }
4180  }
4181  break;
4182  case 'b':
4183  c = '\b';
4184  break;
4185  case 'f':
4186  c = '\f';
4187  break;
4188  case 'n':
4189  c = '\n';
4190  break;
4191  case 'r':
4192  c = '\r';
4193  break;
4194  case 't':
4195  c = '\t';
4196  break;
4197  case 'v':
4198  c = '\v';
4199  break;
4200 
4201  /*
4202  * in all other cases, take the char after '\'
4203  * literally
4204  */
4205  }
4206  }
4207 
4208  /* Add c to output string */
4209  *output_ptr++ = c;
4210  }
4211 
4212  /* Check whether raw input matched null marker */
4213  input_len = end_ptr - start_ptr;
4214  if (input_len == cstate->null_print_len &&
4215  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4216  cstate->raw_fields[fieldno] = NULL;
4217  else
4218  {
4219  /*
4220  * At this point we know the field is supposed to contain data.
4221  *
4222  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4223  * resulting string is valid data for the db encoding.
4224  */
4225  if (saw_non_ascii)
4226  {
4227  char *fld = cstate->raw_fields[fieldno];
4228 
4229  pg_verifymbstr(fld, output_ptr - fld, false);
4230  }
4231  }
4232 
4233  /* Terminate attribute value in output area */
4234  *output_ptr++ = '\0';
4235 
4236  fieldno++;
4237  /* Done if we hit EOL instead of a delim */
4238  if (!found_delim)
4239  break;
4240  }
4241 
4242  /* Clean up state of attribute_buf */
4243  output_ptr--;
4244  Assert(*output_ptr == '\0');
4245  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4246 
4247  return fieldno;
4248 }
4249 
4250 /*
4251  * Parse the current line into separate attributes (fields),
4252  * performing de-escaping as needed. This has exactly the same API as
4253  * CopyReadAttributesText, except we parse the fields according to
4254  * "standard" (i.e. common) CSV usage.
4255  */
4256 static int
4257 CopyReadAttributesCSV(CopyState cstate)
4258 {
4259  char delimc = cstate->delim[0];
4260  char quotec = cstate->quote[0];
4261  char escapec = cstate->escape[0];
4262  int fieldno;
4263  char *output_ptr;
4264  char *cur_ptr;
4265  char *line_end_ptr;
4266 
4267  /*
4268  * We need a special case for zero-column tables: check that the input
4269  * line is empty, and return.
4270  */
4271  if (cstate->max_fields <= 0)
4272  {
4273  if (cstate->line_buf.len != 0)
4274  ereport(ERROR,
4275  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4276  errmsg("extra data after last expected column")));
4277  return 0;
4278  }
4279 
4280  resetStringInfo(&cstate->attribute_buf);
4281 
4282  /*
4283  * The de-escaped attributes will certainly not be longer than the input
4284  * data line, so we can just force attribute_buf to be large enough and
4285  * then transfer data without any checks for enough space. We need to do
4286  * it this way because enlarging attribute_buf mid-stream would invalidate
4287  * pointers already stored into cstate->raw_fields[].
4288  */
4289  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4290  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4291  output_ptr = cstate->attribute_buf.data;
4292 
4293  /* set pointer variables for loop */
4294  cur_ptr = cstate->line_buf.data;
4295  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4296 
4297  /* Outer loop iterates over fields */
4298  fieldno = 0;
4299  for (;;)
4300  {
4301  bool found_delim = false;
4302  bool saw_quote = false;
4303  char *start_ptr;
4304  char *end_ptr;
4305  int input_len;
4306 
4307  /* Make sure there is enough space for the next value */
4308  if (fieldno >= cstate->max_fields)
4309  {
4310  cstate->max_fields *= 2;
4311  cstate->raw_fields =
4312  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4313  }
4314 
4315  /* Remember start of field on both input and output sides */
4316  start_ptr = cur_ptr;
4317  cstate->raw_fields[fieldno] = output_ptr;
4318 
4319  /*
4320  * Scan data for field,
4321  *
4322  * The loop starts in "not quote" mode and then toggles between that
4323  * and "in quote" mode. The loop exits normally if it is in "not
4324  * quote" mode and a delimiter or line end is seen.
4325  */
4326  for (;;)
4327  {
4328  char c;
4329 
4330  /* Not in quote */
4331  for (;;)
4332  {
4333  end_ptr = cur_ptr;
4334  if (cur_ptr >= line_end_ptr)
4335  goto endfield;
4336  c = *cur_ptr++;
4337  /* unquoted field delimiter */
4338  if (c == delimc)
4339  {
4340  found_delim = true;
4341  goto endfield;
4342  }
4343  /* start of quoted field (or part of field) */
4344  if (c == quotec)
4345  {
4346  saw_quote = true;
4347  break;
4348  }
4349  /* Add c to output string */
4350  *output_ptr++ = c;
4351  }
4352 
4353  /* In quote */
4354  for (;;)
4355  {
4356  end_ptr = cur_ptr;
4357  if (cur_ptr >= line_end_ptr)
4358  ereport(ERROR,
4359  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4360  errmsg("unterminated CSV quoted field")));
4361 
4362  c = *cur_ptr++;
4363 
4364  /* escape within a quoted field */
4365  if (c == escapec)
4366  {
4367  /*
4368  * peek at the next char if available, and escape it if it
4369  * is an escape char or a quote char
4370  */
4371  if (cur_ptr < line_end_ptr)
4372  {
4373  char nextc = *cur_ptr;
4374 
4375  if (nextc == escapec || nextc == quotec)
4376  {
4377  *output_ptr++ = nextc;
4378  cur_ptr++;
4379  continue;
4380  }
4381  }
4382  }
4383 
4384  /*
4385  * end of quoted field. Must do this test after testing for
4386  * escape in case quote char and escape char are the same
4387  * (which is the common case).
4388  */
4389  if (c == quotec)
4390  break;
4391 
4392  /* Add c to output string */
4393  *output_ptr++ = c;
4394  }
4395  }
4396 endfield:
4397 
4398  /* Terminate attribute value in output area */
4399  *output_ptr++ = '\0';
4400 
4401  /* Check whether raw input matched null marker */
4402  input_len = end_ptr - start_ptr;
4403  if (!saw_quote && input_len == cstate->null_print_len &&
4404  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4405  cstate->raw_fields[fieldno] = NULL;
4406 
4407  fieldno++;
4408  /* Done if we hit EOL instead of a delim */
4409  if (!found_delim)
4410  break;
4411  }
4412 
4413  /* Clean up state of attribute_buf */
4414  output_ptr--;
4415  Assert(*output_ptr == '\0');
4416  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4417 
4418  return fieldno;
4419 }
4420 
4421 
4422 /*
4423  * Read a binary attribute
4424  */
4425 static Datum
4426 CopyReadBinaryAttribute(CopyState cstate,
4427  int column_no, FmgrInfo *flinfo,
4428  Oid typioparam, int32 typmod,
4429  bool *isnull)
4430 {
4431  int32 fld_size;
4432  Datum result;
4433 
4434  if (!CopyGetInt32(cstate, &fld_size))
4435  ereport(ERROR,
4436  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4437  errmsg("unexpected EOF in COPY data")));
4438  if (fld_size == -1)
4439  {
4440  *isnull = true;
4441  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4442  }
4443  if (fld_size < 0)
4444  ereport(ERROR,
4445  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4446  errmsg("invalid field size")));
4447 
4448  /* reset attribute_buf to empty, and load raw data in it */
4449  resetStringInfo(&cstate->attribute_buf);
4450 
4451  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4452  if (CopyGetData(cstate, cstate->attribute_buf.data,
4453  fld_size, fld_size) != fld_size)
4454  ereport(ERROR,
4455  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4456  errmsg("unexpected EOF in COPY data")));
4457 
4458  cstate->attribute_buf.len = fld_size;
4459  cstate->attribute_buf.data[fld_size] = '\0';
4460 
4461  /* Call the column type's binary input converter */
4462  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4463  typioparam, typmod);
4464 
4465  /* Trouble if it didn't eat the whole buffer */
4466  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4467  ereport(ERROR,
4468  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4469  errmsg("incorrect binary data format")));
4470 
4471  *isnull = false;
4472  return result;
4473 }
4474 
4475 /*
4476  * Send text representation of one attribute, with conversion and escaping
4477  */
4478 #define DUMPSOFAR() \
4479  do { \
4480  if (ptr > start) \
4481  CopySendData(cstate, start, ptr - start); \
4482  } while (0)
4483 
4484 static void
4485 CopyAttributeOutText(CopyState cstate, char *string)
4486 {
4487  char *ptr;
4488  char *start;
4489  char c;
4490  char delimc = cstate->delim[0];
4491 
4492  if (cstate->need_transcoding)
4493  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4494  else
4495  ptr = string;
4496 
4497  /*
4498  * We have to grovel through the string searching for control characters
4499  * and instances of the delimiter character. In most cases, though, these
4500  * are infrequent. To avoid overhead from calling CopySendData once per
4501  * character, we dump out all characters between escaped characters in a
4502  * single call. The loop invariant is that the data from "start" to "ptr"
4503  * can be sent literally, but hasn't yet been.
4504  *
4505  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4506  * in valid backend encodings, extra bytes of a multibyte character never
4507  * look like ASCII. This loop is sufficiently performance-critical that
4508  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4509  * of the normal safe-encoding path.
4510  */
4511  if (cstate->encoding_embeds_ascii)
4512  {
4513  start = ptr;
4514  while ((c = *ptr) != '\0')
4515  {
4516  if ((unsigned char) c < (unsigned char) 0x20)
4517  {
4518  /*
4519  * \r and \n must be escaped, the others are traditional. We
4520  * prefer to dump these using the C-like notation, rather than
4521  * a backslash and the literal character, because it makes the
4522  * dump file a bit more proof against Microsoftish data
4523  * mangling.
4524  */
4525  switch (c)
4526  {
4527  case '\b':
4528  c = 'b';
4529  break;
4530  case '\f':
4531  c = 'f';
4532  break;
4533  case '\n':
4534  c = 'n';
4535  break;
4536  case '\r':
4537  c = 'r';
4538  break;
4539  case '\t':
4540  c = 't';
4541  break;
4542  case '\v':
4543  c = 'v';
4544  break;
4545  default:
4546  /* If it's the delimiter, must backslash it */
4547  if (c == delimc)
4548  break;
4549  /* All ASCII control chars are length 1 */
4550  ptr++;
4551  continue; /* fall to end of loop */
4552  }
4553  /* if we get here, we need to convert the control char */
4554  DUMPSOFAR();
4555  CopySendChar(cstate, '\\');
4556  CopySendChar(cstate, c);
4557  start = ++ptr; /* do not include char in next run */
4558  }
4559  else if (c == '\\' || c == delimc)
4560  {
4561  DUMPSOFAR();
4562  CopySendChar(cstate, '\\');
4563  start = ptr++; /* we include char in next run */
4564  }
4565  else if (IS_HIGHBIT_SET(c))
4566  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4567  else
4568  ptr++;
4569  }
4570  }
4571  else
4572  {
4573  start = ptr;
4574  while ((c = *ptr) != '\0')
4575  {
4576  if ((unsigned char) c < (unsigned char) 0x20)
4577  {
4578  /*
4579  * \r and \n must be escaped, the others are traditional. We
4580  * prefer to dump these using the C-like notation, rather than
4581  * a backslash and the literal character, because it makes the
4582  * dump file a bit more proof against Microsoftish data
4583  * mangling.
4584  */
4585  switch (c)
4586  {
4587  case '\b':
4588  c = 'b';
4589  break;
4590  case '\f':
4591  c = 'f';
4592  break;
4593  case '\n':
4594  c = 'n';
4595  break;
4596  case '\r':
4597  c = 'r';
4598  break;
4599  case '\t':
4600  c = 't';
4601  break;
4602  case '\v':
4603  c = 'v';
4604  break;
4605  default:
4606  /* If it's the delimiter, must backslash it */
4607  if (c == delimc)
4608  break;
4609  /* All ASCII control chars are length 1 */
4610  ptr++;
4611  continue; /* fall to end of loop */
4612  }
4613  /* if we get here, we need to convert the control char */
4614  DUMPSOFAR();
4615  CopySendChar(cstate, '\\');
4616  CopySendChar(cstate, c);
4617  start = ++ptr; /* do not include char in next run */
4618  }
4619  else if (c == '\\' || c == delimc)
4620  {
4621  DUMPSOFAR();
4622  CopySendChar(cstate, '\\');
4623  start = ptr++; /* we include char in next run */
4624  }
4625  else
4626  ptr++;
4627  }
4628  }
4629 
4630  DUMPSOFAR();
4631 }
4632 
4633 /*
4634  * Send text representation of one attribute, with conversion and
4635  * CSV-style escaping
4636  */
4637 static void
4638 CopyAttributeOutCSV(CopyState cstate, char *string,
4639  bool use_quote, bool single_attr)
4640 {
4641  char *ptr;
4642  char *start;
4643  char c;
4644  char delimc = cstate->delim[0];
4645  char quotec = cstate->quote[0];
4646  char escapec = cstate->escape[0];
4647 
4648  /* force quoting if it matches null_print (before conversion!) */
4649  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4650  use_quote = true;
4651 
4652  if (cstate->need_transcoding)
4653  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4654  else
4655  ptr = string;
4656 
4657  /*
4658  * Make a preliminary pass to discover if it needs quoting
4659  */
4660  if (!use_quote)
4661  {
4662  /*
4663  * Because '\.' can be a data value, quote it if it appears alone on a
4664  * line so it is not interpreted as the end-of-data marker.
4665  */
4666  if (single_attr && strcmp(ptr, "\\.") == 0)
4667  use_quote = true;
4668  else
4669  {
4670  char *tptr = ptr;
4671 
4672  while ((c = *tptr) != '\0')
4673  {
4674  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4675  {
4676  use_quote = true;
4677  break;
4678  }
4679  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4680  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4681  else
4682  tptr++;
4683  }
4684  }
4685  }
4686 
4687  if (use_quote)
4688  {
4689  CopySendChar(cstate, quotec);
4690 
4691  /*
4692  * We adopt the same optimization strategy as in CopyAttributeOutText
4693  */
4694  start = ptr;
4695  while ((c = *ptr) != '\0')
4696  {
4697  if (c == quotec || c == escapec)
4698  {
4699  DUMPSOFAR();
4700  CopySendChar(cstate, escapec);
4701  start = ptr; /* we include char in next run */
4702  }
4703  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4704  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4705  else
4706  ptr++;
4707  }
4708  DUMPSOFAR();
4709 
4710  CopySendChar(cstate, quotec);
4711  }
4712  else
4713  {
4714  /* If it doesn't need quoting, we can just dump it as-is */
4715  CopySendString(cstate, ptr);
4716  }
4717 }
4718 
4719 /*
4720  * CopyGetAttnums - build an integer list of attnums to be copied
4721  *
4722  * The input attnamelist is either the user-specified column list,
4723  * or NIL if there was none (in which case we want all the non-dropped
4724  * columns).
4725  *
4726  * rel can be NULL ... it's only used for error reports.
4727  */
4728 static List *
4729 CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
4730 {
4731  List *attnums = NIL;
4732 
4733  if (attnamelist == NIL)
4734  {
4735  /* Generate default column list */
4736  int attr_count = tupDesc->natts;
4737  int i;
4738 
4739  for (i = 0; i < attr_count; i++)
4740  {
4741  if (TupleDescAttr(tupDesc, i)->attisdropped)
4742  continue;
4743  attnums = lappend_int(attnums, i + 1);
4744  }
4745  }
4746  else
4747  {
4748  /* Validate the user-supplied list and extract attnums */
4749  ListCell *l;
4750 
4751  foreach(l, attnamelist)
4752  {
4753  char *name = strVal(lfirst(l));
4754  int attnum;
4755  int i;
4756 
4757  /* Lookup column name */
4758  attnum = InvalidAttrNumber;
4759  for (i = 0; i < tupDesc->natts; i++)
4760  {
4761  Form_pg_attribute att = TupleDescAttr(tupDesc, i);
4762 
4763  if (att->attisdropped)
4764  continue;
4765  if (namestrcmp(&(att->attname), name) == 0)
4766  {
4767  attnum = att->attnum;
4768  break;
4769  }
4770  }
4771  if (attnum == InvalidAttrNumber)
4772  {
4773  if (rel != NULL)
4774  ereport(ERROR,
4775  (errcode(ERRCODE_UNDEFINED_COLUMN),
4776  errmsg("column \"%s\" of relation \"%s\" does not exist",
4777  name, RelationGetRelationName(rel))));
4778  else
4779  ereport(ERROR,
4780  (errcode(ERRCODE_UNDEFINED_COLUMN),
4781  errmsg("column \"%s\" does not exist",
4782  name)));
4783  }
4784  /* Check for duplicates */
4785  if (list_member_int(attnums, attnum))
4786  ereport(ERROR,
4787  (errcode(ERRCODE_DUPLICATE_COLUMN),
4788  errmsg("column \"%s\" specified more than once",
4789  name)));
4790  attnums = lappend_int(attnums, attnum);
4791  }
4792  }
4793 
4794  return attnums;
4795 }
4796 
4797 
4798 /*
4799  * copy_dest_startup --- executor startup
4800  */
4801 static void
4802 copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
4803 {
4804  /* no-op */
4805 }
4806 
4807 /*
4808  * copy_dest_receive --- receive one tuple
4809  */
4810 static bool
4811 copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
4812 {
4813  DR_copy *myState = (DR_copy *) self;
4814  CopyState cstate = myState->cstate;
4815 
4816  /* Make sure the tuple is fully deconstructed */
4817  slot_getallattrs(slot);
4818 
4819  /* And send the data */
4820  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4821  myState->processed++;
4822 
4823  return true;
4824 }
4825 
4826 /*
4827  * copy_dest_shutdown --- executor end
4828  */
4829 static void
4830 copy_dest_shutdown(DestReceiver *self)
4831 {
4832  /* no-op */
4833 }
4834 
4835 /*
4836  * copy_dest_destroy --- release DestReceiver object
4837  */
4838 static void
4839 copy_dest_destroy(DestReceiver *self)
4840 {
4841  pfree(self);
4842 }
4843 
4844 /*
4845  * CreateCopyDestReceiver -- create a suitable DestReceiver object
4846  */
4847 DestReceiver *
4848 CreateCopyDestReceiver(void)
4849 {
4850  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4851 
4852  self->pub.receiveSlot = copy_dest_receive;
4853  self->pub.rStartup = copy_dest_startup;
4854  self->pub.rShutdown = copy_dest_shutdown;
4855  self->pub.rDestroy = copy_dest_destroy;
4856  self->pub.mydest = DestCopyOut;
4857 
4858  self->cstate = NULL; /* will be set later */
4859  self->processed = 0;
4860 
4861  return (DestReceiver *) self;
4862 }
int16
signed short int16
Definition: c.h:312
ResTarget::indirection
List * indirection
Definition: parsenodes.h:442
NextCopyFromRawFields
bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
Definition: copy.c:3244
ResultRelInfo::ri_NumIndices
int ri_NumIndices
Definition: execnodes.h:400
NIL
#define NIL
Definition: pg_list.h:69
CommandId
uint32 CommandId
Definition: c.h:488
ExecFindPartition
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
Definition: execPartition.c:189
CopyReadBinaryAttribute
static Datum CopyReadBinaryAttribute(CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
Definition: copy.c:4426
ExecStoreTuple
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:356
GetDecimalFromHex
static int GetDecimalFromHex(char hex)
Definition: copy.c:4001
FmgrInfo
Definition: fmgr.h:56
MAX_COPY_DATA_DISPLAY
#define MAX_COPY_DATA_DISPLAY
CopyStateData::csv_mode
bool csv_mode
Definition: copy.c:121
SendCopyEnd
static void SendCopyEnd(CopyState cstate)
Definition: copy.c:417
InitResultRelInfo
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1305
ResultRelInfo::ri_RelationDesc
Relation ri_RelationDesc
Definition: execnodes.h:397
CopyStateData::range_table
List * range_table
Definition: copy.c:169
BeginCopy
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1389
UpdateActiveSnapshotCommandId
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:781
IsA
#define IsA(nodeptr, _type_)
Definition: nodes.h:567
CopyReadLineText
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3642
MemoryContextDelete
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:211
HOLD_CANCEL_INTERRUPTS
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:123
contain_volatile_functions_not_nextval
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:1008
ResTarget::val
Node * val
Definition: parsenodes.h:443
CopyGetInt32
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:683
errhint
int errhint(const char *fmt,...)
Definition: elog.c:987
pg_char_to_encoding
int pg_char_to_encoding(const char *name)
Definition: encnames.c:551
CopyStateData::raw_fields
char ** raw_fields
Definition: copy.c:189
EOL_NL
Definition: copy.c:77
getTypeOutputInfo
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2650
VARDATA
#define VARDATA(PTR)
Definition: postgres.h:302
EndCopy
static void EndCopy(CopyState cstate)
Definition: copy.c:1731
CopyStateData::binary
bool binary
Definition: copy.c:118
pq_flush
#define pq_flush()
Definition: libpq.h:39
CopyFromErrorCallback
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2192
heap_endscan
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1572
TupConvMapForLeaf
TupleConversionMap * TupConvMapForLeaf(PartitionTupleRouting *proute, ResultRelInfo *rootRelInfo, int leaf_index)
Definition: execPartition.c:734
PreventCommandIfParallelMode
void PreventCommandIfParallelMode(const char *cmdname)
Definition: utility.c:257
ExecInsertIndexTuples
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
CopyStmt::attlist
List * attlist
Definition: parsenodes.h:1956
SelectStmt::fromClause
List * fromClause
Definition: parsenodes.h:1540
ISOCTAL
#define ISOCTAL(c)
Definition: copy.c:56
OCTVALUE
#define OCTVALUE(c)
Definition: copy.c:57
ResetPerTupleExprContext
#define ResetPerTupleExprContext(estate)
Definition: executor.h:497
RelationGetDescr
#define RelationGetDescr(relation)
Definition: rel.h:433
HEAP_INSERT_FROZEN
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
ResTarget::name
char * name
Definition: parsenodes.h:441
GetUserId
Oid GetUserId(void)
Definition: miscinit.c:379
CopyStateData::need_transcoding
bool need_transcoding
Definition: copy.c:108
castNode
#define castNode(_type_, nodeptr)
Definition: nodes.h:585
FreeQueryDesc
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
CopyStateData::in_functions
FmgrInfo * in_functions
Definition: copy.c:164
CopyStateData::num_defaults
AttrNumber num_defaults
Definition: copy.c:160
CopyStateData::attnumlist
List * attnumlist
Definition: copy.c:114
VARSIZE
#define VARSIZE(PTR)
Definition: postgres.h:303
pq_getmsgstring
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:581
CopyStateData::filename
char * filename
Definition: copy.c:115
FdwRoutine::BeginForeignInsert
BeginForeignInsert_function BeginForeignInsert
Definition: fdwapi.h:214
ModifyTableState::resultRelInfo
ResultRelInfo * resultRelInfo
Definition: execnodes.h:1043
TupleDescAttr
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:93
BeginCopyFrom
CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
Definition: copy.c:2985
VARHDRSZ
#define VARHDRSZ
Definition: c.h:522
FdwRoutine::ExecForeignInsert
ExecForeignInsert_function ExecForeignInsert
Definition: fdwapi.h:210
CopyStateData::file_has_oids
bool file_has_oids
Definition: copy.c:161
DatumGetObjectId
#define DatumGetObjectId(X)
Definition: postgres.h:485
PlannedStmt::relationOids
List * relationOids
Definition: plannodes.h:91
pstrdup
char * pstrdup(const char *in)
Definition: mcxt.c:1161
pg_hton16
#define pg_hton16(x)
Definition: pg_bswap.h:120
ExprContext::ecxt_per_tuple_memory
MemoryContext ecxt_per_tuple_memory
Definition: execnodes.h:226
ReceiveCopyBegin
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:381
XLogIsNeeded
#define XLogIsNeeded()
Definition: xlog.h:146
pg_ntoh16
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
DR_copy
Definition: copy.c:216
makeStringInfo
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
MAX_BUFFERED_TUPLES
#define MAX_BUFFERED_TUPLES
RelationData::rd_islocaltemp
bool rd_islocaltemp
Definition: rel.h:60
ExecIRInsertTriggers
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2596
S_IWOTH
#define S_IWOTH
Definition: win32_port.h:298
expression_planner
Expr * expression_planner(Expr *expr)
Definition: planner.c:5888
ExecutorStart
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:141
DR_copy::pub
DestReceiver pub
Definition: copy.c:218
ProcessCopyOptions
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1031
ExecConstraints
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1971
CopyStateData::line_buf
StringInfoData line_buf
Definition: copy.c:198
MemoryContextSwitchTo
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
IF_NEED_REFILL_AND_EOF_BREAK
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:252
GetActiveSnapshot
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
AccessShareLock
#define AccessShareLock
Definition: lockdefs.h:36
InvalidBuffer
#define InvalidBuffer
Definition: buf.h:25
CopyStateData
struct CopyStateData CopyStateData
Node
Definition: nodes.h:516
strVal
#define strVal(v)
Definition: value.h:54
cur
struct cursor * cur
Definition: ecpg.c:28
CopyStateData::raw_buf_index
int raw_buf_index
Definition: copy.c:211
CopyAttributeOutText
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4485
CopyStateData::line_buf_valid
bool line_buf_valid
Definition: copy.c:200
ThereAreNoPriorRegisteredSnapshots
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1655
DR_copy::cstate
CopyState cstate
Definition: copy.c:219
errcode
int errcode(int sqlerrcode)
Definition: elog.c:575
PG_BINARY_W
#define PG_BINARY_W
Definition: c.h:1083
namestrcmp
int namestrcmp(Name name, const char *str)
Definition: name.c:247
parse_relation.h
MemSet
#define MemSet(start, val, len)
Definition: c.h:908
generate_unaccent_rules.dest
dest
Definition: generate_unaccent_rules.py:207
CopyStateData::fe_eof
bool fe_eof
Definition: copy.c:105
CopyFrom
uint64 CopyFrom(CopyState cstate)
Definition: copy.c:2301
ModifyTableState::operation
CmdType operation
Definition: execnodes.h:1037
ExecSetupChildParentMapForLeaf
void ExecSetupChildParentMapForLeaf(PartitionTupleRouting *proute)
Definition: execPartition.c:712
RelationData::rd_newRelfilenodeSubid
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:81
pq_putemptymessage
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
ExecARInsertTriggers
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2581
heap_sync
void heap_sync(Relation rel)
Definition: heapam.c:9370
TupleTableSlot::tts_values
Datum * tts_values
Definition: tuptable.h:130
snprintf
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
FirstLowInvalidHeapAttributeNumber
#define FirstLowInvalidHeapAttributeNumber
Definition: sysattr.h:28
ClosePipeToProgram
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1708
HEAP_INSERT_SKIP_WAL
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
MemoryContextReset
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:136
CopyReadAttributesCSV
static int CopyReadAttributesCSV(CopyState cstate)
Definition: copy.c:4257
PopActiveSnapshot
void PopActiveSnapshot(void)
Definition: snapmgr.c:812
RangeTblEntry::requiredPerms
AclMode requiredPerms
Definition: parsenodes.h:1070
heap_form_tuple
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:1074
heap_close
#define heap_close(r, l)
Definition: heapam.h:97
DirectFunctionCall1
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:590
pq_sendbyte
static void pq_sendbyte(StringInfo buf, int8 byt)
Definition: pqformat.h:164
PlanState::state
EState * state
Definition: execnodes.h:914
PartitionTupleRouting::partition_tuple_slot
TupleTableSlot * partition_tuple_slot
Definition: execPartition.h:111
CopyStateData::force_quote_flags
bool * force_quote_flags
Definition: copy.c:131
EState::es_range_table
List * es_range_table
Definition: execnodes.h:480
RelationData::rd_rel
Form_pg_class rd_rel
Definition: rel.h:84
Oid
unsigned int Oid
Definition: postgres_ext.h:31
pg_analyze_and_rewrite
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:651
CopyStateData::delim
char * delim
Definition: copy.c:126
pg_type.h
PG_PROTOCOL_MAJOR
#define PG_PROTOCOL_MAJOR(v)
Definition: pqcomm.h:104
Query::utilityStmt
Node * utilityStmt
Definition: parsenodes.h:120
CopyStmt::is_program
bool is_program
Definition: parsenodes.h:1959
linitial_node
#define linitial_node(type, l)
Definition: pg_list.h:114
CopyStateData::volatile_defexprs
bool volatile_defexprs
Definition: copy.c:168
pg_wchar.h
oidout
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
ErrorContextCallback::callback
void(* callback)(void *arg)
Definition: elog.h:239
ErrorContextCallback::previous
struct ErrorContextCallback * previous
Definition: elog.h:238
PG_BINARY_R
#define PG_BINARY_R
Definition: c.h:1082
copy_dest_destroy
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4839
CopyStateData::force_null_flags
bool * force_null_flags
Definition: copy.c:135
wait_result_to_str
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
CopyStateData::rowcontext
MemoryContext rowcontext
Definition: copy.c:155
_DestReceiver
Definition: dest.h:115
tupleDesc::natts
int natts
Definition: tupdesc.h:82
CopyStateData::line_buf_converted
bool line_buf_converted
Definition: copy.c:199
TransitionCaptureState::tcs_original_insert_tuple
HeapTuple tcs_original_insert_tuple
Definition: trigger.h:82
PartitionTupleRouting::partitions
ResultRelInfo ** partitions
Definition: execPartition.h:104
pq_beginmessage
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
ExecInitExtraTupleSlot
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate, TupleDesc tupledesc)
Definition: execTuples.c:931
pg_server_to_any
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:634
int32
signed int int32
Definition: c.h:313
ClosePipeStream
int ClosePipeStream(FILE *file)
Definition: fd.c:2738
errdetail_internal
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
CopyStateData::convert_select_flags
bool * convert_select_flags
Definition: copy.c:138
CopySendChar
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:458
OutputFunctionCall
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1753
ExecOpenIndices
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
ColumnRef::location
int location
Definition: parsenodes.h:237
ExecSetupPartitionTupleRouting
PartitionTupleRouting * ExecSetupPartitionTupleRouting(ModifyTableState *mtstate, Relation rel)
Definition: execPartition.c:76
CopyStateData::copy_dest
CopyDest copy_dest
Definition: copy.c:101
ResTarget::location
int location
Definition: parsenodes.h:444
REFILL_LINEBUF
#define REFILL_LINEBUF
Definition: copy.c:269
HeapTupleSetOid
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:710
error_context_stack
ErrorContextCallback * error_context_stack
Definition: elog.c:88
CopyStateData::cur_lineno
uint64 cur_lineno
Definition: copy.c:142
list_make1
#define list_make1(x1)
Definition: pg_list.h:139
CopyStateData::null_print
char * null_print
Definition: copy.c:123
CopyStateData::cur_attname
const char * cur_attname
Definition: copy.c:143
ExecutorEnd
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:461
COPY_FILE
Definition: copy.c:65
TriggerDesc::trig_insert_instead_row
bool trig_insert_instead_row
Definition: reltrigger.h:57
FreeExecutorState
void FreeExecutorState(EState *estate)
Definition: execUtils.c:188
CopyStateData::rel
Relation rel
Definition: copy.c:112
GetPerTupleExprContext
#define GetPerTupleExprContext(estate)
Definition: executor.h:488
GetBulkInsertState
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2364
CopyStateData::copycontext
MemoryContext copycontext
Definition: copy.c:149
pq_startcopyout
#define pq_startcopyout()
Definition: libpq.h:46
CopyStateData::data_source_cb
copy_data_source_cb data_source_cb
Definition: copy.c:117
defGetBoolean
bool defGetBoolean(DefElem *def)
Definition: define.c:111
appendStringInfoCharMacro
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:127
TriggerDesc::trig_insert_new_table
bool trig_insert_new_table
Definition: reltrigger.h:74
RangeTblEntry::selectedCols
Bitmapset * selectedCols
Definition: parsenodes.h:1072
uint16
unsigned short uint16
Definition: c.h:324
pfree
void pfree(void *pointer)
Definition: mcxt.c:1031
pg_ntoh32
#define pg_ntoh32(x)
Definition: pg_bswap.h:125
IS_HIGHBIT_SET
#define IS_HIGHBIT_SET(ch)
Definition: c.h:994
BeginCopyTo
static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
Definition: copy.c:1754
CopySendInt16
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:700
ThereAreNoReadyPortals
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1198
ObjectIdGetDatum
#define ObjectIdGetDatum(X)
Definition: postgres.h:492
ERROR
#define ERROR
Definition: elog.h:43
attname
NameData attname
Definition: pg_attribute.h:40
ModifyTableState::ps
PlanState ps
Definition: execnodes.h:1036
pq_startmsgread
void pq_startmsgread(void)
Definition: pqcomm.c:1210
DatumGetCString
#define DatumGetCString(X)
Definition: postgres.h:551
CopyGetData
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:557
lfirst_int
#define lfirst_int(lc)
Definition: pg_list.h:107
ExecCleanupTupleRouting
void ExecCleanupTupleRouting(ModifyTableState *mtstate, PartitionTupleRouting *proute)
Definition: execPartition.c:804
ExecutorRun
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:299
pg_hton32
#define pg_hton32(x)
Definition: pg_bswap.h:121
CopyAttributeOutCSV
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4638
ReceiveFunctionCall
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1767
TransitionCaptureState::tcs_map
TupleConversionMap * tcs_map
Definition: trigger.h:73
FATAL
#define FATAL
Definition: elog.h:52
fmgr_info
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:124
defGetString
char * defGetString(DefElem *def)
Definition: define.c:49
CopyStmt::relation
RangeVar * relation
Definition: parsenodes.h:1953
HeapTupleData::t_self
ItemPointerData t_self
Definition: htup.h:65
PushCopiedSnapshot
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:769
CreateQueryDesc
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
pg_mbcliplen
int pg_mbcliplen(const char *mbstr, int len, int limit)
Definition: mbutils.c:820
RelationData::trigdesc
TriggerDesc * trigdesc
Definition: rel.h:90
ALLOCSET_DEFAULT_SIZES
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:192
CheckValidResultRel
void CheckValidResultRel(ResultRelInfo *resultRelInfo, CmdType operation)
Definition: execMain.c:1104
limit_printout_length
static char * limit_printout_length(const char *str)
Definition: copy.c:2272
lfirst_node
#define lfirst_node(type, lc)
Definition: pg_list.h:109
CopyStateData::queryDesc
QueryDesc * queryDesc
Definition: copy.c:113
EolType
EolType
Definition: copy.c:74
list_member_int
bool list_member_int(const List *list, int datum)
Definition: list.c:485
copy_dest_shutdown
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4830
CopyStateData::encoding_embeds_ascii
bool encoding_embeds_ascii
Definition: copy.c:109
HeapTupleData::t_len
uint32 t_len
Definition: htup.h:64
generate_unaccent_rules.str
str
Definition: generate_unaccent_rules.py:207
get_namespace_name
char * get_namespace_name(Oid nspid)
Definition: lsyscache.c:3051
c
char * c
Definition: preproc-cursor.c:31
ExecBSInsertTriggers
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2447
CopyStateData::raw_buf
char * raw_buf
Definition: copy.c:210
CopyLoadRawBuf
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:737
RawStmt::stmt
Node * stmt
Definition: parsenodes.h:1450
NoLock
#define NoLock
Definition: lockdefs.h:34
pq_getbytes
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1094
CopyStateData::quote
char * quote
Definition: copy.c:127
buf
static char * buf
Definition: pg_test_fsync.c:67
memmove
#define memmove(d, s, c)
Definition: c.h:1135
TupleTableSlot::tts_isnull
bool * tts_isnull
Definition: tuptable.h:132
ExecEvalExpr
static Datum ExecEvalExpr(ExprState *state, ExprContext *econtext, bool *isNull)
Definition: executor.h:280
pg_database_encoding_max_length
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
SelectStmt::targetList
List * targetList
Definition: parsenodes.h:1539
EState::es_result_relations
ResultRelInfo * es_result_relations
Definition: execnodes.h:490
DoCopyTo
static uint64 DoCopyTo(CopyState cstate)
Definition: copy.c:1892
CreateDestReceiver
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
copy_dest_startup
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4802
DefElem::location
int location
Definition: parsenodes.h:733
RowExclusiveLock
#define RowExclusiveLock
Definition: lockdefs.h:38
CopyStateData::defexprs
ExprState ** defexprs
Definition: copy.c:167
CopyStateData::cur_relname
const char * cur_relname
Definition: copy.c:141
errcode_for_file_access
int errcode_for_file_access(void)
Definition: elog.c:598
pg_encoding_mblen
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
is_absolute_path
#define is_absolute_path(filename)
Definition: port.h:86
CStringGetDatum
#define CStringGetDatum(X)
Definition: postgres.h:563
string
char string[11]
Definition: preproc-type.c:46
EOL_CR
Definition: copy.c:78
CopyStmt::options
List * options
Definition: parsenodes.h:1961
AllocateFile
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2336
CopyStateData::oid_in_function
FmgrInfo oid_in_function
Definition: copy.c:162
select
#define select(n, r, w, e, timeout)
Definition: win32_port.h:447
enlargeStringInfo
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:264
RelationGetRelationName
#define RelationGetRelationName(relation)
Definition: rel.h:441
BinarySignature
static const char BinarySignature[11]
Definition: copy.c:289
S_IWGRP
#define S_IWGRP
Definition: win32_port.h:286
Form_pg_attribute
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:197
resetStringInfo
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
ResultRelInfo::ri_FdwRoutine
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:421
uint32
unsigned int uint32
Definition: c.h:325
SendFunctionCall
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1814
CopyStateData::raw_buf_len
int raw_buf_len
Definition: copy.c:212
HeapTupleData::t_tableOid
Oid t_tableOid
Definition: htup.h:66
PartitionTupleRouting::partition_dispatch_info
PartitionDispatch * partition_dispatch_info
Definition: execPartition.h:101
TriggerDesc::trig_insert_after_row
bool trig_insert_after_row
Definition: reltrigger.h:56
CurrentMemoryContext
MemoryContext CurrentMemoryContext
Definition: mcxt.c:38
CopyStateData::max_fields
int max_fields
Definition: copy.c:188
CopyStateData::escape
char * escape
Definition: copy.c:128
ParseState::p_sourcetext
const char * p_sourcetext
Definition: parse_node.h:173
Query::returningList
List * returningList
Definition: parsenodes.h:146
EState::es_trig_tuple_slot
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:512
OpenPipeStream
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2435
getTypeBinaryInputInfo
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2683
htup_details.h
getTypeInputInfo
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2617
ereport
#define ereport(elevel, rest)
Definition: elog.h:122
heap_insert
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2441
CopyStateData::null_print_len
int null_print_len
Definition: copy.c:124
A_Star
Definition: parsenodes.h:369
slot_getallattrs
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1612
InvalidateCatalogSnapshot
void InvalidateCatalogSnapshot(void)
Definition: snapmgr.c:510
CopyStateData::force_null
List * force_null
Definition: copy.c:134
ResultRelInfo::ri_TrigDesc
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:409
ExecutorFinish
void ExecutorFinish(QueryDesc *queryDesc)
Definition: execMain.c:401
CreateExecutorState
EState * CreateExecutorState(void)
Definition: execUtils.c:80
lappend_int
List * lappend_int(List *list, int datum)
Definition: list.c:146
DefElem::arg
Node * arg
Definition: parsenodes.h:731
EOL_CRNL
Definition: copy.c:79
lappend
List * lappend(List *list, void *datum)
Definition: list.c:128
CopyStateData::file_encoding
int file_encoding
Definition: copy.c:107
CopyGetAttnums
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4729
AllocSetContextCreate
#define AllocSetContextCreate(parent, name, allocparams)
Definition: memutils.h:170
initStringInfo
void initStringInfo(StringInfo str)
Definition: stringinfo.c:46
QueryDesc::tupDesc
TupleDesc tupDesc
Definition: execdesc.h:47
pq_getmessage
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1272
stat
#define stat(a, b)
Definition: win32_port.h:266
InvalidSnapshot
#define InvalidSnapshot
Definition: snapshot.h:25
RelationData::rd_createSubid
SubTransactionId rd_createSubid
Definition: rel.h:80
CopyStateData::typioparams
Oid * typioparams
Definition: copy.c:165
CopyStateData::is_program
bool is_program
Definition: copy.c:116
build_column_default
Node * build_column_default(Relation rel, int attrno)
Definition: rewriteHandler.c:1106
TriggerDesc::trig_insert_before_row
bool trig_insert_before_row
Definition: reltrigger.h:55
getTypeBinaryOutputInfo
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2716
EState::es_tupleTable
List * es_tupleTable
Definition: execnodes.h:525
ExecResetTupleTable
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:186
palloc0
void * palloc0(Size size)
Definition: mcxt.c:955
CopyStateData::header_line
bool header_line
Definition: copy.c:122
ExecASInsertTriggers
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo, TransitionCaptureState *transition_capture)
Definition: trigger.c:2504
Datum
uintptr_t Datum
Definition: postgres.h:367
GetDatabaseEncoding
int GetDatabaseEncoding(void)
Definition: mbutils.c:1004
pg_get_client_encoding
int pg_get_client_encoding(void)
Definition: mbutils.c:306
ACL_SELECT
#define ACL_SELECT
Definition: parsenodes.h:75
ConvertPartitionTupleSlot
HeapTuple ConvertPartitionTupleSlot(TupleConversionMap *map, HeapTuple tuple, TupleTableSlot *new_slot, TupleTableSlot **p_my_slot)
Definition: execPartition.c:776
MakeTransitionCaptureState
TransitionCaptureState * MakeTransitionCaptureState(TriggerDesc *trigdesc, Oid relid, CmdType cmdType)
Definition: trigger.c:4644
heap_getnext
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1835
RawStmt::stmt_len
int stmt_len
Definition: parsenodes.h:1452
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:240
RawStmt::stmt_location
int stmt_location
Definition: parsenodes.h:1451
EState::es_num_result_relations
int es_num_result_relations
Definition: execnodes.h:491
ResultRelInfo::ri_PartitionCheck
List * ri_PartitionCheck
Definition: execnodes.h:454
value
static struct @131 value
RAW_BUF_SIZE
#define RAW_BUF_SIZE
Definition: copy.c:209
RelationData::rd_att
TupleDesc rd_att
Definition: rel.h:85
CopyStateData::freeze
bool freeze
Definition: copy.c:120
InputFunctionCall
Datum InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod)
Definition: fmgr.c:1709
pq_getbyte
int pq_getbyte(void)
Definition: pqcomm.c:1000
PlanState::plan
Plan * plan
Definition: execnodes.h:912
pq_endmsgread
void pq_endmsgread(void)
Definition: pqcomm.c:1234
heap_openrv
Relation heap_openrv(const RangeVar *relation, LOCKMODE lockmode)
Definition: heapam.c:1323
InvalidOid
#define InvalidOid
Definition: postgres_ext.h:36
copy_dest_receive
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4811
attnum
int16 attnum
Definition: pg_attribute.h:79
XactReadOnly
bool XactReadOnly
Definition: xact.c:76
CopyStateData::force_notnull_flags
bool * force_notnull_flags
Definition: copy.c:133
Query::commandType
CmdType commandType
Definition: parsenodes.h:112
CopyStateData::partition_tuple_routing
PartitionTupleRouting * partition_tuple_routing
Definition: copy.c:172
is_member_of_role
bool is_member_of_role(Oid member, Oid role)
Definition: acl.c:4857
AfterTriggerBeginQuery
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4766
CopyStateData::force_notnull
List * force_notnull
Definition: copy.c:132
check_enable_rls
int check_enable_rls(Oid relid, Oid checkAsUser, bool noError)
Definition: rls.c:52
errmsg_internal
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
EndCopyFrom
void EndCopyFrom(CopyState cstate)
Definition: copy.c:3537