PostgreSQL Source Code  git master
copyto.c File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/tableam.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "commands/copy.h"
#include "commands/progress.h"
#include "executor/execdesc.h"
#include "executor/executor.h"
#include "executor/tuptable.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "optimizer/optimizer.h"
#include "pgstat.h"
#include "rewrite/rewriteHandler.h"
#include "storage/fd.h"
#include "tcop/tcopprot.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/partcache.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
Include dependency graph for copyto.c:

Go to the source code of this file.

Data Structures

struct  CopyToStateData
 
struct  DR_copy
 

Macros

#define DUMPSOFAR()
 

Typedefs

typedef enum CopyDest CopyDest
 
typedef struct CopyToStateData CopyToStateData
 

Enumerations

enum  CopyDest { COPY_FILE, COPY_FRONTEND }
 

Functions

static void EndCopy (CopyToState cstate)
 
static void ClosePipeToProgram (CopyToState cstate)
 
static void CopyOneRowTo (CopyToState cstate, TupleTableSlot *slot)
 
static void CopyAttributeOutText (CopyToState cstate, char *string)
 
static void CopyAttributeOutCSV (CopyToState cstate, char *string, bool use_quote, bool single_attr)
 
static void SendCopyBegin (CopyToState cstate)
 
static void SendCopyEnd (CopyToState cstate)
 
static void CopySendData (CopyToState cstate, const void *databuf, int datasize)
 
static void CopySendString (CopyToState cstate, const char *str)
 
static void CopySendChar (CopyToState cstate, char c)
 
static void CopySendEndOfRow (CopyToState cstate)
 
static void CopySendInt32 (CopyToState cstate, int32 val)
 
static void CopySendInt16 (CopyToState cstate, int16 val)
 
CopyToState BeginCopyTo (ParseState *pstate, Relation rel, RawStmt *raw_query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
 
void EndCopyTo (CopyToState cstate)
 
uint64 DoCopyTo (CopyToState cstate)
 
static void copy_dest_startup (DestReceiver *self, int operation, TupleDesc typeinfo)
 
static bool copy_dest_receive (TupleTableSlot *slot, DestReceiver *self)
 
static void copy_dest_shutdown (DestReceiver *self)
 
static void copy_dest_destroy (DestReceiver *self)
 
DestReceiverCreateCopyDestReceiver (void)
 

Variables

static const char BinarySignature [11] = "PGCOPY\n\377\r\n\0"
 

Macro Definition Documentation

◆ DUMPSOFAR

#define DUMPSOFAR ( )
Value:
do { \
if (ptr > start) \
CopySendData(cstate, start, ptr - start); \
} while (0)

Definition at line 1011 of file copyto.c.

Referenced by CopyAttributeOutCSV(), and CopyAttributeOutText().

Typedef Documentation

◆ CopyDest

typedef enum CopyDest CopyDest

◆ CopyToStateData

Enumeration Type Documentation

◆ CopyDest

enum CopyDest
Enumerator
COPY_FILE 
COPY_FRONTEND 

Definition at line 50 of file copyto.c.

51 {
52  COPY_FILE, /* to file (or a piped program) */
53  COPY_FRONTEND, /* to frontend */
54 } CopyDest;
CopyDest
Definition: copyto.c:50

Function Documentation

◆ BeginCopyTo()

CopyToState BeginCopyTo ( ParseState pstate,
Relation  rel,
RawStmt raw_query,
Oid  queryRelId,
const char *  filename,
bool  is_program,
List attnamelist,
List options 
)

Definition at line 342 of file copyto.c.

References AllocateFile(), ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, Assert, attnum, CopyToStateData::attnumlist, CopyToStateData::bytes_processed, CMD_DELETE, CMD_INSERT, CMD_SELECT, CMD_UPDATE, Query::commandType, CopyToStateData::copy_dest, COPY_FILE, CopyToStateData::copy_file, CopyToStateData::copycontext, CopyGetAttnums(), copyObject, CreateDestReceiver(), CreateQueryDesc(), cur, CurrentMemoryContext, CURSOR_OPT_PARALLEL_OK, generate_unaccent_rules::dest, DestCopyOut, DestRemote, CopyToStateData::encoding_embeds_ascii, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, ExecutorStart(), CopyFormatOptions::file_encoding, CopyToStateData::file_encoding, CopyToStateData::filename, CopyFormatOptions::force_notnull, CopyFormatOptions::force_notnull_flags, CopyFormatOptions::force_null, CopyFormatOptions::force_null_flags, CopyFormatOptions::force_quote, CopyFormatOptions::force_quote_all, CopyFormatOptions::force_quote_flags, fstat, GetActiveSnapshot(), GetDatabaseEncoding(), i, InvalidOid, InvalidSnapshot, is_absolute_path, CopyToStateData::is_program, IsA, lfirst_int, lfirst_node, linitial_node, list_length(), list_member_int(), list_member_oid(), MemoryContextSwitchTo(), NameStr, TupleDescData::natts, CopyToStateData::need_transcoding, NIL, OpenPipeStream(), CopyToStateData::opts, ParseState::p_sourcetext, palloc0(), pg_analyze_and_rewrite(), PG_BINARY_W, pg_database_encoding_max_length(), PG_ENCODING_IS_CLIENT_ONLY, PG_END_TRY, PG_FINALLY, pg_get_client_encoding(), pg_plan_query(), PG_TRY, pgstat_progress_start_command(), pgstat_progress_update_multi_param(), ProcessCopyOptions(), PROGRESS_COMMAND_COPY, PROGRESS_COPY_COMMAND, PROGRESS_COPY_COMMAND_TO, PROGRESS_COPY_TYPE, PROGRESS_COPY_TYPE_FILE, PROGRESS_COPY_TYPE_PIPE, PROGRESS_COPY_TYPE_PROGRAM, pstrdup(), PushCopiedSnapshot(), QSRC_NON_INSTEAD_RULE, QSRC_QUAL_INSTEAD_RULE, CopyToStateData::queryDesc, Query::querySource, RelationData::rd_rel, CopyToStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, PlannedStmt::relationOids, Query::returningList, S_ISDIR, S_IWGRP, S_IWOTH, stat::st_mode, generate_unaccent_rules::stdout, QueryDesc::tupDesc, TupleDescAttr, UpdateActiveSnapshotCommandId(), Query::utilityStmt, and whereToSendOutput.

Referenced by DoCopy().

350 {
351  CopyToState cstate;
352  bool pipe = (filename == NULL);
353  TupleDesc tupDesc;
354  int num_phys_attrs;
355  MemoryContext oldcontext;
356  const int progress_cols[] = {
359  };
360  int64 progress_vals[] = {
362  0
363  };
364 
365  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
366  {
367  if (rel->rd_rel->relkind == RELKIND_VIEW)
368  ereport(ERROR,
369  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
370  errmsg("cannot copy from view \"%s\"",
372  errhint("Try the COPY (SELECT ...) TO variant.")));
373  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
374  ereport(ERROR,
375  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
376  errmsg("cannot copy from materialized view \"%s\"",
378  errhint("Try the COPY (SELECT ...) TO variant.")));
379  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
380  ereport(ERROR,
381  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
382  errmsg("cannot copy from foreign table \"%s\"",
384  errhint("Try the COPY (SELECT ...) TO variant.")));
385  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
386  ereport(ERROR,
387  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
388  errmsg("cannot copy from sequence \"%s\"",
389  RelationGetRelationName(rel))));
390  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
391  ereport(ERROR,
392  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
393  errmsg("cannot copy from partitioned table \"%s\"",
395  errhint("Try the COPY (SELECT ...) TO variant.")));
396  else
397  ereport(ERROR,
398  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
399  errmsg("cannot copy from non-table relation \"%s\"",
400  RelationGetRelationName(rel))));
401  }
402 
403 
404  /* Allocate workspace and zero all fields */
405  cstate = (CopyToStateData *) palloc0(sizeof(CopyToStateData));
406 
407  /*
408  * We allocate everything used by a cstate in a new memory context. This
409  * avoids memory leaks during repeated use of COPY in a query.
410  */
412  "COPY",
414 
415  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
416 
417  /* Extract options from the statement node tree */
418  ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */, options);
419 
420  /* Process the source/target relation or query */
421  if (rel)
422  {
423  Assert(!raw_query);
424 
425  cstate->rel = rel;
426 
427  tupDesc = RelationGetDescr(cstate->rel);
428  }
429  else
430  {
431  List *rewritten;
432  Query *query;
433  PlannedStmt *plan;
435 
436  cstate->rel = NULL;
437 
438  /*
439  * Run parse analysis and rewrite. Note this also acquires sufficient
440  * locks on the source table(s).
441  *
442  * Because the parser and planner tend to scribble on their input, we
443  * make a preliminary copy of the source querytree. This prevents
444  * problems in the case that the COPY is in a portal or plpgsql
445  * function and is executed repeatedly. (See also the same hack in
446  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
447  */
448  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
449  pstate->p_sourcetext, NULL, 0,
450  NULL);
451 
452  /* check that we got back something we can work with */
453  if (rewritten == NIL)
454  {
455  ereport(ERROR,
456  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
457  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
458  }
459  else if (list_length(rewritten) > 1)
460  {
461  ListCell *lc;
462 
463  /* examine queries to determine which error message to issue */
464  foreach(lc, rewritten)
465  {
466  Query *q = lfirst_node(Query, lc);
467 
469  ereport(ERROR,
470  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
471  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
473  ereport(ERROR,
474  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
475  errmsg("DO ALSO rules are not supported for the COPY")));
476  }
477 
478  ereport(ERROR,
479  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
480  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
481  }
482 
483  query = linitial_node(Query, rewritten);
484 
485  /* The grammar allows SELECT INTO, but we don't support that */
486  if (query->utilityStmt != NULL &&
488  ereport(ERROR,
489  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
490  errmsg("COPY (SELECT INTO) is not supported")));
491 
492  Assert(query->utilityStmt == NULL);
493 
494  /*
495  * Similarly the grammar doesn't enforce the presence of a RETURNING
496  * clause, but this is required here.
497  */
498  if (query->commandType != CMD_SELECT &&
499  query->returningList == NIL)
500  {
501  Assert(query->commandType == CMD_INSERT ||
502  query->commandType == CMD_UPDATE ||
503  query->commandType == CMD_DELETE);
504 
505  ereport(ERROR,
506  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
507  errmsg("COPY query must have a RETURNING clause")));
508  }
509 
510  /* plan the query */
511  plan = pg_plan_query(query, pstate->p_sourcetext,
512  CURSOR_OPT_PARALLEL_OK, NULL);
513 
514  /*
515  * With row level security and a user using "COPY relation TO", we
516  * have to convert the "COPY relation TO" to a query-based COPY (eg:
517  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
518  * in any RLS clauses.
519  *
520  * When this happens, we are passed in the relid of the originally
521  * found relation (which we have locked). As the planner will look up
522  * the relation again, we double-check here to make sure it found the
523  * same one that we have locked.
524  */
525  if (queryRelId != InvalidOid)
526  {
527  /*
528  * Note that with RLS involved there may be multiple relations,
529  * and while the one we need is almost certainly first, we don't
530  * make any guarantees of that in the planner, so check the whole
531  * list and make sure we find the original relation.
532  */
533  if (!list_member_oid(plan->relationOids, queryRelId))
534  ereport(ERROR,
535  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
536  errmsg("relation referenced by COPY statement has changed")));
537  }
538 
539  /*
540  * Use a snapshot with an updated command ID to ensure this query sees
541  * results of any previously executed queries.
542  */
545 
546  /* Create dest receiver for COPY OUT */
548  ((DR_copy *) dest)->cstate = cstate;
549 
550  /* Create a QueryDesc requesting no output */
551  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
554  dest, NULL, NULL, 0);
555 
556  /*
557  * Call ExecutorStart to prepare the plan for execution.
558  *
559  * ExecutorStart computes a result tupdesc for us
560  */
561  ExecutorStart(cstate->queryDesc, 0);
562 
563  tupDesc = cstate->queryDesc->tupDesc;
564  }
565 
566  /* Generate or convert list of attributes to process */
567  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
568 
569  num_phys_attrs = tupDesc->natts;
570 
571  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
572  cstate->opts.force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
573  if (cstate->opts.force_quote_all)
574  {
575  int i;
576 
577  for (i = 0; i < num_phys_attrs; i++)
578  cstate->opts.force_quote_flags[i] = true;
579  }
580  else if (cstate->opts.force_quote)
581  {
582  List *attnums;
583  ListCell *cur;
584 
585  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.force_quote);
586 
587  foreach(cur, attnums)
588  {
589  int attnum = lfirst_int(cur);
590  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
591 
592  if (!list_member_int(cstate->attnumlist, attnum))
593  ereport(ERROR,
594  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
595  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
596  NameStr(attr->attname))));
597  cstate->opts.force_quote_flags[attnum - 1] = true;
598  }
599  }
600 
601  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
602  cstate->opts.force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
603  if (cstate->opts.force_notnull)
604  {
605  List *attnums;
606  ListCell *cur;
607 
608  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.force_notnull);
609 
610  foreach(cur, attnums)
611  {
612  int attnum = lfirst_int(cur);
613  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
614 
615  if (!list_member_int(cstate->attnumlist, attnum))
616  ereport(ERROR,
617  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
618  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
619  NameStr(attr->attname))));
620  cstate->opts.force_notnull_flags[attnum - 1] = true;
621  }
622  }
623 
624  /* Convert FORCE_NULL name list to per-column flags, check validity */
625  cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
626  if (cstate->opts.force_null)
627  {
628  List *attnums;
629  ListCell *cur;
630 
631  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->opts.force_null);
632 
633  foreach(cur, attnums)
634  {
635  int attnum = lfirst_int(cur);
636  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
637 
638  if (!list_member_int(cstate->attnumlist, attnum))
639  ereport(ERROR,
640  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
641  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
642  NameStr(attr->attname))));
643  cstate->opts.force_null_flags[attnum - 1] = true;
644  }
645  }
646 
647  /* Use client encoding when ENCODING option is not specified. */
648  if (cstate->opts.file_encoding < 0)
650  else
651  cstate->file_encoding = cstate->opts.file_encoding;
652 
653  /*
654  * Set up encoding conversion info. Even if the file and server encodings
655  * are the same, we must apply pg_any_to_server() to validate data in
656  * multibyte encodings.
657  */
658  cstate->need_transcoding =
659  (cstate->file_encoding != GetDatabaseEncoding() ||
661  /* See Multibyte encoding comment above */
663 
664  cstate->copy_dest = COPY_FILE; /* default */
665 
666  MemoryContextSwitchTo(oldcontext);
667 
668  if (pipe)
669  {
670  progress_vals[1] = PROGRESS_COPY_TYPE_PIPE;
671 
672  Assert(!is_program); /* the grammar does not allow this */
674  cstate->copy_file = stdout;
675  }
676  else
677  {
678  cstate->filename = pstrdup(filename);
679  cstate->is_program = is_program;
680 
681  if (is_program)
682  {
683  progress_vals[1] = PROGRESS_COPY_TYPE_PROGRAM;
684  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
685  if (cstate->copy_file == NULL)
686  ereport(ERROR,
688  errmsg("could not execute command \"%s\": %m",
689  cstate->filename)));
690  }
691  else
692  {
693  mode_t oumask; /* Pre-existing umask value */
694  struct stat st;
695 
696  progress_vals[1] = PROGRESS_COPY_TYPE_FILE;
697 
698  /*
699  * Prevent write to relative path ... too easy to shoot oneself in
700  * the foot by overwriting a database file ...
701  */
703  ereport(ERROR,
704  (errcode(ERRCODE_INVALID_NAME),
705  errmsg("relative path not allowed for COPY to file")));
706 
707  oumask = umask(S_IWGRP | S_IWOTH);
708  PG_TRY();
709  {
710  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
711  }
712  PG_FINALLY();
713  {
714  umask(oumask);
715  }
716  PG_END_TRY();
717  if (cstate->copy_file == NULL)
718  {
719  /* copy errno because ereport subfunctions might change it */
720  int save_errno = errno;
721 
722  ereport(ERROR,
724  errmsg("could not open file \"%s\" for writing: %m",
725  cstate->filename),
726  (save_errno == ENOENT || save_errno == EACCES) ?
727  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
728  "You may want a client-side facility such as psql's \\copy.") : 0));
729  }
730 
731  if (fstat(fileno(cstate->copy_file), &st))
732  ereport(ERROR,
734  errmsg("could not stat file \"%s\": %m",
735  cstate->filename)));
736 
737  if (S_ISDIR(st.st_mode))
738  ereport(ERROR,
739  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
740  errmsg("\"%s\" is a directory", cstate->filename)));
741  }
742  }
743 
744  /* initialize progress */
746  cstate->rel ? RelationGetRelid(cstate->rel) : InvalidOid);
747  pgstat_progress_update_multi_param(2, progress_cols, progress_vals);
748 
749  cstate->bytes_processed = 0;
750 
751  MemoryContextSwitchTo(oldcontext);
752 
753  return cstate;
754 }
int file_encoding
Definition: copyto.c:78
#define NIL
Definition: pg_list.h:65
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:728
#define IsA(nodeptr, _type_)
Definition: nodes.h:590
#define PROGRESS_COPY_COMMAND_TO
Definition: progress.h:146
#define AllocSetContextCreate
Definition: memutils.h:173
#define PROGRESS_COPY_TYPE_FILE
Definition: progress.h:149
int errhint(const char *fmt,...)
Definition: elog.c:1156
Relation rel
Definition: copyto.c:83
bool need_transcoding
Definition: copyto.c:79
FILE * copy_file
Definition: copyto.c:75
#define RelationGetDescr(relation)
Definition: rel.h:483
#define PROGRESS_COPY_COMMAND
Definition: progress.h:141
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
List * relationOids
Definition: plannodes.h:80
char * pstrdup(const char *in)
Definition: mcxt.c:1299
bool * force_quote_flags
Definition: copy.h:44
#define S_IWOTH
Definition: win32_port.h:307
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:130
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:786
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:698
#define PG_BINARY_W
Definition: c.h:1274
uint64 bytes_processed
Definition: copyto.c:99
List * force_notnull
Definition: copy.h:45
bool force_quote_all
Definition: copy.h:43
Form_pg_class rd_rel
Definition: rel.h:110
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:624
Node * utilityStmt
Definition: parsenodes.h:128
#define linitial_node(type, l)
Definition: pg_list.h:177
#define PROGRESS_COPY_TYPE_PIPE
Definition: progress.h:151
bool is_program
Definition: copyto.c:87
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
QueryDesc * queryDesc
Definition: copyto.c:84
#define fstat
Definition: win32_port.h:274
List * attnumlist
Definition: copyto.c:85
#define ERROR
Definition: elog.h:46
#define lfirst_int(lc)
Definition: pg_list.h:170
MemoryContext copycontext
Definition: copyto.c:95
#define PROGRESS_COPY_TYPE
Definition: progress.h:142
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:716
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
List * force_null
Definition: copy.h:47
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:195
#define lfirst_node(type, lc)
Definition: pg_list.h:172
bool list_member_int(const List *list, int datum)
Definition: list.c:669
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:113
int errcode_for_file_access(void)
Definition: elog.c:721
#define is_absolute_path(filename)
Definition: port.h:86
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2373
#define RelationGetRelationName(relation)
Definition: rel.h:491
#define S_IWGRP
Definition: win32_port.h:295
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:203
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
const char * p_sourcetext
Definition: parse_node.h:181
int file_encoding
Definition: copy.h:30
List * returningList
Definition: parsenodes.h:156
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2476
CopyDest copy_dest
Definition: copyto.c:74
TupleDesc tupDesc
Definition: execdesc.h:47
#define InvalidSnapshot
Definition: snapshot.h:123
#define PG_FINALLY()
Definition: elog.h:330
void * palloc0(Size size)
Definition: mcxt.c:1093
List * force_quote
Definition: copy.h:42
#define PROGRESS_COPY_TYPE_PROGRAM
Definition: progress.h:150
int GetDatabaseEncoding(void)
Definition: mbutils.c:1210
int pg_get_client_encoding(void)
Definition: mbutils.c:336
#define InvalidOid
Definition: postgres_ext.h:36
int16 attnum
Definition: pg_attribute.h:83
#define ereport(elevel,...)
Definition: elog.h:157
CmdType commandType
Definition: parsenodes.h:120
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:689
QuerySource querySource
Definition: parsenodes.h:122
#define Assert(condition)
Definition: c.h:804
bool * force_null_flags
Definition: copy.h:48
List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:686
static int list_length(const List *l)
Definition: pg_list.h:149
int pg_database_encoding_max_length(void)
Definition: mbutils.c:1495
CopyFormatOptions opts
Definition: copyto.c:89
#define S_ISDIR(m)
Definition: win32_port.h:316
#define PG_ENCODING_IS_CLIENT_ONLY(_enc)
Definition: pg_wchar.h:298
static char * filename
Definition: pg_dumpall.c:91
#define CURSOR_OPT_PARALLEL_OK
Definition: parsenodes.h:2795
int errmsg(const char *fmt,...)
Definition: elog.c:909
int i
#define NameStr(name)
Definition: c.h:681
PlannedStmt * pg_plan_query(Query *querytree, const char *query_string, int cursorOptions, ParamListInfo boundParams)
Definition: postgres.c:811
bool encoding_embeds_ascii
Definition: copyto.c:80
#define copyObject(obj)
Definition: nodes.h:655
CommandDest whereToSendOutput
Definition: postgres.c:92
char * filename
Definition: copyto.c:86
#define PG_TRY()
Definition: elog.h:313
Definition: pg_list.h:50
#define RelationGetRelid(relation)
Definition: rel.h:457
#define PG_END_TRY()
Definition: elog.h:338
void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options)
Definition: copy.c:334
bool * force_notnull_flags
Definition: copy.h:46

◆ ClosePipeToProgram()

static void ClosePipeToProgram ( CopyToState  cstate)
static

Definition at line 292 of file copyto.c.

References Assert, ClosePipeStream(), CopyToStateData::copy_file, ereport, errcode(), errcode_for_file_access(), errdetail_internal(), errmsg(), ERROR, CopyToStateData::filename, CopyToStateData::is_program, and wait_result_to_str().

Referenced by CopySendEndOfRow(), and EndCopy().

293 {
294  int pclose_rc;
295 
296  Assert(cstate->is_program);
297 
298  pclose_rc = ClosePipeStream(cstate->copy_file);
299  if (pclose_rc == -1)
300  ereport(ERROR,
302  errmsg("could not close pipe to external command: %m")));
303  else if (pclose_rc != 0)
304  {
305  ereport(ERROR,
306  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
307  errmsg("program \"%s\" failed",
308  cstate->filename),
309  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
310  }
311 }
FILE * copy_file
Definition: copyto.c:75
int errcode(int sqlerrcode)
Definition: elog.c:698
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
bool is_program
Definition: copyto.c:87
int ClosePipeStream(FILE *file)
Definition: fd.c:2782
int errdetail_internal(const char *fmt,...)
Definition: elog.c:1069
#define ERROR
Definition: elog.h:46
int errcode_for_file_access(void)
Definition: elog.c:721
#define ereport(elevel,...)
Definition: elog.h:157
#define Assert(condition)
Definition: c.h:804
int errmsg(const char *fmt,...)
Definition: elog.c:909
char * filename
Definition: copyto.c:86

◆ copy_dest_destroy()

static void copy_dest_destroy ( DestReceiver self)
static

Definition at line 1293 of file copyto.c.

References pfree().

Referenced by CreateCopyDestReceiver().

1294 {
1295  pfree(self);
1296 }
void pfree(void *pointer)
Definition: mcxt.c:1169

◆ copy_dest_receive()

static bool copy_dest_receive ( TupleTableSlot slot,
DestReceiver self 
)
static

Definition at line 1265 of file copyto.c.

References CopyOneRowTo(), DR_copy::cstate, pgstat_progress_update_param(), DR_copy::processed, and PROGRESS_COPY_TUPLES_PROCESSED.

Referenced by CreateCopyDestReceiver().

1266 {
1267  DR_copy *myState = (DR_copy *) self;
1268  CopyToState cstate = myState->cstate;
1269 
1270  /* Send the data */
1271  CopyOneRowTo(cstate, slot);
1272 
1273  /* Increment the number of processed tuples, and report the progress */
1275  ++myState->processed);
1276 
1277  return true;
1278 }
static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
Definition: copyto.c:937
#define PROGRESS_COPY_TUPLES_PROCESSED
Definition: progress.h:139
CopyToState cstate
Definition: copyto.c:107
void pgstat_progress_update_param(int index, int64 val)
uint64 processed
Definition: copyto.c:108

◆ copy_dest_shutdown()

static void copy_dest_shutdown ( DestReceiver self)
static

Definition at line 1284 of file copyto.c.

Referenced by CreateCopyDestReceiver().

1285 {
1286  /* no-op */
1287 }

◆ copy_dest_startup()

static void copy_dest_startup ( DestReceiver self,
int  operation,
TupleDesc  typeinfo 
)
static

Definition at line 1256 of file copyto.c.

Referenced by CreateCopyDestReceiver().

1257 {
1258  /* no-op */
1259 }

◆ CopyAttributeOutCSV()

static void CopyAttributeOutCSV ( CopyToState  cstate,
char *  string,
bool  use_quote,
bool  single_attr 
)
static

Definition at line 1171 of file copyto.c.

References CopySendChar(), CopySendString(), CopyFormatOptions::delim, DUMPSOFAR, CopyToStateData::encoding_embeds_ascii, CopyFormatOptions::escape, CopyToStateData::file_encoding, IS_HIGHBIT_SET, CopyToStateData::need_transcoding, CopyFormatOptions::null_print, CopyToStateData::opts, pg_encoding_mblen(), pg_server_to_any(), and CopyFormatOptions::quote.

Referenced by CopyOneRowTo(), and DoCopyTo().

1173 {
1174  char *ptr;
1175  char *start;
1176  char c;
1177  char delimc = cstate->opts.delim[0];
1178  char quotec = cstate->opts.quote[0];
1179  char escapec = cstate->opts.escape[0];
1180 
1181  /* force quoting if it matches null_print (before conversion!) */
1182  if (!use_quote && strcmp(string, cstate->opts.null_print) == 0)
1183  use_quote = true;
1184 
1185  if (cstate->need_transcoding)
1186  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
1187  else
1188  ptr = string;
1189 
1190  /*
1191  * Make a preliminary pass to discover if it needs quoting
1192  */
1193  if (!use_quote)
1194  {
1195  /*
1196  * Because '\.' can be a data value, quote it if it appears alone on a
1197  * line so it is not interpreted as the end-of-data marker.
1198  */
1199  if (single_attr && strcmp(ptr, "\\.") == 0)
1200  use_quote = true;
1201  else
1202  {
1203  char *tptr = ptr;
1204 
1205  while ((c = *tptr) != '\0')
1206  {
1207  if (c == delimc || c == quotec || c == '\n' || c == '\r')
1208  {
1209  use_quote = true;
1210  break;
1211  }
1212  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
1213  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
1214  else
1215  tptr++;
1216  }
1217  }
1218  }
1219 
1220  if (use_quote)
1221  {
1222  CopySendChar(cstate, quotec);
1223 
1224  /*
1225  * We adopt the same optimization strategy as in CopyAttributeOutText
1226  */
1227  start = ptr;
1228  while ((c = *ptr) != '\0')
1229  {
1230  if (c == quotec || c == escapec)
1231  {
1232  DUMPSOFAR();
1233  CopySendChar(cstate, escapec);
1234  start = ptr; /* we include char in next run */
1235  }
1236  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
1237  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
1238  else
1239  ptr++;
1240  }
1241  DUMPSOFAR();
1242 
1243  CopySendChar(cstate, quotec);
1244  }
1245  else
1246  {
1247  /* If it doesn't need quoting, we can just dump it as-is */
1248  CopySendString(cstate, ptr);
1249  }
1250 }
int file_encoding
Definition: copyto.c:78
bool need_transcoding
Definition: copyto.c:79
char * null_print
Definition: copy.h:36
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:749
char * quote
Definition: copy.h:40
#define IS_HIGHBIT_SET(ch)
Definition: c.h:1156
char * c
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1916
char string[11]
Definition: preproc-type.c:46
static void CopySendString(CopyToState cstate, const char *str)
Definition: copyto.c:181
char * delim
Definition: copy.h:39
#define DUMPSOFAR()
Definition: copyto.c:1011
CopyFormatOptions opts
Definition: copyto.c:89
char * escape
Definition: copy.h:41
static void CopySendChar(CopyToState cstate, char c)
Definition: copyto.c:187
bool encoding_embeds_ascii
Definition: copyto.c:80

◆ CopyAttributeOutText()

static void CopyAttributeOutText ( CopyToState  cstate,
char *  string 
)
static

Definition at line 1018 of file copyto.c.

References CopySendChar(), CopyFormatOptions::delim, DUMPSOFAR, CopyToStateData::encoding_embeds_ascii, CopyToStateData::file_encoding, IS_HIGHBIT_SET, CopyToStateData::need_transcoding, CopyToStateData::opts, pg_encoding_mblen(), and pg_server_to_any().

Referenced by CopyOneRowTo().

1019 {
1020  char *ptr;
1021  char *start;
1022  char c;
1023  char delimc = cstate->opts.delim[0];
1024 
1025  if (cstate->need_transcoding)
1026  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
1027  else
1028  ptr = string;
1029 
1030  /*
1031  * We have to grovel through the string searching for control characters
1032  * and instances of the delimiter character. In most cases, though, these
1033  * are infrequent. To avoid overhead from calling CopySendData once per
1034  * character, we dump out all characters between escaped characters in a
1035  * single call. The loop invariant is that the data from "start" to "ptr"
1036  * can be sent literally, but hasn't yet been.
1037  *
1038  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
1039  * in valid backend encodings, extra bytes of a multibyte character never
1040  * look like ASCII. This loop is sufficiently performance-critical that
1041  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
1042  * of the normal safe-encoding path.
1043  */
1044  if (cstate->encoding_embeds_ascii)
1045  {
1046  start = ptr;
1047  while ((c = *ptr) != '\0')
1048  {
1049  if ((unsigned char) c < (unsigned char) 0x20)
1050  {
1051  /*
1052  * \r and \n must be escaped, the others are traditional. We
1053  * prefer to dump these using the C-like notation, rather than
1054  * a backslash and the literal character, because it makes the
1055  * dump file a bit more proof against Microsoftish data
1056  * mangling.
1057  */
1058  switch (c)
1059  {
1060  case '\b':
1061  c = 'b';
1062  break;
1063  case '\f':
1064  c = 'f';
1065  break;
1066  case '\n':
1067  c = 'n';
1068  break;
1069  case '\r':
1070  c = 'r';
1071  break;
1072  case '\t':
1073  c = 't';
1074  break;
1075  case '\v':
1076  c = 'v';
1077  break;
1078  default:
1079  /* If it's the delimiter, must backslash it */
1080  if (c == delimc)
1081  break;
1082  /* All ASCII control chars are length 1 */
1083  ptr++;
1084  continue; /* fall to end of loop */
1085  }
1086  /* if we get here, we need to convert the control char */
1087  DUMPSOFAR();
1088  CopySendChar(cstate, '\\');
1089  CopySendChar(cstate, c);
1090  start = ++ptr; /* do not include char in next run */
1091  }
1092  else if (c == '\\' || c == delimc)
1093  {
1094  DUMPSOFAR();
1095  CopySendChar(cstate, '\\');
1096  start = ptr++; /* we include char in next run */
1097  }
1098  else if (IS_HIGHBIT_SET(c))
1099  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
1100  else
1101  ptr++;
1102  }
1103  }
1104  else
1105  {
1106  start = ptr;
1107  while ((c = *ptr) != '\0')
1108  {
1109  if ((unsigned char) c < (unsigned char) 0x20)
1110  {
1111  /*
1112  * \r and \n must be escaped, the others are traditional. We
1113  * prefer to dump these using the C-like notation, rather than
1114  * a backslash and the literal character, because it makes the
1115  * dump file a bit more proof against Microsoftish data
1116  * mangling.
1117  */
1118  switch (c)
1119  {
1120  case '\b':
1121  c = 'b';
1122  break;
1123  case '\f':
1124  c = 'f';
1125  break;
1126  case '\n':
1127  c = 'n';
1128  break;
1129  case '\r':
1130  c = 'r';
1131  break;
1132  case '\t':
1133  c = 't';
1134  break;
1135  case '\v':
1136  c = 'v';
1137  break;
1138  default:
1139  /* If it's the delimiter, must backslash it */
1140  if (c == delimc)
1141  break;
1142  /* All ASCII control chars are length 1 */
1143  ptr++;
1144  continue; /* fall to end of loop */
1145  }
1146  /* if we get here, we need to convert the control char */
1147  DUMPSOFAR();
1148  CopySendChar(cstate, '\\');
1149  CopySendChar(cstate, c);
1150  start = ++ptr; /* do not include char in next run */
1151  }
1152  else if (c == '\\' || c == delimc)
1153  {
1154  DUMPSOFAR();
1155  CopySendChar(cstate, '\\');
1156  start = ptr++; /* we include char in next run */
1157  }
1158  else
1159  ptr++;
1160  }
1161  }
1162 
1163  DUMPSOFAR();
1164 }
int file_encoding
Definition: copyto.c:78
bool need_transcoding
Definition: copyto.c:79
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:749
#define IS_HIGHBIT_SET(ch)
Definition: c.h:1156
char * c
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1916
char string[11]
Definition: preproc-type.c:46
char * delim
Definition: copy.h:39
#define DUMPSOFAR()
Definition: copyto.c:1011
CopyFormatOptions opts
Definition: copyto.c:89
static void CopySendChar(CopyToState cstate, char c)
Definition: copyto.c:187
bool encoding_embeds_ascii
Definition: copyto.c:80

◆ CopyOneRowTo()

static void CopyOneRowTo ( CopyToState  cstate,
TupleTableSlot slot 
)
static

Definition at line 937 of file copyto.c.

References attnum, CopyToStateData::attnumlist, CopyFormatOptions::binary, CopyAttributeOutCSV(), CopyAttributeOutText(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), CopySendString(), CopyFormatOptions::csv_mode, cur, CopyFormatOptions::delim, CopyFormatOptions::force_quote_flags, lfirst_int, list_length(), MemoryContextReset(), MemoryContextSwitchTo(), CopyFormatOptions::null_print_client, CopyToStateData::opts, CopyToStateData::out_functions, OutputFunctionCall(), CopyToStateData::rowcontext, SendFunctionCall(), slot_getallattrs(), TupleTableSlot::tts_isnull, TupleTableSlot::tts_values, value, VARDATA, VARHDRSZ, and VARSIZE.

Referenced by copy_dest_receive(), and DoCopyTo().

938 {
939  bool need_delim = false;
940  FmgrInfo *out_functions = cstate->out_functions;
941  MemoryContext oldcontext;
942  ListCell *cur;
943  char *string;
944 
946  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
947 
948  if (cstate->opts.binary)
949  {
950  /* Binary per-tuple header */
951  CopySendInt16(cstate, list_length(cstate->attnumlist));
952  }
953 
954  /* Make sure the tuple is fully deconstructed */
955  slot_getallattrs(slot);
956 
957  foreach(cur, cstate->attnumlist)
958  {
959  int attnum = lfirst_int(cur);
960  Datum value = slot->tts_values[attnum - 1];
961  bool isnull = slot->tts_isnull[attnum - 1];
962 
963  if (!cstate->opts.binary)
964  {
965  if (need_delim)
966  CopySendChar(cstate, cstate->opts.delim[0]);
967  need_delim = true;
968  }
969 
970  if (isnull)
971  {
972  if (!cstate->opts.binary)
973  CopySendString(cstate, cstate->opts.null_print_client);
974  else
975  CopySendInt32(cstate, -1);
976  }
977  else
978  {
979  if (!cstate->opts.binary)
980  {
981  string = OutputFunctionCall(&out_functions[attnum - 1],
982  value);
983  if (cstate->opts.csv_mode)
984  CopyAttributeOutCSV(cstate, string,
985  cstate->opts.force_quote_flags[attnum - 1],
986  list_length(cstate->attnumlist) == 1);
987  else
988  CopyAttributeOutText(cstate, string);
989  }
990  else
991  {
992  bytea *outputbytes;
993 
994  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
995  value);
996  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
997  CopySendData(cstate, VARDATA(outputbytes),
998  VARSIZE(outputbytes) - VARHDRSZ);
999  }
1000  }
1001  }
1002 
1003  CopySendEndOfRow(cstate);
1004 
1005  MemoryContextSwitchTo(oldcontext);
1006 }
Definition: fmgr.h:56
#define VARDATA(PTR)
Definition: postgres.h:315
static void CopyAttributeOutCSV(CopyToState cstate, char *string, bool use_quote, bool single_attr)
Definition: copyto.c:1171
static struct @142 value
FmgrInfo * out_functions
Definition: copyto.c:97
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARHDRSZ
Definition: c.h:627
bool * force_quote_flags
Definition: copy.h:44
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
static void CopySendInt16(CopyToState cstate, int16 val)
Definition: copyto.c:280
struct cursor * cur
Definition: ecpg.c:28
char * null_print_client
Definition: copy.h:38
Datum * tts_values
Definition: tuptable.h:126
MemoryContext rowcontext
Definition: copyto.c:98
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:143
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1576
List * attnumlist
Definition: copyto.c:85
#define lfirst_int(lc)
Definition: pg_list.h:170
bool binary
Definition: copy.h:32
static void slot_getallattrs(TupleTableSlot *slot)
Definition: tuptable.h:354
bool csv_mode
Definition: copy.h:34
bool * tts_isnull
Definition: tuptable.h:128
char string[11]
Definition: preproc-type.c:46
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1637
static void CopySendString(CopyToState cstate, const char *str)
Definition: copyto.c:181
static void CopySendInt32(CopyToState cstate, int32 val)
Definition: copyto.c:268
uintptr_t Datum
Definition: postgres.h:411
char * delim
Definition: copy.h:39
static void CopySendData(CopyToState cstate, const void *databuf, int datasize)
Definition: copyto.c:175
int16 attnum
Definition: pg_attribute.h:83
static int list_length(const List *l)
Definition: pg_list.h:149
static void CopyAttributeOutText(CopyToState cstate, char *string)
Definition: copyto.c:1018
CopyFormatOptions opts
Definition: copyto.c:89
static void CopySendChar(CopyToState cstate, char c)
Definition: copyto.c:187
Definition: c.h:621
static void CopySendEndOfRow(CopyToState cstate)
Definition: copyto.c:193

◆ CopySendChar()

static void CopySendChar ( CopyToState  cstate,
char  c 
)
static

Definition at line 187 of file copyto.c.

References appendStringInfoCharMacro, and CopyToStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), CopyOneRowTo(), CopySendEndOfRow(), and DoCopyTo().

188 {
190 }
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:128
char * c
StringInfo fe_msgbuf
Definition: copyto.c:76

◆ CopySendData()

static void CopySendData ( CopyToState  cstate,
const void *  databuf,
int  datasize 
)
static

Definition at line 175 of file copyto.c.

References appendBinaryStringInfo(), and CopyToStateData::fe_msgbuf.

Referenced by CopyOneRowTo(), CopySendInt16(), CopySendInt32(), and DoCopyTo().

176 {
177  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
178 }
StringInfo fe_msgbuf
Definition: copyto.c:76
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:227

◆ CopySendEndOfRow()

static void CopySendEndOfRow ( CopyToState  cstate)
static

Definition at line 193 of file copyto.c.

References CopyFormatOptions::binary, CopyToStateData::bytes_processed, ClosePipeToProgram(), CopyToStateData::copy_dest, COPY_FILE, CopyToStateData::copy_file, COPY_FRONTEND, CopySendChar(), CopySendString(), StringInfoData::data, ereport, errcode_for_file_access(), errmsg(), ERROR, CopyToStateData::fe_msgbuf, CopyToStateData::is_program, StringInfoData::len, CopyToStateData::opts, pgstat_progress_update_param(), pq_putmessage, PROGRESS_COPY_BYTES_PROCESSED, and resetStringInfo().

Referenced by CopyOneRowTo(), and DoCopyTo().

194 {
195  StringInfo fe_msgbuf = cstate->fe_msgbuf;
196 
197  switch (cstate->copy_dest)
198  {
199  case COPY_FILE:
200  if (!cstate->opts.binary)
201  {
202  /* Default line termination depends on platform */
203 #ifndef WIN32
204  CopySendChar(cstate, '\n');
205 #else
206  CopySendString(cstate, "\r\n");
207 #endif
208  }
209 
210  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
211  cstate->copy_file) != 1 ||
212  ferror(cstate->copy_file))
213  {
214  if (cstate->is_program)
215  {
216  if (errno == EPIPE)
217  {
218  /*
219  * The pipe will be closed automatically on error at
220  * the end of transaction, but we might get a better
221  * error message from the subprocess' exit code than
222  * just "Broken Pipe"
223  */
224  ClosePipeToProgram(cstate);
225 
226  /*
227  * If ClosePipeToProgram() didn't throw an error, the
228  * program terminated normally, but closed the pipe
229  * first. Restore errno, and throw an error.
230  */
231  errno = EPIPE;
232  }
233  ereport(ERROR,
235  errmsg("could not write to COPY program: %m")));
236  }
237  else
238  ereport(ERROR,
240  errmsg("could not write to COPY file: %m")));
241  }
242  break;
243  case COPY_FRONTEND:
244  /* The FE/BE protocol uses \n as newline for all platforms */
245  if (!cstate->opts.binary)
246  CopySendChar(cstate, '\n');
247 
248  /* Dump the accumulated row as one CopyData message */
249  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
250  break;
251  }
252 
253  /* Update the progress */
254  cstate->bytes_processed += fe_msgbuf->len;
256 
257  resetStringInfo(fe_msgbuf);
258 }
FILE * copy_file
Definition: copyto.c:75
uint64 bytes_processed
Definition: copyto.c:99
bool is_program
Definition: copyto.c:87
#define ERROR
Definition: elog.h:46
bool binary
Definition: copy.h:32
static void ClosePipeToProgram(CopyToState cstate)
Definition: copyto.c:292
int errcode_for_file_access(void)
Definition: elog.c:721
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:75
static void CopySendString(CopyToState cstate, const char *str)
Definition: copyto.c:181
CopyDest copy_dest
Definition: copyto.c:74
StringInfo fe_msgbuf
Definition: copyto.c:76
#define ereport(elevel,...)
Definition: elog.h:157
void pgstat_progress_update_param(int index, int64 val)
CopyFormatOptions opts
Definition: copyto.c:89
int errmsg(const char *fmt,...)
Definition: elog.c:909
static void CopySendChar(CopyToState cstate, char c)
Definition: copyto.c:187
#define PROGRESS_COPY_BYTES_PROCESSED
Definition: progress.h:137
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:40

◆ CopySendInt16()

static void CopySendInt16 ( CopyToState  cstate,
int16  val 
)
inlinestatic

Definition at line 280 of file copyto.c.

References buf, CopySendData(), and pg_hton16.

Referenced by CopyOneRowTo(), and DoCopyTo().

281 {
282  uint16 buf;
283 
284  buf = pg_hton16((uint16) val);
285  CopySendData(cstate, &buf, sizeof(buf));
286 }
#define pg_hton16(x)
Definition: pg_bswap.h:120
unsigned short uint16
Definition: c.h:440
static char * buf
Definition: pg_test_fsync.c:68
static void CopySendData(CopyToState cstate, const void *databuf, int datasize)
Definition: copyto.c:175
long val
Definition: informix.c:664

◆ CopySendInt32()

static void CopySendInt32 ( CopyToState  cstate,
int32  val 
)
inlinestatic

Definition at line 268 of file copyto.c.

References buf, CopySendData(), and pg_hton32.

Referenced by CopyOneRowTo(), and DoCopyTo().

269 {
270  uint32 buf;
271 
272  buf = pg_hton32((uint32) val);
273  CopySendData(cstate, &buf, sizeof(buf));
274 }
#define pg_hton32(x)
Definition: pg_bswap.h:121
static char * buf
Definition: pg_test_fsync.c:68
unsigned int uint32
Definition: c.h:441
static void CopySendData(CopyToState cstate, const void *databuf, int datasize)
Definition: copyto.c:175
long val
Definition: informix.c:664

◆ CopySendString()

static void CopySendString ( CopyToState  cstate,
const char *  str 
)
static

Definition at line 181 of file copyto.c.

References appendBinaryStringInfo(), and CopyToStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyOneRowTo(), and CopySendEndOfRow().

182 {
183  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
184 }
StringInfo fe_msgbuf
Definition: copyto.c:76
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:227

◆ CreateCopyDestReceiver()

DestReceiver* CreateCopyDestReceiver ( void  )

Definition at line 1302 of file copyto.c.

References copy_dest_destroy(), copy_dest_receive(), copy_dest_shutdown(), copy_dest_startup(), DestCopyOut, and palloc().

Referenced by CreateDestReceiver().

1303 {
1304  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
1305 
1306  self->pub.receiveSlot = copy_dest_receive;
1307  self->pub.rStartup = copy_dest_startup;
1308  self->pub.rShutdown = copy_dest_shutdown;
1309  self->pub.rDestroy = copy_dest_destroy;
1310  self->pub.mydest = DestCopyOut;
1311 
1312  self->cstate = NULL; /* will be set later */
1313  self->processed = 0;
1314 
1315  return (DestReceiver *) self;
1316 }
static void copy_dest_destroy(DestReceiver *self)
Definition: copyto.c:1293
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copyto.c:1265
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copyto.c:1256
void * palloc(Size size)
Definition: mcxt.c:1062
static void copy_dest_shutdown(DestReceiver *self)
Definition: copyto.c:1284

◆ DoCopyTo()

uint64 DoCopyTo ( CopyToState  cstate)

Definition at line 779 of file copyto.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, attname, attnum, CopyToStateData::attnumlist, CopyFormatOptions::binary, BinarySignature, CHECK_FOR_INTERRUPTS, CopyAttributeOutCSV(), CopyOneRowTo(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), cur, CurrentMemoryContext, CopyFormatOptions::delim, QueryDesc::dest, DestRemote, ExecDropSingleTupleTableSlot(), ExecutorRun(), CopyToStateData::fe_msgbuf, CopyToStateData::file_encoding, CopyToStateData::filename, fmgr_info(), ForwardScanDirection, GetActiveSnapshot(), getTypeBinaryOutputInfo(), getTypeOutputInfo(), CopyFormatOptions::header_line, lfirst_int, list_length(), makeStringInfo(), MemoryContextDelete(), NameStr, TupleDescData::natts, CopyToStateData::need_transcoding, CopyFormatOptions::null_print, CopyFormatOptions::null_print_client, CopyFormatOptions::null_print_len, CopyToStateData::opts, CopyToStateData::out_functions, palloc(), pg_server_to_any(), pgstat_progress_update_param(), PROGRESS_COPY_TUPLES_PROCESSED, CopyToStateData::queryDesc, CopyToStateData::rel, RelationGetDescr, CopyToStateData::rowcontext, SendCopyBegin(), SendCopyEnd(), slot_getallattrs(), table_beginscan(), table_endscan(), table_scan_getnextslot(), table_slot_create(), QueryDesc::tupDesc, TupleDescAttr, and whereToSendOutput.

Referenced by DoCopy().

780 {
781  bool pipe = (cstate->filename == NULL);
782  bool fe_copy = (pipe && whereToSendOutput == DestRemote);
783  TupleDesc tupDesc;
784  int num_phys_attrs;
785  ListCell *cur;
786  uint64 processed;
787 
788  if (fe_copy)
789  SendCopyBegin(cstate);
790 
791  if (cstate->rel)
792  tupDesc = RelationGetDescr(cstate->rel);
793  else
794  tupDesc = cstate->queryDesc->tupDesc;
795  num_phys_attrs = tupDesc->natts;
796  cstate->opts.null_print_client = cstate->opts.null_print; /* default */
797 
798  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
799  cstate->fe_msgbuf = makeStringInfo();
800 
801  /* Get info about the columns we need to process. */
802  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
803  foreach(cur, cstate->attnumlist)
804  {
805  int attnum = lfirst_int(cur);
806  Oid out_func_oid;
807  bool isvarlena;
808  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
809 
810  if (cstate->opts.binary)
811  getTypeBinaryOutputInfo(attr->atttypid,
812  &out_func_oid,
813  &isvarlena);
814  else
815  getTypeOutputInfo(attr->atttypid,
816  &out_func_oid,
817  &isvarlena);
818  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
819  }
820 
821  /*
822  * Create a temporary memory context that we can reset once per row to
823  * recover palloc'd memory. This avoids any problems with leaks inside
824  * datatype output routines, and should be faster than retail pfree's
825  * anyway. (We don't need a whole econtext as CopyFrom does.)
826  */
828  "COPY TO",
830 
831  if (cstate->opts.binary)
832  {
833  /* Generate header for a binary copy */
834  int32 tmp;
835 
836  /* Signature */
837  CopySendData(cstate, BinarySignature, 11);
838  /* Flags field */
839  tmp = 0;
840  CopySendInt32(cstate, tmp);
841  /* No header extension */
842  tmp = 0;
843  CopySendInt32(cstate, tmp);
844  }
845  else
846  {
847  /*
848  * For non-binary copy, we need to convert null_print to file
849  * encoding, because it will be sent directly with CopySendString.
850  */
851  if (cstate->need_transcoding)
853  cstate->opts.null_print_len,
854  cstate->file_encoding);
855 
856  /* if a header has been requested send the line */
857  if (cstate->opts.header_line)
858  {
859  bool hdr_delim = false;
860 
861  foreach(cur, cstate->attnumlist)
862  {
863  int attnum = lfirst_int(cur);
864  char *colname;
865 
866  if (hdr_delim)
867  CopySendChar(cstate, cstate->opts.delim[0]);
868  hdr_delim = true;
869 
870  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
871 
872  CopyAttributeOutCSV(cstate, colname, false,
873  list_length(cstate->attnumlist) == 1);
874  }
875 
876  CopySendEndOfRow(cstate);
877  }
878  }
879 
880  if (cstate->rel)
881  {
882  TupleTableSlot *slot;
883  TableScanDesc scandesc;
884 
885  scandesc = table_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
886  slot = table_slot_create(cstate->rel, NULL);
887 
888  processed = 0;
889  while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot))
890  {
892 
893  /* Deconstruct the tuple ... */
894  slot_getallattrs(slot);
895 
896  /* Format and send the data */
897  CopyOneRowTo(cstate, slot);
898 
899  /*
900  * Increment the number of processed tuples, and report the
901  * progress.
902  */
904  ++processed);
905  }
906 
908  table_endscan(scandesc);
909  }
910  else
911  {
912  /* run the plan --- the dest receiver will send tuples */
913  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
914  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
915  }
916 
917  if (cstate->opts.binary)
918  {
919  /* Generate trailer for a binary copy */
920  CopySendInt16(cstate, -1);
921  /* Need to flush out the trailer */
922  CopySendEndOfRow(cstate);
923  }
924 
926 
927  if (fe_copy)
928  SendCopyEnd(cstate);
929 
930  return processed;
931 }
TupleTableSlot * table_slot_create(Relation relation, List **reglist)
Definition: tableam.c:91
int file_encoding
Definition: copyto.c:78
Definition: fmgr.h:56
int null_print_len
Definition: copy.h:37
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:218
#define AllocSetContextCreate
Definition: memutils.h:173
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2854
Relation rel
Definition: copyto.c:83
static void CopyAttributeOutCSV(CopyToState cstate, char *string, bool use_quote, bool single_attr)
Definition: copyto.c:1171
bool need_transcoding
Definition: copyto.c:79
FmgrInfo * out_functions
Definition: copyto.c:97
#define RelationGetDescr(relation)
Definition: rel.h:483
static void SendCopyEnd(CopyToState cstate)
Definition: copyto.c:156
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:92
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:786
static void CopySendInt16(CopyToState cstate, int16 val)
Definition: copyto.c:280
struct cursor * cur
Definition: ecpg.c:28
char * null_print_client
Definition: copy.h:38
MemoryContext rowcontext
Definition: copyto.c:98
static bool table_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot)
Definition: tableam.h:1032
unsigned int Oid
Definition: postgres_ext.h:31
char * null_print
Definition: copy.h:36
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:749
signed int int32
Definition: c.h:429
static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
Definition: copyto.c:937
QueryDesc * queryDesc
Definition: copyto.c:84
static void SendCopyBegin(CopyToState cstate)
Definition: copyto.c:139
List * attnumlist
Definition: copyto.c:85
NameData attname
Definition: pg_attribute.h:41
static TableScanDesc table_beginscan(Relation rel, Snapshot snapshot, int nkeys, struct ScanKeyData *key)
Definition: tableam.h:883
#define lfirst_int(lc)
Definition: pg_list.h:170
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:298
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:126
bool binary
Definition: copy.h:32
static void slot_getallattrs(TupleTableSlot *slot)
Definition: tuptable.h:354
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:195
void ExecDropSingleTupleTableSlot(TupleTableSlot *slot)
Definition: execTuples.c:1224
bool header_line
Definition: copy.h:35
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:203
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
static void CopySendInt32(CopyToState cstate, int32 val)
Definition: copyto.c:268
TupleDesc tupDesc
Definition: execdesc.h:47
StringInfo fe_msgbuf
Definition: copyto.c:76
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2920
char * delim
Definition: copy.h:39
static void CopySendData(CopyToState cstate, const void *databuf, int datasize)
Definition: copyto.c:175
#define PROGRESS_COPY_TUPLES_PROCESSED
Definition: progress.h:139
int16 attnum
Definition: pg_attribute.h:83
void pgstat_progress_update_param(int index, int64 val)
static const char BinarySignature[11]
Definition: copyto.c:112
static int list_length(const List *l)
Definition: pg_list.h:149
CopyFormatOptions opts
Definition: copyto.c:89
static void table_endscan(TableScanDesc scan)
Definition: tableam.h:991
DestReceiver * dest
Definition: execdesc.h:41
void * palloc(Size size)
Definition: mcxt.c:1062
static void CopySendChar(CopyToState cstate, char c)
Definition: copyto.c:187
#define NameStr(name)
Definition: c.h:681
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:102
CommandDest whereToSendOutput
Definition: postgres.c:92
char * filename
Definition: copyto.c:86
static void CopySendEndOfRow(CopyToState cstate)
Definition: copyto.c:193

◆ EndCopy()

static void EndCopy ( CopyToState  cstate)
static

Definition at line 317 of file copyto.c.

References ClosePipeToProgram(), CopyToStateData::copy_file, CopyToStateData::copycontext, ereport, errcode_for_file_access(), errmsg(), ERROR, CopyToStateData::filename, FreeFile(), CopyToStateData::is_program, MemoryContextDelete(), pfree(), and pgstat_progress_end_command().

Referenced by EndCopyTo().

318 {
319  if (cstate->is_program)
320  {
321  ClosePipeToProgram(cstate);
322  }
323  else
324  {
325  if (cstate->filename != NULL && FreeFile(cstate->copy_file))
326  ereport(ERROR,
328  errmsg("could not close file \"%s\": %m",
329  cstate->filename)));
330  }
331 
333 
335  pfree(cstate);
336 }
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:218
FILE * copy_file
Definition: copyto.c:75
void pgstat_progress_end_command(void)
bool is_program
Definition: copyto.c:87
void pfree(void *pointer)
Definition: mcxt.c:1169
#define ERROR
Definition: elog.h:46
MemoryContext copycontext
Definition: copyto.c:95
static void ClosePipeToProgram(CopyToState cstate)
Definition: copyto.c:292
int errcode_for_file_access(void)
Definition: elog.c:721
#define ereport(elevel,...)
Definition: elog.h:157
int FreeFile(FILE *file)
Definition: fd.c:2572
int errmsg(const char *fmt,...)
Definition: elog.c:909
char * filename
Definition: copyto.c:86

◆ EndCopyTo()

void EndCopyTo ( CopyToState  cstate)

Definition at line 760 of file copyto.c.

References EndCopy(), ExecutorEnd(), ExecutorFinish(), FreeQueryDesc(), PopActiveSnapshot(), and CopyToStateData::queryDesc.

Referenced by DoCopy().

761 {
762  if (cstate->queryDesc != NULL)
763  {
764  /* Close down the query and free resources. */
765  ExecutorFinish(cstate->queryDesc);
766  ExecutorEnd(cstate->queryDesc);
767  FreeQueryDesc(cstate->queryDesc);
769  }
770 
771  /* Clean up storage */
772  EndCopy(cstate);
773 }
void FreeQueryDesc(QueryDesc *qdesc)
Definition: pquery.c:105
void PopActiveSnapshot(void)
Definition: snapmgr.c:759
static void EndCopy(CopyToState cstate)
Definition: copyto.c:317
QueryDesc * queryDesc
Definition: copyto.c:84
void ExecutorEnd(QueryDesc *queryDesc)
Definition: execMain.c:459
void ExecutorFinish(QueryDesc *queryDesc)
Definition: execMain.c:399

◆ SendCopyBegin()

static void SendCopyBegin ( CopyToState  cstate)
static

Definition at line 139 of file copyto.c.

References CopyToStateData::attnumlist, CopyFormatOptions::binary, buf, CopyToStateData::copy_dest, COPY_FRONTEND, format, i, list_length(), CopyToStateData::opts, pq_beginmessage(), pq_endmessage(), pq_sendbyte(), and pq_sendint16().

Referenced by DoCopyTo().

140 {
142  int natts = list_length(cstate->attnumlist);
143  int16 format = (cstate->opts.binary ? 1 : 0);
144  int i;
145 
146  pq_beginmessage(&buf, 'H');
147  pq_sendbyte(&buf, format); /* overall format */
148  pq_sendint16(&buf, natts);
149  for (i = 0; i < natts; i++)
150  pq_sendint16(&buf, format); /* per-column formats */
151  pq_endmessage(&buf);
152  cstate->copy_dest = COPY_FRONTEND;
153 }
signed short int16
Definition: c.h:428
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
List * attnumlist
Definition: copyto.c:85
bool binary
Definition: copy.h:32
static char * buf
Definition: pg_test_fsync.c:68
CopyDest copy_dest
Definition: copyto.c:74
static int list_length(const List *l)
Definition: pg_list.h:149
CopyFormatOptions opts
Definition: copyto.c:89
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
int i
static char format

◆ SendCopyEnd()

static void SendCopyEnd ( CopyToState  cstate)
static

Definition at line 156 of file copyto.c.

References Assert, CopyToStateData::fe_msgbuf, StringInfoData::len, and pq_putemptymessage().

Referenced by DoCopyTo().

157 {
158  /* Shouldn't have any unsent data */
159  Assert(cstate->fe_msgbuf->len == 0);
160  /* Send Copy Done message */
161  pq_putemptymessage('c');
162 }
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
StringInfo fe_msgbuf
Definition: copyto.c:76
#define Assert(condition)
Definition: c.h:804

Variable Documentation

◆ BinarySignature

const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"
static

Definition at line 112 of file copyto.c.

Referenced by DoCopyTo().