PostgreSQL Source Code  git master
copy.c File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/sysattr.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/dependency.h"
#include "catalog/pg_type.h"
#include "commands/copy.h"
#include "commands/defrem.h"
#include "commands/trigger.h"
#include "executor/execPartition.h"
#include "executor/executor.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "optimizer/clauses.h"
#include "optimizer/planner.h"
#include "nodes/makefuncs.h"
#include "parser/parse_relation.h"
#include "port/pg_bswap.h"
#include "rewrite/rewriteHandler.h"
#include "storage/fd.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/portal.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
Include dependency graph for copy.c:

Go to the source code of this file.

Data Structures

struct  CopyStateData
 
struct  DR_copy
 

Macros

#define ISOCTAL(c)   (((c) >= '0') && ((c) <= '7'))
 
#define OCTVALUE(c)   ((c) - '0')
 
#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */
 
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
 
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
 
#define REFILL_LINEBUF
 
#define NO_END_OF_COPY_GOTO
 
#define MAX_COPY_DATA_DISPLAY   100
 
#define MAX_BUFFERED_TUPLES   1000
 
#define DUMPSOFAR()
 

Typedefs

typedef enum CopyDest CopyDest
 
typedef enum EolType EolType
 
typedef struct CopyStateData CopyStateData
 

Enumerations

enum  CopyDest { COPY_FILE, COPY_OLD_FE, COPY_NEW_FE, COPY_CALLBACK }
 
enum  EolType { EOL_UNKNOWN, EOL_NL, EOL_CR, EOL_CRNL }
 

Functions

static CopyState BeginCopy (ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
 
static void EndCopy (CopyState cstate)
 
static void ClosePipeToProgram (CopyState cstate)
 
static CopyState BeginCopyTo (ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
 
static void EndCopyTo (CopyState cstate)
 
static uint64 DoCopyTo (CopyState cstate)
 
static uint64 CopyTo (CopyState cstate)
 
static void CopyOneRowTo (CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
 
static void CopyFromInsertBatch (CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
 
static bool CopyReadLine (CopyState cstate)
 
static bool CopyReadLineText (CopyState cstate)
 
static int CopyReadAttributesText (CopyState cstate)
 
static int CopyReadAttributesCSV (CopyState cstate)
 
static Datum CopyReadBinaryAttribute (CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
 
static void CopyAttributeOutText (CopyState cstate, char *string)
 
static void CopyAttributeOutCSV (CopyState cstate, char *string, bool use_quote, bool single_attr)
 
static ListCopyGetAttnums (TupleDesc tupDesc, Relation rel, List *attnamelist)
 
static char * limit_printout_length (const char *str)
 
static void SendCopyBegin (CopyState cstate)
 
static void ReceiveCopyBegin (CopyState cstate)
 
static void SendCopyEnd (CopyState cstate)
 
static void CopySendData (CopyState cstate, const void *databuf, int datasize)
 
static void CopySendString (CopyState cstate, const char *str)
 
static void CopySendChar (CopyState cstate, char c)
 
static void CopySendEndOfRow (CopyState cstate)
 
static int CopyGetData (CopyState cstate, void *databuf, int minread, int maxread)
 
static void CopySendInt32 (CopyState cstate, int32 val)
 
static bool CopyGetInt32 (CopyState cstate, int32 *val)
 
static void CopySendInt16 (CopyState cstate, int16 val)
 
static bool CopyGetInt16 (CopyState cstate, int16 *val)
 
static bool CopyLoadRawBuf (CopyState cstate)
 
void DoCopy (ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
 
void ProcessCopyOptions (ParseState *pstate, CopyState cstate, bool is_from, List *options)
 
void CopyFromErrorCallback (void *arg)
 
uint64 CopyFrom (CopyState cstate)
 
CopyState BeginCopyFrom (ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
 
bool NextCopyFromRawFields (CopyState cstate, char ***fields, int *nfields)
 
bool NextCopyFrom (CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
 
void EndCopyFrom (CopyState cstate)
 
static int GetDecimalFromHex (char hex)
 
static void copy_dest_startup (DestReceiver *self, int operation, TupleDesc typeinfo)
 
static bool copy_dest_receive (TupleTableSlot *slot, DestReceiver *self)
 
static void copy_dest_shutdown (DestReceiver *self)
 
static void copy_dest_destroy (DestReceiver *self)
 
DestReceiverCreateCopyDestReceiver (void)
 

Variables

static const char BinarySignature [11] = "PGCOPY\n\377\r\n\0"
 

Macro Definition Documentation

◆ DUMPSOFAR

#define DUMPSOFAR ( )
Value:
do { \
if (ptr > start) \
CopySendData(cstate, start, ptr - start); \
} while (0)

Definition at line 4447 of file copy.c.

Referenced by CopyAttributeOutCSV(), and CopyAttributeOutText().

◆ IF_NEED_REFILL_AND_EOF_BREAK

#define IF_NEED_REFILL_AND_EOF_BREAK (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
{ \
if (extralen) \
raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
/* backslash just before EOF, treat as data char */ \
result = true; \
break; \
} \
} else ((void) 0)

Definition at line 251 of file copy.c.

Referenced by CopyReadLineText().

◆ IF_NEED_REFILL_AND_NOT_EOF_CONTINUE

#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
{ \
raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
need_data = true; \
continue; \
} \
} else ((void) 0)

Definition at line 239 of file copy.c.

Referenced by CopyReadLineText().

◆ ISOCTAL

#define ISOCTAL (   c)    (((c) >= '0') && ((c) <= '7'))

Definition at line 54 of file copy.c.

Referenced by CopyReadAttributesText().

◆ MAX_BUFFERED_TUPLES

#define MAX_BUFFERED_TUPLES   1000

Referenced by CopyFrom().

◆ MAX_COPY_DATA_DISPLAY

#define MAX_COPY_DATA_DISPLAY   100

Referenced by limit_printout_length().

◆ NO_END_OF_COPY_GOTO

#define NO_END_OF_COPY_GOTO
Value:
if (1) \
{ \
raw_buf_ptr = prev_raw_ptr + 1; \
goto not_end_of_copy; \
} else ((void) 0)

Definition at line 281 of file copy.c.

Referenced by CopyReadLineText().

◆ OCTVALUE

#define OCTVALUE (   c)    ((c) - '0')

Definition at line 55 of file copy.c.

Referenced by CopyReadAttributesText().

◆ RAW_BUF_SIZE

#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */

Definition at line 208 of file copy.c.

Referenced by BeginCopyFrom(), and CopyLoadRawBuf().

◆ REFILL_LINEBUF

#define REFILL_LINEBUF
Value:
if (1) \
{ \
if (raw_buf_ptr > cstate->raw_buf_index) \
{ \
appendBinaryStringInfo(&cstate->line_buf, \
cstate->raw_buf + cstate->raw_buf_index, \
raw_buf_ptr - cstate->raw_buf_index); \
cstate->raw_buf_index = raw_buf_ptr; \
} \
} else ((void) 0)

Definition at line 268 of file copy.c.

Referenced by CopyReadLineText().

Typedef Documentation

◆ CopyDest

◆ CopyStateData

◆ EolType

Enumeration Type Documentation

◆ CopyDest

enum CopyDest
Enumerator
COPY_FILE 
COPY_OLD_FE 
COPY_NEW_FE 
COPY_CALLBACK 

Definition at line 61 of file copy.c.

62 {
63  COPY_FILE, /* to/from file (or a piped program) */
64  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
65  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
66  COPY_CALLBACK /* to/from callback function */
67 } CopyDest;
Definition: copy.c:63
CopyDest
Definition: copy.c:61

◆ EolType

enum EolType
Enumerator
EOL_UNKNOWN 
EOL_NL 
EOL_CR 
EOL_CRNL 

Definition at line 72 of file copy.c.

73 {
75  EOL_NL,
76  EOL_CR,
77  EOL_CRNL
78 } EolType;
Definition: copy.c:75
EolType
Definition: copy.c:72
Definition: copy.c:76
Definition: copy.c:77

Function Documentation

◆ BeginCopy()

static CopyState BeginCopy ( ParseState pstate,
bool  is_from,
Relation  rel,
RawStmt raw_query,
Oid  queryRelId,
List attnamelist,
List options 
)
static

Definition at line 1372 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, Assert, CopyStateData::attnumlist, CMD_DELETE, CMD_INSERT, CMD_SELECT, CMD_UPDATE, Query::commandType, CopyStateData::convert_select, CopyStateData::convert_select_flags, CopyStateData::convert_selectively, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copycontext, CopyGetAttnums(), copyObject, CreateDestReceiver(), CreateQueryDesc(), cur, CurrentMemoryContext, CURSOR_OPT_PARALLEL_OK, generate_unaccent_rules::dest, DestCopyOut, CopyStateData::encoding_embeds_ascii, ereport, errcode(), errmsg(), errmsg_internal(), ERROR, ExecutorStart(), CopyStateData::file_encoding, CopyStateData::force_notnull, CopyStateData::force_notnull_flags, CopyStateData::force_null, CopyStateData::force_null_flags, CopyStateData::force_quote, CopyStateData::force_quote_all, CopyStateData::force_quote_flags, GetActiveSnapshot(), GetDatabaseEncoding(), i, InvalidOid, InvalidSnapshot, IsA, lfirst_int, lfirst_node, linitial_node, list_length(), list_member_int(), list_member_oid(), MemoryContextSwitchTo(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NIL, CopyStateData::oids, ParseState::p_sourcetext, palloc0(), pg_analyze_and_rewrite(), pg_database_encoding_max_length(), PG_ENCODING_IS_CLIENT_ONLY, pg_get_client_encoding(), pg_plan_query(), ProcessCopyOptions(), PushCopiedSnapshot(), QSRC_NON_INSTEAD_RULE, QSRC_QUAL_INSTEAD_RULE, CopyStateData::queryDesc, Query::querySource, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, PlannedStmt::relationOids, Query::returningList, QueryDesc::tupDesc, TupleDescAttr, UpdateActiveSnapshotCommandId(), and Query::utilityStmt.

Referenced by BeginCopyFrom(), and BeginCopyTo().

1379 {
1380  CopyState cstate;
1381  TupleDesc tupDesc;
1382  int num_phys_attrs;
1383  MemoryContext oldcontext;
1384 
1385  /* Allocate workspace and zero all fields */
1386  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1387 
1388  /*
1389  * We allocate everything used by a cstate in a new memory context. This
1390  * avoids memory leaks during repeated use of COPY in a query.
1391  */
1393  "COPY",
1395 
1396  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1397 
1398  /* Extract options from the statement node tree */
1399  ProcessCopyOptions(pstate, cstate, is_from, options);
1400 
1401  /* Process the source/target relation or query */
1402  if (rel)
1403  {
1404  Assert(!raw_query);
1405 
1406  cstate->rel = rel;
1407 
1408  tupDesc = RelationGetDescr(cstate->rel);
1409 
1410  /* Don't allow COPY w/ OIDs to or from a table without them */
1411  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1412  ereport(ERROR,
1413  (errcode(ERRCODE_UNDEFINED_COLUMN),
1414  errmsg("table \"%s\" does not have OIDs",
1415  RelationGetRelationName(cstate->rel))));
1416  }
1417  else
1418  {
1419  List *rewritten;
1420  Query *query;
1421  PlannedStmt *plan;
1422  DestReceiver *dest;
1423 
1424  Assert(!is_from);
1425  cstate->rel = NULL;
1426 
1427  /* Don't allow COPY w/ OIDs from a query */
1428  if (cstate->oids)
1429  ereport(ERROR,
1430  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1431  errmsg("COPY (query) WITH OIDS is not supported")));
1432 
1433  /*
1434  * Run parse analysis and rewrite. Note this also acquires sufficient
1435  * locks on the source table(s).
1436  *
1437  * Because the parser and planner tend to scribble on their input, we
1438  * make a preliminary copy of the source querytree. This prevents
1439  * problems in the case that the COPY is in a portal or plpgsql
1440  * function and is executed repeatedly. (See also the same hack in
1441  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1442  */
1443  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1444  pstate->p_sourcetext, NULL, 0,
1445  NULL);
1446 
1447  /* check that we got back something we can work with */
1448  if (rewritten == NIL)
1449  {
1450  ereport(ERROR,
1451  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1452  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1453  }
1454  else if (list_length(rewritten) > 1)
1455  {
1456  ListCell *lc;
1457 
1458  /* examine queries to determine which error message to issue */
1459  foreach(lc, rewritten)
1460  {
1461  Query *q = lfirst_node(Query, lc);
1462 
1464  ereport(ERROR,
1465  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1466  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1468  ereport(ERROR,
1469  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1470  errmsg("DO ALSO rules are not supported for the COPY")));
1471  }
1472 
1473  ereport(ERROR,
1474  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1475  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1476  }
1477 
1478  query = linitial_node(Query, rewritten);
1479 
1480  /* The grammar allows SELECT INTO, but we don't support that */
1481  if (query->utilityStmt != NULL &&
1483  ereport(ERROR,
1484  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1485  errmsg("COPY (SELECT INTO) is not supported")));
1486 
1487  Assert(query->utilityStmt == NULL);
1488 
1489  /*
1490  * Similarly the grammar doesn't enforce the presence of a RETURNING
1491  * clause, but this is required here.
1492  */
1493  if (query->commandType != CMD_SELECT &&
1494  query->returningList == NIL)
1495  {
1496  Assert(query->commandType == CMD_INSERT ||
1497  query->commandType == CMD_UPDATE ||
1498  query->commandType == CMD_DELETE);
1499 
1500  ereport(ERROR,
1501  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1502  errmsg("COPY query must have a RETURNING clause")));
1503  }
1504 
1505  /* plan the query */
1506  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1507 
1508  /*
1509  * With row level security and a user using "COPY relation TO", we
1510  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1511  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1512  * in any RLS clauses.
1513  *
1514  * When this happens, we are passed in the relid of the originally
1515  * found relation (which we have locked). As the planner will look up
1516  * the relation again, we double-check here to make sure it found the
1517  * same one that we have locked.
1518  */
1519  if (queryRelId != InvalidOid)
1520  {
1521  /*
1522  * Note that with RLS involved there may be multiple relations,
1523  * and while the one we need is almost certainly first, we don't
1524  * make any guarantees of that in the planner, so check the whole
1525  * list and make sure we find the original relation.
1526  */
1527  if (!list_member_oid(plan->relationOids, queryRelId))
1528  ereport(ERROR,
1529  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1530  errmsg("relation referenced by COPY statement has changed")));
1531  }
1532 
1533  /*
1534  * Use a snapshot with an updated command ID to ensure this query sees
1535  * results of any previously executed queries.
1536  */
1539 
1540  /* Create dest receiver for COPY OUT */
1542  ((DR_copy *) dest)->cstate = cstate;
1543 
1544  /* Create a QueryDesc requesting no output */
1545  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1548  dest, NULL, NULL, 0);
1549 
1550  /*
1551  * Call ExecutorStart to prepare the plan for execution.
1552  *
1553  * ExecutorStart computes a result tupdesc for us
1554  */
1555  ExecutorStart(cstate->queryDesc, 0);
1556 
1557  tupDesc = cstate->queryDesc->tupDesc;
1558  }
1559 
1560  /* Generate or convert list of attributes to process */
1561  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1562 
1563  num_phys_attrs = tupDesc->natts;
1564 
1565  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1566  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1567  if (cstate->force_quote_all)
1568  {
1569  int i;
1570 
1571  for (i = 0; i < num_phys_attrs; i++)
1572  cstate->force_quote_flags[i] = true;
1573  }
1574  else if (cstate->force_quote)
1575  {
1576  List *attnums;
1577  ListCell *cur;
1578 
1579  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1580 
1581  foreach(cur, attnums)
1582  {
1583  int attnum = lfirst_int(cur);
1584  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1585 
1586  if (!list_member_int(cstate->attnumlist, attnum))
1587  ereport(ERROR,
1588  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1589  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1590  NameStr(attr->attname))));
1591  cstate->force_quote_flags[attnum - 1] = true;
1592  }
1593  }
1594 
1595  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1596  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1597  if (cstate->force_notnull)
1598  {
1599  List *attnums;
1600  ListCell *cur;
1601 
1602  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1603 
1604  foreach(cur, attnums)
1605  {
1606  int attnum = lfirst_int(cur);
1607  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1608 
1609  if (!list_member_int(cstate->attnumlist, attnum))
1610  ereport(ERROR,
1611  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1612  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1613  NameStr(attr->attname))));
1614  cstate->force_notnull_flags[attnum - 1] = true;
1615  }
1616  }
1617 
1618  /* Convert FORCE_NULL name list to per-column flags, check validity */
1619  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1620  if (cstate->force_null)
1621  {
1622  List *attnums;
1623  ListCell *cur;
1624 
1625  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1626 
1627  foreach(cur, attnums)
1628  {
1629  int attnum = lfirst_int(cur);
1630  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1631 
1632  if (!list_member_int(cstate->attnumlist, attnum))
1633  ereport(ERROR,
1634  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1635  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1636  NameStr(attr->attname))));
1637  cstate->force_null_flags[attnum - 1] = true;
1638  }
1639  }
1640 
1641  /* Convert convert_selectively name list to per-column flags */
1642  if (cstate->convert_selectively)
1643  {
1644  List *attnums;
1645  ListCell *cur;
1646 
1647  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1648 
1649  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1650 
1651  foreach(cur, attnums)
1652  {
1653  int attnum = lfirst_int(cur);
1654  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1655 
1656  if (!list_member_int(cstate->attnumlist, attnum))
1657  ereport(ERROR,
1658  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1659  errmsg_internal("selected column \"%s\" not referenced by COPY",
1660  NameStr(attr->attname))));
1661  cstate->convert_select_flags[attnum - 1] = true;
1662  }
1663  }
1664 
1665  /* Use client encoding when ENCODING option is not specified. */
1666  if (cstate->file_encoding < 0)
1668 
1669  /*
1670  * Set up encoding conversion info. Even if the file and server encodings
1671  * are the same, we must apply pg_any_to_server() to validate data in
1672  * multibyte encodings.
1673  */
1674  cstate->need_transcoding =
1675  (cstate->file_encoding != GetDatabaseEncoding() ||
1677  /* See Multibyte encoding comment above */
1679 
1680  cstate->copy_dest = COPY_FILE; /* default */
1681 
1682  MemoryContextSwitchTo(oldcontext);
1683 
1684  return cstate;
1685 }
#define NIL
Definition: pg_list.h:69
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:781
#define IsA(nodeptr, _type_)
Definition: nodes.h:563
#define RelationGetDescr(relation)
Definition: rel.h:437
bool need_transcoding
Definition: copy.c:106
List * attnumlist
Definition: copy.c:112
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:90
List * relationOids
Definition: plannodes.h:88
Definition: copy.c:215
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:140
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1014
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:575
bool * force_quote_flags
Definition: copy.c:129
Form_pg_class rd_rel
Definition: rel.h:114
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:649
Node * utilityStmt
Definition: parsenodes.h:118
#define linitial_node(type, l)
Definition: pg_list.h:114
bool * force_null_flags
Definition: copy.c:133
int natts
Definition: tupdesc.h:79
bool * convert_select_flags
Definition: copy.c:136
CopyDest copy_dest
Definition: copy.c:99
Definition: copy.c:63
Relation rel
Definition: copy.c:110
MemoryContext copycontext
Definition: copy.c:147
#define ERROR
Definition: elog.h:43
#define lfirst_int(lc)
Definition: pg_list.h:107
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:769
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:197
#define lfirst_node(type, lc)
Definition: pg_list.h:109
QueryDesc * queryDesc
Definition: copy.c:111
bool list_member_int(const List *list, int datum)
Definition: list.c:485
bool encoding_embeds_ascii
Definition: copy.c:107
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
#define RelationGetRelationName(relation)
Definition: rel.h:445
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
const char * p_sourcetext
Definition: parse_node.h:172
List * returningList
Definition: parsenodes.h:144
#define ereport(elevel, rest)
Definition: elog.h:122
List * force_null
Definition: copy.c:132
int file_encoding
Definition: copy.c:105
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4698
#define AllocSetContextCreate(parent, name, allocparams)
Definition: memutils.h:165
TupleDesc tupDesc
Definition: execdesc.h:47
#define InvalidSnapshot
Definition: snapshot.h:25
void * palloc0(Size size)
Definition: mcxt.c:864
int GetDatabaseEncoding(void)
Definition: mbutils.c:1004
int pg_get_client_encoding(void)
Definition: mbutils.c:306
#define InvalidOid
Definition: postgres_ext.h:36
bool * force_notnull_flags
Definition: copy.c:131
CmdType commandType
Definition: parsenodes.h:110
List * force_notnull
Definition: copy.c:130
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:505
QuerySource querySource
Definition: parsenodes.h:112
#define Assert(condition)
Definition: c.h:680
List * convert_select
Definition: copy.c:135
static int list_length(const List *l)
Definition: pg_list.h:89
bool force_quote_all
Definition: copy.c:128
#define PG_ENCODING_IS_CLIENT_ONLY(_enc)
Definition: pg_wchar.h:298
#define CURSOR_OPT_PARALLEL_OK
Definition: parsenodes.h:2650
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
#define NameStr(name)
Definition: c.h:557
#define copyObject(obj)
Definition: nodes.h:625
Definition: pg_list.h:45
List * force_quote
Definition: copy.c:127
bool convert_selectively
Definition: copy.c:134
bool oids
Definition: copy.c:117
PlannedStmt * pg_plan_query(Query *querytree, int cursorOptions, ParamListInfo boundParams)
Definition: postgres.c:790

◆ BeginCopyFrom()

CopyState BeginCopyFrom ( ParseState pstate,
Relation  rel,
const char *  filename,
bool  is_program,
copy_data_source_cb  data_source_cb,
List attnamelist,
List options 
)

Definition at line 2943 of file copy.c.

References AllocateFile(), Assert, CopyStateData::attnumlist, CopyStateData::attribute_buf, BeginCopy(), CopyStateData::binary, BinarySignature, build_column_default(), contain_volatile_functions_not_nextval(), COPY_CALLBACK, CopyStateData::copy_dest, CopyStateData::copy_file, CopyStateData::copycontext, CopyGetData(), CopyGetInt32(), CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, CopyStateData::data_source_cb, CopyStateData::defexprs, CopyStateData::defmap, DestRemote, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, ExecInitExpr(), expression_planner(), CopyStateData::fe_eof, CopyStateData::file_has_oids, CopyStateData::filename, fmgr_info(), getOwnedSequence(), getTypeBinaryInputInfo(), getTypeInputInfo(), CopyStateData::in_functions, initStringInfo(), InvalidOid, CopyStateData::is_program, CopyStateData::line_buf, CopyStateData::line_buf_converted, list_length(), list_member_int(), makeNode, CopyStateData::max_fields, MemoryContextSwitchTo(), tupleDesc::natts, CopyStateData::num_defaults, CopyStateData::oid_in_function, CopyStateData::oid_typioparam, OIDOID, CopyStateData::oids, OpenPipeStream(), ParseState::p_rtable, palloc(), PG_BINARY_R, pstrdup(), CopyStateData::range_table, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, RAW_BUF_SIZE, CopyStateData::raw_fields, ReceiveCopyBegin(), CopyStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, S_ISDIR, NextValueExpr::seqid, stat, TupleDescAttr, NextValueExpr::typeId, CopyStateData::typioparams, CopyStateData::volatile_defexprs, and whereToSendOutput.

Referenced by copy_table(), DoCopy(), file_acquire_sample_rows(), fileBeginForeignScan(), and fileReScanForeignScan().

2950 {
2951  CopyState cstate;
2952  bool pipe = (filename == NULL);
2953  TupleDesc tupDesc;
2954  AttrNumber num_phys_attrs,
2955  num_defaults;
2956  FmgrInfo *in_functions;
2957  Oid *typioparams;
2958  int attnum;
2959  Oid in_func_oid;
2960  int *defmap;
2961  ExprState **defexprs;
2962  MemoryContext oldcontext;
2963  bool volatile_defexprs;
2964 
2965  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
2966  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
2967 
2968  /* Initialize state variables */
2969  cstate->fe_eof = false;
2970  cstate->eol_type = EOL_UNKNOWN;
2971  cstate->cur_relname = RelationGetRelationName(cstate->rel);
2972  cstate->cur_lineno = 0;
2973  cstate->cur_attname = NULL;
2974  cstate->cur_attval = NULL;
2975 
2976  /* Set up variables to avoid per-attribute overhead. */
2977  initStringInfo(&cstate->attribute_buf);
2978  initStringInfo(&cstate->line_buf);
2979  cstate->line_buf_converted = false;
2980  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
2981  cstate->raw_buf_index = cstate->raw_buf_len = 0;
2982 
2983  /* Assign range table, we'll need it in CopyFrom. */
2984  if (pstate)
2985  cstate->range_table = pstate->p_rtable;
2986 
2987  tupDesc = RelationGetDescr(cstate->rel);
2988  num_phys_attrs = tupDesc->natts;
2989  num_defaults = 0;
2990  volatile_defexprs = false;
2991 
2992  /*
2993  * Pick up the required catalog information for each attribute in the
2994  * relation, including the input function, the element type (to pass to
2995  * the input function), and info about defaults and constraints. (Which
2996  * input function we use depends on text/binary format choice.)
2997  */
2998  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2999  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3000  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3001  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3002 
3003  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3004  {
3005  Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
3006 
3007  /* We don't need info for dropped attributes */
3008  if (att->attisdropped)
3009  continue;
3010 
3011  /* Fetch the input function and typioparam info */
3012  if (cstate->binary)
3013  getTypeBinaryInputInfo(att->atttypid,
3014  &in_func_oid, &typioparams[attnum - 1]);
3015  else
3016  getTypeInputInfo(att->atttypid,
3017  &in_func_oid, &typioparams[attnum - 1]);
3018  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3019 
3020  /* Get default info if needed */
3021  if (!list_member_int(cstate->attnumlist, attnum))
3022  {
3023  /* attribute is NOT to be copied from input */
3024  /* use default value if one exists */
3025  Expr *defexpr;
3026 
3027  if (att->attidentity)
3028  {
3030 
3031  nve->seqid = getOwnedSequence(RelationGetRelid(cstate->rel),
3032  attnum);
3033  nve->typeId = att->atttypid;
3034  defexpr = (Expr *) nve;
3035  }
3036  else
3037  defexpr = (Expr *) build_column_default(cstate->rel, attnum);
3038 
3039  if (defexpr != NULL)
3040  {
3041  /* Run the expression through planner */
3042  defexpr = expression_planner(defexpr);
3043 
3044  /* Initialize executable expression in copycontext */
3045  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3046  defmap[num_defaults] = attnum - 1;
3047  num_defaults++;
3048 
3049  /*
3050  * If a default expression looks at the table being loaded,
3051  * then it could give the wrong answer when using
3052  * multi-insert. Since database access can be dynamic this is
3053  * hard to test for exactly, so we use the much wider test of
3054  * whether the default expression is volatile. We allow for
3055  * the special case of when the default expression is the
3056  * nextval() of a sequence which in this specific case is
3057  * known to be safe for use with the multi-insert
3058  * optimization. Hence we use this special case function
3059  * checker rather than the standard check for
3060  * contain_volatile_functions().
3061  */
3062  if (!volatile_defexprs)
3063  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3064  }
3065  }
3066  }
3067 
3068  /* We keep those variables in cstate. */
3069  cstate->in_functions = in_functions;
3070  cstate->typioparams = typioparams;
3071  cstate->defmap = defmap;
3072  cstate->defexprs = defexprs;
3073  cstate->volatile_defexprs = volatile_defexprs;
3074  cstate->num_defaults = num_defaults;
3075  cstate->is_program = is_program;
3076 
3077  if (data_source_cb)
3078  {
3079  cstate->copy_dest = COPY_CALLBACK;
3080  cstate->data_source_cb = data_source_cb;
3081  }
3082  else if (pipe)
3083  {
3084  Assert(!is_program); /* the grammar does not allow this */
3086  ReceiveCopyBegin(cstate);
3087  else
3088  cstate->copy_file = stdin;
3089  }
3090  else
3091  {
3092  cstate->filename = pstrdup(filename);
3093 
3094  if (cstate->is_program)
3095  {
3096  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3097  if (cstate->copy_file == NULL)
3098  ereport(ERROR,
3100  errmsg("could not execute command \"%s\": %m",
3101  cstate->filename)));
3102  }
3103  else
3104  {
3105  struct stat st;
3106 
3107  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3108  if (cstate->copy_file == NULL)
3109  {
3110  /* copy errno because ereport subfunctions might change it */
3111  int save_errno = errno;
3112 
3113  ereport(ERROR,
3115  errmsg("could not open file \"%s\" for reading: %m",
3116  cstate->filename),
3117  (save_errno == ENOENT || save_errno == EACCES) ?
3118  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3119  "You may want a client-side facility such as psql's \\copy.") : 0));
3120  }
3121 
3122  if (fstat(fileno(cstate->copy_file), &st))
3123  ereport(ERROR,
3125  errmsg("could not stat file \"%s\": %m",
3126  cstate->filename)));
3127 
3128  if (S_ISDIR(st.st_mode))
3129  ereport(ERROR,
3130  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3131  errmsg("\"%s\" is a directory", cstate->filename)));
3132  }
3133  }
3134 
3135  if (!cstate->binary)
3136  {
3137  /* must rely on user to tell us... */
3138  cstate->file_has_oids = cstate->oids;
3139  }
3140  else
3141  {
3142  /* Read and verify binary header */
3143  char readSig[11];
3144  int32 tmp;
3145 
3146  /* Signature */
3147  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3148  memcmp(readSig, BinarySignature, 11) != 0)
3149  ereport(ERROR,
3150  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3151  errmsg("COPY file signature not recognized")));
3152  /* Flags field */
3153  if (!CopyGetInt32(cstate, &tmp))
3154  ereport(ERROR,
3155  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3156  errmsg("invalid COPY file header (missing flags)")));
3157  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3158  tmp &= ~(1 << 16);
3159  if ((tmp >> 16) != 0)
3160  ereport(ERROR,
3161  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3162  errmsg("unrecognized critical flags in COPY file header")));
3163  /* Header extension length */
3164  if (!CopyGetInt32(cstate, &tmp) ||
3165  tmp < 0)
3166  ereport(ERROR,
3167  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3168  errmsg("invalid COPY file header (missing length)")));
3169  /* Skip extension header, if present */
3170  while (tmp-- > 0)
3171  {
3172  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3173  ereport(ERROR,
3174  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3175  errmsg("invalid COPY file header (wrong length)")));
3176  }
3177  }
3178 
3179  if (cstate->file_has_oids && cstate->binary)
3180  {
3182  &in_func_oid, &cstate->oid_typioparam);
3183  fmgr_info(in_func_oid, &cstate->oid_in_function);
3184  }
3185 
3186  /* create workspace for CopyReadAttributes results */
3187  if (!cstate->binary)
3188  {
3189  AttrNumber attr_count = list_length(cstate->attnumlist);
3190  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3191 
3192  cstate->max_fields = nfields;
3193  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3194  }
3195 
3196  MemoryContextSwitchTo(oldcontext);
3197 
3198  return cstate;
3199 }
Definition: fmgr.h:56
Oid getOwnedSequence(Oid relid, AttrNumber attnum)
Definition: pg_depend.c:605
List * range_table
Definition: copy.c:167
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1372
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:1010
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:682
int errhint(const char *fmt,...)
Definition: elog.c:987
char ** raw_fields
Definition: copy.c:188
bool binary
Definition: copy.c:116
#define RelationGetDescr(relation)
Definition: rel.h:437
AttrNumber num_defaults
Definition: copy.c:158
FmgrInfo * in_functions
Definition: copy.c:162
List * attnumlist
Definition: copy.c:112
#define OIDOID
Definition: pg_type.h:328
char * filename
Definition: copy.c:113
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:90
bool file_has_oids
Definition: copy.c:159
char * pstrdup(const char *in)
Definition: mcxt.c:1063
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:380
Expr * expression_planner(Expr *expr)
Definition: planner.c:6026
StringInfoData line_buf
Definition: copy.c:197
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Definition: nodes.h:512
int raw_buf_index
Definition: copy.c:210
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:103
unsigned int Oid
Definition: postgres_ext.h:31
bool volatile_defexprs
Definition: copy.c:166
#define PG_BINARY_R
Definition: c.h:1051
int natts
Definition: tupdesc.h:79
bool line_buf_converted
Definition: copy.c:198
signed int int32
Definition: c.h:294
CopyDest copy_dest
Definition: copy.c:99
const char * cur_attname
Definition: copy.c:141
Relation rel
Definition: copy.c:110
MemoryContext copycontext
Definition: copy.c:147
copy_data_source_cb data_source_cb
Definition: copy.c:115
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:122
bool list_member_int(const List *list, int datum)
Definition: list.c:485
char * raw_buf
Definition: copy.c:209
ExprState ** defexprs
Definition: copy.c:165
const char * cur_relname
Definition: copy.c:139
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2343
FmgrInfo oid_in_function
Definition: copy.c:160
#define RelationGetRelationName(relation)
Definition: rel.h:445
static const char BinarySignature[11]
Definition: copy.c:288
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
int raw_buf_len
Definition: copy.c:211
int max_fields
Definition: copy.c:187
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2442
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2698
#define ereport(elevel, rest)
Definition: elog.h:122
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2632
void initStringInfo(StringInfo str)
Definition: stringinfo.c:46
#define stat(a, b)
Definition: win32_port.h:266
Oid * typioparams
Definition: copy.c:163
bool is_program
Definition: copy.c:114
Node * build_column_default(Relation rel, int attrno)
#define RAW_BUF_SIZE
Definition: copy.c:208
#define InvalidOid
Definition: postgres_ext.h:36
#define makeNode(_type_)
Definition: nodes.h:560
EolType eol_type
Definition: copy.c:104
#define Assert(condition)
Definition: c.h:680
static int list_length(const List *l)
Definition: pg_list.h:89
#define S_ISDIR(m)
Definition: win32_port.h:307
static char * filename
Definition: pg_dumpall.c:90
void * palloc(Size size)
Definition: mcxt.c:835
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:100
int cur_lineno
Definition: copy.c:140
StringInfoData attribute_buf
Definition: copy.c:183
ExprState * ExecInitExpr(Expr *node, PlanState *parent)
Definition: execExpr.c:118
const char * cur_attval
Definition: copy.c:142
CommandDest whereToSendOutput
Definition: postgres.c:88
Oid oid_typioparam
Definition: copy.c:161
int16 AttrNumber
Definition: attnum.h:21
#define RelationGetRelid(relation)
Definition: rel.h:425
int * defmap
Definition: copy.c:164
List * p_rtable
Definition: parse_node.h:173
bool oids
Definition: copy.c:117

◆ BeginCopyTo()

static CopyState BeginCopyTo ( ParseState pstate,
Relation  rel,
RawStmt query,
Oid  queryRelId,
const char *  filename,
bool  is_program,
List attnamelist,
List options 
)
static

Definition at line 1737 of file copy.c.

References AllocateFile(), Assert, BeginCopy(), CopyStateData::copy_file, CopyStateData::copycontext, DestRemote, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, CopyStateData::filename, is_absolute_path, CopyStateData::is_program, MemoryContextSwitchTo(), OpenPipeStream(), PG_BINARY_W, PG_CATCH, PG_END_TRY, PG_RE_THROW, PG_TRY, pstrdup(), RelationData::rd_rel, RelationGetRelationName, RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, S_ISDIR, S_IWGRP, S_IWOTH, stat, and whereToSendOutput.

Referenced by DoCopy().

1745 {
1746  CopyState cstate;
1747  bool pipe = (filename == NULL);
1748  MemoryContext oldcontext;
1749 
1750  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1751  {
1752  if (rel->rd_rel->relkind == RELKIND_VIEW)
1753  ereport(ERROR,
1754  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1755  errmsg("cannot copy from view \"%s\"",
1757  errhint("Try the COPY (SELECT ...) TO variant.")));
1758  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1759  ereport(ERROR,
1760  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1761  errmsg("cannot copy from materialized view \"%s\"",
1763  errhint("Try the COPY (SELECT ...) TO variant.")));
1764  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1765  ereport(ERROR,
1766  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1767  errmsg("cannot copy from foreign table \"%s\"",
1769  errhint("Try the COPY (SELECT ...) TO variant.")));
1770  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1771  ereport(ERROR,
1772  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1773  errmsg("cannot copy from sequence \"%s\"",
1774  RelationGetRelationName(rel))));
1775  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1776  ereport(ERROR,
1777  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1778  errmsg("cannot copy from partitioned table \"%s\"",
1780  errhint("Try the COPY (SELECT ...) TO variant.")));
1781  else
1782  ereport(ERROR,
1783  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1784  errmsg("cannot copy from non-table relation \"%s\"",
1785  RelationGetRelationName(rel))));
1786  }
1787 
1788  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1789  options);
1790  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1791 
1792  if (pipe)
1793  {
1794  Assert(!is_program); /* the grammar does not allow this */
1796  cstate->copy_file = stdout;
1797  }
1798  else
1799  {
1800  cstate->filename = pstrdup(filename);
1801  cstate->is_program = is_program;
1802 
1803  if (is_program)
1804  {
1805  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1806  if (cstate->copy_file == NULL)
1807  ereport(ERROR,
1809  errmsg("could not execute command \"%s\": %m",
1810  cstate->filename)));
1811  }
1812  else
1813  {
1814  mode_t oumask; /* Pre-existing umask value */
1815  struct stat st;
1816 
1817  /*
1818  * Prevent write to relative path ... too easy to shoot oneself in
1819  * the foot by overwriting a database file ...
1820  */
1821  if (!is_absolute_path(filename))
1822  ereport(ERROR,
1823  (errcode(ERRCODE_INVALID_NAME),
1824  errmsg("relative path not allowed for COPY to file")));
1825 
1826  oumask = umask(S_IWGRP | S_IWOTH);
1827  PG_TRY();
1828  {
1829  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1830  }
1831  PG_CATCH();
1832  {
1833  umask(oumask);
1834  PG_RE_THROW();
1835  }
1836  PG_END_TRY();
1837  umask(oumask);
1838  if (cstate->copy_file == NULL)
1839  {
1840  /* copy errno because ereport subfunctions might change it */
1841  int save_errno = errno;
1842 
1843  ereport(ERROR,
1845  errmsg("could not open file \"%s\" for writing: %m",
1846  cstate->filename),
1847  (save_errno == ENOENT || save_errno == EACCES) ?
1848  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1849  "You may want a client-side facility such as psql's \\copy.") : 0));
1850  }
1851 
1852  if (fstat(fileno(cstate->copy_file), &st))
1853  ereport(ERROR,
1855  errmsg("could not stat file \"%s\": %m",
1856  cstate->filename)));
1857 
1858  if (S_ISDIR(st.st_mode))
1859  ereport(ERROR,
1860  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1861  errmsg("\"%s\" is a directory", cstate->filename)));
1862  }
1863  }
1864 
1865  MemoryContextSwitchTo(oldcontext);
1866 
1867  return cstate;
1868 }
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1372
int errhint(const char *fmt,...)
Definition: elog.c:987
char * filename
Definition: copy.c:113
char * pstrdup(const char *in)
Definition: mcxt.c:1063
#define S_IWOTH
Definition: win32_port.h:298
#define RELKIND_MATVIEW
Definition: pg_class.h:165
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
int errcode(int sqlerrcode)
Definition: elog.c:575
#define PG_BINARY_W
Definition: c.h:1052
Form_pg_class rd_rel
Definition: rel.h:114
MemoryContext copycontext
Definition: copy.c:147
#define ERROR
Definition: elog.h:43
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:86
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2343
#define RelationGetRelationName(relation)
Definition: rel.h:445
#define S_IWGRP
Definition: win32_port.h:286
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2442
#define ereport(elevel, rest)
Definition: elog.h:122
#define stat(a, b)
Definition: win32_port.h:266
bool is_program
Definition: copy.c:114
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
#define PG_CATCH()
Definition: elog.h:293
#define Assert(condition)
Definition: c.h:680
#define PG_RE_THROW()
Definition: elog.h:314
#define S_ISDIR(m)
Definition: win32_port.h:307
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
FILE * copy_file
Definition: copy.c:100
CommandDest whereToSendOutput
Definition: postgres.c:88
#define PG_TRY()
Definition: elog.h:284
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
#define PG_END_TRY()
Definition: elog.h:300

◆ ClosePipeToProgram()

static void ClosePipeToProgram ( CopyState  cstate)
static

Definition at line 1691 of file copy.c.

References Assert, ClosePipeStream(), CopyStateData::copy_file, ereport, errcode(), errcode_for_file_access(), errdetail_internal(), errmsg(), ERROR, CopyStateData::filename, CopyStateData::is_program, and wait_result_to_str().

Referenced by CopySendEndOfRow(), and EndCopy().

1692 {
1693  int pclose_rc;
1694 
1695  Assert(cstate->is_program);
1696 
1697  pclose_rc = ClosePipeStream(cstate->copy_file);
1698  if (pclose_rc == -1)
1699  ereport(ERROR,
1701  errmsg("could not close pipe to external command: %m")));
1702  else if (pclose_rc != 0)
1703  ereport(ERROR,
1704  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1705  errmsg("program \"%s\" failed",
1706  cstate->filename),
1707  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1708 }
char * filename
Definition: copy.c:113
int errcode(int sqlerrcode)
Definition: elog.c:575
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
int ClosePipeStream(FILE *file)
Definition: fd.c:2745
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
#define ERROR
Definition: elog.h:43
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:114
#define Assert(condition)
Definition: c.h:680
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:100

◆ copy_dest_destroy()

static void copy_dest_destroy ( DestReceiver self)
static

Definition at line 4808 of file copy.c.

References pfree().

Referenced by CreateCopyDestReceiver().

4809 {
4810  pfree(self);
4811 }
void pfree(void *pointer)
Definition: mcxt.c:936

◆ copy_dest_receive()

static bool copy_dest_receive ( TupleTableSlot slot,
DestReceiver self 
)
static

Definition at line 4780 of file copy.c.

References CopyOneRowTo(), DR_copy::cstate, InvalidOid, DR_copy::processed, slot_getallattrs(), TupleTableSlot::tts_isnull, and TupleTableSlot::tts_values.

Referenced by CreateCopyDestReceiver().

4781 {
4782  DR_copy *myState = (DR_copy *) self;
4783  CopyState cstate = myState->cstate;
4784 
4785  /* Make sure the tuple is fully deconstructed */
4786  slot_getallattrs(slot);
4787 
4788  /* And send the data */
4789  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4790  myState->processed++;
4791 
4792  return true;
4793 }
Definition: copy.c:215
CopyState cstate
Definition: copy.c:218
Datum * tts_values
Definition: tuptable.h:125
bool * tts_isnull
Definition: tuptable.h:126
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1238
#define InvalidOid
Definition: postgres_ext.h:36
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2081
uint64 processed
Definition: copy.c:219

◆ copy_dest_shutdown()

static void copy_dest_shutdown ( DestReceiver self)
static

Definition at line 4799 of file copy.c.

Referenced by CreateCopyDestReceiver().

4800 {
4801  /* no-op */
4802 }

◆ copy_dest_startup()

static void copy_dest_startup ( DestReceiver self,
int  operation,
TupleDesc  typeinfo 
)
static

Definition at line 4771 of file copy.c.

Referenced by CreateCopyDestReceiver().

4772 {
4773  /* no-op */
4774 }

◆ CopyAttributeOutCSV()

static void CopyAttributeOutCSV ( CopyState  cstate,
char *  string,
bool  use_quote,
bool  single_attr 
)
static

Definition at line 4607 of file copy.c.

References CopySendChar(), CopySendString(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::escape, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, CopyStateData::null_print, pg_encoding_mblen(), pg_server_to_any(), and CopyStateData::quote.

Referenced by CopyOneRowTo(), and CopyTo().

4609 {
4610  char *ptr;
4611  char *start;
4612  char c;
4613  char delimc = cstate->delim[0];
4614  char quotec = cstate->quote[0];
4615  char escapec = cstate->escape[0];
4616 
4617  /* force quoting if it matches null_print (before conversion!) */
4618  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4619  use_quote = true;
4620 
4621  if (cstate->need_transcoding)
4622  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4623  else
4624  ptr = string;
4625 
4626  /*
4627  * Make a preliminary pass to discover if it needs quoting
4628  */
4629  if (!use_quote)
4630  {
4631  /*
4632  * Because '\.' can be a data value, quote it if it appears alone on a
4633  * line so it is not interpreted as the end-of-data marker.
4634  */
4635  if (single_attr && strcmp(ptr, "\\.") == 0)
4636  use_quote = true;
4637  else
4638  {
4639  char *tptr = ptr;
4640 
4641  while ((c = *tptr) != '\0')
4642  {
4643  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4644  {
4645  use_quote = true;
4646  break;
4647  }
4648  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4649  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4650  else
4651  tptr++;
4652  }
4653  }
4654  }
4655 
4656  if (use_quote)
4657  {
4658  CopySendChar(cstate, quotec);
4659 
4660  /*
4661  * We adopt the same optimization strategy as in CopyAttributeOutText
4662  */
4663  start = ptr;
4664  while ((c = *ptr) != '\0')
4665  {
4666  if (c == quotec || c == escapec)
4667  {
4668  DUMPSOFAR();
4669  CopySendChar(cstate, escapec);
4670  start = ptr; /* we include char in next run */
4671  }
4672  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4673  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4674  else
4675  ptr++;
4676  }
4677  DUMPSOFAR();
4678 
4679  CopySendChar(cstate, quotec);
4680  }
4681  else
4682  {
4683  /* If it doesn't need quoting, we can just dump it as-is */
4684  CopySendString(cstate, ptr);
4685  }
4686 }
bool need_transcoding
Definition: copy.c:106
char * delim
Definition: copy.c:124
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:634
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * null_print
Definition: copy.c:121
#define IS_HIGHBIT_SET(ch)
Definition: c.h:963
bool encoding_embeds_ascii
Definition: copy.c:107
char * c
char * quote
Definition: copy.c:125
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
char * escape
Definition: copy.c:126
int file_encoding
Definition: copy.c:105
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
#define DUMPSOFAR()
Definition: copy.c:4447

◆ CopyAttributeOutText()

static void CopyAttributeOutText ( CopyState  cstate,
char *  string 
)
static

Definition at line 4454 of file copy.c.

References CopySendChar(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, pg_encoding_mblen(), and pg_server_to_any().

Referenced by CopyOneRowTo().

4455 {
4456  char *ptr;
4457  char *start;
4458  char c;
4459  char delimc = cstate->delim[0];
4460 
4461  if (cstate->need_transcoding)
4462  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4463  else
4464  ptr = string;
4465 
4466  /*
4467  * We have to grovel through the string searching for control characters
4468  * and instances of the delimiter character. In most cases, though, these
4469  * are infrequent. To avoid overhead from calling CopySendData once per
4470  * character, we dump out all characters between escaped characters in a
4471  * single call. The loop invariant is that the data from "start" to "ptr"
4472  * can be sent literally, but hasn't yet been.
4473  *
4474  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4475  * in valid backend encodings, extra bytes of a multibyte character never
4476  * look like ASCII. This loop is sufficiently performance-critical that
4477  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4478  * of the normal safe-encoding path.
4479  */
4480  if (cstate->encoding_embeds_ascii)
4481  {
4482  start = ptr;
4483  while ((c = *ptr) != '\0')
4484  {
4485  if ((unsigned char) c < (unsigned char) 0x20)
4486  {
4487  /*
4488  * \r and \n must be escaped, the others are traditional. We
4489  * prefer to dump these using the C-like notation, rather than
4490  * a backslash and the literal character, because it makes the
4491  * dump file a bit more proof against Microsoftish data
4492  * mangling.
4493  */
4494  switch (c)
4495  {
4496  case '\b':
4497  c = 'b';
4498  break;
4499  case '\f':
4500  c = 'f';
4501  break;
4502  case '\n':
4503  c = 'n';
4504  break;
4505  case '\r':
4506  c = 'r';
4507  break;
4508  case '\t':
4509  c = 't';
4510  break;
4511  case '\v':
4512  c = 'v';
4513  break;
4514  default:
4515  /* If it's the delimiter, must backslash it */
4516  if (c == delimc)
4517  break;
4518  /* All ASCII control chars are length 1 */
4519  ptr++;
4520  continue; /* fall to end of loop */
4521  }
4522  /* if we get here, we need to convert the control char */
4523  DUMPSOFAR();
4524  CopySendChar(cstate, '\\');
4525  CopySendChar(cstate, c);
4526  start = ++ptr; /* do not include char in next run */
4527  }
4528  else if (c == '\\' || c == delimc)
4529  {
4530  DUMPSOFAR();
4531  CopySendChar(cstate, '\\');
4532  start = ptr++; /* we include char in next run */
4533  }
4534  else if (IS_HIGHBIT_SET(c))
4535  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4536  else
4537  ptr++;
4538  }
4539  }
4540  else
4541  {
4542  start = ptr;
4543  while ((c = *ptr) != '\0')
4544  {
4545  if ((unsigned char) c < (unsigned char) 0x20)
4546  {
4547  /*
4548  * \r and \n must be escaped, the others are traditional. We
4549  * prefer to dump these using the C-like notation, rather than
4550  * a backslash and the literal character, because it makes the
4551  * dump file a bit more proof against Microsoftish data
4552  * mangling.
4553  */
4554  switch (c)
4555  {
4556  case '\b':
4557  c = 'b';
4558  break;
4559  case '\f':
4560  c = 'f';
4561  break;
4562  case '\n':
4563  c = 'n';
4564  break;
4565  case '\r':
4566  c = 'r';
4567  break;
4568  case '\t':
4569  c = 't';
4570  break;
4571  case '\v':
4572  c = 'v';
4573  break;
4574  default:
4575  /* If it's the delimiter, must backslash it */
4576  if (c == delimc)
4577  break;
4578  /* All ASCII control chars are length 1 */
4579  ptr++;
4580  continue; /* fall to end of loop */
4581  }
4582  /* if we get here, we need to convert the control char */
4583  DUMPSOFAR();
4584  CopySendChar(cstate, '\\');
4585  CopySendChar(cstate, c);
4586  start = ++ptr; /* do not include char in next run */
4587  }
4588  else if (c == '\\' || c == delimc)
4589  {
4590  DUMPSOFAR();
4591  CopySendChar(cstate, '\\');
4592  start = ptr++; /* we include char in next run */
4593  }
4594  else
4595  ptr++;
4596  }
4597  }
4598 
4599  DUMPSOFAR();
4600 }
bool need_transcoding
Definition: copy.c:106
char * delim
Definition: copy.c:124
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:634
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
#define IS_HIGHBIT_SET(ch)
Definition: c.h:963
bool encoding_embeds_ascii
Definition: copy.c:107
char * c
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
int file_encoding
Definition: copy.c:105
#define DUMPSOFAR()
Definition: copy.c:4447

◆ CopyFrom()

uint64 CopyFrom ( CopyState  cstate)

Definition at line 2280 of file copy.c.

References AfterTriggerBeginQuery(), AfterTriggerEndQuery(), ErrorContextCallback::arg, Assert, ErrorContextCallback::callback, CHECK_FOR_INTERRUPTS, CMD_INSERT, tupleDesc::constr, convert_tuples_by_name(), CopyStateData::copy_dest, COPY_OLD_FE, CopyFromErrorCallback(), CopyFromInsertBatch(), CreateExecutorState(), CopyStateData::cur_lineno, CurrentMemoryContext, do_convert_tuple(), ereport, errcode(), errhint(), errmsg(), ERROR, error_context_stack, EState::es_num_result_relations, EState::es_range_table, EState::es_result_relation_info, EState::es_result_relations, EState::es_trig_tuple_slot, EState::es_tupleTable, ExecARInsertTriggers(), ExecASInsertTriggers(), ExecBRInsertTriggers(), ExecBSInsertTriggers(), ExecCleanUpTriggerState(), ExecCleanupTupleRouting(), ExecCloseIndices(), ExecConstraints(), ExecFindPartition(), ExecInitExtraTupleSlot(), ExecInsertIndexTuples(), ExecIRInsertTriggers(), ExecMaterializeSlot(), ExecOpenIndices(), ExecResetTupleTable(), ExecSetSlotDescriptor(), ExecSetupPartitionTupleRouting(), ExecStoreTuple(), FreeBulkInsertState(), FreeExecutorState(), CopyStateData::freeze, GetBulkInsertState(), GetCurrentCommandId(), GetCurrentSubTransactionId(), GetPerTupleExprContext, GetPerTupleMemoryContext, gettext_noop, heap_form_tuple(), heap_insert(), HEAP_INSERT_FROZEN, HEAP_INSERT_SKIP_FSM, HEAP_INSERT_SKIP_WAL, heap_sync(), HeapTupleSetOid, i, InitResultRelInfo(), InvalidateCatalogSnapshot(), InvalidBuffer, InvalidOid, InvalidSubTransactionId, list_free(), makeNode, MakeTransitionCaptureState(), MAX_BUFFERED_TUPLES, MemoryContextSwitchTo(), tupleDesc::natts, NextCopyFrom(), NIL, PartitionTupleRouting::num_partitions, palloc(), palloc0(), PartitionTupleRouting::partition_dispatch_info, PartitionTupleRouting::partition_tupconv_maps, CopyStateData::partition_tuple_routing, PartitionTupleRouting::partition_tuple_slot, PartitionTupleRouting::partitions, pfree(), pq_endmsgread(), ErrorContextCallback::previous, CopyStateData::range_table, RelationData::rd_att, RelationData::rd_createSubid, RelationData::rd_newRelfilenodeSubid, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, ReleaseBulkInsertStatePin(), RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, ResetPerTupleExprContext, ResultRelInfo::ri_FdwRoutine, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_PartitionCheck, ResultRelInfo::ri_RelationDesc, ResultRelInfo::ri_TrigDesc, HeapTupleData::t_len, HeapTupleData::t_self, HeapTupleData::t_tableOid, TransitionCaptureState::tcs_map, TransitionCaptureState::tcs_original_insert_tuple, ThereAreNoPriorRegisteredSnapshots(), ThereAreNoReadyPortals(), CopyStateData::transition_capture, CopyStateData::transition_tupconv_maps, TriggerDesc::trig_insert_before_row, TriggerDesc::trig_insert_instead_row, RelationData::trigdesc, values, CopyStateData::volatile_defexprs, and XLogIsNeeded.

Referenced by copy_table(), and DoCopy().

2281 {
2282  HeapTuple tuple;
2283  TupleDesc tupDesc;
2284  Datum *values;
2285  bool *nulls;
2286  ResultRelInfo *resultRelInfo;
2287  ResultRelInfo *saved_resultRelInfo = NULL;
2288  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2289  ExprContext *econtext;
2290  TupleTableSlot *myslot;
2291  MemoryContext oldcontext = CurrentMemoryContext;
2292 
2293  ErrorContextCallback errcallback;
2294  CommandId mycid = GetCurrentCommandId(true);
2295  int hi_options = 0; /* start with default heap_insert options */
2296  BulkInsertState bistate;
2297  uint64 processed = 0;
2298  bool useHeapMultiInsert;
2299  int nBufferedTuples = 0;
2300  int prev_leaf_part_index = -1;
2301 
2302 #define MAX_BUFFERED_TUPLES 1000
2303  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2304  Size bufferedTuplesSize = 0;
2305  int firstBufferedLineNo = 0;
2306 
2307  Assert(cstate->rel);
2308 
2309  /*
2310  * The target must be a plain relation or have an INSTEAD OF INSERT row
2311  * trigger. (Currently, such triggers are only allowed on views, so we
2312  * only hint about them in the view case.)
2313  */
2314  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2315  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2316  !(cstate->rel->trigdesc &&
2318  {
2319  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2320  ereport(ERROR,
2321  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2322  errmsg("cannot copy to view \"%s\"",
2323  RelationGetRelationName(cstate->rel)),
2324  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2325  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2326  ereport(ERROR,
2327  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2328  errmsg("cannot copy to materialized view \"%s\"",
2329  RelationGetRelationName(cstate->rel))));
2330  else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
2331  ereport(ERROR,
2332  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2333  errmsg("cannot copy to foreign table \"%s\"",
2334  RelationGetRelationName(cstate->rel))));
2335  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2336  ereport(ERROR,
2337  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2338  errmsg("cannot copy to sequence \"%s\"",
2339  RelationGetRelationName(cstate->rel))));
2340  else
2341  ereport(ERROR,
2342  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2343  errmsg("cannot copy to non-table relation \"%s\"",
2344  RelationGetRelationName(cstate->rel))));
2345  }
2346 
2347  tupDesc = RelationGetDescr(cstate->rel);
2348 
2349  /*----------
2350  * Check to see if we can avoid writing WAL
2351  *
2352  * If archive logging/streaming is not enabled *and* either
2353  * - table was created in same transaction as this COPY
2354  * - data is being written to relfilenode created in this transaction
2355  * then we can skip writing WAL. It's safe because if the transaction
2356  * doesn't commit, we'll discard the table (or the new relfilenode file).
2357  * If it does commit, we'll have done the heap_sync at the bottom of this
2358  * routine first.
2359  *
2360  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2361  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2362  * can be cleared before the end of the transaction. The exact case is
2363  * when a relation sets a new relfilenode twice in same transaction, yet
2364  * the second one fails in an aborted subtransaction, e.g.
2365  *
2366  * BEGIN;
2367  * TRUNCATE t;
2368  * SAVEPOINT save;
2369  * TRUNCATE t;
2370  * ROLLBACK TO save;
2371  * COPY ...
2372  *
2373  * Also, if the target file is new-in-transaction, we assume that checking
2374  * FSM for free space is a waste of time, even if we must use WAL because
2375  * of archiving. This could possibly be wrong, but it's unlikely.
2376  *
2377  * The comments for heap_insert and RelationGetBufferForTuple specify that
2378  * skipping WAL logging is only safe if we ensure that our tuples do not
2379  * go into pages containing tuples from any other transactions --- but this
2380  * must be the case if we have a new table or new relfilenode, so we need
2381  * no additional work to enforce that.
2382  *----------
2383  */
2384  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2385  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2387  {
2388  hi_options |= HEAP_INSERT_SKIP_FSM;
2389  if (!XLogIsNeeded())
2390  hi_options |= HEAP_INSERT_SKIP_WAL;
2391  }
2392 
2393  /*
2394  * Optimize if new relfilenode was created in this subxact or one of its
2395  * committed children and we won't see those rows later as part of an
2396  * earlier scan or command. The subxact test ensures that if this subxact
2397  * aborts then the frozen rows won't be visible after xact cleanup. Note
2398  * that the stronger test of exactly which subtransaction created it is
2399  * crucial for correctness of this optimization. The test for an earlier
2400  * scan or command tolerates false negatives. FREEZE causes other sessions
2401  * to see rows they would not see under MVCC, and a false negative merely
2402  * spreads that anomaly to the current session.
2403  */
2404  if (cstate->freeze)
2405  {
2406  /*
2407  * Tolerate one registration for the benefit of FirstXactSnapshot.
2408  * Scan-bearing queries generally create at least two registrations,
2409  * though relying on that is fragile, as is ignoring ActiveSnapshot.
2410  * Clear CatalogSnapshot to avoid counting its registration. We'll
2411  * still detect ongoing catalog scans, each of which separately
2412  * registers the snapshot it uses.
2413  */
2416  ereport(ERROR,
2417  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2418  errmsg("cannot perform FREEZE because of prior transaction activity")));
2419 
2420  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2422  ereport(ERROR,
2423  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2424  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2425 
2426  hi_options |= HEAP_INSERT_FROZEN;
2427  }
2428 
2429  /*
2430  * We need a ResultRelInfo so we can use the regular executor's
2431  * index-entry-making machinery. (There used to be a huge amount of code
2432  * here that basically duplicated execUtils.c ...)
2433  */
2434  resultRelInfo = makeNode(ResultRelInfo);
2435  InitResultRelInfo(resultRelInfo,
2436  cstate->rel,
2437  1, /* dummy rangetable index */
2438  NULL,
2439  0);
2440 
2441  ExecOpenIndices(resultRelInfo, false);
2442 
2443  estate->es_result_relations = resultRelInfo;
2444  estate->es_num_result_relations = 1;
2445  estate->es_result_relation_info = resultRelInfo;
2446  estate->es_range_table = cstate->range_table;
2447 
2448  /* Set up a tuple slot too */
2449  myslot = ExecInitExtraTupleSlot(estate);
2450  ExecSetSlotDescriptor(myslot, tupDesc);
2451  /* Triggers might need a slot as well */
2452  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);
2453 
2454  /* Prepare to catch AFTER triggers. */
2456 
2457  /*
2458  * If there are any triggers with transition tables on the named relation,
2459  * we need to be prepared to capture transition tuples.
2460  */
2461  cstate->transition_capture =
2463  RelationGetRelid(cstate->rel),
2464  CMD_INSERT);
2465 
2466  /*
2467  * If the named relation is a partitioned table, initialize state for
2468  * CopyFrom tuple routing.
2469  */
2470  if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2471  {
2472  PartitionTupleRouting *proute;
2473 
2474  proute = cstate->partition_tuple_routing =
2475  ExecSetupPartitionTupleRouting(NULL, cstate->rel, 1, estate);
2476 
2477  /*
2478  * If we are capturing transition tuples, they may need to be
2479  * converted from partition format back to partitioned table format
2480  * (this is only ever necessary if a BEFORE trigger modifies the
2481  * tuple).
2482  */
2483  if (cstate->transition_capture != NULL)
2484  {
2485  int i;
2486 
2488  palloc0(sizeof(TupleConversionMap *) * proute->num_partitions);
2489  for (i = 0; i < proute->num_partitions; ++i)
2490  {
2491  cstate->transition_tupconv_maps[i] =
2493  RelationGetDescr(cstate->rel),
2494  gettext_noop("could not convert row type"));
2495  }
2496  }
2497  }
2498 
2499  /*
2500  * It's more efficient to prepare a bunch of tuples for insertion, and
2501  * insert them in one heap_multi_insert() call, than call heap_insert()
2502  * separately for every tuple. However, we can't do that if there are
2503  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2504  * expressions. Such triggers or expressions might query the table we're
2505  * inserting to, and act differently if the tuples that have already been
2506  * processed and prepared for insertion are not there. We also can't do
2507  * it if the table is partitioned.
2508  */
2509  if ((resultRelInfo->ri_TrigDesc != NULL &&
2510  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2511  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2512  cstate->partition_tuple_routing != NULL ||
2513  cstate->volatile_defexprs)
2514  {
2515  useHeapMultiInsert = false;
2516  }
2517  else
2518  {
2519  useHeapMultiInsert = true;
2520  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2521  }
2522 
2523  /*
2524  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2525  * should do this for COPY, since it's not really an "INSERT" statement as
2526  * such. However, executing these triggers maintains consistency with the
2527  * EACH ROW triggers that we already fire on COPY.
2528  */
2529  ExecBSInsertTriggers(estate, resultRelInfo);
2530 
2531  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2532  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2533 
2534  bistate = GetBulkInsertState();
2535  econtext = GetPerTupleExprContext(estate);
2536 
2537  /* Set up callback to identify error line number */
2538  errcallback.callback = CopyFromErrorCallback;
2539  errcallback.arg = (void *) cstate;
2540  errcallback.previous = error_context_stack;
2541  error_context_stack = &errcallback;
2542 
2543  for (;;)
2544  {
2545  TupleTableSlot *slot;
2546  bool skip_tuple;
2547  Oid loaded_oid = InvalidOid;
2548 
2550 
2551  if (nBufferedTuples == 0)
2552  {
2553  /*
2554  * Reset the per-tuple exprcontext. We can only do this if the
2555  * tuple buffer is empty. (Calling the context the per-tuple
2556  * memory context is a bit of a misnomer now.)
2557  */
2558  ResetPerTupleExprContext(estate);
2559  }
2560 
2561  /* Switch into its memory context */
2563 
2564  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2565  break;
2566 
2567  /* And now we can form the input tuple. */
2568  tuple = heap_form_tuple(tupDesc, values, nulls);
2569 
2570  if (loaded_oid != InvalidOid)
2571  HeapTupleSetOid(tuple, loaded_oid);
2572 
2573  /*
2574  * Constraints might reference the tableoid column, so initialize
2575  * t_tableOid before evaluating them.
2576  */
2577  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2578 
2579  /* Triggers and stuff need to be invoked in query context. */
2580  MemoryContextSwitchTo(oldcontext);
2581 
2582  /* Place tuple in tuple slot --- but slot shouldn't free it */
2583  slot = myslot;
2584  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2585 
2586  /* Determine the partition to heap_insert the tuple into */
2587  if (cstate->partition_tuple_routing)
2588  {
2589  int leaf_part_index;
2590  TupleConversionMap *map;
2592 
2593  /*
2594  * Away we go ... If we end up not finding a partition after all,
2595  * ExecFindPartition() does not return and errors out instead.
2596  * Otherwise, the returned value is to be used as an index into
2597  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2598  * will get us the ResultRelInfo and TupleConversionMap for the
2599  * partition, respectively.
2600  */
2601  leaf_part_index = ExecFindPartition(resultRelInfo,
2602  proute->partition_dispatch_info,
2603  slot,
2604  estate);
2605  Assert(leaf_part_index >= 0 &&
2606  leaf_part_index < proute->num_partitions);
2607 
2608  /*
2609  * If this tuple is mapped to a partition that is not same as the
2610  * previous one, we'd better make the bulk insert mechanism gets a
2611  * new buffer.
2612  */
2613  if (prev_leaf_part_index != leaf_part_index)
2614  {
2615  ReleaseBulkInsertStatePin(bistate);
2616  prev_leaf_part_index = leaf_part_index;
2617  }
2618 
2619  /*
2620  * Save the old ResultRelInfo and switch to the one corresponding
2621  * to the selected partition.
2622  */
2623  saved_resultRelInfo = resultRelInfo;
2624  resultRelInfo = proute->partitions[leaf_part_index];
2625 
2626  /* We do not yet have a way to insert into a foreign partition */
2627  if (resultRelInfo->ri_FdwRoutine)
2628  ereport(ERROR,
2629  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2630  errmsg("cannot route inserted tuples to a foreign table")));
2631 
2632  /*
2633  * For ExecInsertIndexTuples() to work on the partition's indexes
2634  */
2635  estate->es_result_relation_info = resultRelInfo;
2636 
2637  /*
2638  * If we're capturing transition tuples, we might need to convert
2639  * from the partition rowtype to parent rowtype.
2640  */
2641  if (cstate->transition_capture != NULL)
2642  {
2643  if (resultRelInfo->ri_TrigDesc &&
2644  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2645  resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
2646  {
2647  /*
2648  * If there are any BEFORE or INSTEAD triggers on the
2649  * partition, we'll have to be ready to convert their
2650  * result back to tuplestore format.
2651  */
2653  cstate->transition_capture->tcs_map =
2654  cstate->transition_tupconv_maps[leaf_part_index];
2655  }
2656  else
2657  {
2658  /*
2659  * Otherwise, just remember the original unconverted
2660  * tuple, to avoid a needless round trip conversion.
2661  */
2663  cstate->transition_capture->tcs_map = NULL;
2664  }
2665  }
2666 
2667  /*
2668  * We might need to convert from the parent rowtype to the
2669  * partition rowtype.
2670  */
2671  map = proute->partition_tupconv_maps[leaf_part_index];
2672  if (map)
2673  {
2674  Relation partrel = resultRelInfo->ri_RelationDesc;
2675 
2676  tuple = do_convert_tuple(tuple, map);
2677 
2678  /*
2679  * We must use the partition's tuple descriptor from this
2680  * point on. Use a dedicated slot from this point on until
2681  * we're finished dealing with the partition.
2682  */
2683  slot = proute->partition_tuple_slot;
2684  Assert(slot != NULL);
2685  ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
2686  ExecStoreTuple(tuple, slot, InvalidBuffer, true);
2687  }
2688 
2689  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2690  }
2691 
2692  skip_tuple = false;
2693 
2694  /* BEFORE ROW INSERT Triggers */
2695  if (resultRelInfo->ri_TrigDesc &&
2696  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2697  {
2698  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2699 
2700  if (slot == NULL) /* "do nothing" */
2701  skip_tuple = true;
2702  else /* trigger might have changed tuple */
2703  tuple = ExecMaterializeSlot(slot);
2704  }
2705 
2706  if (!skip_tuple)
2707  {
2708  if (resultRelInfo->ri_TrigDesc &&
2709  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2710  {
2711  /* Pass the data to the INSTEAD ROW INSERT trigger */
2712  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2713  }
2714  else
2715  {
2716  /*
2717  * We always check the partition constraint, including when
2718  * the tuple got here via tuple-routing. However we don't
2719  * need to in the latter case if no BR trigger is defined on
2720  * the partition. Note that a BR trigger might modify the
2721  * tuple such that the partition constraint is no longer
2722  * satisfied, so we need to check in that case.
2723  */
2724  bool check_partition_constr =
2725  (resultRelInfo->ri_PartitionCheck != NIL);
2726 
2727  if (saved_resultRelInfo != NULL &&
2728  !(resultRelInfo->ri_TrigDesc &&
2729  resultRelInfo->ri_TrigDesc->trig_insert_before_row))
2730  check_partition_constr = false;
2731 
2732  /* Check the constraints of the tuple */
2733  if (cstate->rel->rd_att->constr || check_partition_constr)
2734  ExecConstraints(resultRelInfo, slot, estate, true);
2735 
2736  if (useHeapMultiInsert)
2737  {
2738  /* Add this tuple to the tuple buffer */
2739  if (nBufferedTuples == 0)
2740  firstBufferedLineNo = cstate->cur_lineno;
2741  bufferedTuples[nBufferedTuples++] = tuple;
2742  bufferedTuplesSize += tuple->t_len;
2743 
2744  /*
2745  * If the buffer filled up, flush it. Also flush if the
2746  * total size of all the tuples in the buffer becomes
2747  * large, to avoid using large amounts of memory for the
2748  * buffer when the tuples are exceptionally wide.
2749  */
2750  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2751  bufferedTuplesSize > 65535)
2752  {
2753  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2754  resultRelInfo, myslot, bistate,
2755  nBufferedTuples, bufferedTuples,
2756  firstBufferedLineNo);
2757  nBufferedTuples = 0;
2758  bufferedTuplesSize = 0;
2759  }
2760  }
2761  else
2762  {
2763  List *recheckIndexes = NIL;
2764 
2765  /* OK, store the tuple and create index entries for it */
2766  heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
2767  hi_options, bistate);
2768 
2769  if (resultRelInfo->ri_NumIndices > 0)
2770  recheckIndexes = ExecInsertIndexTuples(slot,
2771  &(tuple->t_self),
2772  estate,
2773  false,
2774  NULL,
2775  NIL);
2776 
2777  /* AFTER ROW INSERT Triggers */
2778  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2779  recheckIndexes, cstate->transition_capture);
2780 
2781  list_free(recheckIndexes);
2782  }
2783  }
2784 
2785  /*
2786  * We count only tuples not suppressed by a BEFORE INSERT trigger;
2787  * this is the same definition used by execMain.c for counting
2788  * tuples inserted by an INSERT command.
2789  */
2790  processed++;
2791 
2792  if (saved_resultRelInfo)
2793  {
2794  resultRelInfo = saved_resultRelInfo;
2795  estate->es_result_relation_info = resultRelInfo;
2796  }
2797  }
2798  }
2799 
2800  /* Flush any remaining buffered tuples */
2801  if (nBufferedTuples > 0)
2802  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2803  resultRelInfo, myslot, bistate,
2804  nBufferedTuples, bufferedTuples,
2805  firstBufferedLineNo);
2806 
2807  /* Done, clean up */
2808  error_context_stack = errcallback.previous;
2809 
2810  FreeBulkInsertState(bistate);
2811 
2812  MemoryContextSwitchTo(oldcontext);
2813 
2814  /*
2815  * In the old protocol, tell pqcomm that we can process normal protocol
2816  * messages again.
2817  */
2818  if (cstate->copy_dest == COPY_OLD_FE)
2819  pq_endmsgread();
2820 
2821  /* Execute AFTER STATEMENT insertion triggers */
2822  ExecASInsertTriggers(estate, resultRelInfo, cstate->transition_capture);
2823 
2824  /* Handle queued AFTER triggers */
2825  AfterTriggerEndQuery(estate);
2826 
2827  pfree(values);
2828  pfree(nulls);
2829 
2830  ExecResetTupleTable(estate->es_tupleTable, false);
2831 
2832  ExecCloseIndices(resultRelInfo);
2833 
2834  /* Close all the partitioned tables, leaf partitions, and their indices */
2835  if (cstate->partition_tuple_routing)
2837 
2838  /* Close any trigger target relations */
2839  ExecCleanUpTriggerState(estate);
2840 
2841  FreeExecutorState(estate);
2842 
2843  /*
2844  * If we skipped writing WAL, then we need to sync the heap (but not
2845  * indexes since those use WAL anyway)
2846  */
2847  if (hi_options & HEAP_INSERT_SKIP_WAL)
2848  heap_sync(cstate->rel);
2849 
2850  return processed;
2851 }
int ri_NumIndices
Definition: execnodes.h:368
#define NIL
Definition: pg_list.h:69
uint32 CommandId
Definition: c.h:469
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1306
Relation ri_RelationDesc
Definition: execnodes.h:365
List * range_table
Definition: copy.c:167
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate)
Definition: execTuples.c:852
int errhint(const char *fmt,...)
Definition: elog.c:987
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2175
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
#define ResetPerTupleExprContext(estate)
Definition: executor.h:483
#define RelationGetDescr(relation)
Definition: rel.h:437
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
TupleConversionMap ** partition_tupconv_maps
Definition: execPartition.h:82
#define XLogIsNeeded()
Definition: xlog.h:146
#define MAX_BUFFERED_TUPLES
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2369
#define RELKIND_MATVIEW
Definition: pg_class.h:165
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
#define gettext_noop(x)
Definition: c.h:1005
PartitionTupleRouting * ExecSetupPartitionTupleRouting(ModifyTableState *mtstate, Relation rel, Index resultRTindex, EState *estate)
Definition: execPartition.c:49
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1655
int errcode(int sqlerrcode)
Definition: elog.c:575
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:111
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2354
void heap_sync(Relation rel)
Definition: heapam.c:9209
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:695
TupleTableSlot * partition_tuple_slot
Definition: execPartition.h:83
List * es_range_table
Definition: execnodes.h:442
Form_pg_class rd_rel
Definition: rel.h:114
unsigned int Oid
Definition: postgres_ext.h:31
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate, bool check_partition_constraint)
Definition: execMain.c:1954
bool volatile_defexprs
Definition: copy.c:166
void(* callback)(void *arg)
Definition: elog.h:239
struct ErrorContextCallback * previous
Definition: elog.h:238
int natts
Definition: tupdesc.h:79
HeapTuple tcs_original_insert_tuple
Definition: trigger.h:82
ResultRelInfo ** partitions
Definition: execPartition.h:80
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
CopyDest copy_dest
Definition: copy.c:99
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:703
ErrorContextCallback * error_context_stack
Definition: elog.c:88
bool trig_insert_instead_row
Definition: reltrigger.h:57
void FreeExecutorState(EState *estate)
Definition: execUtils.c:186
Relation rel
Definition: copy.c:110
#define GetPerTupleExprContext(estate)
Definition: executor.h:474
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2336
void pfree(void *pointer)
Definition: mcxt.c:936
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1147
#define ERROR
Definition: elog.h:43
static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
Definition: copy.c:2859
TupleConversionMap * tcs_map
Definition: trigger.h:73
ItemPointerData t_self
Definition: htup.h:65
TriggerDesc * trigdesc
Definition: rel.h:120
uint32 t_len
Definition: htup.h:64
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2220
ResultRelInfo * es_result_relations
Definition: execnodes.h:452
#define RelationGetRelationName(relation)
Definition: rel.h:445
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:389
Oid t_tableOid
Definition: htup.h:66
PartitionDispatch * partition_dispatch_info
Definition: execPartition.h:78
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:471
#define ereport(elevel, rest)
Definition: elog.h:122
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2413
void InvalidateCatalogSnapshot(void)
Definition: snapmgr.c:510
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:377
EState * CreateExecutorState(void)
Definition: execUtils.c:81
TupleConversionMap * convert_tuples_by_name(TupleDesc indesc, TupleDesc outdesc, const char *msg)
Definition: tupconvert.c:210
TupleConversionMap ** transition_tupconv_maps
Definition: copy.c:173
SubTransactionId rd_createSubid
Definition: rel.h:110
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
bool trig_insert_before_row
Definition: reltrigger.h:55
List * es_tupleTable
Definition: execnodes.h:484
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:156
void * palloc0(Size size)
Definition: mcxt.c:864
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo, TransitionCaptureState *transition_capture)
Definition: trigger.c:2277
uintptr_t Datum
Definition: postgres.h:372
void ExecSetSlotDescriptor(TupleTableSlot *slot, TupleDesc tupdesc)
Definition: execTuples.c:247
TransitionCaptureState * MakeTransitionCaptureState(TriggerDesc *trigdesc, Oid relid, CmdType cmdType)
Definition: trigger.c:4404
int es_num_result_relations
Definition: execnodes.h:453
List * ri_PartitionCheck
Definition: execnodes.h:419
TupleDesc rd_att
Definition: rel.h:115
bool freeze
Definition: copy.c:118
void ExecCleanupTupleRouting(PartitionTupleRouting *proute)
void pq_endmsgread(void)
Definition: pqcomm.c:1234
#define InvalidOid
Definition: postgres_ext.h:36
PartitionTupleRouting * partition_tuple_routing
Definition: copy.c:170
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4526
#define makeNode(_type_)
Definition: nodes.h:560
#define Assert(condition)
Definition: c.h:680
void FreeBulkInsertState(BulkInsertState bistate)
Definition: heapam.c:2350
SubTransactionId GetCurrentSubTransactionId(void)
Definition: xact.c:642
TupleConstr * constr
Definition: tupdesc.h:84
size_t Size
Definition: c.h:414
#define InvalidSubTransactionId
Definition: c.h:461
HeapTuple ExecMaterializeSlot(TupleTableSlot *slot)
Definition: execTuples.c:725
void ReleaseBulkInsertStatePin(BulkInsertState bistate)
Definition: heapam.c:2362
HeapTuple do_convert_tuple(HeapTuple tuple, TupleConversionMap *map)
Definition: tupconvert.c:354
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:479
void ExecCleanUpTriggerState(EState *estate)
Definition: execMain.c:1466
#define HEAP_INSERT_SKIP_FSM
Definition: heapam.h:29
static Datum values[MAXATTR]
Definition: bootstrap.c:164
void AfterTriggerEndQuery(EState *estate)
Definition: trigger.c:4546
void * palloc(Size size)
Definition: mcxt.c:835
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
void list_free(List *list)
Definition: list.c:1133
bool NextCopyFrom(CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
Definition: copy.c:3265
int i
int cur_lineno
Definition: copy.c:140
TransitionCaptureState * transition_capture
Definition: copy.c:172
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
TupleTableSlot * ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2288
CommandId GetCurrentCommandId(bool used)
Definition: xact.c:680
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
Definition: pg_list.h:45
#define RelationGetRelid(relation)
Definition: rel.h:425
void ExecCloseIndices(ResultRelInfo *resultRelInfo)
Definition: execIndexing.c:224
ResultRelInfo * es_result_relation_info
Definition: execnodes.h:454

◆ CopyFromErrorCallback()

void CopyFromErrorCallback ( void *  arg)

Definition at line 2175 of file copy.c.

References CopyStateData::binary, CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, StringInfoData::data, errcontext, limit_printout_length(), CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, and pfree().

Referenced by CopyFrom(), file_acquire_sample_rows(), and fileIterateForeignScan().

2176 {
2177  CopyState cstate = (CopyState) arg;
2178 
2179  if (cstate->binary)
2180  {
2181  /* can't usefully display the data */
2182  if (cstate->cur_attname)
2183  errcontext("COPY %s, line %d, column %s",
2184  cstate->cur_relname, cstate->cur_lineno,
2185  cstate->cur_attname);
2186  else
2187  errcontext("COPY %s, line %d",
2188  cstate->cur_relname, cstate->cur_lineno);
2189  }
2190  else
2191  {
2192  if (cstate->cur_attname && cstate->cur_attval)
2193  {
2194  /* error is relevant to a particular column */
2195  char *attval;
2196 
2197  attval = limit_printout_length(cstate->cur_attval);
2198  errcontext("COPY %s, line %d, column %s: \"%s\"",
2199  cstate->cur_relname, cstate->cur_lineno,
2200  cstate->cur_attname, attval);
2201  pfree(attval);
2202  }
2203  else if (cstate->cur_attname)
2204  {
2205  /* error is relevant to a particular column, value is NULL */
2206  errcontext("COPY %s, line %d, column %s: null input",
2207  cstate->cur_relname, cstate->cur_lineno,
2208  cstate->cur_attname);
2209  }
2210  else
2211  {
2212  /*
2213  * Error is relevant to a particular line.
2214  *
2215  * If line_buf still contains the correct line, and it's already
2216  * transcoded, print it. If it's still in a foreign encoding, it's
2217  * quite likely that the error is precisely a failure to do
2218  * encoding conversion (ie, bad data). We dare not try to convert
2219  * it, and at present there's no way to regurgitate it without
2220  * conversion. So we have to punt and just report the line number.
2221  */
2222  if (cstate->line_buf_valid &&
2223  (cstate->line_buf_converted || !cstate->need_transcoding))
2224  {
2225  char *lineval;
2226 
2227  lineval = limit_printout_length(cstate->line_buf.data);
2228  errcontext("COPY %s, line %d: \"%s\"",
2229  cstate->cur_relname, cstate->cur_lineno, lineval);
2230  pfree(lineval);
2231  }
2232  else
2233  {
2234  errcontext("COPY %s, line %d",
2235  cstate->cur_relname, cstate->cur_lineno);
2236  }
2237  }
2238  }
2239 }
bool binary
Definition: copy.c:116
bool need_transcoding
Definition: copy.c:106
StringInfoData line_buf
Definition: copy.c:197
bool line_buf_valid
Definition: copy.c:199
bool line_buf_converted
Definition: copy.c:198
const char * cur_attname
Definition: copy.c:141
void pfree(void *pointer)
Definition: mcxt.c:936
static char * limit_printout_length(const char *str)
Definition: copy.c:2251
const char * cur_relname
Definition: copy.c:139
#define errcontext
Definition: elog.h:164
struct CopyStateData * CopyState
Definition: copy.h:23
void * arg
int cur_lineno
Definition: copy.c:140
const char * cur_attval
Definition: copy.c:142

◆ CopyFromInsertBatch()

static void CopyFromInsertBatch ( CopyState  cstate,
EState estate,
CommandId  mycid,
int  hi_options,
ResultRelInfo resultRelInfo,
TupleTableSlot myslot,
BulkInsertState  bistate,
int  nBufferedTuples,
HeapTuple bufferedTuples,
int  firstBufferedLineNo 
)
static

Definition at line 2859 of file copy.c.

References CopyStateData::cur_lineno, ExecARInsertTriggers(), ExecInsertIndexTuples(), ExecStoreTuple(), GetPerTupleMemoryContext, heap_multi_insert(), i, InvalidBuffer, CopyStateData::line_buf_valid, list_free(), MemoryContextSwitchTo(), NIL, CopyStateData::rel, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_TrigDesc, CopyStateData::transition_capture, TriggerDesc::trig_insert_after_row, and TriggerDesc::trig_insert_new_table.

Referenced by CopyFrom().

2864 {
2865  MemoryContext oldcontext;
2866  int i;
2867  int save_cur_lineno;
2868 
2869  /*
2870  * Print error context information correctly, if one of the operations
2871  * below fail.
2872  */
2873  cstate->line_buf_valid = false;
2874  save_cur_lineno = cstate->cur_lineno;
2875 
2876  /*
2877  * heap_multi_insert leaks memory, so switch to short-lived memory context
2878  * before calling it.
2879  */
2880  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2881  heap_multi_insert(cstate->rel,
2882  bufferedTuples,
2883  nBufferedTuples,
2884  mycid,
2885  hi_options,
2886  bistate);
2887  MemoryContextSwitchTo(oldcontext);
2888 
2889  /*
2890  * If there are any indexes, update them for all the inserted tuples, and
2891  * run AFTER ROW INSERT triggers.
2892  */
2893  if (resultRelInfo->ri_NumIndices > 0)
2894  {
2895  for (i = 0; i < nBufferedTuples; i++)
2896  {
2897  List *recheckIndexes;
2898 
2899  cstate->cur_lineno = firstBufferedLineNo + i;
2900  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2901  recheckIndexes =
2902  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2903  estate, false, NULL, NIL);
2904  ExecARInsertTriggers(estate, resultRelInfo,
2905  bufferedTuples[i],
2906  recheckIndexes, cstate->transition_capture);
2907  list_free(recheckIndexes);
2908  }
2909  }
2910 
2911  /*
2912  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2913  * anyway.
2914  */
2915  else if (resultRelInfo->ri_TrigDesc != NULL &&
2916  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2917  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2918  {
2919  for (i = 0; i < nBufferedTuples; i++)
2920  {
2921  cstate->cur_lineno = firstBufferedLineNo + i;
2922  ExecARInsertTriggers(estate, resultRelInfo,
2923  bufferedTuples[i],
2924  NIL, cstate->transition_capture);
2925  }
2926  }
2927 
2928  /* reset cur_lineno to where we were */
2929  cstate->cur_lineno = save_cur_lineno;
2930 }
int ri_NumIndices
Definition: execnodes.h:368
#define NIL
Definition: pg_list.h:69
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
bool line_buf_valid
Definition: copy.c:199
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2354
Relation rel
Definition: copy.c:110
bool trig_insert_new_table
Definition: reltrigger.h:74
bool trig_insert_after_row
Definition: reltrigger.h:56
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:377
void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2677
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:479
void list_free(List *list)
Definition: list.c:1133
int i
int cur_lineno
Definition: copy.c:140
TransitionCaptureState * transition_capture
Definition: copy.c:172
Definition: pg_list.h:45

◆ CopyGetAttnums()

static List * CopyGetAttnums ( TupleDesc  tupDesc,
Relation  rel,
List attnamelist 
)
static

Definition at line 4698 of file copy.c.

References ereport, errcode(), errmsg(), ERROR, i, InvalidAttrNumber, lappend_int(), lfirst, list_member_int(), name, namestrcmp(), tupleDesc::natts, NIL, RelationGetRelationName, strVal, and TupleDescAttr.

Referenced by BeginCopy(), and DoCopy().

4699 {
4700  List *attnums = NIL;
4701 
4702  if (attnamelist == NIL)
4703  {
4704  /* Generate default column list */
4705  int attr_count = tupDesc->natts;
4706  int i;
4707 
4708  for (i = 0; i < attr_count; i++)
4709  {
4710  if (TupleDescAttr(tupDesc, i)->attisdropped)
4711  continue;
4712  attnums = lappend_int(attnums, i + 1);
4713  }
4714  }
4715  else
4716  {
4717  /* Validate the user-supplied list and extract attnums */
4718  ListCell *l;
4719 
4720  foreach(l, attnamelist)
4721  {
4722  char *name = strVal(lfirst(l));
4723  int attnum;
4724  int i;
4725 
4726  /* Lookup column name */
4727  attnum = InvalidAttrNumber;
4728  for (i = 0; i < tupDesc->natts; i++)
4729  {
4730  Form_pg_attribute att = TupleDescAttr(tupDesc, i);
4731 
4732  if (att->attisdropped)
4733  continue;
4734  if (namestrcmp(&(att->attname), name) == 0)
4735  {
4736  attnum = att->attnum;
4737  break;
4738  }
4739  }
4740  if (attnum == InvalidAttrNumber)
4741  {
4742  if (rel != NULL)
4743  ereport(ERROR,
4744  (errcode(ERRCODE_UNDEFINED_COLUMN),
4745  errmsg("column \"%s\" of relation \"%s\" does not exist",
4746  name, RelationGetRelationName(rel))));
4747  else
4748  ereport(ERROR,
4749  (errcode(ERRCODE_UNDEFINED_COLUMN),
4750  errmsg("column \"%s\" does not exist",
4751  name)));
4752  }
4753  /* Check for duplicates */
4754  if (list_member_int(attnums, attnum))
4755  ereport(ERROR,
4756  (errcode(ERRCODE_DUPLICATE_COLUMN),
4757  errmsg("column \"%s\" specified more than once",
4758  name)));
4759  attnums = lappend_int(attnums, attnum);
4760  }
4761  }
4762 
4763  return attnums;
4764 }
#define NIL
Definition: pg_list.h:69
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:90
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
int namestrcmp(Name name, const char *str)
Definition: name.c:247
int natts
Definition: tupdesc.h:79
#define ERROR
Definition: elog.h:43
bool list_member_int(const List *list, int datum)
Definition: list.c:485
#define RelationGetRelationName(relation)
Definition: rel.h:445
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
#define ereport(elevel, rest)
Definition: elog.h:122
List * lappend_int(List *list, int datum)
Definition: list.c:146
#define lfirst(lc)
Definition: pg_list.h:106
const char * name
Definition: encode.c:521
#define InvalidAttrNumber
Definition: attnum.h:23
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
Definition: pg_list.h:45

◆ CopyGetData()

static int CopyGetData ( CopyState  cstate,
void *  databuf,
int  minread,
int  maxread 
)
static

Definition at line 556 of file copy.c.

References COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, StringInfoData::cursor, CopyStateData::data_source_cb, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, CopyStateData::fe_eof, CopyStateData::fe_msgbuf, HOLD_CANCEL_INTERRUPTS, StringInfoData::len, pq_copymsgbytes(), pq_getbyte(), pq_getbytes(), pq_getmessage(), pq_getmsgstring(), pq_startmsgread(), and RESUME_CANCEL_INTERRUPTS.

Referenced by BeginCopyFrom(), CopyGetInt16(), CopyGetInt32(), CopyLoadRawBuf(), CopyReadBinaryAttribute(), and NextCopyFrom().

557 {
558  int bytesread = 0;
559 
560  switch (cstate->copy_dest)
561  {
562  case COPY_FILE:
563  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
564  if (ferror(cstate->copy_file))
565  ereport(ERROR,
567  errmsg("could not read from COPY file: %m")));
568  break;
569  case COPY_OLD_FE:
570 
571  /*
572  * We cannot read more than minread bytes (which in practice is 1)
573  * because old protocol doesn't have any clear way of separating
574  * the COPY stream from following data. This is slow, but not any
575  * slower than the code path was originally, and we don't care
576  * much anymore about the performance of old protocol.
577  */
578  if (pq_getbytes((char *) databuf, minread))
579  {
580  /* Only a \. terminator is legal EOF in old protocol */
581  ereport(ERROR,
582  (errcode(ERRCODE_CONNECTION_FAILURE),
583  errmsg("unexpected EOF on client connection with an open transaction")));
584  }
585  bytesread = minread;
586  break;
587  case COPY_NEW_FE:
588  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
589  {
590  int avail;
591 
592  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
593  {
594  /* Try to receive another message */
595  int mtype;
596 
597  readmessage:
599  pq_startmsgread();
600  mtype = pq_getbyte();
601  if (mtype == EOF)
602  ereport(ERROR,
603  (errcode(ERRCODE_CONNECTION_FAILURE),
604  errmsg("unexpected EOF on client connection with an open transaction")));
605  if (pq_getmessage(cstate->fe_msgbuf, 0))
606  ereport(ERROR,
607  (errcode(ERRCODE_CONNECTION_FAILURE),
608  errmsg("unexpected EOF on client connection with an open transaction")));
610  switch (mtype)
611  {
612  case 'd': /* CopyData */
613  break;
614  case 'c': /* CopyDone */
615  /* COPY IN correctly terminated by frontend */
616  cstate->fe_eof = true;
617  return bytesread;
618  case 'f': /* CopyFail */
619  ereport(ERROR,
620  (errcode(ERRCODE_QUERY_CANCELED),
621  errmsg("COPY from stdin failed: %s",
622  pq_getmsgstring(cstate->fe_msgbuf))));
623  break;
624  case 'H': /* Flush */
625  case 'S': /* Sync */
626 
627  /*
628  * Ignore Flush/Sync for the convenience of client
629  * libraries (such as libpq) that may send those
630  * without noticing that the command they just
631  * sent was COPY.
632  */
633  goto readmessage;
634  default:
635  ereport(ERROR,
636  (errcode(ERRCODE_PROTOCOL_VIOLATION),
637  errmsg("unexpected message type 0x%02X during COPY from stdin",
638  mtype)));
639  break;
640  }
641  }
642  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
643  if (avail > maxread)
644  avail = maxread;
645  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
646  databuf = (void *) ((char *) databuf + avail);
647  maxread -= avail;
648  bytesread += avail;
649  }
650  break;
651  case COPY_CALLBACK:
652  bytesread = cstate->data_source_cb(databuf, minread, maxread);
653  break;
654  }
655 
656  return bytesread;
657 }
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:123
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:581
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:103
CopyDest copy_dest
Definition: copy.c:99
Definition: copy.c:63
copy_data_source_cb data_source_cb
Definition: copy.c:115
#define ERROR
Definition: elog.h:43
void pq_startmsgread(void)
Definition: pqcomm.c:1210
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1094
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1272
int pq_getbyte(void)
Definition: pqcomm.c:1000
StringInfo fe_msgbuf
Definition: copy.c:101
void pq_copymsgbytes(StringInfo msg, char *buf, int datalen)
Definition: pqformat.c:530
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:100
#define RESUME_CANCEL_INTERRUPTS()
Definition: miscadmin.h:125

◆ CopyGetInt16()

static bool CopyGetInt16 ( CopyState  cstate,
int16 val 
)
static

Definition at line 711 of file copy.c.

References buf, CopyGetData(), and pg_ntoh16.

Referenced by NextCopyFrom().

712 {
713  uint16 buf;
714 
715  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
716  {
717  *val = 0; /* suppress compiler warning */
718  return false;
719  }
720  *val = (int16) pg_ntoh16(buf);
721  return true;
722 }
signed short int16
Definition: c.h:293
#define pg_ntoh16(x)
Definition: pg_bswap.h:124
unsigned short uint16
Definition: c.h:305
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
static char * buf
Definition: pg_test_fsync.c:67
long val
Definition: informix.c:689

◆ CopyGetInt32()

static bool CopyGetInt32 ( CopyState  cstate,
int32 val 
)
static

Definition at line 682 of file copy.c.

References buf, CopyGetData(), and pg_ntoh32.

Referenced by BeginCopyFrom(), and CopyReadBinaryAttribute().

683 {
684  uint32 buf;
685 
686  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
687  {
688  *val = 0; /* suppress compiler warning */
689  return false;
690  }
691  *val = (int32) pg_ntoh32(buf);
692  return true;
693 }
signed int int32
Definition: c.h:294
#define pg_ntoh32(x)
Definition: pg_bswap.h:125
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:306
long val
Definition: informix.c:689

◆ CopyLoadRawBuf()

static bool CopyLoadRawBuf ( CopyState  cstate)
static

Definition at line 736 of file copy.c.

References CopyGetData(), memmove, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and RAW_BUF_SIZE.

Referenced by CopyReadLine(), and CopyReadLineText().

737 {
738  int nbytes;
739  int inbytes;
740 
741  if (cstate->raw_buf_index < cstate->raw_buf_len)
742  {
743  /* Copy down the unprocessed data */
744  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
745  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
746  nbytes);
747  }
748  else
749  nbytes = 0; /* no data need be saved */
750 
751  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
752  1, RAW_BUF_SIZE - nbytes);
753  nbytes += inbytes;
754  cstate->raw_buf[nbytes] = '\0';
755  cstate->raw_buf_index = 0;
756  cstate->raw_buf_len = nbytes;
757  return (inbytes > 0);
758 }
int raw_buf_index
Definition: copy.c:210
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
char * raw_buf
Definition: copy.c:209
#define memmove(d, s, c)
Definition: c.h:1069
int raw_buf_len
Definition: copy.c:211
#define RAW_BUF_SIZE
Definition: copy.c:208

◆ CopyOneRowTo()

static void CopyOneRowTo ( CopyState  cstate,
Oid  tupleOid,
Datum values,
bool nulls 
)
static

Definition at line 2081 of file copy.c.

References CopyStateData::attnumlist, CopyStateData::binary, CopyAttributeOutCSV(), CopyAttributeOutText(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), CopySendString(), CopyStateData::csv_mode, cur, DatumGetCString, CopyStateData::delim, DirectFunctionCall1, CopyStateData::force_quote_flags, lfirst_int, list_length(), MemoryContextReset(), MemoryContextSwitchTo(), CopyStateData::null_print_client, ObjectIdGetDatum, oidout(), CopyStateData::oids, CopyStateData::out_functions, OutputFunctionCall(), CopyStateData::rowcontext, SendFunctionCall(), value, VARDATA, VARHDRSZ, and VARSIZE.

Referenced by copy_dest_receive(), and CopyTo().

2082 {
2083  bool need_delim = false;
2084  FmgrInfo *out_functions = cstate->out_functions;
2085  MemoryContext oldcontext;
2086  ListCell *cur;
2087  char *string;
2088 
2089  MemoryContextReset(cstate->rowcontext);
2090  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2091 
2092  if (cstate->binary)
2093  {
2094  /* Binary per-tuple header */
2095  CopySendInt16(cstate, list_length(cstate->attnumlist));
2096  /* Send OID if wanted --- note attnumlist doesn't include it */
2097  if (cstate->oids)
2098  {
2099  /* Hack --- assume Oid is same size as int32 */
2100  CopySendInt32(cstate, sizeof(int32));
2101  CopySendInt32(cstate, tupleOid);
2102  }
2103  }
2104  else
2105  {
2106  /* Text format has no per-tuple header, but send OID if wanted */
2107  /* Assume digits don't need any quoting or encoding conversion */
2108  if (cstate->oids)
2109  {
2111  ObjectIdGetDatum(tupleOid)));
2112  CopySendString(cstate, string);
2113  need_delim = true;
2114  }
2115  }
2116 
2117  foreach(cur, cstate->attnumlist)
2118  {
2119  int attnum = lfirst_int(cur);
2120  Datum value = values[attnum - 1];
2121  bool isnull = nulls[attnum - 1];
2122 
2123  if (!cstate->binary)
2124  {
2125  if (need_delim)
2126  CopySendChar(cstate, cstate->delim[0]);
2127  need_delim = true;
2128  }
2129 
2130  if (isnull)
2131  {
2132  if (!cstate->binary)
2133  CopySendString(cstate, cstate->null_print_client);
2134  else
2135  CopySendInt32(cstate, -1);
2136  }
2137  else
2138  {
2139  if (!cstate->binary)
2140  {
2141  string = OutputFunctionCall(&out_functions[attnum - 1],
2142  value);
2143  if (cstate->csv_mode)
2144  CopyAttributeOutCSV(cstate, string,
2145  cstate->force_quote_flags[attnum - 1],
2146  list_length(cstate->attnumlist) == 1);
2147  else
2148  CopyAttributeOutText(cstate, string);
2149  }
2150  else
2151  {
2152  bytea *outputbytes;
2153 
2154  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2155  value);
2156  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2157  CopySendData(cstate, VARDATA(outputbytes),
2158  VARSIZE(outputbytes) - VARHDRSZ);
2159  }
2160  }
2161  }
2162 
2163  CopySendEndOfRow(cstate);
2164 
2165  MemoryContextSwitchTo(oldcontext);
2166 }
Definition: fmgr.h:56
bool csv_mode
Definition: copy.c:119
#define VARDATA(PTR)
Definition: postgres.h:303
bool binary
Definition: copy.c:116
static struct @130 value
List * attnumlist
Definition: copy.c:112
#define VARSIZE(PTR)
Definition: postgres.h:304
#define VARHDRSZ
Definition: c.h:503
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
struct cursor * cur
Definition: ecpg.c:28
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4454
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:134
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:585
bool * force_quote_flags
Definition: copy.c:129
char * delim
Definition: copy.c:124
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
MemoryContext rowcontext
Definition: copy.c:153
signed int int32
Definition: c.h:294
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1662
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:699
#define ObjectIdGetDatum(X)
Definition: postgres.h:513
#define DatumGetCString(X)
Definition: postgres.h:572
#define lfirst_int(lc)
Definition: pg_list.h:107
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4607
char string[11]
Definition: preproc-type.c:46
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1723
uintptr_t Datum
Definition: postgres.h:372
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:668
static Datum values[MAXATTR]
Definition: bootstrap.c:164
FmgrInfo * out_functions
Definition: copy.c:152
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:463
Definition: c.h:497
char * null_print_client
Definition: copy.c:123
bool oids
Definition: copy.c:117

◆ CopyReadAttributesCSV()

static int CopyReadAttributesCSV ( CopyState  cstate)
static

Definition at line 4226 of file copy.c.

References Assert, CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, CopyStateData::escape, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, CopyStateData::null_print, CopyStateData::null_print_len, CopyStateData::quote, CopyStateData::raw_fields, repalloc(), and resetStringInfo().

Referenced by NextCopyFromRawFields().

4227 {
4228  char delimc = cstate->delim[0];
4229  char quotec = cstate->quote[0];
4230  char escapec = cstate->escape[0];
4231  int fieldno;
4232  char *output_ptr;
4233  char *cur_ptr;
4234  char *line_end_ptr;
4235 
4236  /*
4237  * We need a special case for zero-column tables: check that the input
4238  * line is empty, and return.
4239  */
4240  if (cstate->max_fields <= 0)
4241  {
4242  if (cstate->line_buf.len != 0)
4243  ereport(ERROR,
4244  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4245  errmsg("extra data after last expected column")));
4246  return 0;
4247  }
4248 
4249  resetStringInfo(&cstate->attribute_buf);
4250 
4251  /*
4252  * The de-escaped attributes will certainly not be longer than the input
4253  * data line, so we can just force attribute_buf to be large enough and
4254  * then transfer data without any checks for enough space. We need to do
4255  * it this way because enlarging attribute_buf mid-stream would invalidate
4256  * pointers already stored into cstate->raw_fields[].
4257  */
4258  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4259  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4260  output_ptr = cstate->attribute_buf.data;
4261 
4262  /* set pointer variables for loop */
4263  cur_ptr = cstate->line_buf.data;
4264  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4265 
4266  /* Outer loop iterates over fields */
4267  fieldno = 0;
4268  for (;;)
4269  {
4270  bool found_delim = false;
4271  bool saw_quote = false;
4272  char *start_ptr;
4273  char *end_ptr;
4274  int input_len;
4275 
4276  /* Make sure there is enough space for the next value */
4277  if (fieldno >= cstate->max_fields)
4278  {
4279  cstate->max_fields *= 2;
4280  cstate->raw_fields =
4281  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4282  }
4283 
4284  /* Remember start of field on both input and output sides */
4285  start_ptr = cur_ptr;
4286  cstate->raw_fields[fieldno] = output_ptr;
4287 
4288  /*
4289  * Scan data for field,
4290  *
4291  * The loop starts in "not quote" mode and then toggles between that
4292  * and "in quote" mode. The loop exits normally if it is in "not
4293  * quote" mode and a delimiter or line end is seen.
4294  */
4295  for (;;)
4296  {
4297  char c;
4298 
4299  /* Not in quote */
4300  for (;;)
4301  {
4302  end_ptr = cur_ptr;
4303  if (cur_ptr >= line_end_ptr)
4304  goto endfield;
4305  c = *cur_ptr++;
4306  /* unquoted field delimiter */
4307  if (c == delimc)
4308  {
4309  found_delim = true;
4310  goto endfield;
4311  }
4312  /* start of quoted field (or part of field) */
4313  if (c == quotec)
4314  {
4315  saw_quote = true;
4316  break;
4317  }
4318  /* Add c to output string */
4319  *output_ptr++ = c;
4320  }
4321 
4322  /* In quote */
4323  for (;;)
4324  {
4325  end_ptr = cur_ptr;
4326  if (cur_ptr >= line_end_ptr)
4327  ereport(ERROR,
4328  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4329  errmsg("unterminated CSV quoted field")));
4330 
4331  c = *cur_ptr++;
4332 
4333  /* escape within a quoted field */
4334  if (c == escapec)
4335  {
4336  /*
4337  * peek at the next char if available, and escape it if it
4338  * is an escape char or a quote char
4339  */
4340  if (cur_ptr < line_end_ptr)
4341  {
4342  char nextc = *cur_ptr;
4343 
4344  if (nextc == escapec || nextc == quotec)
4345  {
4346  *output_ptr++ = nextc;
4347  cur_ptr++;
4348  continue;
4349  }
4350  }
4351  }
4352 
4353  /*
4354  * end of quoted field. Must do this test after testing for
4355  * escape in case quote char and escape char are the same
4356  * (which is the common case).
4357  */
4358  if (c == quotec)
4359  break;
4360 
4361  /* Add c to output string */
4362  *output_ptr++ = c;
4363  }
4364  }
4365 endfield:
4366 
4367  /* Terminate attribute value in output area */
4368  *output_ptr++ = '\0';
4369 
4370  /* Check whether raw input matched null marker */
4371  input_len = end_ptr - start_ptr;
4372  if (!saw_quote && input_len == cstate->null_print_len &&
4373  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4374  cstate->raw_fields[fieldno] = NULL;
4375 
4376  fieldno++;
4377  /* Done if we hit EOL instead of a delim */
4378  if (!found_delim)
4379  break;
4380  }
4381 
4382  /* Clean up state of attribute_buf */
4383  output_ptr--;
4384  Assert(*output_ptr == '\0');
4385  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4386 
4387  return fieldno;
4388 }
char ** raw_fields
Definition: copy.c:188
StringInfoData line_buf
Definition: copy.c:197
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:124
char * null_print
Definition: copy.c:121
#define ERROR
Definition: elog.h:43
char * c
char * quote
Definition: copy.c:125
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:264
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:187
char * escape
Definition: copy.c:126
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:122
#define Assert(condition)
Definition: c.h:680
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:949
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:183

◆ CopyReadAttributesText()

static int CopyReadAttributesText ( CopyState  cstate)
static

Definition at line 3998 of file copy.c.

References Assert, CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, GetDecimalFromHex(), IS_HIGHBIT_SET, ISOCTAL, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, CopyStateData::null_print, CopyStateData::null_print_len, OCTVALUE, pg_verifymbstr(), CopyStateData::raw_fields, repalloc(), resetStringInfo(), and val.

Referenced by NextCopyFromRawFields().

3999 {
4000  char delimc = cstate->delim[0];
4001  int fieldno;
4002  char *output_ptr;
4003  char *cur_ptr;
4004  char *line_end_ptr;
4005 
4006  /*
4007  * We need a special case for zero-column tables: check that the input
4008  * line is empty, and return.
4009  */
4010  if (cstate->max_fields <= 0)
4011  {
4012  if (cstate->line_buf.len != 0)
4013  ereport(ERROR,
4014  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4015  errmsg("extra data after last expected column")));
4016  return 0;
4017  }
4018 
4019  resetStringInfo(&cstate->attribute_buf);
4020 
4021  /*
4022  * The de-escaped attributes will certainly not be longer than the input
4023  * data line, so we can just force attribute_buf to be large enough and
4024  * then transfer data without any checks for enough space. We need to do
4025  * it this way because enlarging attribute_buf mid-stream would invalidate
4026  * pointers already stored into cstate->raw_fields[].
4027  */
4028  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4029  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4030  output_ptr = cstate->attribute_buf.data;
4031 
4032  /* set pointer variables for loop */
4033  cur_ptr = cstate->line_buf.data;
4034  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4035 
4036  /* Outer loop iterates over fields */
4037  fieldno = 0;
4038  for (;;)
4039  {
4040  bool found_delim = false;
4041  char *start_ptr;
4042  char *end_ptr;
4043  int input_len;
4044  bool saw_non_ascii = false;
4045 
4046  /* Make sure there is enough space for the next value */
4047  if (fieldno >= cstate->max_fields)
4048  {
4049  cstate->max_fields *= 2;
4050  cstate->raw_fields =
4051  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4052  }
4053 
4054  /* Remember start of field on both input and output sides */
4055  start_ptr = cur_ptr;
4056  cstate->raw_fields[fieldno] = output_ptr;
4057 
4058  /*
4059  * Scan data for field.
4060  *
4061  * Note that in this loop, we are scanning to locate the end of field
4062  * and also speculatively performing de-escaping. Once we find the
4063  * end-of-field, we can match the raw field contents against the null
4064  * marker string. Only after that comparison fails do we know that
4065  * de-escaping is actually the right thing to do; therefore we *must
4066  * not* throw any syntax errors before we've done the null-marker
4067  * check.
4068  */
4069  for (;;)
4070  {
4071  char c;
4072 
4073  end_ptr = cur_ptr;
4074  if (cur_ptr >= line_end_ptr)
4075  break;
4076  c = *cur_ptr++;
4077  if (c == delimc)
4078  {
4079  found_delim = true;
4080  break;
4081  }
4082  if (c == '\\')
4083  {
4084  if (cur_ptr >= line_end_ptr)
4085  break;
4086  c = *cur_ptr++;
4087  switch (c)
4088  {
4089  case '0':
4090  case '1':
4091  case '2':
4092  case '3':
4093  case '4':
4094  case '5':
4095  case '6':
4096  case '7':
4097  {
4098  /* handle \013 */
4099  int val;
4100 
4101  val = OCTVALUE(c);
4102  if (cur_ptr < line_end_ptr)
4103  {
4104  c = *cur_ptr;
4105  if (ISOCTAL(c))
4106  {
4107  cur_ptr++;
4108  val = (val << 3) + OCTVALUE(c);
4109  if (cur_ptr < line_end_ptr)
4110  {
4111  c = *cur_ptr;
4112  if (ISOCTAL(c))
4113  {
4114  cur_ptr++;
4115  val = (val << 3) + OCTVALUE(c);
4116  }
4117  }
4118  }
4119  }
4120  c = val & 0377;
4121  if (c == '\0' || IS_HIGHBIT_SET(c))
4122  saw_non_ascii = true;
4123  }
4124  break;
4125  case 'x':
4126  /* Handle \x3F */
4127  if (cur_ptr < line_end_ptr)
4128  {
4129  char hexchar = *cur_ptr;
4130 
4131  if (isxdigit((unsigned char) hexchar))
4132  {
4133  int val = GetDecimalFromHex(hexchar);
4134 
4135  cur_ptr++;
4136  if (cur_ptr < line_end_ptr)
4137  {
4138  hexchar = *cur_ptr;
4139  if (isxdigit((unsigned char) hexchar))
4140  {
4141  cur_ptr++;
4142  val = (val << 4) + GetDecimalFromHex(hexchar);
4143  }
4144  }
4145  c = val & 0xff;
4146  if (c == '\0' || IS_HIGHBIT_SET(c))
4147  saw_non_ascii = true;
4148  }
4149  }
4150  break;
4151  case 'b':
4152  c = '\b';
4153  break;
4154  case 'f':
4155  c = '\f';
4156  break;
4157  case 'n':
4158  c = '\n';
4159  break;
4160  case 'r':
4161  c = '\r';
4162  break;
4163  case 't':
4164  c = '\t';
4165  break;
4166  case 'v':
4167  c = '\v';
4168  break;
4169 
4170  /*
4171  * in all other cases, take the char after '\'
4172  * literally
4173  */
4174  }
4175  }
4176 
4177  /* Add c to output string */
4178  *output_ptr++ = c;
4179  }
4180 
4181  /* Check whether raw input matched null marker */
4182  input_len = end_ptr - start_ptr;
4183  if (input_len == cstate->null_print_len &&
4184  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4185  cstate->raw_fields[fieldno] = NULL;
4186  else
4187  {
4188  /*
4189  * At this point we know the field is supposed to contain data.
4190  *
4191  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4192  * resulting string is valid data for the db encoding.
4193  */
4194  if (saw_non_ascii)
4195  {
4196  char *fld = cstate->raw_fields[fieldno];
4197 
4198  pg_verifymbstr(fld, output_ptr - fld, false);
4199  }
4200  }
4201 
4202  /* Terminate attribute value in output area */
4203  *output_ptr++ = '\0';
4204 
4205  fieldno++;
4206  /* Done if we hit EOL instead of a delim */
4207  if (!found_delim)
4208  break;
4209  }
4210 
4211  /* Clean up state of attribute_buf */
4212  output_ptr--;
4213  Assert(*output_ptr == '\0');
4214  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4215 
4216  return fieldno;
4217 }
static int GetDecimalFromHex(char hex)
Definition: copy.c:3970
char ** raw_fields
Definition: copy.c:188
#define ISOCTAL(c)
Definition: copy.c:54
#define OCTVALUE(c)
Definition: copy.c:55
StringInfoData line_buf
Definition: copy.c:197
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:124
char * null_print
Definition: copy.c:121
#define IS_HIGHBIT_SET(ch)
Definition: c.h:963
#define ERROR
Definition: elog.h:43
char * c
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:264
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:187
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:122
#define Assert(condition)
Definition: c.h:680
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:949
int errmsg(const char *fmt,...)
Definition: elog.c:797
bool pg_verifymbstr(const char *mbstr, int len, bool noError)
Definition: wchar.c:1866
StringInfoData attribute_buf
Definition: copy.c:183
long val
Definition: informix.c:689

◆ CopyReadBinaryAttribute()

static Datum CopyReadBinaryAttribute ( CopyState  cstate,
int  column_no,
FmgrInfo flinfo,
Oid  typioparam,
int32  typmod,
bool isnull 
)
static

Definition at line 4395 of file copy.c.

References CopyStateData::attribute_buf, CopyGetData(), CopyGetInt32(), StringInfoData::cursor, StringInfoData::data, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, StringInfoData::len, ReceiveFunctionCall(), and resetStringInfo().

Referenced by NextCopyFrom().

4399 {
4400  int32 fld_size;
4401  Datum result;
4402 
4403  if (!CopyGetInt32(cstate, &fld_size))
4404  ereport(ERROR,
4405  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4406  errmsg("unexpected EOF in COPY data")));
4407  if (fld_size == -1)
4408  {
4409  *isnull = true;
4410  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4411  }
4412  if (fld_size < 0)
4413  ereport(ERROR,
4414  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4415  errmsg("invalid field size")));
4416 
4417  /* reset attribute_buf to empty, and load raw data in it */
4418  resetStringInfo(&cstate->attribute_buf);
4419 
4420  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4421  if (CopyGetData(cstate, cstate->attribute_buf.data,
4422  fld_size, fld_size) != fld_size)
4423  ereport(ERROR,
4424  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4425  errmsg("unexpected EOF in COPY data")));
4426 
4427  cstate->attribute_buf.len = fld_size;
4428  cstate->attribute_buf.data[fld_size] = '\0';
4429 
4430  /* Call the column type's binary input converter */
4431  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4432  typioparam, typmod);
4433 
4434  /* Trouble if it didn't eat the whole buffer */
4435  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4436  ereport(ERROR,
4437  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4438  errmsg("incorrect binary data format")));
4439 
4440  *isnull = false;
4441  return result;
4442 }
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:682
int errcode(int sqlerrcode)
Definition: elog.c:575
signed int int32
Definition: c.h:294
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1676
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:264
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
uintptr_t Datum
Definition: postgres.h:372
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:183

◆ CopyReadLine()

static bool CopyReadLine ( CopyState  cstate)
static

Definition at line 3522 of file copy.c.

References appendBinaryStringInfo(), Assert, CopyStateData::copy_dest, COPY_NEW_FE, CopyLoadRawBuf(), CopyReadLineText(), StringInfoData::data, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, CopyStateData::file_encoding, StringInfoData::len, CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, pfree(), pg_any_to_server(), CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and resetStringInfo().

Referenced by NextCopyFromRawFields().

3523 {
3524  bool result;
3525 
3526  resetStringInfo(&cstate->line_buf);
3527  cstate->line_buf_valid = true;
3528 
3529  /* Mark that encoding conversion hasn't occurred yet */
3530  cstate->line_buf_converted = false;
3531 
3532  /* Parse data and transfer into line_buf */
3533  result = CopyReadLineText(cstate);
3534 
3535  if (result)
3536  {
3537  /*
3538  * Reached EOF. In protocol version 3, we should ignore anything
3539  * after \. up to the protocol end of copy data. (XXX maybe better
3540  * not to treat \. as special?)
3541  */
3542  if (cstate->copy_dest == COPY_NEW_FE)
3543  {
3544  do
3545  {
3546  cstate->raw_buf_index = cstate->raw_buf_len;
3547  } while (CopyLoadRawBuf(cstate));
3548  }
3549  }
3550  else
3551  {
3552  /*
3553  * If we didn't hit EOF, then we must have transferred the EOL marker
3554  * to line_buf along with the data. Get rid of it.
3555  */
3556  switch (cstate->eol_type)
3557  {
3558  case EOL_NL:
3559  Assert(cstate->line_buf.len >= 1);
3560  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3561  cstate->line_buf.len--;
3562  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3563  break;
3564  case EOL_CR:
3565  Assert(cstate->line_buf.len >= 1);
3566  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3567  cstate->line_buf.len--;
3568  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3569  break;
3570  case EOL_CRNL:
3571  Assert(cstate->line_buf.len >= 2);
3572  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3573  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3574  cstate->line_buf.len -= 2;
3575  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3576  break;
3577  case EOL_UNKNOWN:
3578  /* shouldn't get here */
3579  Assert(false);
3580  break;
3581  }
3582  }
3583 
3584  /* Done reading the line. Convert it to server encoding. */
3585  if (cstate->need_transcoding)
3586  {
3587  char *cvt;
3588 
3589  cvt = pg_any_to_server(cstate->line_buf.data,
3590  cstate->line_buf.len,
3591  cstate->file_encoding);
3592  if (cvt != cstate->line_buf.data)
3593  {
3594  /* transfer converted data back to line_buf */
3595  resetStringInfo(&cstate->line_buf);
3596  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3597  pfree(cvt);
3598  }
3599  }
3600 
3601  /* Now it's safe to use the buffer in error messages */
3602  cstate->line_buf_converted = true;
3603 
3604  return result;
3605 }
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3611
Definition: copy.c:75
bool need_transcoding
Definition: copy.c:106
StringInfoData line_buf
Definition: copy.c:197
int raw_buf_index
Definition: copy.c:210
bool line_buf_valid
Definition: copy.c:199
bool line_buf_converted
Definition: copy.c:198
CopyDest copy_dest
Definition: copy.c:99
void pfree(void *pointer)
Definition: mcxt.c:936
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:736
Definition: copy.c:76
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int raw_buf_len
Definition: copy.c:211
Definition: copy.c:77
int file_encoding
Definition: copy.c:105
EolType eol_type
Definition: copy.c:104
#define Assert(condition)
Definition: c.h:680
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:561
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

◆ CopyReadLineText()

static bool CopyReadLineText ( CopyState  cstate)
static

Definition at line 3611 of file copy.c.

References appendBinaryStringInfo(), CopyLoadRawBuf(), CopyStateData::csv_mode, CopyStateData::cur_lineno, CopyStateData::encoding_embeds_ascii, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errhint(), errmsg(), ERROR, CopyStateData::escape, CopyStateData::file_encoding, IF_NEED_REFILL_AND_EOF_BREAK, IF_NEED_REFILL_AND_NOT_EOF_CONTINUE, IS_HIGHBIT_SET, CopyStateData::line_buf, NO_END_OF_COPY_GOTO, pg_encoding_mblen(), CopyStateData::quote, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and REFILL_LINEBUF.

Referenced by CopyReadLine().

3612 {
3613  char *copy_raw_buf;
3614  int raw_buf_ptr;
3615  int copy_buf_len;
3616  bool need_data = false;
3617  bool hit_eof = false;
3618  bool result = false;
3619  char mblen_str[2];
3620 
3621  /* CSV variables */
3622  bool first_char_in_line = true;
3623  bool in_quote = false,
3624  last_was_esc = false;
3625  char quotec = '\0';
3626  char escapec = '\0';
3627 
3628  if (cstate->csv_mode)
3629  {
3630  quotec = cstate->quote[0];
3631  escapec = cstate->escape[0];
3632  /* ignore special escape processing if it's the same as quotec */
3633  if (quotec == escapec)
3634  escapec = '\0';
3635  }
3636 
3637  mblen_str[1] = '\0';
3638 
3639  /*
3640  * The objective of this loop is to transfer the entire next input line
3641  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3642  * \n) and the end-of-copy marker (\.).
3643  *
3644  * In CSV mode, \r and \n inside a quoted field are just part of the data
3645  * value and are put in line_buf. We keep just enough state to know if we
3646  * are currently in a quoted field or not.
3647  *
3648  * These four characters, and the CSV escape and quote characters, are
3649  * assumed the same in frontend and backend encodings.
3650  *
3651  * For speed, we try to move data from raw_buf to line_buf in chunks
3652  * rather than one character at a time. raw_buf_ptr points to the next
3653  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3654  * have been determined to be part of the line, but not yet transferred to
3655  * line_buf.
3656  *
3657  * For a little extra speed within the loop, we copy raw_buf and
3658  * raw_buf_len into local variables.
3659  */
3660  copy_raw_buf = cstate->raw_buf;
3661  raw_buf_ptr = cstate->raw_buf_index;
3662  copy_buf_len = cstate->raw_buf_len;
3663 
3664  for (;;)
3665  {
3666  int prev_raw_ptr;
3667  char c;
3668 
3669  /*
3670  * Load more data if needed. Ideally we would just force four bytes
3671  * of read-ahead and avoid the many calls to
3672  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3673  * does not allow us to read too far ahead or we might read into the
3674  * next data, so we read-ahead only as far we know we can. One
3675  * optimization would be to read-ahead four byte here if
3676  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3677  * considering the size of the buffer.
3678  */
3679  if (raw_buf_ptr >= copy_buf_len || need_data)
3680  {
3682 
3683  /*
3684  * Try to read some more data. This will certainly reset
3685  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3686  */
3687  if (!CopyLoadRawBuf(cstate))
3688  hit_eof = true;
3689  raw_buf_ptr = 0;
3690  copy_buf_len = cstate->raw_buf_len;
3691 
3692  /*
3693  * If we are completely out of data, break out of the loop,
3694  * reporting EOF.
3695  */
3696  if (copy_buf_len <= 0)
3697  {
3698  result = true;
3699  break;
3700  }
3701  need_data = false;
3702  }
3703 
3704  /* OK to fetch a character */
3705  prev_raw_ptr = raw_buf_ptr;
3706  c = copy_raw_buf[raw_buf_ptr++];
3707 
3708  if (cstate->csv_mode)
3709  {
3710  /*
3711  * If character is '\\' or '\r', we may need to look ahead below.
3712  * Force fetch of the next character if we don't already have it.
3713  * We need to do this before changing CSV state, in case one of
3714  * these characters is also the quote or escape character.
3715  *
3716  * Note: old-protocol does not like forced prefetch, but it's OK
3717  * here since we cannot validly be at EOF.
3718  */
3719  if (c == '\\' || c == '\r')
3720  {
3722  }
3723 
3724  /*
3725  * Dealing with quotes and escapes here is mildly tricky. If the
3726  * quote char is also the escape char, there's no problem - we
3727  * just use the char as a toggle. If they are different, we need
3728  * to ensure that we only take account of an escape inside a
3729  * quoted field and immediately preceding a quote char, and not
3730  * the second in an escape-escape sequence.
3731  */
3732  if (in_quote && c == escapec)
3733  last_was_esc = !last_was_esc;
3734  if (c == quotec && !last_was_esc)
3735  in_quote = !in_quote;
3736  if (c != escapec)
3737  last_was_esc = false;
3738 
3739  /*
3740  * Updating the line count for embedded CR and/or LF chars is
3741  * necessarily a little fragile - this test is probably about the
3742  * best we can do. (XXX it's arguable whether we should do this
3743  * at all --- is cur_lineno a physical or logical count?)
3744  */
3745  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3746  cstate->cur_lineno++;
3747  }
3748 
3749  /* Process \r */
3750  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3751  {
3752  /* Check for \r\n on first line, _and_ handle \r\n. */
3753  if (cstate->eol_type == EOL_UNKNOWN ||
3754  cstate->eol_type == EOL_CRNL)
3755  {
3756  /*
3757  * If need more data, go back to loop top to load it.
3758  *
3759  * Note that if we are at EOF, c will wind up as '\0' because
3760  * of the guaranteed pad of raw_buf.
3761  */
3763 
3764  /* get next char */
3765  c = copy_raw_buf[raw_buf_ptr];
3766 
3767  if (c == '\n')
3768  {
3769  raw_buf_ptr++; /* eat newline */
3770  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3771  }
3772  else
3773  {
3774  /* found \r, but no \n */
3775  if (cstate->eol_type == EOL_CRNL)
3776  ereport(ERROR,
3777  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3778  !cstate->csv_mode ?
3779  errmsg("literal carriage return found in data") :
3780  errmsg("unquoted carriage return found in data"),
3781  !cstate->csv_mode ?
3782  errhint("Use \"\\r\" to represent carriage return.") :
3783  errhint("Use quoted CSV field to represent carriage return.")));
3784 
3785  /*
3786  * if we got here, it is the first line and we didn't find
3787  * \n, so don't consume the peeked character
3788  */
3789  cstate->eol_type = EOL_CR;
3790  }
3791  }
3792  else if (cstate->eol_type == EOL_NL)
3793  ereport(ERROR,
3794  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3795  !cstate->csv_mode ?
3796  errmsg("literal carriage return found in data") :
3797  errmsg("unquoted carriage return found in data"),
3798  !cstate->csv_mode ?
3799  errhint("Use \"\\r\" to represent carriage return.") :
3800  errhint("Use quoted CSV field to represent carriage return.")));
3801  /* If reach here, we have found the line terminator */
3802  break;
3803  }
3804 
3805  /* Process \n */
3806  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3807  {
3808  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3809  ereport(ERROR,
3810  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3811  !cstate->csv_mode ?
3812  errmsg("literal newline found in data") :
3813  errmsg("unquoted newline found in data"),
3814  !cstate->csv_mode ?
3815  errhint("Use \"\\n\" to represent newline.") :
3816  errhint("Use quoted CSV field to represent newline.")));
3817  cstate->eol_type = EOL_NL; /* in case not set yet */
3818  /* If reach here, we have found the line terminator */
3819  break;
3820  }
3821 
3822  /*
3823  * In CSV mode, we only recognize \. alone on a line. This is because
3824  * \. is a valid CSV data value.
3825  */
3826  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3827  {
3828  char c2;
3829 
3832 
3833  /* -----
3834  * get next character
3835  * Note: we do not change c so if it isn't \., we can fall
3836  * through and continue processing for file encoding.
3837  * -----
3838  */
3839  c2 = copy_raw_buf[raw_buf_ptr];
3840 
3841  if (c2 == '.')
3842  {
3843  raw_buf_ptr++; /* consume the '.' */
3844 
3845  /*
3846  * Note: if we loop back for more data here, it does not
3847  * matter that the CSV state change checks are re-executed; we
3848  * will come back here with no important state changed.
3849  */
3850  if (cstate->eol_type == EOL_CRNL)
3851  {
3852  /* Get the next character */
3854  /* if hit_eof, c2 will become '\0' */
3855  c2 = copy_raw_buf[raw_buf_ptr++];
3856 
3857  if (c2 == '\n')
3858  {
3859  if (!cstate->csv_mode)
3860  ereport(ERROR,
3861  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3862  errmsg("end-of-copy marker does not match previous newline style")));
3863  else
3865  }
3866  else if (c2 != '\r')
3867  {
3868  if (!cstate->csv_mode)
3869  ereport(ERROR,
3870  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3871  errmsg("end-of-copy marker corrupt")));
3872  else
3874  }
3875  }
3876 
3877  /* Get the next character */
3879  /* if hit_eof, c2 will become '\0' */
3880  c2 = copy_raw_buf[raw_buf_ptr++];
3881 
3882  if (c2 != '\r' && c2 != '\n')
3883  {
3884  if (!cstate->csv_mode)
3885  ereport(ERROR,
3886  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3887  errmsg("end-of-copy marker corrupt")));
3888  else
3890  }
3891 
3892  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3893  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3894  (cstate->eol_type == EOL_CR && c2 != '\r'))
3895  {
3896  ereport(ERROR,
3897  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3898  errmsg("end-of-copy marker does not match previous newline style")));
3899  }
3900 
3901  /*
3902  * Transfer only the data before the \. into line_buf, then
3903  * discard the data and the \. sequence.
3904  */
3905  if (prev_raw_ptr > cstate->raw_buf_index)
3907  cstate->raw_buf + cstate->raw_buf_index,
3908  prev_raw_ptr - cstate->raw_buf_index);
3909  cstate->raw_buf_index = raw_buf_ptr;
3910  result = true; /* report EOF */
3911  break;
3912  }
3913  else if (!cstate->csv_mode)
3914 
3915  /*
3916  * If we are here, it means we found a backslash followed by
3917  * something other than a period. In non-CSV mode, anything
3918  * after a backslash is special, so we skip over that second
3919  * character too. If we didn't do that \\. would be
3920  * considered an eof-of copy, while in non-CSV mode it is a
3921  * literal backslash followed by a period. In CSV mode,
3922  * backslashes are not special, so we want to process the
3923  * character after the backslash just like a normal character,
3924  * so we don't increment in those cases.
3925  */
3926  raw_buf_ptr++;
3927  }
3928 
3929  /*
3930  * This label is for CSV cases where \. appears at the start of a
3931  * line, but there is more text after it, meaning it was a data value.
3932  * We are more strict for \. in CSV mode because \. could be a data
3933  * value, while in non-CSV mode, \. cannot be a data value.
3934  */
3935 not_end_of_copy:
3936 
3937  /*
3938  * Process all bytes of a multi-byte character as a group.
3939  *
3940  * We only support multi-byte sequences where the first byte has the
3941  * high-bit set, so as an optimization we can avoid this block
3942  * entirely if it is not set.
3943  */
3944  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3945  {
3946  int mblen;
3947 
3948  mblen_str[0] = c;
3949  /* All our encodings only read the first byte to get the length */
3950  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3952  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3953  raw_buf_ptr += mblen - 1;
3954  }
3955  first_char_in_line = false;
3956  } /* end of outer loop */
3957 
3958  /*
3959  * Transfer any still-uncopied data to line_buf.
3960  */
3962 
3963  return result;
3964 }
bool csv_mode
Definition: copy.c:119
int errhint(const char *fmt,...)
Definition: elog.c:987
Definition: copy.c:75
StringInfoData line_buf
Definition: copy.c:197
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:251
int raw_buf_index
Definition: copy.c:210
int errcode(int sqlerrcode)
Definition: elog.c:575
#define REFILL_LINEBUF
Definition: copy.c:268
#define IS_HIGHBIT_SET(ch)
Definition: c.h:963
#define ERROR
Definition: elog.h:43
bool encoding_embeds_ascii
Definition: copy.c:107
char * c
char * raw_buf
Definition: copy.c:209
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:736
char * quote
Definition: copy.c:125
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
Definition: copy.c:76
int raw_buf_len
Definition: copy.c:211
char * escape
Definition: copy.c:126
#define ereport(elevel, rest)
Definition: elog.h:122
Definition: copy.c:77
int file_encoding
Definition: copy.c:105
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:239
EolType eol_type
Definition: copy.c:104
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define NO_END_OF_COPY_GOTO
Definition: copy.c:281
int cur_lineno
Definition: copy.c:140
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

◆ CopySendChar()

static void CopySendChar ( CopyState  cstate,
char  c 
)
static

Definition at line 457 of file copy.c.

References appendStringInfoCharMacro, and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), CopyOneRowTo(), CopySendEndOfRow(), and CopyTo().

458 {
460 }
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:127
char * c
StringInfo fe_msgbuf
Definition: copy.c:101

◆ CopySendData()

static void CopySendData ( CopyState  cstate,
const void *  databuf,
int  datasize 
)
static

Definition at line 445 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyOneRowTo(), CopySendInt16(), CopySendInt32(), CopyTo(), and SendCopyEnd().

446 {
447  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
448 }
StringInfo fe_msgbuf
Definition: copy.c:101
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

◆ CopySendEndOfRow()

static void CopySendEndOfRow ( CopyState  cstate)
static

Definition at line 463 of file copy.c.

References Assert, CopyStateData::binary, ClosePipeToProgram(), COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, CopySendChar(), CopySendString(), StringInfoData::data, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, FATAL, CopyStateData::fe_msgbuf, CopyStateData::is_program, StringInfoData::len, pq_putbytes(), pq_putmessage, and resetStringInfo().

Referenced by CopyOneRowTo(), CopyTo(), and SendCopyEnd().

464 {
465  StringInfo fe_msgbuf = cstate->fe_msgbuf;
466 
467  switch (cstate->copy_dest)
468  {
469  case COPY_FILE:
470  if (!cstate->binary)
471  {
472  /* Default line termination depends on platform */
473 #ifndef WIN32
474  CopySendChar(cstate, '\n');
475 #else
476  CopySendString(cstate, "\r\n");
477 #endif
478  }
479 
480  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
481  cstate->copy_file) != 1 ||
482  ferror(cstate->copy_file))
483  {
484  if (cstate->is_program)
485  {
486  if (errno == EPIPE)
487  {
488  /*
489  * The pipe will be closed automatically on error at
490  * the end of transaction, but we might get a better
491  * error message from the subprocess' exit code than
492  * just "Broken Pipe"
493  */
494  ClosePipeToProgram(cstate);
495 
496  /*
497  * If ClosePipeToProgram() didn't throw an error, the
498  * program terminated normally, but closed the pipe
499  * first. Restore errno, and throw an error.
500  */
501  errno = EPIPE;
502  }
503  ereport(ERROR,
505  errmsg("could not write to COPY program: %m")));
506  }
507  else
508  ereport(ERROR,
510  errmsg("could not write to COPY file: %m")));
511  }
512  break;
513  case COPY_OLD_FE:
514  /* The FE/BE protocol uses \n as newline for all platforms */
515  if (!cstate->binary)
516  CopySendChar(cstate, '\n');
517 
518  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
519  {
520  /* no hope of recovering connection sync, so FATAL */
521  ereport(FATAL,
522  (errcode(ERRCODE_CONNECTION_FAILURE),
523  errmsg("connection lost during COPY to stdout")));
524  }
525  break;
526  case COPY_NEW_FE:
527  /* The FE/BE protocol uses \n as newline for all platforms */
528  if (!cstate->binary)
529  CopySendChar(cstate, '\n');
530 
531  /* Dump the accumulated row as one CopyData message */
532  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
533  break;
534  case COPY_CALLBACK:
535  Assert(false); /* Not yet supported. */
536  break;
537  }
538 
539  resetStringInfo(fe_msgbuf);
540 }
bool binary
Definition: copy.c:116
int errcode(int sqlerrcode)
Definition: elog.c:575
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1691
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
CopyDest copy_dest
Definition: copy.c:99
Definition: copy.c:63
#define ERROR
Definition: elog.h:43
#define FATAL
Definition: elog.h:52
int errcode_for_file_access(void)
Definition: elog.c:598
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:114
#define Assert(condition)
Definition: c.h:680
StringInfo fe_msgbuf
Definition: copy.c:101
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:100
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
int pq_putbytes(const char *s, size_t len)
Definition: pqcomm.c:1353

◆ CopySendInt16()

static void CopySendInt16 ( CopyState  cstate,
int16  val 
)
static

Definition at line 699 of file copy.c.

References buf, CopySendData(), and pg_hton16.

Referenced by CopyOneRowTo(), and CopyTo().

700 {
701  uint16 buf;
702 
703  buf = pg_hton16((uint16) val);
704  CopySendData(cstate, &buf, sizeof(buf));
705 }
#define pg_hton16(x)
Definition: pg_bswap.h:120
unsigned short uint16
Definition: c.h:305
static char * buf
Definition: pg_test_fsync.c:67
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
long val
Definition: informix.c:689

◆ CopySendInt32()

static void CopySendInt32 ( CopyState  cstate,
int32  val 
)
static

Definition at line 668 of file copy.c.

References buf, CopySendData(), and pg_hton32.

Referenced by CopyOneRowTo(), and CopyTo().

669 {
670  uint32 buf;
671 
672  buf = pg_hton32((uint32) val);
673  CopySendData(cstate, &buf, sizeof(buf));
674 }
#define pg_hton32(x)
Definition: pg_bswap.h:121
static char * buf
Definition: pg_test_fsync.c:67
unsigned int uint32
Definition: c.h:306
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
long val
Definition: informix.c:689

◆ CopySendString()

static void CopySendString ( CopyState  cstate,
const char *  str 
)
static

Definition at line 451 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyOneRowTo(), and CopySendEndOfRow().

452 {
453  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
454 }
StringInfo fe_msgbuf
Definition: copy.c:101
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

◆ CopyTo()

static uint64 CopyTo ( CopyState  cstate)
static

Definition at line 1929 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, CopyStateData::attnumlist, CopyStateData::binary, BinarySignature, CHECK_FOR_INTERRUPTS, CopyAttributeOutCSV(), CopyOneRowTo(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), cur, CurrentMemoryContext, CopyStateData::delim, QueryDesc::dest, ExecutorRun(), CopyStateData::fe_msgbuf, CopyStateData::file_encoding, fmgr_info(), ForwardScanDirection, GetActiveSnapshot(), getTypeBinaryOutputInfo(), getTypeOutputInfo(), CopyStateData::header_line, heap_beginscan(), heap_deform_tuple(), heap_endscan(), heap_getnext(), HeapTupleGetOid, lfirst_int, list_length(), makeStringInfo(), MemoryContextDelete(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, CopyStateData::null_print, CopyStateData::null_print_client, CopyStateData::null_print_len, CopyStateData::oids, CopyStateData::out_functions, palloc(), pfree(), pg_server_to_any(), CopyStateData::queryDesc, CopyStateData::rel, RelationGetDescr, CopyStateData::rowcontext, QueryDesc::tupDesc, TupleDescAttr, and values.

Referenced by DoCopyTo().

1930 {
1931  TupleDesc tupDesc;
1932  int num_phys_attrs;
1933  ListCell *cur;
1934  uint64 processed;
1935 
1936  if (cstate->rel)
1937  tupDesc = RelationGetDescr(cstate->rel);
1938  else
1939  tupDesc = cstate->queryDesc->tupDesc;
1940  num_phys_attrs = tupDesc->natts;
1941  cstate->null_print_client = cstate->null_print; /* default */
1942 
1943  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1944  cstate->fe_msgbuf = makeStringInfo();
1945 
1946  /* Get info about the columns we need to process. */
1947  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1948  foreach(cur, cstate->attnumlist)
1949  {
1950  int attnum = lfirst_int(cur);
1951  Oid out_func_oid;
1952  bool isvarlena;
1953  Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
1954 
1955  if (cstate->binary)
1956  getTypeBinaryOutputInfo(attr->atttypid,
1957  &out_func_oid,
1958  &isvarlena);
1959  else
1960  getTypeOutputInfo(attr->atttypid,
1961  &out_func_oid,
1962  &isvarlena);
1963  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
1964  }
1965 
1966  /*
1967  * Create a temporary memory context that we can reset once per row to
1968  * recover palloc'd memory. This avoids any problems with leaks inside
1969  * datatype output routines, and should be faster than retail pfree's
1970  * anyway. (We don't need a whole econtext as CopyFrom does.)
1971  */
1973  "COPY TO",
1975 
1976  if (cstate->binary)
1977  {
1978  /* Generate header for a binary copy */
1979  int32 tmp;
1980 
1981  /* Signature */
1982  CopySendData(cstate, BinarySignature, 11);
1983  /* Flags field */
1984  tmp = 0;
1985  if (cstate->oids)
1986  tmp |= (1 << 16);
1987  CopySendInt32(cstate, tmp);
1988  /* No header extension */
1989  tmp = 0;
1990  CopySendInt32(cstate, tmp);
1991  }
1992  else
1993  {
1994  /*
1995  * For non-binary copy, we need to convert null_print to file
1996  * encoding, because it will be sent directly with CopySendString.
1997  */
1998  if (cstate->need_transcoding)
1999  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2000  cstate->null_print_len,
2001  cstate->file_encoding);
2002 
2003  /* if a header has been requested send the line */
2004  if (cstate->header_line)
2005  {
2006  bool hdr_delim = false;
2007 
2008  foreach(cur, cstate->attnumlist)
2009  {
2010  int attnum = lfirst_int(cur);
2011  char *colname;
2012 
2013  if (hdr_delim)
2014  CopySendChar(cstate, cstate->delim[0]);
2015  hdr_delim = true;
2016 
2017  colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
2018 
2019  CopyAttributeOutCSV(cstate, colname, false,
2020  list_length(cstate->attnumlist) == 1);
2021  }
2022 
2023  CopySendEndOfRow(cstate);
2024  }
2025  }
2026 
2027  if (cstate->rel)
2028  {
2029  Datum *values;
2030  bool *nulls;
2031  HeapScanDesc scandesc;
2032  HeapTuple tuple;
2033 
2034  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2035  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2036 
2037  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2038 
2039  processed = 0;
2040  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2041  {
2043 
2044  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2045  heap_deform_tuple(tuple, tupDesc, values, nulls);
2046 
2047  /* Format and send the data */
2048  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2049  processed++;
2050  }
2051 
2052  heap_endscan(scandesc);
2053 
2054  pfree(values);
2055  pfree(nulls);
2056  }
2057  else
2058  {
2059  /* run the plan --- the dest receiver will send tuples */
2060  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2061  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2062  }
2063 
2064  if (cstate->binary)
2065  {
2066  /* Generate trailer for a binary copy */
2067  CopySendInt16(cstate, -1);
2068  /* Need to flush out the trailer */
2069  CopySendEndOfRow(cstate);
2070  }
2071 
2073 
2074  return processed;
2075 }
Definition: fmgr.h:56
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:198
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2665
bool binary
Definition: copy.c:116
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1565
#define RelationGetDescr(relation)
Definition: rel.h:437
bool need_transcoding
Definition: copy.c:106
List * attnumlist
Definition: copy.c:112
#define TupleDescAttr(tupdesc, i)
Definition: tupdesc.h:90
Definition: copy.c:215
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
struct cursor * cur
Definition: ecpg.c:28
unsigned int Oid
Definition: postgres_ext.h:31
char * delim
Definition: copy.c:124
MemoryContext rowcontext
Definition: copy.c:153
int natts
Definition: tupdesc.h:79
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:634
signed int int32
Definition: c.h:294
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * null_print
Definition: copy.c:121
Relation rel
Definition: copy.c:110
void pfree(void *pointer)
Definition: mcxt.c:936
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:699
#define lfirst_int(lc)
Definition: pg_list.h:107
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:297
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4607
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:122
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:197
QueryDesc * queryDesc
Definition: copy.c:111
static const char BinarySignature[11]
Definition: copy.c:288
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
int null_print_len
Definition: copy.c:122
int file_encoding
Definition: copy.c:105
#define AllocSetContextCreate(parent, name, allocparams)
Definition: memutils.h:165
TupleDesc tupDesc
Definition: execdesc.h:47
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2731
bool header_line
Definition: copy.c:120
uintptr_t Datum
Definition: postgres.h:372
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1808
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
StringInfo fe_msgbuf
Definition: copy.c:101
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:668
void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull)
Definition: heaptuple.c:936
static Datum values[MAXATTR]
Definition: bootstrap.c:164
DestReceiver * dest
Definition: execdesc.h:41
FmgrInfo * out_functions
Definition: copy.c:152
void * palloc(Size size)
Definition: mcxt.c:835
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:463
#define NameStr(name)
Definition: c.h:557
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
char * null_print_client
Definition: copy.c:123
#define HeapTupleGetOid(tuple)
Definition: htup_details.h:700
HeapScanDesc heap_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key)
Definition: heapam.c:1397
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2081
bool oids
Definition: copy.c:117

◆ CreateCopyDestReceiver()

DestReceiver* CreateCopyDestReceiver ( void  )

Definition at line 4817 of file copy.c.

References copy_dest_destroy(), copy_dest_receive(), copy_dest_shutdown(), copy_dest_startup(), DestCopyOut, and palloc().

Referenced by CreateDestReceiver().

4818 {
4819  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4820 
4821  self->pub.receiveSlot = copy_dest_receive;
4822  self->pub.rStartup = copy_dest_startup;
4823  self->pub.rShutdown = copy_dest_shutdown;
4824  self->pub.rDestroy = copy_dest_destroy;
4825  self->pub.mydest = DestCopyOut;
4826 
4827  self->cstate = NULL; /* will be set later */
4828  self->processed = 0;
4829 
4830  return (DestReceiver *) self;
4831 }
Definition: copy.c:215
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4808
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4799
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4771
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4780
void * palloc(Size size)
Definition: mcxt.c:835

◆ DoCopy()

void DoCopy ( ParseState pstate,
const CopyStmt stmt,
int  stmt_location,
int  stmt_len,
uint64 *  processed 
)

Definition at line 781 of file copy.c.

References AccessShareLock, ACL_INSERT, ACL_SELECT, addRangeTableEntryForRelation(), Assert, CopyStmt::attlist, BeginCopyFrom(), BeginCopyTo(), bms_add_member(), check_enable_rls(), CopyFrom(), CopyGetAttnums(), cur, DoCopyTo(), EndCopyFrom(), EndCopyTo(), ereport, errcode(), errhint(), errmsg(), ERROR, ExecCheckRTPerms(), ColumnRef::fields, CopyStmt::filename, FirstLowInvalidHeapAttributeNumber, SelectStmt::fromClause, get_namespace_name(), heap_close, heap_openrv(), ResTarget::indirection, RangeTblEntry::insertedCols, InvalidOid, CopyStmt::is_from, CopyStmt::is_program, lappend(), lfirst, lfirst_int, list_make1, ColumnRef::location, ResTarget::location, makeNode, makeRangeVar(), ResTarget::name, NIL, NoLock, CopyStmt::options, ParseState::p_rtable, PreventCommandIfParallelMode(), PreventCommandIfReadOnly(), pstrdup(), CopyStmt::query, RelationData::rd_islocaltemp, CopyStateData::rel, CopyStmt::relation, RelationGetDescr, RelationGetNamespace, RelationGetRelationName, RelationGetRelid, RangeTblEntry::relid, RangeTblEntry::requiredPerms, RLS_ENABLED, RowExclusiveLock, select, RangeTblEntry::selectedCols, RawStmt::stmt, RawStmt::stmt_len, RawStmt::stmt_location, superuser(), SelectStmt::targetList, ResTarget::val, and XactReadOnly.

Referenced by standard_ProcessUtility().

784 {
785  CopyState cstate;
786  bool is_from = stmt->is_from;
787  bool pipe = (stmt->filename == NULL);
788  Relation rel;
789  Oid relid;
790  RawStmt *query = NULL;
791 
792  /* Disallow COPY to/from file or program except to superusers. */
793  if (!pipe && !superuser())
794  {
795  if (stmt->is_program)
796  ereport(ERROR,
797  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
798  errmsg("must be superuser to COPY to or from an external program"),
799  errhint("Anyone can COPY to stdout or from stdin. "
800  "psql's \\copy command also works for anyone.")));
801  else
802  ereport(ERROR,
803  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
804  errmsg("must be superuser to COPY to or from a file"),
805  errhint("Anyone can COPY to stdout or from stdin. "
806  "psql's \\copy command also works for anyone.")));
807  }
808 
809  if (stmt->relation)
810  {
811  TupleDesc tupDesc;
812  List *attnums;
813  ListCell *cur;
814  RangeTblEntry *rte;
815 
816  Assert(!stmt->query);
817 
818  /* Open and lock the relation, using the appropriate lock type. */
819  rel = heap_openrv(stmt->relation,
820  (is_from ? RowExclusiveLock : AccessShareLock));
821 
822  relid = RelationGetRelid(rel);
823 
824  rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
825  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
826 
827  tupDesc = RelationGetDescr(rel);
828  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
829  foreach(cur, attnums)
830  {
831  int attno = lfirst_int(cur) -
833 
834  if (is_from)
835  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
836  else
837  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
838  }
839  ExecCheckRTPerms(pstate->p_rtable, true);
840 
841  /*
842  * Permission check for row security policies.
843  *
844  * check_enable_rls will ereport(ERROR) if the user has requested
845  * something invalid and will otherwise indicate if we should enable
846  * RLS (returns RLS_ENABLED) or not for this COPY statement.
847  *
848  * If the relation has a row security policy and we are to apply it
849  * then perform a "query" copy and allow the normal query processing
850  * to handle the policies.
851  *
852  * If RLS is not enabled for this, then just fall through to the
853  * normal non-filtering relation handling.
854  */
855  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
856  {
858  ColumnRef *cr;
859  ResTarget *target;
860  RangeVar *from;
861  List *targetList = NIL;
862 
863  if (is_from)
864  ereport(ERROR,
865  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
866  errmsg("COPY FROM not supported with row-level security"),
867  errhint("Use INSERT statements instead.")));
868 
869  /*
870  * Build target list
871  *
872  * If no columns are specified in the attribute list of the COPY
873  * command, then the target list is 'all' columns. Therefore, '*'
874  * should be used as the target list for the resulting SELECT
875  * statement.
876  *
877  * In the case that columns are specified in the attribute list,
878  * create a ColumnRef and ResTarget for each column and add them
879  * to the target list for the resulting SELECT statement.
880  */
881  if (!stmt->attlist)
882  {
883  cr = makeNode(ColumnRef);
885  cr->location = -1;
886 
887  target = makeNode(ResTarget);
888  target->name = NULL;
889  target->indirection = NIL;
890  target->val = (Node *) cr;
891  target->location = -1;
892 
893  targetList = list_make1(target);
894  }
895  else
896  {
897  ListCell *lc;
898 
899  foreach(lc, stmt->attlist)
900  {
901  /*
902  * Build the ColumnRef for each column. The ColumnRef
903  * 'fields' property is a String 'Value' node (see