PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy.c File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/sysattr.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/pg_type.h"
#include "commands/copy.h"
#include "commands/defrem.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "optimizer/clauses.h"
#include "optimizer/planner.h"
#include "nodes/makefuncs.h"
#include "parser/parse_relation.h"
#include "rewrite/rewriteHandler.h"
#include "storage/fd.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/portal.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
Include dependency graph for copy.c:

Go to the source code of this file.

Data Structures

struct  CopyStateData
 
struct  DR_copy
 

Macros

#define ISOCTAL(c)   (((c) >= '0') && ((c) <= '7'))
 
#define OCTVALUE(c)   ((c) - '0')
 
#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */
 
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
 
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
 
#define REFILL_LINEBUF
 
#define NO_END_OF_COPY_GOTO
 
#define MAX_COPY_DATA_DISPLAY   100
 
#define MAX_BUFFERED_TUPLES   1000
 
#define DUMPSOFAR()
 

Typedefs

typedef enum CopyDest CopyDest
 
typedef enum EolType EolType
 
typedef struct CopyStateData CopyStateData
 

Enumerations

enum  CopyDest { COPY_FILE, COPY_OLD_FE, COPY_NEW_FE, COPY_CALLBACK }
 
enum  EolType { EOL_UNKNOWN, EOL_NL, EOL_CR, EOL_CRNL }
 

Functions

static CopyState BeginCopy (ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
 
static void EndCopy (CopyState cstate)
 
static void ClosePipeToProgram (CopyState cstate)
 
static CopyState BeginCopyTo (ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
 
static void EndCopyTo (CopyState cstate)
 
static uint64 DoCopyTo (CopyState cstate)
 
static uint64 CopyTo (CopyState cstate)
 
static void CopyOneRowTo (CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
 
static void CopyFromInsertBatch (CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
 
static bool CopyReadLine (CopyState cstate)
 
static bool CopyReadLineText (CopyState cstate)
 
static int CopyReadAttributesText (CopyState cstate)
 
static int CopyReadAttributesCSV (CopyState cstate)
 
static Datum CopyReadBinaryAttribute (CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
 
static void CopyAttributeOutText (CopyState cstate, char *string)
 
static void CopyAttributeOutCSV (CopyState cstate, char *string, bool use_quote, bool single_attr)
 
static ListCopyGetAttnums (TupleDesc tupDesc, Relation rel, List *attnamelist)
 
static char * limit_printout_length (const char *str)
 
static void SendCopyBegin (CopyState cstate)
 
static void ReceiveCopyBegin (CopyState cstate)
 
static void SendCopyEnd (CopyState cstate)
 
static void CopySendData (CopyState cstate, const void *databuf, int datasize)
 
static void CopySendString (CopyState cstate, const char *str)
 
static void CopySendChar (CopyState cstate, char c)
 
static void CopySendEndOfRow (CopyState cstate)
 
static int CopyGetData (CopyState cstate, void *databuf, int minread, int maxread)
 
static void CopySendInt32 (CopyState cstate, int32 val)
 
static bool CopyGetInt32 (CopyState cstate, int32 *val)
 
static void CopySendInt16 (CopyState cstate, int16 val)
 
static bool CopyGetInt16 (CopyState cstate, int16 *val)
 
static bool CopyLoadRawBuf (CopyState cstate)
 
void DoCopy (ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
 
void ProcessCopyOptions (ParseState *pstate, CopyState cstate, bool is_from, List *options)
 
void CopyFromErrorCallback (void *arg)
 
uint64 CopyFrom (CopyState cstate)
 
CopyState BeginCopyFrom (ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
 
bool NextCopyFromRawFields (CopyState cstate, char ***fields, int *nfields)
 
bool NextCopyFrom (CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
 
void EndCopyFrom (CopyState cstate)
 
static int GetDecimalFromHex (char hex)
 
static void copy_dest_startup (DestReceiver *self, int operation, TupleDesc typeinfo)
 
static bool copy_dest_receive (TupleTableSlot *slot, DestReceiver *self)
 
static void copy_dest_shutdown (DestReceiver *self)
 
static void copy_dest_destroy (DestReceiver *self)
 
DestReceiverCreateCopyDestReceiver (void)
 

Variables

static const char BinarySignature [11] = "PGCOPY\n\377\r\n\0"
 

Macro Definition Documentation

#define DUMPSOFAR ( )
Value:
do { \
if (ptr > start) \
CopySendData(cstate, start, ptr - start); \
} while (0)
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:447

Definition at line 4450 of file copy.c.

Referenced by CopyAttributeOutCSV(), and CopyAttributeOutText().

#define IF_NEED_REFILL_AND_EOF_BREAK (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
{ \
if (extralen) \
raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
/* backslash just before EOF, treat as data char */ \
result = true; \
break; \
} \
} else ((void) 0)
return result
Definition: formatting.c:1633

Definition at line 253 of file copy.c.

Referenced by CopyReadLineText().

#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
{ \
raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
need_data = true; \
continue; \
} \
} else ((void) 0)

Definition at line 241 of file copy.c.

Referenced by CopyReadLineText().

#define ISOCTAL (   c)    (((c) >= '0') && ((c) <= '7'))

Definition at line 53 of file copy.c.

Referenced by CopyReadAttributesText().

#define MAX_BUFFERED_TUPLES   1000

Referenced by CopyFrom().

#define MAX_COPY_DATA_DISPLAY   100

Referenced by limit_printout_length().

#define NO_END_OF_COPY_GOTO
Value:
if (1) \
{ \
raw_buf_ptr = prev_raw_ptr + 1; \
goto not_end_of_copy; \
} else ((void) 0)

Definition at line 283 of file copy.c.

Referenced by CopyReadLineText().

#define OCTVALUE (   c)    ((c) - '0')

Definition at line 54 of file copy.c.

Referenced by CopyReadAttributesText().

#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */

Definition at line 210 of file copy.c.

Referenced by BeginCopyFrom(), and CopyLoadRawBuf().

#define REFILL_LINEBUF
Value:
if (1) \
{ \
if (raw_buf_ptr > cstate->raw_buf_index) \
{ \
appendBinaryStringInfo(&cstate->line_buf, \
cstate->raw_buf + cstate->raw_buf_index, \
raw_buf_ptr - cstate->raw_buf_index); \
cstate->raw_buf_index = raw_buf_ptr; \
} \
} else ((void) 0)
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

Definition at line 270 of file copy.c.

Referenced by CopyReadLineText().

Typedef Documentation

Enumeration Type Documentation

enum CopyDest
Enumerator
COPY_FILE 
COPY_OLD_FE 
COPY_NEW_FE 
COPY_CALLBACK 

Definition at line 60 of file copy.c.

61 {
62  COPY_FILE, /* to/from file (or a piped program) */
63  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
64  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
65  COPY_CALLBACK /* to/from callback function */
66 } CopyDest;
Definition: copy.c:62
CopyDest
Definition: copy.c:60
enum EolType
Enumerator
EOL_UNKNOWN 
EOL_NL 
EOL_CR 
EOL_CRNL 

Definition at line 71 of file copy.c.

72 {
74  EOL_NL,
75  EOL_CR,
76  EOL_CRNL
77 } EolType;
Definition: copy.c:74
EolType
Definition: copy.c:71
Definition: copy.c:75
Definition: copy.c:76

Function Documentation

static CopyState BeginCopy ( ParseState pstate,
bool  is_from,
Relation  rel,
RawStmt raw_query,
Oid  queryRelId,
List attnamelist,
List options 
)
static

Definition at line 1374 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), Assert, CopyStateData::attnumlist, tupleDesc::attrs, CMD_DELETE, CMD_INSERT, CMD_SELECT, CMD_UPDATE, Query::commandType, CopyStateData::convert_select, CopyStateData::convert_select_flags, CopyStateData::convert_selectively, convert_tuples_by_name(), CopyStateData::copy_dest, COPY_FILE, CopyStateData::copycontext, CopyGetAttnums(), copyObject, CreateDestReceiver(), CreateQueryDesc(), cur, CurrentMemoryContext, CURSOR_OPT_PARALLEL_OK, DestCopyOut, CopyStateData::encoding_embeds_ascii, ereport, errcode(), errmsg(), errmsg_internal(), ERROR, ExecSetupPartitionTupleRouting(), ExecutorStart(), CopyStateData::file_encoding, CopyStateData::force_notnull, CopyStateData::force_notnull_flags, CopyStateData::force_null, CopyStateData::force_null_flags, CopyStateData::force_quote, CopyStateData::force_quote_all, CopyStateData::force_quote_flags, GetActiveSnapshot(), GetDatabaseEncoding(), gettext_noop, i, InvalidOid, InvalidSnapshot, IsA, lfirst_int, lfirst_node, linitial_node, list_length(), list_member_int(), list_member_oid(), MakeTransitionCaptureState(), MemoryContextSwitchTo(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NIL, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, CopyStateData::oids, ParseState::p_sourcetext, palloc0(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pg_analyze_and_rewrite(), pg_database_encoding_max_length(), PG_ENCODING_IS_CLIENT_ONLY, pg_get_client_encoding(), pg_plan_query(), ProcessCopyOptions(), PushCopiedSnapshot(), QSRC_NON_INSTEAD_RULE, QSRC_QUAL_INSTEAD_RULE, CopyStateData::queryDesc, Query::querySource, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, PlannedStmt::relationOids, RELKIND_PARTITIONED_TABLE, Query::returningList, ResultRelInfo::ri_RelationDesc, CopyStateData::transition_capture, CopyStateData::transition_tupconv_maps, RelationData::trigdesc, QueryDesc::tupDesc, UpdateActiveSnapshotCommandId(), and Query::utilityStmt.

Referenced by BeginCopyFrom(), and BeginCopyTo().

1381 {
1382  CopyState cstate;
1383  TupleDesc tupDesc;
1384  int num_phys_attrs;
1385  MemoryContext oldcontext;
1386 
1387  /* Allocate workspace and zero all fields */
1388  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1389 
1390  /*
1391  * We allocate everything used by a cstate in a new memory context. This
1392  * avoids memory leaks during repeated use of COPY in a query.
1393  */
1395  "COPY",
1397 
1398  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1399 
1400  /* Extract options from the statement node tree */
1401  ProcessCopyOptions(pstate, cstate, is_from, options);
1402 
1403  /* Process the source/target relation or query */
1404  if (rel)
1405  {
1406  Assert(!raw_query);
1407 
1408  cstate->rel = rel;
1409 
1410  tupDesc = RelationGetDescr(cstate->rel);
1411 
1412  /* Don't allow COPY w/ OIDs to or from a table without them */
1413  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1414  ereport(ERROR,
1415  (errcode(ERRCODE_UNDEFINED_COLUMN),
1416  errmsg("table \"%s\" does not have OIDs",
1417  RelationGetRelationName(cstate->rel))));
1418 
1419  /*
1420  * If there are any triggers with transition tables on the named
1421  * relation, we need to be prepared to capture transition tuples.
1422  */
1424 
1425  /* Initialize state for CopyFrom tuple routing. */
1426  if (is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1427  {
1428  PartitionDispatch *partition_dispatch_info;
1429  ResultRelInfo *partitions;
1430  TupleConversionMap **partition_tupconv_maps;
1431  TupleTableSlot *partition_tuple_slot;
1432  int num_parted,
1433  num_partitions;
1434 
1436  1,
1437  &partition_dispatch_info,
1438  &partitions,
1439  &partition_tupconv_maps,
1440  &partition_tuple_slot,
1441  &num_parted, &num_partitions);
1442  cstate->partition_dispatch_info = partition_dispatch_info;
1443  cstate->num_dispatch = num_parted;
1444  cstate->partitions = partitions;
1445  cstate->num_partitions = num_partitions;
1446  cstate->partition_tupconv_maps = partition_tupconv_maps;
1447  cstate->partition_tuple_slot = partition_tuple_slot;
1448 
1449  /*
1450  * If we are capturing transition tuples, they may need to be
1451  * converted from partition format back to partitioned table
1452  * format (this is only ever necessary if a BEFORE trigger
1453  * modifies the tuple).
1454  */
1455  if (cstate->transition_capture != NULL)
1456  {
1457  int i;
1458 
1460  palloc0(sizeof(TupleConversionMap *) *
1461  cstate->num_partitions);
1462  for (i = 0; i < cstate->num_partitions; ++i)
1463  {
1464  cstate->transition_tupconv_maps[i] =
1466  RelationGetDescr(rel),
1467  gettext_noop("could not convert row type"));
1468  }
1469  }
1470  }
1471  }
1472  else
1473  {
1474  List *rewritten;
1475  Query *query;
1476  PlannedStmt *plan;
1477  DestReceiver *dest;
1478 
1479  Assert(!is_from);
1480  cstate->rel = NULL;
1481 
1482  /* Don't allow COPY w/ OIDs from a query */
1483  if (cstate->oids)
1484  ereport(ERROR,
1485  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1486  errmsg("COPY (query) WITH OIDS is not supported")));
1487 
1488  /*
1489  * Run parse analysis and rewrite. Note this also acquires sufficient
1490  * locks on the source table(s).
1491  *
1492  * Because the parser and planner tend to scribble on their input, we
1493  * make a preliminary copy of the source querytree. This prevents
1494  * problems in the case that the COPY is in a portal or plpgsql
1495  * function and is executed repeatedly. (See also the same hack in
1496  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1497  */
1498  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1499  pstate->p_sourcetext, NULL, 0,
1500  NULL);
1501 
1502  /* check that we got back something we can work with */
1503  if (rewritten == NIL)
1504  {
1505  ereport(ERROR,
1506  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1507  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1508  }
1509  else if (list_length(rewritten) > 1)
1510  {
1511  ListCell *lc;
1512 
1513  /* examine queries to determine which error message to issue */
1514  foreach(lc, rewritten)
1515  {
1516  Query *q = lfirst_node(Query, lc);
1517 
1519  ereport(ERROR,
1520  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1521  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1523  ereport(ERROR,
1524  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1525  errmsg("DO ALSO rules are not supported for the COPY")));
1526  }
1527 
1528  ereport(ERROR,
1529  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1530  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1531  }
1532 
1533  query = linitial_node(Query, rewritten);
1534 
1535  /* The grammar allows SELECT INTO, but we don't support that */
1536  if (query->utilityStmt != NULL &&
1538  ereport(ERROR,
1539  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1540  errmsg("COPY (SELECT INTO) is not supported")));
1541 
1542  Assert(query->utilityStmt == NULL);
1543 
1544  /*
1545  * Similarly the grammar doesn't enforce the presence of a RETURNING
1546  * clause, but this is required here.
1547  */
1548  if (query->commandType != CMD_SELECT &&
1549  query->returningList == NIL)
1550  {
1551  Assert(query->commandType == CMD_INSERT ||
1552  query->commandType == CMD_UPDATE ||
1553  query->commandType == CMD_DELETE);
1554 
1555  ereport(ERROR,
1556  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1557  errmsg("COPY query must have a RETURNING clause")));
1558  }
1559 
1560  /* plan the query */
1561  plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);
1562 
1563  /*
1564  * With row level security and a user using "COPY relation TO", we
1565  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1566  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1567  * in any RLS clauses.
1568  *
1569  * When this happens, we are passed in the relid of the originally
1570  * found relation (which we have locked). As the planner will look up
1571  * the relation again, we double-check here to make sure it found the
1572  * same one that we have locked.
1573  */
1574  if (queryRelId != InvalidOid)
1575  {
1576  /*
1577  * Note that with RLS involved there may be multiple relations,
1578  * and while the one we need is almost certainly first, we don't
1579  * make any guarantees of that in the planner, so check the whole
1580  * list and make sure we find the original relation.
1581  */
1582  if (!list_member_oid(plan->relationOids, queryRelId))
1583  ereport(ERROR,
1584  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1585  errmsg("relation referenced by COPY statement has changed")));
1586  }
1587 
1588  /*
1589  * Use a snapshot with an updated command ID to ensure this query sees
1590  * results of any previously executed queries.
1591  */
1594 
1595  /* Create dest receiver for COPY OUT */
1597  ((DR_copy *) dest)->cstate = cstate;
1598 
1599  /* Create a QueryDesc requesting no output */
1600  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1603  dest, NULL, NULL, 0);
1604 
1605  /*
1606  * Call ExecutorStart to prepare the plan for execution.
1607  *
1608  * ExecutorStart computes a result tupdesc for us
1609  */
1610  ExecutorStart(cstate->queryDesc, 0);
1611 
1612  tupDesc = cstate->queryDesc->tupDesc;
1613  }
1614 
1615  /* Generate or convert list of attributes to process */
1616  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1617 
1618  num_phys_attrs = tupDesc->natts;
1619 
1620  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1621  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1622  if (cstate->force_quote_all)
1623  {
1624  int i;
1625 
1626  for (i = 0; i < num_phys_attrs; i++)
1627  cstate->force_quote_flags[i] = true;
1628  }
1629  else if (cstate->force_quote)
1630  {
1631  List *attnums;
1632  ListCell *cur;
1633 
1634  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1635 
1636  foreach(cur, attnums)
1637  {
1638  int attnum = lfirst_int(cur);
1639 
1640  if (!list_member_int(cstate->attnumlist, attnum))
1641  ereport(ERROR,
1642  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1643  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1644  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1645  cstate->force_quote_flags[attnum - 1] = true;
1646  }
1647  }
1648 
1649  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1650  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1651  if (cstate->force_notnull)
1652  {
1653  List *attnums;
1654  ListCell *cur;
1655 
1656  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1657 
1658  foreach(cur, attnums)
1659  {
1660  int attnum = lfirst_int(cur);
1661 
1662  if (!list_member_int(cstate->attnumlist, attnum))
1663  ereport(ERROR,
1664  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1665  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1666  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1667  cstate->force_notnull_flags[attnum - 1] = true;
1668  }
1669  }
1670 
1671  /* Convert FORCE_NULL name list to per-column flags, check validity */
1672  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1673  if (cstate->force_null)
1674  {
1675  List *attnums;
1676  ListCell *cur;
1677 
1678  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1679 
1680  foreach(cur, attnums)
1681  {
1682  int attnum = lfirst_int(cur);
1683 
1684  if (!list_member_int(cstate->attnumlist, attnum))
1685  ereport(ERROR,
1686  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1687  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1688  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1689  cstate->force_null_flags[attnum - 1] = true;
1690  }
1691  }
1692 
1693  /* Convert convert_selectively name list to per-column flags */
1694  if (cstate->convert_selectively)
1695  {
1696  List *attnums;
1697  ListCell *cur;
1698 
1699  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1700 
1701  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1702 
1703  foreach(cur, attnums)
1704  {
1705  int attnum = lfirst_int(cur);
1706 
1707  if (!list_member_int(cstate->attnumlist, attnum))
1708  ereport(ERROR,
1709  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1710  errmsg_internal("selected column \"%s\" not referenced by COPY",
1711  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1712  cstate->convert_select_flags[attnum - 1] = true;
1713  }
1714  }
1715 
1716  /* Use client encoding when ENCODING option is not specified. */
1717  if (cstate->file_encoding < 0)
1719 
1720  /*
1721  * Set up encoding conversion info. Even if the file and server encodings
1722  * are the same, we must apply pg_any_to_server() to validate data in
1723  * multibyte encodings.
1724  */
1725  cstate->need_transcoding =
1726  (cstate->file_encoding != GetDatabaseEncoding() ||
1728  /* See Multibyte encoding comment above */
1730 
1731  cstate->copy_dest = COPY_FILE; /* default */
1732 
1733  MemoryContextSwitchTo(oldcontext);
1734 
1735  return cstate;
1736 }
#define NIL
Definition: pg_list.h:69
Relation ri_RelationDesc
Definition: execnodes.h:354
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:781
#define IsA(nodeptr, _type_)
Definition: nodes.h:560
TransitionCaptureState * MakeTransitionCaptureState(TriggerDesc *trigdesc)
Definition: trigger.c:2104
#define RelationGetDescr(relation)
Definition: rel.h:428
bool need_transcoding
Definition: copy.c:105
ResultRelInfo * partitions
Definition: copy.c:171
List * attnumlist
Definition: copy.c:111
List * relationOids
Definition: plannodes.h:88
Definition: copy.c:217
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:146
Form_pg_attribute * attrs
Definition: tupdesc.h:74
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1016
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
#define gettext_noop(x)
Definition: c.h:139
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:575
bool * force_quote_flags
Definition: copy.c:128
Form_pg_class rd_rel
Definition: rel.h:114
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:638
Node * utilityStmt
Definition: parsenodes.h:118
#define linitial_node(type, l)
Definition: pg_list.h:114
bool * force_null_flags
Definition: copy.c:132
int natts
Definition: tupdesc.h:73
bool * convert_select_flags
Definition: copy.c:135
void ExecSetupPartitionTupleRouting(Relation rel, Index resultRTindex, PartitionDispatch **pd, ResultRelInfo **partitions, TupleConversionMap ***tup_conv_maps, TupleTableSlot **partition_tuple_slot, int *num_parted, int *num_partitions)
Definition: execMain.c:3238
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
Relation rel
Definition: copy.c:109
MemoryContext copycontext
Definition: copy.c:146
TupleTableSlot * partition_tuple_slot
Definition: copy.c:173
#define ERROR
Definition: elog.h:43
#define lfirst_int(lc)
Definition: pg_list.h:107
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:769
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
TriggerDesc * trigdesc
Definition: rel.h:120
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
#define lfirst_node(type, lc)
Definition: pg_list.h:109
QueryDesc * queryDesc
Definition: copy.c:110
bool list_member_int(const List *list, int datum)
Definition: list.c:485
int num_dispatch
Definition: copy.c:169
bool encoding_embeds_ascii
Definition: copy.c:106
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
#define RelationGetRelationName(relation)
Definition: rel.h:436
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
const char * p_sourcetext
Definition: parse_node.h:171
List * returningList
Definition: parsenodes.h:144
#define ereport(elevel, rest)
Definition: elog.h:122
List * force_null
Definition: copy.c:131
TupleConversionMap * convert_tuples_by_name(TupleDesc indesc, TupleDesc outdesc, const char *msg)
Definition: tupconvert.c:205
TupleConversionMap ** transition_tupconv_maps
Definition: copy.c:175
int file_encoding
Definition: copy.c:104
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4701
TupleDesc tupDesc
Definition: execdesc.h:47
#define InvalidSnapshot
Definition: snapshot.h:25
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
void * palloc0(Size size)
Definition: mcxt.c:878
int GetDatabaseEncoding(void)
Definition: mbutils.c:1015
int pg_get_client_encoding(void)
Definition: mbutils.c:317
#define InvalidOid
Definition: postgres_ext.h:36
bool * force_notnull_flags
Definition: copy.c:130
CmdType commandType
Definition: parsenodes.h:110
List * force_notnull
Definition: copy.c:129
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:505
#define NULL
Definition: c.h:229
QuerySource querySource
Definition: parsenodes.h:112
#define Assert(condition)
Definition: c.h:675
List * convert_select
Definition: copy.c:134
static int list_length(const List *l)
Definition: pg_list.h:89
bool force_quote_all
Definition: copy.c:127
#define PG_ENCODING_IS_CLIENT_ONLY(_enc)
Definition: pg_wchar.h:298
#define CURSOR_OPT_PARALLEL_OK
Definition: parsenodes.h:2631
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
#define NameStr(name)
Definition: c.h:499
PartitionDispatch * partition_dispatch_info
Definition: copy.c:168
TransitionCaptureState * transition_capture
Definition: copy.c:174
#define copyObject(obj)
Definition: nodes.h:622
TupleConversionMap ** partition_tupconv_maps
Definition: copy.c:172
Definition: pg_list.h:45
int num_partitions
Definition: copy.c:170
List * force_quote
Definition: copy.c:126
bool convert_selectively
Definition: copy.c:133
bool oids
Definition: copy.c:116
PlannedStmt * pg_plan_query(Query *querytree, int cursorOptions, ParamListInfo boundParams)
Definition: postgres.c:779
CopyState BeginCopyFrom ( ParseState pstate,
Relation  rel,
const char *  filename,
bool  is_program,
copy_data_source_cb  data_source_cb,
List attnamelist,
List options 
)

Definition at line 2957 of file copy.c.

References AllocateFile(), Assert, CopyStateData::attnumlist, CopyStateData::attribute_buf, tupleDesc::attrs, BeginCopy(), CopyStateData::binary, BinarySignature, build_column_default(), contain_volatile_functions_not_nextval(), COPY_CALLBACK, CopyStateData::copy_dest, CopyStateData::copy_file, CopyStateData::copycontext, CopyGetData(), CopyGetInt32(), CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, CopyStateData::data_source_cb, CopyStateData::defexprs, CopyStateData::defmap, DestRemote, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, ExecInitExpr(), expression_planner(), CopyStateData::fe_eof, CopyStateData::file_has_oids, CopyStateData::filename, fmgr_info(), getTypeBinaryInputInfo(), getTypeInputInfo(), CopyStateData::in_functions, initStringInfo(), InvalidOid, CopyStateData::is_program, CopyStateData::line_buf, CopyStateData::line_buf_converted, list_length(), list_member_int(), CopyStateData::max_fields, MemoryContextSwitchTo(), tupleDesc::natts, NULL, CopyStateData::num_defaults, CopyStateData::oid_in_function, CopyStateData::oid_typioparam, OIDOID, CopyStateData::oids, OpenPipeStream(), ParseState::p_rtable, palloc(), PG_BINARY_R, pstrdup(), CopyStateData::range_table, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, RAW_BUF_SIZE, CopyStateData::raw_fields, ReceiveCopyBegin(), CopyStateData::rel, RelationGetDescr, RelationGetRelationName, CopyStateData::typioparams, CopyStateData::volatile_defexprs, and whereToSendOutput.

Referenced by copy_table(), DoCopy(), file_acquire_sample_rows(), fileBeginForeignScan(), and fileReScanForeignScan().

2964 {
2965  CopyState cstate;
2966  bool pipe = (filename == NULL);
2967  TupleDesc tupDesc;
2968  Form_pg_attribute *attr;
2969  AttrNumber num_phys_attrs,
2970  num_defaults;
2971  FmgrInfo *in_functions;
2972  Oid *typioparams;
2973  int attnum;
2974  Oid in_func_oid;
2975  int *defmap;
2976  ExprState **defexprs;
2977  MemoryContext oldcontext;
2978  bool volatile_defexprs;
2979 
2980  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
2981  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
2982 
2983  /* Initialize state variables */
2984  cstate->fe_eof = false;
2985  cstate->eol_type = EOL_UNKNOWN;
2986  cstate->cur_relname = RelationGetRelationName(cstate->rel);
2987  cstate->cur_lineno = 0;
2988  cstate->cur_attname = NULL;
2989  cstate->cur_attval = NULL;
2990 
2991  /* Set up variables to avoid per-attribute overhead. */
2992  initStringInfo(&cstate->attribute_buf);
2993  initStringInfo(&cstate->line_buf);
2994  cstate->line_buf_converted = false;
2995  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
2996  cstate->raw_buf_index = cstate->raw_buf_len = 0;
2997 
2998  /* Assign range table, we'll need it in CopyFrom. */
2999  if (pstate)
3000  cstate->range_table = pstate->p_rtable;
3001 
3002  tupDesc = RelationGetDescr(cstate->rel);
3003  attr = tupDesc->attrs;
3004  num_phys_attrs = tupDesc->natts;
3005  num_defaults = 0;
3006  volatile_defexprs = false;
3007 
3008  /*
3009  * Pick up the required catalog information for each attribute in the
3010  * relation, including the input function, the element type (to pass to
3011  * the input function), and info about defaults and constraints. (Which
3012  * input function we use depends on text/binary format choice.)
3013  */
3014  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
3015  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
3016  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
3017  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
3018 
3019  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
3020  {
3021  /* We don't need info for dropped attributes */
3022  if (attr[attnum - 1]->attisdropped)
3023  continue;
3024 
3025  /* Fetch the input function and typioparam info */
3026  if (cstate->binary)
3027  getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
3028  &in_func_oid, &typioparams[attnum - 1]);
3029  else
3030  getTypeInputInfo(attr[attnum - 1]->atttypid,
3031  &in_func_oid, &typioparams[attnum - 1]);
3032  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
3033 
3034  /* Get default info if needed */
3035  if (!list_member_int(cstate->attnumlist, attnum))
3036  {
3037  /* attribute is NOT to be copied from input */
3038  /* use default value if one exists */
3039  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
3040  attnum);
3041 
3042  if (defexpr != NULL)
3043  {
3044  /* Run the expression through planner */
3045  defexpr = expression_planner(defexpr);
3046 
3047  /* Initialize executable expression in copycontext */
3048  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
3049  defmap[num_defaults] = attnum - 1;
3050  num_defaults++;
3051 
3052  /*
3053  * If a default expression looks at the table being loaded,
3054  * then it could give the wrong answer when using
3055  * multi-insert. Since database access can be dynamic this is
3056  * hard to test for exactly, so we use the much wider test of
3057  * whether the default expression is volatile. We allow for
3058  * the special case of when the default expression is the
3059  * nextval() of a sequence which in this specific case is
3060  * known to be safe for use with the multi-insert
3061  * optimization. Hence we use this special case function
3062  * checker rather than the standard check for
3063  * contain_volatile_functions().
3064  */
3065  if (!volatile_defexprs)
3066  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
3067  }
3068  }
3069  }
3070 
3071  /* We keep those variables in cstate. */
3072  cstate->in_functions = in_functions;
3073  cstate->typioparams = typioparams;
3074  cstate->defmap = defmap;
3075  cstate->defexprs = defexprs;
3076  cstate->volatile_defexprs = volatile_defexprs;
3077  cstate->num_defaults = num_defaults;
3078  cstate->is_program = is_program;
3079 
3080  if (data_source_cb)
3081  {
3082  cstate->copy_dest = COPY_CALLBACK;
3083  cstate->data_source_cb = data_source_cb;
3084  }
3085  else if (pipe)
3086  {
3087  Assert(!is_program); /* the grammar does not allow this */
3089  ReceiveCopyBegin(cstate);
3090  else
3091  cstate->copy_file = stdin;
3092  }
3093  else
3094  {
3095  cstate->filename = pstrdup(filename);
3096 
3097  if (cstate->is_program)
3098  {
3099  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3100  if (cstate->copy_file == NULL)
3101  ereport(ERROR,
3103  errmsg("could not execute command \"%s\": %m",
3104  cstate->filename)));
3105  }
3106  else
3107  {
3108  struct stat st;
3109 
3110  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3111  if (cstate->copy_file == NULL)
3112  {
3113  /* copy errno because ereport subfunctions might change it */
3114  int save_errno = errno;
3115 
3116  ereport(ERROR,
3118  errmsg("could not open file \"%s\" for reading: %m",
3119  cstate->filename),
3120  (save_errno == ENOENT || save_errno == EACCES) ?
3121  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3122  "You may want a client-side facility such as psql's \\copy.") : 0));
3123  }
3124 
3125  if (fstat(fileno(cstate->copy_file), &st))
3126  ereport(ERROR,
3128  errmsg("could not stat file \"%s\": %m",
3129  cstate->filename)));
3130 
3131  if (S_ISDIR(st.st_mode))
3132  ereport(ERROR,
3133  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3134  errmsg("\"%s\" is a directory", cstate->filename)));
3135  }
3136  }
3137 
3138  if (!cstate->binary)
3139  {
3140  /* must rely on user to tell us... */
3141  cstate->file_has_oids = cstate->oids;
3142  }
3143  else
3144  {
3145  /* Read and verify binary header */
3146  char readSig[11];
3147  int32 tmp;
3148 
3149  /* Signature */
3150  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3151  memcmp(readSig, BinarySignature, 11) != 0)
3152  ereport(ERROR,
3153  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3154  errmsg("COPY file signature not recognized")));
3155  /* Flags field */
3156  if (!CopyGetInt32(cstate, &tmp))
3157  ereport(ERROR,
3158  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3159  errmsg("invalid COPY file header (missing flags)")));
3160  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3161  tmp &= ~(1 << 16);
3162  if ((tmp >> 16) != 0)
3163  ereport(ERROR,
3164  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3165  errmsg("unrecognized critical flags in COPY file header")));
3166  /* Header extension length */
3167  if (!CopyGetInt32(cstate, &tmp) ||
3168  tmp < 0)
3169  ereport(ERROR,
3170  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3171  errmsg("invalid COPY file header (missing length)")));
3172  /* Skip extension header, if present */
3173  while (tmp-- > 0)
3174  {
3175  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3176  ereport(ERROR,
3177  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3178  errmsg("invalid COPY file header (wrong length)")));
3179  }
3180  }
3181 
3182  if (cstate->file_has_oids && cstate->binary)
3183  {
3185  &in_func_oid, &cstate->oid_typioparam);
3186  fmgr_info(in_func_oid, &cstate->oid_in_function);
3187  }
3188 
3189  /* create workspace for CopyReadAttributes results */
3190  if (!cstate->binary)
3191  {
3192  AttrNumber attr_count = list_length(cstate->attnumlist);
3193  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3194 
3195  cstate->max_fields = nfields;
3196  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3197  }
3198 
3199  MemoryContextSwitchTo(oldcontext);
3200 
3201  return cstate;
3202 }
Definition: fmgr.h:56
List * range_table
Definition: copy.c:166
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1374
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:1007
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:684
int errhint(const char *fmt,...)
Definition: elog.c:987
char ** raw_fields
Definition: copy.c:190
bool binary
Definition: copy.c:115
#define RelationGetDescr(relation)
Definition: rel.h:428
AttrNumber num_defaults
Definition: copy.c:157
FmgrInfo * in_functions
Definition: copy.c:161
List * attnumlist
Definition: copy.c:111
#define OIDOID
Definition: pg_type.h:328
char * filename
Definition: copy.c:112
bool file_has_oids
Definition: copy.c:158
char * pstrdup(const char *in)
Definition: mcxt.c:1077
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:382
Expr * expression_planner(Expr *expr)
Definition: planner.c:5931
Form_pg_attribute * attrs
Definition: tupdesc.h:74
StringInfoData line_buf
Definition: copy.c:199
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Definition: nodes.h:509
int raw_buf_index
Definition: copy.c:212
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:102
unsigned int Oid
Definition: postgres_ext.h:31
bool volatile_defexprs
Definition: copy.c:165
#define PG_BINARY_R
Definition: c.h:1040
int natts
Definition: tupdesc.h:73
bool line_buf_converted
Definition: copy.c:200
signed int int32
Definition: c.h:256
CopyDest copy_dest
Definition: copy.c:98
const char * cur_attname
Definition: copy.c:140
Relation rel
Definition: copy.c:109
MemoryContext copycontext
Definition: copy.c:146
copy_data_source_cb data_source_cb
Definition: copy.c:114
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:558
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:127
bool list_member_int(const List *list, int datum)
Definition: list.c:485
char * raw_buf
Definition: copy.c:211
ExprState ** defexprs
Definition: copy.c:164
const char * cur_relname
Definition: copy.c:138
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
FmgrInfo oid_in_function
Definition: copy.c:159
#define RelationGetRelationName(relation)
Definition: rel.h:436
static const char BinarySignature[11]
Definition: copy.c:290
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
int raw_buf_len
Definition: copy.c:213
int max_fields
Definition: copy.c:189
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2184
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2665
#define ereport(elevel, rest)
Definition: elog.h:122
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2599
void initStringInfo(StringInfo str)
Definition: stringinfo.c:46
Oid * typioparams
Definition: copy.c:162
bool is_program
Definition: copy.c:113
Node * build_column_default(Relation rel, int attrno)
#define RAW_BUF_SIZE
Definition: copy.c:210
#define InvalidOid
Definition: postgres_ext.h:36
EolType eol_type
Definition: copy.c:103
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
static int list_length(const List *l)
Definition: pg_list.h:89
static char * filename
Definition: pg_dumpall.c:90
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
int cur_lineno
Definition: copy.c:139
StringInfoData attribute_buf
Definition: copy.c:185
ExprState * ExecInitExpr(Expr *node, PlanState *parent)
Definition: execExpr.c:113
const char * cur_attval
Definition: copy.c:141
CommandDest whereToSendOutput
Definition: postgres.c:88
Oid oid_typioparam
Definition: copy.c:160
int16 AttrNumber
Definition: attnum.h:21
int * defmap
Definition: copy.c:163
List * p_rtable
Definition: parse_node.h:172
bool oids
Definition: copy.c:116
static CopyState BeginCopyTo ( ParseState pstate,
Relation  rel,
RawStmt query,
Oid  queryRelId,
const char *  filename,
bool  is_program,
List attnamelist,
List options 
)
static

Definition at line 1788 of file copy.c.

References AllocateFile(), Assert, BeginCopy(), CopyStateData::copy_file, CopyStateData::copycontext, DestRemote, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, CopyStateData::filename, is_absolute_path, CopyStateData::is_program, MemoryContextSwitchTo(), NULL, OpenPipeStream(), PG_BINARY_W, pstrdup(), RelationData::rd_rel, RelationGetRelationName, RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, S_IWGRP, S_IWOTH, and whereToSendOutput.

Referenced by DoCopy().

1796 {
1797  CopyState cstate;
1798  bool pipe = (filename == NULL);
1799  MemoryContext oldcontext;
1800 
1801  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1802  {
1803  if (rel->rd_rel->relkind == RELKIND_VIEW)
1804  ereport(ERROR,
1805  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1806  errmsg("cannot copy from view \"%s\"",
1808  errhint("Try the COPY (SELECT ...) TO variant.")));
1809  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1810  ereport(ERROR,
1811  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1812  errmsg("cannot copy from materialized view \"%s\"",
1814  errhint("Try the COPY (SELECT ...) TO variant.")));
1815  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1816  ereport(ERROR,
1817  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1818  errmsg("cannot copy from foreign table \"%s\"",
1820  errhint("Try the COPY (SELECT ...) TO variant.")));
1821  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1822  ereport(ERROR,
1823  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1824  errmsg("cannot copy from sequence \"%s\"",
1825  RelationGetRelationName(rel))));
1826  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1827  ereport(ERROR,
1828  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1829  errmsg("cannot copy from partitioned table \"%s\"",
1831  errhint("Try the COPY (SELECT ...) TO variant.")));
1832  else
1833  ereport(ERROR,
1834  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1835  errmsg("cannot copy from non-table relation \"%s\"",
1836  RelationGetRelationName(rel))));
1837  }
1838 
1839  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1840  options);
1841  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1842 
1843  if (pipe)
1844  {
1845  Assert(!is_program); /* the grammar does not allow this */
1847  cstate->copy_file = stdout;
1848  }
1849  else
1850  {
1851  cstate->filename = pstrdup(filename);
1852  cstate->is_program = is_program;
1853 
1854  if (is_program)
1855  {
1856  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1857  if (cstate->copy_file == NULL)
1858  ereport(ERROR,
1860  errmsg("could not execute command \"%s\": %m",
1861  cstate->filename)));
1862  }
1863  else
1864  {
1865  mode_t oumask; /* Pre-existing umask value */
1866  struct stat st;
1867 
1868  /*
1869  * Prevent write to relative path ... too easy to shoot oneself in
1870  * the foot by overwriting a database file ...
1871  */
1872  if (!is_absolute_path(filename))
1873  ereport(ERROR,
1874  (errcode(ERRCODE_INVALID_NAME),
1875  errmsg("relative path not allowed for COPY to file")));
1876 
1877  oumask = umask(S_IWGRP | S_IWOTH);
1878  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1879  umask(oumask);
1880  if (cstate->copy_file == NULL)
1881  {
1882  /* copy errno because ereport subfunctions might change it */
1883  int save_errno = errno;
1884 
1885  ereport(ERROR,
1887  errmsg("could not open file \"%s\" for writing: %m",
1888  cstate->filename),
1889  (save_errno == ENOENT || save_errno == EACCES) ?
1890  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1891  "You may want a client-side facility such as psql's \\copy.") : 0));
1892  }
1893 
1894  if (fstat(fileno(cstate->copy_file), &st))
1895  ereport(ERROR,
1897  errmsg("could not stat file \"%s\": %m",
1898  cstate->filename)));
1899 
1900  if (S_ISDIR(st.st_mode))
1901  ereport(ERROR,
1902  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1903  errmsg("\"%s\" is a directory", cstate->filename)));
1904  }
1905  }
1906 
1907  MemoryContextSwitchTo(oldcontext);
1908 
1909  return cstate;
1910 }
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1374
int errhint(const char *fmt,...)
Definition: elog.c:987
char * filename
Definition: copy.c:112
char * pstrdup(const char *in)
Definition: mcxt.c:1077
#define RELKIND_MATVIEW
Definition: pg_class.h:165
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
int errcode(int sqlerrcode)
Definition: elog.c:575
#define PG_BINARY_W
Definition: c.h:1041
Form_pg_class rd_rel
Definition: rel.h:114
MemoryContext copycontext
Definition: copy.c:146
#define ERROR
Definition: elog.h:43
#define S_IWGRP
Definition: win32.h:449
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
#define RelationGetRelationName(relation)
Definition: rel.h:436
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2184
#define ereport(elevel, rest)
Definition: elog.h:122
#define S_IWOTH
Definition: win32.h:453
bool is_program
Definition: copy.c:113
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
static char * filename
Definition: pg_dumpall.c:90
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
FILE * copy_file
Definition: copy.c:99
CommandDest whereToSendOutput
Definition: postgres.c:88
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
static void ClosePipeToProgram ( CopyState  cstate)
static

Definition at line 1742 of file copy.c.

References Assert, ClosePipeStream(), CopyStateData::copy_file, ereport, errcode(), errcode_for_file_access(), errdetail_internal(), errmsg(), ERROR, CopyStateData::filename, CopyStateData::is_program, and wait_result_to_str().

Referenced by CopySendEndOfRow(), and EndCopy().

1743 {
1744  int pclose_rc;
1745 
1746  Assert(cstate->is_program);
1747 
1748  pclose_rc = ClosePipeStream(cstate->copy_file);
1749  if (pclose_rc == -1)
1750  ereport(ERROR,
1752  errmsg("could not close pipe to external command: %m")));
1753  else if (pclose_rc != 0)
1754  ereport(ERROR,
1755  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1756  errmsg("program \"%s\" failed",
1757  cstate->filename),
1758  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1759 }
char * filename
Definition: copy.c:112
int errcode(int sqlerrcode)
Definition: elog.c:575
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
int ClosePipeStream(FILE *file)
Definition: fd.c:2470
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
#define ERROR
Definition: elog.h:43
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:113
#define Assert(condition)
Definition: c.h:675
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
static void copy_dest_destroy ( DestReceiver self)
static

Definition at line 4810 of file copy.c.

References pfree().

Referenced by CreateCopyDestReceiver().

4811 {
4812  pfree(self);
4813 }
void pfree(void *pointer)
Definition: mcxt.c:950
static bool copy_dest_receive ( TupleTableSlot slot,
DestReceiver self 
)
static

Definition at line 4782 of file copy.c.

References CopyOneRowTo(), DR_copy::cstate, InvalidOid, DR_copy::processed, slot_getallattrs(), TupleTableSlot::tts_isnull, and TupleTableSlot::tts_values.

Referenced by CreateCopyDestReceiver().

4783 {
4784  DR_copy *myState = (DR_copy *) self;
4785  CopyState cstate = myState->cstate;
4786 
4787  /* Make sure the tuple is fully deconstructed */
4788  slot_getallattrs(slot);
4789 
4790  /* And send the data */
4791  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4792  myState->processed++;
4793 
4794  return true;
4795 }
Definition: copy.c:217
CopyState cstate
Definition: copy.c:220
Datum * tts_values
Definition: tuptable.h:125
bool * tts_isnull
Definition: tuptable.h:126
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1237
#define InvalidOid
Definition: postgres_ext.h:36
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2124
uint64 processed
Definition: copy.c:221
static void copy_dest_shutdown ( DestReceiver self)
static

Definition at line 4801 of file copy.c.

Referenced by CreateCopyDestReceiver().

4802 {
4803  /* no-op */
4804 }
static void copy_dest_startup ( DestReceiver self,
int  operation,
TupleDesc  typeinfo 
)
static

Definition at line 4773 of file copy.c.

Referenced by CreateCopyDestReceiver().

4774 {
4775  /* no-op */
4776 }
static void CopyAttributeOutCSV ( CopyState  cstate,
char *  string,
bool  use_quote,
bool  single_attr 
)
static

Definition at line 4610 of file copy.c.

References CopySendChar(), CopySendString(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::escape, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, CopyStateData::null_print, pg_encoding_mblen(), pg_server_to_any(), and CopyStateData::quote.

Referenced by CopyOneRowTo(), and CopyTo().

4612 {
4613  char *ptr;
4614  char *start;
4615  char c;
4616  char delimc = cstate->delim[0];
4617  char quotec = cstate->quote[0];
4618  char escapec = cstate->escape[0];
4619 
4620  /* force quoting if it matches null_print (before conversion!) */
4621  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4622  use_quote = true;
4623 
4624  if (cstate->need_transcoding)
4625  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4626  else
4627  ptr = string;
4628 
4629  /*
4630  * Make a preliminary pass to discover if it needs quoting
4631  */
4632  if (!use_quote)
4633  {
4634  /*
4635  * Because '\.' can be a data value, quote it if it appears alone on a
4636  * line so it is not interpreted as the end-of-data marker.
4637  */
4638  if (single_attr && strcmp(ptr, "\\.") == 0)
4639  use_quote = true;
4640  else
4641  {
4642  char *tptr = ptr;
4643 
4644  while ((c = *tptr) != '\0')
4645  {
4646  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4647  {
4648  use_quote = true;
4649  break;
4650  }
4651  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4652  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4653  else
4654  tptr++;
4655  }
4656  }
4657  }
4658 
4659  if (use_quote)
4660  {
4661  CopySendChar(cstate, quotec);
4662 
4663  /*
4664  * We adopt the same optimization strategy as in CopyAttributeOutText
4665  */
4666  start = ptr;
4667  while ((c = *ptr) != '\0')
4668  {
4669  if (c == quotec || c == escapec)
4670  {
4671  DUMPSOFAR();
4672  CopySendChar(cstate, escapec);
4673  start = ptr; /* we include char in next run */
4674  }
4675  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4676  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4677  else
4678  ptr++;
4679  }
4680  DUMPSOFAR();
4681 
4682  CopySendChar(cstate, quotec);
4683  }
4684  else
4685  {
4686  /* If it doesn't need quoting, we can just dump it as-is */
4687  CopySendString(cstate, ptr);
4688  }
4689 }
bool need_transcoding
Definition: copy.c:105
char * delim
Definition: copy.c:123
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:459
char * null_print
Definition: copy.c:120
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
char * quote
Definition: copy.c:124
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
char * escape
Definition: copy.c:125
int file_encoding
Definition: copy.c:104
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:453
#define DUMPSOFAR()
Definition: copy.c:4450
static void CopyAttributeOutText ( CopyState  cstate,
char *  string 
)
static

Definition at line 4457 of file copy.c.

References CopySendChar(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, pg_encoding_mblen(), and pg_server_to_any().

Referenced by CopyOneRowTo().

4458 {
4459  char *ptr;
4460  char *start;
4461  char c;
4462  char delimc = cstate->delim[0];
4463 
4464  if (cstate->need_transcoding)
4465  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4466  else
4467  ptr = string;
4468 
4469  /*
4470  * We have to grovel through the string searching for control characters
4471  * and instances of the delimiter character. In most cases, though, these
4472  * are infrequent. To avoid overhead from calling CopySendData once per
4473  * character, we dump out all characters between escaped characters in a
4474  * single call. The loop invariant is that the data from "start" to "ptr"
4475  * can be sent literally, but hasn't yet been.
4476  *
4477  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4478  * in valid backend encodings, extra bytes of a multibyte character never
4479  * look like ASCII. This loop is sufficiently performance-critical that
4480  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4481  * of the normal safe-encoding path.
4482  */
4483  if (cstate->encoding_embeds_ascii)
4484  {
4485  start = ptr;
4486  while ((c = *ptr) != '\0')
4487  {
4488  if ((unsigned char) c < (unsigned char) 0x20)
4489  {
4490  /*
4491  * \r and \n must be escaped, the others are traditional. We
4492  * prefer to dump these using the C-like notation, rather than
4493  * a backslash and the literal character, because it makes the
4494  * dump file a bit more proof against Microsoftish data
4495  * mangling.
4496  */
4497  switch (c)
4498  {
4499  case '\b':
4500  c = 'b';
4501  break;
4502  case '\f':
4503  c = 'f';
4504  break;
4505  case '\n':
4506  c = 'n';
4507  break;
4508  case '\r':
4509  c = 'r';
4510  break;
4511  case '\t':
4512  c = 't';
4513  break;
4514  case '\v':
4515  c = 'v';
4516  break;
4517  default:
4518  /* If it's the delimiter, must backslash it */
4519  if (c == delimc)
4520  break;
4521  /* All ASCII control chars are length 1 */
4522  ptr++;
4523  continue; /* fall to end of loop */
4524  }
4525  /* if we get here, we need to convert the control char */
4526  DUMPSOFAR();
4527  CopySendChar(cstate, '\\');
4528  CopySendChar(cstate, c);
4529  start = ++ptr; /* do not include char in next run */
4530  }
4531  else if (c == '\\' || c == delimc)
4532  {
4533  DUMPSOFAR();
4534  CopySendChar(cstate, '\\');
4535  start = ptr++; /* we include char in next run */
4536  }
4537  else if (IS_HIGHBIT_SET(c))
4538  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4539  else
4540  ptr++;
4541  }
4542  }
4543  else
4544  {
4545  start = ptr;
4546  while ((c = *ptr) != '\0')
4547  {
4548  if ((unsigned char) c < (unsigned char) 0x20)
4549  {
4550  /*
4551  * \r and \n must be escaped, the others are traditional. We
4552  * prefer to dump these using the C-like notation, rather than
4553  * a backslash and the literal character, because it makes the
4554  * dump file a bit more proof against Microsoftish data
4555  * mangling.
4556  */
4557  switch (c)
4558  {
4559  case '\b':
4560  c = 'b';
4561  break;
4562  case '\f':
4563  c = 'f';
4564  break;
4565  case '\n':
4566  c = 'n';
4567  break;
4568  case '\r':
4569  c = 'r';
4570  break;
4571  case '\t':
4572  c = 't';
4573  break;
4574  case '\v':
4575  c = 'v';
4576  break;
4577  default:
4578  /* If it's the delimiter, must backslash it */
4579  if (c == delimc)
4580  break;
4581  /* All ASCII control chars are length 1 */
4582  ptr++;
4583  continue; /* fall to end of loop */
4584  }
4585  /* if we get here, we need to convert the control char */
4586  DUMPSOFAR();
4587  CopySendChar(cstate, '\\');
4588  CopySendChar(cstate, c);
4589  start = ++ptr; /* do not include char in next run */
4590  }
4591  else if (c == '\\' || c == delimc)
4592  {
4593  DUMPSOFAR();
4594  CopySendChar(cstate, '\\');
4595  start = ptr++; /* we include char in next run */
4596  }
4597  else
4598  ptr++;
4599  }
4600  }
4601 
4602  DUMPSOFAR();
4603 }
bool need_transcoding
Definition: copy.c:105
char * delim
Definition: copy.c:123
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:459
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
int file_encoding
Definition: copy.c:104
#define DUMPSOFAR()
Definition: copy.c:4450
uint64 CopyFrom ( CopyState  cstate)

Definition at line 2323 of file copy.c.

References AfterTriggerBeginQuery(), AfterTriggerEndQuery(), ErrorContextCallback::arg, Assert, ErrorContextCallback::callback, CHECK_FOR_INTERRUPTS, tupleDesc::constr, CopyStateData::copy_dest, COPY_OLD_FE, CopyFromErrorCallback(), CopyFromInsertBatch(), CreateExecutorState(), CopyStateData::cur_lineno, CurrentMemoryContext, do_convert_tuple(), ereport, errcode(), errhint(), errmsg(), ERROR, error_context_stack, EState::es_num_result_relations, EState::es_range_table, EState::es_result_relation_info, EState::es_result_relations, EState::es_trig_tuple_slot, EState::es_tupleTable, ExecARInsertTriggers(), ExecASInsertTriggers(), ExecBRInsertTriggers(), ExecBSInsertTriggers(), ExecCleanUpTriggerState(), ExecCloseIndices(), ExecConstraints(), ExecDropSingleTupleTableSlot(), ExecFindPartition(), ExecInitExtraTupleSlot(), ExecInsertIndexTuples(), ExecIRInsertTriggers(), ExecMaterializeSlot(), ExecOpenIndices(), ExecResetTupleTable(), ExecSetSlotDescriptor(), ExecStoreTuple(), FreeBulkInsertState(), FreeExecutorState(), CopyStateData::freeze, GetBulkInsertState(), GetCurrentCommandId(), GetCurrentSubTransactionId(), GetPerTupleExprContext, GetPerTupleMemoryContext, heap_close, heap_form_tuple(), heap_insert(), HEAP_INSERT_FROZEN, HEAP_INSERT_SKIP_FSM, HEAP_INSERT_SKIP_WAL, heap_sync(), HeapTupleSetOid, i, InitResultRelInfo(), InvalidBuffer, InvalidOid, InvalidSubTransactionId, list_free(), makeNode, MAX_BUFFERED_TUPLES, MemoryContextSwitchTo(), tupleDesc::natts, NextCopyFrom(), NIL, NoLock, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, palloc(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pfree(), pq_endmsgread(), ErrorContextCallback::previous, CopyStateData::range_table, RelationData::rd_att, RelationData::rd_createSubid, RelationData::rd_newRelfilenodeSubid, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, PartitionDispatchData::reldesc, ReleaseBulkInsertStatePin(), RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, ResetPerTupleExprContext, ResultRelInfo::ri_FdwRoutine, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_PartitionCheck, ResultRelInfo::ri_RelationDesc, ResultRelInfo::ri_TrigDesc, HeapTupleData::t_len, HeapTupleData::t_self, HeapTupleData::t_tableOid, TransitionCaptureState::tcs_map, TransitionCaptureState::tcs_original_insert_tuple, ThereAreNoPriorRegisteredSnapshots(), ThereAreNoReadyPortals(), CopyStateData::transition_capture, CopyStateData::transition_tupconv_maps, TriggerDesc::trig_insert_before_row, TriggerDesc::trig_insert_instead_row, RelationData::trigdesc, PartitionDispatchData::tupslot, values, CopyStateData::volatile_defexprs, and XLogIsNeeded.

Referenced by copy_table(), and DoCopy().

2324 {
2325  HeapTuple tuple;
2326  TupleDesc tupDesc;
2327  Datum *values;
2328  bool *nulls;
2329  ResultRelInfo *resultRelInfo;
2330  ResultRelInfo *saved_resultRelInfo = NULL;
2331  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2332  ExprContext *econtext;
2333  TupleTableSlot *myslot;
2334  MemoryContext oldcontext = CurrentMemoryContext;
2335 
2336  ErrorContextCallback errcallback;
2337  CommandId mycid = GetCurrentCommandId(true);
2338  int hi_options = 0; /* start with default heap_insert options */
2339  BulkInsertState bistate;
2340  uint64 processed = 0;
2341  bool useHeapMultiInsert;
2342  int nBufferedTuples = 0;
2343  int prev_leaf_part_index = -1;
2344 
2345 #define MAX_BUFFERED_TUPLES 1000
2346  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2347  Size bufferedTuplesSize = 0;
2348  int firstBufferedLineNo = 0;
2349 
2350  Assert(cstate->rel);
2351 
2352  /*
2353  * The target must be a plain relation or have an INSTEAD OF INSERT row
2354  * trigger. (Currently, such triggers are only allowed on views, so we
2355  * only hint about them in the view case.)
2356  */
2357  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2358  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2359  !(cstate->rel->trigdesc &&
2361  {
2362  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2363  ereport(ERROR,
2364  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2365  errmsg("cannot copy to view \"%s\"",
2366  RelationGetRelationName(cstate->rel)),
2367  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2368  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2369  ereport(ERROR,
2370  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2371  errmsg("cannot copy to materialized view \"%s\"",
2372  RelationGetRelationName(cstate->rel))));
2373  else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
2374  ereport(ERROR,
2375  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2376  errmsg("cannot copy to foreign table \"%s\"",
2377  RelationGetRelationName(cstate->rel))));
2378  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2379  ereport(ERROR,
2380  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2381  errmsg("cannot copy to sequence \"%s\"",
2382  RelationGetRelationName(cstate->rel))));
2383  else
2384  ereport(ERROR,
2385  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2386  errmsg("cannot copy to non-table relation \"%s\"",
2387  RelationGetRelationName(cstate->rel))));
2388  }
2389 
2390  tupDesc = RelationGetDescr(cstate->rel);
2391 
2392  /*----------
2393  * Check to see if we can avoid writing WAL
2394  *
2395  * If archive logging/streaming is not enabled *and* either
2396  * - table was created in same transaction as this COPY
2397  * - data is being written to relfilenode created in this transaction
2398  * then we can skip writing WAL. It's safe because if the transaction
2399  * doesn't commit, we'll discard the table (or the new relfilenode file).
2400  * If it does commit, we'll have done the heap_sync at the bottom of this
2401  * routine first.
2402  *
2403  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2404  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2405  * can be cleared before the end of the transaction. The exact case is
2406  * when a relation sets a new relfilenode twice in same transaction, yet
2407  * the second one fails in an aborted subtransaction, e.g.
2408  *
2409  * BEGIN;
2410  * TRUNCATE t;
2411  * SAVEPOINT save;
2412  * TRUNCATE t;
2413  * ROLLBACK TO save;
2414  * COPY ...
2415  *
2416  * Also, if the target file is new-in-transaction, we assume that checking
2417  * FSM for free space is a waste of time, even if we must use WAL because
2418  * of archiving. This could possibly be wrong, but it's unlikely.
2419  *
2420  * The comments for heap_insert and RelationGetBufferForTuple specify that
2421  * skipping WAL logging is only safe if we ensure that our tuples do not
2422  * go into pages containing tuples from any other transactions --- but this
2423  * must be the case if we have a new table or new relfilenode, so we need
2424  * no additional work to enforce that.
2425  *----------
2426  */
2427  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2428  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2430  {
2431  hi_options |= HEAP_INSERT_SKIP_FSM;
2432  if (!XLogIsNeeded())
2433  hi_options |= HEAP_INSERT_SKIP_WAL;
2434  }
2435 
2436  /*
2437  * Optimize if new relfilenode was created in this subxact or one of its
2438  * committed children and we won't see those rows later as part of an
2439  * earlier scan or command. This ensures that if this subtransaction
2440  * aborts then the frozen rows won't be visible after xact cleanup. Note
2441  * that the stronger test of exactly which subtransaction created it is
2442  * crucial for correctness of this optimization.
2443  */
2444  if (cstate->freeze)
2445  {
2447  ereport(ERROR,
2448  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2449  errmsg("cannot perform FREEZE because of prior transaction activity")));
2450 
2451  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2453  ereport(ERROR,
2454  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2455  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2456 
2457  hi_options |= HEAP_INSERT_FROZEN;
2458  }
2459 
2460  /*
2461  * We need a ResultRelInfo so we can use the regular executor's
2462  * index-entry-making machinery. (There used to be a huge amount of code
2463  * here that basically duplicated execUtils.c ...)
2464  */
2465  resultRelInfo = makeNode(ResultRelInfo);
2466  InitResultRelInfo(resultRelInfo,
2467  cstate->rel,
2468  1, /* dummy rangetable index */
2469  NULL,
2470  0);
2471 
2472  ExecOpenIndices(resultRelInfo, false);
2473 
2474  estate->es_result_relations = resultRelInfo;
2475  estate->es_num_result_relations = 1;
2476  estate->es_result_relation_info = resultRelInfo;
2477  estate->es_range_table = cstate->range_table;
2478 
2479  /* Set up a tuple slot too */
2480  myslot = ExecInitExtraTupleSlot(estate);
2481  ExecSetSlotDescriptor(myslot, tupDesc);
2482  /* Triggers might need a slot as well */
2483  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);
2484 
2485  /*
2486  * It's more efficient to prepare a bunch of tuples for insertion, and
2487  * insert them in one heap_multi_insert() call, than call heap_insert()
2488  * separately for every tuple. However, we can't do that if there are
2489  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2490  * expressions. Such triggers or expressions might query the table we're
2491  * inserting to, and act differently if the tuples that have already been
2492  * processed and prepared for insertion are not there. We also can't do
2493  * it if the table is partitioned.
2494  */
2495  if ((resultRelInfo->ri_TrigDesc != NULL &&
2496  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2497  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2498  cstate->partition_dispatch_info != NULL ||
2499  cstate->volatile_defexprs)
2500  {
2501  useHeapMultiInsert = false;
2502  }
2503  else
2504  {
2505  useHeapMultiInsert = true;
2506  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2507  }
2508 
2509  /* Prepare to catch AFTER triggers. */
2511 
2512  /*
2513  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2514  * should do this for COPY, since it's not really an "INSERT" statement as
2515  * such. However, executing these triggers maintains consistency with the
2516  * EACH ROW triggers that we already fire on COPY.
2517  */
2518  ExecBSInsertTriggers(estate, resultRelInfo);
2519 
2520  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2521  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2522 
2523  bistate = GetBulkInsertState();
2524  econtext = GetPerTupleExprContext(estate);
2525 
2526  /* Set up callback to identify error line number */
2527  errcallback.callback = CopyFromErrorCallback;
2528  errcallback.arg = (void *) cstate;
2529  errcallback.previous = error_context_stack;
2530  error_context_stack = &errcallback;
2531 
2532  for (;;)
2533  {
2534  TupleTableSlot *slot;
2535  bool skip_tuple;
2536  Oid loaded_oid = InvalidOid;
2537 
2539 
2540  if (nBufferedTuples == 0)
2541  {
2542  /*
2543  * Reset the per-tuple exprcontext. We can only do this if the
2544  * tuple buffer is empty. (Calling the context the per-tuple
2545  * memory context is a bit of a misnomer now.)
2546  */
2547  ResetPerTupleExprContext(estate);
2548  }
2549 
2550  /* Switch into its memory context */
2552 
2553  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2554  break;
2555 
2556  /* And now we can form the input tuple. */
2557  tuple = heap_form_tuple(tupDesc, values, nulls);
2558 
2559  if (loaded_oid != InvalidOid)
2560  HeapTupleSetOid(tuple, loaded_oid);
2561 
2562  /*
2563  * Constraints might reference the tableoid column, so initialize
2564  * t_tableOid before evaluating them.
2565  */
2566  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2567 
2568  /* Triggers and stuff need to be invoked in query context. */
2569  MemoryContextSwitchTo(oldcontext);
2570 
2571  /* Place tuple in tuple slot --- but slot shouldn't free it */
2572  slot = myslot;
2573  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2574 
2575  /* Determine the partition to heap_insert the tuple into */
2576  if (cstate->partition_dispatch_info)
2577  {
2578  int leaf_part_index;
2579  TupleConversionMap *map;
2580 
2581  /*
2582  * Away we go ... If we end up not finding a partition after all,
2583  * ExecFindPartition() does not return and errors out instead.
2584  * Otherwise, the returned value is to be used as an index into
2585  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2586  * will get us the ResultRelInfo and TupleConversionMap for the
2587  * partition, respectively.
2588  */
2589  leaf_part_index = ExecFindPartition(resultRelInfo,
2590  cstate->partition_dispatch_info,
2591  slot,
2592  estate);
2593  Assert(leaf_part_index >= 0 &&
2594  leaf_part_index < cstate->num_partitions);
2595 
2596  /*
2597  * If this tuple is mapped to a partition that is not same as the
2598  * previous one, we'd better make the bulk insert mechanism gets a
2599  * new buffer.
2600  */
2601  if (prev_leaf_part_index != leaf_part_index)
2602  {
2603  ReleaseBulkInsertStatePin(bistate);
2604  prev_leaf_part_index = leaf_part_index;
2605  }
2606 
2607  /*
2608  * Save the old ResultRelInfo and switch to the one corresponding
2609  * to the selected partition.
2610  */
2611  saved_resultRelInfo = resultRelInfo;
2612  resultRelInfo = cstate->partitions + leaf_part_index;
2613 
2614  /* We do not yet have a way to insert into a foreign partition */
2615  if (resultRelInfo->ri_FdwRoutine)
2616  ereport(ERROR,
2617  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2618  errmsg("cannot route inserted tuples to a foreign table")));
2619 
2620  /*
2621  * For ExecInsertIndexTuples() to work on the partition's indexes
2622  */
2623  estate->es_result_relation_info = resultRelInfo;
2624 
2625  /*
2626  * If we're capturing transition tuples, we might need to convert
2627  * from the partition rowtype to parent rowtype.
2628  */
2629  if (cstate->transition_capture != NULL)
2630  {
2631  if (resultRelInfo->ri_TrigDesc &&
2632  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2633  resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
2634  {
2635  /*
2636  * If there are any BEFORE or INSTEAD triggers on the
2637  * partition, we'll have to be ready to convert their
2638  * result back to tuplestore format.
2639  */
2641  cstate->transition_capture->tcs_map =
2642  cstate->transition_tupconv_maps[leaf_part_index];
2643  }
2644  else
2645  {
2646  /*
2647  * Otherwise, just remember the original unconverted
2648  * tuple, to avoid a needless round trip conversion.
2649  */
2651  cstate->transition_capture->tcs_map = NULL;
2652  }
2653  }
2654 
2655  /*
2656  * We might need to convert from the parent rowtype to the
2657  * partition rowtype.
2658  */
2659  map = cstate->partition_tupconv_maps[leaf_part_index];
2660  if (map)
2661  {
2662  Relation partrel = resultRelInfo->ri_RelationDesc;
2663 
2664  tuple = do_convert_tuple(tuple, map);
2665 
2666  /*
2667  * We must use the partition's tuple descriptor from this
2668  * point on. Use a dedicated slot from this point on until
2669  * we're finished dealing with the partition.
2670  */
2671  slot = cstate->partition_tuple_slot;
2672  Assert(slot != NULL);
2673  ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
2674  ExecStoreTuple(tuple, slot, InvalidBuffer, true);
2675  }
2676 
2677  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2678  }
2679 
2680  skip_tuple = false;
2681 
2682  /* BEFORE ROW INSERT Triggers */
2683  if (resultRelInfo->ri_TrigDesc &&
2684  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2685  {
2686  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2687 
2688  if (slot == NULL) /* "do nothing" */
2689  skip_tuple = true;
2690  else /* trigger might have changed tuple */
2691  tuple = ExecMaterializeSlot(slot);
2692  }
2693 
2694  if (!skip_tuple)
2695  {
2696  if (resultRelInfo->ri_TrigDesc &&
2697  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2698  {
2699  /* Pass the data to the INSTEAD ROW INSERT trigger */
2700  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2701  }
2702  else
2703  {
2704  /*
2705  * We always check the partition constraint, including when
2706  * the tuple got here via tuple-routing. However we don't
2707  * need to in the latter case if no BR trigger is defined on
2708  * the partition. Note that a BR trigger might modify the
2709  * tuple such that the partition constraint is no longer
2710  * satisfied, so we need to check in that case.
2711  */
2712  bool check_partition_constr =
2713  (resultRelInfo->ri_PartitionCheck != NIL);
2714 
2715  if (saved_resultRelInfo != NULL &&
2716  !(resultRelInfo->ri_TrigDesc &&
2717  resultRelInfo->ri_TrigDesc->trig_insert_before_row))
2718  check_partition_constr = false;
2719 
2720  /* Check the constraints of the tuple */
2721  if (cstate->rel->rd_att->constr || check_partition_constr)
2722  ExecConstraints(resultRelInfo, slot, estate);
2723 
2724  if (useHeapMultiInsert)
2725  {
2726  /* Add this tuple to the tuple buffer */
2727  if (nBufferedTuples == 0)
2728  firstBufferedLineNo = cstate->cur_lineno;
2729  bufferedTuples[nBufferedTuples++] = tuple;
2730  bufferedTuplesSize += tuple->t_len;
2731 
2732  /*
2733  * If the buffer filled up, flush it. Also flush if the
2734  * total size of all the tuples in the buffer becomes
2735  * large, to avoid using large amounts of memory for the
2736  * buffer when the tuples are exceptionally wide.
2737  */
2738  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2739  bufferedTuplesSize > 65535)
2740  {
2741  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2742  resultRelInfo, myslot, bistate,
2743  nBufferedTuples, bufferedTuples,
2744  firstBufferedLineNo);
2745  nBufferedTuples = 0;
2746  bufferedTuplesSize = 0;
2747  }
2748  }
2749  else
2750  {
2751  List *recheckIndexes = NIL;
2752 
2753  /* OK, store the tuple and create index entries for it */
2754  heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
2755  hi_options, bistate);
2756 
2757  if (resultRelInfo->ri_NumIndices > 0)
2758  recheckIndexes = ExecInsertIndexTuples(slot,
2759  &(tuple->t_self),
2760  estate,
2761  false,
2762  NULL,
2763  NIL);
2764 
2765  /* AFTER ROW INSERT Triggers */
2766  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2767  recheckIndexes, cstate->transition_capture);
2768 
2769  list_free(recheckIndexes);
2770  }
2771  }
2772 
2773  /*
2774  * We count only tuples not suppressed by a BEFORE INSERT trigger;
2775  * this is the same definition used by execMain.c for counting
2776  * tuples inserted by an INSERT command.
2777  */
2778  processed++;
2779 
2780  if (saved_resultRelInfo)
2781  {
2782  resultRelInfo = saved_resultRelInfo;
2783  estate->es_result_relation_info = resultRelInfo;
2784  }
2785  }
2786  }
2787 
2788  /* Flush any remaining buffered tuples */
2789  if (nBufferedTuples > 0)
2790  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2791  resultRelInfo, myslot, bistate,
2792  nBufferedTuples, bufferedTuples,
2793  firstBufferedLineNo);
2794 
2795  /* Done, clean up */
2796  error_context_stack = errcallback.previous;
2797 
2798  FreeBulkInsertState(bistate);
2799 
2800  MemoryContextSwitchTo(oldcontext);
2801 
2802  /*
2803  * In the old protocol, tell pqcomm that we can process normal protocol
2804  * messages again.
2805  */
2806  if (cstate->copy_dest == COPY_OLD_FE)
2807  pq_endmsgread();
2808 
2809  /* Execute AFTER STATEMENT insertion triggers */
2810  ExecASInsertTriggers(estate, resultRelInfo, cstate->transition_capture);
2811 
2812  /* Handle queued AFTER triggers */
2813  AfterTriggerEndQuery(estate);
2814 
2815  pfree(values);
2816  pfree(nulls);
2817 
2818  ExecResetTupleTable(estate->es_tupleTable, false);
2819 
2820  ExecCloseIndices(resultRelInfo);
2821 
2822  /* Close all the partitioned tables, leaf partitions, and their indices */
2823  if (cstate->partition_dispatch_info)
2824  {
2825  int i;
2826 
2827  /*
2828  * Remember cstate->partition_dispatch_info[0] corresponds to the root
2829  * partitioned table, which we must not try to close, because it is
2830  * the main target table of COPY that will be closed eventually by
2831  * DoCopy(). Also, tupslot is NULL for the root partitioned table.
2832  */
2833  for (i = 1; i < cstate->num_dispatch; i++)
2834  {
2836 
2837  heap_close(pd->reldesc, NoLock);
2839  }
2840  for (i = 0; i < cstate->num_partitions; i++)
2841  {
2842  ResultRelInfo *resultRelInfo = cstate->partitions + i;
2843 
2844  ExecCloseIndices(resultRelInfo);
2845  heap_close(resultRelInfo->ri_RelationDesc, NoLock);
2846  }
2847 
2848  /* Release the standalone partition tuple descriptor */
2850  }
2851 
2852  /* Close any trigger target relations */
2853  ExecCleanUpTriggerState(estate);
2854 
2855  FreeExecutorState(estate);
2856 
2857  /*
2858  * If we skipped writing WAL, then we need to sync the heap (but not
2859  * indexes since those use WAL anyway)
2860  */
2861  if (hi_options & HEAP_INSERT_SKIP_WAL)
2862  heap_sync(cstate->rel);
2863 
2864  return processed;
2865 }
int ri_NumIndices
Definition: execnodes.h:357
#define NIL
Definition: pg_list.h:69
uint32 CommandId
Definition: c.h:411
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1299
Relation ri_RelationDesc
Definition: execnodes.h:354
List * range_table
Definition: copy.c:166
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate)
Definition: execTuples.c:852
int errhint(const char *fmt,...)
Definition: elog.c:987
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2218
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
#define ResetPerTupleExprContext(estate)
Definition: executor.h:483
#define RelationGetDescr(relation)
Definition: rel.h:428
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
ResultRelInfo * partitions
Definition: copy.c:171
#define XLogIsNeeded()
Definition: xlog.h:145
#define MAX_BUFFERED_TUPLES
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2428
#define RELKIND_MATVIEW
Definition: pg_class.h:165
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1914
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1649
int errcode(int sqlerrcode)
Definition: elog.c:575
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:111
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2413
void heap_sync(Relation rel)
Definition: heapam.c:9118
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:692
#define heap_close(r, l)
Definition: heapam.h:97
List * es_range_table
Definition: execnodes.h:431
Form_pg_class rd_rel
Definition: rel.h:114
unsigned int Oid
Definition: postgres_ext.h:31
bool volatile_defexprs
Definition: copy.c:165
struct ErrorContextCallback * previous
Definition: elog.h:238
int natts
Definition: tupdesc.h:73
HeapTuple tcs_original_insert_tuple
Definition: trigger.h:74
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
CopyDest copy_dest
Definition: copy.c:98
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:698
ErrorContextCallback * error_context_stack
Definition: elog.c:88
bool trig_insert_instead_row
Definition: reltrigger.h:57
void FreeExecutorState(EState *estate)
Definition: execUtils.c:178
Relation rel
Definition: copy.c:109
#define GetPerTupleExprContext(estate)
Definition: executor.h:474
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2319
void pfree(void *pointer)
Definition: mcxt.c:950
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1141
TupleTableSlot * partition_tuple_slot
Definition: copy.c:173
#define ERROR
Definition: elog.h:43
static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
Definition: copy.c:2873
TupleConversionMap * tcs_map
Definition: trigger.h:65
ItemPointerData t_self
Definition: htup.h:65
TriggerDesc * trigdesc
Definition: rel.h:120
int num_dispatch
Definition: copy.c:169
uint32 t_len
Definition: htup.h:64
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2284
#define NoLock
Definition: lockdefs.h:34
void ExecDropSingleTupleTableSlot(TupleTableSlot *slot)
Definition: execTuples.c:216
ResultRelInfo * es_result_relations
Definition: execnodes.h:441
#define RelationGetRelationName(relation)
Definition: rel.h:436
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:378
Oid t_tableOid
Definition: htup.h:66
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:457
#define ereport(elevel, rest)
Definition: elog.h:122
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2396
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:366
EState * CreateExecutorState(void)
Definition: execUtils.c:80
TupleConversionMap ** transition_tupconv_maps
Definition: copy.c:175
SubTransactionId rd_createSubid
Definition: rel.h:110
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
bool trig_insert_before_row
Definition: reltrigger.h:55
List * es_tupleTable
Definition: execnodes.h:470
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:156
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo, TransitionCaptureState *transition_capture)
Definition: trigger.c:2336
uintptr_t Datum
Definition: postgres.h:372
TupleTableSlot * tupslot
Definition: partition.h:66
void ExecSetSlotDescriptor(TupleTableSlot *slot, TupleDesc tupdesc)
Definition: execTuples.c:247
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:3328
int es_num_result_relations
Definition: execnodes.h:442
List * ri_PartitionCheck
Definition: execnodes.h:408
TupleDesc rd_att
Definition: rel.h:115
bool freeze
Definition: copy.c:117
void pq_endmsgread(void)
Definition: pqcomm.c:1234
#define InvalidOid
Definition: postgres_ext.h:36
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4343
#define makeNode(_type_)
Definition: nodes.h:557
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void FreeBulkInsertState(BulkInsertState bistate)
Definition: heapam.c:2333
SubTransactionId GetCurrentSubTransactionId(void)
Definition: xact.c:649
TupleConstr * constr
Definition: tupdesc.h:76
size_t Size
Definition: c.h:356
#define InvalidSubTransactionId
Definition: c.h:403
HeapTuple ExecMaterializeSlot(TupleTableSlot *slot)
Definition: execTuples.c:725
void ReleaseBulkInsertStatePin(BulkInsertState bistate)
Definition: heapam.c:2345
HeapTuple do_convert_tuple(HeapTuple tuple, TupleConversionMap *map)
Definition: tupconvert.c:343
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:479
void ExecCleanUpTriggerState(EState *estate)
Definition: execMain.c:1440
#define HEAP_INSERT_SKIP_FSM
Definition: heapam.h:29
static Datum values[MAXATTR]
Definition: bootstrap.c:163
void AfterTriggerEndQuery(EState *estate)
Definition: trigger.c:4363
void(* callback)(void *arg)
Definition: elog.h:239
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
void list_free(List *list)
Definition: list.c:1133
bool NextCopyFrom(CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
Definition: copy.c:3268
int i
int cur_lineno
Definition: copy.c:139
PartitionDispatch * partition_dispatch_info
Definition: copy.c:168
TransitionCaptureState * transition_capture
Definition: copy.c:174
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
TupleTableSlot * ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2347
CommandId GetCurrentCommandId(bool used)
Definition: xact.c:687
TupleConversionMap ** partition_tupconv_maps
Definition: copy.c:172
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
Definition: pg_list.h:45
#define RelationGetRelid(relation)
Definition: rel.h:416
void ExecCloseIndices(ResultRelInfo *resultRelInfo)
Definition: execIndexing.c:224
int num_partitions
Definition: copy.c:170
ResultRelInfo * es_result_relation_info
Definition: execnodes.h:443
void CopyFromErrorCallback ( void *  arg)

Definition at line 2218 of file copy.c.

References CopyStateData::binary, CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, StringInfoData::data, errcontext, limit_printout_length(), CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, and pfree().

Referenced by CopyFrom(), file_acquire_sample_rows(), and fileIterateForeignScan().

2219 {
2220  CopyState cstate = (CopyState) arg;
2221 
2222  if (cstate->binary)
2223  {
2224  /* can't usefully display the data */
2225  if (cstate->cur_attname)
2226  errcontext("COPY %s, line %d, column %s",
2227  cstate->cur_relname, cstate->cur_lineno,
2228  cstate->cur_attname);
2229  else
2230  errcontext("COPY %s, line %d",
2231  cstate->cur_relname, cstate->cur_lineno);
2232  }
2233  else
2234  {
2235  if (cstate->cur_attname && cstate->cur_attval)
2236  {
2237  /* error is relevant to a particular column */
2238  char *attval;
2239 
2240  attval = limit_printout_length(cstate->cur_attval);
2241  errcontext("COPY %s, line %d, column %s: \"%s\"",
2242  cstate->cur_relname, cstate->cur_lineno,
2243  cstate->cur_attname, attval);
2244  pfree(attval);
2245  }
2246  else if (cstate->cur_attname)
2247  {
2248  /* error is relevant to a particular column, value is NULL */
2249  errcontext("COPY %s, line %d, column %s: null input",
2250  cstate->cur_relname, cstate->cur_lineno,
2251  cstate->cur_attname);
2252  }
2253  else
2254  {
2255  /*
2256  * Error is relevant to a particular line.
2257  *
2258  * If line_buf still contains the correct line, and it's already
2259  * transcoded, print it. If it's still in a foreign encoding, it's
2260  * quite likely that the error is precisely a failure to do
2261  * encoding conversion (ie, bad data). We dare not try to convert
2262  * it, and at present there's no way to regurgitate it without
2263  * conversion. So we have to punt and just report the line number.
2264  */
2265  if (cstate->line_buf_valid &&
2266  (cstate->line_buf_converted || !cstate->need_transcoding))
2267  {
2268  char *lineval;
2269 
2270  lineval = limit_printout_length(cstate->line_buf.data);
2271  errcontext("COPY %s, line %d: \"%s\"",
2272  cstate->cur_relname, cstate->cur_lineno, lineval);
2273  pfree(lineval);
2274  }
2275  else
2276  {
2277  errcontext("COPY %s, line %d",
2278  cstate->cur_relname, cstate->cur_lineno);
2279  }
2280  }
2281  }
2282 }
bool binary
Definition: copy.c:115
bool need_transcoding
Definition: copy.c:105
StringInfoData line_buf
Definition: copy.c:199
bool line_buf_valid
Definition: copy.c:201
bool line_buf_converted
Definition: copy.c:200
const char * cur_attname
Definition: copy.c:140
void pfree(void *pointer)
Definition: mcxt.c:950
static char * limit_printout_length(const char *str)
Definition: copy.c:2294
const char * cur_relname
Definition: copy.c:138
#define errcontext
Definition: elog.h:164
struct CopyStateData * CopyState
Definition: copy.h:23
void * arg
int cur_lineno
Definition: copy.c:139
const char * cur_attval
Definition: copy.c:141
static void CopyFromInsertBatch ( CopyState  cstate,
EState estate,
CommandId  mycid,
int  hi_options,
ResultRelInfo resultRelInfo,
TupleTableSlot myslot,
BulkInsertState  bistate,
int  nBufferedTuples,
HeapTuple bufferedTuples,
int  firstBufferedLineNo 
)
static

Definition at line 2873 of file copy.c.

References CopyStateData::cur_lineno, ExecARInsertTriggers(), ExecInsertIndexTuples(), ExecStoreTuple(), GetPerTupleMemoryContext, heap_multi_insert(), i, InvalidBuffer, CopyStateData::line_buf_valid, list_free(), MemoryContextSwitchTo(), NIL, NULL, CopyStateData::rel, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_TrigDesc, CopyStateData::transition_capture, TriggerDesc::trig_insert_after_row, and TriggerDesc::trig_insert_new_table.

Referenced by CopyFrom().

2878 {
2879  MemoryContext oldcontext;
2880  int i;
2881  int save_cur_lineno;
2882 
2883  /*
2884  * Print error context information correctly, if one of the operations
2885  * below fail.
2886  */
2887  cstate->line_buf_valid = false;
2888  save_cur_lineno = cstate->cur_lineno;
2889 
2890  /*
2891  * heap_multi_insert leaks memory, so switch to short-lived memory context
2892  * before calling it.
2893  */
2894  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2895  heap_multi_insert(cstate->rel,
2896  bufferedTuples,
2897  nBufferedTuples,
2898  mycid,
2899  hi_options,
2900  bistate);
2901  MemoryContextSwitchTo(oldcontext);
2902 
2903  /*
2904  * If there are any indexes, update them for all the inserted tuples, and
2905  * run AFTER ROW INSERT triggers.
2906  */
2907  if (resultRelInfo->ri_NumIndices > 0)
2908  {
2909  for (i = 0; i < nBufferedTuples; i++)
2910  {
2911  List *recheckIndexes;
2912 
2913  cstate->cur_lineno = firstBufferedLineNo + i;
2914  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2915  recheckIndexes =
2916  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2917  estate, false, NULL, NIL);
2918  ExecARInsertTriggers(estate, resultRelInfo,
2919  bufferedTuples[i],
2920  recheckIndexes, cstate->transition_capture);
2921  list_free(recheckIndexes);
2922  }
2923  }
2924 
2925  /*
2926  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2927  * anyway.
2928  */
2929  else if (resultRelInfo->ri_TrigDesc != NULL &&
2930  (resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
2931  resultRelInfo->ri_TrigDesc->trig_insert_new_table))
2932  {
2933  for (i = 0; i < nBufferedTuples; i++)
2934  {
2935  cstate->cur_lineno = firstBufferedLineNo + i;
2936  ExecARInsertTriggers(estate, resultRelInfo,
2937  bufferedTuples[i],
2938  NIL, cstate->transition_capture);
2939  }
2940  }
2941 
2942  /* reset cur_lineno to where we were */
2943  cstate->cur_lineno = save_cur_lineno;
2944 }
int ri_NumIndices
Definition: execnodes.h:357
#define NIL
Definition: pg_list.h:69
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
bool line_buf_valid
Definition: copy.c:201
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes, TransitionCaptureState *transition_capture)
Definition: trigger.c:2413
Relation rel
Definition: copy.c:109
bool trig_insert_new_table
Definition: reltrigger.h:74
bool trig_insert_after_row
Definition: reltrigger.h:56
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:366
#define NULL
Definition: c.h:229
void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2658
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:479
void list_free(List *list)
Definition: list.c:1133
int i
int cur_lineno
Definition: copy.c:139
TransitionCaptureState * transition_capture
Definition: copy.c:174
Definition: pg_list.h:45
static List * CopyGetAttnums ( TupleDesc  tupDesc,
Relation  rel,
List attnamelist 
)
static

Definition at line 4701 of file copy.c.

References tupleDesc::attrs, ereport, errcode(), errmsg(), ERROR, i, InvalidAttrNumber, lappend_int(), lfirst, list_member_int(), name, namestrcmp(), tupleDesc::natts, NIL, NULL, RelationGetRelationName, and strVal.

Referenced by BeginCopy(), and DoCopy().

4702 {
4703  List *attnums = NIL;
4704 
4705  if (attnamelist == NIL)
4706  {
4707  /* Generate default column list */
4708  Form_pg_attribute *attr = tupDesc->attrs;
4709  int attr_count = tupDesc->natts;
4710  int i;
4711 
4712  for (i = 0; i < attr_count; i++)
4713  {
4714  if (attr[i]->attisdropped)
4715  continue;
4716  attnums = lappend_int(attnums, i + 1);
4717  }
4718  }
4719  else
4720  {
4721  /* Validate the user-supplied list and extract attnums */
4722  ListCell *l;
4723 
4724  foreach(l, attnamelist)
4725  {
4726  char *name = strVal(lfirst(l));
4727  int attnum;
4728  int i;
4729 
4730  /* Lookup column name */
4731  attnum = InvalidAttrNumber;
4732  for (i = 0; i < tupDesc->natts; i++)
4733  {
4734  if (tupDesc->attrs[i]->attisdropped)
4735  continue;
4736  if (namestrcmp(&(tupDesc->attrs[i]->attname), name) == 0)
4737  {
4738  attnum = tupDesc->attrs[i]->attnum;
4739  break;
4740  }
4741  }
4742  if (attnum == InvalidAttrNumber)
4743  {
4744  if (rel != NULL)
4745  ereport(ERROR,
4746  (errcode(ERRCODE_UNDEFINED_COLUMN),
4747  errmsg("column \"%s\" of relation \"%s\" does not exist",
4748  name, RelationGetRelationName(rel))));
4749  else
4750  ereport(ERROR,
4751  (errcode(ERRCODE_UNDEFINED_COLUMN),
4752  errmsg("column \"%s\" does not exist",
4753  name)));
4754  }
4755  /* Check for duplicates */
4756  if (list_member_int(attnums, attnum))
4757  ereport(ERROR,
4758  (errcode(ERRCODE_DUPLICATE_COLUMN),
4759  errmsg("column \"%s\" specified more than once",
4760  name)));
4761  attnums = lappend_int(attnums, attnum);
4762  }
4763  }
4764 
4765  return attnums;
4766 }
#define NIL
Definition: pg_list.h:69
Form_pg_attribute * attrs
Definition: tupdesc.h:74
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
int namestrcmp(Name name, const char *str)
Definition: name.c:247
int natts
Definition: tupdesc.h:73
#define ERROR
Definition: elog.h:43
bool list_member_int(const List *list, int datum)
Definition: list.c:485
#define RelationGetRelationName(relation)
Definition: rel.h:436
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
#define ereport(elevel, rest)
Definition: elog.h:122
List * lappend_int(List *list, int datum)
Definition: list.c:146
#define NULL
Definition: c.h:229
#define lfirst(lc)
Definition: pg_list.h:106
const char * name
Definition: encode.c:521
#define InvalidAttrNumber
Definition: attnum.h:23
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
Definition: pg_list.h:45
static int CopyGetData ( CopyState  cstate,
void *  databuf,
int  minread,
int  maxread 
)
static

Definition at line 558 of file copy.c.

References COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, StringInfoData::cursor, CopyStateData::data_source_cb, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, CopyStateData::fe_eof, CopyStateData::fe_msgbuf, HOLD_CANCEL_INTERRUPTS, StringInfoData::len, pq_copymsgbytes(), pq_getbyte(), pq_getbytes(), pq_getmessage(), pq_getmsgstring(), pq_startmsgread(), and RESUME_CANCEL_INTERRUPTS.

Referenced by BeginCopyFrom(), CopyGetInt16(), CopyGetInt32(), CopyLoadRawBuf(), CopyReadBinaryAttribute(), and NextCopyFrom().

559 {
560  int bytesread = 0;
561 
562  switch (cstate->copy_dest)
563  {
564  case COPY_FILE:
565  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
566  if (ferror(cstate->copy_file))
567  ereport(ERROR,
569  errmsg("could not read from COPY file: %m")));
570  break;
571  case COPY_OLD_FE:
572 
573  /*
574  * We cannot read more than minread bytes (which in practice is 1)
575  * because old protocol doesn't have any clear way of separating
576  * the COPY stream from following data. This is slow, but not any
577  * slower than the code path was originally, and we don't care
578  * much anymore about the performance of old protocol.
579  */
580  if (pq_getbytes((char *) databuf, minread))
581  {
582  /* Only a \. terminator is legal EOF in old protocol */
583  ereport(ERROR,
584  (errcode(ERRCODE_CONNECTION_FAILURE),
585  errmsg("unexpected EOF on client connection with an open transaction")));
586  }
587  bytesread = minread;
588  break;
589  case COPY_NEW_FE:
590  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
591  {
592  int avail;
593 
594  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
595  {
596  /* Try to receive another message */
597  int mtype;
598 
599  readmessage:
601  pq_startmsgread();
602  mtype = pq_getbyte();
603  if (mtype == EOF)
604  ereport(ERROR,
605  (errcode(ERRCODE_CONNECTION_FAILURE),
606  errmsg("unexpected EOF on client connection with an open transaction")));
607  if (pq_getmessage(cstate->fe_msgbuf, 0))
608  ereport(ERROR,
609  (errcode(ERRCODE_CONNECTION_FAILURE),
610  errmsg("unexpected EOF on client connection with an open transaction")));
612  switch (mtype)
613  {
614  case 'd': /* CopyData */
615  break;
616  case 'c': /* CopyDone */
617  /* COPY IN correctly terminated by frontend */
618  cstate->fe_eof = true;
619  return bytesread;
620  case 'f': /* CopyFail */
621  ereport(ERROR,
622  (errcode(ERRCODE_QUERY_CANCELED),
623  errmsg("COPY from stdin failed: %s",
624  pq_getmsgstring(cstate->fe_msgbuf))));
625  break;
626  case 'H': /* Flush */
627  case 'S': /* Sync */
628 
629  /*
630  * Ignore Flush/Sync for the convenience of client
631  * libraries (such as libpq) that may send those
632  * without noticing that the command they just
633  * sent was COPY.
634  */
635  goto readmessage;
636  default:
637  ereport(ERROR,
638  (errcode(ERRCODE_PROTOCOL_VIOLATION),
639  errmsg("unexpected message type 0x%02X during COPY from stdin",
640  mtype)));
641  break;
642  }
643  }
644  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
645  if (avail > maxread)
646  avail = maxread;
647  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
648  databuf = (void *) ((char *) databuf + avail);
649  maxread -= avail;
650  bytesread += avail;
651  }
652  break;
653  case COPY_CALLBACK:
654  bytesread = cstate->data_source_cb(databuf, minread, maxread);
655  break;
656  }
657 
658  return bytesread;
659 }
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:123
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:621
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:102
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
copy_data_source_cb data_source_cb
Definition: copy.c:114
#define ERROR
Definition: elog.h:43
void pq_startmsgread(void)
Definition: pqcomm.c:1210
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1094
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1272
int pq_getbyte(void)
Definition: pqcomm.c:1000
StringInfo fe_msgbuf
Definition: copy.c:100
void pq_copymsgbytes(StringInfo msg, char *buf, int datalen)
Definition: pqformat.c:570
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
#define RESUME_CANCEL_INTERRUPTS()
Definition: miscadmin.h:125
static bool CopyGetInt16 ( CopyState  cstate,
int16 val 
)
static

Definition at line 713 of file copy.c.

References buf, and CopyGetData().

Referenced by NextCopyFrom().

714 {
715  uint16 buf;
716 
717  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
718  {
719  *val = 0; /* suppress compiler warning */
720  return false;
721  }
722  *val = (int16) ntohs(buf);
723  return true;
724 }
signed short int16
Definition: c.h:255
unsigned short uint16
Definition: c.h:267
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:558
static char * buf
Definition: pg_test_fsync.c:66
long val
Definition: informix.c:689
static bool CopyGetInt32 ( CopyState  cstate,
int32 val 
)
static

Definition at line 684 of file copy.c.

References buf, and CopyGetData().

Referenced by BeginCopyFrom(), and CopyReadBinaryAttribute().

685 {
686  uint32 buf;
687 
688  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
689  {
690  *val = 0; /* suppress compiler warning */
691  return false;
692  }
693  *val = (int32) ntohl(buf);
694  return true;
695 }
signed int int32
Definition: c.h:256
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:558
static char * buf
Definition: pg_test_fsync.c:66
unsigned int uint32
Definition: c.h:268
long val
Definition: informix.c:689
static bool CopyLoadRawBuf ( CopyState  cstate)
static

Definition at line 738 of file copy.c.

References CopyGetData(), memmove, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and RAW_BUF_SIZE.

Referenced by CopyReadLine(), and CopyReadLineText().

739 {
740  int nbytes;
741  int inbytes;
742 
743  if (cstate->raw_buf_index < cstate->raw_buf_len)
744  {
745  /* Copy down the unprocessed data */
746  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
747  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
748  nbytes);
749  }
750  else
751  nbytes = 0; /* no data need be saved */
752 
753  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
754  1, RAW_BUF_SIZE - nbytes);
755  nbytes += inbytes;
756  cstate->raw_buf[nbytes] = '\0';
757  cstate->raw_buf_index = 0;
758  cstate->raw_buf_len = nbytes;
759  return (inbytes > 0);
760 }
int raw_buf_index
Definition: copy.c:212
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:558
char * raw_buf
Definition: copy.c:211
#define memmove(d, s, c)
Definition: c.h:1058
int raw_buf_len
Definition: copy.c:213
#define RAW_BUF_SIZE
Definition: copy.c:210
static void CopyOneRowTo ( CopyState  cstate,
Oid  tupleOid,
Datum values,
bool nulls 
)
static

Definition at line 2124 of file copy.c.

References CopyStateData::attnumlist, CopyStateData::binary, CopyAttributeOutCSV(), CopyAttributeOutText(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), CopySendString(), CopyStateData::csv_mode, cur, DatumGetCString, CopyStateData::delim, DirectFunctionCall1, CopyStateData::force_quote_flags, lfirst_int, list_length(), MemoryContextReset(), MemoryContextSwitchTo(), CopyStateData::null_print_client, ObjectIdGetDatum, oidout(), CopyStateData::oids, CopyStateData::out_functions, OutputFunctionCall(), CopyStateData::rowcontext, SendFunctionCall(), value, VARDATA, VARHDRSZ, and VARSIZE.

Referenced by copy_dest_receive(), and CopyTo().

2125 {
2126  bool need_delim = false;
2127  FmgrInfo *out_functions = cstate->out_functions;
2128  MemoryContext oldcontext;
2129  ListCell *cur;
2130  char *string;
2131 
2132  MemoryContextReset(cstate->rowcontext);
2133  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2134 
2135  if (cstate->binary)
2136  {
2137  /* Binary per-tuple header */
2138  CopySendInt16(cstate, list_length(cstate->attnumlist));
2139  /* Send OID if wanted --- note attnumlist doesn't include it */
2140  if (cstate->oids)
2141  {
2142  /* Hack --- assume Oid is same size as int32 */
2143  CopySendInt32(cstate, sizeof(int32));
2144  CopySendInt32(cstate, tupleOid);
2145  }
2146  }
2147  else
2148  {
2149  /* Text format has no per-tuple header, but send OID if wanted */
2150  /* Assume digits don't need any quoting or encoding conversion */
2151  if (cstate->oids)
2152  {
2154  ObjectIdGetDatum(tupleOid)));
2155  CopySendString(cstate, string);
2156  need_delim = true;
2157  }
2158  }
2159 
2160  foreach(cur, cstate->attnumlist)
2161  {
2162  int attnum = lfirst_int(cur);
2163  Datum value = values[attnum - 1];
2164  bool isnull = nulls[attnum - 1];
2165 
2166  if (!cstate->binary)
2167  {
2168  if (need_delim)
2169  CopySendChar(cstate, cstate->delim[0]);
2170  need_delim = true;
2171  }
2172 
2173  if (isnull)
2174  {
2175  if (!cstate->binary)
2176  CopySendString(cstate, cstate->null_print_client);
2177  else
2178  CopySendInt32(cstate, -1);
2179  }
2180  else
2181  {
2182  if (!cstate->binary)
2183  {
2184  string = OutputFunctionCall(&out_functions[attnum - 1],
2185  value);
2186  if (cstate->csv_mode)
2187  CopyAttributeOutCSV(cstate, string,
2188  cstate->force_quote_flags[attnum - 1],
2189  list_length(cstate->attnumlist) == 1);
2190  else
2191  CopyAttributeOutText(cstate, string);
2192  }
2193  else
2194  {
2195  bytea *outputbytes;
2196 
2197  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2198  value);
2199  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2200  CopySendData(cstate, VARDATA(outputbytes),
2201  VARSIZE(outputbytes) - VARHDRSZ);
2202  }
2203  }
2204  }
2205 
2206  CopySendEndOfRow(cstate);
2207 
2208  MemoryContextSwitchTo(oldcontext);
2209 }
Definition: fmgr.h:56
bool csv_mode
Definition: copy.c:118
#define VARDATA(PTR)
Definition: postgres.h:303
bool binary
Definition: copy.c:115
List * attnumlist
Definition: copy.c:111
#define VARSIZE(PTR)
Definition: postgres.h:304
#define VARHDRSZ
Definition: c.h:445
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
struct cursor * cur
Definition: ecpg.c:28
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4457
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:135
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:584
bool * force_quote_flags
Definition: copy.c:128
char * delim
Definition: copy.c:123
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
MemoryContext rowcontext
Definition: copy.c:152
signed int int32
Definition: c.h:256
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:459
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1667
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:701
#define ObjectIdGetDatum(X)
Definition: postgres.h:513
#define DatumGetCString(X)
Definition: postgres.h:572
#define lfirst_int(lc)
Definition: pg_list.h:107
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4610
static struct @121 value
char string[11]
Definition: preproc-type.c:46
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1728
uintptr_t Datum
Definition: postgres.h:372
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:447
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:453
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:670
static Datum values[MAXATTR]
Definition: bootstrap.c:163
FmgrInfo * out_functions
Definition: copy.c:151
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:465
Definition: c.h:439
char * null_print_client
Definition: copy.c:122
bool oids
Definition: copy.c:116
static int CopyReadAttributesCSV ( CopyState  cstate)
static

Definition at line 4229 of file copy.c.

References Assert, CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, CopyStateData::escape, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, CopyStateData::quote, CopyStateData::raw_fields, repalloc(), and resetStringInfo().

Referenced by NextCopyFromRawFields().

4230 {
4231  char delimc = cstate->delim[0];
4232  char quotec = cstate->quote[0];
4233  char escapec = cstate->escape[0];
4234  int fieldno;
4235  char *output_ptr;
4236  char *cur_ptr;
4237  char *line_end_ptr;
4238 
4239  /*
4240  * We need a special case for zero-column tables: check that the input
4241  * line is empty, and return.
4242  */
4243  if (cstate->max_fields <= 0)
4244  {
4245  if (cstate->line_buf.len != 0)
4246  ereport(ERROR,
4247  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4248  errmsg("extra data after last expected column")));
4249  return 0;
4250  }
4251 
4252  resetStringInfo(&cstate->attribute_buf);
4253 
4254  /*
4255  * The de-escaped attributes will certainly not be longer than the input
4256  * data line, so we can just force attribute_buf to be large enough and
4257  * then transfer data without any checks for enough space. We need to do
4258  * it this way because enlarging attribute_buf mid-stream would invalidate
4259  * pointers already stored into cstate->raw_fields[].
4260  */
4261  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4262  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4263  output_ptr = cstate->attribute_buf.data;
4264 
4265  /* set pointer variables for loop */
4266  cur_ptr = cstate->line_buf.data;
4267  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4268 
4269  /* Outer loop iterates over fields */
4270  fieldno = 0;
4271  for (;;)
4272  {
4273  bool found_delim = false;
4274  bool saw_quote = false;
4275  char *start_ptr;
4276  char *end_ptr;
4277  int input_len;
4278 
4279  /* Make sure there is enough space for the next value */
4280  if (fieldno >= cstate->max_fields)
4281  {
4282  cstate->max_fields *= 2;
4283  cstate->raw_fields =
4284  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4285  }
4286 
4287  /* Remember start of field on both input and output sides */
4288  start_ptr = cur_ptr;
4289  cstate->raw_fields[fieldno] = output_ptr;
4290 
4291  /*
4292  * Scan data for field,
4293  *
4294  * The loop starts in "not quote" mode and then toggles between that
4295  * and "in quote" mode. The loop exits normally if it is in "not
4296  * quote" mode and a delimiter or line end is seen.
4297  */
4298  for (;;)
4299  {
4300  char c;
4301 
4302  /* Not in quote */
4303  for (;;)
4304  {
4305  end_ptr = cur_ptr;
4306  if (cur_ptr >= line_end_ptr)
4307  goto endfield;
4308  c = *cur_ptr++;
4309  /* unquoted field delimiter */
4310  if (c == delimc)
4311  {
4312  found_delim = true;
4313  goto endfield;
4314  }
4315  /* start of quoted field (or part of field) */
4316  if (c == quotec)
4317  {
4318  saw_quote = true;
4319  break;
4320  }
4321  /* Add c to output string */
4322  *output_ptr++ = c;
4323  }
4324 
4325  /* In quote */
4326  for (;;)
4327  {
4328  end_ptr = cur_ptr;
4329  if (cur_ptr >= line_end_ptr)
4330  ereport(ERROR,
4331  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4332  errmsg("unterminated CSV quoted field")));
4333 
4334  c = *cur_ptr++;
4335 
4336  /* escape within a quoted field */
4337  if (c == escapec)
4338  {
4339  /*
4340  * peek at the next char if available, and escape it if it
4341  * is an escape char or a quote char
4342  */
4343  if (cur_ptr < line_end_ptr)
4344  {
4345  char nextc = *cur_ptr;
4346 
4347  if (nextc == escapec || nextc == quotec)
4348  {
4349  *output_ptr++ = nextc;
4350  cur_ptr++;
4351  continue;
4352  }
4353  }
4354  }
4355 
4356  /*
4357  * end of quoted field. Must do this test after testing for
4358  * escape in case quote char and escape char are the same
4359  * (which is the common case).
4360  */
4361  if (c == quotec)
4362  break;
4363 
4364  /* Add c to output string */
4365  *output_ptr++ = c;
4366  }
4367  }
4368 endfield:
4369 
4370  /* Terminate attribute value in output area */
4371  *output_ptr++ = '\0';
4372 
4373  /* Check whether raw input matched null marker */
4374  input_len = end_ptr - start_ptr;
4375  if (!saw_quote && input_len == cstate->null_print_len &&
4376  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4377  cstate->raw_fields[fieldno] = NULL;
4378 
4379  fieldno++;
4380  /* Done if we hit EOL instead of a delim */
4381  if (!found_delim)
4382  break;
4383  }
4384 
4385  /* Clean up state of attribute_buf */
4386  output_ptr--;
4387  Assert(*output_ptr == '\0');
4388  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4389 
4390  return fieldno;
4391 }
char ** raw_fields
Definition: copy.c:190
StringInfoData line_buf
Definition: copy.c:199
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:123
char * null_print
Definition: copy.c:120
#define ERROR
Definition: elog.h:43
char * c
char * quote
Definition: copy.c:124
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:189
char * escape
Definition: copy.c:125
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:121
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:185
static int CopyReadAttributesText ( CopyState  cstate)
static

Definition at line 4001 of file copy.c.

References CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, GetDecimalFromHex(), IS_HIGHBIT_SET, ISOCTAL, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, OCTVALUE, pg_verifymbstr(), CopyStateData::raw_fields, repalloc(), resetStringInfo(), and val.

Referenced by NextCopyFromRawFields().

4002 {
4003  char delimc = cstate->delim[0];
4004  int fieldno;
4005  char *output_ptr;
4006  char *cur_ptr;
4007  char *line_end_ptr;
4008 
4009  /*
4010  * We need a special case for zero-column tables: check that the input
4011  * line is empty, and return.
4012  */
4013  if (cstate->max_fields <= 0)
4014  {
4015  if (cstate->line_buf.len != 0)
4016  ereport(ERROR,
4017  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4018  errmsg("extra data after last expected column")));
4019  return 0;
4020  }
4021 
4022  resetStringInfo(&cstate->attribute_buf);
4023 
4024  /*
4025  * The de-escaped attributes will certainly not be longer than the input
4026  * data line, so we can just force attribute_buf to be large enough and
4027  * then transfer data without any checks for enough space. We need to do
4028  * it this way because enlarging attribute_buf mid-stream would invalidate
4029  * pointers already stored into cstate->raw_fields[].
4030  */
4031  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4032  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4033  output_ptr = cstate->attribute_buf.data;
4034 
4035  /* set pointer variables for loop */
4036  cur_ptr = cstate->line_buf.data;
4037  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4038 
4039  /* Outer loop iterates over fields */
4040  fieldno = 0;
4041  for (;;)
4042  {
4043  bool found_delim = false;
4044  char *start_ptr;
4045  char *end_ptr;
4046  int input_len;
4047  bool saw_non_ascii = false;
4048 
4049  /* Make sure there is enough space for the next value */
4050  if (fieldno >= cstate->max_fields)
4051  {
4052  cstate->max_fields *= 2;
4053  cstate->raw_fields =
4054  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4055  }
4056 
4057  /* Remember start of field on both input and output sides */
4058  start_ptr = cur_ptr;
4059  cstate->raw_fields[fieldno] = output_ptr;
4060 
4061  /*
4062  * Scan data for field.
4063  *
4064  * Note that in this loop, we are scanning to locate the end of field
4065  * and also speculatively performing de-escaping. Once we find the
4066  * end-of-field, we can match the raw field contents against the null
4067  * marker string. Only after that comparison fails do we know that
4068  * de-escaping is actually the right thing to do; therefore we *must
4069  * not* throw any syntax errors before we've done the null-marker
4070  * check.
4071  */
4072  for (;;)
4073  {
4074  char c;
4075 
4076  end_ptr = cur_ptr;
4077  if (cur_ptr >= line_end_ptr)
4078  break;
4079  c = *cur_ptr++;
4080  if (c == delimc)
4081  {
4082  found_delim = true;
4083  break;
4084  }
4085  if (c == '\\')
4086  {
4087  if (cur_ptr >= line_end_ptr)
4088  break;
4089  c = *cur_ptr++;
4090  switch (c)
4091  {
4092  case '0':
4093  case '1':
4094  case '2':
4095  case '3':
4096  case '4':
4097  case '5':
4098  case '6':
4099  case '7':
4100  {
4101  /* handle \013 */
4102  int val;
4103 
4104  val = OCTVALUE(c);
4105  if (cur_ptr < line_end_ptr)
4106  {
4107  c = *cur_ptr;
4108  if (ISOCTAL(c))
4109  {
4110  cur_ptr++;
4111  val = (val << 3) + OCTVALUE(c);
4112  if (cur_ptr < line_end_ptr)
4113  {
4114  c = *cur_ptr;
4115  if (ISOCTAL(c))
4116  {
4117  cur_ptr++;
4118  val = (val << 3) + OCTVALUE(c);
4119  }
4120  }
4121  }
4122  }
4123  c = val & 0377;
4124  if (c == '\0' || IS_HIGHBIT_SET(c))
4125  saw_non_ascii = true;
4126  }
4127  break;
4128  case 'x':
4129  /* Handle \x3F */
4130  if (cur_ptr < line_end_ptr)
4131  {
4132  char hexchar = *cur_ptr;
4133 
4134  if (isxdigit((unsigned char) hexchar))
4135  {
4136  int val = GetDecimalFromHex(hexchar);
4137 
4138  cur_ptr++;
4139  if (cur_ptr < line_end_ptr)
4140  {
4141  hexchar = *cur_ptr;
4142  if (isxdigit((unsigned char) hexchar))
4143  {
4144  cur_ptr++;
4145  val = (val << 4) + GetDecimalFromHex(hexchar);
4146  }
4147  }
4148  c = val & 0xff;
4149  if (c == '\0' || IS_HIGHBIT_SET(c))
4150  saw_non_ascii = true;
4151  }
4152  }
4153  break;
4154  case 'b':
4155  c = '\b';
4156  break;
4157  case 'f':
4158  c = '\f';
4159  break;
4160  case 'n':
4161  c = '\n';
4162  break;
4163  case 'r':
4164  c = '\r';
4165  break;
4166  case 't':
4167  c = '\t';
4168  break;
4169  case 'v':
4170  c = '\v';
4171  break;
4172 
4173  /*
4174  * in all other cases, take the char after '\'
4175  * literally
4176  */
4177  }
4178  }
4179 
4180  /* Add c to output string */
4181  *output_ptr++ = c;
4182  }
4183 
4184  /* Check whether raw input matched null marker */
4185  input_len = end_ptr - start_ptr;
4186  if (input_len == cstate->null_print_len &&
4187  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4188  cstate->raw_fields[fieldno] = NULL;
4189  else
4190  {
4191  /*
4192  * At this point we know the field is supposed to contain data.
4193  *
4194  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4195  * resulting string is valid data for the db encoding.
4196  */
4197  if (saw_non_ascii)
4198  {
4199  char *fld = cstate->raw_fields[fieldno];
4200 
4201  pg_verifymbstr(fld, output_ptr - fld, false);
4202  }
4203  }
4204 
4205  /* Terminate attribute value in output area */
4206  *output_ptr++ = '\0';
4207 
4208  fieldno++;
4209  /* Done if we hit EOL instead of a delim */
4210  if (!found_delim)
4211  break;
4212  }
4213 
4214  /* Clean up state of attribute_buf */
4215  output_ptr--;
4216  Assert(*output_ptr == '\0');
4217  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4218 
4219  return fieldno;
4220 }
static int GetDecimalFromHex(char hex)
Definition: copy.c:3973
char ** raw_fields
Definition: copy.c:190
#define ISOCTAL(c)
Definition: copy.c:53
#define OCTVALUE(c)
Definition: copy.c:54
StringInfoData line_buf
Definition: copy.c:199
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:123
char * null_print
Definition: copy.c:120
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
#define ERROR
Definition: elog.h:43
char * c
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:189
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:121
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
int errmsg(const char *fmt,...)
Definition: elog.c:797
bool pg_verifymbstr(const char *mbstr, int len, bool noError)
Definition: wchar.c:1866
StringInfoData attribute_buf
Definition: copy.c:185
long val
Definition: informix.c:689
static Datum CopyReadBinaryAttribute ( CopyState  cstate,
int  column_no,
FmgrInfo flinfo,
Oid  typioparam,
int32  typmod,
bool isnull 
)
static

Definition at line 4398 of file copy.c.

References CopyStateData::attribute_buf, CopyGetData(), CopyGetInt32(), StringInfoData::cursor, StringInfoData::data, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, StringInfoData::len, NULL, ReceiveFunctionCall(), resetStringInfo(), and result.

Referenced by NextCopyFrom().

4402 {
4403  int32 fld_size;
4404  Datum result;
4405 
4406  if (!CopyGetInt32(cstate, &fld_size))
4407  ereport(ERROR,
4408  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4409  errmsg("unexpected EOF in COPY data")));
4410  if (fld_size == -1)
4411  {
4412  *isnull = true;
4413  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4414  }
4415  if (fld_size < 0)
4416  ereport(ERROR,
4417  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4418  errmsg("invalid field size")));
4419 
4420  /* reset attribute_buf to empty, and load raw data in it */
4421  resetStringInfo(&cstate->attribute_buf);
4422 
4423  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4424  if (CopyGetData(cstate, cstate->attribute_buf.data,
4425  fld_size, fld_size) != fld_size)
4426  ereport(ERROR,
4427  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4428  errmsg("unexpected EOF in COPY data")));
4429 
4430  cstate->attribute_buf.len = fld_size;
4431  cstate->attribute_buf.data[fld_size] = '\0';
4432 
4433  /* Call the column type's binary input converter */
4434  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4435  typioparam, typmod);
4436 
4437  /* Trouble if it didn't eat the whole buffer */
4438  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4439  ereport(ERROR,
4440  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4441  errmsg("incorrect binary data format")));
4442 
4443  *isnull = false;
4444  return result;
4445 }
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:684
int errcode(int sqlerrcode)
Definition: elog.c:575
return result
Definition: formatting.c:1633
signed int int32
Definition: c.h:256
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:558
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1681
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
uintptr_t Datum
Definition: postgres.h:372
#define NULL
Definition: c.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:185
static bool CopyReadLine ( CopyState  cstate)
static

Definition at line 3525 of file copy.c.

References appendBinaryStringInfo(), Assert, CopyStateData::copy_dest, COPY_NEW_FE, CopyLoadRawBuf(), CopyReadLineText(), StringInfoData::data, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, CopyStateData::file_encoding, StringInfoData::len, CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, pfree(), pg_any_to_server(), CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, resetStringInfo(), and result.

Referenced by NextCopyFromRawFields().

3526 {
3527  bool result;
3528 
3529  resetStringInfo(&cstate->line_buf);
3530  cstate->line_buf_valid = true;
3531 
3532  /* Mark that encoding conversion hasn't occurred yet */
3533  cstate->line_buf_converted = false;
3534 
3535  /* Parse data and transfer into line_buf */
3536  result = CopyReadLineText(cstate);
3537 
3538  if (result)
3539  {
3540  /*
3541  * Reached EOF. In protocol version 3, we should ignore anything
3542  * after \. up to the protocol end of copy data. (XXX maybe better
3543  * not to treat \. as special?)
3544  */
3545  if (cstate->copy_dest == COPY_NEW_FE)
3546  {
3547  do
3548  {
3549  cstate->raw_buf_index = cstate->raw_buf_len;
3550  } while (CopyLoadRawBuf(cstate));
3551  }
3552  }
3553  else
3554  {
3555  /*
3556  * If we didn't hit EOF, then we must have transferred the EOL marker
3557  * to line_buf along with the data. Get rid of it.
3558  */
3559  switch (cstate->eol_type)
3560  {
3561  case EOL_NL:
3562  Assert(cstate->line_buf.len >= 1);
3563  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3564  cstate->line_buf.len--;
3565  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3566  break;
3567  case EOL_CR:
3568  Assert(cstate->line_buf.len >= 1);
3569  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3570  cstate->line_buf.len--;
3571  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3572  break;
3573  case EOL_CRNL:
3574  Assert(cstate->line_buf.len >= 2);
3575  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3576  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3577  cstate->line_buf.len -= 2;
3578  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3579  break;
3580  case EOL_UNKNOWN:
3581  /* shouldn't get here */
3582  Assert(false);
3583  break;
3584  }
3585  }
3586 
3587  /* Done reading the line. Convert it to server encoding. */
3588  if (cstate->need_transcoding)
3589  {
3590  char *cvt;
3591 
3592  cvt = pg_any_to_server(cstate->line_buf.data,
3593  cstate->line_buf.len,
3594  cstate->file_encoding);
3595  if (cvt != cstate->line_buf.data)
3596  {
3597  /* transfer converted data back to line_buf */
3598  resetStringInfo(&cstate->line_buf);
3599  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3600  pfree(cvt);
3601  }
3602  }
3603 
3604  /* Now it's safe to use the buffer in error messages */
3605  cstate->line_buf_converted = true;
3606 
3607  return result;
3608 }
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3614
Definition: copy.c:74
bool need_transcoding
Definition: copy.c:105
StringInfoData line_buf
Definition: copy.c:199
int raw_buf_index
Definition: copy.c:212
bool line_buf_valid
Definition: copy.c:201
return result
Definition: formatting.c:1633
bool line_buf_converted
Definition: copy.c:200
CopyDest copy_dest
Definition: copy.c:98
void pfree(void *pointer)
Definition: mcxt.c:950
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:738
Definition: copy.c:75
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int raw_buf_len
Definition: copy.c:213
Definition: copy.c:76
int file_encoding
Definition: copy.c:104
EolType eol_type
Definition: copy.c:103
#define Assert(condition)
Definition: c.h:675
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:572
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static bool CopyReadLineText ( CopyState  cstate)
static

Definition at line 3614 of file copy.c.

References appendBinaryStringInfo(), CopyLoadRawBuf(), CopyStateData::csv_mode, CopyStateData::cur_lineno, CopyStateData::encoding_embeds_ascii, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errhint(), errmsg(), ERROR, CopyStateData::escape, CopyStateData::file_encoding, IF_NEED_REFILL_AND_EOF_BREAK, IF_NEED_REFILL_AND_NOT_EOF_CONTINUE, IS_HIGHBIT_SET, CopyStateData::line_buf, NO_END_OF_COPY_GOTO, pg_encoding_mblen(), CopyStateData::quote, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, REFILL_LINEBUF, and result.

Referenced by CopyReadLine().

3615 {
3616  char *copy_raw_buf;
3617  int raw_buf_ptr;
3618  int copy_buf_len;
3619  bool need_data = false;
3620  bool hit_eof = false;
3621  bool result = false;
3622  char mblen_str[2];
3623 
3624  /* CSV variables */
3625  bool first_char_in_line = true;
3626  bool in_quote = false,
3627  last_was_esc = false;
3628  char quotec = '\0';
3629  char escapec = '\0';
3630 
3631  if (cstate->csv_mode)
3632  {
3633  quotec = cstate->quote[0];
3634  escapec = cstate->escape[0];
3635  /* ignore special escape processing if it's the same as quotec */
3636  if (quotec == escapec)
3637  escapec = '\0';
3638  }
3639 
3640  mblen_str[1] = '\0';
3641 
3642  /*
3643  * The objective of this loop is to transfer the entire next input line
3644  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3645  * \n) and the end-of-copy marker (\.).
3646  *
3647  * In CSV mode, \r and \n inside a quoted field are just part of the data
3648  * value and are put in line_buf. We keep just enough state to know if we
3649  * are currently in a quoted field or not.
3650  *
3651  * These four characters, and the CSV escape and quote characters, are
3652  * assumed the same in frontend and backend encodings.
3653  *
3654  * For speed, we try to move data from raw_buf to line_buf in chunks
3655  * rather than one character at a time. raw_buf_ptr points to the next
3656  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3657  * have been determined to be part of the line, but not yet transferred to
3658  * line_buf.
3659  *
3660  * For a little extra speed within the loop, we copy raw_buf and
3661  * raw_buf_len into local variables.
3662  */
3663  copy_raw_buf = cstate->raw_buf;
3664  raw_buf_ptr = cstate->raw_buf_index;
3665  copy_buf_len = cstate->raw_buf_len;
3666 
3667  for (;;)
3668  {
3669  int prev_raw_ptr;
3670  char c;
3671 
3672  /*
3673  * Load more data if needed. Ideally we would just force four bytes
3674  * of read-ahead and avoid the many calls to
3675  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3676  * does not allow us to read too far ahead or we might read into the
3677  * next data, so we read-ahead only as far we know we can. One
3678  * optimization would be to read-ahead four byte here if
3679  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3680  * considering the size of the buffer.
3681  */
3682  if (raw_buf_ptr >= copy_buf_len || need_data)
3683  {
3685 
3686  /*
3687  * Try to read some more data. This will certainly reset
3688  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3689  */
3690  if (!CopyLoadRawBuf(cstate))
3691  hit_eof = true;
3692  raw_buf_ptr = 0;
3693  copy_buf_len = cstate->raw_buf_len;
3694 
3695  /*
3696  * If we are completely out of data, break out of the loop,
3697  * reporting EOF.
3698  */
3699  if (copy_buf_len <= 0)
3700  {
3701  result = true;
3702  break;
3703  }
3704  need_data = false;
3705  }
3706 
3707  /* OK to fetch a character */
3708  prev_raw_ptr = raw_buf_ptr;
3709  c = copy_raw_buf[raw_buf_ptr++];
3710 
3711  if (cstate->csv_mode)
3712  {
3713  /*
3714  * If character is '\\' or '\r', we may need to look ahead below.
3715  * Force fetch of the next character if we don't already have it.
3716  * We need to do this before changing CSV state, in case one of
3717  * these characters is also the quote or escape character.
3718  *
3719  * Note: old-protocol does not like forced prefetch, but it's OK
3720  * here since we cannot validly be at EOF.
3721  */
3722  if (c == '\\' || c == '\r')
3723  {
3725  }
3726 
3727  /*
3728  * Dealing with quotes and escapes here is mildly tricky. If the
3729  * quote char is also the escape char, there's no problem - we
3730  * just use the char as a toggle. If they are different, we need
3731  * to ensure that we only take account of an escape inside a
3732  * quoted field and immediately preceding a quote char, and not
3733  * the second in an escape-escape sequence.
3734  */
3735  if (in_quote && c == escapec)
3736  last_was_esc = !last_was_esc;
3737  if (c == quotec && !last_was_esc)
3738  in_quote = !in_quote;
3739  if (c != escapec)
3740  last_was_esc = false;
3741 
3742  /*
3743  * Updating the line count for embedded CR and/or LF chars is
3744  * necessarily a little fragile - this test is probably about the
3745  * best we can do. (XXX it's arguable whether we should do this
3746  * at all --- is cur_lineno a physical or logical count?)
3747  */
3748  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3749  cstate->cur_lineno++;
3750  }
3751 
3752  /* Process \r */
3753  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3754  {
3755  /* Check for \r\n on first line, _and_ handle \r\n. */
3756  if (cstate->eol_type == EOL_UNKNOWN ||
3757  cstate->eol_type == EOL_CRNL)
3758  {
3759  /*
3760  * If need more data, go back to loop top to load it.
3761  *
3762  * Note that if we are at EOF, c will wind up as '\0' because
3763  * of the guaranteed pad of raw_buf.
3764  */
3766 
3767  /* get next char */
3768  c = copy_raw_buf[raw_buf_ptr];
3769 
3770  if (c == '\n')
3771  {
3772  raw_buf_ptr++; /* eat newline */
3773  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3774  }
3775  else
3776  {
3777  /* found \r, but no \n */
3778  if (cstate->eol_type == EOL_CRNL)
3779  ereport(ERROR,
3780  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3781  !cstate->csv_mode ?
3782  errmsg("literal carriage return found in data") :
3783  errmsg("unquoted carriage return found in data"),
3784  !cstate->csv_mode ?
3785  errhint("Use \"\\r\" to represent carriage return.") :
3786  errhint("Use quoted CSV field to represent carriage return.")));
3787 
3788  /*
3789  * if we got here, it is the first line and we didn't find
3790  * \n, so don't consume the peeked character
3791  */
3792  cstate->eol_type = EOL_CR;
3793  }
3794  }
3795  else if (cstate->eol_type == EOL_NL)
3796  ereport(ERROR,
3797  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3798  !cstate->csv_mode ?
3799  errmsg("literal carriage return found in data") :
3800  errmsg("unquoted carriage return found in data"),
3801  !cstate->csv_mode ?
3802  errhint("Use \"\\r\" to represent carriage return.") :
3803  errhint("Use quoted CSV field to represent carriage return.")));
3804  /* If reach here, we have found the line terminator */
3805  break;
3806  }
3807 
3808  /* Process \n */
3809  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3810  {
3811  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3812  ereport(ERROR,
3813  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3814  !cstate->csv_mode ?
3815  errmsg("literal newline found in data") :
3816  errmsg("unquoted newline found in data"),
3817  !cstate->csv_mode ?
3818  errhint("Use \"\\n\" to represent newline.") :
3819  errhint("Use quoted CSV field to represent newline.")));
3820  cstate->eol_type = EOL_NL; /* in case not set yet */
3821  /* If reach here, we have found the line terminator */
3822  break;
3823  }
3824 
3825  /*
3826  * In CSV mode, we only recognize \. alone on a line. This is because
3827  * \. is a valid CSV data value.
3828  */
3829  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3830  {
3831  char c2;
3832 
3835 
3836  /* -----
3837  * get next character
3838  * Note: we do not change c so if it isn't \., we can fall
3839  * through and continue processing for file encoding.
3840  * -----
3841  */
3842  c2 = copy_raw_buf[raw_buf_ptr];
3843 
3844  if (c2 == '.')
3845  {
3846  raw_buf_ptr++; /* consume the '.' */
3847 
3848  /*
3849  * Note: if we loop back for more data here, it does not
3850  * matter that the CSV state change checks are re-executed; we
3851  * will come back here with no important state changed.
3852  */
3853  if (cstate->eol_type == EOL_CRNL)
3854  {
3855  /* Get the next character */
3857  /* if hit_eof, c2 will become '\0' */
3858  c2 = copy_raw_buf[raw_buf_ptr++];
3859 
3860  if (c2 == '\n')
3861  {
3862  if (!cstate->csv_mode)
3863  ereport(ERROR,
3864  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3865  errmsg("end-of-copy marker does not match previous newline style")));
3866  else
3868  }
3869  else if (c2 != '\r')
3870  {
3871  if (!cstate->csv_mode)
3872  ereport(ERROR,
3873  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3874  errmsg("end-of-copy marker corrupt")));
3875  else
3877  }
3878  }
3879 
3880  /* Get the next character */
3882  /* if hit_eof, c2 will become '\0' */
3883  c2 = copy_raw_buf[raw_buf_ptr++];
3884 
3885  if (c2 != '\r' && c2 != '\n')
3886  {
3887  if (!cstate->csv_mode)
3888  ereport(ERROR,
3889  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3890  errmsg("end-of-copy marker corrupt")));
3891  else
3893  }
3894 
3895  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3896  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3897  (cstate->eol_type == EOL_CR && c2 != '\r'))
3898  {
3899  ereport(ERROR,
3900  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3901  errmsg("end-of-copy marker does not match previous newline style")));
3902  }
3903 
3904  /*
3905  * Transfer only the data before the \. into line_buf, then
3906  * discard the data and the \. sequence.
3907  */
3908  if (prev_raw_ptr > cstate->raw_buf_index)
3910  cstate->raw_buf + cstate->raw_buf_index,
3911  prev_raw_ptr - cstate->raw_buf_index);
3912  cstate->raw_buf_index = raw_buf_ptr;
3913  result = true; /* report EOF */
3914  break;
3915  }
3916  else if (!cstate->csv_mode)
3917 
3918  /*
3919  * If we are here, it means we found a backslash followed by
3920  * something other than a period. In non-CSV mode, anything
3921  * after a backslash is special, so we skip over that second
3922  * character too. If we didn't do that \\. would be
3923  * considered an eof-of copy, while in non-CSV mode it is a
3924  * literal backslash followed by a period. In CSV mode,
3925  * backslashes are not special, so we want to process the
3926  * character after the backslash just like a normal character,
3927  * so we don't increment in those cases.
3928  */
3929  raw_buf_ptr++;
3930  }
3931 
3932  /*
3933  * This label is for CSV cases where \. appears at the start of a
3934  * line, but there is more text after it, meaning it was a data value.
3935  * We are more strict for \. in CSV mode because \. could be a data
3936  * value, while in non-CSV mode, \. cannot be a data value.
3937  */
3938 not_end_of_copy:
3939 
3940  /*
3941  * Process all bytes of a multi-byte character as a group.
3942  *
3943  * We only support multi-byte sequences where the first byte has the
3944  * high-bit set, so as an optimization we can avoid this block
3945  * entirely if it is not set.
3946  */
3947  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3948  {
3949  int mblen;
3950 
3951  mblen_str[0] = c;
3952  /* All our encodings only read the first byte to get the length */
3953  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3955  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3956  raw_buf_ptr += mblen - 1;
3957  }
3958  first_char_in_line = false;
3959  } /* end of outer loop */
3960 
3961  /*
3962  * Transfer any still-uncopied data to line_buf.
3963  */
3965 
3966  return result;
3967 }
bool csv_mode
Definition: copy.c:118
int errhint(const char *fmt,...)
Definition: elog.c:987
Definition: copy.c:74
StringInfoData line_buf
Definition: copy.c:199
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:253
int raw_buf_index
Definition: copy.c:212
int errcode(int sqlerrcode)
Definition: elog.c:575
return result
Definition: formatting.c:1633
#define REFILL_LINEBUF
Definition: copy.c:270
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
#define ERROR
Definition: elog.h:43
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
char * raw_buf
Definition: copy.c:211
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:738
char * quote
Definition: copy.c:124
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
Definition: copy.c:75
int raw_buf_len
Definition: copy.c:213
char * escape
Definition: copy.c:125
#define ereport(elevel, rest)
Definition: elog.h:122
Definition: copy.c:76
int file_encoding
Definition: copy.c:104
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:241
EolType eol_type
Definition: copy.c:103
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define NO_END_OF_COPY_GOTO
Definition: copy.c:283
int cur_lineno
Definition: copy.c:139
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static void CopySendChar ( CopyState  cstate,
char  c 
)
static

Definition at line 459 of file copy.c.

References appendStringInfoCharMacro, and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), CopyOneRowTo(), CopySendEndOfRow(), and CopyTo().

460 {
462 }
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:127
char * c
StringInfo fe_msgbuf
Definition: copy.c:100
static void CopySendData ( CopyState  cstate,
const void *  databuf,
int  datasize 
)
static

Definition at line 447 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyOneRowTo(), CopySendInt16(), CopySendInt32(), CopyTo(), and SendCopyEnd().

448 {
449  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
450 }
StringInfo fe_msgbuf
Definition: copy.c:100
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static void CopySendEndOfRow ( CopyState  cstate)
static

Definition at line 465 of file copy.c.

References Assert, CopyStateData::binary, ClosePipeToProgram(), COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, CopySendChar(), CopySendString(), StringInfoData::data, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, FATAL, CopyStateData::fe_msgbuf, CopyStateData::is_program, StringInfoData::len, pq_putbytes(), pq_putmessage, and resetStringInfo().

Referenced by CopyOneRowTo(), CopyTo(), and SendCopyEnd().

466 {
467  StringInfo fe_msgbuf = cstate->fe_msgbuf;
468 
469  switch (cstate->copy_dest)
470  {
471  case COPY_FILE:
472  if (!cstate->binary)
473  {
474  /* Default line termination depends on platform */
475 #ifndef WIN32
476  CopySendChar(cstate, '\n');
477 #else
478  CopySendString(cstate, "\r\n");
479 #endif
480  }
481 
482  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
483  cstate->copy_file) != 1 ||
484  ferror(cstate->copy_file))
485  {
486  if (cstate->is_program)
487  {
488  if (errno == EPIPE)
489  {
490  /*
491  * The pipe will be closed automatically on error at
492  * the end of transaction, but we might get a better
493  * error message from the subprocess' exit code than
494  * just "Broken Pipe"
495  */
496  ClosePipeToProgram(cstate);
497 
498  /*
499  * If ClosePipeToProgram() didn't throw an error, the
500  * program terminated normally, but closed the pipe
501  * first. Restore errno, and throw an error.
502  */
503  errno = EPIPE;
504  }
505  ereport(ERROR,
507  errmsg("could not write to COPY program: %m")));
508  }
509  else
510  ereport(ERROR,
512  errmsg("could not write to COPY file: %m")));
513  }
514  break;
515  case COPY_OLD_FE:
516  /* The FE/BE protocol uses \n as newline for all platforms */
517  if (!cstate->binary)
518  CopySendChar(cstate, '\n');
519 
520  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
521  {
522  /* no hope of recovering connection sync, so FATAL */
523  ereport(FATAL,
524  (errcode(ERRCODE_CONNECTION_FAILURE),
525  errmsg("connection lost during COPY to stdout")));
526  }
527  break;
528  case COPY_NEW_FE:
529  /* The FE/BE protocol uses \n as newline for all platforms */
530  if (!cstate->binary)
531  CopySendChar(cstate, '\n');
532 
533  /* Dump the accumulated row as one CopyData message */
534  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
535  break;
536  case COPY_CALLBACK:
537  Assert(false); /* Not yet supported. */
538  break;
539  }
540 
541  resetStringInfo(fe_msgbuf);
542 }
bool binary
Definition: copy.c:115
int errcode(int sqlerrcode)
Definition: elog.c:575
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1742
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:459
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
#define ERROR
Definition: elog.h:43
#define FATAL
Definition: elog.h:52
int errcode_for_file_access(void)
Definition: elog.c:598
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:113
#define Assert(condition)
Definition: c.h:675
StringInfo fe_msgbuf
Definition: copy.c:100
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:453
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
int pq_putbytes(const char *s, size_t len)
Definition: pqcomm.c:1353
static void CopySendInt16 ( CopyState  cstate,
int16  val 
)
static

Definition at line 701 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

702 {
703  uint16 buf;
704 
705  buf = htons((uint16) val);
706  CopySendData(cstate, &buf, sizeof(buf));
707 }
unsigned short uint16
Definition: c.h:267
static char * buf
Definition: pg_test_fsync.c:66
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:447
long val
Definition: informix.c:689
static void CopySendInt32 ( CopyState  cstate,
int32  val 
)
static

Definition at line 670 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

671 {
672  uint32 buf;
673 
674  buf = htonl((uint32) val);
675  CopySendData(cstate, &buf, sizeof(buf));
676 }
static char * buf
Definition: pg_test_fsync.c:66
unsigned int uint32
Definition: c.h:268
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:447
long val
Definition: informix.c:689
static void CopySendString ( CopyState  cstate,
const char *  str 
)
static

Definition at line 453 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyOneRowTo(), and CopySendEndOfRow().

454 {
455  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
456 }
StringInfo fe_msgbuf
Definition: copy.c:100
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static uint64 CopyTo ( CopyState  cstate)
static

Definition at line 1971 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), CopyStateData::attnumlist, tupleDesc::attrs, CopyStateData::binary, BinarySignature, CHECK_FOR_INTERRUPTS, CopyAttributeOutCSV(), CopyOneRowTo(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), cur, CurrentMemoryContext, CopyStateData::delim, QueryDesc::dest, ExecutorRun(), CopyStateData::fe_msgbuf, CopyStateData::file_encoding, fmgr_info(), ForwardScanDirection, GetActiveSnapshot(), getTypeBinaryOutputInfo(), getTypeOutputInfo(), CopyStateData::header_line, heap_beginscan(), heap_deform_tuple(), heap_endscan(), heap_getnext(), HeapTupleGetOid, lfirst_int, list_length(), makeStringInfo(), MemoryContextDelete(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NULL, CopyStateData::null_print, CopyStateData::null_print_client, CopyStateData::null_print_len, CopyStateData::oids, CopyStateData::out_functions, palloc(), pfree(), pg_server_to_any(), CopyStateData::queryDesc, CopyStateData::rel, RelationGetDescr, CopyStateData::rowcontext, QueryDesc::tupDesc, and values.

Referenced by DoCopyTo().

1972 {
1973  TupleDesc tupDesc;
1974  int num_phys_attrs;
1975  Form_pg_attribute *attr;
1976  ListCell *cur;
1977  uint64 processed;
1978 
1979  if (cstate->rel)
1980  tupDesc = RelationGetDescr(cstate->rel);
1981  else
1982  tupDesc = cstate->queryDesc->tupDesc;
1983  attr = tupDesc->attrs;
1984  num_phys_attrs = tupDesc->natts;
1985  cstate->null_print_client = cstate->null_print; /* default */
1986 
1987  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1988  cstate->fe_msgbuf = makeStringInfo();
1989 
1990  /* Get info about the columns we need to process. */
1991  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1992  foreach(cur, cstate->attnumlist)
1993  {
1994  int attnum = lfirst_int(cur);
1995  Oid out_func_oid;
1996  bool isvarlena;
1997 
1998  if (cstate->binary)
1999  getTypeBinaryOutputInfo(attr[attnum - 1]->atttypid,
2000  &out_func_oid,
2001  &isvarlena);
2002  else
2003  getTypeOutputInfo(attr[attnum - 1]->atttypid,
2004  &out_func_oid,
2005  &isvarlena);
2006  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
2007  }
2008 
2009  /*
2010  * Create a temporary memory context that we can reset once per row to
2011  * recover palloc'd memory. This avoids any problems with leaks inside
2012  * datatype output routines, and should be faster than retail pfree's
2013  * anyway. (We don't need a whole econtext as CopyFrom does.)
2014  */
2016  "COPY TO",
2018 
2019  if (cstate->binary)
2020  {
2021  /* Generate header for a binary copy */
2022  int32 tmp;
2023 
2024  /* Signature */
2025  CopySendData(cstate, BinarySignature, 11);
2026  /* Flags field */
2027  tmp = 0;
2028  if (cstate->oids)
2029  tmp |= (1 << 16);
2030  CopySendInt32(cstate, tmp);
2031  /* No header extension */
2032  tmp = 0;
2033  CopySendInt32(cstate, tmp);
2034  }
2035  else
2036  {
2037  /*
2038  * For non-binary copy, we need to convert null_print to file
2039  * encoding, because it will be sent directly with CopySendString.
2040  */
2041  if (cstate->need_transcoding)
2042  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2043  cstate->null_print_len,
2044  cstate->file_encoding);
2045 
2046  /* if a header has been requested send the line */
2047  if (cstate->header_line)
2048  {
2049  bool hdr_delim = false;
2050 
2051  foreach(cur, cstate->attnumlist)
2052  {
2053  int attnum = lfirst_int(cur);
2054  char *colname;
2055 
2056  if (hdr_delim)
2057  CopySendChar(cstate, cstate->delim[0]);
2058  hdr_delim = true;
2059 
2060  colname = NameStr(attr[attnum - 1]->attname);
2061 
2062  CopyAttributeOutCSV(cstate, colname, false,
2063  list_length(cstate->attnumlist) == 1);
2064  }
2065 
2066  CopySendEndOfRow(cstate);
2067  }
2068  }
2069 
2070  if (cstate->rel)
2071  {
2072  Datum *values;
2073  bool *nulls;
2074  HeapScanDesc scandesc;
2075  HeapTuple tuple;
2076 
2077  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2078  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2079 
2080  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2081 
2082  processed = 0;
2083  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2084  {
2086 
2087  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2088  heap_deform_tuple(tuple, tupDesc, values, nulls);
2089 
2090  /* Format and send the data */
2091  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2092  processed++;
2093  }
2094 
2095  heap_endscan(scandesc);
2096 
2097  pfree(values);
2098  pfree(nulls);
2099  }
2100  else
2101  {
2102  /* run the plan --- the dest receiver will send tuples */
2103  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2104  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2105  }
2106 
2107  if (cstate->binary)
2108  {
2109  /* Generate trailer for a binary copy */
2110  CopySendInt16(cstate, -1);
2111  /* Need to flush out the trailer */
2112  CopySendEndOfRow(cstate);
2113  }
2114 
2116 
2117  return processed;
2118 }
Definition: fmgr.h:56
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:200
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2632
bool binary
Definition: copy.c:115
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1578
#define RelationGetDescr(relation)
Definition: rel.h:428
bool need_transcoding
Definition: copy.c:105
List * attnumlist
Definition: copy.c:111
Definition: copy.c:217
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
Form_pg_attribute * attrs
Definition: tupdesc.h:74
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:839
struct cursor * cur
Definition: ecpg.c:28
unsigned int Oid
Definition: postgres_ext.h:31
char * delim
Definition: copy.c:123
MemoryContext rowcontext
Definition: copy.c:152
int natts
Definition: tupdesc.h:73
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
signed int int32
Definition: c.h:256
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:459
char * null_print
Definition: copy.c:120
Relation rel
Definition: copy.c:109
void pfree(void *pointer)
Definition: mcxt.c:950
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:701
#define lfirst_int(lc)
Definition: pg_list.h:107
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:298
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4610
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:127
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
QueryDesc * queryDesc
Definition: copy.c:110
static const char BinarySignature[11]
Definition: copy.c:290
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
int null_print_len
Definition: copy.c:121
int file_encoding
Definition: copy.c:104
TupleDesc tupDesc
Definition: execdesc.h:47
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2698
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
bool header_line
Definition: copy.c:119
uintptr_t Datum
Definition: postgres.h:372
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1794
#define NULL
Definition: c.h:229
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:447
StringInfo fe_msgbuf
Definition: copy.c:100
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:670
void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull)
Definition: heaptuple.c:933
static Datum values[MAXATTR]
Definition: bootstrap.c:163
DestReceiver * dest
Definition: execdesc.h:41
FmgrInfo * out_functions
Definition: copy.c:151
void * palloc(Size size)
Definition: mcxt.c:849
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:465
#define NameStr(name)
Definition: c.h:499
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
char * null_print_client
Definition: copy.c:122
#define HeapTupleGetOid(tuple)
Definition: htup_details.h:695
HeapScanDesc heap_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key)
Definition: heapam.c:1391
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2124
bool oids
Definition: copy.c:116
DestReceiver* CreateCopyDestReceiver ( void  )

Definition at line 4819 of file copy.c.

References copy_dest_destroy(), copy_dest_receive(), copy_dest_shutdown(), copy_dest_startup(), DestCopyOut, NULL, and palloc().

Referenced by CreateDestReceiver().

4820 {
4821  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4822 
4823  self->pub.receiveSlot = copy_dest_receive;
4824  self->pub.rStartup = copy_dest_startup;
4825  self->pub.rShutdown = copy_dest_shutdown;
4826  self->pub.rDestroy = copy_dest_destroy;
4827  self->pub.mydest = DestCopyOut;
4828 
4829  self->cstate = NULL; /* will be set later */
4830  self->processed = 0;
4831 
4832  return (DestReceiver *) self;
4833 }
Definition: copy.c:217
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4810
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4801
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4773
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4782
#define NULL
Definition: c.h:229
void * palloc(Size size)
Definition: mcxt.c:849
void DoCopy ( ParseState pstate,
const CopyStmt stmt,
int  stmt_location,
int  stmt_len,
uint64 *  processed 
)

Definition at line 783 of file copy.c.

References AccessShareLock, ACL_INSERT, ACL_SELECT, addRangeTableEntryForRelation(), Assert, CopyStmt::attlist, BeginCopyFrom(), BeginCopyTo(), bms_add_member(), check_enable_rls(), CopyFrom(), CopyGetAttnums(), cur, DoCopyTo(), EndCopyFrom(), EndCopyTo(), ereport, errcode(), errhint(), errmsg(), ERROR, ExecCheckRTPerms(), ColumnRef::fields, CopyStmt::filename, FirstLowInvalidHeapAttributeNumber, SelectStmt::fromClause, get_namespace_name(), heap_close, heap_openrv(), ResTarget::indirection, RangeTblEntry::insertedCols, InvalidOid, CopyStmt::is_from, CopyStmt::is_program, lappend(), lfirst, lfirst_int, list_make1, ColumnRef::location, ResTarget::location, makeNode, makeRangeVar(), ResTarget::name, NIL, NoLock, NULL, CopyStmt::options, ParseState::p_rtable, PreventCommandIfParallelMode(), PreventCommandIfReadOnly(), pstrdup(), CopyStmt::query, RelationData::rd_islocaltemp, CopyStmt::relation, RelationGetDescr, RelationGetNamespace, RelationGetRelationName, RelationGetRelid, RangeTblEntry::relid, RangeTblEntry::requiredPerms, RLS_ENABLED, RowExclusiveLock, select, RangeTblEntry::selectedCols, RawStmt::stmt, RawStmt::stmt_len, RawStmt::stmt_location, superuser(), SelectStmt::targetList, ResTarget::val, and XactReadOnly.

Referenced by standard_ProcessUtility().

786 {
787  CopyState cstate;
788  bool is_from = stmt->is_from;
789  bool pipe = (stmt->filename == NULL);
790  Relation rel;
791  Oid relid;
792  RawStmt *query = NULL;
793 
794  /* Disallow COPY to/from file or program except to superusers. */
795  if (!pipe && !superuser())
796  {
797  if (stmt->is_program)
798  ereport(ERROR,
799  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
800  errmsg("must be superuser to COPY to or from an external program"),
801  errhint("Anyone can COPY to stdout or from stdin. "
802  "psql's \\copy command also works for anyone.")));
803  else
804  ereport(ERROR,
805  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
806  errmsg("must be superuser to COPY to or from a file"),
807  errhint("Anyone can COPY to stdout or from stdin. "
808  "psql's \\copy command also works for anyone.")));
809  }
810 
811  if (stmt->relation)
812  {
813  TupleDesc tupDesc;
814  List *attnums;
815  ListCell *cur;
816  RangeTblEntry *rte;
817 
818  Assert(!stmt->query);
819 
820  /* Open and lock the relation, using the appropriate lock type. */
821  rel = heap_openrv(stmt->relation,
822  (is_from ? RowExclusiveLock : AccessShareLock));
823 
824  relid = RelationGetRelid(rel);
825 
826  rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
827  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
828 
829  tupDesc = RelationGetDescr(rel);
830  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
831  foreach(cur, attnums)
832  {
833  int attno = lfirst_int(cur) -
835 
836  if (is_from)
837  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
838  else
839  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
840  }
841  ExecCheckRTPerms(pstate->p_rtable, true);
842 
843  /*
844  * Permission check for row security policies.
845  *
846  * check_enable_rls will ereport(ERROR) if the user has requested
847  * something invalid and will otherwise indicate if we should enable
848  * RLS (returns RLS_ENABLED) or not for this COPY statement.
849  *
850  * If the relation has a row security policy and we are to apply it
851  * then perform a "query" copy and allow the normal query processing
852  * to handle the policies.
853  *
854  * If RLS is not enabled for this, then just fall through to the
855  * normal non-filtering relation handling.
856  */
857  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
858  {
860  ColumnRef *cr;
861  ResTarget *target;
862  RangeVar *from;
863  List *targetList = NIL;
864 
865  if (is_from)
866  ereport(ERROR,
867  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
868  errmsg("COPY FROM not supported with row-level security"),
869  errhint("Use INSERT statements instead.")));
870 
871  /*
872  * Build target list
873  *
874  * If no columns are specified in the attribute list of the COPY
875  * command, then the target list is 'all' columns. Therefore, '*'
876  * should be used as the target list for the resulting