PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
copy.c File Reference
#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/sysattr.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/pg_type.h"
#include "commands/copy.h"
#include "commands/defrem.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "optimizer/clauses.h"
#include "optimizer/planner.h"
#include "nodes/makefuncs.h"
#include "parser/parse_relation.h"
#include "rewrite/rewriteHandler.h"
#include "storage/fd.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/portal.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
Include dependency graph for copy.c:

Go to the source code of this file.

Data Structures

struct  CopyStateData
 
struct  DR_copy
 

Macros

#define ISOCTAL(c)   (((c) >= '0') && ((c) <= '7'))
 
#define OCTVALUE(c)   ((c) - '0')
 
#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */
 
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
 
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
 
#define REFILL_LINEBUF
 
#define NO_END_OF_COPY_GOTO
 
#define MAX_COPY_DATA_DISPLAY   100
 
#define MAX_BUFFERED_TUPLES   1000
 
#define DUMPSOFAR()
 

Typedefs

typedef enum CopyDest CopyDest
 
typedef enum EolType EolType
 
typedef struct CopyStateData CopyStateData
 

Enumerations

enum  CopyDest { COPY_FILE, COPY_OLD_FE, COPY_NEW_FE, COPY_CALLBACK }
 
enum  EolType { EOL_UNKNOWN, EOL_NL, EOL_CR, EOL_CRNL }
 

Functions

static CopyState BeginCopy (ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
 
static void EndCopy (CopyState cstate)
 
static void ClosePipeToProgram (CopyState cstate)
 
static CopyState BeginCopyTo (ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
 
static void EndCopyTo (CopyState cstate)
 
static uint64 DoCopyTo (CopyState cstate)
 
static uint64 CopyTo (CopyState cstate)
 
static void CopyOneRowTo (CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
 
static void CopyFromInsertBatch (CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
 
static bool CopyReadLine (CopyState cstate)
 
static bool CopyReadLineText (CopyState cstate)
 
static int CopyReadAttributesText (CopyState cstate)
 
static int CopyReadAttributesCSV (CopyState cstate)
 
static Datum CopyReadBinaryAttribute (CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
 
static void CopyAttributeOutText (CopyState cstate, char *string)
 
static void CopyAttributeOutCSV (CopyState cstate, char *string, bool use_quote, bool single_attr)
 
static ListCopyGetAttnums (TupleDesc tupDesc, Relation rel, List *attnamelist)
 
static char * limit_printout_length (const char *str)
 
static void SendCopyBegin (CopyState cstate)
 
static void ReceiveCopyBegin (CopyState cstate)
 
static void SendCopyEnd (CopyState cstate)
 
static void CopySendData (CopyState cstate, const void *databuf, int datasize)
 
static void CopySendString (CopyState cstate, const char *str)
 
static void CopySendChar (CopyState cstate, char c)
 
static void CopySendEndOfRow (CopyState cstate)
 
static int CopyGetData (CopyState cstate, void *databuf, int minread, int maxread)
 
static void CopySendInt32 (CopyState cstate, int32 val)
 
static bool CopyGetInt32 (CopyState cstate, int32 *val)
 
static void CopySendInt16 (CopyState cstate, int16 val)
 
static bool CopyGetInt16 (CopyState cstate, int16 *val)
 
static bool CopyLoadRawBuf (CopyState cstate)
 
void DoCopy (ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
 
void ProcessCopyOptions (ParseState *pstate, CopyState cstate, bool is_from, List *options)
 
void CopyFromErrorCallback (void *arg)
 
uint64 CopyFrom (CopyState cstate)
 
CopyState BeginCopyFrom (ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
 
bool NextCopyFromRawFields (CopyState cstate, char ***fields, int *nfields)
 
bool NextCopyFrom (CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
 
void EndCopyFrom (CopyState cstate)
 
static int GetDecimalFromHex (char hex)
 
static void copy_dest_startup (DestReceiver *self, int operation, TupleDesc typeinfo)
 
static bool copy_dest_receive (TupleTableSlot *slot, DestReceiver *self)
 
static void copy_dest_shutdown (DestReceiver *self)
 
static void copy_dest_destroy (DestReceiver *self)
 
DestReceiverCreateCopyDestReceiver (void)
 

Variables

static const char BinarySignature [11] = "PGCOPY\n\377\r\n\0"
 

Macro Definition Documentation

#define DUMPSOFAR ( )
Value:
do { \
if (ptr > start) \
CopySendData(cstate, start, ptr - start); \
} while (0)
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445

Definition at line 4373 of file copy.c.

Referenced by CopyAttributeOutCSV(), and CopyAttributeOutText().

#define IF_NEED_REFILL_AND_EOF_BREAK (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
{ \
if (extralen) \
raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
/* backslash just before EOF, treat as data char */ \
result = true; \
break; \
} \
} else ((void) 0)
return result
Definition: formatting.c:1632

Definition at line 251 of file copy.c.

Referenced by CopyReadLineText().

#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE (   extralen)
Value:
if (1) \
{ \
if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
{ \
raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
need_data = true; \
continue; \
} \
} else ((void) 0)

Definition at line 239 of file copy.c.

Referenced by CopyReadLineText().

#define ISOCTAL (   c)    (((c) >= '0') && ((c) <= '7'))

Definition at line 53 of file copy.c.

Referenced by CopyReadAttributesText().

#define MAX_BUFFERED_TUPLES   1000

Referenced by CopyFrom().

#define MAX_COPY_DATA_DISPLAY   100

Referenced by limit_printout_length().

#define NO_END_OF_COPY_GOTO
Value:
if (1) \
{ \
raw_buf_ptr = prev_raw_ptr + 1; \
goto not_end_of_copy; \
} else ((void) 0)

Definition at line 281 of file copy.c.

Referenced by CopyReadLineText().

#define OCTVALUE (   c)    ((c) - '0')

Definition at line 54 of file copy.c.

Referenced by CopyReadAttributesText().

#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */

Definition at line 208 of file copy.c.

Referenced by BeginCopyFrom(), and CopyLoadRawBuf().

#define REFILL_LINEBUF
Value:
if (1) \
{ \
if (raw_buf_ptr > cstate->raw_buf_index) \
{ \
appendBinaryStringInfo(&cstate->line_buf, \
cstate->raw_buf + cstate->raw_buf_index, \
raw_buf_ptr - cstate->raw_buf_index); \
cstate->raw_buf_index = raw_buf_ptr; \
} \
} else ((void) 0)
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208

Definition at line 268 of file copy.c.

Referenced by CopyReadLineText().

Typedef Documentation

Enumeration Type Documentation

enum CopyDest
Enumerator
COPY_FILE 
COPY_OLD_FE 
COPY_NEW_FE 
COPY_CALLBACK 

Definition at line 60 of file copy.c.

61 {
62  COPY_FILE, /* to/from file (or a piped program) */
63  COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
64  COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
65  COPY_CALLBACK /* to/from callback function */
66 } CopyDest;
Definition: copy.c:62
CopyDest
Definition: copy.c:60
enum EolType
Enumerator
EOL_UNKNOWN 
EOL_NL 
EOL_CR 
EOL_CRNL 

Definition at line 71 of file copy.c.

72 {
74  EOL_NL,
75  EOL_CR,
76  EOL_CRNL
77 } EolType;
Definition: copy.c:74
EolType
Definition: copy.c:71
Definition: copy.c:75
Definition: copy.c:76

Function Documentation

static CopyState BeginCopy ( ParseState pstate,
bool  is_from,
Relation  rel,
RawStmt raw_query,
Oid  queryRelId,
List attnamelist,
List options 
)
static

Definition at line 1372 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), Assert, CopyStateData::attnumlist, tupleDesc::attrs, CMD_DELETE, CMD_INSERT, CMD_SELECT, CMD_UPDATE, Query::commandType, CopyStateData::convert_select, CopyStateData::convert_select_flags, CopyStateData::convert_selectively, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copycontext, CopyGetAttnums(), copyObject, CreateDestReceiver(), CreateQueryDesc(), cur, CurrentMemoryContext, DestCopyOut, CopyStateData::encoding_embeds_ascii, ereport, errcode(), errmsg(), errmsg_internal(), ERROR, ExecSetupPartitionTupleRouting(), ExecutorStart(), CopyStateData::file_encoding, CopyStateData::force_notnull, CopyStateData::force_notnull_flags, CopyStateData::force_null, CopyStateData::force_null_flags, CopyStateData::force_quote, CopyStateData::force_quote_all, CopyStateData::force_quote_flags, GetActiveSnapshot(), GetDatabaseEncoding(), i, InvalidOid, InvalidSnapshot, IsA, lfirst_int, lfirst_node, linitial_node, list_length(), list_member_int(), list_member_oid(), MemoryContextSwitchTo(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NIL, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, CopyStateData::oids, ParseState::p_sourcetext, palloc0(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pg_analyze_and_rewrite(), pg_database_encoding_max_length(), PG_ENCODING_IS_CLIENT_ONLY, pg_get_client_encoding(), pg_plan_query(), ProcessCopyOptions(), PushCopiedSnapshot(), QSRC_NON_INSTEAD_RULE, QSRC_QUAL_INSTEAD_RULE, CopyStateData::queryDesc, Query::querySource, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, PlannedStmt::relationOids, RELKIND_PARTITIONED_TABLE, Query::returningList, QueryDesc::tupDesc, UpdateActiveSnapshotCommandId(), and Query::utilityStmt.

Referenced by BeginCopyFrom(), and BeginCopyTo().

1379 {
1380  CopyState cstate;
1381  TupleDesc tupDesc;
1382  int num_phys_attrs;
1383  MemoryContext oldcontext;
1384 
1385  /* Allocate workspace and zero all fields */
1386  cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
1387 
1388  /*
1389  * We allocate everything used by a cstate in a new memory context. This
1390  * avoids memory leaks during repeated use of COPY in a query.
1391  */
1393  "COPY",
1395 
1396  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1397 
1398  /* Extract options from the statement node tree */
1399  ProcessCopyOptions(pstate, cstate, is_from, options);
1400 
1401  /* Process the source/target relation or query */
1402  if (rel)
1403  {
1404  Assert(!raw_query);
1405 
1406  cstate->rel = rel;
1407 
1408  tupDesc = RelationGetDescr(cstate->rel);
1409 
1410  /* Don't allow COPY w/ OIDs to or from a table without them */
1411  if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
1412  ereport(ERROR,
1413  (errcode(ERRCODE_UNDEFINED_COLUMN),
1414  errmsg("table \"%s\" does not have OIDs",
1415  RelationGetRelationName(cstate->rel))));
1416 
1417  /* Initialize state for CopyFrom tuple routing. */
1418  if (is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1419  {
1420  PartitionDispatch *partition_dispatch_info;
1421  ResultRelInfo *partitions;
1422  TupleConversionMap **partition_tupconv_maps;
1423  TupleTableSlot *partition_tuple_slot;
1424  int num_parted,
1425  num_partitions;
1426 
1428  &partition_dispatch_info,
1429  &partitions,
1430  &partition_tupconv_maps,
1431  &partition_tuple_slot,
1432  &num_parted, &num_partitions);
1433  cstate->partition_dispatch_info = partition_dispatch_info;
1434  cstate->num_dispatch = num_parted;
1435  cstate->partitions = partitions;
1436  cstate->num_partitions = num_partitions;
1437  cstate->partition_tupconv_maps = partition_tupconv_maps;
1438  cstate->partition_tuple_slot = partition_tuple_slot;
1439  }
1440  }
1441  else
1442  {
1443  List *rewritten;
1444  Query *query;
1445  PlannedStmt *plan;
1446  DestReceiver *dest;
1447 
1448  Assert(!is_from);
1449  cstate->rel = NULL;
1450 
1451  /* Don't allow COPY w/ OIDs from a query */
1452  if (cstate->oids)
1453  ereport(ERROR,
1454  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1455  errmsg("COPY (query) WITH OIDS is not supported")));
1456 
1457  /*
1458  * Run parse analysis and rewrite. Note this also acquires sufficient
1459  * locks on the source table(s).
1460  *
1461  * Because the parser and planner tend to scribble on their input, we
1462  * make a preliminary copy of the source querytree. This prevents
1463  * problems in the case that the COPY is in a portal or plpgsql
1464  * function and is executed repeatedly. (See also the same hack in
1465  * DECLARE CURSOR and PREPARE.) XXX FIXME someday.
1466  */
1467  rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
1468  pstate->p_sourcetext, NULL, 0,
1469  NULL);
1470 
1471  /* check that we got back something we can work with */
1472  if (rewritten == NIL)
1473  {
1474  ereport(ERROR,
1475  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1476  errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
1477  }
1478  else if (list_length(rewritten) > 1)
1479  {
1480  ListCell *lc;
1481 
1482  /* examine queries to determine which error message to issue */
1483  foreach(lc, rewritten)
1484  {
1485  Query *q = lfirst_node(Query, lc);
1486 
1488  ereport(ERROR,
1489  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1490  errmsg("conditional DO INSTEAD rules are not supported for COPY")));
1492  ereport(ERROR,
1493  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1494  errmsg("DO ALSO rules are not supported for the COPY")));
1495  }
1496 
1497  ereport(ERROR,
1498  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1499  errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
1500  }
1501 
1502  query = linitial_node(Query, rewritten);
1503 
1504  /* The grammar allows SELECT INTO, but we don't support that */
1505  if (query->utilityStmt != NULL &&
1507  ereport(ERROR,
1508  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1509  errmsg("COPY (SELECT INTO) is not supported")));
1510 
1511  Assert(query->utilityStmt == NULL);
1512 
1513  /*
1514  * Similarly the grammar doesn't enforce the presence of a RETURNING
1515  * clause, but this is required here.
1516  */
1517  if (query->commandType != CMD_SELECT &&
1518  query->returningList == NIL)
1519  {
1520  Assert(query->commandType == CMD_INSERT ||
1521  query->commandType == CMD_UPDATE ||
1522  query->commandType == CMD_DELETE);
1523 
1524  ereport(ERROR,
1525  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1526  errmsg("COPY query must have a RETURNING clause")));
1527  }
1528 
1529  /* plan the query */
1530  plan = pg_plan_query(query, 0, NULL);
1531 
1532  /*
1533  * With row level security and a user using "COPY relation TO", we
1534  * have to convert the "COPY relation TO" to a query-based COPY (eg:
1535  * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
1536  * in any RLS clauses.
1537  *
1538  * When this happens, we are passed in the relid of the originally
1539  * found relation (which we have locked). As the planner will look up
1540  * the relation again, we double-check here to make sure it found the
1541  * same one that we have locked.
1542  */
1543  if (queryRelId != InvalidOid)
1544  {
1545  /*
1546  * Note that with RLS involved there may be multiple relations,
1547  * and while the one we need is almost certainly first, we don't
1548  * make any guarantees of that in the planner, so check the whole
1549  * list and make sure we find the original relation.
1550  */
1551  if (!list_member_oid(plan->relationOids, queryRelId))
1552  ereport(ERROR,
1553  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1554  errmsg("relation referenced by COPY statement has changed")));
1555  }
1556 
1557  /*
1558  * Use a snapshot with an updated command ID to ensure this query sees
1559  * results of any previously executed queries.
1560  */
1563 
1564  /* Create dest receiver for COPY OUT */
1566  ((DR_copy *) dest)->cstate = cstate;
1567 
1568  /* Create a QueryDesc requesting no output */
1569  cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
1572  dest, NULL, NULL, 0);
1573 
1574  /*
1575  * Call ExecutorStart to prepare the plan for execution.
1576  *
1577  * ExecutorStart computes a result tupdesc for us
1578  */
1579  ExecutorStart(cstate->queryDesc, 0);
1580 
1581  tupDesc = cstate->queryDesc->tupDesc;
1582  }
1583 
1584  /* Generate or convert list of attributes to process */
1585  cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
1586 
1587  num_phys_attrs = tupDesc->natts;
1588 
1589  /* Convert FORCE_QUOTE name list to per-column flags, check validity */
1590  cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1591  if (cstate->force_quote_all)
1592  {
1593  int i;
1594 
1595  for (i = 0; i < num_phys_attrs; i++)
1596  cstate->force_quote_flags[i] = true;
1597  }
1598  else if (cstate->force_quote)
1599  {
1600  List *attnums;
1601  ListCell *cur;
1602 
1603  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
1604 
1605  foreach(cur, attnums)
1606  {
1607  int attnum = lfirst_int(cur);
1608 
1609  if (!list_member_int(cstate->attnumlist, attnum))
1610  ereport(ERROR,
1611  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1612  errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
1613  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1614  cstate->force_quote_flags[attnum - 1] = true;
1615  }
1616  }
1617 
1618  /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
1619  cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1620  if (cstate->force_notnull)
1621  {
1622  List *attnums;
1623  ListCell *cur;
1624 
1625  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
1626 
1627  foreach(cur, attnums)
1628  {
1629  int attnum = lfirst_int(cur);
1630 
1631  if (!list_member_int(cstate->attnumlist, attnum))
1632  ereport(ERROR,
1633  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1634  errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
1635  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1636  cstate->force_notnull_flags[attnum - 1] = true;
1637  }
1638  }
1639 
1640  /* Convert FORCE_NULL name list to per-column flags, check validity */
1641  cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1642  if (cstate->force_null)
1643  {
1644  List *attnums;
1645  ListCell *cur;
1646 
1647  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
1648 
1649  foreach(cur, attnums)
1650  {
1651  int attnum = lfirst_int(cur);
1652 
1653  if (!list_member_int(cstate->attnumlist, attnum))
1654  ereport(ERROR,
1655  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1656  errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
1657  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1658  cstate->force_null_flags[attnum - 1] = true;
1659  }
1660  }
1661 
1662  /* Convert convert_selectively name list to per-column flags */
1663  if (cstate->convert_selectively)
1664  {
1665  List *attnums;
1666  ListCell *cur;
1667 
1668  cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
1669 
1670  attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
1671 
1672  foreach(cur, attnums)
1673  {
1674  int attnum = lfirst_int(cur);
1675 
1676  if (!list_member_int(cstate->attnumlist, attnum))
1677  ereport(ERROR,
1678  (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
1679  errmsg_internal("selected column \"%s\" not referenced by COPY",
1680  NameStr(tupDesc->attrs[attnum - 1]->attname))));
1681  cstate->convert_select_flags[attnum - 1] = true;
1682  }
1683  }
1684 
1685  /* Use client encoding when ENCODING option is not specified. */
1686  if (cstate->file_encoding < 0)
1688 
1689  /*
1690  * Set up encoding conversion info. Even if the file and server encodings
1691  * are the same, we must apply pg_any_to_server() to validate data in
1692  * multibyte encodings.
1693  */
1694  cstate->need_transcoding =
1695  (cstate->file_encoding != GetDatabaseEncoding() ||
1697  /* See Multibyte encoding comment above */
1699 
1700  cstate->copy_dest = COPY_FILE; /* default */
1701 
1702  MemoryContextSwitchTo(oldcontext);
1703 
1704  return cstate;
1705 }
#define NIL
Definition: pg_list.h:69
void UpdateActiveSnapshotCommandId(void)
Definition: snapmgr.c:776
#define IsA(nodeptr, _type_)
Definition: nodes.h:560
#define RelationGetDescr(relation)
Definition: rel.h:429
bool need_transcoding
Definition: copy.c:105
ResultRelInfo * partitions
Definition: copy.c:171
List * attnumlist
Definition: copy.c:111
List * relationOids
Definition: plannodes.h:88
Definition: copy.c:215
void ExecutorStart(QueryDesc *queryDesc, int eflags)
Definition: execMain.c:144
Form_pg_attribute * attrs
Definition: tupdesc.h:74
void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options)
Definition: copy.c:1014
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:834
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:575
bool * force_quote_flags
Definition: copy.c:128
Form_pg_class rd_rel
Definition: rel.h:114
List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, Oid *paramTypes, int numParams, QueryEnvironment *queryEnv)
Definition: postgres.c:644
Node * utilityStmt
Definition: parsenodes.h:118
#define linitial_node(type, l)
Definition: pg_list.h:114
bool * force_null_flags
Definition: copy.c:132
int natts
Definition: tupdesc.h:73
bool * convert_select_flags
Definition: copy.c:135
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
Relation rel
Definition: copy.c:109
MemoryContext copycontext
Definition: copy.c:146
void ExecSetupPartitionTupleRouting(Relation rel, PartitionDispatch **pd, ResultRelInfo **partitions, TupleConversionMap ***tup_conv_maps, TupleTableSlot **partition_tuple_slot, int *num_parted, int *num_partitions)
Definition: execMain.c:3224
TupleTableSlot * partition_tuple_slot
Definition: copy.c:173
#define ERROR
Definition: elog.h:43
#define lfirst_int(lc)
Definition: pg_list.h:107
void PushCopiedSnapshot(Snapshot snapshot)
Definition: snapmgr.c:764
QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, QueryEnvironment *queryEnv, int instrument_options)
Definition: pquery.c:67
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
#define lfirst_node(type, lc)
Definition: pg_list.h:109
QueryDesc * queryDesc
Definition: copy.c:110
bool list_member_int(const List *list, int datum)
Definition: list.c:485
int num_dispatch
Definition: copy.c:169
bool encoding_embeds_ascii
Definition: copy.c:106
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
DestReceiver * CreateDestReceiver(CommandDest dest)
Definition: dest.c:109
#define RelationGetRelationName(relation)
Definition: rel.h:437
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
const char * p_sourcetext
Definition: parse_node.h:168
List * returningList
Definition: parsenodes.h:144
#define ereport(elevel, rest)
Definition: elog.h:122
List * force_null
Definition: copy.c:131
int file_encoding
Definition: copy.c:104
static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
Definition: copy.c:4624
TupleDesc tupDesc
Definition: execdesc.h:47
#define InvalidSnapshot
Definition: snapshot.h:25
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
void * palloc0(Size size)
Definition: mcxt.c:878
int GetDatabaseEncoding(void)
Definition: mbutils.c:1015
int pg_get_client_encoding(void)
Definition: mbutils.c:317
#define InvalidOid
Definition: postgres_ext.h:36
bool * force_notnull_flags
Definition: copy.c:130
CmdType commandType
Definition: parsenodes.h:110
List * force_notnull
Definition: copy.c:129
int errmsg_internal(const char *fmt,...)
Definition: elog.c:827
bool list_member_oid(const List *list, Oid datum)
Definition: list.c:505
#define NULL
Definition: c.h:229
QuerySource querySource
Definition: parsenodes.h:112
#define Assert(condition)
Definition: c.h:675
List * convert_select
Definition: copy.c:134
static int list_length(const List *l)
Definition: pg_list.h:89
bool force_quote_all
Definition: copy.c:127
#define PG_ENCODING_IS_CLIENT_ONLY(_enc)
Definition: pg_wchar.h:296
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
#define NameStr(name)
Definition: c.h:499
PartitionDispatch * partition_dispatch_info
Definition: copy.c:168
#define copyObject(obj)
Definition: nodes.h:621
TupleConversionMap ** partition_tupconv_maps
Definition: copy.c:172
Definition: pg_list.h:45
int num_partitions
Definition: copy.c:170
List * force_quote
Definition: copy.c:126
bool convert_selectively
Definition: copy.c:133
bool oids
Definition: copy.c:116
PlannedStmt * pg_plan_query(Query *querytree, int cursorOptions, ParamListInfo boundParams)
Definition: postgres.c:785
CopyState BeginCopyFrom ( ParseState pstate,
Relation  rel,
const char *  filename,
bool  is_program,
copy_data_source_cb  data_source_cb,
List attnamelist,
List options 
)

Definition at line 2880 of file copy.c.

References AllocateFile(), Assert, CopyStateData::attnumlist, CopyStateData::attribute_buf, tupleDesc::attrs, BeginCopy(), CopyStateData::binary, BinarySignature, build_column_default(), contain_volatile_functions_not_nextval(), COPY_CALLBACK, CopyStateData::copy_dest, CopyStateData::copy_file, CopyStateData::copycontext, CopyGetData(), CopyGetInt32(), CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, CopyStateData::data_source_cb, CopyStateData::defexprs, CopyStateData::defmap, DestRemote, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, ExecInitExpr(), expression_planner(), CopyStateData::fe_eof, CopyStateData::file_has_oids, CopyStateData::filename, fmgr_info(), getTypeBinaryInputInfo(), getTypeInputInfo(), CopyStateData::in_functions, initStringInfo(), InvalidOid, CopyStateData::is_program, CopyStateData::line_buf, CopyStateData::line_buf_converted, list_length(), list_member_int(), CopyStateData::max_fields, MemoryContextSwitchTo(), tupleDesc::natts, NULL, CopyStateData::num_defaults, CopyStateData::oid_in_function, CopyStateData::oid_typioparam, OIDOID, CopyStateData::oids, OpenPipeStream(), ParseState::p_rtable, palloc(), PG_BINARY_R, pstrdup(), CopyStateData::range_table, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, RAW_BUF_SIZE, CopyStateData::raw_fields, ReceiveCopyBegin(), CopyStateData::rel, RelationGetDescr, RelationGetRelationName, CopyStateData::typioparams, CopyStateData::volatile_defexprs, and whereToSendOutput.

Referenced by copy_table(), DoCopy(), file_acquire_sample_rows(), fileBeginForeignScan(), and fileReScanForeignScan().

2887 {
2888  CopyState cstate;
2889  bool pipe = (filename == NULL);
2890  TupleDesc tupDesc;
2891  Form_pg_attribute *attr;
2892  AttrNumber num_phys_attrs,
2893  num_defaults;
2894  FmgrInfo *in_functions;
2895  Oid *typioparams;
2896  int attnum;
2897  Oid in_func_oid;
2898  int *defmap;
2899  ExprState **defexprs;
2900  MemoryContext oldcontext;
2901  bool volatile_defexprs;
2902 
2903  cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
2904  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
2905 
2906  /* Initialize state variables */
2907  cstate->fe_eof = false;
2908  cstate->eol_type = EOL_UNKNOWN;
2909  cstate->cur_relname = RelationGetRelationName(cstate->rel);
2910  cstate->cur_lineno = 0;
2911  cstate->cur_attname = NULL;
2912  cstate->cur_attval = NULL;
2913 
2914  /* Set up variables to avoid per-attribute overhead. */
2915  initStringInfo(&cstate->attribute_buf);
2916  initStringInfo(&cstate->line_buf);
2917  cstate->line_buf_converted = false;
2918  cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
2919  cstate->raw_buf_index = cstate->raw_buf_len = 0;
2920 
2921  /* Assign range table, we'll need it in CopyFrom. */
2922  if (pstate)
2923  cstate->range_table = pstate->p_rtable;
2924 
2925  tupDesc = RelationGetDescr(cstate->rel);
2926  attr = tupDesc->attrs;
2927  num_phys_attrs = tupDesc->natts;
2928  num_defaults = 0;
2929  volatile_defexprs = false;
2930 
2931  /*
2932  * Pick up the required catalog information for each attribute in the
2933  * relation, including the input function, the element type (to pass to
2934  * the input function), and info about defaults and constraints. (Which
2935  * input function we use depends on text/binary format choice.)
2936  */
2937  in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
2938  typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
2939  defmap = (int *) palloc(num_phys_attrs * sizeof(int));
2940  defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
2941 
2942  for (attnum = 1; attnum <= num_phys_attrs; attnum++)
2943  {
2944  /* We don't need info for dropped attributes */
2945  if (attr[attnum - 1]->attisdropped)
2946  continue;
2947 
2948  /* Fetch the input function and typioparam info */
2949  if (cstate->binary)
2950  getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
2951  &in_func_oid, &typioparams[attnum - 1]);
2952  else
2953  getTypeInputInfo(attr[attnum - 1]->atttypid,
2954  &in_func_oid, &typioparams[attnum - 1]);
2955  fmgr_info(in_func_oid, &in_functions[attnum - 1]);
2956 
2957  /* Get default info if needed */
2958  if (!list_member_int(cstate->attnumlist, attnum))
2959  {
2960  /* attribute is NOT to be copied from input */
2961  /* use default value if one exists */
2962  Expr *defexpr = (Expr *) build_column_default(cstate->rel,
2963  attnum);
2964 
2965  if (defexpr != NULL)
2966  {
2967  /* Run the expression through planner */
2968  defexpr = expression_planner(defexpr);
2969 
2970  /* Initialize executable expression in copycontext */
2971  defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
2972  defmap[num_defaults] = attnum - 1;
2973  num_defaults++;
2974 
2975  /*
2976  * If a default expression looks at the table being loaded,
2977  * then it could give the wrong answer when using
2978  * multi-insert. Since database access can be dynamic this is
2979  * hard to test for exactly, so we use the much wider test of
2980  * whether the default expression is volatile. We allow for
2981  * the special case of when the default expression is the
2982  * nextval() of a sequence which in this specific case is
2983  * known to be safe for use with the multi-insert
2984  * optimization. Hence we use this special case function
2985  * checker rather than the standard check for
2986  * contain_volatile_functions().
2987  */
2988  if (!volatile_defexprs)
2989  volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
2990  }
2991  }
2992  }
2993 
2994  /* We keep those variables in cstate. */
2995  cstate->in_functions = in_functions;
2996  cstate->typioparams = typioparams;
2997  cstate->defmap = defmap;
2998  cstate->defexprs = defexprs;
2999  cstate->volatile_defexprs = volatile_defexprs;
3000  cstate->num_defaults = num_defaults;
3001  cstate->is_program = is_program;
3002 
3003  if (data_source_cb)
3004  {
3005  cstate->copy_dest = COPY_CALLBACK;
3006  cstate->data_source_cb = data_source_cb;
3007  }
3008  else if (pipe)
3009  {
3010  Assert(!is_program); /* the grammar does not allow this */
3012  ReceiveCopyBegin(cstate);
3013  else
3014  cstate->copy_file = stdin;
3015  }
3016  else
3017  {
3018  cstate->filename = pstrdup(filename);
3019 
3020  if (cstate->is_program)
3021  {
3022  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
3023  if (cstate->copy_file == NULL)
3024  ereport(ERROR,
3026  errmsg("could not execute command \"%s\": %m",
3027  cstate->filename)));
3028  }
3029  else
3030  {
3031  struct stat st;
3032 
3033  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
3034  if (cstate->copy_file == NULL)
3035  {
3036  /* copy errno because ereport subfunctions might change it */
3037  int save_errno = errno;
3038 
3039  ereport(ERROR,
3041  errmsg("could not open file \"%s\" for reading: %m",
3042  cstate->filename),
3043  (save_errno == ENOENT || save_errno == EACCES) ?
3044  errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
3045  "You may want a client-side facility such as psql's \\copy.") : 0));
3046  }
3047 
3048  if (fstat(fileno(cstate->copy_file), &st))
3049  ereport(ERROR,
3051  errmsg("could not stat file \"%s\": %m",
3052  cstate->filename)));
3053 
3054  if (S_ISDIR(st.st_mode))
3055  ereport(ERROR,
3056  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
3057  errmsg("\"%s\" is a directory", cstate->filename)));
3058  }
3059  }
3060 
3061  if (!cstate->binary)
3062  {
3063  /* must rely on user to tell us... */
3064  cstate->file_has_oids = cstate->oids;
3065  }
3066  else
3067  {
3068  /* Read and verify binary header */
3069  char readSig[11];
3070  int32 tmp;
3071 
3072  /* Signature */
3073  if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
3074  memcmp(readSig, BinarySignature, 11) != 0)
3075  ereport(ERROR,
3076  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3077  errmsg("COPY file signature not recognized")));
3078  /* Flags field */
3079  if (!CopyGetInt32(cstate, &tmp))
3080  ereport(ERROR,
3081  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3082  errmsg("invalid COPY file header (missing flags)")));
3083  cstate->file_has_oids = (tmp & (1 << 16)) != 0;
3084  tmp &= ~(1 << 16);
3085  if ((tmp >> 16) != 0)
3086  ereport(ERROR,
3087  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3088  errmsg("unrecognized critical flags in COPY file header")));
3089  /* Header extension length */
3090  if (!CopyGetInt32(cstate, &tmp) ||
3091  tmp < 0)
3092  ereport(ERROR,
3093  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3094  errmsg("invalid COPY file header (missing length)")));
3095  /* Skip extension header, if present */
3096  while (tmp-- > 0)
3097  {
3098  if (CopyGetData(cstate, readSig, 1, 1) != 1)
3099  ereport(ERROR,
3100  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3101  errmsg("invalid COPY file header (wrong length)")));
3102  }
3103  }
3104 
3105  if (cstate->file_has_oids && cstate->binary)
3106  {
3108  &in_func_oid, &cstate->oid_typioparam);
3109  fmgr_info(in_func_oid, &cstate->oid_in_function);
3110  }
3111 
3112  /* create workspace for CopyReadAttributes results */
3113  if (!cstate->binary)
3114  {
3115  AttrNumber attr_count = list_length(cstate->attnumlist);
3116  int nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;
3117 
3118  cstate->max_fields = nfields;
3119  cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
3120  }
3121 
3122  MemoryContextSwitchTo(oldcontext);
3123 
3124  return cstate;
3125 }
Definition: fmgr.h:56
List * range_table
Definition: copy.c:166
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1372
bool contain_volatile_functions_not_nextval(Node *clause)
Definition: clauses.c:995
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:682
int errhint(const char *fmt,...)
Definition: elog.c:987
char ** raw_fields
Definition: copy.c:188
bool binary
Definition: copy.c:115
#define RelationGetDescr(relation)
Definition: rel.h:429
AttrNumber num_defaults
Definition: copy.c:157
FmgrInfo * in_functions
Definition: copy.c:161
List * attnumlist
Definition: copy.c:111
#define OIDOID
Definition: pg_type.h:328
char * filename
Definition: copy.c:112
bool file_has_oids
Definition: copy.c:158
char * pstrdup(const char *in)
Definition: mcxt.c:1077
static void ReceiveCopyBegin(CopyState cstate)
Definition: copy.c:380
Expr * expression_planner(Expr *expr)
Definition: planner.c:5931
Form_pg_attribute * attrs
Definition: tupdesc.h:74
StringInfoData line_buf
Definition: copy.c:197
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Definition: nodes.h:509
int raw_buf_index
Definition: copy.c:210
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:102
unsigned int Oid
Definition: postgres_ext.h:31
bool volatile_defexprs
Definition: copy.c:165
#define PG_BINARY_R
Definition: c.h:1040
int natts
Definition: tupdesc.h:73
bool line_buf_converted
Definition: copy.c:198
signed int int32
Definition: c.h:256
CopyDest copy_dest
Definition: copy.c:98
const char * cur_attname
Definition: copy.c:140
Relation rel
Definition: copy.c:109
MemoryContext copycontext
Definition: copy.c:146
copy_data_source_cb data_source_cb
Definition: copy.c:114
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:127
bool list_member_int(const List *list, int datum)
Definition: list.c:485
char * raw_buf
Definition: copy.c:209
ExprState ** defexprs
Definition: copy.c:164
const char * cur_relname
Definition: copy.c:138
int errcode_for_file_access(void)
Definition: elog.c:598
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
FmgrInfo oid_in_function
Definition: copy.c:159
#define RelationGetRelationName(relation)
Definition: rel.h:437
static const char BinarySignature[11]
Definition: copy.c:288
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
int raw_buf_len
Definition: copy.c:211
int max_fields
Definition: copy.c:187
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2184
void getTypeBinaryInputInfo(Oid type, Oid *typReceive, Oid *typIOParam)
Definition: lsyscache.c:2665
#define ereport(elevel, rest)
Definition: elog.h:122
void getTypeInputInfo(Oid type, Oid *typInput, Oid *typIOParam)
Definition: lsyscache.c:2599
void initStringInfo(StringInfo str)
Definition: stringinfo.c:46
Oid * typioparams
Definition: copy.c:162
bool is_program
Definition: copy.c:113
Node * build_column_default(Relation rel, int attrno)
#define RAW_BUF_SIZE
Definition: copy.c:208
#define InvalidOid
Definition: postgres_ext.h:36
EolType eol_type
Definition: copy.c:103
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
static int list_length(const List *l)
Definition: pg_list.h:89
static char * filename
Definition: pg_dumpall.c:89
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
int cur_lineno
Definition: copy.c:139
StringInfoData attribute_buf
Definition: copy.c:183
ExprState * ExecInitExpr(Expr *node, PlanState *parent)
Definition: execExpr.c:113
const char * cur_attval
Definition: copy.c:141
CommandDest whereToSendOutput
Definition: postgres.c:86
Oid oid_typioparam
Definition: copy.c:160
int16 AttrNumber
Definition: attnum.h:21
int * defmap
Definition: copy.c:163
List * p_rtable
Definition: parse_node.h:169
bool oids
Definition: copy.c:116
static CopyState BeginCopyTo ( ParseState pstate,
Relation  rel,
RawStmt query,
Oid  queryRelId,
const char *  filename,
bool  is_program,
List attnamelist,
List options 
)
static

Definition at line 1757 of file copy.c.

References AllocateFile(), Assert, BeginCopy(), CopyStateData::copy_file, CopyStateData::copycontext, DestRemote, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, CopyStateData::filename, is_absolute_path, CopyStateData::is_program, MemoryContextSwitchTo(), NULL, OpenPipeStream(), PG_BINARY_W, pstrdup(), RelationData::rd_rel, RelationGetRelationName, RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, S_IWGRP, S_IWOTH, and whereToSendOutput.

Referenced by DoCopy().

1765 {
1766  CopyState cstate;
1767  bool pipe = (filename == NULL);
1768  MemoryContext oldcontext;
1769 
1770  if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
1771  {
1772  if (rel->rd_rel->relkind == RELKIND_VIEW)
1773  ereport(ERROR,
1774  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1775  errmsg("cannot copy from view \"%s\"",
1777  errhint("Try the COPY (SELECT ...) TO variant.")));
1778  else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
1779  ereport(ERROR,
1780  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1781  errmsg("cannot copy from materialized view \"%s\"",
1783  errhint("Try the COPY (SELECT ...) TO variant.")));
1784  else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
1785  ereport(ERROR,
1786  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1787  errmsg("cannot copy from foreign table \"%s\"",
1789  errhint("Try the COPY (SELECT ...) TO variant.")));
1790  else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
1791  ereport(ERROR,
1792  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1793  errmsg("cannot copy from sequence \"%s\"",
1794  RelationGetRelationName(rel))));
1795  else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
1796  ereport(ERROR,
1797  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1798  errmsg("cannot copy from partitioned table \"%s\"",
1800  errhint("Try the COPY (SELECT ...) TO variant.")));
1801  else
1802  ereport(ERROR,
1803  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1804  errmsg("cannot copy from non-table relation \"%s\"",
1805  RelationGetRelationName(rel))));
1806  }
1807 
1808  cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
1809  options);
1810  oldcontext = MemoryContextSwitchTo(cstate->copycontext);
1811 
1812  if (pipe)
1813  {
1814  Assert(!is_program); /* the grammar does not allow this */
1816  cstate->copy_file = stdout;
1817  }
1818  else
1819  {
1820  cstate->filename = pstrdup(filename);
1821  cstate->is_program = is_program;
1822 
1823  if (is_program)
1824  {
1825  cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
1826  if (cstate->copy_file == NULL)
1827  ereport(ERROR,
1829  errmsg("could not execute command \"%s\": %m",
1830  cstate->filename)));
1831  }
1832  else
1833  {
1834  mode_t oumask; /* Pre-existing umask value */
1835  struct stat st;
1836 
1837  /*
1838  * Prevent write to relative path ... too easy to shoot oneself in
1839  * the foot by overwriting a database file ...
1840  */
1841  if (!is_absolute_path(filename))
1842  ereport(ERROR,
1843  (errcode(ERRCODE_INVALID_NAME),
1844  errmsg("relative path not allowed for COPY to file")));
1845 
1846  oumask = umask(S_IWGRP | S_IWOTH);
1847  cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
1848  umask(oumask);
1849  if (cstate->copy_file == NULL)
1850  {
1851  /* copy errno because ereport subfunctions might change it */
1852  int save_errno = errno;
1853 
1854  ereport(ERROR,
1856  errmsg("could not open file \"%s\" for writing: %m",
1857  cstate->filename),
1858  (save_errno == ENOENT || save_errno == EACCES) ?
1859  errhint("COPY TO instructs the PostgreSQL server process to write a file. "
1860  "You may want a client-side facility such as psql's \\copy.") : 0));
1861  }
1862 
1863  if (fstat(fileno(cstate->copy_file), &st))
1864  ereport(ERROR,
1866  errmsg("could not stat file \"%s\": %m",
1867  cstate->filename)));
1868 
1869  if (S_ISDIR(st.st_mode))
1870  ereport(ERROR,
1871  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
1872  errmsg("\"%s\" is a directory", cstate->filename)));
1873  }
1874  }
1875 
1876  MemoryContextSwitchTo(oldcontext);
1877 
1878  return cstate;
1879 }
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
Definition: copy.c:1372
int errhint(const char *fmt,...)
Definition: elog.c:987
char * filename
Definition: copy.c:112
char * pstrdup(const char *in)
Definition: mcxt.c:1077
#define RELKIND_MATVIEW
Definition: pg_class.h:165
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
int errcode(int sqlerrcode)
Definition: elog.c:575
#define PG_BINARY_W
Definition: c.h:1041
Form_pg_class rd_rel
Definition: rel.h:114
MemoryContext copycontext
Definition: copy.c:146
#define ERROR
Definition: elog.h:43
#define S_IWGRP
Definition: win32.h:449
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2094
#define RelationGetRelationName(relation)
Definition: rel.h:437
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
FILE * OpenPipeStream(const char *command, const char *mode)
Definition: fd.c:2184
#define ereport(elevel, rest)
Definition: elog.h:122
#define S_IWOTH
Definition: win32.h:453
bool is_program
Definition: copy.c:113
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
static char * filename
Definition: pg_dumpall.c:89
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
FILE * copy_file
Definition: copy.c:99
CommandDest whereToSendOutput
Definition: postgres.c:86
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
static void ClosePipeToProgram ( CopyState  cstate)
static

Definition at line 1711 of file copy.c.

References Assert, ClosePipeStream(), CopyStateData::copy_file, ereport, errcode(), errcode_for_file_access(), errdetail_internal(), errmsg(), ERROR, CopyStateData::filename, CopyStateData::is_program, and wait_result_to_str().

Referenced by CopySendEndOfRow(), and EndCopy().

1712 {
1713  int pclose_rc;
1714 
1715  Assert(cstate->is_program);
1716 
1717  pclose_rc = ClosePipeStream(cstate->copy_file);
1718  if (pclose_rc == -1)
1719  ereport(ERROR,
1721  errmsg("could not close pipe to external command: %m")));
1722  else if (pclose_rc != 0)
1723  ereport(ERROR,
1724  (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
1725  errmsg("program \"%s\" failed",
1726  cstate->filename),
1727  errdetail_internal("%s", wait_result_to_str(pclose_rc))));
1728 }
char * filename
Definition: copy.c:112
int errcode(int sqlerrcode)
Definition: elog.c:575
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
int ClosePipeStream(FILE *file)
Definition: fd.c:2470
int errdetail_internal(const char *fmt,...)
Definition: elog.c:900
#define ERROR
Definition: elog.h:43
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:113
#define Assert(condition)
Definition: c.h:675
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
static void copy_dest_destroy ( DestReceiver self)
static

Definition at line 4733 of file copy.c.

References pfree().

Referenced by CreateCopyDestReceiver().

4734 {
4735  pfree(self);
4736 }
void pfree(void *pointer)
Definition: mcxt.c:950
static bool copy_dest_receive ( TupleTableSlot slot,
DestReceiver self 
)
static

Definition at line 4705 of file copy.c.

References CopyOneRowTo(), DR_copy::cstate, InvalidOid, DR_copy::processed, slot_getallattrs(), TupleTableSlot::tts_isnull, and TupleTableSlot::tts_values.

Referenced by CreateCopyDestReceiver().

4706 {
4707  DR_copy *myState = (DR_copy *) self;
4708  CopyState cstate = myState->cstate;
4709 
4710  /* Make sure the tuple is fully deconstructed */
4711  slot_getallattrs(slot);
4712 
4713  /* And send the data */
4714  CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
4715  myState->processed++;
4716 
4717  return true;
4718 }
Definition: copy.c:215
CopyState cstate
Definition: copy.c:218
Datum * tts_values
Definition: tuptable.h:125
bool * tts_isnull
Definition: tuptable.h:126
void slot_getallattrs(TupleTableSlot *slot)
Definition: heaptuple.c:1237
#define InvalidOid
Definition: postgres_ext.h:36
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2093
uint64 processed
Definition: copy.c:219
static void copy_dest_shutdown ( DestReceiver self)
static

Definition at line 4724 of file copy.c.

Referenced by CreateCopyDestReceiver().

4725 {
4726  /* no-op */
4727 }
static void copy_dest_startup ( DestReceiver self,
int  operation,
TupleDesc  typeinfo 
)
static

Definition at line 4696 of file copy.c.

Referenced by CreateCopyDestReceiver().

4697 {
4698  /* no-op */
4699 }
static void CopyAttributeOutCSV ( CopyState  cstate,
char *  string,
bool  use_quote,
bool  single_attr 
)
static

Definition at line 4533 of file copy.c.

References CopySendChar(), CopySendString(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::escape, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, CopyStateData::null_print, pg_encoding_mblen(), pg_server_to_any(), and CopyStateData::quote.

Referenced by CopyOneRowTo(), and CopyTo().

4535 {
4536  char *ptr;
4537  char *start;
4538  char c;
4539  char delimc = cstate->delim[0];
4540  char quotec = cstate->quote[0];
4541  char escapec = cstate->escape[0];
4542 
4543  /* force quoting if it matches null_print (before conversion!) */
4544  if (!use_quote && strcmp(string, cstate->null_print) == 0)
4545  use_quote = true;
4546 
4547  if (cstate->need_transcoding)
4548  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4549  else
4550  ptr = string;
4551 
4552  /*
4553  * Make a preliminary pass to discover if it needs quoting
4554  */
4555  if (!use_quote)
4556  {
4557  /*
4558  * Because '\.' can be a data value, quote it if it appears alone on a
4559  * line so it is not interpreted as the end-of-data marker.
4560  */
4561  if (single_attr && strcmp(ptr, "\\.") == 0)
4562  use_quote = true;
4563  else
4564  {
4565  char *tptr = ptr;
4566 
4567  while ((c = *tptr) != '\0')
4568  {
4569  if (c == delimc || c == quotec || c == '\n' || c == '\r')
4570  {
4571  use_quote = true;
4572  break;
4573  }
4574  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4575  tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
4576  else
4577  tptr++;
4578  }
4579  }
4580  }
4581 
4582  if (use_quote)
4583  {
4584  CopySendChar(cstate, quotec);
4585 
4586  /*
4587  * We adopt the same optimization strategy as in CopyAttributeOutText
4588  */
4589  start = ptr;
4590  while ((c = *ptr) != '\0')
4591  {
4592  if (c == quotec || c == escapec)
4593  {
4594  DUMPSOFAR();
4595  CopySendChar(cstate, escapec);
4596  start = ptr; /* we include char in next run */
4597  }
4598  if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
4599  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4600  else
4601  ptr++;
4602  }
4603  DUMPSOFAR();
4604 
4605  CopySendChar(cstate, quotec);
4606  }
4607  else
4608  {
4609  /* If it doesn't need quoting, we can just dump it as-is */
4610  CopySendString(cstate, ptr);
4611  }
4612 }
bool need_transcoding
Definition: copy.c:105
char * delim
Definition: copy.c:123
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * null_print
Definition: copy.c:120
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
char * quote
Definition: copy.c:124
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
char * escape
Definition: copy.c:125
int file_encoding
Definition: copy.c:104
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
#define DUMPSOFAR()
Definition: copy.c:4373
static void CopyAttributeOutText ( CopyState  cstate,
char *  string 
)
static

Definition at line 4380 of file copy.c.

References CopySendChar(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, pg_encoding_mblen(), and pg_server_to_any().

Referenced by CopyOneRowTo().

4381 {
4382  char *ptr;
4383  char *start;
4384  char c;
4385  char delimc = cstate->delim[0];
4386 
4387  if (cstate->need_transcoding)
4388  ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
4389  else
4390  ptr = string;
4391 
4392  /*
4393  * We have to grovel through the string searching for control characters
4394  * and instances of the delimiter character. In most cases, though, these
4395  * are infrequent. To avoid overhead from calling CopySendData once per
4396  * character, we dump out all characters between escaped characters in a
4397  * single call. The loop invariant is that the data from "start" to "ptr"
4398  * can be sent literally, but hasn't yet been.
4399  *
4400  * We can skip pg_encoding_mblen() overhead when encoding is safe, because
4401  * in valid backend encodings, extra bytes of a multibyte character never
4402  * look like ASCII. This loop is sufficiently performance-critical that
4403  * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
4404  * of the normal safe-encoding path.
4405  */
4406  if (cstate->encoding_embeds_ascii)
4407  {
4408  start = ptr;
4409  while ((c = *ptr) != '\0')
4410  {
4411  if ((unsigned char) c < (unsigned char) 0x20)
4412  {
4413  /*
4414  * \r and \n must be escaped, the others are traditional. We
4415  * prefer to dump these using the C-like notation, rather than
4416  * a backslash and the literal character, because it makes the
4417  * dump file a bit more proof against Microsoftish data
4418  * mangling.
4419  */
4420  switch (c)
4421  {
4422  case '\b':
4423  c = 'b';
4424  break;
4425  case '\f':
4426  c = 'f';
4427  break;
4428  case '\n':
4429  c = 'n';
4430  break;
4431  case '\r':
4432  c = 'r';
4433  break;
4434  case '\t':
4435  c = 't';
4436  break;
4437  case '\v':
4438  c = 'v';
4439  break;
4440  default:
4441  /* If it's the delimiter, must backslash it */
4442  if (c == delimc)
4443  break;
4444  /* All ASCII control chars are length 1 */
4445  ptr++;
4446  continue; /* fall to end of loop */
4447  }
4448  /* if we get here, we need to convert the control char */
4449  DUMPSOFAR();
4450  CopySendChar(cstate, '\\');
4451  CopySendChar(cstate, c);
4452  start = ++ptr; /* do not include char in next run */
4453  }
4454  else if (c == '\\' || c == delimc)
4455  {
4456  DUMPSOFAR();
4457  CopySendChar(cstate, '\\');
4458  start = ptr++; /* we include char in next run */
4459  }
4460  else if (IS_HIGHBIT_SET(c))
4461  ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
4462  else
4463  ptr++;
4464  }
4465  }
4466  else
4467  {
4468  start = ptr;
4469  while ((c = *ptr) != '\0')
4470  {
4471  if ((unsigned char) c < (unsigned char) 0x20)
4472  {
4473  /*
4474  * \r and \n must be escaped, the others are traditional. We
4475  * prefer to dump these using the C-like notation, rather than
4476  * a backslash and the literal character, because it makes the
4477  * dump file a bit more proof against Microsoftish data
4478  * mangling.
4479  */
4480  switch (c)
4481  {
4482  case '\b':
4483  c = 'b';
4484  break;
4485  case '\f':
4486  c = 'f';
4487  break;
4488  case '\n':
4489  c = 'n';
4490  break;
4491  case '\r':
4492  c = 'r';
4493  break;
4494  case '\t':
4495  c = 't';
4496  break;
4497  case '\v':
4498  c = 'v';
4499  break;
4500  default:
4501  /* If it's the delimiter, must backslash it */
4502  if (c == delimc)
4503  break;
4504  /* All ASCII control chars are length 1 */
4505  ptr++;
4506  continue; /* fall to end of loop */
4507  }
4508  /* if we get here, we need to convert the control char */
4509  DUMPSOFAR();
4510  CopySendChar(cstate, '\\');
4511  CopySendChar(cstate, c);
4512  start = ++ptr; /* do not include char in next run */
4513  }
4514  else if (c == '\\' || c == delimc)
4515  {
4516  DUMPSOFAR();
4517  CopySendChar(cstate, '\\');
4518  start = ptr++; /* we include char in next run */
4519  }
4520  else
4521  ptr++;
4522  }
4523  }
4524 
4525  DUMPSOFAR();
4526 }
bool need_transcoding
Definition: copy.c:105
char * delim
Definition: copy.c:123
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
char string[11]
Definition: preproc-type.c:46
int file_encoding
Definition: copy.c:104
#define DUMPSOFAR()
Definition: copy.c:4373
uint64 CopyFrom ( CopyState  cstate)

Definition at line 2292 of file copy.c.

References AfterTriggerBeginQuery(), AfterTriggerEndQuery(), ErrorContextCallback::arg, Assert, ErrorContextCallback::callback, CHECK_FOR_INTERRUPTS, tupleDesc::constr, CopyStateData::copy_dest, COPY_OLD_FE, CopyFromErrorCallback(), CopyFromInsertBatch(), CreateExecutorState(), CopyStateData::cur_lineno, CurrentMemoryContext, do_convert_tuple(), ereport, errcode(), errhint(), errmsg(), ERROR, error_context_stack, EState::es_num_result_relations, EState::es_range_table, EState::es_result_relation_info, EState::es_result_relations, EState::es_trig_tuple_slot, EState::es_tupleTable, ExecARInsertTriggers(), ExecASInsertTriggers(), ExecBRInsertTriggers(), ExecBSInsertTriggers(), ExecCleanUpTriggerState(), ExecCloseIndices(), ExecConstraints(), ExecDropSingleTupleTableSlot(), ExecFindPartition(), ExecInitExtraTupleSlot(), ExecInsertIndexTuples(), ExecIRInsertTriggers(), ExecMaterializeSlot(), ExecOpenIndices(), ExecResetTupleTable(), ExecSetSlotDescriptor(), ExecStoreTuple(), FreeBulkInsertState(), FreeExecutorState(), CopyStateData::freeze, GetBulkInsertState(), GetCurrentCommandId(), GetCurrentSubTransactionId(), GetPerTupleExprContext, GetPerTupleMemoryContext, heap_close, heap_form_tuple(), heap_insert(), HEAP_INSERT_FROZEN, HEAP_INSERT_SKIP_FSM, HEAP_INSERT_SKIP_WAL, heap_sync(), HeapTupleSetOid, i, InitResultRelInfo(), InvalidBuffer, InvalidOid, InvalidSubTransactionId, list_free(), makeNode, MAX_BUFFERED_TUPLES, MemoryContextSwitchTo(), tupleDesc::natts, NextCopyFrom(), NIL, NoLock, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, palloc(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pfree(), pq_endmsgread(), ErrorContextCallback::previous, CopyStateData::range_table, RelationData::rd_att, RelationData::rd_createSubid, RelationData::rd_newRelfilenodeSubid, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, PartitionDispatchData::reldesc, ReleaseBulkInsertStatePin(), RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, ResetPerTupleExprContext, ResultRelInfo::ri_FdwRoutine, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_PartitionCheck, ResultRelInfo::ri_RelationDesc, ResultRelInfo::ri_TrigDesc, HeapTupleData::t_len, HeapTupleData::t_self, HeapTupleData::t_tableOid, ThereAreNoPriorRegisteredSnapshots(), ThereAreNoReadyPortals(), TriggerDesc::trig_insert_before_row, TriggerDesc::trig_insert_instead_row, RelationData::trigdesc, PartitionDispatchData::tupslot, values, CopyStateData::volatile_defexprs, and XLogIsNeeded.

Referenced by copy_table(), and DoCopy().

2293 {
2294  HeapTuple tuple;
2295  TupleDesc tupDesc;
2296  Datum *values;
2297  bool *nulls;
2298  ResultRelInfo *resultRelInfo;
2299  ResultRelInfo *saved_resultRelInfo = NULL;
2300  EState *estate = CreateExecutorState(); /* for ExecConstraints() */
2301  ExprContext *econtext;
2302  TupleTableSlot *myslot;
2303  MemoryContext oldcontext = CurrentMemoryContext;
2304 
2305  ErrorContextCallback errcallback;
2306  CommandId mycid = GetCurrentCommandId(true);
2307  int hi_options = 0; /* start with default heap_insert options */
2308  BulkInsertState bistate;
2309  uint64 processed = 0;
2310  bool useHeapMultiInsert;
2311  int nBufferedTuples = 0;
2312  int prev_leaf_part_index = -1;
2313 
2314 #define MAX_BUFFERED_TUPLES 1000
2315  HeapTuple *bufferedTuples = NULL; /* initialize to silence warning */
2316  Size bufferedTuplesSize = 0;
2317  int firstBufferedLineNo = 0;
2318 
2319  Assert(cstate->rel);
2320 
2321  /*
2322  * The target must be a plain relation or have an INSTEAD OF INSERT row
2323  * trigger. (Currently, such triggers are only allowed on views, so we
2324  * only hint about them in the view case.)
2325  */
2326  if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
2327  cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
2328  !(cstate->rel->trigdesc &&
2330  {
2331  if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
2332  ereport(ERROR,
2333  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2334  errmsg("cannot copy to view \"%s\"",
2335  RelationGetRelationName(cstate->rel)),
2336  errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
2337  else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
2338  ereport(ERROR,
2339  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2340  errmsg("cannot copy to materialized view \"%s\"",
2341  RelationGetRelationName(cstate->rel))));
2342  else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
2343  ereport(ERROR,
2344  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2345  errmsg("cannot copy to foreign table \"%s\"",
2346  RelationGetRelationName(cstate->rel))));
2347  else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
2348  ereport(ERROR,
2349  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2350  errmsg("cannot copy to sequence \"%s\"",
2351  RelationGetRelationName(cstate->rel))));
2352  else
2353  ereport(ERROR,
2354  (errcode(ERRCODE_WRONG_OBJECT_TYPE),
2355  errmsg("cannot copy to non-table relation \"%s\"",
2356  RelationGetRelationName(cstate->rel))));
2357  }
2358 
2359  tupDesc = RelationGetDescr(cstate->rel);
2360 
2361  /*----------
2362  * Check to see if we can avoid writing WAL
2363  *
2364  * If archive logging/streaming is not enabled *and* either
2365  * - table was created in same transaction as this COPY
2366  * - data is being written to relfilenode created in this transaction
2367  * then we can skip writing WAL. It's safe because if the transaction
2368  * doesn't commit, we'll discard the table (or the new relfilenode file).
2369  * If it does commit, we'll have done the heap_sync at the bottom of this
2370  * routine first.
2371  *
2372  * As mentioned in comments in utils/rel.h, the in-same-transaction test
2373  * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
2374  * can be cleared before the end of the transaction. The exact case is
2375  * when a relation sets a new relfilenode twice in same transaction, yet
2376  * the second one fails in an aborted subtransaction, e.g.
2377  *
2378  * BEGIN;
2379  * TRUNCATE t;
2380  * SAVEPOINT save;
2381  * TRUNCATE t;
2382  * ROLLBACK TO save;
2383  * COPY ...
2384  *
2385  * Also, if the target file is new-in-transaction, we assume that checking
2386  * FSM for free space is a waste of time, even if we must use WAL because
2387  * of archiving. This could possibly be wrong, but it's unlikely.
2388  *
2389  * The comments for heap_insert and RelationGetBufferForTuple specify that
2390  * skipping WAL logging is only safe if we ensure that our tuples do not
2391  * go into pages containing tuples from any other transactions --- but this
2392  * must be the case if we have a new table or new relfilenode, so we need
2393  * no additional work to enforce that.
2394  *----------
2395  */
2396  /* createSubid is creation check, newRelfilenodeSubid is truncation check */
2397  if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
2399  {
2400  hi_options |= HEAP_INSERT_SKIP_FSM;
2401  if (!XLogIsNeeded())
2402  hi_options |= HEAP_INSERT_SKIP_WAL;
2403  }
2404 
2405  /*
2406  * Optimize if new relfilenode was created in this subxact or one of its
2407  * committed children and we won't see those rows later as part of an
2408  * earlier scan or command. This ensures that if this subtransaction
2409  * aborts then the frozen rows won't be visible after xact cleanup. Note
2410  * that the stronger test of exactly which subtransaction created it is
2411  * crucial for correctness of this optimization.
2412  */
2413  if (cstate->freeze)
2414  {
2416  ereport(ERROR,
2417  (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
2418  errmsg("cannot perform FREEZE because of prior transaction activity")));
2419 
2420  if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
2422  ereport(ERROR,
2423  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2424  errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));
2425 
2426  hi_options |= HEAP_INSERT_FROZEN;
2427  }
2428 
2429  /*
2430  * We need a ResultRelInfo so we can use the regular executor's
2431  * index-entry-making machinery. (There used to be a huge amount of code
2432  * here that basically duplicated execUtils.c ...)
2433  */
2434  resultRelInfo = makeNode(ResultRelInfo);
2435  InitResultRelInfo(resultRelInfo,
2436  cstate->rel,
2437  1, /* dummy rangetable index */
2438  NULL,
2439  0);
2440 
2441  ExecOpenIndices(resultRelInfo, false);
2442 
2443  estate->es_result_relations = resultRelInfo;
2444  estate->es_num_result_relations = 1;
2445  estate->es_result_relation_info = resultRelInfo;
2446  estate->es_range_table = cstate->range_table;
2447 
2448  /* Set up a tuple slot too */
2449  myslot = ExecInitExtraTupleSlot(estate);
2450  ExecSetSlotDescriptor(myslot, tupDesc);
2451  /* Triggers might need a slot as well */
2452  estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);
2453 
2454  /*
2455  * It's more efficient to prepare a bunch of tuples for insertion, and
2456  * insert them in one heap_multi_insert() call, than call heap_insert()
2457  * separately for every tuple. However, we can't do that if there are
2458  * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
2459  * expressions. Such triggers or expressions might query the table we're
2460  * inserting to, and act differently if the tuples that have already been
2461  * processed and prepared for insertion are not there. We also can't do
2462  * it if the table is partitioned.
2463  */
2464  if ((resultRelInfo->ri_TrigDesc != NULL &&
2465  (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
2466  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
2467  cstate->partition_dispatch_info != NULL ||
2468  cstate->volatile_defexprs)
2469  {
2470  useHeapMultiInsert = false;
2471  }
2472  else
2473  {
2474  useHeapMultiInsert = true;
2475  bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
2476  }
2477 
2478  /* Prepare to catch AFTER triggers. */
2480 
2481  /*
2482  * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
2483  * should do this for COPY, since it's not really an "INSERT" statement as
2484  * such. However, executing these triggers maintains consistency with the
2485  * EACH ROW triggers that we already fire on COPY.
2486  */
2487  ExecBSInsertTriggers(estate, resultRelInfo);
2488 
2489  values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
2490  nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
2491 
2492  bistate = GetBulkInsertState();
2493  econtext = GetPerTupleExprContext(estate);
2494 
2495  /* Set up callback to identify error line number */
2496  errcallback.callback = CopyFromErrorCallback;
2497  errcallback.arg = (void *) cstate;
2498  errcallback.previous = error_context_stack;
2499  error_context_stack = &errcallback;
2500 
2501  for (;;)
2502  {
2503  TupleTableSlot *slot;
2504  bool skip_tuple;
2505  Oid loaded_oid = InvalidOid;
2506 
2508 
2509  if (nBufferedTuples == 0)
2510  {
2511  /*
2512  * Reset the per-tuple exprcontext. We can only do this if the
2513  * tuple buffer is empty. (Calling the context the per-tuple
2514  * memory context is a bit of a misnomer now.)
2515  */
2516  ResetPerTupleExprContext(estate);
2517  }
2518 
2519  /* Switch into its memory context */
2521 
2522  if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
2523  break;
2524 
2525  /* And now we can form the input tuple. */
2526  tuple = heap_form_tuple(tupDesc, values, nulls);
2527 
2528  if (loaded_oid != InvalidOid)
2529  HeapTupleSetOid(tuple, loaded_oid);
2530 
2531  /*
2532  * Constraints might reference the tableoid column, so initialize
2533  * t_tableOid before evaluating them.
2534  */
2535  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2536 
2537  /* Triggers and stuff need to be invoked in query context. */
2538  MemoryContextSwitchTo(oldcontext);
2539 
2540  /* Place tuple in tuple slot --- but slot shouldn't free it */
2541  slot = myslot;
2542  ExecStoreTuple(tuple, slot, InvalidBuffer, false);
2543 
2544  /* Determine the partition to heap_insert the tuple into */
2545  if (cstate->partition_dispatch_info)
2546  {
2547  int leaf_part_index;
2548  TupleConversionMap *map;
2549 
2550  /*
2551  * Away we go ... If we end up not finding a partition after all,
2552  * ExecFindPartition() does not return and errors out instead.
2553  * Otherwise, the returned value is to be used as an index into
2554  * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
2555  * will get us the ResultRelInfo and TupleConversionMap for the
2556  * partition, respectively.
2557  */
2558  leaf_part_index = ExecFindPartition(resultRelInfo,
2559  cstate->partition_dispatch_info,
2560  slot,
2561  estate);
2562  Assert(leaf_part_index >= 0 &&
2563  leaf_part_index < cstate->num_partitions);
2564 
2565  /*
2566  * If this tuple is mapped to a partition that is not same as the
2567  * previous one, we'd better make the bulk insert mechanism gets a
2568  * new buffer.
2569  */
2570  if (prev_leaf_part_index != leaf_part_index)
2571  {
2572  ReleaseBulkInsertStatePin(bistate);
2573  prev_leaf_part_index = leaf_part_index;
2574  }
2575 
2576  /*
2577  * Save the old ResultRelInfo and switch to the one corresponding
2578  * to the selected partition.
2579  */
2580  saved_resultRelInfo = resultRelInfo;
2581  resultRelInfo = cstate->partitions + leaf_part_index;
2582 
2583  /* We do not yet have a way to insert into a foreign partition */
2584  if (resultRelInfo->ri_FdwRoutine)
2585  ereport(ERROR,
2586  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2587  errmsg("cannot route inserted tuples to a foreign table")));
2588 
2589  /*
2590  * For ExecInsertIndexTuples() to work on the partition's indexes
2591  */
2592  estate->es_result_relation_info = resultRelInfo;
2593 
2594  /*
2595  * We might need to convert from the parent rowtype to the
2596  * partition rowtype.
2597  */
2598  map = cstate->partition_tupconv_maps[leaf_part_index];
2599  if (map)
2600  {
2601  Relation partrel = resultRelInfo->ri_RelationDesc;
2602 
2603  tuple = do_convert_tuple(tuple, map);
2604 
2605  /*
2606  * We must use the partition's tuple descriptor from this
2607  * point on. Use a dedicated slot from this point on until
2608  * we're finished dealing with the partition.
2609  */
2610  slot = cstate->partition_tuple_slot;
2611  Assert(slot != NULL);
2612  ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
2613  ExecStoreTuple(tuple, slot, InvalidBuffer, true);
2614  }
2615 
2616  tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
2617  }
2618 
2619  skip_tuple = false;
2620 
2621  /* BEFORE ROW INSERT Triggers */
2622  if (resultRelInfo->ri_TrigDesc &&
2623  resultRelInfo->ri_TrigDesc->trig_insert_before_row)
2624  {
2625  slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
2626 
2627  if (slot == NULL) /* "do nothing" */
2628  skip_tuple = true;
2629  else /* trigger might have changed tuple */
2630  tuple = ExecMaterializeSlot(slot);
2631  }
2632 
2633  if (!skip_tuple)
2634  {
2635  if (resultRelInfo->ri_TrigDesc &&
2636  resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
2637  {
2638  /* Pass the data to the INSTEAD ROW INSERT trigger */
2639  ExecIRInsertTriggers(estate, resultRelInfo, slot);
2640  }
2641  else
2642  {
2643  /* Check the constraints of the tuple */
2644  if (cstate->rel->rd_att->constr ||
2645  resultRelInfo->ri_PartitionCheck)
2646  ExecConstraints(resultRelInfo, slot, estate);
2647 
2648  if (useHeapMultiInsert)
2649  {
2650  /* Add this tuple to the tuple buffer */
2651  if (nBufferedTuples == 0)
2652  firstBufferedLineNo = cstate->cur_lineno;
2653  bufferedTuples[nBufferedTuples++] = tuple;
2654  bufferedTuplesSize += tuple->t_len;
2655 
2656  /*
2657  * If the buffer filled up, flush it. Also flush if the
2658  * total size of all the tuples in the buffer becomes
2659  * large, to avoid using large amounts of memory for the
2660  * buffer when the tuples are exceptionally wide.
2661  */
2662  if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
2663  bufferedTuplesSize > 65535)
2664  {
2665  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2666  resultRelInfo, myslot, bistate,
2667  nBufferedTuples, bufferedTuples,
2668  firstBufferedLineNo);
2669  nBufferedTuples = 0;
2670  bufferedTuplesSize = 0;
2671  }
2672  }
2673  else
2674  {
2675  List *recheckIndexes = NIL;
2676 
2677  /* OK, store the tuple and create index entries for it */
2678  heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
2679  hi_options, bistate);
2680 
2681  if (resultRelInfo->ri_NumIndices > 0)
2682  recheckIndexes = ExecInsertIndexTuples(slot,
2683  &(tuple->t_self),
2684  estate,
2685  false,
2686  NULL,
2687  NIL);
2688 
2689  /* AFTER ROW INSERT Triggers */
2690  ExecARInsertTriggers(estate, resultRelInfo, tuple,
2691  recheckIndexes);
2692 
2693  list_free(recheckIndexes);
2694  }
2695  }
2696 
2697  /*
2698  * We count only tuples not suppressed by a BEFORE INSERT trigger;
2699  * this is the same definition used by execMain.c for counting
2700  * tuples inserted by an INSERT command.
2701  */
2702  processed++;
2703 
2704  if (saved_resultRelInfo)
2705  {
2706  resultRelInfo = saved_resultRelInfo;
2707  estate->es_result_relation_info = resultRelInfo;
2708  }
2709  }
2710  }
2711 
2712  /* Flush any remaining buffered tuples */
2713  if (nBufferedTuples > 0)
2714  CopyFromInsertBatch(cstate, estate, mycid, hi_options,
2715  resultRelInfo, myslot, bistate,
2716  nBufferedTuples, bufferedTuples,
2717  firstBufferedLineNo);
2718 
2719  /* Done, clean up */
2720  error_context_stack = errcallback.previous;
2721 
2722  FreeBulkInsertState(bistate);
2723 
2724  MemoryContextSwitchTo(oldcontext);
2725 
2726  /*
2727  * In the old protocol, tell pqcomm that we can process normal protocol
2728  * messages again.
2729  */
2730  if (cstate->copy_dest == COPY_OLD_FE)
2731  pq_endmsgread();
2732 
2733  /* Execute AFTER STATEMENT insertion triggers */
2734  ExecASInsertTriggers(estate, resultRelInfo);
2735 
2736  /* Handle queued AFTER triggers */
2737  AfterTriggerEndQuery(estate);
2738 
2739  pfree(values);
2740  pfree(nulls);
2741 
2742  ExecResetTupleTable(estate->es_tupleTable, false);
2743 
2744  ExecCloseIndices(resultRelInfo);
2745 
2746  /* Close all the partitioned tables, leaf partitions, and their indices */
2747  if (cstate->partition_dispatch_info)
2748  {
2749  int i;
2750 
2751  /*
2752  * Remember cstate->partition_dispatch_info[0] corresponds to the root
2753  * partitioned table, which we must not try to close, because it is
2754  * the main target table of COPY that will be closed eventually by
2755  * DoCopy(). Also, tupslot is NULL for the root partitioned table.
2756  */
2757  for (i = 1; i < cstate->num_dispatch; i++)
2758  {
2760 
2761  heap_close(pd->reldesc, NoLock);
2763  }
2764  for (i = 0; i < cstate->num_partitions; i++)
2765  {
2766  ResultRelInfo *resultRelInfo = cstate->partitions + i;
2767 
2768  ExecCloseIndices(resultRelInfo);
2769  heap_close(resultRelInfo->ri_RelationDesc, NoLock);
2770  }
2771 
2772  /* Release the standalone partition tuple descriptor */
2774  }
2775 
2776  /* Close any trigger target relations */
2777  ExecCleanUpTriggerState(estate);
2778 
2779  FreeExecutorState(estate);
2780 
2781  /*
2782  * If we skipped writing WAL, then we need to sync the heap (but not
2783  * indexes since those use WAL anyway)
2784  */
2785  if (hi_options & HEAP_INSERT_SKIP_WAL)
2786  heap_sync(cstate->rel);
2787 
2788  return processed;
2789 }
int ri_NumIndices
Definition: execnodes.h:375
#define NIL
Definition: pg_list.h:69
uint32 CommandId
Definition: c.h:411
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation resultRelationDesc, Index resultRelationIndex, Relation partition_root, int instrument_options)
Definition: execMain.c:1297
Relation ri_RelationDesc
Definition: execnodes.h:374
List * range_table
Definition: copy.c:166
TupleTableSlot * ExecInitExtraTupleSlot(EState *estate)
Definition: execTuples.c:852
int errhint(const char *fmt,...)
Definition: elog.c:987
void CopyFromErrorCallback(void *arg)
Definition: copy.c:2187
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
#define ResetPerTupleExprContext(estate)
Definition: executor.h:465
#define RelationGetDescr(relation)
Definition: rel.h:429
#define HEAP_INSERT_FROZEN
Definition: heapam.h:30
ResultRelInfo * partitions
Definition: copy.c:171
#define XLogIsNeeded()
Definition: xlog.h:145
#define MAX_BUFFERED_TUPLES
TupleTableSlot * ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2277
#define RELKIND_MATVIEW
Definition: pg_class.h:165
void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:1886
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
bool ThereAreNoPriorRegisteredSnapshots(void)
Definition: snapmgr.c:1611
int errcode(int sqlerrcode)
Definition: elog.c:575
SubTransactionId rd_newRelfilenodeSubid
Definition: rel.h:111
void heap_sync(Relation rel)
Definition: heapam.c:9084
#define HEAP_INSERT_SKIP_WAL
Definition: heapam.h:28
HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull)
Definition: heaptuple.c:692
#define heap_close(r, l)
Definition: heapam.h:97
List * es_range_table
Definition: execnodes.h:411
Form_pg_class rd_rel
Definition: rel.h:114
unsigned int Oid
Definition: postgres_ext.h:31
bool volatile_defexprs
Definition: copy.c:165
struct ErrorContextCallback * previous
Definition: elog.h:238
int natts
Definition: tupdesc.h:73
void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative)
Definition: execIndexing.c:149
CopyDest copy_dest
Definition: copy.c:98
#define HeapTupleSetOid(tuple, oid)
Definition: htup_details.h:698
ErrorContextCallback * error_context_stack
Definition: elog.c:88
bool trig_insert_instead_row
Definition: reltrigger.h:57
void FreeExecutorState(EState *estate)
Definition: execUtils.c:178
Relation rel
Definition: copy.c:109
#define GetPerTupleExprContext(estate)
Definition: executor.h:456
BulkInsertState GetBulkInsertState(void)
Definition: heapam.c:2319
void pfree(void *pointer)
Definition: mcxt.c:950
bool ThereAreNoReadyPortals(void)
Definition: portalmem.c:1127
TupleTableSlot * partition_tuple_slot
Definition: copy.c:173
#define ERROR
Definition: elog.h:43
static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
Definition: copy.c:2797
ItemPointerData t_self
Definition: htup.h:65
TriggerDesc * trigdesc
Definition: rel.h:120
int num_dispatch
Definition: copy.c:169
uint32 t_len
Definition: htup.h:64
void ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2137
#define NoLock
Definition: lockdefs.h:34
void ExecDropSingleTupleTableSlot(TupleTableSlot *slot)
Definition: execTuples.c:216
ResultRelInfo * es_result_relations
Definition: execnodes.h:421
#define RelationGetRelationName(relation)
Definition: rel.h:437
struct FdwRoutine * ri_FdwRoutine
Definition: execnodes.h:382
Oid t_tableOid
Definition: htup.h:66
#define RELKIND_FOREIGN_TABLE
Definition: pg_class.h:167
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
TupleTableSlot * es_trig_tuple_slot
Definition: execnodes.h:437
#define ereport(elevel, rest)
Definition: elog.h:122
Oid heap_insert(Relation relation, HeapTuple tup, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2396
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:378
EState * CreateExecutorState(void)
Definition: execUtils.c:80
SubTransactionId rd_createSubid
Definition: rel.h:110
#define RELKIND_PARTITIONED_TABLE
Definition: pg_class.h:168
bool trig_insert_before_row
Definition: reltrigger.h:55
List * es_tupleTable
Definition: execnodes.h:450
void ExecResetTupleTable(List *tupleTable, bool shouldFree)
Definition: execTuples.c:156
uintptr_t Datum
Definition: postgres.h:372
TupleTableSlot * tupslot
Definition: partition.h:66
void ExecSetSlotDescriptor(TupleTableSlot *slot, TupleDesc tupdesc)
Definition: execTuples.c:247
void ExecASInsertTriggers(EState *estate, ResultRelInfo *relinfo)
Definition: trigger.c:2189
int ExecFindPartition(ResultRelInfo *resultRelInfo, PartitionDispatch *pd, TupleTableSlot *slot, EState *estate)
Definition: execMain.c:3313
int es_num_result_relations
Definition: execnodes.h:422
List * ri_PartitionCheck
Definition: execnodes.h:392
TupleDesc rd_att
Definition: rel.h:115
bool freeze
Definition: copy.c:117
void pq_endmsgread(void)
Definition: pqcomm.c:1215
#define InvalidOid
Definition: postgres_ext.h:36
void AfterTriggerBeginQuery(void)
Definition: trigger.c:4168
#define makeNode(_type_)
Definition: nodes.h:557
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void FreeBulkInsertState(BulkInsertState bistate)
Definition: heapam.c:2333
SubTransactionId GetCurrentSubTransactionId(void)
Definition: xact.c:649
TupleConstr * constr
Definition: tupdesc.h:76
size_t Size
Definition: c.h:356
#define InvalidSubTransactionId
Definition: c.h:403
HeapTuple ExecMaterializeSlot(TupleTableSlot *slot)
Definition: execTuples.c:725
void ReleaseBulkInsertStatePin(BulkInsertState bistate)
Definition: heapam.c:2345
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes)
Definition: trigger.c:2265
HeapTuple do_convert_tuple(HeapTuple tuple, TupleConversionMap *map)
Definition: tupconvert.c:343
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:461
void ExecCleanUpTriggerState(EState *estate)
Definition: execMain.c:1453
#define HEAP_INSERT_SKIP_FSM
Definition: heapam.h:29
static Datum values[MAXATTR]
Definition: bootstrap.c:163
void AfterTriggerEndQuery(EState *estate)
Definition: trigger.c:4188
void(* callback)(void *arg)
Definition: elog.h:239
void * palloc(Size size)
Definition: mcxt.c:849
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define RELKIND_VIEW
Definition: pg_class.h:164
void list_free(List *list)
Definition: list.c:1133
bool NextCopyFrom(CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
Definition: copy.c:3191
int i
int cur_lineno
Definition: copy.c:139
PartitionDispatch * partition_dispatch_info
Definition: copy.c:168
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
TupleTableSlot * ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TupleTableSlot *slot)
Definition: trigger.c:2199
CommandId GetCurrentCommandId(bool used)
Definition: xact.c:687
TupleConversionMap ** partition_tupconv_maps
Definition: copy.c:172
#define RELKIND_RELATION
Definition: pg_class.h:160
#define RELKIND_SEQUENCE
Definition: pg_class.h:162
Definition: pg_list.h:45
#define RelationGetRelid(relation)
Definition: rel.h:417
void ExecCloseIndices(ResultRelInfo *resultRelInfo)
Definition: execIndexing.c:224
int num_partitions
Definition: copy.c:170
ResultRelInfo * es_result_relation_info
Definition: execnodes.h:423
void CopyFromErrorCallback ( void *  arg)

Definition at line 2187 of file copy.c.

References CopyStateData::binary, CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, StringInfoData::data, errcontext, limit_printout_length(), CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, and pfree().

Referenced by CopyFrom(), file_acquire_sample_rows(), and fileIterateForeignScan().

2188 {
2189  CopyState cstate = (CopyState) arg;
2190 
2191  if (cstate->binary)
2192  {
2193  /* can't usefully display the data */
2194  if (cstate->cur_attname)
2195  errcontext("COPY %s, line %d, column %s",
2196  cstate->cur_relname, cstate->cur_lineno,
2197  cstate->cur_attname);
2198  else
2199  errcontext("COPY %s, line %d",
2200  cstate->cur_relname, cstate->cur_lineno);
2201  }
2202  else
2203  {
2204  if (cstate->cur_attname && cstate->cur_attval)
2205  {
2206  /* error is relevant to a particular column */
2207  char *attval;
2208 
2209  attval = limit_printout_length(cstate->cur_attval);
2210  errcontext("COPY %s, line %d, column %s: \"%s\"",
2211  cstate->cur_relname, cstate->cur_lineno,
2212  cstate->cur_attname, attval);
2213  pfree(attval);
2214  }
2215  else if (cstate->cur_attname)
2216  {
2217  /* error is relevant to a particular column, value is NULL */
2218  errcontext("COPY %s, line %d, column %s: null input",
2219  cstate->cur_relname, cstate->cur_lineno,
2220  cstate->cur_attname);
2221  }
2222  else
2223  {
2224  /*
2225  * Error is relevant to a particular line.
2226  *
2227  * If line_buf still contains the correct line, and it's already
2228  * transcoded, print it. If it's still in a foreign encoding, it's
2229  * quite likely that the error is precisely a failure to do
2230  * encoding conversion (ie, bad data). We dare not try to convert
2231  * it, and at present there's no way to regurgitate it without
2232  * conversion. So we have to punt and just report the line number.
2233  */
2234  if (cstate->line_buf_valid &&
2235  (cstate->line_buf_converted || !cstate->need_transcoding))
2236  {
2237  char *lineval;
2238 
2239  lineval = limit_printout_length(cstate->line_buf.data);
2240  errcontext("COPY %s, line %d: \"%s\"",
2241  cstate->cur_relname, cstate->cur_lineno, lineval);
2242  pfree(lineval);
2243  }
2244  else
2245  {
2246  errcontext("COPY %s, line %d",
2247  cstate->cur_relname, cstate->cur_lineno);
2248  }
2249  }
2250  }
2251 }
bool binary
Definition: copy.c:115
bool need_transcoding
Definition: copy.c:105
StringInfoData line_buf
Definition: copy.c:197
bool line_buf_valid
Definition: copy.c:199
bool line_buf_converted
Definition: copy.c:198
const char * cur_attname
Definition: copy.c:140
void pfree(void *pointer)
Definition: mcxt.c:950
static char * limit_printout_length(const char *str)
Definition: copy.c:2263
const char * cur_relname
Definition: copy.c:138
#define errcontext
Definition: elog.h:164
struct CopyStateData * CopyState
Definition: copy.h:23
void * arg
int cur_lineno
Definition: copy.c:139
const char * cur_attval
Definition: copy.c:141
static void CopyFromInsertBatch ( CopyState  cstate,
EState estate,
CommandId  mycid,
int  hi_options,
ResultRelInfo resultRelInfo,
TupleTableSlot myslot,
BulkInsertState  bistate,
int  nBufferedTuples,
HeapTuple bufferedTuples,
int  firstBufferedLineNo 
)
static

Definition at line 2797 of file copy.c.

References CopyStateData::cur_lineno, ExecARInsertTriggers(), ExecInsertIndexTuples(), ExecStoreTuple(), GetPerTupleMemoryContext, heap_multi_insert(), i, InvalidBuffer, CopyStateData::line_buf_valid, list_free(), MemoryContextSwitchTo(), NIL, NULL, CopyStateData::rel, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_TrigDesc, and TriggerDesc::trig_insert_after_row.

Referenced by CopyFrom().

2802 {
2803  MemoryContext oldcontext;
2804  int i;
2805  int save_cur_lineno;
2806 
2807  /*
2808  * Print error context information correctly, if one of the operations
2809  * below fail.
2810  */
2811  cstate->line_buf_valid = false;
2812  save_cur_lineno = cstate->cur_lineno;
2813 
2814  /*
2815  * heap_multi_insert leaks memory, so switch to short-lived memory context
2816  * before calling it.
2817  */
2818  oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2819  heap_multi_insert(cstate->rel,
2820  bufferedTuples,
2821  nBufferedTuples,
2822  mycid,
2823  hi_options,
2824  bistate);
2825  MemoryContextSwitchTo(oldcontext);
2826 
2827  /*
2828  * If there are any indexes, update them for all the inserted tuples, and
2829  * run AFTER ROW INSERT triggers.
2830  */
2831  if (resultRelInfo->ri_NumIndices > 0)
2832  {
2833  for (i = 0; i < nBufferedTuples; i++)
2834  {
2835  List *recheckIndexes;
2836 
2837  cstate->cur_lineno = firstBufferedLineNo + i;
2838  ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
2839  recheckIndexes =
2840  ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
2841  estate, false, NULL, NIL);
2842  ExecARInsertTriggers(estate, resultRelInfo,
2843  bufferedTuples[i],
2844  recheckIndexes);
2845  list_free(recheckIndexes);
2846  }
2847  }
2848 
2849  /*
2850  * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
2851  * anyway.
2852  */
2853  else if (resultRelInfo->ri_TrigDesc != NULL &&
2854  resultRelInfo->ri_TrigDesc->trig_insert_after_row)
2855  {
2856  for (i = 0; i < nBufferedTuples; i++)
2857  {
2858  cstate->cur_lineno = firstBufferedLineNo + i;
2859  ExecARInsertTriggers(estate, resultRelInfo,
2860  bufferedTuples[i],
2861  NIL);
2862  }
2863  }
2864 
2865  /* reset cur_lineno to where we were */
2866  cstate->cur_lineno = save_cur_lineno;
2867 }
int ri_NumIndices
Definition: execnodes.h:375
#define NIL
Definition: pg_list.h:69
TupleTableSlot * ExecStoreTuple(HeapTuple tuple, TupleTableSlot *slot, Buffer buffer, bool shouldFree)
Definition: execTuples.c:320
List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, EState *estate, bool noDupErr, bool *specConflict, List *arbiterIndexes)
Definition: execIndexing.c:271
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
#define InvalidBuffer
Definition: buf.h:25
bool line_buf_valid
Definition: copy.c:199
Relation rel
Definition: copy.c:109
bool trig_insert_after_row
Definition: reltrigger.h:56
TriggerDesc * ri_TrigDesc
Definition: execnodes.h:378
#define NULL
Definition: c.h:229
void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, CommandId cid, int options, BulkInsertState bistate)
Definition: heapam.c:2658
void ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, HeapTuple trigtuple, List *recheckIndexes)
Definition: trigger.c:2265
#define GetPerTupleMemoryContext(estate)
Definition: executor.h:461
void list_free(List *list)
Definition: list.c:1133
int i
int cur_lineno
Definition: copy.c:139
Definition: pg_list.h:45
static List * CopyGetAttnums ( TupleDesc  tupDesc,
Relation  rel,
List attnamelist 
)
static

Definition at line 4624 of file copy.c.

References tupleDesc::attrs, ereport, errcode(), errmsg(), ERROR, i, InvalidAttrNumber, lappend_int(), lfirst, list_member_int(), name, namestrcmp(), tupleDesc::natts, NIL, NULL, RelationGetRelationName, and strVal.

Referenced by BeginCopy(), and DoCopy().

4625 {
4626  List *attnums = NIL;
4627 
4628  if (attnamelist == NIL)
4629  {
4630  /* Generate default column list */
4631  Form_pg_attribute *attr = tupDesc->attrs;
4632  int attr_count = tupDesc->natts;
4633  int i;
4634 
4635  for (i = 0; i < attr_count; i++)
4636  {
4637  if (attr[i]->attisdropped)
4638  continue;
4639  attnums = lappend_int(attnums, i + 1);
4640  }
4641  }
4642  else
4643  {
4644  /* Validate the user-supplied list and extract attnums */
4645  ListCell *l;
4646 
4647  foreach(l, attnamelist)
4648  {
4649  char *name = strVal(lfirst(l));
4650  int attnum;
4651  int i;
4652 
4653  /* Lookup column name */
4654  attnum = InvalidAttrNumber;
4655  for (i = 0; i < tupDesc->natts; i++)
4656  {
4657  if (tupDesc->attrs[i]->attisdropped)
4658  continue;
4659  if (namestrcmp(&(tupDesc->attrs[i]->attname), name) == 0)
4660  {
4661  attnum = tupDesc->attrs[i]->attnum;
4662  break;
4663  }
4664  }
4665  if (attnum == InvalidAttrNumber)
4666  {
4667  if (rel != NULL)
4668  ereport(ERROR,
4669  (errcode(ERRCODE_UNDEFINED_COLUMN),
4670  errmsg("column \"%s\" of relation \"%s\" does not exist",
4671  name, RelationGetRelationName(rel))));
4672  else
4673  ereport(ERROR,
4674  (errcode(ERRCODE_UNDEFINED_COLUMN),
4675  errmsg("column \"%s\" does not exist",
4676  name)));
4677  }
4678  /* Check for duplicates */
4679  if (list_member_int(attnums, attnum))
4680  ereport(ERROR,
4681  (errcode(ERRCODE_DUPLICATE_COLUMN),
4682  errmsg("column \"%s\" specified more than once",
4683  name)));
4684  attnums = lappend_int(attnums, attnum);
4685  }
4686  }
4687 
4688  return attnums;
4689 }
#define NIL
Definition: pg_list.h:69
Form_pg_attribute * attrs
Definition: tupdesc.h:74
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
int namestrcmp(Name name, const char *str)
Definition: name.c:248
int natts
Definition: tupdesc.h:73
#define ERROR
Definition: elog.h:43
bool list_member_int(const List *list, int datum)
Definition: list.c:485
#define RelationGetRelationName(relation)
Definition: rel.h:437
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
#define ereport(elevel, rest)
Definition: elog.h:122
List * lappend_int(List *list, int datum)
Definition: list.c:146
#define NULL
Definition: c.h:229
#define lfirst(lc)
Definition: pg_list.h:106
const char * name
Definition: encode.c:521
#define InvalidAttrNumber
Definition: attnum.h:23
int errmsg(const char *fmt,...)
Definition: elog.c:797
int i
Definition: pg_list.h:45
static int CopyGetData ( CopyState  cstate,
void *  databuf,
int  minread,
int  maxread 
)
static

Definition at line 556 of file copy.c.

References COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, StringInfoData::cursor, CopyStateData::data_source_cb, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, CopyStateData::fe_eof, CopyStateData::fe_msgbuf, HOLD_CANCEL_INTERRUPTS, StringInfoData::len, pq_copymsgbytes(), pq_getbyte(), pq_getbytes(), pq_getmessage(), pq_getmsgstring(), pq_startmsgread(), and RESUME_CANCEL_INTERRUPTS.

Referenced by BeginCopyFrom(), CopyGetInt16(), CopyGetInt32(), CopyLoadRawBuf(), CopyReadBinaryAttribute(), and NextCopyFrom().

557 {
558  int bytesread = 0;
559 
560  switch (cstate->copy_dest)
561  {
562  case COPY_FILE:
563  bytesread = fread(databuf, 1, maxread, cstate->copy_file);
564  if (ferror(cstate->copy_file))
565  ereport(ERROR,
567  errmsg("could not read from COPY file: %m")));
568  break;
569  case COPY_OLD_FE:
570 
571  /*
572  * We cannot read more than minread bytes (which in practice is 1)
573  * because old protocol doesn't have any clear way of separating
574  * the COPY stream from following data. This is slow, but not any
575  * slower than the code path was originally, and we don't care
576  * much anymore about the performance of old protocol.
577  */
578  if (pq_getbytes((char *) databuf, minread))
579  {
580  /* Only a \. terminator is legal EOF in old protocol */
581  ereport(ERROR,
582  (errcode(ERRCODE_CONNECTION_FAILURE),
583  errmsg("unexpected EOF on client connection with an open transaction")));
584  }
585  bytesread = minread;
586  break;
587  case COPY_NEW_FE:
588  while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
589  {
590  int avail;
591 
592  while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
593  {
594  /* Try to receive another message */
595  int mtype;
596 
597  readmessage:
599  pq_startmsgread();
600  mtype = pq_getbyte();
601  if (mtype == EOF)
602  ereport(ERROR,
603  (errcode(ERRCODE_CONNECTION_FAILURE),
604  errmsg("unexpected EOF on client connection with an open transaction")));
605  if (pq_getmessage(cstate->fe_msgbuf, 0))
606  ereport(ERROR,
607  (errcode(ERRCODE_CONNECTION_FAILURE),
608  errmsg("unexpected EOF on client connection with an open transaction")));
610  switch (mtype)
611  {
612  case 'd': /* CopyData */
613  break;
614  case 'c': /* CopyDone */
615  /* COPY IN correctly terminated by frontend */
616  cstate->fe_eof = true;
617  return bytesread;
618  case 'f': /* CopyFail */
619  ereport(ERROR,
620  (errcode(ERRCODE_QUERY_CANCELED),
621  errmsg("COPY from stdin failed: %s",
622  pq_getmsgstring(cstate->fe_msgbuf))));
623  break;
624  case 'H': /* Flush */
625  case 'S': /* Sync */
626 
627  /*
628  * Ignore Flush/Sync for the convenience of client
629  * libraries (such as libpq) that may send those
630  * without noticing that the command they just
631  * sent was COPY.
632  */
633  goto readmessage;
634  default:
635  ereport(ERROR,
636  (errcode(ERRCODE_PROTOCOL_VIOLATION),
637  errmsg("unexpected message type 0x%02X during COPY from stdin",
638  mtype)));
639  break;
640  }
641  }
642  avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
643  if (avail > maxread)
644  avail = maxread;
645  pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
646  databuf = (void *) ((char *) databuf + avail);
647  maxread -= avail;
648  bytesread += avail;
649  }
650  break;
651  case COPY_CALLBACK:
652  bytesread = cstate->data_source_cb(databuf, minread, maxread);
653  break;
654  }
655 
656  return bytesread;
657 }
#define HOLD_CANCEL_INTERRUPTS()
Definition: miscadmin.h:122
const char * pq_getmsgstring(StringInfo msg)
Definition: pqformat.c:621
int errcode(int sqlerrcode)
Definition: elog.c:575
bool fe_eof
Definition: copy.c:102
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
copy_data_source_cb data_source_cb
Definition: copy.c:114
#define ERROR
Definition: elog.h:43
void pq_startmsgread(void)
Definition: pqcomm.c:1191
int pq_getbytes(char *s, size_t len)
Definition: pqcomm.c:1075
int errcode_for_file_access(void)
Definition: elog.c:598
#define ereport(elevel, rest)
Definition: elog.h:122
int pq_getmessage(StringInfo s, int maxlen)
Definition: pqcomm.c:1253
int pq_getbyte(void)
Definition: pqcomm.c:981
StringInfo fe_msgbuf
Definition: copy.c:100
void pq_copymsgbytes(StringInfo msg, char *buf, int datalen)
Definition: pqformat.c:570
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
#define RESUME_CANCEL_INTERRUPTS()
Definition: miscadmin.h:124
static bool CopyGetInt16 ( CopyState  cstate,
int16 val 
)
static

Definition at line 711 of file copy.c.

References buf, and CopyGetData().

Referenced by NextCopyFrom().

712 {
713  uint16 buf;
714 
715  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
716  {
717  *val = 0; /* suppress compiler warning */
718  return false;
719  }
720  *val = (int16) ntohs(buf);
721  return true;
722 }
signed short int16
Definition: c.h:255
unsigned short uint16
Definition: c.h:267
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
static char * buf
Definition: pg_test_fsync.c:66
long val
Definition: informix.c:689
static bool CopyGetInt32 ( CopyState  cstate,
int32 val 
)
static

Definition at line 682 of file copy.c.

References buf, and CopyGetData().

Referenced by BeginCopyFrom(), and CopyReadBinaryAttribute().

683 {
684  uint32 buf;
685 
686  if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
687  {
688  *val = 0; /* suppress compiler warning */
689  return false;
690  }
691  *val = (int32) ntohl(buf);
692  return true;
693 }
signed int int32
Definition: c.h:256
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
static char * buf
Definition: pg_test_fsync.c:66
unsigned int uint32
Definition: c.h:268
long val
Definition: informix.c:689
static bool CopyLoadRawBuf ( CopyState  cstate)
static

Definition at line 736 of file copy.c.

References CopyGetData(), memmove, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and RAW_BUF_SIZE.

Referenced by CopyReadLine(), and CopyReadLineText().

737 {
738  int nbytes;
739  int inbytes;
740 
741  if (cstate->raw_buf_index < cstate->raw_buf_len)
742  {
743  /* Copy down the unprocessed data */
744  nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
745  memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
746  nbytes);
747  }
748  else
749  nbytes = 0; /* no data need be saved */
750 
751  inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
752  1, RAW_BUF_SIZE - nbytes);
753  nbytes += inbytes;
754  cstate->raw_buf[nbytes] = '\0';
755  cstate->raw_buf_index = 0;
756  cstate->raw_buf_len = nbytes;
757  return (inbytes > 0);
758 }
int raw_buf_index
Definition: copy.c:210
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
char * raw_buf
Definition: copy.c:209
#define memmove(d, s, c)
Definition: c.h:1058
int raw_buf_len
Definition: copy.c:211
#define RAW_BUF_SIZE
Definition: copy.c:208
static void CopyOneRowTo ( CopyState  cstate,
Oid  tupleOid,
Datum values,
bool nulls 
)
static

Definition at line 2093 of file copy.c.

References CopyStateData::attnumlist, CopyStateData::binary, CopyAttributeOutCSV(), CopyAttributeOutText(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), CopySendString(), CopyStateData::csv_mode, cur, DatumGetCString, CopyStateData::delim, DirectFunctionCall1, CopyStateData::force_quote_flags, lfirst_int, list_length(), MemoryContextReset(), MemoryContextSwitchTo(), CopyStateData::null_print_client, ObjectIdGetDatum, oidout(), CopyStateData::oids, CopyStateData::out_functions, OutputFunctionCall(), CopyStateData::rowcontext, SendFunctionCall(), value, VARDATA, VARHDRSZ, and VARSIZE.

Referenced by copy_dest_receive(), and CopyTo().

2094 {
2095  bool need_delim = false;
2096  FmgrInfo *out_functions = cstate->out_functions;
2097  MemoryContext oldcontext;
2098  ListCell *cur;
2099  char *string;
2100 
2101  MemoryContextReset(cstate->rowcontext);
2102  oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
2103 
2104  if (cstate->binary)
2105  {
2106  /* Binary per-tuple header */
2107  CopySendInt16(cstate, list_length(cstate->attnumlist));
2108  /* Send OID if wanted --- note attnumlist doesn't include it */
2109  if (cstate->oids)
2110  {
2111  /* Hack --- assume Oid is same size as int32 */
2112  CopySendInt32(cstate, sizeof(int32));
2113  CopySendInt32(cstate, tupleOid);
2114  }
2115  }
2116  else
2117  {
2118  /* Text format has no per-tuple header, but send OID if wanted */
2119  /* Assume digits don't need any quoting or encoding conversion */
2120  if (cstate->oids)
2121  {
2123  ObjectIdGetDatum(tupleOid)));
2124  CopySendString(cstate, string);
2125  need_delim = true;
2126  }
2127  }
2128 
2129  foreach(cur, cstate->attnumlist)
2130  {
2131  int attnum = lfirst_int(cur);
2132  Datum value = values[attnum - 1];
2133  bool isnull = nulls[attnum - 1];
2134 
2135  if (!cstate->binary)
2136  {
2137  if (need_delim)
2138  CopySendChar(cstate, cstate->delim[0]);
2139  need_delim = true;
2140  }
2141 
2142  if (isnull)
2143  {
2144  if (!cstate->binary)
2145  CopySendString(cstate, cstate->null_print_client);
2146  else
2147  CopySendInt32(cstate, -1);
2148  }
2149  else
2150  {
2151  if (!cstate->binary)
2152  {
2153  string = OutputFunctionCall(&out_functions[attnum - 1],
2154  value);
2155  if (cstate->csv_mode)
2156  CopyAttributeOutCSV(cstate, string,
2157  cstate->force_quote_flags[attnum - 1],
2158  list_length(cstate->attnumlist) == 1);
2159  else
2160  CopyAttributeOutText(cstate, string);
2161  }
2162  else
2163  {
2164  bytea *outputbytes;
2165 
2166  outputbytes = SendFunctionCall(&out_functions[attnum - 1],
2167  value);
2168  CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
2169  CopySendData(cstate, VARDATA(outputbytes),
2170  VARSIZE(outputbytes) - VARHDRSZ);
2171  }
2172  }
2173  }
2174 
2175  CopySendEndOfRow(cstate);
2176 
2177  MemoryContextSwitchTo(oldcontext);
2178 }
Definition: fmgr.h:56
bool csv_mode
Definition: copy.c:118
#define VARDATA(PTR)
Definition: postgres.h:303
bool binary
Definition: copy.c:115
List * attnumlist
Definition: copy.c:111
#define VARSIZE(PTR)
Definition: postgres.h:304
#define VARHDRSZ
Definition: c.h:445
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
struct cursor * cur
Definition: ecpg.c:28
static void CopyAttributeOutText(CopyState cstate, char *string)
Definition: copy.c:4380
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:135
#define DirectFunctionCall1(func, arg1)
Definition: fmgr.h:584
bool * force_quote_flags
Definition: copy.c:128
char * delim
Definition: copy.c:123
Datum oidout(PG_FUNCTION_ARGS)
Definition: oid.c:127
MemoryContext rowcontext
Definition: copy.c:152
signed int int32
Definition: c.h:256
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * OutputFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1667
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:699
#define ObjectIdGetDatum(X)
Definition: postgres.h:513
#define DatumGetCString(X)
Definition: postgres.h:572
#define lfirst_int(lc)
Definition: pg_list.h:107
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4533
char string[11]
Definition: preproc-type.c:46
bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val)
Definition: fmgr.c:1728
uintptr_t Datum
Definition: postgres.h:372
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:668
static Datum values[MAXATTR]
Definition: bootstrap.c:163
FmgrInfo * out_functions
Definition: copy.c:151
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:463
Definition: c.h:439
char * null_print_client
Definition: copy.c:122
static struct @121 value
bool oids
Definition: copy.c:116
static int CopyReadAttributesCSV ( CopyState  cstate)
static

Definition at line 4152 of file copy.c.

References Assert, CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, CopyStateData::escape, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, CopyStateData::quote, CopyStateData::raw_fields, repalloc(), and resetStringInfo().

Referenced by NextCopyFromRawFields().

4153 {
4154  char delimc = cstate->delim[0];
4155  char quotec = cstate->quote[0];
4156  char escapec = cstate->escape[0];
4157  int fieldno;
4158  char *output_ptr;
4159  char *cur_ptr;
4160  char *line_end_ptr;
4161 
4162  /*
4163  * We need a special case for zero-column tables: check that the input
4164  * line is empty, and return.
4165  */
4166  if (cstate->max_fields <= 0)
4167  {
4168  if (cstate->line_buf.len != 0)
4169  ereport(ERROR,
4170  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4171  errmsg("extra data after last expected column")));
4172  return 0;
4173  }
4174 
4175  resetStringInfo(&cstate->attribute_buf);
4176 
4177  /*
4178  * The de-escaped attributes will certainly not be longer than the input
4179  * data line, so we can just force attribute_buf to be large enough and
4180  * then transfer data without any checks for enough space. We need to do
4181  * it this way because enlarging attribute_buf mid-stream would invalidate
4182  * pointers already stored into cstate->raw_fields[].
4183  */
4184  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
4185  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
4186  output_ptr = cstate->attribute_buf.data;
4187 
4188  /* set pointer variables for loop */
4189  cur_ptr = cstate->line_buf.data;
4190  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
4191 
4192  /* Outer loop iterates over fields */
4193  fieldno = 0;
4194  for (;;)
4195  {
4196  bool found_delim = false;
4197  bool saw_quote = false;
4198  char *start_ptr;
4199  char *end_ptr;
4200  int input_len;
4201 
4202  /* Make sure there is enough space for the next value */
4203  if (fieldno >= cstate->max_fields)
4204  {
4205  cstate->max_fields *= 2;
4206  cstate->raw_fields =
4207  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
4208  }
4209 
4210  /* Remember start of field on both input and output sides */
4211  start_ptr = cur_ptr;
4212  cstate->raw_fields[fieldno] = output_ptr;
4213 
4214  /*
4215  * Scan data for field,
4216  *
4217  * The loop starts in "not quote" mode and then toggles between that
4218  * and "in quote" mode. The loop exits normally if it is in "not
4219  * quote" mode and a delimiter or line end is seen.
4220  */
4221  for (;;)
4222  {
4223  char c;
4224 
4225  /* Not in quote */
4226  for (;;)
4227  {
4228  end_ptr = cur_ptr;
4229  if (cur_ptr >= line_end_ptr)
4230  goto endfield;
4231  c = *cur_ptr++;
4232  /* unquoted field delimiter */
4233  if (c == delimc)
4234  {
4235  found_delim = true;
4236  goto endfield;
4237  }
4238  /* start of quoted field (or part of field) */
4239  if (c == quotec)
4240  {
4241  saw_quote = true;
4242  break;
4243  }
4244  /* Add c to output string */
4245  *output_ptr++ = c;
4246  }
4247 
4248  /* In quote */
4249  for (;;)
4250  {
4251  end_ptr = cur_ptr;
4252  if (cur_ptr >= line_end_ptr)
4253  ereport(ERROR,
4254  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4255  errmsg("unterminated CSV quoted field")));
4256 
4257  c = *cur_ptr++;
4258 
4259  /* escape within a quoted field */
4260  if (c == escapec)
4261  {
4262  /*
4263  * peek at the next char if available, and escape it if it
4264  * is an escape char or a quote char
4265  */
4266  if (cur_ptr < line_end_ptr)
4267  {
4268  char nextc = *cur_ptr;
4269 
4270  if (nextc == escapec || nextc == quotec)
4271  {
4272  *output_ptr++ = nextc;
4273  cur_ptr++;
4274  continue;
4275  }
4276  }
4277  }
4278 
4279  /*
4280  * end of quoted field. Must do this test after testing for
4281  * escape in case quote char and escape char are the same
4282  * (which is the common case).
4283  */
4284  if (c == quotec)
4285  break;
4286 
4287  /* Add c to output string */
4288  *output_ptr++ = c;
4289  }
4290  }
4291 endfield:
4292 
4293  /* Terminate attribute value in output area */
4294  *output_ptr++ = '\0';
4295 
4296  /* Check whether raw input matched null marker */
4297  input_len = end_ptr - start_ptr;
4298  if (!saw_quote && input_len == cstate->null_print_len &&
4299  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4300  cstate->raw_fields[fieldno] = NULL;
4301 
4302  fieldno++;
4303  /* Done if we hit EOL instead of a delim */
4304  if (!found_delim)
4305  break;
4306  }
4307 
4308  /* Clean up state of attribute_buf */
4309  output_ptr--;
4310  Assert(*output_ptr == '\0');
4311  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4312 
4313  return fieldno;
4314 }
char ** raw_fields
Definition: copy.c:188
StringInfoData line_buf
Definition: copy.c:197
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:123
char * null_print
Definition: copy.c:120
#define ERROR
Definition: elog.h:43
char * c
char * quote
Definition: copy.c:124
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:187
char * escape
Definition: copy.c:125
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:121
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:183
static int CopyReadAttributesText ( CopyState  cstate)
static

Definition at line 3924 of file copy.c.

References CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, GetDecimalFromHex(), IS_HIGHBIT_SET, ISOCTAL, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, OCTVALUE, pg_verifymbstr(), CopyStateData::raw_fields, repalloc(), resetStringInfo(), and val.

Referenced by NextCopyFromRawFields().

3925 {
3926  char delimc = cstate->delim[0];
3927  int fieldno;
3928  char *output_ptr;
3929  char *cur_ptr;
3930  char *line_end_ptr;
3931 
3932  /*
3933  * We need a special case for zero-column tables: check that the input
3934  * line is empty, and return.
3935  */
3936  if (cstate->max_fields <= 0)
3937  {
3938  if (cstate->line_buf.len != 0)
3939  ereport(ERROR,
3940  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3941  errmsg("extra data after last expected column")));
3942  return 0;
3943  }
3944 
3945  resetStringInfo(&cstate->attribute_buf);
3946 
3947  /*
3948  * The de-escaped attributes will certainly not be longer than the input
3949  * data line, so we can just force attribute_buf to be large enough and
3950  * then transfer data without any checks for enough space. We need to do
3951  * it this way because enlarging attribute_buf mid-stream would invalidate
3952  * pointers already stored into cstate->raw_fields[].
3953  */
3954  if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
3955  enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
3956  output_ptr = cstate->attribute_buf.data;
3957 
3958  /* set pointer variables for loop */
3959  cur_ptr = cstate->line_buf.data;
3960  line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
3961 
3962  /* Outer loop iterates over fields */
3963  fieldno = 0;
3964  for (;;)
3965  {
3966  bool found_delim = false;
3967  char *start_ptr;
3968  char *end_ptr;
3969  int input_len;
3970  bool saw_non_ascii = false;
3971 
3972  /* Make sure there is enough space for the next value */
3973  if (fieldno >= cstate->max_fields)
3974  {
3975  cstate->max_fields *= 2;
3976  cstate->raw_fields =
3977  repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
3978  }
3979 
3980  /* Remember start of field on both input and output sides */
3981  start_ptr = cur_ptr;
3982  cstate->raw_fields[fieldno] = output_ptr;
3983 
3984  /*
3985  * Scan data for field.
3986  *
3987  * Note that in this loop, we are scanning to locate the end of field
3988  * and also speculatively performing de-escaping. Once we find the
3989  * end-of-field, we can match the raw field contents against the null
3990  * marker string. Only after that comparison fails do we know that
3991  * de-escaping is actually the right thing to do; therefore we *must
3992  * not* throw any syntax errors before we've done the null-marker
3993  * check.
3994  */
3995  for (;;)
3996  {
3997  char c;
3998 
3999  end_ptr = cur_ptr;
4000  if (cur_ptr >= line_end_ptr)
4001  break;
4002  c = *cur_ptr++;
4003  if (c == delimc)
4004  {
4005  found_delim = true;
4006  break;
4007  }
4008  if (c == '\\')
4009  {
4010  if (cur_ptr >= line_end_ptr)
4011  break;
4012  c = *cur_ptr++;
4013  switch (c)
4014  {
4015  case '0':
4016  case '1':
4017  case '2':
4018  case '3':
4019  case '4':
4020  case '5':
4021  case '6':
4022  case '7':
4023  {
4024  /* handle \013 */
4025  int val;
4026 
4027  val = OCTVALUE(c);
4028  if (cur_ptr < line_end_ptr)
4029  {
4030  c = *cur_ptr;
4031  if (ISOCTAL(c))
4032  {
4033  cur_ptr++;
4034  val = (val << 3) + OCTVALUE(c);
4035  if (cur_ptr < line_end_ptr)
4036  {
4037  c = *cur_ptr;
4038  if (ISOCTAL(c))
4039  {
4040  cur_ptr++;
4041  val = (val << 3) + OCTVALUE(c);
4042  }
4043  }
4044  }
4045  }
4046  c = val & 0377;
4047  if (c == '\0' || IS_HIGHBIT_SET(c))
4048  saw_non_ascii = true;
4049  }
4050  break;
4051  case 'x':
4052  /* Handle \x3F */
4053  if (cur_ptr < line_end_ptr)
4054  {
4055  char hexchar = *cur_ptr;
4056 
4057  if (isxdigit((unsigned char) hexchar))
4058  {
4059  int val = GetDecimalFromHex(hexchar);
4060 
4061  cur_ptr++;
4062  if (cur_ptr < line_end_ptr)
4063  {
4064  hexchar = *cur_ptr;
4065  if (isxdigit((unsigned char) hexchar))
4066  {
4067  cur_ptr++;
4068  val = (val << 4) + GetDecimalFromHex(hexchar);
4069  }
4070  }
4071  c = val & 0xff;
4072  if (c == '\0' || IS_HIGHBIT_SET(c))
4073  saw_non_ascii = true;
4074  }
4075  }
4076  break;
4077  case 'b':
4078  c = '\b';
4079  break;
4080  case 'f':
4081  c = '\f';
4082  break;
4083  case 'n':
4084  c = '\n';
4085  break;
4086  case 'r':
4087  c = '\r';
4088  break;
4089  case 't':
4090  c = '\t';
4091  break;
4092  case 'v':
4093  c = '\v';
4094  break;
4095 
4096  /*
4097  * in all other cases, take the char after '\'
4098  * literally
4099  */
4100  }
4101  }
4102 
4103  /* Add c to output string */
4104  *output_ptr++ = c;
4105  }
4106 
4107  /* Check whether raw input matched null marker */
4108  input_len = end_ptr - start_ptr;
4109  if (input_len == cstate->null_print_len &&
4110  strncmp(start_ptr, cstate->null_print, input_len) == 0)
4111  cstate->raw_fields[fieldno] = NULL;
4112  else
4113  {
4114  /*
4115  * At this point we know the field is supposed to contain data.
4116  *
4117  * If we de-escaped any non-7-bit-ASCII chars, make sure the
4118  * resulting string is valid data for the db encoding.
4119  */
4120  if (saw_non_ascii)
4121  {
4122  char *fld = cstate->raw_fields[fieldno];
4123 
4124  pg_verifymbstr(fld, output_ptr - fld, false);
4125  }
4126  }
4127 
4128  /* Terminate attribute value in output area */
4129  *output_ptr++ = '\0';
4130 
4131  fieldno++;
4132  /* Done if we hit EOL instead of a delim */
4133  if (!found_delim)
4134  break;
4135  }
4136 
4137  /* Clean up state of attribute_buf */
4138  output_ptr--;
4139  Assert(*output_ptr == '\0');
4140  cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
4141 
4142  return fieldno;
4143 }
static int GetDecimalFromHex(char hex)
Definition: copy.c:3896
char ** raw_fields
Definition: copy.c:188
#define ISOCTAL(c)
Definition: copy.c:53
#define OCTVALUE(c)
Definition: copy.c:54
StringInfoData line_buf
Definition: copy.c:197
int errcode(int sqlerrcode)
Definition: elog.c:575
char * delim
Definition: copy.c:123
char * null_print
Definition: copy.c:120
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
#define ERROR
Definition: elog.h:43
char * c
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int max_fields
Definition: copy.c:187
#define ereport(elevel, rest)
Definition: elog.h:122
int null_print_len
Definition: copy.c:121
#define NULL
Definition: c.h:229
#define Assert(condition)
Definition: c.h:675
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:963
int errmsg(const char *fmt,...)
Definition: elog.c:797
bool pg_verifymbstr(const char *mbstr, int len, bool noError)
Definition: wchar.c:1866
StringInfoData attribute_buf
Definition: copy.c:183
long val
Definition: informix.c:689
static Datum CopyReadBinaryAttribute ( CopyState  cstate,
int  column_no,
FmgrInfo flinfo,
Oid  typioparam,
int32  typmod,
bool isnull 
)
static

Definition at line 4321 of file copy.c.

References CopyStateData::attribute_buf, CopyGetData(), CopyGetInt32(), StringInfoData::cursor, StringInfoData::data, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, StringInfoData::len, NULL, ReceiveFunctionCall(), resetStringInfo(), and result.

Referenced by NextCopyFrom().

4325 {
4326  int32 fld_size;
4327  Datum result;
4328 
4329  if (!CopyGetInt32(cstate, &fld_size))
4330  ereport(ERROR,
4331  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4332  errmsg("unexpected EOF in COPY data")));
4333  if (fld_size == -1)
4334  {
4335  *isnull = true;
4336  return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
4337  }
4338  if (fld_size < 0)
4339  ereport(ERROR,
4340  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4341  errmsg("invalid field size")));
4342 
4343  /* reset attribute_buf to empty, and load raw data in it */
4344  resetStringInfo(&cstate->attribute_buf);
4345 
4346  enlargeStringInfo(&cstate->attribute_buf, fld_size);
4347  if (CopyGetData(cstate, cstate->attribute_buf.data,
4348  fld_size, fld_size) != fld_size)
4349  ereport(ERROR,
4350  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
4351  errmsg("unexpected EOF in COPY data")));
4352 
4353  cstate->attribute_buf.len = fld_size;
4354  cstate->attribute_buf.data[fld_size] = '\0';
4355 
4356  /* Call the column type's binary input converter */
4357  result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
4358  typioparam, typmod);
4359 
4360  /* Trouble if it didn't eat the whole buffer */
4361  if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
4362  ereport(ERROR,
4363  (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
4364  errmsg("incorrect binary data format")));
4365 
4366  *isnull = false;
4367  return result;
4368 }
static bool CopyGetInt32(CopyState cstate, int32 *val)
Definition: copy.c:682
int errcode(int sqlerrcode)
Definition: elog.c:575
return result
Definition: formatting.c:1632
signed int int32
Definition: c.h:256
#define ERROR
Definition: elog.h:43
static int CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
Definition: copy.c:556
Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, Oid typioparam, int32 typmod)
Definition: fmgr.c:1681
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:245
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
uintptr_t Datum
Definition: postgres.h:372
#define NULL
Definition: c.h:229
int errmsg(const char *fmt,...)
Definition: elog.c:797
StringInfoData attribute_buf
Definition: copy.c:183
static bool CopyReadLine ( CopyState  cstate)
static

Definition at line 3448 of file copy.c.

References appendBinaryStringInfo(), Assert, CopyStateData::copy_dest, COPY_NEW_FE, CopyLoadRawBuf(), CopyReadLineText(), StringInfoData::data, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, CopyStateData::file_encoding, StringInfoData::len, CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, pfree(), pg_any_to_server(), CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, resetStringInfo(), and result.

Referenced by NextCopyFromRawFields().

3449 {
3450  bool result;
3451 
3452  resetStringInfo(&cstate->line_buf);
3453  cstate->line_buf_valid = true;
3454 
3455  /* Mark that encoding conversion hasn't occurred yet */
3456  cstate->line_buf_converted = false;
3457 
3458  /* Parse data and transfer into line_buf */
3459  result = CopyReadLineText(cstate);
3460 
3461  if (result)
3462  {
3463  /*
3464  * Reached EOF. In protocol version 3, we should ignore anything
3465  * after \. up to the protocol end of copy data. (XXX maybe better
3466  * not to treat \. as special?)
3467  */
3468  if (cstate->copy_dest == COPY_NEW_FE)
3469  {
3470  do
3471  {
3472  cstate->raw_buf_index = cstate->raw_buf_len;
3473  } while (CopyLoadRawBuf(cstate));
3474  }
3475  }
3476  else
3477  {
3478  /*
3479  * If we didn't hit EOF, then we must have transferred the EOL marker
3480  * to line_buf along with the data. Get rid of it.
3481  */
3482  switch (cstate->eol_type)
3483  {
3484  case EOL_NL:
3485  Assert(cstate->line_buf.len >= 1);
3486  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3487  cstate->line_buf.len--;
3488  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3489  break;
3490  case EOL_CR:
3491  Assert(cstate->line_buf.len >= 1);
3492  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
3493  cstate->line_buf.len--;
3494  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3495  break;
3496  case EOL_CRNL:
3497  Assert(cstate->line_buf.len >= 2);
3498  Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
3499  Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
3500  cstate->line_buf.len -= 2;
3501  cstate->line_buf.data[cstate->line_buf.len] = '\0';
3502  break;
3503  case EOL_UNKNOWN:
3504  /* shouldn't get here */
3505  Assert(false);
3506  break;
3507  }
3508  }
3509 
3510  /* Done reading the line. Convert it to server encoding. */
3511  if (cstate->need_transcoding)
3512  {
3513  char *cvt;
3514 
3515  cvt = pg_any_to_server(cstate->line_buf.data,
3516  cstate->line_buf.len,
3517  cstate->file_encoding);
3518  if (cvt != cstate->line_buf.data)
3519  {
3520  /* transfer converted data back to line_buf */
3521  resetStringInfo(&cstate->line_buf);
3522  appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
3523  pfree(cvt);
3524  }
3525  }
3526 
3527  /* Now it's safe to use the buffer in error messages */
3528  cstate->line_buf_converted = true;
3529 
3530  return result;
3531 }
static bool CopyReadLineText(CopyState cstate)
Definition: copy.c:3537
Definition: copy.c:74
bool need_transcoding
Definition: copy.c:105
StringInfoData line_buf
Definition: copy.c:197
int raw_buf_index
Definition: copy.c:210
bool line_buf_valid
Definition: copy.c:199
return result
Definition: formatting.c:1632
bool line_buf_converted
Definition: copy.c:198
CopyDest copy_dest
Definition: copy.c:98
void pfree(void *pointer)
Definition: mcxt.c:950
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:736
Definition: copy.c:75
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
int raw_buf_len
Definition: copy.c:211
Definition: copy.c:76
int file_encoding
Definition: copy.c:104
EolType eol_type
Definition: copy.c:103
#define Assert(condition)
Definition: c.h:675
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:572
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static bool CopyReadLineText ( CopyState  cstate)
static

Definition at line 3537 of file copy.c.

References appendBinaryStringInfo(), CopyLoadRawBuf(), CopyStateData::csv_mode, CopyStateData::cur_lineno, CopyStateData::encoding_embeds_ascii, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errhint(), errmsg(), ERROR, CopyStateData::escape, CopyStateData::file_encoding, IF_NEED_REFILL_AND_EOF_BREAK, IF_NEED_REFILL_AND_NOT_EOF_CONTINUE, IS_HIGHBIT_SET, CopyStateData::line_buf, NO_END_OF_COPY_GOTO, pg_encoding_mblen(), CopyStateData::quote, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, REFILL_LINEBUF, and result.

Referenced by CopyReadLine().

3538 {
3539  char *copy_raw_buf;
3540  int raw_buf_ptr;
3541  int copy_buf_len;
3542  bool need_data = false;
3543  bool hit_eof = false;
3544  bool result = false;
3545  char mblen_str[2];
3546 
3547  /* CSV variables */
3548  bool first_char_in_line = true;
3549  bool in_quote = false,
3550  last_was_esc = false;
3551  char quotec = '\0';
3552  char escapec = '\0';
3553 
3554  if (cstate->csv_mode)
3555  {
3556  quotec = cstate->quote[0];
3557  escapec = cstate->escape[0];
3558  /* ignore special escape processing if it's the same as quotec */
3559  if (quotec == escapec)
3560  escapec = '\0';
3561  }
3562 
3563  mblen_str[1] = '\0';
3564 
3565  /*
3566  * The objective of this loop is to transfer the entire next input line
3567  * into line_buf. Hence, we only care for detecting newlines (\r and/or
3568  * \n) and the end-of-copy marker (\.).
3569  *
3570  * In CSV mode, \r and \n inside a quoted field are just part of the data
3571  * value and are put in line_buf. We keep just enough state to know if we
3572  * are currently in a quoted field or not.
3573  *
3574  * These four characters, and the CSV escape and quote characters, are
3575  * assumed the same in frontend and backend encodings.
3576  *
3577  * For speed, we try to move data from raw_buf to line_buf in chunks
3578  * rather than one character at a time. raw_buf_ptr points to the next
3579  * character to examine; any characters from raw_buf_index to raw_buf_ptr
3580  * have been determined to be part of the line, but not yet transferred to
3581  * line_buf.
3582  *
3583  * For a little extra speed within the loop, we copy raw_buf and
3584  * raw_buf_len into local variables.
3585  */
3586  copy_raw_buf = cstate->raw_buf;
3587  raw_buf_ptr = cstate->raw_buf_index;
3588  copy_buf_len = cstate->raw_buf_len;
3589 
3590  for (;;)
3591  {
3592  int prev_raw_ptr;
3593  char c;
3594 
3595  /*
3596  * Load more data if needed. Ideally we would just force four bytes
3597  * of read-ahead and avoid the many calls to
3598  * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
3599  * does not allow us to read too far ahead or we might read into the
3600  * next data, so we read-ahead only as far we know we can. One
3601  * optimization would be to read-ahead four byte here if
3602  * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
3603  * considering the size of the buffer.
3604  */
3605  if (raw_buf_ptr >= copy_buf_len || need_data)
3606  {
3608 
3609  /*
3610  * Try to read some more data. This will certainly reset
3611  * raw_buf_index to zero, and raw_buf_ptr must go with it.
3612  */
3613  if (!CopyLoadRawBuf(cstate))
3614  hit_eof = true;
3615  raw_buf_ptr = 0;
3616  copy_buf_len = cstate->raw_buf_len;
3617 
3618  /*
3619  * If we are completely out of data, break out of the loop,
3620  * reporting EOF.
3621  */
3622  if (copy_buf_len <= 0)
3623  {
3624  result = true;
3625  break;
3626  }
3627  need_data = false;
3628  }
3629 
3630  /* OK to fetch a character */
3631  prev_raw_ptr = raw_buf_ptr;
3632  c = copy_raw_buf[raw_buf_ptr++];
3633 
3634  if (cstate->csv_mode)
3635  {
3636  /*
3637  * If character is '\\' or '\r', we may need to look ahead below.
3638  * Force fetch of the next character if we don't already have it.
3639  * We need to do this before changing CSV state, in case one of
3640  * these characters is also the quote or escape character.
3641  *
3642  * Note: old-protocol does not like forced prefetch, but it's OK
3643  * here since we cannot validly be at EOF.
3644  */
3645  if (c == '\\' || c == '\r')
3646  {
3648  }
3649 
3650  /*
3651  * Dealing with quotes and escapes here is mildly tricky. If the
3652  * quote char is also the escape char, there's no problem - we
3653  * just use the char as a toggle. If they are different, we need
3654  * to ensure that we only take account of an escape inside a
3655  * quoted field and immediately preceding a quote char, and not
3656  * the second in an escape-escape sequence.
3657  */
3658  if (in_quote && c == escapec)
3659  last_was_esc = !last_was_esc;
3660  if (c == quotec && !last_was_esc)
3661  in_quote = !in_quote;
3662  if (c != escapec)
3663  last_was_esc = false;
3664 
3665  /*
3666  * Updating the line count for embedded CR and/or LF chars is
3667  * necessarily a little fragile - this test is probably about the
3668  * best we can do. (XXX it's arguable whether we should do this
3669  * at all --- is cur_lineno a physical or logical count?)
3670  */
3671  if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
3672  cstate->cur_lineno++;
3673  }
3674 
3675  /* Process \r */
3676  if (c == '\r' && (!cstate->csv_mode || !in_quote))
3677  {
3678  /* Check for \r\n on first line, _and_ handle \r\n. */
3679  if (cstate->eol_type == EOL_UNKNOWN ||
3680  cstate->eol_type == EOL_CRNL)
3681  {
3682  /*
3683  * If need more data, go back to loop top to load it.
3684  *
3685  * Note that if we are at EOF, c will wind up as '\0' because
3686  * of the guaranteed pad of raw_buf.
3687  */
3689 
3690  /* get next char */
3691  c = copy_raw_buf[raw_buf_ptr];
3692 
3693  if (c == '\n')
3694  {
3695  raw_buf_ptr++; /* eat newline */
3696  cstate->eol_type = EOL_CRNL; /* in case not set yet */
3697  }
3698  else
3699  {
3700  /* found \r, but no \n */
3701  if (cstate->eol_type == EOL_CRNL)
3702  ereport(ERROR,
3703  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3704  !cstate->csv_mode ?
3705  errmsg("literal carriage return found in data") :
3706  errmsg("unquoted carriage return found in data"),
3707  !cstate->csv_mode ?
3708  errhint("Use \"\\r\" to represent carriage return.") :
3709  errhint("Use quoted CSV field to represent carriage return.")));
3710 
3711  /*
3712  * if we got here, it is the first line and we didn't find
3713  * \n, so don't consume the peeked character
3714  */
3715  cstate->eol_type = EOL_CR;
3716  }
3717  }
3718  else if (cstate->eol_type == EOL_NL)
3719  ereport(ERROR,
3720  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3721  !cstate->csv_mode ?
3722  errmsg("literal carriage return found in data") :
3723  errmsg("unquoted carriage return found in data"),
3724  !cstate->csv_mode ?
3725  errhint("Use \"\\r\" to represent carriage return.") :
3726  errhint("Use quoted CSV field to represent carriage return.")));
3727  /* If reach here, we have found the line terminator */
3728  break;
3729  }
3730 
3731  /* Process \n */
3732  if (c == '\n' && (!cstate->csv_mode || !in_quote))
3733  {
3734  if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
3735  ereport(ERROR,
3736  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3737  !cstate->csv_mode ?
3738  errmsg("literal newline found in data") :
3739  errmsg("unquoted newline found in data"),
3740  !cstate->csv_mode ?
3741  errhint("Use \"\\n\" to represent newline.") :
3742  errhint("Use quoted CSV field to represent newline.")));
3743  cstate->eol_type = EOL_NL; /* in case not set yet */
3744  /* If reach here, we have found the line terminator */
3745  break;
3746  }
3747 
3748  /*
3749  * In CSV mode, we only recognize \. alone on a line. This is because
3750  * \. is a valid CSV data value.
3751  */
3752  if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
3753  {
3754  char c2;
3755 
3758 
3759  /* -----
3760  * get next character
3761  * Note: we do not change c so if it isn't \., we can fall
3762  * through and continue processing for file encoding.
3763  * -----
3764  */
3765  c2 = copy_raw_buf[raw_buf_ptr];
3766 
3767  if (c2 == '.')
3768  {
3769  raw_buf_ptr++; /* consume the '.' */
3770 
3771  /*
3772  * Note: if we loop back for more data here, it does not
3773  * matter that the CSV state change checks are re-executed; we
3774  * will come back here with no important state changed.
3775  */
3776  if (cstate->eol_type == EOL_CRNL)
3777  {
3778  /* Get the next character */
3780  /* if hit_eof, c2 will become '\0' */
3781  c2 = copy_raw_buf[raw_buf_ptr++];
3782 
3783  if (c2 == '\n')
3784  {
3785  if (!cstate->csv_mode)
3786  ereport(ERROR,
3787  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3788  errmsg("end-of-copy marker does not match previous newline style")));
3789  else
3791  }
3792  else if (c2 != '\r')
3793  {
3794  if (!cstate->csv_mode)
3795  ereport(ERROR,
3796  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3797  errmsg("end-of-copy marker corrupt")));
3798  else
3800  }
3801  }
3802 
3803  /* Get the next character */
3805  /* if hit_eof, c2 will become '\0' */
3806  c2 = copy_raw_buf[raw_buf_ptr++];
3807 
3808  if (c2 != '\r' && c2 != '\n')
3809  {
3810  if (!cstate->csv_mode)
3811  ereport(ERROR,
3812  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3813  errmsg("end-of-copy marker corrupt")));
3814  else
3816  }
3817 
3818  if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
3819  (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
3820  (cstate->eol_type == EOL_CR && c2 != '\r'))
3821  {
3822  ereport(ERROR,
3823  (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3824  errmsg("end-of-copy marker does not match previous newline style")));
3825  }
3826 
3827  /*
3828  * Transfer only the data before the \. into line_buf, then
3829  * discard the data and the \. sequence.
3830  */
3831  if (prev_raw_ptr > cstate->raw_buf_index)
3833  cstate->raw_buf + cstate->raw_buf_index,
3834  prev_raw_ptr - cstate->raw_buf_index);
3835  cstate->raw_buf_index = raw_buf_ptr;
3836  result = true; /* report EOF */
3837  break;
3838  }
3839  else if (!cstate->csv_mode)
3840 
3841  /*
3842  * If we are here, it means we found a backslash followed by
3843  * something other than a period. In non-CSV mode, anything
3844  * after a backslash is special, so we skip over that second
3845  * character too. If we didn't do that \\. would be
3846  * considered an eof-of copy, while in non-CSV mode it is a
3847  * literal backslash followed by a period. In CSV mode,
3848  * backslashes are not special, so we want to process the
3849  * character after the backslash just like a normal character,
3850  * so we don't increment in those cases.
3851  */
3852  raw_buf_ptr++;
3853  }
3854 
3855  /*
3856  * This label is for CSV cases where \. appears at the start of a
3857  * line, but there is more text after it, meaning it was a data value.
3858  * We are more strict for \. in CSV mode because \. could be a data
3859  * value, while in non-CSV mode, \. cannot be a data value.
3860  */
3861 not_end_of_copy:
3862 
3863  /*
3864  * Process all bytes of a multi-byte character as a group.
3865  *
3866  * We only support multi-byte sequences where the first byte has the
3867  * high-bit set, so as an optimization we can avoid this block
3868  * entirely if it is not set.
3869  */
3870  if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
3871  {
3872  int mblen;
3873 
3874  mblen_str[0] = c;
3875  /* All our encodings only read the first byte to get the length */
3876  mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
3878  IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
3879  raw_buf_ptr += mblen - 1;
3880  }
3881  first_char_in_line = false;
3882  } /* end of outer loop */
3883 
3884  /*
3885  * Transfer any still-uncopied data to line_buf.
3886  */
3888 
3889  return result;
3890 }
bool csv_mode
Definition: copy.c:118
int errhint(const char *fmt,...)
Definition: elog.c:987
Definition: copy.c:74
StringInfoData line_buf
Definition: copy.c:197
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen)
Definition: copy.c:251
int raw_buf_index
Definition: copy.c:210
int errcode(int sqlerrcode)
Definition: elog.c:575
return result
Definition: formatting.c:1632
#define REFILL_LINEBUF
Definition: copy.c:268
#define IS_HIGHBIT_SET(ch)
Definition: c.h:973
#define ERROR
Definition: elog.h:43
bool encoding_embeds_ascii
Definition: copy.c:106
char * c
char * raw_buf
Definition: copy.c:209
static bool CopyLoadRawBuf(CopyState cstate)
Definition: copy.c:736
char * quote
Definition: copy.c:124
int pg_encoding_mblen(int encoding, const char *mbstr)
Definition: wchar.c:1785
Definition: copy.c:75
int raw_buf_len
Definition: copy.c:211
char * escape
Definition: copy.c:125
#define ereport(elevel, rest)
Definition: elog.h:122
Definition: copy.c:76
int file_encoding
Definition: copy.c:104
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
Definition: copy.c:239
EolType eol_type
Definition: copy.c:103
int errmsg(const char *fmt,...)
Definition: elog.c:797
#define NO_END_OF_COPY_GOTO
Definition: copy.c:281
int cur_lineno
Definition: copy.c:139
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static void CopySendChar ( CopyState  cstate,
char  c 
)
static

Definition at line 457 of file copy.c.

References appendStringInfoCharMacro, and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), CopyOneRowTo(), CopySendEndOfRow(), and CopyTo().

458 {
460 }
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:127
char * c
StringInfo fe_msgbuf
Definition: copy.c:100
static void CopySendData ( CopyState  cstate,
const void *  databuf,
int  datasize 
)
static

Definition at line 445 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyOneRowTo(), CopySendInt16(), CopySendInt32(), CopyTo(), and SendCopyEnd().

446 {
447  appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
448 }
StringInfo fe_msgbuf
Definition: copy.c:100
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static void CopySendEndOfRow ( CopyState  cstate)
static

Definition at line 463 of file copy.c.

References Assert, CopyStateData::binary, ClosePipeToProgram(), COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, CopySendChar(), CopySendString(), StringInfoData::data, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, FATAL, CopyStateData::fe_msgbuf, CopyStateData::is_program, StringInfoData::len, pq_putbytes(), pq_putmessage, and resetStringInfo().

Referenced by CopyOneRowTo(), CopyTo(), and SendCopyEnd().

464 {
465  StringInfo fe_msgbuf = cstate->fe_msgbuf;
466 
467  switch (cstate->copy_dest)
468  {
469  case COPY_FILE:
470  if (!cstate->binary)
471  {
472  /* Default line termination depends on platform */
473 #ifndef WIN32
474  CopySendChar(cstate, '\n');
475 #else
476  CopySendString(cstate, "\r\n");
477 #endif
478  }
479 
480  if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
481  cstate->copy_file) != 1 ||
482  ferror(cstate->copy_file))
483  {
484  if (cstate->is_program)
485  {
486  if (errno == EPIPE)
487  {
488  /*
489  * The pipe will be closed automatically on error at
490  * the end of transaction, but we might get a better
491  * error message from the subprocess' exit code than
492  * just "Broken Pipe"
493  */
494  ClosePipeToProgram(cstate);
495 
496  /*
497  * If ClosePipeToProgram() didn't throw an error, the
498  * program terminated normally, but closed the pipe
499  * first. Restore errno, and throw an error.
500  */
501  errno = EPIPE;
502  }
503  ereport(ERROR,
505  errmsg("could not write to COPY program: %m")));
506  }
507  else
508  ereport(ERROR,
510  errmsg("could not write to COPY file: %m")));
511  }
512  break;
513  case COPY_OLD_FE:
514  /* The FE/BE protocol uses \n as newline for all platforms */
515  if (!cstate->binary)
516  CopySendChar(cstate, '\n');
517 
518  if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
519  {
520  /* no hope of recovering connection sync, so FATAL */
521  ereport(FATAL,
522  (errcode(ERRCODE_CONNECTION_FAILURE),
523  errmsg("connection lost during COPY to stdout")));
524  }
525  break;
526  case COPY_NEW_FE:
527  /* The FE/BE protocol uses \n as newline for all platforms */
528  if (!cstate->binary)
529  CopySendChar(cstate, '\n');
530 
531  /* Dump the accumulated row as one CopyData message */
532  (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
533  break;
534  case COPY_CALLBACK:
535  Assert(false); /* Not yet supported. */
536  break;
537  }
538 
539  resetStringInfo(fe_msgbuf);
540 }
bool binary
Definition: copy.c:115
int errcode(int sqlerrcode)
Definition: elog.c:575
static void ClosePipeToProgram(CopyState cstate)
Definition: copy.c:1711
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
CopyDest copy_dest
Definition: copy.c:98
Definition: copy.c:62
#define ERROR
Definition: elog.h:43
#define FATAL
Definition: elog.h:52
int errcode_for_file_access(void)
Definition: elog.c:598
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:62
#define ereport(elevel, rest)
Definition: elog.h:122
bool is_program
Definition: copy.c:113
#define Assert(condition)
Definition: c.h:675
StringInfo fe_msgbuf
Definition: copy.c:100
static void CopySendString(CopyState cstate, const char *str)
Definition: copy.c:451
int errmsg(const char *fmt,...)
Definition: elog.c:797
FILE * copy_file
Definition: copy.c:99
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
int pq_putbytes(const char *s, size_t len)
Definition: pqcomm.c:1334
static void CopySendInt16 ( CopyState  cstate,
int16  val 
)
static

Definition at line 699 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

700 {
701  uint16 buf;
702 
703  buf = htons((uint16) val);
704  CopySendData(cstate, &buf, sizeof(buf));
705 }
unsigned short uint16
Definition: c.h:267
static char * buf
Definition: pg_test_fsync.c:66
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
long val
Definition: informix.c:689
static void CopySendInt32 ( CopyState  cstate,
int32  val 
)
static

Definition at line 668 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

669 {
670  uint32 buf;
671 
672  buf = htonl((uint32) val);
673  CopySendData(cstate, &buf, sizeof(buf));
674 }
static char * buf
Definition: pg_test_fsync.c:66
unsigned int uint32
Definition: c.h:268
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
long val
Definition: informix.c:689
static void CopySendString ( CopyState  cstate,
const char *  str 
)
static

Definition at line 451 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyOneRowTo(), and CopySendEndOfRow().

452 {
453  appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
454 }
StringInfo fe_msgbuf
Definition: copy.c:100
void appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
Definition: stringinfo.c:208
static uint64 CopyTo ( CopyState  cstate)
static

Definition at line 1940 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), CopyStateData::attnumlist, tupleDesc::attrs, CopyStateData::binary, BinarySignature, CHECK_FOR_INTERRUPTS, CopyAttributeOutCSV(), CopyOneRowTo(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), cur, CurrentMemoryContext, CopyStateData::delim, QueryDesc::dest, ExecutorRun(), CopyStateData::fe_msgbuf, CopyStateData::file_encoding, fmgr_info(), ForwardScanDirection, GetActiveSnapshot(), getTypeBinaryOutputInfo(), getTypeOutputInfo(), CopyStateData::header_line, heap_beginscan(), heap_deform_tuple(), heap_endscan(), heap_getnext(), HeapTupleGetOid, lfirst_int, list_length(), makeStringInfo(), MemoryContextDelete(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NULL, CopyStateData::null_print, CopyStateData::null_print_client, CopyStateData::null_print_len, CopyStateData::oids, CopyStateData::out_functions, palloc(), pfree(), pg_server_to_any(), CopyStateData::queryDesc, CopyStateData::rel, RelationGetDescr, CopyStateData::rowcontext, QueryDesc::tupDesc, and values.

Referenced by DoCopyTo().

1941 {
1942  TupleDesc tupDesc;
1943  int num_phys_attrs;
1944  Form_pg_attribute *attr;
1945  ListCell *cur;
1946  uint64 processed;
1947 
1948  if (cstate->rel)
1949  tupDesc = RelationGetDescr(cstate->rel);
1950  else
1951  tupDesc = cstate->queryDesc->tupDesc;
1952  attr = tupDesc->attrs;
1953  num_phys_attrs = tupDesc->natts;
1954  cstate->null_print_client = cstate->null_print; /* default */
1955 
1956  /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
1957  cstate->fe_msgbuf = makeStringInfo();
1958 
1959  /* Get info about the columns we need to process. */
1960  cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
1961  foreach(cur, cstate->attnumlist)
1962  {
1963  int attnum = lfirst_int(cur);
1964  Oid out_func_oid;
1965  bool isvarlena;
1966 
1967  if (cstate->binary)
1968  getTypeBinaryOutputInfo(attr[attnum - 1]->atttypid,
1969  &out_func_oid,
1970  &isvarlena);
1971  else
1972  getTypeOutputInfo(attr[attnum - 1]->atttypid,
1973  &out_func_oid,
1974  &isvarlena);
1975  fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
1976  }
1977 
1978  /*
1979  * Create a temporary memory context that we can reset once per row to
1980  * recover palloc'd memory. This avoids any problems with leaks inside
1981  * datatype output routines, and should be faster than retail pfree's
1982  * anyway. (We don't need a whole econtext as CopyFrom does.)
1983  */
1985  "COPY TO",
1987 
1988  if (cstate->binary)
1989  {
1990  /* Generate header for a binary copy */
1991  int32 tmp;
1992 
1993  /* Signature */
1994  CopySendData(cstate, BinarySignature, 11);
1995  /* Flags field */
1996  tmp = 0;
1997  if (cstate->oids)
1998  tmp |= (1 << 16);
1999  CopySendInt32(cstate, tmp);
2000  /* No header extension */
2001  tmp = 0;
2002  CopySendInt32(cstate, tmp);
2003  }
2004  else
2005  {
2006  /*
2007  * For non-binary copy, we need to convert null_print to file
2008  * encoding, because it will be sent directly with CopySendString.
2009  */
2010  if (cstate->need_transcoding)
2011  cstate->null_print_client = pg_server_to_any(cstate->null_print,
2012  cstate->null_print_len,
2013  cstate->file_encoding);
2014 
2015  /* if a header has been requested send the line */
2016  if (cstate->header_line)
2017  {
2018  bool hdr_delim = false;
2019 
2020  foreach(cur, cstate->attnumlist)
2021  {
2022  int attnum = lfirst_int(cur);
2023  char *colname;
2024 
2025  if (hdr_delim)
2026  CopySendChar(cstate, cstate->delim[0]);
2027  hdr_delim = true;
2028 
2029  colname = NameStr(attr[attnum - 1]->attname);
2030 
2031  CopyAttributeOutCSV(cstate, colname, false,
2032  list_length(cstate->attnumlist) == 1);
2033  }
2034 
2035  CopySendEndOfRow(cstate);
2036  }
2037  }
2038 
2039  if (cstate->rel)
2040  {
2041  Datum *values;
2042  bool *nulls;
2043  HeapScanDesc scandesc;
2044  HeapTuple tuple;
2045 
2046  values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
2047  nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));
2048 
2049  scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
2050 
2051  processed = 0;
2052  while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
2053  {
2055 
2056  /* Deconstruct the tuple ... faster than repeated heap_getattr */
2057  heap_deform_tuple(tuple, tupDesc, values, nulls);
2058 
2059  /* Format and send the data */
2060  CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
2061  processed++;
2062  }
2063 
2064  heap_endscan(scandesc);
2065 
2066  pfree(values);
2067  pfree(nulls);
2068  }
2069  else
2070  {
2071  /* run the plan --- the dest receiver will send tuples */
2072  ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
2073  processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
2074  }
2075 
2076  if (cstate->binary)
2077  {
2078  /* Generate trailer for a binary copy */
2079  CopySendInt16(cstate, -1);
2080  /* Need to flush out the trailer */
2081  CopySendEndOfRow(cstate);
2082  }
2083 
2085 
2086  return processed;
2087 }
Definition: fmgr.h:56
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:200
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2632
bool binary
Definition: copy.c:115
void heap_endscan(HeapScanDesc scan)
Definition: heapam.c:1578
#define RelationGetDescr(relation)
Definition: rel.h:429
bool need_transcoding
Definition: copy.c:105
List * attnumlist
Definition: copy.c:111
Definition: copy.c:215
StringInfo makeStringInfo(void)
Definition: stringinfo.c:28
Form_pg_attribute * attrs
Definition: tupdesc.h:74
Snapshot GetActiveSnapshot(void)
Definition: snapmgr.c:834
struct cursor * cur
Definition: ecpg.c:28
unsigned int Oid
Definition: postgres_ext.h:31
char * delim
Definition: copy.c:123
MemoryContext rowcontext
Definition: copy.c:152
int natts
Definition: tupdesc.h:73
char * pg_server_to_any(const char *s, int len, int encoding)
Definition: mbutils.c:645
signed int int32
Definition: c.h:256
static void CopySendChar(CopyState cstate, char c)
Definition: copy.c:457
char * null_print
Definition: copy.c:120
Relation rel
Definition: copy.c:109
void pfree(void *pointer)
Definition: mcxt.c:950
static void CopySendInt16(CopyState cstate, int16 val)
Definition: copy.c:699
#define lfirst_int(lc)
Definition: pg_list.h:107
void ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once)
Definition: execMain.c:296
static void CopyAttributeOutCSV(CopyState cstate, char *string, bool use_quote, bool single_attr)
Definition: copy.c:4533
void fmgr_info(Oid functionId, FmgrInfo *finfo)
Definition: fmgr.c:127
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:165
QueryDesc * queryDesc
Definition: copy.c:110
static const char BinarySignature[11]
Definition: copy.c:288
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:187
MemoryContext CurrentMemoryContext
Definition: mcxt.c:37
int null_print_len
Definition: copy.c:121
int file_encoding
Definition: copy.c:104
TupleDesc tupDesc
Definition: execdesc.h:47
void getTypeBinaryOutputInfo(Oid type, Oid *typSend, bool *typIsVarlena)
Definition: lsyscache.c:2698
MemoryContext AllocSetContextCreate(MemoryContext parent, const char *name, Size minContextSize, Size initBlockSize, Size maxBlockSize)
Definition: aset.c:322
bool header_line
Definition: copy.c:119
uintptr_t Datum
Definition: postgres.h:372
HeapTuple heap_getnext(HeapScanDesc scan, ScanDirection direction)
Definition: heapam.c:1794
#define NULL
Definition: c.h:229
static void CopySendData(CopyState cstate, const void *databuf, int datasize)
Definition: copy.c:445
StringInfo fe_msgbuf
Definition: copy.c:100
static int list_length(const List *l)
Definition: pg_list.h:89
static void CopySendInt32(CopyState cstate, int32 val)
Definition: copy.c:668
void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull)
Definition: heaptuple.c:933
static Datum values[MAXATTR]
Definition: bootstrap.c:163
DestReceiver * dest
Definition: execdesc.h:41
FmgrInfo * out_functions
Definition: copy.c:151
void * palloc(Size size)
Definition: mcxt.c:849
static void CopySendEndOfRow(CopyState cstate)
Definition: copy.c:463
#define NameStr(name)
Definition: c.h:499
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
char * null_print_client
Definition: copy.c:122
#define HeapTupleGetOid(tuple)
Definition: htup_details.h:695
HeapScanDesc heap_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key)
Definition: heapam.c:1391
static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
Definition: copy.c:2093
bool oids
Definition: copy.c:116
DestReceiver* CreateCopyDestReceiver ( void  )

Definition at line 4742 of file copy.c.

References copy_dest_destroy(), copy_dest_receive(), copy_dest_shutdown(), copy_dest_startup(), DestCopyOut, NULL, and palloc().

Referenced by CreateDestReceiver().

4743 {
4744  DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
4745 
4746  self->pub.receiveSlot = copy_dest_receive;
4747  self->pub.rStartup = copy_dest_startup;
4748  self->pub.rShutdown = copy_dest_shutdown;
4749  self->pub.rDestroy = copy_dest_destroy;
4750  self->pub.mydest = DestCopyOut;
4751 
4752  self->cstate = NULL; /* will be set later */
4753  self->processed = 0;
4754 
4755  return (DestReceiver *) self;
4756 }
Definition: copy.c:215
static void copy_dest_destroy(DestReceiver *self)
Definition: copy.c:4733
static void copy_dest_shutdown(DestReceiver *self)
Definition: copy.c:4724
static void copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
Definition: copy.c:4696
static bool copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
Definition: copy.c:4705
#define NULL
Definition: c.h:229
void * palloc(Size size)
Definition: mcxt.c:849
void DoCopy ( ParseState pstate,
const CopyStmt stmt,
int  stmt_location,
int  stmt_len,
uint64 *  processed 
)

Definition at line 781 of file copy.c.

References AccessShareLock, ACL_INSERT, ACL_SELECT, addRangeTableEntryForRelation(), Assert, CopyStmt::attlist, BeginCopyFrom(), BeginCopyTo(), bms_add_member(), check_enable_rls(), CopyFrom(), CopyGetAttnums(), cur, DoCopyTo(), EndCopyFrom(), EndCopyTo(), ereport, errcode(), errhint(), errmsg(), ERROR, ExecCheckRTPerms(), ColumnRef::fields, CopyStmt::filename, FirstLowInvalidHeapAttributeNumber, SelectStmt::fromClause, get_namespace_name(), heap_close, heap_openrv(), ResTarget::indirection, RangeTblEntry::insertedCols, InvalidOid, CopyStmt::is_from, CopyStmt::is_program, lappend(), lfirst, lfirst_int, list_make1, ColumnRef::location, ResTarget::location, makeNode, makeRangeVar(), ResTarget::name, NIL, NoLock, NULL, CopyStmt::options, ParseState::p_rtable, PreventCommandIfParallelMode(), PreventCommandIfReadOnly(), pstrdup(), CopyStmt::query, RelationData::rd_islocaltemp, CopyStmt::relation, RelationGetDescr, RelationGetNamespace, RelationGetRelationName, RelationGetRelid, RangeTblEntry::relid, RangeTblEntry::requiredPerms, RLS_ENABLED, RowExclusiveLock, select, RangeTblEntry::selectedCols, RawStmt::stmt, RawStmt::stmt_len, RawStmt::stmt_location, superuser(), SelectStmt::targetList, ResTarget::val, and XactReadOnly.

Referenced by standard_ProcessUtility().

784 {
785  CopyState cstate;
786  bool is_from = stmt->is_from;
787  bool pipe = (stmt->filename == NULL);
788  Relation rel;
789  Oid relid;
790  RawStmt *query = NULL;
791 
792  /* Disallow COPY to/from file or program except to superusers. */
793  if (!pipe && !superuser())
794  {
795  if (stmt->is_program)
796  ereport(ERROR,
797  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
798  errmsg("must be superuser to COPY to or from an external program"),
799  errhint("Anyone can COPY to stdout or from stdin. "
800  "psql's \\copy command also works for anyone.")));
801  else
802  ereport(ERROR,
803  (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
804  errmsg("must be superuser to COPY to or from a file"),
805  errhint("Anyone can COPY to stdout or from stdin. "
806  "psql's \\copy command also works for anyone.")));
807  }
808 
809  if (stmt->relation)
810  {
811  TupleDesc tupDesc;
812  List *attnums;
813  ListCell *cur;
814  RangeTblEntry *rte;
815 
816  Assert(!stmt->query);
817 
818  /* Open and lock the relation, using the appropriate lock type. */
819  rel = heap_openrv(stmt->relation,
820  (is_from ? RowExclusiveLock : AccessShareLock));
821 
822  relid = RelationGetRelid(rel);
823 
824  rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
825  rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
826 
827  tupDesc = RelationGetDescr(rel);
828  attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
829  foreach(cur, attnums)
830  {
831  int attno = lfirst_int(cur) -
833 
834  if (is_from)
835  rte->insertedCols = bms_add_member(rte->insertedCols, attno);
836  else
837  rte->selectedCols = bms_add_member(rte->selectedCols, attno);
838  }
839  ExecCheckRTPerms(pstate->p_rtable, true);
840 
841  /*
842  * Permission check for row security policies.
843  *
844  * check_enable_rls will ereport(ERROR) if the user has requested
845  * something invalid and will otherwise indicate if we should enable
846  * RLS (returns RLS_ENABLED) or not for this COPY statement.
847  *
848  * If the relation has a row security policy and we are to apply it
849  * then perform a "query" copy and allow the normal query processing
850  * to handle the policies.
851  *
852  * If RLS is not enabled for this, then just fall through to the
853  * normal non-filtering relation handling.
854  */
855  if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
856  {
858  ColumnRef *cr;
859  ResTarget *target;
860  RangeVar *from;
861  List *targetList = NIL;
862 
863  if (is_from)
864  ereport(ERROR,
865  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
866  errmsg("COPY FROM not supported with row-level security"),
867  errhint("Use INSERT statements instead.")));
868 
869  /*
870  * Build target list
871  *
872  * If no columns are specified in the attribute list of the COPY
873  * command, then the target list is 'all' columns. Therefore, '*'
874  * should be used as the target list for the resulting SELECT
875  * statement.
876  *
877  * In the case that columns are specified in the attribute list,
878  * create a ColumnRef and ResTarget for each column and add them
879  * to the target list for the resulting SELECT statement.
880  */
881  if (!stmt->attlist)
882  {
883  cr = makeNode(ColumnRef);
885  cr->location = -1;
886 
887  target = makeNode(ResTarget);
888  target->name = NULL;
889  target->indirection = NIL;
890  target->val = (Node *) cr;
891  target->location = -1;
892 
893  targetList = list_make1(target);
894  }
895  else
896  {
897  ListCell *lc;
898 
899  foreach(lc, stmt->attlist)
900  {
901  /*
902  * Build the ColumnRef for each column. The ColumnRef
903  * 'fields' property is a String 'Value' node (see
904  * nodes/value.h) that corresponds to the column name
905  * respectively.
906  */
907  cr = makeNode(ColumnRef);
908  cr->fields = list_make1(lfirst(lc));
909  cr->location = -1;
910 
911  /* Build the ResTarget and add the ColumnRef to it. */
912  target = makeNode(ResTarget);
913  target->name = NULL;
914  target->indirection = NIL;
915  target->val = (Node *) cr;
916  target->location = -1;
917 
918  /* Add each column to the SELECT statement's target list */
919  targetList = lappend(targetList, target);
920  }
921  }
922 
923  /*
924  * Build RangeVar for from clause, fully qualified based on the
925  * relation which we have opened and locked.
926  */
929  -1);
930 
931  /* Build query */
932  select = makeNode(SelectStmt);
933  select->targetList = targetList;
934  select->fromClause = list_make1(from);
935 
936  query = makeNode(RawStmt);
937  query->stmt = (Node *) select;
938  query->stmt_location = stmt_location;
939  query->stmt_len = stmt_len;
940 
941  /*
942  * Close the relation for now, but keep the lock on it to prevent
943  * changes between now and when we start the query-based COPY.
944  *
945  * We'll reopen it later as part of the query-based COPY.
946  */
947  heap_close(rel, NoLock);
948  rel = NULL;
949  }
950  }
951  else
952  {
953  Assert(stmt->query);
954 
955  query = makeNode(RawStmt);
956  query->stmt = stmt->query;
957  query->stmt_location = stmt_location;
958  query->stmt_len = stmt_len;
959 
960  relid = InvalidOid;
961  rel = NULL;
962  }
963 
964  if (is_from)
965  {
966  Assert(rel);
967 
968  /* check read-only transaction and parallel mode */
969  if (XactReadOnly && !rel->rd_islocaltemp)
970  PreventCommandIfReadOnly("COPY FROM");
971  PreventCommandIfParallelMode("COPY FROM");
972 
973  cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
974  NULL, stmt->attlist, stmt->options);
975  *processed = CopyFrom(cstate); /* copy from file to database */
976  EndCopyFrom(cstate);
977  }
978  else
979  {
980  cstate = BeginCopyTo(pstate, rel, query, relid,
981  stmt->filename, stmt->is_program,
982  stmt->attlist, stmt->options);
983  *processed = DoCopyTo(cstate); /* copy from database to file */
984  EndCopyTo(cstate);
985  }
986 
987  /*
988  * Close the relation. If reading, we can release the AccessShareLock we
989  * got; if writing, we should hold the lock until end of transaction to
990  * ensure that updates will be committed before lock is released.
991  */
992  if (rel != NULL)
993  heap_close(rel, (is_from ? NoLock : AccessShareLock));
994 }