copy.c File Reference

#include "postgres.h"
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/sysattr.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/pg_type.h"
#include "commands/copy.h"
#include "commands/defrem.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "optimizer/clauses.h"
#include "optimizer/planner.h"
#include "nodes/makefuncs.h"
#include "parser/parse_relation.h"
#include "rewrite/rewriteHandler.h"
#include "storage/fd.h"
#include "tcop/tcopprot.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/portal.h"
#include "utils/rel.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"

Include dependency graph for copy.c:

Go to the source code of this file.

Data Structures
struct	CopyStateData
struct	DR_copy

Macros
#define	ISOCTAL(c)   (((c) >= '0') && ((c) <= '7'))
#define	OCTVALUE(c)   ((c) - '0')
#define	RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */
#define	IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen)
#define	IF_NEED_REFILL_AND_EOF_BREAK(extralen)
#define	REFILL_LINEBUF
#define	NO_END_OF_COPY_GOTO
#define	MAX_COPY_DATA_DISPLAY   100
#define	MAX_BUFFERED_TUPLES   1000
#define	DUMPSOFAR()

Typedefs
typedef enum CopyDest	CopyDest
typedef enum EolType	EolType
typedef struct CopyStateData	CopyStateData

Enumerations
enum	CopyDest { COPY_FILE, COPY_OLD_FE, COPY_NEW_FE, COPY_CALLBACK }
enum	EolType { EOL_UNKNOWN, EOL_NL, EOL_CR, EOL_CRNL }

Functions
static CopyState	BeginCopy (ParseState *pstate, bool is_from, Relation rel, RawStmt *raw_query, Oid queryRelId, List *attnamelist, List *options)
static void	EndCopy (CopyState cstate)
static void	ClosePipeToProgram (CopyState cstate)
static CopyState	BeginCopyTo (ParseState *pstate, Relation rel, RawStmt *query, Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options)
static void	EndCopyTo (CopyState cstate)
static uint64	DoCopyTo (CopyState cstate)
static uint64	CopyTo (CopyState cstate)
static void	CopyOneRowTo (CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
static void	CopyFromInsertBatch (CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, int firstBufferedLineNo)
static bool	CopyReadLine (CopyState cstate)
static bool	CopyReadLineText (CopyState cstate)
static int	CopyReadAttributesText (CopyState cstate)
static int	CopyReadAttributesCSV (CopyState cstate)
static Datum	CopyReadBinaryAttribute (CopyState cstate, int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull)
static void	CopyAttributeOutText (CopyState cstate, char *string)
static void	CopyAttributeOutCSV (CopyState cstate, char *string, bool use_quote, bool single_attr)
static List *	CopyGetAttnums (TupleDesc tupDesc, Relation rel, List *attnamelist)
static char *	limit_printout_length (const char *str)
static void	SendCopyBegin (CopyState cstate)
static void	ReceiveCopyBegin (CopyState cstate)
static void	SendCopyEnd (CopyState cstate)
static void	CopySendData (CopyState cstate, const void *databuf, int datasize)
static void	CopySendString (CopyState cstate, const char *str)
static void	CopySendChar (CopyState cstate, char c)
static void	CopySendEndOfRow (CopyState cstate)
static int	CopyGetData (CopyState cstate, void *databuf, int minread, int maxread)
static void	CopySendInt32 (CopyState cstate, int32 val)
static bool	CopyGetInt32 (CopyState cstate, int32 *val)
static void	CopySendInt16 (CopyState cstate, int16 val)
static bool	CopyGetInt16 (CopyState cstate, int16 *val)
static bool	CopyLoadRawBuf (CopyState cstate)
void	DoCopy (ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed)
void	ProcessCopyOptions (ParseState *pstate, CopyState cstate, bool is_from, List *options)
void	CopyFromErrorCallback (void *arg)
uint64	CopyFrom (CopyState cstate)
CopyState	BeginCopyFrom (ParseState *pstate, Relation rel, const char *filename, bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options)
bool	NextCopyFromRawFields (CopyState cstate, char ***fields, int *nfields)
bool	NextCopyFrom (CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid)
void	EndCopyFrom (CopyState cstate)
static int	GetDecimalFromHex (char hex)
static void	copy_dest_startup (DestReceiver *self, int operation, TupleDesc typeinfo)
static bool	copy_dest_receive (TupleTableSlot *slot, DestReceiver *self)
static void	copy_dest_shutdown (DestReceiver *self)
static void	copy_dest_destroy (DestReceiver *self)
DestReceiver *	CreateCopyDestReceiver (void)

Variables
static const char	BinarySignature [11] = "PGCOPY\n\377\r\n\0"

Macro Definition Documentation

#define DUMPSOFAR

(

)

Value:

do { \
        if (ptr > start) \
			CopySendData(cstate, start, ptr - start); \
    } while (0)

Definition at line 4388 of file copy.c.

Referenced by CopyAttributeOutCSV(), and CopyAttributeOutText().

#define IF_NEED_REFILL_AND_EOF_BREAK

(

extralen

)

Value:

if (1) \
{ \
    if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
    { \
        if (extralen) \
            raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
        /* backslash just before EOF, treat as data char */ \
		result = true; \
        break; \
    } \
} else ((void) 0)

Definition at line 251 of file copy.c.

Referenced by CopyReadLineText().

#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE

(

extralen

)

Value:

if (1) \
{ \
    if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
    { \
        raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
        need_data = true; \
        continue; \
    } \
} else ((void) 0)

Definition at line 239 of file copy.c.

Referenced by CopyReadLineText().

#define ISOCTAL

(

c

)

(((c) >= '0') && ((c) <= '7'))

Definition at line 53 of file copy.c.

Referenced by CopyReadAttributesText().

#define MAX_BUFFERED_TUPLES   1000

Referenced by CopyFrom().

#define MAX_COPY_DATA_DISPLAY   100

Referenced by limit_printout_length().

#define NO_END_OF_COPY_GOTO

Value:

if (1) \
{ \
    raw_buf_ptr = prev_raw_ptr + 1; \
    goto not_end_of_copy; \
} else ((void) 0)

Definition at line 281 of file copy.c.

Referenced by CopyReadLineText().

#define OCTVALUE

(

c

)

((c) - '0')

Definition at line 54 of file copy.c.

Referenced by CopyReadAttributesText().

#define RAW_BUF_SIZE   65536 /* we palloc RAW_BUF_SIZE+1 bytes */

Definition at line 208 of file copy.c.

Referenced by BeginCopyFrom(), and CopyLoadRawBuf().

#define REFILL_LINEBUF

Value:

if (1) \
{ \
    if (raw_buf_ptr > cstate->raw_buf_index) \
    { \
		appendBinaryStringInfo(&cstate->line_buf, \
                             cstate->raw_buf + cstate->raw_buf_index, \
                               raw_buf_ptr - cstate->raw_buf_index); \
        cstate->raw_buf_index = raw_buf_ptr; \
    } \
} else ((void) 0)

Definition at line 268 of file copy.c.

Referenced by CopyReadLineText().

Typedef Documentation

typedef enum CopyDest CopyDest

typedef struct CopyStateData CopyStateData

typedef enum EolType EolType

Enumeration Type Documentation

enum CopyDest

Enumerator
COPY_FILE
COPY_OLD_FE
COPY_NEW_FE
COPY_CALLBACK

Definition at line 60 of file copy.c.

{
    COPY_FILE,                  /* to/from file (or a piped program) */
    COPY_OLD_FE,                /* to/from frontend (2.0 protocol) */
    COPY_NEW_FE,                /* to/from frontend (3.0 protocol) */
    COPY_CALLBACK               /* to/from callback function */
} CopyDest;

enum EolType

Enumerator
EOL_UNKNOWN
EOL_NL
EOL_CR
EOL_CRNL

Definition at line 71 of file copy.c.

{
    EOL_UNKNOWN,
    EOL_NL,
    EOL_CR,
    EOL_CRNL
} EolType;

Function Documentation

static CopyState BeginCopy ( ParseState *  pstate, bool  is_from, Relation  rel, RawStmt *  raw_query, Oid  queryRelId, List *  attnamelist, List *  options  )

static

Definition at line 1372 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), Assert, CopyStateData::attnumlist, tupleDesc::attrs, CMD_DELETE, CMD_INSERT, CMD_SELECT, CMD_UPDATE, Query::commandType, CopyStateData::convert_select, CopyStateData::convert_select_flags, CopyStateData::convert_selectively, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copycontext, CopyGetAttnums(), copyObject, CreateDestReceiver(), CreateQueryDesc(), cur, CurrentMemoryContext, CURSOR_OPT_PARALLEL_OK, DestCopyOut, CopyStateData::encoding_embeds_ascii, ereport, errcode(), errmsg(), errmsg_internal(), ERROR, ExecSetupPartitionTupleRouting(), ExecutorStart(), CopyStateData::file_encoding, CopyStateData::force_notnull, CopyStateData::force_notnull_flags, CopyStateData::force_null, CopyStateData::force_null_flags, CopyStateData::force_quote, CopyStateData::force_quote_all, CopyStateData::force_quote_flags, GetActiveSnapshot(), GetDatabaseEncoding(), i, InvalidOid, InvalidSnapshot, IsA, lfirst_int, lfirst_node, linitial_node, list_length(), list_member_int(), list_member_oid(), MemoryContextSwitchTo(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NIL, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, CopyStateData::oids, ParseState::p_sourcetext, palloc0(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pg_analyze_and_rewrite(), pg_database_encoding_max_length(), PG_ENCODING_IS_CLIENT_ONLY, pg_get_client_encoding(), pg_plan_query(), ProcessCopyOptions(), PushCopiedSnapshot(), QSRC_NON_INSTEAD_RULE, QSRC_QUAL_INSTEAD_RULE, CopyStateData::queryDesc, Query::querySource, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, PlannedStmt::relationOids, RELKIND_PARTITIONED_TABLE, Query::returningList, QueryDesc::tupDesc, UpdateActiveSnapshotCommandId(), and Query::utilityStmt.

Referenced by BeginCopyFrom(), and BeginCopyTo().

{
    CopyState   cstate;
    TupleDesc   tupDesc;
    int         num_phys_attrs;
    MemoryContext oldcontext;

    /* Allocate workspace and zero all fields */
    cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));

    /*
     * We allocate everything used by a cstate in a new memory context. This
     * avoids memory leaks during repeated use of COPY in a query.
     */
    cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
                                                "COPY",
                                                ALLOCSET_DEFAULT_SIZES);

    oldcontext = MemoryContextSwitchTo(cstate->copycontext);

    /* Extract options from the statement node tree */
    ProcessCopyOptions(pstate, cstate, is_from, options);

    /* Process the source/target relation or query */
    if (rel)
    {
        Assert(!raw_query);

        cstate->rel = rel;

        tupDesc = RelationGetDescr(cstate->rel);

        /* Don't allow COPY w/ OIDs to or from a table without them */
        if (cstate->oids && !cstate->rel->rd_rel->relhasoids)
            ereport(ERROR,
                    (errcode(ERRCODE_UNDEFINED_COLUMN),
                     errmsg("table \"%s\" does not have OIDs",
                            RelationGetRelationName(cstate->rel))));

        /* Initialize state for CopyFrom tuple routing. */
        if (is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
        {
            PartitionDispatch *partition_dispatch_info;
            ResultRelInfo *partitions;
            TupleConversionMap **partition_tupconv_maps;
            TupleTableSlot *partition_tuple_slot;
            int         num_parted,
                        num_partitions;

            ExecSetupPartitionTupleRouting(rel,
                                           &partition_dispatch_info,
                                           &partitions,
                                           &partition_tupconv_maps,
                                           &partition_tuple_slot,
                                           &num_parted, &num_partitions);
            cstate->partition_dispatch_info = partition_dispatch_info;
            cstate->num_dispatch = num_parted;
            cstate->partitions = partitions;
            cstate->num_partitions = num_partitions;
            cstate->partition_tupconv_maps = partition_tupconv_maps;
            cstate->partition_tuple_slot = partition_tuple_slot;
        }
    }
    else
    {
        List       *rewritten;
        Query      *query;
        PlannedStmt *plan;
        DestReceiver *dest;

        Assert(!is_from);
        cstate->rel = NULL;

        /* Don't allow COPY w/ OIDs from a query */
        if (cstate->oids)
            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("COPY (query) WITH OIDS is not supported")));

        /*
         * Run parse analysis and rewrite.  Note this also acquires sufficient
         * locks on the source table(s).
         *
         * Because the parser and planner tend to scribble on their input, we
         * make a preliminary copy of the source querytree.  This prevents
         * problems in the case that the COPY is in a portal or plpgsql
         * function and is executed repeatedly.  (See also the same hack in
         * DECLARE CURSOR and PREPARE.)  XXX FIXME someday.
         */
        rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
                                           pstate->p_sourcetext, NULL, 0,
                                           NULL);

        /* check that we got back something we can work with */
        if (rewritten == NIL)
        {
            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
        }
        else if (list_length(rewritten) > 1)
        {
            ListCell   *lc;

            /* examine queries to determine which error message to issue */
            foreach(lc, rewritten)
            {
                Query      *q = lfirst_node(Query, lc);

                if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
                    ereport(ERROR,
                            (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                             errmsg("conditional DO INSTEAD rules are not supported for COPY")));
                if (q->querySource == QSRC_NON_INSTEAD_RULE)
                    ereport(ERROR,
                            (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                             errmsg("DO ALSO rules are not supported for the COPY")));
            }

            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
        }

        query = linitial_node(Query, rewritten);

        /* The grammar allows SELECT INTO, but we don't support that */
        if (query->utilityStmt != NULL &&
            IsA(query->utilityStmt, CreateTableAsStmt))
            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("COPY (SELECT INTO) is not supported")));

        Assert(query->utilityStmt == NULL);

        /*
         * Similarly the grammar doesn't enforce the presence of a RETURNING
         * clause, but this is required here.
         */
        if (query->commandType != CMD_SELECT &&
            query->returningList == NIL)
        {
            Assert(query->commandType == CMD_INSERT ||
                   query->commandType == CMD_UPDATE ||
                   query->commandType == CMD_DELETE);

            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("COPY query must have a RETURNING clause")));
        }

        /* plan the query */
        plan = pg_plan_query(query, CURSOR_OPT_PARALLEL_OK, NULL);

        /*
         * With row level security and a user using "COPY relation TO", we
         * have to convert the "COPY relation TO" to a query-based COPY (eg:
         * "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
         * in any RLS clauses.
         *
         * When this happens, we are passed in the relid of the originally
         * found relation (which we have locked).  As the planner will look up
         * the relation again, we double-check here to make sure it found the
         * same one that we have locked.
         */
        if (queryRelId != InvalidOid)
        {
            /*
             * Note that with RLS involved there may be multiple relations,
             * and while the one we need is almost certainly first, we don't
             * make any guarantees of that in the planner, so check the whole
             * list and make sure we find the original relation.
             */
            if (!list_member_oid(plan->relationOids, queryRelId))
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("relation referenced by COPY statement has changed")));
        }

        /*
         * Use a snapshot with an updated command ID to ensure this query sees
         * results of any previously executed queries.
         */
        PushCopiedSnapshot(GetActiveSnapshot());
        UpdateActiveSnapshotCommandId();

        /* Create dest receiver for COPY OUT */
        dest = CreateDestReceiver(DestCopyOut);
        ((DR_copy *) dest)->cstate = cstate;

        /* Create a QueryDesc requesting no output */
        cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
                                            GetActiveSnapshot(),
                                            InvalidSnapshot,
                                            dest, NULL, NULL, 0);

        /*
         * Call ExecutorStart to prepare the plan for execution.
         *
         * ExecutorStart computes a result tupdesc for us
         */
        ExecutorStart(cstate->queryDesc, 0);

        tupDesc = cstate->queryDesc->tupDesc;
    }

    /* Generate or convert list of attributes to process */
    cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);

    num_phys_attrs = tupDesc->natts;

    /* Convert FORCE_QUOTE name list to per-column flags, check validity */
    cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
    if (cstate->force_quote_all)
    {
        int         i;

        for (i = 0; i < num_phys_attrs; i++)
            cstate->force_quote_flags[i] = true;
    }
    else if (cstate->force_quote)
    {
        List       *attnums;
        ListCell   *cur;

        attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);

        foreach(cur, attnums)
        {
            int         attnum = lfirst_int(cur);

            if (!list_member_int(cstate->attnumlist, attnum))
                ereport(ERROR,
                        (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
                         errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
                                NameStr(tupDesc->attrs[attnum - 1]->attname))));
            cstate->force_quote_flags[attnum - 1] = true;
        }
    }

    /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
    cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
    if (cstate->force_notnull)
    {
        List       *attnums;
        ListCell   *cur;

        attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);

        foreach(cur, attnums)
        {
            int         attnum = lfirst_int(cur);

            if (!list_member_int(cstate->attnumlist, attnum))
                ereport(ERROR,
                        (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
                         errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
                                NameStr(tupDesc->attrs[attnum - 1]->attname))));
            cstate->force_notnull_flags[attnum - 1] = true;
        }
    }

    /* Convert FORCE_NULL name list to per-column flags, check validity */
    cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
    if (cstate->force_null)
    {
        List       *attnums;
        ListCell   *cur;

        attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);

        foreach(cur, attnums)
        {
            int         attnum = lfirst_int(cur);

            if (!list_member_int(cstate->attnumlist, attnum))
                ereport(ERROR,
                        (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
                         errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
                                NameStr(tupDesc->attrs[attnum - 1]->attname))));
            cstate->force_null_flags[attnum - 1] = true;
        }
    }

    /* Convert convert_selectively name list to per-column flags */
    if (cstate->convert_selectively)
    {
        List       *attnums;
        ListCell   *cur;

        cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));

        attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);

        foreach(cur, attnums)
        {
            int         attnum = lfirst_int(cur);

            if (!list_member_int(cstate->attnumlist, attnum))
                ereport(ERROR,
                        (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
                         errmsg_internal("selected column \"%s\" not referenced by COPY",
                                         NameStr(tupDesc->attrs[attnum - 1]->attname))));
            cstate->convert_select_flags[attnum - 1] = true;
        }
    }

    /* Use client encoding when ENCODING option is not specified. */
    if (cstate->file_encoding < 0)
        cstate->file_encoding = pg_get_client_encoding();

    /*
     * Set up encoding conversion info.  Even if the file and server encodings
     * are the same, we must apply pg_any_to_server() to validate data in
     * multibyte encodings.
     */
    cstate->need_transcoding =
        (cstate->file_encoding != GetDatabaseEncoding() ||
         pg_database_encoding_max_length() > 1);
    /* See Multibyte encoding comment above */
    cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);

    cstate->copy_dest = COPY_FILE;  /* default */

    MemoryContextSwitchTo(oldcontext);

    return cstate;
}

CopyState BeginCopyFrom	(	ParseState *	pstate,
		Relation	rel,
		const char *	filename,
		bool	is_program,
		copy_data_source_cb	data_source_cb,
		List *	attnamelist,
		List *	options
	)

Definition at line 2895 of file copy.c.

References AllocateFile(), Assert, CopyStateData::attnumlist, CopyStateData::attribute_buf, tupleDesc::attrs, BeginCopy(), CopyStateData::binary, BinarySignature, build_column_default(), contain_volatile_functions_not_nextval(), COPY_CALLBACK, CopyStateData::copy_dest, CopyStateData::copy_file, CopyStateData::copycontext, CopyGetData(), CopyGetInt32(), CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, CopyStateData::data_source_cb, CopyStateData::defexprs, CopyStateData::defmap, DestRemote, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, ExecInitExpr(), expression_planner(), CopyStateData::fe_eof, CopyStateData::file_has_oids, CopyStateData::filename, fmgr_info(), getTypeBinaryInputInfo(), getTypeInputInfo(), CopyStateData::in_functions, initStringInfo(), InvalidOid, CopyStateData::is_program, CopyStateData::line_buf, CopyStateData::line_buf_converted, list_length(), list_member_int(), CopyStateData::max_fields, MemoryContextSwitchTo(), tupleDesc::natts, NULL, CopyStateData::num_defaults, CopyStateData::oid_in_function, CopyStateData::oid_typioparam, OIDOID, CopyStateData::oids, OpenPipeStream(), ParseState::p_rtable, palloc(), PG_BINARY_R, pstrdup(), CopyStateData::range_table, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, RAW_BUF_SIZE, CopyStateData::raw_fields, ReceiveCopyBegin(), CopyStateData::rel, RelationGetDescr, RelationGetRelationName, CopyStateData::typioparams, CopyStateData::volatile_defexprs, and whereToSendOutput.

Referenced by copy_table(), DoCopy(), file_acquire_sample_rows(), fileBeginForeignScan(), and fileReScanForeignScan().

{
    CopyState   cstate;
    bool        pipe = (filename == NULL);
    TupleDesc   tupDesc;
    Form_pg_attribute *attr;
    AttrNumber  num_phys_attrs,
                num_defaults;
    FmgrInfo   *in_functions;
    Oid        *typioparams;
    int         attnum;
    Oid         in_func_oid;
    int        *defmap;
    ExprState **defexprs;
    MemoryContext oldcontext;
    bool        volatile_defexprs;

    cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
    oldcontext = MemoryContextSwitchTo(cstate->copycontext);

    /* Initialize state variables */
    cstate->fe_eof = false;
    cstate->eol_type = EOL_UNKNOWN;
    cstate->cur_relname = RelationGetRelationName(cstate->rel);
    cstate->cur_lineno = 0;
    cstate->cur_attname = NULL;
    cstate->cur_attval = NULL;

    /* Set up variables to avoid per-attribute overhead. */
    initStringInfo(&cstate->attribute_buf);
    initStringInfo(&cstate->line_buf);
    cstate->line_buf_converted = false;
    cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
    cstate->raw_buf_index = cstate->raw_buf_len = 0;

    /* Assign range table, we'll need it in CopyFrom. */
    if (pstate)
        cstate->range_table = pstate->p_rtable;

    tupDesc = RelationGetDescr(cstate->rel);
    attr = tupDesc->attrs;
    num_phys_attrs = tupDesc->natts;
    num_defaults = 0;
    volatile_defexprs = false;

    /*
     * Pick up the required catalog information for each attribute in the
     * relation, including the input function, the element type (to pass to
     * the input function), and info about defaults and constraints. (Which
     * input function we use depends on text/binary format choice.)
     */
    in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
    typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
    defmap = (int *) palloc(num_phys_attrs * sizeof(int));
    defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));

    for (attnum = 1; attnum <= num_phys_attrs; attnum++)
    {
        /* We don't need info for dropped attributes */
        if (attr[attnum - 1]->attisdropped)
            continue;

        /* Fetch the input function and typioparam info */
        if (cstate->binary)
            getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
                                   &in_func_oid, &typioparams[attnum - 1]);
        else
            getTypeInputInfo(attr[attnum - 1]->atttypid,
                             &in_func_oid, &typioparams[attnum - 1]);
        fmgr_info(in_func_oid, &in_functions[attnum - 1]);

        /* Get default info if needed */
        if (!list_member_int(cstate->attnumlist, attnum))
        {
            /* attribute is NOT to be copied from input */
            /* use default value if one exists */
            Expr       *defexpr = (Expr *) build_column_default(cstate->rel,
                                                                attnum);

            if (defexpr != NULL)
            {
                /* Run the expression through planner */
                defexpr = expression_planner(defexpr);

                /* Initialize executable expression in copycontext */
                defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
                defmap[num_defaults] = attnum - 1;
                num_defaults++;

                /*
                 * If a default expression looks at the table being loaded,
                 * then it could give the wrong answer when using
                 * multi-insert. Since database access can be dynamic this is
                 * hard to test for exactly, so we use the much wider test of
                 * whether the default expression is volatile. We allow for
                 * the special case of when the default expression is the
                 * nextval() of a sequence which in this specific case is
                 * known to be safe for use with the multi-insert
                 * optimization. Hence we use this special case function
                 * checker rather than the standard check for
                 * contain_volatile_functions().
                 */
                if (!volatile_defexprs)
                    volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
            }
        }
    }

    /* We keep those variables in cstate. */
    cstate->in_functions = in_functions;
    cstate->typioparams = typioparams;
    cstate->defmap = defmap;
    cstate->defexprs = defexprs;
    cstate->volatile_defexprs = volatile_defexprs;
    cstate->num_defaults = num_defaults;
    cstate->is_program = is_program;

    if (data_source_cb)
    {
        cstate->copy_dest = COPY_CALLBACK;
        cstate->data_source_cb = data_source_cb;
    }
    else if (pipe)
    {
        Assert(!is_program);    /* the grammar does not allow this */
        if (whereToSendOutput == DestRemote)
            ReceiveCopyBegin(cstate);
        else
            cstate->copy_file = stdin;
    }
    else
    {
        cstate->filename = pstrdup(filename);

        if (cstate->is_program)
        {
            cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
            if (cstate->copy_file == NULL)
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not execute command \"%s\": %m",
                                cstate->filename)));
        }
        else
        {
            struct stat st;

            cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
            if (cstate->copy_file == NULL)
            {
                /* copy errno because ereport subfunctions might change it */
                int         save_errno = errno;

                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not open file \"%s\" for reading: %m",
                                cstate->filename),
                         (save_errno == ENOENT || save_errno == EACCES) ?
                         errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
                                 "You may want a client-side facility such as psql's \\copy.") : 0));
            }

            if (fstat(fileno(cstate->copy_file), &st))
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not stat file \"%s\": %m",
                                cstate->filename)));

            if (S_ISDIR(st.st_mode))
                ereport(ERROR,
                        (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                         errmsg("\"%s\" is a directory", cstate->filename)));
        }
    }

    if (!cstate->binary)
    {
        /* must rely on user to tell us... */
        cstate->file_has_oids = cstate->oids;
    }
    else
    {
        /* Read and verify binary header */
        char        readSig[11];
        int32       tmp;

        /* Signature */
        if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
            memcmp(readSig, BinarySignature, 11) != 0)
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("COPY file signature not recognized")));
        /* Flags field */
        if (!CopyGetInt32(cstate, &tmp))
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("invalid COPY file header (missing flags)")));
        cstate->file_has_oids = (tmp & (1 << 16)) != 0;
        tmp &= ~(1 << 16);
        if ((tmp >> 16) != 0)
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("unrecognized critical flags in COPY file header")));
        /* Header extension length */
        if (!CopyGetInt32(cstate, &tmp) ||
            tmp < 0)
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("invalid COPY file header (missing length)")));
        /* Skip extension header, if present */
        while (tmp-- > 0)
        {
            if (CopyGetData(cstate, readSig, 1, 1) != 1)
                ereport(ERROR,
                        (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                         errmsg("invalid COPY file header (wrong length)")));
        }
    }

    if (cstate->file_has_oids && cstate->binary)
    {
        getTypeBinaryInputInfo(OIDOID,
                               &in_func_oid, &cstate->oid_typioparam);
        fmgr_info(in_func_oid, &cstate->oid_in_function);
    }

    /* create workspace for CopyReadAttributes results */
    if (!cstate->binary)
    {
        AttrNumber  attr_count = list_length(cstate->attnumlist);
        int         nfields = cstate->file_has_oids ? (attr_count + 1) : attr_count;

        cstate->max_fields = nfields;
        cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
    }

    MemoryContextSwitchTo(oldcontext);

    return cstate;
}

static CopyState BeginCopyTo ( ParseState *  pstate, Relation  rel, RawStmt *  query, Oid  queryRelId, const char *  filename, bool  is_program, List *  attnamelist, List *  options  )

static

Definition at line 1757 of file copy.c.

References AllocateFile(), Assert, BeginCopy(), CopyStateData::copy_file, CopyStateData::copycontext, DestRemote, ereport, errcode(), errcode_for_file_access(), errhint(), errmsg(), ERROR, CopyStateData::filename, is_absolute_path, CopyStateData::is_program, MemoryContextSwitchTo(), NULL, OpenPipeStream(), PG_BINARY_W, pstrdup(), RelationData::rd_rel, RelationGetRelationName, RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, S_IWGRP, S_IWOTH, and whereToSendOutput.

Referenced by DoCopy().

{
    CopyState   cstate;
    bool        pipe = (filename == NULL);
    MemoryContext oldcontext;

    if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
    {
        if (rel->rd_rel->relkind == RELKIND_VIEW)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from view \"%s\"",
                            RelationGetRelationName(rel)),
                     errhint("Try the COPY (SELECT ...) TO variant.")));
        else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from materialized view \"%s\"",
                            RelationGetRelationName(rel)),
                     errhint("Try the COPY (SELECT ...) TO variant.")));
        else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from foreign table \"%s\"",
                            RelationGetRelationName(rel)),
                     errhint("Try the COPY (SELECT ...) TO variant.")));
        else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from sequence \"%s\"",
                            RelationGetRelationName(rel))));
        else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from partitioned table \"%s\"",
                            RelationGetRelationName(rel)),
                     errhint("Try the COPY (SELECT ...) TO variant.")));
        else
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy from non-table relation \"%s\"",
                            RelationGetRelationName(rel))));
    }

    cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
                       options);
    oldcontext = MemoryContextSwitchTo(cstate->copycontext);

    if (pipe)
    {
        Assert(!is_program);    /* the grammar does not allow this */
        if (whereToSendOutput != DestRemote)
            cstate->copy_file = stdout;
    }
    else
    {
        cstate->filename = pstrdup(filename);
        cstate->is_program = is_program;

        if (is_program)
        {
            cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
            if (cstate->copy_file == NULL)
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not execute command \"%s\": %m",
                                cstate->filename)));
        }
        else
        {
            mode_t      oumask; /* Pre-existing umask value */
            struct stat st;

            /*
             * Prevent write to relative path ... too easy to shoot oneself in
             * the foot by overwriting a database file ...
             */
            if (!is_absolute_path(filename))
                ereport(ERROR,
                        (errcode(ERRCODE_INVALID_NAME),
                         errmsg("relative path not allowed for COPY to file")));

            oumask = umask(S_IWGRP | S_IWOTH);
            cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
            umask(oumask);
            if (cstate->copy_file == NULL)
            {
                /* copy errno because ereport subfunctions might change it */
                int         save_errno = errno;

                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not open file \"%s\" for writing: %m",
                                cstate->filename),
                         (save_errno == ENOENT || save_errno == EACCES) ?
                         errhint("COPY TO instructs the PostgreSQL server process to write a file. "
                                 "You may want a client-side facility such as psql's \\copy.") : 0));
            }

            if (fstat(fileno(cstate->copy_file), &st))
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not stat file \"%s\": %m",
                                cstate->filename)));

            if (S_ISDIR(st.st_mode))
                ereport(ERROR,
                        (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                         errmsg("\"%s\" is a directory", cstate->filename)));
        }
    }

    MemoryContextSwitchTo(oldcontext);

    return cstate;
}

static void ClosePipeToProgram ( CopyState  cstate )

static

Definition at line 1711 of file copy.c.

References Assert, ClosePipeStream(), CopyStateData::copy_file, ereport, errcode(), errcode_for_file_access(), errdetail_internal(), errmsg(), ERROR, CopyStateData::filename, CopyStateData::is_program, and wait_result_to_str().

Referenced by CopySendEndOfRow(), and EndCopy().

{
    int         pclose_rc;

    Assert(cstate->is_program);

    pclose_rc = ClosePipeStream(cstate->copy_file);
    if (pclose_rc == -1)
        ereport(ERROR,
                (errcode_for_file_access(),
                 errmsg("could not close pipe to external command: %m")));
    else if (pclose_rc != 0)
        ereport(ERROR,
                (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
                 errmsg("program \"%s\" failed",
                        cstate->filename),
                 errdetail_internal("%s", wait_result_to_str(pclose_rc))));
}

static void copy_dest_destroy ( DestReceiver *  self )

static

Definition at line 4748 of file copy.c.

References pfree().

Referenced by CreateCopyDestReceiver().

{
    pfree(self);
}

static bool copy_dest_receive ( TupleTableSlot *  slot, DestReceiver *  self  )

static

Definition at line 4720 of file copy.c.

References CopyOneRowTo(), DR_copy::cstate, InvalidOid, DR_copy::processed, slot_getallattrs(), TupleTableSlot::tts_isnull, and TupleTableSlot::tts_values.

Referenced by CreateCopyDestReceiver().

{
    DR_copy    *myState = (DR_copy *) self;
    CopyState   cstate = myState->cstate;

    /* Make sure the tuple is fully deconstructed */
    slot_getallattrs(slot);

    /* And send the data */
    CopyOneRowTo(cstate, InvalidOid, slot->tts_values, slot->tts_isnull);
    myState->processed++;

    return true;
}

static void copy_dest_shutdown ( DestReceiver *  self )

static

Definition at line 4739 of file copy.c.

Referenced by CreateCopyDestReceiver().

{
    /* no-op */
}

static void copy_dest_startup ( DestReceiver *  self, int  operation, TupleDesc  typeinfo  )

static

Definition at line 4711 of file copy.c.

Referenced by CreateCopyDestReceiver().

{
    /* no-op */
}

static void CopyAttributeOutCSV ( CopyState  cstate, char *  string, bool  use_quote, bool  single_attr  )

static

Definition at line 4548 of file copy.c.

References CopySendChar(), CopySendString(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::escape, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, CopyStateData::null_print, pg_encoding_mblen(), pg_server_to_any(), and CopyStateData::quote.

Referenced by CopyOneRowTo(), and CopyTo().

{
    char       *ptr;
    char       *start;
    char        c;
    char        delimc = cstate->delim[0];
    char        quotec = cstate->quote[0];
    char        escapec = cstate->escape[0];

    /* force quoting if it matches null_print (before conversion!) */
    if (!use_quote && strcmp(string, cstate->null_print) == 0)
        use_quote = true;

    if (cstate->need_transcoding)
        ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
    else
        ptr = string;

    /*
     * Make a preliminary pass to discover if it needs quoting
     */
    if (!use_quote)
    {
        /*
         * Because '\.' can be a data value, quote it if it appears alone on a
         * line so it is not interpreted as the end-of-data marker.
         */
        if (single_attr && strcmp(ptr, "\\.") == 0)
            use_quote = true;
        else
        {
            char       *tptr = ptr;

            while ((c = *tptr) != '\0')
            {
                if (c == delimc || c == quotec || c == '\n' || c == '\r')
                {
                    use_quote = true;
                    break;
                }
                if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
                    tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
                else
                    tptr++;
            }
        }
    }

    if (use_quote)
    {
        CopySendChar(cstate, quotec);

        /*
         * We adopt the same optimization strategy as in CopyAttributeOutText
         */
        start = ptr;
        while ((c = *ptr) != '\0')
        {
            if (c == quotec || c == escapec)
            {
                DUMPSOFAR();
                CopySendChar(cstate, escapec);
                start = ptr;    /* we include char in next run */
            }
            if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
                ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
            else
                ptr++;
        }
        DUMPSOFAR();

        CopySendChar(cstate, quotec);
    }
    else
    {
        /* If it doesn't need quoting, we can just dump it as-is */
        CopySendString(cstate, ptr);
    }
}

static void CopyAttributeOutText ( CopyState  cstate, char *  string  )

static

Definition at line 4395 of file copy.c.

References CopySendChar(), CopyStateData::delim, DUMPSOFAR, CopyStateData::encoding_embeds_ascii, CopyStateData::file_encoding, IS_HIGHBIT_SET, CopyStateData::need_transcoding, pg_encoding_mblen(), and pg_server_to_any().

Referenced by CopyOneRowTo().

{
    char       *ptr;
    char       *start;
    char        c;
    char        delimc = cstate->delim[0];

    if (cstate->need_transcoding)
        ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
    else
        ptr = string;

    /*
     * We have to grovel through the string searching for control characters
     * and instances of the delimiter character.  In most cases, though, these
     * are infrequent.  To avoid overhead from calling CopySendData once per
     * character, we dump out all characters between escaped characters in a
     * single call.  The loop invariant is that the data from "start" to "ptr"
     * can be sent literally, but hasn't yet been.
     *
     * We can skip pg_encoding_mblen() overhead when encoding is safe, because
     * in valid backend encodings, extra bytes of a multibyte character never
     * look like ASCII.  This loop is sufficiently performance-critical that
     * it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
     * of the normal safe-encoding path.
     */
    if (cstate->encoding_embeds_ascii)
    {
        start = ptr;
        while ((c = *ptr) != '\0')
        {
            if ((unsigned char) c < (unsigned char) 0x20)
            {
                /*
                 * \r and \n must be escaped, the others are traditional. We
                 * prefer to dump these using the C-like notation, rather than
                 * a backslash and the literal character, because it makes the
                 * dump file a bit more proof against Microsoftish data
                 * mangling.
                 */
                switch (c)
                {
                    case '\b':
                        c = 'b';
                        break;
                    case '\f':
                        c = 'f';
                        break;
                    case '\n':
                        c = 'n';
                        break;
                    case '\r':
                        c = 'r';
                        break;
                    case '\t':
                        c = 't';
                        break;
                    case '\v':
                        c = 'v';
                        break;
                    default:
                        /* If it's the delimiter, must backslash it */
                        if (c == delimc)
                            break;
                        /* All ASCII control chars are length 1 */
                        ptr++;
                        continue;   /* fall to end of loop */
                }
                /* if we get here, we need to convert the control char */
                DUMPSOFAR();
                CopySendChar(cstate, '\\');
                CopySendChar(cstate, c);
                start = ++ptr;  /* do not include char in next run */
            }
            else if (c == '\\' || c == delimc)
            {
                DUMPSOFAR();
                CopySendChar(cstate, '\\');
                start = ptr++;  /* we include char in next run */
            }
            else if (IS_HIGHBIT_SET(c))
                ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
            else
                ptr++;
        }
    }
    else
    {
        start = ptr;
        while ((c = *ptr) != '\0')
        {
            if ((unsigned char) c < (unsigned char) 0x20)
            {
                /*
                 * \r and \n must be escaped, the others are traditional. We
                 * prefer to dump these using the C-like notation, rather than
                 * a backslash and the literal character, because it makes the
                 * dump file a bit more proof against Microsoftish data
                 * mangling.
                 */
                switch (c)
                {
                    case '\b':
                        c = 'b';
                        break;
                    case '\f':
                        c = 'f';
                        break;
                    case '\n':
                        c = 'n';
                        break;
                    case '\r':
                        c = 'r';
                        break;
                    case '\t':
                        c = 't';
                        break;
                    case '\v':
                        c = 'v';
                        break;
                    default:
                        /* If it's the delimiter, must backslash it */
                        if (c == delimc)
                            break;
                        /* All ASCII control chars are length 1 */
                        ptr++;
                        continue;   /* fall to end of loop */
                }
                /* if we get here, we need to convert the control char */
                DUMPSOFAR();
                CopySendChar(cstate, '\\');
                CopySendChar(cstate, c);
                start = ++ptr;  /* do not include char in next run */
            }
            else if (c == '\\' || c == delimc)
            {
                DUMPSOFAR();
                CopySendChar(cstate, '\\');
                start = ptr++;  /* we include char in next run */
            }
            else
                ptr++;
        }
    }

    DUMPSOFAR();
}

uint64 CopyFrom

(

CopyState

cstate

)

Definition at line 2292 of file copy.c.

References AfterTriggerBeginQuery(), AfterTriggerEndQuery(), ErrorContextCallback::arg, Assert, ErrorContextCallback::callback, CHECK_FOR_INTERRUPTS, tupleDesc::constr, CopyStateData::copy_dest, COPY_OLD_FE, CopyFromErrorCallback(), CopyFromInsertBatch(), CreateExecutorState(), CopyStateData::cur_lineno, CurrentMemoryContext, do_convert_tuple(), ereport, errcode(), errhint(), errmsg(), ERROR, error_context_stack, EState::es_num_result_relations, EState::es_range_table, EState::es_result_relation_info, EState::es_result_relations, EState::es_trig_tuple_slot, EState::es_tupleTable, ExecARInsertTriggers(), ExecASInsertTriggers(), ExecBRInsertTriggers(), ExecBSInsertTriggers(), ExecCleanUpTriggerState(), ExecCloseIndices(), ExecConstraints(), ExecDropSingleTupleTableSlot(), ExecFindPartition(), ExecInitExtraTupleSlot(), ExecInsertIndexTuples(), ExecIRInsertTriggers(), ExecMaterializeSlot(), ExecOpenIndices(), ExecResetTupleTable(), ExecSetSlotDescriptor(), ExecStoreTuple(), FreeBulkInsertState(), FreeExecutorState(), CopyStateData::freeze, GetBulkInsertState(), GetCurrentCommandId(), GetCurrentSubTransactionId(), GetPerTupleExprContext, GetPerTupleMemoryContext, heap_close, heap_form_tuple(), heap_insert(), HEAP_INSERT_FROZEN, HEAP_INSERT_SKIP_FSM, HEAP_INSERT_SKIP_WAL, heap_sync(), HeapTupleSetOid, i, InitResultRelInfo(), InvalidBuffer, InvalidOid, InvalidSubTransactionId, list_free(), makeNode, MAX_BUFFERED_TUPLES, MemoryContextSwitchTo(), tupleDesc::natts, NextCopyFrom(), NIL, NoLock, NULL, CopyStateData::num_dispatch, CopyStateData::num_partitions, palloc(), CopyStateData::partition_dispatch_info, CopyStateData::partition_tupconv_maps, CopyStateData::partition_tuple_slot, CopyStateData::partitions, pfree(), pq_endmsgread(), ErrorContextCallback::previous, CopyStateData::range_table, RelationData::rd_att, RelationData::rd_createSubid, RelationData::rd_newRelfilenodeSubid, RelationData::rd_rel, CopyStateData::rel, RelationGetDescr, RelationGetRelationName, RelationGetRelid, PartitionDispatchData::reldesc, ReleaseBulkInsertStatePin(), RELKIND_FOREIGN_TABLE, RELKIND_MATVIEW, RELKIND_PARTITIONED_TABLE, RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, ResetPerTupleExprContext, ResultRelInfo::ri_FdwRoutine, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_PartitionCheck, ResultRelInfo::ri_RelationDesc, ResultRelInfo::ri_TrigDesc, HeapTupleData::t_len, HeapTupleData::t_self, HeapTupleData::t_tableOid, ThereAreNoPriorRegisteredSnapshots(), ThereAreNoReadyPortals(), TriggerDesc::trig_insert_before_row, TriggerDesc::trig_insert_instead_row, RelationData::trigdesc, PartitionDispatchData::tupslot, values, CopyStateData::volatile_defexprs, and XLogIsNeeded.

Referenced by copy_table(), and DoCopy().

{
    HeapTuple   tuple;
    TupleDesc   tupDesc;
    Datum      *values;
    bool       *nulls;
    ResultRelInfo *resultRelInfo;
    ResultRelInfo *saved_resultRelInfo = NULL;
    EState     *estate = CreateExecutorState(); /* for ExecConstraints() */
    ExprContext *econtext;
    TupleTableSlot *myslot;
    MemoryContext oldcontext = CurrentMemoryContext;

    ErrorContextCallback errcallback;
    CommandId   mycid = GetCurrentCommandId(true);
    int         hi_options = 0; /* start with default heap_insert options */
    BulkInsertState bistate;
    uint64      processed = 0;
    bool        useHeapMultiInsert;
    int         nBufferedTuples = 0;
    int         prev_leaf_part_index = -1;

#define MAX_BUFFERED_TUPLES 1000
    HeapTuple  *bufferedTuples = NULL;  /* initialize to silence warning */
    Size        bufferedTuplesSize = 0;
    int         firstBufferedLineNo = 0;

    Assert(cstate->rel);

    /*
     * The target must be a plain relation or have an INSTEAD OF INSERT row
     * trigger.  (Currently, such triggers are only allowed on views, so we
     * only hint about them in the view case.)
     */
    if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
        cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
        !(cstate->rel->trigdesc &&
          cstate->rel->trigdesc->trig_insert_instead_row))
    {
        if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy to view \"%s\"",
                            RelationGetRelationName(cstate->rel)),
                     errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
        else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy to materialized view \"%s\"",
                            RelationGetRelationName(cstate->rel))));
        else if (cstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy to foreign table \"%s\"",
                            RelationGetRelationName(cstate->rel))));
        else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy to sequence \"%s\"",
                            RelationGetRelationName(cstate->rel))));
        else
            ereport(ERROR,
                    (errcode(ERRCODE_WRONG_OBJECT_TYPE),
                     errmsg("cannot copy to non-table relation \"%s\"",
                            RelationGetRelationName(cstate->rel))));
    }

    tupDesc = RelationGetDescr(cstate->rel);

    /*----------
     * Check to see if we can avoid writing WAL
     *
     * If archive logging/streaming is not enabled *and* either
     *  - table was created in same transaction as this COPY
     *  - data is being written to relfilenode created in this transaction
     * then we can skip writing WAL.  It's safe because if the transaction
     * doesn't commit, we'll discard the table (or the new relfilenode file).
     * If it does commit, we'll have done the heap_sync at the bottom of this
     * routine first.
     *
     * As mentioned in comments in utils/rel.h, the in-same-transaction test
     * is not always set correctly, since in rare cases rd_newRelfilenodeSubid
     * can be cleared before the end of the transaction. The exact case is
     * when a relation sets a new relfilenode twice in same transaction, yet
     * the second one fails in an aborted subtransaction, e.g.
     *
     * BEGIN;
     * TRUNCATE t;
     * SAVEPOINT save;
     * TRUNCATE t;
     * ROLLBACK TO save;
     * COPY ...
     *
     * Also, if the target file is new-in-transaction, we assume that checking
     * FSM for free space is a waste of time, even if we must use WAL because
     * of archiving.  This could possibly be wrong, but it's unlikely.
     *
     * The comments for heap_insert and RelationGetBufferForTuple specify that
     * skipping WAL logging is only safe if we ensure that our tuples do not
     * go into pages containing tuples from any other transactions --- but this
     * must be the case if we have a new table or new relfilenode, so we need
     * no additional work to enforce that.
     *----------
     */
    /* createSubid is creation check, newRelfilenodeSubid is truncation check */
    if (cstate->rel->rd_createSubid != InvalidSubTransactionId ||
        cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId)
    {
        hi_options |= HEAP_INSERT_SKIP_FSM;
        if (!XLogIsNeeded())
            hi_options |= HEAP_INSERT_SKIP_WAL;
    }

    /*
     * Optimize if new relfilenode was created in this subxact or one of its
     * committed children and we won't see those rows later as part of an
     * earlier scan or command. This ensures that if this subtransaction
     * aborts then the frozen rows won't be visible after xact cleanup. Note
     * that the stronger test of exactly which subtransaction created it is
     * crucial for correctness of this optimization.
     */
    if (cstate->freeze)
    {
        if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
            ereport(ERROR,
                    (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
                     errmsg("cannot perform FREEZE because of prior transaction activity")));

        if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
            cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
            ereport(ERROR,
                    (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                     errmsg("cannot perform FREEZE because the table was not created or truncated in the current subtransaction")));

        hi_options |= HEAP_INSERT_FROZEN;
    }

    /*
     * We need a ResultRelInfo so we can use the regular executor's
     * index-entry-making machinery.  (There used to be a huge amount of code
     * here that basically duplicated execUtils.c ...)
     */
    resultRelInfo = makeNode(ResultRelInfo);
    InitResultRelInfo(resultRelInfo,
                      cstate->rel,
                      1,        /* dummy rangetable index */
                      NULL,
                      0);

    ExecOpenIndices(resultRelInfo, false);

    estate->es_result_relations = resultRelInfo;
    estate->es_num_result_relations = 1;
    estate->es_result_relation_info = resultRelInfo;
    estate->es_range_table = cstate->range_table;

    /* Set up a tuple slot too */
    myslot = ExecInitExtraTupleSlot(estate);
    ExecSetSlotDescriptor(myslot, tupDesc);
    /* Triggers might need a slot as well */
    estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate);

    /*
     * It's more efficient to prepare a bunch of tuples for insertion, and
     * insert them in one heap_multi_insert() call, than call heap_insert()
     * separately for every tuple. However, we can't do that if there are
     * BEFORE/INSTEAD OF triggers, or we need to evaluate volatile default
     * expressions. Such triggers or expressions might query the table we're
     * inserting to, and act differently if the tuples that have already been
     * processed and prepared for insertion are not there.  We also can't do
     * it if the table is partitioned.
     */
    if ((resultRelInfo->ri_TrigDesc != NULL &&
         (resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
          resultRelInfo->ri_TrigDesc->trig_insert_instead_row)) ||
        cstate->partition_dispatch_info != NULL ||
        cstate->volatile_defexprs)
    {
        useHeapMultiInsert = false;
    }
    else
    {
        useHeapMultiInsert = true;
        bufferedTuples = palloc(MAX_BUFFERED_TUPLES * sizeof(HeapTuple));
    }

    /* Prepare to catch AFTER triggers. */
    AfterTriggerBeginQuery();

    /*
     * Check BEFORE STATEMENT insertion triggers. It's debatable whether we
     * should do this for COPY, since it's not really an "INSERT" statement as
     * such. However, executing these triggers maintains consistency with the
     * EACH ROW triggers that we already fire on COPY.
     */
    ExecBSInsertTriggers(estate, resultRelInfo);

    values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
    nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));

    bistate = GetBulkInsertState();
    econtext = GetPerTupleExprContext(estate);

    /* Set up callback to identify error line number */
    errcallback.callback = CopyFromErrorCallback;
    errcallback.arg = (void *) cstate;
    errcallback.previous = error_context_stack;
    error_context_stack = &errcallback;

    for (;;)
    {
        TupleTableSlot *slot;
        bool        skip_tuple;
        Oid         loaded_oid = InvalidOid;

        CHECK_FOR_INTERRUPTS();

        if (nBufferedTuples == 0)
        {
            /*
             * Reset the per-tuple exprcontext. We can only do this if the
             * tuple buffer is empty. (Calling the context the per-tuple
             * memory context is a bit of a misnomer now.)
             */
            ResetPerTupleExprContext(estate);
        }

        /* Switch into its memory context */
        MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));

        if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
            break;

        /* And now we can form the input tuple. */
        tuple = heap_form_tuple(tupDesc, values, nulls);

        if (loaded_oid != InvalidOid)
            HeapTupleSetOid(tuple, loaded_oid);

        /*
         * Constraints might reference the tableoid column, so initialize
         * t_tableOid before evaluating them.
         */
        tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);

        /* Triggers and stuff need to be invoked in query context. */
        MemoryContextSwitchTo(oldcontext);

        /* Place tuple in tuple slot --- but slot shouldn't free it */
        slot = myslot;
        ExecStoreTuple(tuple, slot, InvalidBuffer, false);

        /* Determine the partition to heap_insert the tuple into */
        if (cstate->partition_dispatch_info)
        {
            int         leaf_part_index;
            TupleConversionMap *map;

            /*
             * Away we go ... If we end up not finding a partition after all,
             * ExecFindPartition() does not return and errors out instead.
             * Otherwise, the returned value is to be used as an index into
             * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
             * will get us the ResultRelInfo and TupleConversionMap for the
             * partition, respectively.
             */
            leaf_part_index = ExecFindPartition(resultRelInfo,
                                                cstate->partition_dispatch_info,
                                                slot,
                                                estate);
            Assert(leaf_part_index >= 0 &&
                   leaf_part_index < cstate->num_partitions);

            /*
             * If this tuple is mapped to a partition that is not same as the
             * previous one, we'd better make the bulk insert mechanism gets a
             * new buffer.
             */
            if (prev_leaf_part_index != leaf_part_index)
            {
                ReleaseBulkInsertStatePin(bistate);
                prev_leaf_part_index = leaf_part_index;
            }

            /*
             * Save the old ResultRelInfo and switch to the one corresponding
             * to the selected partition.
             */
            saved_resultRelInfo = resultRelInfo;
            resultRelInfo = cstate->partitions + leaf_part_index;

            /* We do not yet have a way to insert into a foreign partition */
            if (resultRelInfo->ri_FdwRoutine)
                ereport(ERROR,
                        (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                         errmsg("cannot route inserted tuples to a foreign table")));

            /*
             * For ExecInsertIndexTuples() to work on the partition's indexes
             */
            estate->es_result_relation_info = resultRelInfo;

            /*
             * We might need to convert from the parent rowtype to the
             * partition rowtype.
             */
            map = cstate->partition_tupconv_maps[leaf_part_index];
            if (map)
            {
                Relation    partrel = resultRelInfo->ri_RelationDesc;

                tuple = do_convert_tuple(tuple, map);

                /*
                 * We must use the partition's tuple descriptor from this
                 * point on.  Use a dedicated slot from this point on until
                 * we're finished dealing with the partition.
                 */
                slot = cstate->partition_tuple_slot;
                Assert(slot != NULL);
                ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
                ExecStoreTuple(tuple, slot, InvalidBuffer, true);
            }

            tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
        }

        skip_tuple = false;

        /* BEFORE ROW INSERT Triggers */
        if (resultRelInfo->ri_TrigDesc &&
            resultRelInfo->ri_TrigDesc->trig_insert_before_row)
        {
            slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);

            if (slot == NULL)   /* "do nothing" */
                skip_tuple = true;
            else                /* trigger might have changed tuple */
                tuple = ExecMaterializeSlot(slot);
        }

        if (!skip_tuple)
        {
            if (resultRelInfo->ri_TrigDesc &&
                resultRelInfo->ri_TrigDesc->trig_insert_instead_row)
            {
                /* Pass the data to the INSTEAD ROW INSERT trigger */
                ExecIRInsertTriggers(estate, resultRelInfo, slot);
            }
            else
            {
                /*
                 * We always check the partition constraint, including when
                 * the tuple got here via tuple-routing.  However we don't
                 * need to in the latter case if no BR trigger is defined on
                 * the partition.  Note that a BR trigger might modify the
                 * tuple such that the partition constraint is no longer
                 * satisfied, so we need to check in that case.
                 */
                bool        check_partition_constr =
                (resultRelInfo->ri_PartitionCheck != NIL);

                if (saved_resultRelInfo != NULL &&
                    !(resultRelInfo->ri_TrigDesc &&
                      resultRelInfo->ri_TrigDesc->trig_insert_before_row))
                    check_partition_constr = false;

                /* Check the constraints of the tuple */
                if (cstate->rel->rd_att->constr || check_partition_constr)
                    ExecConstraints(resultRelInfo, slot, estate);

                if (useHeapMultiInsert)
                {
                    /* Add this tuple to the tuple buffer */
                    if (nBufferedTuples == 0)
                        firstBufferedLineNo = cstate->cur_lineno;
                    bufferedTuples[nBufferedTuples++] = tuple;
                    bufferedTuplesSize += tuple->t_len;

                    /*
                     * If the buffer filled up, flush it.  Also flush if the
                     * total size of all the tuples in the buffer becomes
                     * large, to avoid using large amounts of memory for the
                     * buffer when the tuples are exceptionally wide.
                     */
                    if (nBufferedTuples == MAX_BUFFERED_TUPLES ||
                        bufferedTuplesSize > 65535)
                    {
                        CopyFromInsertBatch(cstate, estate, mycid, hi_options,
                                            resultRelInfo, myslot, bistate,
                                            nBufferedTuples, bufferedTuples,
                                            firstBufferedLineNo);
                        nBufferedTuples = 0;
                        bufferedTuplesSize = 0;
                    }
                }
                else
                {
                    List       *recheckIndexes = NIL;

                    /* OK, store the tuple and create index entries for it */
                    heap_insert(resultRelInfo->ri_RelationDesc, tuple, mycid,
                                hi_options, bistate);

                    if (resultRelInfo->ri_NumIndices > 0)
                        recheckIndexes = ExecInsertIndexTuples(slot,
                                                               &(tuple->t_self),
                                                               estate,
                                                               false,
                                                               NULL,
                                                               NIL);

                    /* AFTER ROW INSERT Triggers */
                    ExecARInsertTriggers(estate, resultRelInfo, tuple,
                                         recheckIndexes);

                    list_free(recheckIndexes);
                }
            }

            /*
             * We count only tuples not suppressed by a BEFORE INSERT trigger;
             * this is the same definition used by execMain.c for counting
             * tuples inserted by an INSERT command.
             */
            processed++;

            if (saved_resultRelInfo)
            {
                resultRelInfo = saved_resultRelInfo;
                estate->es_result_relation_info = resultRelInfo;
            }
        }
    }

    /* Flush any remaining buffered tuples */
    if (nBufferedTuples > 0)
        CopyFromInsertBatch(cstate, estate, mycid, hi_options,
                            resultRelInfo, myslot, bistate,
                            nBufferedTuples, bufferedTuples,
                            firstBufferedLineNo);

    /* Done, clean up */
    error_context_stack = errcallback.previous;

    FreeBulkInsertState(bistate);

    MemoryContextSwitchTo(oldcontext);

    /*
     * In the old protocol, tell pqcomm that we can process normal protocol
     * messages again.
     */
    if (cstate->copy_dest == COPY_OLD_FE)
        pq_endmsgread();

    /* Execute AFTER STATEMENT insertion triggers */
    ExecASInsertTriggers(estate, resultRelInfo);

    /* Handle queued AFTER triggers */
    AfterTriggerEndQuery(estate);

    pfree(values);
    pfree(nulls);

    ExecResetTupleTable(estate->es_tupleTable, false);

    ExecCloseIndices(resultRelInfo);

    /* Close all the partitioned tables, leaf partitions, and their indices */
    if (cstate->partition_dispatch_info)
    {
        int         i;

        /*
         * Remember cstate->partition_dispatch_info[0] corresponds to the root
         * partitioned table, which we must not try to close, because it is
         * the main target table of COPY that will be closed eventually by
         * DoCopy().  Also, tupslot is NULL for the root partitioned table.
         */
        for (i = 1; i < cstate->num_dispatch; i++)
        {
            PartitionDispatch pd = cstate->partition_dispatch_info[i];

            heap_close(pd->reldesc, NoLock);
            ExecDropSingleTupleTableSlot(pd->tupslot);
        }
        for (i = 0; i < cstate->num_partitions; i++)
        {
            ResultRelInfo *resultRelInfo = cstate->partitions + i;

            ExecCloseIndices(resultRelInfo);
            heap_close(resultRelInfo->ri_RelationDesc, NoLock);
        }

        /* Release the standalone partition tuple descriptor */
        ExecDropSingleTupleTableSlot(cstate->partition_tuple_slot);
    }

    /* Close any trigger target relations */
    ExecCleanUpTriggerState(estate);

    FreeExecutorState(estate);

    /*
     * If we skipped writing WAL, then we need to sync the heap (but not
     * indexes since those use WAL anyway)
     */
    if (hi_options & HEAP_INSERT_SKIP_WAL)
        heap_sync(cstate->rel);

    return processed;
}

void CopyFromErrorCallback

(

void *

arg

)

Definition at line 2187 of file copy.c.

References CopyStateData::binary, CopyStateData::cur_attname, CopyStateData::cur_attval, CopyStateData::cur_lineno, CopyStateData::cur_relname, StringInfoData::data, errcontext, limit_printout_length(), CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, and pfree().

Referenced by CopyFrom(), file_acquire_sample_rows(), and fileIterateForeignScan().

{
    CopyState   cstate = (CopyState) arg;

    if (cstate->binary)
    {
        /* can't usefully display the data */
        if (cstate->cur_attname)
            errcontext("COPY %s, line %d, column %s",
                       cstate->cur_relname, cstate->cur_lineno,
                       cstate->cur_attname);
        else
            errcontext("COPY %s, line %d",
                       cstate->cur_relname, cstate->cur_lineno);
    }
    else
    {
        if (cstate->cur_attname && cstate->cur_attval)
        {
            /* error is relevant to a particular column */
            char       *attval;

            attval = limit_printout_length(cstate->cur_attval);
            errcontext("COPY %s, line %d, column %s: \"%s\"",
                       cstate->cur_relname, cstate->cur_lineno,
                       cstate->cur_attname, attval);
            pfree(attval);
        }
        else if (cstate->cur_attname)
        {
            /* error is relevant to a particular column, value is NULL */
            errcontext("COPY %s, line %d, column %s: null input",
                       cstate->cur_relname, cstate->cur_lineno,
                       cstate->cur_attname);
        }
        else
        {
            /*
             * Error is relevant to a particular line.
             *
             * If line_buf still contains the correct line, and it's already
             * transcoded, print it. If it's still in a foreign encoding, it's
             * quite likely that the error is precisely a failure to do
             * encoding conversion (ie, bad data). We dare not try to convert
             * it, and at present there's no way to regurgitate it without
             * conversion. So we have to punt and just report the line number.
             */
            if (cstate->line_buf_valid &&
                (cstate->line_buf_converted || !cstate->need_transcoding))
            {
                char       *lineval;

                lineval = limit_printout_length(cstate->line_buf.data);
                errcontext("COPY %s, line %d: \"%s\"",
                           cstate->cur_relname, cstate->cur_lineno, lineval);
                pfree(lineval);
            }
            else
            {
                errcontext("COPY %s, line %d",
                           cstate->cur_relname, cstate->cur_lineno);
            }
        }
    }
}

static void CopyFromInsertBatch ( CopyState  cstate, EState *  estate, CommandId  mycid, int  hi_options, ResultRelInfo *  resultRelInfo, TupleTableSlot *  myslot, BulkInsertState  bistate, int  nBufferedTuples, HeapTuple *  bufferedTuples, int  firstBufferedLineNo  )

static

Definition at line 2812 of file copy.c.

References CopyStateData::cur_lineno, ExecARInsertTriggers(), ExecInsertIndexTuples(), ExecStoreTuple(), GetPerTupleMemoryContext, heap_multi_insert(), i, InvalidBuffer, CopyStateData::line_buf_valid, list_free(), MemoryContextSwitchTo(), NIL, NULL, CopyStateData::rel, ResultRelInfo::ri_NumIndices, ResultRelInfo::ri_TrigDesc, and TriggerDesc::trig_insert_after_row.

Referenced by CopyFrom().

{
    MemoryContext oldcontext;
    int         i;
    int         save_cur_lineno;

    /*
     * Print error context information correctly, if one of the operations
     * below fail.
     */
    cstate->line_buf_valid = false;
    save_cur_lineno = cstate->cur_lineno;

    /*
     * heap_multi_insert leaks memory, so switch to short-lived memory context
     * before calling it.
     */
    oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
    heap_multi_insert(cstate->rel,
                      bufferedTuples,
                      nBufferedTuples,
                      mycid,
                      hi_options,
                      bistate);
    MemoryContextSwitchTo(oldcontext);

    /*
     * If there are any indexes, update them for all the inserted tuples, and
     * run AFTER ROW INSERT triggers.
     */
    if (resultRelInfo->ri_NumIndices > 0)
    {
        for (i = 0; i < nBufferedTuples; i++)
        {
            List       *recheckIndexes;

            cstate->cur_lineno = firstBufferedLineNo + i;
            ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false);
            recheckIndexes =
                ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self),
                                      estate, false, NULL, NIL);
            ExecARInsertTriggers(estate, resultRelInfo,
                                 bufferedTuples[i],
                                 recheckIndexes);
            list_free(recheckIndexes);
        }
    }

    /*
     * There's no indexes, but see if we need to run AFTER ROW INSERT triggers
     * anyway.
     */
    else if (resultRelInfo->ri_TrigDesc != NULL &&
             resultRelInfo->ri_TrigDesc->trig_insert_after_row)
    {
        for (i = 0; i < nBufferedTuples; i++)
        {
            cstate->cur_lineno = firstBufferedLineNo + i;
            ExecARInsertTriggers(estate, resultRelInfo,
                                 bufferedTuples[i],
                                 NIL);
        }
    }

    /* reset cur_lineno to where we were */
    cstate->cur_lineno = save_cur_lineno;
}

static List * CopyGetAttnums ( TupleDesc  tupDesc, Relation  rel, List *  attnamelist  )

static

Definition at line 4639 of file copy.c.

References tupleDesc::attrs, ereport, errcode(), errmsg(), ERROR, i, InvalidAttrNumber, lappend_int(), lfirst, list_member_int(), name, namestrcmp(), tupleDesc::natts, NIL, NULL, RelationGetRelationName, and strVal.

Referenced by BeginCopy(), and DoCopy().

{
    List       *attnums = NIL;

    if (attnamelist == NIL)
    {
        /* Generate default column list */
        Form_pg_attribute *attr = tupDesc->attrs;
        int         attr_count = tupDesc->natts;
        int         i;

        for (i = 0; i < attr_count; i++)
        {
            if (attr[i]->attisdropped)
                continue;
            attnums = lappend_int(attnums, i + 1);
        }
    }
    else
    {
        /* Validate the user-supplied list and extract attnums */
        ListCell   *l;

        foreach(l, attnamelist)
        {
            char       *name = strVal(lfirst(l));
            int         attnum;
            int         i;

            /* Lookup column name */
            attnum = InvalidAttrNumber;
            for (i = 0; i < tupDesc->natts; i++)
            {
                if (tupDesc->attrs[i]->attisdropped)
                    continue;
                if (namestrcmp(&(tupDesc->attrs[i]->attname), name) == 0)
                {
                    attnum = tupDesc->attrs[i]->attnum;
                    break;
                }
            }
            if (attnum == InvalidAttrNumber)
            {
                if (rel != NULL)
                    ereport(ERROR,
                            (errcode(ERRCODE_UNDEFINED_COLUMN),
                             errmsg("column \"%s\" of relation \"%s\" does not exist",
                                    name, RelationGetRelationName(rel))));
                else
                    ereport(ERROR,
                            (errcode(ERRCODE_UNDEFINED_COLUMN),
                             errmsg("column \"%s\" does not exist",
                                    name)));
            }
            /* Check for duplicates */
            if (list_member_int(attnums, attnum))
                ereport(ERROR,
                        (errcode(ERRCODE_DUPLICATE_COLUMN),
                         errmsg("column \"%s\" specified more than once",
                                name)));
            attnums = lappend_int(attnums, attnum);
        }
    }

    return attnums;
}

static int CopyGetData ( CopyState  cstate, void *  databuf, int  minread, int  maxread  )

static

Definition at line 556 of file copy.c.

References COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, StringInfoData::cursor, CopyStateData::data_source_cb, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, CopyStateData::fe_eof, CopyStateData::fe_msgbuf, HOLD_CANCEL_INTERRUPTS, StringInfoData::len, pq_copymsgbytes(), pq_getbyte(), pq_getbytes(), pq_getmessage(), pq_getmsgstring(), pq_startmsgread(), and RESUME_CANCEL_INTERRUPTS.

Referenced by BeginCopyFrom(), CopyGetInt16(), CopyGetInt32(), CopyLoadRawBuf(), CopyReadBinaryAttribute(), and NextCopyFrom().

{
    int         bytesread = 0;

    switch (cstate->copy_dest)
    {
        case COPY_FILE:
            bytesread = fread(databuf, 1, maxread, cstate->copy_file);
            if (ferror(cstate->copy_file))
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not read from COPY file: %m")));
            break;
        case COPY_OLD_FE:

            /*
             * We cannot read more than minread bytes (which in practice is 1)
             * because old protocol doesn't have any clear way of separating
             * the COPY stream from following data.  This is slow, but not any
             * slower than the code path was originally, and we don't care
             * much anymore about the performance of old protocol.
             */
            if (pq_getbytes((char *) databuf, minread))
            {
                /* Only a \. terminator is legal EOF in old protocol */
                ereport(ERROR,
                        (errcode(ERRCODE_CONNECTION_FAILURE),
                         errmsg("unexpected EOF on client connection with an open transaction")));
            }
            bytesread = minread;
            break;
        case COPY_NEW_FE:
            while (maxread > 0 && bytesread < minread && !cstate->fe_eof)
            {
                int         avail;

                while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
                {
                    /* Try to receive another message */
                    int         mtype;

            readmessage:
                    HOLD_CANCEL_INTERRUPTS();
                    pq_startmsgread();
                    mtype = pq_getbyte();
                    if (mtype == EOF)
                        ereport(ERROR,
                                (errcode(ERRCODE_CONNECTION_FAILURE),
                                 errmsg("unexpected EOF on client connection with an open transaction")));
                    if (pq_getmessage(cstate->fe_msgbuf, 0))
                        ereport(ERROR,
                                (errcode(ERRCODE_CONNECTION_FAILURE),
                                 errmsg("unexpected EOF on client connection with an open transaction")));
                    RESUME_CANCEL_INTERRUPTS();
                    switch (mtype)
                    {
                        case 'd':   /* CopyData */
                            break;
                        case 'c':   /* CopyDone */
                            /* COPY IN correctly terminated by frontend */
                            cstate->fe_eof = true;
                            return bytesread;
                        case 'f':   /* CopyFail */
                            ereport(ERROR,
                                    (errcode(ERRCODE_QUERY_CANCELED),
                                     errmsg("COPY from stdin failed: %s",
                                            pq_getmsgstring(cstate->fe_msgbuf))));
                            break;
                        case 'H':   /* Flush */
                        case 'S':   /* Sync */

                            /*
                             * Ignore Flush/Sync for the convenience of client
                             * libraries (such as libpq) that may send those
                             * without noticing that the command they just
                             * sent was COPY.
                             */
                            goto readmessage;
                        default:
                            ereport(ERROR,
                                    (errcode(ERRCODE_PROTOCOL_VIOLATION),
                                     errmsg("unexpected message type 0x%02X during COPY from stdin",
                                            mtype)));
                            break;
                    }
                }
                avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
                if (avail > maxread)
                    avail = maxread;
                pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
                databuf = (void *) ((char *) databuf + avail);
                maxread -= avail;
                bytesread += avail;
            }
            break;
        case COPY_CALLBACK:
            bytesread = cstate->data_source_cb(databuf, minread, maxread);
            break;
    }

    return bytesread;
}

static bool CopyGetInt16 ( CopyState  cstate, int16 *  val  )

static

Definition at line 711 of file copy.c.

References buf, and CopyGetData().

Referenced by NextCopyFrom().

{
    uint16      buf;

    if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
    {
        *val = 0;               /* suppress compiler warning */
        return false;
    }
    *val = (int16) ntohs(buf);
    return true;
}

static bool CopyGetInt32 ( CopyState  cstate, int32 *  val  )

static

Definition at line 682 of file copy.c.

References buf, and CopyGetData().

Referenced by BeginCopyFrom(), and CopyReadBinaryAttribute().

{
    uint32      buf;

    if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
    {
        *val = 0;               /* suppress compiler warning */
        return false;
    }
    *val = (int32) ntohl(buf);
    return true;
}

static bool CopyLoadRawBuf ( CopyState  cstate )

static

Definition at line 736 of file copy.c.

References CopyGetData(), memmove, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, and RAW_BUF_SIZE.

Referenced by CopyReadLine(), and CopyReadLineText().

{
    int         nbytes;
    int         inbytes;

    if (cstate->raw_buf_index < cstate->raw_buf_len)
    {
        /* Copy down the unprocessed data */
        nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
        memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
                nbytes);
    }
    else
        nbytes = 0;             /* no data need be saved */

    inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
                          1, RAW_BUF_SIZE - nbytes);
    nbytes += inbytes;
    cstate->raw_buf[nbytes] = '\0';
    cstate->raw_buf_index = 0;
    cstate->raw_buf_len = nbytes;
    return (inbytes > 0);
}

static void CopyOneRowTo ( CopyState  cstate, Oid  tupleOid, Datum *  values, bool *  nulls  )

static

Definition at line 2093 of file copy.c.

References CopyStateData::attnumlist, CopyStateData::binary, CopyAttributeOutCSV(), CopyAttributeOutText(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), CopySendString(), CopyStateData::csv_mode, cur, DatumGetCString, CopyStateData::delim, DirectFunctionCall1, CopyStateData::force_quote_flags, lfirst_int, list_length(), MemoryContextReset(), MemoryContextSwitchTo(), CopyStateData::null_print_client, ObjectIdGetDatum, oidout(), CopyStateData::oids, CopyStateData::out_functions, OutputFunctionCall(), CopyStateData::rowcontext, SendFunctionCall(), value, VARDATA, VARHDRSZ, and VARSIZE.

Referenced by copy_dest_receive(), and CopyTo().

{
    bool        need_delim = false;
    FmgrInfo   *out_functions = cstate->out_functions;
    MemoryContext oldcontext;
    ListCell   *cur;
    char       *string;

    MemoryContextReset(cstate->rowcontext);
    oldcontext = MemoryContextSwitchTo(cstate->rowcontext);

    if (cstate->binary)
    {
        /* Binary per-tuple header */
        CopySendInt16(cstate, list_length(cstate->attnumlist));
        /* Send OID if wanted --- note attnumlist doesn't include it */
        if (cstate->oids)
        {
            /* Hack --- assume Oid is same size as int32 */
            CopySendInt32(cstate, sizeof(int32));
            CopySendInt32(cstate, tupleOid);
        }
    }
    else
    {
        /* Text format has no per-tuple header, but send OID if wanted */
        /* Assume digits don't need any quoting or encoding conversion */
        if (cstate->oids)
        {
            string = DatumGetCString(DirectFunctionCall1(oidout,
                                                         ObjectIdGetDatum(tupleOid)));
            CopySendString(cstate, string);
            need_delim = true;
        }
    }

    foreach(cur, cstate->attnumlist)
    {
        int         attnum = lfirst_int(cur);
        Datum       value = values[attnum - 1];
        bool        isnull = nulls[attnum - 1];

        if (!cstate->binary)
        {
            if (need_delim)
                CopySendChar(cstate, cstate->delim[0]);
            need_delim = true;
        }

        if (isnull)
        {
            if (!cstate->binary)
                CopySendString(cstate, cstate->null_print_client);
            else
                CopySendInt32(cstate, -1);
        }
        else
        {
            if (!cstate->binary)
            {
                string = OutputFunctionCall(&out_functions[attnum - 1],
                                            value);
                if (cstate->csv_mode)
                    CopyAttributeOutCSV(cstate, string,
                                        cstate->force_quote_flags[attnum - 1],
                                        list_length(cstate->attnumlist) == 1);
                else
                    CopyAttributeOutText(cstate, string);
            }
            else
            {
                bytea      *outputbytes;

                outputbytes = SendFunctionCall(&out_functions[attnum - 1],
                                               value);
                CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
                CopySendData(cstate, VARDATA(outputbytes),
                             VARSIZE(outputbytes) - VARHDRSZ);
            }
        }
    }

    CopySendEndOfRow(cstate);

    MemoryContextSwitchTo(oldcontext);
}

static int CopyReadAttributesCSV ( CopyState  cstate )

static

Definition at line 4167 of file copy.c.

References Assert, CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, CopyStateData::escape, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, CopyStateData::quote, CopyStateData::raw_fields, repalloc(), and resetStringInfo().

Referenced by NextCopyFromRawFields().

{
    char        delimc = cstate->delim[0];
    char        quotec = cstate->quote[0];
    char        escapec = cstate->escape[0];
    int         fieldno;
    char       *output_ptr;
    char       *cur_ptr;
    char       *line_end_ptr;

    /*
     * We need a special case for zero-column tables: check that the input
     * line is empty, and return.
     */
    if (cstate->max_fields <= 0)
    {
        if (cstate->line_buf.len != 0)
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("extra data after last expected column")));
        return 0;
    }

    resetStringInfo(&cstate->attribute_buf);

    /*
     * The de-escaped attributes will certainly not be longer than the input
     * data line, so we can just force attribute_buf to be large enough and
     * then transfer data without any checks for enough space.  We need to do
     * it this way because enlarging attribute_buf mid-stream would invalidate
     * pointers already stored into cstate->raw_fields[].
     */
    if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
        enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
    output_ptr = cstate->attribute_buf.data;

    /* set pointer variables for loop */
    cur_ptr = cstate->line_buf.data;
    line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;

    /* Outer loop iterates over fields */
    fieldno = 0;
    for (;;)
    {
        bool        found_delim = false;
        bool        saw_quote = false;
        char       *start_ptr;
        char       *end_ptr;
        int         input_len;

        /* Make sure there is enough space for the next value */
        if (fieldno >= cstate->max_fields)
        {
            cstate->max_fields *= 2;
            cstate->raw_fields =
                repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
        }

        /* Remember start of field on both input and output sides */
        start_ptr = cur_ptr;
        cstate->raw_fields[fieldno] = output_ptr;

        /*
         * Scan data for field,
         *
         * The loop starts in "not quote" mode and then toggles between that
         * and "in quote" mode. The loop exits normally if it is in "not
         * quote" mode and a delimiter or line end is seen.
         */
        for (;;)
        {
            char        c;

            /* Not in quote */
            for (;;)
            {
                end_ptr = cur_ptr;
                if (cur_ptr >= line_end_ptr)
                    goto endfield;
                c = *cur_ptr++;
                /* unquoted field delimiter */
                if (c == delimc)
                {
                    found_delim = true;
                    goto endfield;
                }
                /* start of quoted field (or part of field) */
                if (c == quotec)
                {
                    saw_quote = true;
                    break;
                }
                /* Add c to output string */
                *output_ptr++ = c;
            }

            /* In quote */
            for (;;)
            {
                end_ptr = cur_ptr;
                if (cur_ptr >= line_end_ptr)
                    ereport(ERROR,
                            (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                             errmsg("unterminated CSV quoted field")));

                c = *cur_ptr++;

                /* escape within a quoted field */
                if (c == escapec)
                {
                    /*
                     * peek at the next char if available, and escape it if it
                     * is an escape char or a quote char
                     */
                    if (cur_ptr < line_end_ptr)
                    {
                        char        nextc = *cur_ptr;

                        if (nextc == escapec || nextc == quotec)
                        {
                            *output_ptr++ = nextc;
                            cur_ptr++;
                            continue;
                        }
                    }
                }

                /*
                 * end of quoted field. Must do this test after testing for
                 * escape in case quote char and escape char are the same
                 * (which is the common case).
                 */
                if (c == quotec)
                    break;

                /* Add c to output string */
                *output_ptr++ = c;
            }
        }
endfield:

        /* Terminate attribute value in output area */
        *output_ptr++ = '\0';

        /* Check whether raw input matched null marker */
        input_len = end_ptr - start_ptr;
        if (!saw_quote && input_len == cstate->null_print_len &&
            strncmp(start_ptr, cstate->null_print, input_len) == 0)
            cstate->raw_fields[fieldno] = NULL;

        fieldno++;
        /* Done if we hit EOL instead of a delim */
        if (!found_delim)
            break;
    }

    /* Clean up state of attribute_buf */
    output_ptr--;
    Assert(*output_ptr == '\0');
    cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);

    return fieldno;
}

static int CopyReadAttributesText ( CopyState  cstate )

static

Definition at line 3939 of file copy.c.

References CopyStateData::attribute_buf, StringInfoData::data, CopyStateData::delim, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, GetDecimalFromHex(), IS_HIGHBIT_SET, ISOCTAL, StringInfoData::len, CopyStateData::line_buf, CopyStateData::max_fields, StringInfoData::maxlen, NULL, CopyStateData::null_print, CopyStateData::null_print_len, OCTVALUE, pg_verifymbstr(), CopyStateData::raw_fields, repalloc(), resetStringInfo(), and val.

Referenced by NextCopyFromRawFields().

{
    char        delimc = cstate->delim[0];
    int         fieldno;
    char       *output_ptr;
    char       *cur_ptr;
    char       *line_end_ptr;

    /*
     * We need a special case for zero-column tables: check that the input
     * line is empty, and return.
     */
    if (cstate->max_fields <= 0)
    {
        if (cstate->line_buf.len != 0)
            ereport(ERROR,
                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                     errmsg("extra data after last expected column")));
        return 0;
    }

    resetStringInfo(&cstate->attribute_buf);

    /*
     * The de-escaped attributes will certainly not be longer than the input
     * data line, so we can just force attribute_buf to be large enough and
     * then transfer data without any checks for enough space.  We need to do
     * it this way because enlarging attribute_buf mid-stream would invalidate
     * pointers already stored into cstate->raw_fields[].
     */
    if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
        enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
    output_ptr = cstate->attribute_buf.data;

    /* set pointer variables for loop */
    cur_ptr = cstate->line_buf.data;
    line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;

    /* Outer loop iterates over fields */
    fieldno = 0;
    for (;;)
    {
        bool        found_delim = false;
        char       *start_ptr;
        char       *end_ptr;
        int         input_len;
        bool        saw_non_ascii = false;

        /* Make sure there is enough space for the next value */
        if (fieldno >= cstate->max_fields)
        {
            cstate->max_fields *= 2;
            cstate->raw_fields =
                repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
        }

        /* Remember start of field on both input and output sides */
        start_ptr = cur_ptr;
        cstate->raw_fields[fieldno] = output_ptr;

        /*
         * Scan data for field.
         *
         * Note that in this loop, we are scanning to locate the end of field
         * and also speculatively performing de-escaping.  Once we find the
         * end-of-field, we can match the raw field contents against the null
         * marker string.  Only after that comparison fails do we know that
         * de-escaping is actually the right thing to do; therefore we *must
         * not* throw any syntax errors before we've done the null-marker
         * check.
         */
        for (;;)
        {
            char        c;

            end_ptr = cur_ptr;
            if (cur_ptr >= line_end_ptr)
                break;
            c = *cur_ptr++;
            if (c == delimc)
            {
                found_delim = true;
                break;
            }
            if (c == '\\')
            {
                if (cur_ptr >= line_end_ptr)
                    break;
                c = *cur_ptr++;
                switch (c)
                {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                        {
                            /* handle \013 */
                            int         val;

                            val = OCTVALUE(c);
                            if (cur_ptr < line_end_ptr)
                            {
                                c = *cur_ptr;
                                if (ISOCTAL(c))
                                {
                                    cur_ptr++;
                                    val = (val << 3) + OCTVALUE(c);
                                    if (cur_ptr < line_end_ptr)
                                    {
                                        c = *cur_ptr;
                                        if (ISOCTAL(c))
                                        {
                                            cur_ptr++;
                                            val = (val << 3) + OCTVALUE(c);
                                        }
                                    }
                                }
                            }
                            c = val & 0377;
                            if (c == '\0' || IS_HIGHBIT_SET(c))
                                saw_non_ascii = true;
                        }
                        break;
                    case 'x':
                        /* Handle \x3F */
                        if (cur_ptr < line_end_ptr)
                        {
                            char        hexchar = *cur_ptr;

                            if (isxdigit((unsigned char) hexchar))
                            {
                                int         val = GetDecimalFromHex(hexchar);

                                cur_ptr++;
                                if (cur_ptr < line_end_ptr)
                                {
                                    hexchar = *cur_ptr;
                                    if (isxdigit((unsigned char) hexchar))
                                    {
                                        cur_ptr++;
                                        val = (val << 4) + GetDecimalFromHex(hexchar);
                                    }
                                }
                                c = val & 0xff;
                                if (c == '\0' || IS_HIGHBIT_SET(c))
                                    saw_non_ascii = true;
                            }
                        }
                        break;
                    case 'b':
                        c = '\b';
                        break;
                    case 'f':
                        c = '\f';
                        break;
                    case 'n':
                        c = '\n';
                        break;
                    case 'r':
                        c = '\r';
                        break;
                    case 't':
                        c = '\t';
                        break;
                    case 'v':
                        c = '\v';
                        break;

                        /*
                         * in all other cases, take the char after '\'
                         * literally
                         */
                }
            }

            /* Add c to output string */
            *output_ptr++ = c;
        }

        /* Check whether raw input matched null marker */
        input_len = end_ptr - start_ptr;
        if (input_len == cstate->null_print_len &&
            strncmp(start_ptr, cstate->null_print, input_len) == 0)
            cstate->raw_fields[fieldno] = NULL;
        else
        {
            /*
             * At this point we know the field is supposed to contain data.
             *
             * If we de-escaped any non-7-bit-ASCII chars, make sure the
             * resulting string is valid data for the db encoding.
             */
            if (saw_non_ascii)
            {
                char       *fld = cstate->raw_fields[fieldno];

                pg_verifymbstr(fld, output_ptr - fld, false);
            }
        }

        /* Terminate attribute value in output area */
        *output_ptr++ = '\0';

        fieldno++;
        /* Done if we hit EOL instead of a delim */
        if (!found_delim)
            break;
    }

    /* Clean up state of attribute_buf */
    output_ptr--;
    Assert(*output_ptr == '\0');
    cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);

    return fieldno;
}

static Datum CopyReadBinaryAttribute ( CopyState  cstate, int  column_no, FmgrInfo *  flinfo, Oid  typioparam, int32  typmod, bool *  isnull  )

static

Definition at line 4336 of file copy.c.

References CopyStateData::attribute_buf, CopyGetData(), CopyGetInt32(), StringInfoData::cursor, StringInfoData::data, enlargeStringInfo(), ereport, errcode(), errmsg(), ERROR, StringInfoData::len, NULL, ReceiveFunctionCall(), resetStringInfo(), and result.

Referenced by NextCopyFrom().

{
    int32       fld_size;
    Datum       result;

    if (!CopyGetInt32(cstate, &fld_size))
        ereport(ERROR,
                (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                 errmsg("unexpected EOF in COPY data")));
    if (fld_size == -1)
    {
        *isnull = true;
        return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
    }
    if (fld_size < 0)
        ereport(ERROR,
                (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                 errmsg("invalid field size")));

    /* reset attribute_buf to empty, and load raw data in it */
    resetStringInfo(&cstate->attribute_buf);

    enlargeStringInfo(&cstate->attribute_buf, fld_size);
    if (CopyGetData(cstate, cstate->attribute_buf.data,
                    fld_size, fld_size) != fld_size)
        ereport(ERROR,
                (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                 errmsg("unexpected EOF in COPY data")));

    cstate->attribute_buf.len = fld_size;
    cstate->attribute_buf.data[fld_size] = '\0';

    /* Call the column type's binary input converter */
    result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
                                 typioparam, typmod);

    /* Trouble if it didn't eat the whole buffer */
    if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
                 errmsg("incorrect binary data format")));

    *isnull = false;
    return result;
}

static bool CopyReadLine ( CopyState  cstate )

static

Definition at line 3463 of file copy.c.

References appendBinaryStringInfo(), Assert, CopyStateData::copy_dest, COPY_NEW_FE, CopyLoadRawBuf(), CopyReadLineText(), StringInfoData::data, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, CopyStateData::file_encoding, StringInfoData::len, CopyStateData::line_buf, CopyStateData::line_buf_converted, CopyStateData::line_buf_valid, CopyStateData::need_transcoding, pfree(), pg_any_to_server(), CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, resetStringInfo(), and result.

Referenced by NextCopyFromRawFields().

{
    bool        result;

    resetStringInfo(&cstate->line_buf);
    cstate->line_buf_valid = true;

    /* Mark that encoding conversion hasn't occurred yet */
    cstate->line_buf_converted = false;

    /* Parse data and transfer into line_buf */
    result = CopyReadLineText(cstate);

    if (result)
    {
        /*
         * Reached EOF.  In protocol version 3, we should ignore anything
         * after \. up to the protocol end of copy data.  (XXX maybe better
         * not to treat \. as special?)
         */
        if (cstate->copy_dest == COPY_NEW_FE)
        {
            do
            {
                cstate->raw_buf_index = cstate->raw_buf_len;
            } while (CopyLoadRawBuf(cstate));
        }
    }
    else
    {
        /*
         * If we didn't hit EOF, then we must have transferred the EOL marker
         * to line_buf along with the data.  Get rid of it.
         */
        switch (cstate->eol_type)
        {
            case EOL_NL:
                Assert(cstate->line_buf.len >= 1);
                Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
                cstate->line_buf.len--;
                cstate->line_buf.data[cstate->line_buf.len] = '\0';
                break;
            case EOL_CR:
                Assert(cstate->line_buf.len >= 1);
                Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
                cstate->line_buf.len--;
                cstate->line_buf.data[cstate->line_buf.len] = '\0';
                break;
            case EOL_CRNL:
                Assert(cstate->line_buf.len >= 2);
                Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
                Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
                cstate->line_buf.len -= 2;
                cstate->line_buf.data[cstate->line_buf.len] = '\0';
                break;
            case EOL_UNKNOWN:
                /* shouldn't get here */
                Assert(false);
                break;
        }
    }

    /* Done reading the line.  Convert it to server encoding. */
    if (cstate->need_transcoding)
    {
        char       *cvt;

        cvt = pg_any_to_server(cstate->line_buf.data,
                               cstate->line_buf.len,
                               cstate->file_encoding);
        if (cvt != cstate->line_buf.data)
        {
            /* transfer converted data back to line_buf */
            resetStringInfo(&cstate->line_buf);
            appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
            pfree(cvt);
        }
    }

    /* Now it's safe to use the buffer in error messages */
    cstate->line_buf_converted = true;

    return result;
}

static bool CopyReadLineText ( CopyState  cstate )

static

Definition at line 3552 of file copy.c.

References appendBinaryStringInfo(), CopyLoadRawBuf(), CopyStateData::csv_mode, CopyStateData::cur_lineno, CopyStateData::encoding_embeds_ascii, EOL_CR, EOL_CRNL, EOL_NL, CopyStateData::eol_type, EOL_UNKNOWN, ereport, errcode(), errhint(), errmsg(), ERROR, CopyStateData::escape, CopyStateData::file_encoding, IF_NEED_REFILL_AND_EOF_BREAK, IF_NEED_REFILL_AND_NOT_EOF_CONTINUE, IS_HIGHBIT_SET, CopyStateData::line_buf, NO_END_OF_COPY_GOTO, pg_encoding_mblen(), CopyStateData::quote, CopyStateData::raw_buf, CopyStateData::raw_buf_index, CopyStateData::raw_buf_len, REFILL_LINEBUF, and result.

Referenced by CopyReadLine().

{
    char       *copy_raw_buf;
    int         raw_buf_ptr;
    int         copy_buf_len;
    bool        need_data = false;
    bool        hit_eof = false;
    bool        result = false;
    char        mblen_str[2];

    /* CSV variables */
    bool        first_char_in_line = true;
    bool        in_quote = false,
                last_was_esc = false;
    char        quotec = '\0';
    char        escapec = '\0';

    if (cstate->csv_mode)
    {
        quotec = cstate->quote[0];
        escapec = cstate->escape[0];
        /* ignore special escape processing if it's the same as quotec */
        if (quotec == escapec)
            escapec = '\0';
    }

    mblen_str[1] = '\0';

    /*
     * The objective of this loop is to transfer the entire next input line
     * into line_buf.  Hence, we only care for detecting newlines (\r and/or
     * \n) and the end-of-copy marker (\.).
     *
     * In CSV mode, \r and \n inside a quoted field are just part of the data
     * value and are put in line_buf.  We keep just enough state to know if we
     * are currently in a quoted field or not.
     *
     * These four characters, and the CSV escape and quote characters, are
     * assumed the same in frontend and backend encodings.
     *
     * For speed, we try to move data from raw_buf to line_buf in chunks
     * rather than one character at a time.  raw_buf_ptr points to the next
     * character to examine; any characters from raw_buf_index to raw_buf_ptr
     * have been determined to be part of the line, but not yet transferred to
     * line_buf.
     *
     * For a little extra speed within the loop, we copy raw_buf and
     * raw_buf_len into local variables.
     */
    copy_raw_buf = cstate->raw_buf;
    raw_buf_ptr = cstate->raw_buf_index;
    copy_buf_len = cstate->raw_buf_len;

    for (;;)
    {
        int         prev_raw_ptr;
        char        c;

        /*
         * Load more data if needed.  Ideally we would just force four bytes
         * of read-ahead and avoid the many calls to
         * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
         * does not allow us to read too far ahead or we might read into the
         * next data, so we read-ahead only as far we know we can.  One
         * optimization would be to read-ahead four byte here if
         * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
         * considering the size of the buffer.
         */
        if (raw_buf_ptr >= copy_buf_len || need_data)
        {
            REFILL_LINEBUF;

            /*
             * Try to read some more data.  This will certainly reset
             * raw_buf_index to zero, and raw_buf_ptr must go with it.
             */
            if (!CopyLoadRawBuf(cstate))
                hit_eof = true;
            raw_buf_ptr = 0;
            copy_buf_len = cstate->raw_buf_len;

            /*
             * If we are completely out of data, break out of the loop,
             * reporting EOF.
             */
            if (copy_buf_len <= 0)
            {
                result = true;
                break;
            }
            need_data = false;
        }

        /* OK to fetch a character */
        prev_raw_ptr = raw_buf_ptr;
        c = copy_raw_buf[raw_buf_ptr++];

        if (cstate->csv_mode)
        {
            /*
             * If character is '\\' or '\r', we may need to look ahead below.
             * Force fetch of the next character if we don't already have it.
             * We need to do this before changing CSV state, in case one of
             * these characters is also the quote or escape character.
             *
             * Note: old-protocol does not like forced prefetch, but it's OK
             * here since we cannot validly be at EOF.
             */
            if (c == '\\' || c == '\r')
            {
                IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
            }

            /*
             * Dealing with quotes and escapes here is mildly tricky. If the
             * quote char is also the escape char, there's no problem - we
             * just use the char as a toggle. If they are different, we need
             * to ensure that we only take account of an escape inside a
             * quoted field and immediately preceding a quote char, and not
             * the second in an escape-escape sequence.
             */
            if (in_quote && c == escapec)
                last_was_esc = !last_was_esc;
            if (c == quotec && !last_was_esc)
                in_quote = !in_quote;
            if (c != escapec)
                last_was_esc = false;

            /*
             * Updating the line count for embedded CR and/or LF chars is
             * necessarily a little fragile - this test is probably about the
             * best we can do.  (XXX it's arguable whether we should do this
             * at all --- is cur_lineno a physical or logical count?)
             */
            if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
                cstate->cur_lineno++;
        }

        /* Process \r */
        if (c == '\r' && (!cstate->csv_mode || !in_quote))
        {
            /* Check for \r\n on first line, _and_ handle \r\n. */
            if (cstate->eol_type == EOL_UNKNOWN ||
                cstate->eol_type == EOL_CRNL)
            {
                /*
                 * If need more data, go back to loop top to load it.
                 *
                 * Note that if we are at EOF, c will wind up as '\0' because
                 * of the guaranteed pad of raw_buf.
                 */
                IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);

                /* get next char */
                c = copy_raw_buf[raw_buf_ptr];

                if (c == '\n')
                {
                    raw_buf_ptr++;  /* eat newline */
                    cstate->eol_type = EOL_CRNL;    /* in case not set yet */
                }
                else
                {
                    /* found \r, but no \n */
                    if (cstate->eol_type == EOL_CRNL)
                        ereport(ERROR,
                                (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                                 !cstate->csv_mode ?
                                 errmsg("literal carriage return found in data") :
                                 errmsg("unquoted carriage return found in data"),
                                 !cstate->csv_mode ?
                                 errhint("Use \"\\r\" to represent carriage return.") :
                                 errhint("Use quoted CSV field to represent carriage return.")));

                    /*
                     * if we got here, it is the first line and we didn't find
                     * \n, so don't consume the peeked character
                     */
                    cstate->eol_type = EOL_CR;
                }
            }
            else if (cstate->eol_type == EOL_NL)
                ereport(ERROR,
                        (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                         !cstate->csv_mode ?
                         errmsg("literal carriage return found in data") :
                         errmsg("unquoted carriage return found in data"),
                         !cstate->csv_mode ?
                         errhint("Use \"\\r\" to represent carriage return.") :
                         errhint("Use quoted CSV field to represent carriage return.")));
            /* If reach here, we have found the line terminator */
            break;
        }

        /* Process \n */
        if (c == '\n' && (!cstate->csv_mode || !in_quote))
        {
            if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
                ereport(ERROR,
                        (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                         !cstate->csv_mode ?
                         errmsg("literal newline found in data") :
                         errmsg("unquoted newline found in data"),
                         !cstate->csv_mode ?
                         errhint("Use \"\\n\" to represent newline.") :
                         errhint("Use quoted CSV field to represent newline.")));
            cstate->eol_type = EOL_NL;  /* in case not set yet */
            /* If reach here, we have found the line terminator */
            break;
        }

        /*
         * In CSV mode, we only recognize \. alone on a line.  This is because
         * \. is a valid CSV data value.
         */
        if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
        {
            char        c2;

            IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
            IF_NEED_REFILL_AND_EOF_BREAK(0);

            /* -----
             * get next character
             * Note: we do not change c so if it isn't \., we can fall
             * through and continue processing for file encoding.
             * -----
             */
            c2 = copy_raw_buf[raw_buf_ptr];

            if (c2 == '.')
            {
                raw_buf_ptr++;  /* consume the '.' */

                /*
                 * Note: if we loop back for more data here, it does not
                 * matter that the CSV state change checks are re-executed; we
                 * will come back here with no important state changed.
                 */
                if (cstate->eol_type == EOL_CRNL)
                {
                    /* Get the next character */
                    IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
                    /* if hit_eof, c2 will become '\0' */
                    c2 = copy_raw_buf[raw_buf_ptr++];

                    if (c2 == '\n')
                    {
                        if (!cstate->csv_mode)
                            ereport(ERROR,
                                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                                     errmsg("end-of-copy marker does not match previous newline style")));
                        else
                            NO_END_OF_COPY_GOTO;
                    }
                    else if (c2 != '\r')
                    {
                        if (!cstate->csv_mode)
                            ereport(ERROR,
                                    (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                                     errmsg("end-of-copy marker corrupt")));
                        else
                            NO_END_OF_COPY_GOTO;
                    }
                }

                /* Get the next character */
                IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
                /* if hit_eof, c2 will become '\0' */
                c2 = copy_raw_buf[raw_buf_ptr++];

                if (c2 != '\r' && c2 != '\n')
                {
                    if (!cstate->csv_mode)
                        ereport(ERROR,
                                (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                                 errmsg("end-of-copy marker corrupt")));
                    else
                        NO_END_OF_COPY_GOTO;
                }

                if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
                    (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
                    (cstate->eol_type == EOL_CR && c2 != '\r'))
                {
                    ereport(ERROR,
                            (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
                             errmsg("end-of-copy marker does not match previous newline style")));
                }

                /*
                 * Transfer only the data before the \. into line_buf, then
                 * discard the data and the \. sequence.
                 */
                if (prev_raw_ptr > cstate->raw_buf_index)
                    appendBinaryStringInfo(&cstate->line_buf,
                                           cstate->raw_buf + cstate->raw_buf_index,
                                           prev_raw_ptr - cstate->raw_buf_index);
                cstate->raw_buf_index = raw_buf_ptr;
                result = true;  /* report EOF */
                break;
            }
            else if (!cstate->csv_mode)

                /*
                 * If we are here, it means we found a backslash followed by
                 * something other than a period.  In non-CSV mode, anything
                 * after a backslash is special, so we skip over that second
                 * character too.  If we didn't do that \\. would be
                 * considered an eof-of copy, while in non-CSV mode it is a
                 * literal backslash followed by a period.  In CSV mode,
                 * backslashes are not special, so we want to process the
                 * character after the backslash just like a normal character,
                 * so we don't increment in those cases.
                 */
                raw_buf_ptr++;
        }

        /*
         * This label is for CSV cases where \. appears at the start of a
         * line, but there is more text after it, meaning it was a data value.
         * We are more strict for \. in CSV mode because \. could be a data
         * value, while in non-CSV mode, \. cannot be a data value.
         */
not_end_of_copy:

        /*
         * Process all bytes of a multi-byte character as a group.
         *
         * We only support multi-byte sequences where the first byte has the
         * high-bit set, so as an optimization we can avoid this block
         * entirely if it is not set.
         */
        if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
        {
            int         mblen;

            mblen_str[0] = c;
            /* All our encodings only read the first byte to get the length */
            mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
            IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
            IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
            raw_buf_ptr += mblen - 1;
        }
        first_char_in_line = false;
    }                           /* end of outer loop */

    /*
     * Transfer any still-uncopied data to line_buf.
     */
    REFILL_LINEBUF;

    return result;
}

static void CopySendChar ( CopyState  cstate, char  c  )

static

Definition at line 457 of file copy.c.

References appendStringInfoCharMacro, and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyAttributeOutText(), CopyOneRowTo(), CopySendEndOfRow(), and CopyTo().

{
    appendStringInfoCharMacro(cstate->fe_msgbuf, c);
}

static void CopySendData ( CopyState  cstate, const void *  databuf, int  datasize  )

static

Definition at line 445 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyOneRowTo(), CopySendInt16(), CopySendInt32(), CopyTo(), and SendCopyEnd().

{
    appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
}

static void CopySendEndOfRow ( CopyState  cstate )

static

Definition at line 463 of file copy.c.

References Assert, CopyStateData::binary, ClosePipeToProgram(), COPY_CALLBACK, CopyStateData::copy_dest, COPY_FILE, CopyStateData::copy_file, COPY_NEW_FE, COPY_OLD_FE, CopySendChar(), CopySendString(), StringInfoData::data, ereport, errcode(), errcode_for_file_access(), errmsg(), ERROR, FATAL, CopyStateData::fe_msgbuf, CopyStateData::is_program, StringInfoData::len, pq_putbytes(), pq_putmessage, and resetStringInfo().

Referenced by CopyOneRowTo(), CopyTo(), and SendCopyEnd().

{
    StringInfo  fe_msgbuf = cstate->fe_msgbuf;

    switch (cstate->copy_dest)
    {
        case COPY_FILE:
            if (!cstate->binary)
            {
                /* Default line termination depends on platform */
#ifndef WIN32
                CopySendChar(cstate, '\n');
#else
                CopySendString(cstate, "\r\n");
#endif
            }

            if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
                       cstate->copy_file) != 1 ||
                ferror(cstate->copy_file))
            {
                if (cstate->is_program)
                {
                    if (errno == EPIPE)
                    {
                        /*
                         * The pipe will be closed automatically on error at
                         * the end of transaction, but we might get a better
                         * error message from the subprocess' exit code than
                         * just "Broken Pipe"
                         */
                        ClosePipeToProgram(cstate);

                        /*
                         * If ClosePipeToProgram() didn't throw an error, the
                         * program terminated normally, but closed the pipe
                         * first. Restore errno, and throw an error.
                         */
                        errno = EPIPE;
                    }
                    ereport(ERROR,
                            (errcode_for_file_access(),
                             errmsg("could not write to COPY program: %m")));
                }
                else
                    ereport(ERROR,
                            (errcode_for_file_access(),
                             errmsg("could not write to COPY file: %m")));
            }
            break;
        case COPY_OLD_FE:
            /* The FE/BE protocol uses \n as newline for all platforms */
            if (!cstate->binary)
                CopySendChar(cstate, '\n');

            if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
            {
                /* no hope of recovering connection sync, so FATAL */
                ereport(FATAL,
                        (errcode(ERRCODE_CONNECTION_FAILURE),
                         errmsg("connection lost during COPY to stdout")));
            }
            break;
        case COPY_NEW_FE:
            /* The FE/BE protocol uses \n as newline for all platforms */
            if (!cstate->binary)
                CopySendChar(cstate, '\n');

            /* Dump the accumulated row as one CopyData message */
            (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
            break;
        case COPY_CALLBACK:
            Assert(false);      /* Not yet supported. */
            break;
    }

    resetStringInfo(fe_msgbuf);
}

static void CopySendInt16 ( CopyState  cstate, int16  val  )

static

Definition at line 699 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

{
    uint16      buf;

    buf = htons((uint16) val);
    CopySendData(cstate, &buf, sizeof(buf));
}

static void CopySendInt32 ( CopyState  cstate, int32  val  )

static

Definition at line 668 of file copy.c.

References buf, and CopySendData().

Referenced by CopyOneRowTo(), and CopyTo().

{
    uint32      buf;

    buf = htonl((uint32) val);
    CopySendData(cstate, &buf, sizeof(buf));
}

static void CopySendString ( CopyState  cstate, const char *  str  )

static

Definition at line 451 of file copy.c.

References appendBinaryStringInfo(), and CopyStateData::fe_msgbuf.

Referenced by CopyAttributeOutCSV(), CopyOneRowTo(), and CopySendEndOfRow().

{
    appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
}

static uint64 CopyTo ( CopyState  cstate )

static

Definition at line 1940 of file copy.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate(), CopyStateData::attnumlist, tupleDesc::attrs, CopyStateData::binary, BinarySignature, CHECK_FOR_INTERRUPTS, CopyAttributeOutCSV(), CopyOneRowTo(), CopySendChar(), CopySendData(), CopySendEndOfRow(), CopySendInt16(), CopySendInt32(), cur, CurrentMemoryContext, CopyStateData::delim, QueryDesc::dest, ExecutorRun(), CopyStateData::fe_msgbuf, CopyStateData::file_encoding, fmgr_info(), ForwardScanDirection, GetActiveSnapshot(), getTypeBinaryOutputInfo(), getTypeOutputInfo(), CopyStateData::header_line, heap_beginscan(), heap_deform_tuple(), heap_endscan(), heap_getnext(), HeapTupleGetOid, lfirst_int, list_length(), makeStringInfo(), MemoryContextDelete(), NameStr, tupleDesc::natts, CopyStateData::need_transcoding, NULL, CopyStateData::null_print, CopyStateData::null_print_client, CopyStateData::null_print_len, CopyStateData::oids, CopyStateData::out_functions, palloc(), pfree(), pg_server_to_any(), CopyStateData::queryDesc, CopyStateData::rel, RelationGetDescr, CopyStateData::rowcontext, QueryDesc::tupDesc, and values.

Referenced by DoCopyTo().

{
    TupleDesc   tupDesc;
    int         num_phys_attrs;
    Form_pg_attribute *attr;
    ListCell   *cur;
    uint64      processed;

    if (cstate->rel)
        tupDesc = RelationGetDescr(cstate->rel);
    else
        tupDesc = cstate->queryDesc->tupDesc;
    attr = tupDesc->attrs;
    num_phys_attrs = tupDesc->natts;
    cstate->null_print_client = cstate->null_print; /* default */

    /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
    cstate->fe_msgbuf = makeStringInfo();

    /* Get info about the columns we need to process. */
    cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
    foreach(cur, cstate->attnumlist)
    {
        int         attnum = lfirst_int(cur);
        Oid         out_func_oid;
        bool        isvarlena;

        if (cstate->binary)
            getTypeBinaryOutputInfo(attr[attnum - 1]->atttypid,
                                    &out_func_oid,
                                    &isvarlena);
        else
            getTypeOutputInfo(attr[attnum - 1]->atttypid,
                              &out_func_oid,
                              &isvarlena);
        fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
    }

    /*
     * Create a temporary memory context that we can reset once per row to
     * recover palloc'd memory.  This avoids any problems with leaks inside
     * datatype output routines, and should be faster than retail pfree's
     * anyway.  (We don't need a whole econtext as CopyFrom does.)
     */
    cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
                                               "COPY TO",
                                               ALLOCSET_DEFAULT_SIZES);

    if (cstate->binary)
    {
        /* Generate header for a binary copy */
        int32       tmp;

        /* Signature */
        CopySendData(cstate, BinarySignature, 11);
        /* Flags field */
        tmp = 0;
        if (cstate->oids)
            tmp |= (1 << 16);
        CopySendInt32(cstate, tmp);
        /* No header extension */
        tmp = 0;
        CopySendInt32(cstate, tmp);
    }
    else
    {
        /*
         * For non-binary copy, we need to convert null_print to file
         * encoding, because it will be sent directly with CopySendString.
         */
        if (cstate->need_transcoding)
            cstate->null_print_client = pg_server_to_any(cstate->null_print,
                                                         cstate->null_print_len,
                                                         cstate->file_encoding);

        /* if a header has been requested send the line */
        if (cstate->header_line)
        {
            bool        hdr_delim = false;

            foreach(cur, cstate->attnumlist)
            {
                int         attnum = lfirst_int(cur);
                char       *colname;

                if (hdr_delim)
                    CopySendChar(cstate, cstate->delim[0]);
                hdr_delim = true;

                colname = NameStr(attr[attnum - 1]->attname);

                CopyAttributeOutCSV(cstate, colname, false,
                                    list_length(cstate->attnumlist) == 1);
            }

            CopySendEndOfRow(cstate);
        }
    }

    if (cstate->rel)
    {
        Datum      *values;
        bool       *nulls;
        HeapScanDesc scandesc;
        HeapTuple   tuple;

        values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
        nulls = (bool *) palloc(num_phys_attrs * sizeof(bool));

        scandesc = heap_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);

        processed = 0;
        while ((tuple = heap_getnext(scandesc, ForwardScanDirection)) != NULL)
        {
            CHECK_FOR_INTERRUPTS();

            /* Deconstruct the tuple ... faster than repeated heap_getattr */
            heap_deform_tuple(tuple, tupDesc, values, nulls);

            /* Format and send the data */
            CopyOneRowTo(cstate, HeapTupleGetOid(tuple), values, nulls);
            processed++;
        }

        heap_endscan(scandesc);

        pfree(values);
        pfree(nulls);
    }
    else
    {
        /* run the plan --- the dest receiver will send tuples */
        ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
        processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
    }

    if (cstate->binary)
    {
        /* Generate trailer for a binary copy */
        CopySendInt16(cstate, -1);
        /* Need to flush out the trailer */
        CopySendEndOfRow(cstate);
    }

    MemoryContextDelete(cstate->rowcontext);

    return processed;
}

DestReceiver* CreateCopyDestReceiver

(

void

)

Definition at line 4757 of file copy.c.

References copy_dest_destroy(), copy_dest_receive(), copy_dest_shutdown(), copy_dest_startup(), DestCopyOut, NULL, and palloc().

Referenced by CreateDestReceiver().

{
    DR_copy    *self = (DR_copy *) palloc(sizeof(DR_copy));

    self->pub.receiveSlot = copy_dest_receive;
    self->pub.rStartup = copy_dest_startup;
    self->pub.rShutdown = copy_dest_shutdown;
    self->pub.rDestroy = copy_dest_destroy;
    self->pub.mydest = DestCopyOut;

    self->cstate = NULL;        /* will be set later */
    self->processed = 0;

    return (DestReceiver *) self;
}

void DoCopy	(	ParseState *	pstate,
		const CopyStmt *	stmt,
		int	stmt_location,
		int	stmt_len,
		uint64 *	processed
	)

Definition at line 781 of file copy.c.

References AccessShareLock, ACL_INSERT, ACL_SELECT, addRangeTableEntryForRelation(), Assert, CopyStmt::attlist, BeginCopyFrom(), BeginCopyTo(), bms_add_member(), check_enable_rls(), CopyFrom(), CopyGetAttnums(), cur, DoCopyTo(), EndCopyFrom(), EndCopyTo(), ereport, errcode(), errhint(), errmsg(), ERROR, ExecCheckRTPerms(), ColumnRef::fields, CopyStmt::filename, FirstLowInvalidHeapAttributeNumber, SelectStmt::fromClause, get_namespace_name(), heap_close, heap_openrv(), ResTarget::indirection, RangeTblEntry::insertedCols, InvalidOid, CopyStmt::is_from, CopyStmt::is_program, lappend(), lfirst, lfirst_int, list_make1, ColumnRef::location, ResTarget::location, makeNode, makeRangeVar(), ResTarget::name, NIL, NoLock, NULL, CopyStmt::options, ParseState::p_rtable, PreventCommandIfParallelMode(), PreventCommandIfReadOnly(), pstrdup(), CopyStmt::query, RelationData::rd_islocaltemp, CopyStmt::relation, RelationGetDescr, RelationGetNamespace, RelationGetRelationName, RelationGetRelid, RangeTblEntry::relid, RangeTblEntry::requiredPerms, RLS_ENABLED, RowExclusiveLock, select, RangeTblEntry::selectedCols, RawStmt::stmt, RawStmt::stmt_len, RawStmt::stmt_location, superuser(), SelectStmt::targetList, ResTarget::val, and XactReadOnly.

Referenced by standard_ProcessUtility().

{
    CopyState   cstate;
    bool        is_from = stmt->is_from;
    bool        pipe = (stmt->filename == NULL);
    Relation    rel;
    Oid         relid;
    RawStmt    *query = NULL;

    /* Disallow COPY to/from file or program except to superusers. */
    if (!pipe && !superuser())
    {
        if (stmt->is_program)
            ereport(ERROR,
                    (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
                     errmsg("must be superuser to COPY to or from an external program"),
                     errhint("Anyone can COPY to stdout or from stdin. "
                             "psql's \\copy command also works for anyone.")));
        else
            ereport(ERROR,
                    (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
                     errmsg("must be superuser to COPY to or from a file"),
                     errhint("Anyone can COPY to stdout or from stdin. "
                             "psql's \\copy command also works for anyone.")));
    }

    if (stmt->relation)
    {
        TupleDesc   tupDesc;
        List       *attnums;
        ListCell   *cur;
        RangeTblEntry *rte;

        Assert(!stmt->query);

        /* Open and lock the relation, using the appropriate lock type. */
        rel = heap_openrv(stmt->relation,
                          (is_from ? RowExclusiveLock : AccessShareLock));

        relid = RelationGetRelid(rel);

        rte = addRangeTableEntryForRelation(pstate, rel, NULL, false, false);
        rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);

        tupDesc = RelationGetDescr(rel);
        attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
        foreach(cur, attnums)
        {
            int         attno = lfirst_int(cur) -
            FirstLowInvalidHeapAttributeNumber;

            if (is_from)
                rte->insertedCols = bms_add_member(rte->insertedCols, attno);
            else
                rte->selectedCols = bms_add_member(rte->selectedCols, attno);
        }
        ExecCheckRTPerms(pstate->p_rtable, true);

        /*
         * Permission check for row security policies.
         *
         * check_enable_rls will ereport(ERROR) if the user has requested
         * something invalid and will otherwise indicate if we should enable
         * RLS (returns RLS_ENABLED) or not for this COPY statement.
         *
         * If the relation has a row security policy and we are to apply it
         * then perform a "query" copy and allow the normal query processing
         * to handle the policies.
         *
         * If RLS is not enabled for this, then just fall through to the
         * normal non-filtering relation handling.
         */
        if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
        {
            SelectStmt *select;
            ColumnRef  *cr;
            ResTarget  *target;
            RangeVar   *from;
            List       *targetList = NIL;

            if (is_from)
                ereport(ERROR,
                        (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                         errmsg("COPY FROM not supported with row-level security"),
                         errhint("Use INSERT statements instead.")));

            /*
             * Build target list
             *
             * If no columns are specified in the attribute list of the COPY
             * command, then the target list is 'all' columns. Therefore, '*'
             * should be used as the target list for the resulting SELECT
             * statement.
             *
             * In the case that columns are specified in the attribute list,
             * create a ColumnRef and ResTarget for each column and add them
             * to the target list for the resulting SELECT statement.
             */
            if (!stmt->attlist)
            {
                cr = makeNode(ColumnRef);
                cr->fields = list_make1(makeNode(A_Star));
                cr->location = -1;

                target = makeNode(ResTarget);
                target->name = NULL;
                target->indirection = NIL;
                target->val = (Node *) cr;
                target->location = -1;

                targetList = list_make1(target);
            }
            else
            {
                ListCell   *lc;

                foreach(lc, stmt->attlist)
                {
                    /*
                     * Build the ColumnRef for each column.  The ColumnRef
                     * 'fields' property is a String 'Value' node (see
                     * nodes/value.h) that corresponds to the column name
                     * respectively.
                     */
                    cr = makeNode(ColumnRef);
                    cr->fields = list_make1(lfirst(lc));
                    cr->location = -1;

                    /* Build the ResTarget and add the ColumnRef to it. */
                    target = makeNode(ResTarget);
                    target->name = NULL;
                    target->indirection = NIL;
                    target->val = (Node *) cr;
                    target->location = -1;

                    /* Add each column to the SELECT statement's target list */
                    targetList = lappend(targetList, target);
                }
            }

            /*
             * Build RangeVar for from clause, fully qualified based on the
             * relation which we have opened and locked.
             */
            from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
                                pstrdup(RelationGetRelationName(rel)),
                                -1);

            /* Build query */
            select = makeNode(SelectStmt);
            select->targetList = targetList;
            select->fromClause = list_make1(from);

            query = makeNode(RawStmt);
            query->stmt = (Node *) select;
            query->stmt_location = stmt_location;
            query->stmt_len = stmt_len;

            /*
             * Close the relation for now, but keep the lock on it to prevent
             * changes between now and when we start the query-based COPY.
             *
             * We'll reopen it later as part of the query-based COPY.
             */
            heap_close(rel, NoLock);
            rel = NULL;
        }
    }
    else
    {
        Assert(stmt->query);

        query = makeNode(RawStmt);
        query->stmt = stmt->query;
        query->stmt_location = stmt_location;
        query->stmt_len = stmt_len;

        relid = InvalidOid;
        rel = NULL;
    }

    if (is_from)
    {
        Assert(rel);

        /* check read-only transaction and parallel mode */
        if (XactReadOnly && !rel->rd_islocaltemp)
            PreventCommandIfReadOnly("COPY FROM");
        PreventCommandIfParallelMode("COPY FROM");

        cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
                               NULL, stmt->attlist, stmt->options);
        *processed = CopyFrom(cstate);  /* copy from file to database */