PostgreSQL Source Code  git master
copyfrom_internal.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * copyfrom_internal.h
4  * Internal definitions for COPY FROM command.
5  *
6  *
7  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
8  * Portions Copyright (c) 1994, Regents of the University of California
9  *
10  * src/include/commands/copyfrom_internal.h
11  *
12  *-------------------------------------------------------------------------
13  */
14 #ifndef COPYFROM_INTERNAL_H
15 #define COPYFROM_INTERNAL_H
16 
17 #include "commands/copy.h"
18 #include "commands/trigger.h"
19 
20 /*
21  * Represents the different source cases we need to worry about at
22  * the bottom level
23  */
24 typedef enum CopySource
25 {
26  COPY_FILE, /* from file (or a piped program) */
27  COPY_FRONTEND, /* from frontend */
28  COPY_CALLBACK /* from callback function */
30 
31 /*
32  * Represents the end-of-line terminator type of the input
33  */
34 typedef enum EolType
35 {
39  EOL_CRNL
41 
42 /*
43  * Represents the insert method to be used during COPY FROM.
44  */
45 typedef enum CopyInsertMethod
46 {
47  CIM_SINGLE, /* use table_tuple_insert or
48  * ExecForeignInsert */
49  CIM_MULTI, /* always use table_multi_insert or
50  * ExecForeignBatchInsert */
51  CIM_MULTI_CONDITIONAL /* use table_multi_insert or
52  * ExecForeignBatchInsert only if valid */
54 
55 /*
56  * This struct contains all the state variables used throughout a COPY FROM
57  * operation.
58  */
59 typedef struct CopyFromStateData
60 {
61  /* low-level state data */
62  CopySource copy_src; /* type of copy source */
63  FILE *copy_file; /* used if copy_src == COPY_FILE */
64  StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
65 
66  EolType eol_type; /* EOL type of input */
67  int file_encoding; /* file or remote side's character encoding */
68  bool need_transcoding; /* file encoding diff from server? */
69  Oid conversion_proc; /* encoding conversion function */
70 
71  /* parameters from the COPY command */
72  Relation rel; /* relation to copy from */
73  List *attnumlist; /* integer list of attnums to copy */
74  char *filename; /* filename, or NULL for STDIN */
75  bool is_program; /* is 'filename' a program to popen? */
76  copy_data_source_cb data_source_cb; /* function for reading data */
77 
79  bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
80  Node *whereClause; /* WHERE condition (or NULL) */
81 
82  /* these are just for error messages, see CopyFromErrorCallback */
83  const char *cur_relname; /* table name for error messages */
84  uint64 cur_lineno; /* line number for error messages */
85  const char *cur_attname; /* current att for error messages */
86  const char *cur_attval; /* current att value for error messages */
87  bool relname_only; /* don't output line number, att, etc. */
88 
89  /*
90  * Working state
91  */
92  MemoryContext copycontext; /* per-copy execution context */
93 
95  FmgrInfo *in_functions; /* array of input functions for each attrs */
96  Oid *typioparams; /* array of element types for in_functions */
97  int *defmap; /* array of default att numbers */
98  ExprState **defexprs; /* array of default att expressions */
99  bool volatile_defexprs; /* is any of defexprs volatile? */
100  List *range_table; /* single element list of RangeTblEntry */
101  List *rteperminfos; /* single element list of RTEPermissionInfo */
103 
105 
106  /*
107  * These variables are used to reduce overhead in COPY FROM.
108  *
109  * attribute_buf holds the separated, de-escaped text for each field of
110  * the current line. The CopyReadAttributes functions return arrays of
111  * pointers into this buffer. We avoid palloc/pfree overhead by re-using
112  * the buffer on each cycle.
113  *
114  * In binary COPY FROM, attribute_buf holds the binary data for the
115  * current field, but the usage is otherwise similar.
116  */
118 
119  /* field raw data pointers found by COPY FROM */
120 
122  char **raw_fields;
123 
124  /*
125  * Similarly, line_buf holds the whole input line being processed. The
126  * input cycle is first to read the whole line into line_buf, and then
127  * extract the individual attribute fields into attribute_buf. line_buf
128  * is preserved unmodified so that we can display it in error messages if
129  * appropriate. (In binary mode, line_buf is not used.)
130  */
132  bool line_buf_valid; /* contains the row being processed? */
133 
134  /*
135  * input_buf holds input data, already converted to database encoding.
136  *
137  * In text mode, CopyReadLine parses this data sufficiently to locate line
138  * boundaries, then transfers the data to line_buf. We guarantee that
139  * there is a \0 at input_buf[input_buf_len] at all times. (In binary
140  * mode, input_buf is not used.)
141  *
142  * If encoding conversion is not required, input_buf is not a separate
143  * buffer but points directly to raw_buf. In that case, input_buf_len
144  * tracks the number of bytes that have been verified as valid in the
145  * database encoding, and raw_buf_len is the total number of bytes stored
146  * in the buffer.
147  */
148 #define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
149  char *input_buf;
150  int input_buf_index; /* next byte to process */
151  int input_buf_len; /* total # of bytes stored */
152  bool input_reached_eof; /* true if we reached EOF */
153  bool input_reached_error; /* true if a conversion error happened */
154  /* Shorthand for number of unconsumed bytes available in input_buf */
155 #define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
156 
157  /*
158  * raw_buf holds raw input data read from the data source (file or client
159  * connection), not yet converted to the database encoding. Like with
160  * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
161  */
162 #define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
163  char *raw_buf;
164  int raw_buf_index; /* next byte to process */
165  int raw_buf_len; /* total # of bytes stored */
166  bool raw_reached_eof; /* true if we reached EOF */
167 
168  /* Shorthand for number of unconsumed bytes available in raw_buf */
169 #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
170 
171  uint64 bytes_processed; /* number of bytes processed so far */
173 
174 extern void ReceiveCopyBegin(CopyFromState cstate);
175 extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
176 
177 #endif /* COPYFROM_INTERNAL_H */
int16 AttrNumber
Definition: attnum.h:21
struct CopyFromStateData CopyFromStateData
CopySource
@ COPY_FILE
@ COPY_CALLBACK
@ COPY_FRONTEND
CopyInsertMethod
@ CIM_SINGLE
@ CIM_MULTI_CONDITIONAL
@ CIM_MULTI
void ReceiveCopyBinaryHeader(CopyFromState cstate)
void ReceiveCopyBegin(CopyFromState cstate)
@ EOL_CR
@ EOL_CRNL
@ EOL_UNKNOWN
@ EOL_NL
int(* copy_data_source_cb)(void *outbuf, int minread, int maxread)
Definition: copy.h:68
unsigned int Oid
Definition: postgres_ext.h:31
ExprState ** defexprs
copy_data_source_cb data_source_cb
StringInfoData line_buf
CopyFormatOptions opts
StringInfoData attribute_buf
TransitionCaptureState * transition_capture
MemoryContext copycontext
const char * cur_attval
const char * cur_attname
const char * cur_relname
Definition: fmgr.h:57
Definition: pg_list.h:54
Definition: nodes.h:129