Vince's CSV Parser
Loading...
Searching...
No Matches
csv_reader.cpp
Go to the documentation of this file.
1
5#include "csv_reader.hpp"
6
7namespace csv {
8 void CSVReader::open_file(csv::string_view filename, const CSVFormat& format) {
9#if defined(__EMSCRIPTEN__)
10 owned_stream.reset(new std::ifstream(std::string(filename), std::ios::binary));
11 if (!*owned_stream) {
12 internals::fail(error_, CSVErrorCode::OpenFailed,
13 internals::make_prefixed_message(internals::ERROR_CANNOT_OPEN_FILE, filename));
14 return;
15 }
16 init_from_stream(*owned_stream, format);
17#else
18 init_parser(std::unique_ptr<internals::parser::CSVParserDriverBase>(
19 new internals::parser::MmapParser(filename, format, col_names)));
20#endif
21 }
22
23 std::unique_ptr<CSVReader> CSVReader::try_open(
24 csv::string_view filename, CSVError& error, const CSVFormat& format) {
25 error.clear();
26 std::unique_ptr<CSVReader> reader(new CSVReader(format, initialize_tag{}));
27 reader->open_file(filename, format);
28 error = reader->error_;
29 return error ? nullptr : std::move(reader);
30 }
31
32 std::unique_ptr<CSVReader> CSVReader::try_open(
33 std::unique_ptr<std::istream> source, CSVError& error, const CSVFormat& format) {
34 error.clear();
35 if (!source) {
36 internals::fail_static(error, CSVErrorCode::InvalidArgument, internals::ERROR_READER_NULL_STREAM);
37 return nullptr;
38 }
39 std::unique_ptr<CSVReader> reader(new CSVReader(format, initialize_tag{}));
40 reader->owned_stream = std::move(source);
41 reader->init_from_stream(*reader->owned_stream, format);
42 error = reader->error_;
43 return error ? nullptr : std::move(reader);
44 }
45
46#ifdef _MSC_VER
47#pragma region Reading helpers
48#endif
49 bool CSVReader::check_for_rows() {
50 if (error_) return false;
51 if (!this->records->empty()) return true;
52
53 if (this->read_scheduler_.wait_if_active(
54 [this] { return this->records->is_waitable(); },
55 [this] { this->records->wait(); }
56 )) {
57 return true;
58 }
59
60 this->read_scheduler_.join();
61 this->read_scheduler_.rethrow_exception_if_any();
62 if (parser->error()) {
63 error_ = parser->error();
64 return false;
65 }
66
67 if (this->parser->eof()) return false;
68
69 if (this->_read_requested && this->records->empty()) {
70 return internals::fail(error_, CSVErrorCode::RowTooLarge,
71 internals::make_row_larger_than_chunk_message(this->_chunk_size));
72 }
73
74 this->read_scheduler_.run(
75 [this] { this->read_csv(this->_chunk_size); },
76 [this] { this->records->notify_all(); }
77 );
78 this->read_scheduler_.rethrow_exception_if_any();
79 // The parser may still be running. Inspect its error only after join().
80 this->_read_requested = true;
81 return true;
82 }
83
84#ifdef _MSC_VER
85#pragma endregion Reading helpers
86#endif
87
88#ifdef _MSC_VER
89#pragma region Format and header helpers
90#endif
91 void CSVReader::init_parser(
92 std::unique_ptr<internals::parser::CSVParserDriverBase> parser_impl
93 ) {
94 if (parser_impl->error()) {
95 error_ = parser_impl->error();
96 return;
97 }
98 auto resolved = parser_impl->get_resolved_format();
99 this->_format = resolved.format;
100 this->_chunk_size = this->_format.get_chunk_size();
101 this->n_cols = resolved.n_cols;
102
103 if (!this->_format.col_names.empty()) {
104 this->set_col_names(this->_format.col_names);
105 }
106
107 this->parser = std::move(parser_impl);
108 this->initial_read();
109 }
110
112 CSVFormat new_format = this->_format;
113
114 // Since users are normally not allowed to set
115 // column names and header row simulatenously,
116 // we will set the backing variables directly here
117 new_format.col_names = this->col_names->get_col_names();
118 new_format.header = this->_format.header;
119
120 return new_format;
121 }
122
123 void CSVReader::initial_read() {
124 // Issue #337: A single read window may end before the header row when
125 // earlier rows are long, so keep reading until the header is consumed.
126 do {
127 const int trimmed_before = this->header_rows_trimmed_;
128 this->read_scheduler_.run([this] { this->read_csv(this->_chunk_size); });
129 this->read_scheduler_.join();
130 this->read_scheduler_.rethrow_exception_if_any();
131 if (parser->error()) {
132 error_ = parser->error();
133 return;
134 }
135
136 // trim_header() drains the queue until the header is consumed, so an
137 // untrimmed header with no progress means the read produced no rows.
138 if (!this->header_trimmed && !this->parser->eof()
139 && this->header_rows_trimmed_ == trimmed_before) {
140 internals::fail(error_, CSVErrorCode::RowTooLarge,
141 internals::make_row_larger_than_chunk_message(this->_chunk_size));
142 return;
143 }
144 } while (!this->header_trimmed && !this->parser->eof());
145 }
146
147 void CSVReader::trim_header() {
148 if (!this->header_trimmed) {
149 // The rows up to and including the header may arrive over several
150 // reads, so track progress instead of assuming one read holds them all.
151 while (this->header_rows_trimmed_ <= this->_format.header && !this->records->empty()) {
152 if (this->header_rows_trimmed_ == this->_format.header && this->col_names->empty()) {
153 this->set_col_names(this->records->pop_front());
154 }
155 else {
156 this->records->pop_front();
157 }
158
159 this->header_rows_trimmed_++;
160 }
161
162 this->header_trimmed = this->header_rows_trimmed_ > this->_format.header;
163 }
164 }
165
167 void CSVReader::set_col_names(const std::vector<std::string>& names)
168 {
169 this->col_names->set_policy(this->_format.get_column_name_policy());
170 this->col_names->set_col_names(names);
171 this->n_cols = names.size();
172 }
173#ifdef _MSC_VER
174#pragma endregion Format and header helpers
175#endif
176
177#ifdef _MSC_VER
178#pragma region Reading helpers
179#endif
180 bool CSVReader::row_is_returnable(const CSVRow& candidate) {
181 // Any queued record proves that the previous source window made
182 // progress, even when policy filtering excludes the record.
183 this->_read_requested = false;
184
185 const auto policy = this->_format.variable_column_policy;
186 const size_t next_row_size = candidate.size();
187
188 if (policy == VariableColumnPolicy::KEEP_NON_EMPTY && next_row_size == 0) {
189 return false;
190 }
191
192 if (next_row_size != this->n_cols &&
193 (policy == VariableColumnPolicy::THROW || policy == VariableColumnPolicy::IGNORE_ROW)) {
194 if (policy == VariableColumnPolicy::THROW) {
195 if (candidate.size() < this->n_cols) {
196 return internals::fail(error_, CSVErrorCode::RowLengthMismatch,
197 internals::make_prefixed_message(internals::ERROR_ROW_TOO_SHORT, candidate.raw_str()));
198 }
199
200 return internals::fail(error_, CSVErrorCode::RowLengthMismatch,
201 internals::make_prefixed_message(internals::ERROR_ROW_TOO_LONG, candidate.raw_str()));
202 }
203
204 return false;
205 }
206
207 return true;
208 }
209
210 bool CSVReader::accept_row(CSVRow&& candidate, CSVRow* single_row, std::vector<CSVRow>* batch_rows) {
211 if (!this->row_is_returnable(candidate)) {
212 return false;
213 }
214
215 if (single_row != nullptr) {
216 *single_row = std::move(candidate);
217 } else if (batch_rows != nullptr) {
218 batch_rows->push_back(std::move(candidate));
219 } else {
220 return false;
221 }
222
223 this->_n_rows++;
224 return true;
225 }
226
227 void CSVReader::drain_rows_into_chunk(std::vector<CSVRow>& out, size_t max_rows) {
228 std::vector<CSVRow> drained;
229 drained.reserve(max_rows - out.size());
230 this->records->drain_front(drained, max_rows - out.size());
231
232 for (size_t i = 0; i < drained.size(); ++i) {
233 this->accept_row(std::move(drained[i]), nullptr, &out);
234 if (error_) break;
235 }
236 }
237#ifdef _MSC_VER
238#pragma endregion Reading helpers
239#endif
240
241#ifdef _MSC_VER
242#pragma region Worker reading methods
243#endif
244
255 bool CSVReader::read_csv(size_t bytes) {
256 // SCHEDULED READ FUNCTION: Runs asynchronously when runtime threading
257 // is enabled, or synchronously when CSVFormat::threading(false) is active.
258 //
259 // Threading model:
260 // 1. notify_all() - signals read_row() that worker is active
261 // 2. parser->next() - reads and parses bytes (10MB chunks)
262 // 3. kill_all() - signals read_row() that worker is done
263 //
264 // Operational errors stay on the source adapter until the consumer joins.
265 // In exception-enabled builds, CSVReadScheduler catches exceptions and rethrows
266 // them on the consumer thread. Bug #282 fixed cases where worker
267 // exceptions were swallowed, causing std::terminate().
268
269 // Tell read_row() to listen for CSV rows
270 this->records->notify_all();
271
272 CSV_TRY {
273 this->parser->set_output(*this->records);
274 this->parser->next(bytes);
275
276 if (!parser->error() && !this->header_trimmed) {
277 this->trim_header();
278 }
279 }
280 CSV_CATCH((...), {
281 this->records->kill_all();
282 throw;
283 })
284
285 // Tell read_row() to stop waiting
286 this->records->kill_all();
287
288 return true;
289 }
290
292 error.clear();
293 while (this->check_for_rows()) {
294 if (this->records->empty())
295 continue;
296
297 if (this->accept_row(this->records->pop_front(), &row, nullptr))
298 return true;
299 }
300
301 error = error_;
302 return false;
303 }
304
306 error.clear();
307 while (this->check_for_rows()) {
308 CSVRow candidate;
309 if (!this->records->try_copy_front(candidate)) {
310 continue;
311 }
312
313 if (this->row_is_returnable(candidate)) {
314 return true;
315 }
316
317 if (error_) {
318 error = error_;
319 return false;
320 }
321
322 // Keep this lookahead loop separate from try_read_row(): only callers
323 // requesting a peek should pay for the locked front-row copy. Under
324 // CSVReader's single-consumer contract, the copied front remains the
325 // queue front until this discard.
326 this->records->pop_front();
327 }
328
329 error = error_;
330 return false;
331 }
332
333 bool CSVReader::try_read_chunk(std::vector<CSVRow>& out, size_t max_rows, CSVError& error) {
334 error.clear();
335 out.clear();
336
337 if (max_rows == 0) {
338 return internals::fail_static(error, CSVErrorCode::InvalidArgument, "read_chunk() requires a non-zero row count.");
339 }
340
341 while (out.size() < max_rows) {
342 if (check_for_rows()) {
343 if (this->records->empty()) {
344 continue;
345 }
346
347 const size_t before_size = out.size();
348 this->drain_rows_into_chunk(out, max_rows);
349 if (error_) { error = error_; return false; }
350
351 if (out.size() == before_size) {
352 continue;
353 }
354 }
355 else { error = error_; return !error && !out.empty(); }
356 }
357
358 return true;
359 }
360#if CSV_EXCEPTIONS_ENABLED
362 CSVError error;
363 const bool has_row = try_has_next_row(error);
364 internals::throw_error(error);
365 return has_row;
366 }
367
369 CSVError error;
370 const bool read = try_read_row(row, error);
371 internals::throw_error(error);
372 return read;
373 }
374
375 bool CSVReader::read_chunk(std::vector<CSVRow>& out, size_t max_rows) {
376 if (max_rows == 0) { out.clear(); return false; }
377 CSVError error;
378 const bool read = try_read_chunk(out, max_rows, error);
379 internals::throw_error(error);
380 return read;
381 }
382#endif
383#ifdef _MSC_VER
384#pragma endregion Worker reading methods
385#endif
386}
Stores information about how to parse a CSV file.
Main class for parsing CSVs from files and in-memory sources.
bool read_chunk(std::vector< CSVRow > &out, size_t max_rows)
Read up to max_rows rows into a caller-owned batch buffer.
bool try_read_chunk(std::vector< CSVRow > &out, size_t max_rows, CSVError &error)
Clear out and read a batch.
CSVFormat get_format() const
Return the resolved parsing format for this CSV source.
bool has_next_row()
Check whether a logical next row is available without consuming it.
bool try_has_next_row(CSVError &error)
Check for a logical next row, distinguishing exhaustion from failure.
static std::unique_ptr< CSVReader > try_open(csv::string_view filename, CSVError &error, const CSVFormat &format=CSVFormat::guess_csv())
Construct CSVReader from filename.
bool read_row(CSVRow &row)
Retrieve the next CSV row, returning true while more rows are available.
bool try_read_row(CSVRow &row, CSVError &error)
Retrieve a row, distinguishing EOF (false, no error) from failure.
Data structure for representing CSV rows.
Definition csv_row.hpp:533
csv::string_view raw_str() const noexcept
Return a string_view of the raw bytes of this row as they appear in the underlying parse buffer,...
Definition csv_row.cpp:75
CONSTEXPR size_t size() const noexcept
Return the number of fields in this row.
Definition csv_row.hpp:555
Defines functionality needed for basic CSV parsing.
std::unique_ptr< std::istream > owned_stream
Optional owned stream used by two paths: 1) Emscripten filename-constructor fallback to stream parsin...
std::unique_ptr< RowCollection > records
Queue of parsed CSV rows.
size_t _n_rows
How many rows (minus header) have been read so far.
bool read_csv(size_t bytes=internals::CSV_CHUNK_SIZE_DEFAULT)
Read a chunk of CSV data.
internals::ColNamesPtr col_names
Pointer to a object containing column information.
void set_col_names(const std::vector< std::string > &)
Sets this reader's column names and associated data.
std::unique_ptr< internals::parser::CSVParserDriverBase > parser
Helper class which actually does the parsing.
size_t n_cols
The number of columns in this CSV.
The all encompassing namespace.
@ InvalidArgument
Invalid operation input, such as a null stream or zero batch size.
@ RowTooLarge
A complete record could not fit within the configured read window.
@ RowLengthMismatch
A record's field count violates the configured variable-column policy.
@ OpenFailed
A native file could not be opened, including initial input mapping.
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182
A diagnostic whose text remains valid for the lifetime of the error.
Definition csv_error.hpp:94