Vince's CSV Parser
Loading...
Searching...
No Matches
core.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <algorithm>
8#include <array>
9#include <cstdint>
10#include <memory>
11#include <utility>
12#include <vector>
13
14#include "../basic_csv_parser_simd.hpp"
15#include "../col_names.hpp"
16#include "../common.hpp"
17#include "../csv_exceptions.hpp"
18#include "../csv_format.hpp"
19#include "../csv_row.hpp"
20#include "../row_deque.hpp"
21
22namespace csv {
23 namespace internals {
24 constexpr const int UNINITIALIZED_FIELD = -1;
25
27 template<typename OutArray, typename T = typename OutArray::type>
28 CSV_CONST CONSTEXPR_17 OutArray arrayToDefault(T&& value)
29 {
30 OutArray a {};
31 for (auto& e : a)
32 e = value;
33 return a;
34 }
35
40 CSV_CONST CONSTEXPR_17 ParseFlagMap make_parse_flags(char delimiter) {
41 auto ret = arrayToDefault<ParseFlagMap>(ParseFlags::NOT_SPECIAL);
42 ret[delimiter + CHAR_OFFSET] = ParseFlags::DELIMITER;
43 ret['\r' + CHAR_OFFSET] = ParseFlags::CARRIAGE_RETURN;
44 ret['\n' + CHAR_OFFSET] = ParseFlags::NEWLINE;
45 return ret;
46 }
47
52 CSV_CONST CONSTEXPR_17 ParseFlagMap make_parse_flags(char delimiter, char quote_char) {
53 std::array<ParseFlags, 256> ret = make_parse_flags(delimiter);
54 ret[quote_char + CHAR_OFFSET] = ParseFlags::QUOTE;
55 return ret;
56 }
57
58 inline char infer_char_for_flag(
59 const ParseFlagMap& parse_flags,
60 ParseFlags target,
61 char fallback
62 ) noexcept {
63 for (size_t i = 0; i < parse_flags.size(); ++i) {
64 if (parse_flags[i] == target) {
65 return static_cast<char>(static_cast<int>(i) - CHAR_OFFSET);
66 }
67 }
68
69 return fallback;
70 }
71
72 inline char infer_delimiter(const ParseFlagMap& parse_flags) noexcept {
73 return infer_char_for_flag(parse_flags, ParseFlags::DELIMITER, ',');
74 }
75
76 // fallback is returned when no QUOTE flag exists in parse_flags (e.g. no_quote mode).
77 // Pass the delimiter so SIMD stops there instead of on a byte that is NOT_SPECIAL.
78 inline char infer_quote_char(const ParseFlagMap& parse_flags, char fallback = '"') noexcept {
79 return infer_char_for_flag(parse_flags, ParseFlags::QUOTE, fallback);
80 }
81
86 CSV_CONST CONSTEXPR_17 WhitespaceMap make_ws_flags(const char* ws_chars, size_t n_chars) {
88 for (size_t j = 0; j < n_chars; j++) {
89 ret[ws_chars[j] + CHAR_OFFSET] = true;
90 }
91 return ret;
92 }
93
94 inline WhitespaceMap make_ws_flags(const std::vector<char>& flags) {
95 return make_ws_flags(flags.data(), flags.size());
96 }
97
107 ParserDFAState() noexcept = default;
109 bool quote_escape,
110 bool pending_quote = false,
111 bool pending_linefeed = false
112 ) noexcept
113 : quote_escape(quote_escape),
114 pending_quote(pending_quote),
115 pending_linefeed(pending_linefeed) {}
116
117 bool quote_escape = false;
118 bool pending_quote = false;
119 bool pending_linefeed = false;
120 };
121
122 inline bool parser_dfa_state_equal(ParserDFAState lhs, ParserDFAState rhs) noexcept {
123 return lhs.quote_escape == rhs.quote_escape
124 && lhs.pending_quote == rhs.pending_quote
125 && lhs.pending_linefeed == rhs.pending_linefeed;
126 }
127
129 ParserChunkOptions() noexcept = default;
130 explicit ParserChunkOptions(
131 ParserDFAState initial_state,
132 size_t source_start = 0
133 ) noexcept
134 : initial_state(initial_state),
135 source_start(source_start) {}
136
137 ParserDFAState initial_state;
138 size_t source_start = 0;
139 };
140
142 ParserChunkResult() noexcept = default;
144 ParserDFAState initial_state,
145 ParserDFAState ending_state,
146 size_t complete_prefix_length
147 ) noexcept
148 : initial_state(initial_state),
149 ending_state(ending_state),
150 complete_prefix_length(complete_prefix_length) {}
151
152 ParserDFAState initial_state;
153 ParserDFAState ending_state;
154 size_t complete_prefix_length = 0;
155 };
156
160
163
166 return this->complete_prefix_length > this->skipped_prefix_length;
167 }
168 };
169 }
170
173
174 namespace internals {
182 void begin_chunk(const RawCSVDataPtr&) noexcept {}
183 void begin_row(const CSVRow&) noexcept {}
184 void push_field(const RawCSVField&) noexcept {}
185 void end_row(const CSVRow&) noexcept {}
186 };
187
193 template<bool EagerClassify = false>
195 void begin_chunk(
196 RawCSVDataPtr& data_ptr,
197 RawCSVFieldList*& fields,
198 const ParseFlagMap& parse_flags,
199 const WhitespaceMap& ws_flags,
200 bool has_ws_trimming,
201 const ColNamesPtr& col_names
202 ) const {
203 data_ptr = std::make_shared<RawCSVData>();
204 data_ptr->parse_flags = parse_flags;
205 data_ptr->ws_flags = ws_flags;
206 data_ptr->has_ws_trimming = has_ws_trimming;
207 data_ptr->col_names = col_names;
208 fields = &(data_ptr->fields);
209 }
210
211 const RawCSVField& push_field(
212 RawCSVData& data,
213 RawCSVFieldList& fields,
214 size_t row_start,
215 int field_start,
216 size_t field_length,
218 ) const {
219 const size_t raw_start = field_start == UNINITIALIZED_FIELD
220 ? 0
221 : static_cast<size_t>(field_start);
222 size_t stored_start = raw_start;
224 bool is_realized = false;
225
227 stored_start = this->append_realized_quoted_field(
228 data,
232 );
233 is_realized = true;
234 }
235
236 fields.emplace_back(
239 is_realized
240 );
241 this->append_scalar(
242 data,
243 fields[fields.size() - 1],
244 row_start,
245 std::integral_constant<bool, EagerClassify>()
246 );
247 return fields[fields.size() - 1];
248 }
249
250 private:
251 void append_scalar(
252 RawCSVData&,
253 const RawCSVField&,
254 size_t,
255 std::false_type
256 ) const {}
257
258 void append_scalar(
259 RawCSVData& data,
260 const RawCSVField& field,
261 size_t row_start,
262 std::true_type
263 ) const {
265 if (field.has_realized_storage()) {
266 field_str = data.quote_arena.view(field.start, field.length);
267 }
268 else {
269 field_str = csv::string_view(data.data).substr(row_start + field.start, field.length);
270 }
271
272 if (data.has_ws_trimming) {
273 field_str = internals::get_trimmed(field_str, data.ws_flags);
274 }
275
276 data.field_scalars.emplace_back(internals::classify_field_scalar(field_str));
277 }
278
279 CSVChunkIndex append_realized_quoted_field(
280 RawCSVData& data,
281 size_t field_start,
282 size_t field_length,
283 size_t& realized_length
284 ) const {
286
288 // Allocate the original length as an upper bound, then compact doubled
289 // quotes in one pass. Wasting a byte per escaped quote pair is cheaper
290 // than scanning quote-heavy fields twice in the parser hot path.
291 auto allocation = data.quote_arena.allocate_contiguous(field_str.size());
292 char* out = allocation.data;
293 size_t i = 0;
294 while (i < field_str.size()) {
295 const bool doubled_quote = data.parse_flags[field_str[i] + CHAR_OFFSET] == ParseFlags::QUOTE
296 && i + 1 < field_str.size()
297 && data.parse_flags[field_str[i + 1] + CHAR_OFFSET] == ParseFlags::QUOTE;
298 *(out++) = field_str[i];
299 i += doubled_quote ? 2 : 1;
300 }
301
302 realized_length = static_cast<size_t>(out - allocation.data);
303 return allocation.offset;
304 }
305 };
306
309 CSVRow make_initial_row(const RawCSVDataPtr& data_ptr) const {
310 return CSVRow(data_ptr);
311 }
312
313 CSVRow make_next_row(
314 const RawCSVDataPtr& data_ptr,
315 size_t data_pos,
316 size_t fields_size
317 ) const {
319 }
320
321 void finalize_row(
322 CSVRow& row,
323 const RawCSVFieldList& fields,
324 size_t raw_end
325 ) const {
326 row.row_length = fields.size() - row.fields_start;
327 row.data_end = raw_end;
328 }
329 };
330
331 inline void csv_push_row(RowCollection& sink, CSVRow&& row) {
332 sink.push_back(std::move(row));
333 }
334
335 inline void csv_push_row(std::vector<CSVRow>& sink, CSVRow&& row) {
336 sink.push_back(std::move(row));
337 }
338
339 inline void csv_append_rows(RowCollection& sink, std::vector<CSVRow>&& rows) {
340 sink.append_rows(std::move(rows));
341 }
342
343 inline void csv_append_rows(std::vector<CSVRow>& sink, std::vector<CSVRow>&& rows) {
344 if (rows.empty()) {
345 return;
346 }
347
348 sink.reserve(sink.size() + rows.size());
349 for (auto& row : rows) {
350 sink.push_back(std::move(row));
351 }
352 }
353
354 template<
355 typename RowSink = RowCollection,
356 typename ParsePolicy = PermissiveParsePolicy,
357 typename FieldPolicy = CSVRowFieldPolicy<false>,
358 typename RowPolicy = CSVRowRowPolicy>
360 public:
361 CSVParserCore() = default;
362
365 const ColNamesPtr& col_names
366 ) : col_names_(col_names) {
367 // Only initialize the fields that are stable before format resolution.
368 // parse_flags_ and simd_sentinels_ are always set by resolve_format_from_head,
369 // so there is no point computing them here with a placeholder delimiter.
370 ws_flags_ = internals::make_ws_flags(
371 source_format.trim_chars.data(), source_format.trim_chars.size()
372 );
373 has_ws_trimming_ = !source_format.trim_chars.empty();
374 }
375
377 const ParseFlagMap& parse_flags,
378 const WhitespaceMap& ws_flags
379 ) : parse_flags_(parse_flags),
380 ws_flags_(ws_flags) {
381 const char d = internals::infer_delimiter(parse_flags);
382 simd_sentinels_ = SentinelVecs(d, internals::infer_quote_char(parse_flags, d));
383 has_ws_trimming_ = std::any_of(ws_flags.begin(), ws_flags.end(), [](bool b) { return b; });
384 }
385
387 const ParseFlagMap& parse_flags,
388 const WhitespaceMap& ws_flags,
389 const ColNamesPtr& col_names
390 ) : CSVParserCore(parse_flags, ws_flags) {
391 this->col_names_ = col_names;
392 }
393
394 ~CSVParserCore() {}
395
397 void end_feed() {
399
400 bool empty_last_field = this->data_ptr_
401 && this->data_ptr_->_data
402 && !this->data_ptr_->data.empty()
403 && (parse_flag(this->data_ptr_->data.back()) == ParseFlags::DELIMITER
404 || parse_flag(this->data_ptr_->data.back()) == ParseFlags::QUOTE);
405
406 if (this->field_length_ > 0 || empty_last_field) {
407 this->push_field();
408 }
409
410 if (this->current_row_.size() > 0) {
411 this->push_row(this->data_ptr_ ? this->data_ptr_->data.size() : this->current_row_start());
412 }
413 }
414
415 CONSTEXPR_17 ParseFlags parse_flag(const char ch) const noexcept {
416 return parse_flags_.data()[ch + CHAR_OFFSET];
417 }
418
419 CONSTEXPR_17 ParseFlags compound_parse_flag(const char ch) const noexcept {
420 return quote_escape_flag(parse_flag(ch), this->quote_escape_);
421 }
422
423 void set_output(RowSink& output) noexcept {
424 this->output_ = &output;
425 }
426
427 RowSink& output() noexcept {
428 return *this->output_;
429 }
430
433 this->initial_state_ = state;
434 }
435
440
443 return this->ending_state_;
444 }
445
446 ParserChunkResult parse_chunk(
447 csv::string_view chunk,
448 std::shared_ptr<void> owner
449 ) {
450 return this->parse_prepared_chunk(
451 chunk,
452 std::move(owner),
453 ParserChunkOptions(this->initial_state_)
454 );
455 }
456
457 ParserChunkResult parse_chunk(
458 csv::string_view chunk,
459 std::shared_ptr<void> owner,
460 const ParserChunkOptions& options
461 ) {
462 return this->parse_prepared_chunk(chunk, std::move(owner), options);
463 }
464
465 ParserChunkResult parse_chunk(
466 csv::string_view chunk,
467 std::shared_ptr<void> owner,
468 RowSink& output
469 ) {
470 return this->parse_chunk(
471 chunk,
472 std::move(owner),
473 output,
474 ParserChunkOptions(this->initial_state_)
475 );
476 }
477
478 ParserChunkResult parse_chunk(
479 csv::string_view chunk,
480 std::shared_ptr<void> owner,
481 RowSink& output,
482 size_t source_start
483 ) {
484 return this->parse_chunk(
485 chunk,
486 std::move(owner),
487 output,
488 ParserChunkOptions(this->initial_state_, source_start)
489 );
490 }
491
492 ParserChunkResult parse_chunk(
493 csv::string_view chunk,
494 std::shared_ptr<void> owner,
495 RowSink& output,
496 const ParserChunkOptions& options
497 ) {
498 this->output_ = &output;
499 return this->parse_prepared_chunk(
500 chunk,
501 std::move(owner),
502 options
503 );
504 }
505
506 protected:
507 void set_col_names(const ColNamesPtr& col_names) {
508 this->col_names_ = col_names;
509 }
510
511 void set_whitespace_flags(const WhitespaceMap& ws_flags) {
512 this->ws_flags_ = ws_flags;
513 this->has_ws_trimming_ = std::any_of(ws_flags.begin(), ws_flags.end(), [](bool b) { return b; });
514 }
515
516 void set_parse_flags(const ParseFlagMap& parse_flags) {
517 this->parse_flags_ = parse_flags;
518 const char d = internals::infer_delimiter(parse_flags);
519 this->simd_sentinels_ = SentinelVecs(d, internals::infer_quote_char(parse_flags, d));
520 }
521
522 void set_parse_flags(const ParseFlagMap& parse_flags, char delimiter, char quote_char) {
523 this->parse_flags_ = parse_flags;
524 this->simd_sentinels_ = SentinelVecs(delimiter, quote_char);
525 }
526
529 CSVRow current_row_;
530 RawCSVDataPtr data_ptr_ = nullptr;
531 ColNamesPtr col_names_ = nullptr;
532 RawCSVFieldList* fields_ = nullptr;
533 int field_start_ = UNINITIALIZED_FIELD;
534 size_t field_length_ = 0;
535
538
542
544 size_t parse() {
546
547 const ParserDFAState start_state = this->initial_state_;
548 this->quote_escape_ = start_state.quote_escape;
549 this->pending_quote_ = start_state.pending_quote;
550 this->pending_linefeed_ = start_state.pending_linefeed;
551 this->data_pos_ = 0;
552 this->current_row_start() = 0;
553
554 auto& in = this->data_ptr_->data;
555
556 // Resolve any pending state from the previous chunk in speculative parsing
557 this->resolve_pending_linefeed_at_start(in);
558 this->resolve_pending_quote_at_start(in);
559
560 while (this->data_pos_ < in.size()) {
561 const size_t raw_end = this->data_pos_;
562 switch (compound_parse_flag(in[this->data_pos_])) {
563 case ParseFlags::DELIMITER:
564 this->push_field();
565 this->data_pos_++;
566 break;
567
568 case ParseFlags::CARRIAGE_RETURN:
569 if (this->data_pos_ + 1 == in.size()) {
570 this->pending_linefeed_ = true;
571 this->data_pos_++;
572 this->finish_row(raw_end);
573 break;
574 }
575 else if (parse_flag(in[this->data_pos_ + 1]) == ParseFlags::NEWLINE) {
576 this->data_pos_++;
577 }
578
579 CSV_FALLTHROUGH;
580
581 case ParseFlags::NEWLINE:
582 this->data_pos_++;
583 this->finish_row(raw_end);
584 break;
585
586 case ParseFlags::NOT_SPECIAL:
587 this->parse_field();
588 break;
589
590 case ParseFlags::QUOTE_ESCAPE_QUOTE:
591 if (data_pos_ + 1 == in.size()) {
592 this->pending_quote_ = true;
593 return this->finish_parse(this->current_row_start());
594 }
595 else if (data_pos_ + 1 < in.size()) {
596 auto next_ch = parse_flag(in[data_pos_ + 1]);
597 if (next_ch >= ParseFlags::DELIMITER) {
598 quote_escape_ = false;
599 data_pos_++;
600 break;
601 }
602 else if (next_ch == ParseFlags::QUOTE) {
603 data_pos_ += 2;
604 this->field_length_ += 2;
605 this->field_has_double_quote_ = true;
606 break;
607 }
608 }
609
610 this->field_length_++;
611 data_pos_++;
612 break;
613
614 default:
615 if (this->field_length_ == 0) {
616 quote_escape_ = true;
617 data_pos_++;
618 if (field_start_ == UNINITIALIZED_FIELD && data_pos_ < in.size() && !ws_flag(in[data_pos_]))
619 field_start_ = (int)(data_pos_ - current_row_start());
620 break;
621 }
622
623 this->field_length_++;
624 data_pos_++;
625 break;
626 }
627 }
628
629 return this->finish_parse(this->current_row_start());
630 }
631
634 this->field_policy_.begin_chunk(
635 this->data_ptr_,
636 this->fields_,
637 this->parse_flags_,
638 this->ws_flags_,
639 this->has_ws_trimming_,
640 this->col_names_
641 );
642 }
643
644 const WhitespaceMap& whitespace_flags() const noexcept {
645 return this->ws_flags_;
646 }
647
648 void emit_row(CSVRow&& row) {
649 if (this->output_) {
650 csv_push_row(*this->output_, std::move(row));
651 }
652 }
653
654 private:
658 WhitespaceMap ws_flags_;
659
663 bool has_ws_trimming_ = false;
664 bool quote_escape_ = false;
665 bool pending_quote_ = false;
666 bool pending_linefeed_ = false;
667 bool field_has_double_quote_ = false;
668 ParserDFAState initial_state_;
669 ParserDFAState ending_state_;
670
672 size_t data_pos_ = 0;
673
674 RowSink* output_ = nullptr;
675 ParsePolicy policy_;
676 FieldPolicy field_policy_;
677 RowPolicy row_policy_;
678
679 CONSTEXPR_17 bool ws_flag(const char ch) const noexcept {
680 return ws_flags_.data()[ch + CHAR_OFFSET];
681 }
682
683 size_t& current_row_start() {
684 return this->current_row_.data_start;
685 }
686
687 void resolve_pending_quote_at_start(csv::string_view in) {
688 using internals::ParseFlags;
689
690 if (!this->pending_quote_ || this->data_pos_ >= in.size()) {
691 return;
692 }
693
694 const ParseFlags next_ch = this->parse_flag(in[this->data_pos_]);
695 this->pending_quote_ = false;
696
697 if (next_ch >= ParseFlags::DELIMITER) {
698 this->quote_escape_ = false;
699 return;
700 }
701
702 if (next_ch == ParseFlags::QUOTE) {
703 this->quote_escape_ = true;
704 this->field_has_double_quote_ = true;
705 this->field_length_++;
706 this->data_pos_++;
707 return;
708 }
709
710 this->quote_escape_ = true;
711 this->field_length_++;
712 }
713
714 void resolve_pending_linefeed_at_start(csv::string_view in) {
715 using internals::ParseFlags;
716
717 if (!this->pending_linefeed_ || this->data_pos_ >= in.size()) {
718 return;
719 }
720
721 this->pending_linefeed_ = false;
722 if (this->parse_flag(in[this->data_pos_]) == ParseFlags::NEWLINE) {
723 this->data_pos_++;
724 this->current_row_start() = this->data_pos_;
725 }
726 }
727
728 void parse_field() noexcept {
729 using internals::ParseFlags;
730 auto& in = this->data_ptr_->data;
731
732 if (field_start_ == UNINITIALIZED_FIELD)
733 field_start_ = (int)(data_pos_ - current_row_start());
734
735#if !defined(CSV_NO_SIMD)
736 data_pos_ = find_next_non_special(in, data_pos_, this->simd_sentinels_);
737#endif
738
739 while (data_pos_ < in.size() && compound_parse_flag(in[data_pos_]) == ParseFlags::NOT_SPECIAL)
740 data_pos_++;
741
742 field_length_ = data_pos_ - (field_start_ + current_row_start());
743 }
744
746 void push_field() {
747 const RawCSVField& field = this->field_policy_.push_field(
748 *this->data_ptr_,
749 *this->fields_,
750 this->current_row_start(),
751 this->field_start_,
752 this->field_length_,
753 this->field_has_double_quote_
754 );
755
756 this->policy_.push_field(field);
757 current_row_.row_length++;
758
759 field_has_double_quote_ = false;
760 field_start_ = UNINITIALIZED_FIELD;
761 field_length_ = 0;
762 }
763
765 void finish_row(size_t raw_end) {
766 if (this->field_length_ > 0
767 || this->field_start_ != UNINITIALIZED_FIELD
768 || !this->current_row_.empty()) {
769 this->push_field();
770 }
771
772 this->push_row(raw_end);
773 this->current_row_ = this->row_policy_.make_next_row(
774 this->data_ptr_,
775 this->data_pos_,
776 this->fields_->size()
777 );
778 this->policy_.begin_row(this->current_row_);
779 }
780
782 void push_row(size_t raw_end) {
783 this->row_policy_.finalize_row(this->current_row_, *this->fields_, raw_end);
784 this->policy_.end_row(this->current_row_);
785 this->emit_row(std::move(current_row_));
786 }
787
788 ParserChunkResult parse_prepared_chunk(
789 csv::string_view chunk,
790 std::shared_ptr<void> owner,
791 const ParserChunkOptions& options
792 ) {
793 this->field_start_ = UNINITIALIZED_FIELD;
794 this->field_length_ = 0;
795 this->field_has_double_quote_ = false;
796 this->reset_data_ptr();
797 this->data_ptr_->_data = std::move(owner);
798 this->data_ptr_->data = chunk;
799 this->data_ptr_->source_start = options.source_start;
800 this->data_ptr_->fields.reserve_for_source_size(chunk.size());
801 this->data_ptr_->field_scalars.reserve_for_source_size(chunk.size());
802 this->data_ptr_->quote_arena.reserve_for_source_size(chunk.size());
803 this->initial_state_ = options.initial_state;
804 this->current_row_ = this->row_policy_.make_initial_row(this->data_ptr_);
805 this->policy_.begin_chunk(this->data_ptr_);
806 this->policy_.begin_row(this->current_row_);
807
808 const size_t complete_prefix_length = this->parse();
809 return ParserChunkResult(options.initial_state, this->ending_state_, complete_prefix_length);
810 }
811
812 ParserDFAState current_dfa_state() const noexcept {
813 return ParserDFAState{ this->quote_escape_, this->pending_quote_, this->pending_linefeed_ };
814 }
815
816 size_t finish_parse(size_t remainder) {
817 this->ending_state_ = this->current_dfa_state();
818 this->initial_state_ = this->ending_state_.pending_linefeed
819 ? this->ending_state_
820 : ParserDFAState{};
821 return remainder;
822 }
823 };
824 }
825}
Stores information about how to parse a CSV file.
Data structure for representing CSV rows.
Definition csv_row.hpp:533
CONSTEXPR size_t size() const noexcept
Return the number of fields in this row.
Definition csv_row.hpp:555
ParseFlagMap parse_flags_
An array where the (i + 128)th slot gives the ParseFlags for ASCII character i.
Definition core.hpp:540
void reset_with_initial_state(bool starts_in_quoted, bool in_escape=false) noexcept
Convenience overload for callers that only care about quote state.
Definition core.hpp:437
void end_feed()
Indicate the last block of data has been parsed.
Definition core.hpp:397
size_t parse()
Parse the current chunk of data and return the completed-row prefix length.
Definition core.hpp:544
void reset_with_initial_state(ParserDFAState state) noexcept
Seed the DFA state for the next parse call.
Definition core.hpp:432
SentinelVecs simd_sentinels_
Precomputed SIMD broadcast vectors for find_next_non_special.
Definition core.hpp:537
ParserDFAState ending_state() const noexcept
Return the DFA state observed when the most recent parse call stopped.
Definition core.hpp:442
void reset_data_ptr()
Create a new RawCSVDataPtr for a new chunk of data.
Definition core.hpp:633
Borrow a row and error state for synchronous, single-pass field access.
A std::deque wrapper which allows multiple read and write threads to concurrently access it along wit...
A class used for efficiently storing RawCSVField objects and expanding as necessary.
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
ParseFlags
An enum used for describing the significance of each character with respect to CSV parsing.
Definition common.hpp:456
std::uint32_t CSVChunkIndex
Type used to represent the location of a CSV byte within a larger chunk.
Definition common.hpp:419
CSV_CONST CONSTEXPR_17 ParseFlagMap make_parse_flags(char delimiter)
Create a vector v where each index i corresponds to the ASCII number for a character and,...
Definition core.hpp:40
CSV_CONST CONSTEXPR_17 WhitespaceMap make_ws_flags(const char *ws_chars, size_t n_chars)
Create a vector v where each index i corresponds to the ASCII number for a character c and,...
Definition core.hpp:86
CSV_CONST CONSTEXPR_17 OutArray arrayToDefault(T &&value)
Helper constexpr function to initialize an array with all the elements set to value.
Definition core.hpp:28
The all encompassing namespace.
internals::ThreadSafeDeque< CSVRow > RowCollection
Standard type for storing collection of rows.
Definition core.hpp:172
constexpr unsigned CHAR_OFFSET
Offset to convert char into array index.
Definition common.hpp:505
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182
size_t skipped_prefix_length
Leading source bytes consumed without being parsed (a UTF-8 BOM).
Definition core.hpp:162
size_t complete_prefix_length
Source bytes fully consumed by this window, including skipped_prefix_length.
Definition core.hpp:159
bool completed_row() const noexcept
Whether this window produced at least one complete row.
Definition core.hpp:165
Default field policy for the CSVRow path.
Definition core.hpp:194
Default row policy for the CSVRow path.
Definition core.hpp:308
Explicit DFA state at a parse boundary.
Definition core.hpp:106
Default parse policy hook.
Definition core.hpp:181
A class for storing raw CSV data and associated metadata.
internals::RawCSVQuoteArena quote_arena
Parser-time sidecar bytes for fields whose quoted contents contained doubled quotes.
bool has_ws_trimming
True when at least one whitespace trim character is configured.
internals::CSVFieldScalarList field_scalars
Optional parser-time scalar sidecar; empty unless eager classification is enabled.
A barebones class used for describing CSV fields.