365 const ColNamesPtr& col_names
366 ) : col_names_(col_names) {
370 ws_flags_ = internals::make_ws_flags(
380 ws_flags_(ws_flags) {
381 const char d = internals::infer_delimiter(parse_flags);
383 has_ws_trimming_ = std::any_of(ws_flags.begin(), ws_flags.end(), [](
bool b) { return b; });
389 const ColNamesPtr& col_names
391 this->col_names_ = col_names;
401 && this->data_ptr_->_data
402 && !this->data_ptr_->data.empty()
403 && (parse_flag(this->data_ptr_->data.back()) == ParseFlags::DELIMITER
404 || parse_flag(this->data_ptr_->data.back()) == ParseFlags::QUOTE);
410 if (this->current_row_.
size() > 0) {
411 this->push_row(this->data_ptr_ ? this->data_ptr_->data.size() : this->current_row_start());
415 CONSTEXPR_17
ParseFlags parse_flag(
const char ch)
const noexcept {
419 CONSTEXPR_17 ParseFlags compound_parse_flag(
const char ch)
const noexcept {
420 return quote_escape_flag(parse_flag(ch), this->quote_escape_);
423 void set_output(RowSink& output)
noexcept {
424 this->output_ = &output;
427 RowSink& output() noexcept {
428 return *this->output_;
433 this->initial_state_ =
state;
443 return this->ending_state_;
448 std::shared_ptr<void> owner
450 return this->parse_prepared_chunk(
457 ParserChunkResult parse_chunk(
459 std::shared_ptr<void> owner,
460 const ParserChunkOptions& options
462 return this->parse_prepared_chunk(chunk, std::move(owner), options);
465 ParserChunkResult parse_chunk(
467 std::shared_ptr<void> owner,
470 return this->parse_chunk(
474 ParserChunkOptions(this->initial_state_)
478 ParserChunkResult parse_chunk(
480 std::shared_ptr<void> owner,
484 return this->parse_chunk(
488 ParserChunkOptions(this->initial_state_, source_start)
492 ParserChunkResult parse_chunk(
494 std::shared_ptr<void> owner,
496 const ParserChunkOptions& options
498 this->output_ = &output;
499 return this->parse_prepared_chunk(
507 void set_col_names(
const ColNamesPtr& col_names) {
508 this->col_names_ = col_names;
511 void set_whitespace_flags(
const WhitespaceMap& ws_flags) {
512 this->ws_flags_ = ws_flags;
513 this->has_ws_trimming_ = std::any_of(ws_flags.begin(), ws_flags.end(), [](
bool b) { return b; });
516 void set_parse_flags(
const ParseFlagMap& parse_flags) {
518 const char d = internals::infer_delimiter(parse_flags);
519 this->
simd_sentinels_ = SentinelVecs(d, internals::infer_quote_char(parse_flags, d));
522 void set_parse_flags(
const ParseFlagMap& parse_flags,
char delimiter,
char quote_char) {
530 RawCSVDataPtr data_ptr_ =
nullptr;
531 ColNamesPtr col_names_ =
nullptr;
532 RawCSVFieldList* fields_ =
nullptr;
533 int field_start_ = UNINITIALIZED_FIELD;
534 size_t field_length_ = 0;
550 this->pending_linefeed_ =
start_state.pending_linefeed;
552 this->current_row_start() = 0;
554 auto&
in = this->data_ptr_->data;
557 this->resolve_pending_linefeed_at_start(
in);
558 this->resolve_pending_quote_at_start(
in);
560 while (this->data_pos_ <
in.size()) {
561 const size_t raw_end = this->data_pos_;
562 switch (compound_parse_flag(
in[this->data_pos_])) {
563 case ParseFlags::DELIMITER:
568 case ParseFlags::CARRIAGE_RETURN:
569 if (this->data_pos_ + 1 ==
in.size()) {
570 this->pending_linefeed_ =
true;
575 else if (parse_flag(
in[this->data_pos_ + 1]) == ParseFlags::NEWLINE) {
581 case ParseFlags::NEWLINE:
586 case ParseFlags::NOT_SPECIAL:
590 case ParseFlags::QUOTE_ESCAPE_QUOTE:
591 if (data_pos_ + 1 ==
in.size()) {
592 this->pending_quote_ =
true;
593 return this->finish_parse(this->current_row_start());
595 else if (data_pos_ + 1 <
in.size()) {
596 auto next_ch = parse_flag(
in[data_pos_ + 1]);
597 if (
next_ch >= ParseFlags::DELIMITER) {
598 quote_escape_ =
false;
602 else if (
next_ch == ParseFlags::QUOTE) {
604 this->field_length_ += 2;
605 this->field_has_double_quote_ =
true;
610 this->field_length_++;
615 if (this->field_length_ == 0) {
616 quote_escape_ =
true;
618 if (field_start_ == UNINITIALIZED_FIELD && data_pos_ <
in.size() && !ws_flag(
in[data_pos_]))
619 field_start_ = (
int)(data_pos_ - current_row_start());
623 this->field_length_++;
629 return this->finish_parse(this->current_row_start());
634 this->field_policy_.begin_chunk(
639 this->has_ws_trimming_,
645 return this->ws_flags_;
648 void emit_row(
CSVRow&& row) {
650 csv_push_row(*this->output_, std::move(row));
663 bool has_ws_trimming_ =
false;
664 bool quote_escape_ =
false;
665 bool pending_quote_ =
false;
666 bool pending_linefeed_ =
false;
667 bool field_has_double_quote_ =
false;
668 ParserDFAState initial_state_;
669 ParserDFAState ending_state_;
672 size_t data_pos_ = 0;
674 RowSink* output_ =
nullptr;
676 FieldPolicy field_policy_;
677 RowPolicy row_policy_;
679 CONSTEXPR_17
bool ws_flag(
const char ch)
const noexcept {
683 size_t& current_row_start() {
684 return this->current_row_.data_start;
688 using internals::ParseFlags;
690 if (!this->pending_quote_ || this->data_pos_ >= in.size()) {
694 const ParseFlags next_ch = this->parse_flag(in[this->data_pos_]);
695 this->pending_quote_ =
false;
697 if (next_ch >= ParseFlags::DELIMITER) {
698 this->quote_escape_ =
false;
702 if (next_ch == ParseFlags::QUOTE) {
703 this->quote_escape_ =
true;
704 this->field_has_double_quote_ =
true;
705 this->field_length_++;
710 this->quote_escape_ =
true;
711 this->field_length_++;
715 using internals::ParseFlags;
717 if (!this->pending_linefeed_ || this->data_pos_ >= in.size()) {
721 this->pending_linefeed_ =
false;
722 if (this->parse_flag(in[this->data_pos_]) == ParseFlags::NEWLINE) {
724 this->current_row_start() = this->data_pos_;
728 void parse_field() noexcept {
729 using internals::ParseFlags;
730 auto& in = this->data_ptr_->data;
732 if (field_start_ == UNINITIALIZED_FIELD)
733 field_start_ = (int)(data_pos_ - current_row_start());
735#if !defined(CSV_NO_SIMD)
736 data_pos_ = find_next_non_special(in, data_pos_, this->simd_sentinels_);
739 while (data_pos_ < in.size() && compound_parse_flag(in[data_pos_]) == ParseFlags::NOT_SPECIAL)
742 field_length_ = data_pos_ - (field_start_ + current_row_start());
747 const RawCSVField& field = this->field_policy_.push_field(
750 this->current_row_start(),
753 this->field_has_double_quote_
756 this->policy_.push_field(field);
757 current_row_.row_length++;
759 field_has_double_quote_ =
false;
760 field_start_ = UNINITIALIZED_FIELD;
765 void finish_row(
size_t raw_end) {
766 if (this->field_length_ > 0
767 || this->field_start_ != UNINITIALIZED_FIELD
768 || !this->current_row_.empty()) {
772 this->push_row(raw_end);
773 this->current_row_ = this->row_policy_.make_next_row(
776 this->fields_->size()
778 this->policy_.begin_row(this->current_row_);
782 void push_row(
size_t raw_end) {
783 this->row_policy_.finalize_row(this->current_row_, *this->fields_, raw_end);
784 this->policy_.end_row(this->current_row_);
785 this->emit_row(std::move(current_row_));
788 ParserChunkResult parse_prepared_chunk(
790 std::shared_ptr<void> owner,
791 const ParserChunkOptions& options
793 this->field_start_ = UNINITIALIZED_FIELD;
794 this->field_length_ = 0;
795 this->field_has_double_quote_ =
false;
797 this->data_ptr_->_data = std::move(owner);
798 this->data_ptr_->data = chunk;
799 this->data_ptr_->source_start = options.source_start;
800 this->data_ptr_->fields.reserve_for_source_size(chunk.size());
801 this->data_ptr_->field_scalars.reserve_for_source_size(chunk.size());
802 this->data_ptr_->quote_arena.reserve_for_source_size(chunk.size());
803 this->initial_state_ = options.initial_state;
804 this->current_row_ = this->row_policy_.make_initial_row(this->data_ptr_);
805 this->policy_.begin_chunk(this->data_ptr_);
806 this->policy_.begin_row(this->current_row_);
808 const size_t complete_prefix_length = this->
parse();
809 return ParserChunkResult(options.initial_state, this->ending_state_, complete_prefix_length);
812 ParserDFAState current_dfa_state() const noexcept {
813 return ParserDFAState{ this->quote_escape_, this->pending_quote_, this->pending_linefeed_ };
816 size_t finish_parse(
size_t remainder) {
817 this->ending_state_ = this->current_dfa_state();
818 this->initial_state_ = this->ending_state_.pending_linefeed
819 ? this->ending_state_