Vince's CSV Parser
Loading...
Searching...
No Matches
stream.hpp
1#pragma once
2
3#include "driver.hpp"
4#include "orchestrator.hpp"
5
6namespace csv {
7 namespace internals {
8 namespace parser {
9 constexpr size_t CSV_STREAM_WINDOW_SIZE_MAX = 256 * 1024 * 1024;
10
28 template<typename TStream>
31
32 public:
35 const CSVFormat& format,
36 const ColNamesPtr& col_names = nullptr
37 ) : CSVParserDriverBase(format, col_names),
38 source_(source) {
39 this->resolve_format_from_head(format);
40 if (this->error_) return;
41 this->parse_orchestrator_ = make_csv_parse_orchestrator(
42 this->parse_flags_,
43 this->whitespace_flags(),
44 this->format.format,
45 0,
46 col_names,
47 true,
48 false
49 );
50 }
51
54 ParseFlagMap parse_flags,
55 WhitespaceMap ws_flags
56 ) : CSVParserDriverBase(parse_flags, ws_flags),
57 source_(source) {
58 this->parse_orchestrator_ = make_csv_parse_orchestrator(
59 this->parse_flags_,
60 this->whitespace_flags(),
61 CSVFormat(),
62 0,
63 nullptr,
64 false,
65 false
66 );
67 }
68
69 ~StreamParser() {}
70
71 std::string& get_csv_head() override {
72 leftover_ = get_csv_head_stream(this->source_, this->error_);
73 return this->leftover_;
74 }
75
76 void next(size_t bytes = CSV_CHUNK_SIZE_DEFAULT) override {
77 if (this->eof() || this->error_) return;
78
79 auto chunk_owner = std::make_shared<std::string>();
80 auto& chunk = *chunk_owner;
81
82 // Prepend leftover bytes from the previous chunk's incomplete
83 // trailing row, then read enough bytes to fill the orchestrator
84 // window. The window grows to chunk_size * worker_count only
85 // when speculative parsing is active.
86 chunk = std::move(leftover_);
87 const size_t requested_window_size = this->parse_orchestrator_->read_window_size(bytes);
88 const size_t stream_window_cap = bytes > CSV_STREAM_WINDOW_SIZE_MAX
89 ? bytes
90 : CSV_STREAM_WINDOW_SIZE_MAX;
91 const size_t window_size = std::min(requested_window_size, stream_window_cap);
92 const size_t read_size = chunk.size() < window_size
93 ? window_size - chunk.size()
94 : bytes;
95 std::unique_ptr<char[]> buf(new char[read_size]);
96#if !CSV_EXCEPTIONS_ENABLED
97 if (source_.exceptions() != std::ios::goodbit) {
98 fail_static(this->error_, CSVErrorCode::InvalidArgument,
99 "Explicit CSV reads require a non-throwing stream exception mask.");
100 return;
101 }
102#endif
103 CSV_TRY {
104 source_.read(buf.get(), static_cast<std::streamsize>(read_size));
105 }
106 CSV_CATCH((const std::ios_base::failure&), {
107 if (!source_.eof() || source_.bad()) {
108 fail_static(this->error_, CSVErrorCode::ReadFailed, ERROR_STREAM_READ_FAILURE);
109 return;
110 }
111 })
112
113 const size_t n = static_cast<size_t>(source_.gcount());
114
115 if (n > 0) chunk.append(buf.get(), n);
116
117 // Check for real I/O errors only (bad bit indicates unrecoverable error).
118 // failbit alone is not fatal - it's set on EOF or when requesting bytes
119 // beyond available data, which is normal behavior for stringstreams.
120 if (source_.bad() || (source_.fail() && !source_.eof())) {
121 fail_static(this->error_, CSVErrorCode::ReadFailed, ERROR_STREAM_READ_FAILURE);
122 return;
123 }
124
125 const bool source_exhausted = source_.eof() || chunk.empty();
126 const CSVParseWindowResult result = this->parse_orchestrator_->parse_window(
127 chunk,
129 this->stream_pos_,
130 bytes,
132 this->output(), this->error_
133 );
134 if (this->error_) return;
135
136 if (source_exhausted) {
137 this->eof_ = true;
138 }
139 else {
140 // Save the tail bytes that begin an incomplete row so they
141 // are prepended to the next chunk (see class-level comment).
142 leftover_ = chunk.substr(result.complete_prefix_length);
143 this->stream_pos_ += result.complete_prefix_length;
144 }
145 }
146
147 void reset_with_initial_state(ParserDFAState state) noexcept {
148 if (this->parse_orchestrator_) {
149 this->parse_orchestrator_->reset_with_initial_state(state);
150 }
151 else {
153 }
154 }
155
156 void reset_with_initial_state(bool starts_in_quoted, bool in_escape = false) noexcept {
157 this->reset_with_initial_state(ParserDFAState{ starts_in_quoted, in_escape });
158 }
159
160 ParserDFAState ending_state() const noexcept {
161 return this->parse_orchestrator_
162 ? this->parse_orchestrator_->ending_state()
164 }
165
166 private:
167 // Bytes from the previous chunk that form the start of an incomplete
168 // row, plus the initial head buffer on the first call.
169 std::string leftover_;
170 size_t stream_pos_ = 0;
171
172 TStream& source_;
173 };
174 }
175 }
176}
Stores information about how to parse a CSV file.
ParseFlagMap parse_flags_
An array where the (i + 128)th slot gives the ParseFlags for ASCII character i.
Definition core.hpp:540
void reset_with_initial_state(ParserDFAState state) noexcept
Seed the DFA state for the next parse call.
Definition core.hpp:432
ParserDFAState ending_state() const noexcept
Return the DFA state observed when the most recent parse call stopped.
Definition core.hpp:442
Borrow a row and error state for synchronous, single-pass field access.
Abstract base class for source adapters.
Definition driver.hpp:182
bool eof() const noexcept
Whether or not we have reached the end of source.
Definition driver.hpp:197
A class for parsing CSV data from any std::istream, including non-seekable sources such as pipes and ...
Definition stream.hpp:29
void next(size_t bytes=CSV_CHUNK_SIZE_DEFAULT) override
Parse the next block of data.
Definition stream.hpp:76
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
Contains CSV parser source-adapter declarations and format helpers.
std::string get_csv_head_stream(TStream &source, CSVError &error)
Read the first 500KB from a seekless stream source.
Definition driver.hpp:55
The all encompassing namespace.
@ InvalidArgument
Invalid operation input, such as a null stream or zero batch size.
@ ReadFailed
The input stream reported an I/O failure rather than ordinary EOF.
Explicit DFA state at a parse boundary.
Definition core.hpp:106