Vince's CSV Parser
Loading...
Searching...
No Matches
orchestrator.hpp
1#pragma once
2
3#include <atomic>
4
5#include "driver.hpp"
6#include "../speculative/parallel_parser.hpp"
7
8namespace csv {
9 namespace internals {
10 namespace parser {
11 template<bool EagerClassify = false>
13 public:
15 const ParseFlagMap& parse_flags,
16 const WhitespaceMap& ws_flags,
17 const CSVFormat& format,
18 size_t source_size,
19 const ColNamesPtr& col_names = nullptr,
21 bool source_size_known = true
22 )
23 : serial_parser_(parse_flags, ws_flags, col_names)
24#if CSV_ENABLE_THREADS
25 , parse_flags_(parse_flags),
26 ws_flags_(ws_flags),
27 scanner_(parse_flags)
28#endif
29 {
30#if CSV_ENABLE_THREADS
31 size_t n_threads = 1;
32 if (format.is_threading_enabled()
34 && (!source_size_known || source_size >= format.get_speculative_parallel_min_bytes())) {
35 n_threads = format.get_speculative_parallel_threads();
36 if (n_threads == 0) {
37 const unsigned int hardware_threads = std::thread::hardware_concurrency();
38 n_threads = hardware_threads == 0 ? 2 : static_cast<size_t>(hardware_threads);
39 }
40 }
41
42 this->worker_count_ = n_threads == 0 ? 1 : n_threads;
43 this->use_speculative_parallel_ = format.is_threading_enabled()
45 && this->worker_count_ > 1
46 && (!source_size_known || format.should_use_speculative_parallel(
48 this->worker_count_
49 ));
50 if (this->use_speculative_parallel_) {
51 this->speculative_parser_.reset(new speculative::ParallelCSVParser<EagerClassify>(
52 this->parse_flags_,
53 this->ws_flags_,
54 this->worker_count_,
55 col_names
56 ));
57 }
58#else
59 (void)parse_flags;
60 (void)ws_flags;
61 (void)format;
65#endif
66 }
67
68 size_t worker_count() const noexcept override {
69#if CSV_ENABLE_THREADS
70 return this->use_speculative_parallel_ ? this->worker_count_ : 1;
71#else
72 return 1;
73#endif
74 }
75
76 SpeculativeParseDiagnostics diagnostics() const noexcept override {
77#if CSV_ENABLE_THREADS
79 const SpeculativeParseDiagnostics diagnostics = this->speculative_diagnostics_;
81 return diagnostics;
82#else
83 return this->speculative_diagnostics_;
84#endif
85 }
86
87 size_t read_window_size(size_t chunk_size) const noexcept override {
88#if CSV_ENABLE_THREADS
89 if (!this->use_speculative_parallel_ || this->worker_count_ <= 1) {
90 return chunk_size;
91 }
92
93 const size_t max_size = (std::numeric_limits<size_t>::max)();
94 if (chunk_size > max_size / this->worker_count_) {
95 return max_size;
96 }
97
98 return chunk_size * this->worker_count_;
99#else
100 return chunk_size;
101#endif
102 }
103
104 bool utf8_bom() const noexcept override {
105 return this->utf8_bom_;
106 }
107
108 void reset_with_initial_state(ParserDFAState state) noexcept override {
109 this->serial_parser_.reset_with_initial_state(state);
110 }
111
112 ParserDFAState ending_state() const noexcept override {
113 return this->serial_parser_.ending_state();
114 }
115
116 CSVParseWindowResult parse_window(
117 csv::string_view chunk,
118 std::shared_ptr<void> owner,
119 size_t base_offset,
120 size_t serial_chunk_size,
121 bool source_exhausted,
122 RowCollection& output, CSVError& error
123 ) override {
124 // BOM handling is a source-window concern: detect it once on the
125 // first window, reject unsupported encodings before any parser
126 // (serial or speculative) sees the bytes, and hand parsers a
127 // BOM-free view. Source adapters speak in original source-byte
128 // offsets, so the skipped bytes are added back to the completed
129 // prefix. That also keeps a re-read of the first window (e.g. an
130 // mmap head that completed no row, issue #337) from starting at
131 // the BOM again.
132 //
133 // Both adapters' first windows include the pre-read head buffer
134 // (up to 500KB); streams may append more bytes before parsing.
135 // A first window too short to classify a BOM therefore occurs
136 // only at end of input.
137 size_t bom_skip = 0;
138 if (!this->bom_scanned_) {
139 bom_skip = get_bom_skip(chunk, this->utf8_bom_, error);
140 if (error) return {};
141 this->bom_scanned_ = true;
142 }
143
144 CSVParseWindowResult result = this->dispatch_window(
145 chunk.substr(bom_skip),
146 std::move(owner),
150 output
151 );
152 result.complete_prefix_length += bom_skip;
153 result.skipped_prefix_length = bom_skip;
154 return result;
155 }
156
157 private:
158#if CSV_ENABLE_THREADS
159 void lock_speculative_diagnostics_() const noexcept {
160 while (this->speculative_diagnostics_lock_.test_and_set(std::memory_order_acquire)) {}
161 }
162
163 void unlock_speculative_diagnostics_() const noexcept {
164 this->speculative_diagnostics_lock_.clear(std::memory_order_release);
165 }
166#endif
167
168 CSVParseWindowResult dispatch_window(
169 csv::string_view chunk,
170 std::shared_ptr<void> owner,
171 size_t base_offset,
172 size_t serial_chunk_size,
173 bool source_exhausted,
174 RowCollection& output
175 ) {
176#if CSV_ENABLE_THREADS
178 && this->worker_count_ > 1
179 && serial_chunk_size > 0
180 && chunk.size() > serial_chunk_size) {
181 return this->parse_speculative_window(
182 chunk,
183 std::move(owner),
187 output
188 );
189 }
190#else
192#endif
193
194 return this->parse_serial_window(
195 chunk,
196 std::move(owner),
199 output
200 );
201 }
202
203 CSVParseWindowResult parse_serial_window(
204 csv::string_view chunk,
205 std::shared_ptr<void> owner,
206 size_t base_offset,
207 bool source_exhausted,
208 RowCollection& output
209 ) {
211 const ParserChunkResult parse_result = this->serial_parser_.parse_chunk(
212 chunk,
213 std::move(owner),
214 output,
216 );
217 result.complete_prefix_length = parse_result.complete_prefix_length;
218 if (source_exhausted) {
219 this->serial_parser_.end_feed();
220 }
221 return result;
222 }
223
224#if CSV_ENABLE_THREADS
226 csv::string_view chunk,
227 std::shared_ptr<void> owner,
228 size_t base_offset,
229 size_t serial_chunk_size,
230 bool source_exhausted,
231 RowCollection& output
232 ) {
234 auto chunks = speculative::make_speculative_parse_chunks(
235 chunk,
236 owner,
238 this->scanner_,
240 );
241
242 const speculative::ParallelCSVParseResult parse_result = this->speculative_parser_->parse_chunks(
243 chunks,
244 output,
246 );
247
248 result.complete_prefix_length = parse_result.complete_prefix_length;
250 this->speculative_diagnostics_.merge(parse_result.diagnostics);
252 return result;
253 }
254#endif
255
260 CSVRowRowPolicy> serial_parser_;
261 SpeculativeParseDiagnostics speculative_diagnostics_;
262#if CSV_ENABLE_THREADS
264 mutable std::atomic_flag speculative_diagnostics_lock_ = ATOMIC_FLAG_INIT;
265#endif
266
268 bool bom_scanned_ = false;
269 bool utf8_bom_ = false;
270#if CSV_ENABLE_THREADS
271 ParseFlagMap parse_flags_;
272 WhitespaceMap ws_flags_;
273 speculative::SpeculativeScanner scanner_;
274 bool use_speculative_parallel_ = false;
275 size_t worker_count_ = 1;
276 std::unique_ptr<speculative::ParallelCSVParser<EagerClassify>> speculative_parser_;
277#endif
278 };
279
280 inline std::unique_ptr<ICSVParseOrchestrator> make_csv_parse_orchestrator(
281 const ParseFlagMap& parse_flags,
282 const WhitespaceMap& ws_flags,
283 const CSVFormat& format,
284 size_t source_size,
285 const ColNamesPtr& col_names,
288 ) {
289 if (format.is_eager_field_classification_enabled()) {
290 return std::unique_ptr<ICSVParseOrchestrator>(new CSVParseOrchestrator<true>(
291 parse_flags,
292 ws_flags,
293 format,
295 col_names,
298 ));
299 }
300
301 return std::unique_ptr<ICSVParseOrchestrator>(new CSVParseOrchestrator<false>(
302 parse_flags,
303 ws_flags,
304 format,
306 col_names,
309 ));
310 }
311 }
312 }
313}
Stores information about how to parse a CSV file.
Borrow a row and error state for synchronous, single-pass field access.
A std::deque wrapper which allows multiple read and write threads to concurrently access it along wit...
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
Contains CSV parser source-adapter declarations and format helpers.
The all encompassing namespace.
internals::ThreadSafeDeque< CSVRow > RowCollection
Standard type for storing collection of rows.
Definition core.hpp:172
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182
A diagnostic whose text remains valid for the lifetime of the error.
Definition csv_error.hpp:94
Default row policy for the CSVRow path.
Definition core.hpp:308
Explicit DFA state at a parse boundary.
Definition core.hpp:106
Default parse policy hook.
Definition core.hpp:181