Vince's CSV Parser
Loading...
Searching...
No Matches
parallel_parser.hpp
1#pragma once
2
3#include "../parallel/indexed_task_pool.hpp"
4#include "scanner.hpp"
5#include "validator.hpp"
6
7#if CSV_ENABLE_THREADS
8
9namespace csv {
10 namespace internals {
11 namespace speculative {
12 struct SpeculativeParseChunk {
13 size_t sequence_number = 0;
14 size_t offset = 0;
15 csv::string_view bytes;
16 std::shared_ptr<void> owner;
17 ChunkSpeculation speculation;
18 bool starts_at_record_boundary = false;
19 };
20
21 struct ParallelCSVParseResult {
22 size_t chunks_processed = 0;
23 size_t repair_count = 0;
24 size_t complete_prefix_length = 0;
25 bool has_pending_suffix = false;
26 ParserDFAState ending_state;
27 SpeculativeParseDiagnostics diagnostics;
28 };
29
30 inline std::vector<SpeculativeParseChunk> make_speculative_parse_chunks(
32 std::shared_ptr<void> owner,
33 size_t chunk_size,
34 const SpeculativeScanner& scanner,
35 size_t base_offset = 0,
36 size_t first_sequence_number = 0
37 ) {
38 std::vector<SpeculativeParseChunk> chunks;
39 if (chunk_size == 0) {
40 return chunks;
41 }
42
43 size_t sequence_number = first_sequence_number;
44 for (size_t offset = 0; offset < data.size(); offset += chunk_size) {
45 const size_t length = std::min(chunk_size, data.size() - offset);
46 const csv::string_view bytes(data.data() + offset, length);
47 const bool first_chunk = offset == 0;
48
49 SpeculativeParseChunk chunk;
50 chunk.sequence_number = sequence_number;
51 chunk.offset = base_offset + offset;
52 chunk.bytes = bytes;
53 chunk.owner = owner;
54 chunk.speculation = scanner.speculate(sequence_number, chunk.offset, bytes);
55 chunk.starts_at_record_boundary = first_chunk;
56
57 // The first chunk in a speculative window starts at a known row
58 // boundary because source windows must be aligned to incomplete-row
59 // starts before the next read.
60 if (first_chunk) {
61 chunk.speculation.assumed_start_state = ParserDFAState();
62 }
63
64 chunks.push_back(chunk);
65 sequence_number++;
66 }
67
68 return chunks;
69 }
70
71 template<bool EagerClassify = false>
72 class ParallelCSVParser {
73 public:
74 ParallelCSVParser(
75 const ParseFlagMap& parse_flags,
76 const WhitespaceMap& ws_flags,
77 size_t worker_count = 1,
78 const ColNamesPtr& col_names = nullptr
79 ) : parse_flags_(parse_flags),
80 ws_flags_(ws_flags),
81 col_names_(col_names),
82 task_pool_(worker_count == 0 ? 1 : worker_count) {
83 this->init_worker_parsers();
84 }
85
86 ~ParallelCSVParser() {}
87
88 ParallelCSVParser(const ParallelCSVParser&) = delete;
89 ParallelCSVParser& operator=(const ParallelCSVParser&) = delete;
90
91 ParsedChunkRows parse_chunk(const SpeculativeParseChunk& chunk) const {
92 ChunkParserCoreT<EagerClassify> parser(this->parse_flags_, this->ws_flags_, this->col_names_);
93 return this->parse_chunk_with(parser, chunk);
94 }
95
96 template<typename RowSink>
97 ParallelCSVParseResult parse_chunks(
98 const std::vector<SpeculativeParseChunk>& chunks,
99 RowSink& output,
100 bool finish = true
101 ) {
102 ParallelCSVParseResult result;
103 result.chunks_processed = chunks.size();
104 for (size_t i = 0; i < chunks.size(); ++i) {
105 observe_speculation(result.diagnostics, chunks[i].speculation);
106 }
107
108 std::vector<ParsedChunkRows> parsed(chunks.size());
109 this->parse_chunks_into(chunks, parsed);
110
111 ChunkParserCoreT<EagerClassify> repair_parser(this->parse_flags_, this->ws_flags_, this->col_names_);
112 SpeculativeParseValidator<RowSink, ChunkParserCoreT<EagerClassify>> validator(repair_parser, output);
113 for (size_t i = 0; i < parsed.size(); ++i) {
114 validator.validate_and_release(std::move(parsed[i]));
115 }
116 validator.finish(finish);
117
118 result.repair_count = validator.repair_count();
119 result.diagnostics.validation_repairs += result.repair_count;
120 result.has_pending_suffix = validator.has_pending_suffix();
121 result.ending_state = validator.expected_start_state();
122 if (!chunks.empty()) {
123 if (validator.has_pending_suffix()) {
124 result.complete_prefix_length = validator.pending_suffix().offset - chunks.front().offset;
125 }
126 else {
127 result.complete_prefix_length =
128 chunks.back().offset + chunks.back().bytes.size() - chunks.front().offset;
129 }
130 }
131 return result;
132 }
133
134 private:
135 ParsedChunkRows parse_chunk_with(
136 ChunkParserCoreT<EagerClassify>& parser,
137 const SpeculativeParseChunk& chunk
138 ) const {
139 std::vector<CSVRow> rows;
140 const ParserChunkResult parse_result = parser.parse_chunk(
141 chunk.bytes,
142 chunk.owner,
143 rows,
144 ParserChunkOptions(chunk.speculation.assumed_start_state, chunk.offset)
145 );
146
147 ParsedChunkRows result = split_parsed_chunk_rows(
148 chunk.sequence_number,
149 chunk.bytes,
150 chunk.owner,
151 parse_result,
152 std::move(rows),
153 chunk.starts_at_record_boundary,
154 chunk.offset
155 );
156 return result;
157 }
158
159 void parse_chunks_into(
160 const std::vector<SpeculativeParseChunk>& chunks,
161 std::vector<ParsedChunkRows>& parsed
162 ) {
163 if (this->task_pool_.worker_count() > 1 && chunks.size() > 1) {
164 this->parse_chunks_parallel(chunks, parsed);
165 return;
166 }
167
168 ChunkParserCoreT<EagerClassify> parser(this->parse_flags_, this->ws_flags_, this->col_names_);
169 for (size_t i = 0; i < chunks.size(); ++i) {
170 parsed[i] = this->parse_chunk_with(parser, chunks[i]);
171 }
172 }
173
174 void parse_chunks_parallel(
175 const std::vector<SpeculativeParseChunk>& chunks,
176 std::vector<ParsedChunkRows>& parsed
177 ) {
178 this->task_pool_.parallel_for(chunks.size(), [this, &chunks, &parsed](
179 size_t worker_index,
180 size_t task_index
181 ) {
182 CSV_DEBUG_ASSERT(worker_index < this->worker_parsers_.size());
183 parsed[task_index] = this->parse_chunk_with(
184 this->worker_parsers_[worker_index],
185 chunks[task_index]
186 );
187 });
188 }
189
190 void init_worker_parsers() {
191 const size_t worker_count = this->task_pool_.worker_count();
192 if (worker_count <= 1) {
193 return;
194 }
195
196 this->worker_parsers_.reserve(worker_count);
197 for (size_t i = 0; i < worker_count; ++i) {
198 this->worker_parsers_.push_back(
199 ChunkParserCoreT<EagerClassify>(this->parse_flags_, this->ws_flags_, this->col_names_)
200 );
201 }
202 }
203
204 ParseFlagMap parse_flags_;
205 WhitespaceMap ws_flags_;
206 ColNamesPtr col_names_;
207 internals::parallel::IndexedTaskPool task_pool_;
208 std::vector<ChunkParserCoreT<EagerClassify>> worker_parsers_;
209 };
210 }
211 }
212}
213
214#endif
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
The all encompassing namespace.
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182