3#include "../parallel/indexed_task_pool.hpp"
5#include "validator.hpp"
11 namespace speculative {
12 struct SpeculativeParseChunk {
13 size_t sequence_number = 0;
16 std::shared_ptr<void> owner;
17 ChunkSpeculation speculation;
18 bool starts_at_record_boundary =
false;
21 struct ParallelCSVParseResult {
22 size_t chunks_processed = 0;
23 size_t repair_count = 0;
24 size_t complete_prefix_length = 0;
25 bool has_pending_suffix =
false;
26 ParserDFAState ending_state;
27 SpeculativeParseDiagnostics diagnostics;
30 inline std::vector<SpeculativeParseChunk> make_speculative_parse_chunks(
32 std::shared_ptr<void> owner,
34 const SpeculativeScanner& scanner,
35 size_t base_offset = 0,
36 size_t first_sequence_number = 0
38 std::vector<SpeculativeParseChunk> chunks;
39 if (chunk_size == 0) {
43 size_t sequence_number = first_sequence_number;
44 for (
size_t offset = 0; offset < data.size(); offset += chunk_size) {
45 const size_t length = std::min(chunk_size, data.size() - offset);
47 const bool first_chunk = offset == 0;
49 SpeculativeParseChunk chunk;
50 chunk.sequence_number = sequence_number;
51 chunk.offset = base_offset + offset;
54 chunk.speculation = scanner.speculate(sequence_number, chunk.offset, bytes);
55 chunk.starts_at_record_boundary = first_chunk;
61 chunk.speculation.assumed_start_state = ParserDFAState();
64 chunks.push_back(chunk);
71 template<
bool EagerClassify = false>
72 class ParallelCSVParser {
75 const ParseFlagMap& parse_flags,
76 const WhitespaceMap& ws_flags,
77 size_t worker_count = 1,
78 const ColNamesPtr& col_names =
nullptr
79 ) : parse_flags_(parse_flags),
81 col_names_(col_names),
82 task_pool_(worker_count == 0 ? 1 : worker_count) {
83 this->init_worker_parsers();
86 ~ParallelCSVParser() {}
88 ParallelCSVParser(
const ParallelCSVParser&) =
delete;
89 ParallelCSVParser& operator=(
const ParallelCSVParser&) =
delete;
91 ParsedChunkRows parse_chunk(
const SpeculativeParseChunk& chunk)
const {
92 ChunkParserCoreT<EagerClassify> parser(this->parse_flags_, this->ws_flags_, this->col_names_);
93 return this->parse_chunk_with(parser, chunk);
96 template<
typename RowSink>
97 ParallelCSVParseResult parse_chunks(
98 const std::vector<SpeculativeParseChunk>& chunks,
102 ParallelCSVParseResult result;
103 result.chunks_processed = chunks.size();
104 for (
size_t i = 0; i < chunks.size(); ++i) {
105 observe_speculation(result.diagnostics, chunks[i].speculation);
108 std::vector<ParsedChunkRows> parsed(chunks.size());
109 this->parse_chunks_into(chunks, parsed);
111 ChunkParserCoreT<EagerClassify> repair_parser(this->parse_flags_, this->ws_flags_, this->col_names_);
112 SpeculativeParseValidator<RowSink, ChunkParserCoreT<EagerClassify>> validator(repair_parser, output);
113 for (
size_t i = 0; i < parsed.size(); ++i) {
114 validator.validate_and_release(std::move(parsed[i]));
116 validator.finish(finish);
118 result.repair_count = validator.repair_count();
119 result.diagnostics.validation_repairs += result.repair_count;
120 result.has_pending_suffix = validator.has_pending_suffix();
121 result.ending_state = validator.expected_start_state();
122 if (!chunks.empty()) {
123 if (validator.has_pending_suffix()) {
124 result.complete_prefix_length = validator.pending_suffix().offset - chunks.front().offset;
127 result.complete_prefix_length =
128 chunks.back().offset + chunks.back().bytes.size() - chunks.front().offset;
135 ParsedChunkRows parse_chunk_with(
136 ChunkParserCoreT<EagerClassify>& parser,
137 const SpeculativeParseChunk& chunk
139 std::vector<CSVRow> rows;
140 const ParserChunkResult parse_result = parser.parse_chunk(
144 ParserChunkOptions(chunk.speculation.assumed_start_state, chunk.offset)
147 ParsedChunkRows result = split_parsed_chunk_rows(
148 chunk.sequence_number,
153 chunk.starts_at_record_boundary,
159 void parse_chunks_into(
160 const std::vector<SpeculativeParseChunk>& chunks,
161 std::vector<ParsedChunkRows>& parsed
163 if (this->task_pool_.worker_count() > 1 && chunks.size() > 1) {
164 this->parse_chunks_parallel(chunks, parsed);
168 ChunkParserCoreT<EagerClassify> parser(this->parse_flags_, this->ws_flags_, this->col_names_);
169 for (
size_t i = 0; i < chunks.size(); ++i) {
170 parsed[i] = this->parse_chunk_with(parser, chunks[i]);
174 void parse_chunks_parallel(
175 const std::vector<SpeculativeParseChunk>& chunks,
176 std::vector<ParsedChunkRows>& parsed
178 this->task_pool_.parallel_for(chunks.size(), [
this, &chunks, &parsed](
182 CSV_DEBUG_ASSERT(worker_index < this->worker_parsers_.size());
183 parsed[task_index] = this->parse_chunk_with(
184 this->worker_parsers_[worker_index],
190 void init_worker_parsers() {
191 const size_t worker_count = this->task_pool_.worker_count();
192 if (worker_count <= 1) {
196 this->worker_parsers_.reserve(worker_count);
197 for (
size_t i = 0; i < worker_count; ++i) {
198 this->worker_parsers_.push_back(
199 ChunkParserCoreT<EagerClassify>(this->parse_flags_, this->ws_flags_, this->col_names_)
206 ColNamesPtr col_names_;
207 internals::parallel::IndexedTaskPool task_pool_;
208 std::vector<ChunkParserCoreT<EagerClassify>> worker_parsers_;
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
The all encompassing namespace.
std::string_view string_view
The string_view class used by this library.