Vince's CSV Parser
Loading...
Searching...
No Matches
driver.hpp
Go to the documentation of this file.
1
5#pragma once
6#include <algorithm>
7#include <array>
8#include <cmath>
9#include <exception>
10#include <fstream>
11#include <limits>
12#include <memory>
13#include <unordered_map>
14#include <unordered_set>
15#include <utility>
16#include <vector>
17
18#if !defined(__EMSCRIPTEN__)
19#include "../../external/mio.hpp"
20#endif
21#include "../basic_csv_parser_simd.hpp"
22#include "../col_names.hpp"
23#include "../common.hpp"
24#include "../csv_exceptions.hpp"
25#include "../csv_format.hpp"
26#include "core.hpp"
27#include "../csv_row.hpp"
28#include "../speculative/diagnostics.hpp"
29
30namespace csv {
31 namespace internals {
37 size_t get_bom_skip(csv::string_view data, bool& utf8_bom, CSVError& error);
38
39 namespace parser {
40 struct GuessScore {
41 size_t header;
42 size_t mode_row_length;
43 double score;
44 };
45
46 GuessScore calculate_score(csv::string_view head, const CSVFormat& format);
47
50 csv::string_view head, const std::vector<char>& delims, CSVError& error);
51
53 template<typename TStream,
54 csv::enable_if_t<std::is_base_of<std::istream, TStream>::value, int> = 0>
56 error.clear();
57#if !CSV_EXCEPTIONS_ENABLED
58 if (source.exceptions() != std::ios::goodbit) {
59 fail_static(error, CSVErrorCode::InvalidArgument,
60 "Explicit CSV reads require a non-throwing stream exception mask.");
61 return {};
62 }
63#endif
64 const size_t limit = CSV_HEAD_PREVIEW_SIZE;
65 std::string buf(limit, '\0');
66 CSV_TRY {
67 source.read(&buf[0], static_cast<std::streamsize>(limit));
68 }
69 CSV_CATCH((const std::ios_base::failure&), {
70 if (!source.eof() || source.bad()) {
71 fail_static(error, CSVErrorCode::ReadFailed, ERROR_STREAM_READ_FAILURE);
72 return {};
73 }
74 })
75 if (source.bad() || (source.fail() && !source.eof())) {
76 fail_static(error, CSVErrorCode::ReadFailed, ERROR_STREAM_READ_FAILURE);
77 return {};
78 }
79 buf.resize(static_cast<size_t>(source.gcount()));
80 return buf;
81 }
82
83#if defined(__EMSCRIPTEN__)
85 std::string get_csv_head_stream(csv::string_view filename, CSVError& error);
86#endif
87
88#if !defined(__EMSCRIPTEN__)
92 std::pair<std::string, size_t> get_csv_head_mmap(csv::string_view filename, CSVError& error);
93#endif
94
96 std::string get_csv_head(csv::string_view filename, CSVError& error);
97
99 CSVFormat format;
100 size_t n_cols = 0;
101 };
102
104 }
105 }
106
126 const std::vector<char>& delims = { ',', '|', '\t', ';', '^', '~' }) {
127 error.clear();
128 auto head = internals::parser::get_csv_head(filename, error);
129 if (error) return {',', 0, 0};
130 return internals::parser::guess_format(head, delims, error);
131 }
132
133#if CSV_EXCEPTIONS_ENABLED
134 inline CSVGuessResult guess_format(csv::string_view filename,
135 const std::vector<char>& delims = { ',', '|', '\t', ';', '^', '~' }) {
136 CSVError error;
137 auto result = try_guess_format(filename, error, delims);
138 internals::throw_error(error);
139 return result;
140 }
141#endif
142
143 namespace internals {
144 namespace parser {
145
147 public:
148 virtual ~ICSVParseOrchestrator() {}
149
150 virtual size_t worker_count() const noexcept = 0;
151 virtual SpeculativeParseDiagnostics diagnostics() const noexcept = 0;
152 virtual size_t read_window_size(size_t chunk_size) const noexcept = 0;
153 virtual bool utf8_bom() const noexcept = 0;
154 virtual void reset_with_initial_state(ParserDFAState state) noexcept = 0;
155 virtual ParserDFAState ending_state() const noexcept = 0;
156
157 virtual CSVParseWindowResult parse_window(
158 csv::string_view chunk,
159 std::shared_ptr<void> owner,
160 size_t base_offset,
161 size_t serial_chunk_size,
162 bool source_exhausted,
163 RowCollection& output, CSVError& error
164 ) = 0;
165 };
166
167 inline std::unique_ptr<ICSVParseOrchestrator> make_csv_parse_orchestrator(
168 const ParseFlagMap& parse_flags,
169 const WhitespaceMap& ws_flags,
170 const CSVFormat& format,
171 size_t source_size,
172 const ColNamesPtr& col_names = nullptr,
173 bool enable_speculative_parallel = true,
174 bool source_size_known = true
175 );
176
183 public:
184 CSVParserDriverBase() = default;
185 CSVParserDriverBase(const CSVFormat&, const ColNamesPtr&);
187 const ParseFlagMap& parse_flags,
188 const WhitespaceMap& ws_flags
189 ) : CSVParserCore<>(parse_flags, ws_flags) {}
190
191 virtual ~CSVParserDriverBase() {}
192
194 const CSVError& error() const noexcept { return error_; }
195
197 bool eof() const noexcept { return this->eof_; }
198
199 ResolvedFormat get_resolved_format() { return this->format; }
200
202 virtual void next(size_t bytes) = 0;
203
204 virtual SpeculativeParseDiagnostics speculative_diagnostics() const noexcept {
205 return this->parse_orchestrator_
206 ? this->parse_orchestrator_->diagnostics()
208 }
209
210 virtual size_t parse_worker_count() const noexcept {
211 return this->parse_orchestrator_
212 ? this->parse_orchestrator_->worker_count()
213 : 1;
214 }
215
216 virtual bool utf8_bom() const noexcept {
217 return this->parse_orchestrator_
218 ? this->parse_orchestrator_->utf8_bom()
219 : false;
220 }
221
222 protected:
225 bool eof_ = false;
226 CSVError error_;
227
228 ResolvedFormat format;
229
231 size_t source_size_ = 0;
233
234 std::unique_ptr<ICSVParseOrchestrator> parse_orchestrator_;
235
236 virtual std::string& get_csv_head() = 0;
237
238 void resolve_format_from_head(const CSVFormat& format);
239 };
240 }
241 }
242}
Stores information about how to parse a CSV file.
Borrow a row and error state for synchronous, single-pass field access.
A std::deque wrapper which allows multiple read and write threads to concurrently access it along wit...
Abstract base class for source adapters.
Definition driver.hpp:182
virtual void next(size_t bytes)=0
Parse the next block of data.
bool eof() const noexcept
Whether or not we have reached the end of source.
Definition driver.hpp:197
const CSVError & error() const noexcept
Source errors are read by the consumer only after joining its scheduler.
Definition driver.hpp:194
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
constexpr size_t CSV_HEAD_PREVIEW_SIZE
Bytes read from the start of a source to resolve its format (header, delimiter guessing) before the f...
Definition common.hpp:438
Focused CSV byte parser core.
std::string get_csv_head_stream(TStream &source, CSVError &error)
Read the first 500KB from a seekless stream source.
Definition driver.hpp:55
CSVGuessResult guess_format(csv::string_view head, const std::vector< char > &delims, CSVError &error)
Guess the delimiter used by a delimiter-separated values file.
Definition guessing.cpp:73
The all encompassing namespace.
CSVGuessResult try_guess_format(csv::string_view filename, CSVError &error, const std::vector< char > &delims={ ',', '|', '\t', ';', '^', '~' })
Guess the delimiter, header row, and mode column count of a CSV file.
Definition driver.hpp:125
@ InvalidArgument
Invalid operation input, such as a null stream or zero batch size.
@ ReadFailed
The input stream reported an I/O failure rather than ordinary EOF.
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182
A diagnostic whose text remains valid for the lifetime of the error.
Definition csv_error.hpp:94
Stores the inferred format of a CSV file.
Explicit DFA state at a parse boundary.
Definition core.hpp:106