Vince's CSV Parser
Loading...
Searching...
No Matches
chunks.hpp
1#pragma once
2
3#include "../parser/driver.hpp"
4
5namespace csv {
6 namespace internals {
7 namespace speculative {
9 CSVRowFragment() = default;
11 csv::string_view bytes,
12 std::shared_ptr<void> owner,
13 ParserDFAState initial_state = ParserDFAState(),
14 ParserDFAState ending_state = ParserDFAState(),
15 size_t offset = 0
16 ) : bytes(bytes),
17 owner(std::move(owner)),
18 initial_state(initial_state),
19 ending_state(ending_state),
20 offset(offset),
21 present(true) {}
22
23 bool empty() const noexcept {
24 return !present;
25 }
26
27 static CSVRowFragment from_row(
28 const CSVRow& row,
29 ParserDFAState initial_state = ParserDFAState(),
30 ParserDFAState ending_state = ParserDFAState(),
31 size_t chunk_offset = 0
32 ) {
33 return CSVRowFragment(
34 row.raw_str(),
35 row.data,
36 initial_state,
37 ending_state,
38 chunk_offset + row.data_start
39 );
40 }
41
42 csv::string_view bytes;
43 std::shared_ptr<void> owner;
44 ParserDFAState initial_state;
45 ParserDFAState ending_state;
46 size_t offset = 0;
47 bool present = false;
48 };
49
50 inline CSVRowFragment concatenate_row_fragments(
51 const CSVRowFragment& left,
53 ) {
54 if (left.empty()) {
55 return right;
56 }
57
58 if (right.empty()) {
59 return left;
60 }
61
62 auto bytes = std::make_shared<std::string>();
63 bytes->reserve(left.bytes.size() + right.bytes.size());
64 if (!left.bytes.empty()) {
65 bytes->append(left.bytes.data(), left.bytes.size());
66 }
67 if (!right.bytes.empty()) {
68 bytes->append(right.bytes.data(), right.bytes.size());
69 }
70
71 return CSVRowFragment(
72 csv::string_view(*bytes),
73 bytes,
74 left.initial_state,
75 right.ending_state,
76 left.offset
77 );
78 }
79
81 size_t sequence_number = 0;
82 size_t offset = 0;
83 csv::string_view chunk;
84 std::shared_ptr<void> owner;
85 bool starts_at_record_boundary = true;
86 ParserChunkResult parse_result;
87 CSVRowFragment prefix_fragment;
88 std::vector<CSVRow> complete_rows;
89 CSVRowFragment suffix_fragment;
90 };
91
92 inline ParsedChunkRows split_parsed_chunk_rows(
93 size_t sequence_number,
94 csv::string_view chunk,
95 std::shared_ptr<void> owner,
96 const ParserChunkResult& parse_result,
97 std::vector<CSVRow> parsed_rows,
98 bool starts_at_record_boundary,
99 size_t chunk_offset = 0
100 ) {
102 result.sequence_number = sequence_number;
103 result.offset = chunk_offset;
104 result.chunk = chunk;
105 result.owner = owner;
106 result.starts_at_record_boundary = starts_at_record_boundary
107 || parse_result.initial_state.pending_linefeed;
108 result.parse_result = parse_result;
109
110 size_t first_complete_row = 0;
111 if (!result.starts_at_record_boundary) {
112 if (parsed_rows.empty()) {
113 result.prefix_fragment = CSVRowFragment(
114 chunk,
115 owner,
116 parse_result.initial_state,
117 parse_result.ending_state,
119 );
120 return result;
121 }
122
123 result.prefix_fragment = CSVRowFragment::from_row(
124 parsed_rows.front(),
125 parse_result.initial_state,
128 );
130 }
131
132 result.complete_rows.reserve(parsed_rows.size() - first_complete_row);
133 for (size_t i = first_complete_row; i < parsed_rows.size(); ++i) {
134 result.complete_rows.push_back(std::move(parsed_rows[i]));
135 }
136
137 if (parse_result.complete_prefix_length < chunk.size()) {
138 result.suffix_fragment = CSVRowFragment(
139 chunk.substr(parse_result.complete_prefix_length),
140 owner,
141 ParserDFAState(),
142 parse_result.ending_state,
143 chunk_offset + parse_result.complete_prefix_length
144 );
145 }
146
147 return result;
148 }
149
150 template<typename Parser>
151 inline std::vector<CSVRow> materialize_row_fragment(
152 Parser& parser,
153 const CSVRowFragment& fragment
154 ) {
155 std::vector<CSVRow> rows;
156 if (fragment.empty()) {
157 return rows;
158 }
159
160 parser.parse_chunk(
161 fragment.bytes,
162 fragment.owner,
163 rows,
164 ParserChunkOptions(ParserDFAState(), fragment.offset)
165 );
166 parser.end_feed();
167 return rows;
168 }
169
170 template<typename Parser>
171 inline ParsedChunkRows repair_parsed_chunk_rows(
172 Parser& parser,
173 const ParsedChunkRows& chunk,
174 ParserDFAState corrected_initial_state
175 ) {
176 std::vector<CSVRow> parsed_rows;
177
178 const ParserChunkResult parse_result = parser.parse_chunk(
179 chunk.chunk,
180 chunk.owner,
181 parsed_rows,
182 ParserChunkOptions(corrected_initial_state, chunk.offset)
183 );
184
185 return split_parsed_chunk_rows(
186 chunk.sequence_number,
187 chunk.chunk,
188 chunk.owner,
189 parse_result,
190 std::move(parsed_rows),
191 chunk.starts_at_record_boundary,
192 chunk.offset
193 );
194 }
195
201 template<bool EagerClassify = false>
203 std::vector<CSVRow>,
204 PermissiveParsePolicy,
205 CSVRowFieldPolicy<EagerClassify>,
206 CSVRowRowPolicy> {
207 using Base = CSVParserCore<
208 std::vector<CSVRow>,
212
213 public:
215 const ParseFlagMap& parse_flags,
216 const WhitespaceMap& ws_flags
217 ) : Base(parse_flags, ws_flags) {}
218
220 const ParseFlagMap& parse_flags,
221 const WhitespaceMap& ws_flags,
222 const ColNamesPtr& col_names
223 ) : Base(parse_flags, ws_flags, col_names) {}
224 };
225
227 }
228 }
229}
Data structure for representing CSV rows.
Definition csv_row.hpp:533
csv::string_view raw_str() const noexcept
Return a string_view of the raw bytes of this row as they appear in the underlying parse buffer,...
Definition csv_row.cpp:75
Borrow a row and error state for synchronous, single-pass field access.
Minimal parser shell for caller-owned chunks.
Definition chunks.hpp:206
std::array< ParseFlags, 256 > ParseFlagMap
An array which maps ASCII chars to a parsing flag.
Definition common.hpp:495
std::array< bool, 256 > WhitespaceMap
An array which maps ASCII chars to a flag indicating if it is whitespace.
Definition common.hpp:498
The all encompassing namespace.
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182
Default row policy for the CSVRow path.
Definition core.hpp:308
Explicit DFA state at a parse boundary.
Definition core.hpp:106
Default parse policy hook.
Definition core.hpp:181