Vince's CSV Parser
Loading...
Searching...
No Matches
mmap.cpp
1#include "mmap.hpp"
2#include "orchestrator.hpp"
3
4#if !defined(__EMSCRIPTEN__)
5namespace csv {
6 namespace internals {
7 namespace parser {
8 MmapParser::MmapParser(
9 csv::string_view filename,
10 const CSVFormat& format,
11 const ColNamesPtr& col_names
12 ) : CSVParserDriverBase(format, col_names) {
13 this->_filename = std::string(filename);
14 auto head_and_size = get_csv_head_mmap(filename, error_);
15 if (error_) return;
16 this->head_ = std::move(head_and_size.first);
17 this->source_size_ = head_and_size.second;
18 this->resolve_format_from_head(format);
19 if (error_) return;
20
21 this->parse_orchestrator_ = make_csv_parse_orchestrator(
22 this->parse_flags_,
23 this->whitespace_flags(),
24 format,
25 this->source_size_,
26 col_names,
27 true
28 );
29 }
30
31 MmapParser::~MmapParser() = default;
32
33 bool MmapParser::finalize_loaded_chunk(
34 csv::string_view chunk,
35 std::shared_ptr<void> owner,
36 size_t length,
37 size_t chunk_size
38 ) {
39 const size_t base_offset = this->mmap_pos - length;
40 const bool source_exhausted = this->mmap_pos == this->source_size_;
41 CSVParseWindowResult result = this->parse_orchestrator_->parse_window(
42 chunk,
43 std::move(owner),
44 base_offset,
45 chunk_size,
46 source_exhausted,
47 this->output(), error_
48 );
49 if (error_) return false;
50
51 if (source_exhausted) {
52 this->eof_ = true;
53 }
54
55 this->mmap_pos -= (length - result.complete_prefix_length);
56 return result.completed_row();
57 }
58
59 size_t MmapParser::read_window_size(size_t chunk_size) const noexcept {
60 return this->parse_orchestrator_
61 ? this->parse_orchestrator_->read_window_size(chunk_size)
62 : chunk_size;
63 }
64
65 void MmapParser::next(size_t bytes = CSV_CHUNK_SIZE_DEFAULT) {
66 if (error_) return;
67 // CRITICAL SECTION: Chunk Transition Logic
68 // This function reads 10MB chunks and must correctly handle fields that span
69 // chunk boundaries. The 'remainder' calculation below ensures partial fields
70 // are preserved for the next chunk.
71 //
72 // Bug #280: Field corruption occurred here when chunk transitions incorrectly
73 // split multi-byte characters or field boundaries.
74
75 // Reuse the pre-read head buffer (if any) as the first chunk.
76 // This avoids re-reading the same bytes that were already consumed
77 // for delimiter/header guessing.
78 if (!head_.empty()) {
79 auto head_owner = std::make_shared<std::string>(std::move(head_));
80 const size_t length = head_owner->size();
81 this->mmap_pos += length;
82
83 const bool completed_row = this->finalize_loaded_chunk(*head_owner, head_owner, length, bytes);
84
85 // Issue #337: The head is capped at 500KB regardless of chunk_size,
86 // so it may end before the first row does. Only in that case fall
87 // through and map a full read window from the rewound position;
88 // otherwise return so construction doesn't parse an extra window
89 // (CSVReader::initial_read() keeps reading if the header row is
90 // still pending). Ask the parse result rather than comparing
91 // positions: a skipped UTF-8 BOM advances mmap_pos even when no
92 // row completed.
93 if (error_ || this->eof_ || completed_row) {
94 return;
95 }
96 }
97
98 // Create memory map
99 const size_t offset = this->mmap_pos;
100 const size_t remaining = (offset < this->source_size_)
101 ? (this->source_size_ - offset)
102 : 0;
103 const size_t length = std::min(remaining, this->read_window_size(bytes));
104 if (length == 0) {
105 // No more data to read; mark EOF and end feed
106 // (Prevent exception on empty mmap as reported by #267)
107 this->eof_ = true;
108 this->end_feed();
109 return;
110 }
111
112 std::error_code error;
113 auto mmap = mio::make_mmap_source(this->_filename, offset, length, error);
114 if (error) {
115 fail(error_, CSVErrorCode::MappingFailed,
116 make_mmap_failure_message(this->_filename, offset, length));
117 error_.system = error;
118 return;
119 }
120 auto mmap_owner = std::make_shared<mio::basic_mmap_source<char>>(std::move(mmap));
121 this->mmap_pos += length;
122
123 auto mmap_ptr = mmap_owner.get();
124
125 // Create string view
126 csv::string_view chunk(mmap_ptr->data(), mmap_ptr->length());
127 this->finalize_loaded_chunk(chunk, mmap_owner, length, bytes);
128 }
129 }
130 }
131}
132#endif
Borrow a row and error state for synchronous, single-pass field access.
The all encompassing namespace.
std::string_view string_view
The string_view class used by this library.
Definition common.hpp:182