Vince's CSV Parser
Loading...
Searching...
No Matches
csv::internals::parser::StreamParser< TStream > Class Template Reference

A class for parsing CSV data from any std::istream, including non-seekable sources such as pipes and decompression filters. More...

#include <stream.hpp>

Inherits csv::internals::parser::CSVParserDriverBase.

Public Member Functions

 StreamParser (TStream &source, const CSVFormat &format, const ColNamesPtr &col_names=nullptr)
 
 StreamParser (TStream &source, ParseFlagMap parse_flags, WhitespaceMap ws_flags)
 
std::string & get_csv_head () override
 
void next (size_t bytes=CSV_CHUNK_SIZE_DEFAULT) override
 Parse the next block of data.
 
void reset_with_initial_state (ParserDFAState state) noexcept
 
void reset_with_initial_state (bool starts_in_quoted, bool in_escape=false) noexcept
 
ParserDFAState ending_state () const noexcept
 
- Public Member Functions inherited from csv::internals::parser::CSVParserDriverBase
 CSVParserDriverBase (const CSVFormat &, const ColNamesPtr &)
 
 CSVParserDriverBase (const ParseFlagMap &parse_flags, const WhitespaceMap &ws_flags)
 
const CSVError & error () const noexcept
 Source errors are read by the consumer only after joining its scheduler.
 
bool eof () const noexcept
 Whether or not we have reached the end of source.
 
ResolvedFormat get_resolved_format ()
 
virtual SpeculativeParseDiagnostics speculative_diagnostics () const noexcept
 
virtual size_t parse_worker_count () const noexcept
 
virtual bool utf8_bom () const noexcept
 
- Public Member Functions inherited from csv::internals::CSVParserCore< RowSink, ParsePolicy, FieldPolicy, RowPolicy >
 CSVParserCore (const CSVFormat &source_format, const ColNamesPtr &col_names)
 
 CSVParserCore (const ParseFlagMap &parse_flags, const WhitespaceMap &ws_flags)
 
 CSVParserCore (const ParseFlagMap &parse_flags, const WhitespaceMap &ws_flags, const ColNamesPtr &col_names)
 
void end_feed ()
 Indicate the last block of data has been parsed.
 
CONSTEXPR_17 ParseFlags parse_flag (const char ch) const noexcept
 
CONSTEXPR_17 ParseFlags compound_parse_flag (const char ch) const noexcept
 
void set_output (RowSink &output) noexcept
 
RowSink & output () noexcept
 
void reset_with_initial_state (ParserDFAState state) noexcept
 Seed the DFA state for the next parse call.
 
void reset_with_initial_state (bool starts_in_quoted, bool in_escape=false) noexcept
 Convenience overload for callers that only care about quote state.
 
ParserDFAState ending_state () const noexcept
 Return the DFA state observed when the most recent parse call stopped.
 
ParserChunkResult parse_chunk (csv::string_view chunk, std::shared_ptr< void > owner)
 
ParserChunkResult parse_chunk (csv::string_view chunk, std::shared_ptr< void > owner, const ParserChunkOptions &options)
 
ParserChunkResult parse_chunk (csv::string_view chunk, std::shared_ptr< void > owner, RowSink &output)
 
ParserChunkResult parse_chunk (csv::string_view chunk, std::shared_ptr< void > owner, RowSink &output, size_t source_start)
 
ParserChunkResult parse_chunk (csv::string_view chunk, std::shared_ptr< void > owner, RowSink &output, const ParserChunkOptions &options)
 

Additional Inherited Members

- Protected Member Functions inherited from csv::internals::parser::CSVParserDriverBase
void resolve_format_from_head (const CSVFormat &format)
 
- Protected Member Functions inherited from csv::internals::CSVParserCore< RowSink, ParsePolicy, FieldPolicy, RowPolicy >
void set_col_names (const ColNamesPtr &col_names)
 
void set_whitespace_flags (const WhitespaceMap &ws_flags)
 
void set_parse_flags (const ParseFlagMap &parse_flags)
 
void set_parse_flags (const ParseFlagMap &parse_flags, char delimiter, char quote_char)
 
size_t parse ()
 Parse the current chunk of data and return the completed-row prefix length.
 
void reset_data_ptr ()
 Create a new RawCSVDataPtr for a new chunk of data.
 
const WhitespaceMap & whitespace_flags () const noexcept
 
void emit_row (CSVRow &&row)
 
- Protected Attributes inherited from csv::internals::parser::CSVParserDriverBase
std::unique_ptr< ICSVParseOrchestrator > parse_orchestrator_
 
bool eof_ = false
 
CSVError error_
 
ResolvedFormat format
 
size_t source_size_ = 0
 The size of the incoming CSV.
 
- Protected Attributes inherited from csv::internals::CSVParserCore< RowSink, ParsePolicy, FieldPolicy, RowPolicy >
CSVRow current_row_
 
RawCSVDataPtr data_ptr_ = nullptr
 
ColNamesPtr col_names_ = nullptr
 
RawCSVFieldList * fields_ = nullptr
 
int field_start_ = UNINITIALIZED_FIELD
 
size_t field_length_ = 0
 
SentinelVecs simd_sentinels_
 Precomputed SIMD broadcast vectors for find_next_non_special.
 
ParseFlagMap parse_flags_
 An array where the (i + 128)th slot gives the ParseFlags for ASCII character i.
 

Detailed Description

template<typename TStream>
class csv::internals::parser::StreamParser< TStream >

A class for parsing CSV data from any std::istream, including non-seekable sources such as pipes and decompression filters.

Chunk boundary handling
parse() returns the byte offset of the start of the last incomplete row in the current chunk (the "remainder"). Rather than seeking back to re-read those bytes (which requires a seekable stream), they are saved in leftover_ and prepended to the next chunk. This is semantically identical to the old seek-back approach but works on any istream and avoids the syscall overhead of seekg().
Format resolution
The constructor reads a head buffer from the stream via get_csv_head() and passes it to resolve_format_from_head(), which infers the delimiter and header row when not explicitly set. The head bytes are stored in leftover_ so the first next() call re-parses them without re-reading.

Definition at line 29 of file stream.hpp.

Constructor & Destructor Documentation

◆ StreamParser() [1/2]

template<typename TStream >
csv::internals::parser::StreamParser< TStream >::StreamParser ( TStream &  source,
const CSVFormat &  format,
const ColNamesPtr &  col_names = nullptr 
)
inline

Definition at line 33 of file stream.hpp.

◆ StreamParser() [2/2]

template<typename TStream >
csv::internals::parser::StreamParser< TStream >::StreamParser ( TStream &  source,
ParseFlagMap  parse_flags,
WhitespaceMap  ws_flags 
)
inline

Definition at line 52 of file stream.hpp.

◆ ~StreamParser()

Definition at line 69 of file stream.hpp.

Member Function Documentation

◆ ending_state()

template<typename TStream >
ParserDFAState csv::internals::parser::StreamParser< TStream >::ending_state ( ) const
inlinenoexcept

Definition at line 160 of file stream.hpp.

◆ get_csv_head()

template<typename TStream >
std::string & csv::internals::parser::StreamParser< TStream >::get_csv_head ( )
inlineoverridevirtual

Implements csv::internals::parser::CSVParserDriverBase.

Definition at line 71 of file stream.hpp.

◆ next()

template<typename TStream >
void csv::internals::parser::StreamParser< TStream >::next ( size_t  bytes = CSV_CHUNK_SIZE_DEFAULT)
inlineoverridevirtual

Parse the next block of data.

Implements csv::internals::parser::CSVParserDriverBase.

Definition at line 76 of file stream.hpp.

◆ reset_with_initial_state() [1/2]

template<typename TStream >
void csv::internals::parser::StreamParser< TStream >::reset_with_initial_state ( bool  starts_in_quoted,
bool  in_escape = false 
)
inlinenoexcept

Definition at line 156 of file stream.hpp.

◆ reset_with_initial_state() [2/2]

template<typename TStream >
void csv::internals::parser::StreamParser< TStream >::reset_with_initial_state ( ParserDFAState  state)
inlinenoexcept

Definition at line 147 of file stream.hpp.


The documentation for this class was generated from the following file: