csv_reader.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <algorithm>
#include <deque>
#include <exception>
#include <fstream>
#include <iterator>
#include <memory>
#include <mutex>
#include <thread>
#include <sstream>
#include <string>
#include <vector>
 
#include "../external/mio.hpp"
#include "basic_csv_parser.hpp"
#include "common.hpp"
#include "data_type.hpp"
#include "csv_format.hpp"
 
namespace csv {
    namespace internals {
        std::string format_row(const std::vector<std::string>& row, csv::string_view delim = ", ");
 
        std::vector<std::string> _get_col_names( csv::string_view head, const CSVFormat format = CSVFormat::guess_csv());
 
        struct GuessScore {
            double score;
            size_t header;
        };
 
        CSV_INLINE GuessScore calculate_score(csv::string_view head, const CSVFormat& format);
 
        CSVGuessResult _guess_format(csv::string_view head, const std::vector<char>& delims = { ',', '|', '\t', ';', '^', '~' });
    }
 
    std::vector<std::string> get_col_names(
        csv::string_view filename,
        const CSVFormat format = CSVFormat::guess_csv());
 
    CSVGuessResult guess_format(csv::string_view filename,
        const std::vector<char>& delims = { ',', '|', '\t', ';', '^', '~' });
 
    class CSVReader {
    public:
        class iterator {
        public:
            #ifndef DOXYGEN_SHOULD_SKIP_THIS
            using value_type = CSVRow;
            using difference_type = std::ptrdiff_t;
            using pointer = CSVRow * ;
            using reference = CSVRow & ;
            using iterator_category = std::input_iterator_tag;
            #endif
 
            iterator() = default;
            iterator(CSVReader* reader) : daddy(reader) {}
            iterator(CSVReader*, CSVRow&&);
 
            CONSTEXPR_14 reference operator*() { return this->row; }
            CONSTEXPR_14 reference operator*() const { return const_cast<reference>(this->row); }
 
            CONSTEXPR_14 pointer operator->() { return &(this->row); }
            CONSTEXPR_14 pointer operator->() const { return const_cast<pointer>(&(this->row)); }
 
            iterator& operator++();   
            iterator operator++(int); 
            CONSTEXPR bool operator==(const iterator& other) const noexcept {
                return (this->daddy == other.daddy) && (this->i == other.i);
            }
 
            CONSTEXPR bool operator!=(const iterator& other) const noexcept { return !operator==(other); }
        private:
            CSVReader * daddy = nullptr;  // Pointer to parent
            CSVRow row;                   // Current row
            size_t i = 0;               // Index of current row
        };
 
 
        CSVReader(csv::string_view filename, CSVFormat format = CSVFormat::guess_csv());
 
        template<typename TStream,
            csv::enable_if_t<std::is_base_of<std::istream, TStream>::value, int> = 0>
        CSVReader(TStream &source, CSVFormat format = CSVFormat::guess_csv()) : _format(format) {
            auto head = internals::get_csv_head(source);
            using Parser = internals::StreamParser<TStream>;
 
            // Apply chunk size from format before any reading occurs
            this->_chunk_size = format.get_chunk_size();
 
            if (format.guess_delim()) {
                auto guess_result = internals::_guess_format(head, format.possible_delimiters);
                format.delimiter(guess_result.delim);
                // Only override header if user hasn't explicitly called no_header()
                // Note: column_names() also sets header=-1, but it populates col_names,
                // so we can distinguish: no_header() means header=-1 && col_names.empty()
                if (format.header != -1 || !format.col_names.empty()) {
                    format.header = guess_result.header_row;
                }
                this->_format = format;
            }
 
            if (!format.col_names.empty())
                this->set_col_names(format.col_names);
 
            this->parser = std::unique_ptr<Parser>(
                new Parser(source, format, col_names)); // For C++11
            this->initial_read();
        }
 
        CSVReader(const CSVReader&) = delete;             
        CSVReader(CSVReader&&) = delete;                  
        CSVReader& operator=(const CSVReader&) = delete;  
        CSVReader& operator=(CSVReader&&) = delete;       
        ~CSVReader() {
            if (this->read_csv_worker.joinable()) {
                this->read_csv_worker.join();
            }
        }
 
        bool read_row(CSVRow &row);
        iterator begin();
        CSV_CONST iterator end() const noexcept;
 
        bool eof() const noexcept { return this->parser->eof(); }
 
        CSVFormat get_format() const;
        std::vector<std::string> get_col_names() const;
        int index_of(csv::string_view col_name) const;
 
 
        CONSTEXPR bool empty() const noexcept { return this->n_rows() == 0; }
 
        CONSTEXPR size_t n_rows() const noexcept { return this->_n_rows; }
 
        bool utf8_bom() const noexcept { return this->parser->utf8_bom(); }
 
    protected:
        void set_col_names(const std::vector<std::string>&);
 
        CSVFormat _format;
 
 
        internals::ColNamesPtr col_names = std::make_shared<internals::ColNames>();
 
        std::unique_ptr<internals::IBasicCSVParser> parser = nullptr;
 
        std::unique_ptr<RowCollection> records{new RowCollection(100)};
 
        size_t n_cols = 0;  
        size_t _n_rows = 0; 
        bool read_csv(size_t bytes = internals::ITERATION_CHUNK_SIZE);
 
    private:
        bool header_trimmed = false;
 
        std::thread read_csv_worker; 
        size_t _chunk_size = internals::ITERATION_CHUNK_SIZE; 
        bool _read_requested = false; 
 
        std::exception_ptr read_csv_exception = nullptr;
        std::mutex read_csv_exception_lock;
 
        void set_read_csv_exception(std::exception_ptr eptr) {
            std::lock_guard<std::mutex> lock(this->read_csv_exception_lock);
            this->read_csv_exception = std::move(eptr);
        }
 
        std::exception_ptr take_read_csv_exception() {
            std::lock_guard<std::mutex> lock(this->read_csv_exception_lock);
            auto eptr = this->read_csv_exception;
            this->read_csv_exception = nullptr;
            return eptr;
        }
 
        void rethrow_read_csv_exception_if_any() {
            if (auto eptr = this->take_read_csv_exception()) {
                std::rethrow_exception(eptr);
            }
        }
 
        void initial_read() {
            this->read_csv_worker = std::thread(&CSVReader::read_csv, this, this->_chunk_size);
            this->read_csv_worker.join();
            this->rethrow_read_csv_exception_if_any();
        }
 
        void trim_header();
    };
}