TSAMemTable.h

/*  ===================================================================
*
*                 T R A D I N G   S Y S T E M   A P I ™
*    Copyright © 1999 - 2014 by Peter Ritter ( TradingSystemAPI.com )
*                A L L   R I G H T S   R E S E R V E D
*
*     Consult your license regarding permissions and restrictions.
*             You may obtain a copy of the License at:
*     http://www.TradingSystemAPI.com/licenses/LICENSE-TSAPI-3.0.html
*
*  ====================================================================
*/

#ifndef TSA_MEM_TABLE__INCLUDED
#define TSA_MEM_TABLE__INCLUDED

#include <algorithm>
#include <ostream>
#include <climits>

#include "TSADBRecord.h"
#include "TSASeriesTemplate.h"
#include "TSAStreams.h"
#include "TSADataDef.h"

namespace tsa {
    class column_defs;
    class series_base;
    namespace fast{
        class table_imp_ref;
    }

    /*
    ** ====================================================
    ** >>>             class mem_table                  <<<
    ** ====================================================
    */

    class dll_export mem_table :
        public out_stream_adaptor,
        public in_stream_adaptor,
        public object,
        public appendable {
    private:
        std::vector<variant*> m_records;
        size_t m_num_fields;
        size_t m_total_num_fields; //including timestamp
        mutable size_t m_cursor_position;
        std::vector<variant> m_append_record;
        mutable variant* m_curr_record_ptr;
        std::vector<std::string> m_field_names;
        std::vector<type_t> m_field_types;
        std::vector<size_t> m_field_lengths;
        size_t m_max_record_count;
    public:
        static const size_t default_max_records;
    protected:
        void Deallocate(void);
        void Allocate(size_t num_columns);
    public:
        mem_table(void);

        mem_table(int col_count);

        mem_table(const mem_table& other);

        mem_table(const mem_table&& other);

        // internal
        mem_table(const fast::table_imp_ref&);

        mem_table(const column_defs& col_def);

        virtual ~mem_table(void);

        void operator= (const table& other);

        void name(const std::string& name);

        const std::string& name(void) const;

        void size_limit(size_t max);

        size_t size_limit(void) const;

        bool is_equal(const mem_table& other, double fp_allowance = 0.0);

        void columns(const column_defs& data_def);

        class column_defs columns(void) const;

    public:
        //=============================
        //=============================
        
        variant* operator[](size_t) const;

        virtual class record record(void);    /*class appendable*/

        void read(size_t index, class record&);

        virtual void append(const class record&, append_flag_type flag = error_on_duplicate_timestamp);   /*class appendable*/

        size_t size(void) const;

        bool empty(void) const;

        date_time first_timestamp(void) const;

        date_time last_timestamp(void) const;

        void clear(void);

        const variant& at(size_t record_pos, size_t col_pos) const;

        void update(size_t record_pos, size_t col_pos, const variant& value);

        // Returns the timestamp of the record at @b record_pos.
        date_time timestamp(size_t record_pos) const;

        void delete_range(const date_time& start, const date_time& end);

        void delete_range(size_t start, size_t end);

        void resize__drop_oldest(size_t newSize);

        void truncate(const date_time& new_max_timestamp);

        void append(const mem_table& other);

        void populate_random_records(size_t num_records);
        void populate_random_records(size_t num_records, bool some_as_nil);

        // Turns self into an identical copy of @b other. Copies both its structure and data.
        void copy_from(const mem_table& other);

        // Clears @b other, and then populates @b other with all records between @b from and @b to (inclusive)
        void copy_to(mem_table& other, const date_time& from, const date_time& to);
    public: //Cursor operations

        // Positions the cursor on the records at @a index.
        void cursor__seek(size_t index)const;

        //
        bool cursor__seek_next(void)const;

        //
        bool cursor__seek_prev(void)const;

        // Positions the cursor on the last record.
        void cursor__seek_last(void)const;

        // Returns the index of the record on which the internal iterator is currently positioned.
        size_t cursor__position(void) const;

        // Returns the time of the record on which the cursor is currently positioned.
        date_time cursor__timestamp(void) const;

        //
        void cursor__read_record(date_time& timestamp, std::vector<variant>& rec)const;

        // Returns the value of the field at @a index, of the record on which the cursor is currently positioned.
        const variant& cursor__at(size_t fieldIndex) const;

        // Returns the value of the field with given @a fieldName, of the record on which the cursor is currently positioned.
        const variant& cursor__at(const std::string& fieldName) const;

        // Returns true if Cursor position is not nullptr.
        bool cursor__is_valid(void) const;

        // Modifies the value of the field at @a fieldIndex for the 'current record'
        void cursor__set(size_t fieldIndex, const variant& value);
        date_time cursor__seek(const date_time& target_timestamp)const;
        seek_result cursor__seek_NEW(const date_time& target_timestamp)const;
        date_time test__find_timestamp_LOE(const date_time&);
    public:
        //=====================
        //=====================
        // 
        void copy(fast::table& db_table);

        // @brief Populates self with data from @b db table with @b tableName.
        void read_db_table(fast::database& db, const std::string& tableName);

        void populate_random_records(size_t numRecords, const date_time& start,
            const duration& ts, bool someAsNIL = true);

        
        void populate_random_ohlcv_records(size_t numRecords, const date_time& start,
            const duration& ts);
    public:
        void append_set(size_t fieldIndex, const variant& value);
        void internal_append(const date_time& timestamp,
            const std::vector<variant>& record, append_flag_type flag);
        void Append(const date_time& appendTime);
    public:
        //===========================
        //===========================

        // Internal: Resets to number of fields to @b numFields, all of type float8. All data will be
        // lost.
        void reset_column_count(size_t num_cols);

        void reset_column_type(size_t column_pos, type_t new_type, size_t length = 5);

        void rename_column(size_t column_position, const std::string& new_name);

        void reset_column_properties(size_t column_position, const std::string& new_name, type_t new_type, size_t length = 5);

        void add_column(const std::string& name, const variant& type_and_default_value, size_t length = 5);

        const std::string& column_name(size_t position) const;

        type_t column_type(size_t position) const;

        size_t column_position(const std::string& column_name) const;

        bool column_exists(const std::string& column_name) const;

        virtual size_t column_count(void) const;    /*appendable*/

        size_t column_field_length(size_t column_position) const;
    public:

        // Returns true if it is possible to convert all Variants at @a fieldIndex to @a type, in
        // all rows, without error.
        bool type_conversion_possible(size_t fieldIndex, type_t type);

        // Returns the index of the field with given @b fieldname, by initializing @b index.
        // Will return false if a field with given name does not exist, else true.
        bool field_index(const std::string& fieldName, size_t& index) const;

    public: // Internal

        // @brief Sets all values in @b column_name to @b value. Every record will be affected!
        void reset_all_values_in_column(const std::string& column_name, const variant& value);

        // @brief Sets all values in the column indexed by @b fieldIndex to @b value. Every record will be affected!
        void reset_all_values_in_column(size_t columnIndex, const variant& value);

        // @brief Iterates through all values in @b column_name, and replaces the value with @b newValue if the
        // original value was equal to @b toReplace.
        void replace(const std::string& column_name, const variant& to_replace,const variant& new_value);

        // @brief Iterates through all values in @b column_name, and deletes all records where the
        // value is equal to @b deleteValue.
        size_t delete_where(const std::string& column_name, const variant& deleteValue);

        // @brief Multiples all values at @b column_index by given @b factor.
        void multiply_all_values_in_column(size_t columnIndex, double factor);

        // @brief Adds @b value to all values in @b columnIndex.
        void add_to_all_values_in_column(size_t columnIndex, double value);

        // @brief Truncates all string values in all fields of type string to their respective
        // field size.
        void fix_string_length(void);
    public:
        void purge_deleted_records(void);
        size_t last_valid_record_index(void) const;    //IU Returns the highest valid record index.
        bool test__curr_record_ptr_is_valid(void) const;
        variant* get_record_ptr(void) const;
        void Read(date_time& timestamp, std::vector<variant>& InternalRecord);
        void _ReadWithTimestampAsFirstValue(std::vector<variant>& InternalRecord);
    public:
        std::string to_string(void)const;
        void from_string(const std::string&);

        void write_to(std::stringstream&)const;
        void read_from(std::stringstream&);

    public: //Internal
        // Prints table (with borders). @b maxRec can be used to limit the number of records to print.
        // @b FPPrecision determines the precision of floating point numbers.
        void print(std::ostream& stream, size_t maxRec = tsa::print_no_limit, size_t FPPrecision = 7);
        // Prints the table as comma separated values (CSV). @b maxRec can be used to limit
        // the number of records to print.
        void csv(std::ostream&, size_t maxRec = tsa::print_no_limit);
        // internal
        void print_record(size_t pos, std::ostream&);
        void print_record_header(std::ostream&);

    public: // out_stream_adaptor
        virtual void osa__append(const date_time&, std::vector<variant>&) override;
        virtual void osa__prepare_stream(const std::string& sourceName,
            const tsa::column_defs& dd) override;

    public: // component
        virtual void cmp__evaluate_bar(const date_time&) override;

    public: //in_stream_adaptor
        virtual void        isa__set_position(const date_time&) override;
        virtual bool        isa__field_exists(const std::string&) override;
        virtual size_t      isa__field_index(const std::string&) override;
        virtual type_t      isa__field_type(size_t) override;
        virtual size_t      isa__field_count(void) override;
        virtual const std::string& isa__field_name(size_t _idx) override;
        virtual date_time   isa__get_record_timestamp(void) override;
        virtual date_time   isa__begin(void) override;
        virtual date_time   isa__end(void) override;
        virtual size_t      isa__size(void) override;
        virtual void        isa__on_open(void);
        virtual variant     isa__get_variant(size_t idx) override;
        virtual double      isa__get_double(size_t idx) override;
        virtual void        isa__update_value_at_position(size_t idx) override;
        virtual std::string isa__stream_name(void) const override;
        virtual bool        isa__get_ohlc_fields_exist(void) override;
        virtual void        isa__get_price_bar(const date_time&, instrument_price_bar&) override;
        virtual bool        isa__get_time_of_next_bar(date_time& nextDateTime, const date_time& currentDateTime) override;
        virtual bool        isa__implements_scheduling(void) const override;
    public: //Testing
        bool verify_types_in_column(size_t col_ndex) const;
        bool verify_types(void) const;
        bool VerifyColumnDataEqualTo(const std::string& col_name, const variant& data) const;
        bool VerifyColumnVariantsOfType(size_t col_index, type_t type) const;
        bool VerifyColumnStringsHaveValidLength(size_t col_index) const;
    };
}//tsa

#endif