TSADatabase.h

/*  ===================================================================
*
*                 T R A D I N G   S Y S T E M   A P I ™
*    Copyright © 1999 - 2014 by Peter Ritter ( TradingSystemAPI.com )
*                A L L   R I G H T S   R E S E R V E D
*
*     Consult your license regarding permissions and restrictions.
*             You may obtain a copy of the License at:
*     http://www.TradingSystemAPI.com/licenses/LICENSE-TSAPI-3.0.html
*
*  ====================================================================
*/

#ifndef TSA_DATABASE__INCLUDED
#define TSA_DATABASE__INCLUDED

#include <fstream>
#include <iostream>
#include <ostream>
#include <climits>

#include "TSADBCore.h"
#include "TSALargeFile.h"
#include "TSASimBaseSeries.h"
#include "TSAPriceBar.h"
#include "TSASeriesTemplate.h"
#include "TSAStreams.h"
#include "TSAOS.h"
#include "TSADataDef.h"
#include "TSADBBase.h"
#include "TSADatabaseImp.h"

namespace tsa {


    dll_export enum open_db_flag_type {
        create_db = 0x2,       
        no_create_db = 0x4,    
        truncate_db = 0x8      
    };

    open_db_flag_type operator |(open_db_flag_type left_flag, open_db_flag_type right_flag);


    namespace os {
        class path;
    }
    class table_imp;
    class large_file;
    class mem_table;

    
    namespace fast {
        
        /*
        ** ====================================================
        ** > > >        class database                    < < <
        ** ====================================================
        */

        class dll_export database final : public data_base {
            tsa_declare_testable;
            friend class table_imp_ref;
            friend class table_imp;
            friend class table;
        private:
            friend class table_reader;
        private:
            class db_file_creator {
            private:
                friend class database;
                db_file_creator(void);
                db_file_creator(const std::string& db_name,
                    const os::path& dir, open_db_flag_type flag);
            } m_db_file_maker;
            static const std::string default_db_file_name;
        public:
            enum Result { success, table_does_not_exist, table_is_opened };
        private:
            db_page_file m_page_file;
            db_storage_manager m_storage_man;
            std::map<table_imp_ref*, table_imp_ref*> m_table_ptr_map;
        private:
            identifier_t m_database_ID;
            std::map<std::string, table_imp*> m_name_to_table_map;
            size_t __get_next_page_num_to_write(void) const;
            std::string  m_db_name;
            os::path m_db_path;
        public:
            static bool invalid_shutdown;
        private:
            db_page_file* get_page_file_ptr(void);
            size_t get_page_size(void) const;
        public:
            table_imp_ref get_table_object(const std::string&);
            void save_table_object(const table_imp&);
            size_t get_open_table_count(void) const;
        private:
            static bool can_open_file_exclusively(const os::path& path);
            database(const std::string& data_file, const std::string& index_file, bool tst, bool _tst);
        public:
            virtual std::string type(void)const override;
        public:

            database(const os::path& db_path, open_db_flag_type flag = create_db);

            virtual ~database(void);
            
            // Internal
            database(const std::string&  db_name, const os::path& db_dir, open_db_flag_type flag = create_db);
        public:

            static bool exists(const os::path& db_path);

            static void remove(const os::path& db_path);
        public:
            // internal use
            static void create_db_files(const std::string& name, const os::path& directory, bool bReplace = false);
            static void remove_db_files(const std::string& name, const os::path& directory);
            static bool db_files_exists(const std::string& name, const os::path& directory);
            static void create_index_file(const std::string&);
            static void create_data_file(const std::string&);
        public:
            std::string get_db_name(void) const;
            os::path get_db_path(void) const;
        public:

            table_imp_ref table(const std::string& table_name);

            void create_table(const std::string&  table_name, const tsa::column_defs& columns);

            virtual void create_series_table(const std::string&  name, const tsa::column_defs& columns)override;

            void create_table(const tsa::mem_table& mem_table, const std::string& db_table_name,
                replace_flag_type replace = no_replace_existing);

            void copy_table(const std::string& source_table_name, const std::string& copy_table_name);

            void drop_table(const std::string& table_name, missing_table_flag_type missing_table_flag = silent_on_missing);

            virtual bool drop_series_table(const std::string& table_name)override;

            virtual bool table_exists(const std::string& name) const override;

            bool table_exists_with_data(const std::string& table_name);

            bool table_is_open(const std::string& table_name);

            bool table_is_writeable(const std::string& name)const;

            bool table_is_readable(const std::string& name);

            tsa::column_defs table_column_defs(const std::string& table_name);

            size_t table_count(void) const;

            void rename_table(const std::string& table_name, const std::string& new_table_name);

            void table_names(std::vector<std::string>& table_name_arr) const;

            // IU
            bool tables_are_equal(const std::string& name1, const std::string& name2);
            // IU
            std::string unused_table_name(void);
        public:

            /* Internal 
             *  
             *  Will silently replace an existing table if true, else throw an exception if table already
             *  exists. The Schema with @b schemaName must already exist! Schema definitions are created in
             *  the series_base via the Database::CreateSchema() member. This function will fail if a table
             *  with a given @b tableName already exists, unless @b replace flag is true.
             */
            void create_table_ex(const std::string& table_name, const std::string& column_defs_name);
        public:
            Result _DeleteTable(const std::string& tableName);
            bool GetTableTimeSpanInfo(const std::string& tableName, size_t& numberOfRecords,
                date_time& firstTimestamp, date_time& lastTimestamp);
            table_metrics get_table_metrics(const std::string& tableName);
        public:
            virtual in_stream_adaptor* make_input_stream_adaptor(const std::string& stream)override;
            virtual out_stream_adaptor* make_output_stream_adaptor(void)override;
        public:
            virtual date_time series_table_first_timestamp(const std::string& table_name)override;
            virtual date_time series_table_last_timestamp(const std::string& table_name)override;
            virtual bool table_has_data(const std::string& table_name)override;
            virtual tsa::column_defs series_table_column_defs(const std::string& table_name)override;
            virtual date_time series_table_closest_timestamp_LOE(const date_time& ts, const std::string& table_name)override;
            virtual void series_load(const std::string& table_name, tsa::mem_table& data,
                const date_time& from_timestamp, size_t num_records)override;
            virtual void series_append(const std::string& table_name, const tsa::mem_table& data)override;
            virtual void truncate_series_table(const std::string& table_name, const date_time& new_max_timestamp = date_time::min)override;
            virtual bool table_record_count_is_known(void)const;
            virtual int64_t table_record_count(const std::string& table_name);
        public:

            void export_table(const std::string& name, const os::path& dir_path, bool replace = true);

            bool import_table(const os::path& dir_path, const std::string& file_name,

                bool replace = false, bool error_as_exception = true);

        public:


            void print_table_report(std::ostream& stream);

            void print_table_names(std::ostream&);

            void print_table(const std::string& name, std::ostream& stream, size_t FPPrecision = 8,
                size_t maxNumRec = tsa::print_no_limit, bool bPrintTimestamp = true);

            void save_table_html(const std::string&  name, const os::path& path, size_t FPPrecision = 8,
                size_t maxNumRec = tsa::print_no_limit, bool bPrintTimestamp = true);

            void print_table_csv(const std::string& name,
                std::ostream& stream, size_t maxNumRec = tsa::print_no_limit);

            void print_table_csv(const std::string& name, const date_time& start, 
                const date_time& end, std::ostream& stream);

            void print_table_vertical(const std::string& name, std::ostream&  stream, 
                bool inclTimestamp = true, size_t maxNumRecToPrint = tsa::print_no_limit);

            /* 
             * @brief   Prints an (ascii) graph of the values found in a given @b column of a given @b table to @b Stream
             *
             * @param   table           The name of the table that contains the given @bcolumn name.
             * @param [in,out]  stream  The stream to which the graph will be written.
             * @param   column          The name of the column (field) which is to be graphed.
             * @param   width           The width, in characters, of the resulting graph.
             * @param   height          The height, in characters, of the resulting graph.
             * @param   start           The lowest timestamp to be used (lower bound of data range)
             * @param   end             The highest timestamp to be used (upper bound of data range)
             */

            void print_graph(const std::string&  table,
                std::ostream& stream, const std::string&  column,
                size_t width = 70, size_t height = 20,
                const date_time& start = date_time::min, const date_time& end = date_time::max
                );

            /*
             * @brief   Print hlc graph.
             *
             * @param   table           The name of the table that contains the given 'high', 'low', 'close'
             *                          fields.
             * @param [in,out]  stream  The stream to which the graph will be written.
             * @param   width           The width, in characters, of the resulting graph.
             * @param   height          The height, in characters, of the resulting graph.
             * @param   start           The lowest timestamp to be used (lower bound of data range)
             * @param   end             The highest timestamp to be used (upper bound of data range)
             */

            void print_hlc_graph(const std::string& table, std::ostream& stream,
                size_t width = 70, size_t height = 20,
                const date_time& start = date_time::min, const date_time& end = date_time::max
                );
            void print_table(const std::string& tableName,
                std::ostream& S, TablePrinter& tablePrinter,
                bool bPrintTimestamp, size_t maxNumRec = tsa::print_no_limit);

        public:
            db_storage_manager* get_db_storage_manager(void);
            db_record* create_record_object(const std::string& data_def_name);
        public:
            void create_data_def(const std::string&  schemaName, const tsa::column_defs& schema);
            bool data_def_exists(const std::string& name) const;
            tsa::column_defs GetSchema(const std::string& schemaName) const;
        private:
            bool delete_data_def(const std::string& name);
            void delete_unused_data_def(void);
            std::string data_def_name_from_table_name(const std::string& tableName) const;
            std::string get_unused_data_def_name(void) const;
            void data_def_names_arr(std::vector<std::string>& schemaNameArray) const;
            bool data_def_in_use(const std::string& schemaName) const;
            void print_data_def_report(std::ostream& stream);
            void init_data_def(tsa::column_defs& schema, const std::string& schemaName) const;
        public:
            size_t db_page_count(void) const;
            size_t db_free_page_count(void) const;
        private: 
            friend class tsa::mem_table;
            void read_into_mem_table(tsa::mem_table& series, const std::string& tableID);
        private:    // Internal Use
            void register_table_imp_ptr(table_imp_ref*);
            void unregister_table_imp_ptr(table_imp_ref*);
            void close_table(table_imp* ptable);
            table_imp* LoadCorePtr(unsigned table_id);
            void force_close_table_no_exception(table_imp* pSeries);
            Result load_table(table_imp*&, const std::string& table_name);
            table_imp* load_table(const std::string& table_name);
            void print_db_table_info(std::ostream& Stream);
        public:
            void VerifyTableHasOHLCV_Fields(const std::string& tableName);
            void verify_table_has_PV_fields(const std::string& tableName);
            void verify_has_ohlcv_fields(const std::string& tableName);

            // @name  TimeFrame Conversion
            // @{

            void convert_tick_to_1min(const std::string& in_table_name, 
                const std::string& out_table_name, 
                const std::string& price_column_name = "last", const std::string& volume_column_name = "volume");

            void convert_tick_to_sec(const std::string& in_table_name, 
                size_t seconds_per_bar, const std::string& out_table_name);

            void convert_1min_to_multi_min(const std::string& in_table_name,
                size_t minutes_per_bar, const std::string& out_table_name);

            void convert_to_daily(const std::string& in_table_name, const std::string& out_table_name);

            void convert_to_weekly(const std::string& in_table_name, const std::string& out_table_name);

            void convert_to_monthly(const std::string& in_table_name, const std::string& out_table_name);

            void convert_tick_to_multi_tick(const std::string& in_table_name, 
                size_t numTicksPerBar, const std::string& targetTable, 
                const std::string& price_column_name = "price", const std::string& volume_column_name = "");

            void convert_tick_to_volume(const std::string& source_table, 
                size_t volumePerBar, const std::string& out_table_name);
            // @} END

        public:
            //internal
            void import_daily_data(const std::string& tableName,
                const os::path& file,
                const std::string&  dateFormatString = "yyyymmdd",
                char sep = ','
                );

            // internal
            void import_intraday_data(const std::string& tableName, const os::path& file,
                const std::string&  dateFormatString = "yyyymmdd",
                const std::string& timeFormatString = "hh:mm:ss.f",
                char sep = ',', bool fixDuplicateTime = false);

            // Internal, Describes the data to be imported from file.   
            struct import_info {
                import_info(void) {
                    ;
                }
                os::path path;
                variant table_name;
                bool delete_existing_table = false;
                variant date_format;
                variant time_format;
                bool timestamp_one_field = true;
                char datetime_separator = ' ';
                variant columns;
                char separator = '\n';
                bool fix_duplicate_timestamp = false;
            };
            
            // Iternal 
            void import_file(const import_info& info);
        };
    
    }    // namespace fast


}    // namespace tsa
#endif