blockchain.h

Go to the documentation of this file.

// Copyrights(c) 2017-2020, The Electroneum Project
// Copyrights(c) 2014-2019, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
//    conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
//    of conditions and the following disclaimer in the documentation and/or other
//    materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
//    used to endorse or promote products derived from this software without specific
//    prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
#pragma once
#include <boost/asio/io_service.hpp>
#include <boost/serialization/serialization.hpp>
#include <boost/serialization/version.hpp>
#include <boost/serialization/list.hpp>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/global_fun.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/algorithm/hex.hpp>
#include <atomic>
#include <functional>
#include <unordered_map>
#include <unordered_set>
 
#include "span.h"
#include "syncobj.h"
#include "string_tools.h"
#include "rolling_median.h"
#include "cryptonote_basic/cryptonote_basic.h"
#include "common/util.h"
#include "cryptonote_protocol/cryptonote_protocol_defs.h"
#include "rpc/core_rpc_server_commands_defs.h"
#include "cryptonote_basic/difficulty.h"
#include "cryptonote_tx_utils.h"
#include "cryptonote_basic/verification_context.h"
#include "crypto/hash.h"
#include "checkpoints/checkpoints.h"
#include "cryptonote_basic/hardfork.h"
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_basic/validators.h"
 
namespace tools { class Notify; }
 
namespace cryptonote
{
  class tx_memory_pool;
  struct test_options;
 
  enum blockchain_db_sync_mode
  {
    db_defaultsync, 
    db_sync,  
    db_async, 
    db_nosync 
  };
 
  typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback;
 
  /************************************************************************/
  /*                                                                      */
  /************************************************************************/
  class Blockchain
  {
  public:
    struct transaction_chain_entry
    {
      transaction tx;
      uint64_t m_keeper_block_height;
      size_t m_blob_size;
      std::vector<uint64_t> m_global_output_indexes;
    };
 
    struct block_extended_info
    {
      block   bl; 
      uint64_t height; 
      size_t block_cumulative_weight; 
      difficulty_type cumulative_difficulty; 
      uint64_t already_generated_coins; 
    };
 
    Blockchain(tx_memory_pool& tx_pool);
 
    ~Blockchain();
 
    bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0, const GetCheckpointsCallback& get_checkpoints = nullptr, bool ignore_bsig = false, bool fallback_to_pow = false);
 
    bool init(BlockchainDB* db, HardFork*& hf, const network_type nettype = MAINNET, bool offline = false);
 
    bool deinit();
 
    void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; }
 
    bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const;
 
    bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const;
 
    bool get_alternative_blocks(std::vector<block>& blocks) const;
 
    size_t get_alternative_blocks_count() const;
 
    crypto::hash get_block_id_by_height(uint64_t height) const;
 
    bool get_block_by_hash(const crypto::hash &h, block &blk, bool *orphan = NULL) const;
 
    bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry>  &blocks_entry, std::vector<block> &blocks);
 
    bool cleanup_handle_incoming_blocks(bool force_sync = false);
 
    bool have_tx(const crypto::hash &id) const;
 
    bool have_tx_keyimges_as_spent(const transaction &tx) const;
 
    bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const;
 
    uint64_t get_current_blockchain_height() const;
 
    crypto::hash get_tail_id() const;
 
    crypto::hash get_tail_id(uint64_t& height) const;
 
    difficulty_type get_difficulty_for_next_block();
 
    void normalize_v7_difficulties();   
 
    bool add_new_block(const block& bl_, block_verification_context& bvc);
 
    bool reset_and_set_genesis_block(const block& b);
 
    bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
    bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
 
    bool have_block(const crypto::hash& id) const;
 
    size_t get_total_transactions() const;
 
    bool get_short_chain_history(std::list<crypto::hash>& ids) const;
 
    bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const;
 
    bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
 
    bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const;
 
    bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const;
 
    bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp);
 
    uint64_t get_num_mature_outputs(uint64_t amount) const;
 
    crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const;
 
    bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const;
 
    void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const;
 
    bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
 
    bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const;
    bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const;
 
    bool store_blockchain();
 
    bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false);
 
    static uint64_t get_fee_quantization_mask();
 
    static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version);
 
    uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const;
 
    bool check_fee(size_t tx_weight, uint64_t fee) const;
 
    bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc);
 
    uint64_t get_current_cumulative_block_weight_limit() const;
 
    uint64_t get_next_long_term_block_weight(uint64_t block_weight) const;
 
    uint64_t get_current_cumulative_block_weight_median() const;
 
    difficulty_type block_difficulty(uint64_t i) const;
 
    template<class t_ids_container, class t_blocks_container, class t_missed_container>
    bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const;
 
    template<class t_ids_container, class t_tx_container, class t_missed_container>
    bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const;
    template<class t_ids_container, class t_tx_container, class t_missed_container>
    bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const;
    template<class t_ids_container, class t_tx_container, class t_missed_container>
    bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const;
 
    //debug functions
 
    void check_against_checkpoints(const checkpoints& points, bool enforce);
 
    void set_enforce_dns_checkpoints(bool enforce);
 
    bool update_checkpoints(const std::string& file_path, bool check_dns);
 
    // user options, must be called before calling init()
 
    void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold,
        blockchain_db_sync_mode sync_mode, bool fast_sync, std::string validator_key);
 
    void set_block_notify(const std::shared_ptr<tools::Notify> &notify) { m_block_notify = notify; }
 
    void set_reorg_notify(const std::shared_ptr<tools::Notify> &notify) { m_reorg_notify = notify; }
 
    void safesyncmode(const bool onoff);
 
    void set_show_time_stats(bool stats) { m_show_time_stats = stats; }
 
    HardFork::State get_hard_fork_state() const;
 
    static const std::vector<HardFork::Params>& get_hard_fork_heights(network_type nettype);
 
    uint8_t get_current_hard_fork_version() const { return m_hardfork->get_current_version(); }
 
    uint8_t get_ideal_hard_fork_version() const { return m_hardfork->get_ideal_version(); }
 
    uint8_t get_next_hard_fork_version() const { return m_hardfork->get_next_version(); }
 
    uint8_t get_ideal_hard_fork_version(uint64_t height) const { return m_hardfork->get_ideal_version(height); }
 
    uint8_t get_hard_fork_version(uint64_t height) const { return m_hardfork->get(height); }
 
    uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return m_hardfork->get_earliest_ideal_height_for_version(version); }
 
    bool get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const;
 
    uint64_t get_difficulty_target() const;
 
    bool flush_txes_from_pool(const std::vector<crypto::hash> &txids);
 
    std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count = 0) const;
 
    bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
 
    bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const block&)>) const;
 
    bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const;
 
    bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)>) const;
 
    bool for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)>) const;
 
    const BlockchainDB& get_db() const
    {
      return *m_db;
    }
 
    BlockchainDB& get_db()
    {
      return *m_db;
    }
 
    void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets,
        std::vector<output_data_t> &outputs) const;
 
    void block_longhash_worker(uint64_t height, const epee::span<const block> &blocks,
        std::unordered_map<crypto::hash, crypto::hash> &map) const;
 
    std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const;
 
    void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta);
    void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta);
    void remove_txpool_tx(const crypto::hash &txid);
    uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const;
    bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const;
    bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const;
    cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
    bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const;
    uint32_t get_mempool_tx_livetime() const;
 
    bool is_within_compiled_block_hash_area(uint64_t height) const;
    bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); }
    uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); }
    bool prune_blockchain(uint32_t pruning_seed = 0);
    bool update_blockchain_pruning();
    bool check_blockchain_pruning();
 
    void lock();
    void unlock();
 
    void cancel();
 
    void on_new_tx_from_block(const cryptonote::transaction &tx);
 
    std::vector<time_t> get_last_block_timestamps(unsigned int blocks) const;
 
    void pop_blocks(uint64_t nblocks);
 
    void set_validator_key(std::string key) { m_validator_key = boost::algorithm::unhex(key); }
 
    void set_validators_list_instance(std::unique_ptr<electroneum::basic::Validators> &v) { m_validators = v.get(); }
 
    electroneum::basic::Validator get_validator_by_height(uint64_t height);
 
    network_type get_nettype() const { return m_nettype; };
    
 
#ifndef IN_UNIT_TESTS
  private:
#endif
 
    // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage
    typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index;
 
    typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container;
 
    typedef std::unordered_set<crypto::key_image> key_images_container;
 
    typedef std::vector<block_extended_info> blocks_container;
 
    typedef std::unordered_map<crypto::hash, block_extended_info> blocks_ext_by_hash;
 
    typedef std::unordered_map<crypto::hash, block> blocks_by_hash;
 
    typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction
 
 
    BlockchainDB* m_db;
 
    tx_memory_pool& m_tx_pool;
 
    mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock
 
    // main chain
    transactions_container m_transactions;
    size_t m_current_block_cumul_weight_limit;
    size_t m_current_block_cumul_weight_median;
 
    // metadata containers
    std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table;
    std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table;
    std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table;
 
    // SHA-3 hashes for each block and for fast pow checking
    std::vector<crypto::hash> m_blocks_hash_check;
    std::vector<crypto::hash> m_blocks_txs_check;
 
    blockchain_db_sync_mode m_db_sync_mode;
    bool m_fast_sync;
    bool m_show_time_stats;
    bool m_db_default_sync;
    bool m_db_sync_on_blocks;
    uint64_t m_db_sync_threshold;
    uint64_t m_max_prepare_blocks_threads;
    uint64_t m_fake_pow_calc_time;
    uint64_t m_fake_scan_time;
    uint64_t m_sync_counter;
    uint64_t m_bytes_to_sync;
    std::vector<uint64_t> m_timestamps;
    std::vector<difficulty_type> m_difficulties;
    uint64_t m_timestamps_and_difficulties_height;
    uint64_t m_long_term_block_weights_window;
    uint64_t m_long_term_effective_median_block_weight;
    mutable crypto::hash m_long_term_block_weights_cache_tip_hash;
    mutable epee::misc_utils::rolling_median_t<uint64_t> m_long_term_block_weights_cache_rolling_median;
 
    epee::critical_section m_difficulty_lock;
    crypto::hash m_difficulty_for_next_block_top_hash;
    difficulty_type m_difficulty_for_next_block;
 
    std::string m_validator_key;
    std::vector<std::string> m_validators_public_keys;
 
    boost::asio::io_service m_async_service;
    boost::thread_group m_async_pool;
    std::unique_ptr<boost::asio::io_service::work> m_async_work_idle;
 
    // all alternative chains
    blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info
 
    // some invalid blocks
    blocks_ext_by_hash m_invalid_blocks;     // crypto::hash -> block_extended_info
 
 
    checkpoints m_checkpoints;
    bool m_enforce_dns_checkpoints;
 
    HardFork *m_hardfork;
 
    network_type m_nettype;
    bool m_offline;
    difficulty_type m_fixed_difficulty;
 
    bool m_ignore_bsig;
    bool m_fallback_to_pow;
 
    std::atomic<bool> m_cancel;
 
    // block template cache
    block m_btc;
    account_public_address m_btc_address;
    blobdata m_btc_nonce;
    difficulty_type m_btc_difficulty;
    uint64_t m_btc_height;
    uint64_t m_btc_pool_cookie;
    uint64_t m_btc_expected_reward;
    bool m_btc_valid;
 
 
    bool m_batch_success;
 
    std::shared_ptr<tools::Notify> m_block_notify;
    std::shared_ptr<tools::Notify> m_reorg_notify;
    
    electroneum::basic::Validators* m_validators;
 
    template<class visitor_t>
    inline bool scan_outputkeys_for_indexes(size_t tx_version, const txin_to_key& tx_in_to_key, visitor_t &vis, const crypto::hash &tx_prefix_hash, uint64_t* pmax_related_block_height = NULL) const;
 
    bool check_tx_input(size_t tx_version,const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height);
 
    bool check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height = NULL);
 
    bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::const_iterator>& alt_chain, bool discard_disconnected_chain);
 
    block pop_block_from_blockchain();
 
    bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc);
 
    bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc);
 
    bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc);
 
    bool build_alt_chain(const crypto::hash &prev_id, std::list<blocks_ext_by_hash::const_iterator>& alt_chain, std::vector<uint64_t> &timestamps, block_verification_context& bvc) const;
 
    difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::const_iterator>& alt_chain, block_extended_info& bei) const;
 
    bool prevalidate_miner_transaction(const block& b, uint64_t height);
 
    bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version);
 
    bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height);
 
    void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const;
 
    uint64_t get_long_term_block_weight_median(uint64_t start_height, size_t count) const;
 
    bool is_tx_spendtime_unlocked(uint64_t unlock_time) const;
 
    bool add_block_as_invalid(const block& bl, const crypto::hash& h);
 
    bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h);
 
    bool check_block_timestamp(const block& b, uint64_t& median_ts) const;
    bool check_block_timestamp(const block& b) const { uint64_t median_ts; return check_block_timestamp(b, median_ts); }
 
    bool check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b, uint64_t& median_ts) const;
    bool check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const { uint64_t median_ts; return check_block_timestamp(timestamps, b, median_ts); }
 
    uint64_t get_adjusted_time() const;
 
    bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps) const;
 
    bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL);
    void return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> &txs);
 
    bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const;
 
    void check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image,
        const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature> &sig, uint64_t &result);
 
    void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints);
 
    bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys);
 
    void invalidate_block_template_cache();
 
    void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t pool_cookie);
    
    void sign_block(block& b, std::string privateKey);
 
    bool verify_block_signature(const block& b);
 
  };
}  // namespace cryptonote