# Merged Python Files 
 
#### address.py 
 
import logging
logger = logging.getLogger(__name__)
import struct

import bitcoin
from counterpartylib.lib import config
from counterpartylib.lib import script

def address_scriptpubkey(address):
    try:
        bech32 = bitcoin.bech32.CBech32Data(address)
        return b''.join([b'\x00\x14', bech32.to_bytes()])
    except Exception as e:
        bs58 = bitcoin.base58.decode(address)[1:-4]
        return b''.join([b'\x76\xa9\x14', bs58, b'\x88\xac'])

def pack(address):
    """
    Converts a base58 bitcoin address into a 21 byte bytes object
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support'):
        try:
            bech32 = bitcoin.bech32.CBech32Data(address)
            witver = (0x80 + bech32.witver).to_bytes(1, byteorder='big') # mark the first byte for segwit
            witprog = bech32.to_bytes()
            if len(witprog) > 20:
                raise Exception('p2wsh still not supported for sending')
            return b''.join([witver, witprog])
        except Exception as ne:
            try:
                script.validate(address) #This will check if the address is valid
                short_address_bytes = bitcoin.base58.decode(address)[:-4]
                return short_address_bytes
            except bitcoin.base58.InvalidBase58Error as e:
                raise e
            except Exception as e:
                raise Exception(('The address {} is not a valid bitcoin address ({})').format(address,'testnet' if config.TESTNET or config.REGTEST else 'mainnet'))
    else:
        try:
            short_address_bytes = bitcoin.base58.decode(address)[:-4]
            return short_address_bytes
        except bitcoin.base58.InvalidBase58Error as e:
            raise e

# retuns both the message type id and the remainder of the message data
def unpack(short_address_bytes):
    """
    Converts a 21 byte prefix and public key hash into a full base58 bitcoin address
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support') and short_address_bytes[0] >= 0x80 and short_address_bytes[0] <= 0x8F:
        # we have a segwit address here
        witver = short_address_bytes[0] - 0x80
        witprog = short_address_bytes[1:]
        return str(bitcoin.bech32.CBech32Data.from_bytes(witver, witprog))
    else:
        check = bitcoin.core.Hash(short_address_bytes)[0:4]
        return bitcoin.base58.encode(short_address_bytes + check)
 
 
#### api.py 
 
#! /usr/bin/python3

"""
The database connections are read‐only, so SQL injection attacks can’t be a
problem.
"""

import sys
import os
import threading
import decimal
import time
import json
import re
import requests
import collections
import logging
import traceback
logger = logging.getLogger(__name__)
from logging import handlers as logging_handlers
D = decimal.Decimal
import binascii
import math

import struct
import apsw
import flask
from flask_httpauth import HTTPBasicAuth
import jsonrpc
from jsonrpc import dispatcher
from jsonrpc.exceptions import JSONRPCDispatchException
import inspect
from xmltodict import unparse as serialize_to_xml

from counterpartylib.lib import config
from counterpartylib.lib import exceptions
from counterpartylib.lib import util
from counterpartylib.lib import check
from counterpartylib.lib import backend
from counterpartylib.lib import database
from counterpartylib.lib import transaction
from counterpartylib.lib import blocks
from counterpartylib.lib import script
from counterpartylib.lib import message_type
from counterpartylib.lib.messages import send
from counterpartylib.lib.messages.versions import enhanced_send
from counterpartylib.lib.messages import order
from counterpartylib.lib.messages import btcpay
from counterpartylib.lib.messages import issuance
from counterpartylib.lib.messages import broadcast
from counterpartylib.lib.messages import bet
from counterpartylib.lib.messages import dividend
from counterpartylib.lib.messages import burn
from counterpartylib.lib.messages import destroy
from counterpartylib.lib.messages import cancel
from counterpartylib.lib.messages import rps
from counterpartylib.lib.messages import rpsresolve
from counterpartylib.lib.messages import sweep
from counterpartylib.lib.messages import dispenser

API_TABLES = ['assets', 'balances', 'credits', 'debits', 'bets', 'bet_matches',
              'broadcasts', 'btcpays', 'burns', 'cancels', 'destructions',
              'dividends', 'issuances', 'orders', 'order_matches', 'sends',
              'bet_expirations', 'order_expirations', 'bet_match_expirations',
              'order_match_expirations', 'bet_match_resolutions', 'rps',
              'rpsresolves', 'rps_matches', 'rps_expirations', 'rps_match_expirations',
              'mempool', 'sweeps', 'dispensers', 'dispenses','transactions']

API_TRANSACTIONS = ['bet', 'broadcast', 'btcpay', 'burn', 'cancel', 'destroy',
                    'dividend', 'issuance', 'order', 'send',
                    'rps', 'rpsresolve', 'sweep', 'dispenser']

COMMONS_ARGS = ['encoding', 'fee_per_kb', 'regular_dust_size',
                'multisig_dust_size', 'op_return_value', 'pubkey',
                'allow_unconfirmed_inputs', 'fee', 'fee_provided',
                'estimate_fee_per_kb', 'estimate_fee_per_kb_nblocks', 'estimate_fee_per_kb_conf_target', 'estimate_fee_per_kb_mode',
                'unspent_tx_hash', 'custom_inputs', 'dust_return_pubkey', 'disable_utxo_locks', 'extended_tx_info',
                'p2sh_source_multisig_pubkeys', 'p2sh_source_multisig_pubkeys_required', 'p2sh_pretx_txid']

API_MAX_LOG_SIZE = 10 * 1024 * 1024 #max log size of 20 MB before rotation (make configurable later)
API_MAX_LOG_COUNT = 10
JSON_RPC_ERROR_API_COMPOSE = -32001 #code to use for error composing transaction result

current_api_status_code = None #is updated by the APIStatusPoller
current_api_status_response_json = None #is updated by the APIStatusPoller

class APIError(Exception):
    pass


class BackendError(Exception):
    pass
def check_backend_state():
    """Checks blocktime of last block to see if {} Core is running behind.""".format(config.BTC_NAME)
    block_count = backend.getblockcount()
    block_hash = backend.getblockhash(block_count)
    cblock = backend.getblock(block_hash)
    time_behind = time.time() - cblock.nTime   # TODO: Block times are not very reliable.
    if time_behind > 60 * 60 * 2:   # Two hours.
        raise BackendError('Bitcoind is running about {} hours behind.'.format(round(time_behind / 3600)))

    # check backend index
    blocks_behind = backend.getindexblocksbehind()
    if blocks_behind > 5:
        raise BackendError('Indexd is running {} blocks behind.'.format(blocks_behind))

    logger.debug('Backend state check passed.')

class DatabaseError(Exception):
    pass
def check_database_state(db, blockcount):
    """Checks {} database to see if is caught up with backend.""".format(config.XCP_NAME)
    if util.CURRENT_BLOCK_INDEX + 1 < blockcount:
        raise DatabaseError('{} database is behind backend.'.format(config.XCP_NAME))
    logger.debug('Database state check passed.')
    return


# TODO: ALL queries EVERYWHERE should be done with these methods
def db_query(db, statement, bindings=(), callback=None, **callback_args):
    """Allow direct access to the database in a parametrized manner."""
    cursor = db.cursor()

    # Sanitize.
    forbidden_words = ['pragma', 'attach', 'database', 'begin', 'transaction']
    for word in forbidden_words:
        #This will find if the forbidden word is in the statement as a whole word. For example, "transactions" will be allowed because the "s" at the end
        if re.search(r"\b"+word+"\b", statement.lower()):
            raise APIError("Forbidden word in query: '{}'.".format(word))

    if hasattr(callback, '__call__'):
        cursor.execute(statement, bindings)
        for row in cursor:
            callback(row, **callback_args)
        results = None
    else:
        results = list(cursor.execute(statement, bindings))
    cursor.close()
    return results

def get_rows(db, table, filters=None, filterop='AND', order_by=None, order_dir=None, start_block=None, end_block=None,
              status=None, limit=1000, offset=0, show_expired=True):
    """SELECT * FROM wrapper. Filters results based on a filter data structure (as used by the API)."""

    if filters == None:
        filters = []

    def value_to_marker(value):
        # if value is an array place holder is (?,?,?,..)
        if isinstance(value, list):
            return '''({})'''.format(','.join(['?' for e in range(0, len(value))]))
        else:
            return '''?'''

    # TODO: Document that op can be anything that SQLite3 accepts.
    if not table or table.lower() not in API_TABLES:
        raise APIError('Unknown table')
    if filterop and filterop.upper() not in ['OR', 'AND']:
        raise APIError('Invalid filter operator (OR, AND)')
    if order_dir and order_dir.upper() not in ['ASC', 'DESC']:
        raise APIError('Invalid order direction (ASC, DESC)')
    if not isinstance(limit, int):
        raise APIError('Invalid limit')
    elif config.API_LIMIT_ROWS != 0 and limit > config.API_LIMIT_ROWS:
        raise APIError('Limit should be lower or equal to %i' % config.API_LIMIT_ROWS)
    elif config.API_LIMIT_ROWS != 0 and limit == 0:
        raise APIError('Limit should be greater than 0')
    if not isinstance(offset, int):
        raise APIError('Invalid offset')
    # TODO: accept an object:  {'field1':'ASC', 'field2': 'DESC'}
    if order_by and not re.compile('^[a-z0-9_]+$').match(order_by):
        raise APIError('Invalid order_by, must be a field name')

    if isinstance(filters, dict): #single filter entry, convert to a one entry list
        filters = [filters,]
    elif not isinstance(filters, list):
        filters = []

    # TODO: Document this! (Each filter can be an ordered list.)
    new_filters = []
    for filter_ in filters:
        if type(filter_) in (list, tuple) and len(filter_) in [3, 4]:
            new_filter = {'field': filter_[0], 'op': filter_[1], 'value':  filter_[2]}
            if len(filter_) == 4:
                new_filter['case_sensitive'] = filter_[3]
            new_filters.append(new_filter)
        elif type(filter_) == dict:
            new_filters.append(filter_)
        else:
            raise APIError('Unknown filter type')
    filters = new_filters

    # validate filter(s)
    for filter_ in filters:
        for field in ['field', 'op', 'value']: #should have all fields
            if field not in filter_:
                raise APIError("A specified filter is missing the '%s' field" % field)
        if not isinstance(filter_['value'], (str, int, float, list)):
            raise APIError("Invalid value for the field '%s'" % filter_['field'])
        if isinstance(filter_['value'], list) and filter_['op'].upper() not in ['IN', 'NOT IN']:
            raise APIError("Invalid value for the field '%s'" % filter_['field'])
        if filter_['op'].upper() not in ['=', '==', '!=', '>', '<', '>=', '<=', 'IN', 'LIKE', 'NOT IN', 'NOT LIKE']:
            raise APIError("Invalid operator for the field '%s'" % filter_['field'])
        if 'case_sensitive' in filter_ and not isinstance(filter_['case_sensitive'], bool):
            raise APIError("case_sensitive must be a boolean")

    # special case for memo and memo_hex field searches
    if table == 'sends':
        adjust_get_sends_memo_filters(filters)

    # SELECT
    statement = '''SELECT * FROM {}'''.format(table)
    # WHERE
    bindings = []
    conditions = []
    for filter_ in filters:
        case_sensitive = False if 'case_sensitive' not in filter_ else filter_['case_sensitive']
        if filter_['op'] == 'LIKE' and case_sensitive == False:
            filter_['field'] = '''UPPER({})'''.format(filter_['field'])
            filter_['value'] = filter_['value'].upper()
        marker = value_to_marker(filter_['value'])
        conditions.append('''{} {} {}'''.format(filter_['field'], filter_['op'], marker))
        if isinstance(filter_['value'], list):
            bindings += filter_['value']
        else:
            bindings.append(filter_['value'])
    # AND filters
    more_conditions = []
    if table not in ['balances', 'order_matches', 'bet_matches']:
        if start_block != None:
            more_conditions.append('''block_index >= ?''')
            bindings.append(start_block)
        if end_block != None:
            more_conditions.append('''block_index <= ?''')
            bindings.append(end_block)
    elif table in ['order_matches', 'bet_matches']:
        if start_block != None:
            more_conditions.append('''tx0_block_index >= ?''')
            bindings.append(start_block)
        if end_block != None:
            more_conditions.append('''tx1_block_index <= ?''')
            bindings.append(end_block)

    # status
    if isinstance(status, list) and len(status) > 0:
        more_conditions.append('''status IN {}'''.format(value_to_marker(status)))
        bindings += status
    elif isinstance(status, str) and status != '':
        more_conditions.append('''status == ?''')
        bindings.append(status)

    # legacy filters
    if not show_expired and table == 'orders':
        #Ignore BTC orders one block early.
        expire_index = util.CURRENT_BLOCK_INDEX + 1
        more_conditions.append('''((give_asset == ? AND expire_index > ?) OR give_asset != ?)''')
        bindings += [config.BTC, expire_index, config.BTC]

    if (len(conditions) + len(more_conditions)) > 0:
        statement += ''' WHERE'''
        all_conditions = []
        if len(conditions) > 0:
            all_conditions.append('''({})'''.format(''' {} '''.format(filterop.upper()).join(conditions)))
        if len(more_conditions) > 0:
            all_conditions.append('''({})'''.format(''' AND '''.join(more_conditions)))
        statement += ''' {}'''.format(''' AND '''.join(all_conditions))

    # ORDER BY
    if order_by != None:
        statement += ''' ORDER BY {}'''.format(order_by)
        if order_dir != None:
            statement += ''' {}'''.format(order_dir.upper())
    # LIMIT
    if limit and limit > 0:
        statement += ''' LIMIT {}'''.format(limit)
        if offset:
            statement += ''' OFFSET {}'''.format(offset)


    query_result = db_query(db, statement, tuple(bindings))
    
    if table == 'balances':
        return adjust_get_balances_results(query_result, db)

    if table == 'destructions':
        return adjust_get_destructions_results(query_result)
        
    if table == 'sends':
        # for sends, handle the memo field properly
        return adjust_get_sends_results(query_result)
    
    if table == 'transactions':
        # for transactions, handle the data field properly
        return adjust_get_transactions_results(query_result)    

    return query_result

def adjust_get_balances_results(query_result, db):
    filtered_results = []
    assets = {}
    for balances_row in list(query_result):
        asset = balances_row['asset']
        if not asset in assets:
            assets[asset] = util.is_divisible(db, asset)

        balances_row['divisible'] = assets[asset]
        filtered_results.append(balances_row)

    return filtered_results

def adjust_get_destructions_results(query_result):
    filtered_results = []
    for destruction_row in list(query_result):
        if type(destruction_row['tag']) == bytes:
            destruction_row['tag'] = destruction_row['tag'].decode('utf-8', 'ignore')

        filtered_results.append(destruction_row)

    return filtered_results

def adjust_get_sends_memo_filters(filters):
    """Convert memo to a byte string.  If memo_hex is supplied, attempt to decode it and use that instead."""
    for filter_ in filters:
        if filter_['field'] == 'memo':
            filter_['value'] = bytes(filter_['value'], 'utf-8')
        if filter_['field'] == 'memo_hex':
            # search the indexed memo field with a byte string
            filter_['field'] = 'memo'
            try:
                filter_['value'] = bytes.fromhex(filter_['value'])
            except ValueError as e:
                raise APIError("Invalid memo_hex value")

def adjust_get_sends_results(query_result):
    """Format the memo_hex field.  Try and decode the memo from a utf-8 uncoded string. Invalid utf-8 strings return an empty memo."""
    filtered_results = []
    for send_row in list(query_result):
        try:
            if send_row['memo'] is None:
                send_row['memo_hex'] = None
                send_row['memo'] = None
            else:
                if type(send_row['memo']) == str:
                    send_row['memo'] = bytes(send_row['memo'], 'utf-8')
                    
                send_row['memo_hex'] = binascii.hexlify(send_row['memo']).decode('utf8')
                send_row['memo'] = send_row['memo'].decode('utf-8')
        except UnicodeDecodeError:
            send_row['memo'] = ''
        filtered_results.append(send_row)
    return filtered_results

def adjust_get_transactions_results(query_result):
    """Format the data field.  Try and decode the data from a utf-8 uncoded string. Invalid utf-8 strings return an empty data."""
    filtered_results = []
    for transaction_row in list(query_result):
        transaction_row['data'] = transaction_row['data'].hex()
        filtered_results.append(transaction_row)
    return filtered_results

def compose_transaction(db, name, params,
                        encoding='auto',
                        fee_per_kb=None,
                        estimate_fee_per_kb=None, estimate_fee_per_kb_conf_target=config.ESTIMATE_FEE_CONF_TARGET, estimate_fee_per_kb_mode=config.ESTIMATE_FEE_MODE,
                        regular_dust_size=config.DEFAULT_REGULAR_DUST_SIZE,
                        multisig_dust_size=config.DEFAULT_MULTISIG_DUST_SIZE,
                        op_return_value=config.DEFAULT_OP_RETURN_VALUE,
                        pubkey=None,
                        allow_unconfirmed_inputs=False,
                        fee=None,
                        fee_provided=0,
                        unspent_tx_hash=None, custom_inputs=None, dust_return_pubkey=None, disable_utxo_locks=False, extended_tx_info=False,
                        p2sh_source_multisig_pubkeys=None, p2sh_source_multisig_pubkeys_required=None,
                        p2sh_pretx_txid=None, old_style_api=True, segwit=False):
    """Create and return a transaction."""

    # Get provided pubkeys.
    if type(pubkey) == str:
        provided_pubkeys = [pubkey]
    elif type(pubkey) == list:
        provided_pubkeys = pubkey
    elif pubkey == None:
        provided_pubkeys = []
    else:
        assert False

    # Get additional pubkeys from `source` and `destination` params.
    # Convert `source` and `destination` to pubkeyhash form.
    for address_name in ['source', 'destination']:
        if address_name in params:
            address = params[address_name]
            if isinstance(address, list):
                #pkhshs = []
                #for addr in address:
                #    provided_pubkeys += script.extract_pubkeys(addr)
                #    pkhshs.append(script.make_pubkeyhash(addr))
                #params[address_name] = pkhshs
                pass
            else:
                provided_pubkeys += script.extract_pubkeys(address)
                params[address_name] = script.make_pubkeyhash(address)

    # Check validity of collected pubkeys.
    for pubkey in provided_pubkeys:
        if not script.is_fully_valid(binascii.unhexlify(pubkey)):
            raise script.AddressError('invalid public key: {}'.format(pubkey))

    compose_method = sys.modules['counterpartylib.lib.messages.{}'.format(name)].compose
    compose_params = inspect.getargspec(compose_method)[0]
    missing_params = [p for p in compose_params if p not in params and p != 'db']
    for param in missing_params:
        params[param] = None

    # dont override fee_per_kb if specified
    if fee_per_kb is not None:
        estimate_fee_per_kb = False
    else:
        fee_per_kb = config.DEFAULT_FEE_PER_KB

    if 'extended_tx_info' in params:
      extended_tx_info = params['extended_tx_info']
      del params['extended_tx_info']

    if 'old_style_api' in params:
        old_style_api = params['old_style_api']
        del params['old_style_api']

    if 'segwit' in params:
        segwit = params['segwit']
        del params['segwit']

    tx_info = compose_method(db, **params)
    return transaction.construct(db, tx_info, encoding=encoding,
                                        fee_per_kb=fee_per_kb,
                                        estimate_fee_per_kb=estimate_fee_per_kb, estimate_fee_per_kb_conf_target=estimate_fee_per_kb_conf_target,
                                        regular_dust_size=regular_dust_size,
                                        multisig_dust_size=multisig_dust_size,
                                        op_return_value=op_return_value,
                                        provided_pubkeys=provided_pubkeys,
                                        allow_unconfirmed_inputs=allow_unconfirmed_inputs,
                                        exact_fee=fee,
                                        fee_provided=fee_provided,
                                        unspent_tx_hash=unspent_tx_hash, custom_inputs=custom_inputs,
                                        dust_return_pubkey=dust_return_pubkey,
                                        disable_utxo_locks=disable_utxo_locks,
                                        extended_tx_info=extended_tx_info,
                                        p2sh_source_multisig_pubkeys=p2sh_source_multisig_pubkeys, p2sh_source_multisig_pubkeys_required=p2sh_source_multisig_pubkeys_required,
                                        p2sh_pretx_txid=p2sh_pretx_txid,
                                        old_style_api=old_style_api,
                                        segwit=segwit)

def conditional_decorator(decorator, condition):
    """Checks the condition and if True applies specified decorator."""
    def gen_decorator(f):
        if not condition:
            return f
        return decorator(f)
    return gen_decorator

def init_api_access_log(app):
    """Initialize API logger."""
    loggers = (logging.getLogger('werkzeug'), app.logger)

    # Disable console logging...
    for l in loggers:
        l.setLevel(logging.INFO)
        l.propagate = False

    # Log to file, if configured...
    if config.API_LOG:
        handler = logging_handlers.RotatingFileHandler(config.API_LOG, 'a', API_MAX_LOG_SIZE, API_MAX_LOG_COUNT)
        for l in loggers:
            l.addHandler(handler)

class APIStatusPoller(threading.Thread):
    """Perform regular checks on the state of the backend and the database."""
    def __init__(self):
        self.last_database_check = 0
        threading.Thread.__init__(self)
        self.stop_event = threading.Event()

    def stop(self):
        self.stop_event.set()

    def run(self):
        logger.debug('Starting API Status Poller.')
        global current_api_status_code, current_api_status_response_json
        db = database.get_connection(read_only=True, integrity_check=False)

        while self.stop_event.is_set() != True:
            try:
                # Check that backend is running, communicable, and caught up with the blockchain.
                # Check that the database has caught up with bitcoind.
                if time.time() - self.last_database_check > 10 * 60: # Ten minutes since last check.
                    if not config.FORCE:
                        code = 11
                        logger.debug('Checking backend state.')
                        check_backend_state()
                        code = 12
                        logger.debug('Checking database state.')
                        check_database_state(db, backend.getblockcount())
                        self.last_database_check = time.time()
            except (BackendError, DatabaseError) as e:
                exception_name = e.__class__.__name__
                exception_text = str(e)
                logger.debug("API Status Poller: %s", exception_text)
                jsonrpc_response = jsonrpc.exceptions.JSONRPCServerError(message=exception_name, data=exception_text)
                current_api_status_code = code
                current_api_status_response_json = jsonrpc_response.json.encode()
            else:
                current_api_status_code = None
                current_api_status_response_json = None
            time.sleep(config.BACKEND_POLL_INTERVAL)

class APIServer(threading.Thread):
    """Handle JSON-RPC API calls."""
    def __init__(self, db=None):
        self.db = db
        self.is_ready = False
        threading.Thread.__init__(self)
        self.stop_event = threading.Event()

    def stop(self):
        self.join()
        self.stop_event.set()

    def run(self):
        logger.info('Starting API Server.')
        self.db = self.db or database.get_connection(read_only=True, integrity_check=False)
        app = flask.Flask(__name__)
        auth = HTTPBasicAuth()

        @auth.get_password
        def get_pw(username):
            if username == config.RPC_USER:
                return config.RPC_PASSWORD
            return None

        ######################
        #READ API

        # Generate dynamically get_{table} methods
        def generate_get_method(table):
            def get_method(**kwargs):
                try:
                    return get_rows(self.db, table=table, **kwargs)
                except TypeError as e:          #TODO: generalise for all API methods
                    raise APIError(str(e))
            return get_method

        for table in API_TABLES:
            new_method = generate_get_method(table)
            new_method.__name__ = 'get_{}'.format(table)
            dispatcher.add_method(new_method)

        @dispatcher.add_method
        def sql(query, bindings=None):
            if bindings == None:
                bindings = []
            return db_query(self.db, query, tuple(bindings))


        ######################
        #WRITE/ACTION API

        # Generate dynamically create_{transaction} methods
        def generate_create_method(tx):

            def split_params(**kwargs):
                transaction_args = {}
                common_args = {}
                private_key_wif = None
                for key in kwargs:
                    if key in COMMONS_ARGS:
                        common_args[key] = kwargs[key]
                    elif key == 'privkey':
                        private_key_wif = kwargs[key]
                    else:
                        transaction_args[key] = kwargs[key]
                return transaction_args, common_args, private_key_wif

            def create_method(**kwargs):
                try:
                    transaction_args, common_args, private_key_wif = split_params(**kwargs)
                    return compose_transaction(self.db, name=tx, params=transaction_args, **common_args)
                except (TypeError, script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error:
                    # TypeError happens when unexpected keyword arguments are passed in
                    error_msg = "Error composing {} transaction via API: {}".format(tx, str(error))
                    logging.warning(error_msg)
                    logging.warning(traceback.format_exc())
                    raise JSONRPCDispatchException(code=JSON_RPC_ERROR_API_COMPOSE, message=error_msg)

            return create_method

        for tx in API_TRANSACTIONS:
            create_method = generate_create_method(tx)
            create_method.__name__ = 'create_{}'.format(tx)
            dispatcher.add_method(create_method)

        @dispatcher.add_method
        def get_messages(block_index):
            if not isinstance(block_index, int):
                raise APIError("block_index must be an integer.")

            cursor = self.db.cursor()
            cursor.execute('select * from messages where block_index = ? order by message_index asc', (block_index,))
            messages = cursor.fetchall()
            cursor.close()
            return messages

        @dispatcher.add_method
        def get_messages_by_index(message_indexes):
            """Get specific messages from the feed, based on the message_index.

            @param message_index: A single index, or a list of one or more message indexes to retrieve.
            """
            if not isinstance(message_indexes, list):
                message_indexes = [message_indexes,]
            for idx in message_indexes:  #make sure the data is clean
                if not isinstance(idx, int):
                    raise APIError("All items in message_indexes are not integers")

            cursor = self.db.cursor()
            cursor.execute('SELECT * FROM messages WHERE message_index IN (%s) ORDER BY message_index ASC'
                % (','.join([str(x) for x in message_indexes]),))
            messages = cursor.fetchall()
            cursor.close()
            return messages

        @dispatcher.add_method
        def get_supply(asset):
            if asset == 'BTC':
                return  backend.get_btc_supply(normalize=False)
            elif asset == 'XCP':
                return util.xcp_supply(self.db)
            else:
                asset = util.resolve_subasset_longname(self.db, asset)
                return util.asset_supply(self.db, asset)

        @dispatcher.add_method
        def get_xcp_supply():
            logger.warning("Deprecated method: `get_xcp_supply`")
            return util.xcp_supply(self.db)

        @dispatcher.add_method
        def get_asset_info(assets=None, asset=None):
            if asset is not None:
                assets = [asset]
            
            if not isinstance(assets, list):
                raise APIError("assets must be a list of asset names, even if it just contains one entry")
            assetsInfo = []
            for asset in assets:
                asset = util.resolve_subasset_longname(self.db, asset)

                # BTC and XCP.
                if asset in [config.BTC, config.XCP]:
                    if asset == config.BTC:
                        supply = backend.get_btc_supply(normalize=False)
                    else:
                        supply = util.xcp_supply(self.db)

                    assetsInfo.append({
                        'asset': asset,
                        'asset_longname': None,
                        'owner': None,
                        'divisible': True,
                        'locked': False,
                        'supply': supply,
                        'description': '',
                        'issuer': None
                    })
                    continue

                # User‐created asset.
                cursor = self.db.cursor()
                issuances = list(cursor.execute('''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY block_index ASC''', ('valid', asset)))
                cursor.close()
                if not issuances:
                    continue #asset not found, most likely
                else:
                    last_issuance = issuances[-1]
                locked = False
                for e in issuances:
                    if e['locked']: locked = True
                assetsInfo.append({
                    'asset': asset,
                    'asset_longname': last_issuance['asset_longname'],
                    'owner': last_issuance['issuer'],
                    'divisible': bool(last_issuance['divisible']),
                    'locked': locked,
                    'supply': util.asset_supply(self.db, asset),
                    'description': last_issuance['description'],
                    'issuer': last_issuance['issuer']})
            return assetsInfo

        @dispatcher.add_method
        def get_block_info(block_index):
            assert isinstance(block_index, int)
            cursor = self.db.cursor()
            cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,))
            blocks = list(cursor)
            if len(blocks) == 1:
                block = blocks[0]
            elif len(blocks) == 0:
                raise exceptions.DatabaseError('No blocks found.')
            else:
                assert False
            cursor.close()
            return block

        @dispatcher.add_method
        def fee_per_kb(conf_target=config.ESTIMATE_FEE_CONF_TARGET, mode=config.ESTIMATE_FEE_MODE):
            return backend.fee_per_kb(conf_target, mode)

        @dispatcher.add_method
        def get_blocks(block_indexes, min_message_index=None):
            """fetches block info and messages for the specified block indexes
            @param min_message_index: Retrieve blocks from the message feed on or after this specific message index
              (useful since blocks may appear in the message feed more than once, if a reorg occurred). Note that
              if this parameter is not specified, the messages for the first block will be returned.
            """
            if not isinstance(block_indexes, (list, tuple)):
                raise APIError("block_indexes must be a list of integers.")
            if len(block_indexes) >= 250:
                raise APIError("can only specify up to 250 indexes at a time.")

            block_indexes_str = ','.join([str(x) for x in block_indexes])
            cursor = self.db.cursor()

            # The blocks table gets rolled back from undolog, so min_message_index doesn't matter for this query
            cursor.execute('SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC'
                % (block_indexes_str,))
            blocks = cursor.fetchall()

            cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY message_index ASC'
                % (block_indexes_str,))
            messages = collections.deque(cursor.fetchall())

            # Discard any messages less than min_message_index
            if min_message_index:
                while len(messages) and messages[0]['message_index'] < min_message_index:
                    messages.popleft()

            # Packages messages into their appropriate block in the data structure to be returned
            for block in blocks:
                block['_messages'] = []
                while len(messages) and messages[0]['block_index'] == block['block_index']:
                    block['_messages'].append(messages.popleft())
            #NOTE: if len(messages), then we're only returning the messages for the first set of blocks before the reorg

            cursor.close()
            return blocks

        @dispatcher.add_method
        def get_running_info():
            latestBlockIndex = backend.getblockcount()

            try:
                check_database_state(self.db, latestBlockIndex)
            except DatabaseError:
                caught_up = False
            else:
                caught_up = True

            try:
                cursor = self.db.cursor()
                blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (util.CURRENT_BLOCK_INDEX, )))
                assert len(blocks) == 1
                last_block = blocks[0]
                cursor.close()
            except:
                last_block = None

            try:
                last_message = util.last_message(self.db)
            except:
                last_message = None

            try:
                indexd_blocks_behind = backend.getindexblocksbehind()
            except:
                indexd_blocks_behind = latestBlockIndex if latestBlockIndex > 0 else 999999
            indexd_caught_up = indexd_blocks_behind <= 1

            server_ready = caught_up and indexd_caught_up

            return {
                'server_ready': server_ready,
                'db_caught_up': caught_up,
                'bitcoin_block_count': latestBlockIndex,
                'last_block': last_block,
                'indexd_caught_up': indexd_caught_up,
                'indexd_blocks_behind': indexd_blocks_behind,
                'last_message_index': last_message['message_index'] if last_message else -1,
                'api_limit_rows': config.API_LIMIT_ROWS,
                'running_testnet': config.TESTNET,
                'running_regtest': config.REGTEST,
                'running_testcoin': config.TESTCOIN,
                'version_major': config.VERSION_MAJOR,
                'version_minor': config.VERSION_MINOR,
                'version_revision': config.VERSION_REVISION
            }

        @dispatcher.add_method
        def get_element_counts():
            counts = {}
            cursor = self.db.cursor()
            for element in ['transactions', 'blocks', 'debits', 'credits', 'balances', 'sends', 'orders',
                'order_matches', 'btcpays', 'issuances', 'broadcasts', 'bets', 'bet_matches', 'dividends',
                'burns', 'cancels', 'order_expirations', 'bet_expirations', 'order_match_expirations',
                'bet_match_expirations', 'messages', 'destructions']:
                cursor.execute("SELECT COUNT(*) AS count FROM %s" % element)
                count_list = cursor.fetchall()
                assert len(count_list) == 1
                counts[element] = count_list[0]['count']
            cursor.close()
            return counts

        @dispatcher.add_method
        def get_asset_names(longnames=False):
            cursor = self.db.cursor()
            if longnames:
                names = []
                for row in cursor.execute("SELECT asset, asset_longname FROM issuances WHERE status = 'valid' GROUP BY asset ORDER BY asset ASC"):
                    names.append({'asset': row['asset'], 'asset_longname': row['asset_longname']})
            else:
                names = [row['asset'] for row in cursor.execute("SELECT DISTINCT asset FROM issuances WHERE status = 'valid' ORDER BY asset ASC")]
            cursor.close()
            return names

        @dispatcher.add_method
        def get_asset_longnames():
            return get_asset_names(longnames=True)

        @dispatcher.add_method
        def get_holder_count(asset):
            asset = util.resolve_subasset_longname(self.db, asset)
            holders = util.holders(self.db, asset, True)
            addresses = []
            for holder in holders:
                addresses.append(holder['address'])
            return {asset: len(set(addresses))}

        @dispatcher.add_method
        def get_holders(asset):
            asset = util.resolve_subasset_longname(self.db, asset)
            holders = util.holders(self.db, asset, True)
            return holders

        @dispatcher.add_method
        def search_raw_transactions(address, unconfirmed=True):
            return backend.search_raw_transactions(address, unconfirmed=unconfirmed)

        @dispatcher.add_method
        def get_unspent_txouts(address, unconfirmed=False, unspent_tx_hash=None, order_by=None):
            results = backend.get_unspent_txouts(address, unconfirmed=unconfirmed, unspent_tx_hash=unspent_tx_hash)
            if order_by is None:
                return results
            else:
                order_key = order_by
                reverse = False
                if order_key.startswith('-'):
                    order_key = order_key[1:]
                    reverse = True
                return sorted(results, key=lambda x: x[order_key], reverse=reverse)


        @dispatcher.add_method
        def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
            return backend.getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)

        @dispatcher.add_method
        def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
            return backend.getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)

        @dispatcher.add_method
        def get_tx_info(tx_hex, block_index=None):
            # block_index mandatory for transactions before block 335000
            source, destination, btc_amount, fee, data, extra = blocks.get_tx_info(tx_hex, block_index=block_index)
            return source, destination, btc_amount, fee, util.hexlify(data) if data else ''

        @dispatcher.add_method
        def unpack(data_hex):
            data = binascii.unhexlify(data_hex)
            message_type_id, message = message_type.unpack(data)

            # TODO: Enabled only for `send`.
            if message_type_id == send.ID:
                unpack_method = send.unpack
            elif message_type_id == enhanced_send.ID:
                unpack_method = enhanced_send.unpack
            else:
                raise APIError('unsupported message type')
            unpacked = unpack_method(self.db, message, util.CURRENT_BLOCK_INDEX)
            return message_type_id, unpacked

        @dispatcher.add_method
        # TODO: Rename this method.
        def search_pubkey(pubkeyhash, provided_pubkeys=None):
            return backend.pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys=provided_pubkeys)

        @dispatcher.add_method
        def get_dispenser_info(tx_hash=None, tx_index=None):
            cursor = self.db.cursor()
            
            if tx_hash is None and tx_index is None:
                raise APIError("You must provided a tx hash or a tx index")
            
            if tx_hash is not None:
                cursor.execute('SELECT d.*, a.asset_longname FROM dispensers d LEFT JOIN assets a ON a.asset_name = d.asset WHERE tx_hash=:tx_hash', {"tx_hash":tx_hash})
            else:
                cursor.execute('SELECT d.*, a.asset_longname FROM dispensers d LEFT JOIN assets a ON a.asset_name = d.asset WHERE tx_index=:tx_index', {"tx_index":tx_index})
            
            dispensers = cursor.fetchall()
            
            if len(dispensers) == 1:
                dispenser = dispensers[0]
                oracle_price = ""
                satoshi_price = ""
                fiat_price = ""
                oracle_price_last_updated = ""
                oracle_fiat_label = ""
                
                if dispenser["oracle_address"] != None:
                    fiat_price = util.satoshirate_to_fiat(dispenser["satoshirate"])
                    oracle_price, oracle_fee, oracle_fiat_label, oracle_price_last_updated = util.get_oracle_last_price(self.db, dispenser["oracle_address"], util.CURRENT_BLOCK_INDEX)
                    
                    if (oracle_price > 0):
                        satoshi_price = math.ceil((fiat_price/oracle_price) * config.UNIT)
                    else:
                        raise APIError("Last oracle price is zero")
                
                return {
                    "tx_index": dispenser["tx_index"],
                    "tx_hash": dispenser["tx_hash"],
                    "block_index": dispenser["block_index"],
                    "source": dispenser["source"],
                    "asset": dispenser["asset"],
                    "give_quantity": dispenser["give_quantity"],
                    "escrow_quantity": dispenser["escrow_quantity"],
                    "mainchainrate": dispenser["satoshirate"],
                    "fiat_price": fiat_price,
                    "fiat_unit": oracle_fiat_label,
                    "oracle_price": oracle_price,
                    "satoshi_price": satoshi_price,
                    "status": dispenser["status"],
                    "give_remaining": dispenser["give_remaining"],
                    "oracle_address": dispenser["oracle_address"],
                    "oracle_price_last_updated": oracle_price_last_updated,
                    "asset_longname": dispenser["asset_longname"]
                }
            
            return {}

        

        def _set_cors_headers(response):
            if not config.RPC_NO_ALLOW_CORS:
                response.headers['Access-Control-Allow-Origin'] = '*'
                response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
                response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'

        @app.route('/healthz', methods=['GET'])
        def handle_healthz():
            msg, code = 'Healthy', 200
            try:
                latestBlockIndex = backend.getblockcount()
                check_database_state(self.db, latestBlockIndex)
            except DatabaseError:
                msg, code = 'Unhealthy', 503
            return flask.Response(msg, code, mimetype='text/plain')

        @app.route('/', defaults={'args_path': ''}, methods=['GET', 'POST', 'OPTIONS'])
        @app.route('/<path:args_path>',  methods=['GET', 'POST', 'OPTIONS'])
        # Only require authentication if RPC_PASSWORD is set.
        @conditional_decorator(auth.login_required, hasattr(config, 'RPC_PASSWORD'))
        def handle_root(args_path):
            """Handle all paths, decide where to forward the query."""
            if args_path == '' or args_path.startswith('api/') or args_path.startswith('API/') or \
               args_path.startswith('rpc/') or args_path.startswith('RPC/'):
                if flask.request.method == 'POST':
                    # Need to get those here because it might not be available in this aux function.
                    request_json = flask.request.get_data().decode('utf-8')
                    response = handle_rpc_post(request_json)
                    return response
                elif flask.request.method == 'OPTIONS':
                    response = handle_rpc_options()
                    return response
                else:
                    error = 'Invalid method.'
                    return flask.Response(error, 405, mimetype='application/json')
            elif args_path.startswith('rest/') or args_path.startswith('REST/'):
                if flask.request.method == 'GET' or flask.request.method == 'POST':
                    # Pass the URL path without /REST/ part and Flask request object.
                    rest_path = args_path.split('/', 1)[1]
                    response = handle_rest(rest_path, flask.request)
                    return response
                else:
                    error = 'Invalid method.'
                    return flask.Response(error, 405, mimetype='application/json')
            else:
                # Not found
                return flask.Response(None, 404, mimetype='application/json')

        ######################
        # JSON-RPC API
        ######################
        def handle_rpc_options():
            response = flask.Response('', 204)
            _set_cors_headers(response)
            return response

        def handle_rpc_post(request_json):
            """Handle /API/ POST route. Call relevant get_rows/create_transaction wrapper."""
            # Check for valid request format.
            try:
                request_data = json.loads(request_json)
                assert 'id' in request_data and request_data['jsonrpc'] == "2.0" and request_data['method']
                # params may be omitted
            except:
                obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(data="Invalid JSON-RPC 2.0 request format")
                return flask.Response(obj_error.json.encode(), 400, mimetype='application/json')

            # Only arguments passed as a `dict` are supported.
            if request_data.get('params', None) and not isinstance(request_data['params'], dict):
                obj_error = jsonrpc.exceptions.JSONRPCInvalidRequest(
                    data='Arguments must be passed as a JSON object (list of unnamed arguments not supported)')
                return flask.Response(obj_error.json.encode(), 400, mimetype='application/json')

            # Return an error if the API Status Poller checks fail.
            if not config.FORCE and current_api_status_code:
                return flask.Response(current_api_status_response_json, 503, mimetype='application/json')

            # Answer request normally.
            # NOTE: `UnboundLocalError: local variable 'output' referenced before assignment` means the method doesn’t return anything.
            jsonrpc_response = jsonrpc.JSONRPCResponseManager.handle(request_json, dispatcher)
            response = flask.Response(jsonrpc_response.json.encode(), 200, mimetype='application/json')
            _set_cors_headers(response)
            return response

        ######################
        # HTTP REST API
        ######################
        def handle_rest(path_args, flask_request):
            """Handle /REST/ route. Query the database using get_rows or create transaction using compose_transaction."""
            url_action = flask_request.path.split('/')[-1]
            if url_action == 'compose':
                compose = True
            elif url_action == 'get':
                compose = False
            else:
                error = 'Invalid action "%s".' % url_action
                return flask.Response(error, 400, mimetype='application/json')

            # Get all arguments passed via URL.
            url_args = path_args.split('/')
            try:
                query_type = url_args.pop(0).lower()
            except IndexError:
                error = 'No query_type provided.'
                return flask.Response(error, 400, mimetype='application/json')
            # Check if message type or table name are valid.
            if (compose and query_type not in API_TRANSACTIONS) or \
               (not compose and query_type not in API_TABLES):
                error = 'No such query type in supported queries: "%s".' % query_type
                return flask.Response(error, 400, mimetype='application/json')

            # Parse the additional arguments.
            extra_args = flask_request.args.items()
            query_data = {}

            if compose:
                common_args = {}
                transaction_args = {}
                for (key, value) in extra_args:
                    # Determine value type.
                    try:
                        value = int(value)
                    except ValueError:
                        try:
                            value = float(value)
                        except ValueError:
                            pass
                    # Split keys into common and transaction-specific arguments. Discard the privkey.
                    if key in COMMONS_ARGS:
                        common_args[key] = value
                    elif key == 'privkey':
                        pass
                    else:
                        transaction_args[key] = value

                # Must have some additional transaction arguments.
                if not len(transaction_args):
                    error = 'No transaction arguments provided.'
                    return flask.Response(error, 400, mimetype='application/json')

                # Compose the transaction.
                try:
                    query_data = compose_transaction(self.db, name=query_type, params=transaction_args, **common_args)
                except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error:
                    error_msg = logging.warning("{} -- error composing {} transaction via API: {}".format(
                        str(error.__class__.__name__), query_type, str(error)))
                    return flask.Response(error_msg, 400, mimetype='application/json')
            else:
                # Need to de-generate extra_args to pass it through.
                query_args = dict([item for item in extra_args])
                operator = query_args.pop('op', 'AND')
                # Put the data into specific dictionary format.
                data_filter = [{'field': key, 'op': '==', 'value': value} for (key, value) in query_args.items()]

                # Run the query.
                try:
                    query_data = get_rows(self.db, table=query_type, filters=data_filter, filterop=operator)
                except APIError as error:
                    return flask.Response(str(error), 400, mimetype='application/json')

            # See which encoding to choose from.
            file_format = flask_request.headers['Accept']
            # JSON as default.
            if file_format == 'application/json' or file_format == '*/*':
                response_data = json.dumps(query_data)
            elif file_format == 'application/xml':
                # Add document root for XML. Note when xmltodict encounters a list, it produces separate tags for every item.
                # Hence we end up with multiple query_type roots. To combat this we put it in a separate item dict.
                response_data = serialize_to_xml({query_type: {'item': query_data}})
            else:
                error = 'Invalid file format: "%s".' % file_format
                return flask.Response(error, 400, mimetype='application/json')

            response = flask.Response(response_data, 200, mimetype=file_format)
            return response

        # Init the HTTP Server.
        init_api_access_log(app)

        # Run app server (blocking)
        self.is_ready = True
        app.run(host=config.RPC_HOST, port=config.RPC_PORT, threaded=True)

        self.db.close()
        return

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### arc4.py 
 
import binascii
from Crypto.Cipher import ARC4

def init_arc4(seed):
    if isinstance(seed, str):
        seed = binascii.unhexlify(seed)
    return ARC4.new(seed)
 
 
#### blocks.py 
 
"""
Initialise database.

Sieve blockchain for Counterparty transactions, and add them to the database.
"""

import os
import time
import binascii
import struct
import decimal
D = decimal.Decimal
import logging
logger = logging.getLogger(__name__)
import collections
import platform
import apsw
import csv
import copy
import http

import bitcoin as bitcoinlib
from bitcoin.core.script import CScriptInvalidError

from counterpartylib.lib import config
from counterpartylib.lib import exceptions
from counterpartylib.lib import util
from counterpartylib.lib import check
from counterpartylib.lib import script
from counterpartylib.lib import backend
from counterpartylib.lib import log
from counterpartylib.lib import database
from counterpartylib.lib import message_type
from counterpartylib.lib import arc4
from counterpartylib.lib.transaction_helper import p2sh_encoding

from .messages import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, rps, rpsresolve, destroy, sweep, dispenser)
from .messages.versions import enhanced_send, mpma

from .kickstart.blocks_parser import BlockchainParser, ChainstateParser
from .kickstart.utils import ib2h

from .exceptions import DecodeError, BTCOnlyError

# Order matters for FOREIGN KEY constraints.
TABLES = ['credits', 'debits', 'messages'] + \
         ['bet_match_resolutions', 'order_match_expirations', 'order_matches',
         'order_expirations', 'orders', 'bet_match_expirations', 'bet_matches',
         'bet_expirations', 'bets', 'broadcasts', 'btcpays', 'burns',
         'cancels', 'dividends', 'issuances', 'sends',
         'rps_match_expirations', 'rps_expirations', 'rpsresolves',
         'rps_matches', 'rps',
         'destructions', 'assets', 'addresses', 'sweeps', 'dispensers', 'dispenses']
# Compose list of tables tracked by undolog
UNDOLOG_TABLES = copy.copy(TABLES)
UNDOLOG_TABLES.remove('messages')
UNDOLOG_TABLES += ['balances']

CURR_DIR = os.path.dirname(os.path.realpath(__file__))
with open(CURR_DIR + '/../mainnet_burns.csv', 'r') as f:
    mainnet_burns_reader = csv.DictReader(f)
    MAINNET_BURNS = {}
    for line in mainnet_burns_reader:
        MAINNET_BURNS[line['tx_hash']] = line

def parse_tx(db, tx):
    """Parse the transaction, return True for success."""
    cursor = db.cursor()

    try:
        with db:
            # Only one source and one destination allowed for now.
            if len(tx['source'].split('-')) > 1:
                return
            if tx['destination']:
                if len(tx['destination'].split('-')) > 1:
                    return

            # Burns.
            if tx['destination'] == config.UNSPENDABLE:
                burn.parse(db, tx, MAINNET_BURNS)
                return

            if len(tx['data']) > 1:
                try:
                    message_type_id, message = message_type.unpack(tx['data'], tx['block_index'])
                except struct.error:    # Deterministically raised.
                    message_type_id = None
                    message = None
            else:
                message_type_id = None
                message = None

            # Protocol change.
            rps_enabled = tx['block_index'] >= 308500 or config.TESTNET or config.REGTEST

            if message_type_id == send.ID:
                send.parse(db, tx, message)
            elif message_type_id == enhanced_send.ID and util.enabled('enhanced_sends', block_index=tx['block_index']):
                enhanced_send.parse(db, tx, message)
            elif message_type_id == mpma.ID and util.enabled('mpma_sends', block_index=tx['block_index']):
                mpma.parse(db, tx, message)
            elif message_type_id == order.ID:
                order.parse(db, tx, message)
            elif message_type_id == btcpay.ID:
                btcpay.parse(db, tx, message)
            elif message_type_id == issuance.ID:
                issuance.parse(db, tx, message, message_type_id)
            elif message_type_id == issuance.SUBASSET_ID and util.enabled('subassets', block_index=tx['block_index']):
                issuance.parse(db, tx, message, message_type_id)
            elif message_type_id == broadcast.ID:
                broadcast.parse(db, tx, message)
            elif message_type_id == bet.ID:
                bet.parse(db, tx, message)
            elif message_type_id == dividend.ID:
                dividend.parse(db, tx, message)
            elif message_type_id == cancel.ID:
                cancel.parse(db, tx, message)
            elif message_type_id == rps.ID and rps_enabled:
                rps.parse(db, tx, message)
            elif message_type_id == rpsresolve.ID and rps_enabled:
                rpsresolve.parse(db, tx, message)
            elif message_type_id == destroy.ID and util.enabled('destroy_reactivated', block_index=tx['block_index']):
                destroy.parse(db, tx, message)
            elif message_type_id == sweep.ID and util.enabled('sweep_send', block_index=tx['block_index']):
                sweep.parse(db, tx, message)
            elif message_type_id == dispenser.ID and util.enabled('dispensers', block_index=tx['block_index']):
                dispenser.parse(db, tx, message)
            elif message_type_id == dispenser.DISPENSE_ID and util.enabled('dispensers', block_index=tx['block_index']):
                dispenser.dispense(db, tx)
            else:
                cursor.execute('''UPDATE transactions \
                                           SET supported=? \
                                           WHERE tx_hash=?''',
                                        (False, tx['tx_hash']))
                if tx['block_index'] != config.MEMPOOL_BLOCK_INDEX:
                    logger.info('Unsupported transaction: hash {}; data {}'.format(tx['tx_hash'], tx['data']))
                cursor.close()
                return False

            # NOTE: for debugging (check asset conservation after every `N` transactions).
            # if not tx['tx_index'] % N:
            #     check.asset_conservation(db)

            return True
    except Exception as e:
        raise exceptions.ParseTransactionError("%s" % e)
    finally:
        cursor.close()


def parse_block(db, block_index, block_time,
                previous_ledger_hash=None, ledger_hash=None,
                previous_txlist_hash=None, txlist_hash=None,
                previous_messages_hash=None):
    """Parse the block, return hash of new ledger, txlist and messages.

    The unused arguments `ledger_hash` and `txlist_hash` are for the test suite.
    """
    undolog_cursor = db.cursor()
    #remove the row tracer and exec tracer on this cursor, so we don't utilize them with undolog operations...
    undolog_cursor.setexectrace(None)
    undolog_cursor.setrowtrace(None)

    util.BLOCK_LEDGER = []
    database.BLOCK_MESSAGES = []

    assert block_index == util.CURRENT_BLOCK_INDEX

    # Remove undolog records for any block older than we should be tracking
    undolog_oldest_block_index = block_index - config.UNDOLOG_MAX_PAST_BLOCKS
    first_undo_index = list(undolog_cursor.execute('''SELECT first_undo_index FROM undolog_block WHERE block_index == ?''',
        (undolog_oldest_block_index,)))
    if len(first_undo_index) == 1 and first_undo_index[0] is not None:
        undolog_cursor.execute('''DELETE FROM undolog WHERE undo_index < ?''', (first_undo_index[0][0],))
    undolog_cursor.execute('''DELETE FROM undolog_block WHERE block_index < ?''',
        (undolog_oldest_block_index,))

    # Set undolog barrier for this block
    if block_index != config.BLOCK_FIRST:
        undolog_cursor.execute('''INSERT OR REPLACE INTO undolog_block(block_index, first_undo_index)
            SELECT ?, seq+1 FROM SQLITE_SEQUENCE WHERE name='undolog' ''', (block_index,))
    else:
        undolog_cursor.execute('''INSERT OR REPLACE INTO undolog_block(block_index, first_undo_index)
            VALUES(?,?)''', (block_index, 1,))
    undolog_cursor.close()

    # Expire orders, bets and rps.
    order.expire(db, block_index)
    bet.expire(db, block_index, block_time)
    rps.expire(db, block_index)

    # Parse transactions, sorting them by type.
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM transactions \
                      WHERE block_index=? ORDER BY tx_index''',
                   (block_index,))
    txlist = []
    for tx in list(cursor):
        try:
            parse_tx(db, tx)
            txlist.append('{}{}{}{}{}{}'.format(tx['tx_hash'], tx['source'], tx['destination'],
                                                tx['btc_amount'], tx['fee'],
                                                binascii.hexlify(tx['data']).decode('UTF-8')))
        except exceptions.ParseTransactionError as e:
            logger.warn('ParseTransactionError for tx %s: %s' % (tx['tx_hash'], e))
            raise e
            #pass

    cursor.close()

    # Calculate consensus hashes.
    new_txlist_hash, found_txlist_hash = check.consensus_hash(db, 'txlist_hash', previous_txlist_hash, txlist)
    new_ledger_hash, found_ledger_hash = check.consensus_hash(db, 'ledger_hash', previous_ledger_hash, util.BLOCK_LEDGER)
    new_messages_hash, found_messages_hash = check.consensus_hash(db, 'messages_hash', previous_messages_hash, database.BLOCK_MESSAGES)

    return new_ledger_hash, new_txlist_hash, new_messages_hash, found_messages_hash


def initialise(db):
    """Initialise data, create and populate the database."""
    cursor = db.cursor()

    # Blocks
    cursor.execute('''CREATE TABLE IF NOT EXISTS blocks(
                      block_index INTEGER UNIQUE,
                      block_hash TEXT UNIQUE,
                      block_time INTEGER,
                      previous_block_hash TEXT UNIQUE,
                      difficulty INTEGER,
                      PRIMARY KEY (block_index, block_hash))
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      block_index_idx ON blocks (block_index)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      index_hash_idx ON blocks (block_index, block_hash)
                   ''')

    # SQLite can’t do `ALTER TABLE IF COLUMN NOT EXISTS`.
    columns = [column['name'] for column in cursor.execute('''PRAGMA table_info(blocks)''')]
    if 'ledger_hash' not in columns:
        cursor.execute('''ALTER TABLE blocks ADD COLUMN ledger_hash TEXT''')
    if 'txlist_hash' not in columns:
        cursor.execute('''ALTER TABLE blocks ADD COLUMN txlist_hash TEXT''')
    if 'messages_hash' not in columns:
        cursor.execute('''ALTER TABLE blocks ADD COLUMN messages_hash TEXT''')
    if 'previous_block_hash' not in columns:
        cursor.execute('''ALTER TABLE blocks ADD COLUMN previous_block_hash TEXT''')
    if 'difficulty' not in columns:
        cursor.execute('''ALTER TABLE blocks ADD COLUMN difficulty TEXT''')

    # Check that first block in DB is BLOCK_FIRST.
    cursor.execute('''SELECT * from blocks ORDER BY block_index''')
    blocks = list(cursor)
    if len(blocks):
        if blocks[0]['block_index'] != config.BLOCK_FIRST:
            raise exceptions.DatabaseError('First block in database is not block {}.'.format(config.BLOCK_FIRST))

    # Transactions
    cursor.execute('''CREATE TABLE IF NOT EXISTS transactions(
                      tx_index INTEGER UNIQUE,
                      tx_hash TEXT UNIQUE,
                      block_index INTEGER,
                      block_hash TEXT,
                      block_time INTEGER,
                      source TEXT,
                      destination TEXT,
                      btc_amount INTEGER,
                      fee INTEGER,
                      data BLOB,
                      supported BOOL DEFAULT 1,
                      FOREIGN KEY (block_index, block_hash) REFERENCES blocks(block_index, block_hash),
                      PRIMARY KEY (tx_index, tx_hash, block_index))
                    ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      block_index_idx ON transactions (block_index)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      tx_index_idx ON transactions (tx_index)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      tx_hash_idx ON transactions (tx_hash)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      index_index_idx ON transactions (block_index, tx_index)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      index_hash_index_idx ON transactions (tx_index, tx_hash, block_index)
                   ''')

    # Purge database of blocks, transactions from before BLOCK_FIRST.
    cursor.execute('''DELETE FROM blocks WHERE block_index < ?''', (config.BLOCK_FIRST,))
    cursor.execute('''DELETE FROM transactions WHERE block_index < ?''', (config.BLOCK_FIRST,))


    # (Valid) debits
    cursor.execute('''CREATE TABLE IF NOT EXISTS debits(
                      block_index INTEGER,
                      address TEXT,
                      asset TEXT,
                      quantity INTEGER,
                      action TEXT,
                      event TEXT,
                      FOREIGN KEY (block_index) REFERENCES blocks(block_index))
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      address_idx ON debits (address)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      asset_idx ON debits (asset)
                   ''')

    # (Valid) credits
    cursor.execute('''CREATE TABLE IF NOT EXISTS credits(
                      block_index INTEGER,
                      address TEXT,
                      asset TEXT,
                      quantity INTEGER,
                      calling_function TEXT,
                      event TEXT,
                      FOREIGN KEY (block_index) REFERENCES blocks(block_index))
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      address_idx ON credits (address)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      asset_idx ON credits (asset)
                   ''')

    # Balances
    cursor.execute('''CREATE TABLE IF NOT EXISTS balances(
                      address TEXT,
                      asset TEXT,
                      quantity INTEGER)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      address_asset_idx ON balances (address, asset)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      address_idx ON balances (address)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      asset_idx ON balances (asset)
                   ''')

    # Assets
    # TODO: Store more asset info here?!
    cursor.execute('''CREATE TABLE IF NOT EXISTS assets(
                      asset_id TEXT UNIQUE,
                      asset_name TEXT UNIQUE,
                      block_index INTEGER,
                      asset_longname TEXT)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      name_idx ON assets (asset_name)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      id_idx ON assets (asset_id)
                   ''')

    # Add asset_longname for sub-assets
    #   SQLite can’t do `ALTER TABLE IF COLUMN NOT EXISTS`.
    columns = [column['name'] for column in cursor.execute('''PRAGMA table_info(assets)''')]
    if 'asset_longname' not in columns:
        cursor.execute('''ALTER TABLE assets ADD COLUMN asset_longname TEXT''')
    cursor.execute('''CREATE UNIQUE INDEX IF NOT EXISTS asset_longname_idx ON assets(asset_longname)''')

    cursor.execute('''SELECT * FROM assets WHERE asset_name = ?''', ('BTC',))
    if not list(cursor):
        cursor.execute('''INSERT INTO assets VALUES (?,?,?,?)''', ('0', 'BTC', None, None))
        cursor.execute('''INSERT INTO assets VALUES (?,?,?,?)''', ('1', 'XCP', None, None))

    # Addresses
    # Leaving this here because in the future this could work for other things besides broadcast
    cursor.execute('''CREATE TABLE IF NOT EXISTS addresses(
                      address TEXT UNIQUE,
                      options INTEGER,
                      block_index INTEGER)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      addresses_idx ON addresses (address)
                   ''')

    # Consolidated
    send.initialise(db)
    destroy.initialise(db)
    order.initialise(db)
    btcpay.initialise(db)
    issuance.initialise(db)
    broadcast.initialise(db)
    bet.initialise(db)
    dividend.initialise(db)
    burn.initialise(db)
    cancel.initialise(db)
    rps.initialise(db)
    rpsresolve.initialise(db)
    sweep.initialise(db)
    dispenser.initialise(db)

    # Messages
    cursor.execute('''CREATE TABLE IF NOT EXISTS messages(
                      message_index INTEGER PRIMARY KEY,
                      block_index INTEGER,
                      command TEXT,
                      category TEXT,
                      bindings TEXT,
                      timestamp INTEGER)
                  ''')
                      # TODO: FOREIGN KEY (block_index) REFERENCES blocks(block_index) DEFERRABLE INITIALLY DEFERRED)
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      block_index_idx ON messages (block_index)
                   ''')
    cursor.execute('''CREATE INDEX IF NOT EXISTS
                      block_index_message_index_idx ON messages (block_index, message_index)
                   ''')

    # Create undolog tables
    cursor.execute('''CREATE TABLE IF NOT EXISTS undolog(
                        undo_index INTEGER PRIMARY KEY AUTOINCREMENT,
                        sql TEXT)
                   ''')
    cursor.execute('''CREATE TABLE IF NOT EXISTS undolog_block(
                        block_index INTEGER PRIMARY KEY,
                        first_undo_index INTEGER)
                   ''')
    # Create undolog triggers for all tables in TABLES list, plus the 'balances' table
    for table in UNDOLOG_TABLES:
        columns = [column['name'] for column in cursor.execute('''PRAGMA table_info({})'''.format(table))]
        cursor.execute('''CREATE TRIGGER IF NOT EXISTS _{}_insert AFTER INSERT ON {} BEGIN
                            INSERT INTO undolog VALUES(NULL, 'DELETE FROM {} WHERE rowid='||new.rowid);
                            END;
                       '''.format(table, table, table))

        columns_parts = ["{}='||quote(old.{})||'".format(c, c) for c in columns]
        cursor.execute('''CREATE TRIGGER IF NOT EXISTS _{}_update AFTER UPDATE ON {} BEGIN
                            INSERT INTO undolog VALUES(NULL, 'UPDATE {} SET {} WHERE rowid='||old.rowid);
                            END;
                       '''.format(table, table, table, ','.join(columns_parts)))

        columns_parts = ["'||quote(old.{})||'".format(c) for c in columns]
        cursor.execute('''CREATE TRIGGER IF NOT EXISTS _{}_delete BEFORE DELETE ON {} BEGIN
                            INSERT INTO undolog VALUES(NULL, 'INSERT INTO {}(rowid,{}) VALUES('||old.rowid||',{})');
                            END;
                       '''.format(table, table, table, ','.join(columns), ','.join(columns_parts)))
    # Drop undolog tables on messages table if they exist (fix for adding them in 9.52.0)
    for trigger_type in ('insert', 'update', 'delete'):
        cursor.execute("DROP TRIGGER IF EXISTS _messages_{}".format(trigger_type))

    # Mempool messages
    # NOTE: `status`, 'block_index` are removed from bindings.
    cursor.execute('''DROP TABLE IF EXISTS mempool''')
    cursor.execute('''CREATE TABLE mempool(
                      tx_hash TEXT,
                      command TEXT,
                      category TEXT,
                      bindings TEXT,
                      timestamp INTEGER)
                  ''')

    cursor.close()

def get_tx_info(tx_hex, block_parser=None, block_index=None, db=None):
    """Get the transaction info. Returns normalized None data for DecodeError and BTCOnlyError."""
    try:
        return _get_tx_info(tx_hex, block_parser, block_index)
    except DecodeError as e:
        return b'', None, None, None, None, None
    except BTCOnlyError as e:
        # NOTE: For debugging, logger.debug('Could not decode: ' + str(e))
        if util.enabled('dispensers', block_index):
            try:
                return b'', None, None, None, None, _get_swap_tx(e.decodedTx, block_parser, block_index, db=db)
            except: # (DecodeError, backend.indexd.BackendRPCError) as e:
                return b'', None, None, None, None, None
        else:
            return b'', None, None, None, None, None

def _get_swap_tx(decoded_tx, block_parser=None, block_index=None, db=None):
    def get_pubkeyhash(scriptpubkey):
        asm = script.get_asm(scriptpubkey)
        
        if len(asm) > 0:
            if asm[0] == "OP_DUP":
                if len(asm) != 5 or asm[1] != 'OP_HASH160' or asm[3] != 'OP_EQUALVERIFY' or asm[4] != 'OP_CHECKSIG':            
                    return False
                else:
                    return {"pubkeyhash":asm[2],"address_version":config.ADDRESSVERSION}
            elif (asm[0] == "OP_HASH160") and util.enabled('p2sh_dispensers_support'):
                if len(asm) != 3 or asm[-1] != 'OP_EQUAL':          
                    return False
                else:
                    return {"pubkeyhash":asm[1],"address_version":config.P2SH_ADDRESSVERSION}
                
        return False

    def get_address(scriptpubkey):
        if util.enabled('correct_segwit_txids') and scriptpubkey.is_witness_v0_keyhash():
            pubkey = scriptpubkey[2:]
            address = str(bitcoinlib.bech32.CBech32Data.from_bytes(0, pubkey))
            return address
        else:
            pubkeyhashdict = get_pubkeyhash(scriptpubkey)
            if not pubkeyhashdict:
                return False
            pubkeyhash = pubkeyhashdict["pubkeyhash"]
            address_version = pubkeyhashdict["address_version"]
            
            pubkeyhash = binascii.hexlify(pubkeyhash).decode('utf-8')
            address = script.base58_check_encode(pubkeyhash, address_version)
            # Test decoding of address.
            if address != config.UNSPENDABLE and binascii.unhexlify(bytes(pubkeyhash, 'utf-8')) != script.base58_check_decode(address, address_version):
                return False

            return address

    outputs = []
    check_sources = db == None # If we didn't get passed a database cursor, assume we have to check for dispenser
    for vout in decoded_tx.vout:
        address = get_address(vout.scriptPubKey)
        destination = None
        btc_amount = None
        if address:
            destination = address
            btc_amount = vout.nValue
        elif util.enabled('hotfix_dispensers_with_non_p2pkh'):
            asm = script.get_asm(vout.scriptPubKey)
            if asm[-1] == 'OP_CHECKSIG':
                destination, new_data = decode_checksig(asm, decoded_tx)
            elif asm[-1] == 'OP_CHECKMULTISIG':
                destination, new_data = decode_checkmultisig(asm, decoded_tx)
            elif asm[0] == 'OP_HASH160' and asm[-1] == 'OP_EQUAL' and len(asm) == 3:
                destination, new_data = decode_scripthash(asm)
            elif asm[0] == 'OP_RETURN':
                pass #Just ignore.
            elif util.enabled('segwit_support') and asm[0] == 0:
                # Segwit output
                destination, new_data = decode_p2w(vout.scriptPubKey)
            else:
                logger.error('unrecognised scriptPubkey. Just ignore this: ' + str(asm))

            if destination and not new_data:
                amount = vout.nValue
            else:
                logger.error('cannot parse destination address or new_data found: ' + str(asm))

        if db != None and dispenser.is_dispensable(db, destination, btc_amount):
            check_sources = True
            outputs.append((destination, btc_amount))

    # Collect all (unique) source addresses.
    #   if we haven't found them yet
    sources = []
    if check_sources:
        for vin in decoded_tx.vin[:]:                   # Loop through inputs.
            if block_parser:
                vin_tx = block_parser.read_raw_transaction(ib2h(vin.prevout.hash))
                vin_ctx = backend.deserialize(vin_tx['__data__'])
            else:
                vin_tx = backend.getrawtransaction(ib2h(vin.prevout.hash)) # TODO: Biggest penalty on parsing is here
                vin_ctx = backend.deserialize(vin_tx)
            vout = vin_ctx.vout[vin.prevout.n]

            asm = script.get_asm(vout.scriptPubKey)
            if asm[-1] == 'OP_CHECKSIG':
                new_source, new_data = decode_checksig(asm, decoded_tx)
                if new_data or not new_source:
                    raise DecodeError('data in source')
            elif asm[-1] == 'OP_CHECKMULTISIG':
                new_source, new_data = decode_checkmultisig(asm, decoded_tx)
                if new_data or not new_source:
                    raise DecodeError('data in source')
            elif asm[0] == 'OP_HASH160' and asm[-1] == 'OP_EQUAL' and len(asm) == 3:
                new_source, new_data = decode_scripthash(asm)
                if new_data or not new_source:
                    raise DecodeError('data in source')
            elif util.enabled('segwit_support') and asm[0] == 0:
                # Segwit output
                # Get the full transaction data for this input transaction.
                new_source, new_data = decode_p2w(vout.scriptPubKey)
            else:
                raise DecodeError('unrecognised source type')

            # old; append to sources, results in invalid addresses
            # new; first found source is source, the rest can be anything (to fund the TX for example)
            if not (util.enabled('first_input_is_source') and len(sources)):
                # Collect unique sources.
                if new_source not in sources:
                    sources.append(new_source)

    return (sources, outputs)

def _get_tx_info(tx_hex, block_parser=None, block_index=None, p2sh_is_segwit=False):
    """Get the transaction info. Calls one of two subfunctions depending on signature type."""
    if not block_index:
        block_index = util.CURRENT_BLOCK_INDEX
    if util.enabled('p2sh_addresses', block_index=block_index):   # Protocol change.
        return  get_tx_info3(tx_hex, block_parser=block_parser, p2sh_is_segwit=p2sh_is_segwit)
    elif util.enabled('multisig_addresses', block_index=block_index):   # Protocol change.
        return get_tx_info2(tx_hex, block_parser=block_parser)
    else:
        return get_tx_info1(tx_hex, block_index, block_parser=block_parser)

def get_tx_info1(tx_hex, block_index, block_parser=None):
    """Get singlesig transaction info.
    The destination, if it exists, always comes before the data output; the
    change, if it exists, always comes after.
    """
    ctx = backend.deserialize(tx_hex)

    def get_pubkeyhash(scriptpubkey):
        asm = script.get_asm(scriptpubkey)
        if len(asm) != 5 or asm[0] != 'OP_DUP' or asm[1] != 'OP_HASH160' or asm[3] != 'OP_EQUALVERIFY' or asm[4] != 'OP_CHECKSIG':
            return False
        return asm[2]

    def get_address(scriptpubkey):
        pubkeyhash = get_pubkeyhash(scriptpubkey)
        if not pubkeyhash:
            return False
        pubkeyhash = binascii.hexlify(pubkeyhash).decode('utf-8')
        address = script.base58_check_encode(pubkeyhash, config.ADDRESSVERSION)
        # Test decoding of address.
        if address != config.UNSPENDABLE and binascii.unhexlify(bytes(pubkeyhash, 'utf-8')) != script.base58_check_decode(address, config.ADDRESSVERSION):
            return False

        return address

    # Fee is the input values minus output values.
    fee = 0

    # Get destination output and data output.
    destination, btc_amount, data = None, None, b''
    pubkeyhash_encoding = False
    for vout in ctx.vout:
        fee -= vout.nValue

        # Sum data chunks to get data. (Can mix OP_RETURN and multi-sig.)
        asm = script.get_asm(vout.scriptPubKey)
        if len(asm) == 2 and asm[0] == 'OP_RETURN':                                             # OP_RETURN
            if type(asm[1]) != bytes:
                continue
            data_chunk = asm[1]
            data += data_chunk
        elif len(asm) == 5 and asm[0] == 1 and asm[3] == 2 and asm[4] == 'OP_CHECKMULTISIG':    # Multi-sig
            if type(asm[2]) != bytes:
                continue
            data_pubkey = asm[2]
            data_chunk_length = data_pubkey[0]  # No ord() necessary.
            data_chunk = data_pubkey[1:data_chunk_length + 1]
            data += data_chunk
        elif len(asm) == 5 and (block_index >= 293000 or config.TESTNET or config.REGTEST):    # Protocol change.
            # Be strict.
            pubkeyhash = get_pubkeyhash(vout.scriptPubKey)
            if not pubkeyhash:
                continue

            if ctx.is_coinbase():
                raise DecodeError('coinbase transaction')
            obj1 = arc4.init_arc4(ctx.vin[0].prevout.hash[::-1])
            data_pubkey = obj1.decrypt(pubkeyhash)
            if data_pubkey[1:9] == config.PREFIX or pubkeyhash_encoding:
                pubkeyhash_encoding = True
                data_chunk_length = data_pubkey[0]  # No ord() necessary.
                data_chunk = data_pubkey[1:data_chunk_length + 1]
                if data_chunk[-8:] == config.PREFIX:
                    data += data_chunk[:-8]
                    break
                else:
                    data += data_chunk

        # Destination is the first output before the data.
        if not destination and not btc_amount and not data:
            address = get_address(vout.scriptPubKey)
            if address:
                destination = address
                btc_amount = vout.nValue

    # Check for, and strip away, prefix (except for burns).
    if destination == config.UNSPENDABLE:
        pass
    elif data[:len(config.PREFIX)] == config.PREFIX:
        data = data[len(config.PREFIX):]
    else:
        raise DecodeError('no prefix')

    # Only look for source if data were found or destination is UNSPENDABLE, for speed.
    if not data and destination != config.UNSPENDABLE:
        raise BTCOnlyError('no data and not unspendable')

    # Collect all possible source addresses; ignore coinbase transactions and anything but the simplest Pay‐to‐PubkeyHash inputs.
    source_list = []
    for vin in ctx.vin[:]:                                               # Loop through input transactions.
        if vin.prevout.is_null():
            raise DecodeError('coinbase transaction')
         # Get the full transaction data for this input transaction.
        if block_parser:
            vin_tx = block_parser.read_raw_transaction(ib2h(vin.prevout.hash))
            vin_ctx = backend.deserialize(vin_tx['__data__'])
        else:
            vin_tx = backend.getrawtransaction(ib2h(vin.prevout.hash))
            vin_ctx = backend.deserialize(vin_tx)
        vout = vin_ctx.vout[vin.prevout.n]
        fee += vout.nValue

        address = get_address(vout.scriptPubKey)
        if not address:
            raise DecodeError('invalid scriptpubkey')
        else:
            source_list.append(address)

    # Require that all possible source addresses be the same.
    if all(x == source_list[0] for x in source_list):
        source = source_list[0]
    else:
        source = None

    return source, destination, btc_amount, fee, data, None

def get_tx_info3(tx_hex, block_parser=None, p2sh_is_segwit=False):
    return get_tx_info2(tx_hex, block_parser=block_parser, p2sh_support=True, p2sh_is_segwit=p2sh_is_segwit)

def arc4_decrypt(cyphertext, ctx):
    '''Un‐obfuscate. Initialise key once per attempt.'''
    key = arc4.init_arc4(ctx.vin[0].prevout.hash[::-1])
    return key.decrypt(cyphertext)

def get_opreturn(asm):
    if len(asm) == 2 and asm[0] == 'OP_RETURN':
        pubkeyhash = asm[1]
        if type(pubkeyhash) == bytes:
            return pubkeyhash
    raise DecodeError('invalid OP_RETURN')

def decode_opreturn(asm, ctx):
    chunk = get_opreturn(asm)
    chunk = arc4_decrypt(chunk, ctx)
    if chunk[:len(config.PREFIX)] == config.PREFIX:             # Data
        destination, data = None, chunk[len(config.PREFIX):]
    else:
        raise DecodeError('unrecognised OP_RETURN output')

    return destination, data

def decode_checksig(asm, ctx):
    pubkeyhash = script.get_checksig(asm)
    chunk = arc4_decrypt(pubkeyhash, ctx)
    if chunk[1:len(config.PREFIX) + 1] == config.PREFIX:        # Data
        # Padding byte in each output (instead of just in the last one) so that encoding methods may be mixed. Also, it’s just not very much data.
        chunk_length = chunk[0]
        chunk = chunk[1:chunk_length + 1]
        destination, data = None, chunk[len(config.PREFIX):]
    else:                                                       # Destination
        pubkeyhash = binascii.hexlify(pubkeyhash).decode('utf-8')
        destination, data = script.base58_check_encode(pubkeyhash, config.ADDRESSVERSION), None

    return destination, data

def decode_scripthash(asm):
    destination = script.base58_check_encode(binascii.hexlify(asm[1]).decode('utf-8'), config.P2SH_ADDRESSVERSION)

    return destination, None

def decode_checkmultisig(asm, ctx):
    pubkeys, signatures_required = script.get_checkmultisig(asm)
    chunk = b''
    for pubkey in pubkeys[:-1]:     # (No data in last pubkey.)
        chunk += pubkey[1:-1]       # Skip sign byte and nonce byte.
    chunk = arc4_decrypt(chunk, ctx)
    if chunk[1:len(config.PREFIX) + 1] == config.PREFIX:        # Data
        # Padding byte in each output (instead of just in the last one) so that encoding methods may be mixed. Also, it’s just not very much data.
        chunk_length = chunk[0]
        chunk = chunk[1:chunk_length + 1]
        destination, data = None, chunk[len(config.PREFIX):]
    else:                                                       # Destination
        pubkeyhashes = [script.pubkey_to_pubkeyhash(pubkey) for pubkey in pubkeys]
        destination, data = script.construct_array(signatures_required, pubkeyhashes, len(pubkeyhashes)), None

    return destination, data

def decode_p2w(script_pubkey):
    try:
        bech32 = bitcoinlib.bech32.CBech32Data.from_bytes(0, script_pubkey[2:22])
        return str(bech32), None
    except TypeError as e:
        raise DecodeError('bech32 decoding error')

def get_tx_info2(tx_hex, block_parser=None, p2sh_support=False, p2sh_is_segwit=False):
    """Get multisig transaction info.
    The destinations, if they exists, always comes before the data output; the
    change, if it exists, always comes after.
    """
    # Decode transaction binary.
    ctx = backend.deserialize(tx_hex)

    # Ignore coinbase transactions.
    if ctx.is_coinbase():
        raise DecodeError('coinbase transaction')

    # Get destinations and data outputs.
    destinations, btc_amount, fee, data = [], 0, 0, b''

    for vout in ctx.vout:
        # Fee is the input values minus output values.
        output_value = vout.nValue
        fee -= output_value

        # Ignore transactions with invalid script.
        try:
            asm = script.get_asm(vout.scriptPubKey)
        except CScriptInvalidError as e:
            raise DecodeError(e)

        if asm[0] == 'OP_RETURN':
            new_destination, new_data = decode_opreturn(asm, ctx)
        elif asm[-1] == 'OP_CHECKSIG':
            new_destination, new_data = decode_checksig(asm, ctx)
        elif asm[-1] == 'OP_CHECKMULTISIG':
            try:
                new_destination, new_data = decode_checkmultisig(asm, ctx)
            except:
                raise DecodeError('unrecognised output type')
        elif p2sh_support and asm[0] == 'OP_HASH160' and asm[-1] == 'OP_EQUAL' and len(asm) == 3:
            new_destination, new_data = decode_scripthash(asm)
        elif util.enabled('segwit_support') and asm[0] == 0:
            # Segwit Vout, second param is redeemScript
            #redeemScript = asm[1]
            new_destination, new_data = decode_p2w(vout.scriptPubKey)
        else:
            raise DecodeError('unrecognised output type')
        assert not (new_destination and new_data)
        assert new_destination != None or new_data != None  # `decode_*()` should never return `None, None`.

        if util.enabled('null_data_check'):
            if new_data == []:
                raise DecodeError('new destination is `None`')

        # All destinations come before all data.
        if not data and not new_data and destinations != [config.UNSPENDABLE,]:
            destinations.append(new_destination)
            btc_amount += output_value
        else:
            if new_destination:     # Change.
                break
            else:                   # Data.
                data += new_data

    # source can be determined by parsing the p2sh_data transaction
    #   or from the first spent output
    sources = []

    # P2SH encoding signalling
    p2sh_encoding_source = None
    if util.enabled('p2sh_encoding') and data == b'P2SH':
        data = b''
        for vin in ctx.vin:
            if util.enabled("prevout_segwit_fix"):
                vin_tx = backend.getrawtransaction(ib2h(vin.prevout.hash))
                vin_ctx = backend.deserialize(vin_tx)
                prevout_is_segwit = vin_ctx.has_witness()
            else:
                prevout_is_segwit = p2sh_is_segwit
                
            # Ignore transactions with invalid script.
            try:
                asm = script.get_asm(vin.scriptSig)
            except CScriptInvalidError as e:
                raise DecodeError(e)

            new_source, new_destination, new_data = p2sh_encoding.decode_p2sh_input(asm, p2sh_is_segwit=prevout_is_segwit)
            # this could be a p2sh source address with no encoded data
            if new_data is None:
              continue;

            if new_source is not None:
                if p2sh_encoding_source is not None and new_source != p2sh_encoding_source:
                    # this p2sh data input has a bad source address
                    raise DecodeError('inconsistent p2sh inputs')

                p2sh_encoding_source = new_source

            assert not new_destination

            data += new_data
    # Only look for source if data were found or destination is `UNSPENDABLE`,
    # for speed.
    if not data and destinations != [config.UNSPENDABLE,]:
        raise BTCOnlyError('no data and not unspendable', ctx)

    # Collect all (unique) source addresses.
    #   if we haven't found them yet
    for vin in ctx.vin[:]:                   # Loop through inputs.
        # Get the full transaction data for this input transaction.
        if block_parser:
            vin_tx = block_parser.read_raw_transaction(ib2h(vin.prevout.hash))
            vin_ctx = backend.deserialize(vin_tx['__data__'])
        else:
            vin_tx = backend.getrawtransaction(ib2h(vin.prevout.hash))
            vin_ctx = backend.deserialize(vin_tx)
        vout = vin_ctx.vout[vin.prevout.n]
        fee += vout.nValue

        asm = script.get_asm(vout.scriptPubKey)
        if asm[-1] == 'OP_CHECKSIG':
            new_source, new_data = decode_checksig(asm, ctx)
            if new_data or not new_source:
                raise DecodeError('data in source')
        elif asm[-1] == 'OP_CHECKMULTISIG':
            new_source, new_data = decode_checkmultisig(asm, ctx)
            if new_data or not new_source:
                raise DecodeError('data in source')
        elif p2sh_support and asm[0] == 'OP_HASH160' and asm[-1] == 'OP_EQUAL' and len(asm) == 3:
            new_source, new_data = decode_scripthash(asm)
            if new_data or not new_source:
                raise DecodeError('data in source')
        elif util.enabled('segwit_support') and asm[0] == 0:
            # Segwit output
            new_source, new_data = decode_p2w(vout.scriptPubKey)
        else:
            raise DecodeError('unrecognised source type')

        # old; append to sources, results in invalid addresses
        # new; first found source is source, the rest can be anything (to fund the TX for example)
        if not (util.enabled('first_input_is_source') and len(sources)):
            # Collect unique sources.
            if new_source not in sources:
                sources.append(new_source)

    # use the source from the p2sh data source
    if p2sh_encoding_source is not None:
        sources = p2sh_encoding_source
    else:
        sources = '-'.join(sources)

    destinations = '-'.join(destinations)
    return sources, destinations, btc_amount, round(fee), data, None

def reinitialise(db, block_index=None):
    """Drop all predefined tables and initialise the database once again."""
    cursor = db.cursor()

    # Delete all of the results of parsing (including the undolog)
    for table in TABLES + ['balances', 'undolog', 'undolog_block']:
        cursor.execute('''DROP TABLE IF EXISTS {}'''.format(table))

    # Create missing tables
    initialise(db)

    # clean consensus hashes if first block hash doesn't match with checkpoint.
    if config.TESTNET:
        checkpoints = check.CHECKPOINTS_TESTNET
    elif config.REGTEST:
        checkpoints = check.CHECKPOINTS_REGTEST
    else:
        checkpoints = check.CHECKPOINTS_MAINNET

    columns = [column['name'] for column in cursor.execute('''PRAGMA table_info(blocks)''')]
    for field in ['ledger_hash', 'txlist_hash']:
        if field in columns:
            sql = '''SELECT {} FROM blocks  WHERE block_index = ?'''.format(field)
            first_block = list(cursor.execute(sql, (config.BLOCK_FIRST,)))
            if first_block:
                first_hash = first_block[0][field]
                if first_hash != checkpoints[config.BLOCK_FIRST][field]:
                    logger.info('First hash changed. Cleaning {}.'.format(field))
                    cursor.execute('''UPDATE blocks SET {} = NULL'''.format(field))

    # For rollbacks, just delete new blocks and then reparse what’s left.
    if block_index:
        cursor.execute('''DELETE FROM transactions WHERE block_index > ?''', (block_index,))
        cursor.execute('''DELETE FROM blocks WHERE block_index > ?''', (block_index,))
    elif config.TESTNET or config.REGTEST:  # block_index NOT specified and we are running testnet
        # just blow away the consensus hashes with a full testnet reparse, as we could activate
        # new features retroactively, which could otherwise lead to ConsensusError exceptions being raised.
        logger.info("Testnet/regtest full reparse detected: Clearing all consensus hashes before performing reparse.")
        cursor.execute('''UPDATE blocks SET ledger_hash = NULL, txlist_hash = NULL, messages_hash = NULL''')

    cursor.close()

def reparse(db, block_index=None, quiet=False):
    """Reparse all transactions (atomically). If block_index is set, rollback
    to the end of that block.
    """
    def reparse_from_undolog(db, block_index, quiet):
        """speedy reparse method that utilizes the undolog.
        if fails, fallback to the full reparse method"""
        if not block_index:
            return False # Can't reparse from undolog

        undolog_cursor = db.cursor()
        undolog_cursor.setexectrace(None)
        undolog_cursor.setrowtrace(None)

        def get_block_index_for_undo_index(undo_indexes, undo_index):
            for block_index, first_undo_index in undo_indexes.items(): #in order
                if undo_index < first_undo_index:
                    return block_index - 1
            else:
                return next(reversed(undo_indexes)) #the last inserted block_index

        with db:
            # Check if we can reparse from the undolog
            results = list(undolog_cursor.execute(
                '''SELECT block_index, first_undo_index FROM undolog_block WHERE block_index >= ? ORDER BY block_index ASC''', (block_index,)))
            undo_indexes = collections.OrderedDict()
            for result in results:
                undo_indexes[result[0]] = result[1]

            undo_start_block_index = block_index + 1

            if undo_start_block_index not in undo_indexes:
                if block_index in undo_indexes:
                    # Edge case, should only happen if we're "rolling back" to latest block (e.g. via cmd line)
                    return True #skip undo
                else:
                    return False # Undolog doesn't go that far back, full reparse required...

            # Grab the undolog...
            undolog = list(undolog_cursor.execute(
                '''SELECT undo_index, sql FROM undolog WHERE undo_index >= ? ORDER BY undo_index DESC''',
                (undo_indexes[undo_start_block_index],)))

            # Replay the undolog backwards, from the last entry to first_undo_index...
            for entry in undolog:
                logger.info("Undolog: Block {} (undo_index {}): {}".format(
                    get_block_index_for_undo_index(undo_indexes, entry[0]), entry[0], entry[1]))
                undolog_cursor.execute(entry[1])

            # Trim back tx and blocks
            undolog_cursor.execute('''DELETE FROM transactions WHERE block_index > ?''', (block_index,))
            undolog_cursor.execute('''DELETE FROM blocks WHERE block_index > ?''', (block_index,))
            # As well as undolog entries...
            undolog_cursor.execute('''DELETE FROM undolog WHERE undo_index >= ?''', (undo_indexes[undo_start_block_index],))
            undolog_cursor.execute('''DELETE FROM undolog_block WHERE block_index >= ?''', (undo_start_block_index,))

        undolog_cursor.close()
        return True

    if block_index:
        logger.info('Rolling back transactions to block {}.'.format(block_index))
    else:
        logger.info('Reparsing all transactions.')

    check.software_version()
    reparse_start = time.time()

    # Reparse from the undolog if possible
    reparsed = reparse_from_undolog(db, block_index, quiet)

    cursor = db.cursor()

    if not reparsed:
        if block_index:
            logger.info("Could not roll back from undolog. Performing full reparse instead...")

        if quiet:
            root_logger = logging.getLogger()
            root_level = logger.getEffectiveLevel()

        with db:
            reinitialise(db, block_index)

            # Reparse all blocks, transactions.
            if quiet:
                root_logger.setLevel(logging.WARNING)

            previous_ledger_hash, previous_txlist_hash, previous_messages_hash = None, None, None
            cursor.execute('''SELECT * FROM blocks ORDER BY block_index''')
            for block in cursor.fetchall():
                util.CURRENT_BLOCK_INDEX = block['block_index']
                previous_ledger_hash, previous_txlist_hash, previous_messages_hash, previous_found_messages_hash = parse_block(
                                                                         db, block['block_index'], block['block_time'],
                                                                         previous_ledger_hash=previous_ledger_hash,
                                                                         previous_txlist_hash=previous_txlist_hash,
                                                                         previous_messages_hash=previous_messages_hash)
                if quiet and block['block_index'] % 10 == 0:  # every 10 blocks print status
                    root_logger.setLevel(logging.INFO)
                logger.info('Block (re-parse): %s (hashes: L:%s / TX:%s / M:%s%s)' % (
                    block['block_index'], previous_ledger_hash[-5:], previous_txlist_hash[-5:], previous_messages_hash[-5:],
                    (' [overwrote %s]' % previous_found_messages_hash) if previous_found_messages_hash and previous_found_messages_hash != previous_messages_hash else ''))
                if quiet and block['block_index'] % 10 == 0:
                    root_logger.setLevel(logging.WARNING)

        if quiet:
            root_logger.setLevel(root_level)

    with db:
        # Check for conservation of assets.
        check.asset_conservation(db)

        # Update database version number.
        database.update_version(db)

    cursor.close()
    reparse_end = time.time()
    logger.info("Reparse took {:.3f} minutes.".format((reparse_end - reparse_start) / 60.0))

    # on full reparse - vacuum the DB afterwards for better subsequent performance (especially on non-SSDs)
    if not block_index:
        database.vacuum(db)

def list_tx(db, block_hash, block_index, block_time, tx_hash, tx_index, tx_hex=None):
    assert type(tx_hash) == str
    cursor = db.cursor()

    # Edge case: confirmed tx_hash also in mempool
    cursor.execute('''SELECT * FROM transactions WHERE tx_hash = ?''', (tx_hash,))
    transactions = list(cursor)
    if transactions:
        return tx_index

    # Get the important details about each transaction.
    if tx_hex is None:
        tx_hex = backend.getrawtransaction(tx_hash) # TODO: This is the call that is stalling the process the most

    source, destination, btc_amount, fee, data, decoded_tx = get_tx_info(tx_hex, db=db)

    if not source and decoded_tx and util.enabled('dispensers', block_index):
        outputs = decoded_tx[1]
        for out in outputs:
            if out[0] != decoded_tx[0][0] and dispenser.is_dispensable(db, out[0], out[1]):
                source = decoded_tx[0][0]
                destination = out[0]
                btc_amount = out[1]
                fee = 0
                data = struct.pack(config.SHORT_TXTYPE_FORMAT, dispenser.DISPENSE_ID)
                data += b'\x00'
                break # Prevent inspection of further dispenses (only first one is valid)

    # For mempool
    if block_hash == None:
        block_hash = config.MEMPOOL_BLOCK_HASH
        block_index = config.MEMPOOL_BLOCK_INDEX
    else:
        assert block_index == util.CURRENT_BLOCK_INDEX

    if source and (data or destination == config.UNSPENDABLE or decoded_tx):
        logger.debug('Saving transaction: {}'.format(tx_hash))
        cursor.execute('''INSERT INTO transactions(
                            tx_index,
                            tx_hash,
                            block_index,
                            block_hash,
                            block_time,
                            source,
                            destination,
                            btc_amount,
                            fee,
                            data) VALUES(?,?,?,?,?,?,?,?,?,?)''',
                            (tx_index,
                             tx_hash,
                             block_index,
                             block_hash,
                             block_time,
                             source,
                             destination,
                             btc_amount,
                             fee,
                             data)
                      )
        cursor.close()
        return tx_index + 1
    else:
        logger.getChild('list_tx.skip').debug('Skipping transaction: {}'.format(tx_hash))

    return tx_index

def kickstart(db, bitcoind_dir):
    if bitcoind_dir is None:
        if platform.system() == 'Darwin':
            bitcoind_dir = os.path.expanduser('~/Library/Application Support/Bitcoin/')
        elif platform.system() == 'Windows':
            bitcoind_dir = os.path.join(os.environ['APPDATA'], 'Bitcoin')
        else:
            bitcoind_dir = os.path.expanduser('~/.bitcoin')
    if not os.path.isdir(bitcoind_dir):
        raise Exception('Bitcoin Core data directory not found at {}. Use --bitcoind-dir parameter.'.format(bitcoind_dir))

    cursor = db.cursor()

    logger.warning('''Warning:
- Ensure that bitcoind is stopped.
- You must reindex bitcoind after the initialization is complete (restart with `-reindex=1`)
- The initialization may take a while.''')
    if input('Proceed with the initialization? (y/N) : ') != 'y':
        return

    if config.TESTNET:
        first_hash = config.BLOCK_FIRST_TESTNET_HASH
    elif config.REGTEST:
        first_hash = config.BLOCK_FIRST_REGTEST_HASH
    else:
        first_hash = config.BLOCK_FIRST_MAINNET_HASH

    start_time_total = time.time()

    # Get hash of last known block.
    chain_parser = ChainstateParser(os.path.join(bitcoind_dir, 'chainstate'))
    last_hash = chain_parser.get_last_block_hash()
    chain_parser.close()

    # Start block parser.
    block_parser = BlockchainParser(os.path.join(bitcoind_dir, 'blocks'), os.path.join(bitcoind_dir, 'blocks/index'))

    current_hash = last_hash
    tx_index = 0
    with db:
        # Prepare SQLite database. # TODO: Be more specific!
        logger.info('Preparing database.')
        start_time = time.time()
        reinitialise(db, block_index=config.BLOCK_FIRST - 1)
        logger.info('Prepared database in {:.3f}s'.format(time.time() - start_time))

        # Get blocks and transactions, moving backwards in time.
        while current_hash != None:
            start_time = time.time()
            transactions = []

            # Get `tx_info`s for transactions in this block.
            block = block_parser.read_raw_block(current_hash)
            for tx in block['transactions']:
                source, destination, btc_amount, fee, data = get_tx_info(tx['__data__'], block_parser=block_parser, block_index=block['block_index'])
                if source and (data or destination == config.UNSPENDABLE):
                    transactions.append((
                        tx['tx_hash'], block['block_index'], block['block_hash'], block['block_time'],
                        source, destination, btc_amount, fee, data
                    ))
                    logger.info('Valid transaction: {}'.format(tx['tx_hash']))

            # Insert block and transactions into database.
            cursor.execute('''INSERT INTO blocks(
                                    block_index,
                                    block_hash,
                                    block_time) VALUES(?,?,?)''',
                                    (block['block_index'],
                                    block['block_hash'],
                                    block['block_time']))
            if len(transactions):
                transactions = list(reversed(transactions))
                tx_chunks = [transactions[i:i+90] for i in range(0, len(transactions), 90)]
                for tx_chunk in tx_chunks:
                    sql = '''INSERT INTO transactions
                                (tx_index, tx_hash, block_index, block_hash, block_time, source, destination, btc_amount, fee, data)
                             VALUES '''
                    bindings = ()
                    bindings_place = []
                    # negative tx_index from -1 and inverse order for fast reordering   # TODO: Can this be clearer?
                    for tx in tx_chunk:
                        bindings += (-(tx_index + 1),) + tx
                        bindings_place.append('''(?,?,?,?,?,?,?,?,?,?)''')
                        tx_index += 1
                    sql += ', '.join(bindings_place)
                    cursor.execute(sql, bindings)

            logger.info('Block {} ({}): {}/{} saved in {:.3f}s'.format(
                          block['block_index'], block['block_hash'],
                          len(transactions), len(block['transactions']),
                          time.time() - start_time))

            # Get hash of next block.
            current_hash = block['hash_prev'] if current_hash != first_hash else None

        block_parser.close()

        # Reorder all transactions in database.
        logger.info('Reordering transactions.')
        start_time = time.time()
        cursor.execute('''UPDATE transactions SET tx_index = tx_index + ?''', (tx_index,))
        logger.info('Reordered transactions in {:.3f}s.'.format(time.time() - start_time))

    # Parse all transactions in database.
    reparse(db)

    cursor.close()
    logger.info('Total duration: {:.3f}s'.format(time.time() - start_time_total))

def last_db_index(db):
    cursor = db.cursor()
    try:
        blocks = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = (SELECT MAX(block_index) from blocks)'''))
        try:
            return blocks[0]['block_index']
        except IndexError:
            return 0
    except apsw.SQLError:
        return 0

def get_next_tx_index(db):
    """Return index of next transaction."""
    cursor = db.cursor()
    txes = list(cursor.execute('''SELECT * FROM transactions WHERE tx_index = (SELECT MAX(tx_index) from transactions)'''))
    if txes:
        assert len(txes) == 1
        tx_index = txes[0]['tx_index'] + 1
    else:
        tx_index = 0
    cursor.close()
    return tx_index



class MempoolError(Exception):
    pass
def follow(db):
    # Check software version.
    check.software_version()

    # Initialise.
    initialise(db)

    # Get index of last block.
    if util.CURRENT_BLOCK_INDEX == 0:
        logger.warning('New database.')
        block_index = config.BLOCK_FIRST
    else:
        block_index = util.CURRENT_BLOCK_INDEX + 1

        # Check database version.
        try:
            check.database_version(db)
        except check.DatabaseVersionError as e:
            logger.info(str(e))
            # no need to reparse or rollback a new database
            if block_index != config.BLOCK_FIRST:
                reparse(db, block_index=e.reparse_block_index, quiet=False)
            else: #version update was included in reparse(), so don't do it twice
                database.update_version(db)

    logger.info('Resuming parsing.')

    # Get index of last transaction.
    tx_index = get_next_tx_index(db)

    not_supported = {}   # No false positives. Use a dict to allow for O(1) lookups
    not_supported_sorted = collections.deque()
    # ^ Entries in form of (block_index, tx_hash), oldest first. Allows for easy removal of past, unncessary entries
    cursor = db.cursor()

    # a reorg can happen without the block count increasing, or even for that
    # matter, with the block count decreasing. This should only delay
    # processing of the new blocks a bit.
    while True:
        start_time = time.time()
        # Get block count.
        # If the backend is unreachable and `config.FORCE` is set, just sleep
        # and try again repeatedly.
        try:
            block_count = backend.getblockcount()
        except (ConnectionRefusedError, http.client.CannotSendRequest, backend.addrindexrs.BackendRPCError) as e:
            if config.FORCE:
                time.sleep(config.BACKEND_POLL_INTERVAL)
                continue
            else:
                raise e

        # Get new blocks.
        if block_index <= block_count:

            current_index = block_index
            # Backwards check for incorrect blocks due to chain reorganisation, and stop when a common parent is found.
            if block_count - block_index < 100: # Undolog only saves last 100 blocks, if there's a reorg deeper than that manual reparse should be done
                requires_rollback = False
                while True:
                    if current_index == config.BLOCK_FIRST:
                        break

                    logger.debug('Checking that block {} is not an orphan.'.format(current_index))
                    # Backend parent hash.
                    current_hash = backend.getblockhash(current_index)
                    current_cblock = backend.getblock(current_hash)
                    backend_parent = bitcoinlib.core.b2lx(current_cblock.hashPrevBlock)

                    # DB parent hash.
                    blocks = list(cursor.execute('''SELECT * FROM blocks
                                                    WHERE block_index = ?''', (current_index - 1,)))
                    if len(blocks) != 1:  # For empty DB.
                        break
                    db_parent = blocks[0]['block_hash']

                    # Compare.
                    assert type(db_parent) == str
                    assert type(backend_parent) == str
                    if db_parent == backend_parent:
                        break
                    else:
                        current_index -= 1
                        requires_rollback = True

                # Rollback for reorganisation.
                if requires_rollback:
                    # Record reorganisation.
                    logger.warning('Blockchain reorganisation at block {}.'.format(current_index))
                    log.message(db, block_index, 'reorg', None, {'block_index': current_index})

                    # Rollback the DB.
                    reparse(db, block_index=current_index-1, quiet=True)
                    block_index = current_index
                    tx_index = get_next_tx_index(db)
                    continue

            # Check version. (Don’t add any blocks to the database while
            # running an out‐of‐date client!)
            check.software_version()


            # Get and parse transactions in this block (atomically).
            block_hash = backend.getblockhash(current_index)
            block = backend.getblock(block_hash)
            previous_block_hash = bitcoinlib.core.b2lx(block.hashPrevBlock)
            block_time = block.nTime
            txhash_list, raw_transactions = backend.get_tx_list(block)

            with db:
                util.CURRENT_BLOCK_INDEX = block_index

                # List the block.
                cursor.execute('''INSERT INTO blocks(
                                    block_index,
                                    block_hash,
                                    block_time,
                                    previous_block_hash,
                                    difficulty) VALUES(?,?,?,?,?)''',
                                    (block_index,
                                    block_hash,
                                    block_time,
                                    previous_block_hash,
                                    block.difficulty)
                              )

                # List the transactions in the block.
                for tx_hash in txhash_list:
                    tx_hex = raw_transactions[tx_hash]
                    tx_index = list_tx(db, block_hash, block_index, block_time, tx_hash, tx_index, tx_hex)

                # Parse the transactions in the block.
                new_ledger_hash, new_txlist_hash, new_messages_hash, found_messages_hash = parse_block(db, block_index, block_time)

            # When newly caught up, check for conservation of assets.
            if block_index == block_count:
                if config.CHECK_ASSET_CONSERVATION:
                    check.asset_conservation(db)

            # Remove any non‐supported transactions older than ten blocks.
            while len(not_supported_sorted) and not_supported_sorted[0][0] <= block_index - 10:
                tx_h = not_supported_sorted.popleft()[1]
                del not_supported[tx_h]

            logger.info('Block: %s (%ss, hashes: L:%s / TX:%s / M:%s%s)' % (
                str(block_index), "{:.2f}".format(time.time() - start_time, 3),
                new_ledger_hash[-5:], new_txlist_hash[-5:], new_messages_hash[-5:],
                (' [overwrote %s]' % found_messages_hash) if found_messages_hash and found_messages_hash != new_messages_hash else ''))

            # Increment block index.
            block_count = backend.getblockcount()
            block_index += 1

        else:
            # TODO: add zeromq support here to await TXs and Blocks instead of constantly polling
            # Get old mempool.
            old_mempool = list(cursor.execute('''SELECT * FROM mempool'''))
            old_mempool_hashes = [message['tx_hash'] for message in old_mempool]

            if backend.MEMPOOL_CACHE_INITIALIZED is False:
                backend.init_mempool_cache()
                logger.info("Ready for queries.")

            # Fake values for fake block.
            curr_time = int(time.time())
            mempool_tx_index = tx_index

            xcp_mempool = []
            raw_mempool = backend.getrawmempool()

            # this is a quick fix to make counterparty usable on high mempool situations
            # however, this makes the mempool unreliable on counterparty, a better, larger
            # fix must be done by changing this whole function into a zmq driven loop
            if len(raw_mempool) > config.MEMPOOL_TXCOUNT_UPDATE_LIMIT:
                continue

            # For each transaction in Bitcoin Core mempool, if it’s new, create
            # a fake block, a fake transaction, capture the generated messages,
            # and then save those messages.
            # Every transaction in mempool is parsed independently. (DB is rolled back after each one.)
            # We first filter out which transactions we've already parsed before so we can batch fetch their raw data
            parse_txs = []
            for tx_hash in raw_mempool:
                # If already in mempool, copy to new one.
                if tx_hash in old_mempool_hashes:
                    for message in old_mempool:
                        if message['tx_hash'] == tx_hash:
                            xcp_mempool.append((tx_hash, message))

                # If not a supported XCP transaction, skip.
                elif tx_hash in not_supported:
                    pass

                # Else: list, parse and save it.
                else:
                    parse_txs.append(tx_hash)

            # fetch raw for all transactions that need to be parsed
            # Sometimes the transactions can’t be found: `{'code': -5, 'message': 'No information available about transaction'}`
            #  - is txindex enabled in Bitcoind?
            #  - or was there a block found while batch feting the raw txs
            #  - or was there a double spend for w/e reason accepted into the mempool (replace-by-fee?)
            try:
                raw_transactions = backend.getrawtransaction_batch(parse_txs)
            except Exception as e:
                logger.warning('Failed to fetch raw for mempool TXs, restarting loop; %s', (e, ))
                continue  # restart the follow loop

            for tx_hash in parse_txs:
                try:
                    with db:
                        # List the fake block.
                        cursor.execute('''INSERT INTO blocks(
                                            block_index,
                                            block_hash,
                                            block_time) VALUES(?,?,?)''',
                                       (config.MEMPOOL_BLOCK_INDEX,
                                        config.MEMPOOL_BLOCK_HASH,
                                        curr_time)
                                      )

                        tx_hex = raw_transactions[tx_hash]
                        if tx_hex is None:
                          logger.debug('tx_hash %s not found in backend.  Not adding to mempool.', (tx_hash, ))
                          raise MempoolError
                        mempool_tx_index = list_tx(db, None, block_index, curr_time, tx_hash, tx_index=mempool_tx_index, tx_hex=tx_hex)

                        # Parse transaction.
                        cursor.execute('''SELECT * FROM transactions WHERE tx_hash = ?''', (tx_hash,))
                        transactions = list(cursor)
                        if transactions:
                            assert len(transactions) == 1
                            transaction = transactions[0]
                            supported = parse_tx(db, transaction)
                            if not supported:
                                not_supported[tx_hash] = ''
                                not_supported_sorted.append((block_index, tx_hash))
                        else:
                            # If a transaction hasn’t been added to the
                            # table `transactions`, then it’s not a
                            # Counterparty transaction.
                            not_supported[tx_hash] = ''
                            not_supported_sorted.append((block_index, tx_hash))
                            raise MempoolError

                        # Save transaction and side‐effects in memory.
                        cursor.execute('''SELECT * FROM messages WHERE block_index = ?''', (config.MEMPOOL_BLOCK_INDEX,))
                        for message in list(cursor):
                            xcp_mempool.append((tx_hash, message))

                        # Rollback.
                        raise MempoolError
                except exceptions.ParseTransactionError as e:
                    logger.warn('ParseTransactionError for tx %s: %s' % (tx_hash, e))
                except MempoolError:
                    pass

            # Re‐write mempool messages to database.
            with db:
                cursor.execute('''DELETE FROM mempool''')
                for message in xcp_mempool:
                    tx_hash, new_message = message
                    new_message['tx_hash'] = tx_hash
                    cursor.execute('''INSERT INTO mempool VALUES(:tx_hash, :command, :category, :bindings, :timestamp)''', new_message)

            elapsed_time = time.time() - start_time
            sleep_time = config.BACKEND_POLL_INTERVAL - elapsed_time if elapsed_time <= config.BACKEND_POLL_INTERVAL else 0

            logger.getChild('mempool').debug('Refresh mempool: %s XCP txs seen, out of %s total entries (took %ss, next refresh in %ss)' % (
                len(xcp_mempool), len(raw_mempool),
                "{:.2f}".format(elapsed_time, 3),
                "{:.2f}".format(sleep_time, 3)))

            # Wait
            db.wal_checkpoint(mode=apsw.SQLITE_CHECKPOINT_PASSIVE)
            time.sleep(sleep_time)

    cursor.close()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### check.py 
 
import json
import requests
import logging
logger = logging.getLogger(__name__)
import warnings
import time
import sys

from counterpartylib.lib import config
from counterpartylib.lib import util
from counterpartylib.lib import exceptions
from counterpartylib.lib import backend
from counterpartylib.lib import database

CONSENSUS_HASH_SEED = 'We can only see a short distance ahead, but we can see plenty there that needs to be done.'

CONSENSUS_HASH_VERSION_MAINNET = 2
CHECKPOINTS_MAINNET = {
    config.BLOCK_FIRST_MAINNET: {'ledger_hash': '766ff0a9039521e3628a79fa669477ade241fc4c0ae541c3eae97f34b547b0b7', 'txlist_hash': '766ff0a9039521e3628a79fa669477ade241fc4c0ae541c3eae97f34b547b0b7'},
    280000: {'ledger_hash': '265719e2770d5a6994f6fe49839069183cd842ee14f56c2b870e56641e8a8725', 'txlist_hash': 'a59b33b4633649db4f14586af47e258ed9b8884dbb7aa308fb1f49a653ee60f4'},
    290000: {'ledger_hash': '4612ed7034474b4ff1727eb0e216d533ebe7ac755fb015e0f9a170c063f3e84c', 'txlist_hash': 'c15423c849fd360d38cbd6c6c3ea37a07fece723da92353f3056facc2676d9e7'},
    300000: {'ledger_hash': '9a3dd4949780404d61e5ca1929f94a43f08eb0fa19ccb4b5d6a61cafd7943199', 'txlist_hash': 'efa02dbdcc4158a598e3b476ece5ba9cc8d26f3abc8ac3777ac6dde0f0afc7e6'},
    310000: {'ledger_hash': '45e43d5cc77ea01129df01d7f55b0c89b2d4e18cd3d626fd92f30bfb37a85f4d', 'txlist_hash': '83cdcf75833d828ded09979b601fde87e2fdb0f5eb1cc6ab5d2042b7ec85f90e'},
    320000: {'ledger_hash': '91c1d33626669e8098bc762b1a9e3f616884e4d1cadda4881062c92b0d3d3e98', 'txlist_hash': '761793042d8e7c80e14a16c15bb9d40e237c468a87c207a59730b616bdfde7d4'},
    330000: {'ledger_hash': 'dd56aa97e5ca15841407f383ce1d7814536a594d7cfffcb4cf60bee8b362065a', 'txlist_hash': '3c45b4377a99e020550a198daa45c378c488a72ba199b53deb90b320d55a897b'},
    334000: {'ledger_hash': '24c4fa4097106031267439eb9fbe8ce2a18560169c67726652b608908c1ca9bb', 'txlist_hash': '764ca9e8d3b9546d1c4ff441a39594548989f60daefc6f28e046996e76a273bf'},
    335000: {'ledger_hash': 'e57c9d606a615e7e09bf99148596dd28e64b25cd8b081e226d535a64c1ed08d1', 'txlist_hash': '437d9507185b5e193627edf4998aad2264755af8d13dd3948ce119b32dd50ce2'},
    336000: {'ledger_hash': '1329ff5b80d034b64f6ea3481b7c7176437a8837b2a7cb7b8a265fdd1397572d', 'txlist_hash': '33eb8cacd4c750f8132d81e8e43ca13bd565f1734d7d182346364847414da52f'},
    337000: {'ledger_hash': '607e6a93e8d97cefea9bd55384898ee90c8477ded8a46017f2294feedbc83409', 'txlist_hash': '20b535a55abcc902ca70c19dd648cbe5149af8b4a4157b94f41b71fc422d428e'},
    338000: {'ledger_hash': 'f043914c71e4b711abb1c1002767b9a4e7d605e249facaaf7a2046b0e9741204', 'txlist_hash': 'fa2c3f7f76345278271ed5ec391d582858e10b1f154d9b44e5a1f4896400ee46'},
    339000: {'ledger_hash': '49f7240bc90ebc2f242dd599c7d2c427b9d2ac844992131e6e862b638ae4393a', 'txlist_hash': 'c1e3b497c054dcf67ddd0dc223e8b8a6e09a1a05bacb9fef5c03e48bd01e64e7'},
    340000: {'ledger_hash': '255760e2abfb79fdd76b65759f1590f582c1747f3eeccc4b2ae37d23e30e0729', 'txlist_hash': '8502004bb63e699b243ac8af072d704c69b817905e74787c2031af971e8cd87c'},
    341000: {'ledger_hash': '1369cba3909e564d2e725879a8b2cd987df075db121d1d421c8ce16b65f4bf04', 'txlist_hash': 'd217d0bed190cb27f58fcb96b255f8006bc4b9ed739e1bb08507201c49c426c8'},
    342000: {'ledger_hash': '9e7e9b8620717189ccea697ff2f84fe71bc4ae8d991481ff235164d72a9e6e4f', 'txlist_hash': 'adf75d023760101b2b337f6359dd811b12521c83837eb3f7db3bbfd0b095aa54'},
    343000: {'ledger_hash': 'aa47312ebe94b35504bec6c74713e404e5f36854e0836839344d13debe50558c', 'txlist_hash': '6bdbbc96364b3c92cea132fe66a0925f9445a249f7062326bdcc4ad4711f0c01'},
    344000: {'ledger_hash': '40187263aa96d1362bf7b19c8ba0fff7f0c0f3eb132a40fc90601b5926c7e6e3', 'txlist_hash': '98da8efe705c4b54275bfd25f816a7e7a4ff1f67647e17d7a0aaa2a3fef8bda0'},
    345000: {'ledger_hash': 'e4a1e1be4beea63d9740ca166b75bb4e3ffa2af33e1fe282e5b09c4952a7448c', 'txlist_hash': '777f163eaa5ad79dcb738871d4318a0699defec469d8afe91ab6277ff8d3e8b8'},
    350000: {'ledger_hash': '6a67e9f2e9d07e7bb3277cf9c24f84c857ed1b8fff4a37e589cd56ade276dd95', 'txlist_hash': '96bcbdbce74b782a845d4fda699846d2d3744044c2870a413c018642b8c7c3bf'},
    355000: {'ledger_hash': 'a84b17992217c7845e133a8597dac84eba1ee8c48bcc7f74bcf512837120f463', 'txlist_hash': '210d96b42644432b9e1a3433a29af9acb3bad212b67a7ae1dbc011a11b04bc24'},
    360000: {'ledger_hash': 'ddca07ea43b336b703fb8ebab6c0dc30582eb360d6f0eb0446e1fe58b53dee0a', 'txlist_hash': '31d0ff3e3782cf9464081829c5595b3de5ac477290dc069d98672f3f552767f8'},
    365000: {'ledger_hash': '2d55b126cca3eca15c07b5da683988f9e01d7346d2ca430e940fd7c07ce84fd7', 'txlist_hash': '7988a823cc1e3234953cc87d261d3c1fede8493d0a31b103357eb23cc7dc2eda'},
    366000: {'ledger_hash': '64ce274df2784f9ca88a8d7071613ec6527e506ec31cd434eca64c6a3345a6b7', 'txlist_hash': '0d4374da6100e279b24f4ba4a2d6afbfc4fb0fc2d312330a515806e8c5f49404'},
    370000: {'ledger_hash': 'fabb2a2e91fad3fe7734169d554cca396c1030243044cef42fcf65717cf0fa61', 'txlist_hash': '41d1732868c9ac25951ace5ca9f311a15d5eca9bf8d548e0d988c050bd2aff87'},
    375000: {'ledger_hash': 'a7ac4e2948cea0c426c8fc201cf57d9c313027ea7bff2b32a25ed28d3dbaa581', 'txlist_hash': '96118a7aa2ca753488755b7419a0f44a7fbc371bc58dcc7ab083c70fc14ef8b3'},
    380000: {'ledger_hash': '70453ba04c1c0198c4771e7964cffa25f9456c2f71456a8b05dfe935d5fcdc88', 'txlist_hash': '8bf2070103cca6f0bde507b7d20b0ba0630da6349beb560fa64c926d08dbcaef'},
    385000: {'ledger_hash': '93eb0a6e820bee197e7591edbc5ead7bfa38f32c88aabf4785f080fd6ae96c4c', 'txlist_hash': '1f8f17fd5766382a8c10a2a0e995a7d5a5d1bcd5fc0220d1e2691b2a94dcc78f'},
    390000: {'ledger_hash': '7d42b98eecbc910a67a5f4ac8dc7d6d9b6995ebc5bdf53663b414965fe7d2c5e', 'txlist_hash': 'b50efc4a4241bf3ec33a38c3b5f34756a9f305fe5fa9a80f7f9b70d5d7b2a780'},
    395000: {'ledger_hash': '89f9ac390b35e69dd75d6c34854ba501dce2f662fc707aee63cad5822c7660f2', 'txlist_hash': '2151dd2f0aa14685f3d041727a689d5d242578072a049123b317724fc4f1100c'},
    400000: {'ledger_hash': 'eb681a305125e04b6f044b36045e23ee248ce4eb68433cea2b36d15e7e74d5f1', 'txlist_hash': 'b48e9501e8d6f1f1b4127d868860885d3db76698c2c31a567777257df101cf61'},
    405000: {'ledger_hash': '3725055b37a8958ade6ca1c277cf50fee6036b4a92befb8da2f7c32f0b210881', 'txlist_hash': '871b2adfd246e3fe69f0fe9098e3251045ed6e9712c4cf90ea8dfdd1eb330ed6'},
    410000: {'ledger_hash': '1fa9a34f233695ebd7ebb08703bf8d99812fa099f297efc5d307d1ebef902ffd', 'txlist_hash': 'ee3bd84c728a37e2bbe061c1539c9ee6d71db18733b1ed53ee8d320481f55030'},
    415000: {'ledger_hash': '6772a8a1c784db14c0bf111e415919c9da4e5ca142be0b9e323c82c1b13c74e0', 'txlist_hash': 'cfb81785cd48e9ba0e54fee4d62f49b347489da82139fd5e1555ae0bc11a33d5'},
    420000: {'ledger_hash': '42167117e16943f44bb8117aa0a39bed2d863a454cd694d0bc5006a7aab23b06', 'txlist_hash': 'a1139870bef8eb9bbe60856029a4f01fce5432eb7aeacd088ba2e033757b86e3'},
}

CONSENSUS_HASH_VERSION_TESTNET = 7
CHECKPOINTS_TESTNET = {
    config.BLOCK_FIRST_TESTNET: {'ledger_hash': '63f0fef31d02da85fa779e9a0e1b585b1a6a4e59e14564249e288e074e91c223', 'txlist_hash': '63f0fef31d02da85fa779e9a0e1b585b1a6a4e59e14564249e288e074e91c223'},
    316000: {'ledger_hash': 'f645e6877da416b8b91670ac927df686c5ea6fc1158c150ae49d594222ed504c', 'txlist_hash': '3e29bcbf3873326097024cc26e9296f0164f552dd79c2ee7cfc344e6d64fa87d'},
    319000: {'ledger_hash': '384ca28ac56976bc24a6ab7572b41bc61474e6b87fdee814135701d6a8f5c8a2', 'txlist_hash': '6c05c98418a6daa6de82dd59e000d3f3f5407c5432d4ab7d76047873a38e4d4b'},
    322000: {'ledger_hash': 'f4015c37eb4f31ac42083fd0389cde4868acb5353d3f3abfe2f3a88aba8cae72', 'txlist_hash': '18f278154e9bc3bbcc39da905ab4ad3023742ab7723b55b0fd1c58c36cd3e9bf'},
    325000: {'ledger_hash': 'd7f70a927f5aeed38e559ddc0bc4697601477ea43cde928ad228fefc195b02da', 'txlist_hash': '1a60e38664b39e0f501b3e5a60c6fc0bd4ed311b74872922c2dde4cb2267fd3e'},
    329000: {'ledger_hash': '96637b4400cbe084c2c4f139f59b5bc16770815e96306423aaeb2b2677a9a657', 'txlist_hash': '79d577d8fbba0ad6ae67829dfa5824f758286ccd429d65b7d1d42989134d5b57'},
    350000: {'ledger_hash': 'cae8fec787bba3d5c968a8f4b6fb22a54c96d5acbeadd0425f6b20c3a8813ea3', 'txlist_hash': '097df9c3079df4d96f59518df72492dfd7a79716462e3a4a30d62a37aec6fc16'},
    400000: {'ledger_hash': '94abfd9c00c8462c155f64011e71af141b7d524e17de5aeda26b7469fe79b5f0', 'txlist_hash': 'a9fc42b69f80ec69f3f98e8a3cd81f4f946544fd0561a62a0891254c16970a87'},
    450000: {'ledger_hash': '09eb9f2aa605ce77225362b4b556284acdd9f6d3bc273372dfae4a5be9e9b035', 'txlist_hash': '05af651c1de49d0728834991e50000fbf2286d7928961b71917f682a0f2b7171'},
    500000: {'ledger_hash': '85f3bca8c88246ddfa1a5ec327e71f0696c182ed2a5fedf3712cd2e87e2661ac', 'txlist_hash': '663b34955116a96501e0c1c27f27d24bad7d45995913367553c5cfe4b8b9d0a9'},
    550000: {'ledger_hash': 'c143026133af2d83bc49ef205b4623194466ca3e7c79f95da2ad565359ccb5ad', 'txlist_hash': '097b8bca7a243e0b9bdf089f34de15bd2dcd4727fb4e88aae7bfd96302250326'},
    600000: {'ledger_hash': '82caf720967d0e43a1c49a6c75f255d9056ed1bffe3f96d962478faccdaba8ff', 'txlist_hash': '0d99f42184233426d70102d5ac3c80aaecf804d441a8a0d0ef26038d333ab7a7'},
    650000: {'ledger_hash': 'bef100ae7d5027a8b3f32416c4f26e1f16b21cee2a986c57be1466a3ba338051', 'txlist_hash': '409ed86e4274b511193d187df92e433c734dcc890bf93496e7a7dee770e7035e'},
    700000: {'ledger_hash': 'afe5e9c3f3a8c6f19c4f9feaf09df051c28202c6bae64f3563a09ffea9e79a6e', 'txlist_hash': '4f9765158855d24950c7e076615b0ad5b72738d4d579decfd3b93c998edf4fcb'},
    750000: {'ledger_hash': 'e7c7969a6156facb193b77ef71b5e3fac49c6998e5a94ec3b90292be10ece9cc', 'txlist_hash': '6e511790656d3ffec0c912d697e5d1c2a4e401a1606203c77ab5a5855891bc2c'},
    800000: {'ledger_hash': '42a7c679e51e5e8d38df26b67673b4850e8e6f72723aa19673b3219fcc02b77b', 'txlist_hash': '885ae1e6c21f5fb3645231aaa6bb6910fc21a0ae0ca5dbe9a4011f3b5295b3e7'},
    850000: {'ledger_hash': '35b2a2ab4a8bfbc321d4545292887b4ccaea73415c7674f795aefa6e240890eb', 'txlist_hash': '72d5cfe1e729a22da9eacd4d7752c881c43a191904556b65a0fae82b770dcdf3'},
    900000: {'ledger_hash': 'a5552b4998d2e5a516b9310d6592e7368771c1ad3b6e6330f6bc0baa3db31643', 'txlist_hash': '5a2e9fbd9b52ee32b8e8bfff993ed92dc22510aa7448277a704176cf01e55b04'},
    950000: {'ledger_hash': '5a5e78b55ac294690229abff7ff8f74f390f3a47dc4d08a0bac40e2e89a5bed2', 'txlist_hash': 'f4fa9838fb38d3e5beffb760fae022dcc59c61c506dd28ac83ee48ba814d04b2'},
    1000000: {'ledger_hash': 'eafca6700b9fd8f3992f8a18316e9ad59480ef74a4e7737793c101878aba8e1a', 'txlist_hash': '03deb626e031f30acd394bf49c35e11a487cb11e55dff5ba9a3f6d04b460c7de'},
    1050000: {'ledger_hash': '8012ebaf4c6638173e88ecd3e7bb2242ab88a9bdf877fc32c42dbcd7d2d3bab1', 'txlist_hash': '896274fdba957961083b07b80634126bc9f0434b67d723ed1fa83157ce5cd9a7'},
    1100000: {'ledger_hash': '76357f917235daa180c904cdf5c44366eef3e33539b7b0ba6a38f89582e82d22', 'txlist_hash': '36ecfd4b07f23176cd6960bc0adef97472c13793e53ac3df0eea0dd2e718a570'},
    1150000: {'ledger_hash': '5924f004bfdc3be449401c764808ebced542d2e06ba30c5984830292d1a926aa', 'txlist_hash': '9ff139dacf4b04293074e962153b972d25fa16d862dae05f7f3acc15e83c4fe8'},
    1200000: {'ledger_hash': 'a3d009bd2e0b838c185b8866233d7b4edaff87e5ec4cc4719578d1a8f9f8fe34', 'txlist_hash': '11dcf3a0ab714f05004a4e6c77fe425eb2a6427e4c98b7032412ab29363ffbb2'},
    1250000: {'ledger_hash': '37244453b4eac67d1dbfc0f60116cac90dab7b814d756653ad3d9a072fbac61a', 'txlist_hash': 'c01ed3113f8fd3a6b54f5cefafd842ebf7c314ce82922e36236414d820c5277a'},
    1300000: {'ledger_hash': 'a83c1cd582604130fd46f1304560caf0f4e3300f3ce7c3a89824b8901f13027f', 'txlist_hash': '67e663b75a80940941b8370ada4985be583edaa7ba454d49db9a864a7bb7979c'},
    1350000: {'ledger_hash': 'f96e6aff578896a4568fb69f72aa0a8b52eb9ebffefca4bd7368790341cd821d', 'txlist_hash': '83e7d31217af274b13889bd8b9f8f61afcd7996c2c8913e9b53b1d575f54b7c1'},
    1400000: {'ledger_hash': '85a23f6fee9ce9c80fa335729312183ff014920bbf297095ac77c4105fb67e17', 'txlist_hash': 'eee762f34a3f82e6332c58e0c256757d97ca308719323af78bf5924f08463e12'},
}

CONSENSUS_HASH_VERSION_REGTEST = 1
CHECKPOINTS_REGTEST = {
    config.BLOCK_FIRST_REGTEST: {'ledger_hash': '33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5', 'txlist_hash': '33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5'},
}

class ConsensusError(Exception):
    pass

def consensus_hash(db, field, previous_consensus_hash, content):
    cursor = db.cursor()
    block_index = util.CURRENT_BLOCK_INDEX

    # Initialise previous hash on first block.
    if block_index <= config.BLOCK_FIRST:
        assert not previous_consensus_hash
        previous_consensus_hash = util.dhash_string(CONSENSUS_HASH_SEED)

    # Get previous hash.
    if not previous_consensus_hash:
        try:
            previous_consensus_hash = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index - 1,)))[0][field]
        except IndexError:
            previous_consensus_hash = None
        if not previous_consensus_hash:
            raise ConsensusError('Empty previous {} for block {}. Please launch a `reparse`.'.format(field, block_index))

    # Calculate current hash.
    if config.TESTNET:
        consensus_hash_version = CONSENSUS_HASH_VERSION_TESTNET
    elif config.REGTEST:
        consensus_hash_version = CONSENSUS_HASH_VERSION_REGTEST
    else:
        consensus_hash_version = CONSENSUS_HASH_VERSION_MAINNET

    calculated_hash = util.dhash_string(previous_consensus_hash + '{}{}'.format(consensus_hash_version, ''.join(content)))

    # Verify hash (if already in database) or save hash (if not).
    # NOTE: do not enforce this for messages_hashes, those are more informational (for now at least)
    found_hash = list(cursor.execute('''SELECT * FROM blocks WHERE block_index = ?''', (block_index,)))[0][field] or None
    if found_hash and field != 'messages_hash':
        # Check against existing value.
        if calculated_hash != found_hash:
            raise ConsensusError('Inconsistent {} for block {} (calculated {}, vs {} in database).'.format(
                field, block_index, calculated_hash, found_hash))
    else:
        # Save new hash.
        cursor.execute('''UPDATE blocks SET {} = ? WHERE block_index = ?'''.format(field), (calculated_hash, block_index))

    # Check against checkpoints.
    if config.TESTNET:
        checkpoints = CHECKPOINTS_TESTNET
    elif config.REGTEST:
        checkpoints = CHECKPOINTS_REGTEST
    else:
        checkpoints = CHECKPOINTS_MAINNET

    if field != 'messages_hash' and block_index in checkpoints and checkpoints[block_index][field] != calculated_hash:
        raise ConsensusError('Incorrect {} hash for block {}.  Calculated {} but expected {}'.format(field, block_index, calculated_hash, checkpoints[block_index][field],))

    return calculated_hash, found_hash

class SanityError(Exception):
    pass

def asset_conservation(db):
    logger.debug('Checking for conservation of assets.')
    supplies = util.supplies(db)
    held = util.held(db)
    for asset in supplies.keys():
        asset_issued = supplies[asset]
        asset_held = held[asset] if asset in held and held[asset] != None else 0
        if asset_issued != asset_held:
            raise SanityError('{} {} issued ≠ {} {} held'.format(util.value_out(db, asset_issued, asset), asset, util.value_out(db, asset_held, asset), asset))
        logger.debug('{} has been conserved ({} {} both issued and held)'.format(asset, util.value_out(db, asset_issued, asset), asset))

class VersionError(Exception):
    pass
class VersionUpdateRequiredError(VersionError):
    pass

def check_change(protocol_change, change_name):

    # Check client version.
    passed = True
    if config.VERSION_MAJOR < protocol_change['minimum_version_major']:
        passed = False
    elif config.VERSION_MAJOR == protocol_change['minimum_version_major']:
        if config.VERSION_MINOR < protocol_change['minimum_version_minor']:
            passed = False
        elif config.VERSION_MINOR == protocol_change['minimum_version_minor']:
            if config.VERSION_REVISION < protocol_change['minimum_version_revision']:
                passed = False

    if not passed:
        explanation = 'Your version of {} is v{}, but, as of block {}, the minimum version is v{}.{}.{}. Reason: ‘{}’. Please upgrade to the latest version and restart the server.'.format(
            config.APP_NAME, config.VERSION_STRING, protocol_change['block_index'], protocol_change['minimum_version_major'], protocol_change['minimum_version_minor'],
            protocol_change['minimum_version_revision'], change_name)
        if util.CURRENT_BLOCK_INDEX >= protocol_change['block_index']:
            raise VersionUpdateRequiredError(explanation)
        else:
            warnings.warn(explanation)

def software_version():
    if config.FORCE:
        return
    logger.debug('Checking version.')

    try:
        host = 'https://counterpartyxcp.github.io/counterparty-lib/counterpartylib/protocol_changes.json'
        response = requests.get(host, headers={'cache-control': 'no-cache'})
        versions = json.loads(response.text)
    except (requests.exceptions.ConnectionError, ConnectionRefusedError, ValueError) as e:
        logger.warning('Unable to check version! ' + str(sys.exc_info()[1]))
        return

    for change_name in versions:
        protocol_change = versions[change_name]
        try:
            check_change(protocol_change, change_name)
        except VersionUpdateRequiredError as e:
            logger.error("Version Update Required", exc_info=sys.exc_info())
            sys.exit(config.EXITCODE_UPDATE_REQUIRED)

    logger.debug('Version check passed.')


class DatabaseVersionError(Exception):
    def __init__(self, message, reparse_block_index):
        super(DatabaseVersionError, self).__init__(message)
        self.reparse_block_index = reparse_block_index

def database_version(db):
    if config.FORCE:
        return
    logger.debug('Checking database version.')

    version_major, version_minor = database.version(db)
    if version_major != config.VERSION_MAJOR:
        # Rollback database if major version has changed.
        raise DatabaseVersionError('Client major version number mismatch ({} ≠ {}).'.format(version_major, config.VERSION_MAJOR), config.BLOCK_FIRST)
    elif version_minor != config.VERSION_MINOR:
        # Reparse all transactions if minor version has changed.
        raise DatabaseVersionError('Client minor version number mismatch ({} ≠ {}).'.format(version_minor, config.VERSION_MINOR), None)

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### config.py 
 
import sys

"""Variables prefixed with `DEFAULT` should be able to be overridden by
configuration file and command‐line arguments."""

UNIT = 100000000        # The same across assets.


# Versions
VERSION_MAJOR = 9
VERSION_MINOR = 60
VERSION_REVISION = 1
VERSION_STRING = str(VERSION_MAJOR) + '.' + str(VERSION_MINOR) + '.' + str(VERSION_REVISION)


# Counterparty protocol
TXTYPE_FORMAT = '>I'
SHORT_TXTYPE_FORMAT = 'B'

TWO_WEEKS = 2 * 7 * 24 * 3600
MAX_EXPIRATION = 4 * 2016   # Two months

MEMPOOL_BLOCK_HASH = 'mempool'
MEMPOOL_BLOCK_INDEX = 9999999


# SQLite3
MAX_INT = 2**63 - 1


# Bitcoin Core
OP_RETURN_MAX_SIZE = 80  # bytes


# Currency agnosticism
BTC = 'BTC'
XCP = 'XCP'

BTC_NAME = 'Bitcoin'
XCP_NAME = 'Counterparty'
APP_NAME = XCP_NAME.lower()

DEFAULT_RPC_PORT_REGTEST = 24000
DEFAULT_RPC_PORT_TESTNET = 14000
DEFAULT_RPC_PORT = 4000

DEFAULT_BACKEND_PORT_REGTEST = 28332
DEFAULT_BACKEND_PORT_TESTNET = 18332
DEFAULT_BACKEND_PORT = 8332

DEFAULT_INDEXD_PORT_REGTEST = 28432
DEFAULT_INDEXD_PORT_TESTNET = 18432
DEFAULT_INDEXD_PORT = 8432

UNSPENDABLE_REGTEST = 'mvCounterpartyXXXXXXXXXXXXXXW24Hef'
UNSPENDABLE_TESTNET = 'mvCounterpartyXXXXXXXXXXXXXXW24Hef'
UNSPENDABLE_MAINNET = '1CounterpartyXXXXXXXXXXXXXXXUWLpVr'

ADDRESSVERSION_TESTNET = b'\x6f'
P2SH_ADDRESSVERSION_TESTNET = b'\xc4'
PRIVATEKEY_VERSION_TESTNET = b'\xef'
ADDRESSVERSION_MAINNET = b'\x00'
P2SH_ADDRESSVERSION_MAINNET = b'\x05'
PRIVATEKEY_VERSION_MAINNET = b'\x80'
ADDRESSVERSION_REGTEST = b'\x6f'
P2SH_ADDRESSVERSION_REGTEST = b'\xc4'
PRIVATEKEY_VERSION_REGTEST = b'\xef'
MAGIC_BYTES_TESTNET = b'\xfa\xbf\xb5\xda'   # For bip-0010
MAGIC_BYTES_MAINNET = b'\xf9\xbe\xb4\xd9'   # For bip-0010
MAGIC_BYTES_REGTEST = b'\xda\xb5\xbf\xfa'

BLOCK_FIRST_TESTNET_TESTCOIN = 310000
BURN_START_TESTNET_TESTCOIN = 310000
BURN_END_TESTNET_TESTCOIN = 4017708     # Fifty years, at ten minutes per block.

BLOCK_FIRST_TESTNET = 310000
BLOCK_FIRST_TESTNET_HASH = '000000001f605ec6ee8d2c0d21bf3d3ded0a31ca837acc98893876213828989d'
BURN_START_TESTNET = 310000
BURN_END_TESTNET = 4017708              # Fifty years, at ten minutes per block.

BLOCK_FIRST_MAINNET_TESTCOIN = 278270
BURN_START_MAINNET_TESTCOIN = 278310
BURN_END_MAINNET_TESTCOIN = 2500000     # A long time.

BLOCK_FIRST_MAINNET = 278270
BLOCK_FIRST_MAINNET_HASH = '00000000000000017bac9a8e85660ad348050c789922d5f8fe544d473368be1a'
BURN_START_MAINNET = 278310
BURN_END_MAINNET = 283810

BLOCK_FIRST_REGTEST = 0
BLOCK_FIRST_REGTEST_HASH = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'
BURN_START_REGTEST = 101
BURN_END_REGTEST = 150000000

BLOCK_FIRST_REGTEST_TESTCOIN = 0
BURN_START_REGTEST_TESTCOIN = 101
BURN_END_REGTEST_TESTCOIN = 150

# Protocol defaults
# NOTE: If the DUST_SIZE constants are changed, they MUST also be changed in counterblockd/lib/config.py as well
DEFAULT_REGULAR_DUST_SIZE = 546          # TODO: Revisit when dust size is adjusted in bitcoin core
DEFAULT_MULTISIG_DUST_SIZE = 7800        # <https://bitcointalk.org/index.php?topic=528023.msg7469941#msg7469941>
DEFAULT_OP_RETURN_VALUE = 0
DEFAULT_FEE_PER_KB_ESTIMATE_SMART = 1024
DEFAULT_FEE_PER_KB = 25000               # sane/low default, also used as minimum when estimated fee is used
ESTIMATE_FEE_PER_KB = True               # when True will use `estimatesmartfee` from bitcoind instead of DEFAULT_FEE_PER_KB
ESTIMATE_FEE_CONF_TARGET = 3
ESTIMATE_FEE_MODE = 'CONSERVATIVE'

# UI defaults
DEFAULT_FEE_FRACTION_REQUIRED = .009   # 0.90%
DEFAULT_FEE_FRACTION_PROVIDED = .01    # 1.00%


DEFAULT_REQUESTS_TIMEOUT = 20   # 20 seconds
DEFAULT_RPC_BATCH_SIZE = 20     # A 1 MB block can hold about 4200 transactions.

# Custom exit codes
EXITCODE_UPDATE_REQUIRED = 5


DEFAULT_CHECK_ASSET_CONSERVATION = True

BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 20000
BACKEND_RPC_BATCH_NUM_WORKERS = 6

UNDOLOG_MAX_PAST_BLOCKS = 100 #the number of past blocks that we store undolog history

DEFAULT_UTXO_LOCKS_MAX_ADDRESSES = 1000
DEFAULT_UTXO_LOCKS_MAX_AGE = 3.0 #in seconds

ADDRESS_OPTION_REQUIRE_MEMO = 1
ADDRESS_OPTION_MAX_VALUE = ADDRESS_OPTION_REQUIRE_MEMO # Or list of all the address options
OLD_STYLE_API = True

API_LIMIT_ROWS = 1000
MEMPOOL_TXCOUNT_UPDATE_LIMIT=60000

MPMA_LIMIT = 1000

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### database.py 
 
import apsw
import logging
logger = logging.getLogger(__name__)
import time
import collections
import copy

from counterpartylib.lib import config
from counterpartylib.lib import util
from counterpartylib.lib import exceptions
from counterpartylib.lib import log

BLOCK_MESSAGES = []

def rowtracer(cursor, sql):
    """Converts fetched SQL data into dict-style"""
    dictionary = {}
    for index, (name, type_) in enumerate(cursor.getdescription()):
        dictionary[name] = sql[index]
    return dictionary

def exectracer(cursor, sql, bindings):
    # This means that all changes to database must use a very simple syntax.
    # TODO: Need sanity checks here.
    sql = sql.lower()

    if sql.startswith('create trigger') or sql.startswith('drop trigger'):
        #CREATE TRIGGER stmts may include an "insert" or "update" as part of them
        return True

    # Parse SQL.
    array = sql.split('(')[0].split(' ')
    command = array[0]
    if 'insert' in sql:
        category = array[2]
    elif 'update' in sql:
        category = array[1]
    else:
        #CREATE TABLE, etc
        return True

    db = cursor.getconnection()
    dictionary = {'command': command, 'category': category, 'bindings': bindings}

    skip_tables = [
        'blocks', 'transactions',
        'balances', 'messages', 'mempool', 'assets',
        'new_sends', 'new_issuances' # interim table for CIP10 activation
    ]
    skip_tables_block_messages = copy.copy(skip_tables)
    if command == 'update':
        # List message manually.
        skip_tables += ['orders', 'bets', 'rps', 'order_matches', 'bet_matches', 'rps_matches']

    # Record alteration in database.
    if category not in skip_tables:
        log.message(db, bindings['block_index'], command, category, bindings)
    # Record alteration in computation of message feed hash for the block
    if category not in skip_tables_block_messages:
        # don't include asset_longname as part of the messages hash
        #   until subassets are enabled
        if category == 'issuances' and not util.enabled('subassets'):
            if isinstance(bindings, dict) and 'asset_longname' in bindings: del bindings['asset_longname']

        # don't include memo as part of the messages hash
        #   until enhanced_sends are enabled
        if category == 'sends' and not util.enabled('enhanced_sends'):
            if isinstance(bindings, dict) and 'memo' in bindings: del bindings['memo']

        sorted_bindings = sorted(bindings.items()) if isinstance(bindings, dict) else [bindings,]
        BLOCK_MESSAGES.append('{}{}{}'.format(command, category, sorted_bindings))

    return True

class DatabaseIntegrityError(exceptions.DatabaseError):
    pass
def get_connection(read_only=True, foreign_keys=True, integrity_check=True):
    """Connects to the SQLite database, returning a db `Connection` object"""
    logger.debug('Creating connection to `{}`.'.format(config.DATABASE))

    if read_only:
        db = apsw.Connection(config.DATABASE, flags=apsw.SQLITE_OPEN_READONLY)
    else:
        db = apsw.Connection(config.DATABASE)
    cursor = db.cursor()

    # For integrity, security.
    if foreign_keys and not read_only:
        logger.info('Checking database foreign keys...')
        cursor.execute('''PRAGMA foreign_keys = ON''')
        cursor.execute('''PRAGMA defer_foreign_keys = ON''')
        rows = list(cursor.execute('''PRAGMA foreign_key_check'''))
        if rows:
            for row in rows:
                logger.debug('Foreign Key Error: {}'.format(row))
            raise exceptions.DatabaseError('Foreign key check failed.')

        # So that writers don’t block readers.
        cursor.execute('''PRAGMA journal_mode = WAL''')
        logger.info('Foreign key check completed.')

    # Make case sensitive the `LIKE` operator.
    # For insensitive queries use 'UPPER(fieldname) LIKE value.upper()''
    cursor.execute('''PRAGMA case_sensitive_like = ON''')

    if integrity_check:
        logger.info('Checking database integrity...')
        integral = False
        for i in range(10): # DUPE
            try:
                cursor.execute('''PRAGMA integrity_check''')
                rows = cursor.fetchall()
                if not (len(rows) == 1 and rows[0][0] == 'ok'):
                    raise exceptions.DatabaseError('Integrity check failed.')
                integral = True
                break
            except DatabaseIntegrityError:
                time.sleep(1)
                continue
        if not integral:
            raise exceptions.DatabaseError('Could not perform integrity check.')
        logger.info('Integrity check completed.')

    db.setrowtrace(rowtracer)
    db.setexectrace(exectracer)

    cursor.close()
    return db

def version(db):
    cursor = db.cursor()
    user_version = cursor.execute('PRAGMA user_version').fetchall()[0]['user_version']
    # manage old user_version
    if user_version == config.VERSION_MINOR:
        version_minor = user_version
        version_major = config.VERSION_MAJOR
        user_version = (config.VERSION_MAJOR * 1000) + version_minor
        cursor.execute('PRAGMA user_version = {}'.format(user_version))
    else:
        version_minor = user_version % 1000
        version_major = user_version // 1000
    return version_major, version_minor

def update_version(db):
    cursor = db.cursor()
    user_version = (config.VERSION_MAJOR * 1000) + config.VERSION_MINOR
    cursor.execute('PRAGMA user_version = {}'.format(user_version)) # Syntax?!
    logger.info('Database version number updated.')

def vacuum(db):
    logger.info('Starting database VACUUM. This may take awhile...')
    cursor = db.cursor()
    cursor.execute('VACUUM')
    logger.info('Database VACUUM completed.')

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### exceptions.py 
 
#! /usr/bin/python3

class DatabaseError(Exception):
    pass

class TransactionError(Exception):
    pass

class ParseTransactionError(Exception):
    pass

class AssetError (Exception):
    pass

class AssetNameError(AssetError):
    pass

class AssetIDError(AssetError):
    pass

class MessageError(Exception):
    pass

class ComposeError(MessageError):
    pass

class UnpackError(MessageError):
    pass

class ValidateError(MessageError):
    pass

class DecodeError(MessageError):
    pass

class PushDataDecodeError(DecodeError):
    pass

class BTCOnlyError(MessageError):
    def __init__(self, msg, decodedTx=None):
        super(BTCOnlyError, self).__init__(msg)
        self.decodedTx = decodedTx

class BalanceError(Exception):
    pass

class EncodingError(Exception):
    pass

class OptionsError(Exception):
    pass

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### log.py 
 
import logging
logger = logging.getLogger(__name__)
import decimal
D = decimal.Decimal
import binascii
import collections
import json
import time
from datetime import datetime
from dateutil.tz import tzlocal
import os
from colorlog import ColoredFormatter

from counterpartylib.lib import config
from counterpartylib.lib import exceptions
from counterpartylib.lib import util

class ModuleLoggingFilter(logging.Filter):
    """
    module level logging filter (NodeJS-style), ie:
        filters="*,-counterpartylib.lib,counterpartylib.lib.api"

        will log:
         - counterpartycli.server
         - counterpartylib.lib.api

        but will not log:
         - counterpartylib.lib
         - counterpartylib.lib.backend.indexd
    """

    def __init__(self, filters):
        self.filters = str(filters).split(",")

        self.catchall = "*" in self.filters
        if self.catchall:
            self.filters.remove("*")

    def filter(self, record):
        """
        Determine if specified record should be logged or not
        """
        result = None

        for filter in self.filters:
            if filter[:1] == "-":
                if result is None and ModuleLoggingFilter.ismatch(record, filter[1:]):
                    result = False
            else:
                if ModuleLoggingFilter.ismatch(record, filter):
                    result = True

        if result is None:
            return self.catchall

        return result

    @classmethod
    def ismatch(cls, record, name):
        """
        Determine if the specified record matches the name, in the same way as original logging.Filter does, ie:
            'counterpartylib.lib' will match 'counterpartylib.lib.check'
        """
        nlen = len(name)
        if nlen == 0:
            return True
        elif name == record.name:
            return True
        elif record.name.find(name, 0, nlen) != 0:
            return False
        return record.name[nlen] == "."


ROOT_LOGGER = None
def set_logger(logger):
    global ROOT_LOGGER
    if ROOT_LOGGER is None:
        ROOT_LOGGER = logger


LOGGING_SETUP = False
LOGGING_TOFILE_SETUP = False
def set_up(logger, verbose=False, logfile=None, console_logfilter=None):
    global LOGGING_SETUP
    global LOGGING_TOFILE_SETUP

    def set_up_file_logging():
        assert logfile
        max_log_size = 20 * 1024 * 1024 # 20 MB
        if os.name == 'nt':
            from counterpartylib.lib import util_windows
            fileh = util_windows.SanitizedRotatingFileHandler(logfile, maxBytes=max_log_size, backupCount=5)
        else:
            fileh = logging.handlers.RotatingFileHandler(logfile, maxBytes=max_log_size, backupCount=5)
        fileh.setLevel(logging.DEBUG)
        LOGFORMAT = '%(asctime)s [%(levelname)s] %(message)s'
        formatter = logging.Formatter(LOGFORMAT, '%Y-%m-%d-T%H:%M:%S%z')
        fileh.setFormatter(formatter)
        logger.addHandler(fileh)

    if LOGGING_SETUP:
        if logfile and not LOGGING_TOFILE_SETUP:
             set_up_file_logging()
             LOGGING_TOFILE_SETUP = True
        logger.getChild('log.set_up').debug('logging already setup')
        return
    LOGGING_SETUP = True

    log_level = logging.DEBUG if verbose else logging.INFO
    logger.setLevel(log_level)

    # Console Logging
    console = logging.StreamHandler()
    console.setLevel(log_level)

    # only add [%(name)s] to LOGFORMAT if we're using console_logfilter
    LOGFORMAT = '%(log_color)s[%(asctime)s][%(levelname)s]' + ('' if console_logfilter is None else '[%(name)s]') + ' %(message)s%(reset)s'
    LOGCOLORS = {'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red'}
    formatter = ColoredFormatter(LOGFORMAT, "%Y-%m-%d %H:%M:%S", log_colors=LOGCOLORS)
    console.setFormatter(formatter)
    logger.addHandler(console)

    if console_logfilter:
        console.addFilter(ModuleLoggingFilter(console_logfilter))

    # File Logging
    if logfile:
        set_up_file_logging()
        LOGGING_TOFILE_SETUP = True

    # Quieten noisy libraries.
    requests_log = logging.getLogger("requests")
    requests_log.setLevel(log_level)
    requests_log.propagate = False
    urllib3_log = logging.getLogger('urllib3')
    urllib3_log.setLevel(log_level)
    urllib3_log.propagate = False

    # Disable InsecureRequestWarning
    import requests
    requests.packages.urllib3.disable_warnings()

def curr_time():
    return int(time.time())

def isodt (epoch_time):
    try:
        return datetime.fromtimestamp(epoch_time, tzlocal()).isoformat()
    except OSError:
        return '<datetime>'

def message(db, block_index, command, category, bindings, tx_hash=None):
    cursor = db.cursor()

    # Get last message index.
    messages = list(cursor.execute('''SELECT * FROM messages
                                      WHERE message_index = (SELECT MAX(message_index) from messages)'''))
    if messages:
        assert len(messages) == 1
        message_index = messages[0]['message_index'] + 1
    else:
        message_index = 0

    # Not to be misleading…
    if block_index == config.MEMPOOL_BLOCK_INDEX:
        try:
            del bindings['status']
            del bindings['block_index']
            del bindings['tx_index']
        except KeyError:
            pass

    # Handle binary data.
    items = []
    for item in sorted(bindings.items()):
        if type(item[1]) == bytes:
            items.append((item[0], binascii.hexlify(item[1]).decode('ascii')))
        else:
            items.append(item)

    bindings_string = json.dumps(collections.OrderedDict(items))
    cursor.execute('insert into messages values(:message_index, :block_index, :command, :category, :bindings, :timestamp)',
                   (message_index, block_index, command, category, bindings_string, curr_time()))

    # Log only real transactions.
    if block_index != config.MEMPOOL_BLOCK_INDEX:
        log(db, command, category, bindings)

    cursor.close()


def log (db, command, category, bindings):

    cursor = db.cursor()

    for element in bindings.keys():
        try:
            str(bindings[element])
        except KeyError:
            bindings[element] = '<Error>'

    # Slow?!
    def output (quantity, asset):
        try:
            if asset not in ('fraction', 'leverage'):
                return str(util.value_out(db, quantity, asset)) + ' ' + asset
            else:
                return str(util.value_out(db, quantity, asset))
        except exceptions.AssetError:
            return '<AssetError>'
        except decimal.DivisionByZero:
            return '<DivisionByZero>'
        except TypeError:
            return '<None>'

    if command == 'update':
        if category == 'order':
            logger.debug('Database: set status of order {} to {}.'.format(bindings['tx_hash'], bindings['status']))
        elif category == 'bet':
            logger.debug('Database: set status of bet {} to {}.'.format(bindings['tx_hash'], bindings['status']))
        elif category == 'order_matches':
            logger.debug('Database: set status of order_match {} to {}.'.format(bindings['order_match_id'], bindings['status']))
        elif category == 'bet_matches':
            logger.debug('Database: set status of bet_match {} to {}.'.format(bindings['bet_match_id'], bindings['status']))
        elif category == 'dispensers':
            escrow_quantity = ''
            divisible = get_asset_info(cursor, bindings['asset'])['divisible']
            
            if divisible:
                if "escrow_quantity" in bindings:
                    escrow_quantity = "{:.8f}".format(bindings["escrow_quantity"]/config.UNIT)                 
        
            if ("action" in bindings) and bindings["action"] == 'refill dispenser':
                logger.info("Dispenser: {} refilled a dispenser with {} {}".format(bindings["source"],escrow_quantity,bindings["asset"]))
            elif "prev_status" in bindings: #There was a dispense
                if bindings["prev_status"] == 0:
                    if bindings["status"] == 10:
                        logger.info("Dispenser: {} closed dispenser for {} (dispenser empty)".format(bindings["source"],bindings["asset"]))
            elif bindings["status"] == 10: #Address closed the dispenser
                logger.info("Dispenser: {} closed dispenser for {} (operator closed)".format(bindings["source"],bindings["asset"]))
        # TODO: elif category == 'balances':
            # logger.debug('Database: set balance of {} in {} to {}.'.format(bindings['address'], bindings['asset'], output(bindings['quantity'], bindings['asset']).split(' ')[0]))

    elif command == 'insert':

        if category == 'credits':
            logger.debug('Credit: {} to {} #{}# <{}>'.format(output(bindings['quantity'], bindings['asset']), bindings['address'], bindings['action'], bindings['event']))

        elif category == 'debits':
            logger.debug('Debit: {} from {} #{}# <{}>'.format(output(bindings['quantity'], bindings['asset']), bindings['address'], bindings['action'], bindings['event']))

        elif category == 'sends':
            logger.info('Send: {} from {} to {} ({}) [{}]'.format(output(bindings['quantity'], bindings['asset']), bindings['source'], bindings['destination'], bindings['tx_hash'], bindings['status']))

        elif category == 'orders':
            logger.info('Order: {} ordered {} for {} in {} blocks, with a provided fee of {:.8f} {} and a required fee of {:.8f} {} ({}) [{}]'.format(bindings['source'], output(bindings['give_quantity'], bindings['give_asset']), output(bindings['get_quantity'], bindings['get_asset']), bindings['expiration'], bindings['fee_provided'] / config.UNIT, config.BTC, bindings['fee_required'] / config.UNIT, config.BTC, bindings['tx_hash'], bindings['status']))

        elif category == 'order_matches':
            logger.info('Order Match: {} for {} ({}) [{}]'.format(output(bindings['forward_quantity'], bindings['forward_asset']), output(bindings['backward_quantity'], bindings['backward_asset']), bindings['id'], bindings['status']))

        elif category == 'btcpays':
            logger.info('{} Payment: {} paid {} to {} for order match {} ({}) [{}]'.format(config.BTC, bindings['source'], output(bindings['btc_amount'], config.BTC), bindings['destination'], bindings['order_match_id'], bindings['tx_hash'], bindings['status']))

        elif category == 'issuances':
            if (get_asset_issuances_quantity(cursor, bindings["asset"]) == 0) or (bindings['quantity'] > 0): #This is the first issuance or the creation of more supply, so we have to log the creation of the token
                if bindings['divisible']:
                    divisibility = 'divisible'
                    unit = config.UNIT
                else:
                    divisibility = 'indivisible'
                    unit = 1
                try:
                    quantity = util.value_out(cursor, bindings['quantity'], None, divisible=bindings['divisible'])
                except Exception as e:
                    quantity = '?'
            
                if 'asset_longname' in bindings and bindings['asset_longname'] is not None:
                    logger.info('Subasset Issuance: {} created {} of {} subasset {} as numeric asset {} ({}) [{}]'.format(bindings['source'], quantity, divisibility, bindings['asset_longname'], bindings['asset'], bindings['tx_hash'], bindings['status']))
                else:
                    logger.info('Issuance: {} created {} of {} asset {} ({}) [{}]'.format(bindings['source'], quantity, divisibility, bindings['asset'], bindings['tx_hash'], bindings['status']))
            
            if bindings['locked']:
                lock_issuance = get_lock_issuance(cursor, bindings["asset"])
                
                if (lock_issuance == None) or (lock_issuance['tx_hash'] == bindings['tx_hash']):
                    logger.info('Issuance: {} locked asset {} ({}) [{}]'.format(bindings['source'], bindings['asset'], bindings['tx_hash'], bindings['status']))
            
            if bindings['transfer']:
                logger.info('Issuance: {} transfered asset {} to {} ({}) [{}]'.format(bindings['source'], bindings['asset'], bindings['issuer'], bindings['tx_hash'], bindings['status']))
            
        elif category == 'broadcasts':
            if bindings['locked']:
                logger.info('Broadcast: {} locked his feed ({}) [{}]'.format(bindings['source'], bindings['tx_hash'], bindings['status']))
            else:
                logger.info('Broadcast: ' + bindings['source'] + ' at ' + isodt(bindings['timestamp']) + ' with a fee of {}%'.format(output(D(bindings['fee_fraction_int'] / 1e8), 'fraction')) + ' (' + bindings['tx_hash'] + ')' + ' [{}]'.format(bindings['status']))

        elif category == 'bets':
            logger.info('Bet: {} against {}, by {}, on {}'.format(output(bindings['wager_quantity'], config.XCP), output(bindings['counterwager_quantity'], config.XCP), bindings['source'], bindings['feed_address']))

        elif category == 'bet_matches':
            placeholder = ''
            if bindings['target_value'] >= 0:    # Only non‐negative values are valid.
                placeholder = ' that ' + str(output(bindings['target_value'], 'value'))
            if bindings['leverage']:
                placeholder += ', leveraged {}x'.format(output(bindings['leverage'] / 5040, 'leverage'))
            logger.info('Bet Match: {} for {} against {} for {} on {} at {}{} ({}) [{}]'.format(util.BET_TYPE_NAME[bindings['tx0_bet_type']], output(bindings['forward_quantity'], config.XCP), util.BET_TYPE_NAME[bindings['tx1_bet_type']], output(bindings['backward_quantity'], config.XCP), bindings['feed_address'], isodt(bindings['deadline']), placeholder, bindings['id'], bindings['status']))

        elif category == 'dividends':
            logger.info('Dividend: {} paid {} per unit of {} ({}) [{}]'.format(bindings['source'], output(bindings['quantity_per_unit'], bindings['dividend_asset']), bindings['asset'], bindings['tx_hash'], bindings['status']))

        elif category == 'burns':
            logger.info('Burn: {} burned {} for {} ({}) [{}]'.format(bindings['source'], output(bindings['burned'], config.BTC), output(bindings['earned'], config.XCP), bindings['tx_hash'], bindings['status']))

        elif category == 'cancels':
            logger.info('Cancel: {} ({}) [{}]'.format(bindings['offer_hash'], bindings['tx_hash'], bindings['status']))

        elif category == 'rps':
            log_message = 'RPS: {} opens game with {} possible moves and a wager of {}'.format(bindings['source'], bindings['possible_moves'], output(bindings['wager'], 'XCP'))
            logger.info(log_message)

        elif category == 'rps_matches':
            log_message = 'RPS Match: {} is playing a {}-moves game with {} with a wager of {} ({}) [{}]'.format(bindings['tx0_address'], bindings['possible_moves'], bindings['tx1_address'], output(bindings['wager'], 'XCP'), bindings['id'], bindings['status'])
            logger.info(log_message)

        elif category == 'rpsresolves':

            if bindings['status'] == 'valid':
                rps_matches = list(cursor.execute('''SELECT * FROM rps_matches WHERE id = ?''', (bindings['rps_match_id'],)))
                assert len(rps_matches) == 1
                rps_match = rps_matches[0]
                log_message = 'RPS Resolved: {} is playing {} on a {}-moves game with {} with a wager of {} ({}) [{}]'.format(rps_match['tx0_address'], bindings['move'], rps_match['possible_moves'], rps_match['tx1_address'], output(rps_match['wager'], 'XCP'), rps_match['id'], rps_match['status'])
            else:
                log_message = 'RPS Resolved: {} [{}]'.format(bindings['tx_hash'], bindings['status'])
            logger.info(log_message)

        elif category == 'order_expirations':
            logger.info('Expired order: {}'.format(bindings['order_hash']))

        elif category == 'order_match_expirations':
            logger.info('Expired Order Match awaiting payment: {}'.format(bindings['order_match_id']))

        elif category == 'bet_expirations':
            logger.info('Expired bet: {}'.format(bindings['bet_hash']))

        elif category == 'bet_match_expirations':
            logger.info('Expired Bet Match: {}'.format(bindings['bet_match_id']))

        elif category == 'bet_match_resolutions':
            # DUPE
            cfd_type_id = util.BET_TYPE_ID['BullCFD'] + util.BET_TYPE_ID['BearCFD']
            equal_type_id = util.BET_TYPE_ID['Equal'] + util.BET_TYPE_ID['NotEqual']

            if bindings['bet_match_type_id'] == cfd_type_id:
                if bindings['settled']:
                    logger.info('Bet Match Settled: {} credited to the bull, {} credited to the bear, and {} credited to the feed address ({})'.format(output(bindings['bull_credit'], config.XCP), output(bindings['bear_credit'], config.XCP), output(bindings['fee'], config.XCP), bindings['bet_match_id']))
                else:
                    logger.info('Bet Match Force‐Liquidated: {} credited to the bull, {} credited to the bear, and {} credited to the feed address ({})'.format(output(bindings['bull_credit'], config.XCP), output(bindings['bear_credit'], config.XCP), output(bindings['fee'], config.XCP), bindings['bet_match_id']))

            elif bindings['bet_match_type_id'] == equal_type_id:
                logger.info('Bet Match Settled: {} won the pot of {}; {} credited to the feed address ({})'.format(bindings['winner'], output(bindings['escrow_less_fee'], config.XCP), output(bindings['fee'], config.XCP), bindings['bet_match_id']))

        elif category == 'rps_expirations':
            logger.info('Expired RPS: {}'.format(bindings['rps_hash']))

        elif category == 'rps_match_expirations':
            logger.info('Expired RPS Match: {}'.format(bindings['rps_match_id']))

        elif category == 'destructions':
            asset_info = get_asset_info(cursor, bindings['asset'])
            quantity = bindings['quantity']
            if asset_info['divisible']:
                quantity = "{:.8f}".format(quantity/config.UNIT)

            logger.info('Destruction: {} destroyed {} {} with tag ‘{}’({}) [{}]'.format(bindings['source'], quantity, bindings['asset'], bindings['tag'], bindings['tx_hash'], bindings['status']))

        elif category == 'dispensers':
            each_price = bindings['satoshirate']
            currency = config.BTC
            dispenser_label = 'dispenser'
            escrow_quantity = bindings['escrow_quantity']
            give_quantity = bindings['give_quantity']
            
            if (bindings['oracle_address'] != None) and util.enabled('oracle_dispensers'):
                each_price = "{:.2f}".format(each_price/100.0)
                oracle_last_price, oracle_fee, currency, oracle_last_updated = util.get_oracle_last_price(db, bindings['oracle_address'], bindings['block_index'])
                dispenser_label = 'oracle dispenser using {}'.format(bindings['oracle_address'])
            else:
                each_price = "{:.8f}".format(each_price/config.UNIT) 
            
            divisible = get_asset_info(cursor, bindings['asset'])['divisible']
            
            if divisible:
                escrow_quantity = "{:.8f}".format(escrow_quantity/config.UNIT) 
                give_quantity = "{:.8f}".format(give_quantity/config.UNIT) 
            
            if bindings['status'] == 0:
                logger.info('Dispenser: {} opened a {} for asset {} with {} balance, giving {} {} for each {} {}'.format(bindings['source'], dispenser_label, bindings['asset'], escrow_quantity, give_quantity, bindings['asset'], each_price, currency))
            elif bindings['status'] == 1:
                logger.info('Dispenser: {} (empty address) opened a {} for asset {} with {} balance, giving {} {} for each {} {}'.format(bindings['source'], dispenser_label, bindings['asset'], escrow_quantity, give_quantity, bindings['asset'], each_price, currency))
            elif bindings['status'] == 10:
                logger.info('Dispenser: {} closed a {} for asset {}'.format(bindings['source'], dispenser_label, bindings['asset']))

        elif category == 'dispenses':
            cursor.execute('SELECT * FROM dispensers WHERE tx_hash=:tx_hash', {
                'tx_hash': bindings['dispenser_tx_hash']
            })
            dispensers = cursor.fetchall()
            dispenser = dispensers[0]
        
            if (dispenser["oracle_address"] != None) and util.enabled('oracle_dispensers'):
                tx_btc_amount = get_tx_info(cursor, bindings['tx_hash'])/config.UNIT
                oracle_last_price, oracle_fee, oracle_fiat_label, oracle_last_price_updated = util.get_oracle_last_price(db, dispenser["oracle_address"], bindings['block_index'])
                fiatpaid = round(tx_btc_amount*oracle_last_price,2)
                
                logger.info('Dispense: {} from {} to {} for {:.8f} {} ({} {}) ({})'.format(output(bindings['dispense_quantity'], bindings['asset']), bindings['source'], bindings['destination'], tx_btc_amount, config.BTC, fiatpaid, oracle_fiat_label, bindings['tx_hash']))
            else:
                logger.info('Dispense: {} from {} to {} ({})'.format(output(bindings['dispense_quantity'], bindings['asset']), bindings['source'], bindings['destination'], bindings['tx_hash']))

    cursor.close()

def get_lock_issuance(cursor, asset):
    cursor.execute('''SELECT * FROM issuances \
        WHERE (status = ? AND asset = ? AND locked = ?)
        ORDER BY tx_index ASC''', ('valid', asset, True))
    issuances = cursor.fetchall()
    
    if len(issuances) > 0:
        return issuances[0]
    
    return None

def get_asset_issuances_quantity(cursor, asset):
    cursor.execute('''SELECT COUNT(*) AS issuances_count FROM issuances \
        WHERE (status = ? AND asset = ?)
        ORDER BY tx_index DESC''', ('valid', asset))
    issuances = cursor.fetchall()
    return issuances[0]['issuances_count']  

def get_asset_info(cursor, asset):
    if asset == config.BTC or asset == config.XCP:
        return {'divisible':True}
    
    cursor.execute('''SELECT * FROM issuances \
        WHERE (status = ? AND asset = ?)
        ORDER BY tx_index DESC''', ('valid', asset))
    issuances = cursor.fetchall()
    return issuances[0]

def get_tx_info(cursor, tx_hash):
    cursor.execute('SELECT * FROM transactions WHERE tx_hash=:tx_hash', {
        'tx_hash': tx_hash
    })
    transactions = cursor.fetchall()
    transaction = transactions[0]
    
    return transaction["btc_amount"]

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### message_type.py 
 
import logging
logger = logging.getLogger(__name__)
import struct

from counterpartylib.lib import config
from counterpartylib.lib import util

def pack(message_type_id, block_index=None):
    # pack message ID into 1 byte if not zero
    if util.enabled('short_tx_type_id', block_index) and message_type_id > 0 and message_type_id < 256:
        return struct.pack(config.SHORT_TXTYPE_FORMAT, message_type_id)
        
    # pack into 4 bytes
    return struct.pack(config.TXTYPE_FORMAT, message_type_id)

# retuns both the message type id and the remainder of the message data
def unpack(packed_data, block_index=None):
    message_type_id = None
    message_remainder = None

    if len(packed_data) > 1:
        # try to read 1 byte first
        if util.enabled('short_tx_type_id', block_index):
            message_type_id = struct.unpack(config.SHORT_TXTYPE_FORMAT, packed_data[:1])[0]
            if message_type_id > 0:
                message_remainder = packed_data[1:]
                return (message_type_id, message_remainder)

    # First message byte was 0.  We will read 4 bytes
    if len(packed_data) > 4:
        message_type_id = struct.unpack(config.TXTYPE_FORMAT, packed_data[:4])[0]
        message_remainder = packed_data[4:]

    return (message_type_id, message_remainder)
 
 
#### script.py 
 
"""
None of the functions/objects in this module need be passed `db`.

Naming convention: a `pub` is either a pubkey or a pubkeyhash
"""

import hashlib
import bitcoin as bitcoinlib
import binascii

from bitcoin.core.key import CPubKey
from bitcoin.bech32 import CBech32Data

from counterpartylib.lib import util
from counterpartylib.lib import config
from counterpartylib.lib import exceptions

b58_digits = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'

class InputError (Exception):
    pass
class AddressError(Exception):
    pass
class MultiSigAddressError(AddressError):
    pass
class VersionByteError (AddressError):
    pass
class Base58Error (AddressError):
    pass
class Base58ChecksumError (Base58Error):
    pass


def validate(address, allow_p2sh=True):
    """Make sure the address is valid.

    May throw `AddressError`.
    """
    # Get array of pubkeyhashes to check.
    if is_multisig(address):
        pubkeyhashes = pubkeyhash_array(address)
    else:
        pubkeyhashes = [address]

    # Check validity by attempting to decode.
    for pubkeyhash in pubkeyhashes:
        try:
            if util.enabled('segwit_support'):
                if not is_bech32(pubkeyhash):
                    base58_check_decode(pubkeyhash, config.ADDRESSVERSION)
            else:
                base58_check_decode(pubkeyhash, config.ADDRESSVERSION)
        except VersionByteError as e:
            if not allow_p2sh:
                raise e
            base58_check_decode(pubkeyhash, config.P2SH_ADDRESSVERSION)
        except Base58Error as e:
            if not util.enabled('segwit_support') or not is_bech32(pubkeyhash):
                raise e



def base58_encode(binary):
    """Encode the address in base58."""
    # Convert big‐endian bytes to integer
    n = int('0x0' + util.hexlify(binary), 16)

    # Divide that integer into base58
    res = []
    while n > 0:
        n, r = divmod(n, 58)
        res.append(b58_digits[r])
    res = ''.join(res[::-1])

    return res


def base58_check_encode(original, version):
    """Check if base58 encoding is valid."""
    b = binascii.unhexlify(bytes(original, 'utf-8'))
    d = version + b

    binary = d + util.dhash(d)[:4]
    res = base58_encode(binary)

    # Encode leading zeros as base58 zeros
    czero = 0
    pad = 0
    for c in d:
        if c == czero:
            pad += 1
        else:
            break

    address = b58_digits[0] * pad + res

    if original != util.hexlify(base58_check_decode(address, version)):
        raise AddressError('encoded address does not decode properly')

    return address


def base58_decode(s):
    # Convert the string to an integer
    n = 0
    for c in s:
        n *= 58
        if c not in b58_digits:
            raise Base58Error('Not a valid Base58 character: ‘{}’'.format(c))
        digit = b58_digits.index(c)
        n += digit

    # Convert the integer to bytes
    h = '%x' % n
    if len(h) % 2:
        h = '0' + h
    res = binascii.unhexlify(h.encode('utf8'))

    # Add padding back.
    pad = 0
    for c in s[:-1]:
        if c == b58_digits[0]:
            pad += 1
        else:
            break
    k = b'\x00' * pad + res

    return k


def base58_check_decode_parts(s):
    """Decode from base58 and return parts."""

    k = base58_decode(s)

    addrbyte, data, chk0 = k[0:1], k[1:-4], k[-4:]

    return addrbyte, data, chk0


def base58_check_decode(s, version):
    """Decode from base58 and return data part."""

    addrbyte, data, chk0 = base58_check_decode_parts(s)

    if addrbyte != version:
        raise VersionByteError('incorrect version byte')

    chk1 = util.dhash(addrbyte + data)[:4]
    if chk0 != chk1:
        raise Base58ChecksumError('Checksum mismatch: 0x{} ≠ 0x{}'.format(util.hexlify(chk0), util.hexlify(chk1)))

    return data


def is_multisig(address):
    """Check if the address is multi‐signature."""
    array = address.split('_')
    return len(array) > 1

def is_p2sh(address):
    if is_multisig(address):
        return False

    try:
        base58_check_decode(address, config.P2SH_ADDRESSVERSION)
        return True
    except (VersionByteError, Base58Error):
        return False

def is_bech32(address):
    try:
        b32data = CBech32Data(address)
        return True
    except:
        return False

def is_fully_valid(pubkey_bin):
    """Check if the public key is valid."""
    cpubkey = CPubKey(pubkey_bin)
    return cpubkey.is_fullyvalid

def make_canonical(address):
    """Return canonical version of the address."""
    if is_multisig(address):
        signatures_required, pubkeyhashes, signatures_possible = extract_array(address)
        try:
            [base58_check_decode(pubkeyhash, config.ADDRESSVERSION) for pubkeyhash in pubkeyhashes]
        except Base58Error:
            raise MultiSigAddressError('Multi‐signature address must use PubKeyHashes, not public keys.')
        return construct_array(signatures_required, pubkeyhashes, signatures_possible)
    else:
        return address

def test_array(signatures_required, pubs, signatures_possible):
    """Check if multi‐signature data is valid."""
    try:
        signatures_required, signatures_possible = int(signatures_required), int(signatures_possible)
    except (ValueError, TypeError):
        raise MultiSigAddressError('Signature values not integers.')
    if signatures_required < 1 or signatures_required > 3:
        raise MultiSigAddressError('Invalid signatures_required.')
    if signatures_possible < 2 or signatures_possible > 3:
        raise MultiSigAddressError('Invalid signatures_possible.')
    for pubkey in pubs:
        if '_' in pubkey:
            raise MultiSigAddressError('Invalid characters in pubkeys/pubkeyhashes.')
    if signatures_possible != len(pubs):
        raise InputError('Incorrect number of pubkeys/pubkeyhashes in multi‐signature address.')

def construct_array(signatures_required, pubs, signatures_possible):
    """Create a multi‐signature address."""
    test_array(signatures_required, pubs, signatures_possible)
    address = '_'.join([str(signatures_required)] + sorted(pubs) + [str(signatures_possible)])
    return address

def extract_array(address):
    """Extract data from multi‐signature address."""
    assert is_multisig(address)
    array = address.split('_')
    signatures_required, pubs, signatures_possible = array[0], sorted(array[1:-1]), array[-1]
    test_array(signatures_required, pubs, signatures_possible)
    return int(signatures_required), pubs, int(signatures_possible)

def pubkeyhash_array(address):
    """Return PubKeyHashes from an address."""
    signatures_required, pubs, signatures_possible = extract_array(address)
    if not all([is_pubkeyhash(pub) for pub in pubs]):
        raise MultiSigAddressError('Invalid PubKeyHashes. Multi‐signature address must use PubKeyHashes, not public keys.')
    pubkeyhashes = pubs
    return pubkeyhashes

def hash160(x):
    x = hashlib.sha256(x).digest()
    m = hashlib.new('ripemd160')
    m.update(x)
    return m.digest()

def pubkey_to_pubkeyhash(pubkey):
    """Convert public key to PubKeyHash."""
    pubkeyhash = hash160(pubkey)
    pubkey = base58_check_encode(binascii.hexlify(pubkeyhash).decode('utf-8'), config.ADDRESSVERSION)
    return pubkey

def pubkey_to_p2whash(pubkey):
    """Convert public key to PayToWitness."""
    pubkeyhash = hash160(pubkey)
    pubkey = CBech32Data.from_bytes(0, pubkeyhash)
    return str(pubkey)

def bech32_to_scripthash(address):
    bech32 = CBech32Data(address)
    return bytes(bech32)

def get_asm(scriptpubkey):
    # TODO: When is an exception thrown here? Can this `try` block be tighter? Can it be replaced by a conditional?
    try:
        asm = []
        # TODO: This should be `for element in scriptpubkey`.
        for op in scriptpubkey:
            if type(op) == bitcoinlib.core.script.CScriptOp:
                # TODO: `op = element`
                asm.append(str(op))
            else:
                # TODO: `data = element` (?)
                asm.append(op)
    except bitcoinlib.core.script.CScriptTruncatedPushDataError:
        raise exceptions.PushDataDecodeError('invalid pushdata due to truncation')
    if not asm:
        raise exceptions.DecodeError('empty output')
    return asm

def get_checksig(asm):
    if len(asm) == 5 and asm[0] == 'OP_DUP' and asm[1] == 'OP_HASH160' and asm[3] == 'OP_EQUALVERIFY' and asm[4] == 'OP_CHECKSIG':
        pubkeyhash = asm[2]
        if type(pubkeyhash) == bytes:
            return pubkeyhash
    raise exceptions.DecodeError('invalid OP_CHECKSIG')

def get_checkmultisig(asm):
    # N‐of‐2
    if len(asm) == 5 and asm[3] == 2 and asm[4] == 'OP_CHECKMULTISIG':
        pubkeys, signatures_required = asm[1:3], asm[0]
        if all([type(pubkey) == bytes for pubkey in pubkeys]):
            return pubkeys, signatures_required
    # N‐of‐3
    if len(asm) == 6 and asm[4] == 3 and asm[5] == 'OP_CHECKMULTISIG':
        pubkeys, signatures_required = asm[1:4], asm[0]
        if all([type(pubkey) == bytes for pubkey in pubkeys]):
            return pubkeys, signatures_required
    raise exceptions.DecodeError('invalid OP_CHECKMULTISIG')

def scriptpubkey_to_address(scriptpubkey):
    asm = get_asm(scriptpubkey)

    if asm[-1] == 'OP_CHECKSIG':
        try:
            checksig = get_checksig(asm)
        except exceptions.DecodeError:  # coinbase
            return None

        return base58_check_encode(binascii.hexlify(checksig).decode('utf-8'), config.ADDRESSVERSION)

    elif asm[-1] == 'OP_CHECKMULTISIG':
        pubkeys, signatures_required = get_checkmultisig(asm)
        pubkeyhashes = [pubkey_to_pubkeyhash(pubkey) for pubkey in pubkeys]
        return construct_array(signatures_required, pubkeyhashes, len(pubkeyhashes))

    elif len(asm) == 3 and asm[0] == 'OP_HASH160' and asm[2] == 'OP_EQUAL':
        return base58_check_encode(binascii.hexlify(asm[1]).decode('utf-8'), config.P2SH_ADDRESSVERSION)

    return None


# TODO: Use `python-bitcointools` instead. (Get rid of `pycoin` dependency.)
from pycoin.encoding import wif_to_tuple_of_secret_exponent_compressed, public_pair_to_sec, EncodingError
from pycoin.ecdsa import generator_secp256k1, public_pair_for_secret_exponent

class AltcoinSupportError (Exception): pass
def private_key_to_public_key(private_key_wif):
    """Convert private key to public key."""
    if config.TESTNET:
        allowable_wif_prefixes = [config.PRIVATEKEY_VERSION_TESTNET]
    elif config.REGTEST:
        allowable_wif_prefixes = [config.PRIVATEKEY_VERSION_REGTEST]
    else:
        allowable_wif_prefixes = [config.PRIVATEKEY_VERSION_MAINNET]
    try:
        secret_exponent, compressed = wif_to_tuple_of_secret_exponent_compressed(
                private_key_wif, allowable_wif_prefixes=allowable_wif_prefixes)
    except EncodingError:
        raise AltcoinSupportError('pycoin: unsupported WIF prefix')
    public_pair = public_pair_for_secret_exponent(generator_secp256k1, secret_exponent)
    public_key = public_pair_to_sec(public_pair, compressed=compressed)
    public_key_hex = binascii.hexlify(public_key).decode('utf-8')
    return public_key_hex

def is_pubkeyhash(monosig_address):
    """Check if PubKeyHash is valid P2PKH address. """
    assert not is_multisig(monosig_address)
    try:
        base58_check_decode(monosig_address, config.ADDRESSVERSION)
        return True
    except (Base58Error, VersionByteError):
        return False

def make_pubkeyhash(address):
    """Create a new PubKeyHash."""
    if is_multisig(address):
        signatures_required, pubs, signatures_possible = extract_array(address)
        pubkeyhashes = []
        for pub in pubs:
            if is_pubkeyhash(pub):
                pubkeyhash = pub
            else:
                pubkeyhash = pubkey_to_pubkeyhash(binascii.unhexlify(bytes(pub, 'utf-8')))
            pubkeyhashes.append(pubkeyhash)
        pubkeyhash_address = construct_array(signatures_required, pubkeyhashes, signatures_possible)
    else:
        if util.enabled('segwit_support') and is_bech32(address):
            pubkeyhash_address = address # Some bech32 addresses are valid base58 data
        elif is_pubkeyhash(address):
            pubkeyhash_address = address
        elif is_p2sh(address):
            pubkeyhash_address = address
        else:
            pubkeyhash_address = pubkey_to_pubkeyhash(binascii.unhexlify(bytes(address, 'utf-8')))
    return pubkeyhash_address

def extract_pubkeys(pub):
    """Assume pubkey if not pubkeyhash. (Check validity later.)"""
    pubkeys = []
    if is_multisig(pub):
        _, pubs, _ = extract_array(pub)
        for pub in pubs:
            if not is_pubkeyhash(pub):
                pubkeys.append(pub)
    elif is_p2sh(pub):
        pass
    elif util.enabled('segwit_support') and is_bech32(pub):
        pass
    else:
        if not is_pubkeyhash(pub):
            pubkeys.append(pub)
    return pubkeys

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### transaction.py 
 
"""
Construct and serialize the Bitcoin transactions that are Counterparty transactions.

This module contains no consensus‐critical code.
"""

import os
import sys
import binascii
import json
import hashlib
import re
import time
import decimal
import logging
logger = logging.getLogger(__name__)
import requests
import bitcoin as bitcoinlib
from bitcoin.core.script import CScript
from bitcoin.core import x, CTransaction
from bitcoin.core import b2lx
import cachetools
import math
import io

from counterpartylib.lib import blocks
from counterpartylib.lib import config
from counterpartylib.lib import exceptions
from counterpartylib.lib import util
from counterpartylib.lib import script
from counterpartylib.lib import backend
from counterpartylib.lib import arc4
from counterpartylib.lib.transaction_helper import serializer, p2sh_encoding


# Constants
OP_RETURN = b'\x6a'
OP_PUSHDATA1 = b'\x4c'
OP_DUP = b'\x76'
OP_HASH160 = b'\xa9'
OP_EQUALVERIFY = b'\x88'
OP_CHECKSIG = b'\xac'
OP_0 = b'\x00'
OP_1 = b'\x51'
OP_2 = b'\x52'
OP_3 = b'\x53'
OP_CHECKMULTISIG = b'\xae'
OP_EQUAL = b'\x87'

D = decimal.Decimal

# UTXO_LOCKS is None or DictCache per address
UTXO_LOCKS = None
# set higher than the max number of UTXOs we should expect to
# manage in an aging cache for any one source address, at any one period
UTXO_LOCKS_PER_ADDRESS_MAXSIZE = 5000

# UTXO_P2SH_ENCODING_LOCKS is TTLCache for UTXOs that are used for chaining p2sh encoding
#  instead of a simple (txid, vout) key we use [(vin.prevout.hash, vin.prevout.n) for vin tx1.vin]
UTXO_P2SH_ENCODING_LOCKS = None
# we cache the make_outkey_vin to avoid having to fetch raw txs too often
UTXO_P2SH_ENCODING_LOCKS_CACHE = None


def initialise():
    global UTXO_LOCKS, UTXO_P2SH_ENCODING_LOCKS, UTXO_P2SH_ENCODING_LOCKS_CACHE

    if config.UTXO_LOCKS_MAX_ADDRESSES > 0:  # initialize if configured
        UTXO_LOCKS = util.DictCache(size=config.UTXO_LOCKS_MAX_ADDRESSES)

    UTXO_P2SH_ENCODING_LOCKS = cachetools.TTLCache(10000, 180)
    UTXO_P2SH_ENCODING_LOCKS_CACHE = cachetools.TTLCache(1000, 600)


def print_coin(coin):
    return 'amount: {:.8f}; txid: {}; vout: {}; confirmations: {}'.format(coin['amount'], coin['txid'], coin['vout'], coin.get('confirmations', '?')) # simplify and make deterministic


def chunks(l, n):
    """ Yield successive n‐sized chunks from l.
    """
    for i in range(0, len(l), n):
        yield l[i:i+n]


def make_outkey(output):
    return '{}{}'.format(output['txid'], output['vout'])


def make_outkey_vin_txid(txid, vout):
    global UTXO_P2SH_ENCODING_LOCKS_CACHE

    if (txid, vout) not in UTXO_P2SH_ENCODING_LOCKS_CACHE:
        txhex = backend.getrawtransaction(txid, verbose=False)
        UTXO_P2SH_ENCODING_LOCKS_CACHE[(txid, vout)] = make_outkey_vin(txhex, vout)

    return UTXO_P2SH_ENCODING_LOCKS_CACHE[(txid, vout)]


def make_outkey_vin(txhex, vout):
    txbin = binascii.unhexlify(txhex) if isinstance(txhex, str) else txhex
    assert isinstance(vout, int)

    tx = bitcoinlib.core.CTransaction.deserialize(txbin)
    outkey = [(vin.prevout.hash, vin.prevout.n) for vin in tx.vin]
    outkey = hashlib.sha256(("%s%s" % (outkey, vout)).encode('ascii')).digest()

    return outkey


def get_dust_return_pubkey(source, provided_pubkeys, encoding):
    """Return the pubkey to which dust from data outputs will be sent.

    This pubkey is used in multi-sig data outputs (as the only real pubkey) to
    make those the outputs spendable. It is derived from the source address, so
    that the dust is spendable by the creator of the transaction.
    """
    # Get hex dust return pubkey.
    if script.is_multisig(source):
        a, self_pubkeys, b = script.extract_array(backend.multisig_pubkeyhashes_to_pubkeys(source, provided_pubkeys))
        dust_return_pubkey_hex = self_pubkeys[0]
    else:
        dust_return_pubkey_hex = backend.pubkeyhash_to_pubkey(source, provided_pubkeys)

    # Convert hex public key into the (binary) dust return pubkey.
    try:
        dust_return_pubkey = binascii.unhexlify(dust_return_pubkey_hex)
    except binascii.Error:
        raise script.InputError('Invalid private key.')

    return dust_return_pubkey


def construct_coin_selection(encoding, data_array, source, allow_unconfirmed_inputs, unspent_tx_hash, custom_inputs,
                             fee_per_kb, estimate_fee_per_kb, estimate_fee_per_kb_nblocks, exact_fee, size_for_fee, fee_provided, destination_btc_out, data_btc_out,
                             regular_dust_size, disable_utxo_locks):

    global UTXO_LOCKS, UTXO_P2SH_ENCODING_LOCKS

    # Array of UTXOs, as retrieved by listunspent function from bitcoind
    if custom_inputs:
        use_inputs = unspent = custom_inputs
    else:
        if unspent_tx_hash is not None:
            unspent = backend.get_unspent_txouts(source, unconfirmed=allow_unconfirmed_inputs, unspent_tx_hash=unspent_tx_hash)
        else:
            unspent = backend.get_unspent_txouts(source, unconfirmed=allow_unconfirmed_inputs)

        filter_unspents_utxo_locks = []
        if UTXO_LOCKS is not None and source in UTXO_LOCKS:
            filter_unspents_utxo_locks = UTXO_LOCKS[source].keys()
        filter_unspents_p2sh_locks = UTXO_P2SH_ENCODING_LOCKS.keys()

        # filter out any locked UTXOs to prevent creating transactions that spend the same UTXO when they're created at the same time
        filtered_unspent = []
        for output in unspent:
            if make_outkey(output) not in filter_unspents_utxo_locks and make_outkey_vin_txid(output['txid'], output['vout']) not in filter_unspents_p2sh_locks:
                filtered_unspent.append(output)
        unspent = filtered_unspent

        unspent = backend.sort_unspent_txouts(unspent)
        logger.debug('Sorted candidate UTXOs: {}'.format([print_coin(coin) for coin in unspent]))
        use_inputs = unspent

    # use backend estimated fee_per_kb
    if estimate_fee_per_kb:
        estimated_fee_per_kb = backend.fee_per_kb(estimate_fee_per_kb_nblocks, config.ESTIMATE_FEE_MODE)
        if estimated_fee_per_kb is not None:
            fee_per_kb = max(estimated_fee_per_kb, fee_per_kb)  # never drop below the default fee_per_kb

    logger.debug('Fee/KB {:.8f}'.format(fee_per_kb / config.UNIT))

    inputs = []
    btc_in = 0
    change_quantity = 0
    sufficient_funds = False
    final_fee = fee_per_kb
    desired_input_count = 1

    if encoding == 'multisig' and data_array and util.enabled('bytespersigop'):
        desired_input_count = len(data_array) * 2


    # pop inputs until we can pay for the fee
    for coin in use_inputs:
        logger.debug('New input: {}'.format(print_coin(coin)))
        inputs.append(coin)
        btc_in += round(coin['amount'] * config.UNIT)

        # If exact fee is specified, use that. Otherwise, calculate size of tx
        # and base fee on that (plus provide a minimum fee for selling BTC).
        size = 181 * len(inputs) + size_for_fee + 10
        if exact_fee:
            final_fee = exact_fee
        else:
            necessary_fee = int(size / 1000 * fee_per_kb)
            final_fee = max(fee_provided, necessary_fee)
            logger.getChild('p2shdebug').debug('final_fee inputs: %d size: %d final_fee %s' % (len(inputs), size, final_fee))

        # Check if good.
        btc_out = destination_btc_out + data_btc_out
        change_quantity = btc_in - (btc_out + final_fee)
        logger.debug('Size: {} Fee: {:.8f} Change quantity: {:.8f} BTC'.format(size, final_fee / config.UNIT, change_quantity / config.UNIT))
        # If change is necessary, must not be a dust output.
        if change_quantity == 0 or change_quantity >= regular_dust_size:
            sufficient_funds = True
            if len(inputs) >= desired_input_count:
                break

    if not sufficient_funds:
        # Approximate needed change, fee by with most recently calculated
        # quantities.
        btc_out = destination_btc_out + data_btc_out
        total_btc_out = btc_out + max(change_quantity, 0) + final_fee
        raise exceptions.BalanceError('Insufficient {} at address {}. (Need approximately {} {}.) To spend unconfirmed coins, use the flag `--unconfirmed`. (Unconfirmed coins cannot be spent from multi‐sig addresses.)'.format(config.BTC, source, total_btc_out / config.UNIT, config.BTC))

    # Lock the source's inputs (UTXOs) chosen for this transaction
    if UTXO_LOCKS is not None and not disable_utxo_locks:
        if source not in UTXO_LOCKS:
            UTXO_LOCKS[source] = cachetools.TTLCache(
                UTXO_LOCKS_PER_ADDRESS_MAXSIZE, config.UTXO_LOCKS_MAX_AGE)

        for input in inputs:
            UTXO_LOCKS[source][make_outkey(input)] = input

        logger.debug("UTXO locks: Potentials ({}): {}, Used: {}, locked UTXOs: {}".format(
            len(unspent), [make_outkey(coin) for coin in unspent],
            [make_outkey(input) for input in inputs], list(UTXO_LOCKS[source].keys())))

    # ensure inputs have scriptPubKey
    #   this is not provided by indexd
    inputs = backend.ensure_script_pub_key_for_inputs(inputs)

    return inputs, change_quantity, btc_in, final_fee

def select_any_coin_from_source(source, allow_unconfirmed_inputs=True, disable_utxo_locks=False):
    ''' Get the first (biggest) input from the source address '''
    global UTXO_LOCKS

    # Array of UTXOs, as retrieved by listunspent function from bitcoind
    unspent = backend.get_unspent_txouts(source, unconfirmed=allow_unconfirmed_inputs)

    filter_unspents_utxo_locks = []
    if UTXO_LOCKS is not None and source in UTXO_LOCKS:
        filter_unspents_utxo_locks = UTXO_LOCKS[source].keys()

    # filter out any locked UTXOs to prevent creating transactions that spend the same UTXO when they're created at the same time
    filtered_unspent = []
    for output in unspent:
        if make_outkey(output) not in filter_unspents_utxo_locks:
            filtered_unspent.append(output)
    unspent = filtered_unspent

    # sort
    unspent = backend.sort_unspent_txouts(unspent)

    # use the first input
    input = unspent[0]
    if input is None:
        return None

    # Lock the source's inputs (UTXOs) chosen for this transaction
    if UTXO_LOCKS is not None and not disable_utxo_locks:
        if source not in UTXO_LOCKS:
            UTXO_LOCKS[source] = cachetools.TTLCache(
                UTXO_LOCKS_PER_ADDRESS_MAXSIZE, config.UTXO_LOCKS_MAX_AGE)

        UTXO_LOCKS[source][make_outkey(input)] = input

    return input


def return_result(tx_hexes, old_style_api):
        tx_hexes = list(filter(None, tx_hexes))  # filter out None

        if old_style_api:
            if len(tx_hexes) != 1:
                raise Exception("Can't do 2 TXs with old_style_api")

            return tx_hexes[0]
        else:
            if len(tx_hexes) == 1:
                return tx_hexes[0]
            else:
                return tx_hexes

def construct (db, tx_info, encoding='auto',
               fee_per_kb=config.DEFAULT_FEE_PER_KB,
               estimate_fee_per_kb=None, estimate_fee_per_kb_conf_target=config.ESTIMATE_FEE_CONF_TARGET,
               estimate_fee_per_kb_mode=config.ESTIMATE_FEE_MODE, estimate_fee_per_kb_nblocks=config.ESTIMATE_FEE_CONF_TARGET,
               regular_dust_size=config.DEFAULT_REGULAR_DUST_SIZE,
               multisig_dust_size=config.DEFAULT_MULTISIG_DUST_SIZE,
               op_return_value=config.DEFAULT_OP_RETURN_VALUE,
               exact_fee=None, fee_provided=0, provided_pubkeys=None, dust_return_pubkey=None,
               allow_unconfirmed_inputs=False, unspent_tx_hash=None, custom_inputs=None, disable_utxo_locks=False, extended_tx_info=False,
               old_style_api=None, segwit=False,
               p2sh_source_multisig_pubkeys=None, p2sh_source_multisig_pubkeys_required=None, p2sh_pretx_txid=None,):

    if estimate_fee_per_kb is None:
        estimate_fee_per_kb = config.ESTIMATE_FEE_PER_KB

    global UTXO_LOCKS, UTXO_P2SH_ENCODING_LOCKS

    # lazy assign from config, because when set as default it's evaluated before it's configured
    if old_style_api is None:
        old_style_api = config.OLD_STYLE_API

    (source, destination_outputs, data) = tx_info

    if dust_return_pubkey:
        dust_return_pubkey = binascii.unhexlify(dust_return_pubkey)

    if p2sh_source_multisig_pubkeys:
        p2sh_source_multisig_pubkeys = [binascii.unhexlify(p) for p in p2sh_source_multisig_pubkeys]

    # Source.
        # If public key is necessary for construction of (unsigned)
        # transaction, use the public key provided, or find it from the
        # blockchain.
    if source:
        script.validate(source)

    source_is_p2sh = script.is_p2sh(source)

    # Normalize source
    if script.is_multisig(source):
        source_address = backend.multisig_pubkeyhashes_to_pubkeys(source, provided_pubkeys)
    else:
        source_address = source

    # Sanity checks.
    if exact_fee and not isinstance(exact_fee, int):
        raise exceptions.TransactionError('Exact fees must be in satoshis.')
    if not isinstance(fee_provided, int):
        raise exceptions.TransactionError('Fee provided must be in satoshis.')

    '''Determine encoding method'''

    if data:
        desired_encoding = encoding
        # Data encoding methods (choose and validate).
        if desired_encoding == 'auto':
            if len(data) + len(config.PREFIX) <= config.OP_RETURN_MAX_SIZE:
                encoding = 'opreturn'
            else:
                encoding = 'p2sh' if not old_style_api and util.enabled('p2sh_encoding') else 'multisig'  # p2sh is not possible with old_style_api

        elif desired_encoding == 'p2sh' and not util.enabled('p2sh_encoding'):
            raise exceptions.TransactionError('P2SH encoding not enabled yet')

        elif encoding not in ('pubkeyhash', 'multisig', 'opreturn', 'p2sh'):
            raise exceptions.TransactionError('Unknown encoding‐scheme.')
    else:
        # no data
        encoding = None

    '''Destinations'''

    # Destination outputs.
        # Replace multi‐sig addresses with multi‐sig pubkeys. Check that the
        # destination output isn’t a dust output. Set null values to dust size.
    destination_outputs_new = []
    if encoding != 'p2sh':
        for (address, value) in destination_outputs:

            # Value.
            if script.is_multisig(address):
                dust_size = multisig_dust_size
            else:
                dust_size = regular_dust_size
            if value == None:
                value = dust_size
            elif value < dust_size:
                raise exceptions.TransactionError('Destination output is dust.')

            # Address.
            script.validate(address)
            if script.is_multisig(address):
                destination_outputs_new.append((backend.multisig_pubkeyhashes_to_pubkeys(address, provided_pubkeys), value))
            else:
                destination_outputs_new.append((address, value))

    destination_outputs = destination_outputs_new
    destination_btc_out = sum([value for address, value in destination_outputs])


    '''Data'''

    if data:
        # @TODO: p2sh encoding require signable dust key
        if encoding == 'multisig':
            # dust_return_pubkey should be set or explicitly set to False to use the default configured for the node
            #  the default for the node is optional so could fail
            if (source_is_p2sh and dust_return_pubkey is None) or (dust_return_pubkey is False and config.P2SH_DUST_RETURN_PUBKEY is None):
                raise exceptions.TransactionError("Can't use multisig encoding when source is P2SH and no dust_return_pubkey is provided.")
            elif dust_return_pubkey is False:
                dust_return_pubkey = binascii.unhexlify(config.P2SH_DUST_RETURN_PUBKEY)

        if not dust_return_pubkey:
            if encoding == 'multisig' or encoding == 'p2sh' and not source_is_p2sh:
                dust_return_pubkey = get_dust_return_pubkey(source, provided_pubkeys, encoding)
            else:
                dust_return_pubkey = None

        # Divide data into chunks.
        if encoding == 'pubkeyhash':
            # Prefix is also a suffix here.
            chunk_size = 20 - 1 - 8
        elif encoding == 'multisig':
            # Two pubkeys, minus length byte, minus prefix, minus two nonces,
            # minus two sign bytes.
            chunk_size = (33 * 2) - 1 - 8 - 2 - 2
        elif encoding == 'p2sh':
            pubkeylength = -1
            if dust_return_pubkey is not None:
                pubkeylength = len(dust_return_pubkey)

            chunk_size = p2sh_encoding.maximum_data_chunk_size(pubkeylength)
        elif encoding == 'opreturn':
            chunk_size = config.OP_RETURN_MAX_SIZE
            if len(data) + len(config.PREFIX) > chunk_size:
                raise exceptions.TransactionError('One `OP_RETURN` output per transaction.')
        data_array = list(chunks(data, chunk_size))

        # Data outputs.
        if encoding == 'multisig':
            data_value = multisig_dust_size
        elif encoding == 'p2sh':
            data_value = 0   # this will be calculated later
        elif encoding == 'opreturn':
            data_value = op_return_value
        else:
            # Pay‐to‐PubKeyHash, e.g.
            data_value = regular_dust_size
        data_output = (data_array, data_value)

    else:
        data_value = 0
        data_array = []
        data_output = None
        dust_return_pubkey = None

    data_btc_out = data_value * len(data_array)
    logger.getChild('p2shdebug').debug('data_btc_out=%s (data_value=%d len(data_array)=%d)' % (data_btc_out, data_value, len(data_array)))

    '''Inputs'''
    btc_in = 0
    final_fee = 0
    # Calculate collective size of outputs, for fee calculation.
    p2pkhsize = 25 + 9
    if encoding == 'multisig':
        data_output_size = 81       # 71 for the data
    elif encoding == 'opreturn':
        # prefix + data + 10 bytes script overhead
        data_output_size = len(config.PREFIX) + 10
        if data is not None:
            data_output_size = data_output_size + len(data)
    else:
        data_output_size = p2pkhsize   # Pay‐to‐PubKeyHash (25 for the data?)
    outputs_size = (p2pkhsize * len(destination_outputs)) + (len(data_array) * data_output_size)

    if encoding == 'p2sh':
        # calculate all the p2sh outputs
        size_for_fee, datatx_necessary_fee, data_value, data_btc_out = p2sh_encoding.calculate_outputs(destination_outputs, data_array, fee_per_kb, exact_fee)
        # replace the data value
        data_output = (data_array, data_value)
    else:
        sum_data_output_size = len(data_array) * data_output_size
        size_for_fee = ((25 + 9) * len(destination_outputs)) + sum_data_output_size

    if not (encoding == 'p2sh' and p2sh_pretx_txid):
        inputs, change_quantity, n_btc_in, n_final_fee = construct_coin_selection(
            encoding, data_array,
            source, allow_unconfirmed_inputs, unspent_tx_hash, custom_inputs,
            fee_per_kb, estimate_fee_per_kb, estimate_fee_per_kb_nblocks,
            exact_fee, size_for_fee, fee_provided, destination_btc_out, data_btc_out,
            regular_dust_size, disable_utxo_locks
        )
        btc_in = n_btc_in
        final_fee = n_final_fee
    else:
        # when encoding is P2SH and the pretx txid is passed we can skip coinselection
        inputs, change_quantity = None, None

    '''Finish'''

    if change_quantity:
        change_output = (source_address, change_quantity)
    else:
        change_output = None

    unsigned_pretx_hex = None
    unsigned_tx_hex = None

    pretx_txid = None
    if encoding == 'p2sh':
        assert not (segwit and p2sh_pretx_txid)  # shouldn't do old style with segwit enabled

        if p2sh_pretx_txid:
            pretx_txid = p2sh_pretx_txid if isinstance(p2sh_pretx_txid, bytes) else binascii.unhexlify(p2sh_pretx_txid)
            unsigned_pretx = None
        else:
            destination_value_sum = sum([value for (destination, value) in destination_outputs])
            source_value = destination_value_sum

            if change_output:
                # add the difference between source and destination to the change
                change_value = change_output[1] + (destination_value_sum - source_value)
                change_output = (change_output[0], change_value)

            unsigned_pretx = serializer.serialise_p2sh_pretx(inputs,
                                                             source=source_address,
                                                             source_value=source_value,
                                                             data_output=data_output,
                                                             change_output=change_output,
                                                             pubkey=dust_return_pubkey,
                                                             multisig_pubkeys=p2sh_source_multisig_pubkeys,
                                                             multisig_pubkeys_required=p2sh_source_multisig_pubkeys_required)
            unsigned_pretx_hex = binascii.hexlify(unsigned_pretx).decode('utf-8')

        # with segwit we already know the txid and can return both
        if segwit:
            #pretx_txid = hashlib.sha256(unsigned_pretx).digest()  # this should be segwit txid
            ptx = CTransaction.stream_deserialize(io.BytesIO(unsigned_pretx)) # could be a non-segwit tx anyways
            txid_ba = bytearray(ptx.GetTxid())
            txid_ba.reverse()
            pretx_txid = bytes(txid_ba) # gonna leave the malleability problem to upstream
            logger.getChild('p2shdebug').debug('pretx_txid %s' % pretx_txid)
            print('pretx txid:', binascii.hexlify(pretx_txid))

        if unsigned_pretx:
            # we set a long lock on this, don't want other TXs to spend from it
            UTXO_P2SH_ENCODING_LOCKS[make_outkey_vin(unsigned_pretx, 0)] = True

        # only generate the data TX if we have the pretx txId
        if pretx_txid:
            source_input = None
            if script.is_p2sh(source):
                source_input = select_any_coin_from_source(source)
                if not source_input:
                    raise exceptions.TransactionError('Unable to select source input for p2sh source address')

            unsigned_datatx = serializer.serialise_p2sh_datatx(pretx_txid,
                                                               source=source_address,
                                                               source_input=source_input,
                                                               destination_outputs=destination_outputs,
                                                               data_output=data_output,
                                                               pubkey=dust_return_pubkey,
                                                               multisig_pubkeys=p2sh_source_multisig_pubkeys,
                                                               multisig_pubkeys_required=p2sh_source_multisig_pubkeys_required)
            unsigned_datatx_hex = binascii.hexlify(unsigned_datatx).decode('utf-8')

            # let the rest of the code work it's magic on the data tx
            unsigned_tx_hex = unsigned_datatx_hex
        else:
            # we're just gonna return the pretx, it doesn't require any of the further checks
            logger.warn('old_style_api = %s' % old_style_api)
            return return_result([unsigned_pretx_hex], old_style_api=old_style_api)

    else:
        # Serialise inputs and outputs.
        unsigned_tx = serializer.serialise(encoding, inputs, destination_outputs,
                                data_output, change_output,
                                dust_return_pubkey=dust_return_pubkey)
        unsigned_tx_hex = binascii.hexlify(unsigned_tx).decode('utf-8')


    '''Sanity Check'''

    # Desired transaction info.
    (desired_source, desired_destination_outputs, desired_data) = tx_info
    desired_source = script.make_canonical(desired_source)
    desired_destination = script.make_canonical(desired_destination_outputs[0][0]) if desired_destination_outputs else ''
    # NOTE: Include change in destinations for BTC transactions.
    # if change_output and not desired_data and desired_destination != config.UNSPENDABLE:
    #    if desired_destination == '':
    #        desired_destination = desired_source
    #    else:
    #        desired_destination += '-{}'.format(desired_source)
    # NOTE
    if desired_data == None:
        desired_data = b''

    # Parsed transaction info.
    try:
        if pretx_txid and unsigned_pretx:
            backend.cache_pretx(pretx_txid, unsigned_pretx)
        parsed_source, parsed_destination, x, y, parsed_data, extra = blocks._get_tx_info(unsigned_tx_hex, p2sh_is_segwit=script.is_bech32(desired_source))

        if encoding == 'p2sh':
            # make_canonical can't determine the address, so we blindly change the desired to the parsed
            desired_source = parsed_source

        if pretx_txid and unsigned_pretx:
            backend.clear_pretx(pretx_txid)
    except exceptions.BTCOnlyError:
        # Skip BTC‐only transactions.
        if extended_tx_info:
            return {
                'btc_in': btc_in,
                'btc_out': destination_btc_out + data_btc_out,
                'btc_change': change_quantity,
                'btc_fee': final_fee,
                'tx_hex': unsigned_tx_hex,
            }
        logger.getChild('p2shdebug').debug('BTC-ONLY')
        return return_result([unsigned_pretx_hex, unsigned_tx_hex], old_style_api=old_style_api)
    desired_source = script.make_canonical(desired_source)

    # Check desired info against parsed info.
    desired = (desired_source, desired_destination, desired_data)
    parsed = (parsed_source, parsed_destination, parsed_data)
    if desired != parsed:
        # Unlock (revert) UTXO locks
        if UTXO_LOCKS is not None and inputs:
            for input in inputs:
                UTXO_LOCKS[source].pop(make_outkey(input), None)

        raise exceptions.TransactionError('Constructed transaction does not parse correctly: {} ≠ {}'.format(desired, parsed))

    if extended_tx_info:
        return {
            'btc_in': btc_in,
            'btc_out': destination_btc_out + data_btc_out,
            'btc_change': change_quantity,
            'btc_fee': final_fee,
            'tx_hex': unsigned_tx_hex,
        }
    return return_result([unsigned_pretx_hex, unsigned_tx_hex], old_style_api=old_style_api)

def normalize_custom_inputs(raw_custom_inputs):
    custom_inputs = []
    for custom_input in raw_custom_inputs:
        if 'value' not in custom_input:
            custom_input['value'] = int(custom_input['amount'] * config.UNIT)
        custom_inputs.append(custom_input)
    return custom_inputs

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### util.py 
 
import time
import decimal
import sys
import json
import logging
logger = logging.getLogger(__name__)
import apsw
import inspect
import requests
from datetime import datetime
from dateutil.tz import tzlocal
from operator import itemgetter
import fractions
import warnings
import binascii
import re
import hashlib
import sha3
import bitcoin as bitcoinlib
import os
import collections
import threading
import random
import itertools

from counterpartylib.lib import exceptions
from counterpartylib.lib.exceptions import DecodeError
from counterpartylib.lib import config

D = decimal.Decimal
B26_DIGITS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'

# subasset contain only characters a-zA-Z0-9.-_@!
SUBASSET_DIGITS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_@!'
SUBASSET_REVERSE = {'a':1,'b':2,'c':3,'d':4,'e':5,'f':6,'g':7,'h':8,'i':9,'j':10,'k':11,'l':12,'m':13,'n':14,
                    'o':15,'p':16,'q':17,'r':18,'s':19,'t':20,'u':21,'v':22,'w':23,'x':24,'y':25,'z':26,
                    'A':27,'B':28,'C':29,'D':30,'E':31,'F':32,'G':33,'H':34,'I':35,'J':36,'K':37,'L':38,'M':39,
                    'N':40,'O':41,'P':42,'Q':43,'R':44,'S':45,'T':46,'U':47,'V':48,'W':49,'X':50,'Y':51,'Z':52,
                    '0':53,'1':54,'2':55,'3':56,'4':57,'5':58,'6':59,'7':60,'8':61,'9':62,'.':63,'-':64,'_':65,'@':66,'!':67}

# Obsolete in Python 3.4, with enum module.
BET_TYPE_NAME = {0: 'BullCFD', 1: 'BearCFD', 2: 'Equal', 3: 'NotEqual'}
BET_TYPE_ID = {'BullCFD': 0, 'BearCFD': 1, 'Equal': 2, 'NotEqual': 3}

json_dump = lambda x: json.dumps(x, sort_keys=True, indent=4)
json_print = lambda x: print(json_dump(x))

BLOCK_LEDGER = []

CURRENT_BLOCK_INDEX = None

CURR_DIR = os.path.dirname(os.path.realpath(__file__))
with open(CURR_DIR + '/../protocol_changes.json') as f:
    PROTOCOL_CHANGES = json.load(f)

class RPCError (Exception): pass

# TODO: Move to `util_test.py`.
# TODO: This doesn’t timeout properly. (If server hangs, then unhangs, no result.)
def api(method, params):
    """Poll API via JSON-RPC."""
    headers = {'content-type': 'application/json'}
    payload = {
        "method": method,
        "params": params,
        "jsonrpc": "2.0",
        "id": 0,
    }

    response = requests.post(config.RPC, data=json.dumps(payload), headers=headers)
    if response == None:
        raise RPCError('Cannot communicate with {} server.'.format(config.XCP_NAME))
    elif response.status_code != 200:
        if response.status_code == 500:
            raise RPCError('Malformed API call.')
        else:
            raise RPCError(str(response.status_code) + ' ' + response.reason)

    response_json = response.json()
    if 'error' not in response_json.keys() or response_json['error'] == None:
        try:
            return response_json['result']
        except KeyError:
            raise RPCError(response_json)
    else:
        raise RPCError('{} ({})'.format(response_json['error']['message'], response_json['error']['code']))

def chunkify(l, n):
    n = max(1, n)
    return [l[i:i + n] for i in range(0, len(l), n)]

def flat(z):
    return [x for x in z]

def py34TupleAppend(first_elem, t):
    # Had to do it this way to support python 3.4, if we start
    # using the 3.5 runtime this can be replaced by:
    #  (first_elem, *t)

    l = list(t)
    l.insert(0, first_elem)
    return tuple(l)

def accumulate(l):
    it = itertools.groupby(l, itemgetter(0))
    for key, subiter in it:
       yield key, sum(item[1] for item in subiter)

def date_passed(date):
    """Check if the date has already passed."""
    return date <= int(time.time())

def price (numerator, denominator):
    """Return price as Fraction or Decimal."""
    if CURRENT_BLOCK_INDEX >= 294500 or config.TESTNET or config.REGTEST: # Protocol change.
        return fractions.Fraction(numerator, denominator)
    else:
        numerator = D(numerator)
        denominator = D(denominator)
        return D(numerator / denominator)

def last_message(db):
    """Return latest message from the db."""
    cursor = db.cursor()
    messages = list(cursor.execute('''SELECT * FROM messages WHERE message_index = (SELECT MAX(message_index) from messages)'''))
    if messages:
        assert len(messages) == 1
        last_message = messages[0]
    else:
        raise exceptions.DatabaseError('No messages found.')
    cursor.close()
    return last_message

def generate_asset_id(asset_name, block_index):
    """Create asset_id from asset_name."""
    if asset_name == config.BTC: return 0
    elif asset_name == config.XCP: return 1

    if len(asset_name) < 4:
        raise exceptions.AssetNameError('too short')

    # Numeric asset names.
    if enabled('numeric_asset_names'):  # Protocol change.
        if asset_name[0] == 'A':
            # Must be numeric.
            try:
                asset_id = int(asset_name[1:])
            except ValueError:
                raise exceptions.AssetNameError('non‐numeric asset name starts with ‘A’')

            # Number must be in range.
            if not (26**12 + 1 <= asset_id <= 2**64 - 1):
                raise exceptions.AssetNameError('numeric asset name not in range')

            return asset_id
        elif len(asset_name) >= 13:
            raise exceptions.AssetNameError('long asset names must be numeric')

    if asset_name[0] == 'A': raise exceptions.AssetNameError('non‐numeric asset name starts with ‘A’')

    # Convert the Base 26 string to an integer.
    n = 0
    for c in asset_name:
        n *= 26
        if c not in B26_DIGITS:
            raise exceptions.AssetNameError('invalid character:', c)
        digit = B26_DIGITS.index(c)
        n += digit
    asset_id = n

    if asset_id < 26**3:
        raise exceptions.AssetNameError('too short')

    return asset_id

def generate_asset_name (asset_id, block_index):
    """Create asset_name from asset_id."""
    if asset_id == 0: return config.BTC
    elif asset_id == 1: return config.XCP

    if asset_id < 26**3:
        raise exceptions.AssetIDError('too low')

    if enabled('numeric_asset_names'):  # Protocol change.
        if asset_id <= 2**64 - 1:
            if 26**12 + 1 <= asset_id:
                asset_name = 'A' + str(asset_id)
                return asset_name
        else:
            raise exceptions.AssetIDError('too high')

    # Divide that integer into Base 26 string.
    res = []
    n = asset_id
    while n > 0:
        n, r = divmod (n, 26)
        res.append(B26_DIGITS[r])
    asset_name = ''.join(res[::-1])

    """
    return asset_name + checksum.compute(asset_name)
    """
    return asset_name


def get_asset_id (db, asset_name, block_index):
    """Return asset_id from asset_name."""
    if not enabled('hotfix_numeric_assets'):
        return generate_asset_id(asset_name, block_index)
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM assets WHERE asset_name = ?''', (asset_name,))
    assets = list(cursor)
    if len(assets) == 1:
        return int(assets[0]['asset_id'])
    else:
        raise exceptions.AssetError('No such asset: {}'.format(asset_name))

def get_asset_name (db, asset_id, block_index):
    """Return asset_name from asset_id."""
    if not enabled('hotfix_numeric_assets'):
        return generate_asset_name(asset_id, block_index)
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM assets WHERE asset_id = ?''', (str(asset_id),))
    assets = list(cursor)
    if len(assets) == 1:
        return assets[0]['asset_name']
    elif not assets:
        return 0    # Strange, I know…

# If asset_name is an existing subasset (PARENT.child) then return the corresponding numeric asset name (A12345)
#   If asset_name is not an existing subasset, then return the unmodified asset_name
def resolve_subasset_longname(db, asset_name):
    if enabled('subassets'):
        subasset_longname = None
        try:
            subasset_parent, subasset_longname = parse_subasset_from_asset_name(asset_name)
        except Exception as e:
            logger.warn("Invalid subasset {}".format(asset_name))
            subasset_longname = None

        if subasset_longname is not None:
            cursor = db.cursor()
            cursor.execute('''SELECT asset_name FROM assets WHERE asset_longname = ?''', (subasset_longname,))
            assets = list(cursor)
            cursor.close()
            if len(assets) == 1:
                return assets[0]['asset_name']

    return asset_name


# checks and validates subassets (PARENT.SUBASSET)
#   throws exceptions for assset or subasset names with invalid syntax
#   returns (None, None) if the asset is not a subasset name
def parse_subasset_from_asset_name(asset):
    subasset_parent = None
    subasset_child = None
    subasset_longname = None
    chunks = asset.split('.', 1)
    if (len(chunks) == 2):
        subasset_parent = chunks[0]
        subasset_child = chunks[1]
        subasset_longname = asset

        # validate parent asset
        validate_subasset_parent_name(subasset_parent)

        # validate child asset
        validate_subasset_longname(subasset_longname, subasset_child)

    return (subasset_parent, subasset_longname)

# throws exceptions for invalid subasset names
def validate_subasset_longname(subasset_longname, subasset_child=None):
    if subasset_child is None:
        chunks = subasset_longname.split('.', 1)
        if (len(chunks) == 2):
            subasset_child = chunks[1]
        else:
            subasset_child = ''

    if len(subasset_child) < 1:
        raise exceptions.AssetNameError('subasset name too short')
    if len(subasset_longname) > 250:
        raise exceptions.AssetNameError('subasset name too long')

    # can't start with period, can't have consecutive periods, can't contain anything not in SUBASSET_DIGITS
    previous_digit = '.'
    for c in subasset_child:
        if c not in SUBASSET_DIGITS:
            raise exceptions.AssetNameError('subasset name contains invalid character:', c)
        if c == '.' and previous_digit == '.':
            raise exceptions.AssetNameError('subasset name contains consecutive periods')
        previous_digit = c
    if previous_digit == '.':
        raise exceptions.AssetNameError('subasset name ends with a period')

    return True

# throws exceptions for invalid subasset names
def validate_subasset_parent_name(asset_name):
    if asset_name == config.BTC:
        raise exceptions.AssetNameError('parent asset cannot be {}'.format(config.BTC))
    if asset_name == config.XCP:
        raise exceptions.AssetNameError('parent asset cannot be {}'.format(config.XCP))
    if len(asset_name) < 4:
        raise exceptions.AssetNameError('parent asset name too short')
    if len(asset_name) >= 13:
        raise exceptions.AssetNameError('parent asset name too long')
    if asset_name[0] == 'A':
        raise exceptions.AssetNameError('parent asset name starts with ‘A’')
    for c in asset_name:
        if c not in B26_DIGITS:
            raise exceptions.AssetNameError('parent asset name contains invalid character:', c)
    return True

def compact_subasset_longname(string):
    """Compacts a subasset name string into an array of bytes to save space using a base68 encoding scheme.
    Assumes all characters provided belong to SUBASSET_DIGITS.
    """
    name_int = 0
    for i, c in enumerate(string[::-1]):
        name_int += (68 ** i) * SUBASSET_REVERSE[c]
    return name_int.to_bytes((name_int.bit_length() + 7) // 8, byteorder='big')

def expand_subasset_longname(raw_bytes):
    """Expands an array of bytes into a subasset name string."""
    integer = int.from_bytes(raw_bytes, byteorder='big')
    if integer == 0:
        return ''
    ret = ''
    while integer != 0:
        ret = SUBASSET_DIGITS[integer % 68 - 1] + ret
        integer //= 68
    return ret

def generate_random_asset ():
    return 'A' + str(random.randint(26**12 + 1, 2**64 - 1))

def parse_options_from_string(string):
    """Parse options integer from string, if exists."""
    string_list = string.split(" ")
    if len(string_list) == 2:
        try:
            options = int(string_list.pop())
        except:
            raise exceptions.OptionsError('options not an integer')
        return options
    else:
        return False

def validate_address_options(options):
    """Ensure the options are all valid and in range."""
    if (options > config.MAX_INT) or (options < 0):
        raise exceptions.OptionsError('options integer overflow')
    elif options > config.ADDRESS_OPTION_MAX_VALUE:
        raise exceptions.OptionsError('options out of range')
    elif not active_options(config.ADDRESS_OPTION_MAX_VALUE, options):
        raise exceptions.OptionsError('options not possible')

def active_options(config, options):
    """Checks if options active in some given config."""
    return config & options == options

class DebitError (Exception): pass
def debit (db, address, asset, quantity, action=None, event=None):
    """Debit given address by quantity of asset."""
    block_index = CURRENT_BLOCK_INDEX

    if type(quantity) != int:
        raise DebitError('Quantity must be an integer.')
    if quantity < 0:
        raise DebitError('Negative quantity.')
    if quantity > config.MAX_INT:
        raise DebitError('Quantity can\'t be higher than MAX_INT.')
    if asset == config.BTC:
        raise DebitError('Cannot debit bitcoins.')

    debit_cursor = db.cursor()

    # Contracts can only hold XCP balances.
    if enabled('contracts_only_xcp_balances'): # Protocol change.
        if len(address) == 40:
            assert asset == config.XCP

    if asset == config.BTC:
        raise exceptions.BalanceError('Cannot debit bitcoins from a {} address!'.format(config.XCP_NAME))

    debit_cursor.execute('''SELECT * FROM balances \
                            WHERE (address = ? AND asset = ?)''', (address, asset))
    balances = debit_cursor.fetchall()
    if not len(balances) == 1:
        old_balance = 0
    else:
        old_balance = balances[0]['quantity']

    if old_balance < quantity:
        raise DebitError('Insufficient funds.')

    balance = round(old_balance - quantity)
    balance = min(balance, config.MAX_INT)
    assert balance >= 0

    bindings = {
        'quantity': balance,
        'address': address,
        'asset': asset
    }
    sql='update balances set quantity = :quantity where (address = :address and asset = :asset)'
    debit_cursor.execute(sql, bindings)

    # Record debit.
    bindings = {
        'block_index': block_index,
        'address': address,
        'asset': asset,
        'quantity': quantity,
        'action': action,
        'event': event
    }
    sql='insert into debits values(:block_index, :address, :asset, :quantity, :action, :event)'
    debit_cursor.execute(sql, bindings)
    debit_cursor.close()

    BLOCK_LEDGER.append('{}{}{}{}'.format(block_index, address, asset, quantity))

class CreditError (Exception): pass
def credit (db, address, asset, quantity, action=None, event=None):
    """Credit given address by quantity of asset."""
    block_index = CURRENT_BLOCK_INDEX

    if type(quantity) != int:
        raise CreditError('Quantity must be an integer.')
    if quantity < 0:
        raise CreditError('Negative quantity.')
    if quantity > config.MAX_INT:
        raise CreditError('Quantity can\'t be higher than MAX_INT.')
    if asset == config.BTC:
        raise CreditError('Cannot debit bitcoins.')

    credit_cursor = db.cursor()

    # Contracts can only hold XCP balances.
    if enabled('contracts_only_xcp_balances'): # Protocol change.
        if len(address) == 40:
            assert asset == config.XCP

    credit_cursor.execute('''SELECT * FROM balances \
                             WHERE (address = ? AND asset = ?)''', (address, asset))
    balances = credit_cursor.fetchall()
    if len(balances) == 0:
        assert balances == []

        #update balances table with new balance
        bindings = {
            'address': address,
            'asset': asset,
            'quantity': quantity,
        }
        sql='insert into balances values(:address, :asset, :quantity)'
        credit_cursor.execute(sql, bindings)
    elif len(balances) > 1:
        assert False
    else:
        old_balance = balances[0]['quantity']
        assert type(old_balance) == int
        balance = round(old_balance + quantity)
        balance = min(balance, config.MAX_INT)

        bindings = {
            'quantity': balance,
            'address': address,
            'asset': asset
        }
        sql='update balances set quantity = :quantity where (address = :address and asset = :asset)'
        credit_cursor.execute(sql, bindings)

    # Record credit.
    bindings = {
        'block_index': block_index,
        'address': address,
        'asset': asset,
        'quantity': quantity,
        'action': action,
        'event': event
    }
    sql='insert into credits values(:block_index, :address, :asset, :quantity, :action, :event)'

    credit_cursor.execute(sql, bindings)
    credit_cursor.close()

    BLOCK_LEDGER.append('{}{}{}{}'.format(block_index, address, asset, quantity))

class QuantityError(Exception): pass

def is_divisible(db, asset):
    """Check if the asset is divisible."""
    if asset in (config.BTC, config.XCP):
        return True
    else:
        cursor = db.cursor()
        cursor.execute('''SELECT * FROM issuances \
                          WHERE (status = ? AND asset = ?) ORDER BY tx_index DESC''', ('valid', asset))
        issuances = cursor.fetchall()
        if not issuances: raise exceptions.AssetError('No such asset: {}'.format(asset))
        return issuances[0]['divisible']

def value_input(quantity, asset, divisible):
    if asset == 'leverage':
        return round(quantity)

    if asset in ('value', 'fraction', 'price', 'odds'):
        return float(quantity)  # TODO: Float?!

    if divisible:
        quantity = D(quantity) * config.UNIT
        if quantity == quantity.to_integral():
            return int(quantity)
        else:
            raise QuantityError('Divisible assets have only eight decimal places of precision.')
    else:
        quantity = D(quantity)
        if quantity != round(quantity):
            raise QuantityError('Fractional quantities of indivisible assets.')
        return round(quantity)

def value_in(db, quantity, asset, divisible=None):
    if asset not in ['leverage', 'value', 'fraction', 'price', 'odds'] and divisible == None:
        divisible = is_divisible(db, asset)
    return value_input(quantity, asset, divisible)

def value_output(quantity, asset, divisible):

    def norm(num, places):
        """Round only if necessary."""
        num = round(num, places)
        fmt = '{:.' + str(places) + 'f}'
        num = fmt.format(num)
        return num.rstrip('0')+'0' if num.rstrip('0')[-1] == '.' else num.rstrip('0')

    if asset == 'fraction':
        return str(norm(D(quantity) * D(100), 6)) + '%'

    if asset in ('leverage', 'value', 'price', 'odds'):
        return norm(quantity, 6)

    if divisible:
        quantity = D(quantity) / D(config.UNIT)
        if quantity == quantity.to_integral():
            return str(quantity) + '.0'  # For divisible assets, display the decimal point.
        else:
            return norm(quantity, 8)
    else:
        quantity = D(quantity)
        if quantity != round(quantity):
            raise QuantityError('Fractional quantities of indivisible assets.')
        return round(quantity)

def value_out(db, quantity, asset, divisible=None):
    if asset not in ['leverage', 'value', 'fraction', 'price', 'odds'] and divisible == None:
        divisible = is_divisible(db, asset)
    return value_output(quantity, asset, divisible)

### SUPPLIES ###

def holders(db, asset, exclude_empty_holders=False):
    """Return holders of the asset."""
    holders = []
    cursor = db.cursor()
    # Balances
    if exclude_empty_holders:
        cursor.execute('''SELECT * FROM balances \
                          WHERE asset = ? AND quantity > ?''', (asset, 0))
    else:
        cursor.execute('''SELECT * FROM balances \
                          WHERE asset = ?''', (asset, ))

    for balance in list(cursor):
        holders.append({'address': balance['address'], 'address_quantity': balance['quantity'], 'escrow': None})
    # Funds escrowed in orders. (Protocol change.)
    cursor.execute('''SELECT * FROM orders \
                      WHERE give_asset = ? AND status = ?''', (asset, 'open'))
    for order in list(cursor):
        holders.append({'address': order['source'], 'address_quantity': order['give_remaining'], 'escrow': order['tx_hash']})
    # Funds escrowed in pending order matches. (Protocol change.)
    cursor.execute('''SELECT * FROM order_matches \
                      WHERE (forward_asset = ? AND status = ?)''', (asset, 'pending'))
    for order_match in list(cursor):
        holders.append({'address': order_match['tx0_address'], 'address_quantity': order_match['forward_quantity'], 'escrow': order_match['id']})
    cursor.execute('''SELECT * FROM order_matches \
                      WHERE (backward_asset = ? AND status = ?)''', (asset, 'pending'))
    for order_match in list(cursor):
        holders.append({'address': order_match['tx1_address'], 'address_quantity': order_match['backward_quantity'], 'escrow': order_match['id']})

    # Bets and RPS (and bet/rps matches) only escrow XCP.
    if asset == config.XCP:
        cursor.execute('''SELECT * FROM bets \
                          WHERE status = ?''', ('open',))
        for bet in list(cursor):
            holders.append({'address': bet['source'], 'address_quantity': bet['wager_remaining'], 'escrow': bet['tx_hash']})
        cursor.execute('''SELECT * FROM bet_matches \
                          WHERE status = ?''', ('pending',))
        for bet_match in list(cursor):
            holders.append({'address': bet_match['tx0_address'], 'address_quantity': bet_match['forward_quantity'], 'escrow': bet_match['id']})
            holders.append({'address': bet_match['tx1_address'], 'address_quantity': bet_match['backward_quantity'], 'escrow': bet_match['id']})

        cursor.execute('''SELECT * FROM rps \
                          WHERE status = ?''', ('open',))
        for rps in list(cursor):
            holders.append({'address': rps['source'], 'address_quantity': rps['wager'], 'escrow': rps['tx_hash']})
        cursor.execute('''SELECT * FROM rps_matches \
                          WHERE status IN (?, ?, ?)''', ('pending', 'pending and resolved', 'resolved and pending'))
        for rps_match in list(cursor):
            holders.append({'address': rps_match['tx0_address'], 'address_quantity': rps_match['wager'], 'escrow': rps_match['id']})
            holders.append({'address': rps_match['tx1_address'], 'address_quantity': rps_match['wager'], 'escrow': rps_match['id']})

    if enabled('dispensers_in_holders'):
        # Funds escrowed in dispensers.
        cursor.execute('''SELECT * FROM dispensers \
                          WHERE asset = ? AND status = ?''', (asset, 0))


        for dispenser in list(cursor):
            holders.append({'address': dispenser['source'], 'address_quantity': dispenser['give_remaining'], 'escrow': None})

    cursor.close()
    return holders

def xcp_created (db):
    """Return number of XCP created thus far."""
    cursor = db.cursor()
    cursor.execute('''SELECT SUM(earned) AS total FROM burns \
                      WHERE (status = ?)''', ('valid',))
    total = list(cursor)[0]['total'] or 0
    cursor.close()
    return total

def xcp_destroyed (db):
    """Return number of XCP destroyed thus far."""
    cursor = db.cursor()
    # Destructions
    cursor.execute('''SELECT SUM(quantity) AS total FROM destructions \
                      WHERE (status = ? AND asset = ?)''', ('valid', config.XCP))
    destroyed_total = list(cursor)[0]['total'] or 0
    # Subtract issuance fees.
    cursor.execute('''SELECT SUM(fee_paid) AS total FROM issuances\
                      WHERE status = ?''', ('valid',))
    issuance_fee_total = list(cursor)[0]['total'] or 0
    # Subtract dividend fees.
    cursor.execute('''SELECT SUM(fee_paid) AS total FROM dividends\
                      WHERE status = ?''', ('valid',))
    dividend_fee_total = list(cursor)[0]['total'] or 0
    # Subtract sweep fees.
    cursor.execute('''SELECT SUM(fee_paid) AS total FROM sweeps\
                      WHERE status = ?''', ('valid',))
    sweeps_fee_total = list(cursor)[0]['total'] or 0
    cursor.close()
    return destroyed_total + issuance_fee_total + dividend_fee_total + sweeps_fee_total

def xcp_supply (db):
    """Return the XCP supply."""
    return xcp_created(db) - xcp_destroyed(db)

def creations (db):
    """Return creations."""
    cursor = db.cursor()
    creations = {config.XCP: xcp_created(db)}
    cursor.execute('''SELECT asset, SUM(quantity) AS created FROM issuances \
                      WHERE status = ? GROUP BY asset''', ('valid',))

    for issuance in cursor:
        asset = issuance['asset']
        created = issuance['created']
        creations[asset] = created

    cursor.close()
    return creations

def destructions (db):
    """Return destructions."""
    cursor = db.cursor()
    destructions = {config.XCP: xcp_destroyed(db)}
    cursor.execute('''SELECT asset, SUM(quantity) AS destroyed FROM destructions \
                      WHERE (status = ? AND asset != ?) GROUP BY asset''', ('valid', config.XCP))

    for destruction in cursor:
        asset = destruction['asset']
        destroyed = destruction['destroyed']
        destructions[asset] = destroyed

    cursor.close()
    return destructions

def asset_issued_total (db, asset):
    """Return asset total issued."""
    cursor = db.cursor()
    cursor.execute('''SELECT SUM(quantity) AS total FROM issuances \
                      WHERE (status = ? AND asset = ?)''', ('valid', asset))
    issued_total = list(cursor)[0]['total'] or 0
    cursor.close()
    return issued_total

def asset_destroyed_total (db, asset):
    """Return asset total destroyed."""
    cursor = db.cursor()
    cursor.execute('''SELECT SUM(quantity) AS total FROM destructions \
                      WHERE (status = ? AND asset = ?)''', ('valid', asset))
    destroyed_total = list(cursor)[0]['total'] or 0
    cursor.close()
    return destroyed_total

def asset_supply (db, asset):
    """Return asset supply."""
    return asset_issued_total(db, asset) - asset_destroyed_total(db, asset)

def supplies (db):
    """Return supplies."""
    d1 = creations(db)
    d2 = destructions(db)
    return {key: d1[key] - d2.get(key, 0) for key in d1.keys()}

def held (db): #TODO: Rename ?
    queries = [
        "SELECT asset, SUM(quantity) AS total FROM balances GROUP BY asset",
        "SELECT give_asset AS asset, SUM(give_remaining) AS total FROM orders WHERE status = 'open' GROUP BY asset",
        "SELECT give_asset AS asset, SUM(give_remaining) AS total FROM orders WHERE status = 'filled' and give_asset = 'XCP' and get_asset = 'BTC' GROUP BY asset",
        "SELECT forward_asset AS asset, SUM(forward_quantity) AS total FROM order_matches WHERE status = 'pending' GROUP BY asset",
        "SELECT backward_asset AS asset, SUM(backward_quantity) AS total FROM order_matches WHERE status = 'pending' GROUP BY asset",
        "SELECT 'XCP' AS asset, SUM(wager_remaining) AS total FROM bets WHERE status = 'open'",
        "SELECT 'XCP' AS asset, SUM(forward_quantity) AS total FROM bet_matches WHERE status = 'pending'",
        "SELECT 'XCP' AS asset, SUM(backward_quantity) AS total FROM bet_matches WHERE status = 'pending'",
        "SELECT 'XCP' AS asset, SUM(wager) AS total FROM rps WHERE status = 'open'",
        "SELECT 'XCP' AS asset, SUM(wager * 2) AS total FROM rps_matches WHERE status IN ('pending', 'pending and resolved', 'resolved and pending')",
        "SELECT asset, SUM(give_remaining) AS total FROM dispensers WHERE status=0 OR status=1 GROUP BY asset",
    ]

    sql = "SELECT asset, SUM(total) AS total FROM (" + " UNION ALL ".join(queries) + ") GROUP BY asset;"

    cursor = db.cursor()
    cursor.execute(sql)
    held = {}
    for row in cursor:
        asset = row['asset']
        total = row['total']
        held[asset] = total

    return held

### SUPPLIES ###


class GetURLError (Exception): pass
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5):
    """Fetch URL using requests.get."""
    try:
        r = requests.get(url, timeout=fetch_timeout)
    except Exception as e:
        raise GetURLError("Got get_url request error: %s" % e)
    else:
        if r.status_code != 200 and abort_on_error:
            raise GetURLError("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.text))
        result = json.loads(r.text) if is_json else r.text
    return result


def dhash(text):
    if not isinstance(text, bytes):
        text = bytes(str(text), 'utf-8')

    return hashlib.sha256(hashlib.sha256(text).digest()).digest()


def dhash_string(text):
    return binascii.hexlify(dhash(text)).decode()


def get_balance (db, address, asset):
    """Get balance of contract or address."""
    cursor = db.cursor()
    balances = list(cursor.execute('''SELECT * FROM balances WHERE (address = ? AND asset = ?)''', (address, asset)))
    cursor.close()
    if not balances: return 0
    else: return balances[0]['quantity']

# Why on Earth does `binascii.hexlify()` return bytes?!
def hexlify(x):
    """Return the hexadecimal representation of the binary data. Decode from ASCII to UTF-8."""
    return binascii.hexlify(x).decode('ascii')
def unhexlify(hex_string):
    return binascii.unhexlify(bytes(hex_string, 'utf-8'))

### Protocol Changes ###
def enabled(change_name, block_index=None):
    """Return True if protocol change is enabled."""
    if config.REGTEST:
        return True # All changes are always enabled on REGTEST

    if config.TESTNET:
        index_name = 'testnet_block_index'
    else:
        index_name = 'block_index'

    enable_block_index = PROTOCOL_CHANGES[change_name][index_name]

    if not block_index:
        block_index = CURRENT_BLOCK_INDEX

    if block_index >= enable_block_index:
        return True
    else:
        return False

def get_value_by_block_index(change_name, block_index=None):

    if not block_index:
        block_index = CURRENT_BLOCK_INDEX
    
    if config.REGTEST:
        max_block_index_testnet = -1
        for key, value in PROTOCOL_CHANGES[change_name]["testnet"]:
            if int(key) > int(max_block_index):
                max_block_index = key
            
        return PROTOCOL_CHANGES[change_name]["testnet"][max_block_index]["value"]
    
    if config.TESTNET:
        index_name = 'testnet'        
    else:
        index_name = 'mainnet'

    max_block_index = -1
    for key in PROTOCOL_CHANGES[change_name][index_name]:
        if int(key) > int(max_block_index) and block_index >= int(key):
            max_block_index = key
            
    return PROTOCOL_CHANGES[change_name][index_name][max_block_index]["value"]

def transfer(db, source, destination, asset, quantity, action, event):
    """Transfer quantity of asset from source to destination."""
    debit(db, source, asset, quantity, action=action, event=event)
    credit(db, destination, asset, quantity, action=action, event=event)

ID_SEPARATOR = '_'
def make_id(hash_1, hash_2):
    return hash_1 + ID_SEPARATOR + hash_2
def parse_id(match_id):
    assert match_id[64] == ID_SEPARATOR
    return match_id[:64], match_id[65:] # UTF-8 encoding means that the indices are doubled.

def sizeof(v):
    if isinstance(v, dict) or isinstance(v, DictCache):
        s = 0
        for dk, dv in v.items():
            s += sizeof(dk)
            s += sizeof(dv)

        return s
    else:
        return sys.getsizeof(v)

class DictCache:
    """Threadsafe FIFO dict cache"""
    def __init__(self, size=100):
        if int(size) < 1 :
            raise AttributeError('size < 1 or not a number')
        self.size = size
        self.dict = collections.OrderedDict()
        self.lock = threading.Lock()

    def __getitem__(self,key):
        with self.lock:
            return self.dict[key]

    def __setitem__(self,key,value):
        with self.lock:
            while len(self.dict) >= self.size:
                self.dict.popitem(last=False)
            self.dict[key] = value

    def __delitem__(self,key):
        with self.lock:
            del self.dict[key]

    def __len__(self):
        with self.lock:
            return len(self.dict)

    def __contains__(self, key):
        with self.lock:
            return key in self.dict

    def refresh(self, key):
        with self.lock:
            self.dict.move_to_end(key, last=True)


URL_USERNAMEPASS_REGEX = re.compile('.+://(.+)@')
def clean_url_for_log(url):
    m = URL_USERNAMEPASS_REGEX.match(url)
    if m and m.group(1):
        url = url.replace(m.group(1), 'XXXXXXXX')

    return url

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

# ORACLES
def satoshirate_to_fiat(satoshirate):
    return round(satoshirate/100.0,2)

def get_oracle_last_price(db, oracle_address, block_index):
    cursor = db.cursor()
    cursor.execute('SELECT * FROM broadcasts WHERE source=:source AND status=:status AND block_index<:block_index ORDER by tx_index DESC LIMIT 1', {
        'source': oracle_address,
        'status': 'valid',
        'block_index': block_index
    })
    broadcasts = cursor.fetchall()
    cursor.close()
    
    if len(broadcasts) == 0:
        return None, None
    
    oracle_broadcast = broadcasts[0]
    oracle_label = oracle_broadcast["text"].split("-")
    if len(oracle_label) == 2:
        fiat_label = oracle_label[1]
    else:   
        fiat_label = ""
    
    return oracle_broadcast['value'], oracle_broadcast['fee_fraction_int'], fiat_label, oracle_broadcast['block_index'] 
 
#### util_windows.py 
 
import sys
import copy
import logging
import logging.handlers
logger = logging.getLogger(__name__)
import unicodedata
import codecs
from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int
from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID

class SanitizedRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def emit(self, record):
        # If the message doesn't need to be rendered we take a shortcut.
        if record.levelno < self.level:
            return
        # Make sure the message is a string.
        message = record.msg
        #Sanitize and clean up the message
        message = unicodedata.normalize('NFKD', message).encode('ascii', 'ignore').decode()
        # Copy the original record so we don't break other handlers.
        record = copy.copy(record)
        record.msg = message
        # Use the built-in stream handler to handle output.
        logging.handlers.RotatingFileHandler.emit(self, record)

def fix_win32_unicode():
    """Thanks to http://stackoverflow.com/a/3259271 ! (converted to python3)"""
    if sys.platform != "win32":
        return

    original_stderr = sys.stderr

    # If any exception occurs in this code, we'll probably try to print it on stderr,
    # which makes for frustrating debugging if stderr is directed to our wrapper.
    # So be paranoid about catching errors and reporting them to original_stderr,
    # so that we can at least see them.
    def _complain(message):
        print(message if isinstance(message, str) else repr(message), file=original_stderr)

    # Work around <http://bugs.python.org/issue6058>.
    codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

    # Make Unicode console output work independently of the current code page.
    # This also fixes <http://bugs.python.org/issue1602>.
    # Credit to Michael Kaplan <http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx>
    # and TZOmegaTZIOY
    # <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
    try:
        # <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
        # HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
        # returns INVALID_HANDLE_VALUE, NULL, or a valid handle
        #
        # <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
        # DWORD WINAPI GetFileType(DWORD hFile);
        #
        # <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
        # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);

        GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32))
        STD_OUTPUT_HANDLE = DWORD(-11)
        STD_ERROR_HANDLE = DWORD(-12)
        GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32))
        FILE_TYPE_CHAR = 0x0002
        FILE_TYPE_REMOTE = 0x8000
        GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(("GetConsoleMode", windll.kernel32))
        INVALID_HANDLE_VALUE = DWORD(-1).value

        def not_a_console(handle):
            if handle == INVALID_HANDLE_VALUE or handle is None:
                return True
            return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
                    or GetConsoleMode(handle, byref(DWORD())) == 0)

        old_stdout_fileno = None
        old_stderr_fileno = None
        if hasattr(sys.stdout, 'fileno'):
            old_stdout_fileno = sys.stdout.fileno()
        if hasattr(sys.stderr, 'fileno'):
            old_stderr_fileno = sys.stderr.fileno()

        STDOUT_FILENO = 1
        STDERR_FILENO = 2
        real_stdout = (old_stdout_fileno == STDOUT_FILENO)
        real_stderr = (old_stderr_fileno == STDERR_FILENO)

        if real_stdout:
            hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
            if not_a_console(hStdout):
                real_stdout = False

        if real_stderr:
            hStderr = GetStdHandle(STD_ERROR_HANDLE)
            if not_a_console(hStderr):
                real_stderr = False

        if real_stdout or real_stderr:
            # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars,
            #                           LPDWORD lpCharsWritten, LPVOID lpReserved);

            WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)(("WriteConsoleW", windll.kernel32))

            class UnicodeOutput:
                def __init__(self, hConsole, stream, fileno, name):
                    self._hConsole = hConsole
                    self._stream = stream
                    self._fileno = fileno
                    self.closed = False
                    self.softspace = False
                    self.mode = 'w'
                    self.encoding = 'utf-8'
                    self.name = name
                    self.errors = ''
                    self.flush()

                def isatty(self):
                    return False

                def close(self):
                    # don't really close the handle, that would only cause problems
                    self.closed = True

                def fileno(self):
                    return self._fileno

                def flush(self):
                    if self._hConsole is None:
                        try:
                            self._stream.flush()
                        except Exception as e:
                            _complain("%s.flush: %r from %r" % (self.name, e, self._stream))
                            raise

                def write(self, text):
                    try:
                        if self._hConsole is None:
                            if isinstance(text, str):
                                text = text.encode('utf-8')
                            self._stream.write(text)
                        else:
                            if not isinstance(text, str):
                                text = str(text).decode('utf-8')
                            remaining = len(text)
                            while remaining:
                                n = DWORD(0)
                                # There is a shorter-than-documented limitation on the
                                # length of the string passed to WriteConsoleW (see
                                # <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>.
                                retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None)
                                if retval == 0 or n.value == 0:
                                    raise IOError("WriteConsoleW returned %r, n.value = %r" % (retval, n.value))
                                remaining -= n.value
                                if not remaining:
                                    break
                                text = text[n.value:]
                    except Exception as e:
                        _complain("%s.write: %r" % (self.name, e))
                        raise

                def writelines(self, lines):
                    try:
                        for line in lines:
                            self.write(line)
                    except Exception as e:
                        _complain("%s.writelines: %r" % (self.name, e))
                        raise

            if real_stdout:
                sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>')
            else:
                sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>')

            if real_stderr:
                sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>')
            else:
                sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>')
    except Exception as e:
        _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))


    # While we're at it, let's unmangle the command-line arguments:

    # This works around <http://bugs.python.org/issue2128>.
    GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
    CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(("CommandLineToArgvW", windll.shell32))

    argc = c_int(0)
    argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))

    argv = [argv_unicode[i].encode('utf-8').decode('utf-8') for i in range(0, argc.value)]

    if not hasattr(sys, 'frozen'):
        # If this is an executable produced by py2exe or bbfreeze, then it will
        # have been invoked directly. Otherwise, unicode_argv[0] is the Python
        # interpreter, so skip that.
        argv = argv[1:]

        # Also skip option arguments to the Python interpreter.
        while len(argv) > 0:
            arg = argv[0]
            if not arg.startswith("-") or arg == "-":
                break
            argv = argv[1:]
            if arg == '-m':
                # sys.argv[0] should really be the absolute path of the module source,
                # but never mind
                break
            if arg == '-c':
                argv[0] = '-c'
                break

    # if you like:
    sys.argv = argv
 
 
#### __init__.py 
 
 
 
#### addrindexrs.py 
 
import logging
logger = logging.getLogger(__name__)
import sys
import os
import json
import requests
from requests.exceptions import Timeout, ReadTimeout, ConnectionError
import time
import threading
import socket
import concurrent.futures
import collections
import binascii
import hashlib
import signal
import bitcoin.wallet

from counterpartylib.lib import config, util, address

READ_BUF_SIZE = 65536
SOCKET_TIMEOUT = 5.0
BACKEND_PING_TIME = 30.0

raw_transactions_cache = util.DictCache(size=config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)  # used in getrawtransaction_batch()

class BackendRPCError(Exception):
    pass

class AddrIndexRsRPCError(Exception):
    pass

def rpc_call(payload):
    """Calls to bitcoin core and returns the response"""
    url = config.BACKEND_URL
    response = None
    TRIES = 12

    for i in range(TRIES):
        try:
            response = requests.post(url, data=json.dumps(payload), headers={'content-type': 'application/json'},
                verify=(not config.BACKEND_SSL_NO_VERIFY), timeout=config.REQUESTS_TIMEOUT)
            if i > 0:
                logger.debug('Successfully connected.')
            break
        except (Timeout, ReadTimeout, ConnectionError):
            logger.debug('Could not connect to backend at `{}`. (Try {}/{})'.format(util.clean_url_for_log(url), i+1, TRIES))
            time.sleep(5)

    if response == None:
        if config.TESTNET:
            network = 'testnet'
        elif config.REGTEST:
            network = 'regtest'
        else:
            network = 'mainnet'
        raise BackendRPCError('Cannot communicate with backend at `{}`. (server is set to run on {}, is backend?)'.format(util.clean_url_for_log(url), network))
    elif response.status_code in (401,):
        raise BackendRPCError('Authorization error connecting to {}: {} {}'.format(util.clean_url_for_log(url), response.status_code, response.reason))
    elif response.status_code not in (200, 500):
        raise BackendRPCError(str(response.status_code) + ' ' + response.reason)

    # Handle json decode errors
    try:
        response_json = response.json()
    except json.decoder.JSONDecodeError as e:
        raise BackendRPCError('Received invalid JSON from backend with a response of {}'.format(str(response.status_code) + ' ' + response.reason))

    # Batch query returns a list
    if isinstance(response_json, list):
        return response_json
    if 'error' not in response_json.keys() or response_json['error'] == None:
        return response_json['result']
    elif response_json['error']['code'] == -5:   # RPC_INVALID_ADDRESS_OR_KEY
        raise BackendRPCError('{} Is `txindex` enabled in {} Core?'.format(response_json['error'], config.BTC_NAME))
    elif response_json['error']['code'] in [-28, -8, -2]:
        # “Verifying blocks...” or “Block height out of range” or “The network does not appear to fully agree!“
        logger.debug('Backend not ready. Sleeping for ten seconds.')
        # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970`
        # seconds to start, this’ll hit the maximum recursion depth limit.
        time.sleep(10)
        return rpc_call(payload)
    else:
        raise BackendRPCError('Error connecting to {}: {}'.format(util.clean_url_for_log(url), response_json['error']))

def rpc(method, params):
    payload = {
        "method": method,
        "params": params,
        "jsonrpc": "2.0",
        "id": 0,
    }
    return rpc_call(payload)

def rpc_batch(request_list):
    responses = collections.deque()

    def make_call(chunk):
        #send a list of requests to bitcoind to be executed
        #note that this is list executed serially, in the same thread in bitcoind
        #e.g. see: https://github.com/bitcoin/bitcoin/blob/master/src/rpcserver.cpp#L939
        responses.extend(rpc_call(chunk))

    chunks = util.chunkify(request_list, config.RPC_BATCH_SIZE)
    with concurrent.futures.ThreadPoolExecutor(max_workers=config.BACKEND_RPC_BATCH_NUM_WORKERS) as executor:
        for chunk in chunks:
            executor.submit(make_call, chunk)
    return list(responses)

def extract_addresses(txhash_list):
    logger.debug('extract_addresses, txs: %d' % (len(txhash_list), ))
    tx_hashes_tx = getrawtransaction_batch(txhash_list, verbose=True)

    return extract_addresses_from_txlist(tx_hashes_tx, getrawtransaction_batch)

def extract_addresses_from_txlist(tx_hashes_tx, _getrawtransaction_batch):
    """
    helper for extract_addresses, seperated so we can pass in a mocked _getrawtransaction_batch for test purposes
    """

    logger.debug('extract_addresses_from_txlist, txs: %d' % (len(tx_hashes_tx.keys()), ))
    tx_hashes_addresses = {}
    tx_inputs_hashes = set()  # use set to avoid duplicates

    for tx_hash, tx in tx_hashes_tx.items():
        tx_hashes_addresses[tx_hash] = set()
        for vout in tx['vout']:
            if 'addresses' in vout['scriptPubKey']:
                tx_hashes_addresses[tx_hash].update(tuple(vout['scriptPubKey']['addresses']))

        tx_inputs_hashes.update([vin['txid'] for vin in tx['vin']])

    logger.debug('extract_addresses, input TXs: %d' % (len(tx_inputs_hashes), ))

    # chunk txs to avoid huge memory spikes
    for tx_inputs_hashes_chunk in util.chunkify(list(tx_inputs_hashes), config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE):
        raw_transactions = _getrawtransaction_batch(tx_inputs_hashes_chunk, verbose=True)
        for tx_hash, tx in tx_hashes_tx.items():
            for vin in tx['vin']:
                vin_tx = raw_transactions.get(vin['txid'], None)

                if not vin_tx:
                    continue

                vout = vin_tx['vout'][vin['vout']]
                if 'addresses' in vout['scriptPubKey']:
                    tx_hashes_addresses[tx_hash].update(tuple(vout['scriptPubKey']['addresses']))

    return tx_hashes_addresses, tx_hashes_tx

def getblockcount():
    return rpc('getblockcount', [])

def getblockhash(blockcount):
    return rpc('getblockhash', [blockcount])

def getblock(block_hash):
    return rpc('getblock', [block_hash, False])

def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
    return getrawtransaction_batch([tx_hash], verbose=verbose, skip_missing=skip_missing)[tx_hash]

def getrawmempool():
    return rpc('getrawmempool', [])

def fee_per_kb(conf_target, mode, nblocks=None):
    """
    :param conf_target:
    :param mode:
    :return: fee_per_kb in satoshis, or None when unable to determine
    """
    if nblocks is None and conf_target is None:
        conf_target = nblocks

    feeperkb = rpc('estimatesmartfee', [conf_target, mode])

    if 'errors' in feeperkb and feeperkb['errors'][0] == 'Insufficient data or no feerate found':
        return None

    return int(max(feeperkb['feerate'] * config.UNIT, config.DEFAULT_FEE_PER_KB_ESTIMATE_SMART))

def sendrawtransaction(tx_hex):
    return rpc('sendrawtransaction', [tx_hex])

GETRAWTRANSACTION_MAX_RETRIES=2
monotonic_call_id = 0
def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False, _retry=0):
    _logger = logger.getChild("getrawtransaction_batch")

    if len(txhash_list) > config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE:
        #don't try to load in more than BACKEND_RAW_TRANSACTIONS_CACHE_SIZE entries in a single call
        txhash_list_chunks = util.chunkify(txhash_list, config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)
        txes = {}
        for txhash_list_chunk in txhash_list_chunks:
            txes.update(getrawtransaction_batch(txhash_list_chunk, verbose=verbose, skip_missing=skip_missing))
        return txes

    tx_hash_call_id = {}
    payload = []
    noncached_txhashes = set()

    txhash_list = set(txhash_list)

    # payload for transactions not in cache
    for tx_hash in txhash_list:
        if tx_hash not in raw_transactions_cache:
            #call_id = binascii.hexlify(os.urandom(5)).decode('utf8') # Don't drain urandom
            global monotonic_call_id
            monotonic_call_id = monotonic_call_id + 1
            call_id = "{}".format(monotonic_call_id)
            payload.append({
                "method": 'getrawtransaction',
                "params": [tx_hash, 1],
                "jsonrpc": "2.0",
                "id": call_id
            })
            noncached_txhashes.add(tx_hash)
            tx_hash_call_id[call_id] = tx_hash
    #refresh any/all cache entries that already exist in the cache,
    # so they're not inadvertently removed by another thread before we can consult them
    #(this assumes that the size of the working set for any given workload doesn't exceed the max size of the cache)
    for tx_hash in txhash_list.difference(noncached_txhashes):
        raw_transactions_cache.refresh(tx_hash)

    _logger.debug("getrawtransaction_batch: txhash_list size: {} / raw_transactions_cache size: {} / # getrawtransaction calls: {}".format(
        len(txhash_list), len(raw_transactions_cache), len(payload)))

    # populate cache
    if len(payload) > 0:
        batch_responses = rpc_batch(payload)
        for response in batch_responses:
            if 'error' not in response or response['error'] is None:
                tx_hex = response['result']
                tx_hash = tx_hash_call_id[response['id']]
                raw_transactions_cache[tx_hash] = tx_hex
            elif skip_missing and 'error' in response and response['error']['code'] == -5:
                raw_transactions_cache[tx_hash] = None
                logging.debug('Missing TX with no raw info skipped (txhash: {}): {}'.format(
                    tx_hash_call_id.get(response.get('id', '??'), '??'), response['error']))
            else:
                #TODO: this seems to happen for bogus transactions? Maybe handle it more gracefully than just erroring out?
                raise BackendRPCError('{} (txhash:: {})'.format(response['error'], tx_hash_call_id.get(response.get('id', '??'), '??')))

    # get transactions from cache
    result = {}
    for tx_hash in txhash_list:
        try:
            if verbose:
                result[tx_hash] = raw_transactions_cache[tx_hash]
            else:
                result[tx_hash] = raw_transactions_cache[tx_hash]['hex'] if raw_transactions_cache[tx_hash] is not None else None
        except KeyError as e: #shows up most likely due to finickyness with addrindex not always returning results that we need...
            print("Key error in addrindexrs still exists!!!!!")
            _logger.warning("tx missing in rawtx cache: {} -- txhash_list size: {}, hash: {} / raw_transactions_cache size: {} / # rpc_batch calls: {} / txhash in noncached_txhashes: {} / txhash in txhash_list: {} -- list {}".format(
                e, len(txhash_list), hashlib.md5(json.dumps(list(txhash_list)).encode()).hexdigest(), len(raw_transactions_cache), len(payload),
                tx_hash in noncached_txhashes, tx_hash in txhash_list, list(txhash_list.difference(noncached_txhashes)) ))
            if  _retry < GETRAWTRANSACTION_MAX_RETRIES: #try again
                time.sleep(0.05 * (_retry + 1)) # Wait a bit, hitting the index non-stop may cause it to just break down... TODO: Better handling
                r = getrawtransaction_batch([tx_hash], verbose=verbose, skip_missing=skip_missing, _retry=_retry+1)
                result[tx_hash] = r[tx_hash]
            else:
                raise #already tried again, give up

    return result

class AddrIndexRsThread (threading.Thread):
    def __init__(self, host, port):
        threading.Thread.__init__(self)
        self.host = host
        self.port = port
        self.sock = None
        self.lastId = 0
        self.message_to_send = None
        self.message_result = None
        self.is_killed = False

    def stop(self):
        logging.debug('AddrIndexRs thread closing')
        self.send({"kill": True})

    def connect(self):
        self.lastId = 0
        while True:
            logging.info('AddrIndexRs connecting...')
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.settimeout(SOCKET_TIMEOUT)
            try:
                self.sock.connect((self.host, self.port))
            except:
                logging.info('Error connecting to AddrIndexRs! Retrying in a few seconds')
                time.sleep(5.0)
            else:
                logging.info('Connected to AddrIndexRs!')
                break

    def run(self):
        self.locker = threading.Condition()
        self.locker.acquire()
        self.connect()
        while self.locker.wait():
            if not(self.is_killed) and self.message_to_send != None:
                msg = self.message_to_send
                self.message_to_send = None
                retry_count = 15
                while retry_count > 0:
                    has_sent = False
                    while not(has_sent) and msg:
                        try:
                            logging.debug('AddrIndexRs sending')
                            self.sock.send(msg)
                            has_sent = True
                        except Exception as e:
                            #try:
                            logging.debug('AddrIndexRs error:' + e)
                            self.connect()
                            #except Exception as e2:
                            #logging.debug('AddrIndexRs fatal error:' + e2)

                    self.message_to_send = None
                    data = b""
                    parsed = False
                    while not(parsed):
                        try:
                            data = data + self.sock.recv(READ_BUF_SIZE)
                            self.message_result = json.loads(data.decode('utf-8'))
                            retry_count = 0
                            parsed = True
                            logging.debug('AddrIndexRs Recv complete!')
                        except socket.timeout:
                            logging.debug('AddrIndexRs Recv timeout error sending: '+str(msg))
                            if retry_count <= 0:
                                self.connect()
                            self.message_result = None
                            retry_count -= -1
                        except socket.error as e:
                            logging.debug('AddrIndexRs Recv error:' + str(e)+' with msg '+str(msg))
                            self.connect()
                        except Exception as e:
                            logging.debug('AddrIndexRs Recv error:' + str(e)+' with msg '+str(msg))
                            if retry_count <= 0:
                                raise e
                            self.message_result = None
                            retry_count -= 1
                        finally:
                            self.locker.notify()
            else:
                self.locker.notify()
        self.sock.close()
        logging.debug('AddrIndexRs socket closed normally')

    def send(self, msg):
        self.locker.acquire()
        if not("kill" in msg):
            msg["id"] = self.lastId
            self.lastId += 1
            self.message_to_send = (json.dumps(msg) + "\n").encode('utf8')
        self.locker.notify()
        self.locker.wait()
        self.locker.release()
        return self.message_result


_backend = None

def ensure_addrindexrs_connected():
    global _backend
    backoff = 500
    max_backoff = 5000
    while _backend == None:
        try:
            _backend = AddrIndexRsThread(config.INDEXD_CONNECT, config.INDEXD_PORT)
            _backend.daemon = True
            _backend.start()

            _backend.send({
                "method": "server.version",
                "params": []
            })
        except Exception as e:
            logger.debug(e)
            time.sleep(backoff)
            backoff = min(backoff * 1.5, max_backoff)

def _script_pubkey_to_hash(spk):
    return hashlib.sha256(spk).digest()[::-1].hex()

def _address_to_hash(addr):
    script_pubkey = bitcoin.wallet.CBitcoinAddress(addr).to_scriptPubKey()
    return _script_pubkey_to_hash(script_pubkey)

# Returns an array of UTXOS from an address in the following format
# {
#   "txId": utxo_txid_hex,
#   "vout": num,
#   "height": num,
#   "value": sats,
#   "confirmations": num
# }
# [{"txId":"a0d12eb3716e2e70fd00525486ace0da2947f82d818b7be0285f16ff672cf237","vout":5,"height":647484,"value":30455293,"confirmations":2}]
#
def unpack_outpoint(outpoint):
    txid, vout = outpoint.split(':')
    return (txid, int(vout))

def unpack_vout(outpoint, tx, block_count):
    if tx is None:
        return None

    vout = tx["vout"][outpoint[1]]
    height = -1
    if "confirmations" in tx and tx["confirmations"] > 0:
        height = block_count - tx["confirmations"] + 1
    else:
        tx["confirmations"] = 0

    return {
        "txId": tx["txid"],
        "vout": outpoint[1],
        "height": height,
        "value": int(round(vout["value"] * config.UNIT)),
        "confirmations": tx["confirmations"]
    }

def get_unspent_txouts(source):
    ensure_addrindexrs_connected()

    block_count = getblockcount()
    result = _backend.send({
        "method": "blockchain.scripthash.get_utxos",
        "params": [_address_to_hash(source)]
    })

    if not(result is None) and "result" in result:
        result = result["result"]
        result = [unpack_outpoint(x) for x in result]
        # each item on the result array is like
        # {"tx_hash": hex_encoded_hash}
        batch = getrawtransaction_batch([x[0] for x in result], verbose=True, skip_missing=True)
        batch = [unpack_vout(outpoint, batch[outpoint[0]], block_count) for outpoint in result if outpoint[0] in batch]
        batch = [x for x in batch if x is not None]

        return batch
    else:
        return []

# Returns transactions in the following format
# {
#  "blockhash": hexstring,
#  "blocktime": num,
#  "confirmations": num,
#  "hash": hexstring,
#  "height": num,
#  "hex": hexstring,
#  "locktime": num,
#  "size": num,
#  "time": num,
#  "txid": hexstring,
#  "version": num,
#  "vsize": num,
#  "weight": num,
#  "vin": [
#    {
#      "txinwitness": array of hex_witness_program, // Only if it's a witness-containing tx
#      "vout": num,
#      "txid": hexstring,
#      "sequence": num,
#      "coinbase": X, // contents not important, this is only present if the tx is a coinbase
#      "scriptSig": {
#        "asm": asm_decompiled_program,
#        "hex": hex_program
#      }
#    },...
#  ],
#  "vout": [
#    {
#       "n": num,
#       "value": decimal,
#       "scriptPubKey": {
#           "type": string,
#           "reqSigs": num,
#           "hex": hexstring, // the program in hex
#           "asm": string, // the decompiled program
#           "addresses": [ ...list of found addresses on the program ]
#       }
#    }
#  ]
#
# }
def search_raw_transactions(address, unconfirmed=True):
    ensure_addrindexrs_connected()

    hsh = _address_to_hash(address)
    txs = _backend.send({
        "method": "blockchain.scripthash.get_history",
        "params": [hsh]
    })["result"]

    batch = getrawtransaction_batch([x["tx_hash"] for x in txs], verbose=True)

    if not(unconfirmed):
        batch = [x for x in batch if x.height >= 0]

    return batch

# Returns the number of blocks the backend is behind the node
def getindexblocksbehind():
    # Addrindexrs never "gets behind"
    return 0

def init():
    ensure_addrindexrs_connected()

def stop():
    if '_backend' in globals():
        _backend.stop()
 
 
#### indexd.py 
 
import logging
logger = logging.getLogger(__name__)
import sys
import os
import json
import requests
from requests.exceptions import Timeout, ReadTimeout, ConnectionError
import time
import threading
import concurrent.futures
import collections
import binascii
import hashlib

from counterpartylib.lib import config, util

raw_transactions_cache = util.DictCache(size=config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)  # used in getrawtransaction_batch()


class BackendRPCError(Exception):
    pass

class IndexdRPCError(Exception):
    pass


def rpc_call(payload):
    """Calls to bitcoin core and returns the response"""
    url = config.BACKEND_URL
    response = None
    TRIES = 12

    for i in range(TRIES):
        try:
            response = requests.post(url, data=json.dumps(payload), headers={'content-type': 'application/json'},
                verify=(not config.BACKEND_SSL_NO_VERIFY), timeout=config.REQUESTS_TIMEOUT)
            if i > 0:
                logger.debug('Successfully connected.')
            break
        except (Timeout, ReadTimeout, ConnectionError):
            logger.debug('Could not connect to backend at `{}`. (Try {}/{})'.format(util.clean_url_for_log(url), i+1, TRIES))
            time.sleep(5)

    if response == None:
        if config.TESTNET:
            network = 'testnet'
        elif config.REGTEST:
            network = 'regtest'
        else:
            network = 'mainnet'
        raise BackendRPCError('Cannot communicate with backend at `{}`. (server is set to run on {}, is backend?)'.format(util.clean_url_for_log(url), network))
    elif response.status_code in (401,):
        raise BackendRPCError('Authorization error connecting to {}: {} {}'.format(util.clean_url_for_log(url), response.status_code, response.reason))
    elif response.status_code not in (200, 500):
        raise BackendRPCError(str(response.status_code) + ' ' + response.reason)

    # Handle json decode errors
    try:
        response_json = response.json()
    except json.decoder.JSONDecodeError as e:
        raise BackendRPCError('Received invalid JSON from backend with a response of {}'.format(str(response.status_code) + ' ' + response.reason))

    # Batch query returns a list
    if isinstance(response_json, list):
        return response_json
    if 'error' not in response_json.keys() or response_json['error'] == None:
        return response_json['result']
    elif response_json['error']['code'] == -5:   # RPC_INVALID_ADDRESS_OR_KEY
        raise BackendRPCError('{} Is `txindex` enabled in {} Core?'.format(response_json['error'], config.BTC_NAME))
    elif response_json['error']['code'] in [-28, -8, -2]:
        # “Verifying blocks...” or “Block height out of range” or “The network does not appear to fully agree!“
        logger.debug('Backend not ready. Sleeping for ten seconds.')
        # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970`
        # seconds to start, this’ll hit the maximum recursion depth limit.
        time.sleep(10)
        return rpc_call(payload)
    else:
        raise BackendRPCError('Error connecting to {}: {}'.format(util.clean_url_for_log(url), response_json['error']))

def rpc(method, params):
    payload = {
        "method": method,
        "params": params,
        "jsonrpc": "2.0",
        "id": 0,
    }
    return rpc_call(payload)

def rpc_batch(request_list):
    responses = collections.deque()

    def make_call(chunk):
        #send a list of requests to bitcoind to be executed
        #note that this is list executed serially, in the same thread in bitcoind
        #e.g. see: https://github.com/bitcoin/bitcoin/blob/master/src/rpcserver.cpp#L939
        responses.extend(rpc_call(chunk))

    chunks = util.chunkify(request_list, config.RPC_BATCH_SIZE)
    with concurrent.futures.ThreadPoolExecutor(max_workers=config.BACKEND_RPC_BATCH_NUM_WORKERS) as executor:
        for chunk in chunks:
            executor.submit(make_call, chunk)
    return list(responses)


def extract_addresses(txhash_list):
    logger.debug('extract_addresses, txs: %d' % (len(txhash_list), ))
    tx_hashes_tx = getrawtransaction_batch(txhash_list, verbose=True)

    return extract_addresses_from_txlist(tx_hashes_tx, getrawtransaction_batch)


def extract_addresses_from_txlist(tx_hashes_tx, _getrawtransaction_batch):
    """
    helper for extract_addresses, seperated so we can pass in a mocked _getrawtransaction_batch for test purposes
    """

    logger.debug('extract_addresses_from_txlist, txs: %d' % (len(tx_hashes_tx.keys()), ))
    tx_hashes_addresses = {}
    tx_inputs_hashes = set()  # use set to avoid duplicates

    for tx_hash, tx in tx_hashes_tx.items():
        tx_hashes_addresses[tx_hash] = set()
        for vout in tx['vout']:
            if 'addresses' in vout['scriptPubKey']:
                tx_hashes_addresses[tx_hash].update(tuple(vout['scriptPubKey']['addresses']))

        tx_inputs_hashes.update([vin['txid'] for vin in tx['vin']])

    logger.debug('extract_addresses, input TXs: %d' % (len(tx_inputs_hashes), ))

    # chunk txs to avoid huge memory spikes
    for tx_inputs_hashes_chunk in util.chunkify(list(tx_inputs_hashes), config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE):
        raw_transactions = _getrawtransaction_batch(tx_inputs_hashes_chunk, verbose=True)
        for tx_hash, tx in tx_hashes_tx.items():
            for vin in tx['vin']:
                vin_tx = raw_transactions.get(vin['txid'], None)

                if not vin_tx:
                    continue

                vout = vin_tx['vout'][vin['vout']]
                if 'addresses' in vout['scriptPubKey']:
                    tx_hashes_addresses[tx_hash].update(tuple(vout['scriptPubKey']['addresses']))

    return tx_hashes_addresses, tx_hashes_tx

def getblockcount():
    return rpc('getblockcount', [])

def getblockhash(blockcount):
    return rpc('getblockhash', [blockcount])

def getblock(block_hash):
    return rpc('getblock', [block_hash, False])

def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
    return getrawtransaction_batch([tx_hash], verbose=verbose, skip_missing=skip_missing)[tx_hash]

def getrawmempool():
    return rpc('getrawmempool', [])

def fee_per_kb(conf_target, mode, nblocks=None):
    """
    :param conf_target:
    :param mode:
    :return: fee_per_kb in satoshis, or None when unable to determine
    """
    if nblocks is None and conf_target is None:
        conf_target = nblocks

    feeperkb = rpc('estimatesmartfee', [conf_target, mode])

    if 'errors' in feeperkb and feeperkb['errors'][0] == 'Insufficient data or no feerate found':
        return None

    return int(max(feeperkb['feerate'] * config.UNIT, config.DEFAULT_FEE_PER_KB_ESTIMATE_SMART))

def sendrawtransaction(tx_hex):
    return rpc('sendrawtransaction', [tx_hex])

GETRAWTRANSACTION_MAX_RETRIES=2
def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False, _retry=0):
    _logger = logger.getChild("getrawtransaction_batch")

    if len(txhash_list) > config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE:
        #don't try to load in more than BACKEND_RAW_TRANSACTIONS_CACHE_SIZE entries in a single call
        txhash_list_chunks = util.chunkify(txhash_list, config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)
        txes = {}
        for txhash_list_chunk in txhash_list_chunks:
            txes.update(getrawtransaction_batch(txhash_list_chunk, verbose=verbose, skip_missing=skip_missing))
        return txes

    tx_hash_call_id = {}
    payload = []
    noncached_txhashes = set()

    txhash_list = set(txhash_list)

    # payload for transactions not in cache
    for tx_hash in txhash_list:
        if tx_hash not in raw_transactions_cache:
            call_id = binascii.hexlify(os.urandom(5)).decode('utf8')
            payload.append({
                "method": 'getrawtransaction',
                "params": [tx_hash, 1],
                "jsonrpc": "2.0",
                "id": call_id
            })
            noncached_txhashes.add(tx_hash)
            tx_hash_call_id[call_id] = tx_hash
    #refresh any/all cache entries that already exist in the cache,
    # so they're not inadvertently removed by another thread before we can consult them
    #(this assumes that the size of the working set for any given workload doesn't exceed the max size of the cache)
    for tx_hash in txhash_list.difference(noncached_txhashes):
        raw_transactions_cache.refresh(tx_hash)

    _logger.debug("getrawtransaction_batch: txhash_list size: {} / raw_transactions_cache size: {} / # getrawtransaction calls: {}".format(
        len(txhash_list), len(raw_transactions_cache), len(payload)))

    # populate cache
    if len(payload) > 0:
        batch_responses = rpc_batch(payload)
        for response in batch_responses:
            if 'error' not in response or response['error'] is None:
                tx_hex = response['result']
                tx_hash = tx_hash_call_id[response['id']]
                raw_transactions_cache[tx_hash] = tx_hex
            elif skip_missing and 'error' in response and response['error']['code'] == -5:
                raw_transactions_cache[tx_hash] = None
                logging.debug('Missing TX with no raw info skipped (txhash: {}): {}'.format(
                    tx_hash_call_id.get(response.get('id', '??'), '??'), response['error']))
            else:
                #TODO: this seems to happen for bogus transactions? Maybe handle it more gracefully than just erroring out?
                raise BackendRPCError('{} (txhash:: {})'.format(response['error'], tx_hash_call_id.get(response.get('id', '??'), '??')))

    # get transactions from cache
    result = {}
    for tx_hash in txhash_list:
        try:
            if verbose:
                result[tx_hash] = raw_transactions_cache[tx_hash]
            else:
                result[tx_hash] = raw_transactions_cache[tx_hash]['hex'] if raw_transactions_cache[tx_hash] is not None else None
        except KeyError as e: #shows up most likely due to finickyness with addrindex not always returning results that we need...
            _logger.warning("tx missing in rawtx cache: {} -- txhash_list size: {}, hash: {} / raw_transactions_cache size: {} / # rpc_batch calls: {} / txhash in noncached_txhashes: {} / txhash in txhash_list: {} -- list {}".format(
                e, len(txhash_list), hashlib.md5(json.dumps(list(txhash_list)).encode()).hexdigest(), len(raw_transactions_cache), len(payload),
                tx_hash in noncached_txhashes, tx_hash in txhash_list, list(txhash_list.difference(noncached_txhashes)) ))
            if  _retry < GETRAWTRANSACTION_MAX_RETRIES: #try again
                time.sleep(0.05 * (_retry + 1)) # Wait a bit, hitting the index non-stop may cause it to just break down... TODO: Better handling
                r = getrawtransaction_batch([tx_hash], verbose=verbose, skip_missing=skip_missing, _retry=_retry+1)
                result[tx_hash] = r[tx_hash]
            else:
                raise #already tried again, give up

    return result

def get_unspent_txouts(source):
    return indexd_rpc_call('/a/'+source+'/utxos')

def search_raw_transactions(address, unconfirmed=True):
    all_transactions = indexd_rpc_call('/a/'+address+'/txs?verbose=1')

    if unconfirmed:
        return all_transactions

    # filter for confirmed transactions only
    confirmed_transactions = list(filter(lambda t: 'confirmations' in t and t['confirmations'] > 0, all_transactions))
    return confirmed_transactions



def getindexblocksbehind():
    status = indexd_rpc_call('/status')
    if status['ready']:
        return 0
    if status['blocksBehind']:
        return status['blocksBehind']
    raise IndexdRPCError('Unknown status for indexd')

def indexd_rpc_call(path):
    url = config.INDEXD_URL+path

    response = None
    try:
        response = requests.get(url, headers={'content-type': 'application/json'},
            timeout=config.REQUESTS_TIMEOUT)
    except (Timeout, ReadTimeout, ConnectionError):
        logger.debug('Could not connect to backend at `{}`.'.format(util.clean_url_for_log(url),))

    if response == None:
        if config.TESTNET:
            network = 'testnet'
        elif config.REGTEST:
            network = 'regtest'
        else:
            network = 'mainnet'
        raise IndexdRPCError('Cannot communicate with {} indexd server at `{}`.'.format(network, util.clean_url_for_log(url)))
    elif response.status_code == 400:
        raise IndexdRPCError('Indexd returned error: {} {} {}'.format(response.status_code, response.reason, response.text))
    elif response.status_code not in (200, 500):
        raise IndexdRPCError("Bad response from {}: {} {}".format(util.clean_url_for_log(url), response.status_code, response.reason))

    # Return result, with error handling.
    response_json = response.json()
    if isinstance(response_json, (list, tuple)) or 'error' not in response_json.keys() or response_json['error'] == None:
        return response_json
    else:
        raise IndexdRPCError('{}'.format(response_json['error']))

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### __init__.py 
 
import getpass
import binascii
import logging
logger = logging.getLogger(__name__)
import sys
import json
import time
from decimal import Decimal as D

import bitcoin as bitcoinlib
import bitcoin.rpc as bitcoinlib_rpc
from bitcoin.core import CBlock

from counterpartylib.lib import util
from counterpartylib.lib import script
from counterpartylib.lib import config
from counterpartylib.lib import exceptions

from counterpartylib.lib.backend import addrindexrs

MEMPOOL_CACHE_INITIALIZED = False

PRETX_CACHE = {}

def sortkeypicker(keynames):
    """http://stackoverflow.com/a/1143719"""
    negate = set()
    for i, k in enumerate(keynames):
        if k[:1] == '-':
            keynames[i] = k[1:]
            negate.add(k[1:])
    def getit(adict):
       composite = [adict[k] for k in keynames]
       for i, (k, v) in enumerate(zip(keynames, composite)):
           if k in negate:
               composite[i] = -v
       return composite
    return getit

def BACKEND():
    mdl = sys.modules['counterpartylib.lib.backend.{}'.format(config.BACKEND_NAME)]
    mdl.init()
    return mdl

def stop():
    BACKEND().stop()

def getblockcount():
    return BACKEND().getblockcount()

def getblockhash(blockcount):
    return BACKEND().getblockhash(blockcount)

def getblock(block_hash):
    block_hex = BACKEND().getblock(block_hash)
    return CBlock.deserialize(util.unhexlify(block_hex))

def cache_pretx(txid, rawtx):
    PRETX_CACHE[binascii.hexlify(txid).decode('utf8')] = binascii.hexlify(rawtx).decode('utf8')

def clear_pretx(txid):
    del PRETX_CACHE[binascii.hexlify(txid).decode('utf8')]

def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
    if tx_hash in PRETX_CACHE:
        return PRETX_CACHE[tx_hash]
    else:
        return BACKEND().getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)

def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
    return BACKEND().getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)

def sendrawtransaction(tx_hex):
    return BACKEND().sendrawtransaction(tx_hex)

def getrawmempool():
    return BACKEND().getrawmempool()

def getindexblocksbehind():
    return BACKEND().getindexblocksbehind()

def extract_addresses(txhash_list):
    return BACKEND().extract_addresses(txhash_list)

def ensure_script_pub_key_for_inputs(coins):
    txhash_set = set()
    for coin in coins:
        if 'scriptPubKey' not in coin:
            txhash_set.add(coin['txid'])

    if len(txhash_set) > 0:
        txs = BACKEND().getrawtransaction_batch(list(txhash_set), verbose=True, skip_missing=False)
        for coin in coins:
            if 'scriptPubKey' not in coin:
                # get the scriptPubKey
                txid = coin['txid']
                for vout in txs[txid]['vout']:
                    if vout['n'] == coin['vout']:
                        coin['scriptPubKey'] = vout['scriptPubKey']['hex']

    return coins


def fee_per_kb(conf_target, mode, nblocks=None):
    """
    :param conf_target:
    :param mode:
    :return: fee_per_kb in satoshis, or None when unable to determine
    """

    return BACKEND().fee_per_kb(conf_target, mode, nblocks=nblocks)


def deserialize(tx_hex):
    return bitcoinlib.core.CTransaction.deserialize(binascii.unhexlify(tx_hex))

def serialize(ctx):
    return bitcoinlib.core.CTransaction.serialize(ctx)

def is_valid(address):
    try:
        script.validate(address)
        return True
    except script.AddressError:
        return False

def get_txhash_list(block):
    return [bitcoinlib.core.b2lx(ctx.GetHash()) for ctx in block.vtx]

def get_tx_list(block):
    raw_transactions = {}
    tx_hash_list = []

    for ctx in block.vtx:
        if util.enabled('correct_segwit_txids'):
            hsh = ctx.GetTxid()
        else:
            hsh = ctx.GetHash()
        tx_hash = bitcoinlib.core.b2lx(hsh)
        raw = ctx.serialize()

        tx_hash_list.append(tx_hash)
        raw_transactions[tx_hash] = bitcoinlib.core.b2x(raw)

    return (tx_hash_list, raw_transactions)

def sort_unspent_txouts(unspent, unconfirmed=False):
    # Filter out all dust amounts to avoid bloating the resultant transaction
    unspent = list(filter(lambda x: x['value'] > config.DEFAULT_MULTISIG_DUST_SIZE, unspent))
    # Sort by amount, using the largest UTXOs available
    if config.REGTEST:
        # REGTEST has a lot of coinbase inputs that can't be spent due to maturity
        # this doesn't usually happens on mainnet or testnet because most fednodes aren't mining
        unspent = sorted(unspent, key=lambda x: (x['confirmations'], x['value']), reverse=True)
    else:
        unspent = sorted(unspent, key=lambda x: x['value'], reverse=True)

    return unspent

def get_btc_supply(normalize=False):
    """returns the total supply of {} (based on what Bitcoin Core says the current block height is)""".format(config.BTC)
    block_count = getblockcount()
    blocks_remaining = block_count
    total_supply = 0
    reward = 50.0
    while blocks_remaining > 0:
        if blocks_remaining >= 210000:
            blocks_remaining -= 210000
            total_supply += 210000 * reward
            reward /= 2
        else:
            total_supply += (blocks_remaining * reward)
            blocks_remaining = 0
    return total_supply if normalize else int(total_supply * config.UNIT)

class MempoolError(Exception):
    pass

def get_unspent_txouts(source, unconfirmed=False, unspent_tx_hash=None):
    """returns a list of unspent outputs for a specific address
    @return: A list of dicts, with each entry in the dict having the following keys:
    """

    unspent = BACKEND().get_unspent_txouts(source)

    # filter by unspent_tx_hash
    if unspent_tx_hash is not None:
        unspent = list(filter(lambda x: x['txId'] == unspent_tx_hash, unspent))

    # filter unconfirmed
    if not unconfirmed:
        unspent = [utxo for utxo in unspent if utxo['confirmations'] > 0]

    # format
    for utxo in unspent:
        utxo['amount'] = float(utxo['value'] / config.UNIT)
        utxo['txid'] = utxo['txId']
        del utxo['txId']
        # do not add scriptPubKey

    return unspent

def search_raw_transactions(address, unconfirmed=True):
    return BACKEND().search_raw_transactions(address, unconfirmed)

class UnknownPubKeyError(Exception):
    pass

def pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys=None):
    # Search provided pubkeys.
    if provided_pubkeys:
        if type(provided_pubkeys) != list:
            provided_pubkeys = [provided_pubkeys]
        for pubkey in provided_pubkeys:
            if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                return pubkey
            elif pubkeyhash == script.pubkey_to_p2whash(util.unhexlify(pubkey)):
                return pubkey

    # Search blockchain.
    raw_transactions = search_raw_transactions(pubkeyhash, unconfirmed=True)
    for tx_id in raw_transactions:
        tx = raw_transactions[tx_id]
        for vin in tx['vin']:
            if 'txinwitness' in vin:
                if len(vin['txinwitness']) >= 2:
                    # catch unhexlify errs for when txinwitness[1] isn't a witness program (eg; for P2W)
                    try:
                        pubkey = vin['txinwitness'][1]
                        if pubkeyhash == script.pubkey_to_p2whash(util.unhexlify(pubkey)):
                            return pubkey
                    except binascii.Error:
                        pass
            elif 'coinbase' not in vin:
                scriptsig = vin['scriptSig']
                asm = scriptsig['asm'].split(' ')
                if len(asm) >= 2:
                    # catch unhexlify errs for when asm[1] isn't a pubkey (eg; for P2SH)
                    try:
                        pubkey = asm[1]
                        if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                            return pubkey
                    except binascii.Error:
                        pass

    raise UnknownPubKeyError('Public key was neither provided nor published in blockchain.')


def multisig_pubkeyhashes_to_pubkeys(address, provided_pubkeys=None):
    signatures_required, pubkeyhashes, signatures_possible = script.extract_array(address)
    pubkeys = [pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys) for pubkeyhash in pubkeyhashes]
    return script.construct_array(signatures_required, pubkeys, signatures_possible)


def init_mempool_cache():
    """prime the mempool cache, so that functioning is faster...
    """
    global MEMPOOL_CACHE_INITIALIZED
    logger.debug('Initializing mempool cache...')
    start = time.time()

    mempool_txhash_list = getrawmempool()

    #with this function, don't try to load in more than BACKEND_RAW_TRANSACTIONS_CACHE_SIZE entries
    num_tx = min(len(mempool_txhash_list), config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)
    mempool_tx = BACKEND().getrawtransaction_batch(mempool_txhash_list[:num_tx], skip_missing=True, verbose=True)

    vin_txhash_list = []
    max_remaining_num_tx = config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE - num_tx
    if max_remaining_num_tx:
        for txid in mempool_tx:
            tx = mempool_tx[txid]
            if not(tx is None):
                vin_txhash_list += [vin['txid'] for vin in tx['vin']]
        BACKEND().getrawtransaction_batch(vin_txhash_list[:max_remaining_num_tx], skip_missing=True, verbose=True)

    MEMPOOL_CACHE_INITIALIZED = True
    logger.info('Mempool cache initialized: {:.2f}s for {:,} transactions'.format(time.time() - start, num_tx + min(max_remaining_num_tx, len(vin_txhash_list))))


# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
 
 
#### bc_data_stream.py 
 
#
# Workalike python implementation of Bitcoin's CDataStream class.
#
import struct
import mmap
from .exceptions import SerializationError

class BCDataStream(object):
  def __init__(self):
    self.input = None
    self.read_cursor = 0

  def clear(self):
    self.input = None
    self.read_cursor = 0

  def write(self, bytes):  # Initialize with string of bytes
    if self.input is None:
      self.input = bytes
    else:
      self.input += bytes

  def map_file(self, file, start):  # Initialize with bytes from file
    self.input = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)
    self.read_cursor = start

  def seek_file(self, position):
    self.read_cursor = position

  def close_file(self):
    self.input.close()

  def read_string(self):
    # Strings are encoded depending on length:
    # 0 to 252 :  1-byte-length followed by bytes (if any)
    # 253 to 65,535 : byte'253' 2-byte-length followed by bytes
    # 65,536 to 4,294,967,295 : byte '254' 4-byte-length followed by bytes
    # ... and the Bitcoin client is coded to understand:
    # greater than 4,294,967,295 : byte '255' 8-byte-length followed by bytes of string
    # ... but I don't think it actually handles any strings that big.
    if self.input is None:
      raise SerializationError("call write(bytes) before trying to deserialize")

    try:
      length = self.read_compact_size()
    except IndexError:
      raise SerializationError("attempt to read past end of buffer")

    return self.read_bytes(length)

  def write_string(self, string):
    # Length-encoded as with read-string
    self.write_compact_size(len(string))
    self.write(string)

  def read_bytes(self, length):
    try:
      result = self.input[self.read_cursor:self.read_cursor+length]
      self.read_cursor += length
      return result
    except IndexError:
      raise SerializationError("attempt to read past end of buffer")

    return ''

  def read_boolean(self): return self.read_bytes(1)[0] != chr(0)
  def read_int16(self): return self._read_num('<h')
  def read_uint16(self): return self._read_num('<H')
  def read_int32(self): return self._read_num('<i')
  def read_uint32(self): return self._read_num('<I')
  def read_int64(self): return self._read_num('<q')
  def read_uint64(self): return self._read_num('<Q')

  def write_boolean(self, val): return self.write(chr(1) if val else chr(0))
  def write_int16(self, val): return self._write_num('<h', val)
  def write_uint16(self, val): return self._write_num('<H', val)
  def write_int32(self, val): return self._write_num('<i', val)
  def write_uint32(self, val): return self._write_num('<I', val)
  def write_int64(self, val): return self._write_num('<q', val)
  def write_uint64(self, val): return self._write_num('<Q', val)

  def read_compact_size(self):
    size = self.input[self.read_cursor]
    self.read_cursor += 1
    if size == 253:
      size = self._read_num('<H')
    elif size == 254:
      size = self._read_num('<I')
    elif size == 255:
      size = self._read_num('<Q')
    return size

  def write_compact_size(self, size):
    if size < 0:
      raise SerializationError("attempt to write size < 0")
    elif size < 253:
       self.write(chr(size))
    elif size < 2**16:
      self.write('\xfd')
      self._write_num('<H', size)
    elif size < 2**32:
      self.write('\xfe')
      self._write_num('<I', size)
    elif size < 2**64:
      self.write('\xff')
      self._write_num('<Q', size)

  def _read_num(self, format):
    (i,) = struct.unpack_from(format, self.input, self.read_cursor)
    self.read_cursor += struct.calcsize(format)
    return i

  def _write_num(self, format, num):
    s = struct.pack(format, num)
    self.write(s)

  def read_var_int(self):
    n = 0
    while True:
        cur_byte = self.read_bytes(1)[0]
        n = (n << 7) | (cur_byte & 0x7F)
        if cur_byte & 0x80:
            n += 1
        else:
            return n
 
 
#### blocks_parser.py 
 
import os, json, time, logging, binascii
import logging
logger = logging.getLogger(__name__)

from .bc_data_stream import BCDataStream
from .utils import b2h, double_hash, ib2h, inverse_hash

def open_leveldb(db_dir):
    try:
        import plyvel
    except:
        raise Exception("Please install the plyvel package via pip3.")

    try:
        return plyvel.DB(db_dir, create_if_missing=False)
    except plyvel._plyvel.IOError as e:
        logger.info(str(e))
        raise Exception("Ensure that bitcoind is stopped.")

class BlockchainParser():

    def __init__(self, blocks_dir, leveldb_dir):
        self.blocks_dir = blocks_dir 
        self.leveldb_dir = leveldb_dir
        self.file_num = -1
        self.current_file_size = 0
        self.current_block_file = None
        self.data_stream = None
        self.ldb = open_leveldb(self.leveldb_dir)

    def read_tx_in(self, vds):
        tx_in = {}
        tx_in['txid'] = ib2h(vds.read_bytes(32))
        tx_in['vout'] = vds.read_uint32()
        script_sig_size = vds.read_compact_size()
        tx_in['scriptSig'] = b2h(vds.read_bytes(script_sig_size))
        tx_in['sequence'] = vds.read_uint32()
        if tx_in['txid'] == '0000000000000000000000000000000000000000000000000000000000000000':
            tx_in = {
                'coinbase': tx_in['scriptSig'],
                'sequence': tx_in['sequence']
            }
        return tx_in

    def read_tx_out(self, vds):
        tx_out = {}
        tx_out['value'] = vds.read_int64() / 100000000
        script = vds.read_bytes(vds.read_compact_size())
        tx_out['scriptPubKey'] = {
            'hex': b2h(script)
        }
        return tx_out

    def read_transaction(self, vds):
        transaction = {}
        start_pos = vds.read_cursor
        transaction['version'] = vds.read_int32()

        transaction['vin'] = []
        for i in range(vds.read_compact_size()):
            transaction['vin'].append(self.read_tx_in(vds))

        transaction['vout'] = []
        for i in range(vds.read_compact_size()):
            transaction['vout'].append(self.read_tx_out(vds))

        transaction['lock_time'] = vds.read_uint32()
        data = vds.input[start_pos:vds.read_cursor]
        transaction['tx_hash'] = ib2h(double_hash(data))
        transaction['__data__'] = b2h(data)
        return transaction

    def read_block_header(self, vds):
        block_header = {}
        block_header['magic_bytes'] = vds.read_int32()
        #if block_header['magic_bytes'] != 118034699:
         #   raise Exception('Not a block')
        block_header['block_size'] = vds.read_int32()
        header_start = vds.read_cursor
        block_header['version'] = vds.read_int32()
        block_header['hash_prev'] = ib2h(vds.read_bytes(32))
        block_header['hash_merkle_root'] = ib2h(vds.read_bytes(32))
        block_header['block_time'] = vds.read_uint32()
        block_header['bits'] = vds.read_uint32()
        block_header['nonce'] = vds.read_uint32()
        header_end = vds.read_cursor
        header = vds.input[header_start:header_end]
        block_header['block_hash'] = ib2h(double_hash(header))
        block_header['__header__'] = b2h(header)
        return block_header

    def read_block(self, vds):
        block = self.read_block_header(vds)
        block['transaction_count'] = vds.read_compact_size()
        block['transactions'] = []
        for i in range(block['transaction_count']):
            block['transactions'].append(self.read_transaction(vds))
        return block

    def prepare_data_stream(self, file_num, pos_in_file):
        if self.data_stream is None or file_num != self.file_num:
            self.file_num = file_num
            if self.current_block_file:
                self.current_block_file.close()
            data_file_path = os.path.join(self.blocks_dir, 'blk%05d.dat' % (self.file_num,))
            self.current_block_file = open(data_file_path, "rb")
            self.data_stream = BCDataStream()
            self.data_stream.map_file(self.current_block_file, pos_in_file)
        else:
            self.data_stream.seek_file(pos_in_file)

    def read_raw_block(self, block_hash):
        block_hash = binascii.unhexlify(inverse_hash(block_hash))
        block_data = self.ldb.get(bytes('b', 'utf-8') + block_hash)
        ds = BCDataStream()
        ds.write(block_data)

        version = ds.read_var_int()
        height = ds.read_var_int()
        status = ds.read_var_int()
        tx_count = ds.read_var_int()
        file_num = ds.read_var_int()
        block_pos_in_file = ds.read_var_int() - 8
        block_undo_pos_in_file = ds.read_var_int()
        block_header = ds.read_bytes(80)

        self.prepare_data_stream(file_num, block_pos_in_file)

        block = self.read_block(self.data_stream)
        block['block_index'] = height

        return block

    def read_raw_transaction(self, tx_hash):
        tx_hash = binascii.unhexlify(inverse_hash(tx_hash))
        tx_data = self.ldb.get(bytes('t', 'utf-8') + tx_hash)
 
        ds = BCDataStream()
        ds.write(tx_data)

        file_num = ds.read_var_int()
        block_pos_in_file = ds.read_var_int()
        tx_pos_in_block = ds.read_var_int()
        tx_pos_in_file = block_pos_in_file + 80 + tx_pos_in_block
        
        self.prepare_data_stream(file_num, tx_pos_in_file)

        transaction = self.read_transaction(self.data_stream)

        return transaction

    def close(self):
        if self.current_block_file:
            self.current_block_file.close()
        self.ldb.close()

class ChainstateParser():

    def __init__(self, leveldb_dir):
        self.ldb = open_leveldb(leveldb_dir)

    def get_last_block_hash(self):
        block_hash = self.ldb.get(bytes('B', 'utf-8'))
        block_hash = ib2h(block_hash)
        return block_hash

    def close(self):
        self.ldb.close()
 
 
#### exceptions.py 
 

class SolvingError(Exception): pass

class SerializationError(Exception):
  """ Thrown when there's a problem deserializing or serializing """

 
 
#### utils.py 
 
import binascii, os, random, json, hashlib

bytes_from_int = chr if bytes == str else lambda x: bytes([x])

def b2h(b):
    return binascii.hexlify(b).decode('utf-8')

def random_hex(length):
    return binascii.b2a_hex(os.urandom(length))

def double_hash(b): 
	return hashlib.sha256(hashlib.sha256(b).digest()).digest()

def inverse_hash(hashstring):
	hashstring = hashstring[::-1]
	return ''.join([hashstring[i:i+2][::-1] for i in range(0, len(hashstring), 2)])

def ib2h(b):
	return inverse_hash(b2h(b))

class JsonDecimalEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o,  decimal.Decimal):
            return str(o)
        return super(DecimalEncoder, self).default(o)




 
 
#### __init__.py 
 
 
 
#### p2sh_encoding.py 
 
"""
This module contains p2sh data encoding functions
"""

import binascii
import math
import struct
import logging
import traceback # not needed if not printing exceptions on p2sh decoding

logger = logging.getLogger(__name__)

import bitcoin as bitcoinlib
from bitcoin.core.script import CScript

from counterpartylib.lib import config
from counterpartylib.lib import script
from counterpartylib.lib import exceptions

def maximum_data_chunk_size(pubkeylength):
    if pubkeylength >= 0:
        return bitcoinlib.core.script.MAX_SCRIPT_ELEMENT_SIZE - len(config.PREFIX) - pubkeylength - 12 #Two bytes are for unique offset. This will work for a little more than 1000 outputs
    else:
        return bitcoinlib.core.script.MAX_SCRIPT_ELEMENT_SIZE - len(config.PREFIX) - 44 # Redeemscript size for p2pkh addresses, multisig won't work here

def calculate_outputs(destination_outputs, data_array, fee_per_kb, exact_fee=None):
    datatx_size = 10  # 10 base
    datatx_size += 181  # 181 for source input
    datatx_size += (25 + 9) * len(destination_outputs)  # destination outputs
    datatx_size += 13  # opreturn that signals P2SH encoding
    datatx_size += len(data_array) * (9 + 181)  # size of p2sh inputs, excl data
    datatx_size += sum([len(data_chunk) for data_chunk in data_array])  # data in scriptSig
    datatx_necessary_fee = int(datatx_size / 1000 * fee_per_kb)

    pretx_output_size = 10  # 10 base
    pretx_output_size += len(data_array) * 29  # size of P2SH output

    size_for_fee = pretx_output_size

    # split the tx fee evenly between all datatx outputs
    # data_value = math.ceil(datatx_necessary_fee / len(data_array))
    data_value = config.DEFAULT_REGULAR_DUST_SIZE

    # adjust the data output with the new value and recalculate data_btc_out
    data_btc_out = data_value * len(data_array)

    if exact_fee:
        remain_fee = exact_fee - data_value * len(data_array)
        if remain_fee > 0:
            #if the dust isn't enough to reach the exact_fee, data value will be an array with only the last fee bumped
            data_value = [data_value for i in range(len(data_array))]
            data_value[len(data_array)-1] = data_value[len(data_array)-1] + remain_fee
            data_btc_out = exact_fee
    
    data_output = (data_array, data_value)          

    logger.getChild('p2shdebug').debug('datatx size: %d fee: %d' % (datatx_size, datatx_necessary_fee))
    logger.getChild('p2shdebug').debug('pretx output size: %d' % (pretx_output_size, ))
    logger.getChild('p2shdebug').debug('size_for_fee: %d' % (size_for_fee, ))

    return size_for_fee, datatx_necessary_fee, data_value, data_btc_out

def decode_p2sh_input(asm, p2sh_is_segwit=False):
    ''' Looks at the scriptSig for the input of the p2sh-encoded data transaction
        [signature] [data] [OP_HASH160 ... OP_EQUAL]
    '''
    pubkey, source, redeem_script_is_valid, found_data = decode_data_redeem_script(asm[-1], p2sh_is_segwit)
    if redeem_script_is_valid:
        # this is a signed transaction, so we got {sig[,sig]} {datachunk} {redeemScript}
        datachunk = found_data
        redeemScript = asm[-1] #asm[-2:]
    else:
        #print('ASM:', len(asm))
        pubkey, source, redeem_script_is_valid, found_data = decode_data_redeem_script(asm[-1], p2sh_is_segwit)
        if not redeem_script_is_valid or len(asm) != 3:
            return None, None, None

        # this is an unsigned transaction (last is outputScript), so we got [datachunk] [redeemScript] [temporaryOutputScript]
        datachunk, redeemScript, _substituteScript = asm

    data = datachunk
    if data[:len(config.PREFIX)] == config.PREFIX:
        data = data[len(config.PREFIX):]
    else:
        if data == b'':
            return source, None, None
        raise exceptions.DecodeError('unrecognised P2SH output')

    return source, None, data

def decode_data_push(arr, pos):
    pushlen = 0
    data = b''
    opcode = bitcoinlib.core.script.CScriptOp(arr[pos])
    if opcode > 0 and opcode < bitcoinlib.core.script.OP_PUSHDATA1:
        pushlen = arr[pos]
        pos += 1
    elif opcode == bitcoinlib.core.script.OP_PUSHDATA1:
        pushlen = arr[pos + 1]
        pos += 2
    elif opcode == bitcoinlib.core.script.OP_PUSHDATA2:
        (pushlen, ) = struct.unpack('<H', arr[pos + 1:pos + 3])
        pos += 3
    elif opcode == bitcoinlib.core.script.OP_PUSHDATA4:
        (pushlen, ) = struct.unpack('<L', arr[pos + 1:pos + 5])
        pos += 5

    return pos + pushlen, arr[pos:pos + pushlen]

def decode_data_redeem_script(redeemScript, p2sh_is_segwit=False):
    script_len = len(redeemScript)
    found_data = b''

    if script_len == 41 and \
        redeemScript[0] == bitcoinlib.core.script.OP_DROP and \
        redeemScript[35] == bitcoinlib.core.script.OP_CHECKSIGVERIFY and \
        redeemScript[37] == bitcoinlib.core.script.OP_DROP and \
        redeemScript[38] == bitcoinlib.core.script.OP_DEPTH and \
        redeemScript[39] == bitcoinlib.core.script.OP_0 and \
        redeemScript[40] == bitcoinlib.core.script.OP_EQUAL:
            # - OP_DROP [push] [33-byte pubkey] OP_CHECKSIGVERIFY [n] OP_DROP OP_DEPTH 0 OP_EQUAL
            pubkey = redeemScript[2:35]
            if p2sh_is_segwit:
                source = script.pubkey_to_p2whash(pubkey)
            else:
                source = script.pubkey_to_pubkeyhash(pubkey)
            redeem_script_is_valid = True
    elif script_len > 41 and \
        redeemScript[0] == bitcoinlib.core.script.OP_DROP and \
        redeemScript[script_len-4] == bitcoinlib.core.script.OP_DROP and \
        redeemScript[script_len-3] == bitcoinlib.core.script.OP_DEPTH and \
        redeemScript[script_len-2] == bitcoinlib.core.script.OP_0 and \
        redeemScript[script_len-1] == bitcoinlib.core.script.OP_EQUAL:
            # - OP_DROP {arbitrary multisig script} [n] OP_DROP OP_DEPTH 0 OP_EQUAL
            pubkey = None
            source = None
            redeem_script_is_valid = True
    else:
        pubkey = None
        source = None
        redeem_script_is_valid = False

        try:
            opcode = bitcoinlib.core.script.CScriptOp(redeemScript[0])
            if opcode > bitcoinlib.core.script.OP_0 and opcode < bitcoinlib.core.script.OP_PUSHDATA1 or \
                opcode in (bitcoinlib.core.script.OP_PUSHDATA1, bitcoinlib.core.script.OP_PUSHDATA2, bitcoinlib.core.script.OP_PUSHDATA4):

                pos = 0
                pos, found_data = decode_data_push(redeemScript, 0)

                if redeemScript[pos] == bitcoinlib.core.script.OP_DROP:
                    pos += 1
                    valid_sig = False
                    opcode = redeemScript[pos]
                    if type(opcode) != type(''):
                        if opcode >= bitcoinlib.core.script.OP_2 and opcode <= bitcoinlib.core.script.OP_15:
                            # it's multisig
                            req_sigs = opcode - bitcoinlib.core.script.OP_1 + 1
                            pos += 1
                            pubkey = None
                            num_sigs = 0
                            found_sigs = False
                            while not found_sigs:
                                pos, npubkey = decode_data_push(redeemScript, pos)
                                num_sigs += 1
                                if redeemScript[pos] - bitcoinlib.core.script.OP_1 + 1 == num_sigs:
                                    found_sigs = True

                            pos += 1
                            valid_sig = redeemScript[pos] == bitcoinlib.core.script.OP_CHECKMULTISIGVERIFY
                        else:
                            # it's p2pkh
                            pos, pubkey = decode_data_push(redeemScript, pos)

                            if p2sh_is_segwit:
                                source = script.pubkey_to_p2whash(pubkey)
                            else:
                                source = script.pubkey_to_pubkeyhash(pubkey)

                            valid_sig = redeemScript[pos] == bitcoinlib.core.script.OP_CHECKSIGVERIFY
                        pos += 1

                        if valid_sig:
                            uniqueOffsetLength = 0

                            for i in range(pos+1, len(redeemScript)):
                                if redeemScript[i] == bitcoinlib.core.script.OP_DROP:
                                    uniqueOffsetLength = i-pos-1
                                    break

                            redeem_script_is_valid = redeemScript[pos + 1 + uniqueOffsetLength] == bitcoinlib.core.script.OP_DROP and \
                                redeemScript[pos + 2 + uniqueOffsetLength] == bitcoinlib.core.script.OP_DEPTH and \
                                redeemScript[pos + 3 + uniqueOffsetLength] == 0 and \
                                redeemScript[pos + 4 + uniqueOffsetLength] == bitcoinlib.core.script.OP_EQUAL
        except Exception as e:
            pass #traceback.print_exc()

    return pubkey, source, redeem_script_is_valid, found_data

def make_p2sh_encoding_redeemscript(datachunk, n, pubKey=None, multisig_pubkeys=None, multisig_pubkeys_required=None):
    _logger = logger.getChild('p2sh_encoding')
    assert len(datachunk) <= bitcoinlib.core.script.MAX_SCRIPT_ELEMENT_SIZE

    dataDropScript = [datachunk, bitcoinlib.core.script.OP_DROP] # just drop the data chunk
    cleanupScript = [n, bitcoinlib.core.script.OP_DROP, bitcoinlib.core.script.OP_DEPTH, 0, bitcoinlib.core.script.OP_EQUAL] # unique offset + prevent scriptSig malleability

    if pubKey is not None:
        # a p2pkh script looks like this: {pubkey} OP_CHECKSIGVERIFY
        verifyOwnerScript = [pubKey, bitcoinlib.core.script.OP_CHECKSIGVERIFY]
    elif multisig_pubkeys_required is not None and multisig_pubkeys:
        # a 2-of-3 multisig looks like this:
        #   2 {pubkey1} {pubkey2} {pubkey3} 3 OP_CHECKMULTISIGVERIFY
        multisig_pubkeys_required = int(multisig_pubkeys_required)
        if multisig_pubkeys_required < 2 or multisig_pubkeys_required > 15:
            raise exceptions.TransactionError('invalid multisig pubkeys value')
        verifyOwnerScript = [multisig_pubkeys_required]
        for multisig_pubkey in multisig_pubkeys:
            verifyOwnerScript.append(multisig_pubkey)
        verifyOwnerScript = verifyOwnerScript + [len(multisig_pubkeys), bitcoinlib.core.script.OP_CHECKMULTISIGVERIFY]
    else:
        raise exceptions.TransactionError('Either pubKey or multisig pubKeys must be provided')

    #redeemScript = CScript(datachunk) + CScript(dataDropScript + verifyOwnerScript + cleanupScript)
    redeemScript = CScript(dataDropScript + verifyOwnerScript + cleanupScript)

    _logger.debug('datachunk %s' % (binascii.hexlify(datachunk)))
    _logger.debug('dataDropScript %s (%s)' % (repr(CScript(dataDropScript)), binascii.hexlify(CScript(dataDropScript))))
    _logger.debug('verifyOwnerScript %s (%s)' % (repr(CScript(verifyOwnerScript)), binascii.hexlify(CScript(verifyOwnerScript))))
    _logger.debug('entire redeemScript %s (%s)' % (repr(redeemScript), binascii.hexlify(redeemScript)))

    #scriptSig = CScript([]) + redeemScript  # PUSH(datachunk) + redeemScript
    scriptSig = CScript([redeemScript])
    outputScript = redeemScript.to_p2sh_scriptPubKey()

    _logger.debug('scriptSig %s (%s)' % (repr(scriptSig), binascii.hexlify(scriptSig)))
    _logger.debug('outputScript %s (%s)' % (repr(outputScript), binascii.hexlify(outputScript)))

    # outputScript looks like OP_HASH160 {{ hash160([redeemScript]) }} OP_EQUALVERIFY
    # redeemScript looks like OP_DROP {{ pubkey }} OP_CHECKSIGVERIFY {{ n }} OP_DROP OP_DEPTH 0 OP_EQUAL
    # scriptSig is {{ datachunk }} OP_DROP {{ pubkey }} OP_CHECKSIGVERIFY {{ n }} OP_DROP OP_DEPTH 0 OP_EQUAL
    return scriptSig, redeemScript, outputScript

def make_standard_p2sh_multisig_script(multisig_pubkeys, multisig_pubkeys_required):
    # a 2-of-3 multisig looks like this:
    #   2 {pubkey1} {pubkey2} {pubkey3} 3 OP_CHECKMULTISIG
    multisig_pubkeys_required = int(multisig_pubkeys_required)
    multisig_script = [multisig_pubkeys_required]
    for multisig_pubkey in multisig_pubkeys:
        multisig_script.append(multisig_pubkey)
    multisig_script = multisig_script + [len(multisig_pubkeys), bitcoinlib.core.script.OP_CHECKMULTISIG]
    return multisig_script
 
 
#### serializer.py 
 
"""
Construct and serialize the Bitcoin transactions that are Counterparty transactions.

This module contains no consensus‐critical code.
"""

import os
import sys
import binascii
import json
import hashlib
import re
import time
import decimal
import logging
logger = logging.getLogger(__name__)
import requests
import bitcoin as bitcoinlib
from bitcoin.core import Hash160
from bitcoin.core.script import CScript
from bitcoin.wallet import P2PKHBitcoinAddress, P2SHBitcoinAddress
import cachetools

from counterpartylib.lib import config
from counterpartylib.lib import exceptions
from counterpartylib.lib import util
from counterpartylib.lib import script
from counterpartylib.lib import backend
from counterpartylib.lib import arc4
from counterpartylib.lib.transaction_helper import p2sh_encoding
from bitcoin.bech32 import CBech32Data

# Constants
OP_RETURN = b'\x6a'
OP_PUSHDATA1 = b'\x4c'
OP_DUP = b'\x76'
OP_HASH160 = b'\xa9'
OP_EQUALVERIFY = b'\x88'
OP_CHECKSIG = b'\xac'
OP_0 = b'\x00'
OP_1 = b'\x51'
OP_2 = b'\x52'
OP_3 = b'\x53'
OP_CHECKMULTISIG = b'\xae'
OP_EQUAL = b'\x87'

D = decimal.Decimal
UTXO_LOCKS = None
UTXO_LOCKS_PER_ADDRESS_MAXSIZE = 5000  # set higher than the max number of UTXOs we should expect to
                                       # manage in an aging cache for any one source address, at any one period

def var_int (i):
    if i < 0xfd:
        return (i).to_bytes(1, byteorder='little')
    elif i <= 0xffff:
        return b'\xfd' + (i).to_bytes(2, byteorder='little')
    elif i <= 0xffffffff:
        return b'\xfe' + (i).to_bytes(4, byteorder='little')
    else:
        return b'\xff' + (i).to_bytes(8, byteorder='little')


def op_push (i):
    if i < 0x4c:
        return (i).to_bytes(1, byteorder='little')              # Push i bytes.
    elif i <= 0xff:
        return b'\x4c' + (i).to_bytes(1, byteorder='little')    # OP_PUSHDATA1
    elif i <= 0xffff:
        return b'\x4d' + (i).to_bytes(2, byteorder='little')    # OP_PUSHDATA2
    else:
        return b'\x4e' + (i).to_bytes(4, byteorder='little')    # OP_PUSHDATA4


def get_script(address):
    if script.is_multisig(address):
        return get_multisig_script(address)
    elif script.is_bech32(address):
        return get_p2w_script(address)
    else:
        try:
            return get_monosig_script(address)
        except script.VersionByteError as e:
            return get_p2sh_script(address)


def get_multisig_script(address):

    # Unpack multi‐sig address.
    signatures_required, pubkeys, signatures_possible = script.extract_array(address)

    # Required signatures.
    if signatures_required == 1:
        op_required = OP_1
    elif signatures_required == 2:
        op_required = OP_2
    elif signatures_required == 3:
        op_required = OP_3
    else:
        raise script.InputError('Required signatures must be 1, 2 or 3.')

    # Required signatures.
    # Note 1-of-1 addresses are not supported (they don't go through extract_array anyway).
    if signatures_possible == 2:
        op_total = OP_2
    elif signatures_possible == 3:
        op_total = OP_3
    else:
        raise script.InputError('Total possible signatures must be 2 or 3.')

    # Construct script.
    tx_script = op_required                                # Required signatures
    for public_key in pubkeys:
        public_key = binascii.unhexlify(public_key)
        tx_script += op_push(len(public_key))              # Push bytes of public key
        tx_script += public_key                            # Data chunk (fake) public key
    tx_script += op_total                                  # Total signatures
    tx_script += OP_CHECKMULTISIG                          # OP_CHECKMULTISIG

    return (tx_script, None)


def get_monosig_script(address):

    # Construct script.
    pubkeyhash = script.base58_check_decode(address, config.ADDRESSVERSION)
    tx_script = OP_DUP                                     # OP_DUP
    tx_script += OP_HASH160                                # OP_HASH160
    tx_script += op_push(20)                               # Push 0x14 bytes
    tx_script += pubkeyhash                                # pubKeyHash
    tx_script += OP_EQUALVERIFY                            # OP_EQUALVERIFY
    tx_script += OP_CHECKSIG                               # OP_CHECKSIG

    return (tx_script, None)


def get_p2sh_script(address):

    # Construct script.
    scripthash = script.base58_check_decode(address, config.P2SH_ADDRESSVERSION)
    tx_script = OP_HASH160
    tx_script += op_push(len(scripthash))
    tx_script += scripthash
    tx_script += OP_EQUAL

    return (tx_script, None)


def get_p2w_script(address):

    # Construct script.
    scripthash = bytes(CBech32Data(address))

    if len(scripthash) == 20:
        # P2WPKH encoding

        tx_script = OP_0
        tx_script += b'\x14'
        tx_script += scripthash

        witness_script = OP_HASH160
        witness_script += op_push(len(scripthash))
        witness_script += scripthash
        witness_script += OP_EQUAL

        return (tx_script, witness_script)
    elif len(scripthash) == 32:
        # P2WSH encoding
        raise Exception('P2WSH encoding not yet supported')


def make_fully_valid(pubkey_start):
    """Take a too short data pubkey and make it look like a real pubkey.

    Take an obfuscated chunk of data that is two bytes too short to be a pubkey and
    add a sign byte to its beginning and a nonce byte to its end. Choose these
    bytes so that the resulting sequence of bytes is a fully valid pubkey (i.e. on
    the ECDSA curve). Find the correct bytes by guessing randomly until the check
    passes. (In parsing, these two bytes are ignored.)
    """
    assert type(pubkey_start) == bytes
    assert len(pubkey_start) == 31    # One sign byte and one nonce byte required (for 33 bytes).

    random_bytes = hashlib.sha256(pubkey_start).digest()      # Deterministically generated, for unit tests.
    sign = (random_bytes[0] & 0b1) + 2                  # 0x02 or 0x03
    nonce = initial_nonce = random_bytes[1]

    pubkey = b''
    while not script.is_fully_valid(pubkey):
        # Increment nonce.
        nonce += 1
        assert nonce != initial_nonce

        # Construct a possibly fully valid public key.
        pubkey = bytes([sign]) + pubkey_start + bytes([nonce % 256])

    assert len(pubkey) == 33
    return pubkey


def serialise(encoding, inputs, destination_outputs, data_output=None, change_output=None, dust_return_pubkey=None):
    s  = (1).to_bytes(4, byteorder='little')                # Version

    use_segwit = False
    for i in range(len(inputs)):
        txin = inputs[i]
        spk = txin['scriptPubKey']
        if spk[0:2] == '00': # Witness version 0
            datalen = binascii.unhexlify(spk[2:4])[0]
            if datalen == 20 or datalen == 32:
                # 20 is for P2WPKH and 32 is for P2WSH
                if not(use_segwit):
                    s  = (2).to_bytes(4, byteorder='little') # Rewrite version
                    use_segwit = True

                txin['is_segwit'] = True

    if use_segwit:
        s += b'\x00' # marker
        s += b'\x01' # flag

    # Number of inputs.
    s += var_int(int(len(inputs)))

    witness_txins = []
    witness_data = {}

    # List of Inputs.
    for i in range(len(inputs)):
        txin = inputs[i]
        s += binascii.unhexlify(bytes(txin['txid'], 'utf-8'))[::-1]         # TxOutHash
        s += txin['vout'].to_bytes(4, byteorder='little')   # TxOutIndex

        tx_script = binascii.unhexlify(bytes(txin['scriptPubKey'], 'utf-8'))
        s += var_int(int(len(tx_script)))                      # Script length
        s += tx_script                                         # Script
        s += b'\xff' * 4                                    # Sequence

    # Number of outputs.
    n = 0
    n += len(destination_outputs)
    if data_output:
        data_array, value = data_output
        for data_chunk in data_array: n += 1
    else:
        data_array = []
    if change_output: n += 1
    s += var_int(n)

    # Destination output.
    for destination, value in destination_outputs:
        s += value.to_bytes(8, byteorder='little')          # Value

        tx_script, witness_script = get_script(destination)
        #if use_segwit and destination in witness_data: # Not deleteing, We will need this for P2WSH
        #    witness_data[destination].append(witness_script)
        #    tx_script = witness_script

        #if witness_script:
        #    tx_script = witness_script

        s += var_int(int(len(tx_script)))                      # Script length
        s += tx_script

    # Data output.
    for data_chunk in data_array:
        data_array, value = data_output
        s += value.to_bytes(8, byteorder='little')        # Value

        data_chunk = config.PREFIX + data_chunk

        # Initialise encryption key (once per output).
        assert isinstance(inputs[0]['txid'], str)
        key = arc4.init_arc4(binascii.unhexlify(inputs[0]['txid']))  # Arbitrary, easy‐to‐find, unique key.

        if encoding == 'multisig':
            assert dust_return_pubkey
            # Get data (fake) public key.
            pad_length = (33 * 2) - 1 - 2 - 2 - len(data_chunk)
            assert pad_length >= 0
            data_chunk = bytes([len(data_chunk)]) + data_chunk + (pad_length * b'\x00')
            data_chunk = key.encrypt(data_chunk)
            data_pubkey_1 = make_fully_valid(data_chunk[:31])
            data_pubkey_2 = make_fully_valid(data_chunk[31:])

            # Construct script.
            tx_script = OP_1                                   # OP_1
            tx_script += op_push(33)                           # Push bytes of data chunk (fake) public key    (1/2)
            tx_script += data_pubkey_1                         # (Fake) public key                  (1/2)
            tx_script += op_push(33)                           # Push bytes of data chunk (fake) public key    (2/2)
            tx_script += data_pubkey_2                         # (Fake) public key                  (2/2)
            tx_script += op_push(len(dust_return_pubkey))  # Push bytes of source public key
            tx_script += dust_return_pubkey                       # Source public key
            tx_script += OP_3                                  # OP_3
            tx_script += OP_CHECKMULTISIG                      # OP_CHECKMULTISIG
        elif encoding == 'opreturn':
            data_chunk = key.encrypt(data_chunk)
            tx_script = OP_RETURN                                  # OP_RETURN
            tx_script += op_push(len(data_chunk))                  # Push bytes of data chunk (NOTE: OP_SMALLDATA?)
            tx_script += data_chunk                                # Data
        elif encoding == 'pubkeyhash':
            pad_length = 20 - 1 - len(data_chunk)
            assert pad_length >= 0
            data_chunk = bytes([len(data_chunk)]) + data_chunk + (pad_length * b'\x00')
            data_chunk = key.encrypt(data_chunk)
            # Construct script.
            tx_script = OP_DUP                                     # OP_DUP
            tx_script += OP_HASH160                                # OP_HASH160
            tx_script += op_push(20)                               # Push 0x14 bytes
            tx_script += data_chunk                                # (Fake) pubKeyHash
            tx_script += OP_EQUALVERIFY                            # OP_EQUALVERIFY
            tx_script += OP_CHECKSIG                               # OP_CHECKSIG
        else:
            raise exceptions.TransactionError('Unknown encoding‐scheme.')

        s += var_int(int(len(tx_script)))                      # Script length
        s += tx_script

    # Change output.
    if change_output:
        change_address, change_value = change_output
        s += change_value.to_bytes(8, byteorder='little')   # Value

        tx_script, witness_script = get_script(change_address)
        #print("Change address!", change_address, "\n", witness_data, "\n", tx_script, "\n", witness_script)
        #if witness_script: #use_segwit and change_address in witness_data:
        #    if not(change_address in witness_data):
        #        witness_data[change_address] = []
        #    witness_data[change_address].append(witness_script)
        #    tx_script = witness_script
        #    use_segwit = True

        s += var_int(int(len(tx_script)))                      # Script length
        s += tx_script

    if use_segwit:
        for i in range(len(inputs)):
            txin = inputs[i]
            if txin['is_segwit']:
                s += b'\x02'
                s += b'\x00\x00'
            else:
                s += b'\x00'

    s += (0).to_bytes(4, byteorder='little')                # LockTime
    return s



def serialise_p2sh_pretx(inputs, source, source_value, data_output, change_output=None, pubkey=None, multisig_pubkeys=None, multisig_pubkeys_required=None):
    assert data_output  # we don't do this unless there's data

    data_array, data_value = data_output

    s  = (1).to_bytes(4, byteorder='little')  # Version

    # Number of inputs.
    s += var_int(int(len(inputs)))

    # List of Inputs.
    for i in range(len(inputs)):
        txin = inputs[i]
        s += binascii.unhexlify(bytes(txin['txid'], 'utf-8'))[::-1]  # TxOutHash
        s += txin['vout'].to_bytes(4, byteorder='little')            # TxOutIndex

        tx_script = binascii.unhexlify(bytes(txin['scriptPubKey'], 'utf-8'))
        s += var_int(int(len(tx_script)))  # Script length
        s += tx_script                     # Script
        s += b'\xff' * 4                   # Sequence

    # Number of outputs.
    n = len(data_array)
    if change_output:
        n += 1

    # encode number of outputs
    s += var_int(n)

    # P2SH for data encodeded inputs
    for n, data_chunk in enumerate(data_array):
        data_chunk = config.PREFIX + data_chunk  # prefix the data_chunk

        # get the scripts
        scriptSig, redeemScript, outputScript = p2sh_encoding.make_p2sh_encoding_redeemscript(data_chunk, n, pubkey, multisig_pubkeys, multisig_pubkeys_required)

        #if data_value is an array, then every output fee is specified in it
        if type(data_value) == list:
            s += data_value[n].to_bytes(8, byteorder='little')  # Value
        else:
            s += data_value.to_bytes(8, byteorder='little')  # Value
        s += var_int(int(len(outputScript)))             # Script length
        s += outputScript                                # Script

    # Change output.
    if change_output:
        change_address, change_value = change_output
        tx_script, witness_script = get_script(change_address)

        s += change_value.to_bytes(8, byteorder='little')  # Value
        s += var_int(int(len(tx_script)))                  # Script length
        s += tx_script                                     # Script

    s += (0).to_bytes(4, byteorder='little')  # LockTime

    return s


def serialise_p2sh_datatx(txid, source, source_input, destination_outputs, data_output, pubkey=None, multisig_pubkeys=None, multisig_pubkeys_required=None):
    assert data_output  # we don't do this unless there's data

    txhash = bitcoinlib.core.lx(bitcoinlib.core.b2x(txid))  # reverse txId
    data_array, value = data_output

    # version
    s = (1).to_bytes(4, byteorder='little')

    # number of inputs is the length of data_array (+1 if a source_input exists)
    number_of_inputs = len(data_array)
    if source_input is not None:
        number_of_inputs += 1
    s += var_int(number_of_inputs)

    # Handle a source input here for a P2SH source
    if source_input is not None:
        s += binascii.unhexlify(bytes(source_input['txid'], 'utf-8'))[::-1]  # TxOutHash
        s += source_input['vout'].to_bytes(4, byteorder='little')            # TxOutIndex

        # since pubkey is not returned from indexd, add it from bitcoind
        source_inputs = backend.ensure_script_pub_key_for_inputs([source_input])
        source_input = source_inputs[0]
        tx_script = binascii.unhexlify(bytes(source_input['scriptPubKey'], 'utf-8'))
        s += var_int(int(len(tx_script)))                                    # Script length
        s += tx_script                                                       # Script
        s += b'\xff' * 4                                                     # Sequence

    # list of inputs
    for n, data_chunk in enumerate(data_array):
        data_chunk = config.PREFIX + data_chunk  # prefix the data_chunk

        # get the scripts
        scriptSig, redeemScript, outputScript = p2sh_encoding.make_p2sh_encoding_redeemscript(data_chunk, n, pubkey, multisig_pubkeys, multisig_pubkeys_required)
        #substituteScript = scriptSig + outputScript

        s += txhash                                              # TxOutHash
        s += (n).to_bytes(4, byteorder='little')                 # TxOutIndex (assumes 0-based)

        #s += var_int(len(substituteScript))                      # Script length
        #s += substituteScript                                    # Script

        s += var_int(len(scriptSig))# + len(outputScript))                      # Script length
        s += scriptSig                                    # Script
        #s += outputScript                                    # Script

        s += b'\xff' * 4                                         # Sequence

    # number of outputs, always 1 for the opreturn
    n = 1
    n += len(destination_outputs)

    # encode output length
    s += var_int(n)

    # destination outputs
    for destination, value in destination_outputs:
        tx_script, witness_script = get_script(destination)

        s += value.to_bytes(8, byteorder='little')  # Value
        s += var_int(int(len(tx_script)))           # Script length
        s += tx_script                              # Script

    # opreturn to signal P2SH encoding
    key = arc4.init_arc4(txid)
    data_chunk = config.PREFIX + b'P2SH'
    data_chunk = key.encrypt(data_chunk)
    tx_script = OP_RETURN                  # OP_RETURN
    tx_script += op_push(len(data_chunk))  # Push bytes of data chunk
    tx_script += data_chunk                # Data

    # add opreturn
    s += (0).to_bytes(8, byteorder='little')  # Value
    s += var_int(int(len(tx_script)))         # Script length
    s += tx_script                            # Script

    s += (0).to_bytes(4, byteorder='little')  # LockTime

    return s
 
 
#### __init__.py