
%config InlineBackend.figure_format = 'svg'

import os, sys
sys.path.insert(1, os.path.realpath(os.path.pardir))

from typing import Sequence, Dict

from abm1559.utils import (
    constants,
    basefee_from_csv_history,
    get_basefee_bounds,
    flatten
)
constants["SIMPLE_TRANSACTION_GAS"] = 210000

from abm1559.txpool import TxPool

from abm1559.users import (
    User1559,
    AffineUser,
    User
)

from abm1559.config import rng

from abm1559.txs import Transaction, Tx1559, TxLegacy

from abm1559.userpool import UserPool

from abm1559.chain import (
    Chain,
    Block1559,
    Block
)

from abm1559.simulator import (
    spawn_poisson_heterogeneous_demand,
    update_basefee,
    generate_gbm,
)

import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_rows', 1000)
import numpy as np
import time
import seaborn as sns
from tqdm import tqdm


def normalize_from_legacy(legacy: TxLegacy) -> Tx1559:
    """
    Normalized according to
    https://github.com/ethereum/EIPs/blob/541c8be92fe759aa602b7d06a088ab1a139e37ce/EIPS/eip-1559.md
    """
    normalized_params = {
        "max_fee": legacy.gas_price(),
        "gas_premium": legacy.gas_price(),
        "start_block": legacy.start_block,
    }
    return Tx1559(
        sender=legacy.sender,
        gas_used=constants["SIMPLE_TRANSACTION_GAS"],
        tx_params=normalized_params
    )


MIN_ACCEPTABLE_TIP = 1 * (10 ** 9)
ORACLE_SLOW = "slow"
ORACLE_MEDIUM = "medium"
ORACLE_FAST = "fast"

class LegacyBidder(AffineUser):
    def __init__(self, wakeup_block, **kwargs):
        super().__init__(wakeup_block, cost_per_unit = 0, **kwargs)
        self.value = (1 + self.rng.pareto(2) * 20) * (10 ** 9)

    def percentile(self):
        raise Exception("Not implemented, try using a subclass.")

    def query_oracle(self, env: dict):
        fee_oracles = env["fee_oracles"]
        return fee_oracles[self.oracle()]

    def decide_parameters(self, env: dict):
        gas_price = self.query_oracle(env)
        
        if self.oracle() == ORACLE_SLOW:
            scaling = 0.5
        elif self.oracle() == ORACLE_MEDIUM:
            scaling = 0.75
        else:
            scaling = 0.9
        
        # We add a normal error scaled by the user type
        gas_price += scaling * self.rng.normal() * (10 ** 9)
        
        return {
            "gas_price": int(max(gas_price, MIN_ACCEPTABLE_TIP)), # in wei
            "start_block": self.wakeup_block,
        }
    
    # This function is the main user entry point 
    def create_transaction(self, env: dict):
        tx_params = self.decide_parameters(env)
        
        # If gas price is higher than 1.2 * my value, I balk
        if tx_params["gas_price"] >= 1.2 * self.value:
            return None
        
        tx_params["gas_price"] = min(self.value, tx_params["gas_price"])
                
        tx = TxLegacy(
            sender = self.pub_key,
            gas_used=constants["SIMPLE_TRANSACTION_GAS"],
            tx_params = tx_params,
        )
        return tx


rng = np.random.default_rng(4)
pd.DataFrame({ "value": rng.pareto(2, 1000) * 20 }).hist(log=True, bins = 100)

array([[<AxesSubplot:title={'center':'value'}>]], dtype=object)


class LegacyBidderSlow(LegacyBidder):
    def oracle(self):
        return ORACLE_SLOW
    
    def export(self):
        return {
            **super().export(),
            "user_type": "user_slow",
        }

class LegacyBidderMedium(LegacyBidder):
    def oracle(self):
        return ORACLE_MEDIUM
    
    def export(self):
        return {
            **super().export(),
            "user_type": "user_medium",
        }

class LegacyBidderFast(LegacyBidder):
    def oracle(self):
        return ORACLE_FAST
    
    def export(self):
        return {
            **super().export(),
            "user_type": "user_fast",
        }


def update_oracles(env: dict, block_target: int) -> None:
    blocks_window = len(env["recent_blocks"])
    fees = flatten([[tx.gas_price({ "basefee": block.basefee }) for tx in block.txs] for block in env["recent_blocks"]])
    fees += [0.0] * (blocks_window * block_target - len(fees))

    env["fee_oracles"] = {
        ORACLE_SLOW: np.percentile(fees, 50) if len(fees) > 0 else 0,
        ORACLE_MEDIUM: np.percentile(fees, 75) if len(fees) > 0 else 0,
        ORACLE_FAST: np.percentile(fees, 90) if len(fees) > 0 else 0
    }


class BiddingBot(AffineUser):
    def __init__(self, wakeup_block, **kwargs):
        super().__init__(wakeup_block, cost_per_unit = 0, **kwargs)
        self.value = (1 + self.rng.pareto(2) * 40) * (10 ** 9)

    def percentile(self):
        raise Exception("Not implemented, try using a subclass.")

    def decide_parameters(self, env: dict):
        oracle_price = env["fee_oracles"][ORACLE_FAST]
        gas_price = oracle_price + self.rng.random() * (self.value - oracle_price)
        return {
            "gas_price": int(gas_price), # in wei
            "start_block": self.wakeup_block,
        }
    
    # This function is the main user entry point 
    def create_transaction(self, env: dict):
        tx_params = self.decide_parameters(env)
                
        expected_payoff = self.value - tx_params["gas_price"]
        if expected_payoff <= 0:
            return None
        
        tx = TxLegacy(
            sender = self.pub_key,
            gas_used=constants["SIMPLE_TRANSACTION_GAS"],
            tx_params = tx_params,
        )
        return tx
    
    def export(self):
        return {
            **super().export(),
            "user_type": "bidding_bot",
        }


MAX_TRANSACTIONS_IN_POOL = 2000

class MixedTxPool(TxPool):
    
    def add_txs(self, txs: Sequence[Transaction], env: dict) -> None:
        for tx in txs:
            if type(tx) is TxLegacy:
                tx = normalize_from_legacy(tx)
            self.txs[tx.tx_hash] = tx
        
        self.pool_length += len(txs)
            
        if self.pool_length > MAX_TRANSACTIONS_IN_POOL:
            sorted_txs = sorted(self.txs.values(), key = lambda tx: -tx.tip(env))
            self.empty_pool()
            self.add_txs(sorted_txs[0:MAX_TRANSACTIONS_IN_POOL], env)
            return sorted_txs[MAX_TRANSACTIONS_IN_POOL:]
        
        return []
    
    def select_transactions(self, env, user_pool=None, rng=rng):
        # Miner side
        max_tx_in_block = int(constants["MAX_GAS_EIP1559"] / constants["SIMPLE_TRANSACTION_GAS"])

        valid_txs = [tx for tx in self.txs.values() if tx.is_valid(env) and tx.tip(env) >= MIN_ACCEPTABLE_TIP]
        rng.shuffle(valid_txs)

        sorted_valid_demand = sorted(
            valid_txs,
            key = lambda tx: -tx.tip(env)
        )
        selected_txs = sorted_valid_demand[0:max_tx_in_block]

        return selected_txs


def get_market_price(users: Sequence[User], txs: Sequence[Transaction], env: dict, target: int) -> float:
    values = sorted(
        [user.value for user in users] + [tx.gas_price(env) for tx in txs],
        key = lambda v: -v
    )
    return values[target] if target < len(values) else 0


def simulate(demand_scenario, shares_scenario, extra_metrics = None, rng = rng):
    # Instantiate a couple of things
    txpool = MixedTxPool()
    chain = Chain()
    metrics = []
    user_pool = UserPool()
    blocks_window = 30
    start_time = time.time()
    block_target = int(constants["MAX_GAS_EIP1559"] / constants["SIMPLE_TRANSACTION_GAS"] / 2.0)
    
    # `env` is the "environment" of the simulation
    env = {
        # we start with the historic basefee based on previous txs
        "basefee": constants["INITIAL_BASEFEE"],
        # "basefee": initial_historic_basefee,
        "current_block": None,
        "min_premium": 2 * (10 ** 9),
        "recent_blocks": [],
        "fee_oracles": {
            ORACLE_SLOW: 2 * (10 ** 9),
            ORACLE_MEDIUM: 2 * (10 ** 9),
            ORACLE_FAST: 2 * (10 ** 9)
        }
    }

    for t in tqdm(range(len(demand_scenario))):
        # Sets current block
        env["current_block"] = t
        
        # Reset the random number generator with new seed to generate users with same values across runs
        rng = np.random.default_rng(t)
        
        ### SIMULATION ###

        # We return some demand which on expectation yields `demand_scenario[t]` new users per round
        users = spawn_poisson_heterogeneous_demand(t, demand_scenario[t], shares_scenario[t], rng=rng)
        
        # Record the market price given the current snapshot
        market_price = get_market_price(
            users,
            txpool.txs.values(),
            env,
            block_target
        )

        # Add new users to the pool
        # We query each new user with the current basefee value
        # Users either return a transaction or None if they prefer to balk
        decided_txs = user_pool.decide_transactions(users, env)

        # New transactions are added to the transaction pool
        # `evicted_txs` holds the transactions removed from the pool for lack of space
        evicted_txs = txpool.add_txs(decided_txs, env)

        # The best valid transactions are taken out of the pool for inclusion
        selected_txs = txpool.select_transactions(env)
        txpool.remove_txs([tx.tx_hash for tx in selected_txs])

        # We create a block with these transactions
        block = Block1559(
            txs = selected_txs, parent_hash = chain.current_head,
            height = t, basefee = env["basefee"]
        )
        env["recent_blocks"] = env["recent_blocks"][-(blocks_window-1):] + [block]
        
        # Record the min premium in the block
        env["min_premium"] = block.min_premium()
        
        # The block is added to the chain
        chain.add_block(block)
        
        ### METRICS ###
                
        row_metrics = {
            "block": t,
            "users": len(users),
            "decided_txs": len(decided_txs),
            "included_txs": len(selected_txs),
            "basefee": env["basefee"] / (10 ** 9), # to Gwei
            "slow_oracle": env["fee_oracles"][ORACLE_SLOW] / (10 ** 9), # to Gwei
            "medium_oracle": env["fee_oracles"][ORACLE_MEDIUM] / (10 ** 9), # to Gwei
            "fast_oracle": env["fee_oracles"][ORACLE_FAST] / (10 ** 9), # to Gwei
            "market_price": market_price  / (10 ** 9), # to Gwei
            "blk_min_premium": block.min_premium() / (10 ** 9), # to Gwei
            "blk_max_premium": block.max_premium() / (10 ** 9), # to Gwei
            "blk_min_tip": block.min_tip(env) / (10 ** 9), # to Gwei
            "blk_max_tip": block.max_tip(env) / (10 ** 9), # to Gwei
        }
        
        if not extra_metrics is None:
            row_metrics = {
                **row_metrics,
                **extra_metrics(env, users, user_pool, txpool),
            }
        
        metrics.append(row_metrics)

        ### ORACLES ###
        
        update_oracles(env, block_target)
        
        # Finally, basefee is updated and a new round starts
        env["basefee"] = update_basefee(block, env["basefee"])

    return (pd.DataFrame(metrics), user_pool, chain)


blocks = 6000

rng = np.random.default_rng(4)
sigma = 0.05
mu = 0.5 * sigma**2
gbm = list(generate_gbm(200, blocks, paths=1, mu=mu, sigma=sigma, rng=rng).flatten())

# Plot demand
f, ax = plt.subplots()
plt.title('Demand size over time', color='black')
plt.plot(range(blocks), gbm)
ax.set_xlabel("Block height")
ax.set_ylabel("Number of new users")
plt.show()


def extra_metrics_fpa(env, users, user_pool, txpool):
    pool_legacy_users = len(
        [tx for tx in txpool.txs.values() if isinstance(user_pool.users[tx.sender], LegacyBidder)])

    return {
        "legacy": len([user for user in users if isinstance(user, LegacyBidder)]),
        "pool_legacy": pool_legacy_users
    }

# Number of new users per time step
demand_scenario = [gbm[t] for t in range(6000)]

# Shares of new users per time step
shares_scenario_fpa = [{
    LegacyBidderSlow: 0.50,
    LegacyBidderMedium: 0.30,
    LegacyBidderFast: 0.15,
    BiddingBot: 0.05,
} for t in range(blocks)]


(df, user_pool, chain) = simulate(demand_scenario, shares_scenario_fpa, extra_metrics_fpa)

100%|██████████| 6000/6000 [06:47<00:00, 14.73it/s]


ax = df.plot("block", ["basefee", "market_price", "slow_oracle", "medium_oracle", "fast_oracle"])
ax.set_xlabel("Block height")
ax.set_ylabel("Gas price (Gwei)")

Text(0, 0.5, 'Gas price (Gwei)')


ax = df[(df.block >= 400) & (df.block < 1750)].plot("block", ["basefee", "market_price", "slow_oracle", "medium_oracle", "fast_oracle"])
ax.set_xlabel("Block height")
ax.set_ylabel("Gas price (Gwei)")

Text(0, 0.5, 'Gas price (Gwei)')

TL;DR¶

Simulation components¶

Legacy tooling for EIP 1559 environment¶

Legacy users¶

Price oracles¶

Bidding bots¶

Transaction and user pools¶

The market price¶

The main simulation loop¶

Boundary cases: all legacy and all 1559 users¶

Generating the demand¶

Scenario 1: All legacy users¶

Atrocity exhibition I: Sticky oracles¶

Sender	`gas_price`	`gas_premium`	`max_fee`	`basefee`	User pays	Miner receives
Legacy user	100	100	100	40	100	60
1559 user	x	1	100	40	41	1