diff --git a/monitoring.yaml b/monitoring.yaml index c7362bc8..42a6965b 100644 --- a/monitoring.yaml +++ b/monitoring.yaml @@ -293,9 +293,11 @@ protocols: - name: "Low Liquidity" description: "V1 non-YV-collateral and all V2 vault liquidity <2%; combined V1/V2 YV-collateral unwind liquidity vs collateral at risk" - name: "V1 Governance" - description: "Pending supply caps, market removals, timelock changes, guardian changes" + description: "Pending supply caps, market removals (CRITICAL while the vault still supplies the market), timelock decreases, guardian changes" - name: "V2 Governance" - description: "Pending timelocked operations, owner/curator changes, sentinel/allocator/adapter changes" + description: "Pending timelocked operations (CRITICAL for removeAdapter of a funded adapter), owner/curator changes, sentinel/allocator/adapter changes" + - name: "V2 Adapter Share Burns" + description: "Pending MorphoMarketV1AdapterV2 burnShares (CRITICAL while the adapter holds the market) and burnShares timelock decreases, via eth_call" - name: "V2 Wrapped V1 Coverage" description: "V2 vault wrapping a v1 vault not in config.py:VAULTS_V1_BY_CHAIN" - name: "Safe Multisig" diff --git a/protocols/morpho/README.md b/protocols/morpho/README.md index d85c2be0..076a8108 100644 --- a/protocols/morpho/README.md +++ b/protocols/morpho/README.md @@ -8,16 +8,18 @@ Morpho governance monitoring is defined in the [Python script](./governance.py) The script checks if there are any new values pending in the timelock for a given vault. It detects the following changes: -- Changing timelock value, minimal values is 1 day. +- Decreasing the timelock (increases apply instantly, so only decreases are ever pending), minimal value is 1 day. The HIGH alert shows the current → new value and when it becomes executable. - Changing guardian address. - Changing supply caps, only to higher value than the current one, for both supply and withdraw markets. -- Removing of a market from the vault. +- Removing of a market from the vault (`submitMarketRemoval`). Once `removableAt` passes, `updateWithdrawQueue` can drop the market **even if the vault still supplies it**, writing that supply off. The monitor reads the vault's Morpho Blue position in the market: + - any non-zero supply → **CRITICAL** potential-loss alert with the amount at risk and its share of `totalAssets` (omitted if `totalAssets()` reverts, e.g. on a broken IRM — the check itself still runs), re-alerted once within 24h of `removableAt`; + - no supply → MEDIUM (routine cleanup). ### How to Add a New Vault Add a `VaultConfig` to `VAULTS_V1_BY_CHAIN` in [config.py](./config.py). Every configured V1 vault receives both market and governance monitoring. `governance.py` iterates every chain in that mapping, so a new chain needs no script change; a failure on one chain is logged and the remaining chains are still checked before the run fails. -Monitored chains: Mainnet, Base, Katana, HyperEVM, and Robinhood Chain (V2 only). HyperEVM uses `PROVIDER_URL_HYPEREVM` when set; if no `PROVIDER_URL_HYPEREVM*` variable exists it falls back to the public `https://rpc.hyperliquid.xyz/evm` (`PUBLIC_RPC_URLS` in [utils/chains.py](../../utils/chains.py)). RPC requests time out after 60 seconds so a stalled public endpoint cannot hang the daily profile. +Monitored chains: Mainnet, Base, Katana, HyperEVM, and Robinhood Chain (V2 only). HyperEVM uses `PROVIDER_URL_HYPEREVM` when set; if no `PROVIDER_URL_HYPEREVM*` variable exists it falls back to the public `https://rpc.hyperliquid.xyz/evm` (`PUBLIC_RPC_URLS` in [utils/chains.py](../../utils/chains.py)). Robinhood Chain likewise falls back to its official `https://rpc.mainnet.chain.robinhood.com`, used by the V2 adapter timelock reads. RPC requests time out after 60 seconds so a stalled public endpoint cannot hang the daily profile. ## Vaults & Markets @@ -152,8 +154,23 @@ Morpho's [Vault V2](https://github.com/morpho-org/vault-v2) replaces the v1 sing - [`governance_v2.py`](./governance_v2.py) — daily, pulls a per-vault governance **snapshot** from Morpho's GraphQL API (`vaultV2s.pendingConfigs` + `owner` / `curator` / `sentinels` / `allocators` / `adapters`) and diffs it against the persisted cache. Mirrors v1's pull-based approach (`pendingTimelock` / `pendingGuardian` / `pendingCap`) so RPC usage stays bounded. Alerts on: new pending timelocked operations, executed or revoked operations, owner / curator changes, sentinel / allocator / adapter set changes. - [`markets_v2.py`](./markets_v2.py) — hourly, GraphQL-only (no RPC): one `vaultV2s` query loads TVL, liquidity, and `MorphoMarketV1` adapter positions for every configured vault, then one `markets` query per chain loads state/bad debt. Applies the shared [risk.py](./risk.py) policy using each position's `supplyAssetsUsd`, and checks withdrawable liquidity (idle plus the vault's position in the liquidity adapter's market, capped by market cash) against the 2% V2 threshold for every vault, including YV-collateral ones, which `markets.py` also covers with the combined collateral-at-risk check. Non-`MorphoMarketV1` adapters fail the run (configured vaults are market-adapter only). +- [`adapter_timelocks_v2.py`](./adapter_timelocks_v2.py) — run by `governance_v2.py`, reads each `MorphoMarketV1AdapterV2`'s own timelock on-chain with batched `eth_call`s (no `eth_getLogs`), since the Morpho GraphQL API has no adapter-level timelock data. A pending `burnShares(marketId)` writes off the adapter's supply in that market (the V2 analogue of a V1 market removal): `executableAt(burnShares(marketId))` is read for every held market → CRITICAL with the amount at risk (stored shares at Morpho Blue market totals, never the IRM, which may revert). A drop in `timelock(burnShares)` → HIGH. A failed RPC read still lets the GraphQL diff run, then fails the job. + - **TODO:** a burn submitted with non-canonical calldata (canonical + trailing bytes; Solidity ignores them, but `executableAt` is keyed by the exact bytes) is not detected. It needs the adapter's `Submit` events, which the Morpho API does not expose. - [`v2_decoders.py`](./v2_decoders.py) — selector→signature map and decoders for every v2 timelocked function (and the three `idData` tag prefixes used by `increaseAbsoluteCap`/`increaseRelativeCap`). Absolute caps are denominated in the *vault's* asset whatever id they are keyed by, so the decoder takes the vault's asset decimals/symbol; `type(uint128).max` renders as `unlimited`. +### Potential vault loss alerts + +Operations that can write off vault funds are **CRITICAL** for any non-zero amount, sent as their own message prefixed `🚨 POTENTIAL VAULT LOSS` (never bundled into the grouped diff), and re-alerted once when within 24 hours of execution: + +| Vault | Operation | Condition | +|---|---|---| +| V1 | `submitMarketRemoval` | the vault still supplies the market | +| V2 | `removeAdapter(adapter)` | the adapter holds assets (`totalAssets` is the sum of adapter `realAssets`) | +| V2 | `decreaseTimelock(removeAdapter, …)` | any adapter holds assets | +| V2 adapter | `burnShares(marketId)` | the adapter holds assets in the market | + +Other V2 timelock decreases and `abdicate` (irreversible) are HIGH; remaining pending operations stay MEDIUM. Pending V2 operations can be cancelled by the curator or a sentinel with `revoke(data)`; V1 removals by the curator or guardian with `revokePendingMarketRemoval`. + ### Grouped alerts Both governance monitors buffer their findings per vault in [`_alerts.py`](./_alerts.py) and send them as **one Telegram message per vault** — one header naming the vault and chain, one severity (the highest of the group), sections separated by `---`. A vault with several simultaneous changes no longer produces a burst of near-identical messages. Groups too long for a single message split into numbered `(i/N)` parts rather than being truncated, and the cache cursors that record "we alerted on this" are committed only after the send succeeds, so a Telegram failure is retried on the next run instead of being lost. diff --git a/protocols/morpho/_alerts.py b/protocols/morpho/_alerts.py index 2eb2e4d4..cbbbb489 100644 --- a/protocols/morpho/_alerts.py +++ b/protocols/morpho/_alerts.py @@ -34,12 +34,26 @@ _MESSAGE_OVERHEAD = 64 +# Header prefix for findings that can write off vault funds (market removal, +# adapter removal, share burn). They are sent as their own message so they never +# drown in a routine grouped diff. +LOSS_PREFIX = "🚨 POTENTIAL VAULT LOSS" + +# Pending loss operations are re-alerted once when their execution time is this close. +REMINDER_WINDOW_SECONDS = 24 * 60 * 60 + + @dataclass class VaultAlert: - """One section of a vault's grouped Telegram message.""" + """One section of a vault's grouped Telegram message. + + ``standalone`` sections are sent as their own message with ``LOSS_PREFIX`` + instead of being grouped with the vault's other findings. + """ severity: AlertSeverity body: str + standalone: bool = False @dataclass @@ -53,6 +67,10 @@ def alert(self, severity: AlertSeverity, body: str) -> None: """Buffer one section of the vault's grouped message.""" self.alerts.append(VaultAlert(severity, body)) + def loss(self, body: str) -> None: + """Buffer a CRITICAL potential-loss finding, sent as its own message.""" + self.alerts.append(VaultAlert(AlertSeverity.CRITICAL, body, standalone=True)) + def defer(self, write: Callable[..., Any], *args: Any) -> None: """Buffer a cache write to apply once the alert is delivered.""" self.writes.append(partial(write, *args)) @@ -111,7 +129,18 @@ def send_vault_alerts(header: str, alerts: List[VaultAlert], protocol: str) -> N No-op when ``alerts`` is empty so callers don't have to guard. Every part carries the same header and the highest severity of the whole group, so a LOW section bundled with an owner change still pings the channel. + + Standalone (potential-loss) sections go out first, one message each, under a + ``LOSS_PREFIX`` header; the remaining sections are grouped as usual. """ + for alert in alerts: + if alert.standalone: + _send_group(f"{LOSS_PREFIX} — {header}", [alert], protocol) + _send_group(header, [a for a in alerts if not a.standalone], protocol) + + +def _send_group(header: str, alerts: List[VaultAlert], protocol: str) -> None: + """Send sections under one header at their highest severity, split into parts if long.""" if not alerts: return severity = max((a.severity for a in alerts), key=_SEVERITY_ORDER.index) diff --git a/protocols/morpho/_shared.py b/protocols/morpho/_shared.py index 9f22833d..6384c4d8 100644 --- a/protocols/morpho/_shared.py +++ b/protocols/morpho/_shared.py @@ -2,12 +2,14 @@ import time from dataclasses import dataclass +from datetime import datetime from typing import Any, Dict, Iterable, List, Optional import requests from utils.chains import Chain from utils.config import Config +from utils.formatting import format_duration, format_token_amount, format_with_suffix from utils.http_client import request_with_retry from utils.logger import get_logger @@ -81,6 +83,67 @@ def execute_graphql( raise error_type(f"Morpho GraphQL failed after {retries + 1} attempts for {context}") +MORPHO_BLUE_ABI_PATH = "protocols/morpho/abi/morpho_blue.json" + +# Morpho Blue SharesMathLib virtual offsets, used to convert supply shares to assets. +VIRTUAL_SHARES = 10**6 +VIRTUAL_ASSETS = 1 + + +def to_assets_down(shares: int, total_assets: int, total_shares: int) -> int: + """Convert Morpho Blue supply shares to assets, rounding down like ``SharesMathLib``. + + Uses the market's stored totals rather than an IRM-accrued view, so it can't + revert on a market whose IRM reverts; it only omits interest since the last update. + """ + return shares * (total_assets + VIRTUAL_ASSETS) // (total_shares + VIRTUAL_SHARES) + + +def format_ts(ts: int) -> str: + """Render a unix timestamp as local ``YYYY-MM-DD HH:MM:SS``.""" + return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") + + +def format_countdown(ts: int) -> str: + """Human-friendly time remaining until ``ts``, e.g. ``(3 days)``.""" + seconds = ts - int(datetime.now().timestamp()) + if seconds <= 0: + return "(now)" + days = seconds / 86400 + if days >= 1: + n = round(days) + return f"({n} day{'s' if n != 1 else ''})" + hours = seconds / 3600 + if hours >= 1: + n = round(hours) + return f"({n} hour{'s' if n != 1 else ''})" + n = max(round(seconds / 60), 1) + return f"({n} minute{'s' if n != 1 else ''})" + + +def format_timelock(seconds: int) -> str: + """Render a timelock duration; a zero timelock reads ``0s`` rather than ``now``.""" + return format_duration(seconds) if seconds > 0 else "0s" + + +def format_amount_at_risk(raw: int, decimals: int | None, symbol: str | None, total_assets: int | None) -> str: + """Render an at-risk amount with its share of the vault, e.g. ``1.20M USDC (12.50% of vault)``. + + Dust that would round to ``0.00`` is shown in raw units instead, because any + non-zero amount at risk is worth seeing exactly. + """ + if decimals is None: + amount = f"{raw:,} (raw units)" + else: + human = format_token_amount(raw, decimals) + amount = str(format_with_suffix(human)) if human >= 0.01 else f"{raw:,} raw units" + if symbol: + amount = f"{amount} {symbol}" + if total_assets: + amount = f"{amount} ({raw / total_assets:.2%} of vault)" + return amount + + def require_configured_keys( expected: Iterable[str], found: Iterable[str], diff --git a/protocols/morpho/abi/morpho_blue.json b/protocols/morpho/abi/morpho_blue.json new file mode 100644 index 00000000..fdd2731e --- /dev/null +++ b/protocols/morpho/abi/morpho_blue.json @@ -0,0 +1,30 @@ +[ + { + "inputs": [ + { "internalType": "Id", "name": "", "type": "bytes32" }, + { "internalType": "address", "name": "", "type": "address" } + ], + "name": "position", + "outputs": [ + { "internalType": "uint256", "name": "supplyShares", "type": "uint256" }, + { "internalType": "uint128", "name": "borrowShares", "type": "uint128" }, + { "internalType": "uint128", "name": "collateral", "type": "uint128" } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [{ "internalType": "Id", "name": "", "type": "bytes32" }], + "name": "market", + "outputs": [ + { "internalType": "uint128", "name": "totalSupplyAssets", "type": "uint128" }, + { "internalType": "uint128", "name": "totalSupplyShares", "type": "uint128" }, + { "internalType": "uint128", "name": "totalBorrowAssets", "type": "uint128" }, + { "internalType": "uint128", "name": "totalBorrowShares", "type": "uint128" }, + { "internalType": "uint128", "name": "lastUpdate", "type": "uint128" }, + { "internalType": "uint128", "name": "fee", "type": "uint128" } + ], + "stateMutability": "view", + "type": "function" + } +] diff --git a/protocols/morpho/adapter_timelocks_v2.py b/protocols/morpho/adapter_timelocks_v2.py new file mode 100644 index 00000000..0b3c2d21 --- /dev/null +++ b/protocols/morpho/adapter_timelocks_v2.py @@ -0,0 +1,271 @@ +"""On-chain checks of ``MorphoMarketV1AdapterV2``'s own timelock. + +The adapter's ``burnShares(marketId)`` is the V2 equivalent of a V1 +``submitMarketRemoval``: once executed it zeroes the adapter's supply shares in +that market, so ``realAssets`` drops and the vault takes the loss. It is +timelocked on the adapter itself, which the Morpho GraphQL API does not expose. + +Only batched ``eth_call`` reads are used (never ``eth_getLogs``): + +* **Pending burns** — ``executableAt(abi.encodeCall(burnShares, (id)))`` for every + held market. +* ``timelock(burnShares)`` — a shortened reaction window, after the fact. + +TODO: a burn submitted with non-canonical calldata (canonical + trailing bytes) +is not detected. ``executableAt`` is keyed by the exact calldata and Solidity +ignores trailing bytes, so such a burn executes under a key we can't derive. +Finding it needs the adapter's ``Submit`` events, which the Morpho API does not +expose (no adapter-level timelock data at all); add it once an indexer provides them. + +Amounts use the adapter's stored shares and Morpho Blue's stored market totals, +never ``expectedSupplyAssets``: that goes through the market's IRM, which may +revert, and burning shares of such a market is explicitly supported. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, List + +from web3 import Web3 + +from protocols.morpho._alerts import REMINDER_WINDOW_SECONDS, VaultDiff +from protocols.morpho._shared import ( + MORPHO_BLUE_ABI_PATH, + fetch_market_metadata, + format_amount_at_risk, + format_countdown, + format_timelock, + format_ts, + get_market_url, + to_assets_down, +) +from protocols.morpho.v2_decoders import selector_of +from utils.abi import load_abi +from utils.alert import AlertSeverity +from utils.cache import get_last_value_for_key_from_file, morpho_filename, morpho_key, write_last_value_to_file +from utils.chains import Chain +from utils.logger import get_logger +from utils.web3_wrapper import ChainManager + +logger = get_logger("morpho.adapter_timelocks_v2") + +ABI_MARKET_V1_ADAPTER_V2 = load_abi("protocols/morpho/abi/morpho_market_v1_adapter_v2.json") +ABI_MORPHO_BLUE = load_abi(MORPHO_BLUE_ABI_PATH) +BURN_SHARES_SELECTOR = selector_of("burnShares(bytes32)") + +# Cache value-type tags used with utils.cache.morpho_key. +BURN_TIMELOCK_TYPE = "v2_adapter_burn_timelock" +PENDING_BURN_TYPE = "v2_adapter_pending_burn" +PENDING_BURN_REMINDER_TYPE = "v2_adapter_pending_burn_reminder" +PENDING_BURN_INDEX_TYPE = "v2_adapter_pending_burn_index" + + +@dataclass(frozen=True) +class PendingBurn: + """A submitted, not yet executed canonical ``burnShares`` for one market.""" + + market_id: str # 0x-prefixed hex + executable_at: int + shares: int # the adapter's stored supply shares in the market + assets: int # those shares valued at Morpho Blue's stored market totals + + +@dataclass(frozen=True) +class AdapterTimelockState: + """On-chain timelock state of one ``MorphoMarketV1AdapterV2``.""" + + adapter: str # checksummed + burn_timelock: int + pending_burns: List[PendingBurn] = field(default_factory=list) + + +def burn_shares_calldata(market_id: bytes) -> bytes: + """Canonical calldata of ``burnShares(marketId)``, the key of the adapter's ``executableAt`` mapping.""" + return BURN_SHARES_SELECTOR + market_id + + +def _execute(client: Any, functions: List[Any], block_number: int) -> List[Any]: + """Call contract functions as one JSON-RPC batch and check every call answered. + + ``functions`` are bound but not yet called (``contract.functions.x(args)``): + ``.call()`` must run inside the batch context, or it executes eagerly. + """ + if not functions: + return [] + with client.batch_requests() as batch: + for function in functions: + batch.add(function.call(block_identifier=block_number)) + responses = client.execute_batch(batch) + if len(responses) != len(functions): + raise ValueError(f"Expected {len(functions)} batch responses, got {len(responses)}") + return list(responses) + + +# ---------------------------------------------------------------------------- +# Cache helpers +# ---------------------------------------------------------------------------- + + +def _read_str(key: str) -> str: + raw = get_last_value_for_key_from_file(morpho_filename, key) + return str(raw) if raw else "" + + +def _read_int(key: str) -> int: + try: + return int(get_last_value_for_key_from_file(morpho_filename, key)) + except TypeError, ValueError: + return 0 + + +def _write(key: str, value: Any) -> None: + write_last_value_to_file(morpho_filename, key, value) + + +def _key(adapter: str, item: str, value_type: str) -> str: + return str(morpho_key(adapter.lower(), item, value_type)) + + +# ---------------------------------------------------------------------------- +# Loading +# ---------------------------------------------------------------------------- + + +def load_adapter_state(client: Any, adapter_address: str, block_number: int) -> AdapterTimelockState: + """Read the burn timelock and canonical pending burns of one adapter.""" + adapter = client.eth.contract(address=Web3.to_checksum_address(adapter_address), abi=ABI_MARKET_V1_ADAPTER_V2) + fns = adapter.functions + length, burn_timelock = _execute(client, [fns.marketIdsLength(), fns.timelock(BURN_SHARES_SELECTOR)], block_number) + markets: List[bytes] = _execute(client, [fns.marketIds(i) for i in range(length)], block_number) + executable_at = _execute(client, [fns.executableAt(burn_shares_calldata(m)) for m in markets], block_number) + burned = [(m, at) for m, at in zip(markets, executable_at) if at > 0] + if not burned: + return AdapterTimelockState(adapter=adapter.address, burn_timelock=burn_timelock) + + # Valuation only for the (rare) markets with a pending burn. + blue = client.eth.contract(address=_execute(client, [fns.morpho()], block_number)[0], abi=ABI_MORPHO_BLUE) + values = _execute( + client, [f for m, _ in burned for f in (fns.supplyShares(m), blue.functions.market(m))], block_number + ) + pending = [] + for index, (market_id, at) in enumerate(burned): + shares, market = values[index * 2], values[index * 2 + 1] + pending.append(PendingBurn(Web3.to_hex(market_id), at, shares, to_assets_down(shares, market[0], market[1]))) + return AdapterTimelockState(adapter=adapter.address, burn_timelock=burn_timelock, pending_burns=pending) + + +def load_adapter_states(chain: Chain, adapters: List[str]) -> List[AdapterTimelockState]: + """Load the timelock state of every ``MorphoMarketV1AdapterV2`` of one vault.""" + if not adapters: + return [] + client = ChainManager.get_client(chain) + block_number = int(client.eth.block_number) + return [load_adapter_state(client, adapter, block_number) for adapter in adapters] + + +# ---------------------------------------------------------------------------- +# Diff and alerting +# ---------------------------------------------------------------------------- + + +def _address_link(address: str, chain: Chain) -> str: + base = chain.explorer_url + return f"[{address}]({base}/address/{address})" if base else address + + +def _market_label(market_id: str, chain: Chain) -> tuple[str, int | None, str | None]: + """Return ``(markdown market link, loan decimals, loan symbol)``.""" + metadata = fetch_market_metadata(market_id, chain) + if metadata: + return ( + f"[{metadata['name']}]({get_market_url(market_id, chain)})", + metadata["loan_decimals"], + metadata["loan_symbol"], + ) + return f"[{market_id}]({get_market_url(market_id, chain)})", None, None + + +def _burn_body( + chain: Chain, + adapter: str, + burn: PendingBurn, + total_assets: int | None, + *, + reminder: bool, +) -> str: + market, decimals, symbol = _market_label(burn.market_id, chain) + title = "⏳ Reminder: adapter share burn is close" if reminder else "🔴 Adapter share burn submitted" + return ( + f"{title}: `burnShares` on market {market}\n" + f"🧩 Adapter: {_address_link(adapter, chain)}\n" + f"💸 At risk: {format_amount_at_risk(burn.assets, decimals, symbol, total_assets)}\n" + f"⏰ Executable at: {format_ts(burn.executable_at)} {format_countdown(burn.executable_at)}\n" + "Burning the shares writes off the adapter's supply in this market. " + "If unintended, the curator or a sentinel can call `revoke(data)` on the adapter." + ) + + +def _diff_burn_timelock(chain: Chain, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Alert HIGH when ``timelock(burnShares)`` drops; the first run seeds the cache silently.""" + cache_key = _key(state.adapter, "burn_timelock", BURN_TIMELOCK_TYPE) + last = _read_str(cache_key) + if last and state.burn_timelock < int(last): + diff.alert( + AlertSeverity.HIGH, + f"🚨 Adapter {_address_link(state.adapter, chain)} `burnShares` timelock decreased: " + f"{format_timelock(int(last))} → {format_timelock(state.burn_timelock)}", + ) + if last != str(state.burn_timelock): + diff.defer(_write, cache_key, state.burn_timelock) + + +def _diff_pending_burns(chain: Chain, total_assets: int | None, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Alert on new pending burns (CRITICAL if the adapter holds shares), remind near the deadline, note resolutions.""" + now = int(datetime.now().timestamp()) + current: set[str] = set() + for burn in state.pending_burns: + current.add(burn.market_id) + cache_key = _key(state.adapter, burn.market_id, PENDING_BURN_TYPE) + reminder_key = _key(state.adapter, burn.market_id, PENDING_BURN_REMINDER_TYPE) + in_window = now >= burn.executable_at - REMINDER_WINDOW_SECONDS + if _read_int(cache_key) == burn.executable_at: + if burn.shares > 0 and in_window and _read_int(reminder_key) != burn.executable_at: + diff.loss(_burn_body(chain, state.adapter, burn, total_assets, reminder=True)) + diff.defer(_write, reminder_key, burn.executable_at) + continue + if burn.shares > 0: + diff.loss(_burn_body(chain, state.adapter, burn, total_assets, reminder=False)) + if in_window: + diff.defer(_write, reminder_key, burn.executable_at) + else: + diff.alert( + AlertSeverity.MEDIUM, + f"🔥 Adapter {_address_link(state.adapter, chain)} submitted `burnShares` on market " + f"[{burn.market_id}]({get_market_url(burn.market_id, chain)}) with no shares in it; " + f"executable at {format_ts(burn.executable_at)} {format_countdown(burn.executable_at)}", + ) + diff.defer(_write, cache_key, burn.executable_at) + + index_key = _key(state.adapter, "pending_burns", PENDING_BURN_INDEX_TYPE) + previous = {m for m in _read_str(index_key).split(",") if m} + for market_id in sorted(previous - current): + cache_key = _key(state.adapter, market_id, PENDING_BURN_TYPE) + executable_at = _read_int(cache_key) + verb = "executed" if 0 < executable_at <= now else "revoked" + diff.alert( + AlertSeverity.LOW, + f"{'✅' if verb == 'executed' else '🛑'} Pending `burnShares` on market " + f"[{market_id}]({get_market_url(market_id, chain)}) of adapter " + f"{_address_link(state.adapter, chain)} was {verb}.", + ) + # Clear the cursor so a later re-submit with the same executableAt still alerts. + diff.defer(_write, cache_key, 0) + diff.defer(_write, index_key, ",".join(sorted(current))) + + +def diff_adapter(chain: Chain, total_assets: int | None, state: AdapterTimelockState, diff: VaultDiff) -> None: + """Diff one adapter's on-chain timelock state against the cache into the vault's ``diff``.""" + _diff_burn_timelock(chain, state, diff) + _diff_pending_burns(chain, total_assets, state, diff) diff --git a/protocols/morpho/governance.py b/protocols/morpho/governance.py index d01c3b87..1fe61327 100644 --- a/protocols/morpho/governance.py +++ b/protocols/morpho/governance.py @@ -4,14 +4,20 @@ from web3 import Web3 -from protocols.morpho._alerts import VaultDiff, send_vault_alerts +from protocols.morpho._alerts import REMINDER_WINDOW_SECONDS, VaultDiff, send_vault_alerts from protocols.morpho._shared import ( + MORPHO_BLUE_ABI_PATH, PROTOCOL, MorphoMonitoringError, execute_graphql, fetch_market_metadata, + format_amount_at_risk, + format_countdown, + format_timelock, + format_ts, get_market_url, get_vault_url, + to_assets_down, ) from protocols.morpho.config import VAULTS_V1_BY_CHAIN from utils.abi import load_abi @@ -29,7 +35,9 @@ PENDING_CAP_TYPE = "pending_cap" REMOVABLE_AT_TYPE = "removable_at" +REMOVAL_REMINDER_TYPE = "removable_at_reminder" ABI_MORPHO = load_abi("protocols/morpho/abi/morpho.json") +ABI_MORPHO_BLUE = load_abi(MORPHO_BLUE_ABI_PATH) @dataclass(frozen=True) @@ -42,6 +50,11 @@ class MarketGovernanceState: pending_cap_timestamp: int current_cap: int removable_at: int + # The vault's own supply in the market on Morpho Blue. A removal that executes + # while this is non-zero writes the position off and the vault takes the loss. + supply_shares: int = 0 + supply_assets: int = 0 + vault_total_assets: int | None = None def fetch_pending_cap_market_ids(vault_address: str, chain: Chain) -> list[str]: @@ -151,20 +164,35 @@ def _load_market_governance_states( client: Any, block_number: int, ) -> list[MarketGovernanceState]: - """Batch pending-cap and config reads for V1 vault markets.""" + """Batch pending-cap, config, and vault-supply reads for V1 vault markets. + + None of these reads go through a market's IRM, so they can't revert on a broken + market. ``totalAssets`` does (it values every withdraw-queue market), so it is + read separately and only feeds the optional "% of vault" figure. + """ + morpho_address = morpho_contract.functions.MORPHO().call(block_identifier=block_number) + total_assets = _try_total_assets(morpho_contract, block_number) + blue = client.eth.contract(address=morpho_address, abi=ABI_MORPHO_BLUE) + + per_market = 4 with client.batch_requests() as batch: for market_id in market_ids: batch.add(morpho_contract.functions.pendingCap(market_id).call(block_identifier=block_number)) batch.add(morpho_contract.functions.config(market_id).call(block_identifier=block_number)) + batch.add(blue.functions.position(market_id, morpho_contract.address).call(block_identifier=block_number)) + batch.add(blue.functions.market(market_id).call(block_identifier=block_number)) responses = client.execute_batch(batch) - expected_count = len(market_ids) * 2 + expected_count = len(market_ids) * per_market if len(responses) != expected_count: - raise ValueError(f"Expected {expected_count} pendingCap/config responses, got {len(responses)}") + raise ValueError(f"Expected {expected_count} pendingCap/config/position/market responses, got {len(responses)}") states = [] for index, market_id in enumerate(market_ids): - pending_cap, pending_timestamp = responses[index * 2] - config = responses[index * 2 + 1] + offset = index * per_market + pending_cap, pending_timestamp = responses[offset] + config = responses[offset + 1] + supply_shares = responses[offset + 2][0] + market = responses[offset + 3] states.append( MarketGovernanceState( vault_address=morpho_contract.address, @@ -173,11 +201,23 @@ def _load_market_governance_states( pending_cap_timestamp=pending_timestamp, current_cap=config[0], removable_at=config[2], + supply_shares=supply_shares, + supply_assets=to_assets_down(supply_shares, market[0], market[1]), + vault_total_assets=total_assets, ) ) return states +def _try_total_assets(morpho_contract: Any, block_number: int) -> int | None: + """Best-effort ``totalAssets()``; None when it reverts (e.g. a withdraw-queue market's IRM reverts).""" + try: + return int(morpho_contract.functions.totalAssets().call(block_identifier=block_number)) + except Exception as e: + logger.warning("totalAssets() failed for vault %s, alerts omit the vault share: %s", morpho_contract.address, e) + return None + + def _vault_header(name: str, vault_address: str, chain: Chain) -> str: """One-line header for the vault's grouped alert.""" return f"[{name}]({get_vault_url(vault_address, chain)}) on {chain.name}" @@ -220,8 +260,35 @@ def _check_pending_cap(name: str, state: MarketGovernanceState, chain: Chain, di ) +def _removal_loss_body(state: MarketGovernanceState, chain: Chain, *, reminder: bool) -> str: + """Render the CRITICAL section for a queued removal of a market the vault still supplies.""" + market_url = get_market_url(state.market_id, chain) + metadata = fetch_market_metadata(state.market_id, chain) + if metadata: + market_label = f"{metadata['name']} ({metadata['lltv'] / 1e18 * 100:.2f}%)" + decimals, symbol = metadata["loan_decimals"], metadata["loan_symbol"] + else: + market_label, decimals, symbol = state.market_id, None, None + at_risk = format_amount_at_risk(state.supply_assets, decimals, symbol, state.vault_total_assets) + title = "⏳ Reminder: market removal is close" if reminder else "🔴 Market removal queued with funds still supplied" + return ( + f"{title}\n" + f"Market: [{market_label}]({market_url})\n" + f"💸 At risk: {at_risk}\n" + f"⏰ Removable at: {format_ts(state.removable_at)} {format_countdown(state.removable_at)}\n" + "Removing the market writes off the remaining supply. Withdraw it before the removal, " + "or have the guardian/curator call `revokePendingMarketRemoval`." + ) + + def _check_market_removal(state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: - """Buffer a section for a newly queued V1 market removal.""" + """Buffer a section for a queued V1 market removal. + + A removal of a market the vault still supplies is CRITICAL, for any amount: + once ``removableAt`` passes, ``updateWithdrawQueue`` can drop the market and the + supply leaves ``totalAssets``. It is re-alerted once inside the reminder window. + A removal of an empty market is routine cleanup and stays MEDIUM. + """ if state.removable_at <= 0: return last_timestamp = get_last_executed_morpho_from_file( @@ -229,21 +296,31 @@ def _check_market_removal(state: MarketGovernanceState, chain: Chain, diff: Vaul state.market_id, REMOVABLE_AT_TYPE, ) + at_risk = state.supply_shares > 0 if state.removable_at <= last_timestamp: - logger.info( - "Skipping previously alerted market removal for vault %s market %s", - state.vault_address, - state.market_id, - ) + if at_risk: + _check_removal_reminder(state, chain, diff) + else: + logger.info( + "Skipping previously alerted market removal for vault %s market %s", + state.vault_address, + state.market_id, + ) return - market_url = get_market_url(state.market_id, chain) - market_name, _ = fetch_market_info(state.market_id, chain) - removable_at = datetime.fromtimestamp(state.removable_at).strftime("%Y-%m-%d %H:%M:%S") - diff.alert( - AlertSeverity.MEDIUM, - f"➖ Queued to remove market [{market_name}]({market_url}) at {removable_at}", - ) + if at_risk: + diff.loss(_removal_loss_body(state, chain, reminder=False)) + if _in_reminder_window(state.removable_at): + # The first alert already lands inside the window; don't remind right after it. + _defer_reminder_write(state, diff) + else: + market_url = get_market_url(state.market_id, chain) + market_name, _ = fetch_market_info(state.market_id, chain) + diff.alert( + AlertSeverity.MEDIUM, + f"➖ Queued to remove market [{market_name}]({market_url}) at {format_ts(state.removable_at)} " + "(no vault supply in the market)", + ) diff.defer( write_last_executed_morpho_to_file, state.vault_address, @@ -253,6 +330,31 @@ def _check_market_removal(state: MarketGovernanceState, chain: Chain, diff: Vaul ) +def _in_reminder_window(deadline: int) -> bool: + return int(datetime.now().timestamp()) >= deadline - REMINDER_WINDOW_SECONDS + + +def _defer_reminder_write(state: MarketGovernanceState, diff: VaultDiff) -> None: + diff.defer( + write_last_executed_morpho_to_file, + state.vault_address, + state.market_id, + REMOVAL_REMINDER_TYPE, + state.removable_at, + ) + + +def _check_removal_reminder(state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: + """Re-alert once when an already-alerted removal with supply nears ``removableAt``.""" + if not _in_reminder_window(state.removable_at): + return + reminded_at = get_last_executed_morpho_from_file(state.vault_address, state.market_id, REMOVAL_REMINDER_TYPE) + if state.removable_at <= reminded_at: + return + diff.loss(_removal_loss_body(state, chain, reminder=True)) + _defer_reminder_write(state, diff) + + def check_market_governance_state(name: str, state: MarketGovernanceState, chain: Chain, diff: VaultDiff) -> None: """Check pending cap and removal changes for one V1 market.""" _check_pending_cap(name, state, chain, diff) @@ -279,13 +381,24 @@ def check_pending_role_change( role_type: str, timestamp: int, diff: VaultDiff, + message: str | None = None, ) -> None: + """Alert HIGH once per pending ``role_type`` change, keyed by its ``validAt`` timestamp.""" market_id = "" # use empty string for all markets because the value is used per vault if timestamp > get_last_executed_morpho_from_file(morpho_contract.address, market_id, role_type): - diff.alert(AlertSeverity.HIGH, f"🚨 {role_type.capitalize()} is changing") + diff.alert(AlertSeverity.HIGH, message or f"🚨 {role_type.capitalize()} is changing") diff.defer(write_last_executed_morpho_to_file, morpho_contract.address, market_id, role_type, timestamp) +def format_timelock_decrease(current: int, new: int, valid_at: int) -> str: + """Render a pending V1 timelock change; only decreases are ever pending on V1.""" + return ( + f"🚨 Timelock decrease: {format_timelock(current)} → {format_timelock(new)}\n" + f"⏰ Executable at: {format_ts(valid_at)} {format_countdown(valid_at)}\n" + "A shorter timelock shrinks the window to react to cap changes and market removals." + ) + + def check_timelock_and_guardian( morpho_contract: Any, client: Any, @@ -295,15 +408,20 @@ def check_timelock_and_guardian( with morpho_contract.w3.batch_requests() as batch: batch.add(morpho_contract.functions.pendingTimelock().call(block_identifier=block_number)) batch.add(morpho_contract.functions.pendingGuardian().call(block_identifier=block_number)) + batch.add(morpho_contract.functions.timelock().call(block_identifier=block_number)) responses = client.execute_batch(batch) - if len(responses) != 2: - raise ValueError(f"Expected 2 responses from batch, got {len(responses)}") - - timelock = responses[0][1] # [1] to get the timestamp - guardian = responses[1][1] # [1] to get the timestamp - - check_pending_role_change(morpho_contract, "timelock", timelock, diff) - check_pending_role_change(morpho_contract, "guardian", guardian, diff) + if len(responses) != 3: + raise ValueError(f"Expected 3 responses from batch, got {len(responses)}") + + pending_timelock_value, timelock_valid_at = responses[0] + guardian_valid_at = responses[1][1] # [1] to get the timestamp + current_timelock = responses[2] + + timelock_message = None + if timelock_valid_at > 0: + timelock_message = format_timelock_decrease(current_timelock, pending_timelock_value, timelock_valid_at) + check_pending_role_change(morpho_contract, "timelock", timelock_valid_at, diff, timelock_message) + check_pending_role_change(morpho_contract, "guardian", guardian_valid_at, diff) def get_data_for_chain(chain: Chain) -> None: diff --git a/protocols/morpho/governance_v2.py b/protocols/morpho/governance_v2.py index 3b0ef6f2..800453fd 100644 --- a/protocols/morpho/governance_v2.py +++ b/protocols/morpho/governance_v2.py @@ -13,10 +13,14 @@ * **Owner / curator changes** — instant, no timelock. * **Sentinels / allocators / adapters** — added or removed. -NOT covered: ``MorphoMarketV1AdapterV2``'s own internal timelock system. The -GraphQL API does not surface adapter-internal pending operations; replaying -their Submit/Accept/Revoke events would reintroduce the RPC cost we're -explicitly avoiding. Phase-2 candidate. +Pending operations that can write off vault funds are CRITICAL and sent as their +own message (see ``_alerts.LOSS_PREFIX``), then re-alerted once near their +execution time: ``removeAdapter`` of an adapter that still holds assets, and +``decreaseTimelock(removeAdapter)`` while any adapter holds assets. Other +timelock decreases and ``abdicate`` are HIGH. + +``MorphoMarketV1AdapterV2``'s own timelock (``burnShares``) is not exposed by the +GraphQL API; ``adapter_timelocks_v2`` reads it on-chain with a few batched calls. """ from __future__ import annotations @@ -27,16 +31,21 @@ from web3 import Web3 -from protocols.morpho._alerts import VaultDiff, send_vault_alerts +from protocols.morpho import adapter_timelocks_v2 +from protocols.morpho._alerts import REMINDER_WINDOW_SECONDS, VaultDiff, send_vault_alerts from protocols.morpho._shared import ( PROTOCOL, MorphoV2MonitoringError, execute_graphql, + format_amount_at_risk, + format_countdown, + format_ts, get_vault_url, require_configured_keys, ) +from protocols.morpho.adapter_timelocks_v2 import AdapterTimelockState from protocols.morpho.config import VAULTS_V2_BY_CHAIN, get_vault_query_config -from protocols.morpho.v2_decoders import decode_submit, submit_data_key +from protocols.morpho.v2_decoders import decode_call, decode_submit, selector_of, submit_data_key from utils.alert import AlertSeverity from utils.cache import ( get_last_value_for_key_from_file, @@ -55,6 +64,9 @@ PENDING_FUNCTION_TYPE = "v2_pending_function" ROLE_TYPE = "v2_role" SET_TYPE = "v2_set" +PENDING_REMINDER_TYPE = "v2_pending_reminder" + +REMOVE_ADAPTER_SELECTOR = selector_of("removeAdapter(address)") # Sentinel cache values for pending operations: # * positive int: validAt of the latest pending submission we've alerted on @@ -63,19 +75,24 @@ EXECUTED = -1 REVOKED = 0 +# Morpho caps query complexity at 1,000,000 and the page size multiplies the cost +# of every nested field: first: 200 measured 1,151,000, first: 100 measures 575,500. +GOVERNANCE_PAGE_SIZE = 100 + _GOVERNANCE_QUERY = """ -query GovernanceV2($addresses: [String!]!, $chainIds: [Int!]!) { - vaultV2s(first: 200, where: { address_in: $addresses, chainId_in: $chainIds }) { +query GovernanceV2($addresses: [String!]!, $chainIds: [Int!]!, $first: Int!) { + vaultV2s(first: $first, where: { address_in: $addresses, chainId_in: $chainIds }) { items { address name chain { id } + totalAssets asset { symbol decimals } owner { address } curator { address } sentinels { sentinel { address } } allocators { allocator { address } } - adapters { items { address type } } + adapters { items { address type assets } } pendingConfigs { items { validAt functionName data txHash } } @@ -121,6 +138,15 @@ class V2GovernanceSnapshot: # The vault's own asset — denominates absolute caps in decoded operations. asset_symbol: str = "" asset_decimals: int | None = None + total_assets: int | None = None + # Checksummed adapter address → assets it holds / GraphQL adapter type. + adapter_assets: Dict[str, int] = field(default_factory=dict) + adapter_types: Dict[str, str] = field(default_factory=dict) + + @property + def market_v1_adapters(self) -> List[str]: + """Adapters with their own timelock (``MorphoMarketV1AdapterV2``).""" + return sorted(a for a, t in self.adapter_types.items() if t == "MorphoMarketV1") # ---------------------------------------------------------------------------- @@ -140,6 +166,10 @@ def _asset_decimals(item: Dict[str, Any]) -> int | None: return int(raw) if raw is not None else None +def _optional_int(value: Any) -> int | None: + return int(value) if value is not None else None + + def _checksum_or_empty(value: str) -> str: if not value: return "" @@ -159,7 +189,7 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: data = execute_graphql( _GOVERNANCE_QUERY, - {"addresses": addresses, "chainIds": chain_ids}, + {"addresses": addresses, "chainIds": chain_ids, "first": GOVERNANCE_PAGE_SIZE}, "Vault V2 governance", error_type=MorphoV2MonitoringError, ) @@ -179,7 +209,8 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: sentinels = [_checksum_or_empty(s["sentinel"]["address"]) for s in (item.get("sentinels") or [])] allocators = [_checksum_or_empty(a["allocator"]["address"]) for a in (item.get("allocators") or [])] - adapters = [_checksum_or_empty(a["address"]) for a in ((item.get("adapters") or {}).get("items") or [])] + adapter_items = (item.get("adapters") or {}).get("items") or [] + adapters = [_checksum_or_empty(a["address"]) for a in adapter_items] pending = [ PendingConfig( valid_at=int(pc["validAt"]), @@ -203,6 +234,9 @@ def fetch_governance_snapshots() -> Dict[Chain, List[V2GovernanceSnapshot]]: pending_configs=pending, asset_symbol=(item.get("asset") or {}).get("symbol") or "", asset_decimals=_asset_decimals(item), + total_assets=_optional_int(item.get("totalAssets")), + adapter_assets={_checksum_or_empty(a["address"]): int(a.get("assets") or 0) for a in adapter_items}, + adapter_types={_checksum_or_empty(a["address"]): a.get("type") or "" for a in adapter_items}, ) ) @@ -238,27 +272,6 @@ def _write(key: str, value: Any) -> None: # ---------------------------------------------------------------------------- -def _format_ts(ts: int) -> str: - return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") - - -def _format_countdown(ts: int) -> str: - """Human-friendly time remaining until ``ts``, e.g. ``(3 days)``.""" - seconds = ts - int(datetime.now().timestamp()) - if seconds <= 0: - return "(now)" - days = seconds / 86400 - if days >= 1: - n = round(days) - return f"({n} day{'s' if n != 1 else ''})" - hours = seconds / 3600 - if hours >= 1: - n = round(hours) - return f"({n} hour{'s' if n != 1 else ''})" - n = max(round(seconds / 60), 1) - return f"({n} minute{'s' if n != 1 else ''})" - - def _explorer_link(chain: Chain, tx_hash: str) -> str: base = chain.explorer_url if not base or not tx_hash: @@ -295,10 +308,70 @@ def _vault_header(snapshot: V2GovernanceSnapshot) -> str: return f"V2 [{snapshot.name}]({get_vault_url(snapshot.address, snapshot.chain)}) on {snapshot.chain.name}" +def _held_assets(snapshot: V2GovernanceSnapshot) -> int: + return sum(snapshot.adapter_assets.values()) + + +def classify_pending(snapshot: V2GovernanceSnapshot, pc: PendingConfig) -> tuple[AlertSeverity, int | None]: + """Return the severity of a pending operation and, for potential losses, the assets at risk. + + * ``removeAdapter(a)`` while ``a`` holds assets → CRITICAL: ``totalAssets`` is + the sum of adapter ``realAssets``, so removing it writes those assets off. + * ``decreaseTimelock(removeAdapter, …)`` while any adapter holds assets → + CRITICAL: it shrinks the window to react to such a removal. + * Other timelock decreases and ``abdicate`` (irreversible) → HIGH. + * Everything else → MEDIUM. + """ + decoded = decode_call(pc.data) + if decoded is None: + return AlertSeverity.MEDIUM, None + name, args = decoded + if name == "removeAdapter": + assets = snapshot.adapter_assets.get(str(Web3.to_checksum_address(args[0])), 0) + return (AlertSeverity.CRITICAL, assets) if assets > 0 else (AlertSeverity.MEDIUM, None) + if name == "decreaseTimelock": + held = _held_assets(snapshot) + if bytes(args[0]) == REMOVE_ADAPTER_SELECTOR and held > 0: + return AlertSeverity.CRITICAL, held + return AlertSeverity.HIGH, None + if name == "abdicate": + return AlertSeverity.HIGH, None + return AlertSeverity.MEDIUM, None + + +def _loss_body( + snapshot: V2GovernanceSnapshot, + pc: PendingConfig, + operation_label: str, + at_risk: int, + *, + reminder: bool, +) -> str: + title = "⏳ Reminder: loss-causing operation is close" if reminder else "🔴 Submitted loss-causing operation" + amount = format_amount_at_risk( + at_risk, snapshot.asset_decimals, snapshot.asset_symbol or None, snapshot.total_assets + ) + return ( + f"{title}: {operation_label}\n" + f"💸 At risk: {amount}\n" + f"⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}\n" + f"🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}\n" + f"{_loss_reason(pc)} If unintended, the curator or a sentinel can call `revoke(data)`." + ) + + +def _loss_reason(pc: PendingConfig) -> str: + decoded = decode_call(pc.data) + if decoded and decoded[0] == "decreaseTimelock": + return "Shortens the timelock on `removeAdapter` while adapters hold assets." + return "Removing an adapter that still holds assets writes them off; deallocate first." + + def _alert_pending_new( snapshot: V2GovernanceSnapshot, pending: List[tuple[PendingConfig, str]], diff: VaultDiff, + severity: AlertSeverity = AlertSeverity.MEDIUM, ) -> None: """Buffer a section for newly-submitted timelocked operation(s) on one vault. @@ -313,9 +386,9 @@ def _alert_pending_new( if len(pending) == 1: pc, operation_label = pending[0] diff.alert( - AlertSeverity.MEDIUM, + severity, f"📥 Submitted: {operation_label}\n" - f"⏰ Executable at: {_format_ts(pc.valid_at)} {_format_countdown(pc.valid_at)}\n" + f"⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}\n" f"🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}", ) return @@ -327,15 +400,15 @@ def _alert_pending_new( for _, operation_label in pending: lines.append(f" • {operation_label}") pc0 = pending[0][0] - lines.append(f"⏰ Executable at: {_format_ts(pc0.valid_at)} {_format_countdown(pc0.valid_at)}") + lines.append(f"⏰ Executable at: {format_ts(pc0.valid_at)} {format_countdown(pc0.valid_at)}") lines.append(f"🔗 Tx: {_explorer_link(snapshot.chain, pc0.tx_hash)}") else: for pc, operation_label in pending: lines.append(f" • {operation_label}") - lines.append(f" ⏰ Executable at: {_format_ts(pc.valid_at)} {_format_countdown(pc.valid_at)}") + lines.append(f" ⏰ Executable at: {format_ts(pc.valid_at)} {format_countdown(pc.valid_at)}") lines.append(f" 🔗 Tx: {_explorer_link(snapshot.chain, pc.tx_hash)}") - diff.alert(AlertSeverity.MEDIUM, "\n".join(lines)) + diff.alert(severity, "\n".join(lines)) def _alert_pending_resolved( @@ -356,7 +429,7 @@ def _alert_pending_resolved( operation = f"`{function_name}()`" if function_name else f"`{data_hash[:10]}…`" diff.alert( AlertSeverity.LOW, - f"{icon} Pending operation {operation} was {verb} (was due {_format_ts(last_valid_at)}).", + f"{icon} Pending operation {operation} was {verb} (was due {format_ts(last_valid_at)}).", ) @@ -385,20 +458,37 @@ def _diff_pending(snapshot: V2GovernanceSnapshot, diff: VaultDiff) -> None: current_keys: set[str] = set() new_pending: List[tuple[PendingConfig, str]] = [] + group_severity = AlertSeverity.MEDIUM + now = int(datetime.now().timestamp()) for pc in snapshot.pending_configs: current_keys.add(pc.data_hash) operation_label = _operation_label(snapshot, pc) diff.defer(_write, _pending_function_key(snapshot, pc.data_hash), _operation_function_name(pc, operation_label)) + severity, at_risk = classify_pending(snapshot, pc) cache_key = morpho_key(addr, pc.data_hash, PENDING_TYPE) + reminder_key = morpho_key(addr, pc.data_hash, PENDING_REMINDER_TYPE) + in_window = now >= pc.valid_at - REMINDER_WINDOW_SECONDS last = _read_int(cache_key) - # Already alerted at this validAt, or marked executed. - if last == pc.valid_at or last == EXECUTED: + if last == EXECUTED: + continue + if last == pc.valid_at: + # Already alerted; a still-pending loss gets one reminder near its deadline. + if at_risk is not None and in_window and _read_int(reminder_key) != pc.valid_at: + diff.loss(_loss_body(snapshot, pc, operation_label, at_risk, reminder=True)) + diff.defer(_write, reminder_key, pc.valid_at) continue - new_pending.append((pc, operation_label)) diff.defer(_write, cache_key, pc.valid_at) + if at_risk is not None: + diff.loss(_loss_body(snapshot, pc, operation_label, at_risk, reminder=False)) + if in_window: + diff.defer(_write, reminder_key, pc.valid_at) + continue + new_pending.append((pc, operation_label)) + if severity == AlertSeverity.HIGH: + group_severity = AlertSeverity.HIGH - # Group all newly-submitted operations for this vault into one section. - _alert_pending_new(snapshot, new_pending, diff) + # Group all other newly-submitted operations for this vault into one section. + _alert_pending_new(snapshot, new_pending, diff, group_severity) # Detect resolved entries: anything in last-run's index that isn't in the # current pending list. Reading a function-name key here is safe despite the @@ -444,7 +534,10 @@ def _diff_set(snapshot: V2GovernanceSnapshot, set_name: str, current: List[str], diff.defer(_write, cache_key, ",".join(sorted(current_set))) -def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: +def diff_and_alert( + snapshot: V2GovernanceSnapshot, + adapter_states: List[AdapterTimelockState] | None = None, +) -> None: """Diff a vault's snapshot against persisted state and emit one grouped alert. Every diff category (pending, owner/curator, sentinels/allocators/adapters) @@ -457,6 +550,9 @@ def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: than treating them as delivered. A partial send (part 1 of 2 lands, part 2 fails) therefore repeats the whole group next run — duplicates beat a governance change nobody ever sees. + + ``adapter_states`` are the on-chain adapter timelock reads; ``None`` skips the + adapter diff (e.g. when the RPC read failed) so its cache is left untouched. """ diff = VaultDiff() _diff_pending(snapshot, diff) @@ -465,6 +561,8 @@ def diff_and_alert(snapshot: V2GovernanceSnapshot) -> None: _diff_set(snapshot, "sentinels", snapshot.sentinels, diff) _diff_set(snapshot, "allocators", snapshot.allocators, diff) _diff_set(snapshot, "adapters", snapshot.adapters, diff) + for state in adapter_states or []: + adapter_timelocks_v2.diff_adapter(snapshot.chain, snapshot.total_assets, state, diff) send_vault_alerts(_vault_header(snapshot), diff.alerts, PROTOCOL) diff.commit() @@ -487,8 +585,15 @@ def main() -> None: if not vaults: continue for vault in vaults: + adapter_states: List[AdapterTimelockState] | None = None + try: + adapter_states = adapter_timelocks_v2.load_adapter_states(vault.chain, vault.market_v1_adapters) + except Exception as e: + # Still run the GraphQL diff; the adapter cache is left untouched until a read succeeds. + logger.exception("Failed to read adapter timelocks for %s on %s", vault.address, chain.name) + failures.append(f"{vault.name} adapters on {chain.name}: {type(e).__name__}: {e}") try: - diff_and_alert(vault) + diff_and_alert(vault, adapter_states) except Exception as e: logger.exception("Failed to process governance for %s on %s", vault.address, chain.name) failures.append(f"{vault.name} on {chain.name}: {type(e).__name__}: {e}") diff --git a/protocols/morpho/v2_decoders.py b/protocols/morpho/v2_decoders.py index 7a616f1f..1bd28b83 100644 --- a/protocols/morpho/v2_decoders.py +++ b/protocols/morpho/v2_decoders.py @@ -272,10 +272,8 @@ def _format_args( # noqa: PLR0911,PLR0912 """Render a decoded argument tuple per signature.""" name = _function_name(sig) - if name == "removeAdapter": - return "" - if name == "addAdapter": - return f"{_format_address(args[0])}" + if name in ("addAdapter", "removeAdapter"): + return _address_link(args[0], chain) if name == "setIsAllocator": addr, flag = args return f"{_format_address(addr)} = {bool(flag)}" @@ -370,6 +368,29 @@ def decode_submit( return f"{_function_name(sig)}({formatted})" +def decode_call(data: bytes) -> tuple[str, tuple[Any, ...]] | None: + """ABI-decode a known timelocked call into ``(function name, args)``, or None. + + Unlike ``decode_submit`` this returns raw arguments, so callers can make + severity decisions (e.g. which adapter a ``removeAdapter`` targets). + """ + if len(data) < 4: + return None + sig = SELECTOR_TO_SIG.get(_selector_hex(data)) + if sig is None: + return None + try: + args = abi_decode(_arg_types(sig), data[4:]) + except Exception: + return None + return _function_name(sig), tuple(args) + + +def selector_of(sig: str) -> bytes: + """Return the 4-byte selector for a solidity signature.""" + return bytes(Web3.keccak(text=sig)[:4]) + + def submit_data_key(data: bytes) -> str: """Stable cache key derived from Submit ``data`` (keccak hex, no 0x prefix).""" return bytes(Web3.keccak(data)).hex().lower() diff --git a/tests/test_morpho_adapter_timelocks_v2.py b/tests/test_morpho_adapter_timelocks_v2.py new file mode 100644 index 00000000..82595ab7 --- /dev/null +++ b/tests/test_morpho_adapter_timelocks_v2.py @@ -0,0 +1,231 @@ +"""Behavior tests for on-chain MorphoMarketV1AdapterV2 timelock monitoring.""" + +import unittest +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Iterator +from unittest.mock import MagicMock, patch + +from web3 import Web3 + +from protocols.morpho import adapter_timelocks_v2 as adapters +from protocols.morpho._alerts import VaultDiff +from protocols.morpho.adapter_timelocks_v2 import AdapterTimelockState, PendingBurn +from utils.alert import AlertSeverity +from utils.chains import Chain + +ADAPTER = Web3.to_checksum_address("0x" + "22" * 20) +MORPHO = Web3.to_checksum_address("0x" + "33" * 20) +MARKET_A = bytes.fromhex("ab" * 32) +MARKET_B = bytes.fromhex("cd" * 32) +MARKET_A_HEX = Web3.to_hex(MARKET_A) +EXECUTABLE_AT = 2_000_000_000 +METADATA = {"name": "WETH/USDC", "loan_symbol": "USDC", "loan_decimals": 6, "lltv": 860000000000000000} + + +def _state(burns: list[PendingBurn] | None = None, burn_timelock: int = 3 * 86400) -> AdapterTimelockState: + return AdapterTimelockState(adapter=ADAPTER, burn_timelock=burn_timelock, pending_burns=burns or []) + + +def _burn(shares: int, assets: int = 0, executable_at: int = EXECUTABLE_AT) -> PendingBurn: + return PendingBurn(market_id=MARKET_A_HEX, executable_at=executable_at, shares=shares, assets=assets) + + +class _Cache: + """In-memory stand-in for the morpho cache file.""" + + def __init__(self) -> None: + self.values: dict[str, str] = {} + + def patches(self) -> Any: + return ( + patch( + "protocols.morpho.adapter_timelocks_v2.get_last_value_for_key_from_file", + side_effect=lambda _f, key: self.values.get(key, 0), + ), + patch( + "protocols.morpho.adapter_timelocks_v2.write_last_value_to_file", + side_effect=lambda _f, key, value: self.values.__setitem__(key, str(value)), + ), + ) + + +class TestAdapterDiff(unittest.TestCase): + def setUp(self) -> None: + self.cache = _Cache() + + def _run(self, state: AdapterTimelockState, now: int = 1_000_000_000) -> VaultDiff: + diff = VaultDiff() + read, write = self.cache.patches() + with ( + read, + write, + patch("protocols.morpho.adapter_timelocks_v2.fetch_market_metadata", return_value=METADATA), + patch("protocols.morpho.adapter_timelocks_v2.datetime") as fake_datetime, + ): + fake_datetime.now.return_value = datetime.fromtimestamp(now) + adapters.diff_adapter(Chain.MAINNET, 10_000_000 * 10**6, state, diff) + diff.commit() + return diff + + def test_pending_burn_with_shares_is_critical_then_reminded_once(self) -> None: + state = _state([_burn(shares=10**12, assets=2_500_000 * 10**6)]) + + first = self._run(state, now=EXECUTABLE_AT - 3 * 86400) + quiet = self._run(state, now=EXECUTABLE_AT - 2 * 86400) + reminder = self._run(state, now=EXECUTABLE_AT - 3600) + after = self._run(state, now=EXECUTABLE_AT - 60) + + self.assertEqual(len(first.alerts), 1) + alert = first.alerts[0] + self.assertEqual(alert.severity, AlertSeverity.CRITICAL) + self.assertTrue(alert.standalone) + self.assertIn("`burnShares` on market [WETH/USDC]", alert.body) + self.assertIn("2.50M USDC (25.00% of vault)", alert.body) + self.assertIn(f"[{ADAPTER}](", alert.body) + self.assertEqual(quiet.alerts, []) + self.assertEqual(len(reminder.alerts), 1) + self.assertIn("Reminder", reminder.alerts[0].body) + self.assertEqual(after.alerts, []) + + def test_shares_worth_zero_assets_are_still_critical(self) -> None: + """Criticality follows shares, so a stale/zero valuation can't downgrade a real burn.""" + diff = self._run(_state([_burn(shares=1, assets=0)])) + + self.assertEqual(diff.alerts[0].severity, AlertSeverity.CRITICAL) + + def test_pending_burn_without_shares_is_medium(self) -> None: + diff = self._run(_state([_burn(shares=0)])) + + self.assertEqual([a.severity for a in diff.alerts], [AlertSeverity.MEDIUM]) + self.assertFalse(diff.alerts[0].standalone) + + def test_resolved_burn_reports_executed_and_clears_its_cursor(self) -> None: + self._run(_state([_burn(shares=5)]), now=EXECUTABLE_AT - 86400 * 3) + diff = self._run(_state(), now=EXECUTABLE_AT + 60) + + self.assertEqual(len(diff.alerts), 1) + self.assertEqual(diff.alerts[0].severity, AlertSeverity.LOW) + self.assertIn("was executed", diff.alerts[0].body) + # A re-submit with the same executableAt alerts again. + again = self._run(_state([_burn(shares=5)]), now=EXECUTABLE_AT - 86400 * 3) + self.assertEqual(again.alerts[0].severity, AlertSeverity.CRITICAL) + + def test_burn_timelock_seeds_silently_then_alerts_high_on_decrease(self) -> None: + seed = self._run(_state(burn_timelock=3 * 86400)) + increase = self._run(_state(burn_timelock=7 * 86400)) + decrease = self._run(_state(burn_timelock=0)) + + self.assertEqual(seed.alerts, []) + self.assertEqual(increase.alerts, []) + self.assertEqual(len(decrease.alerts), 1) + self.assertEqual(decrease.alerts[0].severity, AlertSeverity.HIGH) + self.assertIn("7d → 0s", decrease.alerts[0].body) + + +class _FakeBatch: + def __init__(self) -> None: + self.calls: list[Any] = [] + + def add(self, call: Any) -> None: + self.calls.append(call) + + +class _FakeClient: + """Minimal batching client whose contract calls only resolve inside a batch context. + + Mirrors web3's batching: a ``.call()`` made before entering ``batch_requests()`` + would run eagerly, so the fake rejects it. Only the listed functions exist, so + a call to ``expectedSupplyAssets`` (IRM-dependent, may revert) or ``get_logs`` + fails the test. + """ + + def __init__(self, values: dict[tuple[str, Any], Any]) -> None: + self.values = values + self.in_batch = False + names = ("marketIdsLength", "timelock", "morpho", "marketIds", "executableAt", "supplyShares") + functions = MagicMock(spec=list(names)) + for name in names: + setattr(functions, name, self._bound(name)) + blue_functions = MagicMock(spec=["market"]) + blue_functions.market = self._bound("market") + adapter = MagicMock(functions=functions, address=ADAPTER) + blue = MagicMock(functions=blue_functions) + self.eth = MagicMock(spec=["contract"]) + self.eth.contract.side_effect = lambda address, abi: adapter if address == ADAPTER else blue + + def _bound(self, name: str) -> Any: + def bind(*args: Any) -> Any: + def call(**_kwargs: Any) -> Any: + if not self.in_batch: + raise AssertionError(f"{name}.call() ran outside a batch context") + return self.values[(name, args[0] if args else None)] + + return MagicMock(call=call) + + return bind + + @contextmanager + def batch_requests(self) -> Iterator[_FakeBatch]: + self.in_batch = True + try: + yield _FakeBatch() + finally: + self.in_batch = False + + def execute_batch(self, batch: _FakeBatch) -> list[Any]: + return batch.calls + + +def _values(held: list[bytes], executable_at: dict[bytes, int], adapter_shares: dict[bytes, int]) -> dict: + values: dict[tuple[str, Any], Any] = { + ("marketIdsLength", None): len(held), + ("timelock", adapters.BURN_SHARES_SELECTOR): 259200, + ("morpho", None): MORPHO, + } + for index, market in enumerate(held): + values[("marketIds", index)] = market + for market in (MARKET_A, MARKET_B): + values[("executableAt", adapters.burn_shares_calldata(market))] = executable_at.get(market, 0) + values[("supplyShares", market)] = adapter_shares.get(market, 0) + values[("market", market)] = (2 * 10**6, 2 * 10**12, 0, 0, 0, 0) + return values + + +class TestLoadAdapterState(unittest.TestCase): + def setUp(self) -> None: + self.cache = _Cache() + + def _load(self, client: _FakeClient) -> AdapterTimelockState: + read, write = self.cache.patches() + with read, write: + return adapters.load_adapter_state(client, ADAPTER, block_number=1) + + def test_reads_canonical_pending_burns_and_values_only_those(self) -> None: + client = _FakeClient( + _values([MARKET_A, MARKET_B], {MARKET_A: EXECUTABLE_AT}, {MARKET_A: 2 * 10**12, MARKET_B: 2 * 10**12}) + ) + + state = self._load(client) + + self.assertEqual(state.burn_timelock, 259200) + self.assertEqual(state.pending_burns, [PendingBurn(MARKET_A_HEX, EXECUTABLE_AT, 2 * 10**12, 2_000_000)]) + + def test_no_pending_burn_needs_no_valuation_calls(self) -> None: + values = _values([MARKET_A], {}, {}) + del values[("morpho", None)] + client = _FakeClient(values) + + state = self._load(client) + + self.assertEqual(state.pending_burns, []) + + def test_burn_shares_calldata_matches_abi_encoding(self) -> None: + self.assertEqual( + adapters.burn_shares_calldata(MARKET_A), + bytes(Web3.keccak(text="burnShares(bytes32)")[:4]) + MARKET_A, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_morpho_governance.py b/tests/test_morpho_governance.py index 212deafe..eb5941b5 100644 --- a/tests/test_morpho_governance.py +++ b/tests/test_morpho_governance.py @@ -2,11 +2,12 @@ import unittest from typing import Any -from unittest.mock import patch +from unittest.mock import MagicMock, patch from protocols.morpho import governance from protocols.morpho._alerts import VaultDiff from protocols.morpho.governance import MarketGovernanceState +from utils.alert import AlertSeverity from utils.chains import Chain VAULT = "0x" + "11" * 20 @@ -69,6 +70,171 @@ def test_previously_alerted_market_removal_is_not_repeated(self) -> None: write.assert_not_called() +METADATA = {"name": "WETH/USDC", "loan_symbol": "USDC", "loan_decimals": 6, "lltv": 860000000000000000} +REMOVAL_STATE = dict(pending_cap=0, pending_cap_timestamp=0, current_cap=0) + + +class _Cache: + """In-memory stand-in for the morpho cache file, keyed like the real helpers.""" + + def __init__(self) -> None: + self.values: dict[tuple[str, str, str], int] = {} + + def get(self, vault: str, market: str, value_type: str) -> int: + return self.values.get((vault, market, value_type), 0) + + def write(self, vault: str, market: str, value_type: str, value: int) -> None: + self.values[(vault, market, value_type)] = value + + +class TestMorphoV1MarketRemovalLoss(unittest.TestCase): + def _run(self, state: MarketGovernanceState, cache: _Cache, now: int) -> VaultDiff: + diff = VaultDiff() + with ( + patch("protocols.morpho.governance.get_last_executed_morpho_from_file", side_effect=cache.get), + patch("protocols.morpho.governance.write_last_executed_morpho_to_file", side_effect=cache.write), + patch("protocols.morpho.governance.fetch_market_metadata", return_value=METADATA), + patch("protocols.morpho.governance.fetch_market_info", return_value=("WETH/USDC (86.00%)", 6)), + patch("protocols.morpho.governance.datetime") as fake_datetime, + patch("protocols.morpho._shared.datetime") as shared_datetime, + ): + from datetime import datetime as real_datetime + + for fake in (fake_datetime, shared_datetime): + fake.now.return_value = real_datetime.fromtimestamp(now) + fake.fromtimestamp.side_effect = real_datetime.fromtimestamp + governance.check_market_governance_state("Example", state, Chain.MAINNET, diff) + diff.commit() + return diff + + def test_removal_with_supply_is_a_standalone_critical_loss_alert(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, + **REMOVAL_STATE, + removable_at=removable_at, + supply_shares=10**12, + supply_assets=1_500_000 * 10**6, + vault_total_assets=10_000_000 * 10**6, + ) + diff = self._run(state, _Cache(), now=removable_at - 3 * 86400) + + self.assertEqual(len(diff.alerts), 1) + alert = diff.alerts[0] + self.assertEqual(alert.severity, AlertSeverity.CRITICAL) + self.assertTrue(alert.standalone) + self.assertIn("funds still supplied", alert.body) + self.assertIn("1.50M USDC (15.00% of vault)", alert.body) + self.assertIn("revokePendingMarketRemoval", alert.body) + self.assertIn("(3 days)", alert.body) + + def test_any_non_zero_supply_is_critical_and_dust_shows_raw_units(self) -> None: + state = _state("0x" + "ab" * 32, **REMOVAL_STATE, removable_at=2_000_000_000, supply_shares=1, supply_assets=1) + diff = self._run(state, _Cache(), now=1_000_000_000) + + self.assertEqual(diff.alerts[0].severity, AlertSeverity.CRITICAL) + self.assertIn("1 raw units USDC", diff.alerts[0].body) + + def test_removal_of_empty_market_stays_medium(self) -> None: + state = _state("0x" + "ab" * 32, **REMOVAL_STATE, removable_at=2_000_000_000) + diff = self._run(state, _Cache(), now=1_000_000_000) + + self.assertEqual(len(diff.alerts), 1) + self.assertEqual(diff.alerts[0].severity, AlertSeverity.MEDIUM) + self.assertFalse(diff.alerts[0].standalone) + self.assertIn("no vault supply", diff.alerts[0].body) + + def test_still_supplied_removal_is_reminded_once_inside_the_window(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at, supply_shares=5, supply_assets=5 * 10**6 + ) + cache = _Cache() + + first = self._run(state, cache, now=removable_at - 3 * 86400) + quiet = self._run(state, cache, now=removable_at - 2 * 86400) + reminder = self._run(state, cache, now=removable_at - 3600) + after = self._run(state, cache, now=removable_at - 60) + + self.assertEqual(len(first.alerts), 1) + self.assertEqual(quiet.alerts, []) + self.assertEqual(len(reminder.alerts), 1) + self.assertEqual(reminder.alerts[0].severity, AlertSeverity.CRITICAL) + self.assertIn("Reminder", reminder.alerts[0].body) + self.assertEqual(after.alerts, []) + + def test_first_alert_inside_the_window_suppresses_the_reminder(self) -> None: + removable_at = 2_000_000_000 + state = _state( + "0x" + "ab" * 32, **REMOVAL_STATE, removable_at=removable_at, supply_shares=5, supply_assets=5 * 10**6 + ) + cache = _Cache() + + first = self._run(state, cache, now=removable_at - 3600) + second = self._run(state, cache, now=removable_at - 60) + + self.assertEqual(len(first.alerts), 1) + self.assertEqual(second.alerts, []) + + +class _V1Batch: + def __init__(self) -> None: + self.calls: list[Any] = [] + + def add(self, call: Any) -> None: + self.calls.append(call) + + +class TestMorphoV1VaultReads(unittest.TestCase): + def test_reverting_total_assets_only_drops_the_vault_share(self) -> None: + """totalAssets() values every withdraw-queue market and can revert on a broken IRM; + the removal check must still see supply and stay CRITICAL.""" + from contextlib import contextmanager + + market_id = bytes.fromhex("ab" * 32) + vault = MagicMock(address=VAULT) + vault.functions.MORPHO.return_value.call.return_value = "0x" + "33" * 20 + vault.functions.totalAssets.return_value.call.side_effect = ValueError("execution reverted") + vault.functions.pendingCap.return_value.call.return_value = (0, 0) + vault.functions.config.return_value.call.return_value = (0, True, 2_000_000_000) + blue = MagicMock() + blue.functions.position.return_value.call.return_value = (10**12, 0, 0) + blue.functions.market.return_value.call.return_value = (2 * 10**6, 2 * 10**12, 0, 0, 0, 0) + client = MagicMock() + client.eth.contract.return_value = blue + + @contextmanager + def batch_requests() -> Any: + yield _V1Batch() + + client.batch_requests.side_effect = batch_requests + client.execute_batch.side_effect = lambda batch: batch.calls + + [state] = governance._load_market_governance_states(vault, [market_id], client, block_number=1) + + self.assertIsNone(state.vault_total_assets) + self.assertEqual( + (state.supply_shares, state.supply_assets, state.removable_at), (10**12, 1_000_000, 2_000_000_000) + ) + self.assertIn("1.00 USDC", governance.format_amount_at_risk(state.supply_assets, 6, "USDC", None)) + self.assertNotIn("of vault", governance.format_amount_at_risk(state.supply_assets, 6, "USDC", None)) + + +class TestMorphoV1Timelock(unittest.TestCase): + def test_timelock_decrease_message_shows_old_and_new_values(self) -> None: + message = governance.format_timelock_decrease(3 * 86400, 0, 2_000_000_000) + + self.assertIn("Timelock decrease: 3d → 0s", message) + self.assertIn("Executable at:", message) + + def test_to_assets_down_matches_shares_math(self) -> None: + # Virtual offsets: 1e6 shares for 1 asset on an empty market. + self.assertEqual(governance.to_assets_down(10**6, 0, 0), 1) + self.assertEqual(governance.to_assets_down(2 * 10**12, 2 * 10**6, 2 * 10**12), 2_000_000) + # Rounds down: 3 shares of a 10-asset / 1e7-share market are worth less than one unit. + self.assertEqual(governance.to_assets_down(3, 10, 10**7), 0) + + class TestMorphoV1GovernanceGrouping(unittest.TestCase): def test_findings_for_one_vault_collapse_into_a_single_message(self) -> None: """Two new markets on one vault produce one message, not two. diff --git a/tests/test_morpho_v2_decoders.py b/tests/test_morpho_v2_decoders.py index 2805604c..5f50749a 100644 --- a/tests/test_morpho_v2_decoders.py +++ b/tests/test_morpho_v2_decoders.py @@ -68,7 +68,7 @@ def test_remove_adapter(self): data = _build("removeAdapter(address)", ["address"], [A2]) self.assertEqual( decode_submit(data), - "removeAdapter()", + f"removeAdapter({Web3.to_checksum_address(A2)})", ) def test_set_is_allocator_grant(self): diff --git a/tests/test_morpho_v2_governance.py b/tests/test_morpho_v2_governance.py index 3d40f64b..25d3c6af 100644 --- a/tests/test_morpho_v2_governance.py +++ b/tests/test_morpho_v2_governance.py @@ -6,9 +6,11 @@ from web3 import Web3 from protocols.morpho import governance_v2 +from protocols.morpho._alerts import LOSS_PREFIX from protocols.morpho._shared import MorphoV2MonitoringError from protocols.morpho.governance_v2 import PendingConfig, V2GovernanceSnapshot from protocols.morpho.v2_decoders import submit_data_key +from utils.alert import AlertSeverity from utils.chains import Chain from utils.telegram import MAX_MESSAGE_LENGTH @@ -381,3 +383,140 @@ def test_fetch_fails_if_api_omits_configured_vaults(self) -> None: if __name__ == "__main__": unittest.main() + + +ADAPTER = Web3.to_checksum_address("0x" + "22" * 20) + + +def _loss_snapshot(pending_configs: list[PendingConfig], adapter_assets: int) -> V2GovernanceSnapshot: + snapshot = _snapshot(pending_configs) + snapshot.asset_symbol = "USDC" + snapshot.asset_decimals = 6 + snapshot.total_assets = 4_000_000 * 10**6 + snapshot.adapter_assets = {ADAPTER: adapter_assets} + snapshot.adapter_types = {ADAPTER: "MorphoMarketV1"} + return snapshot + + +def _pc(sig: str, types: list[str], values: list[Any], valid_at: int = 2_000_000_000) -> PendingConfig: + return PendingConfig( + valid_at=valid_at, + function_name=sig.split("(", 1)[0], + data=_build(sig, types, values), + tx_hash="0x" + "12" * 32, + ) + + +class TestMorphoV2LossClassification(unittest.TestCase): + def test_remove_adapter_holding_assets_is_critical_with_amount(self) -> None: + pc = _pc("removeAdapter(address)", ["address"], [ADAPTER]) + severity, at_risk = governance_v2.classify_pending(_loss_snapshot([pc], 1), pc) + self.assertEqual((severity, at_risk), (AlertSeverity.CRITICAL, 1)) + + def test_remove_empty_adapter_stays_medium(self) -> None: + pc = _pc("removeAdapter(address)", ["address"], [ADAPTER]) + self.assertEqual(governance_v2.classify_pending(_loss_snapshot([pc], 0), pc), (AlertSeverity.MEDIUM, None)) + + def test_decrease_remove_adapter_timelock_with_held_assets_is_critical(self) -> None: + pc = _pc("decreaseTimelock(bytes4,uint256)", ["bytes4", "uint256"], [_selector("removeAdapter(address)"), 0]) + self.assertEqual( + governance_v2.classify_pending(_loss_snapshot([pc], 5), pc), + (AlertSeverity.CRITICAL, 5), + ) + + def test_other_timelock_decrease_and_abdicate_are_high(self) -> None: + decrease = _pc("decreaseTimelock(bytes4,uint256)", ["bytes4", "uint256"], [_selector("addAdapter(address)"), 0]) + abdicate = _pc("abdicate(bytes4)", ["bytes4"], [_selector("setIsAllocator(address,bool)")]) + snapshot = _loss_snapshot([], 5) + self.assertEqual(governance_v2.classify_pending(snapshot, decrease), (AlertSeverity.HIGH, None)) + self.assertEqual(governance_v2.classify_pending(snapshot, abdicate), (AlertSeverity.HIGH, None)) + + +class TestMorphoV2LossDelivery(unittest.TestCase): + def _run(self, snapshot: V2GovernanceSnapshot, state: dict[str, str], now: int) -> list[Any]: + from datetime import datetime as real_datetime + + sent: list[Any] = [] + with ( + patch( + "protocols.morpho.governance_v2.get_last_value_for_key_from_file", + side_effect=lambda _f, key: state.get(key, 0), + ), + patch( + "protocols.morpho.governance_v2.write_last_value_to_file", + side_effect=lambda _f, key, value: state.__setitem__(key, str(value)), + ), + patch("protocols.morpho._alerts.send_alert", side_effect=sent.append), + patch("protocols.morpho.governance_v2.datetime") as fake_datetime, + ): + fake_datetime.now.return_value = real_datetime.fromtimestamp(now) + governance_v2.diff_and_alert(snapshot) + return sent + + def test_loss_operation_is_sent_alone_and_reminded_once_near_deadline(self) -> None: + valid_at = 2_000_000_000 + remove = _pc("removeAdapter(address)", ["address"], [ADAPTER], valid_at) + add = _pc("addAdapter(address)", ["address"], [A1], valid_at) + snapshot = _loss_snapshot([remove, add], 1_000_000 * 10**6) + state: dict[str, str] = {} + + first = self._run(snapshot, state, now=valid_at - 3 * 86400) + quiet = self._run(snapshot, state, now=valid_at - 2 * 86400) + reminder = self._run(snapshot, state, now=valid_at - 3600) + after = self._run(snapshot, state, now=valid_at - 60) + + # The loss goes out on its own, ahead of the grouped routine message. + self.assertEqual(len(first), 2) + self.assertEqual(first[0].severity, AlertSeverity.CRITICAL) + self.assertTrue(first[0].message.startswith(LOSS_PREFIX)) + self.assertIn(f"removeAdapter([{ADAPTER}]", first[0].message) + self.assertIn("1.00M USDC (25.00% of vault)", first[0].message) + self.assertEqual(first[1].severity, AlertSeverity.MEDIUM) + self.assertNotIn("removeAdapter", first[1].message) + self.assertEqual(quiet, []) + self.assertEqual(len(reminder), 1) + self.assertIn("Reminder", reminder[0].message) + self.assertEqual(after, []) + + def test_high_operation_raises_the_grouped_severity(self) -> None: + abdicate = _pc("abdicate(bytes4)", ["bytes4"], [_selector("setIsAllocator(address,bool)")]) + sent = self._run(_loss_snapshot([abdicate], 0), {}, now=1_000_000_000) + + self.assertEqual(len(sent), 1) + self.assertEqual(sent[0].severity, AlertSeverity.HIGH) + self.assertNotIn("POTENTIAL VAULT LOSS", sent[0].message) + + +class TestMorphoV2GovernanceMainAdapters(unittest.TestCase): + def test_adapter_read_failure_still_runs_graphql_diff_and_fails_the_run(self) -> None: + snapshot = _loss_snapshot([], 1) + diffed: list[Any] = [] + + with ( + patch( + "protocols.morpho.governance_v2.fetch_governance_snapshots", + return_value={Chain.MAINNET: [snapshot]}, + ), + patch( + "protocols.morpho.governance_v2.adapter_timelocks_v2.load_adapter_states", + side_effect=ValueError("No providers found for chain MAINNET"), + ) as load, + patch( + "protocols.morpho.governance_v2.diff_and_alert", + side_effect=lambda vault, states: diffed.append((vault, states)), + ), + self.assertRaisesRegex(MorphoV2MonitoringError, "adapters on MAINNET: ValueError"), + ): + governance_v2.main() + + load.assert_called_once_with(Chain.MAINNET, [ADAPTER]) + # The GraphQL diff still ran, with the adapter diff skipped. + self.assertEqual(diffed, [(snapshot, None)]) + + +class TestMorphoV2GovernanceQuery(unittest.TestCase): + def test_configured_vaults_fit_in_one_page(self) -> None: + """The query fetches a single page; outgrowing it would fail every run.""" + from protocols.morpho.config import VAULTS_V2_BY_CHAIN, iter_vaults + + self.assertLessEqual(len(list(iter_vaults(VAULTS_V2_BY_CHAIN))), governance_v2.GOVERNANCE_PAGE_SIZE) diff --git a/tests/test_web3_wrapper.py b/tests/test_web3_wrapper.py index bcb40601..8b8ead3d 100644 --- a/tests/test_web3_wrapper.py +++ b/tests/test_web3_wrapper.py @@ -136,6 +136,7 @@ def _env_without_providers(self) -> dict[str, str]: def test_production_mapping_has_hyperevm_public_rpc(self): self.assertEqual(PUBLIC_RPC_URLS[Chain.HYPEREVM.chain_id], "https://rpc.hyperliquid.xyz/evm") + self.assertEqual(PUBLIC_RPC_URLS[Chain.ROBINHOOD.chain_id], "https://rpc.mainnet.chain.robinhood.com") def test_hyperevm_falls_back_to_public_rpc(self): with ( diff --git a/utils/chains.py b/utils/chains.py index c6469125..029248ef 100644 --- a/utils/chains.py +++ b/utils/chains.py @@ -15,6 +15,7 @@ # Setting any of those env vars disables the public fallback for that chain. PUBLIC_RPC_URLS: dict[int, str] = { 999: "https://rpc.hyperliquid.xyz/evm", + 4663: "https://rpc.mainnet.chain.robinhood.com", }