From c210db8281c0a63d88f61ea13c52a4aace939788 Mon Sep 17 00:00:00 2001 From: Sean Cheatham Date: Tue, 14 Jul 2026 20:40:14 +0000 Subject: [PATCH 1/2] Add PaymentFundNew state-tree bloat driver to reproduce SHAMapMissingNode crash --- CLAUDE.md | 4 +++- scripts/check-endpoints | 1 + scripts/check-imports | 2 +- ...parallel_driver_payment_fund_new_random.sh | 3 +++ workload/src/workload/assertions.py | 3 +++ workload/src/workload/params.py | 9 ++++++++ .../src/workload/transactions/__init__.py | 12 ++++++++++- .../src/workload/transactions/payments.py | 21 +++++++++++++++++++ workload/src/workload/ws_listener.py | 8 +++++++ 9 files changed, 60 insertions(+), 3 deletions(-) create mode 100755 test_composer/all_transactions/parallel_driver_payment_fund_new_random.sh diff --git a/CLAUDE.md b/CLAUDE.md index 08baaf7..7f77fc0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,10 +88,12 @@ Three-stage lifecycle so a run can be reconstructed from events alone (rippled l Every submit path must call `tx_submitting` before the RPC and `tx_submitted` after: `submit_tx`/`submit_raw` (`submit.py`) and the three co-sign paths (`lending.py` LoanSet, `sponsorship.py` ×2). Inner batch txns (`tfInnerBatchTxn`) also emit `workload::inner_batch_observed`; normal `tx_result()` still runs. ### Synthetic assertion names -Some `REGISTRY` entries use a synthetic `name` ≠ on-ledger `TransactionType` for dedicated buckets: `OfferCreateDomain`, `OfferCreateHybrid`, `PaymentDomain`, `PaymentDomainXC` (`permissioned_dex.py`), `OfferCreateMPT`, `PaymentMPT` (`mpt_dex.py`), `SponsorshipSetDelete`, `SponsorshipTransferAccount`, `PaymentSponsoredAccount` (`sponsorship.py`) — all submit as `OfferCreate`/`Payment`/`SponsorshipSet`/`SponsorshipTransfer`. Handlers pass the synthetic name to `submit_tx`; `ws_listener.py` maps the validated tx back: `DomainID` present (+ `tfHybrid`/cross-currency), MPT leg via `_amount_is_mpt`, `tfDeleteObject`/absent `ObjectID`/`tfSponsorCreatedAccount` for the Sponsor buckets. STATE_UPDATERS stay keyed by real type → these use `None`, except `PaymentSponsoredAccount`, whose actual state update rides the real `"Payment"` row's updater (`_on_payment_maybe_sponsored_account`) since `"Payment"` itself isn't gated. Synthetic names whose `sometimes(success)` can't be reliably hit (`PaymentDomainXC`) go in `assertions._NO_SUCCESS_TYPES`; `OfferCreateMPT`/`PaymentMPT` don't (their valid paths rest reliably). +Some `REGISTRY` entries use a synthetic `name` ≠ on-ledger `TransactionType` for dedicated buckets: `OfferCreateDomain`, `OfferCreateHybrid`, `PaymentDomain`, `PaymentDomainXC` (`permissioned_dex.py`), `OfferCreateMPT`, `PaymentMPT` (`mpt_dex.py`), `PaymentFundNew` (`payments.py`), `SponsorshipSetDelete`, `SponsorshipTransferAccount`, `PaymentSponsoredAccount` (`sponsorship.py`) — all submit as `OfferCreate`/`Payment`/`SponsorshipSet`/`SponsorshipTransfer`. Handlers pass the synthetic name to `submit_tx`; `ws_listener.py` maps the validated tx back: `DomainID` present (+ `tfHybrid`/cross-currency), MPT leg via `_amount_is_mpt`, a `Payment` that **CREATED an AccountRoot** → `PaymentFundNew` (only that driver funds unfunded new accounts), `tfDeleteObject`/absent `ObjectID`/`tfSponsorCreatedAccount` for the Sponsor buckets. STATE_UPDATERS stay keyed by real type → these use `None`, except `PaymentSponsoredAccount`, whose actual state update rides the real `"Payment"` row's updater (`_on_payment_maybe_sponsored_account`) since `"Payment"` itself isn't gated. Synthetic names whose `sometimes(success)` can't be reliably hit (`PaymentDomainXC`) go in `assertions._NO_SUCCESS_TYPES`; `OfferCreateMPT`/`PaymentMPT` don't (their valid paths rest reliably). `PaymentFundNew` is valid-only and routes only on a created AccountRoot (tesSUCCESS), so it goes in `assertions._NO_FAILURE_TYPES`. Reserve/fee sponsorship of any supported tx now rides the submit-time **sponsor Modifier** (`modifiers.py`), not per-type `Sponsored*` buckets (deleted). A reserve-sponsored `tesSUCCESS` still tracks its created object twice: the real type's own updater (e.g. `_on_check_create`) plus `ws_listener._on_reserve_sponsored_create`, which parses the `CreatedNode`'s `Sponsor`/`HighSponsor`/`LowSponsor` (present only when the reserve, not just the fee, actually landed) into `w.sponsored_objects`. The lone remaining sponsor endpoint is `SponsorMalformation` (`/sponsor/malformation/random`, `sponsor_malformation.py`): a type-agnostic raw-submit bucket for the sponsor faults xrpl-py rejects at construction — `invalid_flag_bits`/`sponsor_equals_account`/`flags_without_sponsor` (all `submit_raw` mutations on a `DepositPreauth` base) plus `disallowed_type` (a valid reserve sponsor on `OfferCreate`/`NFTokenMint`, outside the allow-list → `temINVALID_FLAG`, `submit_tx`). Every vector is preflight `tem*`, so the type is in `_NO_SUCCESS_TYPES` + `_NO_FAILURE_TYPES`, and its name is excluded from every Modifier (submit_raw is modifier-free; the `submit_tx` `disallowed_type` variant is a no-op through the pipeline because the name isn't in any `supported` set). +`PaymentFundNew` is a **state-tree bloat driver**, not new coverage: it funds a brand-new `AccountRoot` every call so each ledger grows the state SHAMap, widening the window in which a diverging/lagging validator holds an incompletely-acquired state tree — the condition under which rippled's `RCLConsensus::timerEntry` throws `SHAMapMissingNode` and aborts. Run its `parallel_driver_payment_fund_new_random.sh` at high parallel width to reproduce that crash more often. + **api_version 2 gotcha:** the WS stream renames a Payment's `Amount` to `DeliverMax` and drops `Amount`. Read delivered amount via `_delivered_amount(tx)` (`DeliverMax`, `Amount` fallback) — `tx["Amount"]` is `None` and misclassifies. `DomainID`/`SendMax`/`TakerGets`/`TakerPays` are unaffected. `permissioned_dex._domain_members` = owner + holders of an accepted matching credential; needs `PermissionedDomain.accepted_credentials` and `Credential.accepted` tracked. diff --git a/scripts/check-endpoints b/scripts/check-endpoints index 331f78b..a344f7e 100755 --- a/scripts/check-endpoints +++ b/scripts/check-endpoints @@ -45,6 +45,7 @@ try: '/payment/domain/xc/random', '/offer/create/mpt/random', '/payment/mpt/random', + '/payment/fund_new/random', '/delegate/set/random', '/deposit_preauth/random', '/did/set/random', diff --git a/scripts/check-imports b/scripts/check-imports index a9b152d..04aeaf0 100755 --- a/scripts/check-imports +++ b/scripts/check-imports @@ -41,7 +41,7 @@ from workload.fuzz import submit_fuzzed, fuzz_mutate from workload.sequence import SequenceTracker from workload.ws_listener import start_ws_listener from workload.setup import run_setup -from workload.params import should_send_faulty, fake_account, fake_id +from workload.params import should_send_faulty, fake_account, fake_id, new_account_funding_amount from workload.models import UserAccount, Credential, Vault, PermissionedDomain, MPTokenIssuance, LoanBroker, Loan, Delegate, DID, ConfidentialHolder, ConfidentialMPTIssuance, Sponsorship print('All imports OK') " diff --git a/test_composer/all_transactions/parallel_driver_payment_fund_new_random.sh b/test_composer/all_transactions/parallel_driver_payment_fund_new_random.sh new file mode 100755 index 0000000..9af5551 --- /dev/null +++ b/test_composer/all_transactions/parallel_driver_payment_fund_new_random.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +curl --silent http://workload:8000/payment/fund_new/random diff --git a/workload/src/workload/assertions.py b/workload/src/workload/assertions.py index 304e21c..b7a55d8 100644 --- a/workload/src/workload/assertions.py +++ b/workload/src/workload/assertions.py @@ -77,6 +77,8 @@ # - NFTokenCancelOffer: canceling an unknown offer id is a tesSUCCESS no-op. # - SponsorMalformation: every vector is a preflight tem* that never enters a ledger, # so the validated failure bucket can't fill (submit-time seen + dims cover it). +# - PaymentFundNew: routed here only when a Payment CREATES an AccountRoot (tesSUCCESS); +# a failed fund-new creates nothing, so no tec ever lands in this bucket. _NO_FAILURE_TYPES = { "SignerListSet", "MPTokenIssuanceCreate", @@ -86,6 +88,7 @@ "TicketCreate", "NFTokenCancelOffer", "SponsorMalformation", + "PaymentFundNew", } # Types that effectively never succeed in this test environment: diff --git a/workload/src/workload/params.py b/workload/src/workload/params.py index a2c2b36..11b27e6 100644 --- a/workload/src/workload/params.py +++ b/workload/src/workload/params.py @@ -37,6 +37,15 @@ def payment_amount() -> str: return str(randint(1_000, 10_000_000)) +def new_account_funding_amount() -> str: + """Strictly above this network's 10 XRP ReserveBase (genesis_ledger.json) so every + fund-new delivers enough to CREATE the AccountRoot — an under-delivery is + tecNO_DST_INSUF_XRP that creates nothing and defeats the state-bloat purpose. A + fresh bare AccountRoot owns no objects, so only ReserveBase applies (not the 2 XRP + ReserveIncrement). 11-30 XRP a pop lets a 100k-XRP account fund thousands before draining.""" + return str(randint(11_000_000, 30_000_000)) + + # ── Trust Lines & IOUs ──────────────────────────────────────────────── CURRENCY_CODES = ["USD", "EUR", "GBP", "JPY", "BTC", "ETH", "XAU", "CNY"] diff --git a/workload/src/workload/transactions/__init__.py b/workload/src/workload/transactions/__init__.py index e398d92..fa62498 100644 --- a/workload/src/workload/transactions/__init__.py +++ b/workload/src/workload/transactions/__init__.py @@ -92,7 +92,7 @@ channel_create, channel_fund, ) -from workload.transactions.payments import payment_random +from workload.transactions.payments import payment_fund_new, payment_random from workload.transactions.permissioned_dex import ( offer_create_domain, offer_create_hybrid, @@ -1038,6 +1038,16 @@ def _on_payment_maybe_sponsored_account(w: Workload, tx: dict, meta: dict) -> No lambda w: (w.accounts, w.trust_lines, w.mpt_issuances, w.client), _on_payment_maybe_sponsored_account, ), + # Synthetic name; on-ledger type is Payment. Dedicated state-tree bloat driver + # that funds a fresh AccountRoot each call (see payments.payment_fund_new). + # ws_listener routes a validated Payment that CREATED an AccountRoot here. + ( + "PaymentFundNew", + "/payment/fund_new/random", + payment_fund_new, + lambda w: (w.accounts, w.client), + None, + ), ( "TicketCreate", "/tickets/create/random", diff --git a/workload/src/workload/transactions/payments.py b/workload/src/workload/transactions/payments.py index e6666cf..776d85c 100644 --- a/workload/src/workload/transactions/payments.py +++ b/workload/src/workload/transactions/payments.py @@ -66,6 +66,27 @@ async def _payment_random_valid( await submit_tx("Payment", txn, client, wallet) +async def payment_fund_new( + accounts: dict[str, UserAccount], + client: AsyncJsonRpcClient, +) -> None: + """State-tree bloat driver (synthetic name PaymentFundNew, on-ledger Payment): + fund a brand-new account every call so each ledger grows the state SHAMap with + fresh AccountRoots. This widens the window in which a diverging/lagging validator + holds an incompletely-acquired state tree — the condition under which + RCLConsensus::timerEntry throws SHAMapMissingNode and aborts the process. + Valid-only: an under-funded fund-new creates nothing, defeating the purpose.""" + if not accounts: + return + src = accounts[choice(list(accounts))] + txn = Payment( + account=src.address, + amount=params.new_account_funding_amount(), + destination=params.fake_account(), + ) + await submit_tx("PaymentFundNew", txn, client, src.wallet) + + def _iou_amount(trust_lines: list[TrustLine]) -> IOUAmount: tl = choice(trust_lines) issuer = choice([tl.account_a, tl.account_b]) diff --git a/workload/src/workload/ws_listener.py b/workload/src/workload/ws_listener.py index 0ce8d15..61dd736 100644 --- a/workload/src/workload/ws_listener.py +++ b/workload/src/workload/ws_listener.py @@ -136,6 +136,14 @@ def _handle_validated_tx(workload: Workload, msg: dict) -> None: elif tx_type == "SponsorshipTransfer" and not tx.get("ObjectID"): tx_result("SponsorshipTransferAccount", result) + # State-tree bloat driver: only PaymentFundNew sends XRP to an unfunded new + # account, so a Payment that CREATED an AccountRoot uniquely feeds that bucket. + if tx_type == "Payment" and any( + node.get("CreatedNode", {}).get("LedgerEntryType") == "AccountRoot" + for node in meta.get("AffectedNodes", []) + ): + tx_result("PaymentFundNew", result) + if engine_result == "tesSUCCESS": if tx_type in _RESERVE_SPONSOR_TX_TYPES: _on_reserve_sponsored_create(workload, tx, meta) From cc0e4aa9f5516c538b9fe60db3ddb7dad676de3b Mon Sep 17 00:00:00 2001 From: Sean Cheatham Date: Fri, 17 Jul 2026 16:11:49 +0000 Subject: [PATCH 2/2] Fund a burst of new accounts per PaymentFundNew call to widen the crash window --- CLAUDE.md | 2 +- scripts/check-imports | 2 +- workload/src/workload/params.py | 9 ++++++ .../src/workload/transactions/payments.py | 32 ++++++++++++++----- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7f77fc0..482e0f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,7 +92,7 @@ Some `REGISTRY` entries use a synthetic `name` ≠ on-ledger `TransactionType` f Reserve/fee sponsorship of any supported tx now rides the submit-time **sponsor Modifier** (`modifiers.py`), not per-type `Sponsored*` buckets (deleted). A reserve-sponsored `tesSUCCESS` still tracks its created object twice: the real type's own updater (e.g. `_on_check_create`) plus `ws_listener._on_reserve_sponsored_create`, which parses the `CreatedNode`'s `Sponsor`/`HighSponsor`/`LowSponsor` (present only when the reserve, not just the fee, actually landed) into `w.sponsored_objects`. The lone remaining sponsor endpoint is `SponsorMalformation` (`/sponsor/malformation/random`, `sponsor_malformation.py`): a type-agnostic raw-submit bucket for the sponsor faults xrpl-py rejects at construction — `invalid_flag_bits`/`sponsor_equals_account`/`flags_without_sponsor` (all `submit_raw` mutations on a `DepositPreauth` base) plus `disallowed_type` (a valid reserve sponsor on `OfferCreate`/`NFTokenMint`, outside the allow-list → `temINVALID_FLAG`, `submit_tx`). Every vector is preflight `tem*`, so the type is in `_NO_SUCCESS_TYPES` + `_NO_FAILURE_TYPES`, and its name is excluded from every Modifier (submit_raw is modifier-free; the `submit_tx` `disallowed_type` variant is a no-op through the pipeline because the name isn't in any `supported` set). -`PaymentFundNew` is a **state-tree bloat driver**, not new coverage: it funds a brand-new `AccountRoot` every call so each ledger grows the state SHAMap, widening the window in which a diverging/lagging validator holds an incompletely-acquired state tree — the condition under which rippled's `RCLConsensus::timerEntry` throws `SHAMapMissingNode` and aborts. Run its `parallel_driver_payment_fund_new_random.sh` at high parallel width to reproduce that crash more often. +`PaymentFundNew` is a **state-tree bloat driver**, not new coverage: each call funds a burst of brand-new `AccountRoot`s (one per distinct source account, fired concurrently so they land in one open ledger) so each ledger grows the state SHAMap in a concentrated jump, widening the window in which a diverging/lagging validator holds an incompletely-acquired state tree — the condition under which rippled's `RCLConsensus::timerEntry` throws `SHAMapMissingNode` and aborts. Distinct sources are required: reusing one source would autofill the same `Sequence` for every burst member and collide (`tefPAST_SEQ`). Run its `parallel_driver_payment_fund_new_random.sh` at high parallel width to reproduce that crash more often. **api_version 2 gotcha:** the WS stream renames a Payment's `Amount` to `DeliverMax` and drops `Amount`. Read delivered amount via `_delivered_amount(tx)` (`DeliverMax`, `Amount` fallback) — `tx["Amount"]` is `None` and misclassifies. `DomainID`/`SendMax`/`TakerGets`/`TakerPays` are unaffected. diff --git a/scripts/check-imports b/scripts/check-imports index 04aeaf0..3af8d8f 100755 --- a/scripts/check-imports +++ b/scripts/check-imports @@ -41,7 +41,7 @@ from workload.fuzz import submit_fuzzed, fuzz_mutate from workload.sequence import SequenceTracker from workload.ws_listener import start_ws_listener from workload.setup import run_setup -from workload.params import should_send_faulty, fake_account, fake_id, new_account_funding_amount +from workload.params import should_send_faulty, fake_account, fake_id, new_account_funding_amount, fund_new_burst_count from workload.models import UserAccount, Credential, Vault, PermissionedDomain, MPTokenIssuance, LoanBroker, Loan, Delegate, DID, ConfidentialHolder, ConfidentialMPTIssuance, Sponsorship print('All imports OK') " diff --git a/workload/src/workload/params.py b/workload/src/workload/params.py index 11b27e6..e949028 100644 --- a/workload/src/workload/params.py +++ b/workload/src/workload/params.py @@ -46,6 +46,15 @@ def new_account_funding_amount() -> str: return str(randint(11_000_000, 30_000_000)) +def fund_new_burst_count() -> int: + """Accounts to create per PaymentFundNew call, each from a DISTINCT source (same + source would reuse one Sequence and collide). A burst lands in one open ledger, so + the state SHAMap gains ~2N nodes at once (N created + N modified by fee debit) — + concentrated growth widens the SHAMapMissingNode acquisition window more than the + same creations spread thin. Capped at len(accounts) by the caller.""" + return randint(3, 10) + + # ── Trust Lines & IOUs ──────────────────────────────────────────────── CURRENCY_CODES = ["USD", "EUR", "GBP", "JPY", "BTC", "ETH", "XAU", "CNY"] diff --git a/workload/src/workload/transactions/payments.py b/workload/src/workload/transactions/payments.py index 776d85c..50ebe37 100644 --- a/workload/src/workload/transactions/payments.py +++ b/workload/src/workload/transactions/payments.py @@ -1,5 +1,7 @@ """Payment transaction generators.""" +from asyncio import gather + from xrpl.asyncio.clients import AsyncJsonRpcClient from xrpl.models import IssuedCurrencyAmount as IOUAmount from xrpl.models.amounts import MPTAmount @@ -71,20 +73,34 @@ async def payment_fund_new( client: AsyncJsonRpcClient, ) -> None: """State-tree bloat driver (synthetic name PaymentFundNew, on-ledger Payment): - fund a brand-new account every call so each ledger grows the state SHAMap with - fresh AccountRoots. This widens the window in which a diverging/lagging validator + fund a burst of brand-new accounts every call so each ledger grows the state SHAMap + with fresh AccountRoots. This widens the window in which a diverging/lagging validator holds an incompletely-acquired state tree — the condition under which RCLConsensus::timerEntry throws SHAMapMissingNode and aborts the process. + + Distinct sources per burst: same source would autofill one Sequence for all and + collide (tefPAST_SEQ), creating nothing. Fired concurrently so the burst lands in + one open ledger for maximal per-ledger state growth. Valid-only: an under-funded fund-new creates nothing, defeating the purpose.""" if not accounts: return - src = accounts[choice(list(accounts))] - txn = Payment( - account=src.address, - amount=params.new_account_funding_amount(), - destination=params.fake_account(), + n = min(params.fund_new_burst_count(), len(accounts)) + srcs = sample(list(accounts), n) + await gather( + *( + submit_tx( + "PaymentFundNew", + Payment( + account=accounts[addr].address, + amount=params.new_account_funding_amount(), + destination=params.fake_account(), + ), + client, + accounts[addr].wallet, + ) + for addr in srcs + ) ) - await submit_tx("PaymentFundNew", txn, client, src.wallet) def _iou_amount(trust_lines: list[TrustLine]) -> IOUAmount: