diff --git a/docker/run_json.py b/docker/run_json.py index 4a8e295c4..d3e24a798 100755 --- a/docker/run_json.py +++ b/docker/run_json.py @@ -5282,30 +5282,37 @@ def _new_report_sink(lib, state, args): class _TemporaryReportSink: - """An unlinked file, addressed by a private path for the engine's path ABI. - - TemporaryFile uses O_TMPFILE or unlinks immediately. The descriptor path - lets the engine open that file with its existing path-only API; no named - sink, new wire format or public fd API is introduced. Keep the owner open - until both report_free and strategy_free have returned. + """A named, owner-only (0600) temporary file, addressed by its real path. + + The engine opens the path itself with its existing path-only API + (O_WRONLY | O_CREAT | O_TRUNC, no O_EXCL: src/report_sink_writer.cpp), so + the empty file created here is accepted and truncated by it. Every reader + opens the same path again and so owns its offset. No descriptor path + (/proc/self/fd, /dev/fd) is used: Darwin's /dev/fd/N is a dup that shares + the file offset. No named sink, new wire format or public fd API is + introduced. close() unlinks the file and may be called again, unless + ``keep`` is set (the harness's test-only artifact switch). Keep the owner + open until both report_free and strategy_free have returned. """ def __init__(self, lib, state, *, directory=None, run_id=""): self.directory = os.fspath(directory) if directory is not None else tempfile.gettempdir() self.header, self.counts, self.sha256 = {}, {}, "" + self.keep, self.closed = False, False try: - self.file = tempfile.TemporaryFile(mode="w+b", dir=self.directory) + handle, path = tempfile.mkstemp(prefix="pineforge-report-sink-", suffix=".sink", + dir=self.directory) except OSError as error: raise RunFailure("report sink open failed", "report_sink_failed") from error + self.path = os.path.abspath(path) try: - directory_path = "/proc/self/fd" if sys.platform.startswith("linux") else "/dev/fd" - self.path = f"{directory_path}/{self.file.fileno()}" + os.close(handle) # the engine and every reader open the path themselves descriptor = ReportSinkC(ctypes.sizeof(ReportSinkC), 1, os.fsencode(self.path), run_id.encode("utf-8"), os.fsencode(self.directory)) if lib.strategy_set_report_sink_v1(state, ctypes.byref(descriptor)) != 0: raise _engine_invariant("the fresh strategy refused its temporary report sink") except BaseException: - self.file.close() + self.close() raise def verify(self, lib, state, report): @@ -5340,7 +5347,13 @@ def records(self, tag=None): raise _engine_invariant(f"the engine's report sink is invalid: {error}") from error def close(self): - self.file.close() + if not self.closed: + if not self.keep: + try: + os.unlink(self.path) + except FileNotFoundError: + pass + self.closed = True def _free_sink_run(lib, state, report, sink): diff --git a/docs/adr/0001-kernel-adapter-boundary.md b/docs/adr/0001-kernel-adapter-boundary.md index e625ead10..bda6489f7 100644 --- a/docs/adr/0001-kernel-adapter-boundary.md +++ b/docs/adr/0001-kernel-adapter-boundary.md @@ -93,22 +93,22 @@ summary of it. (`native_execution_consumer.cpp:1100-1114`) in the v19 continuation. The Pine adapter keeps its pending-book and chart-sampling policy in the source layer, reading the named input coordinate through `projection_bar_index` - (`pine_adapter.cpp:1049-1055`) where Pine still needs input cadence, and + (`pine_adapter.cpp:1050-1056`) where Pine still needs input cadence, and does not rewrite the kernel's lot or row indices. - **Requests.** `submit` / `replace` / `cancel` / `submit_market` / `replace_market` (`native_host.hpp:1227-1232`), plus `cancel_all` (`native_host.hpp:1267`) and `cancel_where` (`native_host.hpp:1272`). A request carries one of five triggers — Market, Limit (with a - generic market-if-touched `fill_through`, `native_order.hpp:239-242`), Stop, StopLimit, Trail - (`Trigger` `native_order.hpp:296`); one of **six** intents — `Flatten`, `Reduce`, `Transact`, + generic market-if-touched `fill_through`, `native_order.hpp:241-244`), Stop, StopLimit, Trail + (`Trigger` `native_order.hpp:298`); one of **six** intents — `Flatten`, `Reduce`, `Transact`, `ReverseTo`, `HostSized` and the kernel-resolved `Sized` (`OrderIntent` - `native_order.hpp:229`); one of five owner kinds — `Independent`, `WaitForApplied`, - `BindOpening`, `BindOpenings`, `BindCohort` (`Owner` `native_order.hpp:464`); an OCA-style + `native_order.hpp:231`); one of five owner kinds — `Independent`, `WaitForApplied`, + `BindOpening`, `BindOpenings`, `BindCohort` (`Owner` `native_order.hpp:466`); an OCA-style group with cancel-or-reduce effect; a capacity, `ImmediateRemaining` or `PointBudget`; and a - `TriggerAnchor` (`native_order.hpp:336`) that is either the absolute level or + `TriggerAnchor` (`native_order.hpp:338`) that is either the absolute level or `FromOwnerFill`, the anchored bracket child. - **Matching, fills, and who sizes.** The kernel matches geometrically along the bar path and fills with slippage, fees, admission and settlement. Sizing has **two** routes: `Sized` - (`native_order.hpp:164`) is resolved by the kernel from a cash or equity-fraction basis, so a + (`native_order.hpp:166`) is resolved by the kernel from a cash or equity-fraction basis, so a bare host needs no override at all, while `HostSized` hands the units to the host — and an unresolved `HostSized` whose `resolve_execution_terms` (`native_host.hpp:947`) returns no units is a `TermsUnresolved` rejection @@ -145,7 +145,7 @@ summary of it. (`NativeInstrumentFeed`, lane XSYM-D). `initial_margin_fraction` (`native_run_spec.hpp:666`) remains the one-scalar admission-only spelling and is mutually exclusive with `margin`. Percent and cash sizing are a *request* value, not a spec one - (`Sized` `native_order.hpp:164`). What the spec still owns none of is source strategy policy: + (`Sized` `native_order.hpp:166`). What the spec still owns none of is source strategy policy: the field comments say so, and `scripts/check_adapter_spec_shadowing.py` holds the adapter to the fields it actually needs. - **Feeding data.** Bars enter through the engine's own entry points: `run(bars, n)` @@ -213,8 +213,8 @@ kernel and own these quirks, each at its site: - **Seams the adapter overrides.** The kernel is driven, not patched. `PineStrategyHost` overrides `resolve_execution_terms` (`pine_strategy_host.cpp:844`) for TV sizing and fill spelling; `validate_execution_precommit` (`pine_strategy_host.cpp:869`), whose - `validate_precommit` (`pine_adapter.cpp:16431`) returns `AdmitWithHostMargin` - (`pine_adapter.cpp:16635`) to own the opening margin decision; `on_native_tick` + `validate_precommit` (`pine_adapter.cpp:16435`) returns `AdmitWithHostMargin` + (`pine_adapter.cpp:16639`) to own the opening margin decision; `on_native_tick` (`pine_strategy_host.cpp:571`) / `on_native_applied` (`pine_strategy_host.cpp:643`) for `calc_on_order_fills` re-entry. It does not own lot excursions: since R5 lane H-THIN it keeps the kernel's `owns_lot_excursions()` default, so the kernel's sampler books every Pine lot's @@ -229,19 +229,19 @@ kernel and own these quirks, each at its site: `NativeCloseExecution::AfterCalculation` in the projected spec (`pine_adapter.cpp:2819-2820`); calc cadence and language publication are `PineScheduler`'s (`pine_scheduler.hpp:26`). - **Batching and open-order priority.** Same-bar command batching and its deferred queues - (`PendingSameBarCommand` pine_adapter.hpp:1885, `pending_bracket_legs_` pine_adapter.hpp:2519-2537); the retained parent-before-child ordering of live handles — an + (`PendingSameBarCommand` pine_adapter.hpp:1903, `pending_bracket_legs_` pine_adapter.hpp:2545-2564); the retained parent-before-child ordering of live handles — an exactly-shaped entry and its `from_entry` exit, re-created after a named cancel, on a flat book under `process_orders_on_close` (`src/compat/pine/order_priority.cpp:13-59`, applied by - `update_l4c_priority` `pine_adapter.cpp:1366-1436`). Kernel priority is acceptance/incarnation order. + `update_l4c_priority` `pine_adapter.cpp:1367-1437`). Kernel priority is acceptance/incarnation order. - **Exit-leg lifecycle.** Activation / suspension / revival barriers (`select_exit_activation` src/compat/pine/exit_activation.cpp:44, `select_exit_suspension` src/compat/pine/exit_lifecycle.cpp:11, `select_replacement_revival_definition` src/compat/pine/exit_lifecycle.cpp:51) and historical birth reach (`order_birth.cpp` `src/compat/pine/order_birth.cpp:5-14`). - **Trail and tick conventions.** The half-tick arm threshold measured against tick-quantized - extremes while the raw running best is retained (`has_trail_request` `pine_adapter.cpp:12546-12551`); the half-tick - trigger threshold (`source_trigger_threshold` pine_adapter.cpp:384) and raw-vs-booked fill spelling, behind the - adapter's terms seam (`resolve_terms` `pine_adapter.cpp:14904`). + extremes while the raw running best is retained (`has_trail_request` `pine_adapter.cpp:12550-12555`); the half-tick + trigger threshold (`source_trigger_threshold` pine_adapter.cpp:385) and raw-vs-booked fill spelling, behind the + adapter's terms seam (`resolve_terms` `pine_adapter.cpp:14908`). - **Money arithmetic.** Ten-significant-digit half-up money (`source_money_round` - `pine_adapter.cpp:558-564`, twin `tv_money_round` `pine_policy_support.hpp:9-15`). + `pine_adapter.cpp:559-565`, twin `tv_money_round` `pine_policy_support.hpp:9-15`). - **Quantity dust.** After every applied execution the Pine host erases any lot of at most `kQtyEpsilon` (`1e-10`) from the book it shares with the kernel, with no closing row (`on_native_applied` `pine_strategy_host.cpp:643`), inside a live book as well as at a flat @@ -256,12 +256,12 @@ kernel and own these quirks, each at its site: restore's second floor) fail tape-pinned rows (`test_pine_dust_sweep_paired`, `test_tv_money_band_l4b`); `quantity_tolerance` is not their rule (its row below). - **Close reservations.** `strategy.close` callsite batching, two-call provenance and the entry-id - ledger (`close_logical_units_` `pine_adapter.hpp:2584-2614`); the POOC reservation-growth population predicate + ledger (`close_logical_units_` `pine_adapter.hpp:2602-2632`); the POOC reservation-growth population predicate (`select_reservation_growth_sources` src/compat/pine/reservation_expansion.cpp:9). - **Margin.** The kernel owns the margin *mechanism* — the level solve, the check points, the kernel request, its re-pricing, the receipt — and the adapter answers its three policy hooks - with TradingView's: `margin_check_allowed` (`pine_adapter.cpp:17890`) for the scheduling, - `resolve_margin_requirement` (`pine_adapter.cpp:17959`) for the ten-significant-digit money, + with TradingView's: `margin_check_allowed` (`pine_adapter.cpp:17894`) for the scheduling, + `resolve_margin_requirement` (`pine_adapter.cpp:17963`) for the ten-significant-digit money, and `resolve_margin_call_units` for the lot-floored 4x restore. That hook answers every call on purpose (R5 lane F7): TradingView floors the restore onto the lot grid before the multiple, so on a gridded tape the kernel's own `ShortfallMultiple 4` books 5 lots where TradingView @@ -272,7 +272,7 @@ kernel and own these quirks, each at its site: margin since F7), while the roll point measures at the remaining path's adverse mark (MG-FX: 0.4116 @ 97 there against TradingView's 0.3996 @ 100). What has no kernel check point at all stays adapter-side: the `process_orders_on_close` chronology exception - (`non_pooc_commissioned_short` `pine_adapter.cpp:24245`) and + (`non_pooc_commissioned_short` `pine_adapter.cpp:24249`) and the 1x-long money call — plus the TV admission scopes (`explicit_pair_scope` `src/compat/pine/market_admission.cpp:34-47`) and the review fold they feed (`awaits_pair_review` `:68-73`, `fold_admission_history` `:83-140`). @@ -333,9 +333,9 @@ kernel and own these quirks, each at its site: projects `NativeCalculationTrigger::BarCloseAndFills` with TradingView's guard literal as `max_recalculations_per_point`. What stays are the specifics COOF adds on top — the language-state snapshot/restore around a recalculation, the waypoint-only refill deferral - (`next_source_path_waypoint` `pine_adapter.cpp:10510`), the first-open execution chain and its + (`next_source_path_waypoint` `pine_adapter.cpp:10514`), the first-open execution chain and its own loop guard (`kFirstOpenLoopGuard` `pine_scheduler_native.cpp:888`), and the two fills Pine refuses to - recalculate on (`suppress_grouped_stop_recalc` `pine_adapter.cpp:6790`). + recalculate on (`suppress_grouped_stop_recalc` `pine_adapter.cpp:6794`). - **Pine language state and harness flags.** Series, the tick-level barstate flags and position-view freezing (`PineLanguageState` `pine_language_state.hpp:12`); the session flags, `PineStrategyHost` members since R5 lane F5 (`session_ismarket_` @@ -393,8 +393,8 @@ citations, 264 are in `src/source/` and its headers; 6 sit in kernel files 927-961 of that tree, between `round_to_mintick` and `bar_fill_price`), deleted by R5 lane E6; the lines 73-180 earlier drafts cited are and were live code (`ClosedLotExcursionFacts`, `PyramidEntry`, `Trade`, each with its own doc). The arithmetic is - adapter-side (`source_money_round` `pine_adapter.cpp:558-576`). `strategy.close` batching and the entry-id ledger - are **gone** from the kernel — the ledger is adapter state (`close_logical_units_` `pine_adapter.hpp:2584-2614`) and + adapter-side (`source_money_round` `pine_adapter.cpp:559-577`). `strategy.close` batching and the entry-id ledger + are **gone** from the kernel — the ledger is adapter state (`close_logical_units_` `pine_adapter.hpp:2602-2632`) and only names survive in comments. The TradingView margin-call toggle remains source-owned as `source::PineStrategyHost::set_margin_call_enabled` (`pine_strategy_host.hpp:511`); what enables the generic model is the presence of `NativeRunSpec::margin`. So is the string-sentinel decoding of adapter-written comments: @@ -709,9 +709,9 @@ Every line number in this section names its symbol on this tree, which | `NativeRunSpec` field | ruling | what the adapter runs instead | measurement and ruling of record | executed native consumers | |---|---|---|---|---| -| `price_grid`, `grid_rounding` | **native-only** | TradingView's per-order-kind tick rules on top of `None`: `source_trigger_threshold` (`pine_adapter.cpp:384`), `source_level_on_price_grid` (`:367`), `nearest_tick` (`:274`) / `source_bar_fill_tick` (`:295`) / `directional_tick` (`:327`), behind the terms seam | R5-3 and design risk E7 ruled `None` for the adapter before the lane ran; lanes R7 and N13 measured the alternative anyway (raw levels submitted, `QuantizeFillsAndTriggers` with `HalfUp` declared). L8b closed the first blocker (no run aborts). The second has no remedy on either side: TradingView quantizes per order kind (stop and limit legs and a trail's activation on the quantized bar; the trail stop, the running best, stop-limit entries and the `calc_on_order_fills` cursors raw), the grid is one rule for the run, and the old N13 trial moved 30 pinned checks in 4 units; this is historical evidence, not a current-head result; a per-kind mask would spell that inconsistency into the kernel. The corpus cannot arbitrate (every probe runs a 0.01 tick on an on-grid feed; 5 of 312 differ in an engine-only column). Permanent witness: `tests/test_adapter_grid_relower.cpp`, whose section 5 pins the trail stop the grid fires a bar early. Design row PG and §3.6 | `examples/native/native_price_grid_strategy.cpp`, `examples/native/native_price_grid_c.c`, the private test `test_native_price_grid` | -| `risk` | **native-only** | all of `strategy.risk.*` but the direction: `update_risk_state` (`pine_adapter.cpp:17541`), `SourceDayLedger`, `submit_intraday_loss_close` (`pine_adapter.cpp:20920`), `chart_day_key` (`pine_adapter.cpp:17438`), `compat::pine::IntradayCap` with `IntradayOrderBudget` (337 lines) | Structural first: Pine's risk calls are per-bar statements, so a limit reaches the adapter on script bar 0, after `project()` (`pine_strategy_host.cpp:366`) and the kernel's `configure` (`pine_strategy_host.cpp:375`) have fixed and digested the spec. In substance (lane N12, `tests/test_adapter_risk_relower.cpp`, 62 checks over nine paired scenarios): the drawdown latch samples at the close only and still admits a reversal; the loss-day streak counts trades, not days; the intraday loss closes at the path's adverse extreme, refuses every placement and withdraws the book; the fill cap charges slots, transfers quota and closes at the bar's better extreme on the chart timezone's day. With the kernel seeded on the corpus, 3 of 4 cap probes diverge (3840 of 3916, 312 of 604, 2370 of 2384 rows; re-derived by `scripts/check_seeded_risk_experiment.sh` since R5 lane H-MEASURE). `strategy.risk.allow_entry_in` is the one rule that is the kernel's already (`allowed_open_directions`). Design §3.6 | `examples/native/native_risk_limits_strategy.cpp`, `examples/native/native_trail_risk_strategy.cpp`, `tests/test_native_risk_limits.cpp`, `tests/test_native_c_api.c` | -| `max_abs_units` | **adapter-policy** | `strategy.risk.max_position_size` as a gate on the LIVE book before the fill (`max_position_size` `pine_adapter.cpp:16550-16551`): an entry is refused once the book already holds the limit | design row MG2 (R5-1): the resulting-book cap is the generic one. Measured by N12's scenario `PS` in `tests/test_adapter_risk_relower.cpp`: two-unit entries against a limit of 3 leave the adapter at 4 and the kernel cap at 2 | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_run_spec.cpp` | +| `price_grid`, `grid_rounding` | **native-only** | TradingView's per-order-kind tick rules on top of `None`: `source_trigger_threshold` (`pine_adapter.cpp:385`), `source_level_on_price_grid` (`:368`), `nearest_tick` (`:275`) / `source_bar_fill_tick` (`:296`) / `directional_tick` (`:328`), behind the terms seam | R5-3 and design risk E7 ruled `None` for the adapter before the lane ran; lanes R7 and N13 measured the alternative anyway (raw levels submitted, `QuantizeFillsAndTriggers` with `HalfUp` declared). L8b closed the first blocker (no run aborts). The second has no remedy on either side: TradingView quantizes per order kind (stop and limit legs and a trail's activation on the quantized bar; the trail stop, the running best, stop-limit entries and the `calc_on_order_fills` cursors raw), the grid is one rule for the run, and the old N13 trial moved 30 pinned checks in 4 units; this is historical evidence, not a current-head result; a per-kind mask would spell that inconsistency into the kernel. The corpus cannot arbitrate (every probe runs a 0.01 tick on an on-grid feed; 5 of 312 differ in an engine-only column). Permanent witness: `tests/test_adapter_grid_relower.cpp`, whose section 5 pins the trail stop the grid fires a bar early. Design row PG and §3.6 | `examples/native/native_price_grid_strategy.cpp`, `examples/native/native_price_grid_c.c`, the private test `test_native_price_grid` | +| `risk` | **native-only** | all of `strategy.risk.*` but the direction: `update_risk_state` (`pine_adapter.cpp:17545`), `SourceDayLedger`, `submit_intraday_loss_close` (`pine_adapter.cpp:20924`), `chart_day_key` (`pine_adapter.cpp:17442`), `compat::pine::IntradayCap` with `IntradayOrderBudget` (337 lines) | Structural first: Pine's risk calls are per-bar statements, so a limit reaches the adapter on script bar 0, after `project()` (`pine_strategy_host.cpp:366`) and the kernel's `configure` (`pine_strategy_host.cpp:375`) have fixed and digested the spec. In substance (lane N12, `tests/test_adapter_risk_relower.cpp`, 62 checks over nine paired scenarios): the drawdown latch samples at the close only and still admits a reversal; the loss-day streak counts trades, not days; the intraday loss closes at the path's adverse extreme, refuses every placement and withdraws the book; the fill cap charges slots, transfers quota and closes at the bar's better extreme on the chart timezone's day. With the kernel seeded on the corpus, 3 of 4 cap probes diverge (3840 of 3916, 312 of 604, 2370 of 2384 rows; re-derived by `scripts/check_seeded_risk_experiment.sh` since R5 lane H-MEASURE). `strategy.risk.allow_entry_in` is the one rule that is the kernel's already (`allowed_open_directions`). Design §3.6 | `examples/native/native_risk_limits_strategy.cpp`, `examples/native/native_trail_risk_strategy.cpp`, `tests/test_native_risk_limits.cpp`, `tests/test_native_c_api.c` | +| `max_abs_units` | **adapter-policy** | `strategy.risk.max_position_size` as a gate on the LIVE book before the fill (`max_position_size` `pine_adapter.cpp:16554-16555`): an entry is refused once the book already holds the limit | design row MG2 (R5-1): the resulting-book cap is the generic one. Measured by N12's scenario `PS` in `tests/test_adapter_risk_relower.cpp`: two-unit entries against a limit of 3 leave the adapter at 4 and the kernel cap at 2 | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_run_spec.cpp` | | `max_open_lots` | **adapter-policy** | Pine pyramiding is a per-cycle entry count in the adapter's command policy; a resting source entry must not consume a physical-lot cap before it fills, so `project()` leaves the cap unset (`max_open_lots` `pine_adapter.cpp:2648-2650`) | design row MG3 (R5-1); the contract comment in `project()`. Measured against TradingView by R5 lane H-MEASURE (the private test `test_pyramiding_count_differential`, 15 scenarios on three `lab tv` tapes): TradingView checks an entry once, at its first eligible point, against the trades then open, and does not check a resting entry again at its fill; the kernel's cap books 14 of the 15 as TradingView does, the adapter's per-cycle count 13 (it keeps a slot a close or another entry's exit drained; since lane W8A-SIGSTATE-1's R-B a resting LIMIT entry no longer counts, and one the count admitted fills over the cap, and since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap, as on TradingView's tapes). A recorded divergence: the retention stands pending a re-lowering ruling. R5 lane H-THIN tried the lowering and kept the count: on the kernel's cap four corpus probes moved away from TradingView, for want of the first-eligible-point rule (H-MEASURE's P4b); it fixed the reservation divergence measured beside it (P10), so a default `strategy.exit(from_entry)` under FIFO now reserves its entry's own quantity, the oldest lots first, as TradingView does | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_margin_model.cpp` | | `initial_margin_fraction` | **adapter-policy** | TradingView's ten-significant-digit money admission against the signal-time tuple, answered as `AdmitWithHostMargin`; the `margin` model the adapter does declare is maintenance-only (`NativeMarginModel` `pine_adapter.cpp:2887-2895`, `margin.maintenance_long` `pine_adapter.cpp:2902`) | design row MG4 and the wave-4 ruling recorded in `project()`: a positive initial requirement would decline openings TradingView takes. Measured both ways by R5 lane H-MEASURE (the private test `test_adapter_margin_schedule_differential`, M11): the kernel's gate declines a 1x long whose entry fee the adapter admits (`InitialMargin`), and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times | `tests/test_native_precommit_view.cpp`, `tests/test_native_margin_model.cpp` | | `report_open_position_at_end` | **adapter-policy** | The Pine host owns TradingView's range-end shape: batch re-marks the stored final point and re-folds extremes, while REALTIME retains ordinary marks and `PineStrategyHost::present_report` re-marks only the caller-owned copy before generic metric derivation; the stream hash excludes that terminal extreme fold (`scheduler_record_range_end` pine_strategy_host.cpp:2003): report shape, not a mark-to-market row (`KernelRecordedAtHostMarks` pine_adapter.cpp:2857) | design row RP5; the kernel reads the field under `KernelRecorded` only, which `scripts/check_adapter_spec_shadowing.py` gates | `examples/native/native_sized_report_strategy.cpp`, `tests/test_native_report_truth.cpp` | @@ -799,14 +799,14 @@ The table records, at the tree the doc wave read, what each block described and | the deleted block (lines at the doc wave's tree) | what it documented | the live generic mechanism | |---|---|---| -| `src/engine_orders.cpp`, lines 203-210 | `strategy.oca.reduce`: a filled sibling reduces every other sibling's remaining quantity | `native_order`'s `GroupEffect` (`GroupEffect` `native_order.hpp:466`) with `ReservationReducedEvent`, resolved in `src/native_order.cpp`: a group member's effect on its siblings is cancel-or-reduce, named by the request | +| `src/engine_orders.cpp`, lines 203-210 | `strategy.oca.reduce`: a filled sibling reduces every other sibling's remaining quantity | `native_order`'s `GroupEffect` (`GroupEffect` `native_order.hpp:468`) with `ReservationReducedEvent`, resolved in `src/native_order.cpp`: a group member's effect on its siblings is cancel-or-reduce, named by the request | | `src/engine_orders.cpp`, line 350 | TradingView's deferred-flip growth rule (`tv_carry_qty`) | none in the kernel: the carry survives only as a frozen POD field name, ruled in the residual table | | `src/engine_orders.cpp`, line 350 | the opening margin guard `required_margin = qty * fill_price * margin_pct / 100` | `NativeMarginModel::initial_long` / `initial_short` and `initial_margin_fraction`, with `AdmitWithHostMargin` for a host that owns the check | | `src/engine_orders.cpp`, lines 350-355 | TradingView same-tick multi-entry sequential-fill semantics | the consumer's same-point fill order, by acceptance and incarnation ordinal | | `src/engine_run.cpp`, lines 162-186 | `process_orders_on_close` semantics | the spec field `NativeCloseExecution::AfterCalculation` (`NativeCloseExecution::AfterCalculation` `native_run_spec.hpp:35`) | | `include/pineforge/engine.hpp`, lines 1262-1330 | TradingView freezes default market-order sizing at the signal bar | `native_order::SizePrice` (`Signal`, `SignalOnTick`) with `SizeTime::AtAcceptance` — a request value that names no platform | | `include/pineforge/engine.hpp`, lines 1352-1381 | TradingView liquidates intrabar, before the bar-close script body | `NativeLiquidationCheck` (`NativeLiquidationCheck` `native_run_spec.hpp:158`), whose three values are three broker models | -| `include/pineforge/engine.hpp`, lines 918-952 (the row first named lines 73-180, which were and are live code: `ClosedLotExcursionFacts`, `PyramidEntry`, `Trade`) | the ten-significant-digit money rule | adapter-side arithmetic (`source_money_round` `pine_adapter.cpp:558-564`) | +| `include/pineforge/engine.hpp`, lines 918-952 (the row first named lines 73-180, which were and are live code: `ClosedLotExcursionFacts`, `PyramidEntry`, `Trade`) | the ten-significant-digit money rule | adapter-side arithmetic (`source_money_round` `pine_adapter.cpp:559-565`) | ### B. Live kernel mechanisms with a calibrated number — each passes the amended rule 2 @@ -960,7 +960,7 @@ questions: the mechanism, why it is generic, and why no knob is needed. and state their evidence. 2. The kernel changes only for a *generic* capability with a recorded ruling (e.g. per-lot excursion ownership, `owns_lot_excursions` `native_host.hpp:1023`; the market-if-touched - `fill_through` flag, `native_order.hpp:239-242`). **The test is not whether the word + `fill_through` flag, `native_order.hpp:241-244`). **The test is not whether the word "TradingView" appears — a calibrated number has to say what calibrated it. The test is this pair, and both halves must pass:** - **(a) Mechanism.** Could a second, unrelated venue or platform be driven through this same diff --git a/docs/design/cr150-contracts.md b/docs/design/cr150-contracts.md index 90284d8f8..d3273e89c 100644 --- a/docs/design/cr150-contracts.md +++ b/docs/design/cr150-contracts.md @@ -218,7 +218,7 @@ references; L15a must maintain these anchors and mechanically pin every current | Lower-path digest (count first) and continuation hash | `native_intrabar_path_digest` `src/native_run_spec.cpp:891-919`; `native_intrabar_path_digest` `src/native_execution_consumer.cpp:289` | 371,1223-1224 | | Per-script-bar lower-path binary search/walk | `candidate` `src/native_execution_consumer.cpp:8686-8719` | 371,1225 | | Scheduler sub-bar lookup and one-bar look-ahead | `PineScheduler::lower_path_bar_at` `src/source/pine_scheduler_native.cpp:265-287`; `next_open` `src/source/pine_scheduler_native.cpp:328-329` | 371,1226-1227 | -| Lower-path emptiness test | `inactive_sampler_path` `src/source/pine_adapter.cpp:3431-3435` | 371,1228 | +| Lower-path emptiness test | `inactive_sampler_path` `src/source/pine_adapter.cpp:3430-3434` | 371,1228 | Every count/digest must fold the same bytes/order for copied and borrowed feeds, including cached and uncached paths (2.2:372-375). Views replace storage; diff --git a/docs/design/native-feature-parity.md b/docs/design/native-feature-parity.md index e44755b7a..03f25e767 100644 --- a/docs/design/native-feature-parity.md +++ b/docs/design/native-feature-parity.md @@ -22,7 +22,7 @@ Most status differences are row granularity (one input folds two features into o | # | Ruling | |---|---| | R5-1 | **Classification.** Generic execution feature → kernel native API. TradingView quirk → adapter. Mixed feature → kernel gets the generic mechanism with a policy knob, the adapter selects the TV policy, the bare-host default is the plain generic behaviour. | -| R5-2 | **Brackets are not a new kernel object.** `WaitForApplied{RequestHandle parent}` (native_order.hpp:436-438) and `waiting_children(parent)` (native_order.hpp:1953, *fresh*) exist; L7 delivers a builder/toolkit over them plus relative anchors and the missing offsets. Size M. | +| R5-2 | **Brackets are not a new kernel object.** `WaitForApplied{RequestHandle parent}` (native_order.hpp:438-440) and `waiting_children(parent)` (native_order.hpp:1959, *fresh*) exist; L7 delivers a builder/toolkit over them plus relative anchors and the missing offsets. Size M. | | R5-3 | **Price grid is its own kernel lane (L8).** Quantity grid is native already; the price grid (booked fill on the tick, optional quantized trigger test) is generic and missing. TV's exact half-tick rule stays in the adapter on top. | | R5-4 | **Close reservations split.** Fractional reduce + reservation against sibling exits → kernel, inside the sizing lane. Callsite batching, two-call provenance, the POOC population predicate → adapter. | | R5-5 | **Calc-timing is v1.** Sub-bar hook + callback chronology contract; the adapter keeps its current path (zero drift). | @@ -47,69 +47,69 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| OL1 | Market / limit / stop / stop-limit / trail triggers (`trigger_for` pine_adapter.cpp:3447; `entry()` pine_adapter.cpp:7203, `order()` pine_adapter.cpp:14670) | **yes** — `Trigger` native_order.hpp:296-315; geometric matching `first_region_entry` native_execution_consumer.cpp:6635; `submit` native_host.hpp:1213 | K | — | F:A1 O:A1 S:O2 | — | -| OL2 | Intents: flatten, reduce by units, reduce owner-opened units, transact, reverse-to (`close_all()` pine_adapter.cpp:10587 → `Flatten{}` pine_adapter.cpp:10621; `Reduce{ExplicitUnits}` pine_adapter.cpp:9460; `ReverseTo` pine_adapter.cpp:7879) | **yes** — `OrderIntent` native_order.hpp:229, reduce sizes `ReductionSize` native_order.hpp:215, `ReverseTo` native_order.hpp:60-62, `OpeningShape` native_order.hpp:910-914, `Flatten` native_order.hpp:36 | K | — | F:A4 F:A8 O:A2 O:A3 S:O1 | — | -| OL3 | Amend / cancel one working order (`submit_or_replace` pine_adapter.cpp:4388 → `host.replace` pine_adapter.cpp:4813) | **yes** — `replace` native_host.hpp:1218-1220, `cancel` native_host.hpp:1249; `ReplacedEvent` native_order.hpp:1017-1031; result and event types `SubmitStatus` native_order.hpp:835-861, `ReplacedEvent` native_order.hpp:1001-1067 | K | — | F:A2 O:A4 S:O3 | id → handle map is host bookkeeping (F) → OL4 | -| OL4 | Cancel all / cancel by label; label → handle book (`cancel(id)` pine_adapter.cpp:14564, `cancel_all` pine_adapter.cpp:14637; `live_by_source_key_` pine_adapter.hpp:2525; predecessor state hand-carried `projection_predecessor` pine_adapter.cpp:4826) | **partial** — single-handle `cancel` native_host.hpp:1249; no enumeration (`WorkingRequestCore::live()` is consumer-internal native_order.hpp:1703); labels exist but are unindexed (`Request::label` native_order.hpp:485) **Closed:** `cancel_all` native_host.hpp:1267, `cancel_where(comment)` native_host.hpp:1272 and `cancel_where(text, NativeRequestField::Label)` native_host.hpp:1272 withdraw by predicate and answer how many left the book; `native_working_requests()` native_host.hpp:1263 is the enumeration; the id → handle book is `native_toolkit::OrderBook`, host-side and Pine-free (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | F:A3 F:A2 O:A5 O:A14 S:O3 | **Disagree:** F partial / O no (A5) + partial (A14) / S yes (folded into O3). R5-1: cancel-all / by-label is generic → kernel. TV state inheritance across a same-id re-issue stays A. The adapter's `strategy.cancel` / `strategy.cancel_all` keep their per-handle loops, measured by R5 lane F7 (audit M27): TradingView cancels in the SOURCE book, which holds commands of the same script call that have not reached the kernel yet (and a live `strategy.close` is labelled `__close__` there), so at the same point `cancel_where(id, NativeRequestField::Label)` and `cancel_all()` withdraw 0 — the close flattens at the next open, the reversal lands — where the source cancels withdraw them (`tests/test_adapter_brackets_relower.cpp`, M27 rows); each withdrawn source handle is also retired one by one. | -| OL5 | Per-point fill capacity, partial fills, residual working quantity (Pine uses only `ImmediateRemaining` pine_adapter.cpp:5032; residual closes `enqueue_pooc_fifo_close` pine_adapter.cpp:9206) | **yes** — `Capacity` / `PointBudget` native_order.hpp:379-383, `Remaining` native_order.hpp:557-579, allowances `Allowance` native_order.hpp:662-675 | K | — | O:A16 S:O4 | — | -| OL6 | Ownership: legs bound to a parent / opening(s) / cohort; close the lots of a given entry, FIFO or ANY (`owner_for_close` pine_adapter.cpp:6623, `cohort_for` pine_adapter.cpp:3021; FIFO prefix computed adapter-side `source_fifo_prefix_openings` pine_adapter.cpp:9301) | **yes** — `Owner` native_order.hpp:464-479, `Authority` native_order.hpp:620-621, `CohortClose` native_order.hpp:617-619, `SelectedExposure` native_order.hpp:625-629, `cohort_open` native_host.hpp:1282, `CancelReason::OwnerGone` native_order.hpp:949 | K | — | F:A6 O:A8 O:A9 S:O5 | FIFO *ordering* is host-computed (O) | -| OL7 | OCA cancel / reduce groups (`group_for` pine_adapter.cpp:3456) | **yes** — `Group` / `GroupEffect` native_order.hpp:466-473, `CancelReason::Group` native_order.hpp:948-953, `ReservationReducedEvent` native_order.hpp:1191-1201 (deferred form `DeferredGroupAdjustmentEvent` native_order.hpp:1211-1221) | K | — | F:A12 O:A6 O:A7 S:O7 | — | +| OL1 | Market / limit / stop / stop-limit / trail triggers (`trigger_for` pine_adapter.cpp:3446; `entry()` pine_adapter.cpp:7207, `order()` pine_adapter.cpp:14674) | **yes** — `Trigger` native_order.hpp:298-317; geometric matching `first_region_entry` native_execution_consumer.cpp:6635; `submit` native_host.hpp:1213 | K | — | F:A1 O:A1 S:O2 | — | +| OL2 | Intents: flatten, reduce by units, reduce owner-opened units, transact, reverse-to (`close_all()` pine_adapter.cpp:10591 → `Flatten{}` pine_adapter.cpp:10625; `Reduce{ExplicitUnits}` pine_adapter.cpp:9464; `ReverseTo` pine_adapter.cpp:7883) | **yes** — `OrderIntent` native_order.hpp:231, reduce sizes `ReductionSize` native_order.hpp:217, `ReverseTo` native_order.hpp:62-64, `OpeningShape` native_order.hpp:912-916, `Flatten` native_order.hpp:38 | K | — | F:A4 F:A8 O:A2 O:A3 S:O1 | — | +| OL3 | Amend / cancel one working order (`submit_or_replace` pine_adapter.cpp:4387 → `host.replace` pine_adapter.cpp:4812) | **yes** — `replace` native_host.hpp:1218-1220, `cancel` native_host.hpp:1249; `ReplacedEvent` native_order.hpp:1019-1033; result and event types `SubmitStatus` native_order.hpp:837-863, `ReplacedEvent` native_order.hpp:1003-1069 | K | — | F:A2 O:A4 S:O3 | id → handle map is host bookkeeping (F) → OL4 | +| OL4 | Cancel all / cancel by label; label → handle book (`cancel(id)` pine_adapter.cpp:14568, `cancel_all` pine_adapter.cpp:14641; `live_by_source_key_` pine_adapter.hpp:2543; predecessor state hand-carried `projection_predecessor` pine_adapter.cpp:4825) | **partial** — single-handle `cancel` native_host.hpp:1249; no enumeration (`WorkingRequestCore::live()` is consumer-internal native_order.hpp:1706); labels exist but are unindexed (`Request::label` native_order.hpp:487) **Closed:** `cancel_all` native_host.hpp:1267, `cancel_where(comment)` native_host.hpp:1272 and `cancel_where(text, NativeRequestField::Label)` native_host.hpp:1272 withdraw by predicate and answer how many left the book; `native_working_requests()` native_host.hpp:1263 is the enumeration; the id → handle book is `native_toolkit::OrderBook`, host-side and Pine-free (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | F:A3 F:A2 O:A5 O:A14 S:O3 | **Disagree:** F partial / O no (A5) + partial (A14) / S yes (folded into O3). R5-1: cancel-all / by-label is generic → kernel. TV state inheritance across a same-id re-issue stays A. The adapter's `strategy.cancel` / `strategy.cancel_all` keep their per-handle loops, measured by R5 lane F7 (audit M27): TradingView cancels in the SOURCE book, which holds commands of the same script call that have not reached the kernel yet (and a live `strategy.close` is labelled `__close__` there), so at the same point `cancel_where(id, NativeRequestField::Label)` and `cancel_all()` withdraw 0 — the close flattens at the next open, the reversal lands — where the source cancels withdraw them (`tests/test_adapter_brackets_relower.cpp`, M27 rows); each withdrawn source handle is also retired one by one. | +| OL5 | Per-point fill capacity, partial fills, residual working quantity (Pine uses only `ImmediateRemaining` pine_adapter.cpp:5031; residual closes `enqueue_pooc_fifo_close` pine_adapter.cpp:9210) | **yes** — `Capacity` / `PointBudget` native_order.hpp:381-385, `Remaining` native_order.hpp:559-581, allowances `Allowance` native_order.hpp:664-677 | K | — | O:A16 S:O4 | — | +| OL6 | Ownership: legs bound to a parent / opening(s) / cohort; close the lots of a given entry, FIFO or ANY (`owner_for_close` pine_adapter.cpp:6627, `cohort_for` pine_adapter.cpp:3021; FIFO prefix computed adapter-side `source_fifo_prefix_openings` pine_adapter.cpp:9305) | **yes** — `Owner` native_order.hpp:466-481, `Authority` native_order.hpp:622-623, `CohortClose` native_order.hpp:619-621, `SelectedExposure` native_order.hpp:627-631, `cohort_open` native_host.hpp:1282, `CancelReason::OwnerGone` native_order.hpp:951 | K | — | F:A6 O:A8 O:A9 S:O5 | FIFO *ordering* is host-computed (O) | +| OL7 | OCA cancel / reduce groups (`group_for` pine_adapter.cpp:3455) | **yes** — `Group` / `GroupEffect` native_order.hpp:468-475, `CancelReason::Group` native_order.hpp:950-955, `ReservationReducedEvent` native_order.hpp:1193-1203 (deferred form `DeferredGroupAdjustmentEvent` native_order.hpp:1213-1223) | K | — | F:A12 O:A6 O:A7 S:O7 | — | | OL8 | Immediate execution at the current point (`immediately=true`) | **yes** — `inspect_current_execution` / `execute_current` native_host.hpp:1104-1105 | K | — | F:A9 | (single-source row; O:F2 cites the same call) | -| OL9 | Birth eligibility: an order accepted at bar N cannot fill on N's already-delivered points | **yes** — `point_eligible` native_order.hpp:786-790, the generic birth record `OrderBirth` include/pineforge/order_birth.hpp:59, contract `docs/pages/native-engine.md` ("Acceptance is not a fill") | K+A | — | F:A13 O:A10 S:R4 | **Disagree:** F yes / O partial / S "no, TV quirk". R5-1: the generic core (birth cursor) is native; TV's bar-granular first-bar rule is the A half → OL14. | -| OL10 | TP/SL bracket per entry, armed before the entry fills, OCA between legs (`exit()` pine_adapter.cpp:11080; lowering `submit_one` pine_adapter.cpp:11958, `BindCohort` pine_adapter.cpp:12421, `submit_leg` pine_adapter.cpp:11714; lifecycle `select_exit_suspension` exit_lifecycle.cpp:10-66) | **partial** — every primitive is native (`WaitForApplied{parent}` native_order.hpp:436-438, `Reduce{OwnerOpenedUnits}` native_order.hpp:220-224, `Member{…, Cancel}` native_order.hpp:468-475, `waiting_children` native_order.hpp:1953); no builder **Closed:** the builder is `native_toolkit::submit_bracket(host, BracketSpec)` — take-profit, stop-loss and trail as one parent's `WaitForApplied` children in one OCA group, with `anchor_rounding` and `visibility` knobs — and it adds no kernel behaviour (`tests/test_native_toolkit_bracket.cpp`). | K (toolkit) | L7 | F:A10 O:A8 S:O6 | **Disagree:** F "yes (primitives)" + toolkit / S partial + new kernel bracket object, lane size L and prerequisite for everything. R5-2: builder over the existing primitives, size M. | -| OL11 | Bracket levels relative to the parent's fill (profit / loss ticks; derived `profit_ticks` pine_adapter.cpp:11340; `materialize_relative_exits` pine_adapter.cpp:14029, pine_adapter.hpp:2263-2275) | **partial** — composable: the host computes levels in `on_native_applied` and submits; suffix eligibility makes that same-bar correct `born_on_remaining_path` native_execution_consumer.cpp:6563. `Trigger` itself is absolute-only native_order.hpp:296 **Closed:** `Request::anchor = FromOwnerFill{offset, ticks, rounding}` (`TriggerAnchor` native_order.hpp:336) defers the level to the owner's fill, the kernel materializes it once at the arm under `NativeAnchorRounding`, and `resolve_anchored_level` native_host.hpp:1012 is where a source language's own level arithmetic lives (`tests/test_native_anchored_legs.cpp`, `tests/test_native_arm_options.cpp`). | K | L7 | F:A11 O:A11 S:O6 | **Disagree:** F partial (toolkit is enough) / O no (kernel `TriggerAnchor`). R5-2: L7 delivers relative anchors; kernel anchor primary, toolkit materialization is the zero-kernel fallback. | -| OL12 | Working-order view for the host and outside observers (`strategy_pending_order_*`; `PendingIntentView` pine_adapter.hpp:1453-1488; the POD is filled only by the adapter `pine_birth_reach` pine_adapter.cpp:27141) | **no** — base returns 0 / -1 `observe_last_bar_dual_entry_path_v1` engine_consumer.cpp:124-133; the POD is TV-shaped (`tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135) **Closed for a bare host:** `native_working_requests()` native_host.hpp:1263 returns one owning `NativeWorkingRequest` native_host.hpp:681 per live request — definition, remaining, trigger state — with the C spelling `strategy_native_working_len_v1` / `_get_v1`. The TV-shaped POD stays frozen and stays the source layer's projection (`tests/test_native_limit_fill_through.cpp`, `tests/test_native_arm_options.cpp`, `tests/test_native_c_api.c`). | K+A | L7 (C++), L13 (C) | F:J5 O:J6 S:X4 | **Disagree:** O "partial + TV-shaped" / F, S no. A bare host reads zeros → no. R5-1: neutral view → kernel; POD shape and Pine journal retention stay A (POD frozen, §2.ii j). | -| OL13 | *(quirk)* Same-bar command batching and the seven deferred queues (`pending_bracket_legs_` pine_adapter.hpp:2519-2538); KI-62 open-order priority (`select` order_priority.cpp:13-60, selected `update_l4c_priority` pine_adapter.cpp:1366); per-bar priced-entry throttle, BUY-before-SELL batch, short-seed plan / collision (`maybe_activate_short_seed_plan` pine_adapter.cpp:4219, `materialize_snapshot` pine_adapter.cpp:14003), reversal-gap bracket policy (`ShortSeedPlan` pine_adapter.hpp:1360-1388, `DeferredOpenMarketableSell` pine_adapter.hpp:1849-1872, `CloseCallsiteState` pine_adapter.hpp:1987-2001; `apply_reversal_gap_bracket_policy` pine_adapter.cpp:21679, `reaccept_gapped_bracket_behind_same_id_add` pine_adapter.cpp:21913) | n/a — native priority is acceptance / incarnation order `native_events` native-engine.md:676-678 | A | — | F:A15 O:A15 S:O9 S:R3 | R5-1 names KI-62. S: the kernel exposes deterministic insertion order only. | -| OL14 | *(quirk)* Exit activation / suspension / revival barriers, historical birth reach (`select_exit_activation` exit_activation.cpp:44-90, enforced `leg_activation` pine_adapter.cpp:1263; `select_exit_activation` exit_activation.cpp:20-90, `select_historical_birth_reach` order_birth.cpp:5-14, `capture_order_birth` pine_adapter.cpp:1080) | n/a | A | — | S:R4 O:A10 | TV half of OL9. **V19D-P1 (R5 lanes B-ADAPTER, V19-FIX):** a margin-call revival never restores an exit the script cancelled: `strategy.cancel` / `cancel_all` retire every leg of each exit row they name — the live ones they take off the book (caused at the Cancelled event, after anything the same callback did to the leg), the dormant ones, and the ones the adapter took off the book itself — and drop a gapped stop a declined reversal parked for the next margin-call slice (`retire_cancelled_exits` pine_adapter.cpp:2125); the revival skips a withdrawn exit (`revive_brackets_after_margin` pine_adapter.cpp:2202). What `tests/test_adapter_margin_revival_cancel.cpp` replays on the NYSE:F 15m chart of seven lab tv tapes: badapter-v19dp1-control (no cancel: the margin call closes 928 through X) and v19fix-gapped-stop-control (X's stop at 10.15, gapped by the 05-01 open and parked: the same close), and badapter-v19dp1-cancel, v19fix-cancel-before-pair, v19fix-cancel-at-1000, v19fix-gapped-stop-cancel and v19fix-gapped-stop-cancel-after-pair (a cancel on 05-01 09:30 while X is dormant, on 04-30 15:45 before the reversal pair and on 04-30 10:00 while X is live, and on 05-01 09:30 and after the pair while X is parked; one tape file: margin calls 12 @10.39 and 92 @10.57, 836 open). Untaped, the same row holds the tapes' trades for `strategy.cancel_all` on 05-01 and before the pair and for `strategy.cancel` after the pair, and on the same bars with no quantity grid — where the pair holds X in the cancel's own callback — every spelling books what the cancel on the dormant exit books | +| OL9 | Birth eligibility: an order accepted at bar N cannot fill on N's already-delivered points | **yes** — `point_eligible` native_order.hpp:790-794, the generic birth record `OrderBirth` include/pineforge/order_birth.hpp:59, contract `docs/pages/native-engine.md` ("Acceptance is not a fill") | K+A | — | F:A13 O:A10 S:R4 | **Disagree:** F yes / O partial / S "no, TV quirk". R5-1: the generic core (birth cursor) is native; TV's bar-granular first-bar rule is the A half → OL14. | +| OL10 | TP/SL bracket per entry, armed before the entry fills, OCA between legs (`exit()` pine_adapter.cpp:11084; lowering `submit_one` pine_adapter.cpp:11962, `BindCohort` pine_adapter.cpp:12425, `submit_leg` pine_adapter.cpp:11718; lifecycle `select_exit_suspension` exit_lifecycle.cpp:10-66) | **partial** — every primitive is native (`WaitForApplied{parent}` native_order.hpp:438-440, `Reduce{OwnerOpenedUnits}` native_order.hpp:222-226, `Member{…, Cancel}` native_order.hpp:470-477, `waiting_children` native_order.hpp:1959); no builder **Closed:** the builder is `native_toolkit::submit_bracket(host, BracketSpec)` — take-profit, stop-loss and trail as one parent's `WaitForApplied` children in one OCA group, with `anchor_rounding` and `visibility` knobs — and it adds no kernel behaviour (`tests/test_native_toolkit_bracket.cpp`). | K (toolkit) | L7 | F:A10 O:A8 S:O6 | **Disagree:** F "yes (primitives)" + toolkit / S partial + new kernel bracket object, lane size L and prerequisite for everything. R5-2: builder over the existing primitives, size M. | +| OL11 | Bracket levels relative to the parent's fill (profit / loss ticks; derived `profit_ticks` pine_adapter.cpp:11344; `materialize_relative_exits` pine_adapter.cpp:14033, pine_adapter.hpp:2281-2293) | **partial** — composable: the host computes levels in `on_native_applied` and submits; suffix eligibility makes that same-bar correct `born_on_remaining_path` native_execution_consumer.cpp:6563. `Trigger` itself is absolute-only native_order.hpp:298 **Closed:** `Request::anchor = FromOwnerFill{offset, ticks, rounding}` (`TriggerAnchor` native_order.hpp:338) defers the level to the owner's fill, the kernel materializes it once at the arm under `NativeAnchorRounding`, and `resolve_anchored_level` native_host.hpp:1012 is where a source language's own level arithmetic lives (`tests/test_native_anchored_legs.cpp`, `tests/test_native_arm_options.cpp`). | K | L7 | F:A11 O:A11 S:O6 | **Disagree:** F partial (toolkit is enough) / O no (kernel `TriggerAnchor`). R5-2: L7 delivers relative anchors; kernel anchor primary, toolkit materialization is the zero-kernel fallback. | +| OL12 | Working-order view for the host and outside observers (`strategy_pending_order_*`; `PendingIntentView` pine_adapter.hpp:1490-1506; the POD is filled only by the adapter `pine_birth_reach` pine_adapter.cpp:27146) | **no** — base returns 0 / -1 `observe_last_bar_dual_entry_path_v1` engine_consumer.cpp:124-133; the POD is TV-shaped (`tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135) **Closed for a bare host:** `native_working_requests()` native_host.hpp:1263 returns one owning `NativeWorkingRequest` native_host.hpp:681 per live request — definition, remaining, trigger state — with the C spelling `strategy_native_working_len_v1` / `_get_v1`. The TV-shaped POD stays frozen and stays the source layer's projection (`tests/test_native_limit_fill_through.cpp`, `tests/test_native_arm_options.cpp`, `tests/test_native_c_api.c`). | K+A | L7 (C++), L13 (C) | F:J5 O:J6 S:X4 | **Disagree:** O "partial + TV-shaped" / F, S no. A bare host reads zeros → no. R5-1: neutral view → kernel; POD shape and Pine journal retention stay A (POD frozen, §2.ii j). | +| OL13 | *(quirk)* Same-bar command batching and the seven deferred queues (`pending_bracket_legs_` pine_adapter.hpp:2545-2564); KI-62 open-order priority (`select` order_priority.cpp:13-60, selected `update_l4c_priority` pine_adapter.cpp:1367); per-bar priced-entry throttle, BUY-before-SELL batch, short-seed plan / collision (`maybe_activate_short_seed_plan` pine_adapter.cpp:4218, `materialize_snapshot` pine_adapter.cpp:14007), reversal-gap bracket policy (`ShortSeedPlan` pine_adapter.hpp:1390-1406, `DeferredOpenMarketableSell` pine_adapter.hpp:1875-1881, `CloseCallsiteState` pine_adapter.hpp:2013-2022; `apply_reversal_gap_bracket_policy` pine_adapter.cpp:21683, `reaccept_gapped_bracket_behind_same_id_add` pine_adapter.cpp:21917) | n/a — native priority is acceptance / incarnation order `native_events` native-engine.md:676-678 | A | — | F:A15 O:A15 S:O9 S:R3 | R5-1 names KI-62. S: the kernel exposes deterministic insertion order only. | +| OL14 | *(quirk)* Exit activation / suspension / revival barriers, historical birth reach (`select_exit_activation` exit_activation.cpp:44-90, enforced `leg_activation` pine_adapter.cpp:1264; `select_exit_activation` exit_activation.cpp:20-90, `select_historical_birth_reach` order_birth.cpp:5-14, `capture_order_birth` pine_adapter.cpp:1081) | n/a | A | — | S:R4 O:A10 | TV half of OL9. **V19D-P1 (R5 lanes B-ADAPTER, V19-FIX):** a margin-call revival never restores an exit the script cancelled: `strategy.cancel` / `cancel_all` retire every leg of each exit row they name — the live ones they take off the book (caused at the Cancelled event, after anything the same callback did to the leg), the dormant ones, and the ones the adapter took off the book itself — and drop a gapped stop a declined reversal parked for the next margin-call slice (`retire_cancelled_exits` pine_adapter.cpp:2126); the revival skips a withdrawn exit (`revive_brackets_after_margin` pine_adapter.cpp:2203). What `tests/test_adapter_margin_revival_cancel.cpp` replays on the NYSE:F 15m chart of seven lab tv tapes: badapter-v19dp1-control (no cancel: the margin call closes 928 through X) and v19fix-gapped-stop-control (X's stop at 10.15, gapped by the 05-01 open and parked: the same close), and badapter-v19dp1-cancel, v19fix-cancel-before-pair, v19fix-cancel-at-1000, v19fix-gapped-stop-cancel and v19fix-gapped-stop-cancel-after-pair (a cancel on 05-01 09:30 while X is dormant, on 04-30 15:45 before the reversal pair and on 04-30 10:00 while X is live, and on 05-01 09:30 and after the pair while X is parked; one tape file: margin calls 12 @10.39 and 92 @10.57, 836 open). Untaped, the same row holds the tapes' trades for `strategy.cancel_all` on 05-01 and before the pair and for `strategy.cancel` after the pair, and on the same bars with no quantity grid — where the pair holds X in the cancel's own callback — every spelling books what the cancel on the dormant exit books | ### 1.2 Trailing | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| TR1 | Trail arm → track → trigger, with a state query | **yes** — `Trail{offset, arm_price}` native_order.hpp:290-293; states `TrailWaitArm` native_order.hpp:642-660; arm / track / fire `TrailWaitArm` native_execution_consumer.cpp:6644; `NativeTrailState` native_host.hpp:669-674, `trail_state` native_host.hpp:1092 | K | — | F:B1 O:B1 S:T1 | — | -| TR2 | Trail activation / offset in ticks or points, relative to the entry fill (`trail_points_to_ticks` include/pineforge/compat/pine/trail_ticks.hpp:44 + grid snap exit_lifecycle.cpp:22-27; `trail_activation_level` pine_adapter.hpp:350-352) | **partial** — absolute prices and a raw price distance only **Closed:** `TrailTicks{n}` (`Trail::ticks` native_order.hpp:293) is resolved against `NativeRunSpec::price_tick` at acceptance, so a tick-spelled trail and its price-spelled equal behave identically bar for bar; a relative arm threshold is the `FromOwnerFill` anchor of OL11 (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K | L7 | F:B2 O:B2 S:T2 | **Disagree:** S classes all trail operand conversion as TV quirk. R5-1: tick / relative spelling is generic. | -| TR3 | Zero-distance trail, "ride the best, exit on any adverse tick" (adapter substitutes `tick * 0.5` pine_adapter.cpp:15061) | **no** — the kernel fails the run when `offset <= 0` (`checked_trail_stop` → native_execution_consumer.cpp:5379) **Closed:** a zero offset is legal and means *ride the best* — the level is the running best itself and the exit is the first print strictly past it; negative and nonfinite offsets stay rejected (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B3 F:B3 S:T2 | **Disagree:** O kernel / F, S quirk. R5-1: accepting a zero offset is generic; TV zero-offset *pricing* (`zero_trail_sibling_stop` pine_adapter.cpp:14978) stays A. | -| TR4 | Re-price a trail without losing the running extreme (adapter skips `replace` and reads `trail_state()` pine_adapter.cpp:4781) | **partial** — the read exists `trail_state` native_host.hpp:1092; `replace` always rebuilds trigger state **Closed:** `replace(handle, request, ReplaceOptions{/*retain_trigger_state=*/true})` carries the predecessor's live trigger state — a tracking trail's best, an already active stop — into the successor, refusing the replacement when the retained best leaves no representable level (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B4 F:B3 | Status is single-source (O). R5-1: retain-trail replace is generic; TV's "retained best across re-issue" convention (`retained_trail_best` pine_adapter.hpp:353-355) is A. | -| TR5 | *(quirk)* TV trail conventions: half-tick arm threshold against tick-quantized extremes `has_trail_request` pine_adapter.cpp:11202, zero-offset pricing `zero_trail_sibling_stop` pine_adapter.cpp:14978, retained best `retained_trail_best` pine_adapter.hpp:353-355, operand / restart rules `source_trail_points` pine_adapter.cpp:11199, `trail_one_shot` pine_adapter.cpp:12545 | n/a | A | — | F:B3 S:T2 | R5-1 | +| TR1 | Trail arm → track → trigger, with a state query | **yes** — `Trail{offset, arm_price}` native_order.hpp:292-295; states `TrailWaitArm` native_order.hpp:644-662; arm / track / fire `TrailWaitArm` native_execution_consumer.cpp:6644; `NativeTrailState` native_host.hpp:669-674, `trail_state` native_host.hpp:1092 | K | — | F:B1 O:B1 S:T1 | — | +| TR2 | Trail activation / offset in ticks or points, relative to the entry fill (`trail_points_to_ticks` include/pineforge/compat/pine/trail_ticks.hpp:44 + grid snap exit_lifecycle.cpp:22-27; `trail_activation_level` pine_adapter.hpp:353-355) | **partial** — absolute prices and a raw price distance only **Closed:** `TrailTicks{n}` (`Trail::ticks` native_order.hpp:295) is resolved against `NativeRunSpec::price_tick` at acceptance, so a tick-spelled trail and its price-spelled equal behave identically bar for bar; a relative arm threshold is the `FromOwnerFill` anchor of OL11 (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K | L7 | F:B2 O:B2 S:T2 | **Disagree:** S classes all trail operand conversion as TV quirk. R5-1: tick / relative spelling is generic. | +| TR3 | Zero-distance trail, "ride the best, exit on any adverse tick" (adapter substitutes `tick * 0.5` pine_adapter.cpp:15065) | **no** — the kernel fails the run when `offset <= 0` (`checked_trail_stop` → native_execution_consumer.cpp:5379) **Closed:** a zero offset is legal and means *ride the best* — the level is the running best itself and the exit is the first print strictly past it; negative and nonfinite offsets stay rejected (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B3 F:B3 S:T2 | **Disagree:** O kernel / F, S quirk. R5-1: accepting a zero offset is generic; TV zero-offset *pricing* (`zero_trail_sibling_stop` pine_adapter.cpp:14982) stays A. | +| TR4 | Re-price a trail without losing the running extreme (adapter skips `replace` and reads `trail_state()` pine_adapter.cpp:4780) | **partial** — the read exists `trail_state` native_host.hpp:1092; `replace` always rebuilds trigger state **Closed:** `replace(handle, request, ReplaceOptions{/*retain_trigger_state=*/true})` carries the predecessor's live trigger state — a tracking trail's best, an already active stop — into the successor, refusing the replacement when the retained best leaves no representable level (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B4 F:B3 | Status is single-source (O). R5-1: retain-trail replace is generic; TV's "retained best across re-issue" convention (`retained_trail_best` pine_adapter.hpp:356-358) is A. | +| TR5 | *(quirk)* TV trail conventions: half-tick arm threshold against tick-quantized extremes `has_trail_request` pine_adapter.cpp:11206, zero-offset pricing `zero_trail_sibling_stop` pine_adapter.cpp:14982, retained best `retained_trail_best` pine_adapter.hpp:356-358, operand / restart rules `source_trail_points` pine_adapter.cpp:11203, `trail_one_shot` pine_adapter.cpp:12549 | n/a | A | — | F:B3 S:T2 | R5-1 | ### 1.3 Sizing bases and reservations | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| SZ1 | Explicit units (`Transact{signed}` after a grid floor pine_adapter.cpp:7238, pine_adapter.cpp:7238) | **yes** — `Transact` in `OrderIntent` native_order.hpp:229 | K | — | F:C1 O:C1 S:S1 | — | -| SZ2 | Host override of the resolved price / units (`resolve_terms` pine_adapter.cpp:14904-14913) | **yes** — `HostSized` native_order.hpp:73-78 + `resolve_execution_terms` native_host.hpp:945-949, invoked `resolve_execution_terms` native_execution_consumer.cpp:5864, guarded by `consume_matched_request`'s `try` native_execution_consumer.cpp:5765-5799 | K | — | F:C2 O:H9 S:S2 | **Disagree:** S "partial" (a hook is not a basis). The hook is native; bases are SZ3-SZ5. It stays the custom-policy seam after L3. | -| SZ3 | Percent-of-equity sizing (`default_sizing_units` pine_adapter.cpp:3210) | **no** — `HostSized` without host units is `TermsUnresolved` native_execution_consumer.cpp:5893 **Closed:** `Sized{Side, EquityFraction{f}}` (`OrderIntent` native_order.hpp:229) is the sixth intent — the kernel resolves the units from a fraction of marked equity at the sizing point, so a bare host needs no terms override (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:C3 O:C2 S:S3 | TV parts stay A: the percentage's money rounded `source_money_round` pine_adapter.cpp:558 (not the equity: R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-sz-*), money floor-to-lot `source_money_floor_lot` pine_adapter.cpp:658 (→ SZ10) | -| SZ4 | Cash-value sizing (`default_qty_type` pine_adapter.cpp:3081) | **no** — same **Closed:** `Sized{Side, CashValue{cash}}`, the same resolution over an account-currency amount (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:C2 O:C3 S:S4 | A per-call `qty_type` (cash or percent of equity on one `strategy.entry`) is not Pine — TradingView's compiler refuses it, so only generated or C++ callers reach it — and since R5 lane B-ADAPTER it converts through the kernel's `native_sized_units` exactly as the declared default does (`typed_quantity_units` pine_adapter.cpp:3229, `typed_entry_units` pine_adapter.cpp:3247; `tests/test_adapter_typed_entry_admission.cpp`) | -| SZ5 | Fee-net sizing (commission reserve `/(1+c)`, asked of the kernel as `reserve_percent_fee` pine_adapter.cpp:3202; `percent_commission_live_equity` pine_adapter.cpp:5117) | **no** — only a dead primitive, `reserve_percent_commission` (since deleted), with no caller in `src/` **Closed:** `Sized::reserve_percent_fee` divides the sizing cash by `1 + fee_value / 100` under `NativeFeeKind::Percent` and is an exact no-op for every other fee kind; the dead `reserve_percent_commission` primitive is gone (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K | L3 | F:C3 O:C6 S:S3 | **Disagree:** F lists the reserve under quirk yet offers a knob / O kernel, "partial" / S knob. R5-1: fee-net option is generic. TradingView's cash-commission reserve stays adapter policy over the kernel's `CashValue`: a percent-of-equity default quantity under a cash-per-order or cash-per-contract fee takes its percentage of `strategy.equity` (`PineSizingSnapshot::strategy_equity`, the open entries' cash fees charged) and leaves out the fee its own order pays (`default_sizing_cash`, R5 lane PAR-CASHFEE, §3.10; `tests/test_cash_fee_sizing_tapes.cpp` on the tapes in the private fixture set `cash_fee_sizing`) | -| SZ6 | Placement-time vs fill-time sizing (`sizing_snapshot` pine_adapter.cpp:3036, `PineSizingSnapshot` pine_adapter.hpp:157-169; fill-time exceptions `default_sized` pine_adapter.cpp:8512) | **no** **Closed:** `SizeTime::AtMatch` (default) resolves at the matching candidate and `SizeTime::AtAcceptance` freezes the units against the command point, where the run's own opening admission also runs against the frozen quantity (`RequestRejectReason::PlacementAdmission`). Which price the basis converts at is `SizePrice::{Resolved, Signal, SignalOnTick}` (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K+A | L3 | F:C3 O:C7 S:S3 | **Disagree:** O adapter-only / F, S generic timing knob. R5-11 (L3 scope) + R5-1: the knob is kernel; TV's frozen tuple stays A through `HostSized`. | -| SZ7 | Quantity step | **yes** — `quantity_grid` admission native_run_spec.hpp:660, native_order.hpp:1421-1438; host-sized terms snap (`ExecutionGridPolicy` native_order.hpp:71-85); exact grid validation `execution_terms_grid_representable` native_execution_consumer.cpp:506; the book's own quantities are on the grid -- a whole-scope `ScopeFraction` and a `Reduce` of a FIFO boundary of its scope (R5 lane K-ULP4) | K | — | F:C4 O:C5 S:S1 | **Disagree:** F, O partial / S yes. R5-3: the quantity grid is native already. L3's `Sized` intent reuses the snap policy. | -| SZ8 | Fractional reduce of the position / a cohort (`quantize_close_units` pine_adapter.cpp:5237, `compute_exit_reservation` pine_adapter.cpp:5277) | **no** — sizes are `ExplicitUnits` / `OwnerOpenedUnits` / `HostSized` only native_order.hpp:60-74 **Closed:** `Reduce{ScopeFraction{fraction, claim, basis}}` takes a fraction in `(0, 1]` of the scope the reduce is bound to — the book, one opening, or a cohort roster (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:A7 O:A12 O:C4 S:S5 | F's "partial" counts the units-reduce of OL2. R5-4 → L3. | -| SZ9 | Exit-quantity reservation against sibling exits (`compute_exit_reservation` pine_adapter.cpp:5277, `reserved_exit_qty` pine_adapter.cpp:11609) | **partial** — `PointBudget` native_order.hpp:379-382 and group `Reduce` native_order.hpp:220 give a generic reservation; no net-of-siblings claim for fraction sizes **Closed:** `ScopeClaim::NetOfSiblings` subtracts the units already claimed by live sibling reduces bound to the same scope, and `ScopeBasis::{AtMatch, AtAcceptance}` chooses when that scope is measured (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:A14 S:S5 S:O10 O:A12 | F proposed nothing. R5-4: generic half → L3; the Pine ledger → SZ11, SZ12. | -| SZ10 | *(quirk)* Ten-significant-digit money rule (`source_money_round` pine_adapter.cpp:558, `source_money_floor_lot` pine_adapter.cpp:658; twin `tv_money_round` pine_policy_support.hpp:9-15); exact percent-exit rounding (`quantize_percent_exit_units` pine_adapter.cpp:5259) | n/a — kernel money is plain binary64 `marked_equity` engine_execution.cpp:1753 | A | — | O:C8 F:C3 S:M4 S:S5 | R5-1 | -| SZ10a | *(quirk, measured retention)* The money band's affordable price, `sig10(sig10(E) / notional per unit of price)` against the sizing price — at placement (`entry`, `affordable_price` pine_adapter.cpp:8555), in the paired-reversal whole drop (`close`, `affordable_price` pine_adapter.cpp:10079), at the terms boundary (`resolve_terms`, `affordable_price` pine_adapter.cpp:16272), in the precommit money scopes (`validate_precommit`, `affordable_price` pine_adapter.cpp:16803, pine_adapter.cpp:16856) and in the carried-money check (`apply_reversal_gap_bracket_policy`, `affordable_price` pine_adapter.cpp:21727) | n/a — not a units conversion. Every money-to-units conversion in the adapter is the kernel's `native_sized_units` under the source's floors since R5 lane B-ADAPTER: the default at placement and at the fill (`default_sizing_units` pine_adapter.cpp:3210), the typed per-call quantity at placement, at the fill and in the probe (`typed_quantity_units` pine_adapter.cpp:3229, `typed_entry_units` pine_adapter.cpp:3247), and the pending-order fill-quantity probe (`probe_fill_qty` pine_adapter.cpp:26728) | A | — | AUDIT3-opus2 H4 (G2 R2, R3) | **Measured:** the quotient is TradingView's whole-drop PRICE, pinned by 445 famr3 tapes and 202 famr tapes (two tests that live in the private fixtures); on a seven-digit tie (E 9998.99999996, 10 000 lots at tick(close) 0.9999) the rule admits while the kernel's conversion buys 9999.99999996 lots and binary64 gives E/Q 0.999899999996 (`tests/test_adapter_fill_qty_probe.cpp`, section 2). | -| SZ11 | *(quirk)* Same-bar `strategy.close` callsite batching, two-call provenance, entry-id ledger (`enqueue_pooc_fifo_close` pine_adapter.cpp:9206; ledger `close_batch_bar_` pine_adapter.hpp:2599; the kernel's comment-only residue was deleted by R5 lanes E6 and E10) | n/a | A | — | S:O8 F:A15 O:A15 | **Disagree:** S "generic → kernel" (its P2 `ReplaceByKey` / `Fifo`). R5-4: adapter. | -| SZ12 | *(quirk)* POOC reservation-growth market-entry population predicate (`select_reservation_growth_sources` src/compat/pine/reservation_expansion.cpp:9-34, called from the adapter, `select_reservation_growth_sources` pine_adapter.cpp:1324) | n/a | A | — | S:O10 | (single-source) R5-4 confirms S's own split: concept → SZ9, predicate A. | +| SZ1 | Explicit units (`Transact{signed}` after a grid floor pine_adapter.cpp:7242, pine_adapter.cpp:7242) | **yes** — `Transact` in `OrderIntent` native_order.hpp:231 | K | — | F:C1 O:C1 S:S1 | — | +| SZ2 | Host override of the resolved price / units (`resolve_terms` pine_adapter.cpp:14908-14917) | **yes** — `HostSized` native_order.hpp:75-80 + `resolve_execution_terms` native_host.hpp:945-949, invoked `resolve_execution_terms` native_execution_consumer.cpp:5864, guarded by `consume_matched_request`'s `try` native_execution_consumer.cpp:5765-5799 | K | — | F:C2 O:H9 S:S2 | **Disagree:** S "partial" (a hook is not a basis). The hook is native; bases are SZ3-SZ5. It stays the custom-policy seam after L3. | +| SZ3 | Percent-of-equity sizing (`default_sizing_units` pine_adapter.cpp:3209) | **no** — `HostSized` without host units is `TermsUnresolved` native_execution_consumer.cpp:5893 **Closed:** `Sized{Side, EquityFraction{f}}` (`OrderIntent` native_order.hpp:231) is the sixth intent — the kernel resolves the units from a fraction of marked equity at the sizing point, so a bare host needs no terms override (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:C3 O:C2 S:S3 | TV parts stay A: the percentage's money rounded `source_money_round` pine_adapter.cpp:559 (not the equity: R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-sz-*), money floor-to-lot `source_money_floor_lot` pine_adapter.cpp:659 (→ SZ10) | +| SZ4 | Cash-value sizing (`default_qty_type` pine_adapter.cpp:3080) | **no** — same **Closed:** `Sized{Side, CashValue{cash}}`, the same resolution over an account-currency amount (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:C2 O:C3 S:S4 | A per-call `qty_type` (cash or percent of equity on one `strategy.entry`) is not Pine — TradingView's compiler refuses it, so only generated or C++ callers reach it — and since R5 lane B-ADAPTER it converts through the kernel's `native_sized_units` exactly as the declared default does (`typed_quantity_units` pine_adapter.cpp:3228, `typed_entry_units` pine_adapter.cpp:3246; `tests/test_adapter_typed_entry_admission.cpp`) | +| SZ5 | Fee-net sizing (commission reserve `/(1+c)`, asked of the kernel as `reserve_percent_fee` pine_adapter.cpp:3201; `percent_commission_live_equity` pine_adapter.cpp:5116) | **no** — only a dead primitive, `reserve_percent_commission` (since deleted), with no caller in `src/` **Closed:** `Sized::reserve_percent_fee` divides the sizing cash by `1 + fee_value / 100` under `NativeFeeKind::Percent` and is an exact no-op for every other fee kind; the dead `reserve_percent_commission` primitive is gone (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K | L3 | F:C3 O:C6 S:S3 | **Disagree:** F lists the reserve under quirk yet offers a knob / O kernel, "partial" / S knob. R5-1: fee-net option is generic. TradingView's cash-commission reserve stays adapter policy over the kernel's `CashValue`: a percent-of-equity default quantity under a cash-per-order or cash-per-contract fee takes its percentage of `strategy.equity` (`PineSizingSnapshot::strategy_equity`, the open entries' cash fees charged) and leaves out the fee its own order pays (`default_sizing_cash`, R5 lane PAR-CASHFEE, §3.10; `tests/test_cash_fee_sizing_tapes.cpp` on the tapes in the private fixture set `cash_fee_sizing`) | +| SZ6 | Placement-time vs fill-time sizing (`sizing_snapshot` pine_adapter.cpp:3035, `PineSizingSnapshot` pine_adapter.hpp:163-175; fill-time exceptions `default_sized` pine_adapter.cpp:8516) | **no** **Closed:** `SizeTime::AtMatch` (default) resolves at the matching candidate and `SizeTime::AtAcceptance` freezes the units against the command point, where the run's own opening admission also runs against the frozen quantity (`RequestRejectReason::PlacementAdmission`). Which price the basis converts at is `SizePrice::{Resolved, Signal, SignalOnTick}` (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K+A | L3 | F:C3 O:C7 S:S3 | **Disagree:** O adapter-only / F, S generic timing knob. R5-11 (L3 scope) + R5-1: the knob is kernel; TV's frozen tuple stays A through `HostSized`. | +| SZ7 | Quantity step | **yes** — `quantity_grid` admission native_run_spec.hpp:660, native_order.hpp:1423-1440; host-sized terms snap (`ExecutionGridPolicy` native_order.hpp:73-87); exact grid validation `execution_terms_grid_representable` native_execution_consumer.cpp:506; the book's own quantities are on the grid -- a whole-scope `ScopeFraction` and a `Reduce` of a FIFO boundary of its scope (R5 lane K-ULP4) | K | — | F:C4 O:C5 S:S1 | **Disagree:** F, O partial / S yes. R5-3: the quantity grid is native already. L3's `Sized` intent reuses the snap policy. | +| SZ8 | Fractional reduce of the position / a cohort (`quantize_close_units` pine_adapter.cpp:5236, `compute_exit_reservation` pine_adapter.cpp:5276) | **no** — sizes are `ExplicitUnits` / `OwnerOpenedUnits` / `HostSized` only native_order.hpp:62-76 **Closed:** `Reduce{ScopeFraction{fraction, claim, basis}}` takes a fraction in `(0, 1]` of the scope the reduce is bound to — the book, one opening, or a cohort roster (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:A7 O:A12 O:C4 S:S5 | F's "partial" counts the units-reduce of OL2. R5-4 → L3. | +| SZ9 | Exit-quantity reservation against sibling exits (`compute_exit_reservation` pine_adapter.cpp:5276, `reserved_exit_qty` pine_adapter.cpp:11613) | **partial** — `PointBudget` native_order.hpp:381-384 and group `Reduce` native_order.hpp:222 give a generic reservation; no net-of-siblings claim for fraction sizes **Closed:** `ScopeClaim::NetOfSiblings` subtracts the units already claimed by live sibling reduces bound to the same scope, and `ScopeBasis::{AtMatch, AtAcceptance}` chooses when that scope is measured (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:A14 S:S5 S:O10 O:A12 | F proposed nothing. R5-4: generic half → L3; the Pine ledger → SZ11, SZ12. | +| SZ10 | *(quirk)* Ten-significant-digit money rule (`source_money_round` pine_adapter.cpp:559, `source_money_floor_lot` pine_adapter.cpp:659; twin `tv_money_round` pine_policy_support.hpp:9-15); exact percent-exit rounding (`quantize_percent_exit_units` pine_adapter.cpp:5258) | n/a — kernel money is plain binary64 `marked_equity` engine_execution.cpp:1753 | A | — | O:C8 F:C3 S:M4 S:S5 | R5-1 | +| SZ10a | *(quirk, measured retention)* The money band's affordable price, `sig10(sig10(E) / notional per unit of price)` against the sizing price — at placement (`entry`, `affordable_price` pine_adapter.cpp:8559), in the paired-reversal whole drop (`close`, `affordable_price` pine_adapter.cpp:10083), at the terms boundary (`resolve_terms`, `affordable_price` pine_adapter.cpp:16276), in the precommit money scopes (`validate_precommit`, `affordable_price` pine_adapter.cpp:16807, pine_adapter.cpp:16860) and in the carried-money check (`apply_reversal_gap_bracket_policy`, `affordable_price` pine_adapter.cpp:21731) | n/a — not a units conversion. Every money-to-units conversion in the adapter is the kernel's `native_sized_units` under the source's floors since R5 lane B-ADAPTER: the default at placement and at the fill (`default_sizing_units` pine_adapter.cpp:3209), the typed per-call quantity at placement, at the fill and in the probe (`typed_quantity_units` pine_adapter.cpp:3228, `typed_entry_units` pine_adapter.cpp:3246), and the pending-order fill-quantity probe (`probe_fill_qty` pine_adapter.cpp:26733) | A | — | AUDIT3-opus2 H4 (G2 R2, R3) | **Measured:** the quotient is TradingView's whole-drop PRICE, pinned by 445 famr3 tapes and 202 famr tapes (two tests that live in the private fixtures); on a seven-digit tie (E 9998.99999996, 10 000 lots at tick(close) 0.9999) the rule admits while the kernel's conversion buys 9999.99999996 lots and binary64 gives E/Q 0.999899999996 (`tests/test_adapter_fill_qty_probe.cpp`, section 2). | +| SZ11 | *(quirk)* Same-bar `strategy.close` callsite batching, two-call provenance, entry-id ledger (`enqueue_pooc_fifo_close` pine_adapter.cpp:9210; ledger `close_batch_bar_` pine_adapter.hpp:2617; the kernel's comment-only residue was deleted by R5 lanes E6 and E10) | n/a | A | — | S:O8 F:A15 O:A15 | **Disagree:** S "generic → kernel" (its P2 `ReplaceByKey` / `Fifo`). R5-4: adapter. | +| SZ12 | *(quirk)* POOC reservation-growth market-entry population predicate (`select_reservation_growth_sources` src/compat/pine/reservation_expansion.cpp:9-34, called from the adapter, `select_reservation_growth_sources` pine_adapter.cpp:1325) | n/a | A | — | S:O10 | (single-source) R5-4 confirms S's own split: concept → SZ9, predicate A. | ### 1.4 Margin, admission, liquidation, risk | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| | MG1 | Allowed opening directions (`allowed_open_directions` pine_adapter.cpp:2861) | **yes** — `allowed_open_directions` native_run_spec.hpp:665 → `MatchRejectReason::OpeningDirection` native_execution_consumer.cpp:3745 | K | — | F:D4 O:D7 S:M1 | the one risk rule the adapter delegates (O) | -| MG2 | Max absolute position size | **yes** — `max_abs_units` native_run_spec.hpp:663 tests the resulting book native_execution_consumer.cpp:3752 | K+A | — | F:D5 O:D5 S:M1 S:M5 | **Disagree:** O, S:M5 partial. R5-1: the generic cap is native; the adapter's live-position `>= held` variant (`max_position_size` pine_adapter.cpp:16551) is TV. | -| MG3 | Open-lot cap (pyramiding) | **yes** — `max_open_lots` native_run_spec.hpp:664 → `MaxOpenLots` native_execution_consumer.cpp:3757 | K+A | — | F:A5 O:A13 O:D6 S:M1 | **Disagree:** F, O:A13 partial / O:D6, S yes. R5-1: Pine's per-cycle entry count (`accepted_in_cycle` pine_adapter.cpp:7743, cap left unset `max_open_lots` pine_adapter.cpp:2648) is TV. **Measured (R5 lane H-MEASURE), and not TradingView's rule:** on 15 scenarios of three `lab tv` tapes (the private test `test_pyramiding_count_differential`) TradingView checks an entry once, at its first eligible point, against the trades then open; the kernel's `max_open_lots` books 14 as TradingView does, the adapter's count 13 (it keeps a slot a close or another entry's exit drained; since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap). The fifteenth, a resting entry accepted below the cap and filled at it, the kernel's cap refuses; the adapter books it since lane W8A-SIGSTATE-1's R-B, which also stopped the count from counting a resting LIMIT entry (a pending LIMIT entry does not count at a later call, and one the cap admitted fills over it: the private fixture set `pyramiding_open_order`). A recorded divergence, retained pending a re-lowering ruling. **Re-lowering tried by R5 lane H-THIN and not taken:** on the kernel's cap four corpus probes moved away from TradingView, for want of that first-eligible-point rule, so the count stays. The lane fixed the reservation divergence H-MEASURE measured beside it (P10): a default `strategy.exit(from_entry)` under FIFO reserved the whole book (200 in the adapter, 100 on TradingView's tape); it now reserves the entry's own quantity, the oldest lots first (`fifo_named_entry` pine_adapter.cpp:11633). | -| MG4 | Initial-margin opening gate (adapter answers `AdmitWithHostMargin` from frozen signal-time equity pine_adapter.cpp:17364 and leaves the kernel gate unset pine_adapter.cpp:17383) | **yes** — `initial_margin_fraction` native_run_spec.hpp:666; the requirement `resulting_abs_notional * fraction` native_execution_consumer.cpp:3775 against marked equity. **Conversion instant, R5 follow-up lane E21:** both sites of the gate convert at their own point's rate — the candidate at its cursor, the placement gate at the acceptance point where a `Sized{SizeTime::AtAcceptance}` freezes its units — for the notional, the equity and a percent ticket, and the freeze's equity basis with them. The rate is threaded, never the clock: `make_command_context` native_execution_consumer.cpp:3621 hands the point's rate to the freeze, to `marked_equity_at` and to the engine's rate-explicit settlement inspection `inspect_native_settlement_scoped_at` engine.hpp:650, and the engine's presented clock is never written (`tests/test_native_margin_fx_clock.cpp` sections 8, 10 and 11) | K | — | F:D1 O:D1 S:M2 | **Disagree:** F, O partial (they fold per-side in) / S yes. Row split → MG5. **Measured (R5 lane H-MEASURE, M11):** the two gates part both ways on the same bars — the kernel's declines a 1× long whose entry fee the adapter admits, and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times (the private test `test_adapter_margin_schedule_differential`). **Since R5 lane TAIL-C** an explicit-quantity market buy from flat is costed at its signal close's tick plus the slippage ticks its fill carries, off `process_orders_on_close` (the private test `test_explicit_market_admission_tapes`), and an infinite `strategy.entry` quantity trades the default quantity, as na does (`tests/test_nonfinite_entry_qty_tape.cpp`). | -| MG5 | Per-side initial margin (`margin_pct` pine_adapter.cpp:17617, `margin_pct` pine_adapter.cpp:17119) | **no** — one `std::optional` `initial_margin_fraction` native_run_spec.hpp:666 **Closed:** `NativeMarginModel::initial_long` / `initial_short` native_run_spec.hpp:242, opt-in and mutually exclusive with `initial_margin_fraction`; `initial_ == 0.0` is the maintenance-only spelling that leaves opening admission to the host at both of its gates (`tests/test_native_margin_model.cpp`). | K | L4 | F:D1 O:D2 | R5-1 | -| MG6 | Maintenance margin + forced liquidation (`submit_margin_call_slice` pine_adapter.cpp:18004, `submit_margin_call_units` pine_adapter.cpp:18037, `schedule_margin_call_path` pine_adapter.cpp:20286) | **no** — "no maintenance liquidation" `initial_margin_fraction` native_run_spec.hpp:666-667; margin is checked only when `inspect.would_open` native_execution_consumer.cpp:3734 **Closed:** `maintenance_long` / `maintenance_short` with `NativeLiquidationCheck::{PathAdverseExtreme, PathAdverseExtremeMark, CalculationOnly}`; the kernel solves the level, rests its own `Reduce`/`Flatten` bound to the live book, re-prices it with `CancelReason::Superseded` and books a `MarginCallEvent` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | F:D2 O:D3 S:M3 | TV parts → MG15 | -| MG7 | Liquidation slice sizing (grid floor, ×4 restore, one-contract fallback `source_margin_units` pine_adapter.cpp:17661) | **no** **Closed:** `NativeLiquidationSizing::{RestoreMinimum, ShortfallMultiple, Flatten}` with `shortfall_multiple` and `liquidation_min_units`, capped at the held quantity and floored onto `quantity_grid`; `resolve_margin_call_units` native_host.hpp:995 gives a host the last word, clamped into `(0, held]` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | O:D4 F:D2 S:M4 | **Disagree:** F generic knob `ShortfallMultiple` / O "×4 is A" / S quirk, yet a 4.0 default inside its generic model. R5-1: policy knob in the kernel, plain default (restore-minimum). **Decided by R5 lane F7 (the D2 question):** the adapter selects NO kernel knob — `project()` leaves `sizing` / `shortfall_multiple` / `liquidation_min_units` unset and its `resolve_margin_call_units` answers every call with TradingView's slice (the restore floored onto the lot grid BEFORE the ×4, floored again, the one-contract band, the dust gate). Measured: on a one-unit grid, 20 long at 100 on 1000 at 50 % reaching 93.9, TradingView books 4 lots and the kernel's own `ShortfallMultiple 4` books 5 (`tests/test_native_margin_hooks_twin.cpp` MG-F3, an executed run, no longer a comment); without a grid the two agree bit for bit (MG-F/H, 4.2105263157894735), but declaring the knob for those runs alone folds `NativeMarginModel::sizing` into every margin run's identity — an epoch decision F7 does not take. Since R5 lane H-THIN the pre-open admission slice (`schedule_preopen_margin_slice`) sizes through the same `source_margin_units`, where it kept its own copy of the ×4 without the band or the dust gates (AUDIT4 §6 G2-12; the private test `test_adapter_margin_schedule_differential` M10: band and fudge now 1 and 1, the kernel's and `ab9714be`'s). A four-times call on a whole-unit grid whose book's oldest lot is one unit is followed by one more unit at the next check point (`whole_unit_follow_up_due` pine_adapter.cpp:19193; R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-es-*, w5b-sb-es-*, w5b-f-q1x12). With slippage the one-contract band takes its unit only where that unit restores the book at its own fill: the position's requirement at the liquidation's slipped print, less the equity at the mark, under one unit's margin (`source_margin_units` pine_adapter.cpp:17661; lane INT28-FIX rule OU, the private fixture set `slipped_short`). A one-unit call that leaves the book short at its own fill, by more than one unit's margin and two slippage steps with the book re-marked there, is followed by one more unit at the bar's next path point, where that point's own check calls nothing and its print is inside the call's fill (lane INT28-FIX rule OF). Under a percent commission a call of any size is followed so, on a fractional lot grid too: while the book the call left, re-marked at its fill, is short by more than two slippage steps for each unit called and the exit fees of those units and of one more, the next path point inside that fill takes four times the shortfall's lot floor over the fill, or one unit where that floors to none (`margin_follow_up_units` pine_adapter.cpp:19235, `follow_margin_call` pine_adapter.cpp:19284; R5 lane TAIL-I, the private fixture set `margin_open_print`). | -| MG8 | Liquidation price query (`margin_liquidation_price` pine_strategy_host.cpp:172-188) | **no** **Closed:** `native_liquidation_price()` native_host.hpp:1320 answers the exact solved level, `nullopt` where none exists; `NativeLiquidationLevelBase` chooses whether the solve answers for the open entries' fees (`tests/test_native_margin_model.cpp`, `tests/test_native_open_lots.cpp`). | K+A | L4 | F:D3 S:R5 | **Disagree:** F generic / S adapter-only. R5-1: liquidation price is generic; the Pine spelling stays A. **Lowered by R5 lane H-THIN:** the Pine host's second solve (`compute_liquidation_price`) is gone; `strategy.margin_liquidation_price` reads `native_liquidation_price()`, which the adapter configures with `NativeLiquidationLevelBase::RealizedOnly` (the Pine formula's closed-money solve, bit for bit at every bar of the lane's 1,021 seeded margin-call runs), and keeps only TradingView's tick spelling: floored for a long, ceiled for a short. A run whose margin calls are switched off declares no margin model, so it reads na. | -| MG9 | FX revaluation of margin (`apply_fx_open_margin_slice` pine_adapter.cpp:5704, `apply_fx_opening_margin_slice` pine_adapter.cpp:5716) | **no** — staged FX curve `configure_native_fx_curve` native_host.hpp:1202 but no margin re-check. **Closed by audit lane N6:** `NativeMarginCheckKind::FxRoll` — a step of the declared curve is a check point of its own, measured at the unchanged price before the first point the new rate converts (`tests/test_native_margin_fx_roll.cpp`). The adapter refuses the point by kind, for a measured reason since R5 lane F7: TradingView's rollover is a broker-open checkpoint at the open price, now at any positive margin (`apply_fx_open_margin_slice`, on the one TradingView money / units rule every other checkpoint uses), while the roll point measures at the remaining path's adverse mark and rests the slice there — a carried 1x long on a step bar O 100 L 97 books 0.4116 @ 97 on the adapter's run shape and 0.4036 @ 99, measured a bar early, on a strict calendar, against TradingView's 0.3996 @ 100 (`tests/test_native_margin_hooks_twin.cpp` MG-FX). Both N6 follow-ups are closed (§4.4). **Completed by R5 follow-up lane E3:** the conversion INSTANT is now generic too — every check kind converts at its own cursor's rate (`account_currency_fx_at(cursor.point.effective_time_ms)`) for the requirement, the equity and the level, in batch and stream, so a step landing between a fill's cursor and the clock the engine presents no longer breaches at the fill (`tests/test_native_margin_fx_clock.cpp`; N6's local clock swap inside `fx_roll_margin_check` is removed as redundant). **Extended by R5 follow-up lane E21** to the opening gate's placement site (MG4), and the rate-explicit equity E3 restated beside the engine's accessor is now that accessor's one implementation, `BacktestEngine::marked_equity_at` engine.hpp:730 | K | L4 → N6 → E3 → E21 | O:D8 | (single-source) | -| MG10 | Max drawdown halt (`update_risk_state` pine_adapter.cpp:17541) | **no** — observation field only `drawdown_limit` include/pineforge/source/market_admission.hpp:51; composable today from `native_marked_equity` native_host.hpp:1301 **Closed:** `NativeRiskLimits::max_drawdown` native_run_spec.hpp:311, a `NativeLossLimit{value, percent}` measured against the running peak equity at three points of every script bar, blocking openings for the rest of the run (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E1 S:M6 | R5-9 | -| MG11 | Max consecutive losing days (`SourceDayLedger` pine_adapter.hpp:1423-1431; gate `max_cons_loss_days` pine_adapter.cpp:16544) | **no** — observation only `loss_days_limit` include/pineforge/source/market_admission.hpp:50 **Closed:** `NativeRiskLimits::max_consecutive_loss_days`, settled from each day's realized result at the next day's open (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E2 S:M6 | R5-9 | -| MG12 | Max intraday loss → flatten + block (`intraday_loss_breached` pine_adapter.cpp:17574, forced flatten `submit_intraday_loss_close` pine_adapter.cpp:20920) | **no** — observation only `intraday_loss_limit` include/pineforge/source/market_admission.hpp:52 **Closed:** `NativeRiskLimits::max_intraday_loss` with `NativeRiskAction::FlattenAndBlock`, under which the kernel issues one `Flatten` of its own at the breach point (`RequestOrigin::KernelRisk`) and latches the block before that flatten settles (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D7 O:E3 S:M6 | R5-9 | -| MG13 | Max filled orders per day (`IntradayCap` intraday_cap.hpp:84-263, `IntradayOrderBudget` intraday_order_budget.hpp:27-92; `execute_due_cap_close` pine_adapter.cpp:20963) | **no** **Closed:** `NativeRiskLimits::max_fills_per_day` counts every applied execution of the day as it settles, whatever issued it (`tests/test_native_risk_limits.cpp`). | K+A | L9 | F:D8 O:E4 S:M7 | Count cap K; quota transfer, POOC deferral, TV close price A | -| MG14 | Risk-day boundary (`chart_day_key` pine_adapter.cpp:17438) | **no** — the calendar exists `session_day_ordinal` native_calendar.hpp:416-419, there is no risk-day ledger **Closed:** `NativeRiskDay::{SessionDay, CalendarDayInTimezone}` keys the risk day, and `native_risk_state()` native_host.hpp:1329 reads the whole ledger back — block and reason, day, fills counted, loss-day streak, peak equity, day-opening equity (`tests/test_native_risk_limits.cpp`). | K+A | L9 | O:E5 S:M7 | The chart-day shortcut stays A; the kernel keys on session / calendar day | -| MG15 | *(quirk)* TV margin call: money rounding `source_margin_money` pine_adapter.cpp:17606, 4× shortfall `4.0 * minimum` pine_adapter.cpp:17695, POOC chronology exception `resting_order` pine_adapter.cpp:20306 (the commissioned carried short; the competing-entry and rounded-money exceptions retired by R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-pc-*), 1×-long money call `submit_tv_money_long_margin_call` pine_adapter.cpp:18603, fill pricing (the slice books at its waypoint's mark, `submit_tv_money_long_margin_call` pine_adapter.cpp:18603) | n/a | A | — | F:D2 F:G4 O:D4 S:M4 | R5-1. **Measured (R5 lane H-MEASURE)** against the kernel's own points on the same bars and orders, the kernel's hooks answering TradingView's money and slice (the private test `test_adapter_margin_schedule_differential`): a gap-open breach executes at the open, sized on the open's money, where the kernel's mark check rests at the path's adverse extreme and `CalculationOnly` executes at the calculation (M8); the pre-open slice parts from the kernel's `AfterApplied` point, and from `ab9714be`, on the one-contract band and the restore's `+1e-6` floor (M10; the frozen units it also parted on are TradingView's book since R5 lane PAR-MARGIN froze a default-percent stop above 100 % as eight `lab tv` tapes, the private tapes `pm-m10-*` of `margin_entry_bar`, show TradingView does); the entry bar of a leveraged opening, where four `lab tv` tapes (the private fixture set `margin_entry_bar`) and the kernel's `AfterApplied` point book the margin call at that bar's low, is admitted by `on_applied` since R5 lane PAR-MARGIN on the run shapes `ab9714be` checked it on (the pre-open slice's own opening keeps that slice's verdict), and the adapter books the tapes' rows (M7; H-MEASURE had recorded it as a divergence); R5 lane PAR-MARGIN-2 extended it to limit openings, `process_orders_on_close`, `calc_on_order_fills` and adds to a carried book, and made the kernel's post-fill point reach the low a limit was filled on the way down to, or a short's high on its way up (twenty-eight `lab tv` tapes, the private tapes `pm2-*` of `margin_entry_bar`). Under the magnifier the kernel's `IntrabarSample` point (R5 lane PAR-MARGIN) is admitted on a leveraged long, plain, under `process_orders_on_close` or `calc_on_order_fills`, and on a short outside `process_orders_on_close` (PAR-MARGIN-2), where TradingView books its call at the first lower-timeframe low that crosses -- at its own 2-minute intrabars on a 15-minute chart, which a model reproduces on all 20 magnified tapes (the private fixture set `intrabar_margin`); six tapes part on the one-minute samples the host feeds, a recorded divergence of the magnifier's sampling. A margin call books the nearest tick of its print, as TradingView's market fills do (12.105 books 12.11; the private fixture set `half_tick_rounding`, PAR-MARGIN-2). | -| MG16 | *(quirk)* Historical market-admission review / retention journal (`explicit_pair_scope` src/compat/pine/market_admission.cpp:34 and the review fold it feeds (`original_pair_call` src/compat/pine/market_admission.cpp:48); `admission_allocation` pine_adapter.cpp:4571, `record_market_review` pine_adapter.cpp:21219) | n/a | A | — | S:R2 | (single-source) | +| MG2 | Max absolute position size | **yes** — `max_abs_units` native_run_spec.hpp:663 tests the resulting book native_execution_consumer.cpp:3752 | K+A | — | F:D5 O:D5 S:M1 S:M5 | **Disagree:** O, S:M5 partial. R5-1: the generic cap is native; the adapter's live-position `>= held` variant (`max_position_size` pine_adapter.cpp:16555) is TV. | +| MG3 | Open-lot cap (pyramiding) | **yes** — `max_open_lots` native_run_spec.hpp:664 → `MaxOpenLots` native_execution_consumer.cpp:3757 | K+A | — | F:A5 O:A13 O:D6 S:M1 | **Disagree:** F, O:A13 partial / O:D6, S yes. R5-1: Pine's per-cycle entry count (`accepted_in_cycle` pine_adapter.cpp:7747, cap left unset `max_open_lots` pine_adapter.cpp:2648) is TV. **Measured (R5 lane H-MEASURE), and not TradingView's rule:** on 15 scenarios of three `lab tv` tapes (the private test `test_pyramiding_count_differential`) TradingView checks an entry once, at its first eligible point, against the trades then open; the kernel's `max_open_lots` books 14 as TradingView does, the adapter's count 13 (it keeps a slot a close or another entry's exit drained; since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap). The fifteenth, a resting entry accepted below the cap and filled at it, the kernel's cap refuses; the adapter books it since lane W8A-SIGSTATE-1's R-B, which also stopped the count from counting a resting LIMIT entry (a pending LIMIT entry does not count at a later call, and one the cap admitted fills over it: the private fixture set `pyramiding_open_order`). A recorded divergence, retained pending a re-lowering ruling. **Re-lowering tried by R5 lane H-THIN and not taken:** on the kernel's cap four corpus probes moved away from TradingView, for want of that first-eligible-point rule, so the count stays. The lane fixed the reservation divergence H-MEASURE measured beside it (P10): a default `strategy.exit(from_entry)` under FIFO reserved the whole book (200 in the adapter, 100 on TradingView's tape); it now reserves the entry's own quantity, the oldest lots first (`fifo_named_entry` pine_adapter.cpp:11637). | +| MG4 | Initial-margin opening gate (adapter answers `AdmitWithHostMargin` from frozen signal-time equity pine_adapter.cpp:17368 and leaves the kernel gate unset pine_adapter.cpp:17387) | **yes** — `initial_margin_fraction` native_run_spec.hpp:666; the requirement `resulting_abs_notional * fraction` native_execution_consumer.cpp:3775 against marked equity. **Conversion instant, R5 follow-up lane E21:** both sites of the gate convert at their own point's rate — the candidate at its cursor, the placement gate at the acceptance point where a `Sized{SizeTime::AtAcceptance}` freezes its units — for the notional, the equity and a percent ticket, and the freeze's equity basis with them. The rate is threaded, never the clock: `make_command_context` native_execution_consumer.cpp:3621 hands the point's rate to the freeze, to `marked_equity_at` and to the engine's rate-explicit settlement inspection `inspect_native_settlement_scoped_at` engine.hpp:650, and the engine's presented clock is never written (`tests/test_native_margin_fx_clock.cpp` sections 8, 10 and 11) | K | — | F:D1 O:D1 S:M2 | **Disagree:** F, O partial (they fold per-side in) / S yes. Row split → MG5. **Measured (R5 lane H-MEASURE, M11):** the two gates part both ways on the same bars — the kernel's declines a 1× long whose entry fee the adapter admits, and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times (the private test `test_adapter_margin_schedule_differential`). **Since R5 lane TAIL-C** an explicit-quantity market buy from flat is costed at its signal close's tick plus the slippage ticks its fill carries, off `process_orders_on_close` (the private test `test_explicit_market_admission_tapes`), and an infinite `strategy.entry` quantity trades the default quantity, as na does (`tests/test_nonfinite_entry_qty_tape.cpp`). | +| MG5 | Per-side initial margin (`margin_pct` pine_adapter.cpp:17621, `margin_pct` pine_adapter.cpp:17123) | **no** — one `std::optional` `initial_margin_fraction` native_run_spec.hpp:666 **Closed:** `NativeMarginModel::initial_long` / `initial_short` native_run_spec.hpp:242, opt-in and mutually exclusive with `initial_margin_fraction`; `initial_ == 0.0` is the maintenance-only spelling that leaves opening admission to the host at both of its gates (`tests/test_native_margin_model.cpp`). | K | L4 | F:D1 O:D2 | R5-1 | +| MG6 | Maintenance margin + forced liquidation (`submit_margin_call_slice` pine_adapter.cpp:18008, `submit_margin_call_units` pine_adapter.cpp:18041, `schedule_margin_call_path` pine_adapter.cpp:20290) | **no** — "no maintenance liquidation" `initial_margin_fraction` native_run_spec.hpp:666-667; margin is checked only when `inspect.would_open` native_execution_consumer.cpp:3734 **Closed:** `maintenance_long` / `maintenance_short` with `NativeLiquidationCheck::{PathAdverseExtreme, PathAdverseExtremeMark, CalculationOnly}`; the kernel solves the level, rests its own `Reduce`/`Flatten` bound to the live book, re-prices it with `CancelReason::Superseded` and books a `MarginCallEvent` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | F:D2 O:D3 S:M3 | TV parts → MG15 | +| MG7 | Liquidation slice sizing (grid floor, ×4 restore, one-contract fallback `source_margin_units` pine_adapter.cpp:17665) | **no** **Closed:** `NativeLiquidationSizing::{RestoreMinimum, ShortfallMultiple, Flatten}` with `shortfall_multiple` and `liquidation_min_units`, capped at the held quantity and floored onto `quantity_grid`; `resolve_margin_call_units` native_host.hpp:995 gives a host the last word, clamped into `(0, held]` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | O:D4 F:D2 S:M4 | **Disagree:** F generic knob `ShortfallMultiple` / O "×4 is A" / S quirk, yet a 4.0 default inside its generic model. R5-1: policy knob in the kernel, plain default (restore-minimum). **Decided by R5 lane F7 (the D2 question):** the adapter selects NO kernel knob — `project()` leaves `sizing` / `shortfall_multiple` / `liquidation_min_units` unset and its `resolve_margin_call_units` answers every call with TradingView's slice (the restore floored onto the lot grid BEFORE the ×4, floored again, the one-contract band, the dust gate). Measured: on a one-unit grid, 20 long at 100 on 1000 at 50 % reaching 93.9, TradingView books 4 lots and the kernel's own `ShortfallMultiple 4` books 5 (`tests/test_native_margin_hooks_twin.cpp` MG-F3, an executed run, no longer a comment); without a grid the two agree bit for bit (MG-F/H, 4.2105263157894735), but declaring the knob for those runs alone folds `NativeMarginModel::sizing` into every margin run's identity — an epoch decision F7 does not take. Since R5 lane H-THIN the pre-open admission slice (`schedule_preopen_margin_slice`) sizes through the same `source_margin_units`, where it kept its own copy of the ×4 without the band or the dust gates (AUDIT4 §6 G2-12; the private test `test_adapter_margin_schedule_differential` M10: band and fudge now 1 and 1, the kernel's and `ab9714be`'s). A four-times call on a whole-unit grid whose book's oldest lot is one unit is followed by one more unit at the next check point (`whole_unit_follow_up_due` pine_adapter.cpp:19197; R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-es-*, w5b-sb-es-*, w5b-f-q1x12). With slippage the one-contract band takes its unit only where that unit restores the book at its own fill: the position's requirement at the liquidation's slipped print, less the equity at the mark, under one unit's margin (`source_margin_units` pine_adapter.cpp:17665; lane INT28-FIX rule OU, the private fixture set `slipped_short`). A one-unit call that leaves the book short at its own fill, by more than one unit's margin and two slippage steps with the book re-marked there, is followed by one more unit at the bar's next path point, where that point's own check calls nothing and its print is inside the call's fill (lane INT28-FIX rule OF). Under a percent commission a call of any size is followed so, on a fractional lot grid too: while the book the call left, re-marked at its fill, is short by more than two slippage steps for each unit called and the exit fees of those units and of one more, the next path point inside that fill takes four times the shortfall's lot floor over the fill, or one unit where that floors to none (`margin_follow_up_units` pine_adapter.cpp:19239, `follow_margin_call` pine_adapter.cpp:19288; R5 lane TAIL-I, the private fixture set `margin_open_print`). | +| MG8 | Liquidation price query (`margin_liquidation_price` pine_strategy_host.cpp:181-197) | **no** **Closed:** `native_liquidation_price()` native_host.hpp:1320 answers the exact solved level, `nullopt` where none exists; `NativeLiquidationLevelBase` chooses whether the solve answers for the open entries' fees (`tests/test_native_margin_model.cpp`, `tests/test_native_open_lots.cpp`). | K+A | L4 | F:D3 S:R5 | **Disagree:** F generic / S adapter-only. R5-1: liquidation price is generic; the Pine spelling stays A. **Lowered by R5 lane H-THIN:** the Pine host's second solve (`compute_liquidation_price`) is gone; `strategy.margin_liquidation_price` reads `native_liquidation_price()`, which the adapter configures with `NativeLiquidationLevelBase::RealizedOnly` (the Pine formula's closed-money solve, bit for bit at every bar of the lane's 1,021 seeded margin-call runs), and keeps only TradingView's tick spelling: floored for a long, ceiled for a short. A run whose margin calls are switched off declares no margin model, so it reads na. | +| MG9 | FX revaluation of margin (`apply_fx_open_margin_slice` pine_adapter.cpp:5703, `apply_fx_opening_margin_slice` pine_adapter.cpp:5715) | **no** — staged FX curve `configure_native_fx_curve` native_host.hpp:1202 but no margin re-check. **Closed by audit lane N6:** `NativeMarginCheckKind::FxRoll` — a step of the declared curve is a check point of its own, measured at the unchanged price before the first point the new rate converts (`tests/test_native_margin_fx_roll.cpp`). The adapter refuses the point by kind, for a measured reason since R5 lane F7: TradingView's rollover is a broker-open checkpoint at the open price, now at any positive margin (`apply_fx_open_margin_slice`, on the one TradingView money / units rule every other checkpoint uses), while the roll point measures at the remaining path's adverse mark and rests the slice there — a carried 1x long on a step bar O 100 L 97 books 0.4116 @ 97 on the adapter's run shape and 0.4036 @ 99, measured a bar early, on a strict calendar, against TradingView's 0.3996 @ 100 (`tests/test_native_margin_hooks_twin.cpp` MG-FX). Both N6 follow-ups are closed (§4.4). **Completed by R5 follow-up lane E3:** the conversion INSTANT is now generic too — every check kind converts at its own cursor's rate (`account_currency_fx_at(cursor.point.effective_time_ms)`) for the requirement, the equity and the level, in batch and stream, so a step landing between a fill's cursor and the clock the engine presents no longer breaches at the fill (`tests/test_native_margin_fx_clock.cpp`; N6's local clock swap inside `fx_roll_margin_check` is removed as redundant). **Extended by R5 follow-up lane E21** to the opening gate's placement site (MG4), and the rate-explicit equity E3 restated beside the engine's accessor is now that accessor's one implementation, `BacktestEngine::marked_equity_at` engine.hpp:730 | K | L4 → N6 → E3 → E21 | O:D8 | (single-source) | +| MG10 | Max drawdown halt (`update_risk_state` pine_adapter.cpp:17545) | **no** — observation field only `drawdown_limit` include/pineforge/source/market_admission.hpp:51; composable today from `native_marked_equity` native_host.hpp:1301 **Closed:** `NativeRiskLimits::max_drawdown` native_run_spec.hpp:311, a `NativeLossLimit{value, percent}` measured against the running peak equity at three points of every script bar, blocking openings for the rest of the run (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E1 S:M6 | R5-9 | +| MG11 | Max consecutive losing days (`SourceDayLedger` pine_adapter.hpp:1434-1445; gate `max_cons_loss_days` pine_adapter.cpp:16548) | **no** — observation only `loss_days_limit` include/pineforge/source/market_admission.hpp:50 **Closed:** `NativeRiskLimits::max_consecutive_loss_days`, settled from each day's realized result at the next day's open (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E2 S:M6 | R5-9 | +| MG12 | Max intraday loss → flatten + block (`intraday_loss_breached` pine_adapter.cpp:17578, forced flatten `submit_intraday_loss_close` pine_adapter.cpp:20924) | **no** — observation only `intraday_loss_limit` include/pineforge/source/market_admission.hpp:52 **Closed:** `NativeRiskLimits::max_intraday_loss` with `NativeRiskAction::FlattenAndBlock`, under which the kernel issues one `Flatten` of its own at the breach point (`RequestOrigin::KernelRisk`) and latches the block before that flatten settles (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D7 O:E3 S:M6 | R5-9 | +| MG13 | Max filled orders per day (`IntradayCap` intraday_cap.hpp:84-263, `IntradayOrderBudget` intraday_order_budget.hpp:27-92; `execute_due_cap_close` pine_adapter.cpp:20967) | **no** **Closed:** `NativeRiskLimits::max_fills_per_day` counts every applied execution of the day as it settles, whatever issued it (`tests/test_native_risk_limits.cpp`). | K+A | L9 | F:D8 O:E4 S:M7 | Count cap K; quota transfer, POOC deferral, TV close price A | +| MG14 | Risk-day boundary (`chart_day_key` pine_adapter.cpp:17442) | **no** — the calendar exists `session_day_ordinal` native_calendar.hpp:416-419, there is no risk-day ledger **Closed:** `NativeRiskDay::{SessionDay, CalendarDayInTimezone}` keys the risk day, and `native_risk_state()` native_host.hpp:1329 reads the whole ledger back — block and reason, day, fills counted, loss-day streak, peak equity, day-opening equity (`tests/test_native_risk_limits.cpp`). | K+A | L9 | O:E5 S:M7 | The chart-day shortcut stays A; the kernel keys on session / calendar day | +| MG15 | *(quirk)* TV margin call: money rounding `source_margin_money` pine_adapter.cpp:17610, 4× shortfall `4.0 * minimum` pine_adapter.cpp:17699, POOC chronology exception `resting_order` pine_adapter.cpp:20310 (the commissioned carried short; the competing-entry and rounded-money exceptions retired by R5 lane W5B-ENG-MARGIN-RESIDUAL, the private fixture set `margin_residual` w5b-pc-*), 1×-long money call `submit_tv_money_long_margin_call` pine_adapter.cpp:18607, fill pricing (the slice books at its waypoint's mark, `submit_tv_money_long_margin_call` pine_adapter.cpp:18607) | n/a | A | — | F:D2 F:G4 O:D4 S:M4 | R5-1. **Measured (R5 lane H-MEASURE)** against the kernel's own points on the same bars and orders, the kernel's hooks answering TradingView's money and slice (the private test `test_adapter_margin_schedule_differential`): a gap-open breach executes at the open, sized on the open's money, where the kernel's mark check rests at the path's adverse extreme and `CalculationOnly` executes at the calculation (M8); the pre-open slice parts from the kernel's `AfterApplied` point, and from `ab9714be`, on the one-contract band and the restore's `+1e-6` floor (M10; the frozen units it also parted on are TradingView's book since R5 lane PAR-MARGIN froze a default-percent stop above 100 % as eight `lab tv` tapes, the private tapes `pm-m10-*` of `margin_entry_bar`, show TradingView does); the entry bar of a leveraged opening, where four `lab tv` tapes (the private fixture set `margin_entry_bar`) and the kernel's `AfterApplied` point book the margin call at that bar's low, is admitted by `on_applied` since R5 lane PAR-MARGIN on the run shapes `ab9714be` checked it on (the pre-open slice's own opening keeps that slice's verdict), and the adapter books the tapes' rows (M7; H-MEASURE had recorded it as a divergence); R5 lane PAR-MARGIN-2 extended it to limit openings, `process_orders_on_close`, `calc_on_order_fills` and adds to a carried book, and made the kernel's post-fill point reach the low a limit was filled on the way down to, or a short's high on its way up (twenty-eight `lab tv` tapes, the private tapes `pm2-*` of `margin_entry_bar`). Under the magnifier the kernel's `IntrabarSample` point (R5 lane PAR-MARGIN) is admitted on a leveraged long, plain, under `process_orders_on_close` or `calc_on_order_fills`, and on a short outside `process_orders_on_close` (PAR-MARGIN-2), where TradingView books its call at the first lower-timeframe low that crosses -- at its own 2-minute intrabars on a 15-minute chart, which a model reproduces on all 20 magnified tapes (the private fixture set `intrabar_margin`); six tapes part on the one-minute samples the host feeds, a recorded divergence of the magnifier's sampling. A margin call books the nearest tick of its print, as TradingView's market fills do (12.105 books 12.11; the private fixture set `half_tick_rounding`, PAR-MARGIN-2). | +| MG16 | *(quirk)* Historical market-admission review / retention journal (`explicit_pair_scope` src/compat/pine/market_admission.cpp:34 and the review fold it feeds (`original_pair_call` src/compat/pine/market_admission.cpp:48); `admission_allocation` pine_adapter.cpp:4570, `record_market_review` pine_adapter.cpp:21223) | n/a | A | — | S:R2 | (single-source) | ### 1.5 Calculation timing @@ -117,15 +117,15 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa |---|---|---|---|---|---|---| | CT1 | Calculate at script-bar close | **yes** — `on_native_bar` native_host.hpp:893, dispatched by `invoke_callback` native_execution_consumer.cpp:8205 (call site `on_native_recalculate` native_execution_consumer.cpp:8233) | K | — | F:E1 O:F1 | — | | CT2 | Process orders on close (`close_execution` pine_adapter.cpp:2818) | **yes** — `NativeCloseExecution::AfterCalculation` native_run_spec.hpp:35-38; `AfterCalculation` native_execution_consumer.cpp:8649 | K | — | F:E2 O:F6 S:C1 | — | -| CT3 | Pre-match hook at bar open (`on_bar_open` pine_adapter.cpp:22314) | **yes** — `on_native_bar_open` native_host.hpp:889 runs before the open match native_execution_consumer.cpp:8079; `execute_current` legal native_host.hpp:1104 | K | — | F:E3 O:F2 | hazard → CT4 | +| CT3 | Pre-match hook at bar open (`on_bar_open` pine_adapter.cpp:22318) | **yes** — `on_native_bar_open` native_host.hpp:889 runs before the open match native_execution_consumer.cpp:8079; `execute_current` legal native_host.hpp:1104 | K | — | F:E3 O:F2 | hazard → CT4 | | CT4 | Partial-bar view / open-bar lookahead guard | **no** — the open hook receives the *complete* script bar: `invoke_bar_open_callback(engine, bar, point)` native_execution_consumer.cpp:8022 with `engine.current_bar_ = open_view` native_execution_consumer.cpp:8073 (the complete bar unless `OpenOnly`) **Closed:** `current_partial_bar()` native_host.hpp:1073 is the lookahead-free bar so far, through the last path point consumed (a fill inside a segment reads it at the segment's origin), and `NativeOpenBarView::OpenOnly` native_run_spec.hpp:109 masks `on_native_bar_open`'s bar to `H = L = C = open` as presentation only — the complete bar is restored before the open match (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E3 | (single-source) The adapter needs the full bar, so the guard is opt-in. | | CT5 | Raw input observation before aggregation | **yes** — `on_native_input` native_host.hpp:875, `NativeInputContext` native_host.hpp:773-778 | K | — | O:F3 | (single-source row) | -| CT6 | Calculate on order fills (scheduler `PineScheduler::recalculate` pine_scheduler_native.cpp:767, `coof_recalculation_due` pine_scheduler_native.cpp:733, the first-open chain `kFirstOpenLoopGuard` pine_scheduler_native.cpp:888; `begin_coof_recalc` pine_adapter.cpp:6720, `flush_coof_tail` pine_adapter.cpp:7144, `end_coof_recalc` pine_adapter.cpp:6777) | **partial** — `on_native_applied` native_host.hpp:938 fires mid-path after each fill native_execution_consumer.cpp:7926, native_execution_consumer.cpp:7926; commands are legal there `commands_allowed` native_execution_consumer.cpp:1844; a request born there is eligible on the unconsumed rest of the bar `born_on_remaining_path` native_execution_consumer.cpp:6563, `remaining_path_delivery` native_order.cpp:2094-2102. Missing: calculation re-entry, a documented chronology, a bar-so-far view **Closed:** `NativeCalculationTrigger::BarCloseAndFills` native_run_spec.hpp:96 drives one `OrderFill` recalculation per applied execution at that event's own cursor, delivered as `on_native_recalculate(..., OrderFill, cause)` and bounded by `max_recalculations_per_point` (`tests/test_native_calc_timing.cpp`). | K+A | L5 | F:E4 O:F5 S:C2 | **Disagree:** F partial / O, S no; F, S v1 / O deferrable. R5-5: v1, sub-bar hook + chronology contract, adapter path unchanged. TV parts → CT11. | -| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:8161; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:62-76, the runner rejects it (`calc_on_every_tick` main.cpp:259), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:304. | +| CT6 | Calculate on order fills (scheduler `PineScheduler::recalculate` pine_scheduler_native.cpp:767, `coof_recalculation_due` pine_scheduler_native.cpp:733, the first-open chain `kFirstOpenLoopGuard` pine_scheduler_native.cpp:888; `begin_coof_recalc` pine_adapter.cpp:6724, `flush_coof_tail` pine_adapter.cpp:7148, `end_coof_recalc` pine_adapter.cpp:6781) | **partial** — `on_native_applied` native_host.hpp:938 fires mid-path after each fill native_execution_consumer.cpp:7926, native_execution_consumer.cpp:7926; commands are legal there `commands_allowed` native_execution_consumer.cpp:1844; a request born there is eligible on the unconsumed rest of the bar `born_on_remaining_path` native_execution_consumer.cpp:6563, `remaining_path_delivery` native_order.cpp:2110-2118. Missing: calculation re-entry, a documented chronology, a bar-so-far view **Closed:** `NativeCalculationTrigger::BarCloseAndFills` native_run_spec.hpp:96 drives one `OrderFill` recalculation per applied execution at that event's own cursor, delivered as `on_native_recalculate(..., OrderFill, cause)` and bounded by `max_recalculations_per_point` (`tests/test_native_calc_timing.cpp`). | K+A | L5 | F:E4 O:F5 S:C2 | **Disagree:** F partial / O, S no; F, S v1 / O deferrable. R5-5: v1, sub-bar hook + chronology contract, adapter path unchanged. TV parts → CT11. | +| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:8161; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:65-79, the runner rejects it (`calc_on_every_tick` main.cpp:259), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:304. | | CT8 | Historical intrabar calculation (per lower-TF sub-bar / magnifier sample) | **no** — with an `IntrabarPath` the callback still fires once per script bar native_execution_consumer.cpp:8756 **Closed:** `EveryModeledPoint` recalculates at each intrabar sample, and `on_native_sub_bar` native_host.hpp:928 fires once after each retained lower-timeframe sub-bar's whole matching path (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E6 O:F8 S:C4 | — | | CT9 | Intrabar (magnifier) matching path + path-order policy (`volume_weighted_cap` pine_adapter.cpp:2938) | **yes** — `IntrabarPath` native_run_spec.hpp:362-403, delivery `deliver_confirmed_script` native_execution_consumer.cpp:8308-8574; `NativePathOrder` native_run_spec.hpp:335-339 | K+A | — | F:E7 F:G5 O:F7 S:C4 | **Disagree:** S partial (it wants callback cadence → CT8). TV sampler choice is A: TradingView's own intrabars at its table's timeframe, each owned by the chart bar of its last minute, built from the finer feed (`tradingview_magnifier_bars` magnifier_intrabars.hpp:46, R5 lane MAG-INTRABAR). | -| CT10 | Batch → warmup → realtime forward execution | **yes** — `stream_begin` engine.hpp:1772, `stream_push_bar` engine.hpp:1778, `stream_end` engine.hpp:1782, `NativeRunPhase` native_host.hpp:41, C ABI `strategy_stream_begin` c_abi.cpp:699 | K+A | — | F:E8 O:J3 S:C5 | **Disagree:** S partial. R5-1: the lifecycle is native; Pine trade-start suppression and realtime-tail / probe rules (`trading_window_active` pine_strategy_commands.cpp:15-21, `prepare_native_begin` pine_strategy_host.cpp:210-320) are A. S's "neutral tail + trade-window policy" is single-source and unassigned (§4 Q8). | -| CT11 | *(quirk)* COOF TV specifics: waypoint-only refill (`coof_next_waypoint` pine_adapter.cpp:6993), two-fills-at-open rule, script-state rollback (`PineScheduler` pine_scheduler_native.cpp:99-150; hooks `snapshot_script_state` engine.hpp:1333-1334), the first-open guard `kFirstOpenLoopGuard` pine_scheduler_native.cpp:888, deferral queue of `PendingCoofRequest` pine_adapter.hpp:1965-1995 | n/a | A | — | F:E4 O:F5 S:C2 | R5-5: the adapter keeps its path | +| CT10 | Batch → warmup → realtime forward execution | **yes** — `stream_begin` engine.hpp:1772, `stream_push_bar` engine.hpp:1778, `stream_end` engine.hpp:1782, `NativeRunPhase` native_host.hpp:41, C ABI `strategy_stream_begin` c_abi.cpp:699 | K+A | — | F:E8 O:J3 S:C5 | **Disagree:** S partial. R5-1: the lifecycle is native; Pine trade-start suppression and realtime-tail / probe rules (`trading_window_active` pine_strategy_commands.cpp:15-21, `prepare_native_begin` pine_strategy_host.cpp:249-403) are A. S's "neutral tail + trade-window policy" is single-source and unassigned (§4 Q8). | +| CT11 | *(quirk)* COOF TV specifics: waypoint-only refill (`coof_next_waypoint` pine_adapter.cpp:6997), two-fills-at-open rule, script-state rollback (`PineScheduler` pine_scheduler_native.cpp:99-150; hooks `snapshot_script_state` engine.hpp:1333-1334), the first-open guard `kFirstOpenLoopGuard` pine_scheduler_native.cpp:888, deferral queue of `PendingCoofRequest` pine_adapter.hpp:1991-1998 | n/a | A | — | F:E4 O:F5 S:C2 | R5-5: the adapter keeps its path | ### 1.6 Higher timeframes and auxiliary feeds @@ -143,13 +143,13 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| | FP1 | Trigger geometry on the modeled path; fill at the level or at the gap price; market-if-touched | **yes** — `native_matching` native_execution_consumer.cpp:6635, `first_region_entry` native_matching.hpp:229-257; kind `NativeCandidatePriceKind` native_execution_consumer.cpp:7225; `Limit::fill_through` native_order.hpp:239-244 enforced where the matched price is limit-protected (`protect_limit` native_execution_consumer.cpp:7221) and re-checked after host terms (`InvalidTerms` native_execution_consumer.cpp:5890) | K | — | F:G3 O:H1 O:H2 O:H3 | — | -| FP2 | Slippage in ticks, limit-clamped | **yes** — `slippage_ticks` native_run_spec.hpp:652 applied once native_execution_consumer.cpp:5526, clamped by `protect_limit` native_execution_consumer.cpp:7221 | K+A | — | F:G1 O:H7 S:F2 | TV's round-then-slip order (`source_bar_fill` pine_adapter.cpp:15228) is A | -| FP3 | Commission: percent / cash per unit / cash per execution (`fee_kind_for` pine_adapter.cpp:669) | **yes** — `NativeFeeKind` native_run_spec.hpp:19-23; `quote_execution_commissions` engine_execution.cpp:1690 | K | — | F:G2 O:H8 S:F1 | — | -| FP4 | Booked fill quantized to the instrument tick, nearest or directional (`nearest_tick` pine_adapter.cpp:274, `source_bar_fill` pine_adapter.cpp:15228, `directional_tick` pine_adapter.cpp:327) | **no** — `price_tick` is only a slippage multiplier native_execution_consumer.cpp:7203; `bar_fill_price` engine.hpp:772 is reachable only through `NativeCurrentPriceRule::NearestTick` native_execution_consumer.cpp:7376; the reference helpers were dead (since deleted) **Closed:** `NativeRunSpec::price_grid` (`NativePriceGrid` native_run_spec.hpp:118) with `grid_rounding` (`NativeGridRounding` native_run_spec.hpp:128) rounds the fill basis onto `price_tick` before slippage, `HalfUp` or `Directional` (`tests/test_native_sizing_price_rule.cpp` quantizes the fill basis under `HalfUp`; `tests/test_native_acid_composite.cpp` books the `Directional` fills on the 0.25 ladder; the private test `test_native_price_grid` re-ran the pair under the overlay). | K | L8 | O:H4 O:H6 F:G4 S:F3 | **Disagree:** O kernel / F, S "stays adapter, no promotion". R5-3: own kernel lane. | -| FP5 | Trigger tested against the tick-quantized bar | **no** — the kernel tests raw prices against raw levels **Closed:** `NativePriceGrid::QuantizeFillsAndTriggers` additionally tests and re-validates every activation against the tick-quantized path, one rule for every trigger kind (`tests/test_native_trail_best_seed.cpp` puts an off-ladder trail seed on the ladder; `tests/test_native_acid_composite.cpp` runs a trail's best and its exit level on it; the private test `test_native_price_grid` re-ran the pair under the overlay). | K+A | L8 | O:H5 F:G4 S:F3 | R5-3: the optional quantized trigger test is K; TV's exact half-tick rule (`source_trigger_threshold` pine_adapter.cpp:384) is A on top. | +| FP2 | Slippage in ticks, limit-clamped | **yes** — `slippage_ticks` native_run_spec.hpp:652 applied once native_execution_consumer.cpp:5526, clamped by `protect_limit` native_execution_consumer.cpp:7221 | K+A | — | F:G1 O:H7 S:F2 | TV's round-then-slip order (`source_bar_fill` pine_adapter.cpp:15232) is A | +| FP3 | Commission: percent / cash per unit / cash per execution (`fee_kind_for` pine_adapter.cpp:670) | **yes** — `NativeFeeKind` native_run_spec.hpp:19-23; `quote_execution_commissions` engine_execution.cpp:1690 | K | — | F:G2 O:H8 S:F1 | — | +| FP4 | Booked fill quantized to the instrument tick, nearest or directional (`nearest_tick` pine_adapter.cpp:275, `source_bar_fill` pine_adapter.cpp:15232, `directional_tick` pine_adapter.cpp:328) | **no** — `price_tick` is only a slippage multiplier native_execution_consumer.cpp:7203; `bar_fill_price` engine.hpp:772 is reachable only through `NativeCurrentPriceRule::NearestTick` native_execution_consumer.cpp:7376; the reference helpers were dead (since deleted) **Closed:** `NativeRunSpec::price_grid` (`NativePriceGrid` native_run_spec.hpp:118) with `grid_rounding` (`NativeGridRounding` native_run_spec.hpp:128) rounds the fill basis onto `price_tick` before slippage, `HalfUp` or `Directional` (`tests/test_native_sizing_price_rule.cpp` quantizes the fill basis under `HalfUp`; `tests/test_native_acid_composite.cpp` books the `Directional` fills on the 0.25 ladder; the private test `test_native_price_grid` re-ran the pair under the overlay). | K | L8 | O:H4 O:H6 F:G4 S:F3 | **Disagree:** O kernel / F, S "stays adapter, no promotion". R5-3: own kernel lane. | +| FP5 | Trigger tested against the tick-quantized bar | **no** — the kernel tests raw prices against raw levels **Closed:** `NativePriceGrid::QuantizeFillsAndTriggers` additionally tests and re-validates every activation against the tick-quantized path, one rule for every trigger kind (`tests/test_native_trail_best_seed.cpp` puts an off-ladder trail seed on the ladder; `tests/test_native_acid_composite.cpp` runs a trail's best and its exit level on it; the private test `test_native_price_grid` re-ran the pair under the overlay). | K+A | L8 | O:H5 F:G4 S:F3 | R5-3: the optional quantized trigger test is K; TV's exact half-tick rule (`source_trigger_threshold` pine_adapter.cpp:385) is A on top. | | PG | The Pine adapter on the kernel price grid: raw levels submitted, `project()` declaring `QuantizeFillsAndTriggers` + `HalfUp`, `source_trigger_threshold` deleted (lane R7; re-tested after L8b by gap lane N13) | **measured-infeasible for the Pine adapter** — B1 (the sub-tick cursor-print abort) is closed by L8b's `ActivationGrid`: 0 aborted runs under the trial (R7: 9 zero-offset-trail tapes + 6 POOC panels; now `test_zero_offset_trail_rides_l4c` 449/450, `test_pooc_short_close_tick_l4d` 183/183). B2 (TradingView quantizes per order kind, the grid is one run-wide rule) moved 30 pinned checks in 4 units when N13 last ran the trial, before lanes E14 and E16, with no adapter-side remedy: `test_coof_market_limit_recross_l4c` 24, `test_stop_tick_rounding_l4d` 3, `test_adapter_grid_relower` 2, `test_zero_offset_trail_rides_l4c` 1; 12 more checks in 3 units have an adapter-side cause (`test_trail_fill_snap_l4c` 2, `test_native_margin_hooks` 2, `test_adapter_brackets_relower` 8 book digests). Corpus: 5/312 probes differ, in the engine-only entry-incarnation column only (every probe runs a 0.01 tick on an on-grid feed, so it cannot arbitrate) | A | waived (R7, N13) | — | **Waiver.** No class of triggers is byte-identical on its own: the grid is a run-wide switch and a per-kind mask would spell TradingView's inconsistency into the kernel (not generic, R5-3). The adapter stays on `NativePriceGrid::None` and keeps `source_trigger_threshold` and its call sites; native-engine.md "The Pine adapter's grid re-lowering is waived". **Native-only by ruling (audit lane P6): §3.6.2**, with `examples/native/native_price_grid_strategy.cpp` and its C twin as the feature's hosts. | | FP6 | Account-currency FX | **partial** — scalar `account_fx` + immutable `NativeFxCurve` native_host.hpp:1202 in batch; the stream then refused a non-empty curve (today only tick-driven input is refused under one, `refuse_fx_curve_tick_input` native_execution_consumer.cpp:10003). **Closed by audit lane N6:** the declared curve is the run's immutable FX epoch and a confirmed-bar stream runs under it exactly as a batch does (stream twin in `tests/test_native_margin_fx_roll.cpp`); no realtime rate ingestion and no tick-driven input under a curve are stated as permanent limits with their reasons (native-engine.md, "Stream FX"). **Re-examined by R5 follow-up lane E3 and KEPT:** the tick refusal was never only about the margin check — it guards the SCRIPT-VISIBLE conversions (`open_trade_profit()`, `marked_equity()`, the equity curve), which read `active_account_currency_fx()`, i.e. the presented clock the observation hook has not yet moved (re-measured: a step at T+90s left the T+100s and T+110s prints converting at the pre-step rate). E3's cursor rule reaches the margin model only, so the refusal stands and is pinned in `tests/test_native_margin_fx_clock.cpp` | K | unassigned → N6 (E3: refusal kept) | F:D9 S:F4 | **Disagree:** F out of scope ("O7 clock") / S its P10, a stream-safe FX epoch. No R5 lane → §4 U4 (answered by N6: S's "same curve in batch and stream", without a new clock or callback). | -| FP7 | *(quirk)* TV tick conventions: half-tick threshold `source_bar_fill_tick` pine_adapter.cpp:295 (used `source_trigger_threshold` pine_adapter.cpp:384, `exit_limit_trigger` pine_adapter.cpp:14408), `source_bar_fill_tick` pine_adapter.cpp:295, raw-vs-booked spelling `source_same_point` pine_adapter.cpp:313, margin-call fill pricing `source_margin_fill_price` pine_adapter.cpp:17830 | n/a — already isolated behind the terms seam (adapter `resolve_terms` pine_adapter.cpp:14904) | A | — | F:G4 O:H5 S:F3 | R5-1, R5-3 | +| FP7 | *(quirk)* TV tick conventions: half-tick threshold `source_bar_fill_tick` pine_adapter.cpp:296 (used `source_trigger_threshold` pine_adapter.cpp:385, `exit_limit_trigger` pine_adapter.cpp:14412), `source_bar_fill_tick` pine_adapter.cpp:296, raw-vs-booked spelling `source_same_point` pine_adapter.cpp:314, margin-call fill pricing `source_margin_fill_price` pine_adapter.cpp:17834 | n/a — already isolated behind the terms seam (adapter `resolve_terms` pine_adapter.cpp:14908) | A | — | F:G4 O:H5 S:F3 | R5-1, R5-3 | ### 1.8 Excursion, metrics, report, hash @@ -205,13 +205,13 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | a | `source::StrategyOverrides` opaque pointer | Forward declarations in the kernel headers (since removed: no kernel header names the adapter type now); parameter of `run_rich` (execution_consumer.hpp:46-55, native_execution_consumer.hpp:111-120) and of the rich `BacktestEngine::run` overload (engine.hpp:1797-1802, engine_consumer.cpp:81-93). The kernel only forwards it as `NativeBeginArgs::overrides_opaque` native_host.hpp:758; the source host casts it back `overrides_opaque` pine_strategy_host.cpp:316 | NEUTRAL in behaviour; a consumer-vtable signature change, so it rides the shared `engine_script_run` bump | `const void*` in the kernel signatures (F `host_begin_extras`, O `overrides_opaque`), typed convenience overload in `source/pine_strategy_host.hpp`; then drop the checker's single whitelist entry `test_native_example_batch` check_native_include_independence.py:42-46. S's alternative (`NativeRunOverrides`, or removing the rich overload) not adopted. Unverified: generated code calls the typed overload (codegen repo is not in this tree — F). **Status (L11, `3f4fff5b`): done** — `run_rich` and the rich `BacktestEngine::run` take `const void*` overrides, the rich begin carries them as `overrides_opaque`, and the independence checker whitelists nothing (§2.vi row 6) | F O S | | b | `pine_float_compare.hpp` | 1e-10 absolute equality band, used by kernel TA only: `float_band_le` ta_misc.cpp:30 (percentrank), `float_band_gt` ta_oscillators.cpp:462-463 (MFI), `plus_dm` ta_volatility_trend.cpp:337-340 (DMI); no execution TU includes it (O) | NEUTRAL (rename only) | Rename to a neutral header (F `float_band_compare.hpp`, O `ta_compare_band.hpp`), keep `pine_float_*` inline aliases in a source-layer header for generated code. Numerics must not move. S's behavioural `ComparePolicy` split is not neutral → not adopted (§4 U6). **Status (L11 + N14): done** — `ta_compare_band.hpp` is the kernel header; the alias shim is `include/pineforge/source/pine_float_compare.hpp` (N14 moved it off the kernel include root; it has no in-tree includer, generated code emits its own comparator) | F O S | | c | `NativeSlotLabelPolicy::LegacyTolerant`, `NativeLegacyTolerance` (now `NativeSlotLabelPolicy::FeedTolerant`, `NativeFeedTolerance`) | `NativeSlotLabelPolicy` native_run_spec.hpp:324-353; read by `legacy_tolerant_slot_labels` market_driver.cpp:55-56 and `native_feed_tolerance_enabled` market_driver.cpp:43-49, and by `legacy_tolerant_slot_labels` native_execution_consumer.hpp:1061-1063 (outside a run it answers the spec's `spec_tolerant_slot_labels` native_execution_consumer.cpp:2369; the reads are `legacy_tolerant_slot_labels` native_execution_consumer.cpp:2386, `:10137`); set only by the adapter `slot_label_policy` pine_adapter.cpp:2808. A feed-tolerance policy (raw strictly-increasing labels, zero / NaN-volume legacy bars), not TV semantics (O) | NEUTRAL | Rename with deprecated aliases (F `RawIncreasing` + `NativeInputTolerance::{StructuralBars, WarmupNonNegativeOHLC}`; O `NativeLabelPolicy::ProviderLabels`). Enumerator values stay 0/1 and 1/2: they are hashed `slot_label_policy` native_execution_consumer.cpp:273. **Status:** renamed as `FeedTolerant` / `NativeFeedTolerance` (the enumerator values unchanged), and the C transport carries them now, in the extension's feed-policy block (`PF_NATIVE_SPEC_EXT_FEED_POLICY` native_c_api.h:745) | F O S | -| d | Orphaned comments | The KI-62 block of `engine_orders.cpp` (*fresh*: comment-only, the bodies are deleted; S read it as live code) and a comment of `engine.hpp`; wider deleted-declaration blocks in `engine.hpp`; the money-rule essay in `engine.hpp` (the code is `source_money_round` pine_adapter.cpp:558) | NEUTRAL | Delete, or move to `docs/` next to the adapter code that implements them. **Status: done** — R5 lanes E6 and E10 deleted the detached comment residue (a census of 0 over the kernel's compile closure) | F O | +| d | Orphaned comments | The KI-62 block of `engine_orders.cpp` (*fresh*: comment-only, the bodies are deleted; S read it as live code) and a comment of `engine.hpp`; wider deleted-declaration blocks in `engine.hpp`; the money-rule essay in `engine.hpp` (the code is `source_money_round` pine_adapter.cpp:559) | NEUTRAL | Delete, or move to `docs/` next to the adapter code that implements them. **Status: done** — R5 lanes E6 and E10 deleted the detached comment residue (a census of 0 over the kernel's compile closure) | F O | | e | Dead helpers in the public kernel header | **Done.** All four are gone from `engine.hpp`: `round_to_mintick_directional` and `apply_slippage` were the pre-R5 spelling of the directional snap and the slippage step and the kernel now fills on `native_matching::grid_round_directional` / `native_matching::apply_slippage` src/native_matching.hpp:259 (the engine.hpp comment that named them went with R5 lane E10's residue sweep); `apply_limit_fill` and `reserve_percent_commission` are deleted outright, and only the adapter still spells the first two names, in its own comments. `round_to_mintick` engine.hpp:754 and `bar_fill_price` engine.hpp:772 stay, live through the call `bar_fill_price` native_execution_consumer.cpp:7377 | NEUTRAL | Reuse as the reference arithmetic of the price-grid and sizing lanes, delete the rest | O (single-source) | | f | Dead legacy stream path | **Done.** `stream_feed_input_bar` and `dispatch_source_stream_script_bar` are both gone — neither name is in `src/` or `include/` — and the virtual left with the epoch that carried it. `stream_state_hash` engine.hpp:1791 stays, and the declaration/definition pairing is still enforced by `scripts/test_native_source_guard.py` | NEUTRAL (removing the virtual changes the vtable → shared bump) | Delete both | O; F asked (its Q6) | | g | Legacy path resolver | **Done, and then deleted.** `src/engine_path_resolve.cpp` is 118 lines with five functions: `bar_path_uses_high_first`, both `first_touch_position` overloads, `fill_bar_path_points_ordered`, and `BacktestEngine::declare_opened_lot_entry_bar_mask`. Ten test files still mention `resolve_exit_path_fill` in historical comments; the resolver body and test-only oracle are gone. | NEUTRAL (pure move; verify with the link gate) | Keep the generic path helpers in the kernel TU; the old source resolver is not a live counterpart | F (single-source) | | | h | `pine_*`-named generic API | `timeframe_time` session_time.hpp:35-211 (live, e.g. `session_in_market`, which generated code reaches through `pine_session_ismarket` session_time.hpp:263), `str_utils.hpp`, `deterministic_random` math.hpp:9, `tz_util` timezone.hpp:8-30; the guard bans `pine_*` only inside `IDENTIFIER_ROOTS` (test_native_source_guard.py:34, test_native_source_guard.py:34-42) | NEUTRAL, M | Neutral aliases; `pine_*` kept as deprecated inlines for codegen. Large blast radius in generated code. **Status (L11 + N14): done** — session_time / str_utils / math / timezone in L11; N14 applied the same treatment to the two public matrix types: `PineMatrix` → `NumericMatrix`, `PineGenericMatrix` → `GenericMatrix`, old spellings kept as exact aliases (no version script pins either name) | O (single-source) | -| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:27141 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | -| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:715, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:154, hashed (`skip_entry_bar_high` engine_state_hash.cpp:74) and, since R5 lane E6, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until R5 lane H-THIN left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:355-675) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | +| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:27146 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | +| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:715, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:154, hashed (`skip_entry_bar_high` engine_state_hash.cpp:74) and, since R5 lane E6, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until R5 lane H-THIN left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:574-943) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | | k | `margin_call_enabled_` | **Done.** The generic kernel model is enabled by `NativeRunSpec::margin` native_run_spec.hpp:672. TradingView-specific toggling remains `source::PineStrategyHost::set_margin_call_enabled` pine_strategy_host.hpp:511; the frozen C symbol remains source-owned. | HASH-VISIBLE (R5-8) | Keep the source setter in the adapter; the kernel toggle is the presence of `spec.margin` | F O | | l | Kernel decodes adapter strings | **Done.** `closed_trade_close_cause` engine_trade_accessors.cpp:158 branches on no string at all: it reads `Trade::open_at_end`, then the typed `execution::CloseCause` the closer recorded on the row, then `exit_from_bracket`. A kernel-originated liquidation or risk flatten states its own cause through the settling fill, so a bare host reads `3` / `4` where the retired string derivation answered `1`. The public C contract it implements is `strategy_closed_trade_close_cause` pineforge.h:1309 | HASH-VISIBLE (adds durable state) | Kernel `enum class CloseCause` recorded on the `Trade` at settlement (the kernel sets `MarginCall` when L4 liquidates; a source virtual covers adapter causes). C numbers unchanged. After L4 | O (single-source) | | m | `request.security` machinery | `register_security_eval` engine_security.cpp:27, `set_native_security_feed` engine_aux_security.cpp:77-131, `SecurityEvalState` engine.hpp:1157-1361 carry Pine semantics (lookahead, gaps, Heikin-Ashi, KI-55 range start) and are driven only by the source layer | HASH-VISIBLE / measured path | F planned to move both TUs and the state into the source host after native HTF. **R5-6 narrows this:** L6 reuses the feed store and routing, so those stay kernel; only the Pine-only evaluator semantics remain candidates. Do last. **Status (lane L12b): done** — the evaluator semantics are `source::PineSecurityEvalState` + `src/source/pine_security_eval.cpp`; the kernel keeps the registry, aggregator, feed store and one generic step | F S | @@ -320,13 +320,13 @@ Union of F §2.vi (9), O §2.vi (5 + 1) and S §2.6 (7), deduplicated. S had no | # | ADR claim | Correct statement | Evidence | Src | |---|---|---|---|---| -| 1 | Orders are "Market / Limit / Stop" (ADR L72-75) | Also StopLimit and Trail; five intents, five owner kinds, groups, `PointBudget` | `Trigger` native_order.hpp:296, `OrderIntent` native_order.hpp:229, `Owner` native_order.hpp:464, `Group` native_order.hpp:473, `PointBudget` native_order.hpp:379 | F | +| 1 | Orders are "Market / Limit / Stop" (ADR L72-75) | Also StopLimit and Trail; five intents, five owner kinds, groups, `PointBudget` | `Trigger` native_order.hpp:298, `OrderIntent` native_order.hpp:231, `Owner` native_order.hpp:466, `Group` native_order.hpp:475, `PointBudget` native_order.hpp:381 | F | | 2 | "The kernel matches and fills … with sizing, slippage, fees, margin and settlement" (ADR L74-75) | "with slippage, fees, opening admission and settlement; sizing is resolved by the host". `NativeRunSpec` owns fees, FX, close timing, direction / caps and intrabar paths; it does not own percent / cash sizing, maintenance margin, calc-on-fill, every-tick calculation or bare-host security registration | at `73817c1` (overtaken; ADR-0001, "Current state vs. target") | O S | | 3 | "Read results via `metrics.hpp`" (ADR L76-80) | Trade stats yes. Equity curve and equity metrics are empty for a bare host, silently, and an open final position is missing from the report | at `73817c1` (overtaken; ADR-0001, "Current state vs. target") | F O | | 4 | HTF "comes from the kernel feed `set_native_security_feed` / `prepare_native_security_feeds`" (ADR L84-89) | Not usable by a bare host: a partial boundary, not an absent kernel. `prepare_native_security_feeds` is protected with one caller; `configure_security_evaluators` has one caller; the public setter is inert pre-run and refused in-run | `prepare_native_security_feeds` engine.hpp:1640, `configure_security_evaluators` engine.hpp:1323, `set_native_security_feed` engine.hpp:1732, `guard_native_mutation` engine_aux_security.cpp:83 | F O S | | 5 | "`engine_aux_security.cpp` includes `source/pine_strategy_host.hpp`" (ADR L129-132) | False at `73817c1`: its includes are `engine_internal.hpp`, `ta.hpp` and std headers (*fresh* re-read; the run-failure codes added `run_failure.hpp`, a kernel header). No kernel TU includes a source header | `sha256:560a7f68b243d921382a69ed0390c398f28370ac066b84f30b5f96cbb385fb60` engine_aux_security.cpp:6-14 | F | | 6 | "Kernel TUs reference adapter types (`source::StrategyOverrides` …)" (ADR L129-131) | True only as a forward-declared opaque pointer the kernel never dereferences; the independence checker whitelists exactly that symbol | at `73817c1`; overtaken by `3f4fff5b` (§2.ii a), after which the rich begin carries `overrides_opaque` native_host.hpp:758 and the independence checker whitelists nothing | F O | -| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and R5 lane E6 deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:558 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:153, `skip_entry_bar_low` engine_state_hash.cpp:74, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2584; `closed_trade_close_cause` engine_trade_accessors.cpp:158. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | +| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and R5 lane E6 deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:559 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:153, `skip_entry_bar_low` engine_state_hash.cpp:74, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2602; `closed_trade_close_cause` engine_trade_accessors.cpp:158. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | | 8 | "No kernel-level C API. `c_abi.cpp` exports only version/descriptor symbols" (ADR L136-138) | 84 runtime `PF_API` symbols: native configure, the 12-symbol stream family, pending-order views, broker-state hash, security feeds, FX curve, reports. The 84 compiled-strategy runtime symbols in `c_abi.cpp` are distinct from the native C host API in `native_c_host.cpp`: the latter exposes submit, replace, cancel and a callback table to a C host. | `sha256:ee9340c444259a22f97924a1785783b22d1528baf777ccc8c4be2263c6de8260` c_abi.cpp:336-1339, `sha256:6327f1829019fdd172cd76a96e3d8c9c0af1610ad0dc9878640dda0ca21a686b` check_c_abi_runtime.py:20-81, c_abi.cpp:981, `sha256:d0a0a589206d43b26da5ed5161203cb2412c41b42d14b0d0ade72ce24a31455b` c_abi.cpp:699-820, `sha256:b9c178ef9aa9c8c5233b64b49d1c12da47bb63ca9ab50f0c31ae342830b7dac6` c_abi.cpp:624-697, `sha256:c8b7908a9377f2b3e8fac7965b9e8c8be38081b9cf5a148054bc7b3cc49e981f` c_abi.cpp:580-598, `sha256:1d86995d30c6c7d937ca2099285c5fd3ba3af0abbed056ef112b73ebcbc1b0d1` c_abi.cpp:921-950, c_abi.cpp:1052 | F O S | | 9 | "No examples. `PINEFORGE_BUILD_EXAMPLES` is 'none yet'" (ADR L139) | The option is inert, but two Pine-free `NativeStrategyHost` examples exist, are compiled by the independence checker and tested through the live runner, only under `PINEFORGE_BUILD_LIVE_RUNNER`. Wording: "no top-level examples target" | at `73817c1` (overtaken; ADR-0001, "Current state vs. target") | F O S | | 10 | Follow-ups (1)-(4) (ADR L168-170) | Add native reporting, sizing, margin, HTF — and, per R5, calc-timing, price grid and order ergonomics. Without them "usable without Pine" is a build-system claim, not a feature claim | §3.5 | F | @@ -340,7 +340,7 @@ the campaign's own anchor audit found that the first thing to rot: | Where | Said | Tree says | |---|---|---| | the page's opening, and its "close-only" claims | callbacks are close-only | The kernel has a bar-open hook (`on_native_bar_open` native_host.hpp:889), a tick hook (`on_native_tick` native_host.hpp:878), a post-fill hook (`on_native_applied` native_host.hpp:938), a recalculation hook (`on_native_recalculate` native_host.hpp:913) and a sub-bar hook (`on_native_sub_bar` native_host.hpp:928). The page's opening now tables all eleven, and `on_native_applied` is documented as the calculate-on-fill hook with suffix eligibility | -| the request-epoch sentence | `native_order_v4` | `native_order_v7` native_order.hpp:25 | +| the request-epoch sentence | `native_order_v4` | `native_order_v7` native_order.hpp:27 | | the two host-epoch sentences | host epoch v16 | `engine_script_run_v19` native_host.hpp:20 | | "What still requires Pine compatibility to build" | `BacktestEngine` still holds `CapAttachment` / `OrderPriority` / `IntradayCap` | `engine.hpp` has zero references to them; the cap type is `CapAttachment` intraday_cap.hpp:19 and the constructor is `PineStrategyHost`'s pine_strategy_host.hpp:289 | | the rich-`run` paragraph | the rich `run` overload fails a native host | `run_rich` native_execution_consumer.cpp:10573 shows no such refusal (still unpinned by a test; the page says so) | @@ -354,7 +354,7 @@ the campaign's own anchor audit found that the first thing to rot: - **(a) NEUTRALITY.** Every new behaviour is opt-in by a spec field or a new request kind. The adapter's `project()` (pine_adapter.cpp:2729-3015) never sets the field and never emits the kind, so adapter runs are byte-identical by construction. The four feed-shape fields are folded into `hash_spec` unconditionally today, including defaults (native_execution_consumer.cpp:239-261), so even a defaulted new field would move every continuation hash; the precedent for a conditional digest is `precommit_digest_` (native_execution_consumer.hpp:2002-2006). Each lane adds a test that a default spec hashes to the pre-change constant. Every new durable field is hashed (`check_broker_state_hash_coverage.py`, fail-closed, waivers in `broker_state_hash_waivers.txt`). Proof per PR: `scripts/run_corpus.sh` + `scripts/verify_corpus.py` locally, then ONE campaign sweep on the final tree, run by the maintainers (fixed population, 4190 probes when this plan was written, zero tolerance on the hard band — native-refactor-progress.md:42-48; a fresh exact-HEAD `PASS` verdict is required, `sha256:59adeba8aabaefa740633d52e9216b49065b28b397b87f22c57ad5e19f2948c2` AGENTS.md:144-145). A lane that cannot be shown neutral by construction does not merge. - **(b) TWIN.** One chosen probe per lane runs through the adapter AND a hand-written `NativeStrategyHost` on the same bars and spec. Trade lists are diffed **by identity** (entry / exit time, price, qty, pnl, commission), never by trade number; then event, ownership and hash detail (S's D → E order). A difference is allowed only where a named TV-quirk row of §1 is active, and each one is itemized. Because 430 test files drove the adapter and 14 the native host when this plan was written (O; on this tree 543 `tests/*.cpp` name `source::PineStrategyHost` and 110 name `NativeStrategyHost`), every lane also adds native-only tests: a lane is not accepted on the corpus alone, nor on a unit test alone (S). -- **(c) EPOCH BUDGET.** Three inline-namespace epochs move — the request epoch, the host epoch and (*fresh*, not in the inputs) the run-spec epoch, which every new spec field moves. They stood at `native_order_v5`, `engine_script_run_v17` and `native_run_spec_v2` when this plan was written; on this tree they are `native_order_v7` (native_order.hpp:25), `engine_script_run_v19` (native_host.hpp:20) and `native_run_spec_v3` (native_run_spec.hpp:15): the budget below was spent exactly once, and the v19 value epoch (R5 lane V19-A) took the host epoch one step past it. One bump each per release batch: `native_order_v6` = L3 + L7 + L4's origin / event; `engine_script_run_v18` = L4's host hooks + L5 + L6 virtuals + L2's `hash_host_extension` + §2.ii a, f; `native_run_spec_v3` = the spec blocks of L2, L4, L5, L6, L8 (L9 if it ships in the batch). A lane that lands before its batch closes stages behind the open epoch, so codegen-built strategies rebuild once per batch. Epoch lanes are not neutral-refactor lanes: the brief lists the header extension explicitly; the frozen C++ ABI fixtures (`tests/fixtures/native_cpp_abi/host-*`) and the ABI checkers (`check_native_cpp_abi.py`, `check_settlement_cpp_abi.py`) move with them; `variant_size` pins change (`OrderIntent` native_order.hpp:229, `CommandEvent` native_order.hpp:1308). Every new test unit compiles first against the frozen previous-epoch header closure (fail-before, `CLAUDE.md`). +- **(c) EPOCH BUDGET.** Three inline-namespace epochs move — the request epoch, the host epoch and (*fresh*, not in the inputs) the run-spec epoch, which every new spec field moves. They stood at `native_order_v5`, `engine_script_run_v17` and `native_run_spec_v2` when this plan was written; on this tree they are `native_order_v7` (native_order.hpp:27), `engine_script_run_v19` (native_host.hpp:20) and `native_run_spec_v3` (native_run_spec.hpp:15): the budget below was spent exactly once, and the v19 value epoch (R5 lane V19-A) took the host epoch one step past it. One bump each per release batch: `native_order_v6` = L3 + L7 + L4's origin / event; `engine_script_run_v18` = L4's host hooks + L5 + L6 virtuals + L2's `hash_host_extension` + §2.ii a, f; `native_run_spec_v3` = the spec blocks of L2, L4, L5, L6, L8 (L9 if it ships in the batch). A lane that lands before its batch closes stages behind the open epoch, so codegen-built strategies rebuild once per batch. Epoch lanes are not neutral-refactor lanes: the brief lists the header extension explicitly; the frozen C++ ABI fixtures (`tests/fixtures/native_cpp_abi/host-*`) and the ABI checkers (`check_native_cpp_abi.py`, `check_settlement_cpp_abi.py`) move with them; `variant_size` pins change (`OrderIntent` native_order.hpp:231, `CommandEvent` native_order.hpp:1310). Every new test unit compiles first against the frozen previous-epoch header closure (fail-before, `CLAUDE.md`). - **Worker verification** (`CLAUDE.md`): `ci_preflight.py`, then `ci_verify.py release`; plus `check_c_abi_runtime.py` and `check_native_include_independence.py`. No campaign sweep from a worker. ### 3.2 Lanes: scope, size, dependencies, epochs @@ -417,7 +417,7 @@ closure markers and `scripts/check_design_inventory.py`). | L6 | `NativeTimeframeSubscription`, `on_native_timeframe_bar`, `native_series_bar` → A.5, §2.iv | `subscriptions` empty for the adapter; registration-gated pump; `engine_security.cpp` and the host's `scheduler_prepare_security_sequence` (pine_strategy_host.cpp:1652) untouched; corpus diff before / after (O: highest parity risk of any lane) | HTF-filter probe on a 24x7 symbol: exact (F); `"D"` over a 15m feed equals a `TimeframeAggregator` baseline bar for bar (O); installed-feed and no-feed cases (S) | | L7 | `native_working_requests()`, `cancel_all()`, `cancel_where(label)`, `TriggerAnchor`, `TrailTicks`, zero offset, `ReplaceOptions`; toolkit `submit_bracket`, `OrderBook` → A.6 | new kinds unused by the adapter; anchor defaults to `Absolute`; the adapter's `tick * 0.5` sentinel keeps working | bracket + trailing probe through the builder: exact where TR5 / FP7 are inactive (on-grid levels) (F); the zero-offset trail test asserts the run no longer fails (O); parent-limit + stop sibling, parent rejection, replacement, parent-cycle revival (S) | | L8 | `NativePriceGrid { None, QuantizeFills, QuantizeFillsAndTriggers }`, `NativeGridRounding` → A.7 | `None` for the adapter, permanently — **native-only by ruling, audit lane P6: §3.6.2** | a host with `QuantizeFills` books on-grid prices; a second test pins that `QuantizeFillsAndTriggers` still differs from TV's half-tick threshold (FP5) (O) | -| L9 | `NativeRiskLimits`, `NativeRiskEvent`, `MatchRejectReason::RiskLimit` → A.8 | `risk` unset for the adapter; its ledger (`SourceDayLedger` pine_adapter.hpp:1427) untouched — **retained after measurement, audit lane N12; native-only by ruling, audit lane P6: §3.6.1** | risk probes: same halt bar (F); each limit + the day-boundary basis (O); breach, forced close, cancellation, next-day reset (S) | +| L9 | `NativeRiskLimits`, `NativeRiskEvent`, `MatchRejectReason::RiskLimit` → A.8 | `risk` unset for the adapter; its ledger (`SourceDayLedger` pine_adapter.hpp:1434) untouched — **retained after measurement, audit lane N12; native-only by ruling, audit lane P6: §3.6.1** | risk probes: same halt bar (F); each limit + the day-boundary basis (O); breach, forced close, cancellation, next-day reset (S) | | L10 | `PINEFORGE_EXPORT_NATIVE_STRATEGY(Class)` (§2.v) | build-only | the examples run in ctest; the independence checker compiles the relocated examples | | L11 | — (renames / moves, §2.ii a-i) | rename / move only; hashed enumerator values pinned; sweep unchanged | — | | L12 | — (§2.ii j-m) | **not neutral**: coordinated sweep, waiver updates, hash-domain plan (the `"pineforge-broker-state/v17"` literal of that wave, since moved to `"pineforge-broker-state/v19"` engine_state_hash.cpp:41 and pinned to one occurrence by check_broker_state_hash_coverage.py:229) | trades identical, hashes re-baselined once | @@ -524,11 +524,11 @@ line numbers are re-anchored to this tree (the measurement ran on 683a82f); kern | Pine rule | adapter mechanism | kernel counterpart | class | measured divergence (test scenario) | ruling | |---|---|---|---|---|---| | `strategy.risk.allow_entry_in` | `risk_.direction`; `project()` already declares `spec.allowed_open_directions` (pine_adapter.cpp:2861); the forbidden opposite entry is reshaped to `CloseOpposite` in the adapter's own terms branch | per-opening cap `allowed_open_directions` | (i)+(ii), **already re-lowered** | none to measure: the cap is the kernel's, the close-only reshaping is the retained TV policy | done before N12; nothing to move | -| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:16551; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | -| `strategy.risk.max_drawdown` | `update_risk_state(bar.close)` once per script bar at the close mark (`update_risk_state(bar.close)` pine_adapter.cpp:17541, `update_risk_state` pine_adapter.cpp:17541): peak / running max drawdown at close marks, `>=` exact, percent of the current peak; the latch `risk_.halted` gates flat/same-side entries at precommit (`risk_.halted` pine_adapter.cpp:16543), never an opposite entry | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a`: bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b`: latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | -| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1423-1431): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:25801); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | -| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:22554); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:20920), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:20952, `submit_intraday_loss_close` pine_adapter.cpp:20920), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:24758); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:17536, `:17536`, `:17536`, `:17536`, `:17536`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:14637) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | -| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:17532); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:17438, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77 (2025-04-08); `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293 (2025-10-25); `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17 (2025-04-02 02:45), two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | +| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:16555; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | +| `strategy.risk.max_drawdown` | `update_risk_state(bar.close)` once per script bar at the close mark (`update_risk_state(bar.close)` pine_adapter.cpp:17545, `update_risk_state` pine_adapter.cpp:17545): peak / running max drawdown at close marks, `>=` exact, percent of the current peak; the latch `risk_.halted` gates flat/same-side entries at precommit (`risk_.halted` pine_adapter.cpp:16547), never an opposite entry | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a`: bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b`: latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | +| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1434-1445): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:25806); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | +| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:22558); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:20924), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:20956, `submit_intraday_loss_close` pine_adapter.cpp:20924), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:24762); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:17540, `:17540`, `:17540`, `:17540`, `:17540`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:14641) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | +| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:17536); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:17442, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77 (2025-04-08); `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293 (2025-10-25); `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17 (2025-04-02 02:45), two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | **Proposed generic additions (not implemented — no consumer today).** A risk ticket (`label` / `comment` on `NativeRiskLimits`, folded only when set, @@ -742,9 +742,9 @@ rule (L8b) is the opposite on purpose — one rule for every kind. Rule 2's test applies literally: the mask cannot be justified without the word "TradingView". And what the adapter keeps is not a duplicate of the kernel mechanism that a re-lowering would delete: `source_trigger_threshold` -(pine_adapter.cpp:384), `source_level_on_price_grid` (:367) and the -`nearest_tick` (pine_adapter.cpp:274) / `source_bar_fill_tick` (:295) / -`directional_tick` (:327) spellings are a *different* rule — a half-tick threshold per order kind with +(pine_adapter.cpp:385), `source_level_on_price_grid` (:368) and the +`nearest_tick` (pine_adapter.cpp:275) / `source_bar_fill_tick` (:296) / +`directional_tick` (:328) spellings are a *different* rule — a half-tick threshold per order kind with the fill booked elsewhere (E7) — which is why re-lowering moves outcomes instead of preserving them. @@ -1086,11 +1086,11 @@ row C). row, every placement snapshot the run had accepted (a sorted walk of every key, since replaced by the walk of the retained rows, `placement_` pine_state_hash.cpp:332) — one per accepted request, cleared only by -`reset_for_run` pine_adapter.cpp:2398 — and reflected the whole admission +`reset_for_run` pine_adapter.cpp:2399 — and reflected the whole admission journal (since replaced by a running digest of its events, `admission_journal` pine_state_hash.cpp:787). `PineScheduler::hash_state` pine_state_hash.cpp:893 re-folded the consumed source prefix too (`consumed` -pine_state_hash.cpp:824-846). Measured with recording on and off, in two +pine_state_hash.cpp:845-846). Measured with recording on and off, in two shapes: - the audit's order-and-cancel probe (one `strategy_order` accepted and @@ -1121,22 +1121,22 @@ moving a single value, but not from inside lane F9's files: 2. **The transforms need a home.** A row's transform, or a composed segment's, has to outlive the call. The fold is const; the sink is created per call (engine_state_hash.cpp:32); and none of `PlacementTable` - pine_adapter.hpp:571, `admission::Journal` market_admission.hpp:189 or + pine_adapter.hpp:574, `admission::Journal` market_admission.hpp:189 or `PineScheduler` has a member it could keep one in. Adding one is a header change in `include/pineforge/source/`, outside this lane. 3. **A cache must know which rows changed, and nothing records it.** `PlacementTable` hands out mutable rows — the non-const `find` - pine_adapter.hpp:707, the non-const `at` pine_adapter.hpp:741 and its + pine_adapter.hpp:713, the non-const `at` pine_adapter.hpp:747 and its iterators — and the adapter rewrites retained rows in place through them, not only rows still working. Lane F9's probe hashed every row callback to callback: the order-and-cancel shape rewrites none, the re-issue shape 1,339 rows each one callback old, an OCA re-price and a trail attach rows two callbacks old, and the row of an entry that filled at bar 1 is rewritten at bar 60, when its first exit is attached - (`has_full_entry_bracket` pine_adapter.cpp:11469). Full-table scans - rewrite rows as well (`suspend_brackets_for_reversal` pine_adapter.cpp:1967, - `revive_brackets_after_margin` pine_adapter.cpp:2202, - `preserved_by_close_all` pine_adapter.cpp:14705), and + (`has_full_entry_bracket` pine_adapter.cpp:11473). Full-table scans + rewrite rows as well (`suspend_brackets_for_reversal` pine_adapter.cpp:1968, + `revive_brackets_after_margin` pine_adapter.cpp:2203, + `preserved_by_close_all` pine_adapter.cpp:14709), and `admission::Journal::retain` market_admission.hpp:214 drops events from the middle of the journal. So "a row stops changing once its request is done" is false, and a cache resting on it would move a value. @@ -1221,16 +1221,16 @@ doubling from 4,000 to 32,000 bars (11.2 s at 32,000), and standing one (OL14), and a gapped stop a declined reversal had parked for the next margin-call slice came back after a cancel too. A script cancel now retires every leg of each exit row it names and drops such a parked stop -(`retire_cancelled_exits` pine_adapter.cpp:2125), the revival skips a withdrawn +(`retire_cancelled_exits` pine_adapter.cpp:2126), the revival skips a withdrawn exit, and K1 holds a withdrawn leg only while its origin is live or opened -- the one reader left is `origin_leg_consumed`, and it asks about such origins alone (`erase_retired_rows` -pine_adapter.cpp:3650). Both shapes now cost ×1.89 to ×2.12 per doubling up +pine_adapter.cpp:3649). Both shapes now cost ×1.89 to ×2.12 per doubling up to 64,000 bars and retain 3 rows. Row D was the kernel's half: the adapter enrolled every opening in its id's kernel roster (`cohort_add`) and never took one off, and `hash_cohorts` folds every member at every read. An origin whose request chain no longer works and that holds no open lot can never be bound again, so at every bar open that follows a command event the adapter takes -it off (`release_closed_cohort_origins` pine_adapter.cpp:4112); a lot opened +it off (`release_closed_cohort_origins` pine_adapter.cpp:4111); a lot opened by a replace successor keeps every older member of its roster while it is open, because its root is not a public fact. The flip then costs ×1.99, ×2.01, ×2.02 per doubling, and its rosters end with one member where they @@ -1600,12 +1600,12 @@ the rule gives 9.9 + 9.89). | # | Risk | Why it bites | Containment | Src | |---|---|---|---|---| | E1 | **Spec hashing.** `hash_spec` folded every field unconditionally (native_execution_consumer.cpp:235-257); every block added since folds only when set | Any new field, even defaulted, changes every continuation / broker-state hash, adapter runs included | Fold new blocks only when set, as a tagged extension (precedent `precommit_digest_` native_execution_consumer.hpp:2002); test that a default spec hashes to the pre-change constant | F | -| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:18004); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:20286 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by R5 lane H-THIN: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by R5 lane F7 | F O S | -| E3 | **Sizing.** TV sizes from the tick-rounded signal close, 10-digit-rounded equity and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3193, the COOF `at_fill` exception pine_adapter.cpp:8439) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since R5 lanes N11 and F7 the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the freeze, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | -| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (R5 lane MAG-INTRABAR) (`PineScheduler::recalculate` pine_scheduler_native.cpp:767, `coof_next_waypoint` pine_adapter.cpp:6993) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Freeze `born_on_remaining_path` native_execution_consumer.cpp:6563 and `drain_applied_notifications` (native_execution_consumer.cpp:7246); new behaviour only behind new opt-ins | F | -| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:22314) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | +| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:18008); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:20290 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by R5 lane H-THIN: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by R5 lane F7 | F O S | +| E3 | **Sizing.** TV sizes from the tick-rounded signal close, 10-digit-rounded equity and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3192, the COOF `at_fill` exception pine_adapter.cpp:8443) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since R5 lanes N11 and F7 the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the freeze, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | +| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (R5 lane MAG-INTRABAR) (`PineScheduler::recalculate` pine_scheduler_native.cpp:767, `coof_next_waypoint` pine_adapter.cpp:6997) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Freeze `born_on_remaining_path` native_execution_consumer.cpp:6563 and `drain_applied_notifications` (native_execution_consumer.cpp:7246); new behaviour only behind new opt-ins | F | +| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:22318) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | | E6 | **Double equity recording.** The source host already records per bar (history: only its cadence remains, `scheduler_mark_report_point` pine_strategy_host.cpp:2844) | Kernel recording would duplicate points and shift drawdown | Report policy / ownership (A.1); the adapter declares `KernelRecordedAtHostMarks` (R1): one recorder, at the adapter's own marks | F O | -| E7 | **Price grid.** TV does not merely quantize: it tests triggers against a quantized bar while booking the fill on the *unrounded* level, so the adapter shifts the threshold (`source_trigger_threshold` pine_adapter.cpp:384) and books elsewhere | A kernel grid that quantizes both sides is correct and not TV; moving rounding into the kernel would change both native results and the adapter | `NativePriceGrid::None` for the adapter permanently; never "unify". Before L8: a byte-level experiment that the default price spelling on decimal ticks (one-ULP representations) does not move (S) | O S | +| E7 | **Price grid.** TV does not merely quantize: it tests triggers against a quantized bar while booking the fill on the *unrounded* level, so the adapter shifts the threshold (`source_trigger_threshold` pine_adapter.cpp:385) and books elsewhere | A kernel grid that quantizes both sides is correct and not TV; moving rounding into the kernel would change both native results and the adapter | `NativePriceGrid::None` for the adapter permanently; never "unify". Before L8: a byte-level experiment that the default price spelling on decimal ticks (one-ULP representations) does not move (S) | O S | | E8 | **HTF.** The evaluator machinery carries TV publication semantics (lookahead, boundary deferral, early close, `syminfo.type` branching engine_security.cpp:67-73). The feed store is TV-calibrated too: "W" / "M" from native daily bars, native stamps as the period partition (`set_native_security_feed` `sha256:eca3dbea0baa461982ba89f13d0a5c43277854069fb6eb2dd2f5c5471313204d` engine_aux_security.cpp:51-83, *fresh*). Two aggregators exist: script buckets use `native_calendar`, `request.security` uses `TimeframeAggregator`, whose completion rules are TV-pinned (timeframe.hpp:288-330) | Reuse risks changing the *order* of the Pine scheduler's pumps; a bare host inherits TV partition rules; native HTF may differ from Pine HTF on early-close sessions | Registration-gated pump; no change to `scheduler_feed_security_input` pine_strategy_host.cpp:1832; full corpus diff; supplied-bar provenance preserved, never silently replaced by chart aggregation (S); differences on early-close sessions are recorded, not "fixed" (F). The Pine scheduler keeps one more `TimeframeAggregator` pass, before the run (`PineScheduler::run_begin`, audit M26): the script-bar count it yields (`terminal_source_bar`, the probe-tail suppression) and the per-input completion flags it hands `on_native_input` are needed before the first input arrives, while the kernel seals script intervals only as inputs arrive and offers no pre-run interval query (`NativeExecutionConsumer::script_interval_at` is consumer-private). Retained by R5 lane F7: replacing it needs that kernel capability | F O S | | E9 | **TA numerics are TV-calibrated** (1e-10 band in DMI / MFI / percentrank; conditional-call extremum ring `ExtremeRing` ta.hpp:134) | Non-Pine users may expect IEEE semantics; changing them breaks parity | Rename only (§2.ii b). For an every-bar caller the ring equals a positional window (the ring comment `sha256:fd1f92206558863ad8df33fd3ef3b8cc8b9bd6678a25fc98ca866168c1a81aca` ta.hpp:87-88). Document. → U6 | F O S | | E10 | **Dead-but-hashed lot flags** (§2.ii j) | Deleting them changes `broker_state_hash` | Separate, explicitly non-neutral lane (L12) | F O | @@ -1623,7 +1623,7 @@ the rule gives 9.9 + 9.89). | P5 | **Test asymmetry:** 430 adapter test files vs 14 native when this plan was written (on this tree 543 `tests/*.cpp` name `source::PineStrategyHost`, 110 `NativeStrategyHost`) | native-only tests in every lane (§3.1 b) | O | | P6 | **Stream vs batch.** The stream refuses monthly input, mixed modes, in-session gaps, security feeds and source setters, and tick input under an FX curve (any non-empty curve until audit lane N6) (`native-engine.md`, "Stream limitations (current)") | L5 and L6 must state which refusals are generic and which are deliberate source limits | S | | P7 | **C ABI variant growth** | §2.iii hardening rules | S | -| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:364) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:902 → `set_risk_direction` pine_adapter.cpp:26543 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the lane that touches each | F O | +| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:367) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:902 → `set_risk_direction` pine_adapter.cpp:26548 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the lane that touches each | F O | ### 4.3 Open design questions (supervisor level) @@ -1650,7 +1650,7 @@ All six are settled by what landed: U1 (native subscriptions take `lookahead` an | **U3** | **Do risk limits ship in v1?** | R5-9: in scope, not a blocker. S wanted them as a prerequisite; F accepts a toolkit stand-in; O defers. | | **U4** | **Stream-safe FX (FP6): in the roadmap or not?** It has no R5 lane. | F: out of scope here ("O7 clock"). S: its P10, an FX epoch consumed while flat, same curve in batch and stream. | | — | **U4, answered by audit lane N6.** In: the curve declared through `configure_native_fx_curve` is the FX epoch of the run, batch or stream; the account converts at the rate of each driver point's effective time, so the epoch is consumed by the walk itself, flat or not, and a confirmed-bar stream is the batch of the same bars. No `NativeFxEpoch` type and no `on_native_fx_epoch` callback were needed. Out, permanently: realtime rate ingestion (a new rate is a new run) and tick-driven input under a curve (its hooks and partial slots read a stale conversion clock — measured). | — | -| — | **N6 follow-ups (measured on the adapter; both closed by R5 lane F7).** (1) `apply_fx_open_margin_slice` (pine_adapter.cpp:5704) throws on a rate step over a carried position whose margin is not 100 %: a 2× long carried through a 1.0 → 1.5 step ends the whole run with status 1 and an empty report, where the same book without a series, or with a series that never steps, completes. The kernel's `FxRoll` point is the generic mechanism that case can re-lower onto. (2) `default_sizing_intent` (pine_adapter.cpp:3266) returns nullopt whenever any FX series is staged — even one point that restates the scalar rate — so an FX run silently loses the kernel `Sized` path R2b landed (measured: 1 kernel `Sized` request without a series, 0 with a one-point 1.0 series, same 50-unit book). **Closed by R5 lane F7.** (1) The throw is gone: the carried roll takes TradingView's broker-open checkpoint at any positive margin — its money at the open under the new rate and the side's own maintenance ratio, its lot-floored 4x restore, executed at the open before the step bar's script — and a survivor that is not a 1x long is re-checked over the rest of the bar by the kernel's liquidation. A 2x long or short of 15 at 100 on 1000 across 1 → 1.5 takes 6.666666666666667 at the open and keeps 8.33; at a 25 % side ratio it takes nothing; a constant curve never rolls (`tests/test_affordability_fx.cpp` F7 rows; G6's leveraged pin and its L0 oracle twin moved with an `expectation corrected:` note). The kernel's `FxRoll` point stays refused, measured (MG9). (2) `Sized` is withdrawn only where the kernel would convert at another rate than the source sized at — a curve step between the sub-bar open and the acceptance coordinate — so a one-point series lowers exactly as no series does, and on a stepped curve only the step's own bar keeps its host-resolved intent (`tests/test_adapter_sizing_relower.cpp` case 12). | — | +| — | **N6 follow-ups (measured on the adapter; both closed by R5 lane F7).** (1) `apply_fx_open_margin_slice` (pine_adapter.cpp:5703) throws on a rate step over a carried position whose margin is not 100 %: a 2× long carried through a 1.0 → 1.5 step ends the whole run with status 1 and an empty report, where the same book without a series, or with a series that never steps, completes. The kernel's `FxRoll` point is the generic mechanism that case can re-lower onto. (2) `default_sizing_intent` (pine_adapter.cpp:3265) returns nullopt whenever any FX series is staged — even one point that restates the scalar rate — so an FX run silently loses the kernel `Sized` path R2b landed (measured: 1 kernel `Sized` request without a series, 0 with a one-point 1.0 series, same 50-unit book). **Closed by R5 lane F7.** (1) The throw is gone: the carried roll takes TradingView's broker-open checkpoint at any positive margin — its money at the open under the new rate and the side's own maintenance ratio, its lot-floored 4x restore, executed at the open before the step bar's script — and a survivor that is not a 1x long is re-checked over the rest of the bar by the kernel's liquidation. A 2x long or short of 15 at 100 on 1000 across 1 → 1.5 takes 6.666666666666667 at the open and keeps 8.33; at a 25 % side ratio it takes nothing; a constant curve never rolls (`tests/test_affordability_fx.cpp` F7 rows; G6's leveraged pin and its L0 oracle twin moved with an `expectation corrected:` note). The kernel's `FxRoll` point stays refused, measured (MG9). (2) `Sized` is withdrawn only where the kernel would convert at another rate than the source sized at — a curve step between the sub-bar open and the acceptance coordinate — so a one-point series lowers exactly as no series does, and on a stepped curve only the step's own bar keeps its host-resolved intent (`tests/test_adapter_sizing_relower.cpp` case 12). | — | | **U5** | **Open-bar view default.** Should `OpenOnly` be the default for hosts that are not `LegacyTolerant`? | F: safer for users, but a behaviour change for the ~20 existing native tests; default `Complete` proposed. | | **U6** | **TA comparison band.** Is TradingView's 1e-10 tie-breaking a feature of the TA library, or a default to make configurable? | F, O: rename only; a switch changes indicator outputs and needs its own lane. S: separate policies, hash the policy identity. | @@ -1681,7 +1681,7 @@ NativeMetricsView native_metrics() const; // S: net_pro ### A.2 L3 — sizing, fractional reduce, sibling reservation (F P1, P7 + O G2 + S P2, P3) ```cpp -// native_order.hpp — native_order_v6; OrderIntent grows 5 -> 6 (static_assert native_order.hpp:1388) +// native_order.hpp — native_order_v6; OrderIntent grows 5 -> 6 (static_assert native_order.hpp:2505) struct CashValue { double cash = 0.0; }; // account currency struct EquityFraction { double fraction = 0.0; }; // of marked equity at the sizing point (0.10 = 10 %) using SizeBasis = std::variant; @@ -1690,7 +1690,7 @@ struct Sized { // O's name; F: SizedOpen Side side; SizeBasis basis; SizeTime time = SizeTime::AtMatch; - ExecutionGridPolicy grid_policy = ExecutionGridPolicy::SnapToGrid; // existing enum, `ExecutionGridPolicy` native_order.hpp:82 + ExecutionGridPolicy grid_policy = ExecutionGridPolicy::SnapToGrid; // existing enum, `ExecutionGridPolicy` native_order.hpp:84 bool reserve_percent_fee = false; // divide cash by (1 + fee) for NativeFeeKind::Percent }; using OrderIntent = std::variant; @@ -1699,7 +1699,7 @@ struct ScopeFraction { double fraction = 1.0; ScopeClaim claim = ScopeClaim::Gro using ReductionSize = std::variant; // F ``` -- Resolution: `units = cash / (price * point_value * fx)`, `cash` = the value or `fraction * marked_equity(price)`, `price` = the candidate's `default_resolved_price`. It plugs in where `HostSized` units resolve today (native_execution_consumer.cpp:733; terms resolution `resolve_execution_terms` native_execution_consumer.cpp:5864). A host override of `resolve_execution_terms` keeps the last word. Non-representable → `MatchRejectReason::TermsUnresolved` (native_order.hpp:877); validation reuses `RequestRejectReason::InvalidQuantityBasis` (native_order.hpp:827). +- Resolution: `units = cash / (price * point_value * fx)`, `cash` = the value or `fraction * marked_equity(price)`, `price` = the candidate's `default_resolved_price`. It plugs in where `HostSized` units resolve today (native_execution_consumer.cpp:733; terms resolution `resolve_execution_terms` native_execution_consumer.cpp:5864). A host override of `resolve_execution_terms` keeps the last word. Non-representable → `MatchRejectReason::TermsUnresolved` (native_order.hpp:879); validation reuses `RequestRejectReason::InvalidQuantityBasis` (native_order.hpp:829). - `NetOfSiblings`: a fraction-sized reduce claims the bound scope minus the units already claimed by live sibling reduces on it; keys are opaque, never source IDs (S). - Not adopted: S's spec-level `default_sizing` + virtual `resolve_native_sizing` (duplicates the per-request intent, costs an epoch); S's `PercentFreeMargin` (needs L4's free-margin definition) and `EveryCalculation` timing; O's spec-level `size_net_of_fees` (per-request flag, 2 of 3); O's `PercentOfScope` basis (expressed as `ScopeFraction`); S's `ReplaceByKey` / `Fifo` modes (callsite semantics → adapter, R5-4). - TUs: `native_order.{hpp,cpp}`, `native_execution_consumer.cpp`, `native_run_spec.cpp`, the definition hash. @@ -1726,7 +1726,7 @@ virtual std::optional resolve_margin_call_units(const NativeMarginCallVi virtual void on_native_margin_call(const native_order::MarginCallEvent&) {} // O, S ``` -- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:20273). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1308). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:3404), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). +- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:20277). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1310). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:3404), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). - FX revaluation (MG9) falls out once `native_fx_curve` drives the mark (`native_fx_curve.cpp`). - TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:3728 per side; scheduling next to `invoke_bar_open_callback` / `drain_after_applied`), `native_order.*`, `engine_state_hash.cpp` + the coverage checker, `c_abi.cpp` + `pineforge.h` (`pf_native_run_spec_v2` or an extension struct; `CLAUDE.md` SOP). @@ -1749,7 +1749,7 @@ std::optional current_partial_bar() const; Callback chronology contract (the normative part of the lane; F P4.1 + O G8): 1. At one point: match and settle → `on_native_applied` per applied event, FIFO (rule already at `on_native_applied` native-engine.md:1729-1730) → with `BarCloseAndFills`, one recalculation at the fill cursor (reason `OrderFill`), driven from the existing notification drain (native_execution_consumer.cpp:7990-8002) under its re-entrancy guard, bounded by `max_recalculations_per_point`. S would run it *before* the drain; not chosen, the drain order is frozen (E4). -2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:6563). `born_on_remaining_path` (native_execution_consumer.cpp:6563) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:1965-1995) is calibrated against it. +2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:6563). `born_on_remaining_path` (native_execution_consumer.cpp:6563) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:1991-1998) is calibrated against it. 3. `EveryModeledPoint`: a calculation at each magnifier sample / observed print, in batch as well as stream (every delivery loop asks the per-point guard `recalculate_at_point` native_execution_consumer.hpp:1324-1329, whose body is `recalculate_at_modeled_point` native_execution_consumer.cpp:7836). `on_native_tick` stays the observation hook (S). 4. `current_partial_bar()` derives from the driver points already recorded (`record_driver` native_execution_consumer.cpp:3348); valid in bar-open, applied and tick callbacks. 5. The kernel never attempts language-state rollback; that stays Pine's (`snapshot_coof_script_state` pine_scheduler_native.cpp:99, `restore_coof_script_state` :117, `commit_coof_script_state` :134) (O). @@ -1782,13 +1782,13 @@ struct NativeWorkingRequest { native_order::DefinitionRef definition; native_ord native_order::TriggerState trigger_state; }; std::vector native_working_requests() const; // F: owning snapshot (S: NativePendingView) std::size_t cancel_all(); // F, O: one CancelledEvent per live request -std::size_t cancel_where(std::string_view label); // O: optional kernel label -> live-handle index (labels exist, `Request::label` native_order.hpp:485) +std::size_t cancel_where(std::string_view label); // O: optional kernel label -> live-handle index (labels exist, `Request::label` native_order.hpp:487) // native_order.hpp — native_order_v6, shared with L3 struct Absolute {}; struct FromOwnerFill { double offset = 0.0; bool ticks = false; }; // O: signed, adverse = negative using TriggerAnchor = std::variant; // Request::anchor = Absolute{} struct TrailTicks { double ticks; }; // O: offset spelling resolved against price_tick -struct ReplaceOptions { bool retain_trigger_state = false; }; // O: keeps `TrailTrack::best` (native_order.hpp:651) +struct ReplaceOptions { bool retain_trigger_state = false; }; // O: keeps `TrailTrack::best` (native_order.hpp:653) // Trail::offset == 0.0 is accepted: "exit on the first adverse move past the best" (now accepted: `valid_trail_offset` native_order.cpp:511) // include/pineforge/native_toolkit.hpp — header-only, Pine-free, covered by the include-independence checker (F; shapes from S P1) @@ -1801,7 +1801,7 @@ BracketReceipt submit_bracket(NativeStrategyHost&, const BracketSpec&); // emi template class OrderBook; // id -> handle: submit-or-replace, cancel-by-id ``` -- A leg with owner `WaitForApplied` / `BindOpening` and a non-absolute anchor gets its level when the owner's fill arms it (`ArmedEvent` native_order.hpp:1246-1253 already exists) (O). Fallback with no kernel change: the toolkit materializes relative levels from the host's `on_native_applied` (F). +- A leg with owner `WaitForApplied` / `BindOpening` and a non-absolute anchor gets its level when the owner's fill arms it (`ArmedEvent` native_order.hpp:1248-1255 already exists) (O). Fallback with no kernel change: the toolkit materializes relative levels from the host's `on_native_applied` (F). - Gap lane N13 (after R4d): two facts of the arm were missing and are now opt-in fields appended to `WaitForApplied`, each default byte-identical and folded only when set. `NativeArmFirstMatch { AtArmPrint, AfterArmPrint }`: whether the armed leg is a candidate at its owner's fill print or, like a request born in the owner's fill callback, only after it. `NativeArmScope { OwnerLot, Book }`: whether an armed closing leg closes the lot its owner opened or is bound at the arm to the whole position (a `BookClose`), under which it may be `HostSized`. Ruling: generic — both are broker models of a contingent child, neither names a source language — and they are what makes an anchored child the very request a host would submit from `on_native_applied`. The Pine adapter's relative `strategy.exit` legs use both; R4d's owner-lot spelling left a later same-id add open where the source leg closes the book (`rel-pyramid-set-once`, tests/test_adapter_brackets_relower.cpp). - Not adopted (R5-2): S's kernel bracket object, its C symbols `strategy_native_bracket_*` and its TU list (`engine_orders.cpp`, `engine_execution.cpp`). - TUs: `native_order.{hpp,cpp}` (arming), `native_matching.hpp` (`checked_trail_stop`), `native_execution_consumer.cpp` (`prepare_owner_applied` path, command surface). @@ -1836,7 +1836,7 @@ std::optional risk; // NativeRunSpec - Breach: `MatchRejectReason::RiskLimit` for openings (O; F: `RiskHalted`), an optional kernel-issued flatten through A.3's kernel-originated request (`origin = KernelRisk`, F), and a `NativeRiskEvent` in `native_events` (native_host.hpp:1335) (O) or a callback `on_native_risk_event` (S — a virtual, so only inside an epoch batch). The day key uses the existing calendar (`session_day_ordinal` native_calendar.hpp:416), not a chart clock. - S's `directions` / `max_units` / `max_position_units` are existing spec fields (MG1-MG3), not duplicated. -- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (`update_risk_state` pine_adapter.cpp:17541; `on_applied` pine_adapter.cpp:24463; the `opening_margin_checkpoint` block pine_adapter.cpp:17373-17378); a toolkit helper is an acceptable stand-in until L9. +- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (`update_risk_state` pine_adapter.cpp:17545; `on_applied` pine_adapter.cpp:24467; the `opening_margin_checkpoint` block pine_adapter.cpp:17377-17382); a toolkit helper is an acceptable stand-in until L9. - TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:3728 and the per-point walk), `native_order.hpp`, the state hash. ### A.9 Single-source proposals not adopted diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index 743576a3a..3c73cb761 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -173,17 +173,17 @@ them for the record only. "Linux aarch64" is the maintainers' Cortex-A725 host. | Item | What it is | Measured size | Deferred by | Prototype and evidence | Reopens when | | --- | --- | --- | --- | --- | --- | -| D2-B, the quiet-bar protocol and one receipt read per bar | PERF-D2's lane: a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. D2-A landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:322`). PERF-L4 gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:6492`), which still runs about three times a bar | PERF-D2, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On this tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | "R5 PERF-D2 DONE 2026-09-24 19:56" (D2-B after V19-D); "R5 SUPERVISOR RULING 2026-09-25 00:15 Taipei": "D2-B (quiet-bar protocol) waits for PERF-D3's verdict"; TRIAGE1 §D | An unpublished prototype branch and its patches. PERF-D3's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | +| D2-B, the quiet-bar protocol and one receipt read per bar | PERF-D2's lane: a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. D2-A landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:322`). PERF-L4 gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:6496`), which still runs about three times a bar | PERF-D2, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On this tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | "R5 PERF-D2 DONE 2026-09-24 19:56" (D2-B after V19-D); "R5 SUPERVISOR RULING 2026-09-25 00:15 Taipei": "D2-B (quiet-bar protocol) waits for PERF-D3's verdict"; TRIAGE1 §D | An unpublished prototype branch and its patches. PERF-D3's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | | S1a, kernel quiet-bar driving (PERF-D3) | A bar with nothing live, a flat book and a declined opening is driven in a quiet form: the kernel writes the bookkeeping its four price points would leave (ordinals, the last two driver marks, the floor, the point epochs) without walking them or calling the bar-open callback. Additive host cadence declarations; C hosts declare from their callback table; one C symbol under the PF_API checklist | PERF-D3, Linux aarch64 (Cortex-A725), GBench time and `perf stat` instructions, on the L3 tree `b33ea659`, S1a with S1b: no-order floor 0.764 / 0.660 in time (0.718 / 0.656 in instructions), 14 probes 0.958 / 0.921 (0.892 / 0.856), 100 slots 0.972 / 0.956 per slot. A bare kernel host that opts in: idle 0.545× (35 % fewer instructions a bar), market 0.81×, bracket 0.92×. Byte-identical: corpus 312/312, the 132-configuration battery with its hash columns | "FINDING PERF-D3 (filed for a post-release epoch per the owner's cap)" (ledger, 2026-09-24 16:58 UTC); "OWNER DECISION 2026-09-25 00:20 Taipei"; TRIAGE1 §D: "PERF-D3 S1/S2 (post-release epoch)" | An unpublished prototype branch (five commits on L3's tree) and its patches | The post-release epoch, with the owner calls PERF-D3 lists (the API under the existing ruling, the restated gate count) | | S1b, the Pine lazy opening | The Pine host records what its skipped opening and its uncalled input callback would have written, at the bar's first host callback. Member functions only, no layout change | Measured with S1a. A quiet bar then asks 15 gates, not 30, so the quiet-bar cost check `CHECK(asked >= 20 * o.boundary.size())` (`tests/test_adapter_quiet_bar.cpp:1054`) must be restated | As S1a | As S1a. The in-script hash witnesses caught the prototype's one miss (`awaiting_legacy_script_open_ms_`) | As S1a | | S1c, the held-position wake band | S1 for bars that hold a position with nothing live, 41 % of the median public slot's bars. The host declares a price band, and the kernel calls the opening only when the bar's range leaves it. The adapter derives the band from its margin model and declines when it cannot bound it | PERF-D3, a measurement-only upper bound without the band, which is not sound in general: 100 slots 0.776 / 0.716 per slot, 14 probes 0.830 / 0.761, cumulative against L3 with S1, S2 and D2-A | As S1a; the owner call "Fund the price band" | An unpublished measurement-only script | As S1a. Size L, parity risk high: TradingView's margin money rules | | S2 and S2e, the codegen feature manifest | Codegen states which per-bar hooks and publications a strategy can reach, and the engine skips the others; no manifest means every feature. A median 39 of 52 are statically unreachable, over the 312 corpus strategies and the 100 slots alike. S2e adds two gates that stop the Pine risk ledgers of a strategy with no risk rule | PERF-D3, on top of S1: floor −14 % / −15 % in time (−6.4 % in instructions), 14 probes 0.943 / 0.934, 100 slots 0.942 / 0.945. The nine S2 gates are byte-identical. S2e adds about 2–3 % and moves the adapter hash domain `pineforge-source-adapter/v4` to its next version | As S1a; the supervisor's cap: "any structural S1/S2 redesign is a post-release epoch" | In PERF-D3's unpublished prototype (a regex-based manifest scanner) | A codegen and engine co-change with a corpus re-emission; S2e also needs an owner decision on the hash-domain step | | The kernel quiet-run loop and the Pine quiet calculation | A loop for consecutive quiet bars of a pumped batch at the chart's own timeframe (label and gap checks in the batch preflight, precomputed intervals and session facts, one calculation frame); and skipping a flat bar's post-script hooks when its script issued nothing | Estimated, not prototyped (PERF-D3's report, section 3.7): about 1,000 instructions a bar for the loop and 800 for the quiet calculation, taking the floor from 3,726 instructions (S1, S2 and D2-A) to about 1,900, or 180–200 ns a bar | As S1a (PERF-D3's follow-up row) | The PERF-D3 report only | After S1a and S1b | | PGO | Profile-guided optimisation of the library objects: an instrumented build, a training run over the bench strategies, a `-fprofile-use` rebuild and the identity gates, per compiler and target, with a staleness gate on the profile | LAYOUT1, Linux aarch64 (Cortex-A725), GCC 13.3, `perf stat`, trained on 86 slots and timed on the 14 held-out probes: 14.5 % fewer cycles, CPU time 0.865, 7 % fewer instructions, 26 % fewer L1i refills, 36 % fewer front-end stalls, layout spread 9.0 % → 5.1 %, fingerprints identical | "R5 LAYOUT1 DONE + ACCEPTED 2026-09-24 21:45 Taipei": "PGO APPROVED as a lane (PGO1) scheduled LAST in the perf wave after the code lanes (profile staleness)"; the supervisor's cap: "PGO folds into the release lane"; the owner: "release with PGO in the release build"; TRIAGE1 §D | Unpublished. The tree carries only LAYOUT1's `-falign-functions=64` | The release lane. BENCH4 timed the standard Release build without PGO, and neither BENCH4's nor REL10-ENGINE's brief names it | -| Compact journal records and a history ring buffer (L3) | Smaller command records and a ring buffer for the request core's history. `WorkingRequestCore::history` (`include/pineforge/native_order.hpp:1709`) hands out the whole vector and the core holds `history_` (`include/pineforge/native_order.hpp:2293`) inside `native_order_v7`, so both are a layout and API change | L3, Linux aarch64 (Cortex-A725), `perf` call graphs, inclusive shares: at most 3.68 % of bracket, 11.99 % of re-issue, 4.26 % of market and 3.42 % of the 14 probes (retiring the history alone: 1.64, 4.51, 2.53 and 1.00 %); only part is realisable, since records must still be written | "R5 L3 DONE + ACCEPTED 2026-09-24 22:10 Taipei", ruling 1: "DEFERRED to the PERF-D3 structural plan"; TRIAGE1 §D: "compact journal records" | Lane L3's code, merged by INT22 (#285); its profiles are unpublished | A `native_order` layout epoch | -| A checked-values handoff for executions (L3) | `execution_values` runs twice, in `check_execution` (`src/native_order.cpp:4899`) and again in `apply_execution` (`src/native_order.cpp:4922`); handing the checked values across needs a public type, an API addition with no layout change | L3, as above: 2.21 % of bracket for the two together, about 1.1 % recoverable | L3 ruling 2: "a public checked-values handoff for executions (~1.1% bracket) -> DEFERRED" | As above | An owner decision on the API addition | +| Compact journal records and a history ring buffer (L3) | Smaller command records and a ring buffer for the request core's history. `WorkingRequestCore::history` (`include/pineforge/native_order.hpp:1712`) hands out the whole vector and the core holds `history_` (`include/pineforge/native_order.hpp:2299`) inside `native_order_v7`, so both are a layout and API change | L3, Linux aarch64 (Cortex-A725), `perf` call graphs, inclusive shares: at most 3.68 % of bracket, 11.99 % of re-issue, 4.26 % of market and 3.42 % of the 14 probes (retiring the history alone: 1.64, 4.51, 2.53 and 1.00 %); only part is realisable, since records must still be written | "R5 L3 DONE + ACCEPTED 2026-09-24 22:10 Taipei", ruling 1: "DEFERRED to the PERF-D3 structural plan"; TRIAGE1 §D: "compact journal records" | Lane L3's code, merged by INT22 (#285); its profiles are unpublished | A `native_order` layout epoch | +| A checked-values handoff for executions (L3) | `execution_values` runs twice, in `check_execution` (`src/native_order.cpp:4915`) and again in `apply_execution` (`src/native_order.cpp:4938`); handing the checked values across needs a public type, an API addition with no layout change | L3, as above: 2.21 % of bracket for the two together, about 1.1 % recoverable | L3 ruling 2: "a public checked-values handoff for executions (~1.1% bracket) -> DEFERRED" | As above | An owner decision on the API addition | | A chunked `trades_` store (L3) | Closed trades in chunks rather than one vector; `trades_` (`include/pineforge/engine.hpp:547`) is protected `BacktestEngine` state, so script ABI | L3, as above: on bracket, vector growth 0.64 %, `Trade` moves 0.74 % and `record_close_trade` 0.43 % (at most about 1.4 %); at most 0.35 % on the 14 probes | L3 ruling 4: "chunked trades_ store (ABI) -> DEFERRED" | As above | An `engine_script_run` epoch that changes the engine's protected layout | -| A zoned chart-day memo (D2-D finding 4) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:17411`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:22314`) reads `chart_day_key` (`src/source/pine_adapter.cpp:22551`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:46`), pays one `localtime_r` per equity point. UTC charts use D2-D's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:17395`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before D2-D's memo and 68 after (D2-D, Linux aarch64, callgrind on the floor). PERF-K1: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | TRIAGE1 §D: "D2-D finding 4 (the timezone `localtime_r` memo)", under the owner's cap. D2-D's report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A lane that owns `src/timezone.cpp` and builds an exact zone-transition memo (PERF-K1's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | +| A zoned chart-day memo (D2-D finding 4) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:17415`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:22318`) reads `chart_day_key` (`src/source/pine_adapter.cpp:22555`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:46`), pays one `localtime_r` per equity point. UTC charts use D2-D's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:17399`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before D2-D's memo and 68 after (D2-D, Linux aarch64, callgrind on the floor). PERF-K1: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | TRIAGE1 §D: "D2-D finding 4 (the timezone `localtime_r` memo)", under the owner's cap. D2-D's report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A lane that owns `src/timezone.cpp` and builds an exact zone-transition memo (PERF-K1's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | | Thread-local reads that remain per call | D2-C moved the three scopes' writes onto the pump's block, `pump_ambient` (`src/native_execution_consumer.hpp:530`). A reader still reaches the block through `tl_runtime_ambient` (`src/ta_extremes_volume.cpp:29`): `bar_context` (`src/ta_extremes_volume.cpp:82`), read by every `ExtremeRing::update` (`src/ta_extremes_volume.cpp:117`), which backs `ta.highest`, `ta.lowest`, `ta.highestbars` and `ta.lowestbars` and, through an embedded `Highest` and `Lowest`, `ta.stoch`, `ta.wpr` and `ta.range`; and `matching_day_partition` (`src/timeframe.cpp:628`), read by the default-anchored `ta.vwap` through `session_day_index`, by `crosses_boundary` on a DAY period (`timeframe.change("D")`) and by the session-period helpers. `decompose_ms_local` (`src/session_time.cpp:250`) reads two thread-locals per call on a zoned clock, and the prepared order storage's `free_blocks` (`src/native_order.cpp:620`) one per prepared mutation. `ema_na_warmup_flag` is read once per EMA, at its first compute | One thread-local access per reader call. AUDIT4, macOS arm64, a dlopen'd tutorial-MACD probe, counted by `_tlv_get_addr` breakpoints and by descriptor patching: 1.000 a bar for one `ta::Highest` and for one default `ta::VWAP`, 1.999 for one `pine_hour` on a New York clock, 0.000 on the no-order floor and with orders. On Linux a TLS-descriptor call costs about 20 instructions: D2-C's 13 → 4 calls a bar was 260 → 80 instructions (Linux aarch64, GCC 13, callgrind). INT23: 2.00 a bar on the zoned probe 064, 0.08 on probe 008 | None for the TA and day-partition readers: "RULING: INT23 ACCEPTED" (ledger, 2026-09-24 23:30 UTC) reads "dlopen TLS 13->0/bar", and INT23's report names only the timezone and order-core residuals. This row is their record | Unpublished (AUDIT4's scratch) | A byte-identical lane that hands these readers the pump's block without a thread-local read. `ta::bar_context()` is script ABI, so its public declaration stays as it is | | "audit-off mode", "LTO", "data-oriented layout" | Named in "OWNER DECISION 2026-09-25 00:20 Taipei" among the post-release structural items: "S1 bulk quiet bars, S2 per-strategy specialization, audit-off mode, LTO+PGO, data-oriented layout". No engine or campaign document defines, designs or measures the three | None | The owner decision | None | Struck from this inventory until a design note defines one of them | @@ -210,7 +210,7 @@ or more, and the same run under Full holds 66,000 points in one block of **PERF-L4 reads the receipts about three times a bar, not once.** PERF-L4's brief asked it to "read the receipts ONCE per bar instead of three times". The lane gated the read instead: `observe_terminal_receipts` -(`src/source/pine_adapter.cpp:6492`) returns at its receipt gate when no +(`src/source/pine_adapter.cpp:6496`) returns at its receipt gate when no command lies above its cursor. `every_gate_is_taken_both_ways` (`tests/test_adapter_quiet_bar.cpp:1061`) prints `gate ReceiptRead asked 40996 quiet 33552 live 7444`, against `asked 13248` @@ -241,7 +241,7 @@ Mac's 489 and 412 and 1003 of 1249, and no Mac output moved. - **PERF-L4's retired reader slots.** `event_high_water_reader_` and `terminal_receipt_high_water_reader_` - (`include/pineforge/source/pine_adapter.hpp:2392-2393`), with their type + (`include/pineforge/source/pine_adapter.hpp:2410-2411`), with their type `ReceiptHighWaterReader`, are storage nothing reads or writes since PERF-L4: they keep `PineExecutionAdapter`'s layout, which is part of `PineStrategyHost`'s and so of the generated script's ABI. They go at the next diff --git a/docs/pages/market-admission.md b/docs/pages/market-admission.md index 1578ef3da..1901ae240 100644 --- a/docs/pages/market-admission.md +++ b/docs/pages/market-admission.md @@ -25,7 +25,7 @@ broker `OpeningReceipt` Check/Exempt decision, which belongs to a committed fill Three historical placement views are fixed when an order is placed and kept in the adapter's placement snapshot (`projection_after_close`, `projection_over_pyramiding`, `projection_opposite_market_predecessor` -pine_adapter.hpp:310-312): +pine_adapter.hpp:302-313): - The prior-close view compares the original accepted-close quantity with the existing quantity tolerance. This describes a source-time diff --git a/docs/pages/native-engine.md b/docs/pages/native-engine.md index b35bfcf2c..7ac4209b7 100644 --- a/docs/pages/native-engine.md +++ b/docs/pages/native-engine.md @@ -502,7 +502,7 @@ five commands through `strategy_native_submit_v1` / `_replace_v1` / kernel from C* below. `native_order::Request` values belong to `native_order_v7` -(`native_order.hpp:25`); identity types stay `native_order_v1`. Label/comment +(`native_order.hpp:27`); identity types stay `native_order_v1`. Label/comment remain inert text. The market default path still constructs from: - `order_action::Transact{signed_units}` — finite nonzero @@ -1496,7 +1496,7 @@ adapter answers `margin_check_allowed` with TradingView's scheduling — which includes the post-exit re-size: when a priced bracket leg of the script bar fills, the slice resting at that bar's adverse extreme was sized on the pre-exit book, and the legacy broker cancelled and re-scheduled it there -(`margin_check_allowed` `pine_adapter.cpp:17890-17923`), so the adapter admits +(`margin_check_allowed` `pine_adapter.cpp:17894-17927`), so the adapter admits the kernel's own point for that driver point while (and only while) a slice rests — `resolve_margin_requirement` with its ten-significant-digit money and @@ -2047,11 +2047,11 @@ The kernel never drops an event its own live state still reads. A queued applied notification (and the margin receipt it carries) pins the window at its execution, a live deferred group-adjustment chain pins it at its head, and the rest of what the journal used to answer is state now: a request's -replace-chain root is `RequestDefinition::root` (`native_order.hpp:736`) and +replace-chain root is `RequestDefinition::root` (`native_order.hpp:738`) and the order core's chain index, which is all a cohort command reads about a request that is no longer working; a trail's arm ordinal (`NativeTrailState::activation_ordinal`) is its tracking state's -(`TrailTrack` `native_order.hpp:650`); the FX-roll margin check reads the +(`TrailTrack` `native_order.hpp:652`); the FX-roll margin check reads the last two driver points' instants; and a group-effect receipt whose outcome event was dropped stands on its own record. A host that wants the whole run after it has ended asks for `Full` (or `Commands`); the Pine adapter declares @@ -3696,7 +3696,7 @@ The Pine adapter emits `Sized` for the default-quantity entries above and no Pine/generated hosts derive from `pineforge::source::PineStrategyHost`, which derives from `NativeStrategyHost`; handwritten native hosts also derive from `NativeStrategyHost`. The source adapter/scheduler hash domain is -`kSourceAdapterDomain` `pine_adapter.hpp:43` +`kSourceAdapterDomain` `pine_adapter.hpp:46` (`"pineforge-source-adapter/v4"`), while the public C ABI remains version 4. The ownership switch is complete: the compatibility loop and the source diff --git a/docs/pages/pine-to-native.md b/docs/pages/pine-to-native.md index 831a0fe99..1f7ed88ec 100644 --- a/docs/pages/pine-to-native.md +++ b/docs/pages/pine-to-native.md @@ -66,10 +66,10 @@ gain: clock, instrument, account, fees and caps. Nothing is inferred from the bars and nothing comes from a chart. - **Typed orders and typed outcomes.** A request is a value - (`Request` native_order.hpp:483); its fate is a typed event — `AcceptedEvent` - (native_order.hpp:1001), `ExecutionAppliedEvent` (native_order.hpp:1146), - `MatchRejectedEvent` (native_order.hpp:1108), `CancelledEvent` - (native_order.hpp:1043) — readable in order from `native_events` + (`Request` native_order.hpp:485); its fate is a typed event — `AcceptedEvent` + (native_order.hpp:1003), `ExecutionAppliedEvent` (native_order.hpp:1148), + `MatchRejectedEvent` (native_order.hpp:1110), `CancelledEvent` + (native_order.hpp:1045) — readable in order from `native_events` (native_host.hpp:1335). - **Any language.** The same kernel drives from C through a callback table (`pf_native_callbacks_v1` native_c_api.h:2499), so a host in Rust, Go, Python or Zig needs no C++ @@ -129,14 +129,14 @@ rather than defaulted. | `slippage` | `slippage_ticks` native_run_spec.hpp:652 | `slippage_ticks` pineforge.h:502 | `native_price_grid_strategy.cpp` | Raw `ticks * price_tick`, applied once by `native_matching::apply_slippage` native_execution_consumer.cpp:7203-7207. TradingView's round-then-slip order is adapter-only. | | `process_orders_on_close` | `NativeCloseExecution` native_run_spec.hpp:33 | `close_execution` pineforge.h:503 | `native_open_lots_strategy.cpp` | `AfterCalculation` emits the post-calculation close point native_execution_consumer.cpp:8649-8650, so a request submitted in bar *k*'s calculation fills at bar *k*'s own close. | | `pyramiding` | `max_open_lots` native_run_spec.hpp:664 | `max_open_lots` pineforge.h:505 | `native_open_lots_strategy.cpp` | Caps surviving+new physical lots (`resulting_lot_count` native_execution_consumer.cpp:3756). The Pine adapter keeps its own per-cycle *entry count*, ruled **adapter-policy** in ADR-0001's table row `max_open_lots`; measured against TradingView it is not TradingView's rule, and this cap is the closer one (14 of 15 tape scenarios against 13, the private test `test_pyramiding_count_differential`). Lowered onto the cap, four corpus probes move away from TradingView, for want of its first-eligible-point rule, so the adapter keeps its count. | -| `default_qty_type` | `Sized` native_order.hpp:164 with `CashValue` native_order.hpp:91 or `EquityFraction` native_order.hpp:97 | `PF_NATIVE_INTENT_SIZED` native_c_api.h:348 with `size_basis` native_c_api.h:2147 | `native_sized_report_strategy.cpp` | The size is a property of the *request*, not of the run: a native host may size one order by cash and the next by equity fraction. `Transact` native_order.hpp:41 is the fixed-units form. Pine itself has no per-call `qty_type` — TradingView's compiler refuses one — so a script's `strategy.entry(qty = ...)` is always units; the source layer's typed per-call entry, which only generated or C++ callers reach, converts its money exactly as the declared default does. | -| `default_qty_value` | the basis's own scalar: `CashValue::cash` native_order.hpp:92, `EquityFraction::fraction` native_order.hpp:98 | `intent_value` native_c_api.h:2151 | `native_sized_report_strategy.cpp` | `SizeTime` native_order.hpp:106 chooses *when* the basis resolves and `SizePrice` native_order.hpp:146 *which* price it converts at; both compose. | +| `default_qty_type` | `Sized` native_order.hpp:166 with `CashValue` native_order.hpp:93 or `EquityFraction` native_order.hpp:99 | `PF_NATIVE_INTENT_SIZED` native_c_api.h:348 with `size_basis` native_c_api.h:2147 | `native_sized_report_strategy.cpp` | The size is a property of the *request*, not of the run: a native host may size one order by cash and the next by equity fraction. `Transact` native_order.hpp:43 is the fixed-units form. Pine itself has no per-call `qty_type` — TradingView's compiler refuses one — so a script's `strategy.entry(qty = ...)` is always units; the source layer's typed per-call entry, which only generated or C++ callers reach, converts its money exactly as the declared default does. | +| `default_qty_value` | the basis's own scalar: `CashValue::cash` native_order.hpp:94, `EquityFraction::fraction` native_order.hpp:100 | `intent_value` native_c_api.h:2151 | `native_sized_report_strategy.cpp` | `SizeTime` native_order.hpp:108 chooses *when* the basis resolves and `SizePrice` native_order.hpp:148 *which* price it converts at; both compose. | | `calc_on_order_fills` | `NativeCalculationTrigger` native_run_spec.hpp:94 | `calculation` native_c_api.h:2321 | `native_calc_on_fills_strategy.cpp` | `BarCloseAndFills` recalculates once at the cursor of each applied execution, bounded by `max_recalculations_per_point` native_run_spec.hpp:695. The generic cadence only: TradingView's waypoint-only refill, its two-fills-at-open rule and its script-state rollback stay in the adapter. | | `calc_on_every_tick` | `NativeCalculationTrigger` native_run_spec.hpp:94 | `calculation` native_c_api.h:2321 | `native_calc_on_fills_strategy.cpp` | `EveryModeledPoint` additionally recalculates at every modeled point of the delivered path and at every observed print. In a stream, `on_native_tick` native_host.hpp:878 is one call per accepted realtime print whatever the trigger is. | | `margin_long` | `NativeMarginModel::initial_long` native_run_spec.hpp:241 | `margin_initial_long` native_c_api.h:2331 | `native_margin_strategy.cpp` | A fraction, not a percent. `0.0` is the maintenance-only spelling for that side: the kernel enforces no opening requirement there and the host owns opening admission. | | `margin_short` | `NativeMarginModel::initial_short` native_run_spec.hpp:242 | `margin_initial_short` native_c_api.h:2335 | `native_margin_strategy.cpp` | Per side, unlike the one-scalar `initial_margin_fraction` native_run_spec.hpp:666 the model replaces. `maintenance_long` native_run_spec.hpp:243 / `maintenance_short` native_run_spec.hpp:244 add the liquidation half Pine has no word for. | -| `close_entries_rule` | none — a native close names the lots it closes | — | `native_selected_strategy.cpp` | Pine picks FIFO or ANY for the whole script. A native reduce is explicit instead: `BindOpening` native_order.hpp:445 closes one opening's lots, `BindOpenings` native_order.hpp:454 a chosen set, a bare `Reduce` native_order.hpp:220 the book in FIFO order. The adapter keeps the run-level rule (`close_entries_rule` pine_adapter.hpp:88). | -| `backtest_fill_limits_assumption` | none — a generic limit fills when the level is reached | — | `tests/test_native_order_core.cpp` | TradingView's assumption is "the price must exceed the limit by *n* ticks". The generic knob beside it is `Limit::fill_through` native_order.hpp:239, which makes a limit market-if-touched; the *n*-tick assumption itself is a TradingView rule and stays in the adapter. | +| `close_entries_rule` | none — a native close names the lots it closes | — | `native_selected_strategy.cpp` | Pine picks FIFO or ANY for the whole script. A native reduce is explicit instead: `BindOpening` native_order.hpp:447 closes one opening's lots, `BindOpenings` native_order.hpp:456 a chosen set, a bare `Reduce` native_order.hpp:222 the book in FIFO order. The adapter keeps the run-level rule (`close_entries_rule` pine_adapter.hpp:91). | +| `backtest_fill_limits_assumption` | none — a generic limit fills when the level is reached | — | `tests/test_native_order_core.cpp` | TradingView's assumption is "the price must exceed the limit by *n* ticks". The generic knob beside it is `Limit::fill_through` native_order.hpp:241, which makes a limit market-if-touched; the *n*-tick assumption itself is a TradingView rule and stays in the adapter. | | `use_bar_magnifier` | `IntrabarPath` native_run_spec.hpp:378 | `intrabar_kind` native_c_api.h:2374 | `tests/test_native_calc_timing.cpp` | The host owns the lower bars, as `lower_tf` native_run_spec.hpp:394 or `synthesized` native_run_spec.hpp:408. The C magnifier arguments of `run_backtest_full` pineforge.h:610 configure a *compiled Pine strategy*, never a native host; its lower bars are TradingView's own intrabars, built from the finer feed at TradingView's table's timeframe (2 minutes on a 15-minute chart) and owned by the chart bar holding their last minute (`tradingview_magnifier_bars` magnifier_intrabars.hpp:46, @ref magnifier_intrabars). | | `fill_orders_on_standard_ohlc` | none — the kernel matches the bars you feed it | — | — | TradingView's switch exists because a Heikin-Ashi chart's candles are not the traded prices. A native host feeds the standard OHLC it wants matched, and derives any transformed series itself. | | `max_bars_back` | none — a native host owns its own history | — | `hello_kernel.cpp` | Pine infers a buffer length for every series. `pineforge::Series` series.hpp:94 is a fixed-capacity ring you size yourself, so nothing is inferred and nothing silently truncates. | @@ -258,17 +258,17 @@ source-only features state why a native counterpart does not exist. | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | -| `strategy.fixed` | `Transact` native_order.hpp:41 | `PF_NATIVE_INTENT_TRANSACT` native_c_api.h:339 | `hello_kernel.cpp` | Signed units, the default native sizing: `> 0` long, `< 0` short. | -| `strategy.cash` | `CashValue` native_order.hpp:91 | `PF_NATIVE_SIZE_BASIS_CASH` native_c_api.h:372 | `native_sized_report_strategy.cpp` / `native_fee_reserve_strategy.cpp` | `units = cash / (price * point_value * fx)`. `Sized::reserve_percent_fee` native_order.hpp:179 divides the cash by `(1 + fee)` under `NativeFeeKind::Percent`, which is the fee reserve Pine has no spelling for. The fee-reserve example sizes 46 shares with it where 47 fit without, and asserts the fee each leg pays. | -| `strategy.percent_of_equity` | `EquityFraction` native_order.hpp:97 | `PF_NATIVE_SIZE_BASIS_EQUITY_FRACTION` native_c_api.h:373 | `native_sized_report_strategy.cpp` | A fraction of the marked equity at the sizing point: `0.10` is ten percent, never `10`. | -| `strategy.long` | `Side::Long` native_order.hpp:65 | `PF_NATIVE_SIDE_LONG` native_c_api.h:366 | `native_selected_strategy.cpp` | For `Transact` it is the sign of `signed_units` order_action.hpp:10; `Sized::side` native_order.hpp:165 and `HostSized::side` native_order.hpp:75 name it explicitly. | -| `strategy.short` | `Side::Short` native_order.hpp:65 | `PF_NATIVE_SIDE_SHORT` native_c_api.h:367 | `native_selected_strategy.cpp` | Same. | -| `strategy.default_entry_qty()` | none — the kernel has no declaration-level default quantity | — | `native_sized_report_strategy.cpp` | Every native request carries its own size, so there is no run-level default to read back. The adapter keeps Pine's (`default_qty_value` pine_adapter.hpp:67), and `strategy_set_override` pineforge.h:652 overrides it for a compiled Pine strategy. | - -`HostSized` native_order.hpp:73 is the sixth intent and the escape hatch: the +| `strategy.fixed` | `Transact` native_order.hpp:43 | `PF_NATIVE_INTENT_TRANSACT` native_c_api.h:339 | `hello_kernel.cpp` | Signed units, the default native sizing: `> 0` long, `< 0` short. | +| `strategy.cash` | `CashValue` native_order.hpp:93 | `PF_NATIVE_SIZE_BASIS_CASH` native_c_api.h:372 | `native_sized_report_strategy.cpp` / `native_fee_reserve_strategy.cpp` | `units = cash / (price * point_value * fx)`. `Sized::reserve_percent_fee` native_order.hpp:181 divides the cash by `(1 + fee)` under `NativeFeeKind::Percent`, which is the fee reserve Pine has no spelling for. The fee-reserve example sizes 46 shares with it where 47 fit without, and asserts the fee each leg pays. | +| `strategy.percent_of_equity` | `EquityFraction` native_order.hpp:99 | `PF_NATIVE_SIZE_BASIS_EQUITY_FRACTION` native_c_api.h:373 | `native_sized_report_strategy.cpp` | A fraction of the marked equity at the sizing point: `0.10` is ten percent, never `10`. | +| `strategy.long` | `Side::Long` native_order.hpp:67 | `PF_NATIVE_SIDE_LONG` native_c_api.h:366 | `native_selected_strategy.cpp` | For `Transact` it is the sign of `signed_units` order_action.hpp:10; `Sized::side` native_order.hpp:167 and `HostSized::side` native_order.hpp:77 name it explicitly. | +| `strategy.short` | `Side::Short` native_order.hpp:67 | `PF_NATIVE_SIDE_SHORT` native_c_api.h:367 | `native_selected_strategy.cpp` | Same. | +| `strategy.default_entry_qty()` | none — the kernel has no declaration-level default quantity | — | `native_sized_report_strategy.cpp` | Every native request carries its own size, so there is no run-level default to read back. The adapter keeps Pine's (`default_qty_value` pine_adapter.hpp:70), and `strategy_set_override` pineforge.h:652 overrides it for a compiled Pine strategy. | + +`HostSized` native_order.hpp:75 is the sixth intent and the escape hatch: the host answers the units itself from `resolve_execution_terms` (native_host.hpp:947), which sees the kernel-resolved quantity as the facts' -`RemainingUnits` native_order.hpp:550 and may return its own. A C host answers +`RemainingUnits` native_order.hpp:552 and may return its own. A C host answers it only for the two host-sized closes the C surface admits, a cohort close and a Book-scoped `WAIT_FOR_APPLIED` child, through `on_close_units` native_c_api.h:2591; a host-sized opening, and replacing the kernel's own `Sized` @@ -285,16 +285,16 @@ is `strategy_native_submit_v1` native_c_api.h:2742 over a | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | -| `strategy.entry()` | `Request` native_order.hpp:483 carrying `Transact` native_order.hpp:41 | `strategy_native_submit_v1` native_c_api.h:2742 | `native_market_strategy.cpp` | Pine's entry *reverses* an opposite position; `Transact` does not. Spell the reversal explicitly with the row below. | -| `strategy.order()` | the same `Request` native_order.hpp:483 | `strategy_native_submit_v1` native_c_api.h:2742 | `native_market_strategy.cpp` | `strategy.order` is the non-reversing entry, so it maps 1:1 with no caveat. | +| `strategy.entry()` | `Request` native_order.hpp:485 carrying `Transact` native_order.hpp:43 | `strategy_native_submit_v1` native_c_api.h:2742 | `native_market_strategy.cpp` | Pine's entry *reverses* an opposite position; `Transact` does not. Spell the reversal explicitly with the row below. | +| `strategy.order()` | the same `Request` native_order.hpp:485 | `strategy_native_submit_v1` native_c_api.h:2742 | `native_market_strategy.cpp` | `strategy.order` is the non-reversing entry, so it maps 1:1 with no caveat. | | `strategy.exit()` | `submit_bracket` native_toolkit.hpp:160 over a `BracketSpec` native_toolkit.hpp:37 | `strategy_native_submit_v1` native_c_api.h:2742 per leg | `native_bracket_strategy.cpp` | The full bracket is described below in [Brackets](@ref pine_to_native_brackets). | -| `strategy.close()` | `Reduce` native_order.hpp:220 with `OwnerOpenedUnits` native_order.hpp:191 and `BindOpening` native_order.hpp:445 | `PF_NATIVE_REDUCE_OWNER_OPENED` native_c_api.h:354 with `PF_NATIVE_OWNER_BIND_OPENING` native_c_api.h:459 | `native_selected_strategy.cpp` | Closes the lots one opening produced. The cycle comes from `ExecutionAppliedEvent::cycle_after` native_order.hpp:1165. A fractional close is `ScopeFraction` native_order.hpp:207 — `qty_percent = 50` is `fraction = 0.5`, with `ScopeClaim` native_order.hpp:195 and `ScopeBasis` native_order.hpp:202 saying whether siblings are netted and when the scope is measured. On a `quantity_grid` a fraction below 1 is floored onto the grid, and `qty_percent = 100` (`fraction = 1`) of the gross scope, settled in one fill, closes the whole scope unfloored. | -| `strategy.close_all()` | `Flatten` native_order.hpp:36 | `PF_NATIVE_INTENT_FLATTEN` native_c_api.h:337 | `hello_kernel.cpp` / `hello_kernel_c.c` | The whole book, in one settlement cycle. | -| `strategy.cancel()` | `cancel_where` native_host.hpp:1278 with `NativeRequestField::Label` native_host.hpp:691 | `strategy_native_cancel_where_v1` native_c_api.h:2829 | `native_trail_risk_strategy.cpp` | Withdraws exactly the live requests whose `Request::label` native_order.hpp:485 is that id, and answers how many. `cancel` native_host.hpp:1249 is the single-request form for a host that kept the handle. | -| `strategy.cancel_all()` | `cancel_all` native_host.hpp:1267 | `strategy_native_cancel_all_v1` native_c_api.h:2808 | `native_trail_risk_strategy.cpp` | Neither field is indexed: every form walks the live book once. A `PendingUntilArmed` native_order.hpp:394 child is absent from `native_working_requests` native_host.hpp:1263 before its arm, and all three cancels still address it. | -| `strategy.oca.cancel` | `GroupEffect::Cancel` native_order.hpp:466 | `PF_NATIVE_GROUP_CANCEL` native_c_api.h:478 | `native_bracket_strategy.cpp` | Membership is `Member` native_order.hpp:468 inside `Group` native_order.hpp:473. | -| `strategy.oca.reduce` | `GroupEffect::Reduce` native_order.hpp:466 | `PF_NATIVE_GROUP_REDUCE` native_c_api.h:479 | `tests/test_native_resting_group_contract.cpp` | The sibling's quantity is reduced instead of cancelled. | -| `strategy.oca.none` | `NoGroup` native_order.hpp:467 | `PF_NATIVE_GROUP_NONE` native_c_api.h:472 | `hello_kernel.cpp` | The default: no group, no sibling effect. | +| `strategy.close()` | `Reduce` native_order.hpp:222 with `OwnerOpenedUnits` native_order.hpp:193 and `BindOpening` native_order.hpp:447 | `PF_NATIVE_REDUCE_OWNER_OPENED` native_c_api.h:354 with `PF_NATIVE_OWNER_BIND_OPENING` native_c_api.h:459 | `native_selected_strategy.cpp` | Closes the lots one opening produced. The cycle comes from `ExecutionAppliedEvent::cycle_after` native_order.hpp:1167. A fractional close is `ScopeFraction` native_order.hpp:209 — `qty_percent = 50` is `fraction = 0.5`, with `ScopeClaim` native_order.hpp:197 and `ScopeBasis` native_order.hpp:204 saying whether siblings are netted and when the scope is measured. On a `quantity_grid` a fraction below 1 is floored onto the grid, and `qty_percent = 100` (`fraction = 1`) of the gross scope, settled in one fill, closes the whole scope unfloored. | +| `strategy.close_all()` | `Flatten` native_order.hpp:38 | `PF_NATIVE_INTENT_FLATTEN` native_c_api.h:337 | `hello_kernel.cpp` / `hello_kernel_c.c` | The whole book, in one settlement cycle. | +| `strategy.cancel()` | `cancel_where` native_host.hpp:1278 with `NativeRequestField::Label` native_host.hpp:691 | `strategy_native_cancel_where_v1` native_c_api.h:2829 | `native_trail_risk_strategy.cpp` | Withdraws exactly the live requests whose `Request::label` native_order.hpp:487 is that id, and answers how many. `cancel` native_host.hpp:1249 is the single-request form for a host that kept the handle. | +| `strategy.cancel_all()` | `cancel_all` native_host.hpp:1267 | `strategy_native_cancel_all_v1` native_c_api.h:2808 | `native_trail_risk_strategy.cpp` | Neither field is indexed: every form walks the live book once. A `PendingUntilArmed` native_order.hpp:396 child is absent from `native_working_requests` native_host.hpp:1263 before its arm, and all three cancels still address it. | +| `strategy.oca.cancel` | `GroupEffect::Cancel` native_order.hpp:468 | `PF_NATIVE_GROUP_CANCEL` native_c_api.h:478 | `native_bracket_strategy.cpp` | Membership is `Member` native_order.hpp:470 inside `Group` native_order.hpp:475. | +| `strategy.oca.reduce` | `GroupEffect::Reduce` native_order.hpp:468 | `PF_NATIVE_GROUP_REDUCE` native_c_api.h:479 | `tests/test_native_resting_group_contract.cpp` | The sibling's quantity is reduced instead of cancelled. | +| `strategy.oca.none` | `NoGroup` native_order.hpp:469 | `PF_NATIVE_GROUP_NONE` native_c_api.h:472 | `hello_kernel.cpp` | The default: no group, no sibling effect. | | `strategy.direction.all` | `NativeOpenDirections::Both` native_run_spec.hpp:76 | `allowed_open_directions` pineforge.h:503 | `native_risk_limits_strategy.cpp` | The default. | | `strategy.direction.long` | `NativeOpenDirections::Long` native_run_spec.hpp:74 | `allowed_open_directions` pineforge.h:503 | `tests/test_native_run_spec.cpp` | An opening the direction forbids is refused with `OpeningDirection` native_execution_consumer.cpp:3745. | | `strategy.direction.short` | `NativeOpenDirections::Short` native_run_spec.hpp:75 | `allowed_open_directions` pineforge.h:503 | `tests/test_native_run_spec.cpp` | Same, the other way. | @@ -303,29 +303,29 @@ is `strategy_native_submit_v1` native_c_api.h:2742 over a | `strategy.commission.cash_per_order` | `NativeFeeKind::CashPerExecution` native_run_spec.hpp:22 | `fee_kind` pineforge.h:502 | `native_open_lots_strategy.cpp` | One ticket per execution, split by units over every slice that execution touches — the closed rows and, for a reversal, the lot it opens. | Two more order concepts Pine has no word for, both generic: -`ReverseTo` native_order.hpp:60 is one exact signed target reached in one +`ReverseTo` native_order.hpp:62 is one exact signed target reached in one ticket and one settlement cycle — the explicit form of Pine's implicit -reversal; and `PointBudget` native_order.hpp:379 caps how much of a request may +reversal; and `PointBudget` native_order.hpp:381 caps how much of a request may fill at a single matching point, where Pine only ever uses the whole remaining quantity. Amending a live order is `replace` native_host.hpp:1220, which emits -`ReplacedEvent` native_order.hpp:1017 and can carry the predecessor's live -trigger state forward through `ReplaceOptions` native_order.hpp:367 -- or keep +`ReplacedEvent` native_order.hpp:1019 and can carry the predecessor's live +trigger state forward through `ReplaceOptions` native_order.hpp:369 -- or keep its handle, or carry a close's book binding, from C++ only. Its C spelling is `strategy_native_replace_v1`, or `strategy_native_replace_ext_v1` -when a rejection's `RequestRejectReason` native_order.hpp:821 matters. +when a rejection's `RequestRejectReason` native_order.hpp:823 matters. ### Triggers {#pine_to_native_triggers} | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | -| `limit=` | `Limit` native_order.hpp:237 | `PF_NATIVE_TRIGGER_LIMIT` native_c_api.h:391 | `native_calc_on_fills_strategy.cpp` | `fill_through` native_order.hpp:239 makes it market-if-touched: the level gates when the request becomes executable, and the fill is no longer bounded by it. | -| `stop=` | `Stop` native_order.hpp:246 | `PF_NATIVE_TRIGGER_STOP` native_c_api.h:392 | `native_price_grid_strategy.cpp` | Stop-limit is `StopLimit` native_order.hpp:252. | -| `trail_points` | `FromOwnerFill{+ticks, true}` native_order.hpp:331 | `anchor = PF_NATIVE_ANCHOR_FROM_OWNER_FILL`, `anchor_offset = ticks`, `anchor_offset_in_ticks = 1` native_c_api.h:2131-2138 | `native_bracket_strategy.cpp` | Pine points are the arm distance from the entry, in ticks. The adapter lowers that anchor distance separately from the ride offset; it is not `TrailTicks`. | -| `trail_offset` | `TrailTicks` native_order.hpp:293 (resolved into `Trail::offset` native_order.hpp:291) | `p1 = ticks`, `trail_offset_in_ticks = 1` native_c_api.h:2137-2141 | `native_trail_risk_strategy.cpp` | Pine offset is the ride distance behind the running best, also expressed in ticks. Zero is legal and means "ride the best"; the first adverse move past it exits. | -| `trail_price` | `Trail::arm_price` native_order.hpp:292 | `trail_has_arm_price` native_c_api.h:2161 | `native_trail_risk_strategy.cpp` | The absolute level the trail arms at. Read the live projection with `trail_state` native_host.hpp:1092 (C: `strategy_native_trail_state_v1` native_c_api.h:3054). The kernel compares `arm_price` with the **raw** path. TradingView tests the activation on the **tick-quantized** bar, with or without an offset and the placement close included (R5 lanes E5 and E9's tapes), so the Pine adapter arms at the activation's half-tick threshold — a placement close whose tick reaches the activation arms there — and books from a running best that starts at the activation; a one-shot (`trail_offset` 0) books the tick its resting threshold stands for, rounded away from the position. Since R5 lane E14 that running best is the **kernel's** own: the trailing leg names the activation — or the favourable one of the activation and the placement print when that print armed it — in the generic `Trail::best_seed` native_order.hpp:294, so the ride no longer starts half a tick short of the activation. | +| `limit=` | `Limit` native_order.hpp:239 | `PF_NATIVE_TRIGGER_LIMIT` native_c_api.h:391 | `native_calc_on_fills_strategy.cpp` | `fill_through` native_order.hpp:241 makes it market-if-touched: the level gates when the request becomes executable, and the fill is no longer bounded by it. | +| `stop=` | `Stop` native_order.hpp:248 | `PF_NATIVE_TRIGGER_STOP` native_c_api.h:392 | `native_price_grid_strategy.cpp` | Stop-limit is `StopLimit` native_order.hpp:254. | +| `trail_points` | `FromOwnerFill{+ticks, true}` native_order.hpp:333 | `anchor = PF_NATIVE_ANCHOR_FROM_OWNER_FILL`, `anchor_offset = ticks`, `anchor_offset_in_ticks = 1` native_c_api.h:2131-2138 | `native_bracket_strategy.cpp` | Pine points are the arm distance from the entry, in ticks. The adapter lowers that anchor distance separately from the ride offset; it is not `TrailTicks`. | +| `trail_offset` | `TrailTicks` native_order.hpp:295 (resolved into `Trail::offset` native_order.hpp:293) | `p1 = ticks`, `trail_offset_in_ticks = 1` native_c_api.h:2137-2141 | `native_trail_risk_strategy.cpp` | Pine offset is the ride distance behind the running best, also expressed in ticks. Zero is legal and means "ride the best"; the first adverse move past it exits. | +| `trail_price` | `Trail::arm_price` native_order.hpp:294 | `trail_has_arm_price` native_c_api.h:2161 | `native_trail_risk_strategy.cpp` | The absolute level the trail arms at. Read the live projection with `trail_state` native_host.hpp:1092 (C: `strategy_native_trail_state_v1` native_c_api.h:3054). The kernel compares `arm_price` with the **raw** path. TradingView tests the activation on the **tick-quantized** bar, with or without an offset and the placement close included (R5 lanes E5 and E9's tapes), so the Pine adapter arms at the activation's half-tick threshold — a placement close whose tick reaches the activation arms there — and books from a running best that starts at the activation; a one-shot (`trail_offset` 0) books the tick its resting threshold stands for, rounded away from the position. Since R5 lane E14 that running best is the **kernel's** own: the trailing leg names the activation — or the favourable one of the activation and the placement print when that print armed it — in the generic `Trail::best_seed` native_order.hpp:296, so the ride no longer starts half a tick short of the activation. | `limit=` and `stop=` are listed once here rather than once per command, -because a native trigger is a property of the `Request` native_order.hpp:483 +because a native trigger is a property of the `Request` native_order.hpp:485 and applies to every intent. ### Brackets {#pine_to_native_brackets} @@ -335,30 +335,30 @@ Pine call with no single native counterpart, because it is four decisions at once. Natively they are separate and each is nameable: - **Placed before the entry has a price.** Each leg is a child with - `WaitForApplied` native_order.hpp:436 naming the parent, and a - `FromOwnerFill` native_order.hpp:331 anchor instead of an absolute level. The + `WaitForApplied` native_order.hpp:438 naming the parent, and a + `FromOwnerFill` native_order.hpp:333 anchor instead of an absolute level. The trigger's own level field is then left **unwritten**: the `0.0` a `Limit::price` / `Stop::price` already defaults to, and an absent `Trail::arm_price` — or that same `0.0` — for a trail. Writing any other level is refused, because the arm would overwrite it. At the parent's fill the kernel materializes `fill + offset` — in ticks when - `FromOwnerFill::ticks` native_order.hpp:333 is set — snaps it per - `NativeAnchorRounding` native_order.hpp:309, offers it once to + `FromOwnerFill::ticks` native_order.hpp:335 is set — snaps it per + `NativeAnchorRounding` native_order.hpp:311, offers it once to `resolve_anchored_level` native_host.hpp:1012, and carries the installed level - in `ArmedEvent` native_order.hpp:1246. + in `ArmedEvent` native_order.hpp:1248. - **Invisible until armed.** `NativeArmVisibility::PendingUntilArmed` - native_order.hpp:394 keeps the legs out of `native_working_requests` + native_order.hpp:396 keeps the legs out of `native_working_requests` native_host.hpp:1263 until the fill, like Pine's pending exit and like a broker that shows no working child before the parent trades. -- **First match.** `NativeArmFirstMatch::AfterArmPrint` native_order.hpp:412 +- **First match.** `NativeArmFirstMatch::AfterArmPrint` native_order.hpp:414 gives the leg the birth rule of a request submitted from the fill callback: a level the fill print already satisfies does not match on that print. -- **What it closes.** `NativeArmScope::Book` native_order.hpp:427 binds the leg +- **What it closes.** `NativeArmScope::Book` native_order.hpp:429 binds the leg to the whole position the fill left, later same-id adds included, instead of only the lot its owner opened; `qty_percent=100` is - `Reduce{OwnerOpenedUnits{}}` native_order.hpp:220, bound at the arm. -- **One-cancels-all.** `Member` native_order.hpp:468 with `GroupEffect::Cancel` - native_order.hpp:466 makes the take-profit and the stop-loss exclusive. + `Reduce{OwnerOpenedUnits{}}` native_order.hpp:222, bound at the arm. +- **One-cancels-all.** `Member` native_order.hpp:470 with `GroupEffect::Cancel` + native_order.hpp:468 makes the take-profit and the stop-loss exclusive. `submit_bracket` native_toolkit.hpp:160 places the three legs of a `BracketSpec` native_toolkit.hpp:37 in one call and hands back a @@ -442,7 +442,7 @@ liquidation level, and the ticket the forced close is booked under. | a step of the account FX curve, checked though no price moved | `NativeMarginCheckKind::FxRoll` native_host.hpp:553 | `PF_NATIVE_MARGIN_CHECK_FX_ROLL` native_c_api.h:935 | `native_fx_roll_strategy.cpp` | | a magnified bar's later samples, each checked where the path is (TradingView's bar magnifier) | `NativeMarginCheckKind::IntrabarSample` native_host.hpp:554 | `PF_NATIVE_MARGIN_CHECK_INTRABAR_SAMPLE` native_c_api.h:943 | `tests/test_native_margin_intrabar_samples.cpp` | | the host's own liquidation size | `resolve_margin_call_units` native_host.hpp:995 | `on_margin_call_units` native_c_api.h:2569 | `native_margin_strategy.cpp` | -| the liquidation event | `MarginCallEvent` native_order.hpp:1262 delivered to `on_native_margin_call` native_host.hpp:1001 | `on_margin_call` native_c_api.h:2517 | `native_margin_strategy.cpp` | +| the liquidation event | `MarginCallEvent` native_order.hpp:1264 delivered to `on_native_margin_call` native_host.hpp:1001 | `on_margin_call` native_c_api.h:2517 | `native_margin_strategy.cpp` | The one-scalar `initial_margin_fraction` native_run_spec.hpp:666 predates the model and remains the simple spelling; the two are mutually exclusive. It is @@ -583,13 +583,13 @@ version of this whole table. Each row also carries the identity Pine has no word for: `entry_incarnation` native_host.hpp:330 is the never-reused request record whose fill opened the lot, and `cycle` native_host.hpp:331 the position cycle a later `BindOpening` -native_order.hpp:445 names. +native_order.hpp:447 names. | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | | `strategy.opentrades` | `native_open_lots` native_host.hpp:1297 | `strategy_native_open_lot_count_v1` native_c_api.h:2888 | `native_open_lots_strategy.cpp` | The row count, which equals `lot_count` native_host.hpp:299. | -| `strategy.opentrades.entry_id()` | `entry_label` native_host.hpp:333 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | The opening request's `label` native_order.hpp:485. | -| `strategy.opentrades.entry_comment()` | `entry_comment` native_host.hpp:334 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | The opening request's `comment` native_order.hpp:486. | +| `strategy.opentrades.entry_id()` | `entry_label` native_host.hpp:333 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | The opening request's `label` native_order.hpp:487. | +| `strategy.opentrades.entry_comment()` | `entry_comment` native_host.hpp:334 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | The opening request's `comment` native_order.hpp:488. | | `strategy.opentrades.entry_bar_index()` | `entry_bar_index` native_host.hpp:336 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | Script-bar index of the opening fill. | | `strategy.opentrades.entry_time()` | `entry_time_ms` native_host.hpp:335 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | Effective time of the opening fill, Unix ms. | | `strategy.opentrades.entry_price()` | `entry_price` native_host.hpp:337 | `pf_native_open_lot_v1` native_c_api.h:1767 | `native_open_lots_strategy.cpp` | The booked price: slippage and the price grid are already applied. | @@ -895,9 +895,9 @@ const auto entry = submit({sized, "long", ""}); // reject reason beside it names. ``` -`reserve_percent_fee` native_order.hpp:179 is the thing Pine has no spelling +`reserve_percent_fee` native_order.hpp:181 is the thing Pine has no spelling for: without it, a cash-sized entry that spends the whole 2500 cannot also pay -its commission. `SizePrice::SignalOnTick` native_order.hpp:146 sizes against +its commission. `SizePrice::SignalOnTick` native_order.hpp:148 sizes against the decision price on the instrument's own ladder, which is what a trader means by "2500 at the current price". `native_sized_report_strategy.cpp` is the runnable version of cash sizing; the fee reserve runs in this section's own @@ -935,11 +935,11 @@ submit(take_profit); submit(stop_loss); submit(trail); Three legs placed before the entry has a price. The trigger levels are left unwritten on purpose — `0.0` for the limit and the stop, an absent `arm_price` -for the trail: `FromOwnerFill` native_order.hpp:331 supplies them at the arm as +for the trail: `FromOwnerFill` native_order.hpp:333 supplies them at the arm as `fill ± ticks * price_tick`, snapped per `NativeAnchorRounding::Directional` -native_order.hpp:312. Pine's `trail_points = 30` is the *arm* distance (the +native_order.hpp:314. Pine's `trail_points = 30` is the *arm* distance (the anchor) and `trail_offset = 8` the *ride* distance (`TrailTicks` -native_order.hpp:260) — two different numbers that Pine spells in one call. +native_order.hpp:262) — two different numbers that Pine spells in one call. `native_bracket_strategy.cpp` and `native_trail_risk_strategy.cpp` are the runnable halves. @@ -987,9 +987,9 @@ closes the stop at `101 - 20 × 0.25 = 96`. `native_events` native_host.hpp:1335. 3. **`strategy.entry` reverses, `Transact` does not.** If the Pine strategy relies on an entry flipping a short into a long, spell it `ReverseTo` - native_order.hpp:60. + native_order.hpp:62. 4. **Most percents become fractions — two stay out of 100.** `margin_long = 20` - is `0.20` and `EquityFraction{0.10}` native_order.hpp:97 is ten percent. + is `0.20` and `EquityFraction{0.10}` native_order.hpp:99 is ten percent. The two exceptions keep Pine's own unit: `fee_value` native_run_spec.hpp:658 under `NativeFeeKind::Percent`, so `commission_value = 0.1` is `fee_value = 0.1`, and a `NativeLossLimit` whose `percent` diff --git a/include/pineforge/detail/adapter_exposure_storage.hpp b/include/pineforge/detail/adapter_exposure_storage.hpp new file mode 100644 index 000000000..4236bf02e --- /dev/null +++ b/include/pineforge/detail/adapter_exposure_storage.hpp @@ -0,0 +1,23 @@ +#pragma once + +namespace pineforge::detail { + +// Exposure values/handles have already been erased by their existing owner. +// Empty containers still retain buckets/capacity while a cohort's origin +// history pins its resident row. Release that storage without retiring any +// origin, changing a cohort record, or touching the hash's logical inputs. +template +inline void release_empty_exposure_capacity(Container& container) { + if (container.empty()) { + Container empty; + container.swap(empty); + } +} + +template +inline void clear_exposure_storage(Container& container) { + Container empty; + container.swap(empty); +} + +} // namespace pineforge::detail diff --git a/include/pineforge/detail/disk_id_table.hpp b/include/pineforge/detail/disk_id_table.hpp new file mode 100644 index 000000000..75c85c257 --- /dev/null +++ b/include/pineforge/detail/disk_id_table.hpp @@ -0,0 +1,233 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::detail { + +// An AVL tree of byte-string keys and fixed-size values in an anonymous file. +// A node is 56 + sizeof(Value) bytes followed by the key bytes. Parent links +// allow rotations and sorted walks with O(1) RAM, including adversarial ids. +// The independent next link preserves first insertion order. Updating a value +// overwrites its slot: repeated ids cost no additional disk or resident heap. +// File offsets/tree links are storage metadata; callers fold logical values. +template +class DiskIdTable { + static_assert(std::is_trivially_copyable_v, "disk id values are fixed records"); + struct Links { + std::uint64_t left = 0, right = 0, parent = 0, next = 0; + std::uint64_t key_size = 0, height = 1, reserved = 0; + }; + static_assert(sizeof(Links) == 56, "scratch id node layout"); + +public: + using Offset = std::uint64_t; + DiskIdTable() = default; + DiskIdTable(const DiskIdTable&) = delete; + DiskIdTable& operator=(const DiskIdTable&) = delete; + DiskIdTable(DiskIdTable&& other) noexcept { *this = std::move(other); } + DiskIdTable& operator=(DiskIdTable&& other) noexcept { + if (this != &other) { + file_ = std::move(other.file_); + root_ = std::exchange(other.root_, 0); + first_ = std::exchange(other.first_, 0); + last_ = std::exchange(other.last_, 0); + end_ = std::exchange(other.end_, 1); + size_ = std::exchange(other.size_, 0); + } + return *this; + } + void set_directory(const std::string& directory) { file_.set_directory(directory); } + void clear() noexcept { + file_.clear(); + root_ = first_ = last_ = size_ = 0; + end_ = 1; // offset zero is the null link + } + std::uint64_t size() const noexcept { return size_; } + std::uint64_t file_bytes() const noexcept { return size_ == 0 ? 0 : end_; } + Offset first_inserted() const noexcept { return first_; } + Offset next_inserted(Offset at) const { return links(at).next; } + std::string key(Offset at) const { return read_key(at, links(at)); } + Offset first_sorted() const { return leftmost(root_); } + Offset next_sorted(Offset at) const { + auto row = links(at); + if (row.right != 0) return leftmost(row.right); + Offset child = at; + at = row.parent; + while (at != 0) { + row = links(at); + if (row.left == child) break; + child = at; + at = row.parent; + } + return at; + } + + std::optional find(const std::string& key) const { + Offset at = root_; + while (at != 0) { + const auto row = links(at); + const int order = key.compare(read_key(at, row)); + if (order == 0) return at; + at = order < 0 ? row.left : row.right; + } + return std::nullopt; + } + Value get(Offset at) const { + validate(at); + Value value{}; + file_.read(at + sizeof(Links), &value, sizeof value); + return value; + } + void set(Offset at, const Value& value) { + validate(at); + file_.write(at + sizeof(Links), &value, sizeof value); + } + std::pair insert(const std::string& key, const Value& value) { + Offset parent = 0; + Offset at = root_; + bool left = false; + while (at != 0) { + const auto row = links(at); + const int order = key.compare(read_key(at, row)); + if (order == 0) return {at, false}; + parent = at; + left = order < 0; + at = left ? row.left : row.right; + } + constexpr std::size_t header = sizeof(Links) + sizeof(Value); + ScratchFile::check_extent(end_, header); + ScratchFile::check_extent(end_ + header, key.size()); + if (size_ == std::numeric_limits::max()) + throw ScratchIoError(EFBIG, "scratch id count exhausted"); + file_.open("pineforge-ids"); + at = end_; + Links row; + row.parent = parent; + row.key_size = key.size(); + file_.write(at, &row, sizeof row); + file_.write(at + sizeof(Links), &value, sizeof value); + file_.write(at + header, key.data(), key.size()); + // All data bytes exist before linking the row into the logical table. + // Any partial link/rotation I/O error fails the run; it is never + // converted to a successful absent lookup or retried as another id. + end_ += header + key.size(); + if (parent != 0) { + auto old = links(parent); + (left ? old.left : old.right) = at; + put_links(parent, old); + } else root_ = at; + if (last_ != 0) { + auto previous = links(last_); + previous.next = at; + put_links(last_, previous); + } else first_ = at; + last_ = at; + ++size_; + rebalance(parent); + return {at, true}; + } + template void for_each_sorted(F&& visit) const { + for (Offset at = first_sorted(); at != 0; at = next_sorted(at)) + visit(key(at), get(at)); + } + template void for_each_inserted(F&& visit) const { + for (Offset at = first_; at != 0;) { + const auto row = links(at); + visit(read_key(at, row), get(at)); + at = row.next; + } + } + +private: + Offset leftmost(Offset at) const { + while (at != 0) { + const auto row = links(at); + if (row.left == 0) break; + at = row.left; + } + return at; + } + void validate(Offset at) const { + if (at == 0 || at >= end_ || end_ - at < sizeof(Links) + sizeof(Value)) + throw ScratchIoError(EIO, "scratch id invalid offset"); + } + Links links(Offset at) const { + validate(at); + Links row{}; + file_.read(at, &row, sizeof row); + if (row.height == 0 || row.height > 128 + || row.key_size > end_ - at - sizeof(Links) - sizeof(Value)) + throw ScratchIoError(EIO, "scratch id invalid node"); + return row; + } + void put_links(Offset at, const Links& row) { file_.write(at, &row, sizeof row); } + std::string read_key(Offset at, const Links& row) const { + if (row.key_size > std::string{}.max_size()) + throw ScratchIoError(EFBIG, "scratch id key too long"); + std::string key(static_cast(row.key_size), '\0'); + file_.read(at + sizeof(Links) + sizeof(Value), key.data(), key.size()); + return key; + } + std::uint64_t height(Offset at) const { return at == 0 ? 0 : links(at).height; } + void refresh(Links& row) const { row.height = 1 + std::max(height(row.left), height(row.right)); } + int balance(const Links& row) const { + return static_cast(height(row.left)) - static_cast(height(row.right)); + } + Offset rotate(Offset at, bool left) { + auto old = links(at); + const Offset top_at = left ? old.right : old.left; + auto top = links(top_at); + const Offset middle_at = left ? top.left : top.right; + const Offset parent_at = old.parent; + if (parent_at != 0) { + auto parent = links(parent_at); + (parent.left == at ? parent.left : parent.right) = top_at; + put_links(parent_at, parent); + } else root_ = top_at; + top.parent = parent_at; + old.parent = top_at; + (left ? old.right : old.left) = middle_at; + if (middle_at != 0) { + auto middle = links(middle_at); + middle.parent = at; + put_links(middle_at, middle); + } + refresh(old); + put_links(at, old); + (left ? top.left : top.right) = at; + refresh(top); + put_links(top_at, top); + return top_at; + } + void rebalance(Offset at) { + while (at != 0) { + auto row = links(at); + refresh(row); + put_links(at, row); + const int lean = balance(row); + if (lean > 1) { + if (balance(links(row.left)) < 0) (void)rotate(row.left, true); + at = rotate(at, false); + } else if (lean < -1) { + if (balance(links(row.right)) > 0) (void)rotate(row.right, false); + at = rotate(at, true); + } + at = links(at).parent; + } + } + + ScratchFile file_; + Offset root_ = 0, first_ = 0, last_ = 0, end_ = 1; + std::uint64_t size_ = 0; +}; +} // namespace pineforge::detail diff --git a/include/pineforge/detail/disk_resident_id_table.hpp b/include/pineforge/detail/disk_resident_id_table.hpp new file mode 100644 index 000000000..de70bd709 --- /dev/null +++ b/include/pineforge/detail/disk_resident_id_table.hpp @@ -0,0 +1,210 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::detail { + +// An id index plus the rows that still own live data. Value supplies a fixed +// Record, retired_record(), restore_record(Record), and spillable(). Retired +// scalar history lives on disk; live rosters stay resident. Iterators pin a +// row across nested lookups. No destructor does I/O: eviction writes before +// erasing, at the next lookup or explicit spill(), where failures can throw. +// +// begin()/end() visit RESIDENT rows, newest first. A spilled row +// has no live members, so source decisions over live members can omit it. +// all_sorted() visits EVERY logical row, including retired ones, for hashing. +// size() is the logical count, resident_size() the physical count. +template +class DiskResidentIdTable { + using Index = DiskIdTable; + using Offset = typename Index::Offset; + using Row = std::pair; + +public: + template + class Iterator { + friend class DiskResidentIdTable; + using Owner = std::conditional_t; + Owner* owner_ = nullptr; + Offset at_ = 0; + std::shared_ptr row_; + + Iterator(Owner* owner, Offset at, std::shared_ptr row) noexcept + : owner_(owner), at_(at), row_(std::move(row)) {} + public: + using iterator_category = std::forward_iterator_tag; + using value_type = Row; + using difference_type = std::ptrdiff_t; + using reference = std::conditional_t; + using pointer = std::conditional_t; + Iterator() = default; + reference operator*() const noexcept { return *row_; } + pointer operator->() const noexcept { return row_.get(); } + Iterator& operator++() { + if constexpr (Sorted) { + const auto next = owner_->index_.next_sorted(at_); + row_.reset(); + owner_->spill(); + at_ = next; + if (at_ != 0) row_ = owner_->load(at_); + } else if constexpr (Insertion) { + // Nested lookups can erase other empty rows. Re-seek by + // offset rather than keeping an invalidatable map iterator. + const auto next = owner_->resident_.upper_bound(at_); + at_ = next == owner_->resident_.end() ? 0 : next->first; + row_ = at_ == 0 ? nullptr : next->second; + } else { + auto next = owner_->resident_.lower_bound(at_); + if (next == owner_->resident_.begin()) { + at_ = 0; + row_.reset(); + } else { + --next; + at_ = next->first; + row_ = next->second; + } + } + return *this; + } + Iterator operator++(int) { auto old = *this; ++*this; return old; } + bool operator==(const Iterator& other) const noexcept { + return owner_ == other.owner_ && at_ == other.at_; + } + bool operator!=(const Iterator& other) const noexcept { return !(*this == other); } + }; + using iterator = Iterator; + using const_iterator = Iterator; + using sorted_iterator = Iterator; + using const_sorted_iterator = Iterator; + using insertion_iterator = Iterator; + + template struct Range { + Iter first, last; + Iter begin() const { return first; } + Iter end() const { return last; } + }; + template class KeyIterator { + Iter at_; + public: + using iterator_category = std::forward_iterator_tag; + using value_type = std::string; + using difference_type = std::ptrdiff_t; + using reference = const std::string&; + using pointer = const std::string*; + KeyIterator() = default; + explicit KeyIterator(Iter at) : at_(std::move(at)) {} + reference operator*() const noexcept { return at_->first; } + pointer operator->() const noexcept { return &at_->first; } + KeyIterator& operator++() { ++at_; return *this; } + KeyIterator operator++(int) { auto old = *this; ++*this; return old; } + bool operator==(const KeyIterator& other) const noexcept { return at_ == other.at_; } + bool operator!=(const KeyIterator& other) const noexcept { return !(*this == other); } + }; + + DiskResidentIdTable() = default; + DiskResidentIdTable(const DiskResidentIdTable&) = delete; + DiskResidentIdTable& operator=(const DiskResidentIdTable&) = delete; + DiskResidentIdTable(DiskResidentIdTable&&) noexcept = default; + DiskResidentIdTable& operator=(DiskResidentIdTable&&) noexcept = default; + + void set_directory(const std::string& directory) { index_.set_directory(directory); } + void clear() noexcept { resident_.clear(); index_.clear(); } + std::uint64_t size() const noexcept { return index_.size(); } + std::size_t resident_size() const noexcept { return resident_.size(); } + iterator begin() noexcept { return resident_begin(this); } + iterator end() noexcept { return {this, 0, {}}; } + const_iterator begin() const noexcept { return resident_begin(this); } + const_iterator end() const noexcept { return {this, 0, {}}; } + auto resident_ids() const noexcept { + const auto at = resident_.begin(); + const insertion_iterator first{this, at == resident_.end() ? 0 : at->first, + at == resident_.end() ? nullptr : at->second}; + return Range>{ + KeyIterator(first), + KeyIterator(insertion_iterator{this, 0, {}})}; + } + auto all_sorted() { + const auto at = index_.first_sorted(); + return Range{{this, at, at ? load(at) : nullptr}, {this, 0, {}}}; + } + auto all_sorted() const { + const auto at = index_.first_sorted(); + return Range{{this, at, at ? load(at) : nullptr}, {this, 0, {}}}; + } + // Full insertion-order replay without a growing vector of keys. + template void for_each_id(F&& visit) const { + for (auto at = index_.first_inserted(); at != 0; at = index_.next_inserted(at)) + visit(index_.key(at)); + } + // Decisions over historical ids must not depend on row residency. + // Walk keys alone, without restoring facts or retaining a key vector. + template void for_each_sorted_id(F&& visit) const { + for (auto at = index_.first_sorted(); at != 0; at = index_.next_sorted(at)) + visit(index_.key(at)); + } + iterator find(const std::string& key) { + spill(); + const auto at = index_.find(key); + return at ? iterator{this, *at, load(*at)} : end(); + } + const_iterator find(const std::string& key) const { + spill(); + const auto at = index_.find(key); + return at ? const_iterator{this, *at, load(*at)} : end(); + } + // A retired row cannot carry exposure/opened handles. These reads stay + // allocation-free and noexcept even when the disk is unavailable. + iterator find_resident(const std::string& key) noexcept { + for (const auto& row : resident_) + if (row.second->first == key) return {this, row.first, row.second}; + return end(); + } + const_iterator find_resident(const std::string& key) const noexcept { + for (const auto& row : resident_) + if (row.second->first == key) return {this, row.first, row.second}; + return end(); + } + std::pair emplace(const std::string& key, Value value) { + spill(); + const auto placed = index_.insert(key, value.retired_record()); + if (!placed.second) return {{this, placed.first, load(placed.first)}, false}; + auto row = std::make_shared(key, std::move(value)); + resident_.emplace(placed.first, row); + return {{this, placed.first, std::move(row)}, true}; + } + void spill() const { + for (auto at = resident_.begin(); at != resident_.end();) { + if (at->second.use_count() == 1 && at->second->second.spillable()) { + index_.set(at->first, at->second->second.retired_record()); + at = resident_.erase(at); + } else ++at; + } + } + +private: + template + static Iterator resident_begin(Owner* owner) noexcept { + const auto at = owner->resident_.rbegin(); + return {owner, at == owner->resident_.rend() ? 0 : at->first, + at == owner->resident_.rend() ? nullptr : at->second}; + } + std::shared_ptr load(Offset at) const { + const auto found = resident_.find(at); + if (found != resident_.end()) return found->second; + auto row = std::make_shared(index_.key(at), Value::restore_record(index_.get(at))); + resident_.emplace(at, row); + return row; + } + mutable Index index_; + mutable std::map> resident_; +}; +} // namespace pineforge::detail diff --git a/include/pineforge/detail/id_pair_store.hpp b/include/pineforge/detail/id_pair_store.hpp new file mode 100644 index 000000000..2dfea3348 --- /dev/null +++ b/include/pineforge/detail/id_pair_store.hpp @@ -0,0 +1,508 @@ +#pragma once + +// CR-150 L19l-C. Exact string / string-pair tables with no per-key RAM +// directory. Like L19a's successor-root file, scratch files are private, +// unlinked at creation, close-on-exec, and accessed with pread/pwrite. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::detail { + +// Keep L19a's system_error surface, carrying the run's resource failure too. +class IdPairStoreError final : public std::system_error, public RunFailureInfo { +public: + IdPairStoreError(int error, const char* operation) + : std::system_error(error, std::generic_category(), operation), + RunFailureInfo(RunFailureCode::report_sink_failed, {}) {} +}; + +class IdPairScratchFile { +public: + IdPairScratchFile() = default; + IdPairScratchFile(const IdPairScratchFile&) = delete; + IdPairScratchFile& operator=(const IdPairScratchFile&) = delete; + IdPairScratchFile(IdPairScratchFile&& other) noexcept + : fd_(std::exchange(other.fd_, -1)) {} + IdPairScratchFile& operator=(IdPairScratchFile&& other) noexcept { + if (this != &other) { + reset(); + fd_ = std::exchange(other.fd_, -1); + } + return *this; + } + ~IdPairScratchFile() { reset(); } + + bool opened() const noexcept { return fd_ >= 0; } + void reset() noexcept { + if (fd_ >= 0) ::close(fd_); + fd_ = -1; + } + void open(const std::string& directory) { + if (opened()) return; + const char* temporary = std::getenv("TMPDIR"); + std::string name = directory.empty() + ? (temporary && *temporary ? temporary : "/tmp") : directory; + if (name.back() != '/') name += '/'; + name += "pineforge-id-pairs-XXXXXX"; + const int opened = ::mkstemp(name.data()); + if (opened < 0) throw IdPairStoreError(errno, "id table create"); + if (::unlink(name.c_str()) != 0) { + const int error = errno; + ::close(opened); + throw IdPairStoreError(error, "id table unlink"); + } + if (::fcntl(opened, F_SETFD, FD_CLOEXEC) == -1) { + const int error = errno; + ::close(opened); + throw IdPairStoreError(error, "id table cloexec"); + } + fd_ = opened; + } + void resize(std::uint64_t bytes) { + check_range(0, bytes); + int result; + do { + result = ::ftruncate(fd_, static_cast(bytes)); + } while (result != 0 && errno == EINTR); + if (result != 0) throw IdPairStoreError(errno, "id table resize"); + } + void read(std::uint64_t offset, void* destination, std::size_t length) const { + check_range(offset, length); + auto* bytes = static_cast(destination); + std::size_t received = 0; + while (received < length) { + const ssize_t count = ::pread(fd_, bytes + received, length - received, + static_cast(offset + received)); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) + throw IdPairStoreError(count < 0 ? errno : EIO, "id table read"); + received += static_cast(count); + } + } + void write(std::uint64_t offset, const void* source, std::size_t length) { + check_range(offset, length); + const auto* bytes = static_cast(source); + std::size_t written = 0; + while (written < length) { + const ssize_t count = ::pwrite(fd_, bytes + written, length - written, + static_cast(offset + written)); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) + throw IdPairStoreError(count < 0 ? errno : EIO, "id table write"); + written += static_cast(count); + } + } + +private: + static void check_range(std::uint64_t offset, std::uint64_t length) { + constexpr auto maximum = static_cast(std::numeric_limits::max()); + if (offset > maximum || length > maximum - offset) + throw IdPairStoreError(EFBIG, "id table offset exhausted"); + } + int fd_ = -1; +}; + +// One 40-byte record plus both strings per distinct key ever seen, and a +// 16-byte on-disk hash slot at <= 1/2 load (32..64 bytes/key after warmup). +// Replacing a value, erasing and recurring reuse the original record. +// Single keys and pairs are separate domains; ("x", "") is not single "x". +// Hash collisions are always resolved by comparing the complete byte strings. +template +class IdPairStore { + static_assert(std::is_integral_v && sizeof(Value) == 8, + "id auxiliary values are fixed-width 64-bit integers"); + using MemoryKey = std::pair>; + using MemoryMap = std::map; + + struct Slot { + std::uint64_t hash = 0; + std::uint64_t offset_plus_one = 0; + }; + struct Header { + std::uint64_t hash = 0; + std::uint64_t first_size = 0; + std::uint64_t second_size = 0; + Value value = 0; + std::uint64_t flags = 0; + }; + static_assert(sizeof(Slot) == 16 && sizeof(Header) == 40, + "id scratch layout must have no padding"); + static constexpr std::uint64_t kLive = 1; + static constexpr std::uint64_t kPair = 2; + static constexpr std::uint64_t kInitialSlots = 64; + +public: + struct Entry { + std::string first; + Value second = 0; + std::optional from_entry; + }; + + // Streaming iteration owns only its current strings. It is invalidated + // by mutation; no adapter caller keeps an iterator across a write. + class const_iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = Entry; + using difference_type = std::ptrdiff_t; + using pointer = const Entry*; + using reference = const Entry&; + + const_iterator() = default; + reference operator*() const { return *entry_; } + pointer operator->() const { return &*entry_; } + const_iterator& operator++() { + if (owner_->memory_) { + ++memory_at_; + load_memory(); + } else { + offset_ = next_; + load_disk(); + } + return *this; + } + const_iterator operator++(int) { + const_iterator before = *this; + ++*this; + return before; + } + bool operator==(const const_iterator& other) const { + if (owner_ != other.owner_) return false; + if (!entry_ || !other.entry_) return !entry_ && !other.entry_; + return owner_->memory_ ? memory_at_ == other.memory_at_ : offset_ == other.offset_; + } + bool operator!=(const const_iterator& other) const { return !(*this == other); } + + private: + friend class IdPairStore; + const_iterator(const IdPairStore* owner, bool end) : owner_(owner) { + if (end) return; + if (owner_->memory_) { + memory_at_ = owner_->memory_->begin(); + load_memory(); + } else { + load_disk(); + } + } + void load_memory() { + entry_.reset(); + if (memory_at_ != owner_->memory_->end()) + entry_.emplace(Entry{memory_at_->first.first, memory_at_->second, + memory_at_->first.second}); + } + void load_disk() { + entry_.reset(); + while (offset_ < owner_->data_bytes_) { + const Header header = owner_->read_header(offset_); + next_ = offset_ + sizeof(Header) + header.first_size + header.second_size; + if ((header.flags & kLive) != 0) { + Entry row; + row.first.resize(static_cast(header.first_size)); + owner_->data_.read(offset_ + sizeof(Header), row.first.data(), row.first.size()); + row.second = header.value; + if ((header.flags & kPair) != 0) { + row.from_entry.emplace(static_cast(header.second_size), '\0'); + owner_->data_.read(offset_ + sizeof(Header) + header.first_size, + row.from_entry->data(), row.from_entry->size()); + } + entry_.emplace(std::move(row)); + return; + } + offset_ = next_; + } + } + const IdPairStore* owner_ = nullptr; + std::uint64_t offset_ = 0; + std::uint64_t next_ = 0; + typename MemoryMap::const_iterator memory_at_{}; + std::optional entry_; + }; + + IdPairStore() = default; + IdPairStore(const IdPairStore&) = delete; + IdPairStore& operator=(const IdPairStore&) = delete; + IdPairStore(IdPairStore&& other) noexcept { swap(other); } + IdPairStore& operator=(IdPairStore&& other) noexcept { + if (this != &other) { + IdPairStore moved(std::move(other)); + swap(moved); + } + return *this; + } + + void set_directory(const std::string& directory) { + if ((data_.opened() || index_.opened()) && directory != directory_) + throw std::logic_error("id table directory changed during run"); + directory_ = directory; + } + void clear() noexcept { + data_.reset(); + index_.reset(); + data_bytes_ = 0; + slots_ = 0; + used_ = 0; + size_ = 0; + if (memory_) memory_->clear(); + } + std::size_t size() const noexcept { return size_; } + bool empty() const noexcept { return size_ == 0; } + std::uint64_t disk_bytes() const noexcept { return data_bytes_ + slots_ * sizeof(Slot); } + const_iterator begin() const { return const_iterator(this, false); } + const_iterator end() const { return const_iterator(this, true); } + + // An explicit, per-instance fixture seam; production always uses disk. + // No global switch, environment knob, cached key roster or fallback on IO + // failure. clear() preserves this choice for a second run on the host. + void fixture_use_memory_reference() { + if (used_ != 0 || size_ != 0 || data_.opened() || index_.opened()) + throw std::logic_error("id table fixture backend changed after use"); + memory_ = std::make_unique(); + } + + std::optional lookup(std::string_view id) const { return lookup_key(id, std::nullopt); } + std::optional lookup(std::string_view id, std::string_view from_entry) const { + return lookup_key(id, from_entry); + } + Value at(const std::string& id) const { + const auto found = lookup(id); + if (!found) throw std::out_of_range("id auxiliary key is absent"); + return *found; + } + bool insert_or_assign(std::string_view id, Value value) { + return assign_key(id, std::nullopt, value); + } + bool insert_or_assign(std::string_view id, std::string_view from_entry, Value value) { + return assign_key(id, from_entry, value); + } + void min_assign(std::string_view id, Value value) { + const auto old = lookup(id); + if (!old || value < *old) insert_or_assign(id, value); + } + void min_assign(std::string_view id, std::string_view from_entry, Value value) { + const auto old = lookup(id, from_entry); + if (!old || value < *old) insert_or_assign(id, from_entry, value); + } + bool erase(std::string_view id) { return erase_key(id, std::nullopt); } + bool erase(std::string_view id, std::string_view from_entry) { + return erase_key(id, from_entry); + } + +private: + void swap(IdPairStore& other) noexcept { + using std::swap; + swap(data_, other.data_); + swap(index_, other.index_); + swap(directory_, other.directory_); + swap(data_bytes_, other.data_bytes_); + swap(slots_, other.slots_); + swap(used_, other.used_); + swap(size_, other.size_); + swap(memory_, other.memory_); + } + static MemoryKey memory_key(std::string_view first, std::optional second) { + return {std::string(first), second ? std::optional{std::string(*second)} + : std::nullopt}; + } + static std::uint64_t key_hash(std::string_view first, + std::optional second) noexcept { + std::uint64_t hash = 1469598103934665603ULL; + const auto byte = [&hash](unsigned char value) { + hash ^= value; + hash *= 1099511628211ULL; + }; + const auto string = [&byte](std::string_view value) { + const auto length = static_cast(value.size()); + for (unsigned shift = 0; shift < 64; shift += 8) + byte(static_cast(length >> shift)); + for (const unsigned char value_byte : value) byte(value_byte); + }; + byte(second ? 1 : 0); + string(first); + if (second) string(*second); + return hash; + } + Header read_header(std::uint64_t offset) const { + if (offset > data_bytes_ || data_bytes_ - offset < sizeof(Header)) + throw IdPairStoreError(EIO, "id table truncated record"); + Header header; + data_.read(offset, &header, sizeof(header)); + const auto available = data_bytes_ - offset - sizeof(Header); + if (header.first_size > available || header.second_size > available - header.first_size + || (header.flags & ~(kLive | kPair)) != 0 + || ((header.flags & kPair) == 0 && header.second_size != 0)) + throw IdPairStoreError(EIO, "id table invalid record"); + return header; + } + bool equal_string(std::uint64_t offset, std::uint64_t length, std::string_view key) const { + if (length != key.size()) return false; + std::array buffer{}; + for (std::size_t at = 0; at < key.size();) { + const auto count = std::min(buffer.size(), key.size() - at); + data_.read(offset + at, buffer.data(), count); + if (key.compare(at, count, buffer.data(), count) != 0) return false; + at += count; + } + return true; + } + // The first empty slot or the exact key's slot. Dead records retain + // their slot, so erasing cannot break another key's probe chain. + std::pair locate(std::string_view first, + std::optional second, std::uint64_t hash) const { + std::uint64_t at = hash & (slots_ - 1); + for (std::uint64_t probes = 0; probes < slots_; ++probes) { + Slot slot; + index_.read(at * sizeof(Slot), &slot, sizeof(slot)); + if (slot.offset_plus_one == 0) return {at, slot}; + if (slot.hash == hash) { + const auto offset = slot.offset_plus_one - 1; + const Header header = read_header(offset); + if (header.hash != slot.hash) + throw IdPairStoreError(EIO, "id table mismatched index"); + if (((header.flags & kPair) != 0) == second.has_value() + && equal_string(offset + sizeof(Header), header.first_size, first) + && (!second || equal_string(offset + sizeof(Header) + header.first_size, + header.second_size, *second))) + return {at, slot}; + } + at = (at + 1) & (slots_ - 1); + } + throw IdPairStoreError(EIO, "id table full index"); + } + static void place_slot(IdPairScratchFile& index, std::uint64_t slots, const Slot& slot) { + std::uint64_t at = slot.hash & (slots - 1); + for (std::uint64_t probes = 0; probes < slots; ++probes) { + Slot existing; + index.read(at * sizeof(Slot), &existing, sizeof(existing)); + if (existing.offset_plus_one == 0) { + index.write(at * sizeof(Slot), &slot, sizeof(slot)); + return; + } + at = (at + 1) & (slots - 1); + } + throw IdPairStoreError(EIO, "id table rehash overflow"); + } + void grow_index() { + constexpr auto maximum = static_cast(std::numeric_limits::max()) + / sizeof(Slot); + if (slots_ > maximum / 2) throw IdPairStoreError(EFBIG, "id table index exhausted"); + const auto next_slots = slots_ == 0 ? kInitialSlots : slots_ * 2; + IdPairScratchFile next; + next.open(directory_); + next.resize(next_slots * sizeof(Slot)); // fresh sparse bytes read as zero + std::array buffer{}; + for (std::uint64_t at = 0; at < slots_;) { + const auto count = std::min(buffer.size(), slots_ - at); + index_.read(at * sizeof(Slot), buffer.data(), + static_cast(count) * sizeof(Slot)); + for (std::size_t i = 0; i < static_cast(count); ++i) + if (buffer[i].offset_plus_one != 0) place_slot(next, next_slots, buffer[i]); + at += count; + } + index_ = std::move(next); // publish only a complete replacement + slots_ = next_slots; + } + std::optional lookup_key(std::string_view first, + std::optional second) const { + if (memory_) { + const auto found = memory_->find(memory_key(first, second)); + return found == memory_->end() ? std::nullopt : std::optional{found->second}; + } + if (slots_ == 0) return std::nullopt; + const auto found = locate(first, second, key_hash(first, second)).second; + if (found.offset_plus_one == 0) return std::nullopt; + const Header header = read_header(found.offset_plus_one - 1); + return (header.flags & kLive) != 0 ? std::optional{header.value} : std::nullopt; + } + bool assign_key(std::string_view first, std::optional second, Value value) { + if (memory_) { + const auto inserted = memory_->insert_or_assign(memory_key(first, second), value); + if (inserted.second) ++size_; + return inserted.second; + } + const auto hash = key_hash(first, second); + if (slots_ == 0) grow_index(); + auto found = locate(first, second, hash); + if (found.second.offset_plus_one != 0) { + const auto offset = found.second.offset_plus_one - 1; + Header header = read_header(offset); + const bool inserted = (header.flags & kLive) == 0; + header.value = value; + header.flags |= kLive; + data_.write(offset, &header, sizeof(header)); + if (inserted) ++size_; + return inserted; + } + if (used_ >= slots_ / 2) { + grow_index(); + found = locate(first, second, hash); + } + if (size_ == std::numeric_limits::max()) + throw IdPairStoreError(EFBIG, "id table size exhausted"); + const auto second_size = second ? second->size() : 0; + constexpr auto maximum = static_cast(std::numeric_limits::max()); + if (data_bytes_ > maximum - sizeof(Header) + || first.size() > maximum - data_bytes_ - sizeof(Header) + || second_size > maximum - data_bytes_ - sizeof(Header) - first.size()) + throw IdPairStoreError(EFBIG, "id table data exhausted"); + Header header{hash, static_cast(first.size()), + static_cast(second_size), value, kLive | (second ? kPair : 0)}; + data_.open(directory_); + data_.write(data_bytes_, &header, sizeof(header)); + data_.write(data_bytes_ + sizeof(Header), first.data(), first.size()); + if (second) + data_.write(data_bytes_ + sizeof(Header) + first.size(), second->data(), second->size()); + const Slot slot{hash, data_bytes_ + 1}; + index_.write(found.first * sizeof(Slot), &slot, sizeof(slot)); + data_bytes_ += sizeof(Header) + first.size() + second_size; + ++used_; + ++size_; + return true; + } + bool erase_key(std::string_view first, std::optional second) { + if (memory_) { + const auto count = memory_->erase(memory_key(first, second)); + size_ -= count; + return count != 0; + } + if (slots_ == 0) return false; + const auto found = locate(first, second, key_hash(first, second)).second; + if (found.offset_plus_one == 0) return false; + const auto offset = found.offset_plus_one - 1; + Header header = read_header(offset); + if ((header.flags & kLive) == 0) return false; + header.flags &= ~kLive; + data_.write(offset, &header, sizeof(header)); + --size_; + return true; + } + + IdPairScratchFile data_; + IdPairScratchFile index_; + std::string directory_; + std::uint64_t data_bytes_ = 0; + std::uint64_t slots_ = 0; + std::uint64_t used_ = 0; // distinct keys including erased keys + std::size_t size_ = 0; // live keys + std::unique_ptr memory_; // allocated only by the explicit fixture seam +}; + +} // namespace pineforge::detail diff --git a/include/pineforge/detail/scratch_file.hpp b/include/pineforge/detail/scratch_file.hpp new file mode 100644 index 000000000..e7c2d8878 --- /dev/null +++ b/include/pineforge/detail/scratch_file.hpp @@ -0,0 +1,116 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::detail { + +// Keep the errno and std::system_error contract of the successor-root file, +// and carry the run's typed scratch-I/O failure through the ordinary catcher. +class ScratchIoError final : public std::system_error, public RunFailureInfo { +public: + ScratchIoError(int error, const char* operation) + : std::system_error(error, std::generic_category(), operation), + RunFailureInfo(RunFailureCode::report_sink_failed, {}) {} +}; + +// Run-owned, anonymous, close-on-exec scratch. All reads/writes are positional; +// no shared cursor, global descriptor, mapping, or growing userspace buffer. +class ScratchFile { +public: + ScratchFile() = default; + ScratchFile(const ScratchFile&) = delete; + ScratchFile& operator=(const ScratchFile&) = delete; + ScratchFile(ScratchFile&& other) noexcept { *this = std::move(other); } + ScratchFile& operator=(ScratchFile&& other) noexcept { + if (this != &other) { + clear(); + fd_ = std::exchange(other.fd_, -1); + directory_ = std::move(other.directory_); + } + return *this; + } + ~ScratchFile() { clear(); } + + void set_directory(const std::string& value) { + if (fd_ >= 0 && directory_ != value) + throw std::logic_error("scratch directory changed after spill"); + directory_ = value; + } + void clear() noexcept { + if (fd_ >= 0) ::close(fd_); + fd_ = -1; + } + void open(const char* prefix) { + if (fd_ >= 0) return; + const char* temporary = std::getenv("TMPDIR"); + std::string name = directory_.empty() + ? (temporary && *temporary ? temporary : "/tmp") : directory_; + if (name.back() != '/') name += '/'; + name += prefix; + name += "-XXXXXX"; + const int opened = ::mkstemp(name.data()); + if (opened < 0) throw ScratchIoError(errno, "scratch create"); + if (::unlink(name.c_str()) != 0) { + const int error = errno; + ::close(opened); + throw ScratchIoError(error, "scratch unlink"); + } + if (::fcntl(opened, F_SETFD, FD_CLOEXEC) == -1) { + const int error = errno; + ::close(opened); + throw ScratchIoError(error, "scratch cloexec"); + } + fd_ = opened; + } + static void check_extent(std::uint64_t offset, std::size_t length) { + const auto maximum = static_cast(std::numeric_limits::max()); + if (offset > maximum || length > maximum - offset) + throw ScratchIoError(EFBIG, "scratch offset exhausted"); + } + void write(std::uint64_t offset, const void* data, std::size_t length) { + check_extent(offset, length); + const auto* bytes = static_cast(data); + std::size_t written = 0; + while (written < length) { + const auto chunk = std::min(length - written, + static_cast(std::numeric_limits::max())); + const ssize_t count = ::pwrite(fd_, bytes + written, chunk, + static_cast(offset + written)); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) throw ScratchIoError(count < 0 ? errno : EIO, "scratch write"); + written += static_cast(count); + } + } + void read(std::uint64_t offset, void* data, std::size_t length) const { + check_extent(offset, length); + auto* bytes = static_cast(data); + std::size_t received = 0; + while (received < length) { + const auto chunk = std::min(length - received, + static_cast(std::numeric_limits::max())); + const ssize_t count = ::pread(fd_, bytes + received, chunk, + static_cast(offset + received)); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) throw ScratchIoError(count < 0 ? errno : EIO, "scratch read"); + received += static_cast(count); + } + } + +private: + int fd_ = -1; + std::string directory_; +}; +} // namespace pineforge::detail diff --git a/include/pineforge/native_order.hpp b/include/pineforge/native_order.hpp index cd2584b68..c57a6f78a 100644 --- a/include/pineforge/native_order.hpp +++ b/include/pineforge/native_order.hpp @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -1685,6 +1687,7 @@ struct GroupEffectReceipt { class WorkingRequestCore { public: + class CohortView; explicit WorkingRequestCore(RunIdentity identity); WorkingRequestCore(const WorkingRequestCore&) = delete; WorkingRequestCore& operator=(const WorkingRequestCore&) = delete; @@ -1725,7 +1728,10 @@ class WorkingRequestCore { std::size_t retire_history(uint64_t through_ordinal); /// The ordinal of the newest retired event, 0 while none is. uint64_t retired_through() const noexcept { return retired_through_; } - const std::vector& cohorts() const noexcept { return cohorts_; } + /// Logical roster sequence in allocation order, including empty rosters + /// whose members have been removed. Walking allocates nothing and does + /// no I/O; empty rows are synthesized from the dense handle sequence. + CohortView cohorts() const noexcept; /// The retained cohort-receipt window, in command order. Its absolute /// positions are [cohort_receipt_base(), cohort_receipt_count()). A /// standalone core retains all rows until its reader retires them; the @@ -2157,7 +2163,7 @@ class WorkingRequestCore { /// read from the live row or, for a request that is no longer working, /// from the chain index; nullopt for a handle this run never issued. std::optional canonical_cohort_origin(const RequestHandle& origin) const; - std::size_t cohort_index(CohortHandle cohort) const noexcept; + bool cohort_exists(CohortHandle cohort) const noexcept; /// Records an accepted request's incarnation and, for a replace /// successor, its chain root, in the chain index. Storage is reserved by /// reserve_plan, so this cannot throw inside commit. @@ -2328,13 +2334,70 @@ class WorkingRequestCore { std::vector receipts_; std::size_t receipts_base_ = 0; std::uint64_t next_cohort_handle_ = 1; - std::vector cohorts_; + // Only rosters with members occupy nodes; erasing an empty roster frees + // its member capacity as well. next_cohort_handle_ records the complete + // dense sequence of allocated handles, including every empty roster. + std::map cohorts_; std::vector cohort_receipts_; // Storage cursor only: the consumer hashes the lifetime count and every // receipt, independently of how many of their rows have been retired. std::size_t cohort_receipts_base_ = 0; }; +// A non-owning read view. Iteration preserves CohortRoster references for +// existing typed predicates and iterator arrow reads. An empty row belongs +// to its input iterator and is valid until that iterator advances or dies. +// Indexed reads return a value, so a saved row from a temporary view remains +// valid. Callers that need no copies use the allocation-free iterator walk. +class WorkingRequestCore::CohortView { +public: + class const_iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = CohortRoster; + using difference_type = std::ptrdiff_t; + using pointer = const CohortRoster*; + using reference = const CohortRoster&; + + const_iterator() noexcept = default; + reference operator*() const noexcept; + pointer operator->() const noexcept { return &**this; } + const_iterator& operator++() noexcept { ++index_; return *this; } + const_iterator operator++(int) { + auto previous = *this; + ++*this; + return previous; + } + friend bool operator==(const const_iterator& left, const const_iterator& right) noexcept { + return left.core_ == right.core_ && left.index_ == right.index_; + } + friend bool operator!=(const const_iterator& left, const const_iterator& right) noexcept { + return !(left == right); + } + + private: + friend class CohortView; + const_iterator(const WorkingRequestCore* core, std::size_t index) noexcept + : core_(core), index_(index) {} + const WorkingRequestCore* core_ = nullptr; + std::size_t index_ = 0; + mutable CohortRoster empty_{}; + }; + + std::size_t size() const noexcept { + return static_cast(core_->next_cohort_handle_ - 1); + } + bool empty() const noexcept { return size() == 0; } + const_iterator begin() const noexcept { return const_iterator(core_, 0); } + const_iterator end() const noexcept { return const_iterator(core_, size()); } + CohortRoster operator[](std::size_t index) const { return *const_iterator(core_, index); } + +private: + friend class WorkingRequestCore; + explicit CohortView(const WorkingRequestCore* core) noexcept : core_(core) {} + const WorkingRequestCore* core_; +}; + class PreparedSubmit { public: PreparedSubmit() noexcept; diff --git a/include/pineforge/source/pine_adapter.hpp b/include/pineforge/source/pine_adapter.hpp index 1cb2ebe93..3f381d567 100644 --- a/include/pineforge/source/pine_adapter.hpp +++ b/include/pineforge/source/pine_adapter.hpp @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include #include @@ -637,6 +640,9 @@ class PlacementTable { PlacementTable(PlacementTable&&) noexcept = default; PlacementTable& operator=(PlacementTable&&) noexcept = default; + void set_auxiliary_tables_directory(const std::string& directory); + void fixture_use_memory_auxiliary_tables(); + std::uint64_t auxiliary_disk_bytes() const noexcept; std::size_t size() const noexcept { return size_; } std::size_t max_size() const noexcept { return window_.max_size(); } // Largest incarnation ever inserted, erased rows included. Incarnations @@ -852,11 +858,11 @@ class PlacementTable { static constexpr std::size_t kSpareNodes = 64; static constexpr std::uint64_t kFoldBasis = 1469598103934665603ULL; - struct SequenceMinima { - std::uint64_t any = std::numeric_limits::max(); - std::unordered_map by_from_entry; - }; - using Minima = std::unordered_map; + // Single keys hold the minimum over any from_entry; pair keys hold the + // exact (source_id, from_entry) minimum. Their separate key domains keep + // an empty from_entry distinct from the any-entry summary. Neither the + // directory nor old keys consume per-id RAM. + using Minima = pineforge::detail::IdPairStore; // A reused node takes its new row in one assignment. static void overwrite(PlacementSnapshot& node, PlacementSnapshot&& row) { @@ -870,11 +876,11 @@ class PlacementTable { } static void fold_sequence(Minima& index, const PlacementSnapshot& row) { - auto& minima = index[row.source_id]; - minima.any = std::min(minima.any, row.command_sequence); - const auto inserted = minima.by_from_entry.emplace(row.from_entry, row.command_sequence); - if (!inserted.second) - inserted.first->second = std::min(inserted.first->second, row.command_sequence); + // Keep both original minima, including an explicitly present + // UINT64_MAX: command_sequence_for uses the same fallback for it. + // Recurring pairs update their old disk records in place. + index.min_assign(row.source_id, row.command_sequence); + index.min_assign(row.source_id, row.from_entry, row.command_sequence); } PlacementSnapshot* locate(std::uint64_t incarnation) const noexcept { @@ -962,6 +968,7 @@ class OriginRoster { // R5 lane V19-E: a running digest of the members' incarnations, each // folded once, when it was appended (the v4 extension fold's view). std::uint64_t digest() const noexcept { return digest_; } + static OriginRoster from_retired(std::size_t count, std::uint64_t digest); const native_order::RequestHandle& operator[](std::size_t global) const; const native_order::RequestHandle& back() const { return (*this)[size() - 1]; } bool empty() const noexcept { return size() == 0; } @@ -1531,6 +1538,10 @@ class PineExecutionAdapter { void set_configuration(const PineStrategyConfig& config) noexcept; void set_staged_configuration(const StagedConfiguration& staged); void set_begin_mode(bool is_stream, bool bar_magnifier = false) noexcept; + void set_id_tables_directory(const std::string& directory); // All L19l tables, after reset. + void set_auxiliary_tables_directory(const std::string& directory); // Same binding. + void fixture_use_memory_auxiliary_tables(); + std::uint64_t fixture_auxiliary_disk_bytes() const noexcept; // `instrument_feeds`, when given, is moved into the spec's instrument // feeds (and back into it if the projection throws). @@ -1778,7 +1789,7 @@ class PineExecutionAdapter { const NativeDecisionContext&) const noexcept; std::uint64_t command_sequence_for_exit(const SourceId& exit_id, - const SourceId& from_entry = {}) const noexcept; + const SourceId& from_entry = {}) const; bool is_open_phase_exit(std::size_t trade_index) const noexcept; // Reorders trade_exit_phase_[start..) to follow a same-bar exit-group sort: // slot start + i takes the phase of the trade previously at indices[i]. @@ -1815,6 +1826,10 @@ class PineExecutionAdapter { native_order::CohortHandle handle{}; OriginRoster origins; std::vector opened; + struct Record { + std::uint64_t handle; std::int64_t cycle; + std::uint64_t origin_count, origin_digest, erased_sides; + }; // Live source exposure by opening provenance. This is source-layer // bookkeeping only: the generic core still resolves cohort authority // at every candidate. It lets a command snapshot an existing id's @@ -1822,12 +1837,15 @@ class PineExecutionAdapter { // reduce that cohort. std::unordered_map live_units_by_origin; std::int64_t cycle = 0; + Record retired_record() const noexcept; + static CohortFacts restore_record(const Record&); + bool spillable() const noexcept; // R5 lane V19-E: the sides (bit 0 long, bit 1 short) of the openings // in `origins` whose placement rows were erased, so a question about // every origin the cohort ever accepted still covers them. std::uint8_t erased_opening_sides = 0; }; - + using CohortTable = ::pineforge::detail::DiskResidentIdTable; struct PendingBracketLeg { native_order::Request request; PlacementSnapshot snapshot; @@ -2034,7 +2052,7 @@ class PineExecutionAdapter { double cohort_exposure_for(const SourceId&) const noexcept; // The units an exit's from_entry holds; none when it names no entry. std::optional from_entry_units(const SourceId&) const noexcept; - bool from_entry_filled_this_cycle(const SourceId&) const noexcept; + bool from_entry_filled_this_cycle(const SourceId&) const; double percent_commission_live_equity(double) const noexcept; // The same equity at an explicit account-currency rate: the rate an // execution converts at, which its terms facts carry @@ -2396,7 +2414,7 @@ class PineExecutionAdapter { bool follows_same_bar_declined_reversal( const PlacementSnapshot&, const NativePrecommitView&) const; bool bracket_belongs_to_reversal( - const PlacementSnapshot&, const PlacementSnapshot&) const noexcept; + const PlacementSnapshot&, const PlacementSnapshot&) const; void suspend_brackets_for_reversal( const PlacementSnapshot&, const exit_legs::Frame&, double open_price); void suspend_declined_reversal_brackets( @@ -2519,8 +2537,8 @@ class PineExecutionAdapter { std::uint64_t source_command_sequence_ = 0; std::int32_t entry_attempt_bar_ = -1; std::uint32_t entry_attempts_on_bar_ = 0; - std::unordered_map cohorts_by_id_; - std::vector cohort_order_; + CohortTable cohorts_by_id_; + CohortTable::iterator require_cohort(const SourceId&); PlacementTable placement_; std::unordered_map live_by_source_key_; std::unordered_map bracket_families_; @@ -2559,7 +2577,7 @@ class PineExecutionAdapter { // then suppress just that duplicate debit at notification delivery. std::unordered_set current_debited_applied_ordinals_; std::unordered_set intraday_loss_relabel_ordinals_; - std::unordered_map consumed_partial_exit_cycles_; + pineforge::detail::IdPairStore consumed_partial_exit_cycles_; std::unordered_set bracket_shadowed_openings_; std::unordered_map named_entry_cancel_tokens_; // An exit bound to its entry id whose working order a strategy.cancel of @@ -2741,7 +2759,7 @@ class PineExecutionAdapter { // and the most keys it ever held counting one transient key per live // strategy.close / close_all order -- and, for the alternative the tapes // leave open, per live strategy.order market order too. - std::vector exit_rank_keys_; + ::pineforge::detail::DiskIdTable exit_rank_keys_; std::uint64_t exit_rank_keys_digest_ = 1469598103934665603ULL; std::uint32_t exit_rank_peak_ = 0; std::uint32_t exit_rank_alt_peak_ = 0; diff --git a/scripts/check_corpus_parity.sh b/scripts/check_corpus_parity.sh index 94d20f8ff..7de436aaa 100755 --- a/scripts/check_corpus_parity.sh +++ b/scripts/check_corpus_parity.sh @@ -50,6 +50,8 @@ # (default: scripts/corpus_parity_subset.txt) # SKIP_BUILD=1 reuse an existing BUILD_DIR (developer loop only) # SKIP_RUN=1 judge the trades already on disk (developer loop only) +# PINEFORGE_CANARY_LOG optional sink_canary:/sink_kept: log, checked against +# the probes run; full mode delegates to run_corpus.sh # # Exit codes: 0 parity holds, 1 parity drifted, 2 the check could not run. @@ -90,6 +92,11 @@ warn() { printf '\033[1;33m[corpus-parity]\033[0m %s\n' "$*" >&2; } fail() { printf '\033[1;31m[corpus-parity]\033[0m %s\n' "$*" >&2; exit 1; } unrunnable() { printf '\033[1;31m[corpus-parity]\033[0m %s\n' "$*" >&2; exit 2; } +CANARY_LOG="${PINEFORGE_CANARY_LOG:-}" +if [[ "$MODE" == subset && -n "$CANARY_LOG" ]]; then + : > "$CANARY_LOG" +fi + # --- 1) the corpus, at the gitlink this repository records ------------- GITLINK="$(git rev-parse HEAD:corpus 2>/dev/null || true)" @@ -216,9 +223,24 @@ else fi' corpus-parity-subset || run_rc=$? elapsed=$(( $(date +%s) - started )) log "ran ${#PROBES[@]} probes in ${elapsed}s" + canary_rc=0 + if [[ -n "$CANARY_LOG" ]]; then + # Collect only after every child finishes: no concurrent append + # to the shared log, and failed probes keep their lines too. + probe_logs=() + for probe in "${PROBES[@]}"; do + probe_logs+=("$RUN_LOGS/$(printf %s "$probe" | tr / _).log") + done + "$PY" -I scripts/check_sink_canary_log.py append "$CANARY_LOG" "${probe_logs[@]}" + "$PY" -I scripts/check_sink_canary_log.py check "$CANARY_LOG" \ + --probes "${PROBES[@]}" || canary_rc=$? + fi if (( run_rc != 0 )); then unrunnable "a subset probe failed to run; refusing to judge stale trades" fi + if (( canary_rc != 0 )); then + fail "sink canary coverage failed; see $CANARY_LOG" + fi fi fi diff --git a/scripts/check_sink_canary_log.py b/scripts/check_sink_canary_log.py new file mode 100644 index 000000000..d216a133a --- /dev/null +++ b/scripts/check_sink_canary_log.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Collect sweep canaries and check coverage against the probes actually run. + +Append only ^sink_(canary|kept): lines, from stdin or completed probe logs: + python3 -I scripts/check_sink_canary_log.py append LOG [OUTPUT ...] +Check exactly one successful canary per probe, with no failed, malformed, +duplicate or unexpected canaries: + python3 -I scripts/check_sink_canary_log.py check LOG --probes DIR ... + python3 -I scripts/check_sink_canary_log.py check LOG --probes-file FILE + +Probe files contain one directory/id per line, with optional # comments. +The sweep, not a copied population count, supplies the expectation. +Only the sweep truncates LOG, once before running probes; append never does. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from pathlib import Path +import re +import sys + + +CANARY = re.compile( + r"sink_canary: probe=(?P\S+) bytes=\d+ records=\d+ trades=\d+" + r" equity=\d+ (?Pok|FAIL(?: .*)?)") +PROBE = re.compile(r"^sink_canary: probe=(\S+)") + + +def append_lines(log: Path, lines) -> None: + """Preserve passing and failing canaries and kept paths, and nothing else.""" + with log.open("a", encoding="utf-8", newline="\n") as output: + for line in lines: + if line.startswith(("sink_canary:", "sink_kept:")): + output.write(line.rstrip("\r\n") + "\n") + + +def probe_id(value: str) -> str: + return "_".join(Path(value).name.split()) or "-" + + +def check_log(log: Path, probes: list[str]) -> list[str]: + """Return diagnostics naming every missing, repeated or invalid probe.""" + expected = Counter(probe_id(probe) for probe in probes) + seen = Counter() + problems = [] + if not expected: + problems.append("the sweep supplied no probes") + for probe, count in expected.items(): + if count != 1: + problems.append(f"probe={probe}: repeated in the sweep's probe list") + with log.open(encoding="utf-8") as source: + for number, line in enumerate(source, 1): + line = line.rstrip("\r\n") + if line.startswith("sink_kept:"): + continue + found = CANARY.fullmatch(line) + name = PROBE.match(line) + probe = name[1] if name else f"" + if not found: + problems.append(f"probe={probe}: malformed canary at line {number}") + continue + seen[probe] += 1 + if probe not in expected: + problems.append(f"probe={probe}: not in the sweep's probe list") + if found["verdict"] != "ok": + problems.append(f"probe={probe}: {found['verdict']}") + for probe in expected: + if seen[probe] != 1: + problems.append(f"probe={probe}: expected one canary, found {seen[probe]}") + return problems + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_subparsers(dest="mode", required=True) + append = modes.add_parser("append", help="append only canary/kept lines") + append.add_argument("log", type=Path) + append.add_argument("outputs", type=Path, nargs="*") + check = modes.add_parser("check", help="check coverage for the probes run") + check.add_argument("log", type=Path) + source = check.add_mutually_exclusive_group(required=True) + source.add_argument("--probes", nargs="*") + source.add_argument("--probes-file", type=Path) + args = parser.parse_args(argv) + try: + if args.mode == "append": + if args.outputs: + for path in args.outputs: + with path.open(encoding="utf-8", errors="replace") as lines: + append_lines(args.log, lines) + else: + append_lines(args.log, sys.stdin) + return 0 + probes = args.probes + if args.probes_file is not None: + probes = [ + entry for line in args.probes_file.read_text(encoding="utf-8").splitlines() + if (entry := line.partition("#")[0].strip()) + ] + problems = check_log(args.log, probes) + except (OSError, UnicodeError) as error: + print(f"sink_canary_log: {error}", file=sys.stderr) + return 1 + if problems: + for problem in problems: + print(f"sink_canary_log: {problem}", file=sys.stderr) + return 1 + print(f"sink_canary_log: ok probes={len(probes)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fingerprint_self_test.py b/scripts/fingerprint_self_test.py index c149866a1..5735ab5dd 100644 --- a/scripts/fingerprint_self_test.py +++ b/scripts/fingerprint_self_test.py @@ -892,13 +892,15 @@ def check(name: str, cond: bool) -> None: duplicate_rejected) source_tree = ast.parse(RUN_STRATEGY.read_text(encoding="utf-8")) - main_fn = next((node for node in source_tree.body - if isinstance(node, ast.FunctionDef) and node.name == "main"), None) - main_uses_helper = main_fn is not None and any( + # L9c split main() into main() (argparse + sink ExitStack) and _main_run(). + main_fns = [node for node in source_tree.body + if isinstance(node, ast.FunctionDef) and node.name in ("main", "_main_run")] + main_uses_helper = bool(main_fns) and any( isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "build_runtime_provenance" - for node in ast.walk(main_fn) + for fn in main_fns + for node in ast.walk(fn) ) check("run_strategy: main fingerprint path uses runtime helper", main_uses_helper) diff --git a/scripts/run_corpus.sh b/scripts/run_corpus.sh index 52accad11..c275af77f 100755 --- a/scripts/run_corpus.sh +++ b/scripts/run_corpus.sh @@ -16,6 +16,8 @@ # SKIP_BUILD — set to 1 to reuse an existing build directory # SKIP_RUN — set to 1 to skip the Python harness pass # SKIP_VERIFY — set to 1 to skip the parity verifier +# PINEFORGE_CANARY_LOG — optional sink_canary:/sink_kept: log; truncate once, +# append every probe's lines, then check coverage set -euo pipefail @@ -41,6 +43,11 @@ log() { printf '\033[1;34m[run_corpus]\033[0m %s\n' "$*"; } warn() { printf '\033[1;33m[run_corpus]\033[0m %s\n' "$*" >&2; } fail() { printf '\033[1;31m[run_corpus]\033[0m %s\n' "$*" >&2; exit 1; } +CANARY_LOG="${PINEFORGE_CANARY_LOG:-}" +if [[ -n "$CANARY_LOG" ]]; then + : > "$CANARY_LOG" +fi + if [[ ! -f "$ROOT_DIR/corpus/CMakeLists.txt" ]]; then fail "validation corpus is not checked out (missing corpus/CMakeLists.txt). Run: git submodule update --init corpus @@ -91,6 +98,9 @@ if [[ "${SKIP_RUN:-0}" != "1" ]]; then n_fail=0 n_selected=0 failed=() + # Keep the option even if no command can run (e.g. missing libraries), so + # an empty array expansion under Bash 3 + nounset cannot hide the failure. + canary_args=(--probes) started=$(date +%s) for strat_dir in corpus/validation/*/; do @@ -136,6 +146,7 @@ if [[ "${SKIP_RUN:-0}" != "1" ]]; then run_cmd=("$PY" scripts/run_strategy.py "$strat_dir" --so-name "$so_name") fi + canary_args+=("$strat_dir") run_output="" if run_output="$("${run_cmd[@]}" 2>&1)"; then n_ok=$((n_ok + 1)) @@ -147,6 +158,9 @@ if [[ "${SKIP_RUN:-0}" != "1" ]]; then warn " $line" done <<< "$run_output" fi + if [[ -n "$CANARY_LOG" ]]; then + "$PY" -I scripts/check_sink_canary_log.py append "$CANARY_LOG" <<< "$run_output" + fi done if (( n_selected == 0 )); then @@ -154,6 +168,11 @@ if [[ "${SKIP_RUN:-0}" != "1" ]]; then fi elapsed=$(( $(date +%s) - started )) log "ran $((n_ok + n_fail)) strategies in ${elapsed}s -- ok=$n_ok fail=$n_fail" + canary_rc=0 + if [[ -n "$CANARY_LOG" ]]; then + "$PY" -I scripts/check_sink_canary_log.py check "$CANARY_LOG" \ + "${canary_args[@]}" || canary_rc=$? + fi if (( n_fail > 0 )); then warn "failed strategies:" for f in "${failed[@]}"; do @@ -161,6 +180,9 @@ if [[ "${SKIP_RUN:-0}" != "1" ]]; then done fail "$n_fail strategy run(s) failed; refusing to verify stale engine_trades.csv" fi + if (( canary_rc != 0 )); then + fail "sink canary coverage failed; see $CANARY_LOG" + fi fi # --- 3) verify -------------------------------------------------------- diff --git a/scripts/run_strategy.py b/scripts/run_strategy.py index 3d0161dee..0f6ad2a78 100644 --- a/scripts/run_strategy.py +++ b/scripts/run_strategy.py @@ -27,6 +27,22 @@ # Don't overwrite engine_trades.csv if it already exists python scripts/run_strategy.py corpus/basic/greedy --no-overwrite + # Read trades from a verified v1 report sink; keep the same CSV bytes. + # Every probe prints one machine-greppable canary line, + # sink_canary: probe= bytes= records= trades= equity= ok + # (probe id = the strategy directory's basename); a sink that is absent, + # empty or short prints the same fields with "FAIL " and fails + # the probe. + PINEFORGE_REPORT_SINK=1 python scripts/run_strategy.py corpus/basic/greedy + + # Test-only: keep the sink file of chosen probes as a sweep artifact. + # The value is "all" or comma-separated probe ids; a sweep that wants five + # artifacts names the first five probes of its own order. Each kept file + # is announced as "sink_kept: probe= path=" and lives in + # $TMPDIR; without the variable every sink file is unlinked at close. + PINEFORGE_REPORT_SINK=1 PINEFORGE_KEEP_REPORT_SINK=greedy \\ + python scripts/run_strategy.py corpus/basic/greedy + # All strategies (matches `bash scripts/run_corpus.sh`) for d in corpus/*/*/; do python scripts/run_strategy.py "$d"; done @@ -58,7 +74,9 @@ import re import struct import sys +import tempfile import time +from contextlib import ExitStack from datetime import datetime, timedelta, timezone from pathlib import Path from typing import NamedTuple @@ -1965,6 +1983,223 @@ def _with_run_failure_code(error: Exception, lib, state) -> Exception: # The text of a failed run that reported neither a text nor a code # (strategy_last_run_status 1 alone), as docker/run_json.py spells it. RUN_STATUS_FAILED_TEXT = "the run did not complete and the engine reported no error" +REPORT_SINK_ENV = "PINEFORGE_REPORT_SINK" +# Test-only artifact switch of sink mode (_keep_report_sink): the sink file is +# otherwise unlinked at close. +KEEP_REPORT_SINK_ENV = "PINEFORGE_KEEP_REPORT_SINK" + + +def _new_harness_report_sink(lib, state): + """Reuse run_json's v1 bindings, temporary owner and receipt verification.""" + docker_dir = str(REPO_ROOT / "docker") + if docker_dir not in sys.path: + sys.path.insert(0, docker_dir) + import run_json + + required = ("pf_report_sink_version", "strategy_set_report_sink_v1", + "strategy_report_sink_receipt_v1") + if any(not hasattr(lib, name) for name in required): + raise RuntimeError( + f"{REPORT_SINK_ENV}=1 requires the report sink ABI; rebuild the strategy library") + run_json._bind_report_sink(lib) + if int(lib.pf_report_sink_version()) != 1: + raise RuntimeError("strategy library has an incompatible report sink ABI") + return run_json._TemporaryReportSink(lib, state) + + +def _harness_sink_equity_count(lib, state) -> int: + """Save the engine's E count while its handle is live, not from the sink.""" + from run_json import ReportSinkReceiptC + + receipt = ReportSinkReceiptC() + receipt.struct_size, receipt.version = ctypes.sizeof(receipt), 1 + if (lib.strategy_report_sink_receipt_v1(state, ctypes.byref(receipt)) != 0 + or receipt.complete != 1 or receipt.count_e < 0): + raise RuntimeError("the engine's report sink equity receipt is incomplete") + return int(receipt.count_e) + + +def _harness_sink_trade(record) -> dict: + trade = record.as_dict() + del trade["n"] + trade["is_long"] = trade.pop("side") == "long" + return trade + + +class _SinkRows: + """Repeatable rows backed by the verified sink, with no retained history.""" + + def __init__(self, sink, tag, decode, window=None): + self.sink, self.tag, self.decode, self.window = sink, tag, decode, window + self._length = sink.counts.get(tag, 0) if window is None else None + + def __iter__(self): + time_key = "timestamp" if self.tag == "X" else "entry_time" + count = 0 + for record in self.sink.records(self.tag): + row = self.decode(record) + if (self.window is None + or self.window[0] <= int(row[time_key]) <= self.window[1]): + count += 1 + yield row + self._length = count + + def __len__(self): + if self._length is None: + self._length = sum(1 for _ in self) + return self._length + + def __getitem__(self, index): + if not isinstance(index, int): + raise TypeError("sink rows support integer indexing only") + if index < 0: + index += len(self) + for position, row in enumerate(self): + if position == index: + return row + raise IndexError("sink row index out of range") + + +class _SinkReport(dict): + """A report whose row views remain valid until close(), or context exit.""" + + def __init__(self, values, sink, receipt_count_e: int): + super().__init__(values) + self.sink = sink + self.receipt_count_e = receipt_count_e + + def close(self): + self.sink.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + +class _SinkTraceNames: + """First-use trace names, deduplicated on disk rather than in a set.""" + + def __init__(self, sink): + self.sink = sink + + def __iter__(self): + with tempfile.TemporaryFile("w+", encoding="ascii", newline="\n") as names: + for record in self.sink.records("X"): + name = record.as_dict()["name"] + wire = json.dumps(name) + "\n" + names.seek(0) + if not any(line == wire for line in names): + names.seek(0, os.SEEK_END) + names.write(wire) + yield name + + +def _report_from_sink(report, sink, receipt_count_e: int) -> dict: + if any((report.trades, report.trades_len, + report.equity_curve, report.equity_curve_len, + report.trace, report.trace_len, + report.broker_state_hash, report.broker_state_hash_len)): + raise RuntimeError("a report with a sink must contain summary scalars only") + # This reads scalars and any remaining native trace-name table only. + result = _SinkReport(_report_to_dict(report), sink, receipt_count_e) + count = 0 + for record in sink.records("T"): + count += 1 + trade = record.as_dict() + if trade["n"] != count: + raise RuntimeError("the report sink trade rows are out of report order") + for field in ("entry_price", "exit_price", "qty", "pnl", "pnl_pct", + "max_runup", "max_drawdown", "commission"): + if trade[field] is None: + raise RuntimeError(f"non-finite report sink trade field: {field}") + result["trades"] = _SinkRows(sink, "T", _harness_sink_trade) + result["trace"] = _SinkRows(sink, "X", lambda record: record.as_dict()) + if not result["trace_names"] and sink.counts.get("X", 0): + # X carries the name itself. Preserve first-use order when the native + # summary omits the trace-name table along with its trace array. + result["trace_names"] = _SinkTraceNames(sink) + result["broker_state_hash"] = _SinkRows( + sink, "H", lambda record: int(record.values[0], 16)) + result["equity_curve_time_ms"] = _SinkRows( + sink, "E", lambda record: record.values[0]) + return result + + +def _probe_id(strategy_dir: Path) -> str: + """The name a probe goes by in the sweep's lines: its directory's + basename (the slug PINEFORGE_REQUESTS_ROOT uses), without whitespace so + the canary line splits on spaces.""" + return "_".join(strategy_dir.name.split()) or "-" + + +def _keep_report_sink(probe: str, environ=None) -> bool: + """Test-only artifact switch: does this probe keep its sink file at close? + + ``PINEFORGE_KEEP_REPORT_SINK=all`` keeps every probe's file; any other + non-empty value is a comma-separated list of probe ids (_probe_id) and + keeps exactly those probes' files. Probes run as separate processes and + nothing is counted across them, so a sweep that wants five artifacts + names the first five probes of its own order. Unset or empty: no file is + kept, and close() unlinks every sink as it always does. + """ + value = (os.environ if environ is None else environ).get( + KEEP_REPORT_SINK_ENV, "").strip() + return bool(value) and ( + value == "all" or probe in {part.strip() for part in value.split(",")}) + + +def _sink_canary(probe: str, report: _SinkReport) -> None: + """Prove this probe read a real, non-empty, complete sink, in one line. + + Prints ``sink_canary: probe= bytes= records= trades= + equity= ok`` on stdout. ``bytes`` is the sink's size now; ``records`` + and ``equity`` are the T and E rows a fresh strict reader (own offset, + whole-file SHA-256 check) decodes now. T must equal ReportC.total_trades; + E must equal receipt.count_e saved from the live engine, independently + of the sink. Summary-only ReportC.equity_curve_len is zero with a sink. + No E == script_bars_processed relation is asserted. + + This is Strategy.run's ordinary/observed sink path. Native selected + windows have an anchor and separate E publication rules; Strategy.run + does not admit them, and their E-to-bars relation is deliberately + skipped here. CSV emit-window filtering leaves the full sink intact. + A missing, non-regular, empty, unreadable or short sink prints these + same fields with ``FAIL `` and raises RuntimeError. A run that + never installed a sink prints no canary, so the sweep must match exactly + one successful line to each probe it ran. Call before close() unlinks. + """ + sink, trades = report.sink, int(report["total_trades"]) + size = records = equity = 0 + reason = None + try: + size = os.stat(sink.path).st_size + if not os.path.isfile(sink.path): + reason = "the sink path is not a regular file" + elif size <= 0: + reason = "the sink file is empty" + else: + counts = {"T": 0, "E": 0} + for record in sink.records(): + if record.tag in counts: + counts[record.tag] += 1 + # Publish counts only after the strict pass checked its trailer. + records, equity = counts["T"], counts["E"] + if records != trades: + reason = f"{records} T records decoded for {trades} trades" + elif equity != report.receipt_count_e: + reason = (f"{equity} E records decoded for " + f"{report.receipt_count_e} engine equity points") + except Exception as error: + reason = f"the sink cannot be read back: {error}" + verdict = "ok" if reason is None else "FAIL " + " ".join(str(reason).split()) + print(f"sink_canary: probe={probe} bytes={size} records={records} " + f"trades={trades} equity={equity} {verdict}") + if sink.keep: + print(f"sink_kept: probe={probe} path={sink.path}") + if reason is not None: + raise RuntimeError(f"sink canary failed for probe {probe}: {reason}") def _run_failure_text(lib, state) -> str | None: @@ -2484,6 +2719,16 @@ def run(self, bars_csv: Path, params: dict | None = None, Read-only: the run itself is unchanged. Omitted (not an empty list) when the .so predates the exports; ``derived`` / ``position`` are omitted when it predates only the task-8 accessors. + + With ``PINEFORGE_REPORT_SINK=1``, require the v1 sink ABI and return + a context-managed dict whose trades, trace, broker hashes and equity + timestamps are repeatable disk-backed views. Use ``with strat.run(...) + as report:`` or call ``report.close()`` after consuming these views. + The CLI closes each report, including a discarded emit-window rerun. + ``on_report`` still sees the native report, which has summary scalars + only in this mode. The sink is a named owner-only temporary file that + every view re-opens by its real path (own offset), the same on Linux + and macOS; close() unlinks it. """ source_feed_sha256 = None if preloaded_bars is not None: @@ -2557,7 +2802,11 @@ def run(self, bars_csv: Path, params: dict | None = None, state = self.lib.strategy_create(params_json) report = ReportC() + sink = None + result_ready = False try: + if os.environ.get(REPORT_SINK_ENV) == "1": + sink = _new_harness_report_sink(self.lib, state) # Strategy-property overrides (commission/slippage/pyramiding/...) # via the per-strategy ``strategy_set_override`` export. Keys and # value grammar: codegen emit_top.py set_strategy_override — @@ -2741,7 +2990,12 @@ def run(self, bars_csv: Path, params: dict | None = None, ), self.lib, state) if on_report is not None: on_report(report) - result = _report_to_dict(report) + if sink is not None: + sink.verify(self.lib, state, report) + result = _report_from_sink( + report, sink, _harness_sink_equity_count(self.lib, state)) + else: + result = _report_to_dict(report) incarnation_accessor = getattr( self.lib, "strategy_closed_trade_entry_incarnation", None) # ABI v4 (task 9): entry_id / exit_id / exit_comment / close_cause @@ -2756,7 +3010,9 @@ def run(self, bars_csv: Path, params: dict | None = None, self.lib, "strategy_closed_trade_exit_comment", None) close_cause_accessor = getattr( self.lib, "strategy_closed_trade_close_cause", None) - for i, trade in enumerate(result["trades"]): + # T already owns this metadata, including rows retired by the + # ledger. Accessors are only meaningful for the array path. + for i, trade in enumerate(result["trades"] if sink is None else ()): trade["entry_incarnation"] = ( int(incarnation_accessor(state, i)) if incarnation_accessor is not None else 0 @@ -2813,19 +3069,36 @@ def run(self, bars_csv: Path, params: dict | None = None, "applied": _report_terminal_quote_applied(result, report_terminal_quote, script_tf or input_tf or ""), } + result_ready = True return result finally: - self.lib.report_free(ctypes.byref(report)) - self.lib.strategy_free(state) + cleanup_ok = False + try: + try: + self.lib.report_free(ctypes.byref(report)) + finally: + self.lib.strategy_free(state) + cleanup_ok = True + finally: + # A successful result owns the file until its consumer + # finishes. On every failure, native cleanup precedes it. + if sink is not None and not (result_ready and cleanup_ok): + sink.close() def _report_terminal_quote_applied(report: dict, quote: tuple, script_tf: str) -> bool: - rows = [row for row in report["trades"] if row.get("open_at_end")] times = report["equity_curve_time_ms"] - return bool(60 < _tf_seconds(script_tf) < 86400 - and report["bar_magnifier_enabled"] and rows and times - and times[-1] == int(quote[0]) - and all(row["exit_time"] == int(quote[0]) for row in rows)) + if not (60 < _tf_seconds(script_tf) < 86400 + and report["bar_magnifier_enabled"] and times + and times[-1] == int(quote[0])): + return False + found = False + for row in report["trades"]: + if row.get("open_at_end"): + found = True + if row["exit_time"] != int(quote[0]): + return False + return found def _with_tail_bar(bars, n: int, tail: tuple): @@ -3085,7 +3358,8 @@ def _load_window_ms(csv_path: Path) -> tuple[int, int]: return first, last -def _filter_trades_to_window(trades: list[dict], window: tuple[int, int] | None) -> list[dict]: +def _filter_trades_to_window(trades: list[dict] | _SinkRows, + window: tuple[int, int] | None) -> list[dict] | _SinkRows: """Keep trades whose entry (fill) time lies inside ``window`` (inclusive). main() hands it the REPORT window. With a TV tape in use that is @@ -3101,6 +3375,8 @@ def _filter_trades_to_window(trades: list[dict], window: tuple[int, int] | None) gates.""" if window is None: return trades + if isinstance(trades, _SinkRows): + return _SinkRows(trades.sink, trades.tag, trades.decode, window) start_ms, end_ms = window return [ t for t in trades @@ -3108,7 +3384,7 @@ def _filter_trades_to_window(trades: list[dict], window: tuple[int, int] | None) ] -def _entered_before_tv(trades: list[dict], report_start_ms: int) -> int | None: +def _entered_before_tv(trades: list[dict] | _SinkRows, report_start_ms: int) -> int | None: """The earliest entry the engine filled before ``report_start_ms`` (TV's first entry less one bar interval), or None when it filled none. @@ -3120,13 +3396,16 @@ def _entered_before_tv(trades: list[dict], report_start_ms: int) -> int | None: differently) -- and its position and P&L would carry into the trades that are reported, so main() runs such a probe again from the signal bar, as before. Reported rows start at ``report_start_ms`` either way.""" - early = [int(t["entry_time"]) for t in trades if int(t["entry_time"]) < report_start_ms] - return min(early) if early else None + return min((int(t["entry_time"]) for t in trades + if int(t["entry_time"]) < report_start_ms), default=None) -def _filter_trace_to_window(trace: list[dict], window: tuple[int, int] | None) -> list[dict]: +def _filter_trace_to_window(trace: list[dict] | _SinkRows, + window: tuple[int, int] | None) -> list[dict] | _SinkRows: if window is None: return trace + if isinstance(trace, _SinkRows): + return _SinkRows(trace.sink, trace.tag, trace.decode, window) start_ms, end_ms = window return [ e for e in trace @@ -3451,9 +3730,73 @@ def format_trade_qty(qty: float) -> str: # the tape's own mark of the same lot — the date-only tape's Signal "Open". ENGINE_RANGE_END_COLUMN = "Engine range-end" ENGINE_RANGE_END_OPEN = "open" +_ENGINE_TRADES_HEADER = [ + "Trade #", "Type", "Date and time", "Price", "Qty", + "Net PnL", "Net PnL %", + "Favorable excursion USD", "Adverse excursion USD", + "Cumulative PnL", "Engine entry incarnation", + ENGINE_RANGE_END_COLUMN, +] + + +def _engine_trade_csv_rows(n, trade, cumulative): + direction = "long" if trade["is_long"] else "short" + # TV convention: adverse excursion is negative (or zero). + adverse = -trade["max_drawdown"] or 0.0 + for side, time_key, price_key in ( + (f"Exit {direction}", "exit_time", "exit_price"), + (f"Entry {direction}", "entry_time", "entry_price"), + ): + yield [ + n, side, + _fmt_time_utc(trade[time_key]), + f"{trade[price_key]:.6f}", + format_trade_qty(float(trade["qty"])), + f"{trade['pnl']:.6f}", + f"{trade['pnl_pct']:.4f}", + f"{trade['max_runup']:.6f}", + f"{adverse:.6f}", + f"{cumulative:.6f}", + str(int(trade.get("entry_incarnation", 0))) + if (side.startswith("Entry") + and int(trade.get("entry_incarnation", 0)) > 0) else "", + ENGINE_RANGE_END_OPEN + if (side.startswith("Exit") and trade.get("open_at_end")) else "", + ] + +def _write_sink_engine_trades_csv(trades, path): + """Fold forward, then emit formatted trade pairs backward in O(1) RAM. -def write_engine_trades_csv(trades: list[dict], path: Path) -> None: + Store the *formatted* cumulative PnL of each kept trade. Subtracting PnL + while walking backward would change floating-point rounding. The fixed + width offset/length index and CSV pairs live only in temporary files. + Exhausting the verified reader and staging every row precedes opening + the destination, so a corrupt/truncated sink cannot publish a CSV. + """ + index_record = struct.Struct(" None: """Emit one row per trade *side* (exit then entry) in reverse-chronological order — byte-for-byte alignable with TradingView's `trades.csv` export. @@ -3483,6 +3826,9 @@ def write_engine_trades_csv(trades: list[dict], path: Path) -> None: engines before it wrote, project the cell as ``rangeEnd`` (blank -> null, "open" -> "open") and reject any other spelling, so the column is a bump of that contract on both sides, not a free addition.""" + if isinstance(trades, _SinkRows): + _write_sink_engine_trades_csv(trades, path) + return cum_pnls: dict[int, float] = {} running = 0.0 for n, t in enumerate(trades, 1): @@ -3491,38 +3837,9 @@ def write_engine_trades_csv(trades: list[dict], path: Path) -> None: with path.open("w", newline="") as f: w = csv.writer(f) - w.writerow([ - "Trade #", "Type", "Date and time", "Price", "Qty", - "Net PnL", "Net PnL %", - "Favorable excursion USD", "Adverse excursion USD", - "Cumulative PnL", "Engine entry incarnation", - ENGINE_RANGE_END_COLUMN, - ]) + w.writerow(_ENGINE_TRADES_HEADER) for n, t in reversed(list(enumerate(trades, 1))): - direction = "long" if t["is_long"] else "short" - cum = cum_pnls[n] - # TV convention: adverse excursion is negative (or zero). - adverse = -t["max_drawdown"] or 0.0 - for side, time_key, price_key in ( - (f"Exit {direction}", "exit_time", "exit_price"), - (f"Entry {direction}", "entry_time", "entry_price"), - ): - w.writerow([ - n, side, - _fmt_time_utc(t[time_key]), - f"{t[price_key]:.6f}", - format_trade_qty(float(t["qty"])), - f"{t['pnl']:.6f}", - f"{t['pnl_pct']:.4f}", - f"{t['max_runup']:.6f}", - f"{adverse:.6f}", - f"{cum:.6f}", - str(int(t.get("entry_incarnation", 0))) - if (side.startswith("Entry") - and int(t.get("entry_incarnation", 0)) > 0) else "", - ENGINE_RANGE_END_OPEN - if (side.startswith("Exit") and t.get("open_at_end")) else "", - ]) + w.writerows(_engine_trade_csv_rows(n, t, cum_pnls[n])) @@ -4263,6 +4580,11 @@ def main() -> int: "$PINEFORGE_RELEASE_IMAGE or ghcr .../pineforge-release:latest).") args = ap.parse_args() + with ExitStack() as resources: + return _main_run(args, resources) + + +def _main_run(args, resources) -> int: strategy_dir = args.strategy_dir.resolve() out_path = (args.output.resolve() if args.output else strategy_dir / "engine_trades.csv") @@ -4483,15 +4805,18 @@ def run_engine(start_ms): print(f" magnifier: {plan.status}: {plan.detail}") def run_engine(start_ms): - return strat.run(run_ohlcv, params=params, - trace_enabled=args.trace_json is not None, - trade_start_time_ms=start_ms, - realtime_tail_horizon=args.realtime_tail, - probe_suppress_tail=args.probe_suppress_tail, - path_order=args.path_order, - broker_state_hash_recording=args.broker_state_hash, - dump_book=args.dump_book is not None, - **run_kwargs) + result = strat.run(run_ohlcv, params=params, + trace_enabled=args.trace_json is not None, + trade_start_time_ms=start_ms, + realtime_tail_horizon=args.realtime_tail, + probe_suppress_tail=args.probe_suppress_tail, + path_order=args.path_order, + broker_state_hash_recording=args.broker_state_hash, + dump_book=args.dump_book is not None, + **run_kwargs) + if isinstance(result, _SinkReport): + resources.callback(result.close) + return result report = run_engine(trade_start_ms) if (signal_start_ms is not None and trade_start_ms is not None and trade_start_ms < signal_start_ms and report_window is not None): @@ -4506,6 +4831,11 @@ def run_engine(start_ms): trade_start_ms = signal_start_ms emit_window = (signal_start_ms, emit_window[1]) report = run_engine(trade_start_ms) + if isinstance(report, _SinkReport): + # The final report only: a discarded rerun's sink is always unlinked. + # Set before the outputs, so a probe that fails after this point still + # leaves its file when the switch names it. + report.sink.keep = _keep_report_sink(_probe_id(strategy_dir)) if run_kwargs.get("report_terminal_quote") is not None: quote_result = report.get("report_terminal_quote") or {} quote_status = "APPLIED" if quote_result.get("applied") else "INERT" @@ -4522,14 +4852,22 @@ def run_engine(start_ms): trace_to_write = _filter_trace_to_window(report["trace"], emit_window) args.trace_json.parent.mkdir(parents=True, exist_ok=True) with args.trace_json.open("w", encoding="utf-8") as f: - json.dump({ + document = { "strategy": str(strategy_dir), "ohlcv": str(run_ohlcv), "emit_window": None if emit_window is None else {"start_ms": emit_window[0], "end_ms": emit_window[1]}, "report_start_ms": None if report_window is None else report_window[0], "trace_names": report["trace_names"], "trace": trace_to_write, - }, f) + } + if isinstance(trace_to_write, _SinkRows): + from run_json import _JSONRecordArray + document["trace"] = _JSONRecordArray(lambda: iter(trace_to_write)) + document["trace_names"] = _JSONRecordArray( + lambda: iter(report["trace_names"])) + _dump_harness_sink_json(document, f) + else: + json.dump(document, f) if args.realtime_tail is not None: # Receipt: a consumer (scripts/live_flags_lane.py) asserts this line # is present in a flagged run's stdout so a future regression that @@ -4570,18 +4908,30 @@ def run_engine(start_ms): # JSON-Number, so no consumer can silently truncate a uint64 to # a JS-safe double). Invariant (Strategy.run/_report_to_dict): # len(bsh_values) == len(bsh_times) whenever recording was on. - entries = [ - {"time_ms": bsh_times[i], "hash": format(bsh_values[i], "016x")} - for i in range(len(bsh_values)) - ] + if len(bsh_times) != len(bsh_values): + raise RuntimeError("broker-state hashes and equity timestamps differ in length") + entry_count = len(bsh_values) + def entries_iter(): + # With a sink, two readers are live here at once (E and H): each + # _SinkRows pass opens its own SinkReader on the sink's real path, + # so their offsets are independent on every platform. + return ({"time_ms": timestamp, "hash": format(value, "016x")} + for timestamp, value in zip(bsh_times, bsh_values)) with bsh_path.open("w", encoding="utf-8") as f: - json.dump({ + document = { "strategy": str(strategy_dir), "ohlcv": str(run_ohlcv), "script_bars_processed": report["script_bars_processed"], - "entries": entries, - }, f) - print(f" broker-state-hash: wrote {len(entries)} entries to {bsh_path}") + "entries": None, + } + if isinstance(bsh_values, _SinkRows): + from run_json import _JSONRecordArray + document["entries"] = _JSONRecordArray(entries_iter) + _dump_harness_sink_json(document, f) + else: + document["entries"] = list(entries_iter()) + json.dump(document, f) + print(f" broker-state-hash: wrote {entry_count} entries to {bsh_path}") if args.dump_book is not None: # strat is None only under --runner docker, which sys.exit's above # when --dump-book is set. A .so predating the exports still runs @@ -4650,6 +5000,12 @@ def run_engine(start_ms): print(f" fingerprint -> {args.fingerprint_json} ({fp['digest']})") except Exception as e: print(f" fingerprint: skipped ({e})", file=sys.stderr) + if isinstance(report, _SinkReport): + # Every output is written; the sink is still open (the ExitStack + # closes it after this function returns), so the canary can still + # stat and re-read the file. A failure here fails the probe before + # the summary line below can read like a success. + _sink_canary(_probe_id(strategy_dir), report) elapsed = time.time() - started try: @@ -4668,6 +5024,28 @@ def run_engine(start_ms): return 0 +def _dump_harness_sink_json(document, destination): + """Stream diagnostic arrays with json.dump's existing default whitespace.""" + from run_json import _JSONRecordArray + + destination.write("{") + for index, (key, value) in enumerate(document.items()): + if index: + destination.write(", ") + destination.write(json.dumps(key)) + destination.write(": ") + if isinstance(value, _JSONRecordArray): + destination.write("[") + for row_index, row in enumerate(value.records()): + if row_index: + destination.write(", ") + json.dump(row, destination) + destination.write("]") + else: + json.dump(value, destination) + destination.write("}") + + if __name__ == "__main__": try: sys.exit(main()) diff --git a/src/native_calendar.cpp b/src/native_calendar.cpp index 909b61264..08a55de6e 100644 --- a/src/native_calendar.cpp +++ b/src/native_calendar.cpp @@ -1572,7 +1572,7 @@ struct SessionDayMemo::State { // session_day_from_open by the open date's day number, direct-mapped: the // days one lookup reads -- a date and its neighbours, a month of trading // dates -- never share a slot. - static constexpr std::size_t kDaySlots = 64; + static constexpr std::size_t kDaySlots = SessionDayMemo::kDayCapacity; // The local civil day of the last few whole seconds a zoned calendar was // asked about. static constexpr std::size_t kLocalSlots = 8; @@ -1583,8 +1583,14 @@ struct SessionDayMemo::State { struct DaySlot { bool used = false; + // Whether session_day_from_open answered with a day for `open_day`: + // `day` reads only then. An answer of none is kept like any other. + bool present = false; int64_t open_day = 0; - std::optional day; + // Its spans' buffer is reserved when the memo binds its calendar and + // only assigned into after, so the slot owns the same heap whichever + // day it holds. + SessionDay day; }; struct LocalSlot { bool used = false; @@ -1615,6 +1621,8 @@ struct SessionDayMemo::State { std::uint64_t interval_resolutions = 0; // Times libc's local time was read for this calendar. std::uint64_t libc_local_reads = 0; + // Session days resolved rather than answered from `days`. + std::uint64_t day_resolutions = 0; // The zone's own transitions, for cycle_certificate: a constant offset // (UTC, a fixed offset), or the transition instants of the TZif file libc @@ -1701,13 +1709,24 @@ const SessionDay* memo_session_day_from_open(const SessionCalendar& cal, MemoState::DaySlot& slot = m.days[static_cast(open_day) % MemoState::kDaySlots]; if (!slot.used || slot.open_day != open_day) { - // Resolved before the slot is touched, so a throw leaves it intact. - auto day = session_day_from_open(cal, civil_from_days(open_day)); - slot.day = std::move(day); + // Resolved before the slot is touched, and the slot is a miss while it + // is rewritten, so a throw leaves no half-written day behind. + const auto day = session_day_from_open(cal, civil_from_days(open_day)); + ++m.day_resolutions; + slot.used = false; + slot.present = day.has_value(); + if (day) { + // Copy-assigned, so its spans keep the buffer bind reserved, which + // holds any day of this calendar (session_day_span_bound): no + // reallocation. + slot.day = *day; + } else { + slot.day.spans.clear(); + } slot.open_day = open_day; slot.used = true; } - return slot.day ? &*slot.day : nullptr; + return slot.present ? &slot.day : nullptr; } const SessionDay* memo_session_day_with_trading(const SessionCalendar& cal, @@ -2210,10 +2229,45 @@ std::uint64_t SessionDayMemo::libc_local_reads() const noexcept { return state_ ? state_->libc_local_reads : 0; } +std::uint64_t SessionDayMemo::day_resolutions() const noexcept { + return state_ ? state_->day_resolutions : 0; +} + +std::size_t SessionDayMemo::retained_days() const noexcept { + std::size_t held = 0; + if (state_) { + for (const auto& slot : state_->days) { + if (slot.used) ++held; + } + } + return held; +} + +std::size_t SessionDayMemo::retained_span_capacity() const noexcept { + std::size_t capacity = 0; + if (state_) { + for (const auto& slot : state_->days) capacity += slot.day.spans.capacity(); + } + return capacity; +} + +// The loop in session_day_from_open pushes at most one span per declared +// window for each civil date n in {-1, 0, 1} from the open date, or the one +// span of an all-day calendar; merge_spans and the resolution that follows it +// keep or drop spans and never add one. +std::size_t session_day_span_bound(const SessionCalendar& calendar) noexcept { + if (!calendar.valid()) return 0; + return calendar.all_day() ? std::size_t{1} : 3 * calendar.windows().size(); +} + SessionDayMemo::State& SessionDayMemo::bind(const SessionCalendar& calendar) { auto fresh = std::make_unique(); fresh->valid = calendar.valid(); fresh->utc = is_utc_zone(calendar.timezone()); + // Every day slot's span buffer, now and for good (State::DaySlot): what + // the memo owns is the same after its first lookup as after its millionth. + const std::size_t spans = session_day_span_bound(calendar); + for (auto& slot : fresh->days) slot.day.spans.reserve(spans); state_ = std::move(fresh); calendar_ = &calendar; return *state_; diff --git a/src/native_calendar_memo.hpp b/src/native_calendar_memo.hpp index 1f3105688..2c5d7d1b4 100644 --- a/src/native_calendar_memo.hpp +++ b/src/native_calendar_memo.hpp @@ -41,12 +41,26 @@ // A memo serves one calendar value. The owner calls reset() whenever the // calendar it passes is rebuilt or reassigned; a memo handed a calendar at // another address forgets everything first (the address is identity only, -// never dereferenced). It holds a bounded number of days (kDaySlots), +// never dereferenced). It holds a bounded number of days (kDayCapacity), // intervals and civil dates, and is not thread-safe: one owner, one thread, // like the calendar value it serves. +// +// What it owns does not depend on what it was asked (CR-150 lane L19n). The +// day slots are direct-mapped by the open date's day number: a day asked after +// kDayCapacity others replaces the one in its slot and is resolved again if it +// is asked again. That moves no answer. A lookup reads a day by address and +// copies what it keeps before it reads another, so none waits on a day being +// held, and a resolution is a function of the calendar value and the open date +// (up to the glibc fold above, where the memo keeps the first answer and a day +// evicted takes the one it resolves again). Every slot's span buffer is +// reserved once, when the memo binds its calendar, to the most spans a session +// day of that calendar has (session_day_span_bound), and is only assigned into +// after: the heap is the same after the first lookup as after the millionth, +// whatever days they asked (retained_days, retained_span_capacity). #include +#include #include #include #include @@ -61,6 +75,9 @@ class SessionDayMemo { // Defined in native_calendar.cpp, the only translation unit that reads it. struct State; + // The session days the memo keeps at once. + static constexpr std::size_t kDayCapacity = 64; + SessionDayMemo() noexcept; ~SessionDayMemo(); SessionDayMemo(SessionDayMemo&&) noexcept; @@ -86,6 +103,17 @@ class SessionDayMemo { // How many times, since then, it read libc's local time for its calendar's // zone. The same kind of cost reading. std::uint64_t libc_local_reads() const noexcept; + // How many session days it has resolved since then, as opposed to answered + // from the days it keeps: a day asked again after kDayCapacity others is + // resolved again. The same kind of cost reading. + std::uint64_t day_resolutions() const noexcept; + // Its footprint, for the tests that pin it: how many of its day slots hold + // an answer now (a day, or that the calendar has none), and the capacity, + // in spans, of their buffers together. The first never passes + // kDayCapacity; the second is reserved when the calendar is bound and does + // not move after it. Both are 0 for a memo that has bound no calendar. + std::size_t retained_days() const noexcept; + std::size_t retained_span_capacity() const noexcept; private: State& bind(const SessionCalendar& calendar); @@ -95,6 +123,15 @@ class SessionDayMemo { std::unique_ptr state_; }; +// An upper bound on the spans of one session day of `calendar`, a function of +// the calendar value alone: a day merges at most one span per declared window +// for each of the three civil dates its cycle reads (the date before the open +// date, the open date and the date after), or is the one span of an all-day +// calendar, and merging and resolving never add one. 0 for a calendar that is +// not valid, which resolves no day. The memo reserves it in every day slot, and +// the consumer in every certified cycle it keeps. +std::size_t session_day_span_bound(const SessionCalendar& calendar) noexcept; + // Value for value what interval_containing(calendar, script_tf, input_tf, ms) // answers. std::optional interval_containing(const SessionCalendar& calendar, diff --git a/src/native_execution_consumer.cpp b/src/native_execution_consumer.cpp index 7113ae540..68ae64dc4 100644 --- a/src/native_execution_consumer.cpp +++ b/src/native_execution_consumer.cpp @@ -8446,25 +8446,25 @@ NativeExecutionConsumer::certify_cycle(int64_t ms) const { } } try { + const std::size_t day_span_bound = native_calendar::session_day_span_bound(calendar_); + for (CertifiedCycle& slot : certified_cycles_) slot.spans.reserve(day_span_bound); auto certificate = native_calendar::cycle_certificate(calendar_, ms, calendar_memo_); - CertifiedCycle entry; + CertifiedCycle& kept = certified_cycles_[certified_cycle_next_]; + kept.certified = false; // a half-written entry is read the exact way if (certificate) { - entry.certified = true; - entry.origin_ms = certificate->origin_ms; - entry.next_origin_ms = certificate->next_origin_ms; - entry.ordinal = certificate->ordinal; - entry.bucket_ms = native_calendar::fixed_bucket_ms(script_tf_); - entry.spans = std::move(certificate->spans); + kept.spans.assign(certificate->spans.begin(), certificate->spans.end()); + kept.ordinal = certificate->ordinal; + kept.bucket_ms = native_calendar::fixed_bucket_ms(script_tf_); + kept.origin_ms = certificate->origin_ms; + kept.next_origin_ms = certificate->next_origin_ms; + kept.certified = true; } else { - constexpr int64_t kDayMs = 86'400'000; - entry.origin_ms = ms; - entry.next_origin_ms = ms <= std::numeric_limits::max() - kDayMs - ? ms + kDayMs : std::numeric_limits::max(); + kept.origin_ms = ms; + kept.next_origin_ms = ms <= std::numeric_limits::max() - kMsPerDay + ? ms + kMsPerDay : std::numeric_limits::max(); } - CertifiedCycle& kept = certified_cycles_[certified_cycle_next_]; certified_cycle_last_ = certified_cycle_next_; certified_cycle_next_ = (certified_cycle_next_ + 1) % certified_cycles_.size(); - kept = std::move(entry); return kept.certified ? &kept : nullptr; } catch (...) { return nullptr; diff --git a/src/native_order.cpp b/src/native_order.cpp index a7f3788f4..dd4f3aaf0 100644 --- a/src/native_order.cpp +++ b/src/native_order.cpp @@ -1238,13 +1238,22 @@ std::optional WorkingRequestCore::canonical_cohort_origin( return RequestHandle{identity_, *root}; } -std::size_t WorkingRequestCore::cohort_index(CohortHandle cohort) const noexcept { - const auto it = std::lower_bound( - cohorts_.begin(), cohorts_.end(), cohort, - [](const CohortRoster& roster, CohortHandle value) { return roster.handle < value; }); - return it != cohorts_.end() && it->handle == cohort - ? static_cast(it - cohorts_.begin()) - : cohorts_.size(); +WorkingRequestCore::CohortView WorkingRequestCore::cohorts() const noexcept { + return CohortView(this); +} + +WorkingRequestCore::CohortView::const_iterator::reference +WorkingRequestCore::CohortView::const_iterator::operator*() const noexcept { + assert(core_ && index_ < core_->next_cohort_handle_ - 1); + const auto handle = static_cast(index_) + 1; + const auto found = core_->cohorts_.find(handle); + if (found != core_->cohorts_.end()) return found->second; + empty_.handle = CohortHandle{handle}; + return empty_; +} + +bool WorkingRequestCore::cohort_exists(CohortHandle cohort) const noexcept { + return cohort.value > 0 && cohort.value < next_cohort_handle_; } CohortHandle WorkingRequestCore::cohort_open() { @@ -1254,7 +1263,6 @@ CohortHandle WorkingRequestCore::cohort_open() { throw std::overflow_error("native cohort handle exhausted"); } const CohortHandle handle{next_cohort_handle_}; - cohorts_.push_back(CohortRoster{handle, {}}); ++next_cohort_handle_; if (!bump_epoch()) throw std::overflow_error("native working-request epoch exhausted"); return handle; @@ -1267,17 +1275,21 @@ void WorkingRequestCore::cohort_add(CohortHandle cohort, RequestHandle origin) { receipt.operation = CohortReceiptOperation::Add; receipt.cohort = cohort; receipt.origin = origin; - const std::size_t index = cohort_index(cohort); - if (cohort.value == 0 || index == cohorts_.size()) { + if (!cohort_exists(cohort)) { receipt.status = CohortReceiptStatus::InvalidHandle; } else if (!issued(origin)) { receipt.status = CohortReceiptStatus::UnknownOrigin; } else if (!find_live(origin)) { receipt.status = CohortReceiptStatus::TerminalOrigin; } else if (const auto canonical = canonical_cohort_origin(origin)) { - auto& origins = cohorts_[index].origins; - const auto where = std::lower_bound(origins.begin(), origins.end(), *canonical, handle_less); - if (where == origins.end() || *where != *canonical) origins.insert(where, *canonical); + const auto found = cohorts_.find(cohort.value); + if (found == cohorts_.end()) { + cohorts_.emplace(cohort.value, CohortRoster{cohort, {*canonical}}); + } else { + auto& origins = found->second.origins; + const auto where = std::lower_bound(origins.begin(), origins.end(), *canonical, handle_less); + if (where == origins.end() || *where != *canonical) origins.insert(where, *canonical); + } receipt.status = CohortReceiptStatus::Applied; } else { receipt.status = CohortReceiptStatus::UnknownOrigin; @@ -1293,13 +1305,16 @@ void WorkingRequestCore::cohort_remove(CohortHandle cohort, RequestHandle origin receipt.operation = CohortReceiptOperation::Remove; receipt.cohort = cohort; receipt.origin = origin; - const std::size_t index = cohort_index(cohort); - if (cohort.value == 0 || index == cohorts_.size()) { + if (!cohort_exists(cohort)) { receipt.status = CohortReceiptStatus::InvalidHandle; } else if (const auto canonical = canonical_cohort_origin(origin)) { - auto& origins = cohorts_[index].origins; - const auto where = std::lower_bound(origins.begin(), origins.end(), *canonical, handle_less); - if (where != origins.end() && *where == *canonical) origins.erase(where); + const auto found = cohorts_.find(cohort.value); + if (found != cohorts_.end()) { + auto& origins = found->second.origins; + const auto where = std::lower_bound(origins.begin(), origins.end(), *canonical, handle_less); + if (where != origins.end() && *where == *canonical) origins.erase(where); + if (origins.empty()) cohorts_.erase(found); + } receipt.status = CohortReceiptStatus::Applied; } else { receipt.status = CohortReceiptStatus::UnknownOrigin; @@ -1310,11 +1325,12 @@ void WorkingRequestCore::cohort_remove(CohortHandle cohort, RequestHandle origin bool WorkingRequestCore::cohort_contains( CohortHandle cohort, const RequestHandle& opening) const { - const std::size_t index = cohort_index(cohort); - if (cohort.value == 0 || index == cohorts_.size()) return false; + if (!cohort_exists(cohort)) return false; const auto root = canonical_cohort_origin(opening); if (!root) return false; - const auto& origins = cohorts_[index].origins; + const auto found = cohorts_.find(cohort.value); + if (found == cohorts_.end()) return false; + const auto& origins = found->second.origins; return std::binary_search(origins.begin(), origins.end(), *root, handle_less); } @@ -1784,7 +1800,7 @@ std::optional WorkingRequestCore::validate_request( || live_count != cohort.size()) return RequestRejectReason::InvalidOwner; } else if (const auto* bind = std::get_if(&request.owner)) { if (!host_sized || host_sized->kind != HostSizedKind::Close - || bind->cohort.value == 0 || cohort_index(bind->cohort) == cohorts_.size()) { + || !cohort_exists(bind->cohort)) { return RequestRejectReason::InvalidOwner; } } else { diff --git a/src/source/pine_adapter.cpp b/src/source/pine_adapter.cpp index ecb08240d..93b4de849 100644 --- a/src/source/pine_adapter.cpp +++ b/src/source/pine_adapter.cpp @@ -22,6 +22,7 @@ #include "pine_checked_arith.hpp" #include "pine_value_admission.hpp" +#include #include #include #include @@ -912,14 +913,14 @@ std::uint64_t PlacementTable::command_sequence_for(const std::string& source_id, std::uint64_t paired = absent; const Minima* const indexes[] = {&sequence_index_, &erased_minima_}; for (const Minima* index : indexes) { - const auto minima = index->find(source_id); - if (minima == index->end()) continue; + const auto minima = index->lookup(source_id); + if (!minima) continue; known = true; - any = std::min(any, minima->second.any); - const auto pair = minima->second.by_from_entry.find(from_entry); - if (pair == minima->second.by_from_entry.end()) continue; + any = std::min(any, *minima); + const auto pair = index->lookup(source_id, from_entry); + if (!pair) continue; paired_known = true; - paired = std::min(paired, pair->second); + paired = std::min(paired, *pair); } if (!known) return absent; if (paired_known && paired != absent) return paired; @@ -1186,7 +1187,7 @@ void PineExecutionAdapter::initialize_l4c_policy(PlacementSnapshot& snapshot, std::string_view first_lot_id; std::uint64_t first_lot_incarnation = 0; - for (const auto& id : cohort_order_) { + for (const auto& id : cohorts_by_id_.resident_ids()) { const auto cohort = cohorts_by_id_.find(id); if (cohort == cohorts_by_id_.end()) continue; for (const auto& opening : cohort->second.opened) { @@ -1540,9 +1541,9 @@ class AdapterLookupIndex final : public detail::PineRunCache { // exists), and an opening row's `opening` and `is_long` are never written // after it is remembered, so the two flags only ever turn on and a fold // over the entries appended since the last lookup keeps them exact. Keyed - // by the address of the cohort's facts, whose node the cohort map never - // moves; the map is only cleared by reset_for_run, which clears this - // index too. + // by stable resident facts; retire_origin_prefixes removes the cache row + // before those facts can spill. A restored empty roster answers from its + // erased sides, and reset_for_run clears the entire index. struct CohortSides { std::size_t folded = 0; bool opened_long = false; @@ -1784,7 +1785,7 @@ bool PineExecutionAdapter::cohort_opened_on_side(const CohortFacts& cohort, } return false; }; - if (cohort.origins.size() <= kIndexedFrom) return walk(); + if (cohort.origins.size() <= kIndexedFrom || cohort.origins.members().empty()) return walk(); AdapterLookupIndex* index = adapter_lookup_index(bound_consumer(), this, run_counter_); if (!index) return walk(); try { @@ -1904,7 +1905,7 @@ bool PineExecutionAdapter::immediate_close_placed_on(std::int32_t bar) const noe bool PineExecutionAdapter::bracket_belongs_to_reversal( const PlacementSnapshot& bracket, - const PlacementSnapshot& reversal) const noexcept { + const PlacementSnapshot& reversal) const { if (bracket.projection_position_side == reversal.projection_position_side) return true; const auto cohort = cohorts_by_id_.find(bracket.from_entry); @@ -2193,8 +2194,8 @@ void PineExecutionAdapter::purge_brackets_after_applied_reversal( } for (auto& cohort : cohorts_by_id_) { if (cohort_opened_on_side(cohort.second, prior_long)) { - cohort.second.opened.clear(); - cohort.second.live_units_by_origin.clear(); + ::pineforge::detail::clear_exposure_storage(cohort.second.opened); + ::pineforge::detail::clear_exposure_storage(cohort.second.live_units_by_origin); } } } @@ -2404,7 +2405,6 @@ void PineExecutionAdapter::reset_for_run() { index->clear(); admission_journal.reset(); cohorts_by_id_.clear(); - cohort_order_.clear(); placement_.clear(); live_by_source_key_.clear(); bracket_families_.clear(); @@ -3029,7 +3029,6 @@ native_order::CohortHandle PineExecutionAdapter::cohort_for(const SourceId& id) } const auto result = facts.handle; cohorts_by_id_.emplace(id, std::move(facts)); - cohort_order_.push_back(id); return result; } @@ -4227,8 +4226,8 @@ bool PineExecutionAdapter::short_seed_context_is_live() const noexcept { const auto physical = detail::run_position(*host_); if (!(physical.signed_units < 0.0) || physical.lot_count != 1U) return false; const double held = std::abs(physical.signed_units); - for (const auto& id : cohort_order_) { - const auto cohort = cohorts_by_id_.find(id); + for (const auto& id : cohorts_by_id_.resident_ids()) { + const auto cohort = cohorts_by_id_.find_resident(id); if (cohort == cohorts_by_id_.end()) continue; if (cohort_exposure_for(id) == held) return true; } @@ -5020,7 +5019,7 @@ std::optional PineExecutionAdapter::submit_or_repla const auto source = placement_.at(accepted->incarnation).source_id; const auto cohort = cohort_for(source); host.cohort_add(cohort, *accepted); - cohorts_by_id_.at(source).origins.push_back(*accepted); + require_cohort(source)->second.origins.push_back(*accepted); } const auto& accepted_snapshot = placement_.at(accepted->incarnation); if (coof_recalc_active_ @@ -5042,7 +5041,7 @@ std::vector PineExecutionAdapter::openings_for(cons : found->second.opened; } -bool PineExecutionAdapter::from_entry_filled_this_cycle(const SourceId& id) const noexcept { +bool PineExecutionAdapter::from_entry_filled_this_cycle(const SourceId& id) const { // The source analogue of ab9714be cycle_filled_entry_ids_: the id opened // in the live position cycle (reset when the book goes flat). const auto found = cohorts_by_id_.find(id); @@ -5051,7 +5050,7 @@ bool PineExecutionAdapter::from_entry_filled_this_cycle(const SourceId& id) cons } double PineExecutionAdapter::cohort_exposure_for(const SourceId& id) const noexcept { - const auto found = cohorts_by_id_.find(id); + const auto found = cohorts_by_id_.find_resident(id); if (found == cohorts_by_id_.end()) return 0.0; double total = 0.0; // `live_units_by_origin` is a lookup table. Quantities must follow the @@ -5102,8 +5101,8 @@ std::optional PineExecutionAdapter::from_entry_units( int PineExecutionAdapter::source_entry_slot_count() const noexcept { std::size_t total = 0; - for (const auto& id : cohort_order_) { - const auto found = cohorts_by_id_.find(id); + for (const auto& id : cohorts_by_id_.resident_ids()) { + const auto found = cohorts_by_id_.find_resident(id); if (found != cohorts_by_id_.end()) total += found->second.opened.size(); } return total > static_cast(std::numeric_limits::max()) @@ -5922,6 +5921,7 @@ void PineExecutionAdapter::consume_cohort_units( if (unit->second == 0.0) facts.live_units_by_origin.erase(unit); if (!(remaining > 0.0)) break; } + ::pineforge::detail::release_empty_exposure_capacity(facts.live_units_by_origin); } void PineExecutionAdapter::consume_closed_trade_rows( @@ -5930,6 +5930,7 @@ void PineExecutionAdapter::consume_closed_trade_rows( auto& host = require_host(); const auto settle_slot = [&](CohortFacts& cohort, const Trade& trade, bool drained) { + ::pineforge::detail::release_empty_exposure_capacity(cohort.live_units_by_origin); if (!cause) return; const bool bracket = cause->family == PineOrderFamily::ExitLimit || cause->family == PineOrderFamily::ExitStop @@ -5956,6 +5957,7 @@ void PineExecutionAdapter::consume_closed_trade_rows( return opening.incarnation == trade.entry_incarnation; }), cohort.opened.end()); + ::pineforge::detail::release_empty_exposure_capacity(cohort.opened); bracket_shadowed_openings_.erase(trade.entry_incarnation); }; for (std::size_t row = 0; row < event.closed_trade_count; ++row) { @@ -6315,7 +6317,7 @@ void PineExecutionAdapter::retire_in_position_exits_at_flat( // UINT64_MAX is returned only when the exit id has no placement row at all. // ab9714be pine_fills.cpp:664-670: candidate exit fills tie-break by order creation sequence created_seq std::uint64_t PineExecutionAdapter::command_sequence_for_exit( - const SourceId& exit_id, const SourceId& from_entry) const noexcept { + const SourceId& exit_id, const SourceId& from_entry) const { // The same-bar exit sort asks this on every script bar while its tail // group persists; the table answers from its per-id index instead of a // scan over every placement ever retained (quadratic over a long run). @@ -6371,11 +6373,11 @@ void PineExecutionAdapter::note_exit_rank_key(const SourceId& id) { // Past 26 keys the table needs 64 buckets, which the pin does not reach; // the peak keeps it there, so the ids beyond need no record. if (exit_rank_keys_.size() > 26 - || std::find(exit_rank_keys_.begin(), exit_rank_keys_.end(), id) - != exit_rank_keys_.end()) { + || exit_rank_keys_.find(id).has_value()) { + // The recorded population remains capped at 27 keys, as before. return; } - exit_rank_keys_.push_back(id); + exit_rank_keys_.insert(id, static_cast(exit_rank_keys_.size())); const std::uint64_t size = id.size(); exit_rank_keys_digest_ = fnv_append(exit_rank_keys_digest_, &size, sizeof(size)); exit_rank_keys_digest_ = fnv_append(exit_rank_keys_digest_, id.data(), id.size()); @@ -6414,21 +6416,23 @@ std::optional> PineExecutionAdapter::global_exit_chil if (capacity > 32 || buckets_for(exit_rank_alt_peak_) != capacity) return std::nullopt; std::vector bucket(exit_rank_keys_.size()); std::vector load(capacity, 0); - for (std::size_t age = 0; age < exit_rank_keys_.size(); ++age) { + bool supported = true; + exit_rank_keys_.for_each_inserted([&](const auto& key, auto age) { std::uint32_t hash = 0; - if (!java_string_hash(exit_rank_keys_[age], hash)) return std::nullopt; + if (!java_string_hash(key, hash)) { supported = false; return; } bucket[age] = (hash ^ (hash >> 16)) & (capacity - 1); // A bin of eight would make java.util.HashMap treeify or resize. The // count is of entry ids only: a transient close key landing in a bin // of seven would do the same, which no tape reaches. - if (++load[bucket[age]] >= 8) return std::nullopt; - } + if (++load[bucket[age]] >= 8) supported = false; + }); + if (!supported) return std::nullopt; std::vector ages; ages.reserve(ids.size()); for (const auto& id : ids) { - const auto found = std::find(exit_rank_keys_.begin(), exit_rank_keys_.end(), id); - if (found == exit_rank_keys_.end()) return std::nullopt; - const auto age = static_cast(found - exit_rank_keys_.begin()); + const auto found = exit_rank_keys_.find(id); + if (!found) return std::nullopt; + const auto age = static_cast(exit_rank_keys_.get(*found)); if (capacity > 16) { for (const auto other : ages) if (bucket[other] == bucket[age]) return std::nullopt; @@ -6966,8 +6970,8 @@ bool PineExecutionAdapter::coof_current_fill_was_forced_waypoint() const noexcep if (!coof_recalc_active_) return false; const auto point = detail::callback_point(require_host()); if (!point) return false; - for (const auto& id : cohort_order_) { - const auto cohort = cohorts_by_id_.find(id); + for (const auto& id : cohorts_by_id_.resident_ids()) { + const auto cohort = cohorts_by_id_.find_resident(id); if (cohort == cohorts_by_id_.end()) continue; for (const auto& opening : cohort->second.opened) { const auto units = cohort->second.live_units_by_origin.find( @@ -7164,7 +7168,7 @@ std::size_t PineExecutionAdapter::accepted_entries_in_cycle( bool is_long, const SourceId& id) const { std::size_t accepted_in_cycle = 0; std::vector cohort_ids; - cohort_ids.reserve(cohorts_by_id_.size()); + cohort_ids.reserve(cohorts_by_id_.resident_size()); for (const auto& row : cohorts_by_id_) cohort_ids.push_back(row.first); std::sort(cohort_ids.begin(), cohort_ids.end()); for (const auto& cohort_id : cohort_ids) { @@ -8371,7 +8375,7 @@ void PineExecutionAdapter::place_entry(const SourceId& id, bool is_long, double std::vector carried_brackets; std::vector carried_ids; std::vector cohort_ids; - cohort_ids.reserve(cohorts_by_id_.size()); + cohort_ids.reserve(cohorts_by_id_.resident_size()); for (const auto& row : cohorts_by_id_) cohort_ids.push_back(row.first); std::sort(cohort_ids.begin(), cohort_ids.end()); for (const auto& cohort_id : cohort_ids) { @@ -11217,10 +11221,10 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en if (!std::isnan(qty) && live_exit_units > internal::kQtyEpsilon) reissue_percent = std::min(qty, live_exit_units) / live_exit_units * 100.0; const bool partial_reissue = reissue_percent < 100.0 - internal::kFullPercentEps; - if (const auto consumed = consumed_partial_exit_cycles_.find(partial_exit_key); - consumed != consumed_partial_exit_cycles_.end() && partial_reissue + if (const auto consumed = consumed_partial_exit_cycles_.lookup(partial_exit_key); + consumed && partial_reissue && live_exit_units > internal::kQtyEpsilon - && consumed->second == current_position_cycle_) { + && *consumed == current_position_cycle_) { return; } if (partial_reissue && exit_tombstoned(exit_id, from_entry)) return; @@ -15991,7 +15995,7 @@ native_order::ExecutionTerms PineExecutionAdapter::resolve_source_terms( const bool keep_mc_close_surplus = [&]() { if (!source.affordability_keep_mc_close_surplus) return false; bool receipt_origin_live = false; - for (const auto& cohort_id : cohort_order_) { + for (const auto& cohort_id : cohorts_by_id_.resident_ids()) { const auto cohort = cohorts_by_id_.find(cohort_id); if (cohort == cohorts_by_id_.end()) continue; const auto units = cohort->second.live_units_by_origin.find( @@ -17849,7 +17853,7 @@ bool PineExecutionAdapter::source_margin_rounded_tie_veto() const { return false; } bool rounded_tie_opening = false; - for (const auto& cohort_id : cohort_order_) { + for (const auto& cohort_id : cohorts_by_id_.resident_ids()) { const auto cohort = cohorts_by_id_.find(cohort_id); if (cohort == cohorts_by_id_.end()) continue; for (const auto& origin : cohort->second.opened) { @@ -24774,30 +24778,30 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& // the new one. The generic settlement event owns that fact; mirror // it into every source cohort so a later reuse of an old entry id // cannot reserve against already-closed exposure. - std::vector closed_cohorts; - for (const auto& cohort : cohorts_by_id_) { - const bool nested_new_side_opening = std::any_of( - live_handles_.begin(), live_handles_.end(), - [&](const native_order::RequestHandle& handle) { - const auto pending = placement_.find(handle.incarnation); - return pending != placement_.end() - && pending->second.opening - && pending->second.family == PineOrderFamily::Entry - && pending->second.source_id == cohort.first - && pending->second.is_long == (live_position > 0.0); - }); - if ((!placement_snapshot - || cohort.first != placement_snapshot->source_id) - && !nested_new_side_opening) { - closed_cohorts.push_back(cohort.first); + std::vector preserved_cohorts; + if (placement_snapshot) preserved_cohorts.push_back(placement_snapshot->source_id); + for (const auto& handle : live_handles_) { + const auto pending = placement_.find(handle.incarnation); + if (pending != placement_.end() + && pending->second.opening + && pending->second.family == PineOrderFamily::Entry + && pending->second.is_long == (live_position > 0.0)) { + preserved_cohorts.push_back(pending->second.source_id); } } - std::sort(closed_cohorts.begin(), closed_cohorts.end()); - for (const auto& id : closed_cohorts) - cancel_exit_orders_for_full_close(id); + std::sort(preserved_cohorts.begin(), preserved_cohorts.end()); + // An empty, spilled cohort can still own queued exits. The old map + // cancelled those too, in sorted id order. Snapshot only the live + // exemptions, then stream every logical id without restoring facts. + cohorts_by_id_.for_each_sorted_id([&](const SourceId& id) { + if (!std::binary_search(preserved_cohorts.begin(), preserved_cohorts.end(), id)) { + cancel_exit_orders_for_full_close(id); + } + }); + for (auto& cohort : cohorts_by_id_) { - cohort.second.opened.clear(); - cohort.second.live_units_by_origin.clear(); + ::pineforge::detail::clear_exposure_storage(cohort.second.opened); + ::pineforge::detail::clear_exposure_storage(cohort.second.live_units_by_origin); } // ab9714be pine_strategy_host.cpp:244 / :261 // reset_source_open_position_ledgers_before_book clears the close @@ -24867,9 +24871,9 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& || family == PineOrderFamily::ExitTrail); }); if (!sibling_leg_still_live) { - consumed_partial_exit_cycles_[placement_snapshot->source_id + "\x1f" - + placement_snapshot->from_entry] = - current_position_cycle_; + consumed_partial_exit_cycles_.insert_or_assign( + placement_snapshot->source_id + "\x1f" + placement_snapshot->from_entry, + current_position_cycle_); } } if (placement_snapshot && placement_snapshot->family == PineOrderFamily::Order @@ -24956,7 +24960,8 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& live_by_source_key_[key_for(replacement(event.handle().incarnation))] = handle; } { - auto& facts = cohorts_by_id_[placement_snapshot->source_id]; + const auto cohort = require_cohort(placement_snapshot->source_id); + auto& facts = cohort->second; facts.cycle = event.cycle_after; if (std::find(facts.opened.begin(), facts.opened.end(), event.handle()) == facts.opened.end()) facts.opened.push_back(event.handle()); @@ -25362,7 +25367,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& const auto physical = detail::run_position(require_host()); std::uint64_t sole_live_entry_incarnation = 0; std::size_t live_entry_count = 0; - for (const auto& id : cohort_order_) { + for (const auto& id : cohorts_by_id_.resident_ids()) { const auto cohort = cohorts_by_id_.find(id); if (cohort == cohorts_by_id_.end()) continue; for (const auto& opening : cohort->second.opened) { @@ -25398,7 +25403,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& std::uint64_t sole_entry_incarnation = 0; int live_entry_origins = 0; if (physical.signed_units != 0.0 && physical.lot_count == 1U) { - for (const auto& cohort_id : cohort_order_) { + for (const auto& cohort_id : cohorts_by_id_.resident_ids()) { const auto cohort = cohorts_by_id_.find(cohort_id); if (cohort == cohorts_by_id_.end()) continue; for (const auto& origin : cohort->second.opened) { @@ -25508,7 +25513,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& if (placement_snapshot && placement_snapshot->opening) { const double physical_exposure = std::abs(detail::run_position(require_host()).signed_units); - for (const auto& id : cohort_order_) { + for (const auto& id : cohorts_by_id_.resident_ids()) { const double exposure = cohort_exposure_for(id); const bool paired_source_transaction = placement_snapshot->paired_flat_market_candidate @@ -25615,7 +25620,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& } if (placement_snapshot && placement_snapshot->family == PineOrderFamily::CloseAll && event.closed_units > 0.0) { - for (auto& cohort : cohorts_by_id_) cohort.second.live_units_by_origin.clear(); + for (auto& cohort : cohorts_by_id_) ::pineforge::detail::clear_exposure_storage(cohort.second.live_units_by_origin); } if (placement_snapshot && placement_snapshot->family == PineOrderFamily::Margin && event.closed_units > 0.0 @@ -25739,8 +25744,8 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& pending_margin_revivals_.clear(); for (auto& cohort : cohorts_by_id_) { if (!cohort.second.opened.empty()) ended_sources.push_back(cohort.first); - cohort.second.opened.clear(); - cohort.second.live_units_by_origin.clear(); + ::pineforge::detail::clear_exposure_storage(cohort.second.opened); + ::pineforge::detail::clear_exposure_storage(cohort.second.live_units_by_origin); } const auto handles = live_handles_; for (const auto& handle : handles) { @@ -26858,7 +26863,7 @@ int PendingIntentView::level_resolved(int index) const noexcept { native_order::RequestHandle handle; if (!owner_->projected_pending_at(index, snapshot, handle) || !snapshot) return -1; if (snapshot->from_entry.empty()) return 1; - const auto cohort = owner_->cohorts_by_id_.find(snapshot->from_entry); + const auto cohort = owner_->cohorts_by_id_.find_resident(snapshot->from_entry); return cohort != owner_->cohorts_by_id_.end() && !cohort->second.opened.empty() ? 1 : 0; } @@ -28052,6 +28057,7 @@ void OriginRoster::retire_before(std::size_t global) { // row reader. Explicit names in live/queued placements also pin an origin // whose own row has gone. No persistent release bitmap or lookup is added. void PineExecutionAdapter::retire_origin_prefixes() { + cohorts_by_id_.spill(); if (detail::retain_retired_rows()) return; // The front row alone can pin each retained window. Check before building // named, so a resting bracket/stop pair allocates no retirement scratch. @@ -28134,7 +28140,121 @@ void PineExecutionAdapter::retire_origin_prefixes() { ++through; } origins.retire_before(through); + if (origins.members().empty()) { + if (auto* index = adapter_lookup_index(consumer, this, run_counter_, false)) + index->cohort_sides.erase(&facts); + } } + cohorts_by_id_.spill(); + if (auto* index = adapter_lookup_index(consumer, this, run_counter_, false)) { + auto& sides = index->cohort_sides; + if (sides.bucket_count() / 4 > sides.size()) sides.rehash(sides.size()); + } +} + +} // namespace pineforge::source + +namespace pineforge::source { + +OriginRoster OriginRoster::from_retired(std::size_t count, std::uint64_t digest) { + OriginRoster result; + result.base_ = count; + result.digest_ = digest; + return result; +} + +PineExecutionAdapter::CohortFacts::Record +PineExecutionAdapter::CohortFacts::retired_record() const noexcept { + static_assert(sizeof(Record) == 40, "retired cohort scratch record"); + return {handle.value, cycle, origins.size(), origins.digest(), erased_opening_sides}; +} + +PineExecutionAdapter::CohortFacts +PineExecutionAdapter::CohortFacts::restore_record(const Record& row) { + if (row.handle == 0 || static_cast( + static_cast(row.origin_count)) != row.origin_count + || row.erased_sides > 3) { + throw coded( + RunFailureCode::report_sink_failed, {}, + ::pineforge::detail::ScratchIoError(EIO, "scratch cohort invalid record").what()); + } + CohortFacts facts; + facts.handle.value = row.handle; + facts.cycle = row.cycle; + facts.origins = OriginRoster::from_retired( + static_cast(row.origin_count), row.origin_digest); + facts.erased_opening_sides = static_cast(row.erased_sides); + return facts; +} + +bool PineExecutionAdapter::CohortFacts::spillable() const noexcept { + return origins.members().empty() && opened.empty() && live_units_by_origin.empty(); +} + +// CR-150 L19l merge of parts A and C: the one place that binds the scratch +// directory of every adapter-owned L19l table -- part A's cohort and +// entry-rank tables, part C's placement-minima and consumed-cycle tables -- +// the way set_successor_roots_directory binds every table of the kernel core. +// ScratchFile and IdPairStore accept the directory they already hold, so +// calling both public names (this and set_auxiliary_tables_directory) with +// one directory is harmless. A different directory once a table's file is +// open throws std::logic_error from that table: bind after reset_for_run and +// before the first id, as the declaration says. +void PineExecutionAdapter::set_id_tables_directory(const std::string& directory) { + cohorts_by_id_.set_directory(directory); + exit_rank_keys_.set_directory(directory); + placement_.set_auxiliary_tables_directory(directory); + consumed_partial_exit_cycles_.set_directory(directory); +} + +PineExecutionAdapter::CohortTable::iterator +PineExecutionAdapter::require_cohort(const SourceId& id) { + auto found = cohorts_by_id_.find(id); + if (found == cohorts_by_id_.end()) { + throw coded(RunFailureCode::engine_invariant, {}, + "accepted opening has no source cohort"); + } + return found; +} + +} // namespace pineforge::source + +// CR-150 L19l-C. Keep the new storage wiring together, away from part A's +// cohort / entry-id changes in this translation unit. These paths and file +// offsets describe storage of existing state, not additional broker state. +namespace pineforge::source { + +void PlacementTable::set_auxiliary_tables_directory(const std::string& directory) { + sequence_index_.set_directory(directory); + erased_minima_.set_directory(directory); +} + +void PlacementTable::fixture_use_memory_auxiliary_tables() { + if (high_water_ != 0) + throw coded( + RunFailureCode::engine_invariant, {}, + "placement auxiliary fixture backend changed after use"); + sequence_index_.fixture_use_memory_reference(); + erased_minima_.fixture_use_memory_reference(); +} + +std::uint64_t PlacementTable::auxiliary_disk_bytes() const noexcept { + return sequence_index_.disk_bytes() + erased_minima_.disk_bytes(); +} + +// Part C's name for the merged binding: set_id_tables_directory above binds +// the placement minima and consumed-cycle tables together with part A's tables. +void PineExecutionAdapter::set_auxiliary_tables_directory(const std::string& directory) { + set_id_tables_directory(directory); +} + +void PineExecutionAdapter::fixture_use_memory_auxiliary_tables() { + placement_.fixture_use_memory_auxiliary_tables(); + consumed_partial_exit_cycles_.fixture_use_memory_reference(); +} + +std::uint64_t PineExecutionAdapter::fixture_auxiliary_disk_bytes() const noexcept { + return placement_.auxiliary_disk_bytes() + consumed_partial_exit_cycles_.disk_bytes(); } } // namespace pineforge::source diff --git a/src/source/pine_state_hash.cpp b/src/source/pine_state_hash.cpp index d8ec4c4cf..74d3afecd 100644 --- a/src/source/pine_state_hash.cpp +++ b/src/source/pine_state_hash.cpp @@ -304,11 +304,11 @@ void source::PineExecutionAdapter::hash_state(BrokerStateHashSink& f) const { for (const auto& pair : staged_.inputs) input_keys.push_back(pair.first); std::sort(input_keys.begin(), input_keys.end()); f.u(input_keys.size()); for (const auto& key : input_keys) { f.s(key); f.s(staged_.inputs.at(key)); } - std::vector cohort_keys; - for (const auto& pair : cohorts_by_id_) cohort_keys.push_back(pair.first); - std::sort(cohort_keys.begin(), cohort_keys.end()); f.u(cohort_keys.size()); - for (const auto& key : cohort_keys) { - const auto& cohort = cohorts_by_id_.at(key); + // Replay every logical cohort in the same lexicographic order as before, + // including retired summaries, without materializing an id-sized vector. + f.u(cohorts_by_id_.size()); + for (const auto& row : cohorts_by_id_.all_sorted()) { + const auto& key = row.first; const auto& cohort = row.second; f.s(key); f.u(cohort.handle.value); f.i(cohort.cycle); // v4: the origin roster is append-only; its members fold once each, // when appended (OriginRoster::digest). @@ -324,8 +324,8 @@ void source::PineExecutionAdapter::hash_state(BrokerStateHashSink& f) const { f.u(incarnation); f.d(cohort.live_units_by_origin.at(incarnation)); } } - f.u(cohort_order_.size()); - for (const auto& key : cohort_order_) f.s(key); + f.u(cohorts_by_id_.size()); + cohorts_by_id_.for_each_id([&](const auto& key) { f.s(key); }); // v4: the rows the table still holds, in incarnation order, and what the // erased ones left: the high water, how many, and a running digest of // their identities (PlacementTable::erased_digest). diff --git a/tests/cr150.d/core-cr150-l19l.cmake b/tests/cr150.d/core-cr150-l19l.cmake new file mode 100644 index 000000000..3be0193d6 --- /dev/null +++ b/tests/cr150.d/core-cr150-l19l.cmake @@ -0,0 +1,8 @@ +list(APPEND TEST_SOURCES test_disk_id_table) +pineforge_add_unit_test(test_disk_id_table) +list(APPEND TEST_SOURCES test_id_cohort_storage) +pineforge_add_unit_test(test_id_cohort_storage) +if(PINEFORGE_BUILD_SOURCE_LAYER) + list(APPEND TEST_SOURCES test_id_adapter_storage) + pineforge_add_unit_test(test_id_adapter_storage) +endif() diff --git a/tests/cr150.d/core-cr150-l19lb.cmake b/tests/cr150.d/core-cr150-l19lb.cmake new file mode 100644 index 000000000..99f64defd --- /dev/null +++ b/tests/cr150.d/core-cr150-l19lb.cmake @@ -0,0 +1,3 @@ +# CR-150 L19l part B: nonempty kernel rosters and reference/heap proof. +list(APPEND TEST_SOURCES test_kernel_cohort_rosters) +pineforge_add_unit_test(test_kernel_cohort_rosters) diff --git a/tests/cr150.d/core-cr150-l19lc.cmake b/tests/cr150.d/core-cr150-l19lc.cmake new file mode 100644 index 000000000..6f9b1f0ca --- /dev/null +++ b/tests/cr150.d/core-cr150-l19lc.cmake @@ -0,0 +1,9 @@ +# CR-150 L19l part C: auxiliary disk tables and the real adapter differential. +if(UNIX) + list(APPEND TEST_SOURCES test_id_pair_store) + pineforge_add_unit_test(test_id_pair_store) +endif() +if(PINEFORGE_BUILD_SOURCE_LAYER AND UNIX) + list(APPEND TEST_SOURCES test_adapter_auxiliary_disk) + pineforge_add_unit_test(test_adapter_auxiliary_disk) +endif() diff --git a/tests/cr150.d/core-cr150-l19m.cmake b/tests/cr150.d/core-cr150-l19m.cmake new file mode 100644 index 000000000..f82b54aa9 --- /dev/null +++ b/tests/cr150.d/core-cr150-l19m.cmake @@ -0,0 +1,7 @@ +# CR-150 L19m: the census's empty exposure capacity in real settlements. +if(PINEFORGE_BUILD_SOURCE_LAYER AND UNIX) + list(APPEND TEST_SOURCES test_adapter_growth_storage) + pineforge_add_unit_test(test_adapter_growth_storage) + list(APPEND TEST_SOURCES test_adapter_reversal_storage) + pineforge_add_unit_test(test_adapter_reversal_storage) +endif() diff --git a/tests/cr150.d/core-cr150-l19n.cmake b/tests/cr150.d/core-cr150-l19n.cmake new file mode 100644 index 000000000..e73f2d67a --- /dev/null +++ b/tests/cr150.d/core-cr150-l19n.cmake @@ -0,0 +1,5 @@ +# CR-150 L19N: the kernel calendar memo and the consumer's certified-cycle ring own a fixed heap. +list(APPEND TEST_SOURCES test_native_calendar_memo_bound) +pineforge_add_unit_test(test_native_calendar_memo_bound) +list(APPEND TEST_SOURCES test_native_certified_cycle_bound) +pineforge_add_unit_test(test_native_certified_cycle_bound) diff --git a/tests/cr150.d/core-cr150-l9c.cmake b/tests/cr150.d/core-cr150-l9c.cmake new file mode 100644 index 000000000..94f5ec46c --- /dev/null +++ b/tests/cr150.d/core-cr150-l9c.cmake @@ -0,0 +1 @@ +add_test(NAME test_run_strategy_sink_l9c COMMAND ${Python3_EXECUTABLE} -B -m pytest -q -p no:cacheprovider ${PROJECT_SOURCE_DIR}/tests/test_run_strategy_sink.py) diff --git a/tests/heap_census.py b/tests/heap_census.py index 0e4c4426a..4abd3e96a 100644 --- a/tests/heap_census.py +++ b/tests/heap_census.py @@ -110,7 +110,7 @@ def walk(index: int, path: tuple[str, ...], parent_indent: int) -> tuple[int, in ("L19a", ("WorkingRequestCore::index_committed",)), ("L19l", ("PineExecutionAdapter::cohort_for", "CohortFacts", "CohortRoster", "PineExecutionAdapter::note_exit_rank_key", "PlacementTable::fold_sequence", - "PlacementTable::SequenceMinima", + "IdPairStore<", ("PineExecutionAdapter::on_applied", "basic_string +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pineforge; +constexpr double kNa = std::numeric_limits::quiet_NaN(); +constexpr std::int64_t kStart = 1736121600000LL; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +bool same_bits(double left, double right) { + std::uint64_t a = 0, b = 0; + std::memcpy(&a, &left, sizeof(a)); + std::memcpy(&b, &right, sizeof(b)); + return a == b; +} +struct ScratchDirectory { + std::string path = "/tmp/pineforge-auxiliary-test-XXXXXX"; + ScratchDirectory() { + require(::mkdtemp(path.data()) != nullptr, "create adapter test directory"); + } + ~ScratchDirectory() { ::rmdir(path.c_str()); } +}; + +// Keep the old nested in-memory minima as the reference, independently of +// the store's optional fixture backend. Query retained rows by scanning. +class PlacementReference { + struct Minima { + std::uint64_t any = std::numeric_limits::max(); + std::unordered_map by_from_entry; + }; + using Index = std::unordered_map; + +public: + void insert(std::uint64_t incarnation, const source::PlacementSnapshot& row) { + rows_.emplace(incarnation, row); + } + void erase(std::uint64_t incarnation) { + const auto found = rows_.find(incarnation); + require(found != rows_.end(), "reference row exists before erase"); + fold(erased_, found->second); + rows_.erase(found); + } + void replace(std::uint64_t incarnation, const source::PlacementSnapshot& row) { + rows_.at(incarnation) = row; + } + std::uint64_t sequence(const std::string& id, const std::string& from) const { + Index index = erased_; + for (const auto& row : rows_) fold(index, row.second); + const auto found = index.find(id); + if (found == index.end()) return std::numeric_limits::max(); + const auto paired = found->second.by_from_entry.find(from); + if (paired != found->second.by_from_entry.end() + && paired->second != std::numeric_limits::max()) + return paired->second; + return found->second.any; + } + +private: + static void fold(Index& index, const source::PlacementSnapshot& row) { + auto& minima = index[row.source_id]; + minima.any = std::min(minima.any, row.command_sequence); + const auto inserted = minima.by_from_entry.emplace(row.from_entry, row.command_sequence); + if (!inserted.second) + inserted.first->second = std::min(inserted.first->second, row.command_sequence); + } + std::map rows_; + Index erased_; +}; + +void placement_minima_match_old_nested_reference() { + ScratchDirectory scratch; + source::PlacementTable table; + table.set_auxiliary_tables_directory(scratch.path); + PlacementReference reference; + const auto query = [&](const std::string& id, const std::string& from) { + require(table.command_sequence_for(id, from) == reference.sequence(id, from), + "placement minima match retained scan plus old erased nested index"); + }; + for (std::uint64_t i = 1; i <= 512; ++i) { + source::PlacementSnapshot row; + row.source_id = "exit-" + std::to_string(i % 31); + row.from_entry = "unique-entry-" + std::to_string(i); + row.command_sequence = 10000 - i; + table.try_emplace(i, row); + reference.insert(i, row); + if (i % 2 != 0) query(row.source_id, row.from_entry); // fold before erase + require(table.erase(i), "erase a real placement row"); + reference.erase(i); + table.compact(); + query(row.source_id, row.from_entry); + query(row.source_id, "absent-from-entry"); + } + source::PlacementSnapshot row; + row.source_id = "replace-id"; + row.from_entry = ""; + row.command_sequence = 3; + table.try_emplace(513, row); + reference.insert(513, row); + query(row.source_id, row.from_entry); + row.source_id = "replacement-id"; + row.from_entry = "new-from-entry"; + row.command_sequence = std::numeric_limits::max(); + reference.replace(513, row); + table.replace(513, std::move(row)); + query("replace-id", ""); + query("replacement-id", "new-from-entry"); // present UINT64_MAX + query("exit-1", "unique-entry-1"); // erased minima survive a rebuild + require(table.erase(513), "erase replacement row"); + reference.erase(513); + row = {}; + row.source_id = "replacement-id"; + row.from_entry = "other-entry"; + row.command_sequence = 7; + table.try_emplace(514, row); + reference.insert(514, row); + query("replacement-id", "new-from-entry"); // max pair falls back to any + query("replacement-id", "other-entry"); + query("replacement-id", "unknown"); + require(table.auxiliary_disk_bytes() > 0, "placement witness uses real scratch files"); + table.clear(); + require(table.auxiliary_disk_bytes() == 0 + && table.command_sequence_for("exit-1", "unique-entry-1") + == std::numeric_limits::max(), "clear forgets all old minima"); +} + +class UniqueEntryHost final : public source::PineStrategyHost { +public: + UniqueEntryHost(int cycles, int period, bool long_ids, bool memory, const std::string& directory) + : cycles_(cycles), period_(period), long_ids_(long_ids) { + adapter_.reset_for_run(); + adapter_.set_auxiliary_tables_directory(directory); + if (memory) adapter_.fixture_use_memory_auxiliary_tables(); + source::PineStrategyConfig config; + config.initial_capital = 1000000.0; + config.default_qty_type = static_cast(QtyType::FIXED); + config.default_qty_value = 4.0; + config.pyramiding = 1; + config.process_orders_on_close = true; + configure_pine_strategy(config); + set_syminfo_timezone("UTC"); + set_syminfo_session("24x7"); + set_syminfo_mintick(0.25); + set_broker_state_hash_recording(true); + } + + void on_source_bar(const Bar&) override { + const int bar = pine_bar_index(); + const int cycle = bar / 6; + if (cycle >= cycles_) return; + const auto ordinal = period_ == 0 ? cycle : cycle % period_; + const auto id = std::string(long_ids_ ? "long-unique-entry-" : "E") + + std::to_string(ordinal); + const auto exit_id = cycle % 3 == 0 ? "shared-exit" : "half-" + id; + if (bar % 6 == 0) strategy_entry(id, true, kNa, kNa, 4.0); + if (bar % 6 == 1 || bar % 6 == 3) + strategy_exit(exit_id, id, 100.5, 99.5, kNa, kNa, kNa, 50.0); + if (bar % 6 == 4 && live_position_size() > 0.0) strategy_close(id); + // Exercise lazy retained-row indexing as well as erased minima. + // Observations never steer the strategy or add a broker hash field. + sequences_.push_back(adapter_.command_sequence_for_exit(exit_id, id)); + fallbacks_.push_back(adapter_.command_sequence_for_exit(exit_id, "unbound")); + } + + const std::vector& hashes() const { return broker_state_hashes_; } + const std::vector& sequences() const { return sequences_; } + const std::vector& fallbacks() const { return fallbacks_; } + std::uint64_t auxiliary_disk_bytes() const { return adapter_.fixture_auxiliary_disk_bytes(); } + std::uint64_t sequence(const std::string& id, const std::string& from) const { + return adapter_.command_sequence_for_exit(id, from); + } + void clear_observations() { sequences_.clear(); fallbacks_.clear(); } + void reset_to_memory_auxiliary_tables() { + adapter_.reset_for_run(); + adapter_.fixture_use_memory_auxiliary_tables(); + clear_observations(); + } + +private: + int cycles_; + int period_; + bool long_ids_; + std::vector sequences_, fallbacks_; +}; + +std::vector tape(int cycles) { + std::vector bars; + bars.reserve(static_cast(cycles * 6 + 8)); + for (int i = 0; i < cycles * 6 + 8; ++i) + bars.push_back({100.0, 101.0, 99.0, 100.0, 1000.0, kStart + i * 60000LL}); + return bars; +} + +void compare_trade(const Trade& a, const Trade& b) { + require(a.entry_time == b.entry_time && a.exit_time == b.exit_time + && a.entry_bar_index == b.entry_bar_index && a.exit_bar_index == b.exit_bar_index + && a.entry_id == b.entry_id && a.exit_id == b.exit_id + && a.entry_comment == b.entry_comment && a.exit_comment == b.exit_comment + && a.entry_incarnation == b.entry_incarnation && a.is_long == b.is_long + && a.open_at_end == b.open_at_end && a.exit_from_bracket == b.exit_from_bracket + && a.close_cause == b.close_cause, + "disk/reference trade identity and metadata match"); + require(same_bits(a.entry_price, b.entry_price) && same_bits(a.exit_price, b.exit_price) + && same_bits(a.qty, b.qty) && same_bits(a.pnl, b.pnl) && same_bits(a.pnl_pct, b.pnl_pct) + && same_bits(a.commission, b.commission) && same_bits(a.max_runup, b.max_runup) + && same_bits(a.max_drawdown, b.max_drawdown), + "disk/reference trade doubles match bit for bit"); +} + +void differential(int cycles, int period, bool long_ids) { + ScratchDirectory scratch; + UniqueEntryHost disk(cycles, period, long_ids, false, scratch.path); + UniqueEntryHost memory(cycles, period, long_ids, true, scratch.path); + const auto bars = tape(cycles); + for (int run = 0; run < 2; ++run) { + disk.clear_observations(); + memory.clear_observations(); + disk.run(bars.data(), static_cast(bars.size()), "1", "1"); + memory.run(bars.data(), static_cast(bars.size()), "1", "1"); + require(disk.last_error().empty() && memory.last_error().empty(), + "both adapter runs complete"); + require(disk.trade_count() == cycles * 2 && disk.trade_count() == memory.trade_count(), + "each cycle closes one partial exit and the remainder"); + for (int i = 0; i < disk.trade_count(); ++i) compare_trade(disk.get_trade(i), memory.get_trade(i)); + require(!disk.hashes().empty() && disk.hashes() == memory.hashes(), + "every recorded broker hash is identical"); + require(disk.broker_state_hash() == memory.broker_state_hash(), + "final complete broker hash is identical"); + require(disk.sequences() == memory.sequences() && disk.fallbacks() == memory.fallbacks(), + "paired and fallback sequence minima agree throughout the run"); + require(disk.auxiliary_disk_bytes() > 0 && memory.auxiliary_disk_bytes() == 0, + "differential actually compares disk and in-memory tables"); + const auto first = long_ids ? "long-unique-entry-0" : "E0"; + require(disk.sequence("shared-exit", first) == memory.sequence("shared-exit", first) + && disk.sequence("shared-exit", first) != std::numeric_limits::max(), + "first exit/entry pair still finds its old row after many other ids"); + } +} + +void directory_failure_is_a_typed_run_failure() { + ScratchDirectory scratch; + UniqueEntryHost disk(4, 0, true, false, scratch.path + "/missing"); + const auto bars = tape(4); + disk.run(bars.data(), static_cast(bars.size()), "1", "1"); + require(!disk.last_error().empty(), "configured scratch failure ends the run"); + require(run_failure_value_of(disk).code == RunFailureCode::report_sink_failed, + "scratch IO failure is report_sink_failed at the public run boundary"); +} + +void fixture_backend_requires_reset_after_use() { + ScratchDirectory scratch; + source::PlacementTable table; + table.set_auxiliary_tables_directory(scratch.path); + source::PlacementSnapshot row; + row.source_id = "fixture-exit"; + row.from_entry = "fixture-entry"; + row.command_sequence = 7; + table.try_emplace(1, row); + require(table.command_sequence_for(row.source_id, row.from_entry) == row.command_sequence, + "populate the disk auxiliary index before switching backends"); + require(table.erase(1), "erase the fixture placement while keeping its history"); + bool refused = false; + try { + table.fixture_use_memory_auxiliary_tables(); + } catch (const std::logic_error& error) { + refused = classify_run_failure(error).code == RunFailureCode::engine_invariant; + } + require(refused, "even an empty used placement table requires reset before backend selection"); + require(table.command_sequence_for(row.source_id, row.from_entry) == row.command_sequence + && table.auxiliary_disk_bytes() > 0, + "refused backend selection preserves the disk minima and erased history"); + table.clear(); + table.fixture_use_memory_auxiliary_tables(); + row.command_sequence = 11; + table.try_emplace(1, row); + require(table.command_sequence_for(row.source_id, row.from_entry) == row.command_sequence + && table.auxiliary_disk_bytes() == 0, + "clear permits backend selection and removes the previous run's minima"); +} + +void host_reset_permits_backend_selection() { + ScratchDirectory scratch; + UniqueEntryHost reused(8, 0, true, false, scratch.path); + UniqueEntryHost reference(8, 0, true, true, scratch.path); + const auto bars = tape(8); + reused.run(bars.data(), static_cast(bars.size()), "1", "1"); + require(reused.last_error().empty() && reused.trade_count() > 0 + && reused.auxiliary_disk_bytes() > 0, + "use a real host's disk tables before resetting to the memory backend"); + reused.reset_to_memory_auxiliary_tables(); + require(reused.auxiliary_disk_bytes() == 0, + "reset closes every auxiliary scratch file before fixture backend selection"); + reused.run(bars.data(), static_cast(bars.size()), "1", "1"); + reference.run(bars.data(), static_cast(bars.size()), "1", "1"); + require(reused.last_error().empty() && reference.last_error().empty() + && reused.trade_count() > 0 && reused.trade_count() == reference.trade_count(), + "reset host and fresh memory host both complete the same trades"); + for (int i = 0; i < reused.trade_count(); ++i) + compare_trade(reused.get_trade(i), reference.get_trade(i)); + require(!reused.hashes().empty() && reused.hashes() == reference.hashes() + && reused.broker_state_hash() == reference.broker_state_hash() + && reused.sequences() == reference.sequences() + && reused.fallbacks() == reference.fallbacks() + && reused.auxiliary_disk_bytes() == 0 && reference.auxiliary_disk_bytes() == 0, + "backend selection after reset preserves all trades, hashes and sequence minima"); +} + +void placement_directory_failure_is_part_c_error() { + ScratchDirectory scratch; + source::PlacementTable table; + table.set_auxiliary_tables_directory(scratch.path + "/missing"); + source::PlacementSnapshot row; + row.source_id = "part-c-exit"; + row.from_entry = "part-c-entry"; + row.command_sequence = 1; + table.try_emplace(1, row); + bool failed = false; + try { + (void)table.command_sequence_for(row.source_id, row.from_entry); + } catch (const detail::IdPairStoreError& error) { + failed = error.code().value() == ENOENT + && classify_run_failure(error).code == RunFailureCode::report_sink_failed; + } + require(failed, "part C's placement minima preserve the typed missing-directory error"); + require(table.size() == 1 && table.auxiliary_disk_bytes() == 0, + "failed auxiliary fold retains its placement and publishes no disk bytes"); + table.set_auxiliary_tables_directory(scratch.path); + require(table.command_sequence_for(row.source_id, row.from_entry) == row.command_sequence, + "part C's lazy fold retries after the directory is repaired"); +} +} // namespace + +int main() { + try { + placement_minima_match_old_nested_reference(); + placement_directory_failure_is_part_c_error(); + fixture_backend_requires_reset_after_use(); + host_reset_permits_backend_selection(); + differential(256, 0, false); + differential(256, 0, true); + differential(320, 31, true); + differential(64, 1, false); + directory_failure_is_a_typed_run_failure(); + std::puts("test_adapter_auxiliary_disk: trades, hashes, recurrence and reset agree"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "test_adapter_auxiliary_disk: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_adapter_growth_storage.cpp b/tests/test_adapter_growth_storage.cpp new file mode 100644 index 000000000..163ca0341 --- /dev/null +++ b/tests/test_adapter_growth_storage.cpp @@ -0,0 +1,329 @@ +// CR-150 L19m: real settlements with pinned origin rosters, compared with +// test-local copies of the old retain-capacity operations. No hash pins. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" + +#include +#include +#include "../src/source/pine_placement_retention.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace growth_fixture { +using Handle = pineforge::native_order::RequestHandle; +using Units = std::unordered_map; +using Opened = std::vector; + +struct Probe { + bool old_implementation = false; + std::vector units; + std::vector opened; +}; +inline Probe* active = nullptr; + +template +void remember(std::vector& rows, Container& container) { + if (std::find(rows.begin(), rows.end(), &container) == rows.end()) + rows.push_back(&container); +} +inline void observe(Units& units) { + if (active && units.empty() && units.bucket_count() > Units{}.bucket_count()) + remember(active->units, units); +} +inline void observe(Opened& opened) { + if (active && opened.empty() && opened.capacity() > Opened{}.capacity()) + remember(active->opened, opened); +} + +// Before L19m the per-origin erase left the empty map/vector untouched; +// reversal/flat called clear(), which also kept their allocation. Keep those +// implementations here, outside the installed headers and shipped library. +template +void old_release_empty(Container&) {} +template +void old_clear(Container& container) { container.clear(); } +} // namespace growth_fixture + +namespace pineforge::detail { +template +void growth_fixture_release_empty(Container& container) { + growth_fixture::observe(container); + if (growth_fixture::active && growth_fixture::active->old_implementation) + growth_fixture::old_release_empty(container); + else + release_empty_exposure_capacity(container); +} +template +void growth_fixture_clear(Container& container) { + // Observe the empty container after the old clear, before any release. + if (growth_fixture::active && growth_fixture::active->old_implementation) { + growth_fixture::old_clear(container); + growth_fixture::observe(container); + } else { + clear_exposure_storage(container); + } +} +} // namespace pineforge::detail + +// Compile the actual adapter call sites into this executable, substituting +// only the two operations above. Its definitions satisfy the adapter symbols +// before the static archive is searched, so pine_adapter.cpp.o is not pulled +// from libpineforge.a. All other source/kernel objects are the normal library. +// Removing a production release call also removes this test's observation; +// the live witness then fails. No class, access specifier or state is changed. +#define release_empty_exposure_capacity growth_fixture_release_empty +#define clear_exposure_storage growth_fixture_clear +#include "../src/source/pine_adapter.cpp" +#undef clear_exposure_storage +#undef release_empty_exposure_capacity + +namespace growth_fixture { +namespace storage = pineforge::detail; +using namespace pineforge; +constexpr double kNa = std::numeric_limits::quiet_NaN(); + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct Exposure { + Opened opened; + Units units; +}; + +Handle handle(std::uint64_t incarnation) { + Handle result; + result.incarnation = incarnation; + return result; +} + +void exposure_lifecycles_match_old_containers() { + Exposure actual, old; + for (std::uint64_t cycle = 0; cycle < 128; ++cycle) { + for (std::uint64_t offset = 1; offset <= 9; ++offset) { + const auto origin = cycle * 9 + offset; + actual.opened.push_back(handle(origin)); + old.opened.push_back(handle(origin)); + actual.units[origin] += static_cast(offset); + old.units[origin] += static_cast(offset); + } + while (!old.opened.empty()) { + const auto origin = old.opened.back().incarnation; + actual.units.erase(origin); + old.units.erase(origin); + actual.opened.pop_back(); + old.opened.pop_back(); + storage::release_empty_exposure_capacity(actual.units); + storage::release_empty_exposure_capacity(actual.opened); + require(actual.units == old.units && actual.opened == old.opened, + "release preserves all quantities and complete handle identities"); + } + require(actual.units.bucket_count() == Units{}.bucket_count() + && actual.opened.capacity() == Opened{}.capacity(), + "drained exposure has the capacity of fresh containers"); + actual.opened.push_back(handle(cycle + 1)); + actual.units[cycle + 1] = 2.0; + storage::clear_exposure_storage(actual.opened); + storage::clear_exposure_storage(actual.units); + require(actual.opened.empty() && actual.units.empty(), + "clear preserves the old empty logical state"); + } +} + +struct Recording { + Recording() { + require(!global_allocation::recording, "allocation windows must not nest"); + global_allocation::live_bytes = 0; + global_allocation::recording = true; + } + ~Recording() { global_allocation::recording = false; } +}; + +std::size_t retained_closed_exposures(std::size_t ids, bool release, bool map) { + std::vector cohorts(ids); + const Handle opening = handle(1); + Recording recording; + for (auto& cohort : cohorts) { + if (map) { + cohort.units[opening.incarnation] = 1.0; + cohort.units.erase(opening.incarnation); + if (release) storage::release_empty_exposure_capacity(cohort.units); + } else { + cohort.opened.push_back(opening); + cohort.opened.erase(cohort.opened.begin()); + if (release) storage::release_empty_exposure_capacity(cohort.opened); + } + } + return global_allocation::live_bytes; +} + +void closed_id_heap_is_flat() { + for (const bool map : {false, true}) { + const auto old_low = retained_closed_exposures(10, false, map); + const auto old_high = retained_closed_exposures(10000, false, map); + const auto low = retained_closed_exposures(10, true, map); + const auto high = retained_closed_exposures(10000, true, map); + require(old_high > old_low, "old containers reproduce historical-id growth"); + require(high == low && high == 0, "empty exposure children grow with completed ids"); + } +} + +struct PinnedOrigins { + bool previous = source::detail::retain_retired_rows(); + PinnedOrigins() { source::detail::set_retain_retired_rows(true); } + ~PinnedOrigins() { source::detail::set_retain_retired_rows(previous); } +}; + +class SettlementHost final : public source::PineStrategyHost { +public: + explicit SettlementHost(int cycles) : cycles_(cycles) { + source::PineStrategyConfig config; + config.initial_capital = 1000000.0; + config.default_qty_type = static_cast(QtyType::FIXED); + config.default_qty_value = 1.0; + config.pyramiding = 2; + config.close_entries_rule_any = true; + config.process_orders_on_close = true; + configure_pine_strategy(config); + set_syminfo_timezone("UTC"); + set_syminfo_session("24x7"); + set_syminfo_mintick(0.25); + set_broker_state_hash_recording(true); + } + void on_source_bar(const Bar&) override { + const int bar = pine_bar_index(); + if (bar == 0) { + strategy_entry("keeper", true, kNa, kNa, 1.0); + return; + } + const int cycle = (bar - 1) / 4; + if (cycle >= cycles_) return; + const auto id = "settled-" + std::to_string(cycle); + if ((bar - 1) % 4 == 0) strategy_entry(id, true, kNa, kNa, 1.0); + if ((bar - 1) % 4 == 1) + strategy_exit("exit-" + id, id, 100.5, kNa, kNa, kNa, kNa, 100.0); + } + const std::vector& hashes() const { return broker_state_hashes_; } + double keeper_units() const { return adapter_.source_unclosed_qty_for("keeper"); } +private: + int cycles_; +}; + +struct RunResult { + std::vector trades; + std::vector hashes; + std::uint64_t final_hash = 0; + std::vector unit_buckets, opened_capacity; + double keeper_units = 0.0; +}; + +RunResult live_run(const std::vector& bars, int cycles, bool old) { + // Retained placement rows pin every origin, so the cohorts cannot spill. + // The keeper also prevents a flat-book clear from masking a missing + // per-cohort release. Read the saved container pointers only while both + // the host and this existing retention override are still alive. + PinnedOrigins pinned; + const auto erased_before = source::detail::retired_rows_erased(); + SettlementHost host(cycles); + Probe probe; + probe.old_implementation = old; + active = &probe; + try { + host.run(bars.data(), static_cast(bars.size()), "1", "1"); + } catch (...) { + active = nullptr; + throw; + } + active = nullptr; + require(host.last_error().empty(), "live settlement run completes"); + require(source::detail::retired_rows_erased() == erased_before, + "origin placements remain pinned throughout the run"); + require(host.trade_count() == cycles, "every distinct cohort settles its real trade"); + require(!probe.units.empty() && !probe.opened.empty(), + "real settlement reaches both production release call sites"); + RunResult result; + for (const auto* units : probe.units) { + require(units->empty(), "settled cohort retains no live provenance"); + result.unit_buckets.push_back(units->bucket_count()); + } + for (const auto* opened : probe.opened) { + require(opened->empty(), "settled cohort retains no opened handles"); + result.opened_capacity.push_back(opened->capacity()); + } + for (int i = 0; i < host.trade_count(); ++i) result.trades.push_back(host.get_trade(i)); + result.hashes = host.hashes(); + result.final_hash = host.broker_state_hash(); + result.keeper_units = host.keeper_units(); + return result; +} + +bool same_bits(double left, double right) { + return std::memcmp(&left, &right, sizeof left) == 0; +} + +void live_settlements_release_pinned_cohort_capacity() { + const int cycles = 32; + std::vector bars; + for (int i = 0; i < cycles * 4 + 8; ++i) + bars.push_back({100.0, 101.0, 99.0, 100.0, 1000.0, 1736121600000LL + i * 60000LL}); + const auto reference = live_run(bars, cycles, true); + const auto actual = live_run(bars, cycles, false); + require(actual.unit_buckets.size() == reference.unit_buckets.size() + && actual.opened_capacity.size() == reference.opened_capacity.size(), + "identical live runs visit the same settled cohort containers"); + for (std::size_t i = 0; i < actual.unit_buckets.size(); ++i) + require(actual.unit_buckets[i] == Units{}.bucket_count() + && actual.unit_buckets[i] < reference.unit_buckets[i], + "real settlement releases map buckets retained by the old implementation"); + for (std::size_t i = 0; i < actual.opened_capacity.size(); ++i) + require(actual.opened_capacity[i] == Opened{}.capacity() + && actual.opened_capacity[i] < reference.opened_capacity[i], + "real settlement releases opened capacity retained by the old implementation"); + require(!actual.hashes.empty() && actual.hashes == reference.hashes + && actual.final_hash == reference.final_hash, + "all live broker hashes equal the old implementation"); + require(actual.keeper_units > 0.0 && same_bits(actual.keeper_units, reference.keeper_units), + "the keeper stays live while other cohorts drain"); + require(actual.trades.size() == reference.trades.size(), "live trade populations match"); + for (std::size_t i = 0; i < actual.trades.size(); ++i) { + const auto& a = actual.trades[i]; + const auto& b = reference.trades[i]; + require(a.entry_id == b.entry_id && a.exit_id == b.exit_id + && a.entry_comment == b.entry_comment && a.exit_comment == b.exit_comment + && a.entry_time == b.entry_time && a.exit_time == b.exit_time + && a.entry_bar_index == b.entry_bar_index && a.exit_bar_index == b.exit_bar_index + && a.entry_incarnation == b.entry_incarnation && a.is_long == b.is_long + && a.open_at_end == b.open_at_end && a.exit_from_bracket == b.exit_from_bracket + && a.close_cause == b.close_cause, + "live trade identities and metadata match"); + require(same_bits(a.entry_price, b.entry_price) && same_bits(a.exit_price, b.exit_price) + && same_bits(a.qty, b.qty) && same_bits(a.pnl, b.pnl) + && same_bits(a.pnl_pct, b.pnl_pct) && same_bits(a.commission, b.commission) + && same_bits(a.max_runup, b.max_runup) && same_bits(a.max_drawdown, b.max_drawdown), + "live trade doubles match bit for bit"); + } +} +} // namespace growth_fixture + +int main() { + try { + growth_fixture::exposure_lifecycles_match_old_containers(); + growth_fixture::closed_id_heap_is_flat(); + growth_fixture::live_settlements_release_pinned_cohort_capacity(); + std::puts("test_adapter_growth_storage: live settlements release old exposure capacity"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "test_adapter_growth_storage: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_adapter_reversal_storage.cpp b/tests/test_adapter_reversal_storage.cpp new file mode 100644 index 000000000..06c151017 --- /dev/null +++ b/tests/test_adapter_reversal_storage.cpp @@ -0,0 +1,303 @@ +// CR-150 L19l/L19m: real reversals with storage release/eviction on and off. +// The script/tape are the Flip shape in test_adapter_lookup_index_witness. +// No pinned values: the reference keeps cohort rows and empty capacity, as +// the old implementation did, while placement retirement stays enabled. +#include +#include + +// Load every dependency before the narrowly scoped predicate substitution. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace reversal_storage_fixture { +struct Probe { + bool keep_capacity = false; + bool keep_cohorts = false; + std::size_t spills = 0; + std::size_t retained_rows = 0; + std::size_t releases = 0; + std::size_t clears = 0; +}; +inline Probe* active = nullptr; + +// Test-local copies of the old storage operations: erase kept an empty +// allocation, clear retained capacity, and an empty cohort node stayed in +// the map. None changes the placement-retirement override or logical state. +template void old_release_empty(Container&) {} +template void old_clear(Container& container) { container.clear(); } +inline bool old_retain_cohort() noexcept { return false; } + +inline bool allow_spill() noexcept { + if (active && active->keep_cohorts) { + ++active->retained_rows; + return old_retain_cohort(); + } + if (active) ++active->spills; + return true; +} +} // namespace reversal_storage_fixture + +// Only the predicate call in the storage template is substituted. Both +// modes keep the exact production class layout, public access and codec; +// the reference simply declines eviction, retaining the old map's nodes. +// Counts are taken after the real pin/spillable checks, before write/erase. +#define spillable() spillable() && ::reversal_storage_fixture::allow_spill() +#include +#undef spillable + +#include +#include "../src/source/pine_placement_retention.hpp" + +namespace pineforge::detail { +template +void reversal_fixture_release_empty(Container& container) { + if (reversal_storage_fixture::active) + ++reversal_storage_fixture::active->releases; + if (reversal_storage_fixture::active + && reversal_storage_fixture::active->keep_capacity) + reversal_storage_fixture::old_release_empty(container); + else + release_empty_exposure_capacity(container); +} +template +void reversal_fixture_clear(Container& container) { + if (reversal_storage_fixture::active) + ++reversal_storage_fixture::active->clears; + if (reversal_storage_fixture::active + && reversal_storage_fixture::active->keep_capacity) + reversal_storage_fixture::old_clear(container); + else + clear_exposure_storage(container); +} +} // namespace pineforge::detail + +// As in test_adapter_growth_storage, compile the real adapter with only its +// storage operations interposed. Include its hash TU too: every operation +// that can evict a cohort, including a hash read, uses this test's predicate. +// These definitions satisfy both objects before the static archive search. +#define release_empty_exposure_capacity reversal_fixture_release_empty +#define clear_exposure_storage reversal_fixture_clear +#include "../src/source/pine_adapter.cpp" +#undef clear_exposure_storage +#undef release_empty_exposure_capacity +#include "../src/source/pine_state_hash.cpp" + +namespace reversal_storage_fixture { +using namespace pineforge; +constexpr double kNa = std::numeric_limits::quiet_NaN(); +constexpr std::int64_t kStart = 1736121600000LL; +constexpr int kBars = 96; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +bool same_bits(double left, double right) { + return std::memcmp(&left, &right, sizeof left) == 0; +} + +int triangle(int index, int period) { + const int phase = index % (2 * period); + return phase < period ? phase : 2 * period - phase; +} +double price(int index) { + return 0.25 * (400 + 2 * triangle(index, 9) + + 3 * triangle(index, 23) - triangle(index, 4)); +} +std::vector tape() { + std::vector bars; + for (int i = 0; i < kBars; ++i) { + const double open = price(i), close = price(i + 1); + bars.push_back({open, std::max(open, close) + 0.5, + std::min(open, close) - 0.5, close, 1.0, + kStart + static_cast(i) * 60000}); + } + return bars; +} + +class ReversalHost final : public source::PineStrategyHost { +public: + explicit ReversalHost(bool recording) { + source::PineStrategyConfig config; + config.initial_capital = 100000.0; + config.default_qty_type = static_cast(QtyType::FIXED); + config.default_qty_value = 1.0; + config.pyramiding = 2; + config.commission_type = static_cast(CommissionType::CASH_PER_ORDER); + config.commission_value = 0.5; + configure_pine_strategy(config); + set_syminfo_timezone("UTC"); + set_syminfo_session("24x7"); + set_syminfo_mintick(0.25); + set_broker_state_hash_recording(recording); + } + void on_source_bar(const Bar& bar) override { + const int i = pine_bar_index(); + const double position = live_position_size(); + if (position != 0.0) { + const int side = position > 0.0 ? 1 : -1; + if (last_side_ != 0 && side != last_side_) ++reversals; + last_side_ = side; + } + if (i % 6 == 1) strategy_entry("L", true); + if (i % 6 == 4) strategy_entry("S", false); + if (i % 12 == 2 && position > 0.0) strategy_entry("L2", true); + if (i % 18 == 9) strategy_entry("B", true); + if (i % 18 == 15) strategy_entry("B", false); + if (position > 0.0) + strategy_exit("xl", "L", bar.close + 2.0, bar.close - 2.0); + if (position < 0.0) + strategy_exit("xs", "S", bar.close - 2.0, bar.close + 2.0, + kNa, kNa, kNa, 100.0, {}, 1.0); + if (i % 10 == 7) strategy_close("L", {}, 1.0); + if (i % 14 == 11) strategy_order("o", true, 1.0); + if (i % 14 == 12) strategy_order("o", false, 1.0); + reads.push_back(broker_state_hash()); + } + const std::vector& hashes() const { return broker_state_hashes_; } + std::vector reads; + std::size_t reversals = 0; +private: + int last_side_ = 0; +}; + +struct Result { + std::vector trades; + std::vector hashes, reads; + std::uint64_t final_hash = 0; + std::uint64_t erased_rows = 0; + std::size_t reversals = 0; + Probe operations; +}; + +Result run(bool keep_capacity, bool keep_cohorts, bool magnifier, bool recording) { + require(!source::detail::retain_retired_rows(), + "the witness requires ordinary placement retirement in every mode"); + const auto bars = tape(); + ReversalHost host(recording); + Probe probe; + probe.keep_capacity = keep_capacity; + probe.keep_cohorts = keep_cohorts; + const auto erased_before = source::detail::retired_rows_erased(); + active = &probe; + try { + if (magnifier) + host.run(bars.data(), static_cast(bars.size()), "", "", true, 4, + MagnifierDistribution::ENDPOINTS); + else + host.run(bars.data(), static_cast(bars.size())); + require(host.last_error().empty(), "live reversal run completes"); + Result result; + for (int i = 0; i < host.trade_count(); ++i) + result.trades.push_back(host.get_trade(i)); + result.hashes = host.hashes(); + result.reads = host.reads; + result.final_hash = host.broker_state_hash(); + result.erased_rows = source::detail::retired_rows_erased() - erased_before; + result.reversals = host.reversals; + result.operations = probe; + active = nullptr; + return result; + } catch (...) { + active = nullptr; + throw; + } +} + +void compare_trade(const Trade& a, const Trade& b) { + require(a.entry_time == b.entry_time && a.exit_time == b.exit_time + && a.entry_bar_index == b.entry_bar_index && a.exit_bar_index == b.exit_bar_index + && a.entry_id == b.entry_id && a.exit_id == b.exit_id + && a.entry_comment == b.entry_comment && a.exit_comment == b.exit_comment + && a.entry_incarnation == b.entry_incarnation && a.is_long == b.is_long + && a.open_at_end == b.open_at_end && a.exit_from_bracket == b.exit_from_bracket + && a.close_cause == b.close_cause, + "reversal trade identities and metadata match the old storage operations"); + require(same_bits(a.entry_price, b.entry_price) && same_bits(a.exit_price, b.exit_price) + && same_bits(a.qty, b.qty) && same_bits(a.pnl, b.pnl) + && same_bits(a.pnl_pct, b.pnl_pct) && same_bits(a.commission, b.commission) + && same_bits(a.max_runup, b.max_runup) && same_bits(a.max_drawdown, b.max_drawdown), + "reversal trade doubles match the old storage operations bit for bit"); +} + +void compare(const Result& actual, const Result& old, bool magnifier, bool recording) { + const auto mismatch = [](const auto& left, const auto& right) { + std::size_t i = 0; + for (; i < left.size() && i < right.size() && left[i] == right[i]; ++i) {} + return i; + }; + if (actual.reads != old.reads || actual.hashes != old.hashes + || actual.final_hash != old.final_hash || actual.trades.size() != old.trades.size()) { + std::fprintf(stderr, + "reversal storage: keep_capacity=%d keep_cohorts=%d magnifier=%d recording=%d " + "first read=%zu first row=%zu trades=%zu/%zu final=%llu/%llu " + "spills=%zu retained=%zu\n", + actual.operations.keep_capacity ? 1 : 0, actual.operations.keep_cohorts ? 1 : 0, + magnifier ? 1 : 0, recording ? 1 : 0, + mismatch(actual.reads, old.reads), mismatch(actual.hashes, old.hashes), + actual.trades.size(), old.trades.size(), + static_cast(actual.final_hash), + static_cast(old.final_hash), + actual.operations.spills, actual.operations.retained_rows); + } + require(actual.reads == old.reads + && actual.reads.size() == static_cast(kBars), + "every reversal callback hash matches the old storage operations"); + require(actual.hashes == old.hashes && (recording ? !actual.hashes.empty() + : actual.hashes.empty()), + "every recorded reversal hash matches the old storage operations"); + require(actual.final_hash == old.final_hash, "final reversal hashes match bit for bit"); + require(actual.trades.size() == old.trades.size(), "reversal trade populations match"); + for (std::size_t i = 0; i < actual.trades.size(); ++i) + compare_trade(actual.trades[i], old.trades[i]); + require(actual.erased_rows > 0 && actual.erased_rows == old.erased_rows, + "placement retirement runs identically with spilling enabled and disabled"); + require(actual.reversals > 1 && actual.reversals == old.reversals, + "both runs reverse the live position repeatedly"); + require(actual.operations.releases > 0 && actual.operations.clears > 0, + "reversals exercise both production exposure storage operations"); + if (actual.operations.keep_cohorts) + require(actual.operations.spills == 0 && actual.operations.retained_rows > 0, + "the old cohort retention operation disables eligible evictions"); + else + require(actual.operations.spills > 0 && actual.operations.retained_rows == 0, + "the production run really evicts eligible cohort rows"); +} + +void reversal_storage_matches_the_old_operations() { + for (const bool magnifier : {false, true}) { + for (const bool recording : {false, true}) { + const auto old = run(true, true, magnifier, recording); + require(old.operations.spills == 0 && old.operations.retained_rows > 0 + && old.operations.releases > 0 && old.operations.clears > 0, + "the reference exercises the old retain-storage operations"); + // Isolate L19m and L19l-A as well as their combined live path. + compare(run(false, true, magnifier, recording), old, magnifier, recording); + compare(run(true, false, magnifier, recording), old, magnifier, recording); + compare(run(false, false, magnifier, recording), old, magnifier, recording); + } + } +} +} // namespace reversal_storage_fixture + +int main() { + try { + reversal_storage_fixture::reversal_storage_matches_the_old_operations(); + std::puts("test_adapter_reversal_storage: reversal trades and hashes preserve old storage"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "test_adapter_reversal_storage: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_disk_id_table.cpp b/tests/test_disk_id_table.cpp new file mode 100644 index 000000000..ead6919cb --- /dev/null +++ b/tests/test_disk_id_table.cpp @@ -0,0 +1,205 @@ +// CR-150 L19l. The reference owns the old in-memory ordering/fold; no pins. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +void require(bool value, const char* why) { + if (!value) throw std::runtime_error(why); +} + +using Table = pineforge::detail::DiskIdTable; + +struct Directory { + std::string path = "/tmp/pineforge-ids-test-XXXXXX"; + Directory() { require(::mkdtemp(path.data()) != nullptr, "create scratch directory"); } + ~Directory() { ::rmdir(path.c_str()); } +}; + +std::uint64_t append(std::uint64_t h, const void* data, std::size_t n) { + const auto* p = static_cast(data); + for (std::size_t i = 0; i < n; ++i) h = (h ^ p[i]) * 1099511628211ULL; + return h; +} + +std::uint64_t fold(std::uint64_t h, const std::string& id, std::int64_t value) { + const std::uint64_t n = id.size(); + h = append(h, &n, sizeof n); + h = append(h, id.data(), id.size()); + return append(h, &value, sizeof value); +} + +std::string id_for(std::size_t n) { + // Include non-SSO strings and embedded NULs; the disk key is length based. + return "entry-with-a-long-id/" + std::to_string(n) + std::string("\0tail", 5); +} + +void insert_find_recur_and_folds() { + Directory directory; + Table table; + table.set_directory(directory.path); + std::map reference; + std::vector order; + for (std::size_t i = 0; i < 10000; ++i) { + const auto id = id_for((i * 7919) % 10000); + const auto value = static_cast(i); + const auto placed = table.insert(id, value); + require(placed.second, "new id inserted"); + reference.emplace(id, value); + order.push_back(id); + if (i % 97 == 0) { + require(!table.insert(id, -1).second, "recurring id is not inserted twice"); + require(table.get(placed.first) == value, "recurrence preserves old value"); + } + } + require(table.size() == reference.size(), "logical count equals reference"); + require(!table.find("absent"), "absent key stays absent"); + const auto bytes_before_updates = table.file_bytes(); + for (const auto& id : order) { + const auto found = table.find(id); + require(found.has_value(), "old id found after many insertions"); + table.set(*found, -reference.at(id) - 1); + reference.at(id) = -reference.at(id) - 1; + } + require(table.file_bytes() == bytes_before_updates, "updates do not append history"); + std::uint64_t expected = 1469598103934665603ULL; + for (const auto& row : reference) expected = fold(expected, row.first, row.second); + std::uint64_t actual = 1469598103934665603ULL; + std::size_t visited = 0; + table.for_each_sorted([&](const auto& id, auto value) { + actual = fold(actual, id, value); + ++visited; + }); + require(visited == reference.size() && actual == expected, "sorted fold is byte identical"); + expected = actual = 1469598103934665603ULL; + for (const auto& id : order) expected = fold(expected, id, reference.at(id)); + table.for_each_inserted([&](const auto& id, auto value) { + actual = fold(actual, id, value); + }); + require(actual == expected, "first-insertion order and size survive recurrence"); + Table moved(std::move(table)); + require(moved.size() == reference.size() && table.size() == 0, "move transfers ownership"); + require(moved.get(*moved.find(order.front())) == reference.at(order.front()), "move keeps lookup"); + moved.clear(); + require(moved.size() == 0 && !moved.find(order.front()), "reset forgets old ids"); + require(moved.insert("fresh", 7).second, "reset can create a new file"); +} + +void flat_heap() { + global_allocation::recording = true; + { + Table table; + for (std::size_t i = 0; i < 10; ++i) table.insert(id_for(i), static_cast(i)); + const auto low = global_allocation::live_bytes; + for (std::size_t i = 10; i < 10000; ++i) table.insert(id_for(i), static_cast(i)); + const auto high = global_allocation::live_bytes; + require(high <= low + 1024, "disk id table retains per-id heap"); + } + global_allocation::recording = false; +} + +// Find this test's unlinked table by its exact length. No production fd seam. +int descriptor(std::uint64_t bytes) { + const long limit = ::sysconf(_SC_OPEN_MAX); + const long end = limit > 0 ? std::min(limit, 65536L) : 1024; + for (int fd = 3; fd < end; ++fd) { + struct stat status{}; + if (::fstat(fd, &status) == 0 && S_ISREG(status.st_mode) && status.st_nlink == 0 + && status.st_size >= 0 && static_cast(status.st_size) == bytes) { + return fd; + } + } + throw std::runtime_error("find anonymous table descriptor"); +} + +struct FileLimit { + struct rlimit previous{}; + using Handler = void (*)(int); + Handler handler = SIG_ERR; + FileLimit() { + require(::getrlimit(RLIMIT_FSIZE, &previous) == 0, "read file-size limit"); + handler = ::signal(SIGXFSZ, SIG_IGN); + require(handler != SIG_ERR, "ignore file-limit signal"); + auto limit = previous; + limit.rlim_cur = 0; + if (::setrlimit(RLIMIT_FSIZE, &limit) != 0) { + ::signal(SIGXFSZ, handler); + throw std::runtime_error("set file-size limit"); + } + } + ~FileLimit() { + ::setrlimit(RLIMIT_FSIZE, &previous); + ::signal(SIGXFSZ, handler); + } +}; + +template void io_failure(F action) { + bool failed = false; + try { action(); } + catch (const std::system_error& error) { + failed = pineforge::classify_run_failure(error).code + == pineforge::RunFailureCode::report_sink_failed; + } + require(failed, "I/O error must carry report_sink_failed"); +} + +void file_ownership_and_failures() { + Directory directory; + Table table; + table.set_directory(directory.path + "/missing"); + io_failure([&] { table.insert("first", 1); }); + require(table.size() == 0, "open failure did not publish an id"); + table.set_directory(directory.path); + const auto at = table.insert("first", 1).first; + const int fd = descriptor(table.file_bytes()); + require((::fcntl(fd, F_GETFD) & FD_CLOEXEC) != 0, "close on exec"); + require(::lseek(fd, 3, SEEK_SET) == 3, "set unrelated shared cursor"); + require(table.get(at) == 1 && table.find("first") == at, "read exact row"); + require(::lseek(fd, 0, SEEK_CUR) == 3, "reads use pread"); + // A zero file-size limit rejects pwrite on Linux and Darwin, including + // overwrites in an existing file. Reads remain available for rollback. + { + FileLimit limit; + io_failure([&] { table.set(at, 2); }); + } + require(table.get(at) == 1, "failed write did not silently alter the value"); + table.clear(); + table.set_directory(directory.path); + const auto other = table.insert("other", 9).first; + const int truncated = descriptor(table.file_bytes()); + require(::ftruncate(truncated, 0) == 0, "truncate table to inject short pread"); + io_failure([&] { (void)table.get(other); }); +} +} // namespace + +int main() { + try { + insert_find_recur_and_folds(); + flat_heap(); + file_ownership_and_failures(); + std::puts("disk id table: OK"); + } catch (const std::exception& error) { + global_allocation::recording = false; + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_heap_census.py b/tests/test_heap_census.py index 2a6c0f92d..cf727527f 100644 --- a/tests/test_heap_census.py +++ b/tests/test_heap_census.py @@ -100,8 +100,7 @@ def test_demangled_push_functions_and_element_types_name_known_lanes(self): ("L19l", "pineforge::source::PineExecutionAdapter::note_exit_rank_key(" "std::__cxx11::basic_string const&) (src/source/pine_adapter.cpp:6339)"), ("L19l", "pineforge::source::PlacementTable::fold_sequence(" - "std::unordered_map, " - "pineforge::source::PlacementTable::SequenceMinima>&, " + "pineforge::detail::IdPairStore&, " "pineforge::source::PlacementSnapshot const&) (include/pineforge/source/pine_adapter.hpp:864)"), ("L19k", "pineforge::source::tradingview_multi_day_stamps(" "int, pineforge::Bar const*, int, bool, std::vector const&, " @@ -160,8 +159,7 @@ def test_unfolded_vector_is_distinguished_from_placement_nodes_and_id_minima(sel self.assertEqual(census.owner(vector + " <- " + push), ("L19h", False)) self.assertEqual(census.owner(node + " <- " + push), ("L19j", False)) self.assertEqual(census.owner("pineforge::source::PlacementTable::fold_sequence(" - "std::unordered_map, " - "pineforge::source::PlacementTable::SequenceMinima>&, " + "pineforge::detail::IdPairStore&, " "pineforge::source::PlacementSnapshot const&)"), ("L19l", False)) applied = ("pineforge::source::PineExecutionAdapter::on_applied(" "pineforge::native_order_v7::ExecutionAppliedEvent const&, " diff --git a/tests/test_id_adapter_storage.cpp b/tests/test_id_adapter_storage.cpp new file mode 100644 index 000000000..0c1ff38ac --- /dev/null +++ b/tests/test_id_adapter_storage.cpp @@ -0,0 +1,341 @@ +// CR-150 L19l: the production resident/disk container and real OriginRoster, +// compared with the old unordered-map + insertion-vector logical state. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using pineforge::BrokerStateHashSink; +using pineforge::source::OriginRoster; +namespace no = pineforge::native_order; + +void require(bool value, const char* why) { + if (!value) throw std::runtime_error(why); +} + +// The disk codec has the same five scalar fields as the adapter's codec. +// The reference below retains every origin, rather than using that codec. +struct Facts { + struct Record { + std::uint64_t handle; + std::int64_t cycle; + std::uint64_t count, digest, sides; + }; + no::CohortHandle handle{}; + OriginRoster origins; + std::vector opened; + std::unordered_map units; + std::int64_t cycle = 0; + std::uint64_t sides = 0; + Record retired_record() const noexcept { + return {handle.value, cycle, origins.size(), origins.digest(), sides}; + } + static Facts restore_record(const Record& row) { + Facts result; + result.handle.value = row.handle; + result.cycle = row.cycle; + result.sides = row.sides; + result.origins = OriginRoster::from_retired(static_cast(row.count), row.digest); + return result; + } + bool spillable() const noexcept { + return origins.members().empty() && opened.empty() && units.empty(); + } +}; +using Table = pineforge::detail::DiskResidentIdTable; + +// This is the previous in-memory representation, with no retirement or +// production restore/digest helper in its reference fold. +struct ReferenceFacts { + std::uint64_t handle = 0; + std::int64_t cycle = 0; + std::vector origins; + std::vector opened; + std::map units; + std::uint64_t sides = 0; +}; +using Reference = std::unordered_map; + +void hash_handles(BrokerStateHashSink& sink, const std::vector& handles) { + sink.u(handles.size()); + for (const auto& handle : handles) { + sink.s(handle.run.session_key); + sink.u(0); // The source fold canonicalizes native run generations. + sink.u(handle.incarnation); + } +} +void hash_reference(BrokerStateHashSink& sink, const std::string& id, const ReferenceFacts& facts) { + sink.s(id); sink.u(facts.handle); sink.i(facts.cycle); + std::uint64_t digest = 1469598103934665603ULL; + for (const auto& origin : facts.origins) + digest = (digest ^ origin.incarnation) * 1099511628211ULL; + sink.u(facts.origins.size()); sink.u(digest); + hash_handles(sink, facts.opened); sink.u(facts.sides); + sink.u(facts.units.size()); + for (const auto& unit : facts.units) { sink.u(unit.first); sink.d(unit.second); } +} +void hash_actual(BrokerStateHashSink& sink, const std::string& id, const Facts& facts) { + sink.s(id); sink.u(facts.handle.value); sink.i(facts.cycle); + sink.u(facts.origins.size()); sink.u(facts.origins.digest()); + hash_handles(sink, facts.opened); sink.u(facts.sides); + std::map sorted(facts.units.begin(), facts.units.end()); + sink.u(sorted.size()); + for (const auto& unit : sorted) { sink.u(unit.first); sink.d(unit.second); } +} + +void same_fold(const Table& table, const Reference& reference, const std::vector& order) { + std::vector sorted; + for (const auto& row : reference) sorted.push_back(row.first); + std::sort(sorted.begin(), sorted.end()); + BrokerStateHashSink expected, actual; + expected.u(reference.size()); + for (const auto& id : sorted) hash_reference(expected, id, reference.at(id)); + expected.u(order.size()); + for (const auto& id : order) expected.s(id); + actual.u(table.size()); + for (const auto& row : table.all_sorted()) hash_actual(actual, row.first, row.second); + actual.u(table.size()); + table.for_each_id([&](const auto& id) { actual.s(id); }); + require(actual.h == expected.h, "cohort hash/order/count differs from in-memory fold"); +} + +std::string id_for(std::size_t n) { + return "census-entry-" + std::to_string(n) + std::string("\0tail", 5); +} +no::RequestHandle origin_for(std::uint64_t n) { return {{"disk-cohort-test", 1}, n}; } + +void replay_recurring_ids() { + Table table; + Reference reference; + std::vector order; + for (std::size_t i = 0; i < 2048; ++i) { + const auto id = id_for((i * 137) % 2048); + Facts facts; + facts.handle.value = i + 1; + const auto placed = table.emplace(id, std::move(facts)); + require(placed.second, "new cohort"); + ReferenceFacts old; + old.handle = i + 1; + reference.emplace(id, std::move(old)); + order.push_back(id); + auto& current = placed.first->second; + const auto origin = origin_for(i + 1); + current.origins.push_back(origin); + current.cycle = static_cast(i + 1); + current.sides = (i % 2 == 0) ? 1 : 2; + current.origins.retire_before(current.origins.size()); + auto& expected = reference.at(id); + expected.origins.push_back(origin); + expected.cycle = current.cycle; + expected.sides = current.sides; + } + table.spill(); + require(table.resident_size() == 0, "retired cohorts remain on heap"); + same_fold(table, reference, order); + table.spill(); + for (std::size_t i = 0; i < order.size(); i += 17) { + const auto& id = order[i]; + const auto found = table.find(id); + require(found != table.end(), "old cohort id is lost"); + auto& current = found->second; + auto& expected = reference.at(id); + require(current.handle.value == expected.handle, "recurring id got another handle"); + const auto origin = origin_for(10000 + i); + current.origins.push_back(origin); + current.opened.push_back(origin); + current.units[origin.incarnation] = 3.25; + current.cycle += 5000; + expected.origins.push_back(origin); + expected.opened.push_back(origin); + expected.units[origin.incarnation] = 3.25; + expected.cycle += 5000; + } + same_fold(table, reference, order); + table.spill(); + std::vector expected_live, actual_live; + for (const auto& id : order) + if (!reference.at(id).opened.empty()) expected_live.push_back(id); + for (const auto& id : table.resident_ids()) actual_live.push_back(id); + require(actual_live == expected_live, "live cohort walk lost first-insertion order"); + for (auto& row : table) { + row.second.opened.clear(); + row.second.units.clear(); + row.second.origins.retire_before(row.second.origins.size()); + auto& expected = reference.at(row.first); + expected.opened.clear(); + expected.units.clear(); + } + table.spill(); + require(table.resident_size() == 0, "empty live tables did not leave RAM"); + same_fold(table, reference, order); +} + +void pinned_lookup_and_resident_reads() { + Table table; + Facts initial; + initial.handle.value = 1; + { + const auto first = table.emplace("first", std::move(initial)).first; + first->second.cycle = 7; + Facts next; + next.handle.value = 2; + table.emplace("next", std::move(next)); + const auto again = table.find("first"); + require(&first->second == &again->second, "nested lookup lost pinned identity"); + again->second.cycle = 9; + table.spill(); + require(first->second.cycle == 9, "nested mutation is stale"); + } + table.spill(); + require(table.resident_size() == 0, "released pins retain empty facts"); + require(table.find_resident("first") == table.end(), "retired cohort carries live exposure"); + require(table.find("first")->second.cycle == 9, "writeback lost the mutation"); + table.clear(); + require(table.size() == 0 && table.begin() == table.end(), "run reset retains ids"); +} + +std::size_t retained_bytes(std::size_t count, std::size_t period) { + global_allocation::recording = true; + std::size_t bytes = 0; + { + const auto before = global_allocation::live_bytes; + Table table; + for (std::size_t i = 0; i < count; ++i) { + const auto id = id_for(period == 0 ? i : i % period); + auto found = table.find(id); + if (found == table.end()) { + Facts facts; + facts.handle.value = i + 1; + found = table.emplace(id, std::move(facts)).first; + } + found->second.origins.push_back(origin_for(i + 1)); + found->second.origins.retire_before(found->second.origins.size()); + } + table.spill(); + require(table.resident_size() == 0, "Q9 did not exercise disk eviction"); + bytes = global_allocation::live_bytes - before; + } + global_allocation::recording = false; + return bytes; +} + +void q9_difference_of_differences() { + const auto reused_low = retained_bytes(10, 1); + const auto unique_low = retained_bytes(10, 0); + const auto reused_high = retained_bytes(10000, 1); + const auto unique_high = retained_bytes(10000, 0); + const auto low_delta = static_cast(unique_low) - static_cast(reused_low); + const auto high_delta = static_cast(unique_high) - static_cast(reused_high); + require(high_delta <= low_delta + 1024, "Q9 unique-minus-reused retained-byte slope grows"); +} + +class IdHost final : public pineforge::source::PineStrategyHost { +public: + explicit IdHost(int period) : period_(period) { + pineforge::source::PineStrategyConfig config; + config.initial_capital = 1000000.0; + config.process_orders_on_close = true; + config.default_qty_value = 1.0; + configure_pine_strategy(config); + set_syminfo_timezone("UTC"); + set_syminfo_session("24x7"); + set_syminfo_mintick(0.25); + } + void on_source_bar(const pineforge::Bar&) override { + const int i = pine_bar_index(); + if (i % 4 == 0) { + const auto ordinal = static_cast(i / 4); + active_ = id_for(period_ == 0 ? ordinal : ordinal % static_cast(period_)); + const auto na = std::numeric_limits::quiet_NaN(); + strategy_entry(active_, true, na, na, 1.0); + } else if (i % 4 == 1) { + strategy_close(active_); + } + } +private: + int period_; + std::string active_; +}; + +std::vector chart() { + std::vector bars(256); + for (std::size_t i = 0; i < bars.size(); ++i) + bars[i] = {100.0, 101.0, 99.0, 100.0, 1000.0, + 1736121600000LL + static_cast(i) * 60000}; + return bars; +} + +void real_host_recurrence() { + const auto bars = chart(); + const auto expected_trades = static_cast(bars.size() / 4); + for (const int period : {0, 1, 31}) { + IdHost host(period); + host.run(bars.data(), static_cast(bars.size()), "1", "1", false, 4, + pineforge::MagnifierDistribution::ENDPOINTS); + require(host.last_error().empty(), host.last_error().c_str()); + require(host.trade_count() == expected_trades, "public host did not complete its trade population"); + for (int i = 0; i < host.trade_count(); ++i) { + const auto& trade = host.get_trade(i); + const auto id = id_for(static_cast(period == 0 ? i : i % period)); + require(trade.entry_id == id && trade.qty == 1.0 && !trade.open_at_end, + "public host changed recurring id or lot identity"); + } + // Reset the same adapter and its files, rather than only constructing + // fresh hosts; the second run must not see old handles or rank keys. + host.run(bars.data(), static_cast(bars.size()), "1", "1", false, 4, + pineforge::MagnifierDistribution::ENDPOINTS); + require(host.last_error().empty() && host.trade_count() == expected_trades, "id-table run reset"); + } +} + +struct TemporaryDirectorySetting { + bool was_set = std::getenv("TMPDIR") != nullptr; + std::string previous = was_set ? std::getenv("TMPDIR") : ""; + ~TemporaryDirectorySetting() { + if (was_set) ::setenv("TMPDIR", previous.c_str(), 1); + else ::unsetenv("TMPDIR"); + } +}; + +void real_host_io_failure() { + const auto bars = chart(); + IdHost host(0); + TemporaryDirectorySetting saved; + // /dev/null is a file: a scratch child of it cannot be created on either + // target platform, independent of the run uid's directory permissions. + require(::setenv("TMPDIR", "/dev/null/pineforge-id-test", 1) == 0, "set bad scratch directory"); + host.run(bars.data(), static_cast(bars.size()), "1", "1", false, 4, + pineforge::MagnifierDistribution::ENDPOINTS); + require(!host.last_error().empty(), "disk failure did not fail the run"); + require(std::string(pineforge::run_failure_code_of(host)) == "report_sink_failed", + "scratch failure lost its typed code at the run catcher"); +} +} // namespace + +int main() { + try { + replay_recurring_ids(); + pinned_lookup_and_resident_reads(); + q9_difference_of_differences(); + real_host_recurrence(); + real_host_io_failure(); + std::puts("adapter id storage: OK"); + } catch (const std::exception& error) { + global_allocation::recording = false; + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_id_cohort_storage.cpp b/tests/test_id_cohort_storage.cpp new file mode 100644 index 000000000..3cd471afd --- /dev/null +++ b/tests/test_id_cohort_storage.cpp @@ -0,0 +1,69 @@ +// Uses the previous tree's public API too: its heap bound is a runtime red. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" +#include + +#include +#include +#include +#include + +namespace no = pineforge::native_order; +namespace { +void require(bool value, const char* why) { + if (!value) throw std::runtime_error(why); +} + +void empty_rosters_are_flat_and_reusable() { + global_allocation::recording = true; + { + no::WorkingRequestCore core({"id-storage", 1}); + const auto first = core.cohort_open(); + for (int i = 1; i < 10; ++i) (void)core.cohort_open(); + const auto low = global_allocation::live_bytes; + no::CohortHandle last; + for (int i = 10; i < 10000; ++i) last = core.cohort_open(); + const auto high = global_allocation::live_bytes; + require(high <= low + 4096, "empty cohort rosters grow per distinct id"); + require(first.value == 1 && last.value == 10000, "handle numbering changed"); + require(core.cohorts().size() == 10000, "logical roster count changed"); + std::uint64_t expected = 1; + for (const auto& roster : core.cohorts()) { + require(roster.handle.value == expected++ && roster.origins.empty(), "empty roster view"); + } + std::uint64_t incarnation = 1; + std::uint64_t ordinal = 1; + no::Request request{no::Transact{1.0}, "entry", ""}; + const auto submitted = core.submit(request, 0, incarnation, ordinal); + require(submitted.status == no::SubmitStatus::Accepted, "submit origin"); + const auto origin = *submitted.handle; + core.cohort_add(first, origin); + require(core.cohort_contains(first, origin), "old empty handle can recur"); + core.cohort_remove(first, origin); + require(!core.cohort_contains(first, origin), "empty roster dropped after remove"); + core.cohort_remove(first, origin); + require(core.cohort_receipts().back().status == no::CohortReceiptStatus::Applied, + "removing from dropped empty roster is still Applied"); + core.cohort_add(first, origin); + no::WorkingRequestCore moved(std::move(core)); + require(moved.cohort_contains(first, origin), "move keeps active roster"); + moved.reset({"id-storage", 2}); + require(moved.cohorts().empty(), "reset clears logical roster count"); + moved.cohort_add(first, origin); + require(moved.cohort_receipts().back().status == no::CohortReceiptStatus::InvalidHandle, + "old handle invalid after reset"); + } + global_allocation::recording = false; +} +} // namespace + +int main() { + try { + empty_rosters_are_flat_and_reusable(); + std::puts("id cohort storage: OK"); + } catch (const std::exception& error) { + global_allocation::recording = false; + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_id_pair_store.cpp b/tests/test_id_pair_store.cpp new file mode 100644 index 000000000..435408bb2 --- /dev/null +++ b/tests/test_id_pair_store.cpp @@ -0,0 +1,332 @@ +// CR-150 L19l-C. Reference maps, real anonymous files, and a retained-byte +// witness. Fail-before against b3c39238e: id_pair_store.hpp is absent. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pineforge; +using Store = detail::IdPairStore; +using Key = std::pair>; +using Reference = std::map; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct ScratchDirectory { + std::string path = "/tmp/pineforge-id-pair-test-XXXXXX"; + ScratchDirectory() { + require(::mkdtemp(path.data()) != nullptr, "create test scratch directory"); + } + ~ScratchDirectory() { ::rmdir(path.c_str()); } +}; + +bool put(Store& store, const Key& key, std::uint64_t value) { + return key.second ? store.insert_or_assign(key.first, *key.second, value) + : store.insert_or_assign(key.first, value); +} +std::optional get(const Store& store, const Key& key) { + return key.second ? store.lookup(key.first, *key.second) : store.lookup(key.first); +} +bool erase(Store& store, const Key& key) { + return key.second ? store.erase(key.first, *key.second) : store.erase(key.first); +} +void check_rows(const Store& store, const Reference& reference) { + require(store.size() == reference.size(), "reference/live row counts agree"); + Reference observed; + for (const auto& row : store) { + require(observed.emplace(Key{row.first, row.from_entry}, row.second).second, + "iteration yields each key once"); + } + require(observed == reference, "iteration exactly matches reference keys and values"); + for (const auto& row : reference) + require(get(store, row.first) == row.second, "old key remains findable after rehash"); +} + +void reference_insert_find_erase_recur(bool memory_reference) { + ScratchDirectory scratch; + Store store; + store.set_directory(scratch.path); + if (memory_reference) store.fixture_use_memory_reference(); + Reference reference; + require(store.begin() == store.end() && !store.lookup("absent"), "new table is empty"); + const std::vector adversarial{ + {"", std::nullopt}, {"", std::string{}}, {"x", std::nullopt}, + {"x", std::string{}}, {"x\x1f", std::string("y")}, + {"x", std::string("\x1fy")}, {std::string("a\0b", 3), std::string("c\0d", 3)}, + {std::string(4097, 'L'), std::string(3073, 'R')} + }; + std::uint64_t next = 0; + for (const auto& key : adversarial) { + require(put(store, key, next) == reference.insert_or_assign(key, next).second, + "single/pair domains, binary ids and long strings insert exactly"); + ++next; + } + for (std::uint64_t i = 0; i < 10000; ++i) { + const Key key{"entry-" + std::to_string(i), "from-" + std::to_string(i % 71)}; + require(put(store, key, i) == reference.insert_or_assign(key, i).second, + "many distinct pairs insert exactly"); + const Key shared{"shared-exit", "entry-" + std::to_string(i)}; + require(put(store, shared, i + 1) == reference.insert_or_assign(shared, i + 1).second, + "one id with many from_entry keys inserts exactly"); + } + check_rows(store, reference); + const auto bytes = store.disk_bytes(); + for (std::uint64_t i = 0; i < 10000; i += 3) { + const Key key{"entry-" + std::to_string(i), "from-" + std::to_string(i % 71)}; + require(erase(store, key) == (reference.erase(key) != 0), "erase matches reference"); + require(!get(store, key) && !erase(store, key), "erased key is absent"); + } + check_rows(store, reference); + for (std::uint64_t i = 0; i < 10000; ++i) { + const Key key{"entry-" + std::to_string(i), "from-" + std::to_string(i % 71)}; + require(put(store, key, i + 10000) == reference.insert_or_assign(key, i + 10000).second, + "recur/update matches reference"); + } + require(store.disk_bytes() == bytes, "recur and updates reuse old disk records"); + check_rows(store, reference); + Store moved(std::move(store)); + require(store.empty() && store.disk_bytes() == 0, "moved-from table is reset"); + Store assigned; + assigned.insert_or_assign("discarded", 99); + assigned = std::move(moved); + check_rows(assigned, reference); + assigned.clear(); + require(assigned.empty() && assigned.begin() == assigned.end() && assigned.disk_bytes() == 0, + "clear releases all files and old keys"); + require(assigned.insert_or_assign("fresh", 17), "clear permits a new run"); + require(!assigned.lookup("shared-exit", "entry-0"), "previous run keys do not leak"); + require(memory_reference ? assigned.disk_bytes() == 0 : assigned.disk_bytes() > 0, + "fixture backend choice survives clear and move"); +} + +void minima_and_signed_hash_surface() { + Store minima; + std::unordered_map> reference; + std::unordered_map any; + const auto fold = [&](const std::string& id, const std::string& from, std::uint64_t value) { + const auto whole = any.emplace(id, value); + if (!whole.second) whole.first->second = std::min(whole.first->second, value); + const auto pair = reference[id].emplace(from, value); + if (!pair.second) pair.first->second = std::min(pair.first->second, value); + minima.min_assign(id, value); + minima.min_assign(id, from, value); + }; + for (std::uint64_t i = 0; i < 1000; ++i) { + const auto id = "id-" + std::to_string(i % 31); + const auto from = "entry-" + std::to_string(i % 113); + fold(id, from, 10000 - i); + fold(id, from, i); + } + fold("sentinel", "", std::numeric_limits::max()); + for (const auto& row : any) { + require(minima.lookup(row.first) == row.second, "any-entry minimum matches nested reference"); + for (const auto& pair : reference.at(row.first)) + require(minima.lookup(row.first, pair.first) == pair.second, + "pair minimum matches nested reference"); + } + detail::IdPairStore cycles; + std::unordered_map old_cycles; + for (std::int64_t i = 0; i < 600; ++i) { + const auto key = "exit-" + std::to_string(i) + "\x1f" + "entry-" + std::to_string(i % 13); + const auto value = i % 2 == 0 ? -i : i; + cycles.insert_or_assign(key, value); + old_cycles[key] = value; + } + cycles.insert_or_assign("minimum", std::numeric_limits::min()); + old_cycles["minimum"] = std::numeric_limits::min(); + std::vector disk_keys, old_keys; + disk_keys.reserve(cycles.size()); + old_keys.reserve(old_cycles.size()); + for (const auto& row : cycles) disk_keys.push_back(row.first); + for (const auto& row : old_cycles) old_keys.push_back(row.first); + std::sort(disk_keys.begin(), disk_keys.end()); + std::sort(old_keys.begin(), old_keys.end()); + require(disk_keys == old_keys, "unchanged hash fold observes identical sorted key bytes"); + for (const auto& key : old_keys) + require(cycles.at(key) == old_cycles.at(key), "unchanged hash fold observes signed values"); + bool missing = false; + try { (void)cycles.at("missing"); } + catch (const std::out_of_range&) { missing = true; } + require(missing, "at of absent key fails explicitly"); +} + +struct Recording { + Recording() { global_allocation::recording = true; } + ~Recording() { global_allocation::recording = false; } +}; +struct Refusing { + Refusing() { global_allocation::refusing = true; } + ~Refusing() { global_allocation::refusing = false; } +}; + +void heap_flat_in_distinct_pairs() { + ScratchDirectory scratch; + Recording recording; + Store store; + store.set_directory(scratch.path); + const auto insert = [&](std::uint64_t i) { + const auto id = "long-entry-for-heap-witness-" + std::to_string(i); + const auto from = "long-from-entry-for-heap-witness-" + std::to_string(i); + store.insert_or_assign(id, from, i); + }; + for (std::uint64_t i = 0; i < 10; ++i) insert(i); + const auto low = global_allocation::live_bytes; + for (std::uint64_t i = 10; i < 10000; ++i) insert(i); + const auto high = global_allocation::live_bytes; + require(high <= low + 1024, "auxiliary RAM is flat from 10 to 10000 distinct pairs"); + require(store.size() == 10000 && store.disk_bytes() > 10000 * 40, + "heap witness exercised disk storage of every pair"); + const std::string id = "long-entry-for-heap-witness-0"; + const std::string from = "long-from-entry-for-heap-witness-0"; + { + Refusing refusing; + require(store.lookup(id, from) == 0, "old pair lookup needs no heap allocation"); + require(!store.lookup(id, "different"), "missing pair lookup needs no heap allocation"); + } + std::printf("id-pair heap: 10=%zu 10000=%zu bytes; disk=%llu\n", low, high, + static_cast(store.disk_bytes())); +} + +int data_descriptor(off_t size, std::uint64_t value) { + const long limit = ::sysconf(_SC_OPEN_MAX); + const long end = limit > 0 ? std::min(limit, 65536L) : 1024; + for (int fd = 3; fd < end; ++fd) { + struct stat status{}; + std::array header{}; + if (::fstat(fd, &status) == 0 && S_ISREG(status.st_mode) && status.st_nlink == 0 + && status.st_size == size + && ::pread(fd, header.data(), sizeof(header), 0) == static_cast(sizeof(header)) + && header[3] == value) { + return fd; + } + } + throw std::runtime_error("find this table's anonymous data file"); +} + +template +void expect_io_failure(Operation&& operation, int expected_errno) { + bool caught = false; + try { operation(); } + catch (const std::system_error& error) { + caught = error.code().value() == expected_errno + && classify_run_failure(error).code == RunFailureCode::report_sink_failed; + } + require(caught, "IO failure carries errno and report_sink_failed"); +} + +struct FileSizeLimit { + struct rlimit previous{}; + using Handler = void (*)(int); + Handler previous_handler = SIG_DFL; + explicit FileSizeLimit(rlim_t bytes) { + require(::getrlimit(RLIMIT_FSIZE, &previous) == 0, "read file-size limit"); + previous_handler = std::signal(SIGXFSZ, SIG_IGN); + require(previous_handler != SIG_ERR, "ignore file-size signal for fixture"); + auto limit = previous; + limit.rlim_cur = bytes; + if (::setrlimit(RLIMIT_FSIZE, &limit) != 0) { + std::signal(SIGXFSZ, previous_handler); + throw std::runtime_error("set fixture file-size limit"); + } + } + ~FileSizeLimit() { + ::setrlimit(RLIMIT_FSIZE, &previous); + std::signal(SIGXFSZ, previous_handler); + } +}; + +void file_lifetime_and_failures() { + ScratchDirectory scratch; + Store missing; + missing.set_directory(scratch.path + "/missing"); + expect_io_failure([&] { missing.insert_or_assign("absent", 1); }, ENOENT); + require(missing.empty(), "failed create publishes no logical row"); + missing.set_directory(scratch.path); + require(missing.insert_or_assign("retry", 2), "failed create can be retried"); + + Store store; + store.set_directory(scratch.path); + const std::string id = "descriptor-fixture"; + store.insert_or_assign(id, "from", 919); + const int fd = data_descriptor(static_cast(40 + id.size() + 4), 919); + require((::fcntl(fd, F_GETFD) & FD_CLOEXEC) != 0, "scratch file closes on exec"); + require(::lseek(fd, 7, SEEK_SET) == 7, "set scratch file cursor"); + require(store.lookup(id, "from") == 919 && ::lseek(fd, 0, SEEK_CUR) == 7, + "pread lookup does not move scratch file cursor"); + bool changed = false; + try { store.set_directory(scratch.path + "/different"); } + catch (const std::logic_error&) { changed = true; } + require(changed, "directory cannot silently change after first write"); + require(::ftruncate(fd, 12) == 0, "truncate real scratch data for read fault"); + expect_io_failure([&] { (void)store.lookup(id, "from"); }, EIO); + Store moved(std::move(store)); + moved.clear(); + errno = 0; + require(::fcntl(fd, F_GETFD) == -1 && errno == EBADF, "clear closes moved scratch file"); + + Store full; + full.set_directory(scratch.path); + full.insert_or_assign("first", 3); + { + FileSizeLimit limit(96); + expect_io_failure([&] { full.insert_or_assign(std::string(256, 'W'), 4); }, EFBIG); + } + require(full.size() == 1 && full.lookup("first") == 3, "failed append leaves old rows intact"); + require(full.insert_or_assign(std::string(256, 'W'), 4), "retry overwrites unpublished append"); + require(full.lookup(std::string(256, 'W')) == 4, "retried append is complete"); + + Store resizing; + resizing.set_directory(scratch.path); + for (std::uint64_t i = 0; i < 32; ++i) + resizing.insert_or_assign("resize-" + std::to_string(i), i); + { + FileSizeLimit limit(1024); // old 64-slot index fits, replacement does not + expect_io_failure([&] { resizing.insert_or_assign("resize-new", 33); }, EFBIG); + } + require(resizing.size() == 32, "failed rehash publishes no logical row"); + for (std::uint64_t i = 0; i < 32; ++i) + require(resizing.lookup("resize-" + std::to_string(i)) == i, + "failed rehash keeps every old index slot"); + require(resizing.insert_or_assign("resize-new", 33), "rehash can be retried"); + require(resizing.lookup("resize-new") == 33, "retried rehash keeps new key"); +} +} // namespace + +int main() { + try { + reference_insert_find_erase_recur(false); + reference_insert_find_erase_recur(true); + minima_and_signed_hash_surface(); + heap_flat_in_distinct_pairs(); + file_lifetime_and_failures(); + std::puts("test_id_pair_store: all reference, heap and IO checks passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "test_id_pair_store: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_kernel_cohort_rosters.cpp b/tests/test_kernel_cohort_rosters.cpp new file mode 100644 index 000000000..250e2463e --- /dev/null +++ b/tests/test_kernel_cohort_rosters.cpp @@ -0,0 +1,363 @@ +// CR-150 L19l part B. The reference retains the pre-change vector of every +// roster. The heap witness uses only the base kernel API, without retaining +// its own per-id data. Execute the red/green pair on the lab, not this lane. +#define PINEFORGE_TEST_ALLOCATION_BYTES +#include "global_allocation_replacement.hpp" +#include "native_order_full_fold.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace no = pineforge::native_order; + +namespace { + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +no::Request resting_request() { + no::Request request{no::Transact{1.0}, "roster", ""}; + request.trigger = no::Limit{50.0}; + return request; +} + +template +std::uint64_t roster_fold(const Rosters& rosters) { + full_fold::Fold fold; + fold.u(rosters.size()); + for (const no::CohortRoster& roster : rosters) { + full_fold::fold(fold, roster.handle); + full_fold::fold_vector(fold, roster.origins); + } + return fold.h; +} + +// The original vector storage and receipt rules, with an independent walk +// through predecessor links instead of the production successor-root index. +struct Reference { + no::RunIdentity run{"kernel-rosters", 1}; + std::vector rosters; + std::map> predecessors; + std::set working; + + no::CohortHandle open() { + const no::CohortHandle handle{static_cast(rosters.size()) + 1}; + rosters.push_back(no::CohortRoster{handle, {}}); + return handle; + } + + void issue(no::RequestHandle handle, std::optional predecessor = {}) { + predecessors.emplace(handle.incarnation, predecessor); + working.insert(handle.incarnation); + if (predecessor) working.erase(predecessor->incarnation); + } + + std::optional root(no::RequestHandle handle) const { + if (handle.run != run || predecessors.find(handle.incarnation) == predecessors.end()) { + return std::nullopt; + } + for (;;) { + const auto& previous = predecessors.at(handle.incarnation); + if (!previous) return handle; + handle = *previous; + } + } + + no::CohortRoster* find(no::CohortHandle handle) { + const auto at = std::lower_bound(rosters.begin(), rosters.end(), handle, + [](const no::CohortRoster& row, no::CohortHandle value) { + return row.handle < value; + }); + return at != rosters.end() && at->handle == handle ? &*at : nullptr; + } + + no::CohortReceipt edit(no::CohortReceiptOperation operation, + no::CohortHandle handle, no::RequestHandle origin) { + no::CohortReceipt receipt{operation, no::CohortReceiptStatus::InvalidHandle, handle, origin}; + auto* roster = find(handle); + if (!roster) return receipt; + const auto canonical = root(origin); + if (!canonical) { + receipt.status = no::CohortReceiptStatus::UnknownOrigin; + } else if (operation == no::CohortReceiptOperation::Add + && working.count(origin.incarnation) == 0) { + receipt.status = no::CohortReceiptStatus::TerminalOrigin; + } else { + auto& members = roster->origins; + const auto at = std::lower_bound(members.begin(), members.end(), *canonical, + [](const no::RequestHandle& left, const no::RequestHandle& right) { + return left.incarnation < right.incarnation; + }); + if (operation == no::CohortReceiptOperation::Add) { + if (at == members.end() || *at != *canonical) members.insert(at, *canonical); + } else if (at != members.end() && *at == *canonical) { + members.erase(at); + } + receipt.status = no::CohortReceiptStatus::Applied; + } + return receipt; + } + + bool contains(no::CohortHandle handle, no::RequestHandle origin) { + const auto* roster = find(handle); + const auto canonical = root(origin); + return roster && canonical + && std::find(roster->origins.begin(), roster->origins.end(), *canonical) + != roster->origins.end(); + } +}; + +struct Driver { + no::WorkingRequestCore core{{"kernel-rosters", 1}}; + Reference reference; + std::uint64_t incarnation = 1; + std::uint64_t ordinal = 1; + + no::RequestHandle submit() { + const auto result = core.submit(resting_request(), 0, incarnation, ordinal); + require(result.status == no::SubmitStatus::Accepted && result.handle, "opening accepted"); + reference.issue(*result.handle); + core.retire_history(core.last_ordinal()); + return *result.handle; + } + + no::RequestHandle replace(no::RequestHandle handle) { + const auto result = core.replace(handle, resting_request(), 0, incarnation, ordinal); + require(result.status == no::ReplaceStatus::Replaced && result.successor, "replacement accepted"); + reference.issue(*result.successor, handle); + core.retire_history(core.last_ordinal()); + return *result.successor; + } + + no::CohortHandle open() { + const auto expected = reference.open(); + const auto actual = core.cohort_open(); + require(actual == expected, "cohort numbering matches the retained-vector reference"); + return actual; + } + + void edit(bool add, no::CohortHandle cohort, no::RequestHandle origin) { + const auto operation = add ? no::CohortReceiptOperation::Add : no::CohortReceiptOperation::Remove; + const auto expected = reference.edit(operation, cohort, origin); + if (add) core.cohort_add(cohort, origin); + else core.cohort_remove(cohort, origin); + const auto& actual = core.cohort_receipts().back(); + require(actual.operation == expected.operation && actual.status == expected.status + && actual.cohort == expected.cohort && actual.origin == expected.origin, + "roster receipt matches the original implementation"); + core.retire_cohort_receipts(core.cohort_receipt_count()); + } + + void contains(no::CohortHandle cohort, no::RequestHandle origin) { + require(core.cohort_contains(cohort, origin) == reference.contains(cohort, origin), + "membership matches the original implementation"); + } + + void verify() { + require(core.cohorts().size() == reference.rosters.size(), "logical roster count preserved"); + require(roster_fold(core.cohorts()) == roster_fold(reference.rosters), + "every handle and origin folds in the original order"); + const auto rosters = core.cohorts(); + std::size_t index = 0; + for (const no::CohortRoster& row : rosters) { + const auto& expected = reference.rosters[index++]; + require(row.handle == expected.handle && row.origins == expected.origins, + "logical roster rows equal the retained-vector reference"); + } + require(index == reference.rosters.size(), "iteration visits every empty and live roster"); + } +}; + +void many_distinct_and_recurring_handles() { + Driver driver; + std::array roots{driver.submit(), driver.submit(), driver.submit()}; + auto tail = roots[0]; + no::RequestHandle old_successor; + for (int i = 0; i < 193; ++i) { + tail = driver.replace(tail); + if (i == 0) old_successor = tail; + } + constexpr std::size_t count = 4096; + for (std::size_t i = 0; i < count; ++i) { + const auto cohort = driver.open(); + driver.edit(true, cohort, tail); + driver.edit(true, cohort, tail); // Duplicate insertion. + driver.edit(true, cohort, roots[2]); + driver.contains(cohort, old_successor); // A root read from disk. + driver.edit(false, cohort, old_successor); + if (i % 17 != 0) driver.edit(false, cohort, roots[2]); + if (i % 127 == 0) driver.verify(); + } + driver.verify(); + + // Visit every old id in a permutation, with empty rosters on both sides + // of rosters that have stayed live throughout. No prefix can be retired. + for (std::size_t i = 0; i < count; ++i) { + const no::CohortHandle cohort{static_cast((i * 129) % count) + 1}; + driver.edit(true, cohort, roots[1]); + driver.contains(cohort, roots[1]); + driver.edit(true, cohort, old_successor); // TerminalOrigin, not a new member. + driver.edit(false, cohort, tail); // Applied even when absent. + driver.edit(false, cohort, roots[1]); + driver.contains(cohort, roots[1]); + if (i % 127 == 0) driver.verify(); + } + driver.edit(true, no::CohortHandle{}, roots[1]); + driver.edit(false, no::CohortHandle{count + 1}, roots[1]); + driver.edit(true, no::CohortHandle{1}, no::RequestHandle{driver.core.identity(), 999999}); + driver.edit(false, no::CohortHandle{1}, no::RequestHandle{{"foreign", 1}, roots[1].incarnation}); + require(driver.core.cancel(roots[1], driver.ordinal).status == no::CancelStatus::Cancelled, + "origin cancelled"); + driver.reference.working.erase(roots[1].incarnation); + driver.core.retire_history(driver.core.last_ordinal()); + driver.edit(true, no::CohortHandle{1}, roots[1]); + driver.edit(false, no::CohortHandle{1}, roots[1]); + driver.verify(); + + // Existing readers use typed predicates and iterator arrow access. + const auto rows = driver.core.cohorts(); + const auto found = std::find_if(rows.begin(), rows.end(), [](const no::CohortRoster& row) { + return row.handle == no::CohortHandle{2}; + }); + require(found != rows.end() && found->origins.empty(), "empty roster remains findable by iteration"); + const auto& indexed = driver.core.cohorts()[17]; + require(!indexed.origins.empty() + && indexed.handle == driver.reference.rosters[17].handle + && indexed.origins == driver.reference.rosters[17].origins, + "indexed live roster preserves its members"); + + const auto expected = roster_fold(driver.reference.rosters); + global_allocation::refusing = true; + const auto actual = roster_fold(driver.core.cohorts()); + global_allocation::refusing = false; + require(actual == expected, "logical hash walk needs no heap allocation or file I/O"); + no::WorkingRequestCore moved(std::move(driver.core)); + require(roster_fold(moved.cohorts()) == expected, "move retains live and empty roster identities"); + require(driver.core.cohorts().empty(), "moved-from roster view is empty"); + no::WorkingRequestCore assigned({"discard", 1}); + assigned = std::move(moved); + require(roster_fold(assigned.cohorts()) == expected, "move assignment transfers nonempty rosters"); + assigned.reset({"kernel-rosters", 2}); + require(assigned.cohorts().empty(), "reset discards old roster identities"); + require(!assigned.cohort_contains(no::CohortHandle{1}, roots[0]), "old-run membership is absent"); + require(assigned.cohort_open() == no::CohortHandle{1}, "new run restarts handle numbering"); +} + +struct Recording { + Recording() { global_allocation::recording = true; } + ~Recording() { global_allocation::recording = false; } +}; + +void distinct_id_heap_is_flat() { + Recording recording; + no::WorkingRequestCore core{{"roster-heap", 1}}; + std::uint64_t incarnation = 1; + std::uint64_t ordinal = 1; + const auto submitted = core.submit(resting_request(), 0, incarnation, ordinal); + require(submitted.status == no::SubmitStatus::Accepted && submitted.handle, "heap opening accepted"); + const auto origin = *submitted.handle; + core.retire_history(core.last_ordinal()); + const auto keeper = core.cohort_open(); + core.cohort_add(keeper, origin); + core.retire_cohort_receipts(core.cohort_receipt_count()); + const auto round_trip = [&] { + const auto cohort = core.cohort_open(); + core.cohort_add(cohort, origin); + core.cohort_remove(cohort, origin); + core.retire_cohort_receipts(core.cohort_receipt_count()); + }; + for (int i = 0; i < 10; ++i) round_trip(); + const auto low = global_allocation::live_bytes; + for (int i = 10; i < 10000; ++i) round_trip(); + const auto high = global_allocation::live_bytes; + require(high <= low + 4096, "empty rosters or member capacity grow per distinct id"); + require(core.cohort_contains(keeper, origin), "live roster survives empty-roster eviction"); + core.cohort_add(no::CohortHandle{2}, origin); + require(core.cohort_contains(no::CohortHandle{2}, origin), "old empty handle recurs"); + core.cohort_remove(no::CohortHandle{2}, origin); + core.retire_cohort_receipts(core.cohort_receipt_count()); + + // A historical peak in live rosters must not leave a vector's capacity + // behind once all those rosters are empty. + const std::uint64_t first = static_cast(core.cohorts().size()) + 1; + for (int i = 0; i < 2048; ++i) { + core.cohort_add(core.cohort_open(), origin); + core.retire_cohort_receipts(core.cohort_receipt_count()); + } + for (std::uint64_t i = 0; i < 2048; ++i) { + core.cohort_remove(no::CohortHandle{first + i}, origin); + core.retire_cohort_receipts(core.cohort_receipt_count()); + } + const auto after_peak = global_allocation::live_bytes; + require(after_peak <= high + 4096, "empty rosters retain capacity from a historical live-set peak"); + for (int i = 0; i < 2048; ++i) round_trip(); + require(global_allocation::live_bytes <= after_peak + 4096, + "evicted rosters do not accumulate after a live-set peak"); + std::printf("kernel cohort rosters: 10=%zu 10000=%zu retained bytes\n", low, high); +} + +void empty_roster_is_a_valid_owner() { + no::WorkingRequestCore core{{"roster-owner", 1}}; + const auto first = core.cohort_open(); + for (int i = 0; i < 128; ++i) (void)core.cohort_open(); + no::Request close{no::HostSized{no::HostSizedKind::Close, std::nullopt}, "close", ""}; + close.owner = no::BindCohort{first}; + std::uint64_t incarnation = 1; + std::uint64_t ordinal = 1; + require(core.submit(close, 0, incarnation, ordinal).status == no::SubmitStatus::Accepted, + "an allocated empty cohort is still a valid owner"); + close.owner = no::BindCohort{no::CohortHandle{130}}; + const auto rejected = core.submit(close, 0, incarnation, ordinal); + require(rejected.status == no::SubmitStatus::Rejected + && rejected.reason == no::RequestRejectReason::InvalidOwner, + "an unallocated cohort remains an invalid owner"); +} + +void empty_handles_need_no_allocation() { + Reference reference; + for (int i = 0; i < 10000; ++i) (void)reference.open(); + no::WorkingRequestCore core{reference.run}; + // The old vector must allocate as handles are issued. The current core + // represents empty rosters by its dense sequence alone; the population + // comes from the independent vector, without a private spill threshold. + global_allocation::refusing = true; + try { + for (const auto& row : reference.rosters) + require(core.cohort_open() == row.handle, "empty handles preserve vector numbering"); + } catch (...) { + global_allocation::refusing = false; + throw; + } + global_allocation::refusing = false; + require(roster_fold(core.cohorts()) == roster_fold(reference.rosters), + "allocation-free empty rosters retain the independent vector fold"); +} + +} // namespace + +int main() { + try { + many_distinct_and_recurring_handles(); + distinct_id_heap_is_flat(); + empty_roster_is_a_valid_owner(); + empty_handles_need_no_allocation(); + std::puts("kernel cohort rosters: PASS"); + return 0; + } catch (const std::exception& error) { + global_allocation::refusing = false; + std::fprintf(stderr, "kernel cohort rosters: %s\n", error.what()); + return 1; + } +} diff --git a/tests/test_native_calendar_memo_bound.cpp b/tests/test_native_calendar_memo_bound.cpp new file mode 100644 index 000000000..fe5fff2a3 --- /dev/null +++ b/tests/test_native_calendar_memo_bound.cpp @@ -0,0 +1,322 @@ +// CR-150 lane L19n: the session-day memo owns a fixed heap, and owning it moves +// no answer. +// +// The L0b heap census (4,000 against 16,000 one-minute UTC bars) found a +// growth site in the kernel calendar that no lane owned: the span vectors of +// the memo's session days, +0.012 bytes a bar. The memo was never unbounded: +// it is a direct-mapped table of kDayCapacity days that replaces a day on a +// collision. What grew was the heap its slots allocated one buffer at a time, a +// new day and a new buffer, until the table was full -- three days of bars +// against eleven sit inside that fill. The lane reserves every slot's span +// buffer when the memo binds its calendar, to the most spans a session day of +// that calendar has (session_day_span_bound), and assigns into it after, so the +// heap is the same whatever days were asked, and however many. +// +// The reference is the memo-free calendar of native_calendar.hpp, which keeps +// nothing and is what any memo, bounded or not, must say; the memo of the lane +// base was the same table with a buffer per slot and is not kept. This row +// drives one memo through thousands of consecutive session days, a run's walk, +// for calendars with weekday masks (closed days), a declared break, an +// overnight session, a session of three windows and no declared session at all, +// and asks the questions the memo answers -- the interval of an instant, its +// session day, whether it is in session -- of the memo and of the calendar. +// Every answer must agree bit for bit, presence and every field. Every few +// days a memo that has been asked nothing answers the same question too (the +// recomputation a day the table dropped is answered by); and after the walk the +// instants well behind its front, days the table dropped long ago, are asked of +// it, and then asked again. The footprint is read after every day of every +// walk: the capacity of the day slots' buffers is what the first lookup left, +// the days held never pass kDayCapacity, and by the end the table is full and +// every later day has replaced one, so the cap is exercised, not only stated. +// The calendars of an exchange zone (clock changes inside the walk) go through +// the same walk over fewer days, because every resolution there is a mktime. +// +// Fail-before: at the lane's base SessionDayMemo has no kDayCapacity, no +// retained_span_capacity and no day_resolutions, and native_calendar_memo.hpp +// declares no session_day_span_bound, so this TU does not compile there (the +// lane report records the first diagnostic). +// +// Source-free: this TU links the generic kernel alone and runs in the +// kernel-only profile. + +#include + +#include "../src/native_calendar_memo.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pineforge::native_calendar; + +int failures = 0; +long long compared = 0; +long long compared_cold = 0; +std::string scenario = "initialization"; + +#define CHECK(condition) do { \ + if (!(condition)) { \ + ++failures; \ + std::fprintf(stderr, "FAIL [%s] %s:%d %s\n", scenario.c_str(), \ + __FILE__, __LINE__, #condition); \ + } \ +} while (0) + +constexpr std::int64_t kMinute = 60'000; +constexpr std::int64_t kHour = 60 * kMinute; +constexpr std::int64_t kDay = 24 * kHour; +constexpr std::int64_t kUtcFrom = 1420416000000LL; // 2015-01-05 00:00 UTC, a Monday +constexpr std::int64_t kZonedFrom = 1704067200000LL; // 2024-01-01 00:00 UTC, a Monday + +struct Rng { + std::uint64_t s; + std::uint64_t next() { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + return s; + } + std::int64_t in(std::int64_t lo, std::int64_t hi) { // [lo, hi) + const std::uint64_t span = static_cast(hi - lo); + return lo + static_cast(next() % span); + } +}; + +bool same(const std::optional& a, const std::optional& b) { + if (a.has_value() != b.has_value()) return false; + if (!a) return true; + return a->open_ms == b->open_ms && a->eligible_open_ms == b->eligible_open_ms + && a->last_traded_close_ms == b->last_traded_close_ms + && a->next_period_open_ms == b->next_period_open_ms + && a->next_input_open_ms == b->next_input_open_ms; +} + +bool same(const std::optional& a, const std::optional& b) { + if (a.has_value() != b.has_value()) return false; + if (!a) return true; + return a->origin_ms == b->origin_ms && a->next_origin_ms == b->next_origin_ms + && a->ordinal == b->ordinal && a->spans == b->spans; +} + +Timeframe tf(const char* text) { + auto parsed = parse_timeframe(text); + CHECK(parsed.has_value()); + return parsed ? *parsed : Timeframe{}; +} + +SessionCalendar calendar(const char* session, const char* zone) { + auto parsed = parse_session(session, zone); + CHECK(parsed.has_value()); + return parsed ? *parsed : SessionCalendar{}; +} + +// The questions the memo answers, asked of the memo-free calendar and of the +// memo for the same instant: they must agree field for field. The interval is +// left out of an ask where a zoned calendar makes the reference dear. +void ask(const SessionCalendar& cal, const Timeframe& script, const Timeframe& input, + std::int64_t ms, SessionDayMemo& memo, bool with_interval) { + ++compared; + const auto day = session_day_at(cal, ms); + CHECK(same(day, session_day_at(cal, ms, memo))); + CHECK(in_session(cal, ms) == in_session(cal, ms, memo)); + if (with_interval) { + CHECK(same(interval_containing(cal, script, input, ms), + interval_containing(cal, script, input, ms, memo))); + } + // No session day has more spans than the bound every slot's buffer holds. + if (day) CHECK(day->spans.size() <= session_day_span_bound(cal)); +} + +// The same questions of a memo that has been asked nothing: the recomputation a +// day dropped from the table is answered by, which the memo that kept or +// dropped any days must say the same as. +void ask_cold(const SessionCalendar& cal, const Timeframe& script, const Timeframe& input, + std::int64_t ms, SessionDayMemo& memo) { + ++compared_cold; + SessionDayMemo cold; + CHECK(same(session_day_at(cal, ms, cold), session_day_at(cal, ms, memo))); + CHECK(in_session(cal, ms, cold) == in_session(cal, ms, memo)); + CHECK(same(interval_containing(cal, script, input, ms, cold), + interval_containing(cal, script, input, ms, memo))); +} + +// One calendar under one memo, a run's walk: `days` consecutive days from +// `from`, two instants each (one anywhere in the day, one where sessions +// open), every answer compared with the memo-free calendar's, the interval on +// every `interval_every`th day and a cold memo's on every `cold_every`th (never +// when 0). The footprint is read after every day. Then the look-back. +void sweep(const char* session, const char* zone, const char* script, const char* input, + std::int64_t from, int days, int interval_every, int cold_every, + std::uint64_t seed) { + scenario = std::string(zone) + " '" + session + "' " + script + " over " + input; + const SessionCalendar cal = calendar(session, zone); + const Timeframe script_tf = tf(script); + const Timeframe input_tf = tf(input); + Rng rng{seed}; + SessionDayMemo memo; + CHECK(memo.retained_days() == 0); + CHECK(memo.retained_span_capacity() == 0); + + std::size_t reserved = 0; + for (int d = 0; d < days; ++d) { + const std::int64_t start = from + static_cast(d) * kDay; + const bool with_interval = d % interval_every == 0; + ask(cal, script_tf, input_tf, rng.in(start, start + kDay), memo, with_interval); + ask(cal, script_tf, input_tf, start + 9 * kHour + rng.in(0, 30 * kMinute), memo, + with_interval); + if (cold_every > 0 && d % cold_every == 0) { + ask_cold(cal, script_tf, input_tf, rng.in(start, start + kDay), memo); + } + if (d == 0) { + // What the first lookup reserved: every slot, to the bound. + reserved = memo.retained_span_capacity(); + CHECK(reserved > 0); + CHECK(reserved >= SessionDayMemo::kDayCapacity * session_day_span_bound(cal)); + } + CHECK(memo.retained_span_capacity() == reserved); + CHECK(memo.retained_days() <= SessionDayMemo::kDayCapacity); + } + + // The cap was reached and used: the walk is longer than the table, which + // is full, and the memo resolved more days than it holds. + CHECK(days > static_cast(SessionDayMemo::kDayCapacity)); + CHECK(memo.retained_days() == SessionDayMemo::kDayCapacity); + CHECK(memo.day_resolutions() >= static_cast(days)); + + // The look-back: instants behind the front. The recent ones the table may + // still hold; the ones past a table's width it dropped, and a day dropped + // is resolved again (the count rises) and answered as ever. + const std::int64_t front = from + static_cast(days - 1) * kDay; + for (const int back : {1, 2, 3, 40, 70, 200, 500, 1000}) { + if (back >= days) continue; + const std::int64_t at = front - static_cast(back) * kDay + + rng.in(0, kDay); + const std::uint64_t before = memo.day_resolutions(); + ask(cal, script_tf, input_tf, at, memo, true); + if (back >= 200) CHECK(memo.day_resolutions() > before); + // Asked again, the day is answered from what that resolved. + const std::uint64_t resolved = memo.day_resolutions(); + CHECK(same(session_day_at(cal, at), session_day_at(cal, at, memo))); + CHECK(memo.day_resolutions() == resolved); + CHECK(memo.retained_span_capacity() == reserved); + CHECK(memo.retained_days() <= SessionDayMemo::kDayCapacity); + } +} + +void the_memo_answers_what_the_calendar_answers_over_thousands_of_days() { + std::uint64_t seed = 0x9E3779B97F4A7C15ULL; + // Closed days (a weekday mask), a declared break, an overnight session, a + // session of three windows and no declared session at all, as the other + // calendar rows build them; an hour bucket and a day bucket. + for (const char* session : {"24x7", "0930-1600:23456", "0900-1130,1230-1500", + "1800-1700:23456", "0000-0230,0330-0600,1300-2400"}) { + sweep(session, "UTC", "5", "5", kUtcFrom, 1500, 1, 7, seed += 0x1234567); + sweep(session, "UTC", "D", "60", kUtcFrom, 1500, 1, 7, seed += 0x1234567); + } + // Week and month cores read a whole period's days for one question. + for (const char* session : {"0930-1600:23456", "1800-1700:23456"}) { + sweep(session, "UTC", "W", "D", kUtcFrom, 400, 1, 0, seed += 0x1234567); + sweep(session, "UTC", "M", "D", kUtcFrom, 400, 1, 0, seed += 0x1234567); + } +} + +void the_memo_answers_what_the_calendar_answers_on_exchange_zones() { + std::uint64_t seed = 0xD1B54A32D192ED03ULL; + // New York's 2024 clock changes (March 10 and November 3) fall inside the + // walk, across an overnight session whose origin sits in the evening. + sweep("1800-1700:23456", "America/New_York", "5", "5", kZonedFrom, 330, 3, 0, seed); + sweep("0900-1130,1230-1500", "Asia/Tokyo", "60", "60", kZonedFrom, 120, 3, 0, seed += 7); +} + +// A day the calendar cannot key is an answer too: the slot holds none, hands +// out nothing, and is rewritten by the next day that maps to it. +void the_memo_keeps_an_absent_answer_like_any_other() { + scenario = "absent days"; + const SessionCalendar cal = calendar("0930-1600:23456", "UTC"); + const Timeframe five = tf("5"); + const std::vector instants = { + std::numeric_limits::min(), + -62135596800001LL, // one ms before 0001-01-01 + kUtcFrom, + 253402300800000LL, // 10000-01-01 00:00 UTC + kUtcFrom + kDay, + std::numeric_limits::max(), + kUtcFrom + 2 * kDay, + kUtcFrom, + }; + SessionDayMemo memo; + std::size_t reserved = 0; + for (const std::int64_t ms : instants) { + ask(cal, five, five, ms, memo, true); + if (reserved == 0) reserved = memo.retained_span_capacity(); + CHECK(memo.retained_span_capacity() == reserved); + CHECK(memo.retained_days() <= SessionDayMemo::kDayCapacity); + } + CHECK(reserved >= SessionDayMemo::kDayCapacity * session_day_span_bound(cal)); +} + +void the_footprint_follows_the_calendar_it_is_bound_to() { + scenario = "lifecycle"; + const SessionCalendar narrow = calendar("0930-1600", "UTC"); + const SessionCalendar wide = calendar("0000-0100,0200-0300,0400-0500,0600-0700", "UTC"); + const SessionCalendar all_day = calendar("24x7", "UTC"); + const SessionCalendar invalid{}; + CHECK(session_day_span_bound(invalid) == 0); + CHECK(session_day_span_bound(all_day) >= 1); + CHECK(session_day_span_bound(narrow) >= 1); + CHECK(session_day_span_bound(wide) > session_day_span_bound(narrow)); + + // Nothing is held before a calendar is bound, and a calendar that is not + // valid is bound to nothing: it resolves no day and reserves no buffer. + SessionDayMemo memo; + CHECK(memo.retained_days() == 0); + CHECK(memo.retained_span_capacity() == 0); + CHECK(memo.day_resolutions() == 0); + CHECK(!session_day_at(invalid, kUtcFrom, memo).has_value()); + CHECK(memo.retained_days() == 0); + CHECK(memo.retained_span_capacity() == 0); + CHECK(memo.day_resolutions() == 0); + + // A calendar at another address is a new binding: reserved for what it has. + CHECK(session_day_at(narrow, kUtcFrom, memo).has_value()); + const std::size_t narrow_capacity = memo.retained_span_capacity(); + CHECK(narrow_capacity >= SessionDayMemo::kDayCapacity * session_day_span_bound(narrow)); + CHECK(session_day_at(wide, kUtcFrom, memo).has_value()); + const std::size_t wide_capacity = memo.retained_span_capacity(); + CHECK(wide_capacity >= SessionDayMemo::kDayCapacity * session_day_span_bound(wide)); + CHECK(wide_capacity > narrow_capacity); + + // reset() forgets all of it, and the same calendar is reserved the same. + memo.reset(); + CHECK(memo.retained_days() == 0); + CHECK(memo.retained_span_capacity() == 0); + CHECK(memo.day_resolutions() == 0); + CHECK(session_day_at(wide, kUtcFrom, memo).has_value()); + CHECK(memo.retained_span_capacity() == wide_capacity); + + // A moved memo keeps what it reserved. + SessionDayMemo moved = std::move(memo); + CHECK(moved.retained_span_capacity() == wide_capacity); + CHECK(moved.retained_days() >= 1); + CHECK(moved.retained_days() <= SessionDayMemo::kDayCapacity); +} + +} // namespace + +int main() { + the_memo_answers_what_the_calendar_answers_over_thousands_of_days(); + the_memo_answers_what_the_calendar_answers_on_exchange_zones(); + the_memo_keeps_an_absent_answer_like_any_other(); + the_footprint_follows_the_calendar_it_is_bound_to(); + if (failures == 0) { + std::printf("test_native_calendar_memo_bound: %lld instants against the calendar, " + "%lld against a cold memo, footprint fixed\n", compared, compared_cold); + } + return failures == 0 ? 0 : 1; +} diff --git a/tests/test_native_certified_cycle_bound.cpp b/tests/test_native_certified_cycle_bound.cpp new file mode 100644 index 000000000..f9e943b3c --- /dev/null +++ b/tests/test_native_certified_cycle_bound.cpp @@ -0,0 +1,266 @@ +// CR-150 lane L19n: the consumer's ring of certified cycles owns a fixed heap, +// and the cycles it keeps are the calendar's days. +// +// The L0b heap census (4,000 against 16,000 one-minute UTC bars) found a second +// growth site beside the memo's (test_native_calendar_memo_bound): the span +// vectors of NativeExecutionConsumer::certified_cycles_, +0.003 bytes a bar. +// The ring is four entries and was never unbounded; what grew was the heap its +// entries took one buffer at a time, each from the certificate it was made +// from, until all four were in use -- and a certificate of a closed day owns no +// buffer at all, so even a full ring held what the last four days happened to +// have. Lane L19n reserves every entry's span buffer at the first certificate, +// to the most spans a session day of the run's calendar has +// (native_calendar::session_day_span_bound), and copies each certificate into +// the entry it replaces, so the ring owns the same heap whatever days it holds. +// +// This row configures a bare native host (its calendar is built, nothing is +// run) and asks the consumer for the certified cycle of an instant of each of +// thousands of consecutive days, the ring wrapping at every fourth, then of +// instants well behind the front, whose entries the ring dropped long ago. The +// reference is the memo-free calendar of native_calendar.hpp: a certified cycle +// must be the session day that calendar answers for the instant, its origin, +// its end, its ordinal and its spans, bit for bit. After every ask the heap is +// read: the capacity of the ring's buffers, and of the calendar memo behind it, +// is what the first certificate left, and the memo holds no more days than its +// capacity. A UTC calendar certifies every cycle, and the sessions below give +// it days of no span (a weekday mask), two spans (a break) and four (four +// windows an hour long, whose bound is twelve); a POSIX rule zone certifies +// none, and a certificate refused leaves the ring as constant as one granted: +// it reserves all the same and asks the calendar no day. +// +// Fail-before: at the lane's base the memo has no kDayCapacity, +// retained_span_capacity or retained_days, and native_calendar_memo.hpp +// declares no session_day_span_bound, so this TU does not compile there (the +// lane report records the first diagnostic). +// +// Source-free: this TU runs in the kernel-only profile. +#include "../src/native_calendar_memo.hpp" +#include "../src/native_execution_consumer.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pineforge { +inline namespace engine_script_run_v19 { + +struct NativeExecutionConsumerProbe { + // What certify_cycle answers for an instant, copied out of the ring entry. + struct Cycle { + bool certified = false; + std::int64_t origin_ms = 0; + std::int64_t next_origin_ms = 0; + std::int64_t ordinal = 0; + std::vector> spans; + }; + static Cycle certify(const BacktestEngine& host, std::int64_t ms) { + Cycle out; + const auto* entry = NativeExecutionConsumer::bound(host).certify_cycle(ms); + if (entry == nullptr) return out; + out.certified = true; + out.origin_ms = entry->origin_ms; + out.next_origin_ms = entry->next_origin_ms; + out.ordinal = entry->ordinal; + out.spans = entry->spans; + return out; + } + // The ring: how many entries it keeps, how many of them hold a range, and + // the capacity, in spans, of the buffers they own together. + static std::size_t ring_entries(const BacktestEngine& host) { + return NativeExecutionConsumer::bound(host).certified_cycles_.size(); + } + static std::size_t ring_ranges(const BacktestEngine& host) { + std::size_t held = 0; + for (const auto& cycle : NativeExecutionConsumer::bound(host).certified_cycles_) { + if (cycle.origin_ms < cycle.next_origin_ms) ++held; + } + return held; + } + static std::size_t ring_span_capacity(const BacktestEngine& host) { + std::size_t capacity = 0; + for (const auto& cycle : NativeExecutionConsumer::bound(host).certified_cycles_) { + capacity += cycle.spans.capacity(); + } + return capacity; + } + // The calendar memo behind it. + static std::size_t memo_days(const BacktestEngine& host) { + return NativeExecutionConsumer::bound(host).calendar_memo_.retained_days(); + } + static std::size_t memo_span_capacity(const BacktestEngine& host) { + return NativeExecutionConsumer::bound(host).calendar_memo_.retained_span_capacity(); + } +}; + +} // inline namespace engine_script_run_v19 +} // namespace pineforge + +using namespace pineforge; +namespace nc = pineforge::native_calendar; +using Probe = NativeExecutionConsumerProbe; + +namespace { + +int failures = 0; +long long compared = 0; +std::string scenario = "initialization"; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + ++failures; \ + std::fprintf(stderr, "CHECK FAIL [%s] %s:%d %s\n", scenario.c_str(), \ + __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +constexpr std::int64_t kMinute = 60'000; +constexpr std::int64_t kDay = 1440 * kMinute; +constexpr std::int64_t kFrom = 1420416000000LL; // 2015-01-05 00:00 UTC, a Monday + +struct Rng { + std::uint64_t s; + std::uint64_t next() { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + return s; + } + std::int64_t in(std::int64_t lo, std::int64_t hi) { // [lo, hi) + const std::uint64_t span = static_cast(hi - lo); + return lo + static_cast(next() % span); + } +}; + +class Idle final : public NativeStrategyHost { +public: + void on_native_bar(const Bar&, const NativeDecisionContext&) override {} +}; + +NativeRunSpec spec_for(const char* zone, const char* session) { + NativeRunSpec spec; + spec.identity = {"l19n-certified-ring", 1}; + spec.input_tf = "60"; + spec.script_tf = "60"; + spec.tickerid = "TEST:L19N"; + spec.timezone = zone; + spec.session = session; + spec.initial_capital = 100000.0; + spec.point_value = 1.0; + spec.account_fx = 1.0; + spec.price_tick = 0.01; + spec.fee_kind = NativeFeeKind::CashPerExecution; + spec.fee_value = 0.0; + return spec; +} + +// What the ring must say of `ms` when the zone certifies every cycle: the +// session day the memo-free calendar answers for it. +void expect_the_calendar_day(const nc::SessionCalendar& calendar, const Probe::Cycle& cycle, + std::int64_t ms) { + ++compared; + CHECK(cycle.certified); + CHECK(cycle.origin_ms <= ms && ms < cycle.next_origin_ms); + const auto day = nc::session_day_at(calendar, ms); + CHECK(day.has_value()); + if (!day) return; + CHECK(cycle.origin_ms == day->origin_ms); + CHECK(cycle.next_origin_ms == day->next_origin_ms); + CHECK(cycle.ordinal == day->ordinal); + CHECK(cycle.spans == day->spans); + CHECK(cycle.spans.size() <= nc::session_day_span_bound(calendar)); +} + +// One configured host and `days` consecutive days of instants, a random one in +// each: certify_cycle is asked for it, the ring and the memo behind it are +// read, and the cycle is compared with the calendar's day when the zone +// certifies. Then the instants behind the front, asked of a ring that dropped +// their entries long ago. +void sweep(const char* zone, const char* session, bool certifies, int days, + std::uint64_t seed) { + scenario = std::string(zone) + " '" + session + "' " + + (certifies ? "certified" : "refused"); + Idle host; + CHECK(host.configure_native(spec_for(zone, session)).status == NativeSetupStatus::Applied); + const auto calendar = nc::parse_session(session, zone); + CHECK(calendar.has_value()); + if (!calendar) return; + const std::size_t bound = nc::session_day_span_bound(*calendar); + CHECK(bound > 0); + Rng rng{seed}; + + std::size_t ring_reserved = 0; + std::size_t memo_reserved = 0; + for (int d = 0; d < days; ++d) { + const std::int64_t ms = kFrom + static_cast(d) * kDay + rng.in(0, kDay); + const Probe::Cycle cycle = Probe::certify(host, ms); + if (certifies) { + expect_the_calendar_day(*calendar, cycle, ms); + } else { + CHECK(!cycle.certified); + // A refused cycle asked the calendar for no day. + CHECK(Probe::memo_days(host) == 0); + } + if (d == 0) { + // What the first certificate reserved: every entry, to the bound. + ring_reserved = Probe::ring_span_capacity(host); + memo_reserved = Probe::memo_span_capacity(host); + CHECK(ring_reserved >= Probe::ring_entries(host) * bound); + CHECK(memo_reserved >= nc::SessionDayMemo::kDayCapacity * bound); + } + CHECK(Probe::ring_span_capacity(host) == ring_reserved); + CHECK(Probe::memo_span_capacity(host) == memo_reserved); + CHECK(Probe::memo_days(host) <= nc::SessionDayMemo::kDayCapacity); + } + + // The cap was reached and used: the ring turned over many times and every + // entry holds a range; a certifying walk filled the memo's table. + CHECK(days > static_cast(Probe::ring_entries(host))); + CHECK(Probe::ring_ranges(host) == Probe::ring_entries(host)); + if (certifies) CHECK(Probe::memo_days(host) == nc::SessionDayMemo::kDayCapacity); + + // Behind the front: the entries of these days were replaced long ago, and + // the ring makes each again, to the same capacity. + const std::int64_t front = kFrom + static_cast(days - 1) * kDay; + for (const int back : {0, 1, 2, 3, 4, 5, 50, 500, 2000}) { + if (back >= days) continue; + const std::int64_t ms = front - static_cast(back) * kDay + + rng.in(0, kDay); + const Probe::Cycle cycle = Probe::certify(host, ms); + if (certifies) { + expect_the_calendar_day(*calendar, cycle, ms); + } else { + CHECK(!cycle.certified); + } + CHECK(Probe::ring_span_capacity(host) == ring_reserved); + CHECK(Probe::memo_span_capacity(host) == memo_reserved); + } +} + +void the_ring_keeps_a_fixed_heap_and_the_calendars_days() { + std::uint64_t seed = 0x9E3779B97F4A7C15ULL; + // A day with no declared session, a break (two spans on a weekday, none on + // a closed day), an overnight session whose origin is in the evening, and + // four windows an hour long. + for (const char* session : {"24x7", "0900-1130,1230-1500:23456", "1800-1700:23456", + "0000-0100,0200-0300,0400-0500,0600-0700"}) { + sweep("UTC", session, true, 2500, seed += 0x1234567); + } + // A POSIX rule zone is never certified, and the ring is as constant. + sweep("EST5EDT,M3.2.0,M11.1.0", "0930-1600:23456", false, 2500, seed += 0x1234567); +} + +} // namespace + +int main() { + the_ring_keeps_a_fixed_heap_and_the_calendars_days(); + std::printf("test_native_certified_cycle_bound: %lld certified cycles against the " + "calendar, %d failures\n", compared, failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/test_run_json_sink.py b/tests/test_run_json_sink.py index 1f5a41bc2..9a42cc602 100644 --- a/tests/test_run_json_sink.py +++ b/tests/test_run_json_sink.py @@ -313,12 +313,16 @@ def fail_after_trades(value, destination): assert json.loads(text)["code"] == "report_sink_failed" assert len(text.splitlines()) == 1 and "[partial" not in text assert files and all(file.closed for file in files) + # The sink is a named file, not a TemporaryFile in `files` above: its own + # closure is that nothing of it survives the failed run. + assert not Path(library.installed[0]).exists() def test_sink_open_failure_has_the_existing_sink_failure_code(tmp_path, monkeypatch): def refuse(*args, **kwargs): raise OSError(28, "injected full sink disk") - monkeypatch.setattr(run_json.tempfile, "TemporaryFile", refuse) + # The sink is created with mkstemp (a real, named file), not TemporaryFile. + monkeypatch.setattr(run_json.tempfile, "mkstemp", refuse) status, text, stderr, library = invoke(tmp_path, monkeypatch) assert status == 1 and stderr == "" assert json.loads(text)["code"] == "report_sink_failed" @@ -349,12 +353,15 @@ def temporary(*args, **kwargs): return file monkeypatch.setattr(run_json.tempfile, "TemporaryFile", temporary) outputs = fault == "column-close" + library = SinkLibrary(outputs=outputs) status, text, stderr, _ = invoke(tmp_path, monkeypatch, - library=SinkLibrary(outputs=outputs), extra=["--outputs"] if outputs else []) + library=library, extra=["--outputs"] if outputs else []) assert status == 1 and stderr == "" assert json.loads(text)["code"] == "report_sink_failed" assert len(text.splitlines()) == 1 assert files and all(file.closed for file in files) + # The named sink file is gone as well (it is not one of `files`). + assert library.installed and not Path(library.installed[0]).exists() def test_prepared_report_is_copied_in_bounded_chunks(tmp_path): diff --git a/tests/test_run_strategy_sink.py b/tests/test_run_strategy_sink.py new file mode 100644 index 000000000..6cd00257f --- /dev/null +++ b/tests/test_run_strategy_sink.py @@ -0,0 +1,1097 @@ +"""L9c: a fixture sink must reproduce the array harness's CSV bytes. + +Only native calls are replaced. Strategy.run, L9a's sink owner/receipt checks, +pf_sink's strict reader, report-window filtering and the CSV writers are real. + +The named owner-only sink gives every reader its own offset. Canaries compare +decoded T/E rows with engine counts; sweep logs retain passing/failing canaries +and kept paths, and coverage is checked against the probes actually run. +""" +from __future__ import annotations + +import csv +import ctypes +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +import tempfile + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "docker")) + +import run_strategy as rs +import check_sink_canary_log as canary_log +import run_json +import pf_sink +import test_run_json_bin as bars +from test_run_json_sink import SinkLibrary, TRADE_FIELDS, header, write_sink + + +def trade_fixture(): + """Non-associative PnL, signed zeros, lot precision and unsorted fills.""" + trades = [] + for index, (pnl, qty, minute) in enumerate(( + (.1234567, 923941.16, 1), + (1.e16, .30000000000000004, 3), + (-1.e16, .00000001, 2), + (-.00000051, 897902.68, 3), + (-0., 1., 5), + )): + trades.append({ + "entry_time": bars.T0 + minute * 60000, + "exit_time": bars.T0 + (minute + 1) * 60000, + "entry_price": 1.082320123456789, + "exit_price": 1.07836987654321, + "pnl": pnl, + "pnl_pct": -.000051 if index % 2 else 1.23456789, + "is_long": index % 2 == 0, + "max_runup": -0. if index == 4 else 12.3456789, + "max_drawdown": -0. if index == 4 else .00000051, + "qty": qty, + "commission": .123456789, + "entry_bar_index": minute, + "exit_bar_index": minute + 1, + "open_at_end": index == 4, + "entry_incarnation": 0 if index == 1 else 41 + index, + "entry_id": 'entry "\n漢', + "exit_id": "exit,short" if index % 2 else "", + "exit_comment": "range end" if index == 4 else "", + "close_cause": 6 if index == 4 else 1, + }) + return trades + + +class HarnessLibrary(SinkLibrary): + """The same fixture populates either ReportC arrays or the engine sink.""" + + def __init__(self, trades, *, fault=None, bad_field=None, bad_order=False): + self.trades = trades + self.bad_order = bad_order + # What the engine was handed, and how big the sink was when the engine + # was freed: read from the files themselves, not from harness output. + self.engine_view, self.sizes = [], [] + super().__init__(pnls=tuple(trade["pnl"] for trade in trades), + fault=fault, bad_field=bad_field) + for name in ("strategy_set_trace_enabled", + "strategy_set_broker_state_hash_recording", + "strategy_set_trade_start_time"): + setattr(self, name, self.export(name, lambda *_args: None)) + for field in ("entry_incarnation", "entry_id", "exit_id", + "exit_comment", "close_cause"): + name = "strategy_closed_trade_" + field + setattr(self, name, self.export(name, self.accessor(field))) + + def accessor(self, field): + def read(state, index): + assert self.path is None, "sink rows must not call retained-ledger accessors" + value = self.trades[index][field] + return value.encode("utf-8") if isinstance(value, str) else value + return read + + def install(self, state, pointer): + status = super().install(state, pointer) + info = os.stat(self.path) + self.engine_view.append({ + "path": self.path, "regular": stat.S_ISREG(info.st_mode), + "mode": stat.S_IMODE(info.st_mode), "size": info.st_size, + }) + return status + + def free(self, state): + if self.path is not None: + self.sizes.append(os.path.getsize(self.path)) + super().free(state) + + def receipt(self, state, pointer): + result = super().receipt(state, pointer) + # Independent engine-side expectation from the fixture feed, not a + # number obtained by parsing the sink or its rewritten trailer. + pointer._obj.count_e = self.equity_points + return result + + def run(self, state, feed, count, *args): + bars.Library.run(self, state, feed, count, *args) + self.equity_points = count + report = args[-1]._obj + native_trades = (rs.TradeC * len(self.trades))() + rows = [] + for index, trade in enumerate(self.trades): + for field, _ctype in rs.TradeC._fields_: + setattr(native_trades[index], field, trade[field]) + sink_trade = dict(trade, n=index + 1, + side="long" if trade["is_long"] else "short") + if self.bad_field and index == len(self.trades) - 1: + sink_trade[self.bad_field] = None + if self.bad_order and index == 0: + sink_trade["n"] = 2 + rows.append(["T", *(sink_trade[field] for field in TRADE_FIELDS)]) + curve = (rs.EquityPointC * count)() + hashes = (ctypes.c_uint64 * count)() + trace = (rs.TraceEntryC * count)() + trace_names = ("fixture", 'quote "\n漢') + names = (ctypes.c_char_p * 2)( + *(name.encode("utf-8") for name in trace_names)) + for index in range(count): + curve[index] = rs.EquityPointC(feed[index].timestamp, 10000. + index, -0.) + hashes[index] = (1 << 63) + index + trace[index] = rs.TraceEntryC( + feed[index].timestamp, index, index % 2, float(index)) + rows.insert(index, ["E", curve[index].time_ms, + curve[index].equity, curve[index].open_profit]) + report.total_trades = len(self.trades) + report.net_profit = sum(trade["pnl"] for trade in self.trades) + report.input_bars_processed = report.script_bars_processed = count + report.bar_magnifier_enabled = 1 + if self.path is None: + report.trades, report.trades_len = native_trades, len(self.trades) + report.equity_curve, report.equity_curve_len = curve, count + report.broker_state_hash, report.broker_state_hash_len = hashes, count + report.trace, report.trace_len = trace, count + report.trace_names, report.trace_names_len = names, 2 + else: + for index in range(count): + rows.extend([ + ["X", feed[index].timestamp, index, + trace_names[index % 2], float(index)], + ["H", format(hashes[index], "016x")], + ]) + self.counts, self.sha = write_diagnostic_sink( + self.path, rows, fault=self.fault) + self.backing = [native_trades, curve, hashes, trace, names] + + +def write_diagnostic_sink(path, rows, *, fault=None): + # Reuse L9a's fixture writer and fixed schema; add v1 X/H descriptors so + # the auxiliary CLI outputs are exercised alongside the CSV. + from unittest import mock + import test_run_json_sink as fixture + + def diagnostic_header(outputs=False): + value = header(outputs) + value["records"].extend([ + {"type": "X", "fields": ["timestamp", "bar_index", "name", "value"], + "types": ["i64", "i64", "str", "f64?"], + "units": ["unix_ms", "", "", ""]}, + {"type": "H", "fields": ["hash"], "types": ["hex64"], "units": [""]}, + ]) + return value + + with mock.patch.object(fixture, "header", diagnostic_header): + return write_sink(path, rows, fault=fault) + + +def strategy(library): + result = rs.Strategy.__new__(rs.Strategy) + result.lib = library + result._setup_signatures() + return result + + +@pytest.fixture(autouse=True) +def clean_environment(monkeypatch): + for name in ("PINEFORGE_REPORT_SINK", rs.KEEP_REPORT_SINK_ENV, + "PINEFORGE_CANARY_LOG", + "PINEFORGE_REQUESTS_ROOT", + rs.MAGNIFIER_FEED_ENV, rs.LANE_QTY_STEP_ENV, + rs.SESSION_CALENDAR_ENV): + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture(autouse=True) +def sink_directory(tmp_path, monkeypatch): + """Every sink file of a test is created in this directory (the temp dir + the harness picks). A sink file must not outlive its test unless the + keep switch asked for it, and that test removes it itself.""" + directory = tmp_path / "sinks" + directory.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(directory)) + yield directory + assert not list(directory.glob("pineforge-report-sink-*")), ( + "a sink file was left behind") + + +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize("window", [ + None, + (bars.T0 + 120000, bars.T0 + 180000), + (bars.T0 + 300000, bars.T0 + 300000), + (bars.T0 + 900000, bars.T0 + 960000), +]) +def test_fixture_sink_csv_equals_array_bytes(tmp_path, monkeypatch, empty, window): + source = bars.csv_file(tmp_path, count=8) + trades = [] if empty else trade_fixture() + array_library = HarnessLibrary(trades) + array = strategy(array_library).run(source) + expected = tmp_path / "array.csv" + array_rows = rs._filter_trades_to_window(array["trades"], window) + rs.write_engine_trades_csv(array_rows, expected) + assert not array_library.installed + + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trades) + report = strategy(library).run(source) + assert len(library.installed) == 1, "sink-on must install the native sink" + assert library.queried == 2, "verify the receipt and save its independent E count" + assert report.receipt_count_e == len(array["equity_curve_time_ms"]) + with report: + assert not isinstance(report["trades"], list) + assert list(report["trades"]) == array["trades"] + assert list(report["trace_names"]) == array["trace_names"] + assert list(report["trace"]) == array["trace"] + assert list(report["broker_state_hash"]) == array["broker_state_hash"] + assert list(report["equity_curve_time_ms"]) == array["equity_curve_time_ms"] + assert len(report["trades"]) == len(trades) + assert rs._entered_before_tv(report["trades"], bars.T0 + 180000) == ( + rs._entered_before_tv(array["trades"], bars.T0 + 180000)) + sink_rows = rs._filter_trades_to_window(report["trades"], window) + assert len(sink_rows) == len(array_rows) + actual = tmp_path / "sink.csv" + rs.write_engine_trades_csv(sink_rows, actual) + assert actual.read_bytes() == expected.read_bytes() + # A second pass cannot depend on the first iterator's cursor. + rs.write_engine_trades_csv(sink_rows, actual) + assert actual.read_bytes() == expected.read_bytes() + assert rs._report_terminal_quote_applied( + report, (bars.T0 + 420000, 1.), "15") == ( + rs._report_terminal_quote_applied( + array, (bars.T0 + 420000, 1.), "15")) + assert report.sink.closed and not Path(report.sink.path).exists() + assert library.calls.index("report_free") < library.calls.index("strategy_free") + + +def test_sink_csv_is_forward_only_and_keeps_the_legacy_cells(tmp_path, monkeypatch): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + source = bars.csv_file(tmp_path, count=8) + library = HarnessLibrary(trade_fixture()) + report = strategy(library).run(source) + assert library.installed, "sink-on must install the native sink" + with report: + def refuse(*_args): + pytest.fail("CSV export must not size, index or materialize the trade history") + with monkeypatch.context() as patch: + patch.setattr(type(report["trades"]), "__len__", refuse) + patch.setattr(type(report["trades"]), "__getitem__", refuse) + rs.write_engine_trades_csv(report["trades"], tmp_path / "sink.csv") + content = (tmp_path / "sink.csv").read_bytes() + assert content.count(b"\r\n") == 11 + with (tmp_path / "sink.csv").open(newline="") as file: + reader = csv.DictReader(file) + rows = list(reader) + assert reader.fieldnames == [ + "Trade #", "Type", "Date and time", "Price", "Qty", "Net PnL", + "Net PnL %", "Favorable excursion USD", "Adverse excursion USD", + "Cumulative PnL", "Engine entry incarnation", "Engine range-end", + ] + assert [row["Trade #"] for row in rows] == [ + "5", "5", "4", "4", "3", "3", "2", "2", "1", "1"] + assert [row["Type"] for row in rows[:4]] == [ + "Exit long", "Entry long", "Exit short", "Entry short"] + assert rows[0]["Engine range-end"] == "open" + assert rows[0]["Engine entry incarnation"] == "" + assert rows[1]["Engine entry incarnation"] == "45" + assert rows[1]["Engine range-end"] == "" + assert rows[-1]["Qty"] == "923941.16" + assert rows[0]["Adverse excursion USD"] == "0.000000" + assert rows[0]["Net PnL"] == "-0.000000" + running = 0. + for trade, pair in zip(trade_fixture(), reversed(rows[::2])): + running += trade["pnl"] + assert pair["Cumulative PnL"] == f"{running:.6f}" + + +def invoke_main(tmp_path, monkeypatch, library, output, *, extra=()): + source = tmp_path / "chart.csv" + if not source.exists(): + bars.csv_file(tmp_path, count=8) + probe = tmp_path / "probe" + probe.mkdir(exist_ok=True) + runner = strategy(library) + monkeypatch.setattr(rs, "Strategy", lambda _path: runner) + monkeypatch.setattr(sys, "argv", [ + "run_strategy.py", str(probe), "--ohlcv", str(source), + "--no-trim-output", "--output", str(output), *extra, + ]) + return rs.main() + + +# The canary's one machine-greppable line, exactly as the sweep will grep it. +CANARY = re.compile( + r"sink_canary: probe=(?P\S+) bytes=(?P\d+) records=(?P\d+) " + r"trades=(?P\d+) equity=(?P\d+) (?Pok|FAIL \S.*)") + + +def canary_lines(text): + return [line for line in text.splitlines() if line.startswith("sink_canary:")] + + +def test_cli_csv_and_diagnostic_values_equal_array_path(tmp_path, monkeypatch): + # Retain the class while invoke_main replaces only the CLI constructor. + real_strategy = rs.Strategy + trades = trade_fixture() + array_csv = tmp_path / "array.csv" + array_trace = tmp_path / "array-trace.json" + array_library = HarnessLibrary(trades) + assert invoke_main(tmp_path, monkeypatch, array_library, array_csv, + extra=("--trace-json", str(array_trace), + "--broker-state-hash")) == 0 + array_hash_bytes = (tmp_path / "broker_state_hash.json").read_bytes() + monkeypatch.setattr(rs, "Strategy", real_strategy) + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trades) + owners = [] + make_sink = rs._new_harness_report_sink + def track(*args): + sink = make_sink(*args) + owners.append(sink) + return sink + monkeypatch.setattr(rs, "_new_harness_report_sink", track) + actual_csv = tmp_path / "sink.csv" + sink_trace = tmp_path / "sink-trace.json" + assert invoke_main(tmp_path, monkeypatch, library, actual_csv, + extra=("--trace-json", str(sink_trace), + "--broker-state-hash")) == 0 + assert library.installed, "the CLI must honor PINEFORGE_REPORT_SINK" + assert actual_csv.read_bytes() == array_csv.read_bytes() + assert sink_trace.read_bytes() == array_trace.read_bytes() + assert json.loads(sink_trace.read_text()) == json.loads(array_trace.read_text()) + assert (tmp_path / "broker_state_hash.json").read_bytes() == array_hash_bytes + assert owners and all(sink.closed for sink in owners) + + +@pytest.mark.parametrize("fault", ["hash", "truncated", "receipt", "incomplete"]) +def test_bad_sink_never_overwrites_csv_and_releases_resources( + tmp_path, monkeypatch, fault): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture(), fault=fault) + destination = tmp_path / "engine_trades.csv" + previous = b"previous verified CSV\r\n" + destination.write_bytes(previous) + owners = [] + make_sink = rs._new_harness_report_sink + def track(*args): + sink = make_sink(*args) + owners.append(sink) + return sink + monkeypatch.setattr(rs, "_new_harness_report_sink", track) + with pytest.raises(run_json.RunFailure, match="sink"): + invoke_main(tmp_path, monkeypatch, library, destination) + assert library.installed, "the failure must exercise a real fixture sink" + assert destination.read_bytes() == previous + assert owners and all(sink.closed for sink in owners) + assert "report_free" in library.calls and "strategy_free" in library.calls + + +@pytest.mark.parametrize("field", [ + "entry_price", "exit_price", "qty", "pnl", "pnl_pct", + "max_runup", "max_drawdown", "commission", +]) +def test_null_trade_field_is_refused_before_csv(tmp_path, monkeypatch, field): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture(), bad_field=field) + destination = tmp_path / "engine_trades.csv" + with pytest.raises(RuntimeError, match=f"non-finite.*{field}"): + invoke_main(tmp_path, monkeypatch, library, destination) + assert library.installed + assert not destination.exists() + + +def test_report_order_is_checked_even_with_a_valid_receipt(tmp_path, monkeypatch): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture(), bad_order=True) + with pytest.raises(RuntimeError, match="out of report order"): + strategy(library).run(bars.csv_file(tmp_path)) + assert library.installed and library.queried == 2 + + +def test_csv_rechecks_the_sink_before_replacing_existing_output(tmp_path, monkeypatch): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + report = strategy(library).run(bars.csv_file(tmp_path)) + assert library.installed + destination = tmp_path / "engine_trades.csv" + previous = b"previous verified CSV\r\n" + destination.write_bytes(previous) + with report: + # The sink is an ordinary named file: change it the way any other + # process could, through its real path. + sink_file = Path(report.sink.path) + original = sink_file.read_bytes() + changed = original.replace(b'"long"', b'"short"', 1) + assert changed != original + sink_file.write_bytes(changed) + with pytest.raises(run_json.RunFailure, match="sink"): + rs.write_engine_trades_csv(report["trades"], destination) + assert destination.read_bytes() == previous + + +@pytest.mark.parametrize("missing", [ + "pf_report_sink_version", "strategy_set_report_sink_v1", + "strategy_report_sink_receipt_v1", +]) +def test_requested_sink_cannot_silently_fall_back(tmp_path, monkeypatch, missing): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + delattr(library, missing) + with pytest.raises(RuntimeError, match="requires the report sink ABI"): + strategy(library).run(bars.csv_file(tmp_path)) + assert "run_backtest_full" not in library.calls + assert "report_free" in library.calls and "strategy_free" in library.calls + + +def test_explicit_sink_off_needs_no_sink_exports(tmp_path, monkeypatch): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "0") + library = HarnessLibrary(trade_fixture()) + for name in ("pf_report_sink_version", "strategy_set_report_sink_v1", + "strategy_report_sink_receipt_v1"): + delattr(library, name) + report = strategy(library).run(bars.csv_file(tmp_path)) + assert report["trades"] == trade_fixture() + assert not library.installed + + +def test_csv_write_failure_still_closes_the_sink(tmp_path, monkeypatch): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + destination = tmp_path / "directory.csv" + destination.mkdir() + owners = [] + make_sink = rs._new_harness_report_sink + def track(*args): + sink = make_sink(*args) + owners.append(sink) + return sink + monkeypatch.setattr(rs, "_new_harness_report_sink", track) + with pytest.raises(OSError): + invoke_main(tmp_path, monkeypatch, library, destination) + assert library.installed + assert owners and all(sink.closed for sink in owners) + + +@pytest.mark.parametrize("keep", [False, True]) +def test_cli_rerun_uses_fresh_sinks_and_closes_both( + tmp_path, monkeypatch, sink_directory, capsys, keep): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + if keep: + monkeypatch.setenv(rs.KEEP_REPORT_SINK_ENV, "all") + library = HarnessLibrary(trade_fixture()) + owners = [] + make_sink = rs._new_harness_report_sink + def track(*args): + sink = make_sink(*args) + owners.append(sink) + return sink + monkeypatch.setattr(rs, "_new_harness_report_sink", track) + monkeypatch.setattr(rs, "_load_tv_entry_span", + lambda *_args: (bars.T0 + 240000, bars.T0 + 300000)) + monkeypatch.setattr(rs, "_apply_range_end_regime", lambda *_args: "fixture") + monkeypatch.setattr(rs, "_tv_first_bar", lambda *_args: (bars.T0, "fixture")) + def window(feed, first, last, interval, tv_first=None): + return rs.TvEntryWindow( + bars.T0 if tv_first is not None else bars.T0 + 180000, + last, bars.T0 + 120000, first, first, first - interval, + tv_first) + monkeypatch.setattr(rs, "_tv_entry_emit_window", window) + monkeypatch.setattr(rs, "_describe_tv_entry_window", lambda *_args: "fixture") + source = bars.csv_file(tmp_path, count=8) + probe = tmp_path / "probe" + probe.mkdir() + runner = strategy(library) + monkeypatch.setattr(rs, "Strategy", lambda _path: runner) + destination = tmp_path / "sink.csv" + monkeypatch.setattr(sys, "argv", [ + "run_strategy.py", str(probe), "--ohlcv", str(source), + "--output", str(destination), + ]) + assert rs.main() == 0 + assert library.calls.count("run_backtest_full") == 2 + assert len(library.installed) == 2 + assert library.installed[0] != library.installed[1] + assert len(owners) == 2 and all(sink.closed for sink in owners) + reference = tmp_path / "array.csv" + rs.write_engine_trades_csv( + rs._filter_trades_to_window( + trade_fixture(), (bars.T0 + 120000, bars.T0 + 300000)), + reference) + assert destination.read_bytes() == reference.read_bytes() + # Two sinks, one probe: the canary speaks once, for the final report. + assert len(canary_lines(capsys.readouterr().out)) == 1 + # Only the final report can be a kept artifact: the discarded rerun's + # sink is unlinked even when the switch names every probe. + leftovers = sorted(sink_directory.glob("pineforge-report-sink-*")) + try: + assert [path.name for path in leftovers] == ( + [Path(library.installed[1]).name] if keep else []) + finally: + for path in leftovers: + path.unlink() + + +# --- the portable sink: a named file, one offset per reader -------------------- + +class PathLibrary: + """Accepts any sink the owner installs and remembers the path it was given.""" + + def __init__(self, status=0): + self.status, self.paths = status, [] + + def strategy_set_report_sink_v1(self, state, pointer): + self.paths.append(os.fsdecode(pointer._obj.path)) + return self.status + + +def test_engine_gets_a_named_owner_only_file_and_every_reader_starts_at_zero( + tmp_path, monkeypatch, sink_directory): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + report = strategy(library).run(bars.csv_file(tmp_path, count=8)) + (seen,) = library.engine_view + path = Path(seen["path"]) + # What the engine was handed: a real, absolute, regular, owner-only file in + # the temp directory; never a descriptor path (/proc/self/fd, /dev/fd), + # whose open file description, and with it the offset, can be shared. + assert path.is_absolute() and path.parent == sink_directory + assert not seen["path"].startswith(("/proc/", "/dev/")) + assert seen["regular"] and seen["size"] == 0 + assert seen["mode"] & 0o077 == 0 + with report: + sink = report.sink + assert Path(sink.path) == path and path.is_file() + raw = path.read_bytes() + assert raw.endswith(b"\n") and len(raw) == library.sizes[0] > 0 + # Two live SinkReaders, advanced alternately: each owns its offset and + # sees every row from the start. + pairs = list(zip(sink.records("E"), sink.records("E"))) + assert len(pairs) == sink.counts["E"] == 8 + assert all(left.as_dict() == right.as_dict() for left, right in pairs) + # Plain opens of the real path behave the same: a shared offset would + # hand the second reader the file's second line. + with path.open("rb") as one, path.open("rb") as two: + head = one.readline() + assert head and head == two.readline() + assert one.read() == two.read() == raw[len(head):] + # The broker-hash export zips E with H, two readers live at once. The + # expected pairs come from the fixture's inputs, not from the output. + assert list(zip(report["equity_curve_time_ms"], + report["broker_state_hash"])) == [ + (bars.T0 + index * 60000, (1 << 63) + index) for index in range(8)] + # close() unlinks the file; a closed owner names nothing any more, so a + # view read after close fails loudly instead of reading a stranger's file. + assert sink.closed and not path.exists() + sink.close() + with pytest.raises(run_json.RunFailure, match="sink"): + list(sink.records("T")) + + +def test_sink_owner_unlinks_once_and_tolerates_a_vanished_file(sink_directory): + library = PathLibrary() + sink = run_json._TemporaryReportSink(library, 7, directory=sink_directory) + path = Path(library.paths[0]) + assert path == Path(sink.path) and path.is_file() and path.stat().st_size == 0 + assert sink.directory == str(sink_directory) + sink.close() + assert sink.closed and not path.exists() + sink.close() # idempotent + vanished = run_json._TemporaryReportSink(library, 7, directory=sink_directory) + os.unlink(vanished.path) + vanished.close() # FileNotFoundError is tolerated + assert vanished.closed + + +def test_sink_owner_keeps_its_file_only_when_asked(sink_directory): + sink = run_json._TemporaryReportSink(PathLibrary(), 7, directory=sink_directory) + sink.keep = True + sink.close() + path = Path(sink.path) + try: + assert sink.closed and path.is_file() + finally: + path.unlink() + + +def test_sink_owner_leaves_no_file_when_the_engine_refuses_it(sink_directory): + with pytest.raises(run_json.RunFailure, match="refused"): + run_json._TemporaryReportSink(PathLibrary(status=-1), 7, directory=sink_directory) + assert not list(sink_directory.iterdir()) + + +def test_sink_owner_without_a_directory_has_the_existing_failure_code(tmp_path): + with pytest.raises(run_json.RunFailure) as error: + run_json._TemporaryReportSink(PathLibrary(), 7, directory=tmp_path / "missing") + assert error.value.code == "report_sink_failed" + + +# --- the canary: every sink-on probe proves it read a real, complete sink ------- + +def test_cli_prints_one_canary_line_with_the_fixtures_own_counts( + tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + trades = trade_fixture() + library = HarnessLibrary(trades) + assert invoke_main(tmp_path, monkeypatch, library, tmp_path / "sink.csv") == 0 + lines = canary_lines(capsys.readouterr().out) + assert len(lines) == 1 + found = CANARY.fullmatch(lines[0]) + assert found and found["verdict"] == "ok" + assert found["probe"] == "probe" # the strategy directory's basename + assert int(found["trades"]) == int(found["records"]) == len(trades) + with (tmp_path / "chart.csv").open() as source: + assert int(found["equity"]) == sum(1 for _ in csv.DictReader(source)) + # The size the engine's sink had when the engine was freed, read from the + # file by the stand-in library, not taken from the harness's own output. + assert int(found["bytes"]) == library.sizes[0] > 0 + + +def test_canary_accepts_a_real_sink_that_holds_no_trades(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary([]) + source = bars.csv_file(tmp_path, count=8) + with strategy(library).run(source) as report: + rs._sink_canary("fixture", report) + (line,) = canary_lines(capsys.readouterr().out) + found = CANARY.fullmatch(line) + assert found and found["verdict"] == "ok" and found["probe"] == "fixture" + assert int(found["records"]) == int(found["trades"]) == 0 + with source.open() as feed: + assert int(found["equity"]) == sum(1 for _ in csv.DictReader(feed)) + assert int(found["bytes"]) == library.sizes[0] > 0 + + +def test_canary_checks_equity_against_the_saved_engine_receipt( + tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + source = bars.csv_file(tmp_path) + with source.open() as feed: + expected = sum(1 for _ in csv.DictReader(feed)) + library = HarnessLibrary(trade_fixture()) + with strategy(library).run(source) as report: + assert report.receipt_count_e == expected + # Neither summary script-bar counts nor cached reader counts are the + # canary's E oracle. The independent receipt still matches the file. + report["script_bars_processed"] += 1 + report.sink.counts["E"] += 1 + rs._sink_canary("fixture", report) + (line,) = canary_lines(capsys.readouterr().out) + found = CANARY.fullmatch(line) + assert found and found["verdict"] == "ok" + assert int(found["equity"]) == expected + + +@pytest.mark.parametrize("fault", ["refused", "incomplete", "negative"]) +def test_a_missing_engine_equity_receipt_fails_without_leaving_a_sink( + tmp_path, monkeypatch, sink_directory, fault): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + query = library.strategy_report_sink_receipt_v1 + + def fail_the_second_query(state, pointer): + result = query(state, pointer) + if library.queried == 2: + if fault == "refused": + return -1 + if fault == "incomplete": + pointer._obj.complete = 0 + else: + pointer._obj.count_e = -1 + return result + + monkeypatch.setattr(library, "strategy_report_sink_receipt_v1", fail_the_second_query) + with pytest.raises(RuntimeError, match="equity receipt is incomplete"): + strategy(library).run(bars.csv_file(tmp_path)) + assert library.queried == 2 + assert "report_free" in library.calls and "strategy_free" in library.calls + assert not list(sink_directory.iterdir()) + + +@pytest.mark.parametrize("damage", ["corrupt", "short"]) +def test_canary_fails_for_damaged_equity_even_when_every_trade_is_intact( + tmp_path, monkeypatch, capsys, damage): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + source = bars.csv_file(tmp_path) + with source.open() as feed: + expected = sum(1 for _ in csv.DictReader(feed)) + trades = trade_fixture() + library = HarnessLibrary(trades) + with strategy(library).run(source) as report: + rows = [[record.tag, *record.values] for record in report.sink.records()] + original_trades = [row for row in rows if row[0] == "T"] + last_equity = max(index for index, row in enumerate(rows) if row[0] == "E") + if damage == "corrupt": + rows[last_equity][1] = True # bool is invalid in E's i64 time field + else: + del rows[last_equity] + counts, sha = write_diagnostic_sink(report.sink.path, rows) + # Deliberately let the strict reader accept a newly sealed, short E + # sequence: only the independently saved engine count can reject it. + # Invalid E types must still fail strict decoding, with T untouched. + report.sink.sha256, report.sink.counts = sha, counts + assert report.receipt_count_e == expected + written = [json.loads(line) for line in Path(report.sink.path).read_text().splitlines()] + assert [row for row in written if isinstance(row, list) and row[0] == "T"] == original_trades + if damage == "short": + assert sum(1 for _ in report.sink.records("T")) == len(trades) + with pytest.raises(RuntimeError, match="sink canary failed"): + rs._sink_canary("fixture", report) + (line,) = canary_lines(capsys.readouterr().out) + found = CANARY.fullmatch(line) + assert found and found["verdict"].startswith("FAIL ") + if damage == "short": + assert int(found["records"]) == int(found["trades"]) == len(trades) + assert int(found["equity"]) == expected - 1 + assert f"{expected} engine equity points" in found["verdict"] + else: + assert "the sink cannot be read back" in found["verdict"] + + +@pytest.mark.parametrize("damage, reason", [ + ("empty", "the sink file is empty"), + ("truncated", "the sink cannot be read back"), + ("missing", "the sink cannot be read back"), + ("short", "T records decoded for"), +]) +def test_canary_fails_the_probe_for_an_empty_truncated_missing_or_short_sink( + tmp_path, monkeypatch, capsys, damage, reason): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + trades = trade_fixture() + library = HarnessLibrary(trades) + report = strategy(library).run(bars.csv_file(tmp_path, count=8)) + path = Path(report.sink.path) + with report: + if damage == "empty": + path.write_bytes(b"") + elif damage == "truncated": + content = path.read_bytes() + path.write_bytes(content[:len(content) // 2]) + elif damage == "missing": + path.unlink() + else: + # The engine reports one trade more than the sink holds. + report["total_trades"] += 1 + with pytest.raises(RuntimeError, match="sink canary failed"): + rs._sink_canary("fixture", report) + (line,) = canary_lines(capsys.readouterr().out) + found = CANARY.fullmatch(line) + assert found and found["verdict"].startswith("FAIL ") and reason in found["verdict"] + assert found["probe"] == "fixture" + assert int(found["trades"]) == len(trades) + (damage == "short") + assert int(found["records"]) == (len(trades) if damage == "short" else 0) + + +def test_cli_fails_the_probe_when_the_sink_is_lost_before_close( + tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + library = HarnessLibrary(trade_fixture()) + real_write = rs.write_engine_trades_csv + + def write_then_empty_the_sink(trades, path): + real_write(trades, path) + Path(trades.sink.path).write_bytes(b"") + + monkeypatch.setattr(rs, "write_engine_trades_csv", write_then_empty_the_sink) + with pytest.raises(RuntimeError, match="sink canary failed"): + invoke_main(tmp_path, monkeypatch, library, tmp_path / "sink.csv") + out = capsys.readouterr().out + (line,) = canary_lines(out) + found = CANARY.fullmatch(line) + assert found and found["verdict"] == "FAIL the sink file is empty" + assert (int(found["bytes"]), int(found["records"])) == (0, 0) + assert int(found["trades"]) == len(trade_fixture()) + assert "net_profit=" not in out, "a failed probe must not print its success line" + + +def test_sink_off_run_prints_no_canary(tmp_path, monkeypatch, capsys): + library = HarnessLibrary(trade_fixture()) + assert invoke_main(tmp_path, monkeypatch, library, tmp_path / "array.csv") == 0 + out = capsys.readouterr().out + assert "sink_canary:" not in out and "sink_kept:" not in out + assert not library.installed + + +# --- the test-only keep switch -------------------------------------------------- + +@pytest.mark.parametrize("value, kept", [ + ("all", True), ("probe", True), ("other, probe", True), + ("other", False), ("probe-2", False), ("", False), +]) +def test_keep_switch_keeps_the_files_of_the_probes_it_names( + tmp_path, monkeypatch, capsys, sink_directory, value, kept): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + monkeypatch.setenv(rs.KEEP_REPORT_SINK_ENV, value) + trades = trade_fixture() + library = HarnessLibrary(trades) + assert invoke_main(tmp_path, monkeypatch, library, tmp_path / "sink.csv") == 0 + out = capsys.readouterr().out + leftovers = sorted(sink_directory.glob("pineforge-report-sink-*")) + try: + if not kept: + assert not leftovers and "sink_kept:" not in out + else: + assert [path.name for path in leftovers] == [Path(library.installed[0]).name] + assert f"sink_kept: probe=probe path={leftovers[0]}" in out.splitlines() + # The artifact is the complete sink the probe verified: non-empty, + # with a trailer, and as many T rows as the fixture has trades. + with pf_sink.SinkReader(leftovers[0]) as reader: + assert sum(1 for record in reader if record.tag == "T") == len(trades) + assert reader.complete and leftovers[0].stat().st_size == library.sizes[0] > 0 + finally: + for path in leftovers: + path.unlink() + + +def test_keep_switch_is_off_unless_it_names_the_probe(): + assert rs._keep_report_sink("greedy", {}) is False + assert rs._keep_report_sink("greedy", {rs.KEEP_REPORT_SINK_ENV: " "}) is False + assert rs._keep_report_sink("greedy", {rs.KEEP_REPORT_SINK_ENV: "all"}) is True + assert rs._keep_report_sink("greedy", {rs.KEEP_REPORT_SINK_ENV: "a, greedy ,b"}) is True + assert rs._keep_report_sink("greedy", {rs.KEEP_REPORT_SINK_ENV: "greedy-2,gree"}) is False + + +def test_probe_id_is_the_directory_basename_without_whitespace(): + assert rs._probe_id(Path("/corpus/basic/greedy")) == "greedy" + assert rs._probe_id(Path("/corpus/my probe")) == "my_probe" + assert rs._probe_id(Path("/")) == "-" + + +# --- sweep logging: use actual fixture canaries, never hand-written counts ----- + +CANARY_TOOL = ROOT / "scripts" / "check_sink_canary_log.py" + + +@pytest.fixture +def sweep_probe_outputs(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PINEFORGE_REPORT_SINK", "1") + source = bars.csv_file(tmp_path) + outputs = {} + for probe in ("alpha", "beta", "broken"): + library = HarnessLibrary(trade_fixture()) + with strategy(library).run(source) as report: + # Exercise real sink_kept output too, then let context exit remove + # the fixture file; these tests need only the captured output. + report.sink.keep = True + try: + if probe == "broken": + report["total_trades"] += 1 + with pytest.raises(RuntimeError, match="sink canary failed"): + rs._sink_canary(probe, report) + else: + rs._sink_canary(probe, report) + finally: + report.sink.keep = False + outputs[probe] = capsys.readouterr().out + (line,) = canary_lines(outputs[probe]) + found = CANARY.fullmatch(line) + assert found and found["probe"] == probe + assert (found["verdict"] == "ok") == (probe != "broken") + return outputs + + +def run_canary_tool(*args, input=None): + return subprocess.run( + [sys.executable, "-I", str(CANARY_TOOL), *map(str, args)], + input=input, capture_output=True, text=True, check=False, + ) + + +@pytest.mark.parametrize("from_files", [False, True]) +def test_canary_log_writer_appends_passing_and_failing_lines_only( + tmp_path, sweep_probe_outputs, from_files): + log = tmp_path / "canaries.log" + expected = "" + for probe in ("alpha", "broken"): + output = sweep_probe_outputs[probe] + mixed = ("ordinary runner output\n sink_canary: indented noise\n" + "sink_canary_extra: noise\n" + output + "net_profit=ignored\n") + if from_files: + path = tmp_path / f"{probe}.log" + path.write_text(mixed, encoding="utf-8") + result = run_canary_tool("append", log, path) + else: + result = run_canary_tool("append", log, input=mixed) + assert result.returncode == 0, result.stderr + expected += output + assert log.read_text(encoding="utf-8") == expected + assert "FAIL " in expected and " ok\n" in expected and "sink_kept:" in expected + + +@pytest.mark.parametrize("probe_source", ["arguments", "file"]) +def test_canary_log_checker_accepts_exact_probe_coverage( + tmp_path, sweep_probe_outputs, probe_source): + probes = ["alpha", "beta"] + log = tmp_path / "canaries.log" + for probe in probes: + canary_log.append_lines(log, sweep_probe_outputs[probe].splitlines(True)) + if probe_source == "file": + path = tmp_path / "ran-probes.txt" + path.write_text("# the sweep's actual selection\n" + "".join( + f"corpus/validation/{probe}/ # ran this probe\n" for probe in probes), + encoding="utf-8") + result = run_canary_tool("check", log, "--probes-file", path) + else: + result = run_canary_tool( + "check", log, "--probes", *(f"corpus/validation/{probe}" for probe in probes)) + assert result.returncode == 0, result.stderr + assert f"ok probes={len(probes)}" in result.stdout + + +@pytest.mark.parametrize("damage, offenders", [ + ("missing", ("beta",)), + ("duplicate", ("alpha",)), + ("wrong-names-same-count", ("alpha", "beta")), + ("failed", ("broken",)), + ("unexpected", ("beta",)), + ("malformed", ("beta",)), + ("missing-equity-field", ("beta",)), + ("duplicate-expectation", ("alpha",)), +]) +def test_canary_log_checker_names_every_offending_probe( + tmp_path, sweep_probe_outputs, damage, offenders): + probes = ["alpha", "beta"] + outputs = [sweep_probe_outputs[probe] for probe in probes] + if damage == "missing": + outputs.pop() + elif damage == "duplicate": + outputs.append(outputs[0]) + elif damage == "wrong-names-same-count": + outputs[1] = outputs[0] + elif damage == "failed": + probes.append("broken") + outputs.append(sweep_probe_outputs["broken"]) + elif damage == "unexpected": + probes.pop() + elif damage == "malformed": + outputs[1] = outputs[1].replace(" ok\n", " unrecognized\n") + elif damage == "missing-equity-field": + outputs[1] = re.sub(r" equity=\d+", "", outputs[1]) + else: + probes.append(probes[0]) + log = tmp_path / "canaries.log" + canary_log.append_lines(log, "".join(outputs).splitlines(True)) + result = run_canary_tool("check", log, "--probes", *probes) + assert result.returncode == 1, result.stdout + result.stderr + for probe in offenders: + assert f"probe={probe}:" in result.stderr + + +def corpus_driver_fixture(tmp_path, outputs): + """Run the real shell driver with native runs replaying fixture output. + + No build, feed derivation, report generation or parity script executes. + Only the new canary collector/checker runs as a real Python child. + """ + root = tmp_path / "sweep" + scripts = root / "scripts" + scripts.mkdir(parents=True) + shutil.copy2(ROOT / "scripts" / "run_corpus.sh", scripts / "run_corpus.sh") + shutil.copy2(CANARY_TOOL, scripts / CANARY_TOOL.name) + corpus = root / "corpus" + corpus.mkdir() + (corpus / "CMakeLists.txt").write_text("# fixture\n") + output_dir = root / "outputs" + output_dir.mkdir() + for probe, output in outputs.items(): + directory = corpus / "validation" / probe + directory.mkdir(parents=True) + (directory / "generated.cpp").write_text("// fixture\n") + (directory / "strategy.so").write_bytes(b"fixture") + (output_dir / f"{probe}.log").write_text( + "runner output before the canary\n" + output + "runner output after it\n", + encoding="utf-8") + runner = scripts / "fixture-python" + runner.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "if [[ \"$1\" == -I ]]; then\n" + " exec \"$PF_CANARY_TEST_PYTHON\" \"$@\"\n" + "fi\n" + "case \"$1\" in\n" + " scripts/run_strategy.py)\n" + " probe=\"${2##*/}\"\n" + " cat \"$PF_CANARY_TEST_OUTPUTS/$probe.log\"\n" + " [[ \"$probe\" != broken ]]\n" + " ;;\n" + " scripts/derive_corpus_feeds.py|scripts/verify_corpus.py|" + "scripts/regen_validation_report.py|scripts/validation_report_self_test.py)\n" + " ;;\n" + " *) exit 99 ;;\n" + "esac\n", + encoding="utf-8") + runner.chmod(0o755) + env = os.environ.copy() + for name in ("PINEFORGE_CANARY_LOG", "ONLY", "RUNNER", "REGEN", + "SKIP_RUN", "SKIP_BUILD", "SKIP_VERIFY"): + env.pop(name, None) + env.update({ + "PYTHON": str(runner), "SKIP_BUILD": "1", "SKIP_VERIFY": "1", + "PF_CANARY_TEST_PYTHON": sys.executable, + "PF_CANARY_TEST_OUTPUTS": str(output_dir), + }) + return root, env + + +def run_corpus_driver(root, env): + return subprocess.run( + ["bash", str(root / "scripts" / "run_corpus.sh")], cwd=root, env=env, + capture_output=True, text=True, check=False, + ) + + +@pytest.mark.parametrize("failed", [False, True]) +def test_full_sweep_truncates_once_and_keeps_passing_and_failing_canaries( + tmp_path, sweep_probe_outputs, failed): + probes = ["alpha", "beta"] + (["broken"] if failed else []) + outputs = {probe: sweep_probe_outputs[probe] for probe in probes} + root, env = corpus_driver_fixture(tmp_path, outputs) + log = root / "canaries.log" + log.write_text("stale run must be removed\n", encoding="utf-8") + env["PINEFORGE_CANARY_LOG"] = str(log) + result = run_corpus_driver(root, env) + assert result.returncode == int(failed), result.stdout + result.stderr + expected = "".join(outputs[probe] for probe in sorted(probes)) + assert log.read_text(encoding="utf-8") == expected + checked = run_canary_tool("check", log, "--probes", *probes) + assert checked.returncode == int(failed) + if failed: + assert "probe=broken:" in checked.stderr + assert "refusing to verify stale engine_trades.csv" in result.stderr + else: + assert f"ok probes={len(probes)}" in result.stdout + + +def test_full_sweep_checks_the_filtered_selection_and_rejects_missing_canaries( + tmp_path, sweep_probe_outputs): + outputs = {probe: sweep_probe_outputs[probe] for probe in ("alpha", "beta")} + root, env = corpus_driver_fixture(tmp_path, outputs) + log = root / "canaries.log" + env.update({"PINEFORGE_CANARY_LOG": str(log), "ONLY": "beta"}) + result = run_corpus_driver(root, env) + assert result.returncode == 0, result.stdout + result.stderr + assert log.read_text(encoding="utf-8") == outputs["beta"] + # A runner silently taking its array path must fail coverage even though + # it reports success. Start another sweep with no real canary output. + (root / "outputs" / "beta.log").write_text("net_profit=ignored\n") + result = run_corpus_driver(root, env) + assert result.returncode == 1 + assert log.read_text(encoding="utf-8") == "" + assert "probe=beta:" in result.stderr + + +def test_full_sweep_without_a_canary_log_keeps_existing_output_behavior( + tmp_path, sweep_probe_outputs): + outputs = {"alpha": sweep_probe_outputs["alpha"]} + root, env = corpus_driver_fixture(tmp_path, outputs) + untouched = root / "canaries.log" + previous = b"an unrelated file\n" + untouched.write_bytes(previous) + result = run_corpus_driver(root, env) + assert result.returncode == 0, result.stdout + result.stderr + assert untouched.read_bytes() == previous + assert "sink_canary:" not in result.stdout + assert "sink_kept:" not in result.stdout + assert "sink_canary_log:" not in result.stdout + result.stderr