"""Approve a signal only if the same strategy has worked on this symbol.
Deliberately conservative and deliberately dumb: it asks one question — would
this strategy have made money on this symbol recently, without an unacceptable
drawdown — and answers it with the same engine `trader backtest` uses. Its
value is that it is reproducible, not that it is clever. The LLM evaluator that
replaces it next slice will be judged against these numbers.
"""
import logging
from collections.abc import Sequence
from trader.backtest.engine import BacktestConfig, run_backtest
from trader.config.schema import PipelineConfig
from trader.domain import Bar
from trader.errors import BacktestError
from trader.evaluation.base import Verdict
from trader.strategies.base import Signal, Strategy
__all__ = ["BacktestEvaluator"]
_logger = logging.getLogger("trader.evaluation")
[docs]
class BacktestEvaluator:
"""Gates a signal on a backtest of the same strategy over the same bars."""
def __init__(
self,
strategy: Strategy,
config: PipelineConfig,
backtest_config: BacktestConfig | None = None,
) -> None:
self._strategy = strategy
self._config = config
self._backtest_config = backtest_config or BacktestConfig()
[docs]
def evaluate(self, symbol: str, bars: Sequence[Bar], signal: Signal) -> Verdict:
"""Run the backtest and turn its metrics into an approval decision."""
try:
result = run_backtest(self._strategy, symbol, bars, self._backtest_config)
except BacktestError as exc:
# Neither a crash nor an approval: too little history to judge is a
# reason to decline, recorded as such.
return Verdict(
approved=False,
reason=f"Cannot evaluate {symbol}: {exc}",
evidence={"bars_evaluated": len(bars)},
)
# Every value is JSON-safe: this dict is written to
# `decisions.inputs_json`, and a raw Decimal would raise at
# serialisation time on the one path that matters.
evidence: dict[str, object] = {
"total_return_pct": str(result.total_return_pct),
"max_drawdown_pct": str(result.max_drawdown_pct),
# Recorded, deliberately not gated on. Every stored decision keeps
# the do-nothing result beside the strategy's, so "did this approval
# beat holding?" is a query rather than a re-run against bars Yahoo
# has since revised. Making it a *threshold* would change which
# trades this app places and belongs to its own change, decided on
# the operator's answer to "beat the market, or reduce risk?".
"benchmark_return_pct": str(result.benchmark_return_pct),
"excess_return_pct": str(result.excess_return_pct),
"win_count": result.win_count,
"loss_count": result.loss_count,
"trade_count": len(result.trades),
"bars_evaluated": len(bars),
"strategy_id": result.strategy_id,
}
if not result.trades:
# Zero trades produces a 0.00% return, which would otherwise clear
# a `>= 0` threshold. "Nothing happened" is not evidence that the
# strategy works here — it is the absence of evidence either way.
return Verdict(
approved=False,
reason=(
f"{self._strategy.id} produced no trades on {symbol} over "
f"{len(bars)} bars, so there is no evidence either way."
),
evidence=evidence,
)
if result.total_return_pct < self._config.min_backtest_return_pct:
return Verdict(
approved=False,
reason=(
f"Backtest return {result.total_return_pct:.2f}% is below the "
f"required {self._config.min_backtest_return_pct}%."
),
evidence=evidence,
)
if result.max_drawdown_pct > self._config.max_backtest_drawdown_pct:
# Checked separately from the return: a strategy can finish well up
# having passed through a decline no one would have held through,
# and total return alone cannot see the path.
return Verdict(
approved=False,
reason=(
f"Backtest max drawdown {result.max_drawdown_pct:.2f}% exceeds "
f"the permitted {self._config.max_backtest_drawdown_pct}%."
),
evidence=evidence,
)
reason = (
f"{self._strategy.id} returned {result.total_return_pct:.2f}% on "
f"{symbol} over {len(bars)} bars with a "
f"{result.max_drawdown_pct:.2f}% max drawdown across "
f"{len(result.trades)} trades."
)
_logger.info("evaluator approved %s: %s", symbol, reason)
return Verdict(approved=True, reason=reason, evidence=evidence)