r"""Converts third-party historical bar exports into domain `Bar`\s (issue #41).
`trader seed-bars` reads these functions' output through the same
`BarRepository.save_bars` path `YFinanceProvider`/`BarCache` write through, so
an imported bar is indistinguishable in shape from a live-fetched one — only
`CachedBar.source` tells them apart, per that column's own docstring in
`trader/persistence/models.py`.
**Built and tested against synthetic fixtures, not the real dataset.** The
motivating find (`../backtesting`: 264MB of Feather-cached OHLCV across 21
tickers, plus a 145MB Interactive Brokers 1-minute SPY export) was not
reachable from the checkout this was built in. Every parsing decision below is
therefore pinned against the *documented* shape from issue #41 and this
module's own tests, not against a real file. Anyone pointing this at the real
data for the first time should expect to adjust `_TIMESTAMP_CANDIDATES`/
`_CLOSE_CANDIDATES`/the IB date-format list below to match what the files
actually contain, and to widen the fixtures in
`tests/marketdata/test_seed_import.py` once a real sample is available.
Two independent parsers, deliberately not unified behind one "detect the
format" entry point: the two source files come from two different vendors
with two different shapes (yfinance's `Open`/`High`/`Low`/`Close`/`Volume`
columns and a pandas-native timestamp vs. Interactive Brokers'
`date`/`open`/.../`average`/`barCount` columns and an exchange-local string
timestamp), and guessing which one a file is would be exactly the kind of
silent misinterpretation `_to_decimal` already refuses to do for a bare float.
The caller (`trader seed-bars --format ...`) states the format; nothing here
sniffs it.
"""
from __future__ import annotations
from datetime import UTC, datetime
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
from pathlib import Path
from zoneinfo import ZoneInfo
from trader.domain import Bar
from trader.errors import MarketDataError
__all__ = [
"SOURCE_IBKR_SEED",
"SOURCE_YFINANCE_SEED",
"load_feather_bars",
"load_ibkr_csv_bars",
]
#: Tags a row imported from the 21-ticker Feather cache — built, per issue
#: #41, by the source repo's own `download.py` calling yfinance directly, so
#: it is directly comparable to what `YFinanceProvider` fetches live, just
#: years deeper. Still tagged, never left `source=None`: `None` means "this
#: app fetched it live", and a seeded row did not, regardless of how similar
#: the vendor is.
SOURCE_YFINANCE_SEED = "yfinance_seed"
#: Tags a row imported from an Interactive Brokers historical-data export
#: (`barCount`/`average` columns are IB's own fingerprint, per issue #41) — a
#: different vendor with different adjustment/aggregation conventions than
#: yfinance. `source` is what stops this silently sharing an unlabelled
#: series with `SOURCE_YFINANCE_SEED` rows or a live fetch; the three must
#: stay distinguishable from each other, not just seed-vs-live.
SOURCE_IBKR_SEED = "ibkr_seed"
_PRICE_PRECISION = Decimal("0.0001")
# yfinance's own `download()`/`Ticker.history()` naming, checked in this
# order. "Adj Close" (or its lowercase form) is preferred over "Close" when
# present: this app's own stored bars are dividend-adjusted throughout
# (`YFinanceProvider._history_frame` pins `auto_adjust=True`), so preferring
# the adjusted close keeps a seeded row on the same basis as a live-fetched
# one for the same symbol/interval. Newer yfinance releases fold the
# adjustment into "Close" directly and drop "Adj Close" altogether, which is
# exactly why this checks for "Adj Close" rather than assuming it exists.
_CLOSE_CANDIDATES: tuple[str, ...] = ("Adj Close", "adj close", "Close", "close")
_OPEN_CANDIDATES: tuple[str, ...] = ("Open", "open")
_HIGH_CANDIDATES: tuple[str, ...] = ("High", "high")
_LOW_CANDIDATES: tuple[str, ...] = ("Low", "low")
_VOLUME_CANDIDATES: tuple[str, ...] = ("Volume", "volume")
_TIMESTAMP_CANDIDATES: tuple[str, ...] = (
"Date",
"Datetime",
"date",
"datetime",
"Timestamp",
"timestamp",
"index",
)
# Interactive Brokers' own column names for the same five OHLCV fields, plus
# the two provenance columns (`average`/`barCount`, sometimes `wap` instead of
# `average`) issue #41 names as the vendor's fingerprint. The fingerprint
# columns are read only to confirm the file looks like an IB export; their
# values are never stored, because `CachedBar` has no field for a volume-
# weighted average price or a per-bar tick count.
_IBKR_TIMESTAMP_CANDIDATES: tuple[str, ...] = ("date", "Date", "datetime", "Datetime")
_IBKR_OPEN_CANDIDATES: tuple[str, ...] = ("open", "Open")
_IBKR_HIGH_CANDIDATES: tuple[str, ...] = ("high", "High")
_IBKR_LOW_CANDIDATES: tuple[str, ...] = ("low", "Low")
_IBKR_CLOSE_CANDIDATES: tuple[str, ...] = ("close", "Close")
_IBKR_VOLUME_CANDIDATES: tuple[str, ...] = ("volume", "Volume")
_IBKR_FINGERPRINT_CANDIDATES: tuple[str, ...] = ("average", "Average", "wap", "WAP")
_IBKR_BARCOUNT_CANDIDATES: tuple[str, ...] = ("barCount", "BarCount", "bar_count")
# IB's `reqHistoricalData(formatDate=1)` default, "yyyyMMdd HH:mm:ss" (two
# spaces) in the contract's exchange timezone. Tried first because it is the
# format issue #41's `barCount`/`average` fingerprint implies; `%Y%m%d` alone
# covers a daily-bar export with no time component.
_IBKR_DATETIME_FORMATS: tuple[str, ...] = ("%Y%m%d %H:%M:%S", "%Y%m%d %H:%M:%S")
_IBKR_DATE_ONLY_FORMAT = "%Y%m%d"
def _find_column(
columns: object, candidates: tuple[str, ...], *, what: str, symbol: str
) -> str:
"""The first of `candidates` present in `columns`, or a named failure.
A guessed column is worse than refusing — the same reasoning
`bar_width()` gives for refusing an unknown interval rather than
defaulting one.
"""
for name in candidates:
if name in columns:
return name
raise MarketDataError(
f"{symbol}: could not find a {what} column among {list(columns)!r}; "
f"expected one of {candidates!r}."
)
def _to_decimal(value: object, field: str, symbol: str) -> Decimal:
"""Convert a pandas/numpy scalar to a finite `Decimal`, rounded to 4dp.
Mirrors `brokers/alpaca.py::_to_decimal` and `YFinanceProvider`'s own
helper of the same name: convert via `str()`, never `Decimal(float_value)`
directly, so a float's own binary imprecision (`Decimal(190.24)` is
`Decimal('190.2399999999999...')`) never reaches a money column.
`DecimalText` rejects a bare float outright; this is what stands between a
raw Feather/CSV float and that column.
"""
try:
raw = Decimal(str(value))
except (InvalidOperation, TypeError, ValueError) as exc:
raise MarketDataError(f"{symbol}: unparsable {field}: {value!r}") from exc
if not raw.is_finite():
raise MarketDataError(f"{symbol}: non-finite {field}: {value!r}")
try:
return raw.quantize(_PRICE_PRECISION, rounding=ROUND_HALF_UP)
except InvalidOperation as exc:
raise MarketDataError(
f"{symbol}: {field} too large to represent: {value!r}"
) from exc
def _to_volume(value: object, symbol: str) -> int:
"""Convert a pandas/numpy scalar to `int`, refusing NaN/inf/None.
`int(float("nan"))` raises `ValueError`, `int(None)` raises `TypeError`,
and `int(float("inf"))` raises `OverflowError` — all three surface here as
`MarketDataError` rather than as a raw traceback out of the CLI.
"""
try:
return int(value) # type: ignore[call-overload]
except (TypeError, ValueError, OverflowError) as exc:
raise MarketDataError(f"{symbol}: unusable volume: {value!r}") from exc
def _select_symbol_slice(frame: object, symbol: str) -> object:
"""Narrow a possibly multi-ticker Feather frame to one symbol's columns.
`yfinance.download()` returns a `MultiIndex`-columned frame when called
with more than one ticker at once, in either `(field, ticker)` or
`(ticker, field)` order depending on the yfinance release. If the source
repo's `download.py` fetched several of its 21 tickers in one call before
writing one Feather file per ticker, a single-ticker file would carry no
MultiIndex at all — this only activates when one genuinely is present, and
is a no-op otherwise.
"""
import pandas as pd
if not isinstance(frame.columns, pd.MultiIndex):
return frame
for level in range(frame.columns.nlevels):
if symbol in frame.columns.get_level_values(level):
return frame.xs(symbol, axis=1, level=level)
raise MarketDataError(
f"{symbol}: not found in this file's multi-ticker columns "
f"{list(frame.columns)!r}."
)
def _timestamp_column_or_index(frame: object, symbol: str) -> object:
"""The frame's timestamp values, as a pandas Series, from a column or the index.
Feather has no native index-preservation guarantee, so `download.py`
almost certainly called `reset_index()` before writing — which turns the
`DatetimeIndex` yfinance returns into a plain column named `Date` (daily)
or `Datetime` (intraday). A file that somehow kept the index instead is
also handled, since nothing here can rule that out without a real sample.
"""
import pandas as pd
for name in _TIMESTAMP_CANDIDATES:
if name in frame.columns:
return frame[name]
if isinstance(frame.index, pd.DatetimeIndex):
return frame.index.to_series(index=frame.index)
raise MarketDataError(
f"{symbol}: could not find a timestamp column among "
f"{list(frame.columns)!r}; expected one of {_TIMESTAMP_CANDIDATES!r}."
)
def _yfinance_timestamp_to_utc(value: object, symbol: str) -> datetime:
"""Convert one pandas Timestamp to aware UTC.
A naive value is treated as UTC — the same convention
`YFinanceProvider._to_utc_datetime` uses for the identical ambiguity, so a
seeded row and a live-fetched one interpret a naive yfinance timestamp
identically rather than disagreeing by whatever the local clock happens
to be. `NaT` gets the same self-inequality check that function uses,
for the same reason: `pd.NaT.to_pydatetime()` returns `NaT` again rather
than raising.
"""
try:
converted = value.to_pydatetime()
except AttributeError as exc:
raise MarketDataError(
f"{symbol}: unusable timestamp (not a pandas Timestamp): {value!r}"
) from exc
if converted != converted: # noqa: PLR0124 - NaT/NaN self-inequality
raise MarketDataError(f"{symbol}: unusable (NaT) timestamp: {value!r}")
if converted.tzinfo is None:
return converted.replace(tzinfo=UTC)
return converted.astimezone(UTC)
[docs]
def load_feather_bars(path: Path, symbol: str) -> list[Bar]:
r"""Parse one of `../backtesting`\'s cached yfinance Feather files into `Bar`\s.
Documented shape (issue #41): one Feather file per ticker per interval,
written by the source repo's own `download.py` from
`yfinance.download()`/`Ticker.history()` output — columns named
`Open`/`High`/`Low`/`Close`[/`Adj Close`]/`Volume`, with the timestamp
surviving `reset_index()` as a `Date`/`Datetime` column (or, if it was not
reset, still present as the frame's own `DatetimeIndex`).
Raises:
MarketDataError: the file cannot be read, a required column cannot be
found, or a value cannot be converted — never a raw pandas/pyarrow
exception or a silently wrong `Bar`.
"""
import pandas as pd
ticker = symbol.strip().upper()
try:
frame = pd.read_feather(path)
except Exception as exc: # noqa: BLE001 - pyarrow/pandas raise several types
raise MarketDataError(
f"{ticker}: could not read Feather file {path}: {exc}"
) from exc
# The timestamp must be read from the *un-narrowed* frame: in a
# multi-ticker file the timestamp is a plain top-level column shared by
# every ticker, sitting outside the `MultiIndex` `_select_symbol_slice`
# narrows into — narrowing first would drop it along with every other
# ticker's columns.
timestamps = _timestamp_column_or_index(frame, ticker)
frame = _select_symbol_slice(frame, ticker)
open_col = _find_column(frame.columns, _OPEN_CANDIDATES, what="open", symbol=ticker)
high_col = _find_column(frame.columns, _HIGH_CANDIDATES, what="high", symbol=ticker)
low_col = _find_column(frame.columns, _LOW_CANDIDATES, what="low", symbol=ticker)
close_col = _find_column(
frame.columns, _CLOSE_CANDIDATES, what="close", symbol=ticker
)
volume_col = _find_column(
frame.columns, _VOLUME_CANDIDATES, what="volume", symbol=ticker
)
bars: list[Bar] = []
for i, row in frame.iterrows():
bars.append(
Bar(
symbol=ticker,
timestamp=_yfinance_timestamp_to_utc(timestamps.loc[i], ticker),
open=_to_decimal(row[open_col], "open", ticker),
high=_to_decimal(row[high_col], "high", ticker),
low=_to_decimal(row[low_col], "low", ticker),
close=_to_decimal(row[close_col], "close", ticker),
volume=_to_volume(row[volume_col], ticker),
)
)
return bars
def _parse_ibkr_timestamp(raw: str, symbol: str, tz: ZoneInfo) -> datetime:
"""Parse one Interactive Brokers timestamp string into aware UTC.
IB's `reqHistoricalData` with `formatDate=1` (this dataset's likely
origin — see the module docstring) reports the bar's own exchange-local
time as `"yyyyMMdd HH:mm:ss"` (two spaces) for an intraday bar, or a bare
`"yyyyMMdd"` for a daily one, with no timezone embedded in the string
itself. `tz` (default `America/New_York`, the primary listing venue for
every one of the 21 Feather tickers and for SPY) supplies the zone that
string is implicitly in, before converting to UTC.
"""
text = raw.strip()
for fmt in _IBKR_DATETIME_FORMATS:
try:
# Deliberately naive: IB's string carries no zone of its own, and
# `tz` supplies it two lines below via `.replace(tzinfo=tz)` —
# the DTZ007 lint is right in general but has no way to see the
# `.replace` that immediately follows in the caller.
naive = datetime.strptime(text, fmt) # noqa: DTZ007
break
except ValueError:
continue
else:
try:
naive = datetime.strptime(text, _IBKR_DATE_ONLY_FORMAT) # noqa: DTZ007
except ValueError as exc:
raise MarketDataError(
f"{symbol}: unparsable IB timestamp {raw!r}; expected one of "
f"{[*_IBKR_DATETIME_FORMATS, _IBKR_DATE_ONLY_FORMAT]!r}."
) from exc
return naive.replace(tzinfo=tz).astimezone(UTC)
[docs]
def load_ibkr_csv_bars(
path: Path, symbol: str, *, tz: str = "America/New_York"
) -> list[Bar]:
r"""Parse an Interactive Brokers historical-data CSV export into `Bar`\s.
Documented shape (issue #41): `date,open,high,low,close,volume,average,
barCount` (column order not assumed — read by name), one file per
symbol, timestamps in the contract's exchange-local time with no
embedded zone (`tz` supplies it; see `_parse_ibkr_timestamp`).
`average`/`barCount` (or `wap` in place of `average`) are IB's own
fingerprint per issue #41 and are read only to fail loudly if genuinely
absent from *both* names — see below — never stored, since `CachedBar`
has no field for either.
The fingerprint check is soft on `average`/`wap` specifically (a stripped
export might omit it) but the presence of at least one recognisable
volume-or-count column beyond plain OHLCV is not required either; this
function accepts a bare `date,open,high,low,close,volume` file too, so a
trimmed export is not refused for missing metadata nobody asked this
importer to store.
Raises:
MarketDataError: the file cannot be read, a required OHLCV column
cannot be found, or a value cannot be converted.
"""
import pandas as pd
ticker = symbol.strip().upper()
try:
frame = pd.read_csv(path)
except Exception as exc: # noqa: BLE001 - pandas raises several types
raise MarketDataError(f"{ticker}: could not read CSV file {path}: {exc}") from exc
timestamp_col = _find_column(
frame.columns, _IBKR_TIMESTAMP_CANDIDATES, what="timestamp", symbol=ticker
)
open_col = _find_column(
frame.columns, _IBKR_OPEN_CANDIDATES, what="open", symbol=ticker
)
high_col = _find_column(
frame.columns, _IBKR_HIGH_CANDIDATES, what="high", symbol=ticker
)
low_col = _find_column(frame.columns, _IBKR_LOW_CANDIDATES, what="low", symbol=ticker)
close_col = _find_column(
frame.columns, _IBKR_CLOSE_CANDIDATES, what="close", symbol=ticker
)
volume_col = _find_column(
frame.columns, _IBKR_VOLUME_CANDIDATES, what="volume", symbol=ticker
)
zone = ZoneInfo(tz)
# Read the timestamp column on its own, as a string, before iterating rows.
# `DataFrame.iterrows()` builds each row as a single-dtype Series, so a
# bare-digits date column (a daily export's `20090522`, `int64`) sitting
# beside float OHLC columns gets silently upcast to float64 per row —
# `str(row["date"])` then reads `"20090522.0"`, which `strptime` refuses.
# Reading the column directly, before that per-row upcast ever happens,
# is what `_timestamp_column_or_index` already does for the Feather side.
timestamps = frame[timestamp_col].astype(str)
bars: list[Bar] = []
for i, row in frame.iterrows():
bars.append(
Bar(
symbol=ticker,
timestamp=_parse_ibkr_timestamp(timestamps.loc[i], ticker, zone),
open=_to_decimal(row[open_col], "open", ticker),
high=_to_decimal(row[high_col], "high", ticker),
low=_to_decimal(row[low_col], "low", ticker),
close=_to_decimal(row[close_col], "close", ticker),
volume=_to_volume(row[volume_col], ticker),
)
)
return bars