Source code for trader.ideas.scraper

"""Fetch one web page and ask a local LLM what this app could build from it.

**The model never fetches anything itself.** Issue #75's investigation gave a
large "thinking" model the tool-calling wheel for fetching via `ironclaw` (a
separate agentic framework) and hit a real wall: first-token latency on a
tool-heavy request blew past the framework's fixed client timeout and produced
an infinite retry loop, not a clean failure. The fix, proven end to end in a
throwaway prototype before this module existed, is the same shape
`LlmStrategy` already uses: fetching happens in plain code, the model only
ever sees one finished prompt and writes text back. `synthesize_ideas` below
sends no `tools` field, ever.

**Stdlib `urllib` on purpose**, same reasoning as `trader.llm.ollama`: adding
`httpx` to the runtime dependency list for one GET would be a new dependency
this project does not otherwise carry.
"""

import logging
import re
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path

from bs4 import BeautifulSoup

from trader.errors import TraderError
from trader.llm.base import LlmProvider

__all__ = [
    "ExtractedPage",
    "IdeasScraperError",
    "extract_text",
    "fetch_page",
    "scrape_ideas",
    "synthesize_ideas",
    "write_ideas_markdown",
]

_logger = logging.getLogger("trader.ideas")

_USER_AGENT = (
    "claude-alpaca-trader-ideas-scraper/1.0 "
    "(+https://gitlab.com/hulko/claude-alpaca-trader)"
)

#: Tags whose content is never prose worth mining — navigation chrome, embedded
#: scripts/styles, and forms. Removed before text extraction so the model's
#: prompt is not padded with menu labels and cookie-banner copy.
_STRIP_TAGS = (
    "script",
    "style",
    "nav",
    "header",
    "footer",
    "aside",
    "noscript",
    "form",
    "title",
)

_WHITESPACE_RE = re.compile(r"\s+")
_SLUG_RE = re.compile(r"[^a-z0-9]+")

_SYSTEM_PROMPT = (
    "You are a product analyst for claude-alpaca-trader, an AI-assisted "
    "automated stock trading application. You read external content - blog "
    "posts, architecture writeups, strategy explainers - and extract "
    "concrete, product-specific feature or capability ideas this codebase "
    "could implement. Ground every idea in something the source text "
    "actually says; do not invent generic trading advice the source never "
    "mentioned. Write your reply in Markdown: a short intro paragraph, then "
    "one section per idea with a heading, a 2-4 sentence explanation, and a "
    "one-line note on why it is grounded in the source. If the page has "
    "nothing relevant to a trading app, say so plainly instead of inventing "
    "ideas."
)

_SCHEMA: dict[str, object] = {
    "type": "object",
    "properties": {"markdown": {"type": "string"}},
    "required": ["markdown"],
}




class IdeasScraperError(TraderError):
    """Fetching the page, extracting its text, or synthesizing ideas failed."""





@dataclass(frozen=True)
class ExtractedPage:
    title: str
    text: str



def _open_url(request: urllib.request.Request, timeout: int):
    """GET a URL. A seam, so tests need no network — same pattern as
    `OllamaProvider._open`."""
    return urllib.request.urlopen(request, timeout=timeout)  # noqa: S310




def fetch_page(url: str, *, timeout_seconds: int) -> str:
    """Fetch `url` and return its raw HTML.

    Raises:
        IdeasScraperError: the request failed, timed out, or did not return
            HTML.
    """
    request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
    try:
        with _open_url(request, timeout_seconds) as response:
            raw = response.read()
            content_type = response.headers.get("Content-Type", "")
    except urllib.error.HTTPError as exc:
        raise IdeasScraperError(f"Fetching {url} failed: HTTP {exc.code}.") from exc
    except urllib.error.URLError as exc:
        raise IdeasScraperError(f"Fetching {url} failed: {exc.reason}.") from exc
    except TimeoutError as exc:
        raise IdeasScraperError(
            f"Fetching {url} timed out after {timeout_seconds}s."
        ) from exc
    except Exception as exc:  # noqa: BLE001 - transport raises many types
        raise IdeasScraperError(f"Fetching {url} failed: {exc}.") from exc

    if content_type and "html" not in content_type.lower():
        raise IdeasScraperError(
            f"{url} did not return HTML (Content-Type: {content_type})."
        )
    return raw.decode("utf-8", errors="replace")





def extract_text(html: str, *, max_page_chars: int) -> ExtractedPage:
    """Strip markup and return the page's title and prose, truncated to
    `max_page_chars`.

    Raises:
        IdeasScraperError: nothing readable remains after stripping markup.
    """
    soup = BeautifulSoup(html, "html.parser")
    title_tag = soup.find("title")
    title = title_tag.get_text(strip=True) if title_tag else ""

    for tag in soup(_STRIP_TAGS):
        tag.decompose()

    text = _WHITESPACE_RE.sub(" ", soup.get_text(separator=" ", strip=True)).strip()
    if not text:
        raise IdeasScraperError("Page has no extractable text after stripping markup.")
    return ExtractedPage(title=title, text=text[:max_page_chars])





def synthesize_ideas(llm: LlmProvider, *, url: str, title: str, text: str) -> str:
    """Ask the model for product-feature ideas grounded in `text`.

    Raises:
        IdeasScraperError: the model returned no usable markdown.
        LlmError: the call itself failed (also a `TraderError`).
    """
    user = (
        f"Source URL: {url}\nPage title: {title or '(none found)'}\n\nPage text:\n{text}"
    )
    result = llm.generate_json(_SYSTEM_PROMPT, user, _SCHEMA)
    markdown = result.get("markdown")
    if not isinstance(markdown, str) or not markdown.strip():
        raise IdeasScraperError("Model returned no usable markdown for this page.")
    return markdown.strip()



def _slug(title: str, url: str) -> str:
    basis = title.strip() or urllib.parse.urlparse(url).netloc
    slug = _SLUG_RE.sub("_", basis.lower()).strip("_")
    return slug or "untitled"




def write_ideas_markdown(
    output_dir: Path,
    *,
    url: str,
    title: str,
    model: str,
    markdown_body: str,
    now: datetime | None = None,
) -> Path:
    """Write the synthesized ideas to a new markdown file under `output_dir`.

    Never overwrites an existing file for the same slug — a `_2`, `_3`, ...
    suffix is appended instead, so re-running against the same URL accumulates
    rather than silently discards a prior write this is meant to be a backlog
    of, per issue #75's stage-1/stage-2 split.
    """
    output_dir.mkdir(parents=True, exist_ok=True)
    slug = _slug(title, url)
    path = output_dir / f"{slug}.md"
    suffix = 2
    while path.exists():
        path = output_dir / f"{slug}_{suffix}.md"
        suffix += 1

    fetched_at = (now or datetime.now(UTC)).isoformat()
    heading = title or url
    content = (
        f"# {heading}\n\n"
        f"- Source: {url}\n"
        f"- Fetched: {fetched_at}\n"
        f"- Model: {model}\n\n"
        f"{markdown_body}\n"
    )
    path.write_text(content)
    return path





def scrape_ideas(
    url: str,
    *,
    output_dir: Path,
    fetch_timeout_seconds: int,
    max_page_chars: int,
    llm: LlmProvider,
) -> Path:
    """Fetch `url`, synthesize product-feature ideas, and write them to
    `output_dir`. Returns the path written.

    Raises:
        IdeasScraperError: fetching, extraction, or synthesis produced nothing
            usable.
        LlmError: the model call itself failed (also a `TraderError`).
    """
    html = fetch_page(url, timeout_seconds=fetch_timeout_seconds)
    page = extract_text(html, max_page_chars=max_page_chars)
    markdown_body = synthesize_ideas(llm, url=url, title=page.title, text=page.text)
    return write_ideas_markdown(
        output_dir,
        url=url,
        title=page.title,
        model=llm.model,
        markdown_body=markdown_body,
    )