mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
204 lines
8.5 KiB
Python
204 lines
8.5 KiB
Python
# ==============================================================================
|
||
# COPYRIGHT NO ONE EVERYWHERE LLC (WYOMING HOLDING COMPANY)
|
||
# PROJECT: SOVEREIGN STACK
|
||
# This artifact is entirely proprietary and cryptographically proven.
|
||
# Open-Source usage requires explicit permission from Brandon Scott Schneider.
|
||
# ==============================================================================
|
||
|
||
from __future__ import annotations
|
||
|
||
# [WARDEN BOUNDARY ENFORCEMENT INJECTED]
|
||
import sys
|
||
import os
|
||
try:
|
||
from io_harness_compat import spawn_isolated_process, fetch_network_resource
|
||
except ImportError:
|
||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||
from io_harness_compat import spawn_isolated_process, fetch_network_resource
|
||
|
||
#!/usr/bin/env python3
|
||
"""Download the full ^GSPC (S&P 500 Composite) history from Yahoo Finance v8 API
|
||
and slice it into macro-regime era fixtures for the surprise-determinism test suite.
|
||
|
||
Full available range: 1927-12-30 → today (~24,680 trading days as of 2026)
|
||
|
||
Pre-1928 note
|
||
─────────────
|
||
Data before 1927-12-30 is not available from Yahoo Finance. The NYSE was
|
||
organized in 1792; the DJIA was first published in 1896; systematic daily
|
||
S&P Composite data was only reconstructed back to 1928. Pre-1928 monthly
|
||
data (Shiller, 1871–1927) requires Yale's ie_data.xls, which is not
|
||
accessible from automated scripts via this network.
|
||
|
||
Eras saved
|
||
──────────
|
||
roaring_twenties : 1927-12-30 → 1929-10-28 (YF start → pre-crash)
|
||
great_depression : 1929-10-29 → 1941-12-31 (Black Tuesday → Pearl Harbor)
|
||
wwii_recovery : 1942-01-01 → 1952-12-31 (WWII + postwar reconstruction)
|
||
bretton_woods : 1953-01-01 → 1971-08-14 (stable managed growth)
|
||
stagflation : 1971-08-15 → 1982-12-31 (Nixon shock → Volcker victory)
|
||
reagan_bull : 1983-01-01 → 1994-12-31 (80s/90s expansion)
|
||
pre_hft : 1995-01-03 → 2006-12-29 (human microstructure)
|
||
gfc_transition : 2007-01-01 → 2011-12-31 (Reg NMS / Flash Crash dirty zone)
|
||
post_hft : 2012-01-03 → 2023-12-29 (algorithmic-dominant)
|
||
post_pandemic : 2024-01-01 → latest (current)
|
||
full_history : 1927-12-30 → latest (full corpus)
|
||
|
||
Data source: Yahoo Finance v8 chart API (https://query1.finance.yahoo.com).
|
||
No API key required. Full history is fetched in a single request and sliced
|
||
in memory — avoids per-era network calls and YF rate limiting.
|
||
|
||
Usage:
|
||
python 5-Applications/scripts/fetch_hft_regime_fixtures.py
|
||
python 5-Applications/scripts/fetch_hft_regime_fixtures.py --out-dir 5-Applications/tests/fixtures/hft_regime
|
||
"""
|
||
|
||
|
||
import argparse
|
||
import json
|
||
import math
|
||
import sys
|
||
import urllib.error
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import List, Tuple
|
||
|
||
_YF_URL = (
|
||
"https://query1.finance.yahoo.com/v8/finance/chart/"
|
||
"%5EGSPC?period1={p1}&period2={p2}&interval=1d"
|
||
)
|
||
|
||
# Macro-regime era definitions.
|
||
# "2099-12-31" as end date is intentionally far-future; YF clips to latest available.
|
||
# Eras are non-overlapping except gfc_transition, which covers the deliberate gap
|
||
# between pre_hft and post_hft.
|
||
_ERAS = {
|
||
# ── Historical regimes ────────────────────────────────────────────────────
|
||
"roaring_twenties": ("1927-12-30", "1929-10-28"), # YF start → day before Black Tuesday
|
||
"great_depression": ("1929-10-29", "1941-12-31"), # Black Tuesday → Pearl Harbor
|
||
"wwii_recovery": ("1942-01-01", "1952-12-31"), # Wartime boom → Eisenhower
|
||
"bretton_woods": ("1953-01-01", "1971-08-14"), # Gold-peg era → day before Nixon shock
|
||
"stagflation": ("1971-08-15", "1982-12-31"), # Nixon shock → year of Volcker victory
|
||
"reagan_bull": ("1983-01-01", "1994-12-31"), # 80s/90s expansion
|
||
# ── HFT-era split (backward-compatible, used by existing tests) ───────────
|
||
"pre_hft": ("1995-01-03", "2006-12-29"), # Human-only microstructure
|
||
"gfc_transition": ("2007-01-01", "2011-12-31"), # Dirty zone (Reg NMS, Bear, Flash Crash)
|
||
"post_hft": ("2012-01-03", "2023-12-29"), # Algorithmic-dominant
|
||
# ── Current era ──────────────────────────────────────────────────────────
|
||
"post_pandemic": ("2024-01-01", "2099-12-31"),
|
||
# ── Full corpus ──────────────────────────────────────────────────────────
|
||
"full_history": ("1900-01-01", "2099-12-31"), # YF clips to earliest/latest available
|
||
}
|
||
|
||
_REQUEST_TIMEOUT = 30 # seconds
|
||
|
||
|
||
def _date_to_ts(date_str: str) -> int:
|
||
"""Convert YYYY-MM-DD to a UTC Unix timestamp (midnight)."""
|
||
return int(
|
||
datetime.strptime(date_str, "%Y-%m-%d")
|
||
.replace(tzinfo=timezone.utc)
|
||
.timestamp()
|
||
)
|
||
|
||
|
||
def _fetch_all_closes() -> List[Tuple[str, float]]:
|
||
"""Fetch the complete ^GSPC history in a single Yahoo Finance v8 request.
|
||
|
||
Returns [(date_str, close), ...] oldest-first.
|
||
Uses adjclose (split/dividend-adjusted).
|
||
"""
|
||
# Use a wide window — YF returns the actual earliest/latest available.
|
||
p1 = _date_to_ts("1900-01-01")
|
||
p2 = _date_to_ts("2099-12-31")
|
||
url = _YF_URL.format(p1=p1, p2=p2)
|
||
req = urllib.request.Request(
|
||
url,
|
||
headers={"User-Agent": "Mozilla/5.0 (compatible; research-stack/1.0)"},
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT) as resp:
|
||
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||
except urllib.error.URLError as exc:
|
||
print(f"[fetch_hft_regime_fixtures] network error: {exc}", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
try:
|
||
result = data["chart"]["result"][0]
|
||
timestamps = result["timestamp"]
|
||
indicators = result["indicators"]
|
||
closes_raw = (
|
||
indicators.get("adjclose", [{}])[0].get("adjclose")
|
||
or indicators["quote"][0]["close"]
|
||
)
|
||
except (KeyError, IndexError, TypeError) as exc:
|
||
print(f"[fetch_hft_regime_fixtures] unexpected JSON shape: {exc}", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
rows: List[Tuple[str, float]] = []
|
||
for ts, px in zip(timestamps, closes_raw):
|
||
if px is None or px <= 0:
|
||
continue
|
||
date_s = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
|
||
rows.append((date_s, float(px)))
|
||
|
||
rows.sort(key=lambda r: r[0])
|
||
return rows
|
||
|
||
|
||
def _log_returns(closes: List[float]) -> List[float]:
|
||
"""Compute ln(p_t / p_{t-1}) for consecutive close prices."""
|
||
return [
|
||
math.log(closes[i] / closes[i - 1])
|
||
for i in range(1, len(closes))
|
||
if closes[i] > 0 and closes[i - 1] > 0
|
||
]
|
||
|
||
|
||
def main(out_dir: Path) -> None:
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Single network request for the full history; slice all eras from memory.
|
||
print("Fetching ^GSPC full history from Yahoo Finance v8...", end=" ", flush=True)
|
||
all_rows = _fetch_all_closes()
|
||
if len(all_rows) < 1000:
|
||
print(f"\nERROR: only {len(all_rows)} rows — network or API issue")
|
||
sys.exit(1)
|
||
print(f"{len(all_rows)} days ({all_rows[0][0]} → {all_rows[-1][0]})")
|
||
|
||
for era, (d1, d2) in _ERAS.items():
|
||
rows = [(d, px) for d, px in all_rows if d1 <= d <= d2]
|
||
if len(rows) < 5:
|
||
print(f" SKIP {era}: {len(rows)} rows in [{d1}, {d2}]")
|
||
continue
|
||
|
||
dates = [r[0] for r in rows]
|
||
closes = [r[1] for r in rows]
|
||
log_ret = _log_returns(closes)
|
||
|
||
payload = {
|
||
"symbol": "^GSPC via Yahoo Finance v8",
|
||
"era": era,
|
||
"date_start": dates[0],
|
||
"date_end": dates[-1],
|
||
"n_days": len(rows),
|
||
"log_returns": [round(r, 10) for r in log_ret],
|
||
}
|
||
out_path = out_dir / f"{era}_returns.json"
|
||
out_path.write_text(json.dumps(payload, indent=2))
|
||
print(f" {era:20s} {len(rows):5d} days {dates[0]} → {dates[-1]}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
p = argparse.ArgumentParser(description=__doc__)
|
||
p.add_argument(
|
||
"--out-dir",
|
||
default="5-Applications/tests/fixtures/hft_regime",
|
||
help="Directory to write fixture files (created if absent)",
|
||
)
|
||
args = p.parse_args()
|
||
|
||
|
||
root = Path(__file__).resolve().parent.parent
|
||
main(root / args.out_dir)
|