Fresh upload: Complete RRR contents
Browse files- README.md +193 -0
- app.py +362 -0
- backtest/__init__.py +0 -0
- backtest/backtester.py +148 -0
- config.py +99 -0
- data/__init__.py +0 -0
- data/historical.py +169 -0
- data/live.py +42 -0
- data_cache/.gitkeep +0 -0
- indicators/__init__.py +0 -0
- indicators/indicators.py +180 -0
- live_runner.py +102 -0
- models/__init__.py +0 -0
- models/arima_garch_model.py +139 -0
- models/arima_model.py +73 -0
- models/auto_arima_model.py +139 -0
- models/base_model.py +37 -0
- models/moirai_model.py +88 -0
- models/registry.py +88 -0
- models/timesfm_model.py +78 -0
- requirements.txt +33 -0
- utils/__init__.py +0 -0
- utils/helpers.py +27 -0
README.md
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Forex Crypto Prediction
|
| 3 |
+
emoji: 📈
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: gradio
|
| 7 |
+
app_file: app.py
|
| 8 |
+
pinned: false
|
| 9 |
+
python_version: "3.11"
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# 📈 Forex & Crypto Prediction Dashboard
|
| 13 |
+
|
| 14 |
+
Multi-timeframe forex & crypto price prediction with **ARIMA**,
|
| 15 |
+
**Auto-ARIMA**, **ARIMA-GARCH**, **Moirai** (Salesforce) and **TimesFM**
|
| 16 |
+
(Google) — each model's accuracy is backtested and reported **separately**,
|
| 17 |
+
plus a candle-by-candle correct/incorrect verification log. Pure Python,
|
| 18 |
+
built for a free/low CPU-only Hugging Face Space (2 vCPU / 16GB RAM / ~50GB
|
| 19 |
+
storage).
|
| 20 |
+
|
| 21 |
+
**Educational / research project — not financial advice.** Forex and crypto
|
| 22 |
+
trading carries substantial risk of loss. Nothing this app outputs is a
|
| 23 |
+
recommendation to trade.
|
| 24 |
+
|
| 25 |
+
## What's inside
|
| 26 |
+
|
| 27 |
+
- **Live + historical data**, free, no API key — [yfinance](https://github.com/ranaroussi/yfinance)
|
| 28 |
+
(Yahoo Finance). Live means polling the same free source on a timer, since
|
| 29 |
+
Yahoo has no free websocket feed.
|
| 30 |
+
- **Any symbol** — a dropdown of popular forex pairs (`EURUSD=X`, `GBPUSD=X`, ...)
|
| 31 |
+
and crypto pairs (`BTC-USD`, `ETH-USD`, ...) that also accepts any custom
|
| 32 |
+
yfinance-valid ticker you type in.
|
| 33 |
+
- **Any timeframe** — 1m, 5m, 10m, 15m, 30m, 1h, 4h, 1d out of the box.
|
| 34 |
+
Timeframes Yahoo doesn't serve natively (like 10m) are built by resampling
|
| 35 |
+
a smaller native interval, so you're not limited to Yahoo's fixed list.
|
| 36 |
+
- **5 forecasting models**, one shared interface (`models/base_model.py`):
|
| 37 |
+
- **ARIMA** (`statsmodels`) — fast, classical, always available. Order
|
| 38 |
+
(p,d,q) is found with a brute-force AIC grid search.
|
| 39 |
+
- **Auto-ARIMA** (`statsmodels`) — same idea as ARIMA, but the order is
|
| 40 |
+
found with a Hyndman-Khandakar-style stepwise search (the same
|
| 41 |
+
algorithm behind R's `auto.arima()`) instead of a brute-force grid.
|
| 42 |
+
Implemented directly on statsmodels rather than depending on
|
| 43 |
+
`pmdarima` — that package's compiled Cython extensions are a known
|
| 44 |
+
source of numpy-version install breakage on fresh environments.
|
| 45 |
+
- **ARIMA-GARCH** (`statsmodels` + `arch`) — ARIMA models the conditional
|
| 46 |
+
mean of returns, GARCH(1,1) models the conditional variance (volatility
|
| 47 |
+
clustering) on top of it. Forecasted volatility is kept on the model
|
| 48 |
+
instance (`last_volatility_forecast`) for anyone who wants to build
|
| 49 |
+
confidence bands later.
|
| 50 |
+
- **Moirai** (`uni2ts`) — Salesforce's pretrained time-series foundation model.
|
| 51 |
+
- **TimesFM 2.5** (`timesfm`) — Google's pretrained time-series foundation model.
|
| 52 |
+
- **Top 5 technical indicators** (hand-implemented on pandas, no extra
|
| 53 |
+
dependency): RSI, MACD, Bollinger Bands, SMA/EMA, Stochastic Oscillator.
|
| 54 |
+
These are shown for chart analysis — they're separate from, and not mixed
|
| 55 |
+
into, the 5 models' backtested accuracy.
|
| 56 |
+
- **Walk-forward backtesting** with a **separate** accuracy card per model
|
| 57 |
+
(correct / incorrect / accuracy % / MAE / RMSE) — numbers are never
|
| 58 |
+
averaged together across models — plus a per-candle log of every
|
| 59 |
+
prediction vs. what actually happened, so you can verify exactly how many
|
| 60 |
+
candles each model got right or wrong.
|
| 61 |
+
- **Live tab**: polls for the newest candle, predicts the next one, and once
|
| 62 |
+
that candle closes, checks whether the prediction was right — with a
|
| 63 |
+
running accuracy tally. `live_runner.py` does the same thing unattended
|
| 64 |
+
from the command line, logging to CSV.
|
| 65 |
+
|
| 66 |
+
## Project layout
|
| 67 |
+
|
| 68 |
+
```
|
| 69 |
+
config.py symbols, timeframes, model checkpoints
|
| 70 |
+
app.py Gradio dashboard (entry point for the Space)
|
| 71 |
+
live_runner.py standalone CLI for unattended live logging
|
| 72 |
+
data/
|
| 73 |
+
historical.py yfinance fetch + resampling + 7-day-chunked 1m fetch
|
| 74 |
+
live.py polling loop helper
|
| 75 |
+
indicators/
|
| 76 |
+
indicators.py RSI, MACD, Bollinger Bands, SMA/EMA, Stochastic
|
| 77 |
+
models/
|
| 78 |
+
base_model.py shared interface all 5 models implement
|
| 79 |
+
arima_model.py ARIMA (statsmodels), cached order search
|
| 80 |
+
auto_arima_model.py Auto-ARIMA (statsmodels only), cached stepwise search
|
| 81 |
+
arima_garch_model.py ARIMA-GARCH (statsmodels + arch), mean + volatility
|
| 82 |
+
moirai_model.py Moirai (uni2ts + GluonTS)
|
| 83 |
+
timesfm_model.py TimesFM 2.5 (timesfm[torch])
|
| 84 |
+
registry.py lazy-loading model factory
|
| 85 |
+
backtest/
|
| 86 |
+
backtester.py walk-forward loop + per-model accuracy summary
|
| 87 |
+
utils/
|
| 88 |
+
helpers.py small shared utilities
|
| 89 |
+
requirements.txt
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
## Running locally
|
| 93 |
+
|
| 94 |
+
```bash
|
| 95 |
+
python -m venv venv
|
| 96 |
+
source venv/bin/activate # Windows: venv\Scripts\activate
|
| 97 |
+
pip install -r requirements.txt
|
| 98 |
+
python app.py
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
Open the local URL Gradio prints (usually `http://127.0.0.1:7860`).
|
| 102 |
+
|
| 103 |
+
## Deploying to Hugging Face Spaces
|
| 104 |
+
|
| 105 |
+
1. Create a new Space → SDK: **Gradio** → hardware: **2 vCPU / 16GB RAM**
|
| 106 |
+
(the free CPU tier) is enough for all 5 models at their default (small)
|
| 107 |
+
checkpoints.
|
| 108 |
+
2. Upload every file in this project, keeping the folder structure
|
| 109 |
+
(`data/`, `models/`, etc.) intact — Spaces builds from the repo root.
|
| 110 |
+
3. The YAML block at the top of this README is what Spaces reads to
|
| 111 |
+
configure the app (`sdk: gradio`, `app_file: app.py`). No changes needed
|
| 112 |
+
unless you want a different title/emoji/colors.
|
| 113 |
+
4. Push. The Space installs `requirements.txt` and launches `app.py`
|
| 114 |
+
automatically. First launch is normal Python-only (ARIMA works
|
| 115 |
+
immediately); Moirai/TimesFM download their pretrained weights
|
| 116 |
+
on-demand the first time you actually use them from the UI.
|
| 117 |
+
|
| 118 |
+
## Models — what to expect on CPU-only hardware
|
| 119 |
+
|
| 120 |
+
| Model | Package | Approx. size | Notes |
|
| 121 |
+
|---|---|---|---|
|
| 122 |
+
| ARIMA | `statsmodels` | tiny | Fastest by far. Order (p,d,q) is searched once per backtest run (brute-force AIC grid) and reused, not re-searched every candle. |
|
| 123 |
+
| Auto-ARIMA | `statsmodels` | tiny | Same dependency as plain ARIMA — no extra package. Stepwise order search runs once and is reused; usually visits fewer candidate models than ARIMA's brute-force grid. |
|
| 124 |
+
| ARIMA-GARCH | `statsmodels` + `arch` | tiny | Two fits per prediction (ARIMA for the mean, GARCH for the volatility), so a little slower than plain ARIMA — still fast next to the foundation models. |
|
| 125 |
+
| Moirai (`moirai-1.1-R-small`) | `uni2ts` | small (~14M params) | Zero-shot foundation model, no training needed. |
|
| 126 |
+
| TimesFM 2.5 (`timesfm-2.5-200m`) | `timesfm[torch]` | ~800MB download, ~4GB RAM | Zero-shot foundation model, no training needed. |
|
| 127 |
+
|
| 128 |
+
Both foundation models are **zero-shot** — they were pretrained elsewhere and
|
| 129 |
+
just run inference here, no training step in this project. They're slower
|
| 130 |
+
per-prediction than ARIMA on CPU, which matters most in the **Backtest** tab
|
| 131 |
+
(one model call per candle). Use the **Step** slider there to skip candles
|
| 132 |
+
(e.g. step=5 or 10) and keep Moirai/TimesFM backtests fast on 2 vCPUs.
|
| 133 |
+
Checkpoints are configurable in `config.py` if you later move to GPU hardware
|
| 134 |
+
and want the bigger `base`/`large` variants.
|
| 135 |
+
|
| 136 |
+
## How accuracy is measured
|
| 137 |
+
|
| 138 |
+
For each candle in a backtest (or each live prediction), the model forecasts
|
| 139 |
+
the next close; we compare **predicted direction** (up/down vs. the current
|
| 140 |
+
close) to what **actually** happened once that candle closes. That's one
|
| 141 |
+
verified candle: correct or incorrect. A model's accuracy % is
|
| 142 |
+
`correct / (correct + incorrect)` over the whole run — and each model's
|
| 143 |
+
numbers live in their own card/tab, never averaged with another model's.
|
| 144 |
+
MAE/RMSE (predicted price vs. actual price) are reported alongside as a
|
| 145 |
+
secondary, continuous-error view.
|
| 146 |
+
|
| 147 |
+
"Once that candle closes" is checked strictly: a prediction only gets
|
| 148 |
+
verified once a *newer* candle exists after its target, never merely because
|
| 149 |
+
the target timestamp has appeared. Yahoo's intraday feed includes the
|
| 150 |
+
currently-forming candle as its last row, with a Close that's just the
|
| 151 |
+
latest traded price so far — reading that as final the moment it appears
|
| 152 |
+
would grade some candles against a mid-candle price instead of the true
|
| 153 |
+
close (Live tab and `live_runner.py` both apply this check the same way).
|
| 154 |
+
|
| 155 |
+
## Known limitations (please read before relying on this for anything real)
|
| 156 |
+
|
| 157 |
+
- **Yahoo/yfinance history limits** (enforced by Yahoo, not this project):
|
| 158 |
+
1-minute candles ≈ last 7–30 days only; other intraday intervals ≈ last 60
|
| 159 |
+
days; hourly ≈ last 730 days; daily+ is full history. The app tells you if
|
| 160 |
+
a request falls outside what's available rather than failing silently.
|
| 161 |
+
- **"Live" is polling, not push.** Free forex/crypto data has no public
|
| 162 |
+
websocket firehose, so "live" here means re-fetching on a timer — fine for
|
| 163 |
+
candle-close-level decisions, not for tick-by-tick/HFT use.
|
| 164 |
+
- **"All symbols"** means any symbol yfinance/Yahoo covers — there's no
|
| 165 |
+
finite master list to ship, so the app takes any ticker you type in rather
|
| 166 |
+
than pretending to enumerate everything.
|
| 167 |
+
- **Small foundation-model checkpoints** were chosen to fit 2 vCPU/16GB
|
| 168 |
+
comfortably. Bigger checkpoints (`base`/`large`) may forecast better but
|
| 169 |
+
need more RAM/CPU time — swap them in `config.py` if your hardware allows.
|
| 170 |
+
- This was built and syntax-checked without live internet access in the
|
| 171 |
+
build environment, so package APIs were verified against each project's
|
| 172 |
+
current documentation but not executed end-to-end here. Test on your own
|
| 173 |
+
machine or Space before trusting it, and check each library's docs/repo if
|
| 174 |
+
an API has moved since (`uni2ts` and `timesfm` are active research
|
| 175 |
+
projects that occasionally make breaking changes; `arch` is more stable
|
| 176 |
+
but worth a version check too). Every model also validates its own
|
| 177 |
+
forecast is finite (no NaN/inf) before returning it, and raises a clear
|
| 178 |
+
error instead if it isn't — this surfaces as a "failed" row in a backtest
|
| 179 |
+
rather than silently corrupting the accuracy/MAE numbers, which is how an
|
| 180 |
+
earlier version of this project let a bad TimesFM forecast slip through.
|
| 181 |
+
- MAE/RMSE round to 6 **significant figures**, not 6 decimal places — this
|
| 182 |
+
project covers everything from forex pairs (~1.10) to sub-cent tokens
|
| 183 |
+
(~0.000018), and a fixed decimal count either hides a real, non-zero error
|
| 184 |
+
as a misleading `0.0` on a low-price asset or prints false precision
|
| 185 |
+
(`125.345678`) on a high-price one like BTC.
|
| 186 |
+
|
| 187 |
+
## Disclaimer
|
| 188 |
+
|
| 189 |
+
This project is for education and research. It does not constitute
|
| 190 |
+
financial advice, and past backtested accuracy is not a promise of future
|
| 191 |
+
performance. Forex and crypto markets are volatile and you can lose money
|
| 192 |
+
trading them — always do your own research and consider consulting a
|
| 193 |
+
licensed financial advisor before trading real capital.
|
app.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Forex & Crypto Prediction Dashboard — Gradio app, entry point for the
|
| 3 |
+
Hugging Face Space (see README.md for the Space YAML header / deployment).
|
| 4 |
+
|
| 5 |
+
Tabs:
|
| 6 |
+
1. Chart & Indicators - candlestick chart + the top 5 technical indicators
|
| 7 |
+
2. Prediction - ARIMA / Auto-ARIMA / ARIMA-GARCH / Moirai / TimesFM
|
| 8 |
+
forecasts, shown together on one chart for
|
| 9 |
+
visual comparison
|
| 10 |
+
3. Backtest - walk-forward backtest; each model gets its own
|
| 11 |
+
SEPARATE accuracy card + per-candle verify log
|
| 12 |
+
(numbers are never averaged across models)
|
| 13 |
+
4. Live - poll the newest candle, predict the next one, and
|
| 14 |
+
verify the previous prediction once it lands
|
| 15 |
+
|
| 16 |
+
Educational project — nothing here is financial advice.
|
| 17 |
+
"""
|
| 18 |
+
import numpy as np
|
| 19 |
+
import pandas as pd
|
| 20 |
+
import plotly.graph_objects as go
|
| 21 |
+
import gradio as gr
|
| 22 |
+
|
| 23 |
+
from config import ALL_DEFAULT_SYMBOLS, CUSTOM_TIMEFRAMES, MODEL_NAMES
|
| 24 |
+
from data.historical import get_historical_cached, DataFetchError
|
| 25 |
+
from indicators.indicators import add_all_indicators, latest_signals
|
| 26 |
+
from models.registry import get_model, fresh_model
|
| 27 |
+
from backtest.backtester import backtest_model, summarize_backtest
|
| 28 |
+
from utils.helpers import infer_step, future_index
|
| 29 |
+
|
| 30 |
+
TIMEFRAME_CHOICES = list(CUSTOM_TIMEFRAMES.keys())
|
| 31 |
+
DISCLAIMER = (
|
| 32 |
+
"**Educational / research project — not financial advice.** "
|
| 33 |
+
"Forex and crypto trading carries substantial risk of loss."
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ---------------------------------------------------------------------------
|
| 38 |
+
# Tab 1: Chart & Indicators
|
| 39 |
+
# ---------------------------------------------------------------------------
|
| 40 |
+
def make_candlestick_figure(df: pd.DataFrame, symbol: str, timeframe: str) -> go.Figure:
|
| 41 |
+
fig = go.Figure(data=[go.Candlestick(
|
| 42 |
+
x=df.index, open=df["Open"], high=df["High"], low=df["Low"], close=df["Close"],
|
| 43 |
+
name=symbol,
|
| 44 |
+
)])
|
| 45 |
+
if "SMA_20" in df:
|
| 46 |
+
fig.add_trace(go.Scatter(x=df.index, y=df["SMA_20"], name="SMA 20", line=dict(width=1)))
|
| 47 |
+
if "EMA_20" in df:
|
| 48 |
+
fig.add_trace(go.Scatter(x=df.index, y=df["EMA_20"], name="EMA 20", line=dict(width=1)))
|
| 49 |
+
if "BB_Upper" in df:
|
| 50 |
+
fig.add_trace(go.Scatter(x=df.index, y=df["BB_Upper"], name="BB Upper",
|
| 51 |
+
line=dict(width=1, dash="dot")))
|
| 52 |
+
fig.add_trace(go.Scatter(x=df.index, y=df["BB_Lower"], name="BB Lower",
|
| 53 |
+
line=dict(width=1, dash="dot")))
|
| 54 |
+
fig.update_layout(title=f"{symbol} — {timeframe}", xaxis_rangeslider_visible=False,
|
| 55 |
+
height=480, margin=dict(l=10, r=10, t=40, b=10))
|
| 56 |
+
return fig
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def fetch_and_chart(symbol, timeframe, lookback_days):
|
| 60 |
+
try:
|
| 61 |
+
df = get_historical_cached(symbol, timeframe, lookback_days=int(lookback_days), refresh=True)
|
| 62 |
+
df_ind = add_all_indicators(df)
|
| 63 |
+
fig = make_candlestick_figure(df_ind, symbol, timeframe)
|
| 64 |
+
signals = latest_signals(df_ind)
|
| 65 |
+
signal_rows = [[name, str(v["value"]), v["signal"]] for name, v in signals.items()]
|
| 66 |
+
status = f"Loaded {len(df)} candles for {symbol} ({timeframe})."
|
| 67 |
+
return fig, signal_rows, status
|
| 68 |
+
except DataFetchError as e:
|
| 69 |
+
return None, [], f"Data error: {e}"
|
| 70 |
+
except Exception as e:
|
| 71 |
+
return None, [], f"Unexpected error: {e}"
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# ---------------------------------------------------------------------------
|
| 75 |
+
# Tab 2: Prediction
|
| 76 |
+
# ---------------------------------------------------------------------------
|
| 77 |
+
def run_prediction(symbol, timeframe, selected_models, horizon, lookback_days):
|
| 78 |
+
if not selected_models:
|
| 79 |
+
return None, "Pick at least one model."
|
| 80 |
+
try:
|
| 81 |
+
df = get_historical_cached(symbol, timeframe, lookback_days=int(lookback_days), refresh=True)
|
| 82 |
+
except Exception as e:
|
| 83 |
+
return None, f"Data error: {e}"
|
| 84 |
+
|
| 85 |
+
close = df["Close"]
|
| 86 |
+
horizon = int(horizon)
|
| 87 |
+
step = infer_step(df.index)
|
| 88 |
+
|
| 89 |
+
fig = go.Figure()
|
| 90 |
+
tail = close.iloc[-150:]
|
| 91 |
+
fig.add_trace(go.Scatter(x=tail.index, y=tail.values, name="Actual Close",
|
| 92 |
+
line=dict(color="#888888")))
|
| 93 |
+
|
| 94 |
+
summary_lines = []
|
| 95 |
+
for name in selected_models:
|
| 96 |
+
try:
|
| 97 |
+
model = get_model(name, symbol, timeframe)
|
| 98 |
+
forecast = model.predict(close, horizon=horizon)
|
| 99 |
+
fc_index = future_index(df.index[-1], step, len(forecast))
|
| 100 |
+
fig.add_trace(go.Scatter(x=fc_index, y=forecast, name=f"{name} forecast",
|
| 101 |
+
mode="lines+markers"))
|
| 102 |
+
direction = "UP" if forecast[-1] > close.iloc[-1] else "DOWN"
|
| 103 |
+
summary_lines.append(f"**{name}** — next close (approx.) `{forecast[-1]:.5f}` ({direction})")
|
| 104 |
+
except Exception as e:
|
| 105 |
+
summary_lines.append(f"**{name}** — failed: {e}")
|
| 106 |
+
|
| 107 |
+
fig.update_layout(title=f"{symbol} — {timeframe} forecast ({horizon} candle(s) ahead)",
|
| 108 |
+
height=450, margin=dict(l=10, r=10, t=40, b=10))
|
| 109 |
+
return fig, "\n\n".join(summary_lines) + "\n\n" + DISCLAIMER
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# ---------------------------------------------------------------------------
|
| 113 |
+
# Tab 3: Backtest
|
| 114 |
+
# ---------------------------------------------------------------------------
|
| 115 |
+
def run_backtest_ui(symbol, timeframe, selected_models, window, horizon, step_size, lookback_days):
|
| 116 |
+
empty = pd.DataFrame()
|
| 117 |
+
if not selected_models:
|
| 118 |
+
return [], empty, empty, empty, empty, empty, "Pick at least one model."
|
| 119 |
+
try:
|
| 120 |
+
df = get_historical_cached(symbol, timeframe, lookback_days=int(lookback_days), refresh=True)
|
| 121 |
+
except Exception as e:
|
| 122 |
+
return [], empty, empty, empty, empty, empty, f"Data error: {e}"
|
| 123 |
+
|
| 124 |
+
window, horizon, step_size = int(window), int(horizon), int(step_size)
|
| 125 |
+
if len(df) < window + horizon + 5:
|
| 126 |
+
return [], empty, empty, empty, empty, empty, (
|
| 127 |
+
f"Only {len(df)} candles available — raise 'Backtest lookback (days)' "
|
| 128 |
+
f"or lower the window size to backtest this symbol/timeframe."
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
summary_table = []
|
| 132 |
+
detail = {"ARIMA": empty, "Auto-ARIMA": empty, "ARIMA-GARCH": empty,
|
| 133 |
+
"Moirai": empty, "TimesFM": empty}
|
| 134 |
+
for name in selected_models:
|
| 135 |
+
model = fresh_model(name) # fresh instance per run (its own ARIMA order search, etc.)
|
| 136 |
+
results = backtest_model(df, model, window=window, horizon=horizon, step=step_size)
|
| 137 |
+
summary = summarize_backtest(results, name)
|
| 138 |
+
summary_table.append([
|
| 139 |
+
summary["model"], summary["total_predictions"], summary["correct"],
|
| 140 |
+
summary["incorrect"], f"{summary['accuracy_pct']}%",
|
| 141 |
+
summary["mae"], summary["rmse"], summary["failed_predictions"],
|
| 142 |
+
])
|
| 143 |
+
detail[name] = results
|
| 144 |
+
|
| 145 |
+
status = f"Backtested {', '.join(selected_models)} on {len(df)} candles of {symbol} ({timeframe})."
|
| 146 |
+
return (summary_table, detail["ARIMA"], detail["Auto-ARIMA"], detail["ARIMA-GARCH"],
|
| 147 |
+
detail["Moirai"], detail["TimesFM"], status)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
# ---------------------------------------------------------------------------
|
| 151 |
+
# Tab 4: Live
|
| 152 |
+
# ---------------------------------------------------------------------------
|
| 153 |
+
LIVE_COLUMNS = ["predicted_at", "target_time", "current_close", "predicted_close",
|
| 154 |
+
"predicted_direction", "actual_close", "actual_direction", "correct"]
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _check_live_impl(symbol, timeframe, model_name, log_df):
|
| 158 |
+
if log_df is None or len(log_df) == 0:
|
| 159 |
+
log_df = pd.DataFrame(columns=LIVE_COLUMNS)
|
| 160 |
+
else:
|
| 161 |
+
log_df = log_df.copy()
|
| 162 |
+
|
| 163 |
+
df = get_historical_cached(symbol, timeframe, lookback_days=2, refresh=True)
|
| 164 |
+
latest_time = df.index[-1]
|
| 165 |
+
latest_close = float(df["Close"].iloc[-1])
|
| 166 |
+
|
| 167 |
+
# 1) verify any pending prediction whose target candle has now CLOSED.
|
| 168 |
+
#
|
| 169 |
+
# BUG FIX: the old check was just `row["target_time"] in df.index`. Yahoo's
|
| 170 |
+
# intraday feed includes the current, still-forming candle as the last row
|
| 171 |
+
# (see data/live.py's own docstring: "the most recent (possibly
|
| 172 |
+
# still-forming) candle"), and its Close is just the latest traded price so
|
| 173 |
+
# far, not the candle's eventual final close. The instant the target candle
|
| 174 |
+
# starts forming it satisfies `in df.index`, so the old code would verify
|
| 175 |
+
# right then and there against a mid-candle snapshot -- then never revisit
|
| 176 |
+
# that row again, since it's no longer NaN. Reproduced: a prediction that
|
| 177 |
+
# was directionally CORRECT against the true close got permanently logged
|
| 178 |
+
# as incorrect because the candle had dipped the other way in its first
|
| 179 |
+
# minute. That silently corrupts the accuracy % this tab (and
|
| 180 |
+
# live_runner.py) reports.
|
| 181 |
+
#
|
| 182 |
+
# Fix: also require the target candle to no longer be the latest row --
|
| 183 |
+
# i.e. a strictly newer candle exists after it -- which is only true once
|
| 184 |
+
# it has actually closed.
|
| 185 |
+
pending_mask = log_df["actual_close"].isna()
|
| 186 |
+
for idx, row in log_df[pending_mask].iterrows():
|
| 187 |
+
if row["target_time"] in df.index and row["target_time"] < latest_time:
|
| 188 |
+
actual_close = float(df.loc[row["target_time"], "Close"])
|
| 189 |
+
actual_direction = "up" if actual_close > row["current_close"] else "down"
|
| 190 |
+
log_df.loc[idx, "actual_close"] = actual_close
|
| 191 |
+
log_df.loc[idx, "actual_direction"] = actual_direction
|
| 192 |
+
log_df.loc[idx, "correct"] = bool(actual_direction == row["predicted_direction"])
|
| 193 |
+
|
| 194 |
+
# 2) make a fresh prediction for the next candle, unless one is already
|
| 195 |
+
# pending for this exact latest candle
|
| 196 |
+
#
|
| 197 |
+
# Dedup by TIMESTAMP (predicted_at == latest_time), not by close price.
|
| 198 |
+
# Price equality is the wrong proxy in both directions: an intraday
|
| 199 |
+
# "latest" candle from yfinance is often still forming, so its close
|
| 200 |
+
# can drift between polls even though it's still the SAME candle
|
| 201 |
+
# (price-equality would wrongly log a second, duplicate pending row
|
| 202 |
+
# for it before it even closes) -- and a genuinely NEW candle can by
|
| 203 |
+
# coincidence close at the exact same price as the previous one,
|
| 204 |
+
# especially in a quiet/ranging market (price-equality would then
|
| 205 |
+
# wrongly skip predicting it entirely). Either way corrupts the
|
| 206 |
+
# running-accuracy count this tab reports. Timestamp equality has
|
| 207 |
+
# neither failure mode.
|
| 208 |
+
have_pending_for_latest = False
|
| 209 |
+
if len(log_df):
|
| 210 |
+
last_row = log_df.iloc[-1]
|
| 211 |
+
have_pending_for_latest = (
|
| 212 |
+
pd.isna(last_row["actual_close"]) and last_row["predicted_at"] == latest_time
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
status_extra = ""
|
| 216 |
+
if not have_pending_for_latest:
|
| 217 |
+
try:
|
| 218 |
+
model = get_model(model_name, symbol, timeframe)
|
| 219 |
+
forecast = model.predict(df["Close"], horizon=1)
|
| 220 |
+
predicted_close = float(forecast[-1])
|
| 221 |
+
predicted_direction = "up" if predicted_close > latest_close else "down"
|
| 222 |
+
step = infer_step(df.index)
|
| 223 |
+
new_row = {
|
| 224 |
+
"predicted_at": latest_time, "target_time": latest_time + step,
|
| 225 |
+
"current_close": latest_close, "predicted_close": predicted_close,
|
| 226 |
+
"predicted_direction": predicted_direction, "actual_close": np.nan,
|
| 227 |
+
"actual_direction": None, "correct": None,
|
| 228 |
+
}
|
| 229 |
+
log_df = pd.concat([log_df, pd.DataFrame([new_row])], ignore_index=True)
|
| 230 |
+
except Exception as e:
|
| 231 |
+
status_extra = f" (prediction failed: {e})"
|
| 232 |
+
|
| 233 |
+
verified = log_df.dropna(subset=["correct"])
|
| 234 |
+
if len(verified):
|
| 235 |
+
acc = round(100 * verified["correct"].sum() / len(verified), 2)
|
| 236 |
+
acc_text = f"Running accuracy for {model_name}: {acc}% ({int(verified['correct'].sum())}/{len(verified)})"
|
| 237 |
+
else:
|
| 238 |
+
acc_text = "Collecting data — check again after the next candle closes to see accuracy."
|
| 239 |
+
|
| 240 |
+
status = f"Latest {symbol} candle: {latest_time} — Close {latest_close:.5f}. {acc_text}{status_extra}"
|
| 241 |
+
return status, log_df.tail(20), log_df
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def check_live(symbol, timeframe, model_name, log_df):
|
| 245 |
+
try:
|
| 246 |
+
return _check_live_impl(symbol, timeframe, model_name, log_df)
|
| 247 |
+
except Exception as e:
|
| 248 |
+
safe_log = log_df if (log_df is not None and len(log_df)) else pd.DataFrame(columns=LIVE_COLUMNS)
|
| 249 |
+
return f"Error: {e}", safe_log.tail(20), safe_log
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# ---------------------------------------------------------------------------
|
| 253 |
+
# UI
|
| 254 |
+
# ---------------------------------------------------------------------------
|
| 255 |
+
with gr.Blocks(title="Forex & Crypto Prediction Dashboard") as demo:
|
| 256 |
+
gr.Markdown(
|
| 257 |
+
"# 📈 Forex & Crypto Prediction Dashboard\n"
|
| 258 |
+
"ARIMA, Auto-ARIMA, ARIMA-GARCH, Moirai (Salesforce) & TimesFM (Google) — "
|
| 259 |
+
"each model's accuracy is tracked and shown **separately**, never averaged "
|
| 260 |
+
"together.\n\n" + DISCLAIMER
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
with gr.Tab("📊 Chart & Indicators"):
|
| 264 |
+
with gr.Row():
|
| 265 |
+
symbol_in1 = gr.Dropdown(choices=ALL_DEFAULT_SYMBOLS, value="EURUSD=X",
|
| 266 |
+
allow_custom_value=True, label="Symbol")
|
| 267 |
+
timeframe_in1 = gr.Dropdown(choices=TIMEFRAME_CHOICES, value="15m", label="Timeframe")
|
| 268 |
+
lookback_in1 = gr.Slider(1, 60, value=5, step=1, label="Lookback (days)")
|
| 269 |
+
fetch_btn = gr.Button("Fetch & Analyze", variant="primary")
|
| 270 |
+
chart_out1 = gr.Plot()
|
| 271 |
+
signals_out = gr.Dataframe(headers=["Indicator", "Value", "Signal"],
|
| 272 |
+
label="Top 5 Indicators (latest reading)")
|
| 273 |
+
status_out1 = gr.Markdown()
|
| 274 |
+
fetch_btn.click(fetch_and_chart, [symbol_in1, timeframe_in1, lookback_in1],
|
| 275 |
+
[chart_out1, signals_out, status_out1])
|
| 276 |
+
|
| 277 |
+
with gr.Tab("🔮 Prediction"):
|
| 278 |
+
with gr.Row():
|
| 279 |
+
symbol_in2 = gr.Dropdown(choices=ALL_DEFAULT_SYMBOLS, value="BTC-USD",
|
| 280 |
+
allow_custom_value=True, label="Symbol")
|
| 281 |
+
timeframe_in2 = gr.Dropdown(choices=TIMEFRAME_CHOICES, value="15m", label="Timeframe")
|
| 282 |
+
with gr.Row():
|
| 283 |
+
models_in2 = gr.CheckboxGroup(choices=MODEL_NAMES, value=["ARIMA"], label="Models")
|
| 284 |
+
horizon_in2 = gr.Slider(1, 10, value=1, step=1, label="Candles ahead")
|
| 285 |
+
lookback_in2 = gr.Slider(1, 60, value=5, step=1, label="Lookback (days)")
|
| 286 |
+
gr.Markdown(
|
| 287 |
+
"Moirai and TimesFM download pretrained weights the first time you use them "
|
| 288 |
+
"(needs internet on the Space; a few hundred MB each, one-time download)."
|
| 289 |
+
)
|
| 290 |
+
predict_btn = gr.Button("Predict", variant="primary")
|
| 291 |
+
chart_out2 = gr.Plot()
|
| 292 |
+
summary_out2 = gr.Markdown()
|
| 293 |
+
predict_btn.click(run_prediction,
|
| 294 |
+
[symbol_in2, timeframe_in2, models_in2, horizon_in2, lookback_in2],
|
| 295 |
+
[chart_out2, summary_out2])
|
| 296 |
+
|
| 297 |
+
with gr.Tab("🧪 Backtest"):
|
| 298 |
+
gr.Markdown(
|
| 299 |
+
"Each model's accuracy is computed and shown **separately** below — "
|
| 300 |
+
"results are never averaged together. Raise **Step** to speed up "
|
| 301 |
+
"Moirai/TimesFM (slower per-prediction than ARIMA on CPU-only hardware)."
|
| 302 |
+
)
|
| 303 |
+
with gr.Row():
|
| 304 |
+
symbol_in3 = gr.Dropdown(choices=ALL_DEFAULT_SYMBOLS, value="EURUSD=X",
|
| 305 |
+
allow_custom_value=True, label="Symbol")
|
| 306 |
+
timeframe_in3 = gr.Dropdown(choices=TIMEFRAME_CHOICES, value="15m", label="Timeframe")
|
| 307 |
+
with gr.Row():
|
| 308 |
+
models_in3 = gr.CheckboxGroup(choices=MODEL_NAMES, value=["ARIMA"], label="Models to backtest")
|
| 309 |
+
window_in3 = gr.Slider(20, 300, value=60, step=10, label="History window per prediction")
|
| 310 |
+
horizon_in3 = gr.Slider(1, 5, value=1, step=1, label="Horizon (candles ahead)")
|
| 311 |
+
with gr.Row():
|
| 312 |
+
step_in3 = gr.Slider(1, 20, value=1, step=1, label="Step (skip candles between tests)")
|
| 313 |
+
lookback_in3 = gr.Slider(1, 60, value=10, step=1, label="Backtest lookback (days)")
|
| 314 |
+
backtest_btn = gr.Button("Run Backtest", variant="primary")
|
| 315 |
+
summary_out3 = gr.Dataframe(
|
| 316 |
+
headers=["Model", "Total", "Correct", "Incorrect", "Accuracy", "MAE", "RMSE", "Failed"],
|
| 317 |
+
label="Per-model accuracy — kept separate, never mixed",
|
| 318 |
+
)
|
| 319 |
+
status_out3 = gr.Markdown()
|
| 320 |
+
with gr.Tabs():
|
| 321 |
+
with gr.Tab("ARIMA — verify log"):
|
| 322 |
+
arima_detail = gr.Dataframe(label="Per-candle: predicted vs actual, correct/incorrect")
|
| 323 |
+
with gr.Tab("Auto-ARIMA — verify log"):
|
| 324 |
+
autoarima_detail = gr.Dataframe(label="Per-candle: predicted vs actual, correct/incorrect")
|
| 325 |
+
with gr.Tab("ARIMA-GARCH — verify log"):
|
| 326 |
+
arimagarch_detail = gr.Dataframe(label="Per-candle: predicted vs actual, correct/incorrect")
|
| 327 |
+
with gr.Tab("Moirai — verify log"):
|
| 328 |
+
moirai_detail = gr.Dataframe(label="Per-candle: predicted vs actual, correct/incorrect")
|
| 329 |
+
with gr.Tab("TimesFM — verify log"):
|
| 330 |
+
timesfm_detail = gr.Dataframe(label="Per-candle: predicted vs actual, correct/incorrect")
|
| 331 |
+
|
| 332 |
+
backtest_btn.click(
|
| 333 |
+
run_backtest_ui,
|
| 334 |
+
[symbol_in3, timeframe_in3, models_in3, window_in3, horizon_in3, step_in3, lookback_in3],
|
| 335 |
+
[summary_out3, arima_detail, autoarima_detail, arimagarch_detail,
|
| 336 |
+
moirai_detail, timesfm_detail, status_out3],
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
with gr.Tab("📡 Live"):
|
| 340 |
+
gr.Markdown(
|
| 341 |
+
"Polls Yahoo Finance for the newest candle each time you click the button, "
|
| 342 |
+
"predicts the next one, and verifies the previous prediction once that candle "
|
| 343 |
+
"has closed. For unattended/continuous logging instead of clicking manually, "
|
| 344 |
+
"run `live_runner.py` in the background (see README)."
|
| 345 |
+
)
|
| 346 |
+
with gr.Row():
|
| 347 |
+
symbol_in4 = gr.Dropdown(choices=ALL_DEFAULT_SYMBOLS, value="BTC-USD",
|
| 348 |
+
allow_custom_value=True, label="Symbol")
|
| 349 |
+
timeframe_in4 = gr.Dropdown(choices=TIMEFRAME_CHOICES, value="5m", label="Timeframe")
|
| 350 |
+
model_in4 = gr.Dropdown(choices=MODEL_NAMES, value="ARIMA", label="Model")
|
| 351 |
+
live_btn = gr.Button("🔄 Check Latest & Predict Next", variant="primary")
|
| 352 |
+
live_log_state = gr.State(pd.DataFrame(columns=LIVE_COLUMNS))
|
| 353 |
+
live_status = gr.Markdown()
|
| 354 |
+
live_table = gr.Dataframe(label="Live verification log (this model, this session)")
|
| 355 |
+
live_btn.click(check_live, [symbol_in4, timeframe_in4, model_in4, live_log_state],
|
| 356 |
+
[live_status, live_table, live_log_state])
|
| 357 |
+
|
| 358 |
+
gr.Markdown("---\n" + DISCLAIMER)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
if __name__ == "__main__":
|
| 362 |
+
demo.launch()
|
backtest/__init__.py
ADDED
|
File without changes
|
backtest/backtester.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Walk-forward backtesting + per-candle verification.
|
| 3 |
+
|
| 4 |
+
Every model's results are kept in a SEPARATE dict/DataFrame and are never
|
| 5 |
+
averaged or blended across models — each model's accuracy is reported on its
|
| 6 |
+
own (har model ka accuracy alag hona chahiye, mix nahi).
|
| 7 |
+
|
| 8 |
+
For each step of the walk-forward loop we log the predicted close, the real
|
| 9 |
+
close once it's known, the predicted vs actual direction (up/down), and
|
| 10 |
+
whether that candle's prediction was correct — this is the "verify how many
|
| 11 |
+
candles were right / wrong" log.
|
| 12 |
+
"""
|
| 13 |
+
import numpy as np
|
| 14 |
+
import pandas as pd
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _round_sig(x, sig: int = 6):
|
| 18 |
+
"""Round to `sig` significant figures instead of a fixed decimal count.
|
| 19 |
+
|
| 20 |
+
This app accepts ANY custom yfinance ticker (README: "any custom
|
| 21 |
+
yfinance-valid ticker you type in"), so MAE/RMSE have to stay meaningful
|
| 22 |
+
across wildly different price scales in the same codebase: a forex pair
|
| 23 |
+
(~1.10), BTC (~65,000), and a low-price altcoin (~0.000018) can all show
|
| 24 |
+
up. A fixed `round(x, 6)`:
|
| 25 |
+
- silently prints 0.0 for a real, non-zero error once the asset's
|
| 26 |
+
price is below ~1e-6 (a low-price token with genuine, non-trivial
|
| 27 |
+
prediction error would misleadingly look "perfect"), and
|
| 28 |
+
- prints false precision for a high-price asset like BTC, e.g.
|
| 29 |
+
`125.345678` for a forecast error that has nowhere near six decimal
|
| 30 |
+
digits of real meaning at that price scale.
|
| 31 |
+
Significant-figure rounding keeps both ends honest without needing to
|
| 32 |
+
know the asset's scale ahead of time.
|
| 33 |
+
"""
|
| 34 |
+
if x is None or not np.isfinite(x) or x == 0:
|
| 35 |
+
return x
|
| 36 |
+
from math import floor, log10
|
| 37 |
+
decimals = sig - 1 - floor(log10(abs(x)))
|
| 38 |
+
return round(x, decimals)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def backtest_model(df: pd.DataFrame, model, window: int = 100, horizon: int = 1,
|
| 42 |
+
step: int = 1) -> pd.DataFrame:
|
| 43 |
+
"""
|
| 44 |
+
Walk forward through `df` (needs a 'Close' column, sorted oldest -> newest).
|
| 45 |
+
At each step, feed the last `window` closes into `model.predict()`,
|
| 46 |
+
compare the forecast to the real next close, and log correct/incorrect.
|
| 47 |
+
|
| 48 |
+
`step` > 1 skips candles between predictions — raise this to speed up the
|
| 49 |
+
slower foundation models (Moirai/TimesFM) on CPU-only hardware.
|
| 50 |
+
"""
|
| 51 |
+
close = df["Close"].reset_index(drop=True)
|
| 52 |
+
timestamps = df.index
|
| 53 |
+
|
| 54 |
+
last_i = len(close) - horizon
|
| 55 |
+
rows = []
|
| 56 |
+
for i in range(window, last_i + 1, step):
|
| 57 |
+
history = close.iloc[i - window:i]
|
| 58 |
+
current_close = float(close.iloc[i - 1])
|
| 59 |
+
target_idx = i + horizon - 1
|
| 60 |
+
actual_next_close = float(close.iloc[target_idx])
|
| 61 |
+
|
| 62 |
+
try:
|
| 63 |
+
forecast = model.predict(history, horizon=horizon)
|
| 64 |
+
predicted_next_close = float(forecast[-1])
|
| 65 |
+
if not np.isfinite(predicted_next_close):
|
| 66 |
+
raise ValueError(f"non-finite forecast value: {predicted_next_close!r}")
|
| 67 |
+
except Exception as e:
|
| 68 |
+
rows.append({
|
| 69 |
+
"datetime": timestamps[target_idx],
|
| 70 |
+
"current_close": current_close,
|
| 71 |
+
"predicted_close": np.nan,
|
| 72 |
+
"actual_close": actual_next_close,
|
| 73 |
+
"predicted_direction": None,
|
| 74 |
+
"actual_direction": None,
|
| 75 |
+
"correct": None,
|
| 76 |
+
"error": str(e),
|
| 77 |
+
})
|
| 78 |
+
continue
|
| 79 |
+
|
| 80 |
+
predicted_direction = "up" if predicted_next_close > current_close else "down"
|
| 81 |
+
actual_direction = "up" if actual_next_close > current_close else "down"
|
| 82 |
+
correct = predicted_direction == actual_direction
|
| 83 |
+
|
| 84 |
+
rows.append({
|
| 85 |
+
"datetime": timestamps[target_idx],
|
| 86 |
+
"current_close": current_close,
|
| 87 |
+
"predicted_close": predicted_next_close,
|
| 88 |
+
"actual_close": actual_next_close,
|
| 89 |
+
"predicted_direction": predicted_direction,
|
| 90 |
+
"actual_direction": actual_direction,
|
| 91 |
+
"correct": correct,
|
| 92 |
+
"error": None,
|
| 93 |
+
})
|
| 94 |
+
|
| 95 |
+
return pd.DataFrame(rows)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def summarize_backtest(results: pd.DataFrame, model_name: str) -> dict:
|
| 99 |
+
"""One model's accuracy summary. Always kept separate per model — never
|
| 100 |
+
combined with another model's numbers."""
|
| 101 |
+
if results.empty:
|
| 102 |
+
return {
|
| 103 |
+
"model": model_name, "total_predictions": 0, "correct": 0, "incorrect": 0,
|
| 104 |
+
"accuracy_pct": 0.0, "mae": None, "rmse": None, "failed_predictions": 0,
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
valid = results.dropna(subset=["correct"])
|
| 108 |
+
total = len(valid)
|
| 109 |
+
correct = int(valid["correct"].sum()) if total else 0
|
| 110 |
+
incorrect = total - correct
|
| 111 |
+
accuracy = round(100 * correct / total, 2) if total else 0.0
|
| 112 |
+
|
| 113 |
+
price_valid = results.dropna(subset=["predicted_close", "actual_close"])
|
| 114 |
+
if len(price_valid):
|
| 115 |
+
errors = price_valid["predicted_close"] - price_valid["actual_close"]
|
| 116 |
+
mae = float(np.mean(np.abs(errors)))
|
| 117 |
+
rmse = float(np.sqrt(np.mean(errors ** 2)))
|
| 118 |
+
else:
|
| 119 |
+
mae = rmse = None
|
| 120 |
+
|
| 121 |
+
failed = int(results["error"].notna().sum())
|
| 122 |
+
|
| 123 |
+
return {
|
| 124 |
+
"model": model_name,
|
| 125 |
+
"total_predictions": total,
|
| 126 |
+
"correct": correct,
|
| 127 |
+
"incorrect": incorrect,
|
| 128 |
+
"accuracy_pct": accuracy,
|
| 129 |
+
"mae": _round_sig(mae) if mae is not None else None,
|
| 130 |
+
"rmse": _round_sig(rmse) if rmse is not None else None,
|
| 131 |
+
"failed_predictions": failed,
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def run_all_models_backtest(df: pd.DataFrame, models: dict, window: int = 100,
|
| 136 |
+
horizon: int = 1, step: int = 1):
|
| 137 |
+
"""
|
| 138 |
+
Run a backtest for every model in `models` ({name: model_instance}).
|
| 139 |
+
Returns (per_model_results, per_model_summary) — both dicts keyed by
|
| 140 |
+
model name, so results are never merged/averaged across models.
|
| 141 |
+
"""
|
| 142 |
+
per_model_results = {}
|
| 143 |
+
per_model_summary = {}
|
| 144 |
+
for name, model in models.items():
|
| 145 |
+
results = backtest_model(df, model, window=window, horizon=horizon, step=step)
|
| 146 |
+
per_model_results[name] = results
|
| 147 |
+
per_model_summary[name] = summarize_backtest(results, name)
|
| 148 |
+
return per_model_results, per_model_summary
|
config.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Central configuration for the Forex & Crypto Prediction project.
|
| 3 |
+
Edit this file to add symbols, change default model checkpoints, etc.
|
| 4 |
+
Nothing here needs internet access to import.
|
| 5 |
+
"""
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 9 |
+
DATA_CACHE_DIR = os.path.join(BASE_DIR, "data_cache")
|
| 10 |
+
OUTPUT_DIR = os.path.join(BASE_DIR, "outputs")
|
| 11 |
+
|
| 12 |
+
os.makedirs(DATA_CACHE_DIR, exist_ok=True)
|
| 13 |
+
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
| 14 |
+
|
| 15 |
+
# ---------------------------------------------------------------------------
|
| 16 |
+
# Symbols (yfinance ticker format)
|
| 17 |
+
# ---------------------------------------------------------------------------
|
| 18 |
+
# Forex pairs use the "=X" suffix, crypto pairs use "-USD".
|
| 19 |
+
# NOTE on "all symbols": yfinance/Yahoo Finance covers a huge number of forex
|
| 20 |
+
# and crypto tickers, so there is no fixed master list to hardcode. The app's
|
| 21 |
+
# Symbol dropdown ships with the popular ones below AND accepts any custom
|
| 22 |
+
# yfinance-valid ticker typed in directly (allow_custom_value=True) — so you
|
| 23 |
+
# are not limited to this list, this is just a convenient starting set.
|
| 24 |
+
DEFAULT_FOREX_SYMBOLS = [
|
| 25 |
+
"EURUSD=X", "GBPUSD=X", "USDJPY=X", "USDCHF=X", "AUDUSD=X",
|
| 26 |
+
"USDCAD=X", "NZDUSD=X", "EURJPY=X", "EURGBP=X", "GBPJPY=X",
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
DEFAULT_CRYPTO_SYMBOLS = [
|
| 30 |
+
"BTC-USD", "ETH-USD", "BNB-USD", "XRP-USD", "SOL-USD",
|
| 31 |
+
"ADA-USD", "DOGE-USD", "DOT-USD", "LTC-USD", "MATIC-USD",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
ALL_DEFAULT_SYMBOLS = DEFAULT_FOREX_SYMBOLS + DEFAULT_CRYPTO_SYMBOLS
|
| 35 |
+
|
| 36 |
+
# ---------------------------------------------------------------------------
|
| 37 |
+
# Timeframes
|
| 38 |
+
# ---------------------------------------------------------------------------
|
| 39 |
+
# Yahoo Finance only serves a fixed set of native intraday intervals, each
|
| 40 |
+
# with its own history-depth limit (enforced by Yahoo, not by us):
|
| 41 |
+
# 1m -> last 7 days only
|
| 42 |
+
# 2m/5m/15m/30m/90m -> last 60 days
|
| 43 |
+
# 60m (1h) -> last 730 days
|
| 44 |
+
# 1d/1wk/1mo -> full history
|
| 45 |
+
YF_NATIVE_INTERVALS = {
|
| 46 |
+
"1m": {"max_days": 7},
|
| 47 |
+
"2m": {"max_days": 60},
|
| 48 |
+
"5m": {"max_days": 60},
|
| 49 |
+
"15m": {"max_days": 60},
|
| 50 |
+
"30m": {"max_days": 60},
|
| 51 |
+
"60m": {"max_days": 730},
|
| 52 |
+
"90m": {"max_days": 60},
|
| 53 |
+
"1d": {"max_days": None},
|
| 54 |
+
"1wk": {"max_days": None},
|
| 55 |
+
"1mo": {"max_days": None},
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
# Timeframes offered in the app UI. Ones that aren't native Yahoo intervals
|
| 59 |
+
# (e.g. "10m") are built by resampling a smaller native interval, so you can
|
| 60 |
+
# ask for basically any custom candle size, not just what Yahoo natively has.
|
| 61 |
+
#
|
| 62 |
+
# `rule: None` means this timeframe IS a native Yahoo interval already (just
|
| 63 |
+
# under a friendlier display name) -- return it as-is, no resampling.
|
| 64 |
+
# `rule: "<pandas rule>"` means genuine upsampling from a smaller native
|
| 65 |
+
# interval is needed.
|
| 66 |
+
#
|
| 67 |
+
# "1h" -> source "60m" is the one case where the display name and yfinance's
|
| 68 |
+
# native interval name differ ("1h" vs "60m") even though they're the same
|
| 69 |
+
# candle size. That mismatch used to make get_historical()'s
|
| 70 |
+
# `source_interval != timeframe` check misfire and re-resample already-hourly
|
| 71 |
+
# candles through rule="1h" -- a harmless-looking no-op when Yahoo's 60m
|
| 72 |
+
# candles happen to fall on clean UTC hour boundaries, but not otherwise: it
|
| 73 |
+
# silently re-labels each candle's timestamp to the bucket start (observed
|
| 74 |
+
# shifting real candle times by up to 59 minutes in testing) and, for
|
| 75 |
+
# irregularly-spaced candles (weekend gaps, DST), can merge two distinct
|
| 76 |
+
# candles into one bucket or drop one to a dropna(how="any"). Native
|
| 77 |
+
# passthrough (`rule: None`) sidesteps all of that.
|
| 78 |
+
CUSTOM_TIMEFRAMES = {
|
| 79 |
+
"1m": {"source": "1m", "rule": None},
|
| 80 |
+
"5m": {"source": "5m", "rule": None},
|
| 81 |
+
"10m": {"source": "5m", "rule": "10min"},
|
| 82 |
+
"15m": {"source": "15m", "rule": None},
|
| 83 |
+
"30m": {"source": "30m", "rule": None},
|
| 84 |
+
"1h": {"source": "60m", "rule": None},
|
| 85 |
+
"4h": {"source": "60m", "rule": "4h"},
|
| 86 |
+
"1d": {"source": "1d", "rule": None},
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
# ---------------------------------------------------------------------------
|
| 90 |
+
# Models
|
| 91 |
+
# ---------------------------------------------------------------------------
|
| 92 |
+
MODEL_NAMES = ["ARIMA", "Auto-ARIMA", "ARIMA-GARCH", "Moirai", "TimesFM"]
|
| 93 |
+
|
| 94 |
+
# Small/CPU-friendly checkpoints, picked to fit comfortably in 2 vCPU / 16GB
|
| 95 |
+
# RAM. Change these if you deploy on bigger (e.g. GPU) hardware.
|
| 96 |
+
MOIRAI_CHECKPOINT = "Salesforce/moirai-1.1-R-small"
|
| 97 |
+
TIMESFM_CHECKPOINT = "google/timesfm-2.5-200m-pytorch"
|
| 98 |
+
|
| 99 |
+
RANDOM_SEED = 42
|
data/__init__.py
ADDED
|
File without changes
|
data/historical.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Historical OHLCV data via yfinance — free, no API key needed.
|
| 3 |
+
|
| 4 |
+
Two things this module does beyond a plain yf.download():
|
| 5 |
+
1. Resampling: Yahoo only serves a fixed set of native intervals, so any
|
| 6 |
+
timeframe in config.CUSTOM_TIMEFRAMES that isn't native (e.g. "10m") is
|
| 7 |
+
built by aggregating a smaller native interval with pandas .resample().
|
| 8 |
+
2. Chunked 1m fetching: Yahoo limits a single request for 1m candles to 7
|
| 9 |
+
days, but keeps roughly 30 days of 1m history available in total. If you
|
| 10 |
+
ask for more than 7 days of 1m data, this module makes several 7-day
|
| 11 |
+
requests and stitches them together instead of silently truncating.
|
| 12 |
+
"""
|
| 13 |
+
import os
|
| 14 |
+
import time
|
| 15 |
+
from datetime import datetime, timedelta, timezone
|
| 16 |
+
|
| 17 |
+
import pandas as pd
|
| 18 |
+
import yfinance as yf
|
| 19 |
+
|
| 20 |
+
from config import YF_NATIVE_INTERVALS, CUSTOM_TIMEFRAMES, DATA_CACHE_DIR
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class DataFetchError(Exception):
|
| 24 |
+
"""Raised for anything that stops us returning usable candles."""
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _download_window(symbol: str, interval: str, start: datetime, end: datetime) -> pd.DataFrame:
|
| 28 |
+
df = yf.Ticker(symbol).history(start=start, end=end, interval=interval)
|
| 29 |
+
if df is None or df.empty:
|
| 30 |
+
return pd.DataFrame()
|
| 31 |
+
df = df.rename(columns=str.title)
|
| 32 |
+
df.index.name = "Datetime"
|
| 33 |
+
keep = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in df.columns]
|
| 34 |
+
return df[keep]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _native_fetch(symbol: str, interval: str, days: int) -> pd.DataFrame:
|
| 38 |
+
"""Fetch raw OHLCV candles at a yfinance-native interval."""
|
| 39 |
+
if interval not in YF_NATIVE_INTERVALS:
|
| 40 |
+
raise DataFetchError(f"'{interval}' is not a native yfinance interval.")
|
| 41 |
+
|
| 42 |
+
max_days = YF_NATIVE_INTERVALS[interval]["max_days"]
|
| 43 |
+
end = datetime.now(timezone.utc)
|
| 44 |
+
|
| 45 |
+
if max_days is None:
|
| 46 |
+
# 1d / 1wk / 1mo — Yahoo has no rolling-window limit, just ask for it all.
|
| 47 |
+
df = yf.Ticker(symbol).history(period="max", interval=interval)
|
| 48 |
+
if df is None or df.empty:
|
| 49 |
+
raise DataFetchError(
|
| 50 |
+
f"No data returned for '{symbol}' at interval '{interval}'. "
|
| 51 |
+
f"Check it's a valid yfinance ticker (e.g. EURUSD=X, BTC-USD)."
|
| 52 |
+
)
|
| 53 |
+
df = df.rename(columns=str.title)
|
| 54 |
+
df.index.name = "Datetime"
|
| 55 |
+
return df[["Open", "High", "Low", "Close", "Volume"]]
|
| 56 |
+
|
| 57 |
+
days = min(days, max_days)
|
| 58 |
+
|
| 59 |
+
if interval == "1m" and days > 7:
|
| 60 |
+
# Yahoo caps a single 1m request at 7 days — stitch several requests
|
| 61 |
+
# together to cover the full requested window (up to its ~30 day limit).
|
| 62 |
+
frames = []
|
| 63 |
+
window_end = end
|
| 64 |
+
remaining = days
|
| 65 |
+
while remaining > 0:
|
| 66 |
+
window_days = min(7, remaining)
|
| 67 |
+
window_start = window_end - timedelta(days=window_days)
|
| 68 |
+
frame = _download_window(symbol, interval, window_start, window_end)
|
| 69 |
+
if not frame.empty:
|
| 70 |
+
frames.append(frame)
|
| 71 |
+
window_end = window_start
|
| 72 |
+
remaining -= window_days
|
| 73 |
+
time.sleep(0.2) # be polite to Yahoo's endpoint between chunks
|
| 74 |
+
if not frames:
|
| 75 |
+
raise DataFetchError(
|
| 76 |
+
f"No 1m data returned for '{symbol}'. Yahoo only keeps ~30 days "
|
| 77 |
+
f"of 1-minute history — try a shorter lookback or a bigger timeframe."
|
| 78 |
+
)
|
| 79 |
+
df = pd.concat(frames).sort_index()
|
| 80 |
+
df = df[~df.index.duplicated(keep="last")]
|
| 81 |
+
return df
|
| 82 |
+
|
| 83 |
+
start = end - timedelta(days=days)
|
| 84 |
+
df = _download_window(symbol, interval, start, end)
|
| 85 |
+
if df.empty:
|
| 86 |
+
raise DataFetchError(
|
| 87 |
+
f"No data returned for '{symbol}' at interval '{interval}'. "
|
| 88 |
+
f"Check it's a valid yfinance ticker (e.g. EURUSD=X, BTC-USD) and "
|
| 89 |
+
f"that the market has traded recently."
|
| 90 |
+
)
|
| 91 |
+
return df
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def resample_ohlc(df: pd.DataFrame, rule: str) -> pd.DataFrame:
|
| 95 |
+
"""Aggregate candles up to a coarser timeframe, e.g. 5m candles -> 10m candles."""
|
| 96 |
+
agg = {"Open": "first", "High": "max", "Low": "min", "Close": "last", "Volume": "sum"}
|
| 97 |
+
agg = {k: v for k, v in agg.items() if k in df.columns}
|
| 98 |
+
out = df.resample(rule).agg(agg).dropna(how="any")
|
| 99 |
+
return out
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def get_historical(symbol: str, timeframe: str, lookback_days: int = None) -> pd.DataFrame:
|
| 103 |
+
"""
|
| 104 |
+
Fetch historical candles for `symbol` at `timeframe`.
|
| 105 |
+
|
| 106 |
+
`timeframe` can be a native yfinance interval (see config.YF_NATIVE_INTERVALS)
|
| 107 |
+
OR one of config.CUSTOM_TIMEFRAMES (e.g. "10m", "4h"), which is built by
|
| 108 |
+
resampling a smaller native interval.
|
| 109 |
+
"""
|
| 110 |
+
if not symbol or not symbol.strip():
|
| 111 |
+
raise DataFetchError("No symbol given.")
|
| 112 |
+
symbol = symbol.strip()
|
| 113 |
+
|
| 114 |
+
if timeframe in CUSTOM_TIMEFRAMES:
|
| 115 |
+
spec = CUSTOM_TIMEFRAMES[timeframe]
|
| 116 |
+
source_interval, rule = spec["source"], spec["rule"]
|
| 117 |
+
elif timeframe in YF_NATIVE_INTERVALS:
|
| 118 |
+
source_interval, rule = timeframe, None
|
| 119 |
+
else:
|
| 120 |
+
raise DataFetchError(
|
| 121 |
+
f"Unknown timeframe '{timeframe}'. Valid options: "
|
| 122 |
+
f"{', '.join(CUSTOM_TIMEFRAMES.keys())}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
max_days = YF_NATIVE_INTERVALS[source_interval]["max_days"]
|
| 126 |
+
days = lookback_days or (max_days if max_days else 730)
|
| 127 |
+
|
| 128 |
+
df = _native_fetch(symbol, source_interval, days)
|
| 129 |
+
|
| 130 |
+
if rule:
|
| 131 |
+
df = resample_ohlc(df, rule)
|
| 132 |
+
if df.empty:
|
| 133 |
+
raise DataFetchError(
|
| 134 |
+
f"Not enough '{source_interval}' candles to build '{timeframe}' bars. "
|
| 135 |
+
f"Try a longer lookback."
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
return df
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _cache_path(symbol: str, timeframe: str) -> str:
|
| 142 |
+
safe = symbol.replace("=", "_").replace("^", "_").replace("/", "_")
|
| 143 |
+
return os.path.join(DATA_CACHE_DIR, f"{safe}_{timeframe}.csv")
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def get_historical_cached(symbol: str, timeframe: str, lookback_days: int = None,
|
| 147 |
+
refresh: bool = True) -> pd.DataFrame:
|
| 148 |
+
"""
|
| 149 |
+
Same as get_historical, but reads/writes a CSV cache under data_cache/ so
|
| 150 |
+
repeated UI actions don't all need a fresh network round trip.
|
| 151 |
+
`refresh=True` (default) always fetches live and overwrites the cache —
|
| 152 |
+
pass refresh=False to reuse whatever is cached, if anything.
|
| 153 |
+
"""
|
| 154 |
+
path = _cache_path(symbol, timeframe)
|
| 155 |
+
|
| 156 |
+
if not refresh and os.path.exists(path):
|
| 157 |
+
try:
|
| 158 |
+
cached = pd.read_csv(path, index_col="Datetime", parse_dates=True)
|
| 159 |
+
if not cached.empty:
|
| 160 |
+
return cached
|
| 161 |
+
except Exception:
|
| 162 |
+
pass # fall through and fetch fresh
|
| 163 |
+
|
| 164 |
+
df = get_historical(symbol, timeframe, lookback_days)
|
| 165 |
+
try:
|
| 166 |
+
df.to_csv(path)
|
| 167 |
+
except Exception:
|
| 168 |
+
pass # caching is a convenience, never fatal
|
| 169 |
+
return df
|
data/live.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
'Live' data — Yahoo Finance has no free websocket feed, so live data here
|
| 3 |
+
means polling the same free yfinance source used for history, on a timer,
|
| 4 |
+
and watching for a new candle to appear. That's the standard approach for
|
| 5 |
+
free-tier forex/crypto data and is what config.py / app.py assume.
|
| 6 |
+
"""
|
| 7 |
+
import time
|
| 8 |
+
|
| 9 |
+
import pandas as pd
|
| 10 |
+
|
| 11 |
+
from data.historical import get_historical, DataFetchError
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def get_latest_candle(symbol: str, timeframe: str) -> pd.Series:
|
| 15 |
+
"""Return the most recent (possibly still-forming) candle."""
|
| 16 |
+
df = get_historical(symbol, timeframe, lookback_days=2)
|
| 17 |
+
if df.empty:
|
| 18 |
+
raise DataFetchError(f"No live data available for {symbol}.")
|
| 19 |
+
return df.iloc[-1]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def poll_live(symbol: str, timeframe: str, interval_seconds: int, on_new_candle, stop_flag=None):
|
| 23 |
+
"""
|
| 24 |
+
Blocking loop: polls for a new candle every `interval_seconds` and calls
|
| 25 |
+
`on_new_candle(candle)` whenever the latest timestamp changes.
|
| 26 |
+
|
| 27 |
+
Pass a `stop_flag` object with a mutable `.stop` attribute to end the loop
|
| 28 |
+
gracefully from another thread (set stop_flag.stop = True); otherwise this
|
| 29 |
+
runs until interrupted. Used by live_runner.py, not required by the
|
| 30 |
+
Gradio UI (which polls on a button click instead of a background thread).
|
| 31 |
+
"""
|
| 32 |
+
last_ts = None
|
| 33 |
+
while stop_flag is None or not getattr(stop_flag, "stop", False):
|
| 34 |
+
try:
|
| 35 |
+
candle = get_latest_candle(symbol, timeframe)
|
| 36 |
+
ts = candle.name
|
| 37 |
+
if ts != last_ts:
|
| 38 |
+
last_ts = ts
|
| 39 |
+
on_new_candle(candle)
|
| 40 |
+
except Exception as e:
|
| 41 |
+
print(f"[live] fetch error for {symbol}: {e}")
|
| 42 |
+
time.sleep(interval_seconds)
|
data_cache/.gitkeep
ADDED
|
File without changes
|
indicators/__init__.py
ADDED
|
File without changes
|
indicators/indicators.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Top 5 technical indicators for the chart/analysis tab.
|
| 3 |
+
|
| 4 |
+
Implemented directly with pandas/numpy (no extra TA library dependency, so
|
| 5 |
+
there's one less thing that can fail to install):
|
| 6 |
+
1. RSI - Relative Strength Index
|
| 7 |
+
2. MACD - Moving Average Convergence Divergence
|
| 8 |
+
3. Bollinger Bands
|
| 9 |
+
4. Moving Averages - SMA & EMA
|
| 10 |
+
5. Stochastic Oscillator
|
| 11 |
+
|
| 12 |
+
These are informational/analysis indicators shown alongside the chart. They
|
| 13 |
+
are NOT part of the backtested model accuracy numbers — those come only from
|
| 14 |
+
the 5 forecasting models (ARIMA / Auto-ARIMA / ARIMA-GARCH / Moirai /
|
| 15 |
+
TimesFM) in backtest/backtester.py, kept deliberately separate so nothing
|
| 16 |
+
gets mixed together.
|
| 17 |
+
"""
|
| 18 |
+
import numpy as np
|
| 19 |
+
import pandas as pd
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def sma(series: pd.Series, window: int) -> pd.Series:
|
| 23 |
+
return series.rolling(window=window).mean()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def ema(series: pd.Series, window: int) -> pd.Series:
|
| 27 |
+
return series.ewm(span=window, adjust=False).mean()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _wilder_rma(values: pd.Series, window: int) -> pd.Series:
|
| 31 |
+
"""
|
| 32 |
+
Wilder's exact smoothed moving average (the "RMA" his RSI/ATR/ADX are
|
| 33 |
+
all defined with): the FIRST output is a plain SIMPLE average of the
|
| 34 |
+
first `window` values, and every value after that is
|
| 35 |
+
`prev*(window-1)/window + current/window`.
|
| 36 |
+
|
| 37 |
+
`series.ewm(alpha=1/window, adjust=False).mean()` looks equivalent but
|
| 38 |
+
is NOT: `adjust=False` seeds the recursion from the very first raw
|
| 39 |
+
observation instead of a `window`-length simple-average, so for window
|
| 40 |
+
periods it takes roughly 6-7x `window` more bars to converge to what
|
| 41 |
+
the simple-average seed gives immediately. Verified against the
|
| 42 |
+
classic 14-period Wilder RSI reference example (StockCharts'
|
| 43 |
+
44.34/44.09/.../46.28 series): the seeded version reproduces the
|
| 44 |
+
textbook 70.46 exactly; plain `ewm(adjust=False)` gives 50.66 at that
|
| 45 |
+
same point -- a ~20-point error right at the point a short lookback
|
| 46 |
+
would actually display. The two converge (difference < 0.05) only
|
| 47 |
+
after roughly 100 bars, so this only silently matters on short
|
| 48 |
+
lookbacks -- which the Chart tab's lookback slider allows down to 1 day.
|
| 49 |
+
"""
|
| 50 |
+
arr = values.to_numpy(dtype=float)
|
| 51 |
+
n = len(arr)
|
| 52 |
+
out = np.full(n, np.nan)
|
| 53 |
+
first_valid_pos = 0
|
| 54 |
+
while first_valid_pos < n and np.isnan(arr[first_valid_pos]):
|
| 55 |
+
first_valid_pos += 1
|
| 56 |
+
seed_end = first_valid_pos + window
|
| 57 |
+
if seed_end > n:
|
| 58 |
+
return pd.Series(out, index=values.index)
|
| 59 |
+
prev = arr[first_valid_pos:seed_end].mean()
|
| 60 |
+
out[seed_end - 1] = prev
|
| 61 |
+
for i in range(seed_end, n):
|
| 62 |
+
prev = (prev * (window - 1) + arr[i]) / window
|
| 63 |
+
out[i] = prev
|
| 64 |
+
return pd.Series(out, index=values.index)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def rsi(series: pd.Series, window: int = 14) -> pd.Series:
|
| 68 |
+
delta = series.diff()
|
| 69 |
+
gain = delta.clip(lower=0)
|
| 70 |
+
loss = -delta.clip(upper=0)
|
| 71 |
+
avg_gain = _wilder_rma(gain, window)
|
| 72 |
+
avg_loss = _wilder_rma(loss, window)
|
| 73 |
+
rs = avg_gain / avg_loss.replace(0, np.nan)
|
| 74 |
+
out = 100 - (100 / (1 + rs))
|
| 75 |
+
# Wilder's defined edge case (see StockCharts/Wilder RSI reference): a
|
| 76 |
+
# window with gains but zero losses is a divide-by-zero for RS, and RSI
|
| 77 |
+
# is defined as 100 by convention -- NOT neutral. Without this, an
|
| 78 |
+
# all-gains window (the exact moment RSI should scream "overbought")
|
| 79 |
+
# was silently reported as a neutral 50. A genuinely flat window (zero
|
| 80 |
+
# gain AND zero loss) has no such convention and still falls through to
|
| 81 |
+
# the neutral fillna(50) below, same as the warm-up NaN period.
|
| 82 |
+
out = out.mask((avg_loss == 0) & (avg_gain > 0), 100.0)
|
| 83 |
+
return out.fillna(50)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def macd(series: pd.Series, fast: int = 12, slow: int = 26, signal: int = 9):
|
| 87 |
+
fast_ema = ema(series, fast)
|
| 88 |
+
slow_ema = ema(series, slow)
|
| 89 |
+
macd_line = fast_ema - slow_ema
|
| 90 |
+
signal_line = ema(macd_line, signal)
|
| 91 |
+
histogram = macd_line - signal_line
|
| 92 |
+
return macd_line, signal_line, histogram
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def bollinger_bands(series: pd.Series, window: int = 20, num_std: float = 2.0):
|
| 96 |
+
mid = sma(series, window)
|
| 97 |
+
# ddof=0 (population std): traditional Bollinger Bands, as defined by
|
| 98 |
+
# John Bollinger, use the population standard deviation of the SAME
|
| 99 |
+
# `window` closes as the SMA -- not pandas' default sample std
|
| 100 |
+
# (ddof=1), which divides by (window-1) and makes the bands ~2-3%
|
| 101 |
+
# too wide at the default window=20, shifting the "above/below/inside
|
| 102 |
+
# the band" reading right at the margin.
|
| 103 |
+
std = series.rolling(window=window).std(ddof=0)
|
| 104 |
+
upper = mid + num_std * std
|
| 105 |
+
lower = mid - num_std * std
|
| 106 |
+
return upper, mid, lower
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def stochastic_oscillator(high: pd.Series, low: pd.Series, close: pd.Series,
|
| 110 |
+
k_window: int = 14, d_window: int = 3):
|
| 111 |
+
lowest_low = low.rolling(window=k_window).min()
|
| 112 |
+
highest_high = high.rolling(window=k_window).max()
|
| 113 |
+
denom = (highest_high - lowest_low).replace(0, np.nan)
|
| 114 |
+
percent_k = 100 * (close - lowest_low) / denom
|
| 115 |
+
percent_d = percent_k.rolling(window=d_window).mean()
|
| 116 |
+
return percent_k.fillna(50), percent_d.fillna(50)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def add_all_indicators(df: pd.DataFrame) -> pd.DataFrame:
|
| 120 |
+
"""Return a copy of df with all 5 indicators added as columns."""
|
| 121 |
+
out = df.copy()
|
| 122 |
+
out["SMA_20"] = sma(out["Close"], 20)
|
| 123 |
+
out["EMA_20"] = ema(out["Close"], 20)
|
| 124 |
+
out["RSI_14"] = rsi(out["Close"], 14)
|
| 125 |
+
|
| 126 |
+
macd_line, signal_line, hist = macd(out["Close"])
|
| 127 |
+
out["MACD"] = macd_line
|
| 128 |
+
out["MACD_Signal"] = signal_line
|
| 129 |
+
out["MACD_Hist"] = hist
|
| 130 |
+
|
| 131 |
+
upper, mid, lower = bollinger_bands(out["Close"])
|
| 132 |
+
out["BB_Upper"] = upper
|
| 133 |
+
out["BB_Mid"] = mid
|
| 134 |
+
out["BB_Lower"] = lower
|
| 135 |
+
|
| 136 |
+
k, d = stochastic_oscillator(out["High"], out["Low"], out["Close"])
|
| 137 |
+
out["Stoch_%K"] = k
|
| 138 |
+
out["Stoch_%D"] = d
|
| 139 |
+
|
| 140 |
+
return out
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def latest_signals(df_with_indicators: pd.DataFrame) -> dict:
|
| 144 |
+
"""A simple rule-based read of the latest indicator values, for display only."""
|
| 145 |
+
last = df_with_indicators.iloc[-1]
|
| 146 |
+
signals = {}
|
| 147 |
+
|
| 148 |
+
signals["RSI (14)"] = {
|
| 149 |
+
"value": round(float(last["RSI_14"]), 2),
|
| 150 |
+
"signal": "Overbought" if last["RSI_14"] > 70 else "Oversold" if last["RSI_14"] < 30 else "Neutral",
|
| 151 |
+
}
|
| 152 |
+
signals["MACD"] = {
|
| 153 |
+
"value": round(float(last["MACD"]), 5),
|
| 154 |
+
"signal": "Bullish (above signal)" if last["MACD"] > last["MACD_Signal"] else "Bearish (below signal)",
|
| 155 |
+
}
|
| 156 |
+
bb_ready = pd.notna(last["BB_Upper"]) and pd.notna(last["BB_Lower"])
|
| 157 |
+
signals["Bollinger Bands"] = {
|
| 158 |
+
"value": round(float(last["Close"]), 5),
|
| 159 |
+
"signal": (
|
| 160 |
+
"Insufficient data" if not bb_ready
|
| 161 |
+
else "Above Upper Band" if last["Close"] > last["BB_Upper"]
|
| 162 |
+
else "Below Lower Band" if last["Close"] < last["BB_Lower"]
|
| 163 |
+
else "Inside Bands"
|
| 164 |
+
),
|
| 165 |
+
}
|
| 166 |
+
sma_ready = pd.notna(last["SMA_20"])
|
| 167 |
+
signals["Moving Averages"] = {
|
| 168 |
+
"value": f"SMA20={round(float(last['SMA_20']), 5) if sma_ready else 'n/a'}, "
|
| 169 |
+
f"EMA20={round(float(last['EMA_20']), 5)}",
|
| 170 |
+
"signal": (
|
| 171 |
+
"Insufficient data for SMA" if not sma_ready
|
| 172 |
+
else "Price Above MA (Uptrend)" if last["Close"] > last["SMA_20"]
|
| 173 |
+
else "Price Below MA (Downtrend)"
|
| 174 |
+
),
|
| 175 |
+
}
|
| 176 |
+
signals["Stochastic (14,3)"] = {
|
| 177 |
+
"value": round(float(last["Stoch_%K"]), 2),
|
| 178 |
+
"signal": "Overbought" if last["Stoch_%K"] > 80 else "Oversold" if last["Stoch_%K"] < 20 else "Neutral",
|
| 179 |
+
}
|
| 180 |
+
return signals
|
live_runner.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Standalone continuous 'live' prediction logger — an alternative to clicking
|
| 3 |
+
the Live tab in app.py by hand. Run this separately, e.g.:
|
| 4 |
+
|
| 5 |
+
python live_runner.py --symbol BTC-USD --timeframe 5m --model ARIMA
|
| 6 |
+
python live_runner.py --symbol EURUSD=X --timeframe 15m --model ARIMA-GARCH
|
| 7 |
+
python live_runner.py --symbol EURUSD=X --timeframe 15m --model TimesFM --poll-seconds 120
|
| 8 |
+
|
| 9 |
+
Every prediction and its later verification (correct/incorrect once the
|
| 10 |
+
target candle closes) is appended to a CSV under outputs/, one row per
|
| 11 |
+
verified candle: outputs/live_<symbol>_<timeframe>_<model>.csv
|
| 12 |
+
"""
|
| 13 |
+
import argparse
|
| 14 |
+
import os
|
| 15 |
+
from datetime import datetime, timezone
|
| 16 |
+
|
| 17 |
+
import pandas as pd
|
| 18 |
+
|
| 19 |
+
from data.historical import get_historical_cached, DataFetchError
|
| 20 |
+
from models.registry import get_model
|
| 21 |
+
from utils.helpers import infer_step
|
| 22 |
+
from config import OUTPUT_DIR
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def main():
|
| 26 |
+
parser = argparse.ArgumentParser(description="Continuous live prediction logger.")
|
| 27 |
+
parser.add_argument("--symbol", required=True, help="e.g. BTC-USD, EURUSD=X")
|
| 28 |
+
parser.add_argument("--timeframe", required=True, help="e.g. 1m, 5m, 15m, 1h")
|
| 29 |
+
parser.add_argument("--model", required=True,
|
| 30 |
+
choices=["ARIMA", "Auto-ARIMA", "ARIMA-GARCH", "Moirai", "TimesFM"])
|
| 31 |
+
parser.add_argument("--poll-seconds", type=int, default=60,
|
| 32 |
+
help="How often to check for a new candle (default: 60s)")
|
| 33 |
+
args = parser.parse_args()
|
| 34 |
+
|
| 35 |
+
safe_symbol = args.symbol.replace("=", "_").replace("/", "_")
|
| 36 |
+
log_path = os.path.join(OUTPUT_DIR, f"live_{safe_symbol}_{args.timeframe}_{args.model}.csv")
|
| 37 |
+
model = get_model(args.model, args.symbol, args.timeframe)
|
| 38 |
+
|
| 39 |
+
pending = None # dict: target_time, current_close, predicted_close, predicted_direction
|
| 40 |
+
correct_count = 0
|
| 41 |
+
total_count = 0
|
| 42 |
+
|
| 43 |
+
print(f"[live_runner] {args.symbol} [{args.timeframe}] using {args.model}. "
|
| 44 |
+
f"Logging to {log_path}. Ctrl+C to stop.")
|
| 45 |
+
|
| 46 |
+
import time
|
| 47 |
+
while True:
|
| 48 |
+
try:
|
| 49 |
+
df = get_historical_cached(args.symbol, args.timeframe, lookback_days=2, refresh=True)
|
| 50 |
+
latest_time = df.index[-1]
|
| 51 |
+
latest_close = float(df["Close"].iloc[-1])
|
| 52 |
+
|
| 53 |
+
# Only verify once the target candle has actually CLOSED -- i.e. a
|
| 54 |
+
# strictly newer candle exists after it in the freshly fetched data.
|
| 55 |
+
# `in df.index` alone is true the instant the candle starts forming,
|
| 56 |
+
# which would read its still-changing in-progress price as if it
|
| 57 |
+
# were the final close (see app.py's _check_live_impl for the full
|
| 58 |
+
# writeup of this bug and how it silently corrupts the accuracy %).
|
| 59 |
+
if (pending is not None and pending["target_time"] in df.index
|
| 60 |
+
and pending["target_time"] < latest_time):
|
| 61 |
+
actual_close = float(df.loc[pending["target_time"], "Close"])
|
| 62 |
+
actual_direction = "up" if actual_close > pending["current_close"] else "down"
|
| 63 |
+
correct = actual_direction == pending["predicted_direction"]
|
| 64 |
+
total_count += 1
|
| 65 |
+
correct_count += int(correct)
|
| 66 |
+
|
| 67 |
+
row = {
|
| 68 |
+
**pending, "actual_close": actual_close, "actual_direction": actual_direction,
|
| 69 |
+
"correct": correct, "logged_at": datetime.now(timezone.utc).isoformat(),
|
| 70 |
+
}
|
| 71 |
+
pd.DataFrame([row]).to_csv(log_path, mode="a", header=not os.path.exists(log_path), index=False)
|
| 72 |
+
acc = round(100 * correct_count / total_count, 2)
|
| 73 |
+
print(f"[{datetime.now(timezone.utc).isoformat()}] Verified: predicted "
|
| 74 |
+
f"{pending['predicted_direction']}, actual {actual_direction} -> "
|
| 75 |
+
f"{'CORRECT' if correct else 'WRONG'} (running accuracy: {acc}%, n={total_count})")
|
| 76 |
+
pending = None
|
| 77 |
+
|
| 78 |
+
if pending is None:
|
| 79 |
+
forecast = model.predict(df["Close"], horizon=1)
|
| 80 |
+
predicted_close = float(forecast[-1])
|
| 81 |
+
predicted_direction = "up" if predicted_close > latest_close else "down"
|
| 82 |
+
step = infer_step(df.index)
|
| 83 |
+
pending = {
|
| 84 |
+
"target_time": latest_time + step, "current_close": latest_close,
|
| 85 |
+
"predicted_close": predicted_close, "predicted_direction": predicted_direction,
|
| 86 |
+
}
|
| 87 |
+
print(f"[{datetime.now(timezone.utc).isoformat()}] New prediction: "
|
| 88 |
+
f"{args.symbol} next candle -> {predicted_direction} ({predicted_close:.5f})")
|
| 89 |
+
|
| 90 |
+
except DataFetchError as e:
|
| 91 |
+
print(f"[{datetime.now(timezone.utc).isoformat()}] Data error: {e}")
|
| 92 |
+
except KeyboardInterrupt:
|
| 93 |
+
print("Stopped.")
|
| 94 |
+
break
|
| 95 |
+
except Exception as e:
|
| 96 |
+
print(f"[{datetime.now(timezone.utc).isoformat()}] Error: {e}")
|
| 97 |
+
|
| 98 |
+
time.sleep(args.poll_seconds)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if __name__ == "__main__":
|
| 102 |
+
main()
|
models/__init__.py
ADDED
|
File without changes
|
models/arima_garch_model.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ARIMA-GARCH — the classical two-step "mean + volatility" combo for return
|
| 3 |
+
series: `statsmodels` (ARIMA) models the conditional MEAN of returns, and
|
| 4 |
+
`arch` (GARCH) models the conditional VARIANCE — the volatility clustering
|
| 5 |
+
that plain ARIMA can't see, since it assumes constant variance.
|
| 6 |
+
|
| 7 |
+
Why not just run ARIMA on the raw price directly, like arima_model.py does?
|
| 8 |
+
GARCH models volatility, and volatility clustering shows up in RETURNS, not
|
| 9 |
+
raw price levels (price isn't stationary in variance; returns usually are
|
| 10 |
+
much closer to it). So this model:
|
| 11 |
+
1. converts the closing-price history to log returns,
|
| 12 |
+
2. fits ARIMA(p, 0, q) to the returns for the conditional mean,
|
| 13 |
+
3. fits GARCH(1, 1) on the ARIMA residuals for the conditional variance,
|
| 14 |
+
4. turns the forecasted mean return back into a price forecast:
|
| 15 |
+
price[t+1] = price[t] * exp(forecasted_log_return[t+1])
|
| 16 |
+
(compounding forward for horizon > 1).
|
| 17 |
+
|
| 18 |
+
Point-forecast-wise this lands close to plain ARIMA-on-returns, because a
|
| 19 |
+
GARCH model forecasts the SIZE of the next shock (its variance), not its
|
| 20 |
+
sign/mean — ARIMA still supplies the point forecast. Where GARCH actually
|
| 21 |
+
earns its keep is the volatility forecast itself, kept here as
|
| 22 |
+
`last_volatility_forecast` (per-step forecasted std-dev of returns, updated
|
| 23 |
+
on every `.predict()` call). That isn't part of the BaseForecastModel
|
| 24 |
+
interface so it's not forced into predict()'s return value, but it's there
|
| 25 |
+
if you want to build confidence bands or size positions by risk later.
|
| 26 |
+
|
| 27 |
+
Speed note: same story as arima_model.py/auto_arima_model.py — searching a
|
| 28 |
+
(p, q) grid on every `.predict()` call would make backtesting slow, so the
|
| 29 |
+
ARIMA order (for the returns) is searched ONCE and reused. Pass
|
| 30 |
+
`order=(p, 0, q)` yourself to skip that search.
|
| 31 |
+
"""
|
| 32 |
+
import warnings
|
| 33 |
+
|
| 34 |
+
import numpy as np
|
| 35 |
+
import pandas as pd
|
| 36 |
+
from statsmodels.tsa.arima.model import ARIMA
|
| 37 |
+
|
| 38 |
+
from models.base_model import BaseForecastModel
|
| 39 |
+
|
| 40 |
+
warnings.filterwarnings("ignore")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class ArimaGarchModel(BaseForecastModel):
|
| 44 |
+
name = "ARIMA-GARCH"
|
| 45 |
+
|
| 46 |
+
def __init__(self, max_p=3, max_q=3, order=None, garch_p=1, garch_q=1,
|
| 47 |
+
refit_order_every=None):
|
| 48 |
+
self.max_p = max_p
|
| 49 |
+
self.max_q = max_q
|
| 50 |
+
self._cached_order = order # (p, 0, q), searched on the RETURNS series
|
| 51 |
+
self.garch_p = garch_p
|
| 52 |
+
self.garch_q = garch_q
|
| 53 |
+
self.refit_order_every = refit_order_every # None = search once, then always reuse
|
| 54 |
+
self._calls_since_search = 0
|
| 55 |
+
# Per-step forecasted std-dev of returns from the most recent
|
| 56 |
+
# predict() call — informational only, see module docstring.
|
| 57 |
+
self.last_volatility_forecast = None
|
| 58 |
+
|
| 59 |
+
def _check_dependencies(self):
|
| 60 |
+
import arch # noqa
|
| 61 |
+
|
| 62 |
+
def _import_arch_model(self):
|
| 63 |
+
try:
|
| 64 |
+
from arch import arch_model
|
| 65 |
+
except ImportError as e:
|
| 66 |
+
raise ImportError(
|
| 67 |
+
"ARIMA-GARCH needs the 'arch' package. Install with:\n"
|
| 68 |
+
" pip install arch\n"
|
| 69 |
+
f"(original error: {e})"
|
| 70 |
+
)
|
| 71 |
+
return arch_model
|
| 72 |
+
|
| 73 |
+
def _best_order(self, returns: pd.Series):
|
| 74 |
+
"""Small AIC grid search over (p, q) for the returns series, d fixed
|
| 75 |
+
at 0 since returns should already be roughly stationary."""
|
| 76 |
+
best_aic = np.inf
|
| 77 |
+
best_order = (1, 0, 1)
|
| 78 |
+
for p in range(self.max_p + 1):
|
| 79 |
+
for q in range(self.max_q + 1):
|
| 80 |
+
if p == 0 and q == 0:
|
| 81 |
+
continue
|
| 82 |
+
try:
|
| 83 |
+
fit = ARIMA(returns, order=(p, 0, q)).fit()
|
| 84 |
+
if fit.aic < best_aic:
|
| 85 |
+
best_aic = fit.aic
|
| 86 |
+
best_order = (p, 0, q)
|
| 87 |
+
except Exception:
|
| 88 |
+
continue
|
| 89 |
+
return best_order
|
| 90 |
+
|
| 91 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 92 |
+
arch_model = self._import_arch_model()
|
| 93 |
+
|
| 94 |
+
prices = pd.Series(history).astype(float).reset_index(drop=True)
|
| 95 |
+
if len(prices) < 30:
|
| 96 |
+
raise ValueError("ARIMA-GARCH needs at least 30 candles of history.")
|
| 97 |
+
if (prices <= 0).any():
|
| 98 |
+
raise ValueError("ARIMA-GARCH needs strictly positive prices (it uses log returns).")
|
| 99 |
+
|
| 100 |
+
log_returns = np.log(prices / prices.shift(1)).dropna().reset_index(drop=True)
|
| 101 |
+
|
| 102 |
+
need_search = self._cached_order is None
|
| 103 |
+
if self.refit_order_every and self._calls_since_search >= self.refit_order_every:
|
| 104 |
+
need_search = True
|
| 105 |
+
if need_search:
|
| 106 |
+
self._cached_order = self._best_order(log_returns)
|
| 107 |
+
self._calls_since_search = 0
|
| 108 |
+
self._calls_since_search += 1
|
| 109 |
+
|
| 110 |
+
# --- 1) conditional mean: ARIMA on log returns ---
|
| 111 |
+
mean_fit = ARIMA(log_returns, order=self._cached_order).fit()
|
| 112 |
+
mean_forecast = np.asarray(mean_fit.forecast(steps=horizon), dtype=float)
|
| 113 |
+
|
| 114 |
+
# --- 2) conditional variance: GARCH on the ARIMA residuals ---
|
| 115 |
+
# arch_model optimizes more reliably on returns of order ~1 than on
|
| 116 |
+
# tiny raw log-returns (~1e-4), so scale up for fitting and scale
|
| 117 |
+
# the resulting variance back down afterwards.
|
| 118 |
+
resid = pd.Series(mean_fit.resid).dropna()
|
| 119 |
+
scale = 100.0
|
| 120 |
+
garch_fit = arch_model(
|
| 121 |
+
resid * scale, mean="Zero", vol="GARCH", p=self.garch_p, q=self.garch_q,
|
| 122 |
+
dist="normal", rescale=False,
|
| 123 |
+
).fit(disp="off")
|
| 124 |
+
garch_forecast = garch_fit.forecast(horizon=horizon, reindex=False)
|
| 125 |
+
variance = np.asarray(garch_forecast.variance.values[-1], dtype=float) / (scale ** 2)
|
| 126 |
+
self.last_volatility_forecast = [float(x) for x in np.sqrt(variance)]
|
| 127 |
+
|
| 128 |
+
# --- 3) mean log-return forecast -> price forecast (compounded) ---
|
| 129 |
+
last_price = float(prices.iloc[-1])
|
| 130 |
+
prices_out = []
|
| 131 |
+
cum_log_return = 0.0
|
| 132 |
+
for r in mean_forecast:
|
| 133 |
+
cum_log_return += float(r)
|
| 134 |
+
prices_out.append(last_price * float(np.exp(cum_log_return)))
|
| 135 |
+
|
| 136 |
+
if not all(np.isfinite(x) for x in prices_out):
|
| 137 |
+
raise ValueError(f"ARIMA-GARCH produced a non-finite forecast for order {self._cached_order}.")
|
| 138 |
+
|
| 139 |
+
return prices_out
|
models/arima_model.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ARIMA — the classic statistical baseline. Needs only `statsmodels`, so it
|
| 3 |
+
always works even if the two deep-learning models below aren't installed.
|
| 4 |
+
|
| 5 |
+
Speed note: a full (p,d,q) grid search on every single prediction would make
|
| 6 |
+
backtesting painfully slow (hundreds of candles x dozens of candidate orders).
|
| 7 |
+
So by default this class searches for the best order ONCE (on the first
|
| 8 |
+
`.predict()` call) and reuses it for every later call in the same instance —
|
| 9 |
+
that's what makes ARIMA backtesting fast. Pass `order=(p, d, q)` yourself to
|
| 10 |
+
skip the search entirely.
|
| 11 |
+
"""
|
| 12 |
+
import warnings
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
from statsmodels.tsa.arima.model import ARIMA
|
| 17 |
+
|
| 18 |
+
from models.base_model import BaseForecastModel
|
| 19 |
+
|
| 20 |
+
warnings.filterwarnings("ignore")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class ArimaModel(BaseForecastModel):
|
| 24 |
+
name = "ARIMA"
|
| 25 |
+
|
| 26 |
+
def __init__(self, max_p=3, max_d=2, max_q=3, order=None, refit_order_every=None):
|
| 27 |
+
self.max_p = max_p
|
| 28 |
+
self.max_d = max_d
|
| 29 |
+
self.max_q = max_q
|
| 30 |
+
self._cached_order = order
|
| 31 |
+
self.refit_order_every = refit_order_every # None = search once, then always reuse
|
| 32 |
+
self._calls_since_search = 0
|
| 33 |
+
|
| 34 |
+
def _check_dependencies(self):
|
| 35 |
+
import statsmodels # noqa
|
| 36 |
+
|
| 37 |
+
def _best_order(self, series: pd.Series):
|
| 38 |
+
best_aic = np.inf
|
| 39 |
+
best_order = (1, 1, 1)
|
| 40 |
+
for p in range(self.max_p + 1):
|
| 41 |
+
for d in range(self.max_d + 1):
|
| 42 |
+
for q in range(self.max_q + 1):
|
| 43 |
+
if p == 0 and q == 0:
|
| 44 |
+
continue
|
| 45 |
+
try:
|
| 46 |
+
fit = ARIMA(series, order=(p, d, q)).fit()
|
| 47 |
+
if fit.aic < best_aic:
|
| 48 |
+
best_aic = fit.aic
|
| 49 |
+
best_order = (p, d, q)
|
| 50 |
+
except Exception:
|
| 51 |
+
continue
|
| 52 |
+
return best_order
|
| 53 |
+
|
| 54 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 55 |
+
series = pd.Series(history).astype(float).reset_index(drop=True)
|
| 56 |
+
if len(series) < 20:
|
| 57 |
+
raise ValueError("ARIMA needs at least 20 candles of history.")
|
| 58 |
+
|
| 59 |
+
need_search = self._cached_order is None
|
| 60 |
+
if self.refit_order_every and self._calls_since_search >= self.refit_order_every:
|
| 61 |
+
need_search = True
|
| 62 |
+
|
| 63 |
+
if need_search:
|
| 64 |
+
self._cached_order = self._best_order(series)
|
| 65 |
+
self._calls_since_search = 0
|
| 66 |
+
|
| 67 |
+
self._calls_since_search += 1
|
| 68 |
+
|
| 69 |
+
fit = ARIMA(series, order=self._cached_order).fit()
|
| 70 |
+
forecast = np.asarray(fit.forecast(steps=horizon), dtype=float)
|
| 71 |
+
if not np.all(np.isfinite(forecast)):
|
| 72 |
+
raise ValueError(f"ARIMA produced a non-finite forecast for order {self._cached_order}.")
|
| 73 |
+
return [float(x) for x in forecast]
|
models/auto_arima_model.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Auto-ARIMA — automatic (p, d, q) order selection for ARIMA, implemented
|
| 3 |
+
directly on top of `statsmodels` (no extra dependency beyond what plain
|
| 4 |
+
`arima_model.py` already needs).
|
| 5 |
+
|
| 6 |
+
An earlier version of this file used the `pmdarima` package. It's been
|
| 7 |
+
removed: pmdarima ships compiled Cython extensions that are notorious for
|
| 8 |
+
breaking on environments where its build-time numpy version doesn't match
|
| 9 |
+
the runtime one ("binary incompatibility" / import errors) — exactly the
|
| 10 |
+
kind of thing that silently fails every single prediction on a fresh
|
| 11 |
+
Space/container without touching any of your actual code. Since
|
| 12 |
+
`statsmodels` was already confirmed working (plain ARIMA and ARIMA-GARCH
|
| 13 |
+
both depend on it), doing the search ourselves with statsmodels removes
|
| 14 |
+
that whole failure class.
|
| 15 |
+
|
| 16 |
+
How this differs from the plain `ARIMA` model in `arima_model.py`:
|
| 17 |
+
- `arima_model.py` picks (p, d, q) with a brute-force grid search over
|
| 18 |
+
every combination up to max_p/max_d/max_q, scored by AIC.
|
| 19 |
+
- This model instead follows the Hyndman-Khandakar (2008) stepwise
|
| 20 |
+
procedure — the same algorithm R's `forecast::auto.arima()` and
|
| 21 |
+
`pmdarima.auto_arima()` are built on:
|
| 22 |
+
1. pick the differencing order `d` with a KPSS unit-root test
|
| 23 |
+
(difference until the series looks stationary, up to max_d times)
|
| 24 |
+
instead of trying every `d` blindly,
|
| 25 |
+
2. fit a handful of starting (p, q) candidates at that `d`,
|
| 26 |
+
3. greedily step to whichever neighbouring (p, q) improves AIC,
|
| 27 |
+
repeating until no neighbour improves on the current best.
|
| 28 |
+
This visits far fewer models than the brute-force grid, and tends to
|
| 29 |
+
land on the same or a better order.
|
| 30 |
+
|
| 31 |
+
Speed note: same reasoning as arima_model.py — running the full search on
|
| 32 |
+
every single `.predict()` call would make backtesting painfully slow
|
| 33 |
+
(hundreds of candles x a search each time). So the order is searched ONCE
|
| 34 |
+
(on the first call) and reused for every later call on this instance — pass
|
| 35 |
+
`refit_order_every=N` to re-search every N calls, or pass `order=(p, d, q)`
|
| 36 |
+
yourself to skip the search entirely.
|
| 37 |
+
"""
|
| 38 |
+
import warnings
|
| 39 |
+
|
| 40 |
+
import numpy as np
|
| 41 |
+
import pandas as pd
|
| 42 |
+
from statsmodels.tsa.arima.model import ARIMA
|
| 43 |
+
from statsmodels.tsa.stattools import kpss
|
| 44 |
+
|
| 45 |
+
from models.base_model import BaseForecastModel
|
| 46 |
+
|
| 47 |
+
warnings.filterwarnings("ignore")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class AutoArimaModel(BaseForecastModel):
|
| 51 |
+
name = "Auto-ARIMA"
|
| 52 |
+
|
| 53 |
+
def __init__(self, max_p=5, max_q=5, max_d=2, order=None, refit_order_every=None):
|
| 54 |
+
self.max_p = max_p
|
| 55 |
+
self.max_q = max_q
|
| 56 |
+
self.max_d = max_d
|
| 57 |
+
self._cached_order = order
|
| 58 |
+
self.refit_order_every = refit_order_every # None = search once, then always reuse
|
| 59 |
+
self._calls_since_search = 0
|
| 60 |
+
|
| 61 |
+
def _check_dependencies(self):
|
| 62 |
+
import statsmodels # noqa
|
| 63 |
+
|
| 64 |
+
def _find_d(self, series: pd.Series) -> int:
|
| 65 |
+
"""KPSS-based differencing order, capped at max_d."""
|
| 66 |
+
s = series.copy()
|
| 67 |
+
d = 0
|
| 68 |
+
while d < self.max_d:
|
| 69 |
+
try:
|
| 70 |
+
_, p_value, _, _ = kpss(s, regression="c", nlags="auto")
|
| 71 |
+
except Exception:
|
| 72 |
+
break
|
| 73 |
+
if p_value < 0.05: # KPSS null = stationary -> low p-value = reject -> difference again
|
| 74 |
+
s = s.diff().dropna()
|
| 75 |
+
d += 1
|
| 76 |
+
else:
|
| 77 |
+
break
|
| 78 |
+
return d
|
| 79 |
+
|
| 80 |
+
def _aic(self, series: pd.Series, order) -> float:
|
| 81 |
+
try:
|
| 82 |
+
return ARIMA(series, order=order).fit().aic
|
| 83 |
+
except Exception:
|
| 84 |
+
return np.inf
|
| 85 |
+
|
| 86 |
+
def _stepwise_search(self, series: pd.Series, d: int):
|
| 87 |
+
"""Hyndman-Khandakar-style stepwise search: a few starting points,
|
| 88 |
+
then greedy steps to the best-AIC neighbour until nothing improves."""
|
| 89 |
+
starting_points = [(0, 0), (1, 0), (0, 1), (2, 2)]
|
| 90 |
+
best_order, best_aic = None, np.inf
|
| 91 |
+
for p, q in starting_points:
|
| 92 |
+
if p > self.max_p or q > self.max_q:
|
| 93 |
+
continue
|
| 94 |
+
aic = self._aic(series, (p, d, q))
|
| 95 |
+
if aic < best_aic:
|
| 96 |
+
best_aic, best_order = aic, (p, q)
|
| 97 |
+
|
| 98 |
+
if best_order is None:
|
| 99 |
+
best_order, best_aic = (0, 0), self._aic(series, (0, d, 0))
|
| 100 |
+
|
| 101 |
+
improved = True
|
| 102 |
+
while improved:
|
| 103 |
+
improved = False
|
| 104 |
+
p0, q0 = best_order
|
| 105 |
+
for p, q in [(p0 + 1, q0), (p0 - 1, q0), (p0, q0 + 1), (p0, q0 - 1),
|
| 106 |
+
(p0 + 1, q0 + 1), (p0 - 1, q0 - 1)]:
|
| 107 |
+
if p < 0 or q < 0 or p > self.max_p or q > self.max_q:
|
| 108 |
+
continue
|
| 109 |
+
aic = self._aic(series, (p, d, q))
|
| 110 |
+
if aic < best_aic - 1e-6:
|
| 111 |
+
best_aic, best_order = aic, (p, q)
|
| 112 |
+
improved = True
|
| 113 |
+
|
| 114 |
+
p, q = best_order
|
| 115 |
+
return (p, d, q)
|
| 116 |
+
|
| 117 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 118 |
+
series = pd.Series(history).astype(float).reset_index(drop=True)
|
| 119 |
+
if len(series) < 20:
|
| 120 |
+
raise ValueError("Auto-ARIMA needs at least 20 candles of history.")
|
| 121 |
+
|
| 122 |
+
need_search = self._cached_order is None
|
| 123 |
+
if self.refit_order_every and self._calls_since_search >= self.refit_order_every:
|
| 124 |
+
need_search = True
|
| 125 |
+
|
| 126 |
+
if need_search:
|
| 127 |
+
d = self._find_d(series)
|
| 128 |
+
self._cached_order = self._stepwise_search(series, d)
|
| 129 |
+
self._calls_since_search = 0
|
| 130 |
+
|
| 131 |
+
self._calls_since_search += 1
|
| 132 |
+
|
| 133 |
+
fit = ARIMA(series, order=self._cached_order).fit()
|
| 134 |
+
forecast = np.asarray(fit.forecast(steps=horizon), dtype=float)
|
| 135 |
+
|
| 136 |
+
if not np.all(np.isfinite(forecast)):
|
| 137 |
+
raise ValueError(f"Auto-ARIMA produced a non-finite forecast for order {self._cached_order}.")
|
| 138 |
+
|
| 139 |
+
return [float(x) for x in forecast]
|
models/base_model.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Common interface every forecasting model implements, so the rest of the app
|
| 3 |
+
(backtester, Gradio UI) can treat ARIMA / Auto-ARIMA / ARIMA-GARCH / Moirai /
|
| 4 |
+
TimesFM interchangeably without knowing their internal details.
|
| 5 |
+
"""
|
| 6 |
+
from abc import ABC, abstractmethod
|
| 7 |
+
|
| 8 |
+
import pandas as pd
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class BaseForecastModel(ABC):
|
| 12 |
+
name = "base"
|
| 13 |
+
|
| 14 |
+
@abstractmethod
|
| 15 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 16 |
+
"""
|
| 17 |
+
Given a 1-D series of past closing prices (oldest -> newest), return
|
| 18 |
+
a list of `horizon` forecasted future closing prices.
|
| 19 |
+
|
| 20 |
+
Implementations should raise a clear, specific exception (ImportError
|
| 21 |
+
for a missing dependency, ValueError for bad input) rather than
|
| 22 |
+
silently returning a wrong/empty result — the backtester and UI both
|
| 23 |
+
catch and display these per-model, without crashing other models.
|
| 24 |
+
"""
|
| 25 |
+
raise NotImplementedError
|
| 26 |
+
|
| 27 |
+
def is_available(self) -> bool:
|
| 28 |
+
"""True if this model's dependencies are installed and importable."""
|
| 29 |
+
try:
|
| 30 |
+
self._check_dependencies()
|
| 31 |
+
return True
|
| 32 |
+
except Exception:
|
| 33 |
+
return False
|
| 34 |
+
|
| 35 |
+
def _check_dependencies(self):
|
| 36 |
+
"""Override to raise ImportError if a required package is missing."""
|
| 37 |
+
return None
|
models/moirai_model.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Salesforce Moirai — a pretrained time-series foundation model, via the
|
| 3 |
+
`uni2ts` package (https://github.com/SalesforceAIResearch/uni2ts).
|
| 4 |
+
|
| 5 |
+
Needs: pip install uni2ts torch
|
| 6 |
+
First use downloads pretrained weights from Hugging Face (needs internet),
|
| 7 |
+
cached under ~/.cache/huggingface/ afterwards.
|
| 8 |
+
|
| 9 |
+
Uni2TS uses GluonTS for inference. The pattern below (PandasDataset ->
|
| 10 |
+
MoiraiForecast -> create_predictor -> predict) matches the official
|
| 11 |
+
"Getting Started" example in the uni2ts README.
|
| 12 |
+
"""
|
| 13 |
+
import numpy as np
|
| 14 |
+
import pandas as pd
|
| 15 |
+
|
| 16 |
+
from models.base_model import BaseForecastModel
|
| 17 |
+
from config import MOIRAI_CHECKPOINT
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class MoiraiModel(BaseForecastModel):
|
| 21 |
+
name = "Moirai"
|
| 22 |
+
|
| 23 |
+
def __init__(self, checkpoint: str = MOIRAI_CHECKPOINT, context_length: int = 200,
|
| 24 |
+
num_samples: int = 100):
|
| 25 |
+
self.checkpoint = checkpoint
|
| 26 |
+
self.context_length = context_length
|
| 27 |
+
self.num_samples = num_samples
|
| 28 |
+
self._module = None # cached after first load so repeat calls in the
|
| 29 |
+
# same running app don't re-download / re-init the pretrained weights
|
| 30 |
+
|
| 31 |
+
def _check_dependencies(self):
|
| 32 |
+
import uni2ts # noqa
|
| 33 |
+
import gluonts # noqa
|
| 34 |
+
|
| 35 |
+
def _get_module(self):
|
| 36 |
+
from uni2ts.model.moirai import MoiraiModule
|
| 37 |
+
if self._module is None:
|
| 38 |
+
self._module = MoiraiModule.from_pretrained(self.checkpoint)
|
| 39 |
+
return self._module
|
| 40 |
+
|
| 41 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 42 |
+
try:
|
| 43 |
+
from uni2ts.model.moirai import MoiraiForecast
|
| 44 |
+
from gluonts.dataset.pandas import PandasDataset
|
| 45 |
+
except ImportError as e:
|
| 46 |
+
raise ImportError(
|
| 47 |
+
"Moirai needs the 'uni2ts' and 'gluonts' packages. Install with:\n"
|
| 48 |
+
" pip install uni2ts torch\n"
|
| 49 |
+
f"(original error: {e})"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
values = np.asarray(history, dtype=np.float32)
|
| 53 |
+
if len(values) < 10:
|
| 54 |
+
raise ValueError("Moirai needs at least 10 candles of history.")
|
| 55 |
+
|
| 56 |
+
context = values[-self.context_length:] if len(values) > self.context_length else values
|
| 57 |
+
|
| 58 |
+
# Moirai/GluonTS need a DatetimeIndex but only the VALUES matter for a
|
| 59 |
+
# point forecast — the real candle timestamps are re-attached in
|
| 60 |
+
# app.py, so a clean synthetic per-minute index is fine here.
|
| 61 |
+
# (Timestamp.now("UTC"), not the deprecated Timestamp.utcnow() —
|
| 62 |
+
# pandas already warns that utcnow() will be removed in a future
|
| 63 |
+
# release.)
|
| 64 |
+
idx = pd.date_range(end=pd.Timestamp.now("UTC").floor("min"), periods=len(context), freq="min")
|
| 65 |
+
series_df = pd.DataFrame({"target": context}, index=idx)
|
| 66 |
+
dataset = PandasDataset(series_df, target="target")
|
| 67 |
+
|
| 68 |
+
module = self._get_module()
|
| 69 |
+
forecaster = MoiraiForecast(
|
| 70 |
+
module=module,
|
| 71 |
+
prediction_length=horizon,
|
| 72 |
+
context_length=len(context),
|
| 73 |
+
patch_size="auto",
|
| 74 |
+
num_samples=self.num_samples,
|
| 75 |
+
target_dim=1,
|
| 76 |
+
feat_dynamic_real_dim=0,
|
| 77 |
+
past_feat_dynamic_real_dim=0,
|
| 78 |
+
)
|
| 79 |
+
predictor = forecaster.create_predictor(batch_size=1)
|
| 80 |
+
forecast = next(iter(predictor.predict(dataset)))
|
| 81 |
+
|
| 82 |
+
# forecast.samples has shape (num_samples, horizon) — take the
|
| 83 |
+
# per-step median across samples as the point forecast.
|
| 84 |
+
point_forecast = np.median(forecast.samples, axis=0)
|
| 85 |
+
out = [float(x) for x in point_forecast]
|
| 86 |
+
if not all(np.isfinite(x) for x in out):
|
| 87 |
+
raise ValueError("Moirai produced a non-finite forecast (NaN/inf) for this window.")
|
| 88 |
+
return out
|
models/registry.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Model factory. Models are created lazily and cached here — importing this
|
| 3 |
+
module does NOT import torch/uni2ts/timesfm, and calling get_model("ARIMA")
|
| 4 |
+
never touches the (heavier) Moirai/TimesFM code paths. This matters on a
|
| 5 |
+
shared 2 vCPU / 16GB Space: a session that only ever uses ARIMA should never
|
| 6 |
+
pay the cost of loading foundation-model weights.
|
| 7 |
+
|
| 8 |
+
Caching strategy differs by model type, because "cached" means something
|
| 9 |
+
different for each:
|
| 10 |
+
|
| 11 |
+
- ARIMA / Auto-ARIMA / ARIMA-GARCH search and cache a (p, d, q) order on
|
| 12 |
+
their first `.predict()` call, then reuse it for every later call (see
|
| 13 |
+
each model's own docstring) — that order is fit to whichever
|
| 14 |
+
symbol/timeframe supplied the FIRST window of history it ever saw. A
|
| 15 |
+
cache keyed only on model name would keep reusing e.g. EURUSD's order
|
| 16 |
+
after switching to BTC-USD, silently fitting the wrong symbol's dynamics
|
| 17 |
+
with no error or obviously-bad output — just a quietly worse forecast.
|
| 18 |
+
So these three are cached per (name, symbol, timeframe): switching
|
| 19 |
+
either gets its own fresh order search, same symbol+timeframe keeps
|
| 20 |
+
reusing the cached order (the whole reason this cache exists).
|
| 21 |
+
- Moirai / TimesFM are zero-shot foundation models: `.predict()` takes the
|
| 22 |
+
history fresh every call and never caches anything symbol-specific — the
|
| 23 |
+
only thing cached on the instance is the pretrained weights themselves
|
| 24 |
+
(large, symbol-agnostic, and expensive to reload — TimesFM alone is a
|
| 25 |
+
~800MB/compile-step load). There's no staleness to fix here, so these
|
| 26 |
+
stay cached by name only and are always shared/reused across symbols.
|
| 27 |
+
"""
|
| 28 |
+
from models.base_model import BaseForecastModel
|
| 29 |
+
|
| 30 |
+
# Model names that internally cache a fitted (p, d, q) order — see module
|
| 31 |
+
# docstring above for why these need a (name, symbol, timeframe) cache key.
|
| 32 |
+
_ORDER_SEARCH_MODELS = {"ARIMA", "Auto-ARIMA", "ARIMA-GARCH"}
|
| 33 |
+
|
| 34 |
+
_CACHE: dict = {}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _build_model(name: str) -> BaseForecastModel:
|
| 38 |
+
if name == "ARIMA":
|
| 39 |
+
from models.arima_model import ArimaModel
|
| 40 |
+
return ArimaModel()
|
| 41 |
+
elif name == "Auto-ARIMA":
|
| 42 |
+
from models.auto_arima_model import AutoArimaModel
|
| 43 |
+
return AutoArimaModel()
|
| 44 |
+
elif name == "ARIMA-GARCH":
|
| 45 |
+
from models.arima_garch_model import ArimaGarchModel
|
| 46 |
+
return ArimaGarchModel()
|
| 47 |
+
elif name == "Moirai":
|
| 48 |
+
from models.moirai_model import MoiraiModel
|
| 49 |
+
return MoiraiModel()
|
| 50 |
+
elif name == "TimesFM":
|
| 51 |
+
from models.timesfm_model import TimesFMModel
|
| 52 |
+
return TimesFMModel()
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"Unknown model '{name}'. Expected one of: "
|
| 55 |
+
"ARIMA, Auto-ARIMA, ARIMA-GARCH, Moirai, TimesFM."
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def get_model(name: str, symbol: str = None, timeframe: str = None) -> BaseForecastModel:
|
| 60 |
+
"""
|
| 61 |
+
Cached model factory.
|
| 62 |
+
|
| 63 |
+
Pass the symbol/timeframe you're about to predict on — for ARIMA,
|
| 64 |
+
Auto-ARIMA and ARIMA-GARCH this keys the cache so a symbol or
|
| 65 |
+
timeframe switch gets its own fresh instance (and therefore its own
|
| 66 |
+
fresh order search) instead of silently reusing another symbol's
|
| 67 |
+
cached order. Moirai/TimesFM ignore symbol/timeframe for caching
|
| 68 |
+
purposes and are always shared by name only (see module docstring).
|
| 69 |
+
|
| 70 |
+
Calling with just `name` (no symbol/timeframe) still works and behaves
|
| 71 |
+
like before — fine for scripts that only ever handle one symbol.
|
| 72 |
+
"""
|
| 73 |
+
key = (name, symbol, timeframe) if name in _ORDER_SEARCH_MODELS else name
|
| 74 |
+
if key in _CACHE:
|
| 75 |
+
return _CACHE[key]
|
| 76 |
+
|
| 77 |
+
model = _build_model(name)
|
| 78 |
+
_CACHE[key] = model
|
| 79 |
+
return model
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def fresh_model(name: str) -> BaseForecastModel:
|
| 83 |
+
"""
|
| 84 |
+
Like get_model, but always returns a brand-new instance instead of the
|
| 85 |
+
cache — useful for ARIMA in a backtest, where you want a fresh order
|
| 86 |
+
search rather than reusing whatever order a previous run settled on.
|
| 87 |
+
"""
|
| 88 |
+
return _build_model(name)
|
models/timesfm_model.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Google TimesFM 2.5 — a pretrained time-series foundation model, via the
|
| 3 |
+
`timesfm` package (https://github.com/google-research/timesfm).
|
| 4 |
+
|
| 5 |
+
Needs: pip install timesfm[torch]
|
| 6 |
+
First use downloads pretrained weights from Hugging Face (~800MB, needs
|
| 7 |
+
internet), cached under ~/.cache/huggingface/ afterwards. The 200M-parameter
|
| 8 |
+
2.5 checkpoint needs roughly 4GB RAM on CPU, which comfortably fits a 16GB
|
| 9 |
+
Space alongside the other two models.
|
| 10 |
+
"""
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
from models.base_model import BaseForecastModel
|
| 15 |
+
from config import TIMESFM_CHECKPOINT
|
| 16 |
+
|
| 17 |
+
_MAX_HORIZON = 64 # generous fixed ceiling set once at compile time
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class TimesFMModel(BaseForecastModel):
|
| 21 |
+
name = "TimesFM"
|
| 22 |
+
|
| 23 |
+
def __init__(self, checkpoint: str = TIMESFM_CHECKPOINT, max_context: int = 512):
|
| 24 |
+
self.checkpoint = checkpoint
|
| 25 |
+
self.max_context = max_context
|
| 26 |
+
self._model = None # cached after first load+compile (same process)
|
| 27 |
+
|
| 28 |
+
def _check_dependencies(self):
|
| 29 |
+
import timesfm # noqa
|
| 30 |
+
import torch # noqa
|
| 31 |
+
|
| 32 |
+
def _load(self):
|
| 33 |
+
import torch
|
| 34 |
+
import timesfm
|
| 35 |
+
|
| 36 |
+
if self._model is None:
|
| 37 |
+
torch.set_float32_matmul_precision("high")
|
| 38 |
+
model = timesfm.TimesFM_2p5_200M_torch.from_pretrained(self.checkpoint)
|
| 39 |
+
model.compile(timesfm.ForecastConfig(
|
| 40 |
+
max_context=self.max_context,
|
| 41 |
+
max_horizon=_MAX_HORIZON,
|
| 42 |
+
normalize_inputs=True,
|
| 43 |
+
use_continuous_quantile_head=True,
|
| 44 |
+
force_flip_invariance=True,
|
| 45 |
+
infer_is_positive=True, # forex/crypto prices are always > 0
|
| 46 |
+
fix_quantile_crossing=True,
|
| 47 |
+
))
|
| 48 |
+
self._model = model
|
| 49 |
+
return self._model
|
| 50 |
+
|
| 51 |
+
def predict(self, history: pd.Series, horizon: int = 1) -> list:
|
| 52 |
+
try:
|
| 53 |
+
import timesfm # noqa
|
| 54 |
+
except ImportError as e:
|
| 55 |
+
raise ImportError(
|
| 56 |
+
"TimesFM needs the 'timesfm' package. Install with:\n"
|
| 57 |
+
" pip install timesfm[torch]\n"
|
| 58 |
+
f"(original error: {e})"
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
values = np.asarray(history, dtype=np.float32)
|
| 62 |
+
if len(values) < 32:
|
| 63 |
+
raise ValueError("TimesFM needs at least 32 candles of history.")
|
| 64 |
+
if horizon > _MAX_HORIZON:
|
| 65 |
+
raise ValueError(f"This TimesFM wrapper supports at most {_MAX_HORIZON} candles ahead.")
|
| 66 |
+
|
| 67 |
+
context = values[-self.max_context:] if len(values) > self.max_context else values
|
| 68 |
+
model = self._load()
|
| 69 |
+
point_forecast, _quantiles = model.forecast(horizon=horizon, inputs=[context])
|
| 70 |
+
out = [float(x) for x in point_forecast[0]]
|
| 71 |
+
|
| 72 |
+
if not all(np.isfinite(x) for x in out):
|
| 73 |
+
raise ValueError(
|
| 74 |
+
"TimesFM returned a non-finite forecast (NaN/inf) for this window. "
|
| 75 |
+
"This is usually a sign the input window is degenerate (e.g. a flat "
|
| 76 |
+
"or near-constant price run) — try a different window size or symbol."
|
| 77 |
+
)
|
| 78 |
+
return out
|
requirements.txt
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
--extra-index-url https://download.pytorch.org/whl/cpu
|
| 2 |
+
|
| 3 |
+
# --- core data / numerics ---
|
| 4 |
+
# floor is 2.1, not just "2.0": the code's resample/date_range rules use the
|
| 5 |
+
# lowercase offset aliases ("10min", "4h", "min") -- confirmed these work on
|
| 6 |
+
# 2.1.4 through the current 3.0.x, and confirmed pandas 3.0 now HARD-ERRORS
|
| 7 |
+
# on the old uppercase forms ("4H", "10T") rather than just warning, so
|
| 8 |
+
# lowercase was the right choice; 2.1 is the safe, verified floor for it.
|
| 9 |
+
pandas>=2.1
|
| 10 |
+
numpy>=1.24
|
| 11 |
+
|
| 12 |
+
# --- market data (free, no API key) ---
|
| 13 |
+
yfinance>=0.2.40
|
| 14 |
+
|
| 15 |
+
# --- UI & charts ---
|
| 16 |
+
gradio>=4.36
|
| 17 |
+
plotly>=5.20
|
| 18 |
+
|
| 19 |
+
# --- ARIMA / Auto-ARIMA / ARIMA-GARCH ---
|
| 20 |
+
statsmodels>=0.14
|
| 21 |
+
arch>=6.3
|
| 22 |
+
|
| 23 |
+
# --- Moirai (Salesforce) — https://github.com/SalesforceAIResearch/uni2ts ---
|
| 24 |
+
uni2ts
|
| 25 |
+
gluonts
|
| 26 |
+
|
| 27 |
+
# --- TimesFM 2.5 (Google) — https://github.com/google-research/timesfm ---
|
| 28 |
+
timesfm[torch]
|
| 29 |
+
|
| 30 |
+
# --- shared deep-learning backend ---
|
| 31 |
+
torch
|
| 32 |
+
einops
|
| 33 |
+
huggingface_hub
|
utils/__init__.py
ADDED
|
File without changes
|
utils/helpers.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small shared helpers used by app.py and live_runner.py."""
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
import pandas as pd
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def is_probably_valid_symbol(symbol: str) -> bool:
|
| 8 |
+
"""Loose sanity check for a yfinance-style ticker (not exhaustive)."""
|
| 9 |
+
if not symbol or len(symbol) > 20:
|
| 10 |
+
return False
|
| 11 |
+
return bool(re.match(r"^[A-Za-z0-9=\-.^]+$", symbol.strip()))
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def infer_step(index: pd.DatetimeIndex):
|
| 15 |
+
"""Best-effort candle spacing (timedelta) from a DatetimeIndex, used to
|
| 16 |
+
project a forecast's index forward beyond the last known candle."""
|
| 17 |
+
if len(index) < 2:
|
| 18 |
+
return pd.Timedelta(minutes=1)
|
| 19 |
+
diffs = index.to_series().diff().dropna()
|
| 20 |
+
if diffs.empty:
|
| 21 |
+
return pd.Timedelta(minutes=1)
|
| 22 |
+
return diffs.median()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def future_index(last_timestamp, step, n):
|
| 26 |
+
"""Build `n` future timestamps spaced `step` apart, starting after last_timestamp."""
|
| 27 |
+
return [last_timestamp + step * (i + 1) for i in range(n)]
|