add scraper

deno.json

@@ -0,0 +1,8 @@
+{
+  "tasks": {
+    "scrape": "deno run --watch scraper.ts"
+  },
+  "imports": {
+    "@std/assert": "jsr:@std/assert@1"
+  }
+}

scraper.ts

@@ -0,0 +1,111 @@
+#!/usr/bin/env -S deno run --allow-net
+
+// ------------------------------------------------------------
+// Utility: simple linear regression
+// ------------------------------------------------------------
+function linearRegression(xs: number[], ys: number[]) {
+  const n = xs.length;
+  const meanX = xs.reduce((a, b) => a + b, 0) / n;
+  const meanY = ys.reduce((a, b) => a + b, 0) / n;
+
+  let num = 0;
+  let den = 0;
+
+  for (let i = 0; i < n; i++) {
+    num += (xs[i] - meanX) * (ys[i] - meanY);
+    den += (xs[i] - meanX) ** 2;
+  }
+
+  const slope = num / den;
+  const intercept = meanY - slope * meanX;
+
+  // Compute R²
+  let ssTot = 0;
+  let ssRes = 0;
+  for (let i = 0; i < n; i++) {
+    const yPred = slope * xs[i] + intercept;
+    ssTot += (ys[i] - meanY) ** 2;
+    ssRes += (ys[i] - yPred) ** 2;
+  }
+  const r2 = 1 - ssRes / ssTot;
+
+  return { slope, intercept, r2 };
+}
+
+// ------------------------------------------------------------
+// Fetch S&P 500 constituents
+// ------------------------------------------------------------
+async function fetchSP500Tickers(): Promise<string[]> {
+  const url =
+    "https://datahub.io/core/s-and-p-500-companies/_r/-/data/constituents.csv";
+
+  const res = await fetch(url);
+  if (!res.ok) throw new Error("Failed to fetch S&P 500 CSV");
+
+  const csv = await res.text();
+
+  // Simple CSV parsing
+  const lines = csv.trim().split("\n");
+  const header = lines.shift(); // remove header row
+
+  const tickers: string[] = [];
+
+  for (const line of lines) {
+    const [symbol] = line.split(",");
+    tickers.push(symbol);
+  }
+
+  return tickers;
+}
+
+// ------------------------------------------------------------
+// Fetch last 30 days of closes for a ticker
+// ------------------------------------------------------------
+async function fetchLast30Closes(ticker: string): Promise<number[]> {
+  const now = Math.floor(Date.now() / 1000);
+  const thirtyDaysAgo = now - 30 * 24 * 60 * 60;
+
+  const url =
+    `https://query1.finance.yahoo.com/v8/finance/chart/${ticker}?period1=${thirtyDaysAgo}&period2=${now}&interval=1d`;
+
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to fetch data for ${ticker}`);
+
+  const json = await res.json();
+  const closes = json.chart.result?.[0]?.indicators?.quote?.[0]?.close;
+
+  if (!closes) return [];
+  return closes.filter((x: number | null) => x != null);
+}
+
+// ------------------------------------------------------------
+// Main
+// ------------------------------------------------------------
+const tickers = await fetchSP500Tickers();
+
+console.log(`Loaded ${tickers.length} S&P 500 tickers\n`);
+
+for (const ticker of tickers) {
+  try {
+    const closes = await fetchLast30Closes(ticker);
+
+    if (closes.length < 5) {
+      console.log(`${ticker}: insufficient data`);
+      continue;
+    }
+
+    // xs = 0..n-1
+    const xs = closes.map((_, i) => i);
+    const ys = closes;
+
+    const { slope, intercept, r2 } = linearRegression(xs, ys);
+
+    console.log(
+      `${ticker}: slope=${slope.toFixed(4)}, intercept=${intercept.toFixed(
+        2,
+      )}, r2=${r2.toFixed(3)}`,
+    );
+  } catch (err) {
+    console.log(`${ticker}: error (${err.message})`);
+  }
+}