stock-scraper/scraper.ts

#!/usr/bin/env -S deno run --allow-net --allow-write

// ------------------------------------------------------------
// Linear Regression
// ------------------------------------------------------------
function linearRegression(xs: number[], ys: number[]) {
  const n = xs.length;
  const meanX = xs.reduce((a, b) => a + b, 0) / n;
  const meanY = ys.reduce((a, b) => a + b, 0) / n;

  let num = 0;
  let den = 0;

  for (let i = 0; i < n; i++) {
    num += (xs[i] - meanX) * (ys[i] - meanY);
    den += (xs[i] - meanX) ** 2;
  }

  if (den === 0) return null; // flat line → no regression possible

  const slope = num / den;
  const intercept = meanY - slope * meanX;

  let ssTot = 0;
  let ssRes = 0;

  for (let i = 0; i < n; i++) {
    const yPred = slope * xs[i] + intercept;
    ssTot += (ys[i] - meanY) ** 2;
    ssRes += (ys[i] - yPred) ** 2;
  }

  const r2 = ssTot === 0 ? 0 : 1 - ssRes / ssTot;

  const start = intercept;
  const end = start + slope*xs.length;
  const growth = (end/start) - 1;

  return { slope, intercept, r2,  growth};
}

// ------------------------------------------------------------
// Fetch S&P 500 tickers
// ------------------------------------------------------------
async function fetchSP500Tickers(): Promise<string[]> {
  const url =
    "https://datahub.io/core/s-and-p-500-companies/_r/-/data/constituents.csv";

  const res = await fetch(url);
  if (!res.ok) throw new Error("Failed to fetch S&P 500 CSV");

  const csv = await res.text();
  const lines = csv.trim().split("\n");
  lines.shift(); // header

  return lines.map((line) => line.split(",")[0]);
}

// ------------------------------------------------------------
// Yahoo Finance fetch
// ------------------------------------------------------------
const now = Math.floor(Date.now() / 1000);
const thirtyDaysAgo = now - 30 * 24 * 60 * 60;
const args = `?period1=${thirtyDaysAgo}&period2=${now}&interval=1d`;
const closingEndpoint =(ticker:string)=>`https://query1.finance.yahoo.com/v8/finance/chart/${ticker}${args}`;

async function fetchLast30Closes(ticker: string): Promise<number[]> {

  const res = await fetch(closingEndpoint(ticker));
  if (!res.ok) return [];

  const json = await res.json();
  const result = json.chart?.result?.[0];
  if (!result) return [];

  const closes = result.indicators?.quote?.[0]?.close;
  return closes?.filter((x: number | null) => x != null) ?? [];
}

// ------------------------------------------------------------
// Compute regression for a ticker
// ------------------------------------------------------------
async function ComputeTicker(ticker: string) {
  const closes = await fetchLast30Closes(ticker);
  if (closes.length < 5) return null;

  const xs = closes.map( (_, index) => index );
  const ys = closes;

  return linearRegression(xs, ys);
}

// ------------------------------------------------------------
// Concurrency Throttler
// ------------------------------------------------------------
async function throttle<T>(
  items: T[],
  limit: number,
  fn: (item: T) => Promise<void>,
) {
  const queue: Promise<void>[] = [];

  for (const item of items) {
    const p = fn(item);
    queue.push(p);

    if (queue.length >= limit) {
      await Promise.race(queue);
      // Remove settled promises
      for (let i = queue.length - 1; i >= 0; i--) {
        if (queue[i].catch(() => {}) && true) queue.splice(i, 1);
      }
    }
  }

  await Promise.all(queue);
}

// ------------------------------------------------------------
// Main Dump
// ------------------------------------------------------------
async function Dump() {
  const spx = await ComputeTicker("^GSPC");

  if (!spx) {
    console.error("Could not get S&P Index data");
    return;
  }

  const rows: string[] = ["ticker,slope,intercept,r2,growth"];
  const addRow = (ticker: string, model: ReturnType<typeof linearRegression>) => {
    rows.push(
      `${ticker},${model.slope.toFixed(6)},${model.intercept.toFixed(6)},${model.r2.toFixed(6)},${model?.growth.toFixed(2)}`,
    );
  };

  addRow("SPX", spx);

  const tickers = await fetchSP500Tickers();
  console.log(`${tickers.length} S&P 500 stocks found...`);
  console.log(
    `Finding stocks with slope better than the S&P Index slope (${spx.slope.toFixed(6)})...`,
  );

  // Throttle to avoid Yahoo soft throttling
  const limit = 5; // adjust as needed

  await throttle(tickers, limit, async (ticker) => {
    try {
      const model = await ComputeTicker(ticker);
      if (model && model.growth > spx.growth) {
        addRow(ticker, model);
        console.log(`${ticker}`);
      }
      else{
        //console.log("bad: ", ticker, model?.slope)
      }
    } catch (e) {
      console.log(`Skipping "${ticker}" because: ${e}`);
    }
  });

  await Deno.writeTextFile("sp500_regression.csv", rows.join("\n"));
  console.log("Dumped output to sp500_regression.csv");
}

Dump();
//console.log(closingEndpoint("^GSPC"))