diff --git a/.gitignore b/.gitignore index c9e80ad..9a157b0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,5 @@ Thumbs.db *.swp *.swo -# Uploads -uploads/* -!uploads/.gitkeep - -*.tsv +# Scratch data exports +/*.tsv diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 7b06486..0000000 --- a/deploy.sh +++ /dev/null @@ -1,409 +0,0 @@ -#!/bin/bash -# -# Dataflow Deploy Script -# First run: full install (database, schema, UI, nginx, systemd) -# Subsequent runs: update functions, UI, and restart service -# - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -BOLD='\033[1m' -DIM='\033[2m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -RED='\033[0;31m' -RESET='\033[0m' - -section() { echo ""; echo -e "${BOLD}── $1 ──${RESET}"; } -step() { printf " %-42s" "$1..."; } -ok() { echo -e "${GREEN}✓${RESET}"; } -fail() { echo -e "${RED}✗ $1${RESET}"; exit 1; } -info() { echo -e " ${DIM}$1${RESET}"; } -warn() { echo -e " ${YELLOW}$1${RESET}"; } - -confirm() { - # confirm "Section name" — prints section header, asks to proceed - # Returns 0 to proceed, 1 to skip - section "$1" - read -p " Proceed? [Y/n]: " _yn - [[ ! "$_yn" =~ ^[Nn]$ ]] -} - -# ── Mode detection ──────────────────────────────────────────────────────────── - -echo "" -echo " This script covers:" -echo " 1. Database — create user/database, deploy schema + functions" -echo " 2. UI — build from source" -echo " 3. Nginx — reverse proxy config + SSL via certbot" -echo " 4. Service — install and enable systemd unit" -echo " 5. Restart — start or restart the API server" -echo "" -echo " Each step will be confirmed before running." -echo " Press Ctrl+C at any time to abort." - -section "Mode" -if [ ! -f .env ]; then - echo " No .env found — first-time install" - MODE=install -else - echo " .env found — update mode" - MODE=update - export $(cat .env | grep -v '^#' | xargs) -fi - -# ── Phase 1: Collect all config ─────────────────────────────────────────────── - -if [ "$MODE" = "install" ]; then - - section "PostgreSQL Admin" - info "Needed to create the app user and database" - read -p " Admin username [postgres]: " ADMIN_USER; ADMIN_USER=${ADMIN_USER:-postgres} - read -s -p " Admin password: " ADMIN_PASS; echo "" - - section "Application Database" - read -p " Host [localhost]: " DB_HOST; DB_HOST=${DB_HOST:-localhost} - read -p " Port [5432]: " DB_PORT; DB_PORT=${DB_PORT:-5432} - read -p " Database [dataflow]: " DB_NAME; DB_NAME=${DB_NAME:-dataflow} - read -p " App user [dataflow]: " DB_USER; DB_USER=${DB_USER:-dataflow} - read -s -p " App password: " DB_PASSWORD; echo "" - - section "API" - read -p " Port [3020]: " API_PORT; API_PORT=${API_PORT:-3020} - read -p " Environment [production]:" NODE_ENV; NODE_ENV=${NODE_ENV:-production} - - section "Nginx" - read -p " Set up nginx reverse proxy? [Y/n]: " _yn - if [[ ! "$_yn" =~ ^[Nn]$ ]]; then - read -p " Domain (e.g. dataflow.example.com): " NGINX_DOMAIN - fi - - DO_DEPS=y; DO_DB=y; DO_SCHEMA=y; DO_FN=y; DO_BUILD=y; DO_SERVICE=y; DO_RESTART=y - -else - - section "Database" - echo " Current: ${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}" - read -p " Change target? [y/N]: " _yn - if [[ "$_yn" =~ ^[Yy]$ ]]; then - read -p " Host [${DB_HOST}]: " _in; DB_HOST=${_in:-$DB_HOST} - read -p " Port [${DB_PORT}]: " _in; DB_PORT=${_in:-$DB_PORT} - read -p " Database [${DB_NAME}]: " _in; DB_NAME=${_in:-$DB_NAME} - read -p " User [${DB_USER}]: " _in; DB_USER=${_in:-$DB_USER} - read -s -p " Password (blank = keep): " _in; echo "" - if [ -n "$_in" ]; then DB_PASSWORD=$_in; fi - CHANGE_DB=y - fi - - section "Select steps to run" - read -p " Redeploy SQL functions? [Y/n]: " DO_FN; DO_FN=${DO_FN:-y} - read -p " Rebuild UI? [Y/n]: " DO_BUILD; DO_BUILD=${DO_BUILD:-y} - read -p " Set up / update nginx? [y/N]: " DO_NGINX - read -p " Restart API service? [Y/n]: " DO_RESTART; DO_RESTART=${DO_RESTART:-y} - - if [[ "$DO_NGINX" =~ ^[Yy]$ ]]; then - read -p " Nginx domain: " NGINX_DOMAIN - fi -fi - -# ── Phase 2: Plan summary ───────────────────────────────────────────────────── - -section "Plan" -echo " Mode: $( [ "$MODE" = "install" ] && echo "First-time install" || echo "Update" )" -echo " Database: ${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}" -echo " API port: ${API_PORT:-3020}" -echo "" -echo " Steps:" - -if [ "$MODE" = "install" ]; then - echo " • Install Node.js dependencies" - echo " • Test admin connection and create DB user/database" - echo " • Deploy schema and SQL functions" - [ -n "$NGINX_DOMAIN" ] && echo " • Configure nginx → $NGINX_DOMAIN" \ - || echo " • Nginx — skipped (no domain provided)" - echo " • Build UI" - echo " • Install systemd service" - echo " • Start service" -else - [ "${CHANGE_DB}" = "y" ] && echo " • Update .env with new database target" - [[ ! "$DO_FN" =~ ^[Nn]$ ]] && echo " • Redeploy SQL functions" \ - || echo " • SQL functions — skipped" - [[ ! "$DO_BUILD" =~ ^[Nn]$ ]] && echo " • Rebuild UI" \ - || echo " • UI build — skipped" - [ -n "$NGINX_DOMAIN" ] && echo " • Configure nginx → $NGINX_DOMAIN" \ - || echo " • Nginx — skipped" - [[ ! "$DO_RESTART" =~ ^[Nn]$ ]] && echo " • Restart API service" \ - || echo " • Service restart — skipped" -fi - -echo "" -read -p " Continue? [Y/n]: " _yn -[[ "$_yn" =~ ^[Nn]$ ]] && echo " Aborted." && exit 0 - -# ── Phase 3: Execute ────────────────────────────────────────────────────────── - -if [ "$MODE" = "install" ]; then - - # Dependencies - if confirm "Dependencies"; then - step "API (npm install)" - npm install --omit=dev -q && ok || fail "npm install failed" - - step "UI (npm install)" - cd ui && npm install -q && cd .. && ok || fail "ui npm install failed" - else - info "skipped" - fi - - # PostgreSQL - if confirm "PostgreSQL"; then - step "Testing admin connection" - export PGPASSWORD="$ADMIN_PASS" - psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -c '\q' 2>/dev/null && ok \ - || fail "Cannot connect as $ADMIN_USER" - - step "Creating user '$DB_USER'" - if psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -tAc \ - "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" 2>/dev/null | grep -q 1; then - echo -e "${DIM}already exists${RESET}" - else - psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres \ - -c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASSWORD';" > /dev/null && ok \ - || fail "Could not create user" - fi - - step "Creating database '$DB_NAME'" - if psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -lqt \ - | cut -d'|' -f1 | grep -qw "$DB_NAME"; then - echo -e "${DIM}already exists${RESET}" - else - psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres \ - -c "CREATE DATABASE $DB_NAME OWNER $DB_USER;" > /dev/null && ok \ - || fail "Could not create database" - fi - unset PGPASSWORD - - step "Writing .env" - cat > .env << ENVEOF -# Database Configuration -DB_HOST=$DB_HOST -DB_PORT=$DB_PORT -DB_NAME=$DB_NAME -DB_USER=$DB_USER -DB_PASSWORD=$DB_PASSWORD - -# API Configuration -API_PORT=$API_PORT -NODE_ENV=$NODE_ENV -ENVEOF - ok - - export PGPASSWORD="$DB_PASSWORD" - - step "Deploying schema" - psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/schema.sql -q && ok \ - || fail "Schema deploy failed" - - step "Deploying functions" - psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/functions.sql -q && ok \ - || fail "Functions deploy failed" - else - info "skipped" - fi - -else - - # Update: save .env if DB target changed - section ".env" - if [ "${CHANGE_DB}" = "y" ]; then - step "Writing updated .env" - cat > .env << ENVEOF -# Database Configuration -DB_HOST=$DB_HOST -DB_PORT=$DB_PORT -DB_NAME=$DB_NAME -DB_USER=$DB_USER -DB_PASSWORD=$DB_PASSWORD - -# API Configuration -API_PORT=${API_PORT:-3020} -NODE_ENV=${NODE_ENV:-production} -ENVEOF - ok - else - info "no changes" - fi - - # Test connection - section "Database Connection" - step "Testing" - export PGPASSWORD="$DB_PASSWORD" - psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -c '\q' 2>/dev/null && ok \ - || fail "Cannot connect — check credentials" - - # SQL functions - if [[ ! "$DO_FN" =~ ^[Nn]$ ]]; then - if confirm "SQL Functions"; then - step "Deploying functions" - psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/functions.sql -q && ok \ - || fail "Functions deploy failed" - else - info "skipped" - fi - else - section "SQL Functions" - info "skipped" - fi -fi - -# ── UI ──────────────────────────────────────────────────────────────────────── - -if [ "$MODE" = "install" ] || [[ ! "$DO_BUILD" =~ ^[Nn]$ ]]; then - if confirm "UI Build"; then - step "Building" - cd ui && npm run build > /dev/null 2>&1 && cd .. && ok \ - || fail "UI build failed" - else - info "skipped" - fi -else - section "UI Build" - info "skipped" -fi - -# ── Nginx ───────────────────────────────────────────────────────────────────── - -if [ -n "$NGINX_DOMAIN" ]; then - if confirm "Nginx ($NGINX_DOMAIN)"; then - CONF_NAME=$(echo "$NGINX_DOMAIN" | cut -d. -f1) - CONF_PATH="/etc/nginx/sites-enabled/$CONF_NAME" - CERT_PATH="/etc/letsencrypt/live/$NGINX_DOMAIN/fullchain.pem" - TMP_CONF=$(mktemp) - - if [ -f "$CERT_PATH" ]; then - cat > "$TMP_CONF" << NGINXEOF -server { - listen 80; - listen [::]:80; - server_name $NGINX_DOMAIN; - location / { return 301 https://\$host\$request_uri; } -} - -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - server_name $NGINX_DOMAIN; - - ssl_certificate $CERT_PATH; - ssl_certificate_key /etc/letsencrypt/live/$NGINX_DOMAIN/privkey.pem; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:SSL:10m; - - keepalive_timeout 70; - sendfile on; - client_max_body_size 80m; - - location / { - proxy_pass http://localhost:${API_PORT:-3020}; - } -} -NGINXEOF - else - cat > "$TMP_CONF" << NGINXEOF -server { - listen 80; - listen [::]:80; - server_name $NGINX_DOMAIN; - location / { - proxy_pass http://localhost:${API_PORT:-3020}; - } -} -NGINXEOF - fi - - step "Writing /etc/nginx/sites-enabled/$CONF_NAME" - sudo cp "$TMP_CONF" "$CONF_PATH" && rm "$TMP_CONF" && ok \ - || fail "Could not write nginx config (check sudo)" - - step "Testing nginx config" - sudo nginx -t > /dev/null 2>&1 && ok \ - || fail "nginx config invalid — run: sudo nginx -t" - - step "Reloading nginx" - sudo systemctl reload nginx && ok \ - || fail "nginx reload failed" - - if [ ! -f "$CERT_PATH" ]; then - warn "No SSL cert found for $NGINX_DOMAIN" - read -p " Run certbot now? [Y/n]: " _yn - if [[ ! "$_yn" =~ ^[Nn]$ ]]; then - step "Running certbot" - sudo certbot --nginx -d "$NGINX_DOMAIN" --non-interactive --agree-tos \ - --redirect -m "admin@$NGINX_DOMAIN" > /dev/null 2>&1 && ok \ - || fail "certbot failed — run manually: sudo certbot --nginx -d $NGINX_DOMAIN" - fi - fi - else - info "skipped" - fi -else - section "Nginx" - info "skipped" -fi - -# ── Systemd service ─────────────────────────────────────────────────────────── - -SERVICE_FILE="/etc/systemd/system/dataflow.service" - -section "Systemd Service" -if [ -f "$SERVICE_FILE" ]; then - info "already installed" -else - read -p " Not installed. Install now? (requires sudo) [y/N]: " _yn - if [[ "$_yn" =~ ^[Yy]$ ]]; then - step "Installing service" - sudo cp "$SCRIPT_DIR/dataflow.service" "$SERVICE_FILE" && ok \ - || fail "Could not install service" - - step "Enabling on boot" - sudo systemctl daemon-reload && sudo systemctl enable dataflow > /dev/null 2>&1 && ok \ - || fail "Could not enable service" - else - info "skipped" - fi -fi - -# ── API Server ──────────────────────────────────────────────────────────────── - -if [ "$MODE" = "install" ] || [[ ! "$DO_RESTART" =~ ^[Nn]$ ]]; then - if confirm "API Server"; then - if [ -f "$SERVICE_FILE" ]; then - step "Restarting dataflow service" - sudo systemctl restart dataflow && sleep 1 - systemctl is-active --quiet dataflow && ok \ - || fail "Service failed — check: journalctl -u dataflow -n 30" - else - info "systemd service not installed — start manually: node api/server.js" - fi - else - info "skipped" - fi -else - section "API Server" - info "skipped" -fi - -# ── Done ────────────────────────────────────────────────────────────────────── - -section "Done" -echo " API: http://localhost:${API_PORT:-3020}" -[ -n "$NGINX_DOMAIN" ] && echo " Web: https://$NGINX_DOMAIN" -echo " Logs: journalctl -u dataflow -f" -echo "" diff --git a/manage.py b/manage.py index 9eee6dc..efa33dd 100755 --- a/manage.py +++ b/manage.py @@ -18,6 +18,17 @@ SERVICE_FILE = Path('/etc/systemd/system/dataflow.service') SERVICE_SRC = ROOT / 'dataflow.service' NGINX_DIR = Path('/etc/nginx/sites-enabled') +# Deployed in order — stacks.sql creates tables that reference sources +QUERIES_DIR = ROOT / 'database' / 'queries' +QUERY_FILES = [ + QUERIES_DIR / 'sources.sql', + QUERIES_DIR / 'rules.sql', + QUERIES_DIR / 'mappings.sql', + QUERIES_DIR / 'records.sql', + QUERIES_DIR / 'stacks.sql', + QUERIES_DIR / 'status.sql', +] + # ── Terminal helpers ────────────────────────────────────────────────────────── BOLD = '\033[1m' @@ -332,13 +343,8 @@ def action_configure(cfg): db_location = f'database "{new_cfg["DB_NAME"]}" on {new_cfg["DB_HOST"]}:{new_cfg["DB_PORT"]}' schema_file = ROOT / 'database' / 'schema.sql' - queries_dir = ROOT / 'database' / 'queries' - query_files = [ - queries_dir / 'sources.sql', - queries_dir / 'rules.sql', - queries_dir / 'mappings.sql', - queries_dir / 'records.sql', - ] + queries_dir = QUERIES_DIR + query_files = QUERY_FILES # Offer schema deployment print() @@ -433,13 +439,8 @@ def action_deploy_functions(cfg): return db_location = f'database "{cfg["DB_NAME"]}" on {cfg["DB_HOST"]}:{cfg["DB_PORT"]}' - queries_dir = ROOT / 'database' / 'queries' - query_files = [ - queries_dir / 'sources.sql', - queries_dir / 'rules.sql', - queries_dir / 'mappings.sql', - queries_dir / 'records.sql', - ] + queries_dir = QUERIES_DIR + query_files = QUERY_FILES print(f' Source files: {queries_dir}/') for f in query_files: diff --git a/migrate/dataflow.pg.sql b/migrate/dataflow.pg.sql deleted file mode 100644 index fe914c3..0000000 --- a/migrate/dataflow.pg.sql +++ /dev/null @@ -1 +0,0 @@ -select id, source, constrain_key, data from dataflow.records diff --git a/migrate/reimport_dcard_from_tps.sh b/migrate/reimport_dcard_from_tps.sh deleted file mode 100644 index 3cf2343..0000000 --- a/migrate/reimport_dcard_from_tps.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Reimport dcard records from ubm.tps.trans into dataflow.records -# -# Step 1: exports raw rec JSON from ubm -# Step 2: wipes existing dcard data in dataflow and reloads from the export -# -# Usage: bash migrate/reimport_dcard_from_tps.sh - -set -e - -EXPORT_FILE="/tmp/tps_dcard_rec.csv" -echo "==> Exporting dcard from ubm.tps.trans..." -psql -U ptrowbridge -d ubm -p 54329 -h hptrow.me -c "\COPY (SELECT rec FROM tps.trans WHERE srce = 'dcard' ORDER BY id) TO '${EXPORT_FILE}' CSV" -echo " Exported $(wc -l < ${EXPORT_FILE}) rows" - -echo "==> Reimporting into dataflow.records..." -$PG -d dataflow <>'Trans. Date', - 'Post Date', s.rec->>'Post Date', - 'Description', s.rec->>'Description' - ), - i.import_id - FROM _dcard_import s, new_import i - RETURNING id -) -UPDATE dataflow.import_log -SET records_imported = (SELECT COUNT(*) FROM inserted) -WHERE source_name = 'dcard' - AND id = (SELECT id FROM dataflow.import_log WHERE source_name = 'dcard' ORDER BY id DESC LIMIT 1); - -COMMIT; -SELECT records_imported FROM dataflow.import_log WHERE source_name = 'dcard' ORDER BY id DESC LIMIT 1; -SQL - -echo "==> Done. Run transformations to repopulate the transformed column." diff --git a/package.json b/package.json index 6fdca4c..b2d66e8 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "Simple data transformation tool for ingesting, mapping, and transforming data", "main": "api/server.js", "scripts": { - "start": "nodemon api/server.js", - "dev": "node api/server.js", + "start": "node api/server.js", + "dev": "nodemon api/server.js", "test": "echo \"Tests coming soon\" && exit 0" }, "keywords": [ diff --git a/scripts/setup-service.sh b/scripts/setup-service.sh deleted file mode 100755 index b930ef9..0000000 --- a/scripts/setup-service.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e - -SERVICE_NAME="dataflow" -SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service" -WORKDIR=$(pwd) - -echo "Creating systemd service file..." -sudo tee "$SERVICE_FILE" > /dev/null <