Remove obsolete deploy scripts, migration helpers, and unused UI assets
Drop deploy.sh, scripts/setup-service.sh, and migrate/ — all superseded by manage.py. Remove leftover Vite template assets (App.css, hero.png, react.svg, vite.svg), none of which are imported. Hoist the database/queries/ file list in manage.py to a module-level QUERY_FILES constant so configure and deploy-functions share one definition, and add the stacks/status query files that were missing from it. Swap npm start/dev so start runs node and dev runs nodemon. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
442c38d3c4
commit
1260873316
7
.gitignore
vendored
7
.gitignore
vendored
@ -28,8 +28,5 @@ Thumbs.db
|
|||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
|
|
||||||
# Uploads
|
# Scratch data exports
|
||||||
uploads/*
|
/*.tsv
|
||||||
!uploads/.gitkeep
|
|
||||||
|
|
||||||
*.tsv
|
|
||||||
|
|||||||
409
deploy.sh
409
deploy.sh
@ -1,409 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# Dataflow Deploy Script
|
|
||||||
# First run: full install (database, schema, UI, nginx, systemd)
|
|
||||||
# Subsequent runs: update functions, UI, and restart service
|
|
||||||
#
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
cd "$SCRIPT_DIR"
|
|
||||||
|
|
||||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
BOLD='\033[1m'
|
|
||||||
DIM='\033[2m'
|
|
||||||
GREEN='\033[0;32m'
|
|
||||||
YELLOW='\033[0;33m'
|
|
||||||
RED='\033[0;31m'
|
|
||||||
RESET='\033[0m'
|
|
||||||
|
|
||||||
section() { echo ""; echo -e "${BOLD}── $1 ──${RESET}"; }
|
|
||||||
step() { printf " %-42s" "$1..."; }
|
|
||||||
ok() { echo -e "${GREEN}✓${RESET}"; }
|
|
||||||
fail() { echo -e "${RED}✗ $1${RESET}"; exit 1; }
|
|
||||||
info() { echo -e " ${DIM}$1${RESET}"; }
|
|
||||||
warn() { echo -e " ${YELLOW}$1${RESET}"; }
|
|
||||||
|
|
||||||
confirm() {
|
|
||||||
# confirm "Section name" — prints section header, asks to proceed
|
|
||||||
# Returns 0 to proceed, 1 to skip
|
|
||||||
section "$1"
|
|
||||||
read -p " Proceed? [Y/n]: " _yn
|
|
||||||
[[ ! "$_yn" =~ ^[Nn]$ ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Mode detection ────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo " This script covers:"
|
|
||||||
echo " 1. Database — create user/database, deploy schema + functions"
|
|
||||||
echo " 2. UI — build from source"
|
|
||||||
echo " 3. Nginx — reverse proxy config + SSL via certbot"
|
|
||||||
echo " 4. Service — install and enable systemd unit"
|
|
||||||
echo " 5. Restart — start or restart the API server"
|
|
||||||
echo ""
|
|
||||||
echo " Each step will be confirmed before running."
|
|
||||||
echo " Press Ctrl+C at any time to abort."
|
|
||||||
|
|
||||||
section "Mode"
|
|
||||||
if [ ! -f .env ]; then
|
|
||||||
echo " No .env found — first-time install"
|
|
||||||
MODE=install
|
|
||||||
else
|
|
||||||
echo " .env found — update mode"
|
|
||||||
MODE=update
|
|
||||||
export $(cat .env | grep -v '^#' | xargs)
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Phase 1: Collect all config ───────────────────────────────────────────────
|
|
||||||
|
|
||||||
if [ "$MODE" = "install" ]; then
|
|
||||||
|
|
||||||
section "PostgreSQL Admin"
|
|
||||||
info "Needed to create the app user and database"
|
|
||||||
read -p " Admin username [postgres]: " ADMIN_USER; ADMIN_USER=${ADMIN_USER:-postgres}
|
|
||||||
read -s -p " Admin password: " ADMIN_PASS; echo ""
|
|
||||||
|
|
||||||
section "Application Database"
|
|
||||||
read -p " Host [localhost]: " DB_HOST; DB_HOST=${DB_HOST:-localhost}
|
|
||||||
read -p " Port [5432]: " DB_PORT; DB_PORT=${DB_PORT:-5432}
|
|
||||||
read -p " Database [dataflow]: " DB_NAME; DB_NAME=${DB_NAME:-dataflow}
|
|
||||||
read -p " App user [dataflow]: " DB_USER; DB_USER=${DB_USER:-dataflow}
|
|
||||||
read -s -p " App password: " DB_PASSWORD; echo ""
|
|
||||||
|
|
||||||
section "API"
|
|
||||||
read -p " Port [3020]: " API_PORT; API_PORT=${API_PORT:-3020}
|
|
||||||
read -p " Environment [production]:" NODE_ENV; NODE_ENV=${NODE_ENV:-production}
|
|
||||||
|
|
||||||
section "Nginx"
|
|
||||||
read -p " Set up nginx reverse proxy? [Y/n]: " _yn
|
|
||||||
if [[ ! "$_yn" =~ ^[Nn]$ ]]; then
|
|
||||||
read -p " Domain (e.g. dataflow.example.com): " NGINX_DOMAIN
|
|
||||||
fi
|
|
||||||
|
|
||||||
DO_DEPS=y; DO_DB=y; DO_SCHEMA=y; DO_FN=y; DO_BUILD=y; DO_SERVICE=y; DO_RESTART=y
|
|
||||||
|
|
||||||
else
|
|
||||||
|
|
||||||
section "Database"
|
|
||||||
echo " Current: ${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}"
|
|
||||||
read -p " Change target? [y/N]: " _yn
|
|
||||||
if [[ "$_yn" =~ ^[Yy]$ ]]; then
|
|
||||||
read -p " Host [${DB_HOST}]: " _in; DB_HOST=${_in:-$DB_HOST}
|
|
||||||
read -p " Port [${DB_PORT}]: " _in; DB_PORT=${_in:-$DB_PORT}
|
|
||||||
read -p " Database [${DB_NAME}]: " _in; DB_NAME=${_in:-$DB_NAME}
|
|
||||||
read -p " User [${DB_USER}]: " _in; DB_USER=${_in:-$DB_USER}
|
|
||||||
read -s -p " Password (blank = keep): " _in; echo ""
|
|
||||||
if [ -n "$_in" ]; then DB_PASSWORD=$_in; fi
|
|
||||||
CHANGE_DB=y
|
|
||||||
fi
|
|
||||||
|
|
||||||
section "Select steps to run"
|
|
||||||
read -p " Redeploy SQL functions? [Y/n]: " DO_FN; DO_FN=${DO_FN:-y}
|
|
||||||
read -p " Rebuild UI? [Y/n]: " DO_BUILD; DO_BUILD=${DO_BUILD:-y}
|
|
||||||
read -p " Set up / update nginx? [y/N]: " DO_NGINX
|
|
||||||
read -p " Restart API service? [Y/n]: " DO_RESTART; DO_RESTART=${DO_RESTART:-y}
|
|
||||||
|
|
||||||
if [[ "$DO_NGINX" =~ ^[Yy]$ ]]; then
|
|
||||||
read -p " Nginx domain: " NGINX_DOMAIN
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Phase 2: Plan summary ─────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
section "Plan"
|
|
||||||
echo " Mode: $( [ "$MODE" = "install" ] && echo "First-time install" || echo "Update" )"
|
|
||||||
echo " Database: ${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}"
|
|
||||||
echo " API port: ${API_PORT:-3020}"
|
|
||||||
echo ""
|
|
||||||
echo " Steps:"
|
|
||||||
|
|
||||||
if [ "$MODE" = "install" ]; then
|
|
||||||
echo " • Install Node.js dependencies"
|
|
||||||
echo " • Test admin connection and create DB user/database"
|
|
||||||
echo " • Deploy schema and SQL functions"
|
|
||||||
[ -n "$NGINX_DOMAIN" ] && echo " • Configure nginx → $NGINX_DOMAIN" \
|
|
||||||
|| echo " • Nginx — skipped (no domain provided)"
|
|
||||||
echo " • Build UI"
|
|
||||||
echo " • Install systemd service"
|
|
||||||
echo " • Start service"
|
|
||||||
else
|
|
||||||
[ "${CHANGE_DB}" = "y" ] && echo " • Update .env with new database target"
|
|
||||||
[[ ! "$DO_FN" =~ ^[Nn]$ ]] && echo " • Redeploy SQL functions" \
|
|
||||||
|| echo " • SQL functions — skipped"
|
|
||||||
[[ ! "$DO_BUILD" =~ ^[Nn]$ ]] && echo " • Rebuild UI" \
|
|
||||||
|| echo " • UI build — skipped"
|
|
||||||
[ -n "$NGINX_DOMAIN" ] && echo " • Configure nginx → $NGINX_DOMAIN" \
|
|
||||||
|| echo " • Nginx — skipped"
|
|
||||||
[[ ! "$DO_RESTART" =~ ^[Nn]$ ]] && echo " • Restart API service" \
|
|
||||||
|| echo " • Service restart — skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
read -p " Continue? [Y/n]: " _yn
|
|
||||||
[[ "$_yn" =~ ^[Nn]$ ]] && echo " Aborted." && exit 0
|
|
||||||
|
|
||||||
# ── Phase 3: Execute ──────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
if [ "$MODE" = "install" ]; then
|
|
||||||
|
|
||||||
# Dependencies
|
|
||||||
if confirm "Dependencies"; then
|
|
||||||
step "API (npm install)"
|
|
||||||
npm install --omit=dev -q && ok || fail "npm install failed"
|
|
||||||
|
|
||||||
step "UI (npm install)"
|
|
||||||
cd ui && npm install -q && cd .. && ok || fail "ui npm install failed"
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# PostgreSQL
|
|
||||||
if confirm "PostgreSQL"; then
|
|
||||||
step "Testing admin connection"
|
|
||||||
export PGPASSWORD="$ADMIN_PASS"
|
|
||||||
psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -c '\q' 2>/dev/null && ok \
|
|
||||||
|| fail "Cannot connect as $ADMIN_USER"
|
|
||||||
|
|
||||||
step "Creating user '$DB_USER'"
|
|
||||||
if psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -tAc \
|
|
||||||
"SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" 2>/dev/null | grep -q 1; then
|
|
||||||
echo -e "${DIM}already exists${RESET}"
|
|
||||||
else
|
|
||||||
psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres \
|
|
||||||
-c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASSWORD';" > /dev/null && ok \
|
|
||||||
|| fail "Could not create user"
|
|
||||||
fi
|
|
||||||
|
|
||||||
step "Creating database '$DB_NAME'"
|
|
||||||
if psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -lqt \
|
|
||||||
| cut -d'|' -f1 | grep -qw "$DB_NAME"; then
|
|
||||||
echo -e "${DIM}already exists${RESET}"
|
|
||||||
else
|
|
||||||
psql -U "$ADMIN_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres \
|
|
||||||
-c "CREATE DATABASE $DB_NAME OWNER $DB_USER;" > /dev/null && ok \
|
|
||||||
|| fail "Could not create database"
|
|
||||||
fi
|
|
||||||
unset PGPASSWORD
|
|
||||||
|
|
||||||
step "Writing .env"
|
|
||||||
cat > .env << ENVEOF
|
|
||||||
# Database Configuration
|
|
||||||
DB_HOST=$DB_HOST
|
|
||||||
DB_PORT=$DB_PORT
|
|
||||||
DB_NAME=$DB_NAME
|
|
||||||
DB_USER=$DB_USER
|
|
||||||
DB_PASSWORD=$DB_PASSWORD
|
|
||||||
|
|
||||||
# API Configuration
|
|
||||||
API_PORT=$API_PORT
|
|
||||||
NODE_ENV=$NODE_ENV
|
|
||||||
ENVEOF
|
|
||||||
ok
|
|
||||||
|
|
||||||
export PGPASSWORD="$DB_PASSWORD"
|
|
||||||
|
|
||||||
step "Deploying schema"
|
|
||||||
psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/schema.sql -q && ok \
|
|
||||||
|| fail "Schema deploy failed"
|
|
||||||
|
|
||||||
step "Deploying functions"
|
|
||||||
psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/functions.sql -q && ok \
|
|
||||||
|| fail "Functions deploy failed"
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
else
|
|
||||||
|
|
||||||
# Update: save .env if DB target changed
|
|
||||||
section ".env"
|
|
||||||
if [ "${CHANGE_DB}" = "y" ]; then
|
|
||||||
step "Writing updated .env"
|
|
||||||
cat > .env << ENVEOF
|
|
||||||
# Database Configuration
|
|
||||||
DB_HOST=$DB_HOST
|
|
||||||
DB_PORT=$DB_PORT
|
|
||||||
DB_NAME=$DB_NAME
|
|
||||||
DB_USER=$DB_USER
|
|
||||||
DB_PASSWORD=$DB_PASSWORD
|
|
||||||
|
|
||||||
# API Configuration
|
|
||||||
API_PORT=${API_PORT:-3020}
|
|
||||||
NODE_ENV=${NODE_ENV:-production}
|
|
||||||
ENVEOF
|
|
||||||
ok
|
|
||||||
else
|
|
||||||
info "no changes"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Test connection
|
|
||||||
section "Database Connection"
|
|
||||||
step "Testing"
|
|
||||||
export PGPASSWORD="$DB_PASSWORD"
|
|
||||||
psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -c '\q' 2>/dev/null && ok \
|
|
||||||
|| fail "Cannot connect — check credentials"
|
|
||||||
|
|
||||||
# SQL functions
|
|
||||||
if [[ ! "$DO_FN" =~ ^[Nn]$ ]]; then
|
|
||||||
if confirm "SQL Functions"; then
|
|
||||||
step "Deploying functions"
|
|
||||||
psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -f database/functions.sql -q && ok \
|
|
||||||
|| fail "Functions deploy failed"
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
section "SQL Functions"
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── UI ────────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
if [ "$MODE" = "install" ] || [[ ! "$DO_BUILD" =~ ^[Nn]$ ]]; then
|
|
||||||
if confirm "UI Build"; then
|
|
||||||
step "Building"
|
|
||||||
cd ui && npm run build > /dev/null 2>&1 && cd .. && ok \
|
|
||||||
|| fail "UI build failed"
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
section "UI Build"
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Nginx ─────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
if [ -n "$NGINX_DOMAIN" ]; then
|
|
||||||
if confirm "Nginx ($NGINX_DOMAIN)"; then
|
|
||||||
CONF_NAME=$(echo "$NGINX_DOMAIN" | cut -d. -f1)
|
|
||||||
CONF_PATH="/etc/nginx/sites-enabled/$CONF_NAME"
|
|
||||||
CERT_PATH="/etc/letsencrypt/live/$NGINX_DOMAIN/fullchain.pem"
|
|
||||||
TMP_CONF=$(mktemp)
|
|
||||||
|
|
||||||
if [ -f "$CERT_PATH" ]; then
|
|
||||||
cat > "$TMP_CONF" << NGINXEOF
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
listen [::]:80;
|
|
||||||
server_name $NGINX_DOMAIN;
|
|
||||||
location / { return 301 https://\$host\$request_uri; }
|
|
||||||
}
|
|
||||||
|
|
||||||
server {
|
|
||||||
listen 443 ssl http2;
|
|
||||||
listen [::]:443 ssl http2;
|
|
||||||
server_name $NGINX_DOMAIN;
|
|
||||||
|
|
||||||
ssl_certificate $CERT_PATH;
|
|
||||||
ssl_certificate_key /etc/letsencrypt/live/$NGINX_DOMAIN/privkey.pem;
|
|
||||||
ssl_protocols TLSv1.2 TLSv1.3;
|
|
||||||
ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
|
|
||||||
ssl_prefer_server_ciphers on;
|
|
||||||
ssl_session_cache shared:SSL:10m;
|
|
||||||
|
|
||||||
keepalive_timeout 70;
|
|
||||||
sendfile on;
|
|
||||||
client_max_body_size 80m;
|
|
||||||
|
|
||||||
location / {
|
|
||||||
proxy_pass http://localhost:${API_PORT:-3020};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NGINXEOF
|
|
||||||
else
|
|
||||||
cat > "$TMP_CONF" << NGINXEOF
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
listen [::]:80;
|
|
||||||
server_name $NGINX_DOMAIN;
|
|
||||||
location / {
|
|
||||||
proxy_pass http://localhost:${API_PORT:-3020};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NGINXEOF
|
|
||||||
fi
|
|
||||||
|
|
||||||
step "Writing /etc/nginx/sites-enabled/$CONF_NAME"
|
|
||||||
sudo cp "$TMP_CONF" "$CONF_PATH" && rm "$TMP_CONF" && ok \
|
|
||||||
|| fail "Could not write nginx config (check sudo)"
|
|
||||||
|
|
||||||
step "Testing nginx config"
|
|
||||||
sudo nginx -t > /dev/null 2>&1 && ok \
|
|
||||||
|| fail "nginx config invalid — run: sudo nginx -t"
|
|
||||||
|
|
||||||
step "Reloading nginx"
|
|
||||||
sudo systemctl reload nginx && ok \
|
|
||||||
|| fail "nginx reload failed"
|
|
||||||
|
|
||||||
if [ ! -f "$CERT_PATH" ]; then
|
|
||||||
warn "No SSL cert found for $NGINX_DOMAIN"
|
|
||||||
read -p " Run certbot now? [Y/n]: " _yn
|
|
||||||
if [[ ! "$_yn" =~ ^[Nn]$ ]]; then
|
|
||||||
step "Running certbot"
|
|
||||||
sudo certbot --nginx -d "$NGINX_DOMAIN" --non-interactive --agree-tos \
|
|
||||||
--redirect -m "admin@$NGINX_DOMAIN" > /dev/null 2>&1 && ok \
|
|
||||||
|| fail "certbot failed — run manually: sudo certbot --nginx -d $NGINX_DOMAIN"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
section "Nginx"
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Systemd service ───────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
SERVICE_FILE="/etc/systemd/system/dataflow.service"
|
|
||||||
|
|
||||||
section "Systemd Service"
|
|
||||||
if [ -f "$SERVICE_FILE" ]; then
|
|
||||||
info "already installed"
|
|
||||||
else
|
|
||||||
read -p " Not installed. Install now? (requires sudo) [y/N]: " _yn
|
|
||||||
if [[ "$_yn" =~ ^[Yy]$ ]]; then
|
|
||||||
step "Installing service"
|
|
||||||
sudo cp "$SCRIPT_DIR/dataflow.service" "$SERVICE_FILE" && ok \
|
|
||||||
|| fail "Could not install service"
|
|
||||||
|
|
||||||
step "Enabling on boot"
|
|
||||||
sudo systemctl daemon-reload && sudo systemctl enable dataflow > /dev/null 2>&1 && ok \
|
|
||||||
|| fail "Could not enable service"
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── API Server ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
if [ "$MODE" = "install" ] || [[ ! "$DO_RESTART" =~ ^[Nn]$ ]]; then
|
|
||||||
if confirm "API Server"; then
|
|
||||||
if [ -f "$SERVICE_FILE" ]; then
|
|
||||||
step "Restarting dataflow service"
|
|
||||||
sudo systemctl restart dataflow && sleep 1
|
|
||||||
systemctl is-active --quiet dataflow && ok \
|
|
||||||
|| fail "Service failed — check: journalctl -u dataflow -n 30"
|
|
||||||
else
|
|
||||||
info "systemd service not installed — start manually: node api/server.js"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
section "API Server"
|
|
||||||
info "skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
section "Done"
|
|
||||||
echo " API: http://localhost:${API_PORT:-3020}"
|
|
||||||
[ -n "$NGINX_DOMAIN" ] && echo " Web: https://$NGINX_DOMAIN"
|
|
||||||
echo " Logs: journalctl -u dataflow -f"
|
|
||||||
echo ""
|
|
||||||
29
manage.py
29
manage.py
@ -18,6 +18,17 @@ SERVICE_FILE = Path('/etc/systemd/system/dataflow.service')
|
|||||||
SERVICE_SRC = ROOT / 'dataflow.service'
|
SERVICE_SRC = ROOT / 'dataflow.service'
|
||||||
NGINX_DIR = Path('/etc/nginx/sites-enabled')
|
NGINX_DIR = Path('/etc/nginx/sites-enabled')
|
||||||
|
|
||||||
|
# Deployed in order — stacks.sql creates tables that reference sources
|
||||||
|
QUERIES_DIR = ROOT / 'database' / 'queries'
|
||||||
|
QUERY_FILES = [
|
||||||
|
QUERIES_DIR / 'sources.sql',
|
||||||
|
QUERIES_DIR / 'rules.sql',
|
||||||
|
QUERIES_DIR / 'mappings.sql',
|
||||||
|
QUERIES_DIR / 'records.sql',
|
||||||
|
QUERIES_DIR / 'stacks.sql',
|
||||||
|
QUERIES_DIR / 'status.sql',
|
||||||
|
]
|
||||||
|
|
||||||
# ── Terminal helpers ──────────────────────────────────────────────────────────
|
# ── Terminal helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
BOLD = '\033[1m'
|
BOLD = '\033[1m'
|
||||||
@ -332,13 +343,8 @@ def action_configure(cfg):
|
|||||||
|
|
||||||
db_location = f'database "{new_cfg["DB_NAME"]}" on {new_cfg["DB_HOST"]}:{new_cfg["DB_PORT"]}'
|
db_location = f'database "{new_cfg["DB_NAME"]}" on {new_cfg["DB_HOST"]}:{new_cfg["DB_PORT"]}'
|
||||||
schema_file = ROOT / 'database' / 'schema.sql'
|
schema_file = ROOT / 'database' / 'schema.sql'
|
||||||
queries_dir = ROOT / 'database' / 'queries'
|
queries_dir = QUERIES_DIR
|
||||||
query_files = [
|
query_files = QUERY_FILES
|
||||||
queries_dir / 'sources.sql',
|
|
||||||
queries_dir / 'rules.sql',
|
|
||||||
queries_dir / 'mappings.sql',
|
|
||||||
queries_dir / 'records.sql',
|
|
||||||
]
|
|
||||||
|
|
||||||
# Offer schema deployment
|
# Offer schema deployment
|
||||||
print()
|
print()
|
||||||
@ -433,13 +439,8 @@ def action_deploy_functions(cfg):
|
|||||||
return
|
return
|
||||||
|
|
||||||
db_location = f'database "{cfg["DB_NAME"]}" on {cfg["DB_HOST"]}:{cfg["DB_PORT"]}'
|
db_location = f'database "{cfg["DB_NAME"]}" on {cfg["DB_HOST"]}:{cfg["DB_PORT"]}'
|
||||||
queries_dir = ROOT / 'database' / 'queries'
|
queries_dir = QUERIES_DIR
|
||||||
query_files = [
|
query_files = QUERY_FILES
|
||||||
queries_dir / 'sources.sql',
|
|
||||||
queries_dir / 'rules.sql',
|
|
||||||
queries_dir / 'mappings.sql',
|
|
||||||
queries_dir / 'records.sql',
|
|
||||||
]
|
|
||||||
|
|
||||||
print(f' Source files: {queries_dir}/')
|
print(f' Source files: {queries_dir}/')
|
||||||
for f in query_files:
|
for f in query_files:
|
||||||
|
|||||||
@ -1 +0,0 @@
|
|||||||
select id, source, constrain_key, data from dataflow.records
|
|
||||||
@ -1,62 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Reimport dcard records from ubm.tps.trans into dataflow.records
|
|
||||||
#
|
|
||||||
# Step 1: exports raw rec JSON from ubm
|
|
||||||
# Step 2: wipes existing dcard data in dataflow and reloads from the export
|
|
||||||
#
|
|
||||||
# Usage: bash migrate/reimport_dcard_from_tps.sh
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
EXPORT_FILE="/tmp/tps_dcard_rec.csv"
|
|
||||||
echo "==> Exporting dcard from ubm.tps.trans..."
|
|
||||||
psql -U ptrowbridge -d ubm -p 54329 -h hptrow.me -c "\COPY (SELECT rec FROM tps.trans WHERE srce = 'dcard' ORDER BY id) TO '${EXPORT_FILE}' CSV"
|
|
||||||
echo " Exported $(wc -l < ${EXPORT_FILE}) rows"
|
|
||||||
|
|
||||||
echo "==> Reimporting into dataflow.records..."
|
|
||||||
$PG -d dataflow <<SQL
|
|
||||||
BEGIN;
|
|
||||||
|
|
||||||
-- Wipe existing dcard records (FK cascade deletes records too)
|
|
||||||
DELETE FROM dataflow.import_log WHERE source_name = 'dcard';
|
|
||||||
|
|
||||||
-- Staging table for the exported rec JSON
|
|
||||||
CREATE TEMP TABLE _dcard_import (rec jsonb);
|
|
||||||
\COPY _dcard_import FROM '${EXPORT_FILE}' CSV
|
|
||||||
|
|
||||||
-- New import_log entry
|
|
||||||
INSERT INTO dataflow.import_log (source_name, records_imported, records_duplicate)
|
|
||||||
VALUES ('dcard', 0, 0);
|
|
||||||
|
|
||||||
-- Insert records; constraint_key matches source constraint_fields:
|
|
||||||
-- {"Trans. Date","Post Date",Description}
|
|
||||||
WITH new_import AS (
|
|
||||||
SELECT id AS import_id FROM dataflow.import_log
|
|
||||||
WHERE source_name = 'dcard'
|
|
||||||
ORDER BY id DESC LIMIT 1
|
|
||||||
),
|
|
||||||
inserted AS (
|
|
||||||
INSERT INTO dataflow.records (source_name, data, transformed, constraint_key, import_id)
|
|
||||||
SELECT
|
|
||||||
'dcard',
|
|
||||||
s.rec,
|
|
||||||
NULL,
|
|
||||||
jsonb_build_object(
|
|
||||||
'Trans. Date', s.rec->>'Trans. Date',
|
|
||||||
'Post Date', s.rec->>'Post Date',
|
|
||||||
'Description', s.rec->>'Description'
|
|
||||||
),
|
|
||||||
i.import_id
|
|
||||||
FROM _dcard_import s, new_import i
|
|
||||||
RETURNING id
|
|
||||||
)
|
|
||||||
UPDATE dataflow.import_log
|
|
||||||
SET records_imported = (SELECT COUNT(*) FROM inserted)
|
|
||||||
WHERE source_name = 'dcard'
|
|
||||||
AND id = (SELECT id FROM dataflow.import_log WHERE source_name = 'dcard' ORDER BY id DESC LIMIT 1);
|
|
||||||
|
|
||||||
COMMIT;
|
|
||||||
SELECT records_imported FROM dataflow.import_log WHERE source_name = 'dcard' ORDER BY id DESC LIMIT 1;
|
|
||||||
SQL
|
|
||||||
|
|
||||||
echo "==> Done. Run transformations to repopulate the transformed column."
|
|
||||||
@ -4,8 +4,8 @@
|
|||||||
"description": "Simple data transformation tool for ingesting, mapping, and transforming data",
|
"description": "Simple data transformation tool for ingesting, mapping, and transforming data",
|
||||||
"main": "api/server.js",
|
"main": "api/server.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "nodemon api/server.js",
|
"start": "node api/server.js",
|
||||||
"dev": "node api/server.js",
|
"dev": "nodemon api/server.js",
|
||||||
"test": "echo \"Tests coming soon\" && exit 0"
|
"test": "echo \"Tests coming soon\" && exit 0"
|
||||||
},
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
|
|||||||
@ -1,38 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SERVICE_NAME="dataflow"
|
|
||||||
SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
|
|
||||||
WORKDIR=$(pwd)
|
|
||||||
|
|
||||||
echo "Creating systemd service file..."
|
|
||||||
sudo tee "$SERVICE_FILE" > /dev/null <<EOF
|
|
||||||
[Unit]
|
|
||||||
Description=Dataflow API Server
|
|
||||||
After=postgresql.service
|
|
||||||
Wants=postgresql.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=www-data
|
|
||||||
WorkingDirectory=${WORKDIR}
|
|
||||||
Environment=NODE_ENV=production
|
|
||||||
ExecStart=/usr/bin/node api/server.js
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
EOF
|
|
||||||
|
|
||||||
echo "Reloading systemd daemon..."
|
|
||||||
sudo systemctl daemon-reload
|
|
||||||
|
|
||||||
echo "Enabling dataflow service..."
|
|
||||||
sudo systemctl enable "$SERVICE_NAME"
|
|
||||||
|
|
||||||
echo "Starting dataflow service..."
|
|
||||||
sudo systemctl start "$SERVICE_NAME"
|
|
||||||
|
|
||||||
echo "Done! Service status:"
|
|
||||||
sudo systemctl status "$SERVICE_NAME" --no-pager
|
|
||||||
@ -1 +0,0 @@
|
|||||||
/* App-level styles — layout handled by Tailwind */
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 44 KiB |
@ -1 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
|
|
||||||
|
Before Width: | Height: | Size: 4.0 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 8.5 KiB |
Loading…
Reference in New Issue
Block a user