dataflow/api/routes/mappings.js

/**
 * Mappings Routes
 * Manage value mappings
 */

const express = require('express');
const multer = require('multer');
const { parse } = require('csv-parse/sync');
const { lit, jsonLit } = require('../lib/sql');

const upload = multer({ storage: multer.memoryStorage() });

const SYSTEM_COLS = new Set(['source_name', 'rule_name', 'input_value', 'record_count', 'sample']);

module.exports = (pool) => {
    const router = express.Router();

    // List all mappings for a source
    router.get('/source/:source_name', async (req, res, next) => {
        try {
            const result = await pool.query(
                `SELECT * FROM list_mappings(${lit(req.params.source_name)}, ${lit(req.query.rule_name || null)})`
            );
            res.json(result.rows);
        } catch (err) {
            next(err);
        }
    });

    // Get record counts for existing mappings
    router.get('/source/:source_name/counts', async (req, res, next) => {
        try {
            const result = await pool.query(
                `SELECT * FROM get_mapping_counts(${lit(req.params.source_name)}, ${lit(req.query.rule_name || null)})`
            );
            res.json(result.rows);
        } catch (err) {
            next(err);
        }
    });

    // Get unmapped values
    router.get('/source/:source_name/unmapped', async (req, res, next) => {
        try {
            const result = await pool.query(
                `SELECT * FROM get_unmapped_values(${lit(req.params.source_name)}, ${lit(req.query.rule_name || null)})`
            );
            res.json(result.rows);
        } catch (err) {
            next(err);
        }
    });

    // Get all extracted values (mapped + unmapped) with counts
    router.get('/source/:source_name/all-values', async (req, res, next) => {
        try {
            const result = await pool.query(
                `SELECT * FROM get_all_values(${lit(req.params.source_name)}, ${lit(req.query.rule_name || null)})`
            );
            res.json(result.rows);
        } catch (err) {
            next(err);
        }
    });

    // Export all extracted values as TSV
    router.get('/source/:source_name/export.tsv', async (req, res, next) => {
        try {
            const { rule_name } = req.query;
            const source_name = req.params.source_name;

            const result = await pool.query(
                `SELECT * FROM get_all_values(${lit(source_name)}, ${lit(rule_name || null)})`
            );

            const outputKeys = [];
            for (const row of result.rows) {
                for (const key of Object.keys(row.output || {})) {
                    if (!outputKeys.includes(key)) outputKeys.push(key);
                }
            }

            const escape = (val) => String(val ?? '').replace(/\t/g, ' ');
            const allCols = ['source_name', 'rule_name', 'input_value', 'record_count', ...outputKeys, 'sample'];

            const dataRows = result.rows.map(row => {
                const input_value = Array.isArray(row.extracted_value)
                    ? JSON.stringify(row.extracted_value)
                    : String(row.extracted_value ?? '');
                const sample = Array.isArray(row.sample)
                    ? row.sample.map(r => r[row.source_field] ?? '').filter(Boolean).slice(0, 3).join(' | ')
                    : String(row.sample ?? '');
                const r = { source_name, rule_name: row.rule_name, input_value, record_count: row.record_count, sample };
                for (const key of outputKeys) r[key] = row.output?.[key] ?? '';
                return r;
            });

            const tsv = [
                allCols.map(escape).join('\t'),
                ...dataRows.map(r => allCols.map(c => escape(r[c])).join('\t'))
            ].join('\n');

            res.setHeader('Content-Type', 'text/tab-separated-values');
            res.setHeader('Content-Disposition', `attachment; filename="mappings_${source_name}.tsv"`);
            res.send(tsv);
        } catch (err) {
            next(err);
        }
    });

    // Import mappings from uploaded TSV
    router.post('/source/:source_name/import-csv', upload.single('file'), async (req, res, next) => {
        const client = await pool.connect();
        try {
            if (!req.file) {
                return res.status(400).json({ error: 'No file uploaded. Send TSV as multipart field named "file".' });
            }

            const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true, delimiter: '\t' });
            if (records.length === 0) return res.status(400).json({ error: 'File is empty.' });

            const outputKeys = Object.keys(records[0]).filter(k => !SYSTEM_COLS.has(k));
            const mappings = [];
            for (const row of records) {
                const { source_name, rule_name, input_value } = row;
                const output = {};
                for (const key of outputKeys) {
                    if (row[key] && row[key].trim() !== '') output[key] = row[key].trim();
                }
                if (Object.keys(output).length === 0) continue;
                let parsedInput;
                try { parsedInput = JSON.parse(input_value); } catch { parsedInput = input_value; }
                mappings.push({ source_name, rule_name, input_value: parsedInput, output });
            }

            if (mappings.length === 0) return res.status(400).json({ error: 'No rows with output values filled in.' });

            await client.query('BEGIN');
            const results = [];
            for (const { source_name, rule_name, input_value, output } of mappings) {
                const result = await client.query(
                    `SELECT * FROM upsert_mapping(${lit(source_name)}, ${lit(rule_name)}, ${jsonLit(input_value)}, ${jsonLit(output)})`
                );
                results.push(result.rows[0]);
            }
            await client.query('COMMIT');

            res.status(201).json({ count: results.length, mappings: results });
        } catch (err) {
            await client.query('ROLLBACK');
            next(err);
        } finally {
            client.release();
        }
    });

    // Get single mapping
    router.get('/:id', async (req, res, next) => {
        try {
            const result = await pool.query(`SELECT * FROM get_mapping(${lit(parseInt(req.params.id))})`);
            if (result.rows.length === 0) return res.status(404).json({ error: 'Mapping not found' });
            res.json(result.rows[0]);
        } catch (err) {
            next(err);
        }
    });

    // Create mapping
    router.post('/', async (req, res, next) => {
        try {
            const { source_name, rule_name, input_value, output } = req.body;
            if (!source_name || !rule_name || !input_value || !output) {
                return res.status(400).json({ error: 'Missing required fields: source_name, rule_name, input_value, output' });
            }
            const result = await pool.query(
                `SELECT * FROM create_mapping(${lit(source_name)}, ${lit(rule_name)}, ${jsonLit(input_value)}, ${jsonLit(output)})`
            );
            res.status(201).json(result.rows[0]);
        } catch (err) {
            if (err.code === '23505') return res.status(409).json({ error: 'Mapping already exists' });
            if (err.code === '23503') return res.status(404).json({ error: 'Source or rule not found' });
            next(err);
        }
    });

    // Bulk create/update mappings
    router.post('/bulk', async (req, res, next) => {
        const client = await pool.connect();
        try {
            const { mappings } = req.body;
            if (!Array.isArray(mappings)) return res.status(400).json({ error: 'Expected array of mappings' });

            await client.query('BEGIN');
            const results = [];
            for (const { source_name, rule_name, input_value, output } of mappings) {
                const result = await client.query(
                    `SELECT * FROM upsert_mapping(${lit(source_name)}, ${lit(rule_name)}, ${jsonLit(input_value)}, ${jsonLit(output)})`
                );
                results.push(result.rows[0]);
            }
            await client.query('COMMIT');
            res.status(201).json({ count: results.length, mappings: results });
        } catch (err) {
            await client.query('ROLLBACK');
            next(err);
        } finally {
            client.release();
        }
    });

    // Update mapping
    router.put('/:id', async (req, res, next) => {
        try {
            const { input_value, output } = req.body;
            const result = await pool.query(
                `SELECT * FROM update_mapping(${lit(parseInt(req.params.id))}, ${input_value != null ? jsonLit(input_value) : 'NULL'}, ${output != null ? jsonLit(output) : 'NULL'})`
            );
            if (result.rows.length === 0) return res.status(404).json({ error: 'Mapping not found' });
            res.json(result.rows[0]);
        } catch (err) {
            next(err);
        }
    });

    // Delete mapping
    router.delete('/:id', async (req, res, next) => {
        try {
            const result = await pool.query(`SELECT * FROM delete_mapping(${lit(parseInt(req.params.id))})`);
            if (result.rows.length === 0) return res.status(404).json({ error: 'Mapping not found' });
            res.json({ success: true, deleted: result.rows[0].id });
        } catch (err) {
            next(err);
        }
    });

    return router;
};