/** * Mappings Routes * Manage value mappings */ const express = require('express'); const multer = require('multer'); const { parse } = require('csv-parse/sync'); const upload = multer({ storage: multer.memoryStorage() }); const SYSTEM_COLS = new Set(['source_name', 'rule_name', 'input_value', 'record_count', 'sample']); module.exports = (pool) => { const router = express.Router(); // List all mappings for a source router.get('/source/:source_name', async (req, res, next) => { try { const { rule_name } = req.query; let query = 'SELECT * FROM mappings WHERE source_name = $1'; const params = [req.params.source_name]; if (rule_name) { query += ' AND rule_name = $2'; params.push(rule_name); } query += ' ORDER BY rule_name, input_value'; const result = await pool.query(query, params); res.json(result.rows); } catch (err) { next(err); } }); // Get record counts for existing mappings router.get('/source/:source_name/counts', async (req, res, next) => { try { const { rule_name } = req.query; const params = [req.params.source_name]; let ruleFilter = ''; if (rule_name) { ruleFilter = 'AND m.rule_name = $2'; params.push(rule_name); } const result = await pool.query(` SELECT m.rule_name, m.input_value, COUNT(rec.id) AS record_count FROM mappings m JOIN rules r ON r.source_name = m.source_name AND r.name = m.rule_name LEFT JOIN records rec ON rec.source_name = m.source_name AND rec.transformed ? r.output_field AND rec.transformed -> r.output_field = m.input_value WHERE m.source_name = $1 ${ruleFilter} GROUP BY m.rule_name, m.input_value `, params); res.json(result.rows); } catch (err) { next(err); } }); // Get unmapped values router.get('/source/:source_name/unmapped', async (req, res, next) => { try { const { rule_name } = req.query; const result = await pool.query( 'SELECT * FROM get_unmapped_values($1, $2)', [req.params.source_name, rule_name || null] ); res.json(result.rows); } catch (err) { next(err); } }); // Get all extracted values (mapped + unmapped) with counts — single SQL run router.get('/source/:source_name/all-values', async (req, res, next) => { try { const { rule_name } = req.query; const result = await pool.query( 'SELECT * FROM get_all_values($1, $2)', [req.params.source_name, rule_name || null] ); res.json(result.rows); } catch (err) { next(err); } }); // Export all extracted values (mapped + unmapped) as TSV via get_all_values // Columns: source_name, rule_name, input_value, record_count, , sample // sample is always last and is discarded on import router.get('/source/:source_name/export.tsv', async (req, res, next) => { try { const { rule_name } = req.query; const source_name = req.params.source_name; const result = await pool.query( 'SELECT * FROM get_all_values($1, $2)', [source_name, rule_name || null] ); // Collect output keys from mapped rows const outputKeys = []; for (const row of result.rows) { for (const key of Object.keys(row.output || {})) { if (!outputKeys.includes(key)) outputKeys.push(key); } } const escape = (val) => String(val ?? '').replace(/\t/g, ' '); // sample is always last const allCols = ['source_name', 'rule_name', 'input_value', 'record_count', ...outputKeys, 'sample']; const dataRows = result.rows.map(row => { const input_value = Array.isArray(row.extracted_value) ? JSON.stringify(row.extracted_value) : String(row.extracted_value ?? ''); const sample = Array.isArray(row.sample) ? row.sample.map(r => r[row.source_field] ?? '').filter(Boolean).slice(0, 3).join(' | ') : String(row.sample ?? ''); const r = { source_name, rule_name: row.rule_name, input_value, record_count: row.record_count, sample }; for (const key of outputKeys) r[key] = row.output?.[key] ?? ''; return r; }); const tsv = [ allCols.map(escape).join('\t'), ...dataRows.map(r => allCols.map(c => escape(r[c])).join('\t')) ].join('\n'); res.setHeader('Content-Type', 'text/tab-separated-values'); res.setHeader('Content-Disposition', `attachment; filename="mappings_${source_name}.tsv"`); res.send(tsv); } catch (err) { next(err); } }); // Import mappings from uploaded TSV // Any column that isn't a system field (source_name, rule_name, input_value, record_count, sample) // is treated as an output key. sample is discarded wherever it appears. router.post('/source/:source_name/import-csv', upload.single('file'), async (req, res, next) => { const client = await pool.connect(); try { if (!req.file) { return res.status(400).json({ error: 'No file uploaded. Send TSV as multipart field named "file".' }); } const records = parse(req.file.buffer, { columns: true, skip_empty_lines: true, trim: true, delimiter: '\t' }); if (records.length === 0) { return res.status(400).json({ error: 'File is empty.' }); } const outputKeys = Object.keys(records[0]).filter(k => !SYSTEM_COLS.has(k)); const mappings = []; for (const row of records) { const { source_name, rule_name, input_value } = row; const output = {}; for (const key of outputKeys) { if (row[key] && row[key].trim() !== '') output[key] = row[key].trim(); } if (Object.keys(output).length === 0) continue; let parsedInput; try { parsedInput = JSON.parse(input_value); } catch { parsedInput = input_value; } mappings.push({ source_name, rule_name, input_value: parsedInput, output }); } if (mappings.length === 0) { return res.status(400).json({ error: 'No rows with output values filled in.' }); } await client.query('BEGIN'); const results = []; for (const { source_name, rule_name, input_value, output } of mappings) { const result = await client.query( `INSERT INTO mappings (source_name, rule_name, input_value, output) VALUES ($1, $2, $3, $4) ON CONFLICT (source_name, rule_name, input_value) DO UPDATE SET output = EXCLUDED.output RETURNING *`, [source_name, rule_name, JSON.stringify(input_value), JSON.stringify(output)] ); results.push(result.rows[0]); } await client.query('COMMIT'); res.status(201).json({ count: results.length, mappings: results }); } catch (err) { await client.query('ROLLBACK'); next(err); } finally { client.release(); } }); // Get single mapping router.get('/:id', async (req, res, next) => { try { const result = await pool.query( 'SELECT * FROM mappings WHERE id = $1', [req.params.id] ); if (result.rows.length === 0) { return res.status(404).json({ error: 'Mapping not found' }); } res.json(result.rows[0]); } catch (err) { next(err); } }); // Create mapping router.post('/', async (req, res, next) => { try { const { source_name, rule_name, input_value, output } = req.body; if (!source_name || !rule_name || !input_value || !output) { return res.status(400).json({ error: 'Missing required fields: source_name, rule_name, input_value, output' }); } const result = await pool.query( `INSERT INTO mappings (source_name, rule_name, input_value, output) VALUES ($1, $2, $3, $4) RETURNING *`, [source_name, rule_name, JSON.stringify(input_value), JSON.stringify(output)] ); res.status(201).json(result.rows[0]); } catch (err) { if (err.code === '23505') { // Unique violation return res.status(409).json({ error: 'Mapping already exists' }); } if (err.code === '23503') { // Foreign key violation return res.status(404).json({ error: 'Source or rule not found' }); } next(err); } }); // Bulk create mappings router.post('/bulk', async (req, res, next) => { const client = await pool.connect(); try { const { mappings } = req.body; if (!Array.isArray(mappings)) { return res.status(400).json({ error: 'Expected array of mappings' }); } await client.query('BEGIN'); const results = []; for (const mapping of mappings) { const { source_name, rule_name, input_value, output } = mapping; const result = await client.query( `INSERT INTO mappings (source_name, rule_name, input_value, output) VALUES ($1, $2, $3, $4) ON CONFLICT (source_name, rule_name, input_value) DO UPDATE SET output = EXCLUDED.output RETURNING *`, [source_name, rule_name, JSON.stringify(input_value), JSON.stringify(output)] ); results.push(result.rows[0]); } await client.query('COMMIT'); res.status(201).json({ count: results.length, mappings: results }); } catch (err) { await client.query('ROLLBACK'); next(err); } finally { client.release(); } }); // Update mapping router.put('/:id', async (req, res, next) => { try { const { input_value, output } = req.body; const result = await pool.query( `UPDATE mappings SET input_value = COALESCE($2, input_value), output = COALESCE($3, output) WHERE id = $1 RETURNING *`, [req.params.id, input_value, output ? JSON.stringify(output) : null] ); if (result.rows.length === 0) { return res.status(404).json({ error: 'Mapping not found' }); } res.json(result.rows[0]); } catch (err) { next(err); } }); // Delete mapping router.delete('/:id', async (req, res, next) => { try { const result = await pool.query( 'DELETE FROM mappings WHERE id = $1 RETURNING id', [req.params.id] ); if (result.rows.length === 0) { return res.status(404).json({ error: 'Mapping not found' }); } res.json({ success: true, deleted: result.rows[0].id }); } catch (err) { next(err); } }); return router; };