From d9087da9a10cd35a6da2b1b939d2dee3da634bd8 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 26 Jun 2018 00:33:07 -0400 Subject: [PATCH] dont bake parameters directly into sql, clean up a little --- server.js | 108 ++++++++++++++++++------------------------------------ 1 file changed, 36 insertions(+), 72 deletions(-) diff --git a/server.js b/server.js index 2ebeaab..c39b334 100644 --- a/server.js +++ b/server.js @@ -27,12 +27,12 @@ var Postgres = new pg.Client({ host: process.env.host, port: process.env.port, database: process.env.database, - application_name: "tps_etl_api", - ssl: true + ssl: false, + application_name: "tps_etl_api" }); -Postgres.FirstRow = function(inSQL, inResponse) +Postgres.FirstRow = function(inSQL,args, inResponse) { - Postgres.query(inSQL, (err, res) => { + Postgres.query(inSQL,args, (err, res) => { if (err === null) { inResponse.json(res.rows[0]); @@ -43,96 +43,66 @@ Postgres.FirstRow = function(inSQL, inResponse) }; Postgres.connect(); +//----------------------------------------------------------source definitions------------------------------------------------------------ +//returns array of all sources server.get("/source", function (inReq, inRes) { var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce"; - Postgres.FirstRow(sql, inRes); + Postgres.FirstRow(sql,[], inRes); }); +//returns message about status and error description server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build { - var sql = "SELECT x.message FROM tps.srce_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes); }); -//-------------------------------------------------------------list maps-------------------------------------------------------------------------- -server.get("/map_list", function (inReq, inRes) +//----------------------------------------------------------regex instractions-------------------------------------------------------------------------- +//list all regex operations +server.get("/regex", function (inReq, inRes) { var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm"; - Postgres.FirstRow(sql, inRes); + Postgres.FirstRow(sql, [], inRes); }); +//set one or more map definitions +server.post("/regex", bodyParser.json(), function (inReq, inRes) +{ + var sql = "SELECT x.message FROM tps.srce_map_def_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes); +}); + +//------------------------------------------------------------mappings------------------------------------------------------------------------------- + //list unmapped items flagged to be mapped ?srce= server.get("/unmapped", function (inReq, inRes) { - var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs('"+ inReq.query.srce + "') x"; - Postgres.FirstRow(sql, inRes); -}); - - -//set one or more map definitions -server.post("/mapdef_set", bodyParser.json(), function (inReq, inRes) -{ - var sql = "SELECT x.message FROM tps.srce_map_def_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs($1::text) x"; + Postgres.FirstRow(sql,[inReq.query.srce], inRes); }); //add entries to lookup table -server.post("/mapval_set", bodyParser.json(), function (inReq, inRes) +server.post("/mapping", bodyParser.json(), function (inReq, inRes) { - var sql = "SELECT x.message FROM tps.map_rv_set($$" + JSON.stringify( inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT x.message FROM tps.map_rv_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql,[JSON.stringify( inReq.body)], inRes); }); -/* -send a csv with powershell: -wget -uri http://localhost/import -Method Post -InFile "C:\Users\fleet\Downloads\d.csv" -bash -curl -v -F upload=@//mnt/c/Users/fleet/Downloads/d.csv localhost/import -*/ //-------------------------------------------------------------import data-------------------------------------------------------------------------- server.use("/import", upload.single('upload'), function (inReq, inRes) { - //console.log(inReq.file); console.log("should have gotten file as post body here"); var csv = inReq.file.buffer.toString('utf8') - // create a new converter object - //var jobj = csvtojson.fromString(csv). //{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"} csvtojson({ flatKeys: "true" }).fromString(csv).then( (x) => { - //console.log(x); - //inRes.json(x); - - //push to db - var sql = "SELECT x.message FROM tps.srce_import($$"; - sql += inReq.query.srce; - sql += "$$, $$" - sql += JSON.stringify(x) - sql += "$$::jsonb) as x(message)" - console.log("sql for insert here"); - //console.log(sql); - - Postgres.query(sql, (err, res) => { - - //Postgres.end(); - - if (err === null) { - inRes.json(res.rows[0]); - Postgres.end(); - return; - } - inRes.json(err.message); - //Postgres.end(); - //handle error - } - ); + var sql = "SELECT x.message FROM tps.srce_import($1, $2::jsonb) as x(message)" + console.log(sql); + Postgres.FirstRow(sql, [inReq.query.srce, JSON.stringify(x)], inRes); } - //const jsonArray = csv().fromFile(csvFilePath); - //csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) }); - //validate the body contents before pushing to sql? ); } ); @@ -141,42 +111,36 @@ server.use("/import", upload.single('upload'), function (inReq, inRes) { server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) { - //console.log(inReq.file); console.log("should have gotten file as post body here"); var csv = inReq.file.buffer.toString('utf8') - // create a new converter object - //var jobj = csvtojson.fromString(csv). //{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"} csvtojson({ flatKeys: "true" }).fromString(csv).then( (x) => { - //console.log(x); - //inRes.json(x); - - //push to db var sug = {}; for (var key in x[0]) { + //test if number if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) { + //if is a number but leading character is -0- then it's text if (x[0][key].charAt(0) == "0"){ sug[key] = "text"; } + //if number and leadign character is not 0 then numeric else { sug[key] = "numeric"; } } + //if can cast to a date within a hundred years its probably a date else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) { sug[key] = "date"; } + //otherwise its text else { sug[key] = "text"; } } console.log(sug); inRes.json(sug); - //console.log(sql); } - //const jsonArray = csv().fromFile(csvFilePath); - //csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) }); - //validate the body contents before pushing to sql? ); } );