From ebc296366ef8be2ac7780a6b27631f15359b3ba3 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 24 Jun 2018 14:12:34 -0400 Subject: [PATCH 1/5] CORS setup and FirstRow method added to postgres object --- package.json | 2 +- server.js | 183 ++++++++++++++++----------------------------------- 2 files changed, 58 insertions(+), 127 deletions(-) diff --git a/package.json b/package.json index 5bff3e5..9232dea 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "test": "node_modules/mocha/bin/mocha", - "start": "node index.js" + "start": "nodemon index.js" }, "author": "", "license": "ISC", diff --git a/server.js b/server.js index e7ca1f1..2ebeaab 100644 --- a/server.js +++ b/server.js @@ -11,12 +11,16 @@ var pg = require('pg'); var server = express(); server.engine('handlebars', handlebars()); server.set('view engine', 'handlebars'); -server.use(function(req, res, next) { - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - next(); + +server.use(function(inReq, inRes, inNext) +{ + inRes.header("Access-Control-Allow-Origin", "*"); + inRes.header("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS"); + inRes.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + inNext(); }); + var Postgres = new pg.Client({ user: process.env.user, password: process.env.password, @@ -26,133 +30,59 @@ var Postgres = new pg.Client({ application_name: "tps_etl_api", ssl: true }); - +Postgres.FirstRow = function(inSQL, inResponse) +{ + Postgres.query(inSQL, (err, res) => { + if (err === null) + { + inResponse.json(res.rows[0]); + return; + } + inResponse.json(err.message); + }); +}; Postgres.connect(); -//-------------------------------------------------------------list source-------------------------------------------------------------------------- -server.use("/srce_list", function (inReq, inRes) { - - var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce" - console.log(sql); - - Postgres.query(sql, (err, res) => { - inRes.json(res.rows[0]); - console.log("source list request complete"); - }); -} -); +server.get("/source", function (inReq, inRes) +{ + var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce"; + Postgres.FirstRow(sql, inRes); +}); +server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build +{ + var sql = "SELECT x.message FROM tps.srce_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; + Postgres.FirstRow(sql, inRes); +}); //-------------------------------------------------------------list maps-------------------------------------------------------------------------- +server.get("/map_list", function (inReq, inRes) +{ + var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm"; + Postgres.FirstRow(sql, inRes); +}); -server.use("/map_list", function (inReq, inRes) { +//list unmapped items flagged to be mapped ?srce= +server.get("/unmapped", function (inReq, inRes) +{ + var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs('"+ inReq.query.srce + "') x"; + Postgres.FirstRow(sql, inRes); +}); - var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm" - console.log(sql); - Postgres.query(sql, (err, res) => { +//set one or more map definitions +server.post("/mapdef_set", bodyParser.json(), function (inReq, inRes) +{ + var sql = "SELECT x.message FROM tps.srce_map_def_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; + Postgres.FirstRow(sql, inRes); +}); - if (err === null) { - inRes.json(res.rows[0]); - return; - } - inRes.json(err.message); - }); -} -); - -//--------------------------------------------------------list unmapped items flagged to be mapped--------------------------------------------------- - -server.use("/unmapped", function (inReq, inRes) { - - var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs('"; - sql += inReq.query.srce + "') x" - console.log(sql); - - Postgres.query(sql, (err, res) => { - - if (err === null) { - inRes.json(res.rows[0]); - return; - } - inRes.json(err.message); - }); -} -); - -//-------------------------------------------------------------set source via json in body-------------------------------------------------------------------------- - -server.use("/srce_set", bodyParser.json(), function (inReq, inRes) { - - //validate the body contents before pushing to sql? - var sql = "SELECT x.message FROM tps.srce_set($$"; - sql += JSON.stringify( inReq.body); - sql += "$$::jsonb) as x(message)"; - console.log(sql); - - Postgres.query(sql, (err, res) => { - - //Postgres.end(); - - if (err === null) { - inRes.json(res.rows[0]); - return; - } - inRes.json(err.message); - //handle error - }); -} -); - -//-------------------------------------------------------------set one or more map definitions-------------------------------------------------------------------------- - -server.use("/mapdef_set", bodyParser.json(), function (inReq, inRes) { - - //validate the body contents before pushing to sql? - var sql = "SELECT x.message FROM tps.srce_map_def_set($$"; - sql += JSON.stringify( inReq.body); - sql += "$$::jsonb) as x(message)"; - console.log(sql); - - Postgres.query(sql, (err, res) => { - - //Postgres.end(); - - if (err === null) { - inRes.json(res.rows[0]); - return; - } - inRes.json(err.message); - //handle error - }); - -} -); - -//-------------------------------------------------------------add entries to lookup table-------------------------------------------------------------------------- - -server.use("/mapval_set", bodyParser.json(), function (inReq, inRes) { - - //validate the body contents before pushing to sql? - var sql = "SELECT x.message FROM tps.map_rv_set($$"; - sql += JSON.stringify( inReq.body); - sql += "$$::jsonb) as x(message)"; - console.log(sql); - - Postgres.query(sql, (err, res) => { - - //Postgres.end(); - - if (err === null) { - inRes.json(res.rows[0]); - return; - } - inRes.json(err.message); - //handle error - }); - -} -); +//add entries to lookup table +server.post("/mapval_set", bodyParser.json(), function (inReq, inRes) +{ + var sql = "SELECT x.message FROM tps.map_rv_set($$" + JSON.stringify( inReq.body) + "$$::jsonb) as x(message)"; + Postgres.FirstRow(sql, inRes); +}); /* send a csv with powershell: @@ -252,8 +182,9 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) { ); - server.get("/", function (inReq, inRes) { - inRes.render("definition", { title: "definition", layout: "main" }); - }) +server.get("/", function (inReq, inRes) +{ + inRes.render("definition", { title: "definition", layout: "main" }); +}); - module.exports = server; \ No newline at end of file +module.exports = server; \ No newline at end of file From bf4a867ddbddf472a1ed90a20c1542647d354775 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 26 Jun 2018 00:00:13 -0400 Subject: [PATCH 2/5] add sample source --- test/dcard_source/srce.json | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/dcard_source/srce.json diff --git a/test/dcard_source/srce.json b/test/dcard_source/srce.json new file mode 100644 index 0000000..bea9cbd --- /dev/null +++ b/test/dcard_source/srce.json @@ -0,0 +1,76 @@ +{ + "name": "dcard", + "source": "client_file", + "loading_function": "csv", + "constraint": [ + "{Trans. Date}", + "{Post Date}", + "{Description}" + ], + "schemas": { + "default": [ + { + "path": "{Trans. Date}", + "type": "date", + "column_name": "Trans. Date" + }, + { + "path": "{Post Date}", + "type": "date", + "column_name": "Post Date" + }, + { + "path": "{Description}", + "type": "text", + "column_name": "Description" + }, + { + "path": "{Amount}", + "type": "numeric", + "column_name": "Amount" + }, + { + "path": "{Category}", + "type": "text", + "column_name": "Category" + } + ], + "mapped": [ + { + "path": "{Trans. Date}", + "type": "date", + "column_name": "Trans. Date" + }, + { + "path": "{Post Date}", + "type": "date", + "column_name": "Post Date" + }, + { + "path": "{Description}", + "type": "text", + "column_name": "Description" + }, + { + "path": "{Amount}", + "type": "numeric", + "column_name": "Amount" + }, + { + "path": "{Category}", + "type": "text", + "column_name": "Category" + }, + { + "path": "{party}", + "type": "text", + "column_name": "Party" + }, + { + "path": "{reason}", + "type": "text", + "column_name": "Reason" + } + ] + } +} \ No newline at end of file From 645639203be464eb76b62c94020ac2c8ca092284 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 26 Jun 2018 00:13:48 -0400 Subject: [PATCH 3/5] add tests --- test/dcard_regex/curl | 1 + test/dcard_regex/regex.json | 24 ++++++++++++++++++++++++ test/dcard_source/curl | 1 + 3 files changed, 26 insertions(+) create mode 100644 test/dcard_regex/curl create mode 100644 test/dcard_regex/regex.json create mode 100644 test/dcard_source/curl diff --git a/test/dcard_regex/curl b/test/dcard_regex/curl new file mode 100644 index 0000000..07372e6 --- /dev/null +++ b/test/dcard_regex/curl @@ -0,0 +1 @@ +curl -H "Content-Type: application/json" -X POST -d@./regex.json http://localhost/regex \ No newline at end of file diff --git a/test/dcard_regex/regex.json b/test/dcard_regex/regex.json new file mode 100644 index 0000000..4a17a95 --- /dev/null +++ b/test/dcard_regex/regex.json @@ -0,0 +1,24 @@ +[ + { + "regex": { + "function": "extract", + "description": "pull first 20 characters from description for mapping", + "where": [ + {} + ], + "defn": [ + { + "regex": ".{1,20}", + "map": "y", + "field": "f20", + "flag": "", + "key": "{Description}", + "retain": "y" + } + ] + }, + "sequence": 2, + "name": "First 20", + "srce": "dcard" + } +] \ No newline at end of file diff --git a/test/dcard_source/curl b/test/dcard_source/curl new file mode 100644 index 0000000..b92fbae --- /dev/null +++ b/test/dcard_source/curl @@ -0,0 +1 @@ +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/srce_set \ No newline at end of file From d9087da9a10cd35a6da2b1b939d2dee3da634bd8 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 26 Jun 2018 00:33:07 -0400 Subject: [PATCH 4/5] dont bake parameters directly into sql, clean up a little --- server.js | 108 ++++++++++++++++++------------------------------------ 1 file changed, 36 insertions(+), 72 deletions(-) diff --git a/server.js b/server.js index 2ebeaab..c39b334 100644 --- a/server.js +++ b/server.js @@ -27,12 +27,12 @@ var Postgres = new pg.Client({ host: process.env.host, port: process.env.port, database: process.env.database, - application_name: "tps_etl_api", - ssl: true + ssl: false, + application_name: "tps_etl_api" }); -Postgres.FirstRow = function(inSQL, inResponse) +Postgres.FirstRow = function(inSQL,args, inResponse) { - Postgres.query(inSQL, (err, res) => { + Postgres.query(inSQL,args, (err, res) => { if (err === null) { inResponse.json(res.rows[0]); @@ -43,96 +43,66 @@ Postgres.FirstRow = function(inSQL, inResponse) }; Postgres.connect(); +//----------------------------------------------------------source definitions------------------------------------------------------------ +//returns array of all sources server.get("/source", function (inReq, inRes) { var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce"; - Postgres.FirstRow(sql, inRes); + Postgres.FirstRow(sql,[], inRes); }); +//returns message about status and error description server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build { - var sql = "SELECT x.message FROM tps.srce_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes); }); -//-------------------------------------------------------------list maps-------------------------------------------------------------------------- -server.get("/map_list", function (inReq, inRes) +//----------------------------------------------------------regex instractions-------------------------------------------------------------------------- +//list all regex operations +server.get("/regex", function (inReq, inRes) { var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm"; - Postgres.FirstRow(sql, inRes); + Postgres.FirstRow(sql, [], inRes); }); +//set one or more map definitions +server.post("/regex", bodyParser.json(), function (inReq, inRes) +{ + var sql = "SELECT x.message FROM tps.srce_map_def_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes); +}); + +//------------------------------------------------------------mappings------------------------------------------------------------------------------- + //list unmapped items flagged to be mapped ?srce= server.get("/unmapped", function (inReq, inRes) { - var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs('"+ inReq.query.srce + "') x"; - Postgres.FirstRow(sql, inRes); -}); - - -//set one or more map definitions -server.post("/mapdef_set", bodyParser.json(), function (inReq, inRes) -{ - var sql = "SELECT x.message FROM tps.srce_map_def_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs($1::text) x"; + Postgres.FirstRow(sql,[inReq.query.srce], inRes); }); //add entries to lookup table -server.post("/mapval_set", bodyParser.json(), function (inReq, inRes) +server.post("/mapping", bodyParser.json(), function (inReq, inRes) { - var sql = "SELECT x.message FROM tps.map_rv_set($$" + JSON.stringify( inReq.body) + "$$::jsonb) as x(message)"; - Postgres.FirstRow(sql, inRes); + var sql = "SELECT x.message FROM tps.map_rv_set($1::jsonb) as x(message)"; + Postgres.FirstRow(sql,[JSON.stringify( inReq.body)], inRes); }); -/* -send a csv with powershell: -wget -uri http://localhost/import -Method Post -InFile "C:\Users\fleet\Downloads\d.csv" -bash -curl -v -F upload=@//mnt/c/Users/fleet/Downloads/d.csv localhost/import -*/ //-------------------------------------------------------------import data-------------------------------------------------------------------------- server.use("/import", upload.single('upload'), function (inReq, inRes) { - //console.log(inReq.file); console.log("should have gotten file as post body here"); var csv = inReq.file.buffer.toString('utf8') - // create a new converter object - //var jobj = csvtojson.fromString(csv). //{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"} csvtojson({ flatKeys: "true" }).fromString(csv).then( (x) => { - //console.log(x); - //inRes.json(x); - - //push to db - var sql = "SELECT x.message FROM tps.srce_import($$"; - sql += inReq.query.srce; - sql += "$$, $$" - sql += JSON.stringify(x) - sql += "$$::jsonb) as x(message)" - console.log("sql for insert here"); - //console.log(sql); - - Postgres.query(sql, (err, res) => { - - //Postgres.end(); - - if (err === null) { - inRes.json(res.rows[0]); - Postgres.end(); - return; - } - inRes.json(err.message); - //Postgres.end(); - //handle error - } - ); + var sql = "SELECT x.message FROM tps.srce_import($1, $2::jsonb) as x(message)" + console.log(sql); + Postgres.FirstRow(sql, [inReq.query.srce, JSON.stringify(x)], inRes); } - //const jsonArray = csv().fromFile(csvFilePath); - //csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) }); - //validate the body contents before pushing to sql? ); } ); @@ -141,42 +111,36 @@ server.use("/import", upload.single('upload'), function (inReq, inRes) { server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) { - //console.log(inReq.file); console.log("should have gotten file as post body here"); var csv = inReq.file.buffer.toString('utf8') - // create a new converter object - //var jobj = csvtojson.fromString(csv). //{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"} csvtojson({ flatKeys: "true" }).fromString(csv).then( (x) => { - //console.log(x); - //inRes.json(x); - - //push to db var sug = {}; for (var key in x[0]) { + //test if number if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) { + //if is a number but leading character is -0- then it's text if (x[0][key].charAt(0) == "0"){ sug[key] = "text"; } + //if number and leadign character is not 0 then numeric else { sug[key] = "numeric"; } } + //if can cast to a date within a hundred years its probably a date else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) { sug[key] = "date"; } + //otherwise its text else { sug[key] = "text"; } } console.log(sug); inRes.json(sug); - //console.log(sql); } - //const jsonArray = csv().fromFile(csvFilePath); - //csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) }); - //validate the body contents before pushing to sql? ); } ); From fe025f6bf3fb86c060c039411bd66c8edb3ad6f1 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 26 Jun 2018 01:12:16 -0400 Subject: [PATCH 5/5] make csv suggest return full source-type json --- server.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/server.js b/server.js index c39b334..38f1afd 100644 --- a/server.js +++ b/server.js @@ -116,27 +116,39 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) { //{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"} csvtojson({ flatKeys: "true" }).fromString(csv).then( (x) => { - var sug = {}; + var sug = { + schemas: { + default: [] + }, + loading_function: "csv", + source:"client_file", + name: "", + constraint: [] + }; for (var key in x[0]) { + var col = {}; //test if number if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) { //if is a number but leading character is -0- then it's text if (x[0][key].charAt(0) == "0"){ - sug[key] = "text"; + col["type"] = "text"; } //if number and leadign character is not 0 then numeric else { - sug[key] = "numeric"; + col["type"] = "numeric"; } } //if can cast to a date within a hundred years its probably a date else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) { - sug[key] = "date"; + col["type"] = "date"; } //otherwise its text else { - sug[key] = "text"; + col["type"] = "text"; } + col["path"] = "{" + key + "}"; + col["column_name"] = key; + sug.schemas.default.push(col); } console.log(sug); inRes.json(sug);