dont bake parameters directly into sql, clean up a little
This commit is contained in:
Paul Trowbridge
parent
645639203b
commit
d9087da9a1
108
server.js
108
server.js
@ -27,12 +27,12 @@ var Postgres = new pg.Client({
|
||||
host: process.env.host,
|
||||
port: process.env.port,
|
||||
database: process.env.database,
|
||||
application_name: "tps_etl_api",
|
||||
ssl: true
|
||||
ssl: false,
|
||||
application_name: "tps_etl_api"
|
||||
});
|
||||
Postgres.FirstRow = function(inSQL, inResponse)
|
||||
Postgres.FirstRow = function(inSQL,args, inResponse)
|
||||
{
|
||||
Postgres.query(inSQL, (err, res) => {
|
||||
Postgres.query(inSQL,args, (err, res) => {
|
||||
if (err === null)
|
||||
{
|
||||
inResponse.json(res.rows[0]);
|
||||
@ -43,96 +43,66 @@ Postgres.FirstRow = function(inSQL, inResponse)
|
||||
};
|
||||
Postgres.connect();
|
||||
|
||||
//----------------------------------------------------------source definitions------------------------------------------------------------
|
||||
|
||||
//returns array of all sources
|
||||
server.get("/source", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
Postgres.FirstRow(sql,[], inRes);
|
||||
});
|
||||
//returns message about status and error description
|
||||
server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.srce_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes);
|
||||
});
|
||||
|
||||
//-------------------------------------------------------------list maps--------------------------------------------------------------------------
|
||||
server.get("/map_list", function (inReq, inRes)
|
||||
//----------------------------------------------------------regex instractions--------------------------------------------------------------------------
|
||||
//list all regex operations
|
||||
server.get("/regex", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
Postgres.FirstRow(sql, [], inRes);
|
||||
});
|
||||
|
||||
//set one or more map definitions
|
||||
server.post("/regex", bodyParser.json(), function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.srce_map_def_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
|
||||
});
|
||||
|
||||
//------------------------------------------------------------mappings-------------------------------------------------------------------------------
|
||||
|
||||
//list unmapped items flagged to be mapped ?srce=
|
||||
server.get("/unmapped", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs('"+ inReq.query.srce + "') x";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
});
|
||||
|
||||
|
||||
//set one or more map definitions
|
||||
server.post("/mapdef_set", bodyParser.json(), function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.srce_map_def_set($$" + JSON.stringify(inReq.body) + "$$::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs($1::text) x";
|
||||
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
|
||||
});
|
||||
|
||||
//add entries to lookup table
|
||||
server.post("/mapval_set", bodyParser.json(), function (inReq, inRes)
|
||||
server.post("/mapping", bodyParser.json(), function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.map_rv_set($$" + JSON.stringify( inReq.body) + "$$::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql, inRes);
|
||||
var sql = "SELECT x.message FROM tps.map_rv_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql,[JSON.stringify( inReq.body)], inRes);
|
||||
});
|
||||
|
||||
/*
|
||||
send a csv with powershell:
|
||||
wget -uri http://localhost/import -Method Post -InFile "C:\Users\fleet\Downloads\d.csv"
|
||||
bash
|
||||
curl -v -F upload=@//mnt/c/Users/fleet/Downloads/d.csv localhost/import
|
||||
*/
|
||||
|
||||
//-------------------------------------------------------------import data--------------------------------------------------------------------------
|
||||
|
||||
server.use("/import", upload.single('upload'), function (inReq, inRes) {
|
||||
|
||||
//console.log(inReq.file);
|
||||
console.log("should have gotten file as post body here");
|
||||
var csv = inReq.file.buffer.toString('utf8')
|
||||
// create a new converter object
|
||||
//var jobj = csvtojson.fromString(csv).
|
||||
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
|
||||
csvtojson({ flatKeys: "true" }).fromString(csv).then(
|
||||
(x) => {
|
||||
//console.log(x);
|
||||
//inRes.json(x);
|
||||
|
||||
//push to db
|
||||
var sql = "SELECT x.message FROM tps.srce_import($$";
|
||||
sql += inReq.query.srce;
|
||||
sql += "$$, $$"
|
||||
sql += JSON.stringify(x)
|
||||
sql += "$$::jsonb) as x(message)"
|
||||
console.log("sql for insert here");
|
||||
//console.log(sql);
|
||||
|
||||
Postgres.query(sql, (err, res) => {
|
||||
|
||||
//Postgres.end();
|
||||
|
||||
if (err === null) {
|
||||
inRes.json(res.rows[0]);
|
||||
Postgres.end();
|
||||
return;
|
||||
}
|
||||
inRes.json(err.message);
|
||||
//Postgres.end();
|
||||
//handle error
|
||||
}
|
||||
);
|
||||
var sql = "SELECT x.message FROM tps.srce_import($1, $2::jsonb) as x(message)"
|
||||
console.log(sql);
|
||||
Postgres.FirstRow(sql, [inReq.query.srce, JSON.stringify(x)], inRes);
|
||||
}
|
||||
//const jsonArray = csv().fromFile(csvFilePath);
|
||||
//csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) });
|
||||
//validate the body contents before pushing to sql?
|
||||
);
|
||||
}
|
||||
);
|
||||
@ -141,42 +111,36 @@ server.use("/import", upload.single('upload'), function (inReq, inRes) {
|
||||
|
||||
server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
|
||||
|
||||
//console.log(inReq.file);
|
||||
console.log("should have gotten file as post body here");
|
||||
var csv = inReq.file.buffer.toString('utf8')
|
||||
// create a new converter object
|
||||
//var jobj = csvtojson.fromString(csv).
|
||||
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
|
||||
csvtojson({ flatKeys: "true" }).fromString(csv).then(
|
||||
(x) => {
|
||||
//console.log(x);
|
||||
//inRes.json(x);
|
||||
|
||||
//push to db
|
||||
var sug = {};
|
||||
for (var key in x[0]) {
|
||||
//test if number
|
||||
if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) {
|
||||
//if is a number but leading character is -0- then it's text
|
||||
if (x[0][key].charAt(0) == "0"){
|
||||
sug[key] = "text";
|
||||
}
|
||||
//if number and leadign character is not 0 then numeric
|
||||
else {
|
||||
sug[key] = "numeric";
|
||||
}
|
||||
}
|
||||
//if can cast to a date within a hundred years its probably a date
|
||||
else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) {
|
||||
sug[key] = "date";
|
||||
}
|
||||
//otherwise its text
|
||||
else {
|
||||
sug[key] = "text";
|
||||
}
|
||||
}
|
||||
console.log(sug);
|
||||
inRes.json(sug);
|
||||
//console.log(sql);
|
||||
}
|
||||
//const jsonArray = csv().fromFile(csvFilePath);
|
||||
//csvtojson({ output: "csv" }).fromString(csv).then((jsonObj) => { console.log(jsonObj) });
|
||||
//validate the body contents before pushing to sql?
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user