copy code from local instance
This commit is contained in:
Paul Trowbridge
parent
f0b1daa002
commit
90b65e2fed
183
server.js
183
server.js
@ -6,12 +6,16 @@ var https = require('https');
|
||||
var bodyParser = require('body-parser');
|
||||
const server = express();
|
||||
const pg = require('pg');
|
||||
var mult = require('multer');
|
||||
var upload = mult({ encoding: "utf8" });
|
||||
var csvtojson = require('csvtojson');
|
||||
|
||||
//---------read sql files into variables----------------
|
||||
var fs = require('fs');
|
||||
var readline = require('readline');
|
||||
//-------------------------------------------------------
|
||||
|
||||
|
||||
//---------------setup TLS------------------------------------
|
||||
var options = {
|
||||
key: fs.readFileSync(process.env.wd + 'key.pem'),
|
||||
cert: fs.readFileSync(process.env.wd + 'cert.pem'),
|
||||
@ -21,3 +25,180 @@ var options = {
|
||||
https.createServer(options,server).listen(process.env.nodeport, () => {
|
||||
console.log('started on '+ process.env.nodeport)
|
||||
});
|
||||
|
||||
|
||||
//---------------database connection-----------------------
|
||||
var Postgres = new pg.Client({
|
||||
user: process.env.user,
|
||||
password: process.env.password,
|
||||
host: process.env.host,
|
||||
port: process.env.port,
|
||||
database: process.env.database,
|
||||
ssl: false,
|
||||
application_name: "tps_etl_api"
|
||||
});
|
||||
Postgres.FirstRow = function(inSQL,args, inResponse)
|
||||
{
|
||||
Postgres.query(inSQL,args, (err, res) => {
|
||||
if (err === null)
|
||||
{
|
||||
inResponse.json(res.rows[0]);
|
||||
return;
|
||||
}
|
||||
inResponse.json(err.stack);
|
||||
});
|
||||
};
|
||||
Postgres.connect();
|
||||
|
||||
|
||||
//----------------------------------------------------------source definitions-------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
//----------returns array of all sources--------------------------
|
||||
server.get("/source", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce";
|
||||
Postgres.FirstRow(sql,[], inRes);
|
||||
});
|
||||
//----------returns message about status and error description--
|
||||
server.post("/source_single", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes);
|
||||
});
|
||||
//------assume inboud info is json array of definitions to set--
|
||||
server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build
|
||||
{
|
||||
x = inReq.body;
|
||||
var sql = "SELECT x.message FROM tps.srce_overwrite_all($1::jsonb) x(message)";
|
||||
console.log(JSON.stringify(inReq.body));
|
||||
Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes);
|
||||
});
|
||||
|
||||
//----------------------------------------------------------regex instrUctions-------------------------------------------------------------------------------------------------------------------------
|
||||
//list all regex operations
|
||||
server.get("/regex", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm WHERE srce = $1::text";
|
||||
Postgres.FirstRow(sql, [inReq.query.srce], inRes);
|
||||
});
|
||||
|
||||
//set one or more map definitions
|
||||
server.post("/regex", bodyParser.json(), function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.srce_map_def_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
|
||||
});
|
||||
|
||||
//------------------------------------------------------------mappings---------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
//list unmapped items flagged to be mapped ?srce=
|
||||
server.get("/unmapped_all", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs($1::text) x";
|
||||
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
|
||||
});
|
||||
|
||||
//list unmapped items flagged to be mapped ?srce=
|
||||
server.get("/unmapped", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped($1::text) x";
|
||||
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
|
||||
});
|
||||
|
||||
server.get("/mapping", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.map_rv x WHERE srce = $1::text";
|
||||
|
||||
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
|
||||
});
|
||||
|
||||
//add entries to lookup table
|
||||
server.post("/mapping", bodyParser.json(), function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT x.message FROM tps.map_rv_set($1::jsonb) as x(message)";
|
||||
Postgres.FirstRow(sql,[JSON.stringify( inReq.body)], inRes);
|
||||
});
|
||||
|
||||
//---------------------------------------------------------list imports--------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
server.get("/import_log", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(row_to_json(l)::jsonb) regex FROM tps.trans_log l";
|
||||
Postgres.FirstRow(sql,[], inRes);
|
||||
});
|
||||
|
||||
|
||||
//-------------------------------------------------------------import data-----------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
server.use("/import", upload.single('upload'), function (inReq, inRes) {
|
||||
|
||||
console.log("should have gotten file as post body here");
|
||||
var csv = inReq.file.buffer.toString('utf8')
|
||||
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
|
||||
csvtojson({ flatKeys: "true" }).fromString(csv).then(
|
||||
(x) => {
|
||||
var sql = "SELECT x.message FROM tps.srce_import($1, $2::jsonb) as x(message)"
|
||||
console.log(sql);
|
||||
Postgres.FirstRow(sql, [inReq.query.srce, JSON.stringify(x)], inRes);
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
//----------------------------------------------------------list import logs---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
server.get("/import_log", function (inReq, inRes)
|
||||
{
|
||||
var sql = "SELECT jsonb_agg(info) info FROM tps.trans_log WHERE info @> $1::jsonb";
|
||||
Postgres.FirstRow(sql, [inReq.query], inRes);
|
||||
});
|
||||
|
||||
//-------------------------------------------------------------suggest source def----------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
|
||||
|
||||
console.log("should have gotten file as post body here");
|
||||
var csv = inReq.file.buffer.toString('utf8')
|
||||
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
|
||||
csvtojson({ flatKeys: "true" }).fromString(csv).then(
|
||||
(x) => {
|
||||
var sug = {
|
||||
schemas: {
|
||||
default: []
|
||||
},
|
||||
loading_function: "csv",
|
||||
source:"client_file",
|
||||
name: "",
|
||||
constraint: []
|
||||
};
|
||||
for (var key in x[0]) {
|
||||
var col = {};
|
||||
//test if number
|
||||
if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) {
|
||||
//if is a number but leading character is -0- then it's text
|
||||
if (x[0][key].charAt(0) == "0"){
|
||||
col["type"] = "text";
|
||||
}
|
||||
//if number and leadign character is not 0 then numeric
|
||||
else {
|
||||
col["type"] = "numeric";
|
||||
}
|
||||
}
|
||||
//if can cast to a date within a hundred years its probably a date
|
||||
else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) {
|
||||
col["type"] = "date";
|
||||
}
|
||||
//otherwise its text
|
||||
else {
|
||||
col["type"] = "text";
|
||||
}
|
||||
col["path"] = "{" + key + "}";
|
||||
col["column_name"] = key;
|
||||
sug.schemas.default.push(col);
|
||||
}
|
||||
console.log(sug);
|
||||
inRes.json(sug);
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
Loading…
Reference in New Issue
Block a user