work on turning into service

This commit is contained in:
Paul Trowbridge 2020-03-25 23:30:25 -04:00
parent b8c1535cf8
commit 8a40417de9
6 changed files with 143 additions and 76 deletions

19
cert.pem Normal file
View File

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDETCCAfkCFDygOtYyPxZgKLgMp/wIeQ2yGHweMA0GCSqGSIb3DQEBCwUAMEUx
CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjAwMzE3MDQ0MzI2WhcNNDcwODAyMDQ0
MzI2WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
AQ8AMIIBCgKCAQEAruKzxqy7Zdk1odLAtnKs60eu5/rLGMdsLjbB+V7R7v3bIdi/
TaoKD0oYOHjJSEfpKUEqva+W45Yx8A9dt2OE2jD2Rr3sCNub6m2vY4iB9xXGA5Wy
qtMr83Die225TOtMgGnTmYIU2D7VfOZ08xFu4rwU6NlvjHx/xMByRQ7N1QAMmlkr
F1/KRd6i4+OmFoY81ErsLL4P/rhJ8jbzbRmJGzz3DGOXFvE9Qk+DdMxk8WnzfiZc
8HBVBMYPYifSvuoARtxigQOwodjOTrb+asw3AG5B4Yh1NnZMDa8ujaYgid0RiN8a
Y18GcXZmtGdl1h88Zmlv+sqJbSK5ThOj8pPsxQIDAQABMA0GCSqGSIb3DQEBCwUA
A4IBAQBpLPJ1YZYPERYPYQbczMgcv+GaT7mqJNo7ATAgvMQYIhk2PjbS1FU8+A1X
DBuSb4vfJ6C2zEAWoduncNbKXw4Q9UkZS6/fer8HS1oYOWe9gli/V+hisEPfF1DB
6jyvbp1PZYd39LccovQ9d1ujEOdch+I2iQP4BfpiQohXePDXJA1eDCg2kQBI7aAF
IQ9ZB4ywe6IivLsv1hM37EWEAf6/wO8k2HrZf+LvQAf+fuk/SIhNFav7oRg/FuBJ
p+56Itc8M+Qd6fIEFyj6GOAUhUzbuA1TH9h/XP6dmMk1XIWrFjhUQ4QC0scLdKTF
+xO3dWJaJnrTa+8Ex7KK6Akgo9rc
-----END CERTIFICATE-----

4
create_certs.sh Normal file
View File

@ -0,0 +1,4 @@
openssl genrsa -out key.pem
openssl req -new -key key.pem -out csr.pem
openssl x509 -req -days 9999 -in csr.pem -signkey key.pem -out cert.pem
rm csr.pem

View File

@ -1,2 +1,12 @@
var server = require('./server');
server.listen(80);
//var server = require('./server');
//server.listen(process.env.nodeport);
var options = {
key: fs.readFileSync(process.env.wd + 'key.pem'),
cert: fs.readFileSync(process.env.wd + 'cert.pem'),
passprase: []
};
https.createServer(options, server).listen(process.env.nodeport, () => {
console.log('started on ' + process.env.nodeport)
});

27
key.pem Normal file
View File

@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAruKzxqy7Zdk1odLAtnKs60eu5/rLGMdsLjbB+V7R7v3bIdi/
TaoKD0oYOHjJSEfpKUEqva+W45Yx8A9dt2OE2jD2Rr3sCNub6m2vY4iB9xXGA5Wy
qtMr83Die225TOtMgGnTmYIU2D7VfOZ08xFu4rwU6NlvjHx/xMByRQ7N1QAMmlkr
F1/KRd6i4+OmFoY81ErsLL4P/rhJ8jbzbRmJGzz3DGOXFvE9Qk+DdMxk8WnzfiZc
8HBVBMYPYifSvuoARtxigQOwodjOTrb+asw3AG5B4Yh1NnZMDa8ujaYgid0RiN8a
Y18GcXZmtGdl1h88Zmlv+sqJbSK5ThOj8pPsxQIDAQABAoIBAF28hy1q8frSJIv7
AW4JIyPsxRPz+Z3rbdGLaga2SwG1MRoAKfF/vWqahUbnfsNuP8vr8PPAxLhxjYyl
P88G2XN1oNsj+vp4cyaA6ewIg9ii5AKR3zTugPU3wTz0/8f5qoR9tHvoHDIX46fR
gkeyRWPj+bUgrkh3hqg3mkfnem26eQ7/lBLrwhscCqDnxg7CneoFfvdqdlYO1HTO
gU+zmwZ6LPXoeV2wQpJy0yf+FZkOd4mUiSlK5C+DMOrfjWcvdVG931h91xzQUHDh
MumjbEx5tSJXMeb4zXofBbPLUZZCHl4VkgY/C7JjG7KGrl/GN6aSkjfSCnzzBBrb
3B7lcKECgYEA20g9TfFMLp7rlaap5WAUh2wDgg7klIhLYlqx+No5YgAcnkC4ec6y
97CV9jrAdLzod9i0WVczN9d4dfA30vu45Vp9k6k+XUgnKe00PFAHQheAOY+yG+Am
KUxvjvVwHolYSTq+6dVbMAfN0rfiJgvABKuxTmdFEloQWlzMpG7Re+kCgYEAzCta
0NJEhDrgf7AUPDkeFx45ZmwcTIwGff5jOlr8WfW8/u24qRVrqnMc7JyA7Pub8NlX
b+tFMVUKfWTJbCnqOiwQjstuXrUUkL78vIBcAyETRrXln11kRM+/aOLWm3F9WgCR
qRqxEehSlpVjM3w98hLfElWjbjSMZekboNjvjH0CgYEAqoWexsHiUE7dPN8CN5Fj
5XulrroOH0Nt++ozhCWPgho8JwSFcJPAsMTtmTFPAHj2/lvSw7b/6WjyECTiBHwj
6JdaGD4AdWHqsrv3Zv+E5WyJFjocc3j3tB3wkudibRy5Pxkj5Ys5AjzPXhIJFzUD
+Z3an/HG0OKg3ORVb3mOBiECgYA9EEZ0KD9H0Rgt+GrE7Excm7SNNo5sMoEKk9f3
rxjEiyPAlzbgk0CFVTJ0bgZ9Wm90ZHyrQ5GJRKcDg+5eRFcCmQVGv/89oikJdaW/
dyCN6wmJ34NS26rOjdfwtc975ECad4sZKZuGAJca5Ikr79/TE+PV5vkfT+Yh4Mtb
myZiPQKBgHGM9ZSXCQRGeZqhbSZeXBC/oBSdI2+R8HSR9O7x9yLiRd5JGDJS82q9
4aKQRCAGQ7AILEBmdwWXQuFqrroUYFWR+YYIGwBZhms1Fdcb1WvjV0Ag/GBce93x
XQgdFrcA0mZ5BUXR5MFPCOeYfi/d4GjQbMPls36tz7DnAFMHeKbF
-----END RSA PRIVATE KEY-----

142
server.js
View File

@ -7,13 +7,15 @@ var mult = require('multer');
var upload = mult({ encoding: "utf8" });
var csvtojson = require('csvtojson');
var pg = require('pg');
var https = require('https');
var fs = require('fs');
var readline = require('readline');
var server = express();
server.engine('handlebars', handlebars());
server.set('view engine', 'handlebars');
server.use(function(inReq, inRes, inNext)
{
server.use(function(inReq, inRes, inNext) {
inRes.header("Access-Control-Allow-Origin", "*");
inRes.header("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS");
inRes.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
@ -21,6 +23,17 @@ server.use(function(inReq, inRes, inNext)
});
var options = {
key: fs.readFileSync(process.env.wd + 'key.pem'),
cert: fs.readFileSync(process.env.wd + 'cert.pem'),
passprase: []
};
https.createServer(options, server).listen(process.env.nodeport, () => {
console.log('started on ' + process.env.nodeport)
});
var Postgres = new pg.Client({
user: process.env.user,
password: process.env.password,
@ -30,11 +43,9 @@ var Postgres = new pg.Client({
ssl: false,
application_name: "tps_etl_api"
});
Postgres.FirstRow = function(inSQL,args, inResponse)
{
Postgres.query(inSQL,args, (err, res) => {
if (err === null)
{
Postgres.FirstRow = function(inSQL, args, inResponse) {
Postgres.query(inSQL, args, (err, res) => {
if (err === null) {
inResponse.json(res.rows[0]);
return;
}
@ -46,44 +57,40 @@ Postgres.connect();
//----------------------------------------------------------source definitions-------------------------------------------------------------------------------------------------------------------------
//returns array of all sources
server.get("/source", function (inReq, inRes)
{
server.get("/source", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(defn) source_list FROM tps.srce";
Postgres.FirstRow(sql,[], inRes);
Postgres.FirstRow(sql, [], inRes);
});
//returns message about status and error description
server.post("/source_single", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build
{
var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)";
Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes);
});
server.post("/source_single", bodyParser.json(), function(inReq, inRes) // remove body parsing, just pass post body to the sql string build
{
var sql = "SELECT x.message FROM tps.srce_set($1::jsonb) as x(message)";
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
});
//assume inboud info is json array of definitions to set
server.post("/source", bodyParser.json(), function (inReq, inRes)// remove body parsing, just pass post body to the sql string build
{
//x = inReq.body;
var sql = "SELECT x.message FROM tps.srce_overwrite_all($1::jsonb) x(message)";
//console.log(JSON.stringify(inReq.body));
Postgres.FirstRow(sql,[JSON.stringify(inReq.body)], inRes);
});
server.post("/source", bodyParser.json(), function(inReq, inRes) // remove body parsing, just pass post body to the sql string build
{
//x = inReq.body;
var sql = "SELECT x.message FROM tps.srce_overwrite_all($1::jsonb) x(message)";
//console.log(JSON.stringify(inReq.body));
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
});
//----------------------------------------------------------regex instrUctions-------------------------------------------------------------------------------------------------------------------------
//list all regex operations
server.get("/regex", function (inReq, inRes)
{
server.get("/regex", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(regex) regex FROM tps.map_rm WHERE srce = $1::text";
Postgres.FirstRow(sql, [inReq.query.srce], inRes);
});
//set one or more map definitions
server.post("/regex", bodyParser.json(), function (inReq, inRes)
{
server.post("/regex", bodyParser.json(), function(inReq, inRes) {
var sql = "SELECT x.message FROM tps.srce_map_def_set($1::jsonb) as x(message)";
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
});
//takes an ad-hoc regex definition in curly braces
server.get("/regex_test", bodyParser.json(), function (inReq, inRes)
{
server.get("/regex_test", bodyParser.json(), function(inReq, inRes) {
var sql = "SELECT x.message FROM tps.test_regex_recs($1::jsonb) as x(message)";
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
});
@ -91,49 +98,44 @@ server.get("/regex_test", bodyParser.json(), function (inReq, inRes)
//------------------------------------------------------------mappings---------------------------------------------------------------------------------------------------------------------------------
//list unmapped items flagged to be mapped ?srce=
server.get("/unmapped_all", function (inReq, inRes)
{
server.get("/unmapped_all", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped_recs($1::text) x";
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
Postgres.FirstRow(sql, [inReq.query.srce], inRes);
});
//list unmapped items flagged to be mapped ?srce=
server.get("/unmapped", function (inReq, inRes)
{
server.get("/unmapped", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.report_unmapped($1::text) x";
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
Postgres.FirstRow(sql, [inReq.query.srce], inRes);
});
server.get("/mapping", function (inReq, inRes)
{
server.get("/mapping", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(row_to_json(x)::jsonb) regex FROM tps.map_rv x WHERE srce = $1::text";
Postgres.FirstRow(sql,[inReq.query.srce], inRes);
Postgres.FirstRow(sql, [inReq.query.srce], inRes);
});
//add entries to lookup table
server.post("/mapping", bodyParser.json(), function (inReq, inRes)
{
server.post("/mapping", bodyParser.json(), function(inReq, inRes) {
var sql = "SELECT x.message FROM tps.map_rv_set($1::jsonb) as x(message)";
Postgres.FirstRow(sql,[JSON.stringify( inReq.body)], inRes);
Postgres.FirstRow(sql, [JSON.stringify(inReq.body)], inRes);
});
//---------------------------------------------------------list imports--------------------------------------------------------------------------------------------------------------------------------
server.get("/import_log", function (inReq, inRes)
{
server.get("/import_log", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(row_to_json(l)::jsonb) regex FROM tps.trans_log l";
Postgres.FirstRow(sql,[], inRes);
Postgres.FirstRow(sql, [], inRes);
});
//-------------------------------------------------------------import data-----------------------------------------------------------------------------------------------------------------------------
server.use("/import", upload.single('upload'), function (inReq, inRes) {
server.use("/import", upload.single('upload'), function(inReq, inRes) {
console.log("should have gotten file as post body here");
var csv = inReq.file.buffer.toString('utf8')
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
csvtojson({ flatKeys: "true" }).fromString(csv).then(
(x) => {
var sql = "SELECT x.message FROM tps.srce_import($1, $2::jsonb) as x(message)"
@ -141,24 +143,22 @@ server.use("/import", upload.single('upload'), function (inReq, inRes) {
Postgres.FirstRow(sql, [inReq.query.srce, JSON.stringify(x)], inRes);
}
);
}
);
});
//----------------------------------------------------------list import logs---------------------------------------------------------------------------------------------------------------------------
server.get("/import_log", function (inReq, inRes)
{
server.get("/import_log", function(inReq, inRes) {
var sql = "SELECT jsonb_agg(info) info FROM tps.trans_log WHERE info @> $1::jsonb";
Postgres.FirstRow(sql, [inReq.query], inRes);
});
//-------------------------------------------------------------suggest source def----------------------------------------------------------------------------------------------------------------------
server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
server.use("/csv_suggest", upload.single('upload'), function(inReq, inRes) {
console.log("should have gotten file as post body here");
var csv = inReq.file.buffer.toString('utf8')
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
//{headers: "true", delimiter: ",", output: "jsonObj", flatKeys: "true"}
csvtojson({ flatKeys: "true" }).fromString(csv).then(
(x) => {
var sug = {
@ -166,7 +166,7 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
default: []
},
loading_function: "csv",
source:"client_file",
source: "client_file",
name: "",
constraint: []
};
@ -175,14 +175,14 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
//test if number
if (!isNaN(parseFloat(x[0][key])) && isFinite(x[0][key])) {
//if is a number but leading character is -0- then it's text
if (x[0][key].charAt(0) == "0"){
if (x[0][key].charAt(0) == "0") {
col["type"] = "text";
}
//if number and leadign character is not 0 then numeric
else {
col["type"] = "numeric";
}
}
}
//if can cast to a date within a hundred years its probably a date
else if (Date.parse(x[0][key]) > Date.parse('1950-01-01') && Date.parse(x[0][key]) < Date.parse('2050-01-01')) {
col["type"] = "date";
@ -199,8 +199,7 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
inRes.json(sug);
}
);
}
);
});
//--------------------------------------------------------------------------------------------------------------------------------------------------------------------
@ -209,21 +208,20 @@ server.use("/csv_suggest", upload.single('upload'), function (inReq, inRes) {
//add ledger array and create offset account for every line
server.get("/gl_mhi_multi_post", bodyParser.json(), function (inReq, inRes)
{
server.get("/gl_mhi_multi_post", bodyParser.json(), function(inReq, inRes) {
var l = 0;
console.log(inReq.body);
x = inReq.body;
x.gl = {};
x.gl.lines = [];
x.gl.jpath = [];
for (var i in x.item){
for (var i in x.item) {
//copy the current item to the gl array
var line = x.item[i];
x.gl.lines.push(line);
//build references to 'item' array
var ref = [];
ref.push("{item,"+i+"}");
ref.push("{item," + i + "}");
ref.push("{header}");
x.gl.jpath.push(ref);
//copy the current item to the gl array again, but swap account with supplied 'account' in header
@ -236,37 +234,35 @@ server.get("/gl_mhi_multi_post", bodyParser.json(), function (inReq, inRes)
}
var sql = "INSERT INTO evt.bpr (bpr) SELECT $1";
console.log(JSON.stringify(x));
Postgres.FirstRow(sql,[JSON.stringify(x)], inRes);
Postgres.FirstRow(sql, [JSON.stringify(x)], inRes);
});
//add ledger array and create offset account for every line
server.get("/gl_mje_build", bodyParser.json(), function (inReq, inRes)
{
server.get("/gl_mje_build", bodyParser.json(), function(inReq, inRes) {
var l = 0;
console.log(inReq.body);
x = inReq.body;
x.gl = {};
x.gl.lines = [];
x.gl.jpath = [];
for (var i in x.items){
for (var i in x.items) {
//copy the current item to the gl array
var line = x.items[i];
x.gl.lines.push(line);
//build references to 'item' array
var ref = [];
ref.push("{items,"+i+"}");
ref.push("{items," + i + "}");
ref.push("{header}");
x.gl.jpath.push(ref);
}
var sql = "INSERT INTO evt.bpr (bpr) SELECT $1";
console.log(JSON.stringify(x));
Postgres.FirstRow(sql,[JSON.stringify(x)], inRes);
Postgres.FirstRow(sql, [JSON.stringify(x)], inRes);
});
//add ledger array and create offset account for total of all lines
server.get("/gl_mhi_single_build", bodyParser.json(), function (inReq, inRes)
{
server.get("/gl_mhi_single_build", bodyParser.json(), function(inReq, inRes) {
var l = 0;
var tot = 0.00;
var bomb = false;
@ -274,13 +270,12 @@ server.get("/gl_mhi_single_build", bodyParser.json(), function (inReq, inRes)
x = inReq.body;
//add GL array
x.GL = [];
for (var i in x.item){
for (var i in x.item) {
var line = x.item[i];
if ((line.account != null) && (line.amount != null)) {
x.GL.push(line);
x.GL.push(line);
tot = tot + (line.amount || 0);
}
else {
} else {
bomb = true;
}
//add the whole line as-is
@ -296,8 +291,7 @@ server.get("/gl_mhi_single_build", bodyParser.json(), function (inReq, inRes)
});
server.get("/", function (inReq, inRes)
{
server.get("/", function(inReq, inRes) {
inRes.render("definition", { title: "definition", layout: "main" });
});

13
template.service Normal file
View File

@ -0,0 +1,13 @@
[Unit]
Description=forecast_api
After=network.target
[Service]
ExecStart=/usr/bin/node //opt/forecast_api/index.js
Restart=always
User=fc_api
Environemnt=NODE_ENV=production
WorkingDirectory=//opt/forecast_api
[Install]
WantedBy=multi-user.target