pt/pivot_forecast
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Compare commits

base: pt:9c6ed6b3040543517dcc08032a3a58acef280a82
Branches Tags
pt:master
pt:nofile
pt:refactor
..
compare: pt:80bcb231704b7e89831ee928364a6f7089c5f201
Branches Tags
pt:nofile
pt:master
pt:refactor

No commits in common. "9c6ed6b3040543517dcc08032a3a58acef280a82" and "80bcb231704b7e89831ee928364a6f7089c5f201" have entirely different histories.
9c6ed6b304 ... 80bcb23170

5 changed files with 93 additions and 207 deletions
Show Stats Expand all files Collapse all files

3
index.js
View File

@ -55,7 +55,7 @@ server.get('/baseline', bodyParser.json(), function(req, res) {
var path = './routes/baseline/baseline.sql';
var args = [];
var app_req = JSON.stringify(req.body);
var app_baseline_from_date = req.body.app_baseline_from_date;
var app_baseline_to_date = req.body.app_baseline_to_date;
var app_first_forecast_date = req.body.app_first_forecast_date;
@ -71,7 +71,6 @@ server.get('/baseline', bodyParser.json(), function(req, res) {
console.log(req.body);
//parse the where clause into the main sql statement
//sql = sql.replace(new RegExp("where_clause", 'g'), w)
sql = sql.replace(new RegExp("app_req", 'g'), app_req);
sql = sql.replace(new RegExp("app_baseline_from_date", 'g'), app_baseline_from_date);
sql = sql.replace(new RegExp("app_baseline_to_date", 'g'), app_baseline_to_date);
sql = sql.replace(new RegExp("app_first_forecast_date", 'g'), app_first_forecast_date);

15
readme.md
View File

@ -26,15 +26,11 @@ setup
| | | | | | | | | iter | |
| | | | | | | | | logid | |
* func:
* a foreign key and table is setup (except date types)
* baseline function does a group-by to compact data for any func/appcol and value types are aggregated
* fkey:
* primary key of assoicated data and foreign key basis
* for date types no foreign key is setup, but dynamic table joins use the fkey field to join on
* func: table name of associated data
* fkey: primary key of assoicated dat
* fcol: associated field from the master data table if it is different (oseas would refer to ssyr in fc.perd)
* pretty: display column name (user friendly)
* appcol: flag columns that will be needed by the application
* appcol: parameters that will have to be supplied but the application
* order_date
* ship_date
* customer
@ -48,11 +44,6 @@ setup
* logid (added if missing)
issues
- [ ] baseline request json is not depostiing correctly into the sql
- [ ] what if you create a func table but some of the values are null? (orders without quotes)
- [x] what if you want certain things like invoice numbers included in the baseline, is a func table necesary? no, just leave fkey null and no table will be setup
- [ ] baseline needs to populate all columns in sequential order like the scale route
- [ ] what if there is more than one table in fc.target_meta
- [x] validate the ability to join to associated date tables, for example order season that needs incremented
- [ ] there is not currently any initial grouping to limit excess data from all the document# scenarios
- [ ] how to handle a target value adjustment, which currency is it in?

84
routes/baseline/baseline.sql
View File

@ -1,30 +1,20 @@
DELETE FROM tpsv.dcard WHERE app_version = 'app_forecast_name';
WITH
req AS (SELECT $$app_req$$::jsonb j)
,ilog AS (
INSERT INTO
fc.log (doc)
baseline AS (
SELECT
req.j
FROM
req
RETURNING *
)
,baseline AS (
SELECT
null::integer AS id
,null::integer AS logid
o.id
,o.logid
,o."Trans. Date"
,o."Post Date"
,o."Description"
,sum(o."Amount") AS "Amount"
,o."Amount"
,o."Category"
,o."Party"
,o."Reason"
,sum(o.app_units) AS app_units
,'app_forecast_name' AS app_version
,'baseline' AS app_iter
,(SELECT id FROM ilog) AS app_logid
,o.app_units
,'app_forecast_name' "version"
,'actuals' iter
,null::bigint app_logid
FROM
tpsv.dcard o
WHERE
@ -37,72 +27,52 @@ WHERE
OR ("Post Date" BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date)
)
--be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
GROUP BY
o."Trans. Date"
,o."Post Date"
,o."Description"
,o."Category"
,o."Party"
,o."Reason"
UNION ALL
SELECT
null::integer AS id
,null::integer AS logid
o.id
,o.logid
,o."Trans. Date" + interval '1 year' AS "Trans. Date"
,o."Post Date" + interval '1 year' AS "Post Date"
,o."Description"
,sum(o."Amount") AS "Amount"
,o."Amount"
,o."Category"
,o."Party"
,o."Reason"
,sum(o.app_units) AS app_units
,'app_forecast_name' AS app_version
,'baseline' AS app_iter
,(SELECT id FROM ilog) AS app_logid
,o.app_units
,'app_forecast_name' "version"
,'plug' iter
,null::bigint app_logid
FROM
tpsv.dcard o
LEFT OUTER JOIN fc.perd tdate ON
(o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
LEFT OUTER JOIN fc.perd pdate ON
(o."Post Date" + interval '1 year' )::date <@ pdate.drange
LEFT OUTER JOIN fc.perd tdate ON
(o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
WHERE
"Trans. Date" BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date
--be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
GROUP BY
o."Trans. Date"
,o."Post Date"
,o."Description"
,o."Category"
,o."Party"
,o."Reason")
)
,incr AS (
SELECT
null::integer AS id
,null::integer AS logid
o.id
,o.logid
,o."Trans. Date" + interval '1 year' AS "Trans. Date"
,o."Post Date" + interval '1 year' AS "Post Date"
,o."Description"
,sum(o."Amount") AS "Amount"
,o."Amount"
,o."Category"
,o."Party"
,o."Reason"
,sum(o.app_units) AS app_units
,'app_forecast_name' AS app_version
,'baseline' AS app_iter
,(SELECT id FROM ilog) AS app_logid
,o.app_units
,'app_forecast_name' "version"
,'baseline' iter
,null::bigint app_logid
FROM
baseline o
LEFT OUTER JOIN fc.perd tdate ON
(o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
LEFT OUTER JOIN fc.perd pdate ON
(o."Post Date" + interval '1 year' )::date <@ pdate.drange
GROUP BY
o."Trans. Date"
,o."Post Date"
,o."Description"
,o."Category"
,o."Party"
,o."Reason"
LEFT OUTER JOIN fc.perd tdate ON
(o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
)
,ins AS (
INSERT INTO

162
routes/baseline/gen_baseline.sql
View File

@ -2,18 +2,11 @@ DO
$func$
DECLARE
_clist text;
_clist_sum text;
_clist_group text;
_clist_inc text;
_version_col text;
_iter_col text;
_logid_col text;
_ytdbody text;
_order_date text;
_ship_date text;
_order_status text;
_units_col text;
_value_col text;
_ytdbody text;
_actpy text;
_sql text;
_baseline text;
@ -21,6 +14,7 @@ DECLARE
_perd_joins text;
_interval interval;
_target_table text;
_version_col text;
/*----------------parameters listing--------------
app_baseline_from_date
@ -35,22 +29,14 @@ app_plug_todate
BEGIN
-----------------populate application variables--------------------------------------------
SELECT cname INTO _order_date FROM fc.target_meta WHERE appcol = 'order_date' ;
SELECT cname INTO _ship_date FROM fc.target_meta WHERE appcol = 'ship_date' ;
SELECT cname INTO _order_status FROM fc.target_meta WHERE appcol = 'order_status' ;
SELECT cname INTO _units_col FROM fc.target_meta WHERE appcol = 'units' ;
SELECT cname INTO _value_col FROM fc.target_meta WHERE appcol = 'value' ;
SELECT cname INTO _version_col FROM fc.target_meta WHERE appcol = 'version' ;
SELECT cname INTO _iter_col FROM fc.target_meta WHERE appcol = 'iter' ;
SELECT cname INTO _logid_col FROM fc.target_meta WHERE appcol = 'logid' ;
SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date;
SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date;
SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status;
SELECT format('%I',max(schema))||'.'||format('%I',max(tname)) INTO _target_table FROM fc.target_meta;
--the target interval
SELECT interval '1 year' INTO _interval;
----------------------setup listing of date joins required-----------------------------------------------------
SELECT jsonb_agg(DISTINCT func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND fkey is NOT null;
raise notice E'date list -----> \n %',_date_funcs;
-----------------------create the date based table joins--------------------------------------------------------
SELECT cname INTO _version_col FROM fc.target_meta WHERE appcol = 'version';
SELECT jsonb_agg(func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND appcol is NOT null;
--create table join for each date based func in target_meta joining to fc.perd static table
--the join, though, should be based on the target date, which is needs an interval added to get to the target
SELECT
@ -67,60 +53,19 @@ WHERE
dtype = 'date'
AND func IS NOT NULL;
raise notice E'period table joins -----> \n %',_perd_joins;
--raise notice '%',_perd_joins;
-------------------------------build a column for sums-----------------------------------------
-------------------------------build a column list-----------------------------------------
SELECT
--string_agg('o.'||format('%I',cname),E'\n ,' ORDER BY opos ASC)
string_agg(
--create the column reference
CASE appcol
----aggregation columns-----------------------------------------------------------
WHEN 'units' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
WHEN 'value' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
WHEN 'cost' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
----create singular values for fingle forecast step-------------------------------
WHEN 'version' THEN '''app_forecast_name'' AS '||format('%I',_version_col)
WHEN 'iter' THEN '''baseline'' AS '||format('%I',_iter_col)
WHEN 'logid' THEN '(SELECT id FROM ilog) AS '||format('%I',_logid_col)
ELSE
---if there is no function, make the column null or the default value---
CASE WHEN t.func IS NULL
THEN 'null::'||t.dtype||' AS '||format('%I',cname)
ELSE 'o.'||format('%I',cname)
END
END
--delimiter
,E'\n ,'
--sort column ordinal
ORDER BY opos ASC
)
string_agg('o.'||format('%I',cname),E'\n ,' ORDER BY opos ASC)
INTO
_clist_sum
_clist
FROM
fc.target_meta t;
raise notice E'column list for sums -----> \n %',_clist_sum;
-------------------------------build a column for groupby-----------------------------------------
SELECT
--string_agg('o.'||format('%I',cname),E'\n ,' ORDER BY opos ASC)
string_agg(
--create the column reference
'o.'||format('%I',cname)
--delimiter
,E'\n ,'
--sort column ordinal
ORDER BY opos ASC
)
INTO
_clist_group
FROM
fc.target_meta t
fc.target_meta
WHERE
COALESCE(appcol,'') NOT IN ('units','value','cost','version','iter','logid')
AND func IS NOT NULL;
COALESCE(appcol,'') NOT IN ('version','iter','logid');
raise notice E'column list for group by-----> \n %',_clist_group;
--raise notice '%',_clist;
---------------------------build column to increment dates---------------------------------
SELECT
@ -130,7 +75,7 @@ SELECT
WHEN _date_funcs ? func THEN
CASE
--...but it's not the date itself...
WHEN fkey IS NULL THEN
WHEN appcol IS NULL THEN
--...pull the associated date field from perd table
'perd.'||m.fcol
--...and it's the primary key date...
@ -140,41 +85,29 @@ SELECT
'o.'||format('%I',cname)||' + interval '||format('%L',_interval) ||' AS '||format('%I',cname)
END
ELSE
CASE appcol
----aggregation columns-----------------------------------------------------------
WHEN 'units' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
WHEN 'value' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
WHEN 'cost' THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname)
----create singular values for fingle forecast step-------------------------------
WHEN 'version' THEN '''app_forecast_name'' AS '||format('%I',_version_col)
WHEN 'iter' THEN '''baseline'' AS '||format('%I',_iter_col)
WHEN 'logid' THEN '(SELECT id FROM ilog) AS '||format('%I',_logid_col)
ELSE
---if there is no function, make the column null or the default value---
CASE WHEN m.func IS NULL
THEN 'null::'||m.dtype||' AS '||format('%I',cname)
ELSE 'o.'||format('%I',cname)
END
END
'o.'||format('%I',cname)
END
,E'\n ,' ORDER BY opos ASC
)
INTO
_clist_inc
FROM
fc.target_meta m;
--WHERE
-- COALESCE(appcol,'') NOT IN ('version','iter','logid');
fc.target_meta m
WHERE
COALESCE(appcol,'') NOT IN ('version','iter','logid');
RAISE NOTICE E'DATES INCREMENTED: ------> \n %',_clist_inc;
--RAISE NOTICE 'DATES INCREMENTED: %',_clist_inc;
--------------------------------------clone the actual baseline-----------------------------------------------
SELECT
$$SELECT
$$||
_clist_sum||
$$::text||
_clist||
$$
,'app_forecast_name' "version"
,'actuals' iter
,null::bigint app_logid
FROM
$$||_target_table||$$ o
WHERE
@ -187,13 +120,12 @@ WHERE
OR ($$||format('%I',_ship_date)||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date)
)
--be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
GROUP BY
$$||
_clist_group
$$::text
INTO
_ytdbody;
RAISE NOTICE E'_ytdbody \n%', _ytdbody;
--RAISE NOTICE '_ytdbody %', _ytdbody;
------------------------------------pull a plug from actuals to create a full year baseline------------------
@ -201,17 +133,19 @@ SELECT
$$SELECT
$$||_clist_inc||
$$
,'app_forecast_name' "version"
,'plug' iter
,null::bigint app_logid
FROM
$$||_target_table||' o'||E'\n'||_perd_joins||$$
WHERE
$$||format('%I',_order_date)||$$ BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date
--be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
GROUP BY
$$||_clist_group
$$
INTO
_actpy;
RAISE NOTICE E'plug body --------> \n%',_actpy;
--RAISE NOTICE '_actpf %',_actpy;
------------------------------copy a full year and increment by 1 year for the baseline-------------------------
SELECT
@ -219,11 +153,13 @@ SELECT
-- fc.live
$$,incr AS (
SELECT
$$||_clist_inc||$$
$$||_clist_inc||
$$
,'app_forecast_name' "version"
,'baseline' iter
,null::bigint app_logid
FROM
baseline o$$||E'\n'||_perd_joins||$$
GROUP BY
$$||_clist_group||$$
)
,ins AS (
INSERT INTO
@ -243,32 +179,22 @@ INTO
_baseline;
RAISE NOTICE E'_baseline -------> \n%',_baseline;
--RAISE NOTICE '_baseline %',_baseline;
------------------------------stack the sql into the final format------------------------------------------------
SELECT
$$DELETE FROM $$||_target_table||$$ WHERE $$||_version_col||$$ = 'app_forecast_name';
WITH
req AS (SELECT $$||'$$app_req$$::jsonb j)'||$$
,ilog AS (
INSERT INTO
fc.log (doc)
SELECT
req.j
FROM
req
RETURNING *
)
,baseline AS (
$$||_ytdbody||$$
UNION ALL
baseline AS (
$$||_ytdbody||
$$UNION ALL
$$||_actpy
||$$)
$$||_baseline
INTO
_sql;
RAISE NOTICE E'_sql ---------> \n%',_sql;
--RAISE NOTICE '_sql %',_sql;
INSERT INTO fc.sql SELECT 'baseline', _sql ON CONFLICT ON CONSTRAINT sql_pkey DO UPDATE SET t = EXCLUDED.t;

2
setup_sql/03_build_master_tables.sql
View File

@ -35,7 +35,7 @@ BEGIN
--HAVING
-- string_agg(cname,', ') FILTER (WHERE fkey = func) <> ''
loop
INSERT INTO fc.sql SELECT f.func, f.ddl ON CONFLICT ON CONSTRAINT sql_pkey DO UPDATE SET t = EXCLUDED.t;
INSERT INTO fc.sql SELECT f.func, f.ddl;
EXECUTE format('%s',f.ddl);
EXECUTE format('%s',f.pop);
EXECUTE format('%s',f.fk);