index.js

@@ -55,7 +55,7 @@ server.get('/baseline', bodyParser.json(), function(req, res) {
     var path = './routes/baseline/baseline.sql';
     var args = [];
    
-    var app_req                      =  JSON.stringify(req.body);
+
     var app_baseline_from_date =        req.body.app_baseline_from_date;
     var app_baseline_to_date =          req.body.app_baseline_to_date;
     var app_first_forecast_date =       req.body.app_first_forecast_date;
@@ -71,7 +71,6 @@ server.get('/baseline', bodyParser.json(), function(req, res) {
         console.log(req.body);
         //parse the where clause into the main sql statement
         //sql = sql.replace(new RegExp("where_clause", 'g'), w)
-        sql = sql.replace(new RegExp("app_req", 'g'),                   app_req);
         sql = sql.replace(new RegExp("app_baseline_from_date", 'g'),    app_baseline_from_date);
         sql = sql.replace(new RegExp("app_baseline_to_date", 'g'),      app_baseline_to_date);
         sql = sql.replace(new RegExp("app_first_forecast_date", 'g'),   app_first_forecast_date);

readme.md

@@ -26,15 +26,11 @@ setup
 |         |         |                |      |              |              |                |         | iter         |        | 
 |         |         |                |      |              |              |                |         | logid        |        | 
 
-* func: 
+* func: table name of associated data
-    * a foreign key and table is setup (except date types)
+* fkey: primary key of assoicated dat
-    * baseline function does a group-by to compact data for any func/appcol and value types are aggregated
-* fkey: 
-    * primary key of assoicated data and foreign key basis
-    * for date types no foreign key is setup, but dynamic table joins use the fkey field to join on
 * fcol: associated field from the master data table if it is different (oseas would refer to ssyr in fc.perd)
 * pretty: display column name (user friendly)
-* appcol: flag columns that will be needed by the application
+* appcol: parameters that will have to be supplied but the application
     * order_date
     * ship_date
     * customer
@@ -48,11 +44,6 @@ setup
     * logid   (added if missing)
 
 issues
- - [ ] baseline request json is not depostiing correctly into the sql
- - [ ] what if you create a func table but some of the values are null? (orders without quotes)
- - [x] what if you want certain things like invoice numbers included in the baseline, is a func table necesary? no, just leave fkey null and no table will be setup
- - [ ] baseline needs to populate all columns in sequential order like the scale route
- - [ ] what if there is more than one table in fc.target_meta
  - [x] validate the ability to join to associated date tables, for example order season that needs incremented
  - [ ] there is not currently any initial grouping to limit excess data from all the document# scenarios
  - [ ] how to handle a target value adjustment, which currency is it in?

routes/baseline/baseline.sql

@@ -1,30 +1,20 @@
 DELETE FROM tpsv.dcard WHERE app_version = 'app_forecast_name';
 WITH
-req AS  (SELECT $$app_req$$::jsonb j)
+baseline AS (
-,ilog AS (
-    INSERT INTO
-        fc.log (doc)
-    SELECT
-        req.j 
-    FROM
-        req
-    RETURNING *
-)
-,baseline AS (
 SELECT
-    null::integer AS id
+    o.id
-    ,null::integer AS logid
+    ,o.logid
     ,o."Trans. Date"
     ,o."Post Date"
     ,o."Description"
-    ,sum(o."Amount") AS "Amount"
+    ,o."Amount"
     ,o."Category"
     ,o."Party"
     ,o."Reason"
-    ,sum(o.app_units) AS app_units
+    ,o.app_units
-    ,'app_forecast_name' AS app_version
+    ,'app_forecast_name' "version"
-    ,'baseline' AS app_iter
+    ,'actuals' iter
-    ,(SELECT id FROM ilog) AS app_logid
+    ,null::bigint app_logid
 FROM
     tpsv.dcard o
 WHERE
@@ -37,72 +27,52 @@ WHERE
         OR ("Post Date" BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date)
     )
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-GROUP BY
-    o."Trans. Date"
-    ,o."Post Date"
-    ,o."Description"
-    ,o."Category"
-    ,o."Party"
-    ,o."Reason"
 UNION ALL
 SELECT
-    null::integer AS id
+    o.id
-    ,null::integer AS logid
+    ,o.logid
     ,o."Trans. Date" + interval '1 year' AS "Trans. Date"
     ,o."Post Date" + interval '1 year' AS "Post Date"
     ,o."Description"
-    ,sum(o."Amount") AS "Amount"
+    ,o."Amount"
     ,o."Category"
     ,o."Party"
     ,o."Reason"
-    ,sum(o.app_units) AS app_units
+    ,o.app_units
-    ,'app_forecast_name' AS app_version
+    ,'app_forecast_name' "version"
-    ,'baseline' AS app_iter
+    ,'plug' iter
-    ,(SELECT id FROM ilog) AS app_logid
+    ,null::bigint app_logid
 FROM
     tpsv.dcard o
-    LEFT OUTER JOIN fc.perd tdate ON
+LEFT OUTER JOIN fc.perd pdate ON
-        (o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
-    LEFT OUTER JOIN fc.perd pdate ON
         (o."Post Date" + interval '1 year' )::date <@ pdate.drange
+LEFT OUTER JOIN fc.perd tdate ON
+        (o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
 WHERE
     "Trans. Date" BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-GROUP BY
+)
-    o."Trans. Date"
-    ,o."Post Date"
-    ,o."Description"
-    ,o."Category"
-    ,o."Party"
-    ,o."Reason")
 ,incr AS (
 SELECT
-    null::integer AS id
+    o.id
-    ,null::integer AS logid
+    ,o.logid
     ,o."Trans. Date" + interval '1 year' AS "Trans. Date"
     ,o."Post Date" + interval '1 year' AS "Post Date"
     ,o."Description"
-    ,sum(o."Amount") AS "Amount"
+    ,o."Amount"
     ,o."Category"
     ,o."Party"
     ,o."Reason"
-    ,sum(o.app_units) AS app_units
+    ,o.app_units
-    ,'app_forecast_name' AS app_version
+    ,'app_forecast_name' "version"
-    ,'baseline' AS app_iter
+    ,'baseline' iter
-    ,(SELECT id FROM ilog) AS app_logid
+    ,null::bigint app_logid
 FROM
     baseline o
+LEFT OUTER JOIN fc.perd pdate ON
+        (o."Post Date" + interval '1 year' )::date <@ pdate.drange
 LEFT OUTER JOIN fc.perd tdate ON
         (o."Trans. Date" + interval '1 year' )::date <@ tdate.drange
-    LEFT OUTER JOIN fc.perd pdate ON
-        (o."Post Date" + interval '1 year' )::date <@ pdate.drange
-GROUP BY
-    o."Trans. Date"
-    ,o."Post Date"
-    ,o."Description"
-    ,o."Category"
-    ,o."Party"
-    ,o."Reason"
 )
 ,ins AS (
 INSERT INTO

routes/baseline/gen_baseline.sql

@@ -1,26 +1,20 @@
 DO
 $func$
 DECLARE
-    _clist          text;
+    _clist text;
-    _clist_sum      text;
+    _clist_inc text;
-    _clist_group    text;
+    _ytdbody text;
-    _clist_inc      text;
+    _order_date text;
-    _version_col    text;
+    _ship_date text;
-    _iter_col       text;
+    _order_status text;
-    _logid_col      text;
+    _actpy text;
-    _order_date     text;
+    _sql text;
-    _ship_date      text;
+    _baseline text;
-    _order_status   text;
+    _date_funcs jsonb;
-    _units_col      text;
+    _perd_joins text;
-    _value_col      text;
+    _interval interval;
-    _ytdbody        text;
+    _target_table text;
-    _actpy          text;
+    _version_col text;
-    _sql            text;
-    _baseline       text;
-    _date_funcs     jsonb;
-    _perd_joins     text;
-    _interval       interval;
-    _target_table   text;
 
 /*----------------parameters listing--------------
 app_baseline_from_date
@@ -35,22 +29,14 @@ app_plug_todate
 BEGIN
 
 -----------------populate application variables--------------------------------------------
-SELECT cname INTO _order_date   FROM fc.target_meta WHERE appcol = 'order_date'      ;
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date;
-SELECT cname INTO _ship_date    FROM fc.target_meta WHERE appcol = 'ship_date'       ;
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date;
-SELECT cname INTO _order_status FROM fc.target_meta WHERE appcol = 'order_status'    ;
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status;
-SELECT cname INTO _units_col    FROM fc.target_meta WHERE appcol = 'units'           ;
+SELECT format('%I',max(schema))||'.'||format('%I',max(tname)) INTO _target_table FROM fc.target_meta;
-SELECT cname INTO _value_col    FROM fc.target_meta WHERE appcol = 'value'           ;
+--the target interval
-SELECT cname INTO _version_col  FROM fc.target_meta WHERE appcol = 'version'         ;
-SELECT cname INTO _iter_col     FROM fc.target_meta WHERE appcol = 'iter'            ;
-SELECT cname INTO _logid_col    FROM fc.target_meta WHERE appcol = 'logid'           ;
-SELECT format('%I',max(schema))||'.'||format('%I',max(tname))   INTO _target_table FROM fc.target_meta;
 SELECT interval '1 year' INTO _interval;
-
+SELECT cname INTO _version_col FROM fc.target_meta WHERE appcol = 'version';
-----------------------setup listing of date joins required-----------------------------------------------------
+SELECT jsonb_agg(func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND appcol is NOT null;
-SELECT jsonb_agg(DISTINCT func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND fkey is NOT null;
-raise notice E'date list  -----> \n    %',_date_funcs;
-
------------------------create the date based table joins--------------------------------------------------------
 --create table join for each date based func in target_meta joining to fc.perd static table
 --the join, though, should be based on the target date, which is needs an interval added to get to the target
 SELECT
@@ -58,7 +44,7 @@ SELECT
         'LEFT OUTER JOIN fc.perd '||func||' ON'||
         $$
         $$||'(o.'||format('%I',cname)||' + interval '||format('%L',_interval) ||' )::date <@ '||func||'.drange'
-    ,E'\n    ')
+    ,E'\n')
 INTO
     _perd_joins
 FROM 
@@ -67,60 +53,19 @@ WHERE
     dtype = 'date' 
     AND func IS NOT NULL;
 
-raise notice E'period table joins   -----> \n    %',_perd_joins;
+--raise notice '%',_perd_joins;
 
--------------------------------build a column for sums-----------------------------------------
+-------------------------------build a column list-----------------------------------------
 SELECT 
-    --string_agg('o.'||format('%I',cname),E'\n    ,' ORDER BY opos ASC)
+    string_agg('o.'||format('%I',cname),E'\n    ,' ORDER BY opos ASC)
-    string_agg(
-        --create the column reference
-        CASE appcol 
-            ----aggregation columns-----------------------------------------------------------
-            WHEN 'units'    THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-            WHEN 'value'    THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-            WHEN 'cost'     THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-            ----create singular values for fingle forecast step-------------------------------
-            WHEN 'version'  THEN '''app_forecast_name'' AS '||format('%I',_version_col)
-            WHEN 'iter'     THEN '''baseline'' AS '||format('%I',_iter_col)
-            WHEN 'logid'    THEN '(SELECT id FROM ilog) AS '||format('%I',_logid_col)
-            ELSE 
-                ---if there is no function, make the column null or the default value---
-                CASE WHEN t.func IS NULL 
-                    THEN 'null::'||t.dtype||' AS '||format('%I',cname)
-                    ELSE 'o.'||format('%I',cname)
-                END
-        END
-        --delimiter
-        ,E'\n    ,' 
-        --sort column ordinal
-        ORDER BY opos ASC
-    )
 INTO
-    _clist_sum
+    _clist
 FROM 
-    fc.target_meta t;
+    fc.target_meta 
+WHERE 
+    COALESCE(appcol,'') NOT IN ('version','iter','logid');
 
-raise notice E'column list for sums -----> \n    %',_clist_sum;
+--raise notice '%',_clist;
--------------------------------build a column for groupby-----------------------------------------
-SELECT 
-    --string_agg('o.'||format('%I',cname),E'\n    ,' ORDER BY opos ASC)
-    string_agg(
-        --create the column reference
-        'o.'||format('%I',cname)
-        --delimiter
-        ,E'\n    ,' 
-        --sort column ordinal
-        ORDER BY opos ASC
-    )
-INTO
-    _clist_group
-FROM 
-    fc.target_meta t
-WHERE
-    COALESCE(appcol,'') NOT IN ('units','value','cost','version','iter','logid')
-    AND func IS NOT NULL;
-
-raise notice E'column list for group by-----> \n    %',_clist_group;
 ---------------------------build column to increment dates---------------------------------
 
 SELECT 
@@ -130,7 +75,7 @@ SELECT
             WHEN _date_funcs ? func THEN
                 CASE 
                     --...but it's not the date itself...
-                    WHEN fkey IS NULL THEN 
+                    WHEN appcol IS NULL THEN 
                         --...pull the associated date field from perd table
                         'perd.'||m.fcol
                     --...and it's the primary key date...
@@ -140,41 +85,29 @@ SELECT
                         'o.'||format('%I',cname)||' + interval '||format('%L',_interval) ||' AS '||format('%I',cname)
                 END
             ELSE
-                CASE appcol 
+                'o.'||format('%I',cname)
-                    ----aggregation columns-----------------------------------------------------------
-                    WHEN 'units'    THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-                    WHEN 'value'    THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-                    WHEN 'cost'     THEN 'sum(o.'||format('%I',cname)||') AS '||format('%I',cname) 
-                    ----create singular values for fingle forecast step-------------------------------
-                    WHEN 'version'  THEN '''app_forecast_name'' AS '||format('%I',_version_col)
-                    WHEN 'iter'     THEN '''baseline'' AS '||format('%I',_iter_col)
-                    WHEN 'logid'    THEN '(SELECT id FROM ilog) AS '||format('%I',_logid_col)
-                    ELSE 
-                        ---if there is no function, make the column null or the default value---
-                        CASE WHEN m.func IS NULL 
-                            THEN 'null::'||m.dtype||' AS '||format('%I',cname)
-                            ELSE 'o.'||format('%I',cname)
-                        END
-                END
         END
         ,E'\n    ,' ORDER BY opos ASC
     )
 INTO
     _clist_inc
 FROM 
-    fc.target_meta m;
+    fc.target_meta m
---WHERE 
+WHERE 
---    COALESCE(appcol,'') NOT IN ('version','iter','logid');
+    COALESCE(appcol,'') NOT IN ('version','iter','logid');
 
-RAISE NOTICE E'DATES INCREMENTED: ------> \n    %',_clist_inc;
+--RAISE NOTICE 'DATES INCREMENTED: %',_clist_inc;
 
 --------------------------------------clone the actual baseline-----------------------------------------------
 
 SELECT 
 $$SELECT
-    $$||
+    $$::text||    
-    _clist_sum||
+    _clist||
     $$
+    ,'app_forecast_name' "version"
+    ,'actuals' iter
+    ,null::bigint app_logid
 FROM
     $$||_target_table||$$ o
 WHERE
@@ -187,13 +120,12 @@ WHERE
         OR ($$||format('%I',_ship_date)||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date)
     )
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-GROUP BY
+$$::text
-    $$||
-    _clist_group
 INTO
     _ytdbody;
 
-RAISE NOTICE E'_ytdbody \n%', _ytdbody;
+--RAISE NOTICE '_ytdbody %', _ytdbody;
+
 
 ------------------------------------pull a plug from actuals to create a full year baseline------------------
 
@@ -201,17 +133,19 @@ SELECT
 $$SELECT
     $$||_clist_inc||
 $$
+    ,'app_forecast_name' "version"
+    ,'plug' iter
+    ,null::bigint app_logid
 FROM
-    $$||_target_table||' o'||E'\n    '||_perd_joins||$$
+    $$||_target_table||' o'||E'\n'||_perd_joins||$$
 WHERE
     $$||format('%I',_order_date)||$$ BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-GROUP BY
+$$ 
-    $$||_clist_group
 INTO
     _actpy;
 
-RAISE NOTICE E'plug body --------> \n%',_actpy;
+--RAISE NOTICE '_actpf %',_actpy;
 ------------------------------copy a full year and increment by 1 year for the baseline-------------------------
 
 SELECT
@@ -219,11 +153,13 @@ SELECT
 --    fc.live
 $$,incr AS (
 SELECT
-    $$||_clist_inc||$$
+    $$||_clist_inc||
+    $$
+    ,'app_forecast_name' "version"
+    ,'baseline' iter
+    ,null::bigint app_logid
 FROM
     baseline o$$||E'\n'||_perd_joins||$$
-GROUP BY
-    $$||_clist_group||$$
 )
 ,ins AS (
 INSERT INTO
@@ -243,32 +179,22 @@ INTO
     _baseline;
     
 
-RAISE NOTICE E'_baseline -------> \n%',_baseline;
+--RAISE NOTICE '_baseline %',_baseline;
 ------------------------------stack the sql into the final format------------------------------------------------
 
 SELECT
 $$DELETE FROM $$||_target_table||$$ WHERE $$||_version_col||$$ = 'app_forecast_name';
 WITH
-req AS  (SELECT $$||'$$app_req$$::jsonb j)'||$$
+baseline AS (
-,ilog AS (
+$$||_ytdbody||
-    INSERT INTO
+$$UNION ALL
-        fc.log (doc)
-    SELECT
-        req.j 
-    FROM
-        req
-    RETURNING *
-)
-,baseline AS (
-$$||_ytdbody||$$
-UNION ALL
 $$||_actpy
 ||$$)
 $$||_baseline
 INTO
     _sql;
 
-RAISE NOTICE E'_sql ---------> \n%',_sql;
+--RAISE NOTICE '_sql %',_sql;
 
 INSERT INTO fc.sql SELECT 'baseline', _sql ON CONFLICT ON CONSTRAINT sql_pkey  DO UPDATE SET t = EXCLUDED.t;
 

setup_sql/03_build_master_tables.sql

@@ -35,7 +35,7 @@ BEGIN
         --HAVING
         --    string_agg(cname,', ') FILTER (WHERE fkey = func) <> ''
     loop 
-        INSERT INTO fc.sql SELECT f.func, f.ddl ON CONFLICT ON CONSTRAINT sql_pkey  DO UPDATE SET t = EXCLUDED.t;
+        INSERT INTO fc.sql SELECT f.func, f.ddl;
         EXECUTE format('%s',f.ddl);
         EXECUTE format('%s',f.pop);
         EXECUTE format('%s',f.fk);