start working on scale route, note encountered issues

readme.md

@@ -28,13 +28,16 @@ route baseline
 * regular updates to baseline may be required to keep up with canceled/altered orders
 * copy some period of actual sales and increment all the dates to serve as a baseline forecast
 
-TO-DO:
 - [x] join to period tables to populate season; requires variance number oof table joins, based on howmany date functions there are 🙄
 - [ ] some of the app parameters can be consolidated, the baseline period could be one large range potentially, instead of 2 stacked periods
 - [x] setup something to fill in sql parameters to do testing on the function
 - [ ] update node to handle forecast name parameter
-- [ ] clean up SQL generation to prevent injection
 - [ ] calc status is hard-coded right now in the json request -> probably needs to be manuall supplied up front
+- [ ] the sales data has to have a column for module
+
+scale
+----------------------------------------------------------------------------------------------------------------------------------------------------
+- [ ] how to handle a target value adjustment, which currency is it in?
 
 running problem list
 ----------------------------------------------------------------------------------------------------------------------------------------------------
@@ -45,3 +48,5 @@ running problem list
     - [x] problem: the target sales data has to map have concepts like order_date, and the application needs to know which col is order date
         * add column called application hook
     - [ ] there is not currently any initial grouping to limit excess data from all the document# scenarios
+* general
+    - [ ] clean up SQL generation to prevent injection

routes/scale/gen_scale.sql

@@ -14,177 +14,26 @@ DECLARE
     _perd_joins text;
     _interval interval;
 
-/*----------------parameters listing--------------
-app_baseline_from_date
-app_baseline_to_date
-app_first_forecast_date
-app_openorder_cutoff
-app_openstatus_code
-app_plug_fromdate
-app_plug_todate
-------------------------------------------------*/
 
 BEGIN
 
------------------populate application variables--------------------------------------------
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date;
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date;
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status;
---the target interval
-SELECT interval '1 year' INTO _interval;
-SELECT jsonb_agg(func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND fkey is NOT null;
---create table join for each date based func in target_meta joining to fc.perd static table
---the join, though, should be based on the target date, which is needs an interval added to get to the target
 SELECT
-    string_agg(
+$$WITH
-        'LEFT OUTER JOIN fc.perd '||func||' ON'||
+req AS  (SELECT $$||'$$app_req$$::jsonb)'||$$
-        $$
+test AS (
-        $$||'(o.'||fkey||' + interval '||format('%L',_interval) ||' )::date <@ '||func||'.drange'
+    SELECT
-    ,E'\n')
+        sum(app_units) FILTER WHERE (version <> 'ACTUALS') total
-INTO
+        ,sum(app_units) FILTER (WHERE iter = 'baseline') base
-    _perd_joins
+    FROM
-FROM 
+        fc.live
-    fc.target_meta 
+    WHERE
-WHERE 
+        app_where
-    dtype = 'date' 
-    AND fkey IS NOT NULL;
-
-CREATE TABLE IF NOT EXISTS fc.sql(cmd text PRIMARY KEY, t text );
-
--------------------------------build a column list-----------------------------------------
-SELECT 
-    string_agg('o.'||format('%I',cname),E'\n    ,' ORDER BY opos ASC)
-INTO
-    _clist
-FROM 
-    fc.target_meta 
-WHERE 
-    func NOT IN ('version');
-
----------------------------build column to increment dates---------------------------------
-
-SELECT 
-    string_agg(
-        CASE
-            --if you're dealing with a date function...
-            WHEN _date_funcs ? func THEN
-                CASE 
-                    --...but it's not the date itself...
-                    WHEN fkey IS NULL THEN 
-                        --...pull the associated date field from perd table
-                        func||'.'||m.dateref
-                    --...and it's the primary key date...
-                    ELSE 
-                        --use the date key but increment by the target interval
-                        --this assumes that the primary key for the func is a date, but it has to be or it wont join anyways
-                        'o.'||fkey||' + interval '||format('%L',_interval) ||' AS '||fkey
-                END
-            ELSE
-                'o.'||format('%I',cname)
-        END
-        ,E'\n    ,' ORDER BY opos ASC
-    )
-INTO
-    _clist_inc
-FROM 
-    fc.target_meta m
-WHERE 
-    func NOT IN ('version');
-
---RAISE NOTICE 'build list: %',clist;
-
---------------------------------------clone the actual baseline-----------------------------------------------
-
-SELECT 
-$$SELECT
-    $$::text||    
-    _clist||
-    $$
-    ,'forecast_name' "version"
-    ,'actuals' iter
-FROM
-    fc.live o
-WHERE
-    (
-        --base period orders booked....
-        $$||_order_date||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date
-        --...or any open orders currently booked before cutoff....
-        OR ($$||_order_status||$$ IN (app_openstatus_code) and $$||_order_date||$$ <= 'app_openorder_cutoff'::date)
-        --...or anything that shipped in that period
-        OR ($$||_ship_date||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date)
-    )
-    --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-$$::text
-INTO
-    _ytdbody;
-
---RAISE NOTICE '%', _ytdbody;
-
-
-------------------------------------pull a plug from actuals to create a full year baseline------------------
-
-SELECT
-$$SELECT
-    $$||_clist_inc||
-$$
-    ,'forecast_name' "version"
-    ,'plug' iter
-FROM
-    fc.live o$$||E'\n'||_perd_joins||$$
-WHERE
-    $$||_order_date||$$ BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date
-    --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-$$ 
-INTO
-    _actpy;
-
-------------------------------copy a full year and increment by 1 year for the baseline-------------------------
-
-SELECT
---$$INSERT INTO 
---    fc.live
-$$,incr AS (
-SELECT
-    $$||_clist_inc||
-    $$
-    ,'forecast_name' "version"
-    ,'baseline' iter
-FROM
-    baseline o$$||E'\n'||_perd_joins||$$
 )
-,ins AS (
-INSERT INTO
-    fc.live
-SELECT
-    *
-FROM
-    incr i
-WHERE
-    i.$$||_order_date||$$ >= 'app_first_forecast_date'::date$$||$$
-    OR i.$$||_ship_date||$$ >= 'app_first_forecast_date'::date$$||$$
-RETURNING *
-)
-SELECT COUNT(*) num_rows  FROM ins$$
-    --any orders in the forecast period, or any sales in the forecast period (from open orders)
-INTO
-    _baseline;
 
 
-------------------------------stack the sql into the final format------------------------------------------------
+SELECT 'HI' into    _sql;
 
-SELECT
+INSERT INTO fc.sql SELECT 'scale', _sql ON CONFLICT ON CONSTRAINT sql_pkey  DO UPDATE SET t = EXCLUDED.t;
-$$DELETE FROM fc.live WHERE version = 'forecast_name';
-WITH
-baseline AS (
-$$||_ytdbody||
-$$UNION ALL
-$$||_actpy
-||$$)
-$$||_baseline
-INTO
-    _sql;
-
-INSERT INTO fc.sql SELECT 'baseline', _sql ON CONFLICT ON CONSTRAINT sql_pkey  DO UPDATE SET t = EXCLUDED.t;
 
 END
 $func$;

routes/scale/scale_vupd.sql

@@ -0,0 +1,378 @@
+WITH 
+target AS (select target_vol vincr, target_prc pincr)
+,testv AS (
+    SELECT
+        sum(units) tot
+        ,sum(units) FILTER (WHERE iter = 'copy') base
+        ,sum(units) FILTER (WHERE module = 'new basket') newpart
+    FROM
+        rlarp.osm_pool
+    WHERE
+        -----------------scenario----------------------------
+        where_clause
+        -----------------additional params-------------------
+        AND calc_status||flag <> 'CLOSEDREMAINDER' --exclude short ships when building order adjustments
+        AND order_date <= ship_date
+
+)
+,flagv AS (
+    SELECT
+        tot
+        ,base
+        ,newpart
+        ,CASE WHEN tot = 0 THEN
+            CASE WHEN base = 0 THEN
+                CASE WHEN newpart = 0 THEN
+                    'unclean data. tested -> does not exist'
+                ELSE
+                    'scale new part'
+                END
+            ELSE
+                'scale copy'
+            END
+        ELSE
+            'scale all'
+        END flag
+    FROM
+        testv
+)
+,basemix AS (
+    SELECT
+        fspr
+        ,plnt          ---master data 
+        ,promo          --history date mix
+        ,terms
+        ,bill_cust_descr          --history cust mix
+        ,ship_cust_descr          --history cust mix
+        ,dsm
+        ,quota_rep_descr          --master data 
+        ,director
+        ,billto_group          --master data 
+        ,shipto_group
+        ,chan          --master data 
+        ,chansub
+        ,chan_retail
+        ,part
+        ,part_descr
+        ,part_group
+        ,branding
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        ,segm
+        ,substance
+        ,fs_line          --master data 
+        ,r_currency          --history cust mix
+        ,r_rate          --master data 
+        ,c_currency          --master data 
+        ,c_rate          --master data 
+        ,sum(coalesce(units,0)) units          --history value
+        ,sum(coalesce(value_loc,0)) value_loc          --history value
+        ,sum(coalesce(value_usd,0)) value_usd          --0 
+        ,sum(coalesce(cost_loc,0)) cost_loc          --history part mix
+        ,sum(coalesce(cost_usd,0)) cost_usd
+        ,calc_status          --0 
+        ,flag          --0 
+        ,order_date          --history date mix
+        ,order_month
+        ,order_season
+        ,request_date          --history date mix
+        ,request_month
+        ,request_season
+        ,ship_date          --history date mix
+        ,ship_month
+        ,ship_season
+    FROM
+        rlarp.osm_pool
+    WHERE
+        -----------------scenario----------------------------
+        where_clause
+        -----------------additional params-------------------
+        AND CASE (SELECT flag FROM flagv) 
+                WHEN 'scale all' THEN true
+                WHEN 'scale copy' THEN iter = 'copy'
+                WHEN 'scale new part' THEN module = 'new basket'
+            END
+        AND calc_status||flag <> 'CLOSEDREMAINDER' --exclude short ships when building order adjustments
+        AND order_date <= ship_date
+    GROUP BY
+                fspr
+        ,plnt          ---master data 
+        ,promo          --history date mix
+        ,terms
+        ,bill_cust_descr          --history cust mix
+        ,ship_cust_descr          --history cust mix
+        ,dsm
+        ,quota_rep_descr          --master data 
+        ,director
+        ,billto_group          --master data 
+        ,shipto_group
+        ,chan          --master data 
+        ,chansub
+        ,chan_retail
+        ,part
+        ,part_descr
+        ,part_group
+        ,branding
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        ,segm
+        ,substance
+        ,fs_line          --master data 
+        ,r_currency          --history cust mix
+        ,r_rate          --master data 
+        ,c_currency          --master data 
+        ,c_rate          --master data 
+        ,calc_status          --0 
+        ,flag          --0 
+        ,order_date          --history date mix
+        ,order_month
+        ,order_season
+        ,request_date          --history date mix
+        ,request_month
+        ,request_season
+        ,ship_date          --history date mix
+        ,ship_month
+        ,ship_season
+)
+,vscale AS (
+    SELECT
+        (SELECT vincr::numeric FROM target) incr
+        ,(SELECT sum(units)::numeric FROM basemix) base
+        ,(SELECT vincr::numeric FROM target)/(SELECT sum(units)::numeric FROM basemix) factor
+)
+--select * from vscale
+,log AS (
+    INSERT INTO rlarp.osm_log(doc) SELECT $$replace_iterdef$$::jsonb doc RETURNING *
+)
+,volume AS (
+    SELECT
+        fspr
+        ,plnt          ---master data 
+        ,promo          --history date mix
+        ,terms
+        ,bill_cust_descr          --history cust mix
+        ,ship_cust_descr          --history cust mix
+        ,dsm
+        ,quota_rep_descr          --master data 
+        ,director
+        ,billto_group          --master data 
+        ,shipto_group
+        ,chan          --master data 
+        ,chansub
+        ,chan_retail
+        ,part
+        ,part_descr
+        ,part_group
+        ,branding
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        ,segm
+        ,substance
+        ,fs_line          --master data 
+        ,r_currency          --history cust mix
+        ,r_rate          --master data 
+        ,c_currency          --master data 
+        ,c_rate          --master data 
+        ,units*s.factor units
+        ,value_loc*s.factor value_loc
+        ,value_usd*s.factor value_usd
+        ,cost_loc*s.factor cost_loc
+        ,cost_usd*s.factor cost_usd
+        ,calc_status          --0 
+        ,flag          --0 
+        ,order_date          --history date mix
+        ,order_month
+        ,order_season
+        ,request_date          --history date mix
+        ,request_month
+        ,request_season
+        ,ship_date          --history date mix
+        ,ship_month
+        ,ship_season
+        ,'replace_version' "version"
+        ,'replace_source'||' volume'  iter
+        ,log.id
+        ,COALESCE(log.doc->>'tag','') "tag"
+        ,log.doc->>'message' "comment"
+        ,log.doc->>'type' module
+FROM
+    basemix b
+    CROSS JOIN vscale s
+    CROSS JOIN log
+)
+,pscale AS (
+    SELECT
+        (SELECT pincr::numeric FROM target) incr
+        ,(SELECT sum(value_loc * r_rate) FROM volume) base
+        ,CASE WHEN (SELECT sum(value_loc * r_rate) FROM volume) = 0 THEN
+            --if the base value is -0- scaling will not work, need to generate price, factor goes to -0-
+            0
+        ELSE
+            --if the target $amount is not achieved, adjust further
+            ((SELECT pincr::numeric FROM target)-(SELECT sum(value_loc * r_rate) FROM volume))/(SELECT sum(value_loc * r_rate) FROM volume)
+        END factor
+        ,CASE WHEN (SELECT sum(value_loc * r_rate) FROM volume) = 0 THEN
+            CASE WHEN ((SELECT pincr::numeric FROM target) - (SELECT sum(value_loc * r_rate) FROM volume)) <> 0 THEN
+                --if the base value is -0- but the target value hasn't been achieved, derive a price to apply
+                ((SELECT pincr::numeric FROM target) - (SELECT sum(value_loc * r_rate) FROM volume))/(SELECT sum(units) FROM volume)
+            ELSE
+                0
+            END
+        ELSE
+            0
+        END mod_price
+)
+--select * from pscale
+,pricing AS (
+    SELECT
+        fspr
+        ,plnt          ---master data 
+        ,promo          --history date mix
+        ,terms
+        ,bill_cust_descr          --history cust mix
+        ,ship_cust_descr          --history cust mix
+        ,dsm
+        ,quota_rep_descr          --master data 
+        ,director
+        ,billto_group          --master data 
+        ,shipto_group
+        ,chan          --master data 
+        ,chansub
+        ,chan_retail
+        ,part
+        ,part_descr
+        ,part_group
+        ,branding
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        ,segm
+        ,substance
+        ,fs_line          --master data 
+        ,r_currency          --history cust mix
+        ,r_rate          --master data 
+        ,c_currency          --master data 
+        ,c_rate          --master data 
+        ,0::numeric units
+        ,(CASE WHEN s.factor = 0 THEN b.units * s.mod_price/b.r_rate ELSE b.value_loc*s.factor END)::numeric value_loc
+        ,(CASE WHEN s.factor = 0 THEN b.units * s.mod_price ELSE b.value_usd*s.factor END)::numeric value_usd
+        ,0::numeric cost_loc
+        ,0::numeric cost_usd
+        ,calc_status          --0 
+        ,flag          --0 
+        ,order_date          --history date mix
+        ,order_month
+        ,order_season
+        ,request_date          --history date mix
+        ,request_month
+        ,request_season
+        ,ship_date          --history date mix
+        ,ship_month
+        ,ship_season
+        ,'replace_version' "version"
+        ,'replace_source'||' price'  iter
+        ,log.id
+        ,COALESCE(log.doc->>'tag','') "tag"
+        ,log.doc->>'message' "comment"
+        ,log.doc->>'type' module
+    FROM
+        volume b
+        CROSS JOIN pscale s
+        CROSS JOIN log
+    WHERE
+        s.factor <> 0 or s.mod_price <> 0
+)
+--select sum(value_usd), sum(units) from pricing
+, ins AS (
+    INSERT INTO rlarp.osm_pool (SELECT * FROM pricing UNION ALL SELECT * FROM volume) RETURNING *
+)
+,insagg AS (
+    SELECT
+        ---------customer info-----------------
+        bill_cust_descr
+        ,billto_group
+        ,ship_cust_descr
+        ,shipto_group
+        ,quota_rep_descr
+        ,director
+        ,segm
+        ,substance
+        ,chan
+        ,chansub
+        ---------product info------------------
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        --,brand
+        --,part_family
+        ,part_group
+        ,branding
+        --,color
+        ,part_descr
+        ---------dates-------------------------
+        ,order_season
+        ,order_month
+        ,ship_season
+        ,ship_month
+        ,request_season
+        ,request_month
+        ,promo
+        ,version
+        ,iter
+        ,logid
+        ,tag
+        ,comment
+        --------values-------------------------
+        ,sum(value_loc) value_loc
+        ,sum(value_usd) value_usd
+        ,sum(cost_loc) cost_loc
+        ,sum(cost_usd) cost_usd
+        ,sum(units) units
+    FROM
+        ins
+    GROUP BY
+        ---------customer info-----------------
+        bill_cust_descr
+        ,billto_group
+        ,ship_cust_descr
+        ,shipto_group
+        ,quota_rep_descr
+        ,director
+        ,segm
+        ,substance
+        ,chan
+        ,chansub
+        ---------product info------------------
+        ,majg_descr
+        ,ming_descr
+        ,majs_descr
+        ,mins_descr
+        --,brand
+        --,part_family
+        ,part_group
+        ,branding
+        --,color
+        ,part_descr
+        ---------dates-------------------------
+        ,order_season
+        ,order_month
+        ,ship_season
+        ,ship_month
+        ,request_season
+        ,request_month
+        ,promo
+        ,version
+        ,iter
+        ,logid
+        ,tag
+        ,comment
+)
+SELECT json_agg(row_to_json(insagg)) x from insagg