From 0415dad1f254bed7a1e3d9cf80e44f66bb949b97 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 28 Nov 2020 03:00:23 -0500 Subject: [PATCH] start working on scale route, note encountered issues --- readme.md | 9 +- routes/scale/gen_scale.sql | 175 ++--------------- routes/scale/scale_vupd.sql | 378 ++++++++++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+), 165 deletions(-) create mode 100644 routes/scale/scale_vupd.sql diff --git a/readme.md b/readme.md index c94946f..6f46270 100644 --- a/readme.md +++ b/readme.md @@ -28,13 +28,16 @@ route baseline * regular updates to baseline may be required to keep up with canceled/altered orders * copy some period of actual sales and increment all the dates to serve as a baseline forecast -TO-DO: - [x] join to period tables to populate season; requires variance number oof table joins, based on howmany date functions there are 🙄 - [ ] some of the app parameters can be consolidated, the baseline period could be one large range potentially, instead of 2 stacked periods - [x] setup something to fill in sql parameters to do testing on the function - [ ] update node to handle forecast name parameter -- [ ] clean up SQL generation to prevent injection - [ ] calc status is hard-coded right now in the json request -> probably needs to be manuall supplied up front +- [ ] the sales data has to have a column for module + +scale +---------------------------------------------------------------------------------------------------------------------------------------------------- +- [ ] how to handle a target value adjustment, which currency is it in? running problem list ---------------------------------------------------------------------------------------------------------------------------------------------------- @@ -45,3 +48,5 @@ running problem list - [x] problem: the target sales data has to map have concepts like order_date, and the application needs to know which col is order date * add column called application hook - [ ] there is not currently any initial grouping to limit excess data from all the document# scenarios +* general + - [ ] clean up SQL generation to prevent injection diff --git a/routes/scale/gen_scale.sql b/routes/scale/gen_scale.sql index fcb5aa1..2d203cd 100644 --- a/routes/scale/gen_scale.sql +++ b/routes/scale/gen_scale.sql @@ -14,177 +14,26 @@ DECLARE _perd_joins text; _interval interval; -/*----------------parameters listing-------------- -app_baseline_from_date -app_baseline_to_date -app_first_forecast_date -app_openorder_cutoff -app_openstatus_code -app_plug_fromdate -app_plug_todate -------------------------------------------------*/ BEGIN ------------------populate application variables-------------------------------------------- -SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date; -SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date; -SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status; ---the target interval -SELECT interval '1 year' INTO _interval; -SELECT jsonb_agg(func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND fkey is NOT null; ---create table join for each date based func in target_meta joining to fc.perd static table ---the join, though, should be based on the target date, which is needs an interval added to get to the target SELECT - string_agg( - 'LEFT OUTER JOIN fc.perd '||func||' ON'|| - $$ - $$||'(o.'||fkey||' + interval '||format('%L',_interval) ||' )::date <@ '||func||'.drange' - ,E'\n') -INTO - _perd_joins -FROM - fc.target_meta -WHERE - dtype = 'date' - AND fkey IS NOT NULL; - -CREATE TABLE IF NOT EXISTS fc.sql(cmd text PRIMARY KEY, t text ); - --------------------------------build a column list----------------------------------------- -SELECT - string_agg('o.'||format('%I',cname),E'\n ,' ORDER BY opos ASC) -INTO - _clist -FROM - fc.target_meta -WHERE - func NOT IN ('version'); - ----------------------------build column to increment dates--------------------------------- - -SELECT - string_agg( - CASE - --if you're dealing with a date function... - WHEN _date_funcs ? func THEN - CASE - --...but it's not the date itself... - WHEN fkey IS NULL THEN - --...pull the associated date field from perd table - func||'.'||m.dateref - --...and it's the primary key date... - ELSE - --use the date key but increment by the target interval - --this assumes that the primary key for the func is a date, but it has to be or it wont join anyways - 'o.'||fkey||' + interval '||format('%L',_interval) ||' AS '||fkey - END - ELSE - 'o.'||format('%I',cname) - END - ,E'\n ,' ORDER BY opos ASC - ) -INTO - _clist_inc -FROM - fc.target_meta m -WHERE - func NOT IN ('version'); - ---RAISE NOTICE 'build list: %',clist; - ---------------------------------------clone the actual baseline----------------------------------------------- - -SELECT -$$SELECT - $$::text|| - _clist|| - $$ - ,'forecast_name' "version" - ,'actuals' iter -FROM - fc.live o -WHERE - ( - --base period orders booked.... - $$||_order_date||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date - --...or any open orders currently booked before cutoff.... - OR ($$||_order_status||$$ IN (app_openstatus_code) and $$||_order_date||$$ <= 'app_openorder_cutoff'::date) - --...or anything that shipped in that period - OR ($$||_ship_date||$$ BETWEEN 'app_baseline_from_date'::date AND 'app_baseline_to_date'::date) - ) - --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships -$$::text -INTO - _ytdbody; - ---RAISE NOTICE '%', _ytdbody; - - -------------------------------------pull a plug from actuals to create a full year baseline------------------ - -SELECT -$$SELECT - $$||_clist_inc|| -$$ - ,'forecast_name' "version" - ,'plug' iter -FROM - fc.live o$$||E'\n'||_perd_joins||$$ -WHERE - $$||_order_date||$$ BETWEEN 'app_plug_fromdate'::date AND 'app_plug_todate'::date - --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships -$$ -INTO - _actpy; - -------------------------------copy a full year and increment by 1 year for the baseline------------------------- - -SELECT ---$$INSERT INTO --- fc.live -$$,incr AS ( -SELECT - $$||_clist_inc|| - $$ - ,'forecast_name' "version" - ,'baseline' iter -FROM - baseline o$$||E'\n'||_perd_joins||$$ +$$WITH +req AS (SELECT $$||'$$app_req$$::jsonb)'||$$ +test AS ( + SELECT + sum(app_units) FILTER WHERE (version <> 'ACTUALS') total + ,sum(app_units) FILTER (WHERE iter = 'baseline') base + FROM + fc.live + WHERE + app_where ) -,ins AS ( -INSERT INTO - fc.live -SELECT - * -FROM - incr i -WHERE - i.$$||_order_date||$$ >= 'app_first_forecast_date'::date$$||$$ - OR i.$$||_ship_date||$$ >= 'app_first_forecast_date'::date$$||$$ -RETURNING * -) -SELECT COUNT(*) num_rows FROM ins$$ - --any orders in the forecast period, or any sales in the forecast period (from open orders) -INTO - _baseline; - -------------------------------stack the sql into the final format------------------------------------------------ -SELECT -$$DELETE FROM fc.live WHERE version = 'forecast_name'; -WITH -baseline AS ( -$$||_ytdbody|| -$$UNION ALL -$$||_actpy -||$$) -$$||_baseline -INTO - _sql; +SELECT 'HI' into _sql; -INSERT INTO fc.sql SELECT 'baseline', _sql ON CONFLICT ON CONSTRAINT sql_pkey DO UPDATE SET t = EXCLUDED.t; +INSERT INTO fc.sql SELECT 'scale', _sql ON CONFLICT ON CONSTRAINT sql_pkey DO UPDATE SET t = EXCLUDED.t; END $func$; diff --git a/routes/scale/scale_vupd.sql b/routes/scale/scale_vupd.sql new file mode 100644 index 0000000..b25755f --- /dev/null +++ b/routes/scale/scale_vupd.sql @@ -0,0 +1,378 @@ +WITH +target AS (select target_vol vincr, target_prc pincr) +,testv AS ( + SELECT + sum(units) tot + ,sum(units) FILTER (WHERE iter = 'copy') base + ,sum(units) FILTER (WHERE module = 'new basket') newpart + FROM + rlarp.osm_pool + WHERE + -----------------scenario---------------------------- + where_clause + -----------------additional params------------------- + AND calc_status||flag <> 'CLOSEDREMAINDER' --exclude short ships when building order adjustments + AND order_date <= ship_date + +) +,flagv AS ( + SELECT + tot + ,base + ,newpart + ,CASE WHEN tot = 0 THEN + CASE WHEN base = 0 THEN + CASE WHEN newpart = 0 THEN + 'unclean data. tested -> does not exist' + ELSE + 'scale new part' + END + ELSE + 'scale copy' + END + ELSE + 'scale all' + END flag + FROM + testv +) +,basemix AS ( + SELECT + fspr + ,plnt ---master data + ,promo --history date mix + ,terms + ,bill_cust_descr --history cust mix + ,ship_cust_descr --history cust mix + ,dsm + ,quota_rep_descr --master data + ,director + ,billto_group --master data + ,shipto_group + ,chan --master data + ,chansub + ,chan_retail + ,part + ,part_descr + ,part_group + ,branding + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + ,segm + ,substance + ,fs_line --master data + ,r_currency --history cust mix + ,r_rate --master data + ,c_currency --master data + ,c_rate --master data + ,sum(coalesce(units,0)) units --history value + ,sum(coalesce(value_loc,0)) value_loc --history value + ,sum(coalesce(value_usd,0)) value_usd --0 + ,sum(coalesce(cost_loc,0)) cost_loc --history part mix + ,sum(coalesce(cost_usd,0)) cost_usd + ,calc_status --0 + ,flag --0 + ,order_date --history date mix + ,order_month + ,order_season + ,request_date --history date mix + ,request_month + ,request_season + ,ship_date --history date mix + ,ship_month + ,ship_season + FROM + rlarp.osm_pool + WHERE + -----------------scenario---------------------------- + where_clause + -----------------additional params------------------- + AND CASE (SELECT flag FROM flagv) + WHEN 'scale all' THEN true + WHEN 'scale copy' THEN iter = 'copy' + WHEN 'scale new part' THEN module = 'new basket' + END + AND calc_status||flag <> 'CLOSEDREMAINDER' --exclude short ships when building order adjustments + AND order_date <= ship_date + GROUP BY + fspr + ,plnt ---master data + ,promo --history date mix + ,terms + ,bill_cust_descr --history cust mix + ,ship_cust_descr --history cust mix + ,dsm + ,quota_rep_descr --master data + ,director + ,billto_group --master data + ,shipto_group + ,chan --master data + ,chansub + ,chan_retail + ,part + ,part_descr + ,part_group + ,branding + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + ,segm + ,substance + ,fs_line --master data + ,r_currency --history cust mix + ,r_rate --master data + ,c_currency --master data + ,c_rate --master data + ,calc_status --0 + ,flag --0 + ,order_date --history date mix + ,order_month + ,order_season + ,request_date --history date mix + ,request_month + ,request_season + ,ship_date --history date mix + ,ship_month + ,ship_season +) +,vscale AS ( + SELECT + (SELECT vincr::numeric FROM target) incr + ,(SELECT sum(units)::numeric FROM basemix) base + ,(SELECT vincr::numeric FROM target)/(SELECT sum(units)::numeric FROM basemix) factor +) +--select * from vscale +,log AS ( + INSERT INTO rlarp.osm_log(doc) SELECT $$replace_iterdef$$::jsonb doc RETURNING * +) +,volume AS ( + SELECT + fspr + ,plnt ---master data + ,promo --history date mix + ,terms + ,bill_cust_descr --history cust mix + ,ship_cust_descr --history cust mix + ,dsm + ,quota_rep_descr --master data + ,director + ,billto_group --master data + ,shipto_group + ,chan --master data + ,chansub + ,chan_retail + ,part + ,part_descr + ,part_group + ,branding + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + ,segm + ,substance + ,fs_line --master data + ,r_currency --history cust mix + ,r_rate --master data + ,c_currency --master data + ,c_rate --master data + ,units*s.factor units + ,value_loc*s.factor value_loc + ,value_usd*s.factor value_usd + ,cost_loc*s.factor cost_loc + ,cost_usd*s.factor cost_usd + ,calc_status --0 + ,flag --0 + ,order_date --history date mix + ,order_month + ,order_season + ,request_date --history date mix + ,request_month + ,request_season + ,ship_date --history date mix + ,ship_month + ,ship_season + ,'replace_version' "version" + ,'replace_source'||' volume' iter + ,log.id + ,COALESCE(log.doc->>'tag','') "tag" + ,log.doc->>'message' "comment" + ,log.doc->>'type' module +FROM + basemix b + CROSS JOIN vscale s + CROSS JOIN log +) +,pscale AS ( + SELECT + (SELECT pincr::numeric FROM target) incr + ,(SELECT sum(value_loc * r_rate) FROM volume) base + ,CASE WHEN (SELECT sum(value_loc * r_rate) FROM volume) = 0 THEN + --if the base value is -0- scaling will not work, need to generate price, factor goes to -0- + 0 + ELSE + --if the target $amount is not achieved, adjust further + ((SELECT pincr::numeric FROM target)-(SELECT sum(value_loc * r_rate) FROM volume))/(SELECT sum(value_loc * r_rate) FROM volume) + END factor + ,CASE WHEN (SELECT sum(value_loc * r_rate) FROM volume) = 0 THEN + CASE WHEN ((SELECT pincr::numeric FROM target) - (SELECT sum(value_loc * r_rate) FROM volume)) <> 0 THEN + --if the base value is -0- but the target value hasn't been achieved, derive a price to apply + ((SELECT pincr::numeric FROM target) - (SELECT sum(value_loc * r_rate) FROM volume))/(SELECT sum(units) FROM volume) + ELSE + 0 + END + ELSE + 0 + END mod_price +) +--select * from pscale +,pricing AS ( + SELECT + fspr + ,plnt ---master data + ,promo --history date mix + ,terms + ,bill_cust_descr --history cust mix + ,ship_cust_descr --history cust mix + ,dsm + ,quota_rep_descr --master data + ,director + ,billto_group --master data + ,shipto_group + ,chan --master data + ,chansub + ,chan_retail + ,part + ,part_descr + ,part_group + ,branding + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + ,segm + ,substance + ,fs_line --master data + ,r_currency --history cust mix + ,r_rate --master data + ,c_currency --master data + ,c_rate --master data + ,0::numeric units + ,(CASE WHEN s.factor = 0 THEN b.units * s.mod_price/b.r_rate ELSE b.value_loc*s.factor END)::numeric value_loc + ,(CASE WHEN s.factor = 0 THEN b.units * s.mod_price ELSE b.value_usd*s.factor END)::numeric value_usd + ,0::numeric cost_loc + ,0::numeric cost_usd + ,calc_status --0 + ,flag --0 + ,order_date --history date mix + ,order_month + ,order_season + ,request_date --history date mix + ,request_month + ,request_season + ,ship_date --history date mix + ,ship_month + ,ship_season + ,'replace_version' "version" + ,'replace_source'||' price' iter + ,log.id + ,COALESCE(log.doc->>'tag','') "tag" + ,log.doc->>'message' "comment" + ,log.doc->>'type' module + FROM + volume b + CROSS JOIN pscale s + CROSS JOIN log + WHERE + s.factor <> 0 or s.mod_price <> 0 +) +--select sum(value_usd), sum(units) from pricing +, ins AS ( + INSERT INTO rlarp.osm_pool (SELECT * FROM pricing UNION ALL SELECT * FROM volume) RETURNING * +) +,insagg AS ( + SELECT + ---------customer info----------------- + bill_cust_descr + ,billto_group + ,ship_cust_descr + ,shipto_group + ,quota_rep_descr + ,director + ,segm + ,substance + ,chan + ,chansub + ---------product info------------------ + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + --,brand + --,part_family + ,part_group + ,branding + --,color + ,part_descr + ---------dates------------------------- + ,order_season + ,order_month + ,ship_season + ,ship_month + ,request_season + ,request_month + ,promo + ,version + ,iter + ,logid + ,tag + ,comment + --------values------------------------- + ,sum(value_loc) value_loc + ,sum(value_usd) value_usd + ,sum(cost_loc) cost_loc + ,sum(cost_usd) cost_usd + ,sum(units) units + FROM + ins + GROUP BY + ---------customer info----------------- + bill_cust_descr + ,billto_group + ,ship_cust_descr + ,shipto_group + ,quota_rep_descr + ,director + ,segm + ,substance + ,chan + ,chansub + ---------product info------------------ + ,majg_descr + ,ming_descr + ,majs_descr + ,mins_descr + --,brand + --,part_family + ,part_group + ,branding + --,color + ,part_descr + ---------dates------------------------- + ,order_season + ,order_month + ,ship_season + ,ship_month + ,request_season + ,request_month + ,promo + ,version + ,iter + ,logid + ,tag + ,comment +) +SELECT json_agg(row_to_json(insagg)) x from insagg