functional but only 10k rows something wrong

generate_sql/gen_baseline.sql

@@ -10,14 +10,40 @@ DECLARE
     _actpy text;
     _sql text;
     _baseline text;
+    _date_funcs jsonb;
+    _perd_joins text;
+    _interval interval;
 
 BEGIN
 
+-----------------populate application variables--------------------------------------------
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date;
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date;
+SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status;
+--the target interval
+SELECT interval '1 year' INTO _interval;
+SELECT jsonb_agg(func) INTO _date_funcs FROM fc.target_meta WHERE dtype = 'date' AND fkey is NOT null;
+--create table join for each date based func in target_meta joining to fc.perd static table
+--the join, though, should be based on the target date, which is needs an interval added to get to the target
+SELECT
+    string_agg(
+        'LEFT OUTER JOIN fc.perd '||func||' ON'||
+        $$
+        $$||'(o.'||fkey||' + interval '||format('%L',_interval) ||' )::date <@ '||func||'.drange'
+    ,E'\n')
+INTO
+    _perd_joins
+FROM 
+    fc.target_meta 
+WHERE 
+    dtype = 'date' 
+    AND fkey IS NOT NULL;
+
 CREATE TABLE IF NOT EXISTS fc.sql(cmd text PRIMARY KEY, t text );
 
--------------------------------build a column list----------------------------------------
+-------------------------------build a column list-----------------------------------------
 SELECT 
-    string_agg(format('%I',cname),E'\n    ,' ORDER BY opos ASC)
+    string_agg('o.'||format('%I',cname),E'\n    ,' ORDER BY opos ASC)
 INTO
     _clist
 FROM 
@@ -25,46 +51,60 @@ FROM
 WHERE 
     func NOT IN ('version');
 
----------------------------build column to increment dates--------------------------------
+---------------------------build column to increment dates---------------------------------
 
 SELECT 
     string_agg(
-        format('%I',cname) || CASE WHEN func IN ('odate','sdate') AND dtype = 'date' THEN ' + interval ''1 year''' ELSE '' END,E'\n    ,' ORDER BY opos ASC)
+        CASE
+            --if you're dealing with a date function...
+            WHEN _date_funcs ? func THEN
+                CASE 
+                    --...but it's not the date itself...
+                    WHEN fkey IS NULL THEN 
+                        --...pull the associated date field from perd table
+                        func||'.'||m.dateref
+                    --...and it's the primary key date...
+                    ELSE 
+                        --use the date key but increment by the target interval
+                        --this assumes that the primary key for the func is a date, but it has to be or it wont join anyways
+                        'o.'||fkey||' + interval '||format('%L',_interval)
+                END
+            ELSE
+                'o.'||format('%I',cname)
+        END
+        ,E'\n    ,' ORDER BY opos ASC
+    )
 INTO
     _clist_inc
 FROM 
-    fc.target_meta 
+    fc.target_meta m
 WHERE 
     func NOT IN ('version');
 
 --RAISE NOTICE 'build list: %',clist;
 
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_date') INTO _order_date;
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'ship_date') INTO _ship_date;
-SELECT (SELECT cname FROM fc.target_meta WHERE appcol = 'order_status') INTO _order_status;
-
 --------------------------------------clone the actual baseline-----------------------------------------------
 
 SELECT 
-$a$SELECT
-    $a$::text||    
+$$SELECT
+    $$::text||    
     _clist||
-    $b$
+    $$
     ,'forecast_name' "version"
     ,'actuals' iter
 FROM
-    rlarp.osm_dev o
+    fc.live o
 WHERE
     (
         --base period orders booked....
-        $b$||_order_date||$c$ BETWEEN [app_baseline_from_date] AND [app_baseline_to_date]
+        $$||_order_date||$$ BETWEEN [app_baseline_from_date] AND [app_baseline_to_date]
         --...or any open orders currently booked before cutoff....
-        OR ($c$||_order_status||$d$ IN ([app_openstatus_code]) and $d$||_order_date||$e$ <= [app_openorder_cutoff])
+        OR ($$||_order_status||$$ IN ([app_openstatus_code]) and $$||_order_date||$$ <= [app_openorder_cutoff])
         --...or anything that shipped in that period
-        OR ($e$||_ship_date||$f$ BETWEEN [app_baseline_from_date] AND [app_baseline_to_date])
+        OR ($$||_ship_date||$$ BETWEEN [app_baseline_from_date] AND [app_baseline_to_date])
     )
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
-$f$::text
+$$::text
 INTO
     _ytdbody;
 
@@ -80,7 +120,7 @@ $$
     ,'forecast_name' "version"
     ,'plug' iter
 FROM
-    rlarp.osm_dev o
+    fc.live o$$||E'\n'||_perd_joins||$$
 WHERE
     $$||_order_date||$$ BETWEEN [app_plug_fromdate] AND [app_plug_todate]
     --be sure to pre-exclude unwanted items, like canceled orders, non-gross sales, and short-ships
@@ -91,20 +131,19 @@ INTO
 ------------------------------copy a full year and increment by 1 year for the baseline-------------------------
 
 SELECT
-$a$
-INSERT INTO 
+$$INSERT INTO 
     fc.live
 SELECT
-    $a$||_clist_inc||
-    $b$
+    $$||_clist_inc||
+    $$
     ,'forecast_name' "version"
     ,'baseline' iter
 FROM
-    baseline
+    baseline o$$||E'\n'||_perd_joins||$$
 WHERE
-    $b$||_order_date||$c$ + interval '1 year' >= $c$||'[app_first_order_date_year]'
-    --the final forecast baseline should have orders greater than or equal to the
-    --start of the year since new orders is the intended forecast
+    $$||_order_date||' >= [app_first_forecast_date]'||$$
+    OR $$||_ship_date||' >= [app_first_forecast_date]'
+    --any orders in the forecast period, or any sales in the forecast period (from open orders)
 INTO
     _baseline;
     
@@ -119,8 +158,7 @@ $$||_ytdbody||
 $$UNION ALL
 $$||_actpy
 ||$$)
-$$
-||_baseline
+$$||_baseline
 INTO
     _sql;