vault backup: 2024-09-28 17:13:57

database/reports/all_map_return_values.sql

@@ -1,13 +1,5 @@
-/*
-first get distinct target json values
-then apply regex
-*/
-
-
 WITH
-
+--------------------apply regex operations to transactions-----------------------------------------------------------------------------------
---------------------apply regex operations to transactions---------------------------------------------------------------------------------
-
 rx AS (
 SELECT 
     t.srce,
@@ -15,28 +7,26 @@ SELECT
     t.rec,
     m.target,
     m.seq,
-    regex->>'function' regex_function,
+    regex->'regex'->>'function' regex_function,
-    e.v ->> 'field' result_key_name,
+    defn.v ->> 'field'          result_key_name,
-    e.v ->> 'key' target_json_path,
+    defn.v ->> 'key'            target_json_path,
-    e.v ->> 'flag' regex_options_flag,
+    defn.v ->> 'flag'           regex_options_flag,
-    e.v->>'map' map_intention,
+    defn.v->>'map'              map_intention,
-    e.v->>'retain' retain_result,
+    defn.v->>'retain'           retain_result,
-    e.v->>'regex' regex_expression,
+    defn.v->>'regex'            regex_expression,
-    e.rn target_item_number,
+    defn.rn                     target_item_number,
-    COALESCE(mt.rn,rp.rn,1) result_number,
+    COALESCE(mt.rn,rp.rn,1)     result_number,
-    mt.mt rx_match,
+    mt.mt                       rx_match,
-    rp.rp rx_replace,
+    rp.rp                       rx_replace,
-    --------------------------json key name assigned to return value-----------------------------------------------------------------------
+    CASE defn.v->>'map'
-    CASE e.v->>'map'
         WHEN 'y' THEN
-            e.v->>'field'
+            defn.v->>'field'
         ELSE
             null
-    END map_key,
+    END                         map_key,
-    --------------------------json value resulting from regular expression-----------------------------------------------------------------
+    CASE defn.v->>'map'
-    CASE e.v->>'map'
         WHEN 'y' THEN
-            CASE regex->>'function'
+            CASE regex->'regex'->>'function'
                 WHEN 'extract' THEN
                     CASE WHEN array_upper(mt.mt,1)=1 
                         THEN to_json(mt.mt[1])
@@ -49,18 +39,16 @@ SELECT
             END
         ELSE
             NULL
-    END map_val,
+    END                         map_val,
-    --------------------------flag for if retruned regex result is stored as a new part of the final json output---------------------------
+    CASE defn.v->>'retain'
-    CASE e.v->>'retain'
         WHEN 'y' THEN
-            e.v->>'field'
+            defn.v->>'field'
         ELSE
             NULL
-    END retain_key,
+    END                         retain_key,
-    --------------------------push regex result into json object---------------------------------------------------------------------------
+    CASE defn.v->>'retain'
-    CASE e.v->>'retain'
         WHEN 'y' THEN
-            CASE regex->>'function'
+            CASE regex->'regex'->>'function'
                 WHEN 'extract' THEN
                     CASE WHEN array_upper(mt.mt,1)=1 
                         THEN to_json(trim(mt.mt[1]))
@@ -73,42 +61,37 @@ SELECT
             END
         ELSE
             NULL
-    END retain_val
+    END                         retain_val
 FROM 
     --------------------------start with all regex maps------------------------------------------------------------------------------------
     tps.map_rm m
     --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
-    JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') wh(v) ON TRUE
-    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
-    JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
     --------------------------join to main transaction table but only certain key/values are included--------------------------------------
     INNER JOIN tps.trans t ON 
         t.srce = m.srce AND
-        t.rec @> w.v
+        t.rec @> wh.v
+    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
     --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
-    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text,COALESCE(defn.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
-        m.regex->>'function' = 'extract'
+        m.regex->'regex'->>'function' = 'extract'
     --------------------------same as above but for a replacement type function------------------------------------------------------------
-    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text, defn.v ->> 'replace'::text,defn.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
-        m.regex->>'function' = 'replace'
+        m.regex->'regex'->>'function' = 'replace'
 WHERE
     --t.allj IS NULL
-    --t.srce = 'PNCC' AND
+    t.srce = 'dcard'
-    e.v @> '{"map":"y"}'::jsonb
     --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
     --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
-/*
 ORDER BY 
     t.id DESC,
     m.target,
-    e.rn,
+    defn.rn,
     COALESCE(mt.rn,rp.rn,1)
-*/
 )
-
+--SELECT * FROM rx
---SELECT * FROM rx LIMIT 100
+--SELECT count(*) FROM rx LIMIT 100
-
-
 , agg_to_target_items AS (
 SELECT 
     srce
@@ -163,10 +146,6 @@ GROUP BY
     ,map_key
     ,retain_key
 )
-
---SELECT * FROM agg_to_target_items LIMIT 100
-
-
 , agg_to_target AS (
 SELECT
     srce
@@ -184,62 +163,27 @@ GROUP BY
     ,target
     ,seq
     ,map_intention
+ORDER BY
+    id
 )
-
-
-, agg_to_ret AS (
-SELECT
-	srce
-	,target
-	,seq
-	,map_intention
-	,map_val
-	,retain_val
-	,count(*) "count"
-FROM 
-	agg_to_target
-GROUP BY
-	srce
-	,target
-	,seq
-	,map_intention
-	,map_val
-	,retain_val
-)
-
 , link_map AS (
 SELECT
     a.srce
+    ,a.id
     ,a.target
     ,a.seq
     ,a.map_intention
     ,a.map_val
-    ,a."count"
+    ,a.retain_val retain_value
-    ,a.retain_val
+    ,v.map
-    ,v.map mapped_val
 FROM
-    agg_to_ret a
+    agg_to_target a
     LEFT OUTER JOIN tps.map_rv v ON
         v.srce = a.srce AND
         v.target = a.target AND
         v.retval = a.map_val
 )
-
+--SELECT * FROM rx
-SELECT
+--SELECT * FROM agg_to_target_items
-    l.srce
+--SELECT * FROM agg_to_target
-    ,l.target
+SELECT * FROM link_map
-    ,l.seq
-    ,l.map_intention
-    ,l.map_val
-    ,l."count"
-    ,l.retain_val
-    ,l.mapped_val
-FROM
-    link_map l
-ORDER BY
-    l.srce
-    ,l.target
-    ,l.seq
-    ,l."count" desc
-    ,l.map_val
-    ,l.mapped_val

database/reports/list_maps.sql

@@ -1,19 +1,21 @@
 SELECT 
-    m.srce,
+     m.srce
-    m.target,
+    ,m.target
-    regex->>'function' regex_function,
+    ,m.regex->'regex'->>'function' func_name
-    regex->>'where' where_clause,
+    ,m.regex->'regex'->>'description' descr
-    e.v ->> 'field' result_key_name,
+    ,filter.v  filters
-    e.v ->> 'key' target_json_path,
+    ,defn.v->> 'field' result_key_name
-    e.v ->> 'flag' regex_options_flag,
+    ,defn.v->> 'key' target_json_path
-    e.v->>'map' map_intention,
+    ,defn.v->>'regex' regex_expression
-    e.v->>'retain' retain_result,
+    ,defn.v->> 'flag' regex_options_flag
-    e.v->>'regex' regex_expression,
+    ,defn.v->>'map' map_intention
-    e.rn target_item_number
+    ,defn.v->>'retain' retain_result
+    ,defn.rn target_item_number
 FROM 
     tps.map_rm m
-    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') WITH ORDINALITY filter(v, rn) ON true
 ORDER BY 
-    m.srce,
+     m.srce
-    m.target,
+    ,m.target
-    e.rn
+    ,defn.rn

database/reports/stacked.sql

@@ -61,6 +61,7 @@ dcard AS (
     UNION ALL
     SELECT * FROM chase 
 )
+,format AS (
 SELECT 
      s.srce
     ,s.id
@@ -81,3 +82,5 @@ FROM
 ORDER BY 
     s.pdate desc
     ,s.id desc
+)
+SELECT * FROM format