database/reports/all_map_return_values.sql

@@ -1,5 +1,13 @@
+/*
+first get distinct target json values
+then apply regex
+*/
+
+
 WITH
---------------------apply regex operations to transactions-----------------------------------------------------------------------------------
+
+--------------------apply regex operations to transactions---------------------------------------------------------------------------------
+
 rx AS (
 SELECT 
     t.srce,
@@ -7,26 +15,28 @@ SELECT
     t.rec,
     m.target,
     m.seq,
-    regex->'regex'->>'function' regex_function,
-    defn.v ->> 'field'          result_key_name,
-    defn.v ->> 'key'            target_json_path,
-    defn.v ->> 'flag'           regex_options_flag,
-    defn.v->>'map'              map_intention,
-    defn.v->>'retain'           retain_result,
-    defn.v->>'regex'            regex_expression,
-    defn.rn                     target_item_number,
-    COALESCE(mt.rn,rp.rn,1)     result_number,
-    mt.mt                       rx_match,
-    rp.rp                       rx_replace,
-    CASE defn.v->>'map'
+    regex->>'function' regex_function,
+    e.v ->> 'field' result_key_name,
+    e.v ->> 'key' target_json_path,
+    e.v ->> 'flag' regex_options_flag,
+    e.v->>'map' map_intention,
+    e.v->>'retain' retain_result,
+    e.v->>'regex' regex_expression,
+    e.rn target_item_number,
+    COALESCE(mt.rn,rp.rn,1) result_number,
+    mt.mt rx_match,
+    rp.rp rx_replace,
+    --------------------------json key name assigned to return value-----------------------------------------------------------------------
+    CASE e.v->>'map'
         WHEN 'y' THEN
-            defn.v->>'field'
+            e.v->>'field'
         ELSE
             null
-    END                         map_key,
-    CASE defn.v->>'map'
+    END map_key,
+    --------------------------json value resulting from regular expression-----------------------------------------------------------------
+    CASE e.v->>'map'
         WHEN 'y' THEN
-            CASE regex->'regex'->>'function'
+            CASE regex->>'function'
                 WHEN 'extract' THEN
                     CASE WHEN array_upper(mt.mt,1)=1 
                         THEN to_json(mt.mt[1])
@@ -39,16 +49,18 @@ SELECT
             END
         ELSE
             NULL
-    END                         map_val,
-    CASE defn.v->>'retain'
+    END map_val,
+    --------------------------flag for if retruned regex result is stored as a new part of the final json output---------------------------
+    CASE e.v->>'retain'
         WHEN 'y' THEN
-            defn.v->>'field'
+            e.v->>'field'
         ELSE
             NULL
-    END                         retain_key,
-    CASE defn.v->>'retain'
+    END retain_key,
+    --------------------------push regex result into json object---------------------------------------------------------------------------
+    CASE e.v->>'retain'
         WHEN 'y' THEN
-            CASE regex->'regex'->>'function'
+            CASE regex->>'function'
                 WHEN 'extract' THEN
                     CASE WHEN array_upper(mt.mt,1)=1 
                         THEN to_json(trim(mt.mt[1]))
@@ -61,37 +73,42 @@ SELECT
             END
         ELSE
             NULL
-    END                         retain_val
+    END retain_val
 FROM 
     --------------------------start with all regex maps------------------------------------------------------------------------------------
     tps.map_rm m
     --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
-    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') wh(v) ON TRUE
+    JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE
+    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+    JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
     --------------------------join to main transaction table but only certain key/values are included--------------------------------------
     INNER JOIN tps.trans t ON 
         t.srce = m.srce AND
-        t.rec @> wh.v
-    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
-    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
+        t.rec @> w.v
     --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
-    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text,COALESCE(defn.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
-        m.regex->'regex'->>'function' = 'extract'
+    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+        m.regex->>'function' = 'extract'
     --------------------------same as above but for a replacement type function------------------------------------------------------------
-    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text, defn.v ->> 'replace'::text,defn.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
-        m.regex->'regex'->>'function' = 'replace'
+    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+        m.regex->>'function' = 'replace'
 WHERE
     --t.allj IS NULL
-    t.srce = 'dcard'
+    --t.srce = 'PNCC' AND
+    e.v @> '{"map":"y"}'::jsonb
     --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
     --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
+/*
 ORDER BY 
     t.id DESC,
     m.target,
-    defn.rn,
+    e.rn,
     COALESCE(mt.rn,rp.rn,1)
+*/
 )
---SELECT * FROM rx
---SELECT count(*) FROM rx LIMIT 100
+
+--SELECT * FROM rx LIMIT 100
+
+
 , agg_to_target_items AS (
 SELECT 
     srce
@@ -146,6 +163,10 @@ GROUP BY
     ,map_key
     ,retain_key
 )
+
+--SELECT * FROM agg_to_target_items LIMIT 100
+
+
 , agg_to_target AS (
 SELECT
     srce
@@ -163,27 +184,62 @@ GROUP BY
     ,target
     ,seq
     ,map_intention
-ORDER BY
-    id
 )
+
+
+, agg_to_ret AS (
+SELECT
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+	,count(*) "count"
+FROM 
+	agg_to_target
+GROUP BY
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+)
+
 , link_map AS (
 SELECT
     a.srce
-    ,a.id
     ,a.target
     ,a.seq
     ,a.map_intention
     ,a.map_val
-    ,a.retain_val retain_value
-    ,v.map
+    ,a."count"
+    ,a.retain_val
+    ,v.map mapped_val
 FROM
-    agg_to_target a
+    agg_to_ret a
     LEFT OUTER JOIN tps.map_rv v ON
         v.srce = a.srce AND
         v.target = a.target AND
         v.retval = a.map_val
 )
---SELECT * FROM rx
---SELECT * FROM agg_to_target_items
---SELECT * FROM agg_to_target
-SELECT * FROM link_map
+
+SELECT
+    l.srce
+    ,l.target
+    ,l.seq
+    ,l.map_intention
+    ,l.map_val
+    ,l."count"
+    ,l.retain_val
+    ,l.mapped_val
+FROM
+    link_map l
+ORDER BY
+    l.srce
+    ,l.target
+    ,l.seq
+    ,l."count" desc
+    ,l.map_val
+    ,l.mapped_val

database/reports/list_maps.sql

@@ -1,21 +1,19 @@
 SELECT 
-     m.srce
-    ,m.target
-    ,m.regex->'regex'->>'function' func_name
-    ,m.regex->'regex'->>'description' descr
-    ,filter.v  filters
-    ,defn.v->> 'field' result_key_name
-    ,defn.v->> 'key' target_json_path
-    ,defn.v->>'regex' regex_expression
-    ,defn.v->> 'flag' regex_options_flag
-    ,defn.v->>'map' map_intention
-    ,defn.v->>'retain' retain_result
-    ,defn.rn target_item_number
+    m.srce,
+    m.target,
+    regex->>'function' regex_function,
+    regex->>'where' where_clause,
+    e.v ->> 'field' result_key_name,
+    e.v ->> 'key' target_json_path,
+    e.v ->> 'flag' regex_options_flag,
+    e.v->>'map' map_intention,
+    e.v->>'retain' retain_result,
+    e.v->>'regex' regex_expression,
+    e.rn target_item_number
 FROM 
     tps.map_rm m
-    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
-    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') WITH ORDINALITY filter(v, rn) ON true
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
 ORDER BY 
-     m.srce
-    ,m.target
-    ,defn.rn
+    m.srce,
+    m.target,
+    e.rn

database/reports/stacked.sql

@@ -61,7 +61,6 @@ dcard AS (
     UNION ALL
     SELECT * FROM chase 
 )
-,format AS (
 SELECT 
      s.srce
     ,s.id
@@ -82,5 +81,3 @@ FROM
 ORDER BY 
     s.pdate desc
     ,s.id desc
-)
-SELECT * FROM format