diff --git a/database/reports/all_map_return_values.sql b/database/reports/all_map_return_values.sql index b8b50c7..6dd948e 100644 --- a/database/reports/all_map_return_values.sql +++ b/database/reports/all_map_return_values.sql @@ -1,13 +1,5 @@ -/* -first get distinct target json values -then apply regex -*/ - - WITH - ---------------------apply regex operations to transactions--------------------------------------------------------------------------------- - +--------------------apply regex operations to transactions----------------------------------------------------------------------------------- rx AS ( SELECT t.srce, @@ -15,28 +7,26 @@ SELECT t.rec, m.target, m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' + regex->'regex'->>'function' regex_function, + defn.v ->> 'field' result_key_name, + defn.v ->> 'key' target_json_path, + defn.v ->> 'flag' regex_options_flag, + defn.v->>'map' map_intention, + defn.v->>'retain' retain_result, + defn.v->>'regex' regex_expression, + defn.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE defn.v->>'map' WHEN 'y' THEN - e.v->>'field' + defn.v->>'field' ELSE null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' + END map_key, + CASE defn.v->>'map' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(mt.mt[1]) @@ -49,18 +39,16 @@ SELECT END ELSE NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' + END map_val, + CASE defn.v->>'retain' WHEN 'y' THEN - e.v->>'field' + defn.v->>'field' ELSE NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' + END retain_key, + CASE defn.v->>'retain' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(trim(mt.mt[1])) @@ -73,42 +61,37 @@ SELECT END ELSE NULL - END retain_val + END retain_val FROM --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') wh(v) ON TRUE --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND - t.rec @> w.v + t.rec @> wh.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text,COALESCE(defn.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text, defn.v ->> 'replace'::text,defn.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' WHERE --t.allj IS NULL - --t.srce = 'PNCC' AND - e.v @> '{"map":"y"}'::jsonb + t.srce = 'dcard' --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* ORDER BY t.id DESC, m.target, - e.rn, + defn.rn, COALESCE(mt.rn,rp.rn,1) -*/ ) - ---SELECT * FROM rx LIMIT 100 - - +--SELECT * FROM rx +--SELECT count(*) FROM rx LIMIT 100 , agg_to_target_items AS ( SELECT srce @@ -163,10 +146,6 @@ GROUP BY ,map_key ,retain_key ) - ---SELECT * FROM agg_to_target_items LIMIT 100 - - , agg_to_target AS ( SELECT srce @@ -184,62 +163,27 @@ GROUP BY ,target ,seq ,map_intention +ORDER BY + id ) - - -, agg_to_ret AS ( -SELECT - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ,count(*) "count" -FROM - agg_to_target -GROUP BY - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val -) - , link_map AS ( SELECT a.srce + ,a.id ,a.target ,a.seq ,a.map_intention ,a.map_val - ,a."count" - ,a.retain_val - ,v.map mapped_val + ,a.retain_val retain_value + ,v.map FROM - agg_to_ret a + agg_to_target a LEFT OUTER JOIN tps.map_rv v ON v.srce = a.srce AND v.target = a.target AND v.retval = a.map_val ) - -SELECT - l.srce - ,l.target - ,l.seq - ,l.map_intention - ,l.map_val - ,l."count" - ,l.retain_val - ,l.mapped_val -FROM - link_map l -ORDER BY - l.srce - ,l.target - ,l.seq - ,l."count" desc - ,l.map_val - ,l.mapped_val \ No newline at end of file +--SELECT * FROM rx +--SELECT * FROM agg_to_target_items +--SELECT * FROM agg_to_target +SELECT * FROM link_map diff --git a/database/reports/list_maps.sql b/database/reports/list_maps.sql index f422e7e..5df867c 100644 --- a/database/reports/list_maps.sql +++ b/database/reports/list_maps.sql @@ -1,19 +1,21 @@ SELECT - m.srce, - m.target, - regex->>'function' regex_function, - regex->>'where' where_clause, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number + m.srce + ,m.target + ,m.regex->'regex'->>'function' func_name + ,m.regex->'regex'->>'description' descr + ,filter.v filters + ,defn.v->> 'field' result_key_name + ,defn.v->> 'key' target_json_path + ,defn.v->>'regex' regex_expression + ,defn.v->> 'flag' regex_options_flag + ,defn.v->>'map' map_intention + ,defn.v->>'retain' retain_result + ,defn.rn target_item_number FROM tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') WITH ORDINALITY filter(v, rn) ON true ORDER BY - m.srce, - m.target, - e.rn \ No newline at end of file + m.srce + ,m.target + ,defn.rn