vault backup: 2024-09-28 17:13:57
This commit is contained in:
parent
51599ee526
commit
8bba1d95d9
@ -1,13 +1,5 @@
|
||||
/*
|
||||
first get distinct target json values
|
||||
then apply regex
|
||||
*/
|
||||
|
||||
|
||||
WITH
|
||||
|
||||
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
|
||||
|
||||
--------------------apply regex operations to transactions-----------------------------------------------------------------------------------
|
||||
rx AS (
|
||||
SELECT
|
||||
t.srce,
|
||||
@ -15,28 +7,26 @@ SELECT
|
||||
t.rec,
|
||||
m.target,
|
||||
m.seq,
|
||||
regex->>'function' regex_function,
|
||||
e.v ->> 'field' result_key_name,
|
||||
e.v ->> 'key' target_json_path,
|
||||
e.v ->> 'flag' regex_options_flag,
|
||||
e.v->>'map' map_intention,
|
||||
e.v->>'retain' retain_result,
|
||||
e.v->>'regex' regex_expression,
|
||||
e.rn target_item_number,
|
||||
COALESCE(mt.rn,rp.rn,1) result_number,
|
||||
mt.mt rx_match,
|
||||
rp.rp rx_replace,
|
||||
--------------------------json key name assigned to return value-----------------------------------------------------------------------
|
||||
CASE e.v->>'map'
|
||||
regex->'regex'->>'function' regex_function,
|
||||
defn.v ->> 'field' result_key_name,
|
||||
defn.v ->> 'key' target_json_path,
|
||||
defn.v ->> 'flag' regex_options_flag,
|
||||
defn.v->>'map' map_intention,
|
||||
defn.v->>'retain' retain_result,
|
||||
defn.v->>'regex' regex_expression,
|
||||
defn.rn target_item_number,
|
||||
COALESCE(mt.rn,rp.rn,1) result_number,
|
||||
mt.mt rx_match,
|
||||
rp.rp rx_replace,
|
||||
CASE defn.v->>'map'
|
||||
WHEN 'y' THEN
|
||||
e.v->>'field'
|
||||
defn.v->>'field'
|
||||
ELSE
|
||||
null
|
||||
END map_key,
|
||||
--------------------------json value resulting from regular expression-----------------------------------------------------------------
|
||||
CASE e.v->>'map'
|
||||
END map_key,
|
||||
CASE defn.v->>'map'
|
||||
WHEN 'y' THEN
|
||||
CASE regex->>'function'
|
||||
CASE regex->'regex'->>'function'
|
||||
WHEN 'extract' THEN
|
||||
CASE WHEN array_upper(mt.mt,1)=1
|
||||
THEN to_json(mt.mt[1])
|
||||
@ -49,18 +39,16 @@ SELECT
|
||||
END
|
||||
ELSE
|
||||
NULL
|
||||
END map_val,
|
||||
--------------------------flag for if retruned regex result is stored as a new part of the final json output---------------------------
|
||||
CASE e.v->>'retain'
|
||||
END map_val,
|
||||
CASE defn.v->>'retain'
|
||||
WHEN 'y' THEN
|
||||
e.v->>'field'
|
||||
defn.v->>'field'
|
||||
ELSE
|
||||
NULL
|
||||
END retain_key,
|
||||
--------------------------push regex result into json object---------------------------------------------------------------------------
|
||||
CASE e.v->>'retain'
|
||||
END retain_key,
|
||||
CASE defn.v->>'retain'
|
||||
WHEN 'y' THEN
|
||||
CASE regex->>'function'
|
||||
CASE regex->'regex'->>'function'
|
||||
WHEN 'extract' THEN
|
||||
CASE WHEN array_upper(mt.mt,1)=1
|
||||
THEN to_json(trim(mt.mt[1]))
|
||||
@ -73,42 +61,37 @@ SELECT
|
||||
END
|
||||
ELSE
|
||||
NULL
|
||||
END retain_val
|
||||
END retain_val
|
||||
FROM
|
||||
--------------------------start with all regex maps------------------------------------------------------------------------------------
|
||||
tps.map_rm m
|
||||
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
|
||||
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE
|
||||
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
|
||||
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
|
||||
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') wh(v) ON TRUE
|
||||
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
|
||||
INNER JOIN tps.trans t ON
|
||||
t.srce = m.srce AND
|
||||
t.rec @> w.v
|
||||
t.rec @> wh.v
|
||||
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
|
||||
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
|
||||
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
|
||||
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
|
||||
m.regex->>'function' = 'extract'
|
||||
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text,COALESCE(defn.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
|
||||
m.regex->'regex'->>'function' = 'extract'
|
||||
--------------------------same as above but for a replacement type function------------------------------------------------------------
|
||||
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
|
||||
m.regex->>'function' = 'replace'
|
||||
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text, defn.v ->> 'replace'::text,defn.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
|
||||
m.regex->'regex'->>'function' = 'replace'
|
||||
WHERE
|
||||
--t.allj IS NULL
|
||||
--t.srce = 'PNCC' AND
|
||||
e.v @> '{"map":"y"}'::jsonb
|
||||
t.srce = 'dcard'
|
||||
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
|
||||
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
|
||||
/*
|
||||
ORDER BY
|
||||
t.id DESC,
|
||||
m.target,
|
||||
e.rn,
|
||||
defn.rn,
|
||||
COALESCE(mt.rn,rp.rn,1)
|
||||
*/
|
||||
)
|
||||
|
||||
--SELECT * FROM rx LIMIT 100
|
||||
|
||||
|
||||
--SELECT * FROM rx
|
||||
--SELECT count(*) FROM rx LIMIT 100
|
||||
, agg_to_target_items AS (
|
||||
SELECT
|
||||
srce
|
||||
@ -163,10 +146,6 @@ GROUP BY
|
||||
,map_key
|
||||
,retain_key
|
||||
)
|
||||
|
||||
--SELECT * FROM agg_to_target_items LIMIT 100
|
||||
|
||||
|
||||
, agg_to_target AS (
|
||||
SELECT
|
||||
srce
|
||||
@ -184,62 +163,27 @@ GROUP BY
|
||||
,target
|
||||
,seq
|
||||
,map_intention
|
||||
ORDER BY
|
||||
id
|
||||
)
|
||||
|
||||
|
||||
, agg_to_ret AS (
|
||||
SELECT
|
||||
srce
|
||||
,target
|
||||
,seq
|
||||
,map_intention
|
||||
,map_val
|
||||
,retain_val
|
||||
,count(*) "count"
|
||||
FROM
|
||||
agg_to_target
|
||||
GROUP BY
|
||||
srce
|
||||
,target
|
||||
,seq
|
||||
,map_intention
|
||||
,map_val
|
||||
,retain_val
|
||||
)
|
||||
|
||||
, link_map AS (
|
||||
SELECT
|
||||
a.srce
|
||||
,a.id
|
||||
,a.target
|
||||
,a.seq
|
||||
,a.map_intention
|
||||
,a.map_val
|
||||
,a."count"
|
||||
,a.retain_val
|
||||
,v.map mapped_val
|
||||
,a.retain_val retain_value
|
||||
,v.map
|
||||
FROM
|
||||
agg_to_ret a
|
||||
agg_to_target a
|
||||
LEFT OUTER JOIN tps.map_rv v ON
|
||||
v.srce = a.srce AND
|
||||
v.target = a.target AND
|
||||
v.retval = a.map_val
|
||||
)
|
||||
|
||||
SELECT
|
||||
l.srce
|
||||
,l.target
|
||||
,l.seq
|
||||
,l.map_intention
|
||||
,l.map_val
|
||||
,l."count"
|
||||
,l.retain_val
|
||||
,l.mapped_val
|
||||
FROM
|
||||
link_map l
|
||||
ORDER BY
|
||||
l.srce
|
||||
,l.target
|
||||
,l.seq
|
||||
,l."count" desc
|
||||
,l.map_val
|
||||
,l.mapped_val
|
||||
--SELECT * FROM rx
|
||||
--SELECT * FROM agg_to_target_items
|
||||
--SELECT * FROM agg_to_target
|
||||
SELECT * FROM link_map
|
||||
|
||||
@ -1,19 +1,21 @@
|
||||
SELECT
|
||||
m.srce,
|
||||
m.target,
|
||||
regex->>'function' regex_function,
|
||||
regex->>'where' where_clause,
|
||||
e.v ->> 'field' result_key_name,
|
||||
e.v ->> 'key' target_json_path,
|
||||
e.v ->> 'flag' regex_options_flag,
|
||||
e.v->>'map' map_intention,
|
||||
e.v->>'retain' retain_result,
|
||||
e.v->>'regex' regex_expression,
|
||||
e.rn target_item_number
|
||||
m.srce
|
||||
,m.target
|
||||
,m.regex->'regex'->>'function' func_name
|
||||
,m.regex->'regex'->>'description' descr
|
||||
,filter.v filters
|
||||
,defn.v->> 'field' result_key_name
|
||||
,defn.v->> 'key' target_json_path
|
||||
,defn.v->>'regex' regex_expression
|
||||
,defn.v->> 'flag' regex_options_flag
|
||||
,defn.v->>'map' map_intention
|
||||
,defn.v->>'retain' retain_result
|
||||
,defn.rn target_item_number
|
||||
FROM
|
||||
tps.map_rm m
|
||||
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
|
||||
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
|
||||
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') WITH ORDINALITY filter(v, rn) ON true
|
||||
ORDER BY
|
||||
m.srce,
|
||||
m.target,
|
||||
e.rn
|
||||
m.srce
|
||||
,m.target
|
||||
,defn.rn
|
||||
|
||||
Loading…
Reference in New Issue
Block a user