From f9aa2a31a6bd48c789f83bcdd144fdfd233595fc Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Mon, 4 Jun 2018 00:54:23 -0400 Subject: [PATCH 1/6] add single def set --- interface/map_def/srce_map_def_set_single.sql | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 interface/map_def/srce_map_def_set_single.sql diff --git a/interface/map_def/srce_map_def_set_single.sql b/interface/map_def/srce_map_def_set_single.sql new file mode 100644 index 0000000..72ee223 --- /dev/null +++ b/interface/map_def/srce_map_def_set_single.sql @@ -0,0 +1,94 @@ +CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb) RETURNS jsonb +AS +$f$ + +DECLARE + _message jsonb; + _rebuild boolean; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + +BEGIN + + ---------test if anythign is changing-------------------------------------------------------------------------------------------- + + IF _defn = (SELECT regex FROM tps.map_rm WHERE srce = _defn->>'name') THEN + _message:= + ( + $$ + { + "status":"complete", + "message":"source was not different no action taken" + } + $$::jsonb + ); + RETURN _message; + END IF; + + ---------do the rebuild----------------------------------------------------------------------------------------------------------- + + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + --data source + _defn->>'srce' + --map name + ,_defn->>'name' + --map definition + ,_defn + --map aggregation sequence + ,(_defn->>'sequence')::INTEGER + --history definition + ,jsonb_build_object( + 'hist_defn',_defn + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); + + --------------if rebuild was flag call the rebuild-------------------------------------------------------------------------------- + + SELECT + x.message + INTO + _message + FROM + tps.srce_map_overwrite as X(message) + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting definition" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; + + + +END; +$f$ +language plpgsql \ No newline at end of file From e122c4987bb34eb20c4e35b8117fa4d85fafcf06 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 8 Jun 2018 00:47:48 -0400 Subject: [PATCH 2/6] try modifying the mult map set function --- interface/map_def/srce_map_def_set.sql | 109 ++++++++++++------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/interface/map_def/srce_map_def_set.sql b/interface/map_def/srce_map_def_set.sql index a752dac..471f7ec 100644 --- a/interface/map_def/srce_map_def_set.sql +++ b/interface/map_def/srce_map_def_set.sql @@ -10,65 +10,64 @@ DECLARE BEGIN - BEGIN - INSERT INTO - tps.map_rm (srce, target, regex, seq, hist) - SELECT - --data source - ae.r->>'srce' - --map name - ,ae.r->>'name' - --map definition - ,ae.r - --map aggregation sequence - ,(ae.r->>'sequence')::INTEGER - --history definition - ,jsonb_build_object( - 'hist_defn',ae.r - ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) - ) || '[]'::jsonb - FROM - jsonb_array_elements(_defn) ae(r) - ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET - srce = excluded.srce - ,target = excluded.target - ,regex = excluded.regex - ,seq = excluded.seq - ,hist = - --the new definition going to position -0- - jsonb_build_object( - 'hist_defn',excluded.regex - ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) - ) - --the previous definition, set upper bound of effective range which was previously null - || jsonb_set( - map_rm.hist - ,'{0,effective,1}'::text[] - ,to_jsonb(CURRENT_TIMESTAMP) - ); + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + --data source + ae.r->>'srce' + --map name + ,ae.r->>'name' + --map definition + ,ae.r + --map aggregation sequence + ,(ae.r->>'sequence')::INTEGER + --history definition + ,jsonb_build_object( + 'hist_defn',ae.r + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb + FROM + jsonb_array_elements(_defn) ae(r) + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ); - EXCEPTION WHEN OTHERS THEN - - GET STACKED DIAGNOSTICS - _MESSAGE_TEXT = MESSAGE_TEXT, - _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, - _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; - _message:= - ($$ - { - "status":"fail", - "message":"error setting definition" - } - $$::jsonb) - ||jsonb_build_object('message_text',_MESSAGE_TEXT) - ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); - return _message; - END; - - _message:= jsonb_build_object('status','complete','message','definition has been set'); + _message:= jsonb_build_object('status','complete','message','definition has been set'); return _message; + + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error setting definition" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + return _message; + + + END; $f$ language plpgsql \ No newline at end of file From 5027c792ae571de0b3cfc627bdc7abac2f5aea3f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 8 Jun 2018 00:48:07 -0400 Subject: [PATCH 3/6] try modifying the mult map set function --- interface/map_def/srce_map_def_set.sql | 109 ++++++++++++------ interface/map_def/srce_map_def_set_single.sql | 9 +- 2 files changed, 78 insertions(+), 40 deletions(-) diff --git a/interface/map_def/srce_map_def_set.sql b/interface/map_def/srce_map_def_set.sql index 471f7ec..62a7288 100644 --- a/interface/map_def/srce_map_def_set.sql +++ b/interface/map_def/srce_map_def_set.sql @@ -10,44 +10,81 @@ DECLARE BEGIN - - INSERT INTO - tps.map_rm (srce, target, regex, seq, hist) - SELECT - --data source - ae.r->>'srce' - --map name - ,ae.r->>'name' - --map definition - ,ae.r - --map aggregation sequence - ,(ae.r->>'sequence')::INTEGER - --history definition - ,jsonb_build_object( - 'hist_defn',ae.r - ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) - ) || '[]'::jsonb + WITH + ------------------------------------------stage rows to insert----------------------------------------------------- + stg AS ( + SELECT + --data source + ae.r->>'srce' srce + --map name + ,ae.r->>'name' target + --map definition + ,ae.r regex + --map aggregation sequence + ,(ae.r->>'sequence')::INTEGER seq + --history definition + ,jsonb_build_object( + 'hist_defn',ae.r + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) || '[]'::jsonb hist + --determine if the rows are new or match + ,(m.regex->>'regex' = ae.r->>'regex')::BOOLEAN rebuild + FROM + jsonb_array_elements(_defn) ae(r) + LEFT OUTER JOIN tps.map_rm m ON + m.srce = ae.r->>'srce' + AND m.target = ae.t->>'name' + ) + ---------------------------------------do the upsert------------------------------------------------------------------- + ,ins AS ( + INSERT INTO + tps.map_rm (srce, target, regex, seq, hist) + SELECT + srce + ,target + ,regex + ,seq + ,hist + FROM + stg + ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET + srce = excluded.srce + ,target = excluded.target + ,regex = excluded.regex + ,seq = excluded.seq + ,hist = + --the new definition going to position -0- + jsonb_build_object( + 'hist_defn',excluded.regex + ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) + ) + --the previous definition, set upper bound of effective range which was previously null + || jsonb_set( + map_rm.hist + ,'{0,effective,1}'::text[] + ,to_jsonb(CURRENT_TIMESTAMP) + ) + ) + ---------------------------get list of sources that had maps change-------------------------------------------------------- + , to_update AS ( + SELECT DISTINCT + srce FROM - jsonb_array_elements(_defn) ae(r) - ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET - srce = excluded.srce - ,target = excluded.target - ,regex = excluded.regex - ,seq = excluded.seq - ,hist = - --the new definition going to position -0- - jsonb_build_object( - 'hist_defn',excluded.regex - ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz) - ) - --the previous definition, set upper bound of effective range which was previously null - || jsonb_set( - map_rm.hist - ,'{0,effective,1}'::text[] - ,to_jsonb(CURRENT_TIMESTAMP) - ); + ins + WHERE + rebuild = TRUE + ) + --------------------------call the map overwrite for each source and return all the messages into message---------------- + /*the whole source must be overwritten because if an element is no longer returned it shoudl be wiped from the data*/ + SELECT + jsonb_agg(x.message) + INTO + _message + FROM + to_update + JOIN LATERAL tps.srce_map_overwrite(to_update.srce) AS x(message) ON TRUE; - _message:= jsonb_build_object('status','complete','message','definition has been set'); + _message:= jsonb_build_object('status','complete','message','definition has been set'); return _message; diff --git a/interface/map_def/srce_map_def_set_single.sql b/interface/map_def/srce_map_def_set_single.sql index 72ee223..fa9afb7 100644 --- a/interface/map_def/srce_map_def_set_single.sql +++ b/interface/map_def/srce_map_def_set_single.sql @@ -19,7 +19,7 @@ BEGIN $$ { "status":"complete", - "message":"source was not different no action taken" + "message":"map was not different no action taken" } $$::jsonb ); @@ -69,14 +69,15 @@ BEGIN INTO _message FROM - tps.srce_map_overwrite as X(message) + tps.srce_map_overwrite as X(message); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _MESSAGE_TEXT = MESSAGE_TEXT, _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; - _message:= + + _message:= ($$ { "status":"fail", @@ -85,7 +86,7 @@ BEGIN $$::jsonb) ||jsonb_build_object('message_text',_MESSAGE_TEXT) ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); - return _message; + return _message; From 13929a6e32901610eb222d1d9c68896dd19e6670 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 8 Jun 2018 17:42:48 -0400 Subject: [PATCH 4/6] rewrite mapping script for new level in json def and two-part join to test if regex def has changed --- interface/import/map_trigger.sql | 154 +++++++++--------- interface/map_def/srce_map_def_set_single.sql | 9 +- interface/map_values/srce_map_overwrite.sql | 14 +- 3 files changed, 89 insertions(+), 88 deletions(-) diff --git a/interface/import/map_trigger.sql b/interface/import/map_trigger.sql index 86f8304..6002dc1 100644 --- a/interface/import/map_trigger.sql +++ b/interface/import/map_trigger.sql @@ -12,83 +12,83 @@ $f$ --------------------apply regex operations to transactions----------------------------------------------------------------------------------- rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - INNER JOIN new_table t ON - t.srce = m.srce AND - t.rec @> w.v - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' - ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + tps.map_rm m + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + INNER JOIN new_table t ON + t.srce = m.srce AND + t.rec @> w.v + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT count(*) FROM rx LIMIT 100 diff --git a/interface/map_def/srce_map_def_set_single.sql b/interface/map_def/srce_map_def_set_single.sql index fa9afb7..e16709e 100644 --- a/interface/map_def/srce_map_def_set_single.sql +++ b/interface/map_def/srce_map_def_set_single.sql @@ -4,7 +4,6 @@ $f$ DECLARE _message jsonb; - _rebuild boolean; _MESSAGE_TEXT text; _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; @@ -13,7 +12,7 @@ BEGIN ---------test if anythign is changing-------------------------------------------------------------------------------------------- - IF _defn = (SELECT regex FROM tps.map_rm WHERE srce = _defn->>'name') THEN + IF _defn->'regex' = (SELECT regex->'regex' FROM tps.map_rm WHERE srce = _defn->>'srce' and target = _defn->>'name') THEN _message:= ( $$ @@ -65,11 +64,13 @@ BEGIN --------------if rebuild was flag call the rebuild-------------------------------------------------------------------------------- SELECT - x.message + x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb INTO _message FROM - tps.srce_map_overwrite as X(message); + tps.srce_map_overwrite(_defn->>'srce') as X(message); + + return _message; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS diff --git a/interface/map_values/srce_map_overwrite.sql b/interface/map_values/srce_map_overwrite.sql index f11841f..408f8a9 100644 --- a/interface/map_values/srce_map_overwrite.sql +++ b/interface/map_values/srce_map_overwrite.sql @@ -18,7 +18,7 @@ BEGIN t.rec, m.target, m.seq, - regex->>'function' regex_function, + regex->'regex'->>'function' regex_function, e.v ->> 'field' result_key_name, e.v ->> 'key' target_json_path, e.v ->> 'flag' regex_options_flag, @@ -37,7 +37,7 @@ BEGIN END map_key, CASE e.v->>'map' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(mt.mt[1]) @@ -59,7 +59,7 @@ BEGIN END retain_key, CASE e.v->>'retain' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(trim(mt.mt[1])) @@ -75,15 +75,15 @@ BEGIN END retain_val FROM tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' + m.regex->'regex'->>'function' = 'extract' LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + m.regex->'regex'->>'function' = 'replace' WHERE --t.allj IS NULL t.srce = _srce From 8c2936ee54e4957e8d535293eadc3df53df31042 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 12 Jun 2018 22:07:31 -0400 Subject: [PATCH 5/6] include mods to mapping function in initial deploy sql --- deploy/reload/hunt/map.json | 2 +- deploy/setup.sql | 166 +++++++++++++++++++----------------- 2 files changed, 88 insertions(+), 80 deletions(-) diff --git a/deploy/reload/hunt/map.json b/deploy/reload/hunt/map.json index aa796be..a595267 100644 --- a/deploy/reload/hunt/map.json +++ b/deploy/reload/hunt/map.json @@ -17,7 +17,7 @@ "where": [ {} ], - "function": "replace", + "function": "extract", "description": "pull first 20 characters from description for mapping" }, "sequence": 1 diff --git a/deploy/setup.sql b/deploy/setup.sql index e8cd9d7..2cb3f5e 100644 --- a/deploy/setup.sql +++ b/deploy/setup.sql @@ -697,90 +697,97 @@ $f$; -------------------create trigger to map imported items------------------------------------------------------------------------------------------------------ -CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER AS $f$ +CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER +AS +$f$ + DECLARE + _cnt INTEGER; + BEGIN IF (TG_OP = 'INSERT') THEN + + WITH --------------------apply regex operations to transactions----------------------------------------------------------------------------------- rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - INNER JOIN new_table t ON - t.srce = m.srce AND - t.rec @> w.v - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' - ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + tps.map_rm m + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + INNER JOIN new_table t ON + t.srce = m.srce AND + t.rec @> w.v + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT count(*) FROM rx LIMIT 100 @@ -903,7 +910,7 @@ CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER AS $f$ ) --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 - + --create a complete list of all new inserts assuming some do not have maps (left join) ,join_all AS ( SELECT @@ -930,6 +937,7 @@ CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER AS $f$ join_all a WHERE t.id = a.id; + END IF; RETURN NULL; From 271a55d6e31a03f1f4645fc28e973c7a09b7551a Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 12 Jun 2018 22:21:16 -0400 Subject: [PATCH 6/6] include flag to rebuild; may not want to if due to layout change --- interface/map_def/srce_map_def_set_single.sql | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/interface/map_def/srce_map_def_set_single.sql b/interface/map_def/srce_map_def_set_single.sql index e16709e..c22c488 100644 --- a/interface/map_def/srce_map_def_set_single.sql +++ b/interface/map_def/srce_map_def_set_single.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb) RETURNS jsonb +CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb, _rebuild BOOLEAN) RETURNS jsonb AS $f$ @@ -63,12 +63,14 @@ BEGIN --------------if rebuild was flag call the rebuild-------------------------------------------------------------------------------- - SELECT - x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb - INTO - _message - FROM - tps.srce_map_overwrite(_defn->>'srce') as X(message); + IF _rebuild THEN + SELECT + x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb + INTO + _message + FROM + tps.srce_map_overwrite(_defn->>'srce') as X(message); + END IF; return _message;