pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rewrite mapping script for new level in json def and two-part join to test if regex def has changed

Browse Source
This commit is contained in:
Paul Trowbridge 2018-06-08 17:42:48 -04:00
parent 5027c792ae
commit 13929a6e32
3 changed files with 89 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
interface/import/map_trigger.sql
View File

@ -18,7 +18,7 @@ $f$
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
@ -37,7 +37,7 @@ $f$
END map_key, END map_key,
CASE e.v->>'map' CASE e.v->>'map'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1]) THEN to_json(mt.mt[1])
@ -59,7 +59,7 @@ $f$
END retain_key, END retain_key,
CASE e.v->>'retain' CASE e.v->>'retain'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1])) THEN to_json(trim(mt.mt[1]))
@ -75,15 +75,15 @@ $f$
END retain_val END retain_val
FROM FROM
tps.map_rm m tps.map_rm m
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
INNER JOIN new_table t ON INNER JOIN new_table t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
ORDER BY ORDER BY
t.id DESC, t.id DESC,
m.target, m.target,

9
interface/map_def/srce_map_def_set_single.sql
View File

@ -4,7 +4,6 @@ $f$
DECLARE DECLARE
_message jsonb; _message jsonb;
_rebuild boolean;
_MESSAGE_TEXT text; _MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text; _PG_EXCEPTION_HINT text;
@ -13,7 +12,7 @@ BEGIN
---------test if anythign is changing-------------------------------------------------------------------------------------------- ---------test if anythign is changing--------------------------------------------------------------------------------------------
IF _defn = (SELECT regex FROM tps.map_rm WHERE srce = _defn->>'name') THEN IF _defn->'regex' = (SELECT regex->'regex' FROM tps.map_rm WHERE srce = _defn->>'srce' and target = _defn->>'name') THEN
_message:= _message:=
( (
$$ $$
@ -65,11 +64,13 @@ BEGIN
--------------if rebuild was flag call the rebuild-------------------------------------------------------------------------------- --------------if rebuild was flag call the rebuild--------------------------------------------------------------------------------
SELECT SELECT
x.message x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb
INTO INTO
_message _message
FROM FROM
tps.srce_map_overwrite as X(message); tps.srce_map_overwrite(_defn->>'srce') as X(message);
return _message;
EXCEPTION WHEN OTHERS THEN EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS GET STACKED DIAGNOSTICS

14
interface/map_values/srce_map_overwrite.sql
View File

@ -18,7 +18,7 @@ BEGIN
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
@ -37,7 +37,7 @@ BEGIN
END map_key, END map_key,
CASE e.v->>'map' CASE e.v->>'map'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1]) THEN to_json(mt.mt[1])
@ -59,7 +59,7 @@ BEGIN
END retain_key, END retain_key,
CASE e.v->>'retain' CASE e.v->>'retain'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1])) THEN to_json(trim(mt.mt[1]))
@ -75,15 +75,15 @@ BEGIN
END retain_val END retain_val
FROM FROM
tps.map_rm m tps.map_rm m
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
INNER JOIN tps.trans t ON INNER JOIN tps.trans t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
WHERE WHERE
--t.allj IS NULL --t.allj IS NULL
t.srce = _srce t.srce = _srce