pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add notes on mapping and adjust regex testing for json schema

Browse Source
This commit is contained in:
Paul Trowbridge 2018-06-13 01:46:38 -04:00
parent db5f3d6133
commit 3caa2b3887
5 changed files with 360 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

354
deploy/setup.sql
View File

@ -302,7 +302,7 @@ BEGIN
RETURN _message; RETURN _message;
END; END;
$f$ $f$
LANGUAGE plpgsql LANGUAGE plpgsql;
-----generate sql to create select based on schema -----generate sql to create select based on schema
DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text);
@ -773,14 +773,20 @@ $f$
NULL NULL
END retain_val END retain_val
FROM FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN new_table t ON INNER JOIN new_table t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
ORDER BY ORDER BY
@ -1190,7 +1196,7 @@ BEGIN
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
@ -1209,7 +1215,7 @@ BEGIN
END map_key, END map_key,
CASE e.v->>'map' CASE e.v->>'map'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1]) THEN to_json(mt.mt[1])
@ -1231,7 +1237,7 @@ BEGIN
END retain_key, END retain_key,
CASE e.v->>'retain' CASE e.v->>'retain'
WHEN 'y' THEN WHEN 'y' THEN
CASE regex->>'function' CASE regex->'regex'->>'function'
WHEN 'extract' THEN WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1 CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1])) THEN to_json(trim(mt.mt[1]))
@ -1246,16 +1252,22 @@ BEGIN
NULL NULL
END retain_val END retain_val
FROM FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m tps.map_rm m
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON INNER JOIN tps.trans t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true --------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
WHERE WHERE
--t.allj IS NULL --t.allj IS NULL
t.srce = _srce t.srce = _srce
@ -1510,88 +1522,89 @@ BEGIN
--------------------apply regex operations to transactions--------------------------------------------------------------------------------- --------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS ( rx AS (
SELECT SELECT
t.srce, t.srce,
t.id, t.id,
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention, e.v->>'map' map_intention,
e.v->>'retain' retain_result, e.v->>'retain' retain_result,
e.v->>'regex' regex_expression, e.v->>'regex' regex_expression,
e.rn target_item_number, e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number, COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match, mt.mt rx_match,
rp.rp rx_replace, rp.rp rx_replace,
--------------------------json key name assigned to return value----------------------------------------------------------------------- CASE e.v->>'map'
CASE e.v->>'map' WHEN 'y' THEN
WHEN 'y' THEN e.v->>'field'
e.v->>'field' ELSE
ELSE null
null END map_key,
END map_key, CASE e.v->>'map'
--------------------------json value resulting from regular expression----------------------------------------------------------------- WHEN 'y' THEN
CASE e.v->>'map' CASE regex->'regex'->>'function'
WHEN 'y' THEN WHEN 'extract' THEN
CASE regex->>'function' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'extract' THEN THEN to_json(mt.mt[1])
CASE WHEN array_upper(mt.mt,1)=1 ELSE array_to_json(mt.mt)
THEN to_json(mt.mt[1]) END::jsonb
ELSE array_to_json(mt.mt) WHEN 'replace' THEN
END::jsonb to_jsonb(rp.rp)
WHEN 'replace' THEN ELSE
to_jsonb(rp.rp) '{}'::jsonb
ELSE END
'{}'::jsonb ELSE
END NULL
ELSE END map_val,
NULL CASE e.v->>'retain'
END map_val, WHEN 'y' THEN
--------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- e.v->>'field'
CASE e.v->>'retain' ELSE
WHEN 'y' THEN NULL
e.v->>'field' END retain_key,
ELSE CASE e.v->>'retain'
NULL WHEN 'y' THEN
END retain_key, CASE regex->'regex'->>'function'
--------------------------push regex result into json object--------------------------------------------------------------------------- WHEN 'extract' THEN
CASE e.v->>'retain' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'y' THEN THEN to_json(trim(mt.mt[1]))
CASE regex->>'function' ELSE array_to_json(mt.mt)
WHEN 'extract' THEN END::jsonb
CASE WHEN array_upper(mt.mt,1)=1 WHEN 'replace' THEN
THEN to_json(trim(mt.mt[1])) to_jsonb(rtrim(rp.rp))
ELSE array_to_json(mt.mt) ELSE
END::jsonb '{}'::jsonb
WHEN 'replace' THEN END
to_jsonb(rtrim(rp.rp)) ELSE
ELSE NULL
'{}'::jsonb END retain_val
END FROM
ELSE --------------------------start with all regex maps------------------------------------------------------------------------------------
NULL (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m
END retain_val --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
FROM LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------start with all regex maps------------------------------------------------------------------------------------ --------------------------break out array of regluar expressions in the map------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- --------------------------join to main transaction table but only certain key/values are included--------------------------------------
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE INNER JOIN tps.trans t ON
--------------------------break out array of regluar expressions in the map------------------------------------------------------------ t.srce = m.srce AND
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true t.rec @> w.v
--------------------------join to main transaction table but only certain key/values are included-------------------------------------- --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
INNER JOIN tps.trans t ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
t.srce = m.srce AND m.regex->'regex'->>'function' = 'extract'
t.rec @> w.v --------------------------same as above but for a replacement type function------------------------------------------------------------
--------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'replace'
m.regex->>'function' = 'extract' ORDER BY
--------------------------same as above but for a replacement type function------------------------------------------------------------ t.id DESC,
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.target,
m.regex->>'function' = 'replace' e.rn,
COALESCE(mt.rn,rp.rn,1)
) )
--SELECT * FROM rx LIMIT 100 --SELECT * FROM rx LIMIT 100
@ -1734,88 +1747,89 @@ WITH
--------------------apply regex operations to transactions--------------------------------------------------------------------------------- --------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS ( rx AS (
SELECT SELECT
t.srce, t.srce,
t.id, t.id,
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention, e.v->>'map' map_intention,
e.v->>'retain' retain_result, e.v->>'retain' retain_result,
e.v->>'regex' regex_expression, e.v->>'regex' regex_expression,
e.rn target_item_number, e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number, COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match, mt.mt rx_match,
rp.rp rx_replace, rp.rp rx_replace,
--------------------------json key name assigned to return value----------------------------------------------------------------------- CASE e.v->>'map'
CASE e.v->>'map' WHEN 'y' THEN
WHEN 'y' THEN e.v->>'field'
e.v->>'field'
ELSE
null
END map_key,
--------------------------json value resulting from regular expression-----------------------------------------------------------------
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE ELSE
'{}'::jsonb null
END END map_key,
ELSE CASE e.v->>'map'
NULL WHEN 'y' THEN
END map_val, CASE regex->'regex'->>'function'
--------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- WHEN 'extract' THEN
CASE e.v->>'retain' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'y' THEN THEN to_json(mt.mt[1])
e.v->>'field' ELSE array_to_json(mt.mt)
ELSE END::jsonb
NULL WHEN 'replace' THEN
END retain_key, to_jsonb(rp.rp)
--------------------------push regex result into json object--------------------------------------------------------------------------- ELSE
CASE e.v->>'retain' '{}'::jsonb
WHEN 'y' THEN END
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE ELSE
'{}'::jsonb NULL
END END map_val,
ELSE CASE e.v->>'retain'
NULL WHEN 'y' THEN
END retain_val e.v->>'field'
FROM ELSE
--------------------------start with all regex maps------------------------------------------------------------------------------------ NULL
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m END retain_key,
--------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- CASE e.v->>'retain'
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE WHEN 'y' THEN
--------------------------break out array of regluar expressions in the map------------------------------------------------------------ CASE regex->'regex'->>'function'
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true WHEN 'extract' THEN
--------------------------join to main transaction table but only certain key/values are included-------------------------------------- CASE WHEN array_upper(mt.mt,1)=1
INNER JOIN tps.trans t ON THEN to_json(trim(mt.mt[1]))
t.srce = m.srce AND ELSE array_to_json(mt.mt)
t.rec @> w.v END::jsonb
--------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- WHEN 'replace' THEN
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON to_jsonb(rtrim(rp.rp))
m.regex->>'function' = 'extract' ELSE
--------------------------same as above but for a replacement type function------------------------------------------------------------ '{}'::jsonb
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON END
m.regex->>'function' = 'replace' ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
) )
--SELECT * FROM rx LIMIT 100 --SELECT * FROM rx LIMIT 100

6
interface/import/map_trigger.sql
View File

@ -74,14 +74,20 @@ $f$
NULL NULL
END retain_val END retain_val
FROM FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN new_table t ON INNER JOIN new_table t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
ORDER BY ORDER BY

165
interface/map_def/test_regex.sql
View File

@ -12,88 +12,89 @@ BEGIN
--------------------apply regex operations to transactions--------------------------------------------------------------------------------- --------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS ( rx AS (
SELECT SELECT
t.srce, t.srce,
t.id, t.id,
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention, e.v->>'map' map_intention,
e.v->>'retain' retain_result, e.v->>'retain' retain_result,
e.v->>'regex' regex_expression, e.v->>'regex' regex_expression,
e.rn target_item_number, e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number, COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match, mt.mt rx_match,
rp.rp rx_replace, rp.rp rx_replace,
--------------------------json key name assigned to return value----------------------------------------------------------------------- CASE e.v->>'map'
CASE e.v->>'map' WHEN 'y' THEN
WHEN 'y' THEN e.v->>'field'
e.v->>'field' ELSE
ELSE null
null END map_key,
END map_key, CASE e.v->>'map'
--------------------------json value resulting from regular expression----------------------------------------------------------------- WHEN 'y' THEN
CASE e.v->>'map' CASE regex->'regex'->>'function'
WHEN 'y' THEN WHEN 'extract' THEN
CASE regex->>'function' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'extract' THEN THEN to_json(mt.mt[1])
CASE WHEN array_upper(mt.mt,1)=1 ELSE array_to_json(mt.mt)
THEN to_json(mt.mt[1]) END::jsonb
ELSE array_to_json(mt.mt) WHEN 'replace' THEN
END::jsonb to_jsonb(rp.rp)
WHEN 'replace' THEN ELSE
to_jsonb(rp.rp) '{}'::jsonb
ELSE END
'{}'::jsonb ELSE
END NULL
ELSE END map_val,
NULL CASE e.v->>'retain'
END map_val, WHEN 'y' THEN
--------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- e.v->>'field'
CASE e.v->>'retain' ELSE
WHEN 'y' THEN NULL
e.v->>'field' END retain_key,
ELSE CASE e.v->>'retain'
NULL WHEN 'y' THEN
END retain_key, CASE regex->'regex'->>'function'
--------------------------push regex result into json object--------------------------------------------------------------------------- WHEN 'extract' THEN
CASE e.v->>'retain' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'y' THEN THEN to_json(trim(mt.mt[1]))
CASE regex->>'function' ELSE array_to_json(mt.mt)
WHEN 'extract' THEN END::jsonb
CASE WHEN array_upper(mt.mt,1)=1 WHEN 'replace' THEN
THEN to_json(trim(mt.mt[1])) to_jsonb(rtrim(rp.rp))
ELSE array_to_json(mt.mt) ELSE
END::jsonb '{}'::jsonb
WHEN 'replace' THEN END
to_jsonb(rtrim(rp.rp)) ELSE
ELSE NULL
'{}'::jsonb END retain_val
END FROM
ELSE --------------------------start with all regex maps------------------------------------------------------------------------------------
NULL (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
END retain_val --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
FROM LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------start with all regex maps------------------------------------------------------------------------------------ --------------------------break out array of regluar expressions in the map------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- --------------------------join to main transaction table but only certain key/values are included--------------------------------------
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE INNER JOIN tps.trans t ON
--------------------------break out array of regluar expressions in the map------------------------------------------------------------ t.srce = m.srce AND
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true t.rec @> w.v
--------------------------join to main transaction table but only certain key/values are included-------------------------------------- --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
INNER JOIN tps.trans t ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
t.srce = m.srce AND m.regex->'regex'->>'function' = 'extract'
t.rec @> w.v --------------------------same as above but for a replacement type function------------------------------------------------------------
--------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'replace'
m.regex->>'function' = 'extract' ORDER BY
--------------------------same as above but for a replacement type function------------------------------------------------------------ t.id DESC,
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.target,
m.regex->>'function' = 'replace' e.rn,
COALESCE(mt.rn,rp.rn,1)
) )
--SELECT * FROM rx LIMIT 100 --SELECT * FROM rx LIMIT 100

161
interface/map_def/test_regex_recs.sql
View File

@ -12,88 +12,89 @@ WITH
--------------------apply regex operations to transactions--------------------------------------------------------------------------------- --------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS ( rx AS (
SELECT SELECT
t.srce, t.srce,
t.id, t.id,
t.rec, t.rec,
m.target, m.target,
m.seq, m.seq,
regex->>'function' regex_function, regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name, e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path, e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag, e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention, e.v->>'map' map_intention,
e.v->>'retain' retain_result, e.v->>'retain' retain_result,
e.v->>'regex' regex_expression, e.v->>'regex' regex_expression,
e.rn target_item_number, e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number, COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match, mt.mt rx_match,
rp.rp rx_replace, rp.rp rx_replace,
--------------------------json key name assigned to return value----------------------------------------------------------------------- CASE e.v->>'map'
CASE e.v->>'map' WHEN 'y' THEN
WHEN 'y' THEN e.v->>'field'
e.v->>'field'
ELSE
null
END map_key,
--------------------------json value resulting from regular expression-----------------------------------------------------------------
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE ELSE
'{}'::jsonb null
END END map_key,
ELSE CASE e.v->>'map'
NULL WHEN 'y' THEN
END map_val, CASE regex->'regex'->>'function'
--------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- WHEN 'extract' THEN
CASE e.v->>'retain' CASE WHEN array_upper(mt.mt,1)=1
WHEN 'y' THEN THEN to_json(mt.mt[1])
e.v->>'field' ELSE array_to_json(mt.mt)
ELSE END::jsonb
NULL WHEN 'replace' THEN
END retain_key, to_jsonb(rp.rp)
--------------------------push regex result into json object--------------------------------------------------------------------------- ELSE
CASE e.v->>'retain' '{}'::jsonb
WHEN 'y' THEN END
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE ELSE
'{}'::jsonb NULL
END END map_val,
ELSE CASE e.v->>'retain'
NULL WHEN 'y' THEN
END retain_val e.v->>'field'
FROM ELSE
--------------------------start with all regex maps------------------------------------------------------------------------------------ NULL
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m END retain_key,
--------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- CASE e.v->>'retain'
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE WHEN 'y' THEN
--------------------------break out array of regluar expressions in the map------------------------------------------------------------ CASE regex->'regex'->>'function'
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true WHEN 'extract' THEN
--------------------------join to main transaction table but only certain key/values are included-------------------------------------- CASE WHEN array_upper(mt.mt,1)=1
INNER JOIN tps.trans t ON THEN to_json(trim(mt.mt[1]))
t.srce = m.srce AND ELSE array_to_json(mt.mt)
t.rec @> w.v END::jsonb
--------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- WHEN 'replace' THEN
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON to_jsonb(rtrim(rp.rp))
m.regex->>'function' = 'extract' ELSE
--------------------------same as above but for a replacement type function------------------------------------------------------------ '{}'::jsonb
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON END
m.regex->>'function' = 'replace' ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
) )
--SELECT * FROM rx LIMIT 100 --SELECT * FROM rx LIMIT 100

6
interface/map_values/srce_map_overwrite.sql
View File

@ -74,14 +74,20 @@ BEGIN
NULL NULL
END retain_val END retain_val
FROM FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON INNER JOIN tps.trans t ON
t.srce = m.srce AND t.srce = m.srce AND
t.rec @> w.v t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract' m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace' m.regex->'regex'->>'function' = 'replace'
WHERE WHERE