From 908a63b1c7ccdca35e31aec991f8de89a64b89f7 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 27 Dec 2018 00:56:40 -0500 Subject: [PATCH 1/2] isolate issue with source not being inserted --- .../interface/source_maint/srce_overwrite_all.sql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index 5318d02..47bbd29 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -12,6 +12,7 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH @@ -40,11 +41,12 @@ BEGIN SELECT f.srce ,f.actn - ,COALESCE(setd.message, deld.message) message + ,COALESCE(setd.message, '{"message":"not inserted"}'::jsonb/*deld.message*/) message FROM _full f LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- @@ -57,8 +59,14 @@ BEGIN FROM _do; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; + RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN From 0909c7da1998f7bea40b0680627b27e3f576028f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 27 Dec 2018 11:04:53 -0500 Subject: [PATCH 2/2] break out set and delete calls into seperate lateral join calls --- database/deploy/setup.sql | 63 +++++++++++++------ .../source_maint/srce_overwrite_all.sql | 55 ++++++++++------ .../004.source_overwrite.sql | 61 +++++++++++++----- 3 files changed, 126 insertions(+), 53 deletions(-) diff --git a/database/deploy/setup.sql b/database/deploy/setup.sql index 9014fe9..e184a2e 100644 --- a/database/deploy/setup.sql +++ b/database/deploy/setup.sql @@ -2054,17 +2054,18 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -2078,29 +2079,55 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, deld.message) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN @@ -2120,4 +2147,4 @@ BEGIN RETURN _message; END; $f$ -LANGUAGE plpgsql; \ No newline at end of file +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index 47bbd29..ee0446a 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -18,12 +18,12 @@ BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -37,27 +37,46 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, '{"message":"not inserted"}'::jsonb/*deld.message*/) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - --dual left joins for functions that touch the same table causes the first left join actions to be undone - --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; SELECT string_agg(srce,',') INTO _list FROM tps.srce; RAISE NOTICE 'multi source list: %', _list; diff --git a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql index 5318d02..ee0446a 100644 --- a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql +++ b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql @@ -12,17 +12,18 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -36,29 +37,55 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, deld.message) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN