From b94fedf1401856c1cf8bc9f31e17df9367fe046e Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 15 Dec 2018 01:32:34 -0500 Subject: [PATCH 01/11] tempate to start working on function that loads array of all sources --- .../interface/source_maint/srce_set_multi.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 database/interface/source_maint/srce_set_multi.sql diff --git a/database/interface/source_maint/srce_set_multi.sql b/database/interface/source_maint/srce_set_multi.sql new file mode 100644 index 0000000..ff7b478 --- /dev/null +++ b/database/interface/source_maint/srce_set_multi.sql @@ -0,0 +1,41 @@ +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_set(jsonb); +CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + --do a lateral join and expand the array + --do another lateral join calling the single set function for each row and aggregating the result messages + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error importing data" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file From 7d22b95472c1462c237d54892d5d4fdbad3312ed Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Dec 2018 23:14:56 -0500 Subject: [PATCH 02/11] setup a function to delete a source --- .../interface/source_maint/srce_delete.sql | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 database/interface/source_maint/srce_delete.sql diff --git a/database/interface/source_maint/srce_delete.sql b/database/interface/source_maint/srce_delete.sql new file mode 100644 index 0000000..7328fdb --- /dev/null +++ b/database/interface/source_maint/srce_delete.sql @@ -0,0 +1,51 @@ +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name' + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was moved to the recycle bin which has not been implemented so...hope you're in dev" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file From 8e2e20274bb61383196c934eee2af3761f51f004 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Dec 2018 23:15:32 -0500 Subject: [PATCH 03/11] semicolon --- database/interface/source_maint/srce_delete.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/interface/source_maint/srce_delete.sql b/database/interface/source_maint/srce_delete.sql index 7328fdb..7038718 100644 --- a/database/interface/source_maint/srce_delete.sql +++ b/database/interface/source_maint/srce_delete.sql @@ -12,7 +12,7 @@ BEGIN -------------------------------do delete--------------------------------- - DELETE FROM tps.srce WHERE srce = _defn->>'name' + DELETE FROM tps.srce WHERE srce = _defn->>'name'; --could move this record to a "recycle bin" table for a certain period of time --need to handle cascading record deletes From e005fcd42633a5b1cb090d6b6a5d4b5419eec60d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Dec 2018 23:40:27 -0500 Subject: [PATCH 04/11] rename --- .../source_maint/srce_overwrite_all.sql | 81 +++++++++++++++++++ .../interface/source_maint/srce_set_multi.sql | 41 ---------- 2 files changed, 81 insertions(+), 41 deletions(-) create mode 100644 database/interface/source_maint/srce_overwrite_all.sql delete mode 100644 database/interface/source_maint/srce_set_multi.sql diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql new file mode 100644 index 0000000..4431906 --- /dev/null +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -0,0 +1,81 @@ +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ON TRUE + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + l + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do AS ( + SELECT + f.srce + ,f.actn + ,COALESCE(setd.message, deld.message) message + FROM + _full f + LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + INTO + _message + FROM + _do; + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_set_multi.sql b/database/interface/source_maint/srce_set_multi.sql deleted file mode 100644 index ff7b478..0000000 --- a/database/interface/source_maint/srce_set_multi.sql +++ /dev/null @@ -1,41 +0,0 @@ -/* -This function takes and array of definition object where "name" object is the primary key -It will force the entire body of sources to match what is received -*/ -DROP FUNCTION IF EXISTS tps.srce_set(jsonb); -CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb -AS -$f$ -DECLARE - _message jsonb; - _MESSAGE_TEXT text; - _PG_EXCEPTION_DETAIL text; - _PG_EXCEPTION_HINT text; - _rebuild BOOLEAN; -BEGIN - - --do a lateral join and expand the array - --do another lateral join calling the single set function for each row and aggregating the result messages - - RETURN _message; - - - - EXCEPTION WHEN OTHERS THEN - GET STACKED DIAGNOSTICS - _MESSAGE_TEXT = MESSAGE_TEXT, - _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, - _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; - _message:= - ($$ - { - "status":"fail", - "message":"error importing data" - } - $$::jsonb) - ||jsonb_build_object('message_text',_MESSAGE_TEXT) - ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); - RETURN _message; -END; -$f$ -LANGUAGE plpgsql \ No newline at end of file From cb15d928c0d4ee9c9d8b420af8a40796111e2c17 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Dec 2018 23:40:39 -0500 Subject: [PATCH 05/11] change message --- database/interface/source_maint/srce_delete.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/interface/source_maint/srce_delete.sql b/database/interface/source_maint/srce_delete.sql index 7038718..004b795 100644 --- a/database/interface/source_maint/srce_delete.sql +++ b/database/interface/source_maint/srce_delete.sql @@ -22,7 +22,7 @@ BEGIN $$ { "status":"complete", - "message":"source was moved to the recycle bin which has not been implemented so...hope you're in dev" + "message":"source was permanently deleted" } $$::jsonb ); From 605c1c11303799704c8bacd4c64eb633387e2040 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 18 Dec 2018 23:41:12 -0500 Subject: [PATCH 06/11] no on statement required --- database/interface/source_maint/srce_overwrite_all.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index 4431906..c7018e5 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -22,7 +22,7 @@ BEGIN ,j.e->>'name' srce ,j.e defn FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ON TRUE + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( From 5b2bfa99d46f1befec146f36f7a9a46b672e8162 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 19 Dec 2018 00:12:07 -0500 Subject: [PATCH 07/11] parameter name and refernce to tps.srce got deleted --- database/interface/source_maint/srce_overwrite_all.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index c7018e5..5318d02 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -3,7 +3,7 @@ This function takes and array of definition object where "name" object is the pr It will force the entire body of sources to match what is received */ DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); -CREATE FUNCTION tps.srce_overwrite_all(jsonb) RETURNS jsonb +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb AS $f$ DECLARE @@ -31,7 +31,7 @@ BEGIN ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn ,COALESCE(_set.defn,_srce.defn) defn FROM - l + tps.srce _srce FULL OUTER JOIN _set ON _set.srce = _srce.srce ) From 1b2fc97cfd1bceb0dea25857966e36a2b4e4d6e7 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 19 Dec 2018 08:52:03 -0500 Subject: [PATCH 08/11] update --- .../001.extract_schemas.sql | 0 .../002.schema_build_sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename database/upgrade_scripts/{source_front_end_build => 01.source_front_end_build}/001.extract_schemas.sql (100%) rename database/upgrade_scripts/{source_front_end_build => 01.source_front_end_build}/002.schema_build_sql (100%) diff --git a/database/upgrade_scripts/source_front_end_build/001.extract_schemas.sql b/database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql similarity index 100% rename from database/upgrade_scripts/source_front_end_build/001.extract_schemas.sql rename to database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql diff --git a/database/upgrade_scripts/source_front_end_build/002.schema_build_sql b/database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql similarity index 100% rename from database/upgrade_scripts/source_front_end_build/002.schema_build_sql rename to database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql From 1d9532395b14be069fa7569cd97d4b7bae3755ea Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 21 Dec 2018 01:01:45 -0500 Subject: [PATCH 09/11] add scripts to support overwriting all source definitions with single json array --- database/deploy/setup.sql | 138 +++++++++++++++++- .../interface/source_maint/srce_delete.sql | 1 + .../003.srce_delete.sql | 52 +++++++ .../004.source_overwrite.sql | 81 ++++++++++ 4 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql create mode 100644 database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql diff --git a/database/deploy/setup.sql b/database/deploy/setup.sql index 9d84069..9014fe9 100644 --- a/database/deploy/setup.sql +++ b/database/deploy/setup.sql @@ -1984,4 +1984,140 @@ ORDER BY ,l.target ,l."count" desc; END; -$f$ \ No newline at end of file +$f$; + + +--setup function to delete a single source +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name'; + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was permanently deleted" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; + +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + tps.srce _srce + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do AS ( + SELECT + f.srce + ,f.actn + ,COALESCE(setd.message, deld.message) message + FROM + _full f + LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + INTO + _message + FROM + _do; + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql; \ No newline at end of file diff --git a/database/interface/source_maint/srce_delete.sql b/database/interface/source_maint/srce_delete.sql index 004b795..bbdcce9 100644 --- a/database/interface/source_maint/srce_delete.sql +++ b/database/interface/source_maint/srce_delete.sql @@ -1,3 +1,4 @@ +--setup function to delete a single source DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb AS diff --git a/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql b/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql new file mode 100644 index 0000000..bbdcce9 --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql @@ -0,0 +1,52 @@ +--setup function to delete a single source +DROP FUNCTION IF EXISTS tps.srce_delete(jsonb); +CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + -------------------------------do delete--------------------------------- + + DELETE FROM tps.srce WHERE srce = _defn->>'name'; + --could move this record to a "recycle bin" table for a certain period of time + --need to handle cascading record deletes + + ---------------------------set message----------------------------------- + _message:= + ( + $$ + { + "status":"complete", + "message":"source was permanently deleted" + } + $$::jsonb + ); + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error dropping the source" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql new file mode 100644 index 0000000..5318d02 --- /dev/null +++ b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql @@ -0,0 +1,81 @@ +/* +This function takes and array of definition object where "name" object is the primary key +It will force the entire body of sources to match what is received +*/ +DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb); +CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb +AS +$f$ +DECLARE + _message jsonb; + _MESSAGE_TEXT text; + _PG_EXCEPTION_DETAIL text; + _PG_EXCEPTION_HINT text; + _rebuild BOOLEAN; +BEGIN + + WITH + --retain the results of the update by srce + _set AS ( + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + ) + --full join + ,_full AS ( + SELECT + COALESCE(_srce.srce,_set.srce) srce + ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn + ,COALESCE(_set.defn,_srce.defn) defn + FROM + tps.srce _srce + FULL OUTER JOIN _set ON + _set.srce = _srce.srce + ) + --call functions from list + ,_do AS ( + SELECT + f.srce + ,f.actn + ,COALESCE(setd.message, deld.message) message + FROM + _full f + LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + --aggregate all the messages into one message + ---- + ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. + ---- + SELECT + jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + INTO + _message + FROM + _do; + + RETURN _message; + + + + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS + _MESSAGE_TEXT = MESSAGE_TEXT, + _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL, + _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT; + _message:= + ($$ + { + "status":"fail", + "message":"error updating sources" + } + $$::jsonb) + ||jsonb_build_object('message_text',_MESSAGE_TEXT) + ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL); + RETURN _message; +END; +$f$ +LANGUAGE plpgsql \ No newline at end of file From 908a63b1c7ccdca35e31aec991f8de89a64b89f7 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 27 Dec 2018 00:56:40 -0500 Subject: [PATCH 10/11] isolate issue with source not being inserted --- .../interface/source_maint/srce_overwrite_all.sql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index 5318d02..47bbd29 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -12,6 +12,7 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH @@ -40,11 +41,12 @@ BEGIN SELECT f.srce ,f.actn - ,COALESCE(setd.message, deld.message) message + ,COALESCE(setd.message, '{"message":"not inserted"}'::jsonb/*deld.message*/) message FROM _full f LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- @@ -57,8 +59,14 @@ BEGIN FROM _do; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; + RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN From 0909c7da1998f7bea40b0680627b27e3f576028f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 27 Dec 2018 11:04:53 -0500 Subject: [PATCH 11/11] break out set and delete calls into seperate lateral join calls --- database/deploy/setup.sql | 63 +++++++++++++------ .../source_maint/srce_overwrite_all.sql | 55 ++++++++++------ .../004.source_overwrite.sql | 61 +++++++++++++----- 3 files changed, 126 insertions(+), 53 deletions(-) diff --git a/database/deploy/setup.sql b/database/deploy/setup.sql index 9014fe9..e184a2e 100644 --- a/database/deploy/setup.sql +++ b/database/deploy/setup.sql @@ -2054,17 +2054,18 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -2078,29 +2079,55 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, deld.message) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN @@ -2120,4 +2147,4 @@ BEGIN RETURN _message; END; $f$ -LANGUAGE plpgsql; \ No newline at end of file +LANGUAGE plpgsql \ No newline at end of file diff --git a/database/interface/source_maint/srce_overwrite_all.sql b/database/interface/source_maint/srce_overwrite_all.sql index 47bbd29..ee0446a 100644 --- a/database/interface/source_maint/srce_overwrite_all.sql +++ b/database/interface/source_maint/srce_overwrite_all.sql @@ -18,12 +18,12 @@ BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -37,27 +37,46 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, '{"message":"not inserted"}'::jsonb/*deld.message*/) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - --dual left joins for functions that touch the same table causes the first left join actions to be undone - --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; SELECT string_agg(srce,',') INTO _list FROM tps.srce; RAISE NOTICE 'multi source list: %', _list; diff --git a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql index 5318d02..ee0446a 100644 --- a/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql +++ b/database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql @@ -12,17 +12,18 @@ DECLARE _PG_EXCEPTION_DETAIL text; _PG_EXCEPTION_HINT text; _rebuild BOOLEAN; + _list text; BEGIN WITH --retain the results of the update by srce _set AS ( - SELECT - j.rn rn - ,j.e->>'name' srce - ,j.e defn - FROM - jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) + SELECT + j.rn rn + ,j.e->>'name' srce + ,j.e defn + FROM + jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn) ) --full join ,_full AS ( @@ -36,29 +37,55 @@ BEGIN _set.srce = _srce.srce ) --call functions from list - ,_do AS ( - SELECT - f.srce - ,f.actn - ,COALESCE(setd.message, deld.message) message - FROM - _full f - LEFT JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' - LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ,_do_set AS ( + SELECT + f.srce + ,f.actn + ,setd.message + FROM + _full f + JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET' + --dual left joins for functions that touch the same table causes the first left join actions to be undone + --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' + ) + ,_do_del AS ( + SELECT + f.srce + ,f.actn + ,deld.message + FROM + _full f + JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE' ) --aggregate all the messages into one message ---- ---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this. ---- SELECT - jsonb_agg(jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message')) + jsonb_agg(m) INTO _message FROM - _do; + ( + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_set + UNION ALL + SELECT + jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m + FROM + _do_del + ) x; + + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'multi source list: %', _list; RETURN _message; + SELECT string_agg(srce,',') INTO _list FROM tps.srce; + RAISE NOTICE 'after return: %', _list; + EXCEPTION WHEN OTHERS THEN