diff --git a/deploy/dump.cmd b/deploy/dump.cmd index fa5a5df..aab7d1e 100644 --- a/deploy/dump.cmd +++ b/deploy/dump.cmd @@ -1,5 +1,4 @@ -"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_schema.sql" +"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -n "tps" -O -F p -f "C:\users\fleet\Documents\tps_etl\deploy\ubm_schema.sql" -"/home/ubuntu/workspace/bigsql/pg10/bin/psql" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "/home/ubuntu/workspace/tps_etl/ubm_schema.sql" -"/home/ubuntu/workspace/bigsql/pg10/bin/psql" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "/home/ubuntu/workspace/tps_etl/ubm_data.sql" +"/home/ubuntu/workspace/bigsql/pg10/bin/psql" -h localhost -p 5433 -U ptrowbridge -d ubm -s -n "tps" -O -F p -f "/home/ubuntu/workspace/tps_etl/deploy/ubm_schema.sql" diff --git a/deploy/ubm_schema.sql b/deploy/ubm_schema.sql index 0c1c241..42b204a 100644 --- a/deploy/ubm_schema.sql +++ b/deploy/ubm_schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10beta4 --- Dumped by pg_dump version 10beta4 +-- Dumped from database version 10.2 +-- Dumped by pg_dump version 10.2 SET statement_timeout = 0; SET lock_timeout = 0; @@ -14,27 +14,6 @@ SET check_function_bodies = false; SET client_min_messages = warning; SET row_security = off; --- --- Name: bank; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA bank; - - --- --- Name: evt; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA evt; - - --- --- Name: SCHEMA evt; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON SCHEMA evt IS 'events'; - - -- -- Name: tps; Type: SCHEMA; Schema: -; Owner: - -- @@ -49,41 +28,6 @@ CREATE SCHEMA tps; COMMENT ON SCHEMA tps IS 'third party source'; --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - -SET search_path = bank, pg_catalog; - --- --- Name: pncc; Type: TYPE; Schema: bank; Owner: - --- - -CREATE TYPE pncc AS ( - "AsOfDate" date, - "BankId" text, - "AccountNumber" text, - "AccountName" text, - "BaiControl" text, - "Currency" text, - "Transaction" text, - "Reference" text, - "Amount" numeric, - "Description" text, - "AdditionalRemittance" text -); - - SET search_path = tps, pg_catalog; -- @@ -123,112 +67,6 @@ CREATE TYPE srce_defn_schema AS ( ); -SET search_path = evt, pg_catalog; - --- --- Name: build_hdr_item_mje_gl(jsonb); Type: FUNCTION; Schema: evt; Owner: - --- - -CREATE FUNCTION build_hdr_item_mje_gl(_j jsonb) RETURNS jsonb - LANGUAGE plpgsql - AS $_$ -DECLARE _m text; - -BEGIN - ---_j := $${"header":{"vendor":"Target","date":"10/12/2017","instrument":"Discover Card","module":"hdrio","total":47.74,"location":"Stow, OH","transaction":"purchase","offset":"dcard"},"item":[{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"shampoo","reason":"hygiene"},{"vend item":"HERBAL","amt":7.99,"account":"home supplies","item":"conditioner","reason":"hygiene"},{"vend item":"BUILDING SET","amt":28.74,"account":"recreation","item":"legos","reason":"toys","qty":6,"uom":"ea"},{"vend item":"OH TAX","amt":3.02,"account":"sales tax","item":"sales tax","reason":"sales tax","rate":"0.0675"}]}$$; - -WITH -j AS ( - SELECT - _j jb -) - ---------build a duplicating cross join table------------------ - - ,os AS ( - SELECT - flag, - sign, - x.offs - FROM - j - JOIN LATERAL - ( - VALUES - ('ITEM',1,null), - ('OFFSET',-1,j.jb->'header'->>'offset') - ) x (flag, sign, offs) ON TRUE - ) - - -------------do the cross join against all the item elements------------------- - -,build AS ( -SELECT - array['item',rn::text]::text jpath - ,COALESCE(os.offs,ae.e->>'account') acct - ,(ae.e->>'amt')::numeric * os.sign amount -FROM - j - LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(J.JB->'item') WITH ORDINALITY ae(e,rn) ON TRUE - CROSS JOIN os -ORDER BY - ae.rn ASC, - os.flag ASC -) - --------------re-aggregate the items into a single array point called 'gl'--------------- - -,agg AS ( -SELECT - jsonb_build_object('gl',jsonb_agg(row_to_json(b))) gl -FROM - build b -) - -------------take the new 'gl' with array key-value pair and combine it with the original--------------- - -SELECT - jsonb_pretty(agg.gl||j.jb) -INTO - _j -FROM - agg - CROSS JOIN j; - -RETURN _j; - -END -$_$; - - -SET search_path = public, pg_catalog; - --- --- Name: jsonb_extract(jsonb, text[]); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION jsonb_extract(rec jsonb, key_list text[]) RETURNS jsonb - LANGUAGE plpgsql - AS $$ -DECLARE - t text[]; - j jsonb := '{}'::jsonb; - -BEGIN - FOREACH t SLICE 1 IN ARRAY key_list LOOP - --RAISE NOTICE '%', t; - --RAISE NOTICE '%', t[1]; - j := j || jsonb_build_object(t[1],rec#>t); - END LOOP; - RETURN j; -END; -$$; - - -SET search_path = tps, pg_catalog; - -- -- Name: jsonb_concat(jsonb, jsonb); Type: FUNCTION; Schema: tps; Owner: - -- @@ -255,39 +93,10 @@ CREATE AGGREGATE jsonb_concat_obj(jsonb) ( ); -SET search_path = evt, pg_catalog; - SET default_tablespace = ''; SET default_with_oids = false; --- --- Name: log; Type: TABLE; Schema: evt; Owner: - --- - -CREATE TABLE log ( - id integer NOT NULL, - rec jsonb, - post_stmp timestamp with time zone DEFAULT now() -); - - --- --- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: - --- - -ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( - SEQUENCE NAME log_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1 -); - - -SET search_path = tps, pg_catalog; - -- -- Name: map_rm; Type: TABLE; Schema: tps; Owner: - -- @@ -374,18 +183,6 @@ ALTER TABLE trans_log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( ); -SET search_path = evt, pg_catalog; - --- --- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: - --- - -ALTER TABLE ONLY log - ADD CONSTRAINT log_pkey PRIMARY KEY (id); - - -SET search_path = tps, pg_catalog; - -- -- Name: map_rm map_rm_pk; Type: CONSTRAINT; Schema: tps; Owner: - -- @@ -440,13 +237,6 @@ CREATE INDEX trans_allj ON trans USING gin (allj); CREATE INDEX trans_rec ON trans USING gin (rec); --- --- Name: trans_srce; Type: INDEX; Schema: tps; Owner: - --- - -CREATE INDEX trans_srce ON trans USING btree (srce); - - -- -- Name: map_rm map_rm_fk_srce; Type: FK CONSTRAINT; Schema: tps; Owner: - --