diff --git a/build_json.xlsx b/build_json.xlsx index 858f7c2..0c7022c 100644 Binary files a/build_json.xlsx and b/build_json.xlsx differ diff --git a/new_format.pgsql b/new_format.pgsql index 9cd33fd..a0d3c8c 100644 --- a/new_format.pgsql +++ b/new_format.pgsql @@ -1,3 +1,4 @@ + WITH j AS ( SELECT $${"vendor":"Acme","date":"9/19/2017","instrument":"Discover Card","module":"MJE_O","offset":"dcard","total":127.66,"item":[{"item":"sun chips","amt":3.29,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cheetos paws","amt":3.69,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif","amt":4.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif whips","amt":3.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif","amt":4.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"goya chipotle pepp","amt":2.29,"account":"food","reason":"chili peppers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"quick oats","amt":4.69,"account":"food","reason":"oats","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"quick oats","amt":4.69,"account":"food","reason":"oats","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"sara lee artesano","amt":2.99,"account":"food","reason":"bread","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"dan lgt yog","amt":3.29,"account":"food","reason":"yogurt","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"bananas","amt":3.7,"account":"food","reason":"bananas","qty":7.11,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"yellow onions","amt":2.99,"account":"food","reason":"onions","qty":3,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"orange peppers","amt":4.11,"account":"food","reason":"peppers","qty":1.03,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"asparagus","amt":5.14,"account":"food","reason":"asparagus","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"broccoli","amt":2.99,"account":"food","reason":"broccoli","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"hn fat free frank","amt":5.49,"account":"food","reason":"hot dogs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"hn fat free frank","amt":5.49,"account":"food","reason":"hot dogs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"butterscotch morsels","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"seedless grapes","amt":11.45,"account":"food","reason":"grapes","qty":2.87,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"honecrisp apples","amt":14.76,"account":"food","reason":"apples","qty":3.7,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"garlic expressions","amt":4.99,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"olivegarden dressing","amt":3.49,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"garlic expressions","amt":4.99,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"thomas cinamon bagels","amt":4.69,"account":"food","reason":"bagels","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"stawberries","amt":1.25,"account":"food","reason":"strawberries","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"stawberries","amt":1.25,"account":"food","reason":"strawberries","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"provolone cheese","amt":6.99,"account":"food","reason":"cheese","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"lemon juice","amt":1.99,"account":"food","reason":"lemon juice","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cucumbers","amt":1.99,"account":"food","reason":"cucumbers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cucumbers","amt":1.99,"account":"food","reason":"cucumbers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.7,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-0.99,"account":"food","reason":"broccoli","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-8.03,"account":"food","reason":"grapes","qty":2.87,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-5.55,"account":"food","reason":"apples","qty":3.7,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"}]}$$::JSONB as r diff --git a/srce.pgsql b/srce.pgsql index cec92cd..1bf3a42 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -1,42 +1,50 @@ -SELECT -jsonb_pretty( -$$ -{ - "name": "GOOGDM", - "type": "json_csv", - "schema": { - "rows": [ - { - "elements": [ - { - "status": "text", - "distance": { - "text": "text", - "value": "numeric" - }, - "duration": { - "text": "text", - "value": "value" - } - } - ] - } - ], - "status": "text", - "origin_addresses": [ - "text" - ], - "destination_addresses": [ - "text" - ] - }, - "unique_constraint": { - "type": "key", - "fields": [ - "{origin_adresses,0}", - "{destination_adresses,0}" - ] - } -} -$$::jsonb -) \ No newline at end of file + +DO $$ + +declare _t text; + +begin + +----------------------------------------------------build the column list of the temp table---------------------------------------------------------------- + + SELECT + string_agg(quote_ident(prs.key)||' '||prs.type,',') + INTO + _t + FROM + TPS.srce + --unwrap the schema definition array + LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE + WHERE + srce = 'DCARD' + GROUP BY + srce; + +----------------------------------------------------add create table verbage in front of column list-------------------------------------------------------- + + _t := format('CREATE TEMP TABLE csv_i (%s)', _t); + raise notice '%', _t; + + +----------------------------------------------------build the table----------------------------------------------------------------------------------------- + + DROP TABLE IF EXISTS csv_i; + + EXECUTE _t; + + COPY csv_i FROM 'C:\Users\fleet\downloads\dc.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"'); + + +end +$$; + +SELECT * FROM csv_i; + + + +/* +INSERT INTO + tps.trans (srce, rec) +SELECT + 'DCARD', row_to_json(csv_i) FROM csv_i; +*/ \ No newline at end of file diff --git a/ubm.backup b/ubm.backup deleted file mode 100644 index 2c02768..0000000 --- a/ubm.backup +++ /dev/null @@ -1,100 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 10beta4 --- Dumped by pg_dump version 10beta4 - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'WIN1252'; -SET standard_conforming_strings = on; -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: evt; Type: SCHEMA; Schema: -; Owner: ptrowbridge --- - -CREATE SCHEMA evt; - - -ALTER SCHEMA evt OWNER TO ptrowbridge; - --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - -SET search_path = evt, pg_catalog; - -SET default_tablespace = ''; - -SET default_with_oids = false; - --- --- Name: log; Type: TABLE; Schema: evt; Owner: ptrowbridge --- - -CREATE TABLE log ( - id integer NOT NULL, - rec jsonb, - module text -); - - -ALTER TABLE log OWNER TO ptrowbridge; - --- --- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: ptrowbridge --- - -ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( - SEQUENCE NAME log_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1 -); - - --- --- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: ptrowbridge --- - -COPY log (id, rec, module) FROM stdin; -1 {"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"} MJE -\. - - --- --- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: ptrowbridge --- - -SELECT pg_catalog.setval('log_id_seq', 1, true); - - --- --- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: ptrowbridge --- - -ALTER TABLE ONLY log - ADD CONSTRAINT log_pkey PRIMARY KEY (id); - - --- --- PostgreSQL database dump complete --- - diff --git a/ubm_backup.cmd b/ubm_backup.cmd new file mode 100644 index 0000000..e8e9242 --- /dev/null +++ b/ubm_backup.cmd @@ -0,0 +1,2 @@ +"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_schema.sql" +"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_data.sql" diff --git a/ubm_data.sql b/ubm_data.sql new file mode 100644 index 0000000..93538cf --- /dev/null +++ b/ubm_data.sql @@ -0,0 +1,64 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10beta4 +-- Dumped by pg_dump version 10beta4 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'WIN1252'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +SET search_path = evt, pg_catalog; + +-- +-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: - +-- + +INSERT INTO log (id, rec) VALUES (1, '{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}'); + + +SET search_path = tps, pg_catalog; + +-- +-- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: - +-- + +INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('DCARD', '{"name": "DCARD", "type": "csv", "schema": [{"key": "Trans. Date", "type": "date"}, {"key": "Post Date", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Category", "type": "text"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}"]}}'); + + +-- +-- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: - +-- + + + +SET search_path = evt, pg_catalog; + +-- +-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: - +-- + +SELECT pg_catalog.setval('log_id_seq', 1, true); + + +SET search_path = tps, pg_catalog; + +-- +-- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: - +-- + +SELECT pg_catalog.setval('trans_id_seq', 1, false); +---why is this here? + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/ubm_schema.sql b/ubm_schema.sql new file mode 100644 index 0000000..964a317 --- /dev/null +++ b/ubm_schema.sql @@ -0,0 +1,178 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10beta4 +-- Dumped by pg_dump version 10beta4 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'WIN1252'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: evt; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA evt; + + +-- +-- Name: SCHEMA evt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON SCHEMA evt IS 'events'; + + +-- +-- Name: tps; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA tps; + + +-- +-- Name: SCHEMA tps; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON SCHEMA tps IS 'third party source'; + + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = tps, pg_catalog; + +-- +-- Name: srce_defn_schema; Type: TYPE; Schema: tps; Owner: - +-- + +CREATE TYPE srce_defn_schema AS ( + key text, + type text +); + + +SET search_path = evt, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: log; Type: TABLE; Schema: evt; Owner: - +-- + +CREATE TABLE log ( + id integer NOT NULL, + rec jsonb +); + + +-- +-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: - +-- + +ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +SET search_path = tps, pg_catalog; + +-- +-- Name: srce; Type: TABLE; Schema: tps; Owner: - +-- + +CREATE TABLE srce ( + srce text NOT NULL, + defn jsonb +); + + +-- +-- Name: trans; Type: TABLE; Schema: tps; Owner: - +-- + +CREATE TABLE trans ( + id integer NOT NULL, + srce text, + rec jsonb, + map jsonb +); + + +-- +-- Name: trans_id_seq; Type: SEQUENCE; Schema: tps; Owner: - +-- + +ALTER TABLE trans ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME trans_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +SET search_path = evt, pg_catalog; + +-- +-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: - +-- + +ALTER TABLE ONLY log + ADD CONSTRAINT log_pkey PRIMARY KEY (id); + + +SET search_path = tps, pg_catalog; + +-- +-- Name: srce srce_pkey; Type: CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY srce + ADD CONSTRAINT srce_pkey PRIMARY KEY (srce); + + +-- +-- Name: trans trans_pkey; Type: CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY trans + ADD CONSTRAINT trans_pkey PRIMARY KEY (id); + + +-- +-- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY trans + ADD CONSTRAINT trans_srce_fkey FOREIGN KEY (srce) REFERENCES srce(srce); + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/~$build_json.xlsx b/~$build_json.xlsx deleted file mode 100644 index a902500..0000000 Binary files a/~$build_json.xlsx and /dev/null differ