Merge branch 'pt'

srce.pgsql

@@ -1,42 +1,50 @@
-SELECT
+
-jsonb_pretty(
+DO $$
-$$
+
-{
+declare _t text;
-    "name": "GOOGDM",
+
-    "type": "json_csv",
+begin
-    "schema": {
+	
-        "rows": [
+----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
-            {
+
-                "elements": [
+	SELECT
-                    {
+        string_agg(quote_ident(prs.key)||' '||prs.type,',')
-                        "status": "text",
+    INTO
-                        "distance": {
+    	_t
-                            "text": "text",
+    FROM 
-                            "value": "numeric"
+        TPS.srce
-                        },
+        --unwrap the schema definition array
-                        "duration": {
+        LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
-                            "text": "text",
+    WHERE   
-                            "value": "value"
+        srce = 'DCARD'
-                       }
+    GROUP BY
-                    }
+        srce;
-                ]
+        
-            }
+----------------------------------------------------add create table verbage in front of column list--------------------------------------------------------
-        ],
+
-        "status": "text",
+    _t := format('CREATE TEMP TABLE csv_i (%s)', _t);
-        "origin_addresses": [
+    raise notice '%', _t;
-            "text"
+    
-        ],
+
-        "destination_addresses": [
+----------------------------------------------------build the table-----------------------------------------------------------------------------------------
-            "text"
+
-        ]
+    DROP TABLE IF EXISTS csv_i;
-    },
+    
-    "unique_constraint": {
+    EXECUTE _t;
-        "type": "key",
+
-        "fields": [
+    COPY csv_i FROM 'C:\Users\fleet\downloads\dc.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"');
-            "{origin_adresses,0}",
+
-            "{destination_adresses,0}"
+
-        ]
+end
-    }
+$$;
-}
+
-$$::jsonb
+SELECT * FROM csv_i;
-)
+
+
+
+/*
+INSERT INTO
+    tps.trans (srce, rec)
+SELECT 
+    'DCARD', row_to_json(csv_i) FROM csv_i;
+*/

ubm.backup

@@ -1,100 +0,0 @@
---
--- PostgreSQL database dump
---
-
--- Dumped from database version 10beta4
--- Dumped by pg_dump version 10beta4
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET client_encoding = 'WIN1252';
-SET standard_conforming_strings = on;
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET row_security = off;
-
---
--- Name: evt; Type: SCHEMA; Schema: -; Owner: ptrowbridge
---
-
-CREATE SCHEMA evt;
-
-
-ALTER SCHEMA evt OWNER TO ptrowbridge;
-
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: 
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: 
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
-SET search_path = evt, pg_catalog;
-
-SET default_tablespace = '';
-
-SET default_with_oids = false;
-
---
--- Name: log; Type: TABLE; Schema: evt; Owner: ptrowbridge
---
-
-CREATE TABLE log (
-    id integer NOT NULL,
-    rec jsonb,
-    module text
-);
-
-
-ALTER TABLE log OWNER TO ptrowbridge;
-
---
--- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: ptrowbridge
---
-
-ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
-    SEQUENCE NAME log_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1
-);
-
-
---
--- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: ptrowbridge
---
-
-COPY log (id, rec, module) FROM stdin;
-1	{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}	MJE
-\.
-
-
---
--- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: ptrowbridge
---
-
-SELECT pg_catalog.setval('log_id_seq', 1, true);
-
-
---
--- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: ptrowbridge
---
-
-ALTER TABLE ONLY log
-    ADD CONSTRAINT log_pkey PRIMARY KEY (id);
-
-
---
--- PostgreSQL database dump complete
---
-

ubm_backup.cmd

@@ -0,0 +1,2 @@
+"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_schema.sql"
+"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_data.sql"

ubm_data.sql

@@ -0,0 +1,64 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 10beta4
+-- Dumped by pg_dump version 10beta4
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'WIN1252';
+SET standard_conforming_strings = on;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET row_security = off;
+
+SET search_path = evt, pg_catalog;
+
+--
+-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: -
+--
+
+INSERT INTO log (id, rec) VALUES (1, '{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}');
+
+
+SET search_path = tps, pg_catalog;
+
+--
+-- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: -
+--
+
+INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}');
+INSERT INTO srce (srce, defn) VALUES ('DCARD', '{"name": "DCARD", "type": "csv", "schema": [{"key": "Trans. Date", "type": "date"}, {"key": "Post Date", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Category", "type": "text"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}"]}}');
+
+
+--
+-- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: -
+--
+
+
+
+SET search_path = evt, pg_catalog;
+
+--
+-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: -
+--
+
+SELECT pg_catalog.setval('log_id_seq', 1, true);
+
+
+SET search_path = tps, pg_catalog;
+
+--
+-- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: -
+--
+
+SELECT pg_catalog.setval('trans_id_seq', 1, false);
+---why is this here?
+
+
+--
+-- PostgreSQL database dump complete
+--
+

ubm_schema.sql

@@ -0,0 +1,178 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 10beta4
+-- Dumped by pg_dump version 10beta4
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'WIN1252';
+SET standard_conforming_strings = on;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- Name: evt; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA evt;
+
+
+--
+-- Name: SCHEMA evt; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON SCHEMA evt IS 'events';
+
+
+--
+-- Name: tps; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA tps;
+
+
+--
+-- Name: SCHEMA tps; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON SCHEMA tps IS 'third party source';
+
+
+--
+-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
+
+
+--
+-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
+
+
+SET search_path = tps, pg_catalog;
+
+--
+-- Name: srce_defn_schema; Type: TYPE; Schema: tps; Owner: -
+--
+
+CREATE TYPE srce_defn_schema AS (
+	key text,
+	type text
+);
+
+
+SET search_path = evt, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+--
+-- Name: log; Type: TABLE; Schema: evt; Owner: -
+--
+
+CREATE TABLE log (
+    id integer NOT NULL,
+    rec jsonb
+);
+
+
+--
+-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: -
+--
+
+ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME log_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
+
+SET search_path = tps, pg_catalog;
+
+--
+-- Name: srce; Type: TABLE; Schema: tps; Owner: -
+--
+
+CREATE TABLE srce (
+    srce text NOT NULL,
+    defn jsonb
+);
+
+
+--
+-- Name: trans; Type: TABLE; Schema: tps; Owner: -
+--
+
+CREATE TABLE trans (
+    id integer NOT NULL,
+    srce text,
+    rec jsonb,
+    map jsonb
+);
+
+
+--
+-- Name: trans_id_seq; Type: SEQUENCE; Schema: tps; Owner: -
+--
+
+ALTER TABLE trans ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
+    SEQUENCE NAME trans_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
+
+SET search_path = evt, pg_catalog;
+
+--
+-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: -
+--
+
+ALTER TABLE ONLY log
+    ADD CONSTRAINT log_pkey PRIMARY KEY (id);
+
+
+SET search_path = tps, pg_catalog;
+
+--
+-- Name: srce srce_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
+--
+
+ALTER TABLE ONLY srce
+    ADD CONSTRAINT srce_pkey PRIMARY KEY (srce);
+
+
+--
+-- Name: trans trans_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
+--
+
+ALTER TABLE ONLY trans
+    ADD CONSTRAINT trans_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: -
+--
+
+ALTER TABLE ONLY trans
+    ADD CONSTRAINT trans_srce_fkey FOREIGN KEY (srce) REFERENCES srce(srce);
+
+
+--
+-- PostgreSQL database dump complete
+--
+