pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'pt'

Browse Source
This commit is contained in:
Paul Trowbridge 2017-10-13 02:59:45 -04:00
commit a41ea3cfa8
8 changed files with 295 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
build_json.xlsx
View File

Binary file not shown.

1
new_format.pgsql
View File

File diff suppressed because one or more lines are too long

90
srce.pgsql
View File

@ -1,42 +1,50 @@
DO $$
declare _t text;
begin
----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
SELECT
string_agg(quote_ident(prs.key)||' '||prs.type,',')
INTO
_t
FROM
TPS.srce
--unwrap the schema definition array
LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
WHERE
srce = 'DCARD'
GROUP BY
srce;
----------------------------------------------------add create table verbage in front of column list--------------------------------------------------------
_t := format('CREATE TEMP TABLE csv_i (%s)', _t);
raise notice '%', _t;
----------------------------------------------------build the table-----------------------------------------------------------------------------------------
DROP TABLE IF EXISTS csv_i;
EXECUTE _t;
COPY csv_i FROM 'C:\Users\fleet\downloads\dc.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"');
end
$$;
SELECT * FROM csv_i;
/*
INSERT INTO
tps.trans (srce, rec)
SELECT SELECT
jsonb_pretty( 'DCARD', row_to_json(csv_i) FROM csv_i;
$$ */
{
"name": "GOOGDM",
"type": "json_csv",
"schema": {
"rows": [
{
"elements": [
{
"status": "text",
"distance": {
"text": "text",
"value": "numeric"
},
"duration": {
"text": "text",
"value": "value"
}
}
]
}
],
"status": "text",
"origin_addresses": [
"text"
],
"destination_addresses": [
"text"
]
},
"unique_constraint": {
"type": "key",
"fields": [
"{origin_adresses,0}",
"{destination_adresses,0}"
]
}
}
$$::jsonb
)

100
ubm.backup
View File

@ -1,100 +0,0 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 10beta4
-- Dumped by pg_dump version 10beta4
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'WIN1252';
SET standard_conforming_strings = on;
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: evt; Type: SCHEMA; Schema: -; Owner: ptrowbridge
--
CREATE SCHEMA evt;
ALTER SCHEMA evt OWNER TO ptrowbridge;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner:
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner:
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
SET search_path = evt, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: log; Type: TABLE; Schema: evt; Owner: ptrowbridge
--
CREATE TABLE log (
id integer NOT NULL,
rec jsonb,
module text
);
ALTER TABLE log OWNER TO ptrowbridge;
--
-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: ptrowbridge
--
ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
SEQUENCE NAME log_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
--
-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: ptrowbridge
--
COPY log (id, rec, module) FROM stdin;
1 {"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"} MJE
\.
--
-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: ptrowbridge
--
SELECT pg_catalog.setval('log_id_seq', 1, true);
--
-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: ptrowbridge
--
ALTER TABLE ONLY log
ADD CONSTRAINT log_pkey PRIMARY KEY (id);
--
-- PostgreSQL database dump complete
--

2
ubm_backup.cmd Normal file
View File

@ -0,0 +1,2 @@
"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_schema.sql"
"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_data.sql"

64
ubm_data.sql Normal file
View File

@ -0,0 +1,64 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 10beta4
-- Dumped by pg_dump version 10beta4
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'WIN1252';
SET standard_conforming_strings = on;
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
SET search_path = evt, pg_catalog;
--
-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: -
--
INSERT INTO log (id, rec) VALUES (1, '{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}');
SET search_path = tps, pg_catalog;
--
-- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: -
--
INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}');
INSERT INTO srce (srce, defn) VALUES ('DCARD', '{"name": "DCARD", "type": "csv", "schema": [{"key": "Trans. Date", "type": "date"}, {"key": "Post Date", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Category", "type": "text"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}"]}}');
--
-- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: -
--
SET search_path = evt, pg_catalog;
--
-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: -
--
SELECT pg_catalog.setval('log_id_seq', 1, true);
SET search_path = tps, pg_catalog;
--
-- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: -
--
SELECT pg_catalog.setval('trans_id_seq', 1, false);
---why is this here?
--
-- PostgreSQL database dump complete
--

178
ubm_schema.sql Normal file
View File

@ -0,0 +1,178 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 10beta4
-- Dumped by pg_dump version 10beta4
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'WIN1252';
SET standard_conforming_strings = on;
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: evt; Type: SCHEMA; Schema: -; Owner: -
--
CREATE SCHEMA evt;
--
-- Name: SCHEMA evt; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON SCHEMA evt IS 'events';
--
-- Name: tps; Type: SCHEMA; Schema: -; Owner: -
--
CREATE SCHEMA tps;
--
-- Name: SCHEMA tps; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON SCHEMA tps IS 'third party source';
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
SET search_path = tps, pg_catalog;
--
-- Name: srce_defn_schema; Type: TYPE; Schema: tps; Owner: -
--
CREATE TYPE srce_defn_schema AS (
key text,
type text
);
SET search_path = evt, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: log; Type: TABLE; Schema: evt; Owner: -
--
CREATE TABLE log (
id integer NOT NULL,
rec jsonb
);
--
-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: -
--
ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
SEQUENCE NAME log_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
SET search_path = tps, pg_catalog;
--
-- Name: srce; Type: TABLE; Schema: tps; Owner: -
--
CREATE TABLE srce (
srce text NOT NULL,
defn jsonb
);
--
-- Name: trans; Type: TABLE; Schema: tps; Owner: -
--
CREATE TABLE trans (
id integer NOT NULL,
srce text,
rec jsonb,
map jsonb
);
--
-- Name: trans_id_seq; Type: SEQUENCE; Schema: tps; Owner: -
--
ALTER TABLE trans ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
SEQUENCE NAME trans_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
SET search_path = evt, pg_catalog;
--
-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: -
--
ALTER TABLE ONLY log
ADD CONSTRAINT log_pkey PRIMARY KEY (id);
SET search_path = tps, pg_catalog;
--
-- Name: srce srce_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
--
ALTER TABLE ONLY srce
ADD CONSTRAINT srce_pkey PRIMARY KEY (srce);
--
-- Name: trans trans_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
--
ALTER TABLE ONLY trans
ADD CONSTRAINT trans_pkey PRIMARY KEY (id);
--
-- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: -
--
ALTER TABLE ONLY trans
ADD CONSTRAINT trans_srce_fkey FOREIGN KEY (srce) REFERENCES srce(srce);
--
-- PostgreSQL database dump complete
--

BIN
~$build_json.xlsx
View File

Binary file not shown.