Merge branch 'pt'
This commit is contained in:
Paul Trowbridge
commit
a41ea3cfa8
BIN
build_json.xlsx
BIN
build_json.xlsx
Binary file not shown.
File diff suppressed because one or more lines are too long
90
srce.pgsql
90
srce.pgsql
@ -1,42 +1,50 @@
|
||||
|
||||
DO $$
|
||||
|
||||
declare _t text;
|
||||
|
||||
begin
|
||||
|
||||
----------------------------------------------------build the column list of the temp table----------------------------------------------------------------
|
||||
|
||||
SELECT
|
||||
string_agg(quote_ident(prs.key)||' '||prs.type,',')
|
||||
INTO
|
||||
_t
|
||||
FROM
|
||||
TPS.srce
|
||||
--unwrap the schema definition array
|
||||
LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE
|
||||
WHERE
|
||||
srce = 'DCARD'
|
||||
GROUP BY
|
||||
srce;
|
||||
|
||||
----------------------------------------------------add create table verbage in front of column list--------------------------------------------------------
|
||||
|
||||
_t := format('CREATE TEMP TABLE csv_i (%s)', _t);
|
||||
raise notice '%', _t;
|
||||
|
||||
|
||||
----------------------------------------------------build the table-----------------------------------------------------------------------------------------
|
||||
|
||||
DROP TABLE IF EXISTS csv_i;
|
||||
|
||||
EXECUTE _t;
|
||||
|
||||
COPY csv_i FROM 'C:\Users\fleet\downloads\dc.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"');
|
||||
|
||||
|
||||
end
|
||||
$$;
|
||||
|
||||
SELECT * FROM csv_i;
|
||||
|
||||
|
||||
|
||||
/*
|
||||
INSERT INTO
|
||||
tps.trans (srce, rec)
|
||||
SELECT
|
||||
jsonb_pretty(
|
||||
$$
|
||||
{
|
||||
"name": "GOOGDM",
|
||||
"type": "json_csv",
|
||||
"schema": {
|
||||
"rows": [
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"status": "text",
|
||||
"distance": {
|
||||
"text": "text",
|
||||
"value": "numeric"
|
||||
},
|
||||
"duration": {
|
||||
"text": "text",
|
||||
"value": "value"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"status": "text",
|
||||
"origin_addresses": [
|
||||
"text"
|
||||
],
|
||||
"destination_addresses": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"unique_constraint": {
|
||||
"type": "key",
|
||||
"fields": [
|
||||
"{origin_adresses,0}",
|
||||
"{destination_adresses,0}"
|
||||
]
|
||||
}
|
||||
}
|
||||
$$::jsonb
|
||||
)
|
||||
'DCARD', row_to_json(csv_i) FROM csv_i;
|
||||
*/
|
||||
100
ubm.backup
100
ubm.backup
@ -1,100 +0,0 @@
|
||||
--
|
||||
-- PostgreSQL database dump
|
||||
--
|
||||
|
||||
-- Dumped from database version 10beta4
|
||||
-- Dumped by pg_dump version 10beta4
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
SET idle_in_transaction_session_timeout = 0;
|
||||
SET client_encoding = 'WIN1252';
|
||||
SET standard_conforming_strings = on;
|
||||
SET check_function_bodies = false;
|
||||
SET client_min_messages = warning;
|
||||
SET row_security = off;
|
||||
|
||||
--
|
||||
-- Name: evt; Type: SCHEMA; Schema: -; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
CREATE SCHEMA evt;
|
||||
|
||||
|
||||
ALTER SCHEMA evt OWNER TO ptrowbridge;
|
||||
|
||||
--
|
||||
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner:
|
||||
--
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
|
||||
|
||||
|
||||
--
|
||||
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner:
|
||||
--
|
||||
|
||||
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
|
||||
|
||||
|
||||
SET search_path = evt, pg_catalog;
|
||||
|
||||
SET default_tablespace = '';
|
||||
|
||||
SET default_with_oids = false;
|
||||
|
||||
--
|
||||
-- Name: log; Type: TABLE; Schema: evt; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
CREATE TABLE log (
|
||||
id integer NOT NULL,
|
||||
rec jsonb,
|
||||
module text
|
||||
);
|
||||
|
||||
|
||||
ALTER TABLE log OWNER TO ptrowbridge;
|
||||
|
||||
--
|
||||
-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
|
||||
SEQUENCE NAME log_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
COPY log (id, rec, module) FROM stdin;
|
||||
1 {"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"} MJE
|
||||
\.
|
||||
|
||||
|
||||
--
|
||||
-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
SELECT pg_catalog.setval('log_id_seq', 1, true);
|
||||
|
||||
|
||||
--
|
||||
-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: ptrowbridge
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY log
|
||||
ADD CONSTRAINT log_pkey PRIMARY KEY (id);
|
||||
|
||||
|
||||
--
|
||||
-- PostgreSQL database dump complete
|
||||
--
|
||||
|
||||
2
ubm_backup.cmd
Normal file
2
ubm_backup.cmd
Normal file
@ -0,0 +1,2 @@
|
||||
"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_schema.sql"
|
||||
"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "C:\users\fleet\Documents\tps_etl\ubm_data.sql"
|
||||
64
ubm_data.sql
Normal file
64
ubm_data.sql
Normal file
@ -0,0 +1,64 @@
|
||||
--
|
||||
-- PostgreSQL database dump
|
||||
--
|
||||
|
||||
-- Dumped from database version 10beta4
|
||||
-- Dumped by pg_dump version 10beta4
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
SET idle_in_transaction_session_timeout = 0;
|
||||
SET client_encoding = 'WIN1252';
|
||||
SET standard_conforming_strings = on;
|
||||
SET check_function_bodies = false;
|
||||
SET client_min_messages = warning;
|
||||
SET row_security = off;
|
||||
|
||||
SET search_path = evt, pg_catalog;
|
||||
|
||||
--
|
||||
-- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: -
|
||||
--
|
||||
|
||||
INSERT INTO log (id, rec) VALUES (1, '{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}');
|
||||
|
||||
|
||||
SET search_path = tps, pg_catalog;
|
||||
|
||||
--
|
||||
-- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}');
|
||||
INSERT INTO srce (srce, defn) VALUES ('DCARD', '{"name": "DCARD", "type": "csv", "schema": [{"key": "Trans. Date", "type": "date"}, {"key": "Post Date", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Category", "type": "text"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}"]}}');
|
||||
|
||||
|
||||
--
|
||||
-- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
|
||||
|
||||
SET search_path = evt, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: -
|
||||
--
|
||||
|
||||
SELECT pg_catalog.setval('log_id_seq', 1, true);
|
||||
|
||||
|
||||
SET search_path = tps, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
SELECT pg_catalog.setval('trans_id_seq', 1, false);
|
||||
---why is this here?
|
||||
|
||||
|
||||
--
|
||||
-- PostgreSQL database dump complete
|
||||
--
|
||||
|
||||
178
ubm_schema.sql
Normal file
178
ubm_schema.sql
Normal file
@ -0,0 +1,178 @@
|
||||
--
|
||||
-- PostgreSQL database dump
|
||||
--
|
||||
|
||||
-- Dumped from database version 10beta4
|
||||
-- Dumped by pg_dump version 10beta4
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
SET idle_in_transaction_session_timeout = 0;
|
||||
SET client_encoding = 'WIN1252';
|
||||
SET standard_conforming_strings = on;
|
||||
SET check_function_bodies = false;
|
||||
SET client_min_messages = warning;
|
||||
SET row_security = off;
|
||||
|
||||
--
|
||||
-- Name: evt; Type: SCHEMA; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
CREATE SCHEMA evt;
|
||||
|
||||
|
||||
--
|
||||
-- Name: SCHEMA evt; Type: COMMENT; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON SCHEMA evt IS 'events';
|
||||
|
||||
|
||||
--
|
||||
-- Name: tps; Type: SCHEMA; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
CREATE SCHEMA tps;
|
||||
|
||||
|
||||
--
|
||||
-- Name: SCHEMA tps; Type: COMMENT; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON SCHEMA tps IS 'third party source';
|
||||
|
||||
|
||||
--
|
||||
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
|
||||
|
||||
|
||||
--
|
||||
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
|
||||
|
||||
|
||||
SET search_path = tps, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: srce_defn_schema; Type: TYPE; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
CREATE TYPE srce_defn_schema AS (
|
||||
key text,
|
||||
type text
|
||||
);
|
||||
|
||||
|
||||
SET search_path = evt, pg_catalog;
|
||||
|
||||
SET default_tablespace = '';
|
||||
|
||||
SET default_with_oids = false;
|
||||
|
||||
--
|
||||
-- Name: log; Type: TABLE; Schema: evt; Owner: -
|
||||
--
|
||||
|
||||
CREATE TABLE log (
|
||||
id integer NOT NULL,
|
||||
rec jsonb
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: log_id_seq; Type: SEQUENCE; Schema: evt; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
|
||||
SEQUENCE NAME log_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1
|
||||
);
|
||||
|
||||
|
||||
SET search_path = tps, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: srce; Type: TABLE; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
CREATE TABLE srce (
|
||||
srce text NOT NULL,
|
||||
defn jsonb
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: trans; Type: TABLE; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
CREATE TABLE trans (
|
||||
id integer NOT NULL,
|
||||
srce text,
|
||||
rec jsonb,
|
||||
map jsonb
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: trans_id_seq; Type: SEQUENCE; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE trans ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
|
||||
SEQUENCE NAME trans_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1
|
||||
);
|
||||
|
||||
|
||||
SET search_path = evt, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: log log_pkey; Type: CONSTRAINT; Schema: evt; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY log
|
||||
ADD CONSTRAINT log_pkey PRIMARY KEY (id);
|
||||
|
||||
|
||||
SET search_path = tps, pg_catalog;
|
||||
|
||||
--
|
||||
-- Name: srce srce_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY srce
|
||||
ADD CONSTRAINT srce_pkey PRIMARY KEY (srce);
|
||||
|
||||
|
||||
--
|
||||
-- Name: trans trans_pkey; Type: CONSTRAINT; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY trans
|
||||
ADD CONSTRAINT trans_pkey PRIMARY KEY (id);
|
||||
|
||||
|
||||
--
|
||||
-- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY trans
|
||||
ADD CONSTRAINT trans_srce_fkey FOREIGN KEY (srce) REFERENCES srce(srce);
|
||||
|
||||
|
||||
--
|
||||
-- PostgreSQL database dump complete
|
||||
--
|
||||
|
||||
Binary file not shown.
Loading…
Reference in New Issue
Block a user