From b3c0b3355c85fe8440bda12b6f86fbf548aceaf3 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 13 Oct 2017 09:18:48 -0400 Subject: [PATCH 01/27] add credit cards --- coa.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/coa.md b/coa.md index 27b6f0c..ca7ec47 100644 --- a/coa.md +++ b/coa.md @@ -1,3 +1,5 @@ +Balance Sheet +----------------------- * Cash - On-hand - Hunt Checking @@ -20,5 +22,10 @@ - Discover - Kohls - Lowes + - BestBuy + - Target + - TheHomeDepot * Mortgage - Principle + + From 811fbb90f2761ff7a93fe786acaf87e3a630819d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 13 Oct 2017 11:41:43 -0400 Subject: [PATCH 02/27] add rebuild script --- rebuild_pg.cmd | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 rebuild_pg.cmd diff --git a/rebuild_pg.cmd b/rebuild_pg.cmd new file mode 100644 index 0000000..2c3a491 --- /dev/null +++ b/rebuild_pg.cmd @@ -0,0 +1,4 @@ +"C:\PostgreSQL\pg10\bin\psql" -h localhost -p 5433 -d postgres -U postgres -c "DROP DATABASE ubm" +"C:\PostgreSQL\pg10\bin\psql" -h localhost -p 5433 -d postgres -U postgres -c "CREATE DATABASE ubm" +"C:\PostgreSQL\pg10\bin\psql" -h localhost -p 5433 -d ubm -U postgres -f "C:\users\ptrowbridge\documents\tps_etl\ubm_schema.sql" +"C:\PostgreSQL\pg10\bin\psql" -h localhost -p 5433 -d ubm -U postgres -f "C:\users\ptrowbridge\documents\tps_etl\ubm_data.sql" From 5e8768e4a6fe21cbdc7358905f7e9d68734e3402 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 13 Oct 2017 11:57:25 -0400 Subject: [PATCH 03/27] get rid of test insert data --- ubm_data.sql | 119 --------------------------------------------------- 1 file changed, 119 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index 4a66cac..a7d62b3 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -36,125 +36,6 @@ INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", " -- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: - -- -INSERT INTO trans (id, srce, rec, map) VALUES (2, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871559", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (3, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871557", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (4, 'PNCO', '{"Loan#": "606780191", "Advances": 27992.25, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016556", "Loan Balance": 27992.25, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (5, 'PNCO', '{"Loan#": "606780191", "Advances": 8000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016551", "Loan Balance": 35992.25, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (6, 'PNCO', '{"Loan#": "606780191", "Advances": 1656860.21, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016546", "Loan Balance": 1692852.46, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (7, 'PNCO', '{"Loan#": "606780191", "Advances": 2233.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016541", "Loan Balance": 1695085.46, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (8, 'PNCO', '{"Loan#": "606780191", "Advances": 17000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016536", "Loan Balance": 1712085.46, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (9, 'PNCO', '{"Loan#": "606780191", "Advances": 2535261.04, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016531", "Loan Balance": 4247346.50, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (10, 'PNCO', '{"Loan#": "606780191", "Advances": 25919.62, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016526", "Loan Balance": 4273266.12, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (11, 'PNCO', '{"Loan#": "606780191", "Advances": 4581.25, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016521", "Loan Balance": 4277847.37, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (12, 'PNCO', '{"Loan#": "606780191", "Advances": 90000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016516", "Loan Balance": 4367847.37, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (13, 'PNCO', '{"Loan#": "606780191", "Advances": 1000000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016511", "Loan Balance": 5367847.37, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (14, 'PNCO', '{"Loan#": "606780191", "Advances": 6280808.31, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016506", "Loan Balance": 11648655.68, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (15, 'PNCO', '{"Loan#": "606780191", "Advances": 11275.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016501", "Loan Balance": 11659930.68, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (16, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 20000000.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016496", "Loan Balance": -8340069.32, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (17, 'PNCO', '{"Loan#": "606780191", "Advances": 720000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016491", "Loan Balance": -7620069.32, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (18, 'PNCO', '{"Loan#": "606780191", "Advances": 118449.84, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016486", "Loan Balance": -7501619.48, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (19, 'PNCO', '{"Loan#": "606780191", "Advances": 100000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016481", "Loan Balance": -7401619.48, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (20, 'PNCO', '{"Loan#": "606780191", "Advances": 148449.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016476", "Loan Balance": -7253170.48, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (21, 'PNCO', '{"Loan#": "606780191", "Advances": 21585.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016471", "Loan Balance": -7231585.48, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (22, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 30873305.71, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016466", "Loan Balance": -38104891.19, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (23, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 647868.87, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016461", "Loan Balance": -38752760.06, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (24, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 1095801.30, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016456", "Loan Balance": -39848561.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (25, 'PNCO', '{"Loan#": "606780191", "Advances": 165000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016451", "Loan Balance": -39683561.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (26, 'PNCO', '{"Loan#": "606780191", "Advances": 20127.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016446", "Loan Balance": -39663434.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (27, 'PNCO', '{"Loan#": "606780191", "Advances": 8625000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016441", "Loan Balance": -31038434.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (28, 'PNCO', '{"Loan#": "606780191", "Advances": 2500.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016436", "Loan Balance": -31035934.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (29, 'PNCO', '{"Loan#": "606780191", "Advances": 2500.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016431", "Loan Balance": -31033434.36, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (30, 'PNCO', '{"Loan#": "606780191", "Advances": 63505950.65, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016426", "Loan Balance": 32472516.29, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (31, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "240990470", "Loan Balance": 32472516.29, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (32, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 526831.23, "Post Date": "2015-02-20", "Adjustments": 0.00, "Description": "", "Reference #": "241372087", "Loan Balance": 31945685.06, "Effective Date": "2015-02-20"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (33, 'PNCO', '{"Loan#": "606780191", "Advances": 175000.00, "Payments": 0.00, "Post Date": "2015-02-20", "Adjustments": 0.00, "Description": "", "Reference #": "241372082", "Loan Balance": 32120685.06, "Effective Date": "2015-02-20"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (34, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 377629.52, "Post Date": "2015-02-23", "Adjustments": 0.00, "Description": "", "Reference #": "241489522", "Loan Balance": 31743055.54, "Effective Date": "2015-02-23"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (35, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 481204.25, "Post Date": "2015-02-24", "Adjustments": 0.00, "Description": "", "Reference #": "241609117", "Loan Balance": 31261851.29, "Effective Date": "2015-02-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (36, 'PNCO', '{"Loan#": "606780191", "Advances": 1963839.00, "Payments": 0.00, "Post Date": "2015-02-24", "Adjustments": 0.00, "Description": "", "Reference #": "241609112", "Loan Balance": 33225690.29, "Effective Date": "2015-02-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (37, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 963062.01, "Post Date": "2015-02-25", "Adjustments": 0.00, "Description": "", "Reference #": "241748333", "Loan Balance": 32262628.28, "Effective Date": "2015-02-25"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (38, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 71500.00, "Post Date": "2015-02-26", "Adjustments": 0.00, "Description": "", "Reference #": "241856901", "Loan Balance": 32191128.28, "Effective Date": "2015-02-26"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (39, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 928834.96, "Post Date": "2015-02-26", "Adjustments": 0.00, "Description": "", "Reference #": "241856896", "Loan Balance": 31262293.32, "Effective Date": "2015-02-26"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (40, 'PNCO', '{"Loan#": "606780191", "Advances": 15742.26, "Payments": 0.00, "Post Date": "2015-02-27", "Adjustments": 0.00, "Description": "", "Reference #": "241979563", "Loan Balance": 31278035.58, "Effective Date": "2015-02-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (41, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 93093.63, "Post Date": "2015-02-27", "Adjustments": 0.00, "Description": "", "Reference #": "241979558", "Loan Balance": 31184941.95, "Effective Date": "2015-02-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (42, 'PNCO', '{"Loan#": "606780191", "Advances": 3150.00, "Payments": 0.00, "Post Date": "2015-02-27", "Adjustments": 0.00, "Description": "", "Reference #": "241979553", "Loan Balance": 31188091.95, "Effective Date": "2015-02-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (43, 'PNCO', '{"Loan#": "606780191", "Advances": 71500.00, "Payments": 0.00, "Post Date": "2015-02-27", "Adjustments": 0.00, "Description": "", "Reference #": "241979548", "Loan Balance": 31259591.95, "Effective Date": "2015-02-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (44, 'PNCO', '{"Loan#": "606780191", "Advances": 608.33, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146434", "Loan Balance": 31260200.28, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (45, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 750360.24, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146429", "Loan Balance": 30509840.04, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (46, 'PNCO', '{"Loan#": "606780191", "Advances": 125000.00, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146424", "Loan Balance": 30634840.04, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (47, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146421", "Loan Balance": 30634840.04, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (48, 'PNCO', '{"Loan#": "606780191", "Advances": 39722.22, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146416", "Loan Balance": 30674562.26, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (49, 'PNCO', '{"Loan#": "606780191", "Advances": 54895.76, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146411", "Loan Balance": 30729458.02, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (50, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 388182.11, "Post Date": "2015-03-03", "Adjustments": 0.00, "Description": "", "Reference #": "242376851", "Loan Balance": 30341275.91, "Effective Date": "2015-03-03"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (51, 'PNCO', '{"Loan#": "606780191", "Advances": 2758322.00, "Payments": 0.00, "Post Date": "2015-03-03", "Adjustments": 0.00, "Description": "", "Reference #": "242376846", "Loan Balance": 33099597.91, "Effective Date": "2015-03-03"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (52, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 604637.39, "Post Date": "2015-03-04", "Adjustments": 0.00, "Description": "", "Reference #": "242488845", "Loan Balance": 32494960.52, "Effective Date": "2015-03-04"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (53, 'PNCO', '{"Loan#": "606780191", "Advances": 444000.00, "Payments": 0.00, "Post Date": "2015-03-04", "Adjustments": 0.00, "Description": "", "Reference #": "242488840", "Loan Balance": 32938960.52, "Effective Date": "2015-03-04"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (54, 'PNCO', '{"Loan#": "606780191", "Advances": 226000.00, "Payments": 0.00, "Post Date": "2015-03-05", "Adjustments": 0.00, "Description": "", "Reference #": "242600430", "Loan Balance": 33164960.52, "Effective Date": "2015-03-05"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (55, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 1028329.75, "Post Date": "2015-03-06", "Adjustments": 0.00, "Description": "", "Reference #": "242708090", "Loan Balance": 32136630.77, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (56, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": -25000000.00, "Description": "", "Reference #": "242812271", "Loan Balance": 7136630.77, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (57, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 304435.57, "Post Date": "2015-03-09", "Adjustments": 0.00, "Description": "", "Reference #": "242825473", "Loan Balance": 6832195.20, "Effective Date": "2015-03-09"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (58, 'PNCO', '{"Loan#": "606780191", "Advances": 210464.46, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": 0.00, "Description": "", "Reference #": "242825468", "Loan Balance": 7042659.66, "Effective Date": "2015-03-09"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (59, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 908124.34, "Post Date": "2015-03-10", "Adjustments": 0.00, "Description": "", "Reference #": "242947218", "Loan Balance": 6134535.32, "Effective Date": "2015-03-10"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (60, 'PNCO', '{"Loan#": "606780191", "Advances": 24905.07, "Payments": 0.00, "Post Date": "2015-03-10", "Adjustments": 0.00, "Description": "", "Reference #": "242947213", "Loan Balance": 6159440.39, "Effective Date": "2015-03-10"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (61, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 927081.96, "Post Date": "2015-03-11", "Adjustments": 0.00, "Description": "", "Reference #": "243061937", "Loan Balance": 5232358.43, "Effective Date": "2015-03-11"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (62, 'PNCO', '{"Loan#": "606780191", "Advances": 3095988.00, "Payments": 0.00, "Post Date": "2015-03-11", "Adjustments": 0.00, "Description": "", "Reference #": "243061932", "Loan Balance": 8328346.43, "Effective Date": "2015-03-11"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (63, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 76272.73, "Post Date": "2015-03-12", "Adjustments": 0.00, "Description": "", "Reference #": "243178426", "Loan Balance": 8252073.70, "Effective Date": "2015-03-12"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (64, 'PNCO', '{"Loan#": "606780191", "Advances": 106300.00, "Payments": 0.00, "Post Date": "2015-03-12", "Adjustments": 0.00, "Description": "", "Reference #": "243178421", "Loan Balance": 8358373.70, "Effective Date": "2015-03-12"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (65, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 101648.46, "Post Date": "2015-03-13", "Adjustments": 0.00, "Description": "", "Reference #": "243288682", "Loan Balance": 8256725.24, "Effective Date": "2015-03-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (66, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 350.00, "Post Date": "2015-03-16", "Adjustments": 0.00, "Description": "", "Reference #": "243408568", "Loan Balance": 8256375.24, "Effective Date": "2015-03-16"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (67, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 385774.81, "Post Date": "2015-03-16", "Adjustments": 0.00, "Description": "", "Reference #": "243408563", "Loan Balance": 7870600.43, "Effective Date": "2015-03-16"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (68, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 232042.23, "Post Date": "2015-03-16", "Adjustments": 0.00, "Description": "", "Reference #": "243408558", "Loan Balance": 7638558.20, "Effective Date": "2015-03-16"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (69, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 90853.40, "Post Date": "2015-03-17", "Adjustments": 0.00, "Description": "", "Reference #": "243546995", "Loan Balance": 7547704.80, "Effective Date": "2015-03-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (70, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 1100373.44, "Post Date": "2015-03-17", "Adjustments": 0.00, "Description": "", "Reference #": "243546990", "Loan Balance": 6447331.36, "Effective Date": "2015-03-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (71, 'PNCO', '{"Loan#": "606780191", "Advances": 1750223.00, "Payments": 0.00, "Post Date": "2015-03-17", "Adjustments": 0.00, "Description": "", "Reference #": "243546985", "Loan Balance": 8197554.36, "Effective Date": "2015-03-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (72, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 874291.47, "Post Date": "2015-03-18", "Adjustments": 0.00, "Description": "", "Reference #": "243664486", "Loan Balance": 7323262.89, "Effective Date": "2015-03-18"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (73, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 526443.58, "Post Date": "2015-03-18", "Adjustments": 0.00, "Description": "", "Reference #": "243664481", "Loan Balance": 6796819.31, "Effective Date": "2015-03-18"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (74, 'PNCO', '{"Loan#": "606780191", "Advances": 928418.71, "Payments": 0.00, "Post Date": "2015-03-18", "Adjustments": 0.00, "Description": "", "Reference #": "243664476", "Loan Balance": 7725238.02, "Effective Date": "2015-03-18"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (75, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 2809.94, "Post Date": "2015-03-19", "Adjustments": 0.00, "Description": "", "Reference #": "243772989", "Loan Balance": 7722428.08, "Effective Date": "2015-03-19"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (76, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 727091.26, "Post Date": "2015-03-19", "Adjustments": 0.00, "Description": "", "Reference #": "243772984", "Loan Balance": 6995336.82, "Effective Date": "2015-03-19"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (77, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 177215.75, "Post Date": "2015-03-20", "Adjustments": 0.00, "Description": "", "Reference #": "243877202", "Loan Balance": 6818121.07, "Effective Date": "2015-03-20"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (78, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 186772.26, "Post Date": "2015-03-20", "Adjustments": 0.00, "Description": "", "Reference #": "243877197", "Loan Balance": 6631348.81, "Effective Date": "2015-03-20"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (79, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 78218.74, "Post Date": "2015-03-23", "Adjustments": 0.00, "Description": "", "Reference #": "243990906", "Loan Balance": 6553130.07, "Effective Date": "2015-03-23"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (80, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 225525.95, "Post Date": "2015-03-23", "Adjustments": 0.00, "Description": "", "Reference #": "243990901", "Loan Balance": 6327604.12, "Effective Date": "2015-03-23"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (81, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 69784.36, "Post Date": "2015-03-23", "Adjustments": 0.00, "Description": "", "Reference #": "243990896", "Loan Balance": 6257819.76, "Effective Date": "2015-03-23"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (82, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 879711.41, "Post Date": "2015-03-24", "Adjustments": 0.00, "Description": "", "Reference #": "244114417", "Loan Balance": 5378108.35, "Effective Date": "2015-03-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (83, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 41213.54, "Post Date": "2015-03-24", "Adjustments": 0.00, "Description": "", "Reference #": "244114412", "Loan Balance": 5336894.81, "Effective Date": "2015-03-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (84, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 230755.54, "Post Date": "2015-03-24", "Adjustments": 0.00, "Description": "", "Reference #": "244114407", "Loan Balance": 5106139.27, "Effective Date": "2015-03-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (85, 'PNCO', '{"Loan#": "606780191", "Advances": 3877213.00, "Payments": 0.00, "Post Date": "2015-03-24", "Adjustments": 0.00, "Description": "", "Reference #": "244114402", "Loan Balance": 8983352.27, "Effective Date": "2015-03-24"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (86, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 226129.00, "Post Date": "2015-03-25", "Adjustments": 0.00, "Description": "", "Reference #": "244220348", "Loan Balance": 8757223.27, "Effective Date": "2015-03-25"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (87, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 316633.43, "Post Date": "2015-03-25", "Adjustments": 0.00, "Description": "", "Reference #": "244220343", "Loan Balance": 8440589.84, "Effective Date": "2015-03-25"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (88, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 655688.78, "Post Date": "2015-03-26", "Adjustments": 0.00, "Description": "", "Reference #": "244330144", "Loan Balance": 7784901.06, "Effective Date": "2015-03-26"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (89, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 322217.94, "Post Date": "2015-03-26", "Adjustments": 0.00, "Description": "", "Reference #": "244330139", "Loan Balance": 7462683.12, "Effective Date": "2015-03-26"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (90, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 45600.92, "Post Date": "2015-03-27", "Adjustments": 0.00, "Description": "", "Reference #": "244472455", "Loan Balance": 7417082.20, "Effective Date": "2015-03-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (91, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 29545.69, "Post Date": "2015-03-27", "Adjustments": 0.00, "Description": "", "Reference #": "244472450", "Loan Balance": 7387536.51, "Effective Date": "2015-03-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (92, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 8500.38, "Post Date": "2015-03-27", "Adjustments": 0.00, "Description": "", "Reference #": "244472445", "Loan Balance": 7379036.13, "Effective Date": "2015-03-27"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (93, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 162414.01, "Post Date": "2015-03-30", "Adjustments": 0.00, "Description": "", "Reference #": "244588483", "Loan Balance": 7216622.12, "Effective Date": "2015-03-30"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (94, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 153618.23, "Post Date": "2015-03-30", "Adjustments": 0.00, "Description": "", "Reference #": "244588478", "Loan Balance": 7063003.89, "Effective Date": "2015-03-30"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (95, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 74556.66, "Post Date": "2015-03-30", "Adjustments": 0.00, "Description": "", "Reference #": "244588473", "Loan Balance": 6988447.23, "Effective Date": "2015-03-30"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (96, 'PNCO', '{"Loan#": "606780191", "Advances": 75000.00, "Payments": 0.00, "Post Date": "2015-03-30", "Adjustments": 0.00, "Description": "", "Reference #": "244588468", "Loan Balance": 7063447.23, "Effective Date": "2015-03-30"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (97, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 1462007.22, "Post Date": "2015-03-31", "Adjustments": 0.00, "Description": "", "Reference #": "244782046", "Loan Balance": 5601440.01, "Effective Date": "2015-03-31"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (98, 'PNCO', '{"Loan#": "606780191", "Advances": 0.00, "Payments": 197734.52, "Post Date": "2015-03-31", "Adjustments": 0.00, "Description": "", "Reference #": "244782041", "Loan Balance": 5403705.49, "Effective Date": "2015-03-31"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (99, 'PNCO', '{"Loan#": "606780191", "Advances": 3737051.00, "Payments": 0.00, "Post Date": "2015-03-31", "Adjustments": 0.00, "Description": "", "Reference #": "244782036", "Loan Balance": 9140756.49, "Effective Date": "2015-03-31"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (100, 'PNCO', '{"Loan#": "606780206", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871723", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (101, 'PNCO', '{"Loan#": "606780206", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871725", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (102, 'PNCO', '{"Loan#": "606780206", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "240990522", "Loan Balance": 0.00, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (103, 'PNCO', '{"Loan#": "606780214", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871891", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (104, 'PNCO', '{"Loan#": "606780214", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240871889", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (105, 'PNCO', '{"Loan#": "606780214", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "240990558", "Loan Balance": 0.00, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (106, 'PNCO', '{"Loan#": "606780214", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": 0.00, "Description": "", "Reference #": "242812278", "Loan Balance": 0.00, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (107, 'PNCO', '{"Loan#": "606780214", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": 25000000.00, "Description": "", "Reference #": "242812280", "Loan Balance": 25000000.00, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (108, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240873773", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (109, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240873771", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (110, 'PNCO', '{"Loan#": "606780275", "Advances": 20000000.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "241016561", "Loan Balance": 20000000.00, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (111, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "240990781", "Loan Balance": 20000000.00, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (112, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 125000.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146442", "Loan Balance": 19875000.00, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (113, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-02", "Adjustments": 0.00, "Description": "", "Reference #": "242146439", "Loan Balance": 19875000.00, "Effective Date": "2015-03-02"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (114, 'PNCO', '{"Loan#": "606780275", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": -19500000.00, "Description": "", "Reference #": "242813139", "Loan Balance": 375000.00, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (115, 'PNCO', '{"Loan#": "606780281", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240874075", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (116, 'PNCO', '{"Loan#": "606780281", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-13", "Adjustments": 0.00, "Description": "", "Reference #": "240874073", "Loan Balance": 0.00, "Effective Date": "2015-02-13"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (117, 'PNCO', '{"Loan#": "606780281", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-02-17", "Adjustments": 0.00, "Description": "", "Reference #": "240991019", "Loan Balance": 0.00, "Effective Date": "2015-02-17"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (118, 'PNCO', '{"Loan#": "606780281", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": 19500000.00, "Description": "", "Reference #": "242813148", "Loan Balance": 19500000.00, "Effective Date": "2015-03-06"}', NULL); -INSERT INTO trans (id, srce, rec, map) VALUES (119, 'PNCO', '{"Loan#": "606780281", "Advances": 0.00, "Payments": 0.00, "Post Date": "2015-03-09", "Adjustments": 0.00, "Description": "", "Reference #": "242813146", "Loan Balance": 19500000.00, "Effective Date": "2015-03-06"}', NULL); - SET search_path = evt, pg_catalog; From dc9e160c3ef729b0c98ca3f8b6ff3aa782c7c9e4 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 13 Oct 2017 12:00:34 -0400 Subject: [PATCH 04/27] set sequence to 1 --- ubm_data.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubm_data.sql b/ubm_data.sql index a7d62b3..17d4ee9 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -52,7 +52,7 @@ SET search_path = tps, pg_catalog; -- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: - -- -SELECT pg_catalog.setval('trans_id_seq', 119, true); +SELECT pg_catalog.setval('trans_id_seq', 1, true); -- From c81a7c7dc48522683abdd4bb0ac7ca5ba8060f58 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Fri, 13 Oct 2017 17:17:56 -0400 Subject: [PATCH 05/27] extract key at import --- srce.pgsql | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index beaa58a..2fef279 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -15,7 +15,7 @@ begin --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = 'PNCO' + srce = 'PNCL' GROUP BY srce; @@ -31,14 +31,31 @@ begin EXECUTE _t; + COPY csv_i FROM 'C:\Users\ptrowbridge\Documents\OneDrive - The HC Companies, Inc\Cash\build_hist\pncl.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"'); + end $$; ---SELECT * FROM csv_i; - -COPY csv_i FROM 'C:\Users\ptrowbridge\Documents\OneDrive - The HC Companies, Inc\Cash\build_hist\full_dl\15Q1bal.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"'); +--SELECT row_to_json(csv_i) FROM csv_i; +/* INSERT INTO tps.trans (srce, rec) SELECT - 'PNCO', row_to_json(csv_i) FROM csv_i; \ No newline at end of file + 'PNCO', row_to_json(csv_i) FROM csv_i; +*/ + + +SELECT + (row_to_json(i)::jsonb) #> ae.e::text[], + srce, + defn->'unique_constraint'->'type', + defn->'unique_constraint'->'fields', + ae.e, + ae.rn +FROM + csv_i i + INNER JOIN tps.srce s ON + s.srce = 'PNCL' + LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE +LIMIT 10 \ No newline at end of file From fe761c6975f281a081449fd802f6814df2438110 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 18 Oct 2017 01:06:00 -0400 Subject: [PATCH 06/27] include pt branch srce.pgsql changes & transaction range query --- srce.pgsql | 110 ++++++++++++++++++++++++++++------------ transaction_range.pgsql | 14 +++++ 2 files changed, 92 insertions(+), 32 deletions(-) create mode 100644 transaction_range.pgsql diff --git a/srce.pgsql b/srce.pgsql index 2fef279..16209a2 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -1,61 +1,107 @@ DO $$ -declare _t text; +DECLARE _t text; +DECLARE _c text; -begin +BEGIN ----------------------------------------------------build the column list of the temp table---------------------------------------------------------------- SELECT - string_agg(quote_ident(prs.key)||' '||prs.type,',') + string_agg(quote_ident(prs.key)||' '||prs.type,','), + string_agg(quote_ident(prs.key),',') INTO - _t + _t, + _c FROM TPS.srce --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = 'PNCL' + srce = 'DCARD' GROUP BY srce; ----------------------------------------------------add create table verbage in front of column list-------------------------------------------------------- - _t := format('CREATE TEMP TABLE csv_i (%s)', _t); - raise notice '%', _t; - - -----------------------------------------------------build the table----------------------------------------------------------------------------------------- + _t := format('CREATE TEMP TABLE csv_i (%s, id SERIAL)', _t); + --RAISE NOTICE '%', _t; + --RAISE NOTICE '%', _c; DROP TABLE IF EXISTS csv_i; EXECUTE _t; - COPY csv_i FROM 'C:\Users\ptrowbridge\Documents\OneDrive - The HC Companies, Inc\Cash\build_hist\pncl.csv' WITH (HEADER TRUE,DELIMITER ',', FORMAT CSV, ENCODING 'SQL_ASCII',QUOTE '"'); +----------------------------------------------------do the insert------------------------------------------------------------------------------------------- -end + --the column list needs to be dynamic forcing this whole line to be dynamic + _t := format('COPY csv_i (%s) FROM ''C:\Users\fleet\downloads\dc.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); + + --RAISE NOTICE '%', _t; + + EXECUTE _t; + + +END $$; ---SELECT row_to_json(csv_i) FROM csv_i; - -/* -INSERT INTO - tps.trans (srce, rec) -SELECT - 'PNCO', row_to_json(csv_i) FROM csv_i; -*/ - +--******************************************* +--this needs to aggregate on id sequence +--******************************************* +WITH pending_list AS ( + SELECT + ---creates a key value pair and then aggregates rows of key value pairs + jsonb_object_agg( + (ae.e::text[])[1], --the key name + (row_to_json(i)::jsonb) #> ae.e::text[] --get the target value from the key from the csv row that has been converted to json + ) json_key, + row_to_json(i)::JSONB rec, + srce, + --ae.rn, + id + FROM + csv_i i + INNER JOIN tps.srce s ON + s.srce = 'DCARD' + LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE + GROUP BY + i.*, + srce, + id + ORDER BY + id +) +------results of an insert operation-------------- +, inserted AS ( + INSERT INTO + tps.trans (srce, rec) + SELECT + pl.srce + ,pl.rec + FROM + pending_list pl + LEFT JOIN tps.trans t ON + t.srce = pl.srce + AND t.rec @> pl.json_key + WHERE + t IS NULL + ----this conflict is only if an exact duplicate rec json happens, which will be rejected + ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified + ON CONFLICT ON CONSTRAINT uc_rec DO NOTHING + RETURNING * +) +----records not inserted------ SELECT - (row_to_json(i)::jsonb) #> ae.e::text[], - srce, - defn->'unique_constraint'->'type', - defn->'unique_constraint'->'fields', - ae.e, - ae.rn + srce + ,rec FROM - csv_i i - INNER JOIN tps.srce s ON - s.srce = 'PNCL' - LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE -LIMIT 10 \ No newline at end of file + pending_list + +EXCEPT ALL + +SELECT + srce + ,rec +FROM + inserted; \ No newline at end of file diff --git a/transaction_range.pgsql b/transaction_range.pgsql new file mode 100644 index 0000000..b4f469e --- /dev/null +++ b/transaction_range.pgsql @@ -0,0 +1,14 @@ +SELECT + t.srce + ,(ae.e::text[])[1] unq_constr + ,MIN(rec #>> ae.e::text[]) min_text + ,COUNT(*) cnt + ,MAX(rec #>> ae.e::text[]) max_text +FROM + tps.trans t + INNER JOIN tps.srce s ON + s.srce = t.srce + LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE +GROUP BY + t.srce + ,(ae.e::text[])[1] \ No newline at end of file From 748d6ddb3c4c902aa16625c1e2de26309e30ea35 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 18 Oct 2017 14:10:07 -0400 Subject: [PATCH 07/27] merge summary of records not inserted from pt branch --- srce.pgsql | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index 70a6ccf..98b4a21 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -89,18 +89,34 @@ WITH pending_list AS ( ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified RETURNING * ) +---------raw list of records not inserted---------- +, not_inserted AS ( + SELECT + srce + ,rec + FROM + pending_list -----records not inserted------ + EXCEPT ALL + + SELECT + srce + ,rec + FROM + inserted +) +--------summarize records not inserted------------------ SELECT - srce - ,rec + t.srce + ,(ae.e::text[])[1] unq_constr + ,MIN(rec #>> ae.e::text[]) min_text + ,MAX(rec #>> ae.e::text[]) max_text + ,JSONB_PRETTY(JSON_AGG(rec #> ae.e::text[] ORDER BY rec #>> ae.e::text[])::JSONB) FROM - pending_list - -EXCEPT ALL - -SELECT - srce - ,rec -FROM - inserted; \ No newline at end of file + not_inserted t + INNER JOIN tps.srce s ON + s.srce = t.srce + LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE +GROUP BY + t.srce + ,(ae.e::text[])[1]; \ No newline at end of file From 419017f98ce3202e8f03378ec35308472bb05478 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 18 Oct 2017 14:11:57 -0400 Subject: [PATCH 08/27] switch to PNCC --- srce.pgsql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index 98b4a21..d8edc0a 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -18,7 +18,7 @@ BEGIN --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = 'DCARD' + srce = 'PNCC' GROUP BY srce; @@ -35,7 +35,7 @@ BEGIN ----------------------------------------------------do the insert------------------------------------------------------------------------------------------- --the column list needs to be dynamic forcing this whole line to be dynamic - _t := format('COPY csv_i (%s) FROM ''C:\Users\fleet\downloads\dfs.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); + _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); --RAISE NOTICE '%', _t; @@ -62,7 +62,7 @@ WITH pending_list AS ( FROM csv_i i INNER JOIN tps.srce s ON - s.srce = 'DCARD' + s.srce = 'PNCC' LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE GROUP BY i.*, From e0163382f974c7b4a4295d42b5db7fa20aa4f143 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 08:49:33 -0400 Subject: [PATCH 09/27] added sources --- ubm_data.sql | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index 6d6d168..e643208 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10beta4 --- Dumped by pg_dump version 10beta4 +-- Dumped from database version 10rc1 +-- Dumped by pg_dump version 10rc1 SET statement_timeout = 0; SET lock_timeout = 0; @@ -20,7 +20,6 @@ SET search_path = evt, pg_catalog; -- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: - -- -INSERT INTO log (id, rec) VALUES (1, '{"date": "2017-08-20", "item": [{"item": "Green Chili", "amount": 1.49, "account": "food"}, {"item": "Black Beans", "amount": 1.6, "account": "food"}, {"item": "Distilled Water", "amount": 7.12, "account": "food"}, {"item": "Fruit Preservative", "amount": 3.99, "account": "food"}, {"item": "Watch Battery", "amount": 3.79, "account": "stuff"}, {"item": "Sales Tax", "amount": "0.26", "account": "taxes"}, {"item": "Green Chili", "amount": -1.49, "account": "dcard"}, {"item": "Black Beans", "amount": -1.6, "account": "dcard"}, {"item": "Distilled Water", "amount": -7.12, "account": "dcard"}, {"item": "Fruit Preservative", "amount": -3.99, "account": "dcard"}, {"item": "Watch Battery", "amount": -3.79, "account": "dcard"}, {"item": "Sales Tax", "amount": -0.26, "account": "dcard"}], "vendor": "Drug Mart", "instrument": "Discover Card"}'); SET search_path = tps, pg_catalog; @@ -29,13 +28,9 @@ SET search_path = tps, pg_catalog; -- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: - -- -INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('DCARD', '{"name": "DCARD", "type": "csv", "schema": [{"key": "Trans. Date", "type": "date"}, {"key": "Post Date", "type": "date"}, {"key": "Description", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Category", "type": "text"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}", "{Trans. Date}"]}}'); - - --- --- Data for Name: trans; Type: TABLE DATA; Schema: tps; Owner: - --- +INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}", "{Schedule#}"]}}'); SET search_path = evt, pg_catalog; @@ -44,7 +39,7 @@ SET search_path = evt, pg_catalog; -- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: - -- -SELECT pg_catalog.setval('log_id_seq', 1, true); +SELECT pg_catalog.setval('log_id_seq', 1, false); SET search_path = tps, pg_catalog; From 995eb181c10aaae918fc18e845af5f6f4bcdd7ba Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 09:24:55 -0400 Subject: [PATCH 10/27] dont export tps.trans table --- ubm_backup.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubm_backup.cmd b/ubm_backup.cmd index 3d120ca..79cb548 100644 --- a/ubm_backup.cmd +++ b/ubm_backup.cmd @@ -1,2 +1,2 @@ "C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm -s -O -F p -f "C:\users\ptrowbridge\Documents\tps_etl\ubm_schema.sql" -"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -F p -f "C:\users\ptrowbridge\Documents\tps_etl\ubm_data.sql" \ No newline at end of file +"C:\PostgreSQL\pg10\bin\pg_dump" -h localhost -p 5433 -U ptrowbridge -d ubm --column-inserts -a -O -T "tps.trans" -F p -f "C:\users\ptrowbridge\Documents\tps_etl\ubm_data.sql" \ No newline at end of file From bc3cecec098da02fce4cde00e2bc0a86be62bbea Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 09:25:07 -0400 Subject: [PATCH 11/27] add order by clause --- transaction_range.pgsql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/transaction_range.pgsql b/transaction_range.pgsql index b4f469e..c55a083 100644 --- a/transaction_range.pgsql +++ b/transaction_range.pgsql @@ -10,5 +10,8 @@ FROM s.srce = t.srce LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE GROUP BY + t.srce + ,(ae.e::text[])[1] +ORDER BY t.srce ,(ae.e::text[])[1] \ No newline at end of file From dcf2fcbb6cee0bbc30c45db6ea2176436b0f06e0 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 09:25:31 -0400 Subject: [PATCH 12/27] update srce definitions --- ubm_data.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index e643208..a8688a7 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -29,8 +29,8 @@ SET search_path = tps, pg_catalog; -- INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{Post Date}", "{Schedule#}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCL", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{PostDate}", "{Schedule#}"]}}'); SET search_path = evt, pg_catalog; From 1ae529855708be641463a749428da0a2f5cc7872 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 09:25:42 -0400 Subject: [PATCH 13/27] add index on tps.rec --- ubm_schema.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ubm_schema.sql b/ubm_schema.sql index 3e04f5f..49c36e3 100644 --- a/ubm_schema.sql +++ b/ubm_schema.sql @@ -164,6 +164,13 @@ ALTER TABLE ONLY trans ADD CONSTRAINT trans_pkey PRIMARY KEY (id); +-- +-- Name: trans_rec; Type: INDEX; Schema: tps; Owner: - +-- + +CREATE INDEX trans_rec ON trans USING gin (rec); + + -- -- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: - -- From e0f8bad70ac663f2c9f41cdcbde509f0968d014e Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 09:53:52 -0400 Subject: [PATCH 14/27] add ADPRP --- ubm_data.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/ubm_data.sql b/ubm_data.sql index a8688a7..0041faa 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -31,6 +31,7 @@ SET search_path = tps, pg_catalog; INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCL", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{PostDate}", "{Schedule#}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", "descr": "ADP Detail Report", "schema": [{"key": "batch", "type": "text"}, {"key": "week", "type": "text"}, {"key": "period_end", "type": "text"}, {"key": "pay_date", "type": "text"}, {"key": "adp_comp", "type": "text"}, {"key": "hours_reg", "type": "numeric"}, {"key": "hours_ot", "type": "numeric"}, {"key": "adp_dep_home", "type": "text"}, {"key": "adp_dep_worked", "type": "text"}, {"key": "adp_dep", "type": "text"}, {"key": "gl_dep", "type": "text"}, {"key": "checkn", "type": "text"}, {"key": "employee", "type": "text"}, {"key": "title", "type": "text"}, {"key": "prim_offset", "type": "text"}, {"key": "cms_tb", "type": "text"}, {"key": "cms_acct", "type": "text"}, {"key": "gl_descr", "type": "text"}, {"key": "amount", "type": "numeric"}], "unique_constraints": {"type": "key", "fields": ["{pay_date}", "{adp_comp}"]}}'); SET search_path = evt, pg_catalog; From e03a5cdbacf6041b47964e96454ddd3897c30a34 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 10:29:33 -0400 Subject: [PATCH 15/27] backup --- ubm_data.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index 0041faa..ca33e22 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -28,10 +28,10 @@ SET search_path = tps, pg_catalog; -- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: - -- -INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCL", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{PostDate}", "{Schedule#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", "descr": "ADP Detail Report", "schema": [{"key": "batch", "type": "text"}, {"key": "week", "type": "text"}, {"key": "period_end", "type": "text"}, {"key": "pay_date", "type": "text"}, {"key": "adp_comp", "type": "text"}, {"key": "hours_reg", "type": "numeric"}, {"key": "hours_ot", "type": "numeric"}, {"key": "adp_dep_home", "type": "text"}, {"key": "adp_dep_worked", "type": "text"}, {"key": "adp_dep", "type": "text"}, {"key": "gl_dep", "type": "text"}, {"key": "checkn", "type": "text"}, {"key": "employee", "type": "text"}, {"key": "title", "type": "text"}, {"key": "prim_offset", "type": "text"}, {"key": "cms_tb", "type": "text"}, {"key": "cms_acct", "type": "text"}, {"key": "gl_descr", "type": "text"}, {"key": "amount", "type": "numeric"}], "unique_constraints": {"type": "key", "fields": ["{pay_date}", "{adp_comp}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); SET search_path = evt, pg_catalog; @@ -49,7 +49,7 @@ SET search_path = tps, pg_catalog; -- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: - -- -SELECT pg_catalog.setval('trans_id_seq', 1, true); +SELECT pg_catalog.setval('trans_id_seq', 1543110, true); -- From 774540a83e57399caf32bb33983d7ec0178475e9 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 10:30:40 -0400 Subject: [PATCH 16/27] remove assn & coll --- ubm_data.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index ca33e22..c0a7c37 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -28,10 +28,10 @@ SET search_path = tps, pg_catalog; -- Data for Name: srce; Type: TABLE DATA; Schema: tps; Owner: - -- +INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCL", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{PostDate}", "{Schedule#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", "descr": "ADP Detail Report", "schema": [{"key": "batch", "type": "text"}, {"key": "week", "type": "text"}, {"key": "period_end", "type": "text"}, {"key": "pay_date", "type": "text"}, {"key": "adp_comp", "type": "text"}, {"key": "hours_reg", "type": "numeric"}, {"key": "hours_ot", "type": "numeric"}, {"key": "adp_dep_home", "type": "text"}, {"key": "adp_dep_worked", "type": "text"}, {"key": "adp_dep", "type": "text"}, {"key": "gl_dep", "type": "text"}, {"key": "checkn", "type": "text"}, {"key": "employee", "type": "text"}, {"key": "title", "type": "text"}, {"key": "prim_offset", "type": "text"}, {"key": "cms_tb", "type": "text"}, {"key": "cms_acct", "type": "text"}, {"key": "gl_descr", "type": "text"}, {"key": "amount", "type": "numeric"}], "unique_constraints": {"type": "key", "fields": ["{pay_date}", "{adp_comp}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); SET search_path = evt, pg_catalog; From 2190f55acb56d0c44aa591d2db3ce70fe626cdde Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 11:20:52 -0400 Subject: [PATCH 17/27] add running collateral & loan balances --- col_balance.pgsql | 17 +++++++++++++++++ loan_bal.pgsql | 12 ++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 col_balance.pgsql create mode 100644 loan_bal.pgsql diff --git a/col_balance.pgsql b/col_balance.pgsql new file mode 100644 index 0000000..ee2ada9 --- /dev/null +++ b/col_balance.pgsql @@ -0,0 +1,17 @@ + +SELECT + id, + rec->>'id', + r.*, + CASE "Schedule#" + WHEN '02IN Raw Material' THEN 13097563.42 + WHEN '03IN Finished Goods' THEN 35790696.52 + ELSE 0 + END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (ORDER BY "PostDate" ASC, rec->>'id' ASC) +FROM + tps.trans + LEFT JOIN LATERAL jsonb_populate_record(null::tps.pncl, rec) r ON TRUE +WHERE + rec @> '{"Schedule#":"01AR"}' +ORDER BY + r."PostDate" asc \ No newline at end of file diff --git a/loan_bal.pgsql b/loan_bal.pgsql new file mode 100644 index 0000000..240c296 --- /dev/null +++ b/loan_bal.pgsql @@ -0,0 +1,12 @@ + +SELECT + r.*, + SUM(r."Advances"+r."Adjustments"-r."Payments") OVER (ORDER BY r."Post Date" asc ,r."Reference #" asc) +FROM + tps.trans + LEFT JOIN LATERAL jsonb_populate_record(null::tps.pnco, rec) r ON TRUE +WHERE + rec @> '{"Loan#":"606780191"}' +ORDER BY + r."Post Date" asc + ,r."Reference #" asc From f0838b28c08b1b09802cf053f6a74e9adc960d94 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 11:22:03 -0400 Subject: [PATCH 18/27] retain the id number in the json rec, and order by id asc before inserting to trans. if mult records per key, this helps sort --- srce.pgsql | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index d8edc0a..dfa1ae8 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -18,7 +18,7 @@ BEGIN --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = 'PNCC' + srce = 'PNCL' GROUP BY srce; @@ -35,7 +35,7 @@ BEGIN ----------------------------------------------------do the insert------------------------------------------------------------------------------------------- --the column list needs to be dynamic forcing this whole line to be dynamic - _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); + _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\llcol.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); --RAISE NOTICE '%', _t; @@ -55,21 +55,21 @@ WITH pending_list AS ( (ae.e::text[])[1], --the key name (row_to_json(i)::jsonb) #> ae.e::text[] --get the target value from the key from the csv row that has been converted to json ) json_key, - row_to_json(i)::JSONB - 'id' rec, + row_to_json(i)::JSONB rec, srce, --ae.rn, id FROM csv_i i INNER JOIN tps.srce s ON - s.srce = 'PNCC' + s.srce = 'PNCL' LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE GROUP BY i.*, srce, id ORDER BY - id + id ASC ) ------results of an insert operation-------------- , inserted AS ( @@ -85,6 +85,8 @@ WITH pending_list AS ( AND t.rec @> pl.json_key WHERE t IS NULL + ORDER BY + id ASC ----this conflict is only if an exact duplicate rec json happens, which will be rejected ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified RETURNING * From 2a2652e397c8be819f2ea02706e8c2e85b869414 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 11:31:54 -0400 Subject: [PATCH 19/27] qualify the order by id --- srce.pgsql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index dfa1ae8..3ea9d21 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -18,7 +18,7 @@ BEGIN --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = 'PNCL' + srce = 'PNCC' GROUP BY srce; @@ -35,7 +35,7 @@ BEGIN ----------------------------------------------------do the insert------------------------------------------------------------------------------------------- --the column list needs to be dynamic forcing this whole line to be dynamic - _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\llcol.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); + _t := format('COPY csv_i (%s) FROM ''C:\Users\ptrowbridge\downloads\transsearchcsv.csv'' WITH (HEADER TRUE,DELIMITER '','', FORMAT CSV, ENCODING ''SQL_ASCII'',QUOTE ''"'');',_c); --RAISE NOTICE '%', _t; @@ -62,7 +62,7 @@ WITH pending_list AS ( FROM csv_i i INNER JOIN tps.srce s ON - s.srce = 'PNCL' + s.srce = 'PNCC' LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(defn->'unique_constraint'->'fields') WITH ORDINALITY ae(e, rn) ON TRUE GROUP BY i.*, @@ -86,7 +86,7 @@ WITH pending_list AS ( WHERE t IS NULL ORDER BY - id ASC + pl.id ASC ----this conflict is only if an exact duplicate rec json happens, which will be rejected ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified RETURNING * From e68b3e034936fded95615840867ee540bde5c63d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 13:10:25 -0400 Subject: [PATCH 20/27] revise import to work with a set of unique keys in determining which rows to insert. prior used many of the same key, creating large duplication on join to tps --- srce.pgsql | 69 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/srce.pgsql b/srce.pgsql index 3ea9d21..0e8ac0a 100644 --- a/srce.pgsql +++ b/srce.pgsql @@ -1,3 +1,16 @@ +\timing + +/*-------------------------------------------------------- +0. load target import to temp table +1. create pending list +2. get unqiue pending keys +3. see which keys not already in tps.trans +4. insert pending records associated with keys that are not already in trans +5. get list of recors not inserted +6. summarize records not inserted +*/--------------------------------------------------------- + + DO $$ DECLARE _t text; @@ -45,10 +58,12 @@ BEGIN END $$; ---******************************************* ---this needs to aggregate on id sequence ---******************************************* -WITH pending_list AS ( + +WITH + +-------------for each imported row in the COPY table, genereate the json rec, and a column for the json key specified in the srce.defn----------- + +pending_list AS ( SELECT ---creates a key value pair and then aggregates rows of key value pairs jsonb_object_agg( @@ -71,7 +86,36 @@ WITH pending_list AS ( ORDER BY id ASC ) -------results of an insert operation-------------- + +-----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------ + +, pending_keys AS ( + SELECT DISTINCT + json_key + FROM + pending_list +) + +-----------return unique keys that are not already in tps.trans----------------------------------------------------------------------------------- + +, unmatched_keys AS ( +SELECT + json_key +FROM + pending_keys + +EXCEPT + +SELECT DISTINCT + k.json_key +FROM + pending_keys k + INNER JOIN tps.trans t ON + t.rec @> k.json_key +) + +-----------insert pending rows that have key with no trans match----------------------------------------------------------------------------------- + , inserted AS ( INSERT INTO tps.trans (srce, rec) @@ -80,18 +124,17 @@ WITH pending_list AS ( ,pl.rec FROM pending_list pl - LEFT JOIN tps.trans t ON - t.srce = pl.srce - AND t.rec @> pl.json_key - WHERE - t IS NULL + INNER JOIN unmatched_keys u ON + u.json_key = pl.json_key ORDER BY pl.id ASC ----this conflict is only if an exact duplicate rec json happens, which will be rejected ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified RETURNING * ) ----------raw list of records not inserted---------- + +-----------list of records not inserted-------------------------------------------------------------------------------------------------------------- + , not_inserted AS ( SELECT srce @@ -107,7 +150,9 @@ WITH pending_list AS ( FROM inserted ) ---------summarize records not inserted------------------ + +--------summarize records not inserted-------------------+------------------------------------------------------------------------------------------------ + SELECT t.srce ,(ae.e::text[])[1] unq_constr From f203c7010a8a00a94b4a2e9d3c2c300608ede8db Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 13:10:36 -0400 Subject: [PATCH 21/27] add calc columns to collateral --- col_balance.pgsql | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/col_balance.pgsql b/col_balance.pgsql index ee2ada9..09e726a 100644 --- a/col_balance.pgsql +++ b/col_balance.pgsql @@ -1,17 +1,21 @@ SELECT - id, - rec->>'id', - r.*, - CASE "Schedule#" + id + ,rec->>'id' + ,r.* + ,CASE "Schedule#" WHEN '02IN Raw Material' THEN 13097563.42 WHEN '03IN Finished Goods' THEN 35790696.52 ELSE 0 - END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (ORDER BY "PostDate" ASC, rec->>'id' ASC) + END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (ORDER BY "PostDate" ASC, rec->>'id' ASC) running_bal + ,(LEAST("CollateralBalance" - "Ineligible Amount","MaxEligible")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral + ,(("CollateralBalance" - "Ineligible Amount")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral_nl FROM tps.trans LEFT JOIN LATERAL jsonb_populate_record(null::tps.pncl, rec) r ON TRUE WHERE - rec @> '{"Schedule#":"01AR"}' + srce = 'PNCL' + AND "PostDate" >= '2017-10-12' + --rec @> '{"Schedule#":"03IN Finished Goods"}' ORDER BY r."PostDate" asc \ No newline at end of file From 5369a39f34b95c4c9ac3d0e30b057981f3114081 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 14:13:57 -0400 Subject: [PATCH 22/27] add mapping tables & data --- ubm_data.sql | 117 ++++++++++++++++++++++++++++++++++++++++++++++++- ubm_schema.sql | 107 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 2 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index c0a7c37..4732b92 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -31,7 +31,120 @@ SET search_path = tps, pg_catalog; INSERT INTO srce (srce, defn) VALUES ('PNCC', '{"name": "PNCC", "type": "csv", "descr": "PNC Cash Accounts", "schema": [{"key": "AsOfDate", "type": "date"}, {"key": "BankId", "type": "text"}, {"key": "AccountNumber", "type": "text"}, {"key": "AccountName", "type": "text"}, {"key": "BaiControl", "type": "text"}, {"key": "Currency", "type": "text"}, {"key": "Transaction", "type": "text"}, {"key": "Reference", "type": "text"}, {"key": "Amount", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "AdditionalRemittance", "type": "text"}], "unique_constraint": {"type": "range", "fields": ["{AsOfDate}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCO', '{"name": "PNCO", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Loan#", "type": "text"}, {"key": "Post Date", "type": "date"}, {"key": "Effective Date", "type": "date"}, {"key": "Reference #", "type": "text"}, {"key": "Description", "type": "text"}, {"key": "Advances", "type": "numeric"}, {"key": "Adjustments", "type": "numeric"}, {"key": "Payments", "type": "numeric"}, {"key": "Loan Balance", "type": "numeric"}], "unique_constraint": {"type": "range", "fields": ["{Post Date}", "{Effective Date}", "{Loan#}"]}}'); INSERT INTO srce (srce, defn) VALUES ('PNCL', '{"name": "PNCL", "type": "csv", "descr": "PNC Loan Ledger", "schema": [{"key": "Schedule#", "type": "text"}, {"key": "PostDate", "type": "date"}, {"key": "Assn#", "type": "text"}, {"key": "Coll#", "type": "text"}, {"key": "AdvanceRate", "type": "numeric"}, {"key": "Sales", "type": "numeric"}, {"key": "Credits & Adjustments", "type": "numeric"}, {"key": "Gross Collections", "type": "numeric"}, {"key": "CollateralBalance", "type": "numeric"}, {"key": "MaxEligible", "type": "numeric"}, {"key": "Ineligible Amount", "type": "numeric"}, {"key": "Reserve Amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{PostDate}", "{Schedule#}"]}}'); -INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", "descr": "ADP Detail Report", "schema": [{"key": "batch", "type": "text"}, {"key": "week", "type": "text"}, {"key": "period_end", "type": "text"}, {"key": "pay_date", "type": "text"}, {"key": "adp_comp", "type": "text"}, {"key": "hours_reg", "type": "numeric"}, {"key": "hours_ot", "type": "numeric"}, {"key": "adp_dep_home", "type": "text"}, {"key": "adp_dep_worked", "type": "text"}, {"key": "adp_dep", "type": "text"}, {"key": "gl_dep", "type": "text"}, {"key": "checkn", "type": "text"}, {"key": "employee", "type": "text"}, {"key": "title", "type": "text"}, {"key": "prim_offset", "type": "text"}, {"key": "cms_tb", "type": "text"}, {"key": "cms_acct", "type": "text"}, {"key": "gl_descr", "type": "text"}, {"key": "amount", "type": "numeric"}], "unique_constraints": {"type": "key", "fields": ["{pay_date}", "{adp_comp}"]}}'); +INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", "descr": "ADP Detail Report", "schema": [{"key": "batch", "type": "text"}, {"key": "week", "type": "text"}, {"key": "period_end", "type": "text"}, {"key": "pay_date", "type": "text"}, {"key": "adp_comp", "type": "text"}, {"key": "hours_reg", "type": "numeric"}, {"key": "hours_ot", "type": "numeric"}, {"key": "adp_dep_home", "type": "text"}, {"key": "adp_dep_worked", "type": "text"}, {"key": "adp_dep", "type": "text"}, {"key": "gl_dep", "type": "text"}, {"key": "checkn", "type": "text"}, {"key": "employee", "type": "text"}, {"key": "title", "type": "text"}, {"key": "prim_offset", "type": "text"}, {"key": "cms_tb", "type": "text"}, {"key": "cms_acct", "type": "text"}, {"key": "gl_descr", "type": "text"}, {"key": "amount", "type": "numeric"}], "unique_constraint": {"type": "key", "fields": ["{pay_date}", "{adp_comp}"]}}'); + + +-- +-- Data for Name: map_rm; Type: TABLE DATA; Schema: tps; Owner: - +-- + +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'ACH Debits', '{"where":[{"Transaction":"ACH Debits"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":"Comp Name:(.+?)(?=$| Comp|\\w+?:)","field":"compn","key":"{Description}"},{"regex":"Cust ID:.*?(B3X|UDV|U7E|U7C|U7H|U7J).*?(?=$|\\w+?:)","field":"adp_comp","key":"{Description}"},{"regex":"Desc:(.+?) Comp","field":"desc","key":"{Description}"},{"regex":"Discr:(.+?)(?=$| SEC:|\\w+?:)","field":"discr","key":"{Description}"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Trans Type', '{"where":[{}],"type":"extract","defn":[{"regex":"(.*)","field":"acctn","key":"{AccountName}"},{"regex":"(.*)","field":"trans","key":"{Transaction}"},{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"}]}', 1); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Wires Out', '{"where":[{"Transaction":"Intl Money Transfer Debits"},{"Transaction":"Money Transfer DB - Wire"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":"BENEFICIARY:(.+?) AC/","field":"bene","key":"{Description}"},{"regex":"AC/(\\w*) .*AC/(\\w*) ","field":"accts","key":"{Description}"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Currency', '{"where":[{"Transaction":"Miscellaneous Credits"},{"Transaction":"Miscellaneous Debits"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*","field":"curr1","key":"{Description}"},{"regex":".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*","field":"curr2","key":"{Description}"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Check Number', '{"where":[{"Transaction":"Checks Paid"}],"type":"extract","defn":[{"regex":"[^0-9]*([0-9]*)\\s|$","field":"checkn","key":"{Description}"}]}', 2); + + +-- +-- Data for Name: map_rv; Type: TABLE DATA; Schema: tps; Owner: - +-- + +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CANADA TAX", "acctn": "The HC Operating Company OPERA", "trans": "Detail Debit Adjustments"}', '{"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH DEBIT SETTLEMENT", "acctn": "The HC Operating Company OPERA", "trans": "ACH Debits"}', '{"sign": "-1", "ledger": "AP - ACH", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "RET DEP ITEM RTM", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"}', '{"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit RTM", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "RET DEP ITEM STOP", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"}', '{"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit STOP", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CREDIT ADJUSTMENT", "acctn": "The HC Operating Company FBO P", "trans": "Detail Credit Adjustments"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "REFER TO MAKER OF CK RETURN CK", "acctn": "The HC Operating Company OPERA", "trans": "Detail Credit Adjustments"}', '{"sign": "1", "ledger": "Manual", "reason": "Returned Check", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "DEBIT ADJUSTMENT", "acctn": "The HC Operating Company PAYR", "trans": "Detail Debit Adjustments"}', '{"sign": "-1", "ledger": "Manual", "reason": "Payroll Adjustment", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "DEPOSIT", "acctn": "The HC Operating Company FBO P", "trans": "Detail Deposits"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "INTL WIRE OUT", "acctn": "The HC Operating Company OPERA", "trans": "Intl Money Transfer Debits"}', '{"sign": "-1", "ledger": "AP - Wire", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "INTL WIRES OUT", "acctn": "The HC Operating Company OPERA", "trans": "Intl Money Transfer Debits"}', '{"sign": "-1", "ledger": "AP - Wire", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "WHLS LBX DEP", "acctn": "The HC Operating Company FBO P", "trans": "Lockbox Deposits"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "WHLS LBX DEP932855", "acctn": "The HC Operating Company FBO P", "trans": "Lockbox Deposits"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ADVANCE", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "Revolver Advance", "trantype": "Revolver Borrow"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "DEPOSIT:", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Credits"}', '{"sign": "1", "ledger": "Manual", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "MISC CREDIT", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "Misc Credit", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PAYMENT", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "Revolver Payment", "trantype": "Revolver Borrow"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PAYMENT", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "Revolver Payment", "trantype": "Revolver Payment"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "INTTL WIRES IN", "acctn": "The HC Operating Company FBO P", "trans": "Intl Money Transfer Credits"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PNC BANK- NJ LOAN PROCEEDS", "acctn": "The HC Operating Company FBO P", "trans": "Money Transfer CR-Other"}', '{"sign": "1", "ledger": "Manual", "reason": "Revolver Advance", "trantype": "Revolver Borrow"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PNC BANK-PGH LOAN PROCEEDS", "acctn": "The HC Operating Company OPERA", "trans": "Money Transfer CR-Other"}', '{"sign": "1", "ledger": "Manual", "reason": "Revolver Advance", "trantype": "Revolver Borrow"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FED WIRE IN", "acctn": "The HC Operating Company FBO P", "trans": "Money Transfer CR-Wire"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FED WIRE IN", "acctn": "The HC Operating Company OPERA", "trans": "Money Transfer CR-Wire"}', '{"sign": "1", "ledger": "Manual", "reason": "Returned Wires", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "BOOK TRANSFER DEBIT", "acctn": "The HC Operating Company OPERA", "trans": "Money Transfer DB - Other"}', '{"sign": "-1", "ledger": "Manual", "reason": "Returned Item", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FED WIRE OUT", "acctn": "The HC Operating Company FREIG", "trans": "Money Transfer DB - Wire"}', '{"sign": "-1", "ledger": "Manual", "reason": "Freight Wires", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FED WIRE OUT", "acctn": "The HC Operating Company OPERA", "trans": "Money Transfer DB - Wire"}', '{"sign": "-1", "ledger": "AP - Wire", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FED WIRE OUT", "acctn": "The HC Operating Company PAYR", "trans": "Money Transfer DB - Wire"}', '{"sign": "-1", "ledger": "Manual", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FUNDS TRANSFER FROM ACCT", "acctn": "The HC Operating Company FREIG", "trans": "ZBA Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "ZBA Funding", "trantype": "Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FUNDS TRANSFER FROM ACCT", "acctn": "The HC Operating Company OPERA", "trans": "ZBA Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "ZBA Funding", "trantype": "Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FUNDS TRANSFER FROM ACCT", "acctn": "The HC Operating Company PAYR", "trans": "ZBA Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "ZBA Funding", "trantype": "Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FUNDS TRANSFER TO ACCT", "acctn": "The HC Operating Company OPERA", "trans": "ZBA Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "ZBA Funding", "trantype": "Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "FUNDS TRANSFER TO ACCT", "acctn": "The HC Operating Company PAYR", "trans": "ZBA Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "ZBA Funding", "trantype": "Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Currency', '{"ini": "DEPOSIT:", "curr1": ["CREDIT", "USD"], "curr2": ["DEBIT", "CAD"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - Can to US Settlement", "trantype": "Interco Collection"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "INTL WIRES OUT", "accts": ["8026322346", "010241000355"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - permanent AP funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "INTL WIRES OUT", "accts": ["8026322346", "010244001145"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - round-trip settlement outbound"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "INTL WIRES OUT", "accts": ["8026322346", "010244001152"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - permanent AP funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "FED WIRE OUT", "bene": "ADP LLC", "accts": ["8026322346", "00153170"]}', '{"party": "ADP", "ledger": "Manual", "reason": "Payroll"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "FED WIRE OUT", "bene": "ADP LLC", "accts": ["8026322346", "00412283"]}', '{"party": "ADP", "ledger": "Manual", "reason": "Payroll"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "FED WIRE OUT", "bene": "ADP PAYROLL TAX DEPOSIT CUSTODIAN", "accts": ["8026322338", "00153170"]}', '{"party": "ADP", "ledger": "Manual", "reason": "Payroll"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Wires Out', '{"ini": "FED WIRE OUT", "bene": "ADP TAX SVCS INC. REV. WIRE IMPOUND", "accts": ["8026322338", "00416217"]}', '{"party": "ADP", "ledger": "Manual", "reason": "Payroll Direct Deposit"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "compn": " NEVADA TAX"}', '{"party": "State of Nevada", "reason": "Sales & Use Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " ADP - TAX", "compn": " ADP TX/FINCL SVC"}', '{"party": "ADP", "reason": "Payroll"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " ANTHEM", "compn": " ANTHEM"}', '{"party": "Anthem", "reason": "Healthcare Costs"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " TAX/401K", "compn": " ADP TAX/401K"}', '{"party": "ADP", "reason": "Payroll Taxes"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CORPORATE ACCOUNT ANALYSIS CHARGE", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Fees"}', '{"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Fees"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PNC MERCHANT FINCL ADJ", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Fees"}', '{"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Fees"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " ASO CLAIMS", "compn": " UNUM STD"}', '{"party": "Unum", "reason": "Short Term Disability"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "PNC BANK- NJ LOAN PMTS", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "Revolver Payment", "trantype": "Revolver Payment"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "WITHDRAWAL:", "acctn": "The HC Operating Company FBO P", "trans": "Miscellaneous Debits"}', '{"sign": "-1", "ledger": "Manual", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "BOOK TRANSFER CREDIT", "acctn": "The HC Operating Company FBO P", "trans": "Money Transfer CR-Other"}', '{"sign": "1", "ledger": "Manual", "reason": "Returned Item", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "BOOK TRANSFER CREDIT", "acctn": "The HC Operating Company OPERA", "trans": "Money Transfer CR-Other"}', '{"sign": "1", "ledger": "Manual", "reason": "Returned Item", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "BOOK TRANSFER CREDIT GHFTDD DDA CREDIT", "acctn": "The HC Operating Company FBO P", "trans": "Money Transfer CR-Other"}', '{"sign": "1", "ledger": "Manual", "reason": "Returned Item", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " AXP DISCNT", "compn": " AMERICAN EXPRESS"}', '{"party": "American Express", "reason": "Credit Card Fees"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " BNKCRD DEP", "compn": " WORLDPAY"}', '{"party": "Worldpay", "reason": "Credit Card Fees"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " C01", "compn": " FLA DEPT REVENUE"}', '{"party": "Florida", "reason": "Sales & Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " DBI ADMIN", "compn": " DISCOVERY BENEFI"}', '{"party": "Discovery Benefits", "reason": "Benefits Administration"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " DEBITS", "compn": " OHIO BWC", "discr": " OHIO BWC PREMIUM"}', '{"party": "Ohio BWC", "reason": "Workers Comp Premium"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " INSURANCE", "compn": " UNUMGROUP955"}', '{"party": "Unum", "reason": "Insurance"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " INTEREST", "compn": " DERIVATIVES"}', '{"party": "PNC", "reason": "Derivatives Interest"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " LEASE RENT", "compn": " RAYMOND LEASING"}', '{"party": "Raymond Leasing", "reason": "Lease Expense"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " MTHLY CHGS", "compn": " WORLDPAY"}', '{"party": "Worlpay", "reason": "Credit Card Fees"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " OH CAT RTN", "compn": " 8012OHIO-TAXOCAT"}', '{"party": "State of Ohio", "reason": "CAT"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " OH REG VL", "compn": " 8008OHIO-TAXORVL"}', '{"party": "State of Ohio", "reason": "Vendor License"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " OH SALESTX", "compn": " 8013OHIO-TAXOSUT"}', '{"party": "State of Ohio", "reason": "Sales & Use Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " OH TAXASMT", "compn": " 8001OHIO-TAXSUAP"}', '{"party": "State of Ohio", "reason": "Sales & Use Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " OHATTYGN", "compn": " OH ATTORNYGENRAL"}', '{"party": "Ohio Attorney General", "reason": "Sales & Use Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " PLIC-PERIS", "compn": " PRINCIPAL LIFE P"}', '{"party": "Principal", "reason": "401k"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " PROGRAM", "compn": " USDEPTHHSCMS"}', '{"party": "US Dept of HHS", "reason": "Health tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " RIPAYMENT", "compn": " USDEPTHHSCMS"}', '{"party": "US Dept of HHS", "reason": "Health tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " TAX PMT", "compn": " WA ST DEPT REV"}', '{"party": "Washington State", "reason": "Sales & Use Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " UCC FILING", "compn": " RAYMOND LEASING"}', '{"party": "Raymond Leasing", "reason": "Lease Expense"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " USATAXPYMT", "compn": " IRS"}', '{"party": "IRS", "reason": "Federal Tax"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "19UDV", "acctn": "The HC Operating Company PAYR", "trans": "ACH Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "Payroll Credits", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH CREDIT RECEIVED", "acctn": "The HC Operating Company FBO P", "trans": "ACH Credits"}', '{"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH CREDIT RECEIVED", "acctn": "The HC Operating Company PAYR", "trans": "ACH Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "Payroll Credits", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH CREDIT RETURN", "acctn": "The HC Operating Company OPERA", "trans": "ACH Credits"}', '{"sign": "1", "ledger": "Manual", "reason": "AP ACH Returned", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH DEBIT RECEIVED", "acctn": "The HC Operating Company FBO P", "trans": "ACH Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "Auto ACH Out", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH DEBIT RECEIVED", "acctn": "The HC Operating Company OPERA", "trans": "ACH Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "Auto ACH Out", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "ACH DEBIT RECEIVED", "acctn": "The HC Operating Company PAYR", "trans": "ACH Debits"}', '{"sign": "-1", "ledger": "Manual", "reason": "Auto ACH Out", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CASHED CHECK", "acctn": "The HC Operating Company OPERA", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "AP - Check Run", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CASHED CHECK", "acctn": "The HC Operating Company PAYR", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "Manual", "reason": "Payroll Checks", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CHECK", "acctn": "The HC Operating Company FREIG", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "Manual", "reason": "Freight Checks", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CHECK", "acctn": "The HC Operating Company OPERA", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "AP - Check Run", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "CHECK", "acctn": "The HC Operating Company PAYR", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "Manual", "reason": "Payroll Checks", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "SUBSTITUTE CHK", "acctn": "The HC Operating Company FREIG", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "Manual", "reason": "Freight Checks", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "SUBSTITUTE CHK", "acctn": "The HC Operating Company OPERA", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "AP - Check Run", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "RET DEP ITEM NSF UN", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"}', '{"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit NSF", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Currency', '{"ini": "WITHDRAWAL:", "curr1": ["DEBIT", "USD"], "curr2": ["CREDIT", "CAD"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - US to CAN Settlement", "trantype": "Interco Funding"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " EEPAY/GARN", "compn": " ADP EEPAY/GARNWC"}', '{"party": "ADP", "reason": "Payroll Direct Deposit & Garnishments"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Currency', '{"ini": "DEPOSIT:", "curr1": ["CREDIT", "USD"], "curr2": ["DEBIT", "USD"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - round-trip settlement return", "trantype": "Interco Collection"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Currency', '{"ini": "DEPOSIT:", "curr1": ["DEBIT", "USD"], "curr2": ["CREDIT", "CAD"]}', '{"party": "The HC Canada Operating Company, Ltd.", "ledger": "Manual", "reason": "IC - Can to US Settlement", "trantype": "Interco Collection"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " ADP TAX", "compn": " ADP TAX"}', '{"party": "ADP", "reason": "Payroll Taxes"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "DEPOSIT:", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Credits"}', '{"sign": "1", "ledger": "Manual", "trantype": "Collections"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'Trans Type', '{"ini": "SUBSTITUTE CHK", "acctn": "The HC Operating Company PAYR", "trans": "Checks Paid"}', '{"sign": "-1", "ledger": "Manual", "reason": "Payroll Checks", "trantype": "Disbursement"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " WAGE GARN", "compn": " ADP WAGE GARN"}', '{"party": "ADP", "reason": "Payroll Garnishments"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " WAGE PAY", "compn": " ADP WAGE PAY"}', '{"party": "ADP", "reason": "Payroll Direct Deposit"}'); +INSERT INTO map_rv (srce, target, retval, map) VALUES ('PNCC', 'ACH Debits', '{"ini": "ACH DEBIT RECEIVED", "desc": " DBI COBRA", "compn": " DBI COBRA"}', '{"party": "ADP", "reason": "Payroll Direct Deposit"}'); SET search_path = evt, pg_catalog; @@ -49,7 +162,7 @@ SET search_path = tps, pg_catalog; -- Name: trans_id_seq; Type: SEQUENCE SET; Schema: tps; Owner: - -- -SELECT pg_catalog.setval('trans_id_seq', 1543110, true); +SELECT pg_catalog.setval('trans_id_seq', 1543757, true); -- diff --git a/ubm_schema.sql b/ubm_schema.sql index 49c36e3..d12ce95 100644 --- a/ubm_schema.sql +++ b/ubm_schema.sql @@ -56,8 +56,59 @@ CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; +-- +-- Name: plprofiler; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plprofiler WITH SCHEMA public; + + +-- +-- Name: EXTENSION plprofiler; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plprofiler IS 'server-side support for profiling PL/pgSQL functions'; + + SET search_path = tps, pg_catalog; +-- +-- Name: pncl; Type: TYPE; Schema: tps; Owner: - +-- + +CREATE TYPE pncl AS ( + "Schedule#" text, + "PostDate" date, + "Assn#" text, + "Coll#" text, + "AdvanceRate" numeric, + "Sales" numeric, + "Credits & Adjustments" numeric, + "Gross Collections" numeric, + "CollateralBalance" numeric, + "MaxEligible" numeric, + "Ineligible Amount" numeric, + "Reserve Amount" numeric +); + + +-- +-- Name: pnco; Type: TYPE; Schema: tps; Owner: - +-- + +CREATE TYPE pnco AS ( + "Loan#" text, + "Post Date" date, + "Effective Date" date, + "Reference #" text, + "Description" text, + "Advances" numeric, + "Adjustments" numeric, + "Payments" numeric, + "Loan Balance" numeric +); + + -- -- Name: srce_defn_schema; Type: TYPE; Schema: tps; Owner: - -- @@ -100,6 +151,30 @@ ALTER TABLE log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( SET search_path = tps, pg_catalog; +-- +-- Name: map_rm; Type: TABLE; Schema: tps; Owner: - +-- + +CREATE TABLE map_rm ( + srce text NOT NULL, + target text NOT NULL, + regex text, + seq integer NOT NULL +); + + +-- +-- Name: map_rv; Type: TABLE; Schema: tps; Owner: - +-- + +CREATE TABLE map_rv ( + srce text NOT NULL, + target text NOT NULL, + retval jsonb NOT NULL, + map jsonb +); + + -- -- Name: srce; Type: TABLE; Schema: tps; Owner: - -- @@ -148,6 +223,22 @@ ALTER TABLE ONLY log SET search_path = tps, pg_catalog; +-- +-- Name: map_rm map_rm_pk; Type: CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY map_rm + ADD CONSTRAINT map_rm_pk PRIMARY KEY (srce, target, seq); + + +-- +-- Name: map_rv map_rv_pk; Type: CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY map_rv + ADD CONSTRAINT map_rv_pk PRIMARY KEY (srce, target, retval); + + -- -- Name: srce srce_pkey; Type: CONSTRAINT; Schema: tps; Owner: - -- @@ -171,6 +262,22 @@ ALTER TABLE ONLY trans CREATE INDEX trans_rec ON trans USING gin (rec); +-- +-- Name: map_rm map_rm_fk_srce; Type: FK CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY map_rm + ADD CONSTRAINT map_rm_fk_srce FOREIGN KEY (srce) REFERENCES srce(srce); + + +-- +-- Name: map_rv map_rv_fk_srce; Type: FK CONSTRAINT; Schema: tps; Owner: - +-- + +ALTER TABLE ONLY map_rv + ADD CONSTRAINT map_rv_fk_srce FOREIGN KEY (srce) REFERENCES srce(srce); + + -- -- Name: trans trans_srce_fkey; Type: FK CONSTRAINT; Schema: tps; Owner: - -- From 351b83a20648f28d7f55e05bb6a88291710c255b Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 17:37:46 -0400 Subject: [PATCH 23/27] alter regex defs to have a retention flag indicating if the return value shoudl be mapped or thrown away --- ubm_data.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ubm_data.sql b/ubm_data.sql index 4732b92..96a1f74 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -38,11 +38,11 @@ INSERT INTO srce (srce, defn) VALUES ('ADPRP', '{"name": "ADPRP", "type": "csv", -- Data for Name: map_rm; Type: TABLE DATA; Schema: tps; Owner: - -- -INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'ACH Debits', '{"where":[{"Transaction":"ACH Debits"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":"Comp Name:(.+?)(?=$| Comp|\\w+?:)","field":"compn","key":"{Description}"},{"regex":"Cust ID:.*?(B3X|UDV|U7E|U7C|U7H|U7J).*?(?=$|\\w+?:)","field":"adp_comp","key":"{Description}"},{"regex":"Desc:(.+?) Comp","field":"desc","key":"{Description}"},{"regex":"Discr:(.+?)(?=$| SEC:|\\w+?:)","field":"discr","key":"{Description}"}]}', 2); -INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Trans Type', '{"where":[{}],"type":"extract","defn":[{"regex":"(.*)","field":"acctn","key":"{AccountName}"},{"regex":"(.*)","field":"trans","key":"{Transaction}"},{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"}]}', 1); -INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Wires Out', '{"where":[{"Transaction":"Intl Money Transfer Debits"},{"Transaction":"Money Transfer DB - Wire"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":"BENEFICIARY:(.+?) AC/","field":"bene","key":"{Description}"},{"regex":"AC/(\\w*) .*AC/(\\w*) ","field":"accts","key":"{Description}"}]}', 2); -INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Currency', '{"where":[{"Transaction":"Miscellaneous Credits"},{"Transaction":"Miscellaneous Debits"}],"type":"extract","defn":[{"regex":"([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)","field":"ini","key":"{Description}"},{"regex":".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*","field":"curr1","key":"{Description}"},{"regex":".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*","field":"curr2","key":"{Description}"}]}', 2); -INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Check Number', '{"where":[{"Transaction":"Checks Paid"}],"type":"extract","defn":[{"regex":"[^0-9]*([0-9]*)\\s|$","field":"checkn","key":"{Description}"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'ACH Debits', '{"defn": [{"key": "{Description}", "field": "ini", "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", "retain": "y"}, {"key": "{Description}", "field": "compn", "regex": "Comp Name:(.+?)(?=$| Comp|\\w+?:)", "retain": "y"}, {"key": "{Description}", "field": "adp_comp", "regex": "Cust ID:.*?(B3X|UDV|U7E|U7C|U7H|U7J).*?(?=$|\\w+?:)", "retain": "y"}, {"key": "{Description}", "field": "desc", "regex": "Desc:(.+?) Comp", "retain": "y"}, {"key": "{Description}", "field": "discr", "regex": "Discr:(.+?)(?=$| SEC:|\\w+?:)", "retain": "y"}], "where": [{"Transaction": "ACH Debits"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Trans Type', '{"defn": [{"key": "{AccountName}", "field": "acctn", "regex": "(.*)", "retain": "n"}, {"key": "{Transaction}", "field": "trans", "regex": "(.*)", "retain": "n"}, {"key": "{Description}", "field": "ini", "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", "retain": "y"}], "where": [{}]}', 1); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Wires Out', '{"defn": [{"key": "{Description}", "field": "ini", "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", "retain": "y"}, {"key": "{Description}", "field": "bene", "regex": "BENEFICIARY:(.+?) AC/", "retain": "y"}, {"key": "{Description}", "field": "accts", "regex": "AC/(\\w*) .*AC/(\\w*) ", "retain": "y"}], "where": [{"Transaction": "Intl Money Transfer Debits"}, {"Transaction": "Money Transfer DB - Wire"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Currency', '{"defn": [{"key": "{Description}", "field": "ini", "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", "retain": "y"}, {"key": "{Description}", "field": "curr1", "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", "retain": "y"}, {"key": "{Description}", "field": "curr2", "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", "retain": "y"}], "where": [{"Transaction": "Miscellaneous Credits"}, {"Transaction": "Miscellaneous Debits"}]}', 2); +INSERT INTO map_rm (srce, target, regex, seq) VALUES ('PNCC', 'Check Number', '{"defn": [{"key": "{Description}", "field": "checkn", "regex": "[^0-9]*([0-9]*)\\s|$", "retain": "y"}], "where": [{"Transaction": "Checks Paid"}]}', 2); -- From 2979fd3437286299ae3397639183cdcbf895e93c Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 17:38:58 -0400 Subject: [PATCH 24/27] add jsonb object aggregate functions, change regex to jsonb, remove seq from primary key of map_rm, add foreign key to map_rv pointing map_rm --- ubm_schema.sql | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/ubm_schema.sql b/ubm_schema.sql index d12ce95..fcda4b0 100644 --- a/ubm_schema.sql +++ b/ubm_schema.sql @@ -119,6 +119,32 @@ CREATE TYPE srce_defn_schema AS ( ); +-- +-- Name: jsonb_concat(jsonb, jsonb); Type: FUNCTION; Schema: tps; Owner: - +-- + +CREATE FUNCTION jsonb_concat(state jsonb, concat jsonb) RETURNS jsonb + LANGUAGE plpgsql + AS $$ +BEGIN + --RAISE notice 'state is %', state; + --RAISE notice 'concat is %', concat; + RETURN state || concat; +END; +$$; + + +-- +-- Name: jsonb_concat_obj(jsonb); Type: AGGREGATE; Schema: tps; Owner: - +-- + +CREATE AGGREGATE jsonb_concat_obj(jsonb) ( + SFUNC = jsonb_concat, + STYPE = jsonb, + INITCOND = '{}' +); + + SET search_path = evt, pg_catalog; SET default_tablespace = ''; @@ -158,7 +184,7 @@ SET search_path = tps, pg_catalog; CREATE TABLE map_rm ( srce text NOT NULL, target text NOT NULL, - regex text, + regex jsonb, seq integer NOT NULL ); @@ -228,7 +254,7 @@ SET search_path = tps, pg_catalog; -- ALTER TABLE ONLY map_rm - ADD CONSTRAINT map_rm_pk PRIMARY KEY (srce, target, seq); + ADD CONSTRAINT map_rm_pk PRIMARY KEY (srce, target); -- @@ -271,11 +297,11 @@ ALTER TABLE ONLY map_rm -- --- Name: map_rv map_rv_fk_srce; Type: FK CONSTRAINT; Schema: tps; Owner: - +-- Name: map_rv map_rv_fk_rm; Type: FK CONSTRAINT; Schema: tps; Owner: - -- ALTER TABLE ONLY map_rv - ADD CONSTRAINT map_rv_fk_srce FOREIGN KEY (srce) REFERENCES srce(srce); + ADD CONSTRAINT map_rv_fk_rm FOREIGN KEY (srce, target) REFERENCES map_rm(srce, target); -- From 75d511f6a40999ef0875edbdf89b79758c52a6f9 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 17:39:29 -0400 Subject: [PATCH 25/27] regex modification template --- map_rm.pgsql | 206 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 166 insertions(+), 40 deletions(-) diff --git a/map_rm.pgsql b/map_rm.pgsql index 99eaf3a..f116884 100644 --- a/map_rm.pgsql +++ b/map_rm.pgsql @@ -1,40 +1,166 @@ -SELECT -jsonb_pretty( -$$ -{ - "defn": [ - { - "key": "{Description}", - "field": "ini", - "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)" - }, - { - "key": "{Description}", - "field": "compn", - "regex": "Comp Name:(.+?)(?=$| Comp|\\w+?:)" - }, - { - "key": "{Description}", - "field": "adp_comp", - "regex": "Cust ID:.*?(B3X|UDV|U7E|U7C|U7H|U7J).*?(?=$|\\w+?:)" - }, - { - "key": "{Description}", - "field": "desc", - "regex": "Desc:(.+?) Comp" - }, - { - "key": "{Description}", - "field": "discr", - "regex": "Discr:(.+?)(?=$| SEC:|\\w+?:)" - } - ], - "type": "extract", - "where": [ - { - "Transaction": "ACH Debits" - } - ] -} -$$::jsonb -) \ No newline at end of file +INSERT INTO +tps.map_rm +SELECT * +FROM +(VALUES + ('PNCC', 'ACH Debits', + $j$ + { + "defn": [ + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "compn", + "regex": "Comp Name:(.+?)(?=$| Comp|\\w+?:)" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "adp_comp", + "regex": "Cust ID:.*?(B3X|UDV|U7E|U7C|U7H|U7J).*?(?=$|\\w+?:)" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "desc", + "regex": "Desc:(.+?) Comp" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "discr", + "regex": "Discr:(.+?)(?=$| SEC:|\\w+?:)" + ,"retain":"y" + } + ], + "where": [ + { + "Transaction": "ACH Debits" + } + ] + } + $j$::jsonb + , 2) + ,('PNCC', 'Trans Type', + $j$ + { + "defn": [ + { + "key": "{AccountName}", + "field": "acctn", + "regex": "(.*)" + ,"retain":"n" + }, + { + "key": "{Transaction}", + "field": "trans", + "regex": "(.*)" + ,"retain":"n" + }, + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)" + ,"retain":"y" + } + ], + "where": [ + { + } + ] + } + $j$::jsonb + , 1) + ,('PNCC', 'Wires Out', + $j$ + { + "defn": [ + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "bene", + "regex": "BENEFICIARY:(.+?) AC/" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "accts", + "regex": "AC/(\\w*) .*AC/(\\w*) " + ,"retain":"y" + } + ], + "where": [ + { + "Transaction": "Intl Money Transfer Debits" + }, + { + "Transaction": "Money Transfer DB - Wire" + } + ] + } + $j$::jsonb + , 2) + ,('PNCC', 'Currency', + $j$ + { + "defn": [ + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "curr1", + "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*" + ,"retain":"y" + }, + { + "key": "{Description}", + "field": "curr2", + "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*" + ,"retain":"y" + } + ], + "where": [ + { + "Transaction": "Miscellaneous Credits" + }, + { + "Transaction": "Miscellaneous Debits" + } + ] + } + $j$::jsonb + , 2) + ,('PNCC', 'Check Number', + $j$ + { + "defn": [ + { + "key": "{Description}", + "field": "checkn", + "regex": "[^0-9]*([0-9]*)\\s|$" + ,"retain":"y" + } + ], + "where": [ + { + "Transaction": "Checks Paid" + } + ] + } + $j$::jsonb + , 2) +) x \ No newline at end of file From 9001c44b46e9520625c578c6deb5b7cfe815249a Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 17:39:52 -0400 Subject: [PATCH 26/27] regex mapping script --- do_map.pgsql | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 do_map.pgsql diff --git a/do_map.pgsql b/do_map.pgsql new file mode 100644 index 0000000..cace0b6 --- /dev/null +++ b/do_map.pgsql @@ -0,0 +1,83 @@ + +WITH + +--------------------apply regex operations to transactions----------------------------------------------------------------------------------- + +rx AS ( +SELECT + m.srce, + m.target, + t.id, + jsonb_build_object( + e.v ->> 'key', + (t.rec #> ((e.v ->> 'key')::text[])) + ) AS rkey, + jsonb_build_object( + e.v->>'field', + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END + ) retval, + m.seq +FROM + tps.map_rm m + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE + JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text) WITH ORDINALITY mt(mt, rn) ON true +WHERE + t.srce = 'PNCC' +ORDER BY + m.srce, + m.seq, + m.target, + t.id, + e.rn +), + +----------aggregate regex back to the target level (may be several targets per row)--------------------------------------------------------------- + + +agg_rx AS ( + SELECT + rx.srce, + rx.target, + rx.id, + tps.jsonb_concat_obj(rx.rkey) rkey, + tps.jsonb_concat_obj(rx.retval) AS retval, + rx.seq + FROM + --unwrap json instruction and apply regex using a count per original line for re-aggregation + --need to look at integrating regex option like 'g' that would then need aggegated back as an array, or adding the ordinality number to the title + rx + GROUP BY + rx.srce, + rx.target, + rx.id, + rx.seq +) + + +-------------aggregate all targets back to row level (id)------------------------------------------------------------------------------------------------ + + + SELECT + u.srce, + u.id, + string_agg(u.target,',') target, + jsonb_pretty(tps.jsonb_concat_obj(coalesce(v.map,'{}'::jsonb) ORDER BY seq )) map, + jsonb_pretty(tps.jsonb_concat_obj(u.retval||coalesce(v.map,'{}'::jsonb) ORDER BY seq)) comb + FROM + --re-aggregate return values and explude any records where one or more regex failed with a null result + agg_rx u + LEFT OUTER JOIN tps.map_rv v ON + v.target = u.target AND + v.srce = u.srce AND + v.retval <@ u.retval + GROUP BY + u.srce, + u.id + LIMIT 1000 \ No newline at end of file From ccd685ca22c431bb86dc1b7b63ded7009f002609 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 19 Oct 2017 18:02:52 -0400 Subject: [PATCH 27/27] implement retain flag to drop keys used only for linking to the appropriate map value, but not acutally included in the map --- do_map.pgsql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/do_map.pgsql b/do_map.pgsql index cace0b6..5767e93 100644 --- a/do_map.pgsql +++ b/do_map.pgsql @@ -19,7 +19,8 @@ SELECT ELSE array_to_json(mt.mt) END ) retval, - m.seq + m.seq, + e.v->>'retain' retain FROM tps.map_rm m LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE @@ -48,7 +49,8 @@ agg_rx AS ( rx.id, tps.jsonb_concat_obj(rx.rkey) rkey, tps.jsonb_concat_obj(rx.retval) AS retval, - rx.seq + tps.jsonb_concat_obj(CASE rx.retain WHEN 'y' THEN rx.retval ELSE '{}'::jsonb END) retain, + rx.seq FROM --unwrap json instruction and apply regex using a count per original line for re-aggregation --need to look at integrating regex option like 'g' that would then need aggegated back as an array, or adding the ordinality number to the title @@ -63,13 +65,13 @@ agg_rx AS ( -------------aggregate all targets back to row level (id)------------------------------------------------------------------------------------------------ - SELECT u.srce, u.id, string_agg(u.target,',') target, jsonb_pretty(tps.jsonb_concat_obj(coalesce(v.map,'{}'::jsonb) ORDER BY seq )) map, - jsonb_pretty(tps.jsonb_concat_obj(u.retval||coalesce(v.map,'{}'::jsonb) ORDER BY seq)) comb + jsonb_pretty(tps.jsonb_concat_obj(u.retval||coalesce(v.map,'{}'::jsonb) ORDER BY seq)) comb, + jsonb_pretty(tps.jsonb_concat_obj(u.retain||coalesce(v.map,'{}'::jsonb) ORDER BY seq)) retain FROM --re-aggregate return values and explude any records where one or more regex failed with a null result agg_rx u