pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

current state

Browse Source
This commit is contained in:
Paul Trowbridge 2023-03-24 21:43:28 -04:00
parent 1cca9dafc3
commit d2dbe2deb1
61 changed files with 12723 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
database/build_maps.xlsm Normal file
View File

Binary file not shown.

4
database/deploy/reload/dcard/curl_dcard.cmd Normal file
View File

@ -0,0 +1,4 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/source
curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/regex
curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapping
curl -v -F upload=@./d.csv http://localhost/import?srce=dcard

139
database/deploy/reload/dcard/d.csv Normal file
View File

@ -0,0 +1,139 @@
Trans. Date,Post Date,Description,Amount,Category
01/02/2018,01/02/2018,"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV",4.26,"Services"
01/02/2018,01/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
01/03/2018,01/03/2018,"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3",200.00,"Medical Services"
01/04/2018,01/04/2018,"AT&T *PAYMENT 800-288-2020 TX",57.14,"Services"
01/04/2018,01/07/2018,"WWW.KOHLS.COM #0873 MIDDLETOWN OH",-7.90,"Payments and Credits"
01/05/2018,01/07/2018,"PIZZA HUT 007946 STOW OH",9.24,"Restaurants"
01/05/2018,01/07/2018,"SUBWAY 00044289255 STOW OH",10.25,"Restaurants"
01/06/2018,01/07/2018,"ACME NO. 17 STOW OH",103.98,"Supermarkets"
01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",1.69,"Merchandise"
01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",2.19,"Merchandise"
01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00947R",3.94,"Gasoline"
01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00915R",52.99,"Gasoline"
01/13/2018,01/13/2018,"AUTOZONE #0722 STOW OH",85.36,"Automotive"
01/13/2018,01/13/2018,"DISCOUNT DRUG MART 32 STOW OH",26.68,"Merchandise"
01/13/2018,01/13/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
01/13/2018,01/13/2018,"TARGET STOW OH",197.90,"Merchandise"
01/14/2018,01/14/2018,"DISCOUNT DRUG MART 32 STOW OH",13.48,"Merchandise"
01/15/2018,01/15/2018,"TARGET.COM * 800-591-3869 MN",22.41,"Merchandise"
01/16/2018,01/16/2018,"BUFFALO WILD WINGS KENT KENT OH",63.22,"Restaurants"
01/16/2018,01/16/2018,"PARTA - KCG KENT OH",4.00,"Government Services"
01/16/2018,01/16/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
01/16/2018,01/16/2018,"TARGET.COM * 800-591-3869 MN",44.81,"Merchandise"
01/16/2018,01/16/2018,"TREE CITY COFFEE & PASTR KENT OH",17.75,"Restaurants"
01/17/2018,01/17/2018,"BESTBUYCOM805526794885 888-BESTBUY MN",343.72,"Merchandise"
01/19/2018,01/19/2018,"DISCOUNT DRUG MART 32 STOW OH",5.98,"Merchandise"
01/19/2018,01/19/2018,"U-HAUL OF KENT-STOW KENT OH",15.88,"Travel/ Entertainment"
01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",5.99,"Supermarkets"
01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",17.16,"Supermarkets"
01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",500.97,"Supermarkets"
01/20/2018,01/20/2018,"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W",2.12,"Services"
01/20/2018,01/20/2018,"LOWE'S OF STOW, OH. STOW OH",256.48,"Home Improvement"
01/22/2018,01/22/2018,"HOBBY LOBBY #405 STOW OHITEM TRANSFERRED FROM PREV ACCOUNT",38.49,"Merchandise"
01/23/2018,01/23/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-32.20,"Awards and Rebate Credits"
01/23/2018,01/23/2018,"INTERNET PAYMENT - THANK YOU",-2394.51,"Payments and Credits"
01/27/2018,01/27/2018,"GIANT-EAGLE #4096 STOW OH",67.81,"Supermarkets"
01/27/2018,01/27/2018,"OFFICEMAX/OFFICE DEPOT63 STOW OH",21.06,"Merchandise"
01/27/2018,01/27/2018,"TARGET STOW OH",71.00,"Merchandise"
01/29/2018,01/29/2018,"NETFLIX.COM NETFLIX.COM CA19899514437",14.93,"Services"
01/30/2018,01/30/2018,"PARTA - KCG KENT OH",1.00,"Government Services"
01/30/2018,01/30/2018,"SPEEDWAY 09303 KEN KENT OH",46.57,"Gasoline"
01/30/2018,01/30/2018,"SQ *TWISTED MELTZ KENT OH0002305843011416898511",16.87,"Restaurants"
01/30/2018,01/30/2018,"TARGET STOW OH",49.37,"Merchandise"
01/31/2018,01/31/2018,"TARGET STOW OH",4.14,"Merchandise"
01/31/2018,01/31/2018,"TARGET STREETSBORO OH",14.28,"Merchandise"
01/31/2018,02/01/2018,"TARGET STOW OH",-21.34,"Payments and Credits"
01/31/2018,02/01/2018,"TARGET STREETSBORO OH",-9.60,"Payments and Credits"
02/01/2018,02/01/2018,"EL CAMPESINO STOW OH",42.24,"Restaurants"
02/02/2018,02/02/2018,"CASH ADVANCE FEE",5.00,"Fees"
02/02/2018,02/02/2018,"GOOGLE *ASCIIFLOW.COM G.CO/HELPPAY#CAP0HQTYN5",5.00,"Cash Advances"
02/03/2018,02/03/2018,"TARGET STREETSBORO OH",71.69,"Merchandise"
02/03/2018,02/07/2018,"SAMS CLUB - #4750 CUYAHOGA FALLOH",371.90,"Warehouse Clubs"
02/04/2018,02/04/2018,"ACME NO. 17 STOW OH",8.98,"Supermarkets"
02/04/2018,02/04/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
02/06/2018,02/06/2018,"MINIMUM INTEREST CHARGE FEE",0.50,"Fees"
02/06/2018,02/07/2018,"BP#954778736210 7-ELEVEN STOW OH",52.80,"Gasoline"
02/06/2018,02/07/2018,"CVS/PHARMACY #08932 TWINSBURG OH",13.87,"Merchandise"
02/07/2018,02/07/2018,"AT&T *PAYMENT 800-288-2020 TXX51Z5QX7SMT2U01",57.14,"Services"
02/07/2018,02/07/2018,"TOYS R US #9203 CUYAHOGA FALLOH",193.32,"Merchandise"
02/08/2018,02/08/2018,"GIANT-EAGLE #4096 STOW OH",66.13,"Supermarkets"
02/08/2018,02/08/2018,"TARGET STOW OH",121.32,"Merchandise"
02/09/2018,02/09/2018,"GUIDOS ORIGINAL PIZZA KENT OH",11.75,"Restaurants"
02/09/2018,02/09/2018,"MARATHON PETRO73601 TWINSBURG OH",44.30,"Gasoline"
02/10/2018,02/10/2018,"RSVP NO. 36 STOW OH",14.43,"Supermarkets"
02/10/2018,02/10/2018,"TARGET STOW OH",77.90,"Merchandise"
02/11/2018,02/11/2018,"SUBWAY 00044289255 STOW OH",21.00,"Restaurants"
02/13/2018,02/13/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",12.79,"Restaurants"
02/13/2018,02/13/2018,"IN *MR. BULKY'S FOODS AKRON OHAJ16V8Q6",3.39,"Supermarkets"
02/13/2018,02/13/2018,"TARGET CUYAHOGA FALLOH",5.33,"Supermarkets"
02/14/2018,02/14/2018,"DISCOUNT DRUG MART 32 STOW OH",4.29,"Merchandise"
02/14/2018,02/14/2018,"HANDELS ICE CREAM STOW STOW OH",7.95,"Supermarkets"
02/15/2018,02/15/2018,"BATH&BODY STOW OH",47.19,"Merchandise"
02/15/2018,02/15/2018,"TARGET STOW OH",76.35,"Merchandise"
02/17/2018,02/17/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
02/17/2018,02/17/2018,"WALMART GROCERY 800-966-6546 AR",461.36,"Supermarkets"
02/18/2018,02/18/2018,"ACME NO. 17 STOW OH",32.68,"Supermarkets"
02/18/2018,02/18/2018,"CHIPOTLE ONLINE 303-595-4000 CO",20.75,"Restaurants"
02/19/2018,02/19/2018,"GIANT EAGLE #5863 STREETSBORO OH",25.00,"Supermarkets"
02/20/2018,02/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
02/21/2018,02/21/2018,"BP#954635936241 7-ELEVEN STOW OH",30.04,"Gasoline"
02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",3.19,"Restaurants"
02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",18.22,"Restaurants"
02/22/2018,02/22/2018,"PET SUPPLIES PLUS #68 STOW OH",45.88,"Merchandise"
02/22/2018,02/22/2018,"TOYS R US #9203 CUYAHOGA FALLOH",21.31,"Merchandise"
02/23/2018,02/23/2018,"SUMMIT CO PARKING GAR AKRON OH",6.00,"Services"
02/24/2018,02/24/2018,"GET GO #3396 STOW OH",26.46,"Gasoline"
02/25/2018,02/25/2018,"DISCOUNT DRUG MART 32 STOW OH",19.70,"Merchandise"
02/25/2018,02/25/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
02/25/2018,02/25/2018,"SQ *CORNER CUP COFFEEH STOW OH0001152921507942036274",2.30,"Supermarkets"
02/25/2018,02/25/2018,"TARGET STOW OH",18.49,"Merchandise"
02/28/2018,02/28/2018,"NETFLIX.COM NETFLIX.COM CA20475539512",14.93,"Services"
03/01/2018,03/01/2018,"GIANT-EAGLE #4032 STOW OH",2.99,"Supermarkets"
03/01/2018,03/01/2018,"TARGET STOW OH",72.46,"Merchandise"
03/02/2018,03/02/2018,"LATE FEE",27.00,"Fees"
03/02/2018,03/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
03/02/2018,03/02/2018,"PIZZA HUT 007946 STOW OH",9.89,"Restaurants"
03/03/2018,03/03/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-23.28,"Awards and Rebate Credits"
03/03/2018,03/03/2018,"INTERNET PAYMENT - THANK YOU",-2451.43,"Payments and Credits"
03/03/2018,03/03/2018,"LOWE'S OF STOW, OH. STOW OH",6.93,"Home Improvement"
03/03/2018,03/07/2018,"WAL-MART SC - #2323 STOW OH",150.32,"Merchandise"
03/04/2018,03/04/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",13.66,"Merchandise"
03/06/2018,03/07/2018,"MFW BOOKS LLC 5732022000 MO",86.90,"Education"
03/06/2018,03/07/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",127.46,"Merchandise"
03/08/2018,03/08/2018,"ACME NO. 17 STOW OH",8.58,"Supermarkets"
03/08/2018,03/08/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",2.12,"Restaurants"
03/08/2018,03/08/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
03/08/2018,03/08/2018,"SPEEDWAY 03686 496 STOW OH",45.24,"Gasoline"
03/08/2018,03/08/2018,"SWENSONS STOW KENT STOW OH",8.30,"Restaurants"
03/08/2018,03/08/2018,"TOYS R US #9203 CUYAHOGA FALLOH",5.33,"Merchandise"
03/09/2018,03/09/2018,"SPEEDWAY 03686 496 STOW OH",48.29,"Gasoline"
03/10/2018,03/10/2018,"WALMART GROCERY 800-966-6546 AR",522.22,"Supermarkets"
03/11/2018,03/11/2018,"AT&T *PAYMENT 800-288-2020 TXQ8F55RY7SMT2N04",57.14,"Services"
03/11/2018,03/11/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011470140810",2.30,"Supermarkets"
03/12/2018,03/12/2018,"MICROSOFT *STORE 800-642-7676 WA",1.06,"Services"
03/15/2018,03/15/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011475075512",2.30,"Supermarkets"
03/16/2018,03/16/2018,"ACME NO. 17 STOW OH",15.85,"Supermarkets"
03/16/2018,03/16/2018,"CHIPOTLE 1152 STOW OH",3.85,"Restaurants"
03/16/2018,03/16/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
03/16/2018,03/16/2018,"PIZZA HUT 007946 STOW OH",13.98,"Restaurants"
03/17/2018,03/17/2018,"CHIPOTLE ONLINE 303-595-4000 CO",15.75,"Restaurants"
03/17/2018,03/17/2018,"DISCOUNT DRUG MART 32 STOW OH",9.89,"Merchandise"
03/17/2018,03/17/2018,"MFW BOOKS LLC 5732022000 MO",66.75,"Education"
03/18/2018,03/18/2018,"ACME NO. 17 STOW OH",27.78,"Supermarkets"
03/18/2018,03/18/2018,"GIANT-EAGLE #4032 STOW OH",28.34,"Supermarkets"
03/20/2018,03/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
03/20/2018,03/20/2018,"SONLIGHT CURRICULUM LTD 303-730-8193 CO",762.87,"Education"
03/21/2018,03/21/2018,"BP#954635936241 7-ELEVEN STOW OH",8.87,"Gasoline"
03/21/2018,03/21/2018,"DISCOUNT DRUG MART 32 STOW OH",18.07,"Merchandise"
03/21/2018,03/21/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011484061091",2.30,"Supermarkets"
03/21/2018,03/21/2018,"TARGET STOW OH",1.95,"Merchandise"
03/21/2018,03/21/2018,"TARGET STOW OH",224.85,"Merchandise"
03/22/2018,03/22/2018,"JUSTICE #0639 STOW OH",16.01,"Merchandise"
03/22/2018,03/22/2018,"SPEEDWAY 03686 496 STOW OH",32.54,"Gasoline"
03/22/2018,03/22/2018,"SQ *TWISTED MELTZ KENT OH0002305843011486528725",6.74,"Restaurants"
03/22/2018,03/22/2018,"TARGET STOW OH",6.60,"Merchandise"
03/25/2018,03/25/2018,"ACME NO. 17 STOW OH",95.42,"Supermarkets"
03/25/2018,03/25/2018,"ASIAN-GREEK CUISINES STOW OH",70.25,"Restaurants"
03/25/2018,03/25/2018,"MARATHON PETRO73601 TWINSBURG OH",11.09,"Gasoline"
03/25/2018,03/25/2018,"SPEEDWAY 09303 KEN KENT OH",53.28,"Gasoline"
1 Trans. Date Post Date Description Amount Category
2 01/02/2018 01/02/2018 GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV 4.26 Services
3 01/02/2018 01/02/2018 MICROSOFT *ONEDRIVE 800-642-7676 WA 4.26 Services
4 01/03/2018 01/03/2018 CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3 200.00 Medical Services
5 01/04/2018 01/04/2018 AT&T *PAYMENT 800-288-2020 TX 57.14 Services
6 01/04/2018 01/07/2018 WWW.KOHLS.COM #0873 MIDDLETOWN OH -7.90 Payments and Credits
7 01/05/2018 01/07/2018 PIZZA HUT 007946 STOW OH 9.24 Restaurants
8 01/05/2018 01/07/2018 SUBWAY 00044289255 STOW OH 10.25 Restaurants
9 01/06/2018 01/07/2018 ACME NO. 17 STOW OH 103.98 Supermarkets
10 01/06/2018 01/07/2018 DISCOUNT DRUG MART 32 STOW OH 1.69 Merchandise
11 01/06/2018 01/07/2018 DISCOUNT DRUG MART 32 STOW OH 2.19 Merchandise
12 01/09/2018 01/09/2018 CIRCLE K 05416 STOW OH00947R 3.94 Gasoline
13 01/09/2018 01/09/2018 CIRCLE K 05416 STOW OH00915R 52.99 Gasoline
14 01/13/2018 01/13/2018 AUTOZONE #0722 STOW OH 85.36 Automotive
15 01/13/2018 01/13/2018 DISCOUNT DRUG MART 32 STOW OH 26.68 Merchandise
16 01/13/2018 01/13/2018 EL CAMPESINO STOW OH 6.50 Restaurants
17 01/13/2018 01/13/2018 TARGET STOW OH 197.90 Merchandise
18 01/14/2018 01/14/2018 DISCOUNT DRUG MART 32 STOW OH 13.48 Merchandise
19 01/15/2018 01/15/2018 TARGET.COM * 800-591-3869 MN 22.41 Merchandise
20 01/16/2018 01/16/2018 BUFFALO WILD WINGS KENT KENT OH 63.22 Restaurants
21 01/16/2018 01/16/2018 PARTA - KCG KENT OH 4.00 Government Services
22 01/16/2018 01/16/2018 REMEMBERNHU 402-935-7733 IA 60.00 Services
23 01/16/2018 01/16/2018 TARGET.COM * 800-591-3869 MN 44.81 Merchandise
24 01/16/2018 01/16/2018 TREE CITY COFFEE & PASTR KENT OH 17.75 Restaurants
25 01/17/2018 01/17/2018 BESTBUYCOM805526794885 888-BESTBUY MN 343.72 Merchandise
26 01/19/2018 01/19/2018 DISCOUNT DRUG MART 32 STOW OH 5.98 Merchandise
27 01/19/2018 01/19/2018 U-HAUL OF KENT-STOW KENT OH 15.88 Travel/ Entertainment
28 01/19/2018 01/19/2018 WALMART GROCERY 800-966-6546 AR 5.99 Supermarkets
29 01/19/2018 01/19/2018 WALMART GROCERY 800-966-6546 AR 17.16 Supermarkets
30 01/19/2018 01/19/2018 WALMART GROCERY 800-966-6546 AR 500.97 Supermarkets
31 01/20/2018 01/20/2018 GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W 2.12 Services
32 01/20/2018 01/20/2018 LOWE'S OF STOW, OH. STOW OH 256.48 Home Improvement
33 01/22/2018 01/22/2018 HOBBY LOBBY #405 STOW OHITEM TRANSFERRED FROM PREV ACCOUNT 38.49 Merchandise
34 01/23/2018 01/23/2018 CASHBACK BONUS REDEMPTION PYMT/STMT CRDT -32.20 Awards and Rebate Credits
35 01/23/2018 01/23/2018 INTERNET PAYMENT - THANK YOU -2394.51 Payments and Credits
36 01/27/2018 01/27/2018 GIANT-EAGLE #4096 STOW OH 67.81 Supermarkets
37 01/27/2018 01/27/2018 OFFICEMAX/OFFICE DEPOT63 STOW OH 21.06 Merchandise
38 01/27/2018 01/27/2018 TARGET STOW OH 71.00 Merchandise
39 01/29/2018 01/29/2018 NETFLIX.COM NETFLIX.COM CA19899514437 14.93 Services
40 01/30/2018 01/30/2018 PARTA - KCG KENT OH 1.00 Government Services
41 01/30/2018 01/30/2018 SPEEDWAY 09303 KEN KENT OH 46.57 Gasoline
42 01/30/2018 01/30/2018 SQ *TWISTED MELTZ KENT OH0002305843011416898511 16.87 Restaurants
43 01/30/2018 01/30/2018 TARGET STOW OH 49.37 Merchandise
44 01/31/2018 01/31/2018 TARGET STOW OH 4.14 Merchandise
45 01/31/2018 01/31/2018 TARGET STREETSBORO OH 14.28 Merchandise
46 01/31/2018 02/01/2018 TARGET STOW OH -21.34 Payments and Credits
47 01/31/2018 02/01/2018 TARGET STREETSBORO OH -9.60 Payments and Credits
48 02/01/2018 02/01/2018 EL CAMPESINO STOW OH 42.24 Restaurants
49 02/02/2018 02/02/2018 CASH ADVANCE FEE 5.00 Fees
50 02/02/2018 02/02/2018 GOOGLE *ASCIIFLOW.COM G.CO/HELPPAY#CAP0HQTYN5 5.00 Cash Advances
51 02/03/2018 02/03/2018 TARGET STREETSBORO OH 71.69 Merchandise
52 02/03/2018 02/07/2018 SAMS CLUB - #4750 CUYAHOGA FALLOH 371.90 Warehouse Clubs
53 02/04/2018 02/04/2018 ACME NO. 17 STOW OH 8.98 Supermarkets
54 02/04/2018 02/04/2018 MICROSOFT *ONEDRIVE 800-642-7676 WA 4.26 Services
55 02/06/2018 02/06/2018 MINIMUM INTEREST CHARGE FEE 0.50 Fees
56 02/06/2018 02/07/2018 BP#954778736210 7-ELEVEN STOW OH 52.80 Gasoline
57 02/06/2018 02/07/2018 CVS/PHARMACY #08932 TWINSBURG OH 13.87 Merchandise
58 02/07/2018 02/07/2018 AT&T *PAYMENT 800-288-2020 TXX51Z5QX7SMT2U01 57.14 Services
59 02/07/2018 02/07/2018 TOYS R US #9203 CUYAHOGA FALLOH 193.32 Merchandise
60 02/08/2018 02/08/2018 GIANT-EAGLE #4096 STOW OH 66.13 Supermarkets
61 02/08/2018 02/08/2018 TARGET STOW OH 121.32 Merchandise
62 02/09/2018 02/09/2018 GUIDOS ORIGINAL PIZZA KENT OH 11.75 Restaurants
63 02/09/2018 02/09/2018 MARATHON PETRO73601 TWINSBURG OH 44.30 Gasoline
64 02/10/2018 02/10/2018 RSVP NO. 36 STOW OH 14.43 Supermarkets
65 02/10/2018 02/10/2018 TARGET STOW OH 77.90 Merchandise
66 02/11/2018 02/11/2018 SUBWAY 00044289255 STOW OH 21.00 Restaurants
67 02/13/2018 02/13/2018 CHICK-FIL-A #02197 CUYAHOGA FLS OH 12.79 Restaurants
68 02/13/2018 02/13/2018 IN *MR. BULKY'S FOODS AKRON OHAJ16V8Q6 3.39 Supermarkets
69 02/13/2018 02/13/2018 TARGET CUYAHOGA FALLOH 5.33 Supermarkets
70 02/14/2018 02/14/2018 DISCOUNT DRUG MART 32 STOW OH 4.29 Merchandise
71 02/14/2018 02/14/2018 HANDELS ICE CREAM STOW STOW OH 7.95 Supermarkets
72 02/15/2018 02/15/2018 BATH&BODY STOW OH 47.19 Merchandise
73 02/15/2018 02/15/2018 TARGET STOW OH 76.35 Merchandise
74 02/17/2018 02/17/2018 EL CAMPESINO STOW OH 6.50 Restaurants
75 02/17/2018 02/17/2018 WALMART GROCERY 800-966-6546 AR 461.36 Supermarkets
76 02/18/2018 02/18/2018 ACME NO. 17 STOW OH 32.68 Supermarkets
77 02/18/2018 02/18/2018 CHIPOTLE ONLINE 303-595-4000 CO 20.75 Restaurants
78 02/19/2018 02/19/2018 GIANT EAGLE #5863 STREETSBORO OH 25.00 Supermarkets
79 02/20/2018 02/20/2018 REMEMBERNHU 402-935-7733 IA 60.00 Services
80 02/21/2018 02/21/2018 BP#954635936241 7-ELEVEN STOW OH 30.04 Gasoline
81 02/22/2018 02/22/2018 CHICK-FIL-A #02197 CUYAHOGA FLS OH 3.19 Restaurants
82 02/22/2018 02/22/2018 CHICK-FIL-A #02197 CUYAHOGA FLS OH 18.22 Restaurants
83 02/22/2018 02/22/2018 PET SUPPLIES PLUS #68 STOW OH 45.88 Merchandise
84 02/22/2018 02/22/2018 TOYS R US #9203 CUYAHOGA FALLOH 21.31 Merchandise
85 02/23/2018 02/23/2018 SUMMIT CO PARKING GAR AKRON OH 6.00 Services
86 02/24/2018 02/24/2018 GET GO #3396 STOW OH 26.46 Gasoline
87 02/25/2018 02/25/2018 DISCOUNT DRUG MART 32 STOW OH 19.70 Merchandise
88 02/25/2018 02/25/2018 EL CAMPESINO STOW OH 6.50 Restaurants
89 02/25/2018 02/25/2018 SQ *CORNER CUP COFFEEH STOW OH0001152921507942036274 2.30 Supermarkets
90 02/25/2018 02/25/2018 TARGET STOW OH 18.49 Merchandise
91 02/28/2018 02/28/2018 NETFLIX.COM NETFLIX.COM CA20475539512 14.93 Services
92 03/01/2018 03/01/2018 GIANT-EAGLE #4032 STOW OH 2.99 Supermarkets
93 03/01/2018 03/01/2018 TARGET STOW OH 72.46 Merchandise
94 03/02/2018 03/02/2018 LATE FEE 27.00 Fees
95 03/02/2018 03/02/2018 MICROSOFT *ONEDRIVE 800-642-7676 WA 4.26 Services
96 03/02/2018 03/02/2018 PIZZA HUT 007946 STOW OH 9.89 Restaurants
97 03/03/2018 03/03/2018 CASHBACK BONUS REDEMPTION PYMT/STMT CRDT -23.28 Awards and Rebate Credits
98 03/03/2018 03/03/2018 INTERNET PAYMENT - THANK YOU -2451.43 Payments and Credits
99 03/03/2018 03/03/2018 LOWE'S OF STOW, OH. STOW OH 6.93 Home Improvement
100 03/03/2018 03/07/2018 WAL-MART SC - #2323 STOW OH 150.32 Merchandise
101 03/04/2018 03/04/2018 OLD NAVY ON-LINE 800-OLDNAVY OH 13.66 Merchandise
102 03/06/2018 03/07/2018 MFW BOOKS LLC 5732022000 MO 86.90 Education
103 03/06/2018 03/07/2018 OLD NAVY ON-LINE 800-OLDNAVY OH 127.46 Merchandise
104 03/08/2018 03/08/2018 ACME NO. 17 STOW OH 8.58 Supermarkets
105 03/08/2018 03/08/2018 CHICK-FIL-A #02197 CUYAHOGA FLS OH 2.12 Restaurants
106 03/08/2018 03/08/2018 EL CAMPESINO STOW OH 6.50 Restaurants
107 03/08/2018 03/08/2018 SPEEDWAY 03686 496 STOW OH 45.24 Gasoline
108 03/08/2018 03/08/2018 SWENSONS STOW KENT STOW OH 8.30 Restaurants
109 03/08/2018 03/08/2018 TOYS R US #9203 CUYAHOGA FALLOH 5.33 Merchandise
110 03/09/2018 03/09/2018 SPEEDWAY 03686 496 STOW OH 48.29 Gasoline
111 03/10/2018 03/10/2018 WALMART GROCERY 800-966-6546 AR 522.22 Supermarkets
112 03/11/2018 03/11/2018 AT&T *PAYMENT 800-288-2020 TXQ8F55RY7SMT2N04 57.14 Services
113 03/11/2018 03/11/2018 SQ *CORNER CUP COFFEEH STOW OH0002305843011470140810 2.30 Supermarkets
114 03/12/2018 03/12/2018 MICROSOFT *STORE 800-642-7676 WA 1.06 Services
115 03/15/2018 03/15/2018 SQ *CORNER CUP COFFEEH STOW OH0002305843011475075512 2.30 Supermarkets
116 03/16/2018 03/16/2018 ACME NO. 17 STOW OH 15.85 Supermarkets
117 03/16/2018 03/16/2018 CHIPOTLE 1152 STOW OH 3.85 Restaurants
118 03/16/2018 03/16/2018 EL CAMPESINO STOW OH 6.50 Restaurants
119 03/16/2018 03/16/2018 PIZZA HUT 007946 STOW OH 13.98 Restaurants
120 03/17/2018 03/17/2018 CHIPOTLE ONLINE 303-595-4000 CO 15.75 Restaurants
121 03/17/2018 03/17/2018 DISCOUNT DRUG MART 32 STOW OH 9.89 Merchandise
122 03/17/2018 03/17/2018 MFW BOOKS LLC 5732022000 MO 66.75 Education
123 03/18/2018 03/18/2018 ACME NO. 17 STOW OH 27.78 Supermarkets
124 03/18/2018 03/18/2018 GIANT-EAGLE #4032 STOW OH 28.34 Supermarkets
125 03/20/2018 03/20/2018 REMEMBERNHU 402-935-7733 IA 60.00 Services
126 03/20/2018 03/20/2018 SONLIGHT CURRICULUM LTD 303-730-8193 CO 762.87 Education
127 03/21/2018 03/21/2018 BP#954635936241 7-ELEVEN STOW OH 8.87 Gasoline
128 03/21/2018 03/21/2018 DISCOUNT DRUG MART 32 STOW OH 18.07 Merchandise
129 03/21/2018 03/21/2018 SQ *CORNER CUP COFFEEH STOW OH0002305843011484061091 2.30 Supermarkets
130 03/21/2018 03/21/2018 TARGET STOW OH 1.95 Merchandise
131 03/21/2018 03/21/2018 TARGET STOW OH 224.85 Merchandise
132 03/22/2018 03/22/2018 JUSTICE #0639 STOW OH 16.01 Merchandise
133 03/22/2018 03/22/2018 SPEEDWAY 03686 496 STOW OH 32.54 Gasoline
134 03/22/2018 03/22/2018 SQ *TWISTED MELTZ KENT OH0002305843011486528725 6.74 Restaurants
135 03/22/2018 03/22/2018 TARGET STOW OH 6.60 Merchandise
136 03/25/2018 03/25/2018 ACME NO. 17 STOW OH 95.42 Supermarkets
137 03/25/2018 03/25/2018 ASIAN-GREEK CUISINES STOW OH 70.25 Restaurants
138 03/25/2018 03/25/2018 MARATHON PETRO73601 TWINSBURG OH 11.09 Gasoline
139 03/25/2018 03/25/2018 SPEEDWAY 09303 KEN KENT OH 53.28 Gasoline

24
database/deploy/reload/dcard/extract.sql Normal file
View File

@ -0,0 +1,24 @@
--source
SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'DCARD'
--mapdef
SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'DCARD') x
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'DCARD') X
--records
copy (
select
to_char(r."Trans. Date",'mm/dd/yyyy') "Trans. Date"
,to_char(r."Post Date",'mm/dd/yyyy') "Post Date"
,r."Description"
,r."Amount"
,r."Category"
from
tps.trans
join lateral jsonb_populate_record(null::tps.dcard, rec) r on true
where
srce = 'DCARD'
) to
'C:\users\fleet\downloads\dcard.csv' with (format csv, header true);

25
database/deploy/reload/dcard/map.json Normal file
View File

@ -0,0 +1,25 @@
[
{
"regex": {
"function": "extract",
"description": "pull first 20 characters from description for mapping",
"where": [
{}
],
"defn": [
{
"regex": ".{1,20}",
"map": "y",
"field": "f20",
"flag": "",
"key": "{Description}",
"retain": "y"
}
],
"name": "First 20"
},
"sequence": 2,
"name": "First 20",
"srce": "dcard"
}
]

76
database/deploy/reload/dcard/srce.json Normal file
View File

@ -0,0 +1,76 @@
{
"name": "dcard",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Trans. Date}",
"{Post Date}",
"{Description}"
],
"schemas": {
"default": [
{
"path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
}
],
"mapped": [
{
"path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
},
{
"path": "{party}",
"type": "text",
"column_name": "Party"
},
{
"path": "{reason}",
"type": "text",
"column_name": "Reason"
}
]
}
}

4299
database/deploy/reload/dcard/vals.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
database/deploy/reload/hunt/curl_hunt.cmd Normal file
View File

@ -0,0 +1,3 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set
curl -v -F upload=@//mnt/c/Users/fleet/Downloads/hunt.csv http://localhost:81/import?srce=hunt

27
database/deploy/reload/hunt/extract.sql Normal file
View File

@ -0,0 +1,27 @@
--source
COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'HUNT') TO 'C:\users\fleet\documents\tps_etl\reload\hunt\srce.json' WITH (FORMAT TEXT, HEADER FALSE)
--mapdef
COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'HUNT') x) TO 'C:\users\fleet\documents\tps_etl\reload\hunt\map.json' WITH (FORMAT TEXT, HEADER FALSE)
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'HUNT') X
--records
copy (
select
to_char(r."Date",'mm/dd/yy') "Date"
,r."Reference Number"
,r."Payee Name"
,r."Memo"
,r."Amount"
,r."Category Name"
from
tps.trans
join lateral jsonb_populate_record(null::tps.hunt, rec) r on true
where
srce = 'HUNT'
order by
r."Date" asc
) to
'C:\users\fleet\downloads\hunt.csv' with (format csv, header true);

25
database/deploy/reload/hunt/map.json Normal file
View File

@ -0,0 +1,25 @@
[
{
"name": "First 20",
"srce": "hunt",
"regex": {
"defn": [
{
"key": "{Memo}",
"map": "y",
"flag": "",
"field": "f20",
"regex": ".{1,20}",
"retain": "y"
}
],
"name": "First 20",
"where": [
{}
],
"function": "extract",
"description": "pull first 20 characters from description for mapping"
},
"sequence": 1
}
]

42
database/deploy/reload/hunt/srce.json Normal file
View File

@ -0,0 +1,42 @@
{
"name": "hunt",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Date}"
],
"schemas": {
"default": [
{
"path": "{Date}",
"type": "date",
"column_name": "Date"
},
{
"path": "{Reference Number}",
"type": "numeric",
"column_name": "Reference Number"
},
{
"path": "{Payee Name}",
"type": "text",
"column_name": "Payee Name"
},
{
"path": "{Memo}",
"type": "text",
"column_name": "Memo"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category Name}",
"type": "text",
"column_name": "Cateogry Name"
}
]
}
}

58
database/deploy/reload/paycom/extract.sql Normal file
View File

@ -0,0 +1,58 @@
--transactions with date in download format for constraint
COPY
(
SELECT
r."perd_start",
r."perd_end",
r."check_date",
r."loc_code",
r."loc_descr",
r."loc_glseg",
r."loc_over",
r."dep_code",
r."dep_descr",
r."dep_nat",
r."dep_over",
r."di_code",
r."di_descr",
r."di_glseg",
r."di_over",
r."title_code",
r."title_descr",
r."title_glseg",
r."title_over",
r."ee_code",
r."ee_glseg",
r."ee_over",
r."acct_type_code",
r."hours",
r."nat_code",
r."nat_over",
r."gl_ref",
r."gl_group",
r."gl_descr",
r."gl_code",
r."gl_amount",
r."pp_code",
r."pp_descr",
r."pp_gl",
r."pp_over",
r."transaction"
FROM
tps.trans
JOIN LATERAL jsonb_populate_record(NULL::tps.PAYCOM, rec) r ON TRUE
WHERE
srce = 'PAYCOM'
)
TO 'C:\users\ptrowbridge\downloads\PAYCOM.csv' WITH (format csv, header TRUE)
--source
SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PAYCOM'
--mapdef
SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PAYCOM') x
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PAYCOM') X

4
database/deploy/reload/paycom/load.cmd Normal file
View File

@ -0,0 +1,4 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set
curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/PAYCOM.csv http://localhost:81/import?srce=PAYCOM

64
database/deploy/reload/paycom/map.json Normal file
View File

@ -0,0 +1,64 @@
[
{
"name": "Extract month and code",
"srce": "PAYCOM",
"regex": {
"defn": [
{
"key": "{check_date}",
"map": "n",
"flag": "",
"field": "check_month",
"regex": "(\\d{4})-(\\d{2})-\\d{2}",
"retain": "y"
},
{
"key": "{loc_code}",
"map": "y",
"flag": "",
"field": "loc_code_rx",
"regex": ".*",
"retain": "n"
}
],
"name": "Paycom",
"where": [
{}
],
"function": "extract",
"description": "extract month and code"
},
"sequence": 1
},
{
"name": "SUBSEQUENT PAYOUT",
"srce": "PAYCOM",
"regex": {
"defn": [
{
"key": "{gl_group}",
"map": "y",
"flag": "",
"field": "gl_group",
"regex": ".*",
"retain": "n"
},
{
"key": "{acct_type_code}",
"map": "y",
"flag": "",
"field": "acct_type_code",
"regex": ".*",
"retain": "n"
}
],
"name": "Paycom",
"where": [
{}
],
"function": "extract",
"description": "map payroll code and account group to subsequent payout"
},
"sequence": 2
}
]

392
database/deploy/reload/paycom/srce.json Normal file
View File

@ -0,0 +1,392 @@
{
"constraint": [
"{transaction}",
"{loc_code}",
"{perd_start}",
"{perd_end}"
],
"name": "PAYCOM",
"source": "client_file",
"loading_function": "csv",
"schemas": {
"default": [
{
"path": "{perd_start}",
"type": "date",
"column_name": "perd_start"
},
{
"path": "{perd_end}",
"type": "date",
"column_name": "perd_end"
},
{
"path": "{check_date}",
"type": "date",
"column_name": "check_date"
},
{
"path": "{loc_code}",
"type": "text",
"column_name": "loc_code"
},
{
"path": "{loc_descr}",
"type": "text",
"column_name": "loc_descr"
},
{
"path": "{loc_glseg}",
"type": "text",
"column_name": "loc_glseg"
},
{
"path": "{loc_over}",
"type": "text",
"column_name": "loc_over"
},
{
"path": "{dep_code}",
"type": "text",
"column_name": "dep_code"
},
{
"path": "{dep_descr}",
"type": "text",
"column_name": "dep_descr"
},
{
"path": "{dep_nat}",
"type": "text",
"column_name": "dep_nat"
},
{
"path": "{dep_over}",
"type": "text",
"column_name": "dep_over"
},
{
"path": "{di_code}",
"type": "text",
"column_name": "di_code"
},
{
"path": "{di_descr}",
"type": "text",
"column_name": "di_descr"
},
{
"path": "{di_glseg}",
"type": "text",
"column_name": "di_glseg"
},
{
"path": "{di_over}",
"type": "text",
"column_name": "di_over"
},
{
"path": "{title_code}",
"type": "text",
"column_name": "title_code"
},
{
"path": "{title_descr}",
"type": "text",
"column_name": "title_descr"
},
{
"path": "{title_glseg}",
"type": "text",
"column_name": "title_glseg"
},
{
"path": "{title_over}",
"type": "text",
"column_name": "title_over"
},
{
"path": "{ee_code}",
"type": "text",
"column_name": "ee_code"
},
{
"path": "{ee_glseg}",
"type": "text",
"column_name": "ee_glseg"
},
{
"path": "{ee_over}",
"type": "text",
"column_name": "ee_over"
},
{
"path": "{acct_type_code}",
"type": "text",
"column_name": "acct_type_code"
},
{
"path": "{hours}",
"type": "numeric",
"column_name": "hours"
},
{
"path": "{nat_code}",
"type": "text",
"column_name": "nat_code"
},
{
"path": "{nat_over}",
"type": "text",
"column_name": "nat_over"
},
{
"path": "{gl_ref}",
"type": "text",
"column_name": "gl_ref"
},
{
"path": "{gl_group}",
"type": "text",
"column_name": "gl_group"
},
{
"path": "{gl_descr}",
"type": "text",
"column_name": "gl_descr"
},
{
"path": "{gl_code}",
"type": "text",
"column_name": "gl_code"
},
{
"path": "{gl_amount}",
"type": "numeric",
"column_name": "gl_amount"
},
{
"path": "{pp_code}",
"type": "text",
"column_name": "pp_code"
},
{
"path": "{pp_descr}",
"type": "text",
"column_name": "pp_descr"
},
{
"path": "{pp_gl}",
"type": "text",
"column_name": "pp_gl"
},
{
"path": "{pp_over}",
"type": "text",
"column_name": "pp_over"
},
{
"path": "{transaction}",
"type": "text",
"column_name": "transaction"
}
],
"mapped":[
{
"path": "{perd_start}",
"type": "date",
"column_name": "perd_start"
},
{
"path": "{perd_end}",
"type": "date",
"column_name": "perd_end"
},
{
"path": "{check_date}",
"type": "date",
"column_name": "check_date"
},
{
"path": "{loc_code}",
"type": "text",
"column_name": "loc_code"
},
{
"path": "{loc_descr}",
"type": "text",
"column_name": "loc_descr"
},
{
"path": "{loc_glseg}",
"type": "text",
"column_name": "loc_glseg"
},
{
"path": "{loc_over}",
"type": "text",
"column_name": "loc_over"
},
{
"path": "{dep_code}",
"type": "text",
"column_name": "dep_code"
},
{
"path": "{dep_descr}",
"type": "text",
"column_name": "dep_descr"
},
{
"path": "{dep_nat}",
"type": "text",
"column_name": "dep_nat"
},
{
"path": "{dep_over}",
"type": "text",
"column_name": "dep_over"
},
{
"path": "{di_code}",
"type": "text",
"column_name": "di_code"
},
{
"path": "{di_descr}",
"type": "text",
"column_name": "di_descr"
},
{
"path": "{di_glseg}",
"type": "text",
"column_name": "di_glseg"
},
{
"path": "{di_over}",
"type": "text",
"column_name": "di_over"
},
{
"path": "{title_code}",
"type": "text",
"column_name": "title_code"
},
{
"path": "{title_descr}",
"type": "text",
"column_name": "title_descr"
},
{
"path": "{title_glseg}",
"type": "text",
"column_name": "title_glseg"
},
{
"path": "{title_over}",
"type": "text",
"column_name": "title_over"
},
{
"path": "{ee_code}",
"type": "text",
"column_name": "ee_code"
},
{
"path": "{ee_glseg}",
"type": "text",
"column_name": "ee_glseg"
},
{
"path": "{ee_over}",
"type": "text",
"column_name": "ee_over"
},
{
"path": "{acct_type_code}",
"type": "text",
"column_name": "acct_type_code"
},
{
"path": "{hours}",
"type": "numeric",
"column_name": "hours"
},
{
"path": "{nat_code}",
"type": "text",
"column_name": "nat_code"
},
{
"path": "{nat_over}",
"type": "text",
"column_name": "nat_over"
},
{
"path": "{gl_ref}",
"type": "text",
"column_name": "gl_ref"
},
{
"path": "{gl_group}",
"type": "text",
"column_name": "gl_group"
},
{
"path": "{gl_descr}",
"type": "text",
"column_name": "gl_descr"
},
{
"path": "{gl_code}",
"type": "text",
"column_name": "gl_code"
},
{
"path": "{gl_amount}",
"type": "numeric",
"column_name": "gl_amount"
},
{
"path": "{pp_code}",
"type": "text",
"column_name": "pp_code"
},
{
"path": "{pp_descr}",
"type": "text",
"column_name": "pp_descr"
},
{
"path": "{pp_gl}",
"type": "text",
"column_name": "pp_gl"
},
{
"path": "{pp_over}",
"type": "text",
"column_name": "pp_over"
},
{
"path": "{transaction}",
"type": "text",
"column_name": "transaction"
},
{
"path": "{trial_bal}",
"type": "text",
"column_name": "trial_bal"
},
{
"path": "{third_party}",
"type": "text",
"column_name": "third_party"
},
{
"path": "{check_month}",
"type": "jsonb",
"column_name": "check_month"
}
]
}
}

139
database/deploy/reload/paycom/vals.json Normal file
View File

@ -0,0 +1,139 @@
[
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "65"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "SLS"
}
},
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "59"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "SEB"
}
},
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "93"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "MDF"
}
},
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "93"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "ELY"
}
},
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "93"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "TWN"
}
},
{
"map": "Extract month and code",
"mapped": {
"trial_bal": "88"
},
"source": "PAYCOM",
"ret_val": {
"loc_code_rx": "SPK"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "PRINCIPAL LIFE P"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "4.Liab",
"acct_type_code": "40P"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "PRINCIPAL LIFE P"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "4.Liab",
"acct_type_code": "CUF"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "PRINCIPAL LIFE P"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "4.Liab",
"acct_type_code": "K4L"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "PRINCIPAL LIFE P"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "5.Liab",
"acct_type_code": "40P"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "PRINCIPAL LIFE P"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "5.Liab",
"acct_type_code": "CUF"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "Basic NEO"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "4.Liab",
"acct_type_code": "FSA"
}
},
{
"map": "SUBSEQUENT PAYOUT",
"mapped": {
"third_party": "Basic NEO"
},
"source": "PAYCOM",
"ret_val": {
"gl_group": "4.Liab",
"acct_type_code": "FSD"
}
}
]

32
database/deploy/reload/pncc/extract.sql Normal file
View File

@ -0,0 +1,32 @@
--transactions with date in download format for constraint
COPY
(
SELECT
to_char(r."AsOfDate",'mm/dd/yyyy') "AsOfDate"
,r."BankId"
,r."AccountNumber"
,r."AccountName"
,r."BaiControl"
,r."Currency"
,r."Transaction"
,r."Reference"
,r."Amount"
,r."Description"
FROM
tps.trans
JOIN LATERAL jsonb_populate_record(NULL::tps.pncc, rec) r ON TRUE
WHERE
srce = 'PNCC'
)
TO 'C:\users\ptrowbridge\downloads\pncc.csv' WITH (format csv, header TRUE)
--source
SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCC'
--mapdef
SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCC') x
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCC') X

4
database/deploy/reload/pncc/load.cmd Normal file
View File

@ -0,0 +1,4 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set
curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncc.csv http://localhost:81/import?srce=PNCC

455
database/deploy/reload/pncc/map.json Normal file
View File

@ -0,0 +1,455 @@
[
{
"name": "Check Number",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Description}",
"map": "n",
"field": "checkn",
"regex": "[^0-9]*([0-9]*)\\s|$",
"retain": "y"
}
],
"where": [
{
"Transaction": "Checks Paid"
}
],
"function": "extract"
},
"sequence": 2
},
{
"name": "Strip Amount Commas",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Amount}",
"map": "n",
"flag": "g",
"field": "amount",
"regex": ",",
"retain": "y",
"replace": ""
}
],
"name": "Strip Amount Commas",
"where": [
{}
],
"function": "replace",
"description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric"
},
"sequence": 1
},
{
"name": "Trans Type",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{AccountName}",
"map": "y",
"field": "acctn",
"regex": "(.*)",
"retain": "n"
},
{
"key": "{Transaction}",
"map": "y",
"field": "trans",
"regex": "(.*)",
"retain": "n"
},
{
"key": "{Description}",
"map": "y",
"field": "ini",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y"
}
],
"name": "Trans Type",
"where": [
{}
],
"function": "extract",
"description": "extract intial description in conjunction with account name and transaction type for mapping"
},
"sequence": 1
},
{
"name": "Currency",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Description}",
"map": "y",
"field": "ini",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y"
},
{
"key": "{Description}",
"map": "y",
"field": "curr1",
"regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*",
"retain": "y"
},
{
"key": "{Description}",
"map": "y",
"field": "curr2",
"regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*",
"retain": "y"
}
],
"name": "Currency",
"where": [
{
"Transaction": "Miscellaneous Credits"
},
{
"Transaction": "Miscellaneous Debits"
}
],
"function": "extract",
"description": "pull out currency indicators from description of misc items and map"
},
"sequence": 2
},
{
"name": "Parse ACH Credits",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "beneficiary",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Cust ID",
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Desc",
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "originator",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Batch Discr",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Comp ID",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Addenda",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "SETT",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Date",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Time",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
}
],
"name": "Parse ACH Credits",
"where": [
{
"Transaction": "ACH Credits"
}
],
"function": "extract",
"description": "parse select components of the description for ACH Credits Receieved"
},
"sequence": 2
},
{
"name": "Parse ACH Debits",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "originator",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Cust ID",
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Desc",
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "beneficiary",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Batch Discr",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Comp ID",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Addenda",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "SETT",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Date",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "Time",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"retain": "y"
}
],
"name": "Parse ACH Debits",
"where": [
{
"Transaction": "ACH Debits"
}
],
"function": "extract",
"description": "parse select components of the description for ACH Credits Receieved"
},
"sequence": 2
},
{
"name": "Parse Wires",
"srce": "PNCC",
"regex": {
"defn": [
{
"key": "{Description}",
"map": "n",
"flag": "g",
"field": "dparse",
"regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "beneficiary_components",
"regex": "BENEFICIARY:(.*?)AC/([\\d-]*) (.*)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "originator_components",
"regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "beneficiary",
"regex": "BENEFICIARY:(.*?)AC/[\\d-]* .*(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "originator",
"regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "OBI",
"regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "RFB",
"regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "ABA",
"regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "BBI",
"regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "BENEBNK",
"regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "IBK",
"regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "RATE",
"regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
},
{
"key": "{Description}",
"map": "n",
"flag": "",
"field": "RECVBNK",
"regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)",
"retain": "y"
}
],
"name": "Parse Wires",
"where": [
{
"Transaction": "Money Transfer DB - Wire"
},
{
"Transaction": "Money Transfer DB - Other"
},
{
"Transaction": "Money Transfer CR-Wire"
},
{
"Transaction": "Money Transfer CR-Other"
},
{
"Transaction": "Intl Money Transfer Debits"
},
{
"Transaction": "Intl Money Transfer Credits"
}
],
"function": "extract",
"description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered"
},
"sequence": 2
}
]

67
database/deploy/reload/pncc/srce.json Normal file
View File

@ -0,0 +1,67 @@
{
"name": "PNCC",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{AsOfDate}"
],
"schemas": {
"default": [
{
"path": "{AsOfDate}",
"type": "date",
"column_name": "AsOfDate"
},
{
"path": "{BankId}",
"type": "text",
"column_name": "BankId"
},
{
"path": "{AccountNumber}",
"type": "text",
"column_name": "AccountNumber"
},
{
"path": "{AccountName}",
"type": "text",
"column_name": "AccountName"
},
{
"path": "{BaiControl}",
"type": "text",
"column_name": "BaiControl"
},
{
"path": "{Currency}",
"type": "text",
"column_name": "Currency"
},
{
"path": "{Transaction}",
"type": "text",
"column_name": "Transaction"
},
{
"path": "{Reference}",
"type": "text",
"column_name": "Reference"
},
{
"path": "{Amount}",
"type": "text",
"column_name": "Amount"
},
{
"path":"{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{AdditionalRemittance}",
"type": "text",
"column_name": "AdditionalRemittance"
}
]
}
}

952
database/deploy/reload/pncc/vals.json Normal file
View File

@ -0,0 +1,952 @@
[
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"party": "PNC",
"ledger": "Manual",
"reason": "Bank Fees",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CANADA TAX",
"acctn": "The HC Operating Company OPERA",
"trans": "Detail Debit Adjustments"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - ACH",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH DEBIT SETTLEMENT",
"acctn": "The HC Operating Company OPERA",
"trans": "ACH Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Returned Deposit RTM",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "RET DEP ITEM RTM",
"acctn": "The HC Operating Company FBO P",
"trans": "Deposited Items Returned"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Returned Deposit STOP",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "RET DEP ITEM STOP",
"acctn": "The HC Operating Company FBO P",
"trans": "Deposited Items Returned"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "CREDIT ADJUSTMENT",
"acctn": "The HC Operating Company FBO P",
"trans": "Detail Credit Adjustments"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Returned Check",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "REFER TO MAKER OF CK RETURN CK",
"acctn": "The HC Operating Company OPERA",
"trans": "Detail Credit Adjustments"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Payroll Adjustment",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "DEBIT ADJUSTMENT",
"acctn": "The HC Operating Company PAYR",
"trans": "Detail Debit Adjustments"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT",
"acctn": "The HC Operating Company FBO P",
"trans": "Detail Deposits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Wire",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "INTL WIRE OUT",
"acctn": "The HC Operating Company OPERA",
"trans": "Intl Money Transfer Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Wire",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "INTL WIRES OUT",
"acctn": "The HC Operating Company OPERA",
"trans": "Intl Money Transfer Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "WHLS LBX DEP",
"acctn": "The HC Operating Company FBO P",
"trans": "Lockbox Deposits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "WHLS LBX DEP932855",
"acctn": "The HC Operating Company FBO P",
"trans": "Lockbox Deposits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Revolver Advance",
"trantype": "Revolver Borrow"
},
"source": "PNCC",
"ret_val": {
"ini": "ADVANCE",
"acctn": "The HC Operating Company OPERA",
"trans": "Miscellaneous Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT:",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Misc Credit",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "MISC CREDIT",
"acctn": "The HC Operating Company OPERA",
"trans": "Miscellaneous Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Revolver Payment",
"trantype": "Revolver Borrow"
},
"source": "PNCC",
"ret_val": {
"ini": "PAYMENT",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Revolver Payment",
"trantype": "Revolver Payment"
},
"source": "PNCC",
"ret_val": {
"ini": "PAYMENT",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "INTTL WIRES IN",
"acctn": "The HC Operating Company FBO P",
"trans": "Intl Money Transfer Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Revolver Advance",
"trantype": "Revolver Borrow"
},
"source": "PNCC",
"ret_val": {
"ini": "PNC BANK- NJ LOAN PROCEEDS",
"acctn": "The HC Operating Company FBO P",
"trans": "Money Transfer CR-Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Revolver Advance",
"trantype": "Revolver Borrow"
},
"source": "PNCC",
"ret_val": {
"ini": "PNC BANK-PGH LOAN PROCEEDS",
"acctn": "The HC Operating Company OPERA",
"trans": "Money Transfer CR-Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "FED WIRE IN",
"acctn": "The HC Operating Company FBO P",
"trans": "Money Transfer CR-Wire"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Returned Wires",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "FED WIRE IN",
"acctn": "The HC Operating Company OPERA",
"trans": "Money Transfer CR-Wire"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Returned Item",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "BOOK TRANSFER DEBIT",
"acctn": "The HC Operating Company OPERA",
"trans": "Money Transfer DB - Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Freight Wires",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "FED WIRE OUT",
"acctn": "The HC Operating Company FREIG",
"trans": "Money Transfer DB - Wire"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Wire",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "FED WIRE OUT",
"acctn": "The HC Operating Company OPERA",
"trans": "Money Transfer DB - Wire"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "FED WIRE OUT",
"acctn": "The HC Operating Company PAYR",
"trans": "Money Transfer DB - Wire"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "ZBA Funding",
"trantype": "Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "FUNDS TRANSFER FROM ACCT",
"acctn": "The HC Operating Company FREIG",
"trans": "ZBA Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "ZBA Funding",
"trantype": "Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "FUNDS TRANSFER FROM ACCT",
"acctn": "The HC Operating Company OPERA",
"trans": "ZBA Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "ZBA Funding",
"trantype": "Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "FUNDS TRANSFER FROM ACCT",
"acctn": "The HC Operating Company PAYR",
"trans": "ZBA Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "ZBA Funding",
"trantype": "Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "FUNDS TRANSFER TO ACCT",
"acctn": "The HC Operating Company OPERA",
"trans": "ZBA Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "ZBA Funding",
"trantype": "Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "FUNDS TRANSFER TO ACCT",
"acctn": "The HC Operating Company PAYR",
"trans": "ZBA Debits"
}
},
{
"map": "Currency",
"mapped": {
"party": "The HC Canada Operating Company, Ltd.",
"ledger": "Manual",
"reason": "IC - Can to US Settlement",
"trantype": "Interco Collection"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT:",
"curr1": [
"CREDIT",
"USD"
],
"curr2": [
"DEBIT",
"CAD"
]
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"party": "PNC",
"ledger": "Manual",
"reason": "Bank Fees",
"trantype": "Fees"
},
"source": "PNCC",
"ret_val": {
"ini": "CORPORATE ACCOUNT ANALYSIS CHARGE",
"acctn": "The HC Operating Company OPERA",
"trans": "Miscellaneous Fees"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"party": "PNC",
"ledger": "Manual",
"reason": "Bank Fees",
"trantype": "Fees"
},
"source": "PNCC",
"ret_val": {
"ini": "PNC MERCHANT FINCL ADJ",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Fees"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Revolver Payment",
"trantype": "Revolver Payment"
},
"source": "PNCC",
"ret_val": {
"ini": "PNC BANK- NJ LOAN PMTS",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "WITHDRAWAL:",
"acctn": "The HC Operating Company FBO P",
"trans": "Miscellaneous Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Returned Item",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "BOOK TRANSFER CREDIT",
"acctn": "The HC Operating Company FBO P",
"trans": "Money Transfer CR-Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Returned Item",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "BOOK TRANSFER CREDIT",
"acctn": "The HC Operating Company OPERA",
"trans": "Money Transfer CR-Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Returned Item",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "BOOK TRANSFER CREDIT GHFTDD DDA CREDIT",
"acctn": "The HC Operating Company FBO P",
"trans": "Money Transfer CR-Other"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Payroll Credits",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "19UDV",
"acctn": "The HC Operating Company PAYR",
"trans": "ACH Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH CREDIT RECEIVED",
"acctn": "The HC Operating Company FBO P",
"trans": "ACH Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "Payroll Credits",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH CREDIT RECEIVED",
"acctn": "The HC Operating Company PAYR",
"trans": "ACH Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"reason": "AP ACH Returned",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH CREDIT RETURN",
"acctn": "The HC Operating Company OPERA",
"trans": "ACH Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Auto ACH Out",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH DEBIT RECEIVED",
"acctn": "The HC Operating Company FBO P",
"trans": "ACH Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Auto ACH Out",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH DEBIT RECEIVED",
"acctn": "The HC Operating Company OPERA",
"trans": "ACH Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Auto ACH Out",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH DEBIT RECEIVED",
"acctn": "The HC Operating Company PAYR",
"trans": "ACH Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Check Run",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CASHED CHECK",
"acctn": "The HC Operating Company OPERA",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Payroll Checks",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CASHED CHECK",
"acctn": "The HC Operating Company PAYR",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Freight Checks",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CHECK",
"acctn": "The HC Operating Company FREIG",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Check Run",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CHECK",
"acctn": "The HC Operating Company OPERA",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Payroll Checks",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CHECK",
"acctn": "The HC Operating Company PAYR",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Freight Checks",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "SUBSTITUTE CHK",
"acctn": "The HC Operating Company FREIG",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "AP - Check Run",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "SUBSTITUTE CHK",
"acctn": "The HC Operating Company OPERA",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Returned Deposit NSF",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "RET DEP ITEM NSF UN",
"acctn": "The HC Operating Company FBO P",
"trans": "Deposited Items Returned"
}
},
{
"map": "Currency",
"mapped": {
"party": "The HC Canada Operating Company, Ltd.",
"ledger": "Manual",
"reason": "IC - US to CAN Settlement",
"trantype": "Interco Funding"
},
"source": "PNCC",
"ret_val": {
"ini": "WITHDRAWAL:",
"curr1": [
"DEBIT",
"USD"
],
"curr2": [
"CREDIT",
"CAD"
]
}
},
{
"map": "Currency",
"mapped": {
"party": "The HC Canada Operating Company, Ltd.",
"ledger": "Manual",
"reason": "IC - round-trip settlement return",
"trantype": "Interco Collection"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT:",
"curr1": [
"CREDIT",
"USD"
],
"curr2": [
"DEBIT",
"USD"
]
}
},
{
"map": "Currency",
"mapped": {
"party": "The HC Canada Operating Company, Ltd.",
"ledger": "Manual",
"reason": "IC - Can to US Settlement",
"trantype": "Interco Collection"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT:",
"curr1": [
"DEBIT",
"USD"
],
"curr2": [
"CREDIT",
"CAD"
]
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "Manual",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "DEPOSIT:",
"acctn": "The HC Operating Company OPERA",
"trans": "Miscellaneous Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"reason": "Payroll Checks",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "SUBSTITUTE CHK",
"acctn": "The HC Operating Company PAYR",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "manual",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH DEBIT RETURN",
"acctn": "The HC Operating Company OPERA",
"trans": "ACH Debits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "manual",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "ACH CREDIT SETTLEMENT",
"acctn": "The HC Operating Company OPERA",
"trans": "ACH Credits"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "-1",
"ledger": "Manual",
"trantype": "Disbursement"
},
"source": "PNCC",
"ret_val": {
"ini": "CHECK",
"acctn": "The HC Operating Company FBO P",
"trans": "Checks Paid"
}
},
{
"map": "Trans Type",
"mapped": {
"sign": "1",
"ledger": "AR - Collections",
"trantype": "Collections"
},
"source": "PNCC",
"ret_val": {
"ini": "POSTING CORRECTION RETURN CK",
"acctn": "The HC Operating Company FBO P",
"trans": "Detail Credit Adjustments"
}
}
]

30
database/deploy/reload/pncl/extract.sql Normal file
View File

@ -0,0 +1,30 @@
--transactions with date in download format for constraint
--transactions with date in download format for constraint
COPY
(
SELECT
r."Schedule#"
,to_char(r."PostDate",'mm/dd/yyyy') "PostDate"
,r."Assn#"
,r."Coll#"
,COALESCE(r."AdvanceRate",0) "AdvanceRate"
,COALESCE(r."Sales",0) "Sales"
,COALESCE(r."Credits & Adjustments",0) "Credits & Adjustments"
,COALESCE(r."Gross Collections",0) "Gross Collections"
,COALESCE(r."CollateralBalance",0) "CollateralBalance"
,COALESCE(r."MaxEligible",0) "MaxEligible"
,COALESCE(r."Ineligible Amount",0) "Ineligible Amount"
,COALESCE(r."Reserve Amount",0) "Reserve Amount"
FROM
tps.trans
JOIN LATERAL jsonb_populate_record(NULL::tps.pncl, rec) r ON TRUE
WHERE
srce = 'PNCL'
--and case when rec->>'Credits & Adjustments' is null then 'null' else '' end <> 'null'
)
TO 'C:\users\ptrowbridge\downloads\pncl.csv' WITH (format csv, header TRUE)
--source
SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCL'

2
database/deploy/reload/pncl/load.cmd Normal file
View File

@ -0,0 +1,2 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncl.csv http://localhost:81/import?srce=PNCL

73
database/deploy/reload/pncl/srce.json Normal file
View File

@ -0,0 +1,73 @@
{
"constraint": [
"{PostDate}",
"{Schedule#}"
],
"source": "client_file",
"loading_function": "csv",
"name": "PNCL",
"schemas": {
"default": [
{
"path": "{Schedule#}",
"type": "text",
"column_name": "Schedule#"
},
{
"type": "date",
"column_name": "PostDate",
"path": "{PostDate}"
},
{
"type": "text",
"column_name": "Assn#",
"path": "{Assn#}"
},
{
"type": "text",
"column_name": "Coll#",
"path": "{Coll#}"
},
{
"type": "numeric",
"column_name": "AdvanceRate",
"path": "{AdvanceRate}"
},
{
"type": "numeric",
"column_name": "Sales",
"path": "{Sales}"
},
{
"type": "numeric",
"column_name": "Credits & Adjustments",
"path": "{Credits & Adjustments}"
},
{
"type": "numeric",
"column_name": "Gross Collections",
"path": "{Gross Collections}"
},
{
"type": "numeric",
"column_name": "CollateralBalance",
"path": "{CollateralBalance}"
},
{
"type": "numeric",
"column_name": "MaxEligible",
"path": "{MaxEligible}"
},
{
"type": "numeric",
"column_name": "Ineligible Amount",
"path": "{Ineligible Amount}"
},
{
"type": "numeric",
"column_name": "Reserve Amount",
"path": "{Reserve Amount}"
}
]
}
}

32
database/deploy/reload/pnco/extract.sql Normal file
View File

@ -0,0 +1,32 @@
--transactions with date in download format for constraint
--transactions with date in download format for constraint
COPY
(
SELECT
r."Loan#"
,to_char(r."Post Date",'mm/dd/yyyy') "Post Date"
,to_char(r."Effective Date",'mm/dd/yyyy') "Effective Date"
,r."Reference #"
,r."Description"
,r."Advances"
,r."Adjustments"
,r."Payments"
,r."Loan Balance"
FROM
tps.trans
JOIN LATERAL jsonb_populate_record(NULL::tps.pnco, rec) r ON TRUE
WHERE
srce = 'PNCO'
)
TO 'C:\users\ptrowbridge\downloads\pnco.csv' WITH (format csv, header TRUE)
--source
SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCO'
--mapdef
SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCO') x
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCO') X

2
database/deploy/reload/pnco/load.cmd Normal file
View File

@ -0,0 +1,2 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pnco.csv http://localhost:81/import?srce=PNCO

60
database/deploy/reload/pnco/srce.json Normal file
View File

@ -0,0 +1,60 @@
{
"name": "PNCO",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Post Date}",
"{Effective Date}",
"{Loan#}",
"{Reference #}"
],
"schemas": {
"default": [
{
"path": "{Loan#}",
"type": "text",
"column_name":"Loan#"
},
{
"path": "{Post Date}",
"type": "date",
"column_name":"Post Date"
},
{
"path": "{Effective Date}",
"type": "date",
"column_name":"Effective Date"
},
{
"path": "{Reference #}",
"type": "text",
"column_name":"Reference #"
},
{
"path": "{Description}",
"type": "text",
"column_name":"Description"
},
{
"path": "{Advances}",
"type": "numeric",
"column_name":"Advances"
},
{
"path": "{Adjustments}",
"type": "numeric",
"column_name":"Adjustments"
},
{
"path": "{Payments}",
"type": "numeric",
"column_name":"Payments"
},
{
"path": "{Loan Balance}",
"type": "numeric",
"column_name":"Loan Balance"
}
]
}
}

3
database/deploy/reload/wmpd/curl_hunt.cmd Normal file
View File

@ -0,0 +1,3 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/WMPD.csv http://localhost:81/import?srce=WMPD

63
database/deploy/reload/wmpd/extract.sql Normal file
View File

@ -0,0 +1,63 @@
--source
COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'WMPD') TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\srce.json' WITH (FORMAT TEXT, HEADER FALSE)
--mapdef
COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'WMPD') x) TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\map.json' WITH (FORMAT TEXT, HEADER FALSE)
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'WMPD') X
--records
copy (
select
r."Carrier",
r."SCAC",
r."Mode",
r."Pro #",
r."B/L",
r."Pd Amt",
r."Loc#",
r."Pcs",
r."Wgt",
r."Chk#",
r."Pay Dt",
r."Acct #",
r."I/O",
r."Sh Nm",
r."Sh City",
r."Sh St",
r."Sh Zip",
r."Cons Nm",
r."D City ",
r."D St",
r."D Zip",
r."Sh Dt",
r."Inv Dt",
r."Customs Entry#",
r."Miles",
r."Frt Class",
r."Master B/L"
from
tps.trans
join lateral jsonb_populate_record(null::tps.WMPD, rec) r on true
where
srce = 'WMPD'
order by
r."Pay Dt" asc
) to
'C:\users\ptrowbridge\downloads\WMPD.csv' with (format csv, header true);
--rebuild source def to include PATH
SELECT
ae.r
||jsonb_build_object(
'path',
(
'{'||(ae.r->>'column_name')||'}'
)
)
FROM
tps.srce
JOIN LATERAL jsonb_array_elements(defn->'schemas'->'default') ae(r) ON TRUE
WHERE
srce = 'WMPD'

148
database/deploy/reload/wmpd/srce.json Normal file
View File

@ -0,0 +1,148 @@
{
"name": "WMPD",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Pay Dt}",
"{Carrier}"
],
"schemas": {
"default": [
{
"path": "{Carrier}",
"type": "text",
"column_name": "Carrier"
},
{
"path": "{SCAC}",
"type": "text",
"column_name": "SCAC"
},
{
"path": "{Mode}",
"type": "text",
"column_name": "Mode"
},
{
"path": "{Pro #}",
"type": "text",
"column_name": "Pro #"
},
{
"path": "{B/L}",
"type": "text",
"column_name": "B/L"
},
{
"path": "{Pd Amt}",
"type": "numeric",
"column_name": "Pd Amt"
},
{
"path": "{Loc#}",
"type": "text",
"column_name": "Loc#"
},
{
"path": "{Pcs}",
"type": "numeric",
"column_name": "Pcs"
},
{
"path": "{Wgt}",
"type": "numeric",
"column_name": "Wgt"
},
{
"path": "{Chk#}",
"type": "numeric",
"column_name": "Chk#"
},
{
"path": "{Pay Dt}",
"type": "date",
"column_name": "Pay Dt"
},
{
"path": "{Acct #}",
"type": "text",
"column_name": "Acct #"
},
{
"path": "{I/O}",
"type": "text",
"column_name": "I/O"
},
{
"path": "{Sh Nm}",
"type": "text",
"column_name": "Sh Nm"
},
{
"path": "{Sh City}",
"type": "text",
"column_name": "Sh City"
},
{
"path": "{Sh St}",
"type": "text",
"column_name": "Sh St"
},
{
"path": "{Sh Zip}",
"type": "text",
"column_name": "Sh Zip"
},
{
"path": "{Cons Nm}",
"type": "text",
"column_name": "Cons Nm"
},
{
"path": "{D City }",
"type": "text",
"column_name": "D City "
},
{
"path": "{D St}",
"type": "text",
"column_name": "D St"
},
{
"path": "{D Zip}",
"type": "text",
"column_name": "D Zip"
},
{
"path": "{Sh Dt}",
"type": "date",
"column_name": "Sh Dt"
},
{
"path": "{Inv Dt}",
"type": "date",
"column_name": "Inv Dt"
},
{
"path": "{Customs Entry#}",
"type": "text",
"column_name": "Customs Entry#"
},
{
"path": "{Miles}",
"type": "numeric",
"column_name": "Miles"
},
{
"path": "{Frt Class}",
"type": "text",
"column_name": "Frt Class"
},
{
"path": "{Master B/L}",
"type": "text",
"column_name": "Master B/L"
}
]
}
}

2150
database/deploy/setup.sql Normal file
View File

File diff suppressed because one or more lines are too long

257
database/interface/import/map_trigger.sql Normal file
View File

@ -0,0 +1,257 @@
CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER
AS
$f$
DECLARE
_cnt INTEGER;
BEGIN
IF (TG_OP = 'INSERT') THEN
WITH
--------------------apply regex operations to transactions-----------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN new_table t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT count(*) FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,target
,seq
,map_intention
ORDER BY
id
)
--SELECT * FROM agg_to_target
, link_map AS (
SELECT
a.srce
,a.id
,a.target
,a.seq
,a.map_intention
,a.map_val
,a.retain_val retain_value
,v.map
FROM
agg_to_target a
LEFT OUTER JOIN tps.map_rv v ON
v.srce = a.srce AND
v.target = a.target AND
v.retval = a.map_val
)
--SELECT * FROM link_map
, agg_to_id AS (
SELECT
srce
,id
,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val
,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map
FROM
link_map
GROUP BY
srce
,id
)
--SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100
--create a complete list of all new inserts assuming some do not have maps (left join)
,join_all AS (
SELECT
n.srce
,n.id
,n.rec
,a.retain_val parse
,a.map
,n.rec||COALESCE(a.map||a.retain_val,'{}'::jsonb) allj
FROM
new_table n
LEFT OUTER JOIN agg_to_id a ON
a.id = n.id
)
--update trans with join_all recs
UPDATE
tps.trans t
SET
parse = a.parse
,map = a.map
,allj = a.allj
FROM
join_all a
WHERE
t.id = a.id;
END IF;
RETURN NULL;
END;
$f$ LANGUAGE plpgsql;
CREATE TRIGGER trans_insert
AFTER INSERT ON tps.trans
REFERENCING NEW TABLE AS new_table
FOR EACH STATEMENT EXECUTE PROCEDURE tps.trans_insert_map();

215
database/interface/import/srce_import.sql Normal file
View File

File diff suppressed because one or more lines are too long

184
database/interface/import/srce_import_do.sql Normal file
View File

@ -0,0 +1,184 @@
DO
$f$
DECLARE
_t text;
_c text;
_log_info jsonb;
_log_id text;
_cnt numeric;
_message jsonb;
_recs jsonb;
_srce text;
_defn jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
_srce := 'DMAPI';
_recs:= $${"id":1,"doc":{"rows":[{"elements":[{"status":"OK","distance":{"text":"225 mi","value":361940},"duration":{"text":"3 hours 50 mins","value":13812}}]}],"status":"OK","origin_addresses":["Washington, DC, USA"],"destination_addresses":["New York, NY, USA"]}}$$::jsonb;
----------------------------------------------------test if source exists----------------------------------------------------------------------------------
SELECT
defn
INTO
_defn
FROM
tps.srce
WHERE
srce = _srce;
IF _defn IS NULL THEN
_message:=
format(
$$
{
"status":"fail",
"message":"source %L does not exists"
}
$$,
_srce
)::jsonb;
RAISE NOTICE '%s', _message;
END IF;
-------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record-----------------------------------------------------------------------------------
WITH
pending_list AS (
SELECT
_srce srce
,j.rec
,j.id
--aggregate back to the record since multiple paths may be listed in the constraint
--it is unclear why the "->>0" is required to correctly extract the text array from the jsonb
,tps.jsonb_concat_obj(
jsonb_build_object(
--the new json key is the path itself
cons.path->>0
,j.rec#>((cons.path->>0)::text[])
)
) json_key
FROM
jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id)
JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE
GROUP BY
j.rec
,j.id
)
-----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
, pending_keys AS (
SELECT DISTINCT
json_key
FROM
pending_list
)
-----------list of keys already loaded to tps-----------------------------------------------------------------------------------------------------
, matched_keys AS (
SELECT DISTINCT
k.json_key
FROM
pending_keys k
INNER JOIN tps.trans t ON
t.ic = k.json_key
)
-----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
, unmatched_keys AS (
SELECT
json_key
FROM
pending_keys
EXCEPT
SELECT
json_key
FROM
matched_keys
)
--------build log record-------------------+------------------------------------------------------------------------------------------------
, logged AS (
INSERT INTO
tps.trans_log (info)
SELECT
JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP)
||JSONB_BUILD_OBJECT('srce',_srce)
--||JSONB_BUILD_OBJECT('path',_path)
||JSONB_BUILD_OBJECT('not_inserted',
(
SELECT
jsonb_agg(json_key)
FROM
matched_keys
)
)
||JSONB_BUILD_OBJECT('inserted',
(
SELECT
jsonb_agg(json_key)
FROM
unmatched_keys
)
)
RETURNING *
)
-----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
--need to look into mapping the transactions prior to loading
, inserted AS (
INSERT INTO
tps.trans (srce, rec, ic, logid)
SELECT
pl.srce
,pl.rec
,pl.json_key
,logged.id
FROM
pending_list pl
INNER JOIN unmatched_keys u ON
u.json_key = pl.json_key
CROSS JOIN logged
ORDER BY
pl.id ASC
----this conflict is only if an exact duplicate rec json happens, which will be rejected
----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
RETURNING *
)
SELECT
id
,info
INTO
_log_id
,_log_info
FROM
logged;
--RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info;
_message:=
(
$$
{
"status":"complete"
}
$$::jsonb
)||jsonb_build_object('details',_log_info);
RAISE NOTICE '%s', _message;
END;
$f$
LANGUAGE plpgsql

110
database/interface/map_def/srce_map_def_set.sql Normal file
View File

@ -0,0 +1,110 @@
CREATE OR REPLACE FUNCTION tps.srce_map_def_set(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
WITH
------------------------------------------stage rows to insert-----------------------------------------------------
stg AS (
SELECT
--data source
ae.r->>'srce' srce
--map name
,ae.r->>'name' target
--map definition
,ae.r regex
--map aggregation sequence
,(ae.r->>'sequence')::INTEGER seq
--history definition
,jsonb_build_object(
'hist_defn',ae.r
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
) || '[]'::jsonb hist
--determine if the rows are new or match
,(m.regex->>'regex' = ae.r->>'regex')::BOOLEAN rebuild
FROM
jsonb_array_elements(_defn) ae(r)
LEFT OUTER JOIN tps.map_rm m ON
m.srce = ae.r->>'srce'
AND m.target = ae.t->>'name'
)
---------------------------------------do the upsert-------------------------------------------------------------------
,ins AS (
INSERT INTO
tps.map_rm (srce, target, regex, seq, hist)
SELECT
srce
,target
,regex
,seq
,hist
FROM
stg
ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET
srce = excluded.srce
,target = excluded.target
,regex = excluded.regex
,seq = excluded.seq
,hist =
--the new definition going to position -0-
jsonb_build_object(
'hist_defn',excluded.regex
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
)
--the previous definition, set upper bound of effective range which was previously null
|| jsonb_set(
map_rm.hist
,'{0,effective,1}'::text[]
,to_jsonb(CURRENT_TIMESTAMP)
)
)
---------------------------get list of sources that had maps change--------------------------------------------------------
, to_update AS (
SELECT DISTINCT
srce
FROM
ins
WHERE
rebuild = TRUE
)
--------------------------call the map overwrite for each source and return all the messages into message----------------
/*the whole source must be overwritten because if an element is no longer returned it shoudl be wiped from the data*/
SELECT
jsonb_agg(x.message)
INTO
_message
FROM
to_update
JOIN LATERAL tps.srce_map_overwrite(to_update.srce) AS x(message) ON TRUE;
_message:= jsonb_build_object('status','complete','message','definition has been set');
return _message;
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error setting definition"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
return _message;
END;
$f$
language plpgsql

98
database/interface/map_def/srce_map_def_set_single.sql Normal file
View File

@ -0,0 +1,98 @@
CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb, _rebuild BOOLEAN) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
---------test if anythign is changing--------------------------------------------------------------------------------------------
IF _defn->'regex' = (SELECT regex->'regex' FROM tps.map_rm WHERE srce = _defn->>'srce' and target = _defn->>'name') THEN
_message:=
(
$$
{
"status":"complete",
"message":"map was not different no action taken"
}
$$::jsonb
);
RETURN _message;
END IF;
---------do the rebuild-----------------------------------------------------------------------------------------------------------
INSERT INTO
tps.map_rm (srce, target, regex, seq, hist)
SELECT
--data source
_defn->>'srce'
--map name
,_defn->>'name'
--map definition
,_defn
--map aggregation sequence
,(_defn->>'sequence')::INTEGER
--history definition
,jsonb_build_object(
'hist_defn',_defn
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
) || '[]'::jsonb
ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET
srce = excluded.srce
,target = excluded.target
,regex = excluded.regex
,seq = excluded.seq
,hist =
--the new definition going to position -0-
jsonb_build_object(
'hist_defn',excluded.regex
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
)
--the previous definition, set upper bound of effective range which was previously null
|| jsonb_set(
map_rm.hist
,'{0,effective,1}'::text[]
,to_jsonb(CURRENT_TIMESTAMP)
);
--------------if rebuild was flag call the rebuild--------------------------------------------------------------------------------
IF _rebuild THEN
SELECT
x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb
INTO
_message
FROM
tps.srce_map_overwrite(_defn->>'srce') as X(message);
END IF;
return _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error setting definition"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
return _message;
END;
$f$
language plpgsql

222
database/interface/map_def/test_regex.sql Normal file
View File

@ -0,0 +1,222 @@
DROP FUNCTION IF EXISTS tps.test_regex(jsonb);
CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb
LANGUAGE plpgsql
AS
$f$
DECLARE
_rslt jsonb;
BEGIN
WITH
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT * FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,target
,seq
,map_intention
)
, agg_to_ret AS (
SELECT
srce
,target
,seq
,map_intention
,map_val
,retain_val
,count(*) "count"
FROM
agg_to_target
GROUP BY
srce
,target
,seq
,map_intention
,map_val
,retain_val
)
,agg_to_id AS (
SELECT
l.srce
,l.target
,l.map_val
,l."count"
FROM
agg_to_ret l
ORDER BY
l.srce
,l.target
,l."count" desc
)
SELECT
jsonb_agg(row_to_json(agg_to_id)::jsonb)
INTO
_rslt
FROM
agg_to_id;
RETURN _rslt;
END;
$f$;

211
database/interface/map_def/test_regex_recs.sql Normal file
View File

@ -0,0 +1,211 @@
DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb);
CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb
LANGUAGE plpgsql
AS
$f$
DECLARE
_rslt jsonb;
BEGIN
WITH
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
(SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT * FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,rec
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,rec
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,rec
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,rec
,target
,seq
,map_intention
)
, agg_to_ret AS (
SELECT
srce
,target
,seq
,map_intention
,map_val
,retain_val
,count(*) "count"
,jsonb_agg(rec) rec
FROM
agg_to_target
GROUP BY
srce
,target
,seq
,map_intention
,map_val
,retain_val
)
SELECT
jsonb_agg(row_to_json(agg_to_ret)::jsonb)
INTO
_rslt
FROM
agg_to_ret;
RETURN _rslt;
END;
$f$;

64
database/interface/map_values/map_rv_set.sql Normal file
View File

@ -0,0 +1,64 @@
DROP FUNCTION IF EXISTS tps.map_rv_set;
CREATE OR REPLACE FUNCTION tps.map_rv_set(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
INSERT INTO
tps.map_rv (srce, target, retval, map, hist)
SELECT
r.source
,r.map
,r.ret_val
,r.mapped
,jsonb_build_object(
'hist_defn',mapped
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
) || '[]'::jsonb
FROM
JSONB_ARRAY_ELEMENTS(_defn) WITH ORDINALITY ae(r,s)
JOIN LATERAL jsonb_to_record(ae.r) r(source TEXT,map TEXT, ret_val jsonb, mapped jsonb) ON TRUE
ON CONFLICT ON CONSTRAINT map_rv_pk DO UPDATE
SET
map = excluded.map
,hist =
--the new definition going to position -0-
jsonb_build_object(
'hist_defn',excluded.map
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
)
--the previous definition, set upper bound of effective range which was previously null
|| jsonb_set(
map_rv.hist
,'{0,effective,1}'::text[]
,to_jsonb(CURRENT_TIMESTAMP)
);
-------return message--------------------------------------------------------------------------------------------------
_message:= jsonb_build_object('status','complete');
RETURN _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error setting map value"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql;

249
database/interface/map_values/report_unmapped.sql Normal file
View File

@ -0,0 +1,249 @@
DROP FUNCTION IF EXISTS tps.report_unmapped;
CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE
(
source text,
map text,
ret_val jsonb,
"count" bigint
)
LANGUAGE plpgsql
AS
$f$
BEGIN
/*
first get distinct target json values
then apply regex
*/
RETURN QUERY
WITH
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
WHERE
--t.allj IS NULL
t.srce = _srce AND
e.v @> '{"map":"y"}'::jsonb
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT * FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,target
,seq
,map_intention
)
, agg_to_ret AS (
SELECT
srce
,target
,seq
,map_intention
,map_val
,retain_val
,count(*) "count"
FROM
agg_to_target
GROUP BY
srce
,target
,seq
,map_intention
,map_val
,retain_val
)
, link_map AS (
SELECT
a.srce
,a.target
,a.seq
,a.map_intention
,a.map_val
,a."count"
,a.retain_val
,v.map mapped_val
FROM
agg_to_ret a
LEFT OUTER JOIN tps.map_rv v ON
v.srce = a.srce AND
v.target = a.target AND
v.retval = a.map_val
)
SELECT
l.srce
,l.target
,l.map_val
,l."count"
FROM
link_map l
WHERE
l.mapped_val IS NULL
ORDER BY
l.srce
,l.target
,l."count" desc;
END;
$f$

257
database/interface/map_values/report_unmapped_recs.sql Normal file
View File

@ -0,0 +1,257 @@
DROP FUNCTION IF EXISTS tps.report_unmapped_recs;
CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE
(
source text,
map text,
ret_val jsonb,
"count" bigint,
recs jsonb
)
LANGUAGE plpgsql
AS
$f$
BEGIN
/*
first get distinct target json values
then apply regex
*/
RETURN QUERY
WITH
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
WHERE
--t.allj IS NULL
t.srce = _srce AND
e.v @> '{"map":"y"}'::jsonb
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT * FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,rec
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,rec
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,rec
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,rec
,target
,seq
,map_intention
)
, agg_to_ret AS (
SELECT
srce
,target
,seq
,map_intention
,map_val
,retain_val
,count(*) "count"
,jsonb_agg(rec) rec
FROM
agg_to_target
GROUP BY
srce
,target
,seq
,map_intention
,map_val
,retain_val
)
, link_map AS (
SELECT
a.srce
,a.target
,a.seq
,a.map_intention
,a.map_val
,a."count"
,a.rec
,a.retain_val
,v.map mapped_val
FROM
agg_to_ret a
LEFT OUTER JOIN tps.map_rv v ON
v.srce = a.srce AND
v.target = a.target AND
v.retval = a.map_val
)
SELECT
l.srce
,l.target
,l.map_val
,l."count"
,l.rec
FROM
link_map l
WHERE
l.mapped_val IS NULL
ORDER BY
l.srce
,l.target
,l."count" desc;
END;
$f$

261
database/interface/map_values/srce_map_overwrite.sql Normal file
View File

@ -0,0 +1,261 @@
CREATE OR REPLACE FUNCTION tps.srce_map_overwrite(_srce text) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
BEGIN
WITH
--------------------apply regex operations to transactions-----------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->'regex'->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->'regex'->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->'regex'->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->'regex'->>'function' = 'replace'
WHERE
--t.allj IS NULL
t.srce = _srce
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
)
--SELECT count(*) FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,target
,seq
,map_intention
ORDER BY
id
)
--SELECT * FROM agg_to_target
, link_map AS (
SELECT
a.srce
,a.id
,a.target
,a.seq
,a.map_intention
,a.map_val
,a.retain_val retain_value
,v.map
FROM
agg_to_target a
LEFT OUTER JOIN tps.map_rv v ON
v.srce = a.srce AND
v.target = a.target AND
v.retval = a.map_val
)
--SELECT * FROM link_map
, agg_to_id AS (
SELECT
srce
,id
,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val
,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map
FROM
link_map
GROUP BY
srce
,id
)
--SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100
UPDATE
tps.trans t
SET
map = o.map,
parse = o.retain_val,
allj = t.rec||o.map||o.retain_val
FROM
agg_to_id o
WHERE
o.id = t.id;
_message:= jsonb_build_object('status','complete');
RETURN _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error setting map value"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
language plpgsql

33
database/interface/source_maint/srce_build_view.sql Normal file
View File

@ -0,0 +1,33 @@
DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text);
CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT
AS
$f$
DECLARE
--_schema text;
--_srce text;
_sql text;
BEGIN
--_schema:= 'default';
--_srce:= 'dcard';
SELECT
'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';'
INTO
_sql
FROM
tps.srce s
JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE
JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE
JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE
WHERE
srce = _srce
AND list.e->>'name' = _schema
GROUP BY
s.srce
,list.e;
RETURN _sql;
RAISE NOTICE '%',_sql;
END
$f$
LANGUAGE plpgsql;

52
database/interface/source_maint/srce_delete.sql Normal file
View File

@ -0,0 +1,52 @@
--setup function to delete a single source
DROP FUNCTION IF EXISTS tps.srce_delete(jsonb);
CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
_rebuild BOOLEAN;
BEGIN
-------------------------------do delete---------------------------------
DELETE FROM tps.srce WHERE srce = _defn->>'name';
--could move this record to a "recycle bin" table for a certain period of time
--need to handle cascading record deletes
---------------------------set message-----------------------------------
_message:=
(
$$
{
"status":"complete",
"message":"source was permanently deleted"
}
$$::jsonb
);
RETURN _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error dropping the source"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql

108
database/interface/source_maint/srce_overwrite_all.sql Normal file
View File

@ -0,0 +1,108 @@
/*
This function takes and array of definition object where "name" object is the primary key
It will force the entire body of sources to match what is received
*/
DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb);
CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
_rebuild BOOLEAN;
_list text;
BEGIN
WITH
--retain the results of the update by srce
_set AS (
SELECT
j.rn rn
,j.e->>'name' srce
,j.e defn
FROM
jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn)
)
--full join
,_full AS (
SELECT
COALESCE(_srce.srce,_set.srce) srce
,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn
,COALESCE(_set.defn,_srce.defn) defn
FROM
tps.srce _srce
FULL OUTER JOIN _set ON
_set.srce = _srce.srce
)
--call functions from list
,_do_set AS (
SELECT
f.srce
,f.actn
,setd.message
FROM
_full f
JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET'
--dual left joins for functions that touch the same table causes the first left join actions to be undone
--LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
)
,_do_del AS (
SELECT
f.srce
,f.actn
,deld.message
FROM
_full f
JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
)
--aggregate all the messages into one message
----
---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this.
----
SELECT
jsonb_agg(m)
INTO
_message
FROM
(
SELECT
jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
FROM
_do_set
UNION ALL
SELECT
jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
FROM
_do_del
) x;
SELECT string_agg(srce,',') INTO _list FROM tps.srce;
RAISE NOTICE 'multi source list: %', _list;
RETURN _message;
SELECT string_agg(srce,',') INTO _list FROM tps.srce;
RAISE NOTICE 'after return: %', _list;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error updating sources"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql

144
database/interface/source_maint/srce_set.sql Normal file
View File

@ -0,0 +1,144 @@
DROP FUNCTION IF EXISTS tps.srce_set(jsonb);
CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
_rebuild BOOLEAN;
BEGIN
---------test if anythign is changing--------------------------------------------------------------------------------------------
IF _defn = (SELECT defn FROM tps.srce WHERE srce = _defn->>'name') THEN
_message:=
(
$$
{
"status":"complete",
"message":"source was not different no action taken"
}
$$::jsonb
);
RETURN _message;
END IF;
---------if the constraint definition is changing, rebuild for existing records---------------------------------------------------
SELECT
NOT (_defn->'constraint' = (SELECT defn->'constraint' FROM tps.srce WHERE srce = _defn->>'name'))
INTO
_rebuild;
RAISE NOTICE '%',_rebuild::text;
---------do merge-----------------------------------------------------------------------------------------------------------------
INSERT INTO
tps.srce (srce, defn, hist)
SELECT
--extract name from defintion
_defn->>'name'
--add current timestamp to defintions
,_defn
--add definition
,jsonb_build_object(
'hist_defn',_defn
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
) || '[]'::jsonb
ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE
SET
defn = _defn
,hist =
--the new definition going to position -0-
jsonb_build_object(
'hist_defn',_defn
,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
)
--the previous definition, set upper bound of effective range which was previously null
|| jsonb_set(
srce.hist
,'{0,effective,1}'::text[]
,to_jsonb(CURRENT_TIMESTAMP)
);
--rebuild constraint key if necessary---------------------------------------------------------------------------------------
IF _rebuild THEN
WITH
rebuild AS (
SELECT
j.srce
,j.rec
,j.id
--aggregate back to the record since multiple paths may be listed in the constraint
,tps.jsonb_concat_obj(
jsonb_build_object(
--the new json key is the path itself
cons.path->>0
,j.rec#>((cons.path->>0)::text[])
)
) json_key
FROM
tps.trans j
INNER JOIN tps.srce s ON
s.srce = j.srce
JOIN LATERAL jsonb_array_elements(s.defn->'constraint') WITH ORDINALITY cons(path, seq) ON TRUE
WHERE
s.srce = _defn->>'name'
GROUP BY
j.rec
,j.id
)
UPDATE
tps.trans t
SET
ic = r.json_key
FROM
rebuild r
WHERE
t.id = r.id;
_message:=
(
$$
{
"status":"complete",
"message":"source set and constraint rebuilt on existing records"
}
$$::jsonb
);
ELSE
_message:=
(
$$
{
"status":"complete",
"message":"source set"
}
$$::jsonb
);
END IF;
RETURN _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error importing data"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql

128
database/readme.md Normal file
View File

@ -0,0 +1,128 @@
Generic Data Transformation Tool
=======================================================
The goal is to:
1. house external data and prevent duplication on insert
2. facilitate regular exression operations to extract meaningful data
3. be able to reference it from outside sources (no action required) and maintain reference to original data
It is well suited for data from outside systems that
* requires complex transformation (parsing and mapping)
* original data is retained for reference
* don't feel like writing a map-reduce
use cases:
* on-going bank feeds
* jumbled product lists
* storing api results
The data is converted to json by the importing program and inserted to the database.
Regex expressions are applied to specified json components and the results can be mapped to other values.
Major Interactions
------------------------
* Source Definitions (Maint/Inquire)
* Regex Instructions (Maint/Inquire)
* Cross Reference List (Maint/Inquire)
* Run Import (Run Job)
### Interaction Details
* _Source Definitions (Maint/Inquire)_
* display a list of existing sources with display detials/edit options
* create new option
* underlying function is `tps.srce_set(_name text, _defn jsonb)`
* the current definition of a source includes data based on bad presumptions:
* how to load from a csv file using `COPY`
* setup a Postgres type to reflect the associated columns (if applicable)
* _Regex Instructions (Maint/Inquire)_
* display a list of existing instruction sets with display details/edit options
* create new option
* underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json
* _Cross Reference List (Maint/Inquire)_
* first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)`
* the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs
* function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)`
* _Run Import_
* underlying function is `tps.srce_import(_path text, _srce text)`
source definition
----------------------------------------------------------------------
* **load data**
* the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion`
* the backend builds a json array of all the rows to be added and sends as an argument to a database insert function
* build constraint key `based on srce definition`
* handle violations
* increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed)
* build an import log
* run maps (as opposed to relying on trigger)
* **read data**
* the `schema` key contains either a text element or a text array in curly braces
* forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record`
* it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table
* top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema`
* custom function parsing contents based on #> operator and extracting from `srce.defn->schema`
* view that `uses the source definiton` to extrapolate a table?
* a materialized table is built `based on the source definition` and any addtional regex?
* add regex = alter table add column with historic updates?
* no primary key?
* every document must work out to one row
```
{
"name":"dcard",
"source":"client_file",
"loading_function":"csv"
"constraint":[
"{Trans. Date}",
"{Post Date}"
],
"schemas":{
"default":[
{
"path":"{doc,origin_addresses,0}",
"type":"text",
"column_name":"origin_address"
},
{
"path":"{doc,destination_addresses,0}",
"type":"text",
"column_name":"origin_address"
},
{
"path":"{doc,status}",
"type":"text",
"column_name":"status"
}
{
"path":"{doc,rows,0,elements,0,distance,value}",
"type":"numeric",
"column_name":"distance"
}
{
"path":"{doc,rows,0,elements,0,duration,value}",
"type":"numeric",
"column_name":"duration"
}
],
"version2":[]
}
}
```

245
database/reports/all_map_return_values.sql Normal file
View File

@ -0,0 +1,245 @@
/*
first get distinct target json values
then apply regex
*/
WITH
--------------------apply regex operations to transactions---------------------------------------------------------------------------------
rx AS (
SELECT
t.srce,
t.id,
t.rec,
m.target,
m.seq,
regex->>'function' regex_function,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number,
COALESCE(mt.rn,rp.rn,1) result_number,
mt.mt rx_match,
rp.rp rx_replace,
--------------------------json key name assigned to return value-----------------------------------------------------------------------
CASE e.v->>'map'
WHEN 'y' THEN
e.v->>'field'
ELSE
null
END map_key,
--------------------------json value resulting from regular expression-----------------------------------------------------------------
CASE e.v->>'map'
WHEN 'y' THEN
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(mt.mt[1])
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rp.rp)
ELSE
'{}'::jsonb
END
ELSE
NULL
END map_val,
--------------------------flag for if retruned regex result is stored as a new part of the final json output---------------------------
CASE e.v->>'retain'
WHEN 'y' THEN
e.v->>'field'
ELSE
NULL
END retain_key,
--------------------------push regex result into json object---------------------------------------------------------------------------
CASE e.v->>'retain'
WHEN 'y' THEN
CASE regex->>'function'
WHEN 'extract' THEN
CASE WHEN array_upper(mt.mt,1)=1
THEN to_json(trim(mt.mt[1]))
ELSE array_to_json(mt.mt)
END::jsonb
WHEN 'replace' THEN
to_jsonb(rtrim(rp.rp))
ELSE
'{}'::jsonb
END
ELSE
NULL
END retain_val
FROM
--------------------------start with all regex maps------------------------------------------------------------------------------------
tps.map_rm m
--------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE
--------------------------break out array of regluar expressions in the map------------------------------------------------------------
JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
--------------------------join to main transaction table but only certain key/values are included--------------------------------------
INNER JOIN tps.trans t ON
t.srce = m.srce AND
t.rec @> w.v
--------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
m.regex->>'function' = 'extract'
--------------------------same as above but for a replacement type function------------------------------------------------------------
LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
m.regex->>'function' = 'replace'
WHERE
--t.allj IS NULL
--t.srce = 'PNCC' AND
e.v @> '{"map":"y"}'::jsonb
--rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
--rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
/*
ORDER BY
t.id DESC,
m.target,
e.rn,
COALESCE(mt.rn,rp.rn,1)
*/
)
--SELECT * FROM rx LIMIT 100
, agg_to_target_items AS (
SELECT
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,CASE WHEN map_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
map_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(map_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(map_val ORDER BY result_number)
END
)
END map_val
,CASE WHEN retain_key IS NULL
THEN
NULL
ELSE
jsonb_build_object(
retain_key,
CASE WHEN max(result_number) = 1
THEN
jsonb_agg(retain_val ORDER BY result_number) -> 0
ELSE
jsonb_agg(retain_val ORDER BY result_number)
END
)
END retain_val
FROM
rx
GROUP BY
srce
,id
,target
,seq
,map_intention
,regex_function
,target_item_number
,result_key_name
,target_json_path
,map_key
,retain_key
)
--SELECT * FROM agg_to_target_items LIMIT 100
, agg_to_target AS (
SELECT
srce
,id
,target
,seq
,map_intention
,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
FROM
agg_to_target_items
GROUP BY
srce
,id
,target
,seq
,map_intention
)
, agg_to_ret AS (
SELECT
srce
,target
,seq
,map_intention
,map_val
,retain_val
,count(*) "count"
FROM
agg_to_target
GROUP BY
srce
,target
,seq
,map_intention
,map_val
,retain_val
)
, link_map AS (
SELECT
a.srce
,a.target
,a.seq
,a.map_intention
,a.map_val
,a."count"
,a.retain_val
,v.map mapped_val
FROM
agg_to_ret a
LEFT OUTER JOIN tps.map_rv v ON
v.srce = a.srce AND
v.target = a.target AND
v.retval = a.map_val
)
SELECT
l.srce
,l.target
,l.seq
,l.map_intention
,l.map_val
,l."count"
,l.retain_val
,l.mapped_val
FROM
link_map l
ORDER BY
l.srce
,l.target
,l.seq
,l."count" desc
,l.map_val
,l.mapped_val

19
database/reports/colateral_balalance.sql Normal file
View File

@ -0,0 +1,19 @@
SELECT
r.*
,CASE "Schedule#"
WHEN '02IN Raw Material' THEN 13097563.42
WHEN '03IN Finished Goods' THEN 35790696.52
ELSE 0
END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (PARTITION BY "Schedule#" ORDER BY "Schedule#" ASC, "PostDate" ASC) running_bal
,(LEAST("CollateralBalance" - "Ineligible Amount","MaxEligible")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral
,(("CollateralBalance" - "Ineligible Amount")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral_nl
FROM
tpsv.pncl_default r
WHERE
"Schedule#" = '01AR'
--"Schedule#" = '02IN Raw Material'
--"Schedule#" = '03IN Finished Goods'
ORDER BY
"Schedule#" asc
,r."PostDate" asc
,id

8
database/reports/dcard_bal.sql Normal file
View File

@ -0,0 +1,8 @@
\timing
SELECT
r.*
,SUM(r."Amount") OVER (ORDER BY r."Post Date" asc , r."Description") + 1061.1 + 22.40 balance
FROM
tpsv.dcard_default r
ORDER BY
r."Post Date" asc

34
database/reports/key_list.sql Normal file
View File

@ -0,0 +1,34 @@
\timing
/*--------------------------------------------------
maintain statment level triggers to update a master log of keys
* table based listing
* composite type maintenance
potential updates sources/events
* tps.trans insert
* tps.trans re-map
--------------------------------------------------*/
WITH ok AS (
SELECT
srce,
ok.k,
jsonb_typeof(allj->ok.k) typeof,
COUNT(*)
FROM
tps.trans
JOIN LATERAL jsonb_object_keys(allj) ok(k) ON TRUE
GROUP BY
srce,
ok.k,
jsonb_typeof(allj->ok.k)
ORDER BY
srce
)
SELECT
srce
,k
,typeof
FROM
ok

19
database/reports/list_maps.sql Normal file
View File

@ -0,0 +1,19 @@
SELECT
m.srce,
m.target,
regex->>'function' regex_function,
regex->>'where' where_clause,
e.v ->> 'field' result_key_name,
e.v ->> 'key' target_json_path,
e.v ->> 'flag' regex_options_flag,
e.v->>'map' map_intention,
e.v->>'retain' retain_result,
e.v->>'regex' regex_expression,
e.rn target_item_number
FROM
tps.map_rm m
LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true
ORDER BY
m.srce,
m.target,
e.rn

12
database/reports/loan_balance.sql Normal file
View File

@ -0,0 +1,12 @@
\timing
SELECT
r.*,
SUM(r."Advances"+r."Adjustments"-r."Payments") OVER (PARTITION BY "Loan#" ORDER BY r."Post Date" asc, r."Reference #" asc)
FROM
tpsv.pnco_default r
WHERE
"Loan#" = '606780191'
ORDER BY
r."Loan#"
,r."Post Date" ASC
,r."Reference #" ASC

25
database/templates/regex.json Normal file
View File

@ -0,0 +1,25 @@
[
{
"regex": {
"function": "extract",
"description": "pull first 20 characters from description for mapping",
"where": [
{}
],
"defn": [
{
"regex": ".{1,20}",
"map": "y",
"field": "f20",
"flag": "",
"key": "{Description}",
"retain": "y"
}
],
"name": "First 20"
},
"sequence": 2,
"name": "First 20",
"srce": "dcard"
}
]

82
database/templates/srce.json Normal file
View File

@ -0,0 +1,82 @@
{
"name": "dcard",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Trans. Date}",
"{Post Date}",
"{Description}"
],
"schemas": [
{
"name": "default",
"columns": [
{
"path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
}
]
},
{
"name": "mapped",
"columns": [
{
"path": "{Trans. Date}",
"type": "date",
"column_name": "Trans. Date"
},
{
"path": "{Post Date}",
"type": "date",
"column_name": "Post Date"
},
{
"path": "{Description}",
"type": "text",
"column_name": "Description"
},
{
"path": "{Amount}",
"type": "numeric",
"column_name": "Amount"
},
{
"path": "{Category}",
"type": "text",
"column_name": "Category"
},
{
"path": "{party}",
"type": "text",
"column_name": "Party"
},
{
"path": "{reason}",
"type": "text",
"column_name": "Reason"
}
]
}
]
}

24
database/templates/strip_commas.jsonc Normal file
View File

@ -0,0 +1,24 @@
{
"name": "Strip Amount Commas", //the name here currently also serves as the primary key in the database
"srce": "PNCC", //name of the target source
"sequence": 1 , //only for edge cases where the instructions returns two keys of the same name, this determines priority. pretty much 1.
"regex": { //instruction set
"where": [ //only apply this regex to these specified key value pairs, if none then use empty object {}
{
"example_key":"example_value"
}
],
"function": "replace", //even though there is an array of definitions they all have to operate under the same premise(extract or replace)
"defn": [ //there is an array of instructions
{
"key": "{Amount}", //key= the path to the json key/value pair to operate on. path woudl be a better term.
"map": "n", //y or n to indicate if the returned value will be used to search a lookup table
"flag": "g", //g indicates find all values, null or empty would be the other option I guess
"field": "amount", //the key name to give the value that comes out of this instruction
"regex": ",", //the reg expression itself
"retain": "y", //flag to indicate if the returned value should be retained and included with the data
"replace": "" //this key is only evaluated if the function is defined as replace
}
]
}
}

45
database/templates/transaction_type.jsonc Normal file
View File

@ -0,0 +1,45 @@
{
"name": "Trans Type",
"srce": "PNCC",
"regex": {
"function": "extract",
"defn": [
{
"key": "{AccountName}",
"map": "y",
"field": "acctn",
"regex": "(.*)",
"retain": "n"
},
{
"key": "{Transaction}",
"map": "y",
"field": "trans",
"regex": "(.*)",
"retain": "n"
},
{
"key": "{Description}",
"map": "y",
"field": "ini",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y"
}
],
"where": [
{}
]
},
"sequence": 1
}
/*
target | retval | map
------------+----------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------
Trans Type | {"ini": "01346", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Fees"} | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"}
Trans Type | {"ini": "CANADA TAX", "acctn": "The HC Operating Company OPERA", "trans": "Detail Debit Adjustments"} | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"}
Trans Type | {"ini": "ACH DEBIT SETTLEMENT", "acctn": "The HC Operating Company OPERA", "trans": "ACH Debits"} | {"sign": "-1", "ledger": "AP - ACH", "trantype": "Disbursement"}
Trans Type | {"ini": "RET DEP ITEM RTM", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"} | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit RTM", "trantype": "Collections"}
Trans Type | {"ini": "RET DEP ITEM STOP", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"} | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit STOP", "trantype": "Collections"}
Trans Type | {"ini": "CREDIT ADJUSTMENT", "acctn": "The HC Operating Company FBO P", "trans": "Detail Credit Adjustments"} | {"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}
*/

30
database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql Normal file
View File

@ -0,0 +1,30 @@
WITH
mod AS (
SELECT
srce
,jsonb_pretty(defn) orig
,(defn - 'schemas')||
--rebuild the schemas key value from below
jsonb_build_object(
'schemas'
--aggregate all the new key values for a single soure
,jsonb_agg(
--combine a new key 'name' with the columns for that name
jsonb_build_object('name',k)||jsonb_build_object('columns',v)
)
) rebuild
FROM
tps.srce
LEFT JOIN LATERAL jsonb_each(defn->'schemas') WITH ORDINALITY je(k,v, rn) ON TRUE
GROUP BY
srce
,defn
)
UPDATE
tps.srce s
SET
defn = rebuild
FROM
mod
WHERE
mod.srce = s.srce

33
database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql Normal file
View File

@ -0,0 +1,33 @@
DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text);
CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT
AS
$f$
DECLARE
--_schema text;
--_srce text;
_sql text;
BEGIN
--_schema:= 'default';
--_srce:= 'dcard';
SELECT
'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';'
INTO
_sql
FROM
tps.srce s
JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE
JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE
JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE
WHERE
srce = _srce
AND list.e->>'name' = _schema
GROUP BY
s.srce
,list.e;
RETURN _sql;
RAISE NOTICE '%',_sql;
END
$f$
LANGUAGE plpgsql;

52
database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql Normal file
View File

@ -0,0 +1,52 @@
--setup function to delete a single source
DROP FUNCTION IF EXISTS tps.srce_delete(jsonb);
CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
_rebuild BOOLEAN;
BEGIN
-------------------------------do delete---------------------------------
DELETE FROM tps.srce WHERE srce = _defn->>'name';
--could move this record to a "recycle bin" table for a certain period of time
--need to handle cascading record deletes
---------------------------set message-----------------------------------
_message:=
(
$$
{
"status":"complete",
"message":"source was permanently deleted"
}
$$::jsonb
);
RETURN _message;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error dropping the source"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql

108
database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql Normal file
View File

@ -0,0 +1,108 @@
/*
This function takes and array of definition object where "name" object is the primary key
It will force the entire body of sources to match what is received
*/
DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb);
CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb
AS
$f$
DECLARE
_message jsonb;
_MESSAGE_TEXT text;
_PG_EXCEPTION_DETAIL text;
_PG_EXCEPTION_HINT text;
_rebuild BOOLEAN;
_list text;
BEGIN
WITH
--retain the results of the update by srce
_set AS (
SELECT
j.rn rn
,j.e->>'name' srce
,j.e defn
FROM
jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn)
)
--full join
,_full AS (
SELECT
COALESCE(_srce.srce,_set.srce) srce
,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn
,COALESCE(_set.defn,_srce.defn) defn
FROM
tps.srce _srce
FULL OUTER JOIN _set ON
_set.srce = _srce.srce
)
--call functions from list
,_do_set AS (
SELECT
f.srce
,f.actn
,setd.message
FROM
_full f
JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET'
--dual left joins for functions that touch the same table causes the first left join actions to be undone
--LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
)
,_do_del AS (
SELECT
f.srce
,f.actn
,deld.message
FROM
_full f
JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
)
--aggregate all the messages into one message
----
---- should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this.
----
SELECT
jsonb_agg(m)
INTO
_message
FROM
(
SELECT
jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
FROM
_do_set
UNION ALL
SELECT
jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
FROM
_do_del
) x;
SELECT string_agg(srce,',') INTO _list FROM tps.srce;
RAISE NOTICE 'multi source list: %', _list;
RETURN _message;
SELECT string_agg(srce,',') INTO _list FROM tps.srce;
RAISE NOTICE 'after return: %', _list;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_MESSAGE_TEXT = MESSAGE_TEXT,
_PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
_PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
_message:=
($$
{
"status":"fail",
"message":"error updating sources"
}
$$::jsonb)
||jsonb_build_object('message_text',_MESSAGE_TEXT)
||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
RETURN _message;
END;
$f$
LANGUAGE plpgsql