diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 911ea4a..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/build_maps.xlsm b/database/build_maps.xlsm similarity index 100% rename from build_maps.xlsm rename to database/build_maps.xlsm diff --git a/deploy/reload/dcard/curl_dcard.cmd b/database/deploy/reload/dcard/curl_dcard.cmd similarity index 50% rename from deploy/reload/dcard/curl_dcard.cmd rename to database/deploy/reload/dcard/curl_dcard.cmd index 3e2d43c..8a77b83 100644 --- a/deploy/reload/dcard/curl_dcard.cmd +++ b/database/deploy/reload/dcard/curl_dcard.cmd @@ -1,4 +1,4 @@ -curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set -curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set -curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set -curl -v -F upload=@//mnt/c/Users/fleet/Downloads/dcard.csv http://localhost:81/import?srce=dcard \ No newline at end of file +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/source +curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/regex +curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapping +curl -v -F upload=@./d.csv http://localhost/import?srce=dcard \ No newline at end of file diff --git a/database/deploy/reload/dcard/d.csv b/database/deploy/reload/dcard/d.csv new file mode 100644 index 0000000..f0151f9 --- /dev/null +++ b/database/deploy/reload/dcard/d.csv @@ -0,0 +1,139 @@ +Trans. Date,Post Date,Description,Amount,Category +01/02/2018,01/02/2018,"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV",4.26,"Services" +01/02/2018,01/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +01/03/2018,01/03/2018,"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3",200.00,"Medical Services" +01/04/2018,01/04/2018,"AT&T *PAYMENT 800-288-2020 TX",57.14,"Services" +01/04/2018,01/07/2018,"WWW.KOHLS.COM #0873 MIDDLETOWN OH",-7.90,"Payments and Credits" +01/05/2018,01/07/2018,"PIZZA HUT 007946 STOW OH",9.24,"Restaurants" +01/05/2018,01/07/2018,"SUBWAY 00044289255 STOW OH",10.25,"Restaurants" +01/06/2018,01/07/2018,"ACME NO. 17 STOW OH",103.98,"Supermarkets" +01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",1.69,"Merchandise" +01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",2.19,"Merchandise" +01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00947R",3.94,"Gasoline" +01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00915R",52.99,"Gasoline" +01/13/2018,01/13/2018,"AUTOZONE #0722 STOW OH",85.36,"Automotive" +01/13/2018,01/13/2018,"DISCOUNT DRUG MART 32 STOW OH",26.68,"Merchandise" +01/13/2018,01/13/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +01/13/2018,01/13/2018,"TARGET STOW OH",197.90,"Merchandise" +01/14/2018,01/14/2018,"DISCOUNT DRUG MART 32 STOW OH",13.48,"Merchandise" +01/15/2018,01/15/2018,"TARGET.COM * 800-591-3869 MN",22.41,"Merchandise" +01/16/2018,01/16/2018,"BUFFALO WILD WINGS KENT KENT OH",63.22,"Restaurants" +01/16/2018,01/16/2018,"PARTA - KCG KENT OH",4.00,"Government Services" +01/16/2018,01/16/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +01/16/2018,01/16/2018,"TARGET.COM * 800-591-3869 MN",44.81,"Merchandise" +01/16/2018,01/16/2018,"TREE CITY COFFEE & PASTR KENT OH",17.75,"Restaurants" +01/17/2018,01/17/2018,"BESTBUYCOM805526794885 888-BESTBUY MN",343.72,"Merchandise" +01/19/2018,01/19/2018,"DISCOUNT DRUG MART 32 STOW OH",5.98,"Merchandise" +01/19/2018,01/19/2018,"U-HAUL OF KENT-STOW KENT OH",15.88,"Travel/ Entertainment" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",5.99,"Supermarkets" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",17.16,"Supermarkets" +01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",500.97,"Supermarkets" +01/20/2018,01/20/2018,"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W",2.12,"Services" +01/20/2018,01/20/2018,"LOWE'S OF STOW, OH. STOW OH",256.48,"Home Improvement" +01/22/2018,01/22/2018,"HOBBY LOBBY #405 STOW OHITEM TRANSFERRED FROM PREV ACCOUNT",38.49,"Merchandise" +01/23/2018,01/23/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-32.20,"Awards and Rebate Credits" +01/23/2018,01/23/2018,"INTERNET PAYMENT - THANK YOU",-2394.51,"Payments and Credits" +01/27/2018,01/27/2018,"GIANT-EAGLE #4096 STOW OH",67.81,"Supermarkets" +01/27/2018,01/27/2018,"OFFICEMAX/OFFICE DEPOT63 STOW OH",21.06,"Merchandise" +01/27/2018,01/27/2018,"TARGET STOW OH",71.00,"Merchandise" +01/29/2018,01/29/2018,"NETFLIX.COM NETFLIX.COM CA19899514437",14.93,"Services" +01/30/2018,01/30/2018,"PARTA - KCG KENT OH",1.00,"Government Services" +01/30/2018,01/30/2018,"SPEEDWAY 09303 KEN KENT OH",46.57,"Gasoline" +01/30/2018,01/30/2018,"SQ *TWISTED MELTZ KENT OH0002305843011416898511",16.87,"Restaurants" +01/30/2018,01/30/2018,"TARGET STOW OH",49.37,"Merchandise" +01/31/2018,01/31/2018,"TARGET STOW OH",4.14,"Merchandise" +01/31/2018,01/31/2018,"TARGET STREETSBORO OH",14.28,"Merchandise" +01/31/2018,02/01/2018,"TARGET STOW OH",-21.34,"Payments and Credits" +01/31/2018,02/01/2018,"TARGET STREETSBORO OH",-9.60,"Payments and Credits" +02/01/2018,02/01/2018,"EL CAMPESINO STOW OH",42.24,"Restaurants" +02/02/2018,02/02/2018,"CASH ADVANCE FEE",5.00,"Fees" +02/02/2018,02/02/2018,"GOOGLE *ASCIIFLOW.COM G.CO/HELPPAY#CAP0HQTYN5",5.00,"Cash Advances" +02/03/2018,02/03/2018,"TARGET STREETSBORO OH",71.69,"Merchandise" +02/03/2018,02/07/2018,"SAMS CLUB - #4750 CUYAHOGA FALLOH",371.90,"Warehouse Clubs" +02/04/2018,02/04/2018,"ACME NO. 17 STOW OH",8.98,"Supermarkets" +02/04/2018,02/04/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +02/06/2018,02/06/2018,"MINIMUM INTEREST CHARGE FEE",0.50,"Fees" +02/06/2018,02/07/2018,"BP#954778736210 7-ELEVEN STOW OH",52.80,"Gasoline" +02/06/2018,02/07/2018,"CVS/PHARMACY #08932 TWINSBURG OH",13.87,"Merchandise" +02/07/2018,02/07/2018,"AT&T *PAYMENT 800-288-2020 TXX51Z5QX7SMT2U01",57.14,"Services" +02/07/2018,02/07/2018,"TOYS R US #9203 CUYAHOGA FALLOH",193.32,"Merchandise" +02/08/2018,02/08/2018,"GIANT-EAGLE #4096 STOW OH",66.13,"Supermarkets" +02/08/2018,02/08/2018,"TARGET STOW OH",121.32,"Merchandise" +02/09/2018,02/09/2018,"GUIDOS ORIGINAL PIZZA KENT OH",11.75,"Restaurants" +02/09/2018,02/09/2018,"MARATHON PETRO73601 TWINSBURG OH",44.30,"Gasoline" +02/10/2018,02/10/2018,"RSVP NO. 36 STOW OH",14.43,"Supermarkets" +02/10/2018,02/10/2018,"TARGET STOW OH",77.90,"Merchandise" +02/11/2018,02/11/2018,"SUBWAY 00044289255 STOW OH",21.00,"Restaurants" +02/13/2018,02/13/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",12.79,"Restaurants" +02/13/2018,02/13/2018,"IN *MR. BULKY'S FOODS AKRON OHAJ16V8Q6",3.39,"Supermarkets" +02/13/2018,02/13/2018,"TARGET CUYAHOGA FALLOH",5.33,"Supermarkets" +02/14/2018,02/14/2018,"DISCOUNT DRUG MART 32 STOW OH",4.29,"Merchandise" +02/14/2018,02/14/2018,"HANDELS ICE CREAM STOW STOW OH",7.95,"Supermarkets" +02/15/2018,02/15/2018,"BATH&BODY STOW OH",47.19,"Merchandise" +02/15/2018,02/15/2018,"TARGET STOW OH",76.35,"Merchandise" +02/17/2018,02/17/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +02/17/2018,02/17/2018,"WALMART GROCERY 800-966-6546 AR",461.36,"Supermarkets" +02/18/2018,02/18/2018,"ACME NO. 17 STOW OH",32.68,"Supermarkets" +02/18/2018,02/18/2018,"CHIPOTLE ONLINE 303-595-4000 CO",20.75,"Restaurants" +02/19/2018,02/19/2018,"GIANT EAGLE #5863 STREETSBORO OH",25.00,"Supermarkets" +02/20/2018,02/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +02/21/2018,02/21/2018,"BP#954635936241 7-ELEVEN STOW OH",30.04,"Gasoline" +02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",3.19,"Restaurants" +02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",18.22,"Restaurants" +02/22/2018,02/22/2018,"PET SUPPLIES PLUS #68 STOW OH",45.88,"Merchandise" +02/22/2018,02/22/2018,"TOYS R US #9203 CUYAHOGA FALLOH",21.31,"Merchandise" +02/23/2018,02/23/2018,"SUMMIT CO PARKING GAR AKRON OH",6.00,"Services" +02/24/2018,02/24/2018,"GET GO #3396 STOW OH",26.46,"Gasoline" +02/25/2018,02/25/2018,"DISCOUNT DRUG MART 32 STOW OH",19.70,"Merchandise" +02/25/2018,02/25/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +02/25/2018,02/25/2018,"SQ *CORNER CUP COFFEEH STOW OH0001152921507942036274",2.30,"Supermarkets" +02/25/2018,02/25/2018,"TARGET STOW OH",18.49,"Merchandise" +02/28/2018,02/28/2018,"NETFLIX.COM NETFLIX.COM CA20475539512",14.93,"Services" +03/01/2018,03/01/2018,"GIANT-EAGLE #4032 STOW OH",2.99,"Supermarkets" +03/01/2018,03/01/2018,"TARGET STOW OH",72.46,"Merchandise" +03/02/2018,03/02/2018,"LATE FEE",27.00,"Fees" +03/02/2018,03/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services" +03/02/2018,03/02/2018,"PIZZA HUT 007946 STOW OH",9.89,"Restaurants" +03/03/2018,03/03/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-23.28,"Awards and Rebate Credits" +03/03/2018,03/03/2018,"INTERNET PAYMENT - THANK YOU",-2451.43,"Payments and Credits" +03/03/2018,03/03/2018,"LOWE'S OF STOW, OH. STOW OH",6.93,"Home Improvement" +03/03/2018,03/07/2018,"WAL-MART SC - #2323 STOW OH",150.32,"Merchandise" +03/04/2018,03/04/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",13.66,"Merchandise" +03/06/2018,03/07/2018,"MFW BOOKS LLC 5732022000 MO",86.90,"Education" +03/06/2018,03/07/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",127.46,"Merchandise" +03/08/2018,03/08/2018,"ACME NO. 17 STOW OH",8.58,"Supermarkets" +03/08/2018,03/08/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",2.12,"Restaurants" +03/08/2018,03/08/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +03/08/2018,03/08/2018,"SPEEDWAY 03686 496 STOW OH",45.24,"Gasoline" +03/08/2018,03/08/2018,"SWENSONS STOW KENT STOW OH",8.30,"Restaurants" +03/08/2018,03/08/2018,"TOYS R US #9203 CUYAHOGA FALLOH",5.33,"Merchandise" +03/09/2018,03/09/2018,"SPEEDWAY 03686 496 STOW OH",48.29,"Gasoline" +03/10/2018,03/10/2018,"WALMART GROCERY 800-966-6546 AR",522.22,"Supermarkets" +03/11/2018,03/11/2018,"AT&T *PAYMENT 800-288-2020 TXQ8F55RY7SMT2N04",57.14,"Services" +03/11/2018,03/11/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011470140810",2.30,"Supermarkets" +03/12/2018,03/12/2018,"MICROSOFT *STORE 800-642-7676 WA",1.06,"Services" +03/15/2018,03/15/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011475075512",2.30,"Supermarkets" +03/16/2018,03/16/2018,"ACME NO. 17 STOW OH",15.85,"Supermarkets" +03/16/2018,03/16/2018,"CHIPOTLE 1152 STOW OH",3.85,"Restaurants" +03/16/2018,03/16/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants" +03/16/2018,03/16/2018,"PIZZA HUT 007946 STOW OH",13.98,"Restaurants" +03/17/2018,03/17/2018,"CHIPOTLE ONLINE 303-595-4000 CO",15.75,"Restaurants" +03/17/2018,03/17/2018,"DISCOUNT DRUG MART 32 STOW OH",9.89,"Merchandise" +03/17/2018,03/17/2018,"MFW BOOKS LLC 5732022000 MO",66.75,"Education" +03/18/2018,03/18/2018,"ACME NO. 17 STOW OH",27.78,"Supermarkets" +03/18/2018,03/18/2018,"GIANT-EAGLE #4032 STOW OH",28.34,"Supermarkets" +03/20/2018,03/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services" +03/20/2018,03/20/2018,"SONLIGHT CURRICULUM LTD 303-730-8193 CO",762.87,"Education" +03/21/2018,03/21/2018,"BP#954635936241 7-ELEVEN STOW OH",8.87,"Gasoline" +03/21/2018,03/21/2018,"DISCOUNT DRUG MART 32 STOW OH",18.07,"Merchandise" +03/21/2018,03/21/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011484061091",2.30,"Supermarkets" +03/21/2018,03/21/2018,"TARGET STOW OH",1.95,"Merchandise" +03/21/2018,03/21/2018,"TARGET STOW OH",224.85,"Merchandise" +03/22/2018,03/22/2018,"JUSTICE #0639 STOW OH",16.01,"Merchandise" +03/22/2018,03/22/2018,"SPEEDWAY 03686 496 STOW OH",32.54,"Gasoline" +03/22/2018,03/22/2018,"SQ *TWISTED MELTZ KENT OH0002305843011486528725",6.74,"Restaurants" +03/22/2018,03/22/2018,"TARGET STOW OH",6.60,"Merchandise" +03/25/2018,03/25/2018,"ACME NO. 17 STOW OH",95.42,"Supermarkets" +03/25/2018,03/25/2018,"ASIAN-GREEK CUISINES STOW OH",70.25,"Restaurants" +03/25/2018,03/25/2018,"MARATHON PETRO73601 TWINSBURG OH",11.09,"Gasoline" +03/25/2018,03/25/2018,"SPEEDWAY 09303 KEN KENT OH",53.28,"Gasoline" diff --git a/deploy/reload/dcard/extract.sql b/database/deploy/reload/dcard/extract.sql similarity index 100% rename from deploy/reload/dcard/extract.sql rename to database/deploy/reload/dcard/extract.sql diff --git a/deploy/reload/dcard/map.json b/database/deploy/reload/dcard/map.json similarity index 100% rename from deploy/reload/dcard/map.json rename to database/deploy/reload/dcard/map.json diff --git a/deploy/reload/dcard/srce.json b/database/deploy/reload/dcard/srce.json similarity index 100% rename from deploy/reload/dcard/srce.json rename to database/deploy/reload/dcard/srce.json diff --git a/deploy/reload/dcard/vals.json b/database/deploy/reload/dcard/vals.json similarity index 100% rename from deploy/reload/dcard/vals.json rename to database/deploy/reload/dcard/vals.json diff --git a/deploy/reload/hunt/curl_hunt.cmd b/database/deploy/reload/hunt/curl_hunt.cmd similarity index 100% rename from deploy/reload/hunt/curl_hunt.cmd rename to database/deploy/reload/hunt/curl_hunt.cmd diff --git a/deploy/reload/hunt/extract.sql b/database/deploy/reload/hunt/extract.sql similarity index 100% rename from deploy/reload/hunt/extract.sql rename to database/deploy/reload/hunt/extract.sql diff --git a/deploy/reload/hunt/map.json b/database/deploy/reload/hunt/map.json similarity index 100% rename from deploy/reload/hunt/map.json rename to database/deploy/reload/hunt/map.json diff --git a/deploy/reload/hunt/srce.json b/database/deploy/reload/hunt/srce.json similarity index 100% rename from deploy/reload/hunt/srce.json rename to database/deploy/reload/hunt/srce.json diff --git a/deploy/reload/paycom/extract.sql b/database/deploy/reload/paycom/extract.sql similarity index 100% rename from deploy/reload/paycom/extract.sql rename to database/deploy/reload/paycom/extract.sql diff --git a/deploy/reload/paycom/load.cmd b/database/deploy/reload/paycom/load.cmd similarity index 100% rename from deploy/reload/paycom/load.cmd rename to database/deploy/reload/paycom/load.cmd diff --git a/deploy/reload/paycom/map.json b/database/deploy/reload/paycom/map.json similarity index 100% rename from deploy/reload/paycom/map.json rename to database/deploy/reload/paycom/map.json diff --git a/deploy/reload/paycom/srce.json b/database/deploy/reload/paycom/srce.json similarity index 100% rename from deploy/reload/paycom/srce.json rename to database/deploy/reload/paycom/srce.json diff --git a/deploy/reload/paycom/vals.json b/database/deploy/reload/paycom/vals.json similarity index 100% rename from deploy/reload/paycom/vals.json rename to database/deploy/reload/paycom/vals.json diff --git a/deploy/reload/pncc/extract.sql b/database/deploy/reload/pncc/extract.sql similarity index 100% rename from deploy/reload/pncc/extract.sql rename to database/deploy/reload/pncc/extract.sql diff --git a/deploy/reload/pncc/load.cmd b/database/deploy/reload/pncc/load.cmd similarity index 100% rename from deploy/reload/pncc/load.cmd rename to database/deploy/reload/pncc/load.cmd diff --git a/deploy/reload/pncc/map.json b/database/deploy/reload/pncc/map.json similarity index 100% rename from deploy/reload/pncc/map.json rename to database/deploy/reload/pncc/map.json diff --git a/deploy/reload/pncc/srce.json b/database/deploy/reload/pncc/srce.json similarity index 100% rename from deploy/reload/pncc/srce.json rename to database/deploy/reload/pncc/srce.json diff --git a/deploy/reload/pncc/vals.json b/database/deploy/reload/pncc/vals.json similarity index 100% rename from deploy/reload/pncc/vals.json rename to database/deploy/reload/pncc/vals.json diff --git a/deploy/reload/pncl/extract.sql b/database/deploy/reload/pncl/extract.sql similarity index 100% rename from deploy/reload/pncl/extract.sql rename to database/deploy/reload/pncl/extract.sql diff --git a/deploy/reload/pncl/load.cmd b/database/deploy/reload/pncl/load.cmd similarity index 100% rename from deploy/reload/pncl/load.cmd rename to database/deploy/reload/pncl/load.cmd diff --git a/deploy/reload/pncl/srce.json b/database/deploy/reload/pncl/srce.json similarity index 100% rename from deploy/reload/pncl/srce.json rename to database/deploy/reload/pncl/srce.json diff --git a/deploy/reload/pnco/extract.sql b/database/deploy/reload/pnco/extract.sql similarity index 100% rename from deploy/reload/pnco/extract.sql rename to database/deploy/reload/pnco/extract.sql diff --git a/deploy/reload/pnco/load.cmd b/database/deploy/reload/pnco/load.cmd similarity index 100% rename from deploy/reload/pnco/load.cmd rename to database/deploy/reload/pnco/load.cmd diff --git a/deploy/reload/pnco/srce.json b/database/deploy/reload/pnco/srce.json similarity index 100% rename from deploy/reload/pnco/srce.json rename to database/deploy/reload/pnco/srce.json diff --git a/deploy/reload/wmpd/curl_hunt.cmd b/database/deploy/reload/wmpd/curl_hunt.cmd similarity index 100% rename from deploy/reload/wmpd/curl_hunt.cmd rename to database/deploy/reload/wmpd/curl_hunt.cmd diff --git a/deploy/reload/wmpd/extract.sql b/database/deploy/reload/wmpd/extract.sql similarity index 100% rename from deploy/reload/wmpd/extract.sql rename to database/deploy/reload/wmpd/extract.sql diff --git a/deploy/reload/wmpd/srce.json b/database/deploy/reload/wmpd/srce.json similarity index 100% rename from deploy/reload/wmpd/srce.json rename to database/deploy/reload/wmpd/srce.json diff --git a/deploy/setup.sql b/database/deploy/setup.sql similarity index 82% rename from deploy/setup.sql rename to database/deploy/setup.sql index 2cb3f5e..0493a4a 100644 --- a/deploy/setup.sql +++ b/database/deploy/setup.sql @@ -302,7 +302,7 @@ BEGIN RETURN _message; END; $f$ -LANGUAGE plpgsql +LANGUAGE plpgsql; -----generate sql to create select based on schema DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text); @@ -439,7 +439,7 @@ CREATE AGGREGATE tps.jsonb_concat_obj(jsonb) ( ); -DROP FUNCTION IF EXISTS tps.report_unmapped(text); +DROP FUNCTION IF EXISTS tps.report_unmapped; CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE ( source text, @@ -463,101 +463,95 @@ WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE --t.allj IS NULL t.srce = _srce AND e.v @> '{"map":"y"}'::jsonb --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 @@ -773,14 +767,20 @@ $f$ NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN new_table t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' ORDER BY @@ -1190,7 +1190,7 @@ BEGIN t.rec, m.target, m.seq, - regex->>'function' regex_function, + regex->'regex'->>'function' regex_function, e.v ->> 'field' result_key_name, e.v ->> 'key' target_json_path, e.v ->> 'flag' regex_options_flag, @@ -1209,7 +1209,7 @@ BEGIN END map_key, CASE e.v->>'map' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(mt.mt[1]) @@ -1231,7 +1231,7 @@ BEGIN END retain_key, CASE e.v->>'retain' WHEN 'y' THEN - CASE regex->>'function' + CASE regex->'regex'->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 THEN to_json(trim(mt.mt[1])) @@ -1246,16 +1246,22 @@ BEGIN NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v - LEFT JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + m.regex->'regex'->>'function' = 'replace' WHERE --t.allj IS NULL t.srce = _srce @@ -1510,88 +1516,89 @@ BEGIN --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 @@ -1720,102 +1727,121 @@ $f$; ------------------------------test regex with all original records-------------------------------------------------------------- -DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb); -CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb +DROP FUNCTION IF EXISTS tps.report_unmapped_recs; +CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint, + recs jsonb + +) LANGUAGE plpgsql AS -$f$ -DECLARE - _rslt jsonb; +$f$ BEGIN +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY WITH --------------------apply regex operations to transactions--------------------------------------------------------------------------------- rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) ) --SELECT * FROM rx LIMIT 100 @@ -1901,6 +1927,8 @@ GROUP BY ,seq ,map_intention ) + + , agg_to_ret AS ( SELECT srce @@ -1921,12 +1949,38 @@ GROUP BY ,map_val ,retain_val ) + +, link_map AS ( SELECT - jsonb_agg(row_to_json(agg_to_ret)::jsonb) -INTO - _rslt + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.rec + ,a.retain_val + ,v.map mapped_val FROM - agg_to_ret; -RETURN _rslt; + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + ,l.rec +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; END; -$f$; \ No newline at end of file +$f$ \ No newline at end of file diff --git a/interface/import/map_trigger.sql b/database/interface/import/map_trigger.sql similarity index 90% rename from interface/import/map_trigger.sql rename to database/interface/import/map_trigger.sql index 6002dc1..f9215e3 100644 --- a/interface/import/map_trigger.sql +++ b/database/interface/import/map_trigger.sql @@ -74,14 +74,20 @@ $f$ NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN new_table t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' ORDER BY diff --git a/interface/import/srce_import.sql b/database/interface/import/srce_import.sql similarity index 100% rename from interface/import/srce_import.sql rename to database/interface/import/srce_import.sql diff --git a/interface/import/srce_import_do.sql b/database/interface/import/srce_import_do.sql similarity index 100% rename from interface/import/srce_import_do.sql rename to database/interface/import/srce_import_do.sql diff --git a/interface/map_def/srce_map_def_set.sql b/database/interface/map_def/srce_map_def_set.sql similarity index 100% rename from interface/map_def/srce_map_def_set.sql rename to database/interface/map_def/srce_map_def_set.sql diff --git a/interface/map_def/srce_map_def_set_single.sql b/database/interface/map_def/srce_map_def_set_single.sql similarity index 100% rename from interface/map_def/srce_map_def_set_single.sql rename to database/interface/map_def/srce_map_def_set_single.sql diff --git a/database/interface/map_def/test_regex.sql b/database/interface/map_def/test_regex.sql new file mode 100644 index 0000000..d09b6e9 --- /dev/null +++ b/database/interface/map_def/test_regex.sql @@ -0,0 +1,222 @@ +DROP FUNCTION IF EXISTS tps.test_regex(jsonb); +CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb +LANGUAGE plpgsql +AS +$f$ +DECLARE + _rslt jsonb; +BEGIN + + WITH + + --------------------apply regex operations to transactions--------------------------------------------------------------------------------- + + rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) + ) + + --SELECT * FROM rx LIMIT 100 + + + , agg_to_target_items AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val + FROM + rx + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key + ) + + --SELECT * FROM agg_to_target_items LIMIT 100 + + + , agg_to_target AS ( + SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val + FROM + agg_to_target_items + GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ) + + + , agg_to_ret AS ( + SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + FROM + agg_to_target + GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ) + ,agg_to_id AS ( + SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + FROM + agg_to_ret l + ORDER BY + l.srce + ,l.target + ,l."count" desc + ) + SELECT + jsonb_agg(row_to_json(agg_to_id)::jsonb) + INTO + _rslt + FROM + agg_to_id; + + RETURN _rslt; +END; +$f$; \ No newline at end of file diff --git a/database/interface/map_def/test_regex_recs.sql b/database/interface/map_def/test_regex_recs.sql new file mode 100644 index 0000000..2844ba3 --- /dev/null +++ b/database/interface/map_def/test_regex_recs.sql @@ -0,0 +1,211 @@ +DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb); +CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb +LANGUAGE plpgsql +AS +$f$ +DECLARE + _rslt jsonb; +BEGIN + +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention +) +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + ,jsonb_agg(rec) rec +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) +SELECT + jsonb_agg(row_to_json(agg_to_ret)::jsonb) +INTO + _rslt +FROM + agg_to_ret; +RETURN _rslt; +END; +$f$; \ No newline at end of file diff --git a/interface/map_values/map_rv_set.sql b/database/interface/map_values/map_rv_set.sql similarity index 100% rename from interface/map_values/map_rv_set.sql rename to database/interface/map_values/map_rv_set.sql diff --git a/database/interface/map_values/report_unmapped.sql b/database/interface/map_values/report_unmapped.sql new file mode 100644 index 0000000..30110fe --- /dev/null +++ b/database/interface/map_values/report_unmapped.sql @@ -0,0 +1,249 @@ + +DROP FUNCTION IF EXISTS tps.report_unmapped; +CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$ \ No newline at end of file diff --git a/database/interface/map_values/report_unmapped_recs.sql b/database/interface/map_values/report_unmapped_recs.sql new file mode 100644 index 0000000..8ef5899 --- /dev/null +++ b/database/interface/map_values/report_unmapped_recs.sql @@ -0,0 +1,257 @@ +DROP FUNCTION IF EXISTS tps.report_unmapped_recs; +CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE +( + source text, + map text, + ret_val jsonb, + "count" bigint, + recs jsonb + +) +LANGUAGE plpgsql +AS +$f$ +BEGIN + +/* +first get distinct target json values +then apply regex +*/ + +RETURN QUERY +WITH + +--------------------apply regex operations to transactions--------------------------------------------------------------------------------- + +rx AS ( + SELECT + t.srce, + t.id, + t.rec, + m.target, + m.seq, + regex->'regex'->>'function' regex_function, + e.v ->> 'field' result_key_name, + e.v ->> 'key' target_json_path, + e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, + e.v->>'retain' retain_result, + e.v->>'regex' regex_expression, + e.rn target_item_number, + COALESCE(mt.rn,rp.rn,1) result_number, + mt.mt rx_match, + rp.rp rx_replace, + CASE e.v->>'map' + WHEN 'y' THEN + e.v->>'field' + ELSE + null + END map_key, + CASE e.v->>'map' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(mt.mt[1]) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rp.rp) + ELSE + '{}'::jsonb + END + ELSE + NULL + END map_val, + CASE e.v->>'retain' + WHEN 'y' THEN + e.v->>'field' + ELSE + NULL + END retain_key, + CASE e.v->>'retain' + WHEN 'y' THEN + CASE regex->'regex'->>'function' + WHEN 'extract' THEN + CASE WHEN array_upper(mt.mt,1)=1 + THEN to_json(trim(mt.mt[1])) + ELSE array_to_json(mt.mt) + END::jsonb + WHEN 'replace' THEN + to_jsonb(rtrim(rp.rp)) + ELSE + '{}'::jsonb + END + ELSE + NULL + END retain_val + FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ + tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- + INNER JOIN tps.trans t ON + t.srce = m.srce AND + t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ + LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- + LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON + m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ + LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON + m.regex->'regex'->>'function' = 'replace' + WHERE + --t.allj IS NULL + t.srce = _srce AND + e.v @> '{"map":"y"}'::jsonb + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb + ORDER BY + t.id DESC, + m.target, + e.rn, + COALESCE(mt.rn,rp.rn,1) +) + +--SELECT * FROM rx LIMIT 100 + + +, agg_to_target_items AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,CASE WHEN map_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + map_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(map_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(map_val ORDER BY result_number) + END + ) + END map_val + ,CASE WHEN retain_key IS NULL + THEN + NULL + ELSE + jsonb_build_object( + retain_key, + CASE WHEN max(result_number) = 1 + THEN + jsonb_agg(retain_val ORDER BY result_number) -> 0 + ELSE + jsonb_agg(retain_val ORDER BY result_number) + END + ) + END retain_val +FROM + rx +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,regex_function + ,target_item_number + ,result_key_name + ,target_json_path + ,map_key + ,retain_key +) + +--SELECT * FROM agg_to_target_items LIMIT 100 + + +, agg_to_target AS ( +SELECT + srce + ,id + ,rec + ,target + ,seq + ,map_intention + ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val + ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val +FROM + agg_to_target_items +GROUP BY + srce + ,id + ,rec + ,target + ,seq + ,map_intention +) + + +, agg_to_ret AS ( +SELECT + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val + ,count(*) "count" + ,jsonb_agg(rec) rec +FROM + agg_to_target +GROUP BY + srce + ,target + ,seq + ,map_intention + ,map_val + ,retain_val +) + +, link_map AS ( +SELECT + a.srce + ,a.target + ,a.seq + ,a.map_intention + ,a.map_val + ,a."count" + ,a.rec + ,a.retain_val + ,v.map mapped_val +FROM + agg_to_ret a + LEFT OUTER JOIN tps.map_rv v ON + v.srce = a.srce AND + v.target = a.target AND + v.retval = a.map_val +) +SELECT + l.srce + ,l.target + ,l.map_val + ,l."count" + ,l.rec +FROM + link_map l +WHERE + l.mapped_val IS NULL +ORDER BY + l.srce + ,l.target + ,l."count" desc; +END; +$f$ \ No newline at end of file diff --git a/interface/map_values/srce_map_overwrite.sql b/database/interface/map_values/srce_map_overwrite.sql similarity index 88% rename from interface/map_values/srce_map_overwrite.sql rename to database/interface/map_values/srce_map_overwrite.sql index 408f8a9..4706f44 100644 --- a/interface/map_values/srce_map_overwrite.sql +++ b/database/interface/map_values/srce_map_overwrite.sql @@ -74,14 +74,20 @@ BEGIN NULL END retain_val FROM + --------------------------start with all regex maps------------------------------------------------------------------------------------ tps.map_rm m + --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE + --------------------------join to main transaction table but only certain key/values are included-------------------------------------- INNER JOIN tps.trans t ON t.srce = m.srce AND t.rec @> w.v + --------------------------break out array of regluar expressions in the map------------------------------------------------------------ LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true + --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON m.regex->'regex'->>'function' = 'extract' + --------------------------same as above but for a replacement type function------------------------------------------------------------ LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->'regex'->>'function' = 'replace' WHERE diff --git a/interface/source_maint/srce_build_view.sql b/database/interface/source_maint/srce_build_view.sql similarity index 100% rename from interface/source_maint/srce_build_view.sql rename to database/interface/source_maint/srce_build_view.sql diff --git a/interface/source_maint/srce_set.sql b/database/interface/source_maint/srce_set.sql similarity index 100% rename from interface/source_maint/srce_set.sql rename to database/interface/source_maint/srce_set.sql diff --git a/readme.md b/database/readme.md similarity index 97% rename from readme.md rename to database/readme.md index 71e3a1f..4cb3cc9 100644 --- a/readme.md +++ b/database/readme.md @@ -1,128 +1,128 @@ -Generic Data Transformation Tool -======================================================= - -The goal is to: -1. house external data and prevent duplication on insert -2. facilitate regular exression operations to extract meaningful data -3. be able to reference it from outside sources (no action required) and maintain reference to original data - - -It is well suited for data from outside systems that -* requires complex transformation (parsing and mapping) -* original data is retained for reference -* don't feel like writing a map-reduce - -use cases: -* on-going bank feeds -* jumbled product lists -* storing api results - - -The data is converted to json by the importing program and inserted to the database. -Regex expressions are applied to specified json components and the results can be mapped to other values. - - -Major Interactions ------------------------- - -* Source Definitions (Maint/Inquire) -* Regex Instructions (Maint/Inquire) -* Cross Reference List (Maint/Inquire) -* Run Import (Run Job) - - - -### Interaction Details -* _Source Definitions (Maint/Inquire)_ - - * display a list of existing sources with display detials/edit options - * create new option - * underlying function is `tps.srce_set(_name text, _defn jsonb)` - - * the current definition of a source includes data based on bad presumptions: - * how to load from a csv file using `COPY` - * setup a Postgres type to reflect the associated columns (if applicable) - - -* _Regex Instructions (Maint/Inquire)_ - - * display a list of existing instruction sets with display details/edit options - * create new option - * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json - -* _Cross Reference List (Maint/Inquire)_ - - * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` - * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs - * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` - -* _Run Import_ - - * underlying function is `tps.srce_import(_path text, _srce text)` - - - -source definition ----------------------------------------------------------------------- - -* **load data** - * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` - * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function - * build constraint key `based on srce definition` - * handle violations - * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) - * build an import log - * run maps (as opposed to relying on trigger) -* **read data** - * the `schema` key contains either a text element or a text array in curly braces - * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` - * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table - * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` - * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` - * view that `uses the source definiton` to extrapolate a table? - * a materialized table is built `based on the source definition` and any addtional regex? - * add regex = alter table add column with historic updates? - * no primary key? - * every document must work out to one row - -``` -{ - "name":"dcard", - "source":"client_file", - "loading_function":"csv" - "constraint":[ - "{Trans. Date}", - "{Post Date}" - ], - "schemas":{ - "default":[ - { - "path":"{doc,origin_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,destination_addresses,0}", - "type":"text", - "column_name":"origin_address" - }, - { - "path":"{doc,status}", - "type":"text", - "column_name":"status" - } - { - "path":"{doc,rows,0,elements,0,distance,value}", - "type":"numeric", - "column_name":"distance" - } - { - "path":"{doc,rows,0,elements,0,duration,value}", - "type":"numeric", - "column_name":"duration" - } - ], - "version2":[] - } -} +Generic Data Transformation Tool +======================================================= + +The goal is to: +1. house external data and prevent duplication on insert +2. facilitate regular exression operations to extract meaningful data +3. be able to reference it from outside sources (no action required) and maintain reference to original data + + +It is well suited for data from outside systems that +* requires complex transformation (parsing and mapping) +* original data is retained for reference +* don't feel like writing a map-reduce + +use cases: +* on-going bank feeds +* jumbled product lists +* storing api results + + +The data is converted to json by the importing program and inserted to the database. +Regex expressions are applied to specified json components and the results can be mapped to other values. + + +Major Interactions +------------------------ + +* Source Definitions (Maint/Inquire) +* Regex Instructions (Maint/Inquire) +* Cross Reference List (Maint/Inquire) +* Run Import (Run Job) + + + +### Interaction Details +* _Source Definitions (Maint/Inquire)_ + + * display a list of existing sources with display detials/edit options + * create new option + * underlying function is `tps.srce_set(_name text, _defn jsonb)` + + * the current definition of a source includes data based on bad presumptions: + * how to load from a csv file using `COPY` + * setup a Postgres type to reflect the associated columns (if applicable) + + +* _Regex Instructions (Maint/Inquire)_ + + * display a list of existing instruction sets with display details/edit options + * create new option + * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json + +* _Cross Reference List (Maint/Inquire)_ + + * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)` + * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs + * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)` + +* _Run Import_ + + * underlying function is `tps.srce_import(_path text, _srce text)` + + + +source definition +---------------------------------------------------------------------- + +* **load data** + * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion` + * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function + * build constraint key `based on srce definition` + * handle violations + * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed) + * build an import log + * run maps (as opposed to relying on trigger) +* **read data** + * the `schema` key contains either a text element or a text array in curly braces + * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record` + * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table + * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema` + * custom function parsing contents based on #> operator and extracting from `srce.defn->schema` + * view that `uses the source definiton` to extrapolate a table? + * a materialized table is built `based on the source definition` and any addtional regex? + * add regex = alter table add column with historic updates? + * no primary key? + * every document must work out to one row + +``` +{ + "name":"dcard", + "source":"client_file", + "loading_function":"csv" + "constraint":[ + "{Trans. Date}", + "{Post Date}" + ], + "schemas":{ + "default":[ + { + "path":"{doc,origin_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,destination_addresses,0}", + "type":"text", + "column_name":"origin_address" + }, + { + "path":"{doc,status}", + "type":"text", + "column_name":"status" + } + { + "path":"{doc,rows,0,elements,0,distance,value}", + "type":"numeric", + "column_name":"distance" + } + { + "path":"{doc,rows,0,elements,0,duration,value}", + "type":"numeric", + "column_name":"duration" + } + ], + "version2":[] + } +} ``` \ No newline at end of file diff --git a/reports/all_map_return_values.sql b/database/reports/all_map_return_values.sql similarity index 100% rename from reports/all_map_return_values.sql rename to database/reports/all_map_return_values.sql diff --git a/reports/colateral_balalance.sql b/database/reports/colateral_balalance.sql similarity index 100% rename from reports/colateral_balalance.sql rename to database/reports/colateral_balalance.sql diff --git a/reports/dcard_bal.sql b/database/reports/dcard_bal.sql similarity index 100% rename from reports/dcard_bal.sql rename to database/reports/dcard_bal.sql diff --git a/reports/key_list.sql b/database/reports/key_list.sql similarity index 100% rename from reports/key_list.sql rename to database/reports/key_list.sql diff --git a/reports/list_maps.sql b/database/reports/list_maps.sql similarity index 100% rename from reports/list_maps.sql rename to database/reports/list_maps.sql diff --git a/reports/loan_balance.sql b/database/reports/loan_balance.sql similarity index 100% rename from reports/loan_balance.sql rename to database/reports/loan_balance.sql diff --git a/templates/regex.json b/database/templates/regex.json similarity index 100% rename from templates/regex.json rename to database/templates/regex.json diff --git a/templates/srce.json b/database/templates/srce.json similarity index 100% rename from templates/srce.json rename to database/templates/srce.json diff --git a/interface/map_def/test_regex.sql b/interface/map_def/test_regex.sql deleted file mode 100644 index 71e928d..0000000 --- a/interface/map_def/test_regex.sql +++ /dev/null @@ -1,221 +0,0 @@ -DROP FUNCTION IF EXISTS tps.test_regex(jsonb); -CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb -LANGUAGE plpgsql -AS -$f$ -DECLARE - _rslt jsonb; -BEGIN - - WITH - - --------------------apply regex operations to transactions--------------------------------------------------------------------------------- - - rx AS ( - SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val - FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' - ) - - --SELECT * FROM rx LIMIT 100 - - - , agg_to_target_items AS ( - SELECT - srce - ,id - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,CASE WHEN map_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - map_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(map_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(map_val ORDER BY result_number) - END - ) - END map_val - ,CASE WHEN retain_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - retain_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(retain_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(retain_val ORDER BY result_number) - END - ) - END retain_val - FROM - rx - GROUP BY - srce - ,id - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,map_key - ,retain_key - ) - - --SELECT * FROM agg_to_target_items LIMIT 100 - - - , agg_to_target AS ( - SELECT - srce - ,id - ,target - ,seq - ,map_intention - ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val - ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val - FROM - agg_to_target_items - GROUP BY - srce - ,id - ,target - ,seq - ,map_intention - ) - - - , agg_to_ret AS ( - SELECT - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ,count(*) "count" - FROM - agg_to_target - GROUP BY - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ) - ,agg_to_id AS ( - SELECT - l.srce - ,l.target - ,l.map_val - ,l."count" - FROM - agg_to_ret l - ORDER BY - l.srce - ,l.target - ,l."count" desc - ) - SELECT - jsonb_agg(row_to_json(agg_to_id)::jsonb) - INTO - _rslt - FROM - agg_to_id; - - RETURN _rslt; -END; -$f$; \ No newline at end of file diff --git a/interface/map_def/test_regex_recs.sql b/interface/map_def/test_regex_recs.sql deleted file mode 100644 index 3522b0e..0000000 --- a/interface/map_def/test_regex_recs.sql +++ /dev/null @@ -1,210 +0,0 @@ -DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb); -CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb -LANGUAGE plpgsql -AS -$f$ -DECLARE - _rslt jsonb; -BEGIN - -WITH - ---------------------apply regex operations to transactions--------------------------------------------------------------------------------- - -rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn->'regex' regex, (_defn->>'sequence')::numeric seq) m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -) - ---SELECT * FROM rx LIMIT 100 - - -, agg_to_target_items AS ( -SELECT - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,CASE WHEN map_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - map_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(map_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(map_val ORDER BY result_number) - END - ) - END map_val - ,CASE WHEN retain_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - retain_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(retain_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(retain_val ORDER BY result_number) - END - ) - END retain_val -FROM - rx -GROUP BY - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,map_key - ,retain_key -) - ---SELECT * FROM agg_to_target_items LIMIT 100 - - -, agg_to_target AS ( -SELECT - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val - ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val -FROM - agg_to_target_items -GROUP BY - srce - ,id - ,rec - ,target - ,seq - ,map_intention -) -, agg_to_ret AS ( -SELECT - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ,count(*) "count" - ,jsonb_agg(rec) rec -FROM - agg_to_target -GROUP BY - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val -) -SELECT - jsonb_agg(row_to_json(agg_to_ret)::jsonb) -INTO - _rslt -FROM - agg_to_ret; -RETURN _rslt; -END; -$f$; \ No newline at end of file diff --git a/interface/map_values/report_unmapped.sql b/interface/map_values/report_unmapped.sql deleted file mode 100644 index 5c10966..0000000 --- a/interface/map_values/report_unmapped.sql +++ /dev/null @@ -1,276 +0,0 @@ -CREATE OR REPLACE FUNCTION tps.jsonb_concat( - state jsonb, - concat jsonb) - RETURNS jsonb AS -$BODY$ -BEGIN - --RAISE notice 'state is %', state; - --RAISE notice 'concat is %', concat; - RETURN state || concat; -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; - -DROP AGGREGATE IF EXISTS tps.jsonb_concat_obj(jsonb); -CREATE AGGREGATE tps.jsonb_concat_obj(jsonb) ( - SFUNC=tps.jsonb_concat, - STYPE=jsonb, - INITCOND='{}' -); - - -DROP FUNCTION IF EXISTS tps.report_unmapped; -CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE -( - source text, - map text, - ret_val jsonb, - "count" bigint -) -LANGUAGE plpgsql -AS -$f$ -BEGIN - -/* -first get distinct target json values -then apply regex -*/ - -RETURN QUERY -WITH - ---------------------apply regex operations to transactions--------------------------------------------------------------------------------- - -rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE - --t.allj IS NULL - t.srce = _srce AND - e.v @> '{"map":"y"}'::jsonb - --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' - --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ -) - ---SELECT * FROM rx LIMIT 100 - - -, agg_to_target_items AS ( -SELECT - srce - ,id - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,CASE WHEN map_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - map_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(map_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(map_val ORDER BY result_number) - END - ) - END map_val - ,CASE WHEN retain_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - retain_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(retain_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(retain_val ORDER BY result_number) - END - ) - END retain_val -FROM - rx -GROUP BY - srce - ,id - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,map_key - ,retain_key -) - ---SELECT * FROM agg_to_target_items LIMIT 100 - - -, agg_to_target AS ( -SELECT - srce - ,id - ,target - ,seq - ,map_intention - ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val - ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val -FROM - agg_to_target_items -GROUP BY - srce - ,id - ,target - ,seq - ,map_intention -) - - -, agg_to_ret AS ( -SELECT - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ,count(*) "count" -FROM - agg_to_target -GROUP BY - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val -) - -, link_map AS ( -SELECT - a.srce - ,a.target - ,a.seq - ,a.map_intention - ,a.map_val - ,a."count" - ,a.retain_val - ,v.map mapped_val -FROM - agg_to_ret a - LEFT OUTER JOIN tps.map_rv v ON - v.srce = a.srce AND - v.target = a.target AND - v.retval = a.map_val -) -SELECT - l.srce - ,l.target - ,l.map_val - ,l."count" -FROM - link_map l -WHERE - l.mapped_val IS NULL -ORDER BY - l.srce - ,l.target - ,l."count" desc; -END; -$f$ \ No newline at end of file diff --git a/interface/map_values/report_unmapped_recs.sql b/interface/map_values/report_unmapped_recs.sql deleted file mode 100644 index cceee91..0000000 --- a/interface/map_values/report_unmapped_recs.sql +++ /dev/null @@ -1,263 +0,0 @@ -DROP FUNCTION IF EXISTS tps.report_unmapped_recs; -CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE -( - source text, - map text, - ret_val jsonb, - "count" bigint, - recs jsonb - -) -LANGUAGE plpgsql -AS -$f$ -BEGIN - -/* -first get distinct target json values -then apply regex -*/ - -RETURN QUERY -WITH - ---------------------apply regex operations to transactions--------------------------------------------------------------------------------- - -rx AS ( -SELECT - t.srce, - t.id, - t.rec, - m.target, - m.seq, - regex->>'function' regex_function, - e.v ->> 'field' result_key_name, - e.v ->> 'key' target_json_path, - e.v ->> 'flag' regex_options_flag, - e.v->>'map' map_intention, - e.v->>'retain' retain_result, - e.v->>'regex' regex_expression, - e.rn target_item_number, - COALESCE(mt.rn,rp.rn,1) result_number, - mt.mt rx_match, - rp.rp rx_replace, - --------------------------json key name assigned to return value----------------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - e.v->>'field' - ELSE - null - END map_key, - --------------------------json value resulting from regular expression----------------------------------------------------------------- - CASE e.v->>'map' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(mt.mt[1]) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rp.rp) - ELSE - '{}'::jsonb - END - ELSE - NULL - END map_val, - --------------------------flag for if retruned regex result is stored as a new part of the final json output--------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - e.v->>'field' - ELSE - NULL - END retain_key, - --------------------------push regex result into json object--------------------------------------------------------------------------- - CASE e.v->>'retain' - WHEN 'y' THEN - CASE regex->>'function' - WHEN 'extract' THEN - CASE WHEN array_upper(mt.mt,1)=1 - THEN to_json(trim(mt.mt[1])) - ELSE array_to_json(mt.mt) - END::jsonb - WHEN 'replace' THEN - to_jsonb(rtrim(rp.rp)) - ELSE - '{}'::jsonb - END - ELSE - NULL - END retain_val -FROM - --------------------------start with all regex maps------------------------------------------------------------------------------------ - tps.map_rm m - --------------------------isolate matching basis to limit map to only look at certain json--------------------------------------------- - JOIN LATERAL jsonb_array_elements(m.regex->'where') w(v) ON TRUE - --------------------------break out array of regluar expressions in the map------------------------------------------------------------ - JOIN LATERAL jsonb_array_elements(m.regex->'defn') WITH ORDINALITY e(v, rn) ON true - --------------------------join to main transaction table but only certain key/values are included-------------------------------------- - INNER JOIN tps.trans t ON - t.srce = m.srce AND - t.rec @> w.v - --------------------------each regex references a path to the target value, extract the target from the reference and do regex--------- - LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON - m.regex->>'function' = 'extract' - --------------------------same as above but for a replacement type function------------------------------------------------------------ - LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON - m.regex->>'function' = 'replace' -WHERE - --t.allj IS NULL - t.srce = _srce AND - e.v @> '{"map":"y"}'::jsonb - --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' - --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb -/* -ORDER BY - t.id DESC, - m.target, - e.rn, - COALESCE(mt.rn,rp.rn,1) -*/ -) - ---SELECT * FROM rx LIMIT 100 - - -, agg_to_target_items AS ( -SELECT - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,CASE WHEN map_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - map_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(map_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(map_val ORDER BY result_number) - END - ) - END map_val - ,CASE WHEN retain_key IS NULL - THEN - NULL - ELSE - jsonb_build_object( - retain_key, - CASE WHEN max(result_number) = 1 - THEN - jsonb_agg(retain_val ORDER BY result_number) -> 0 - ELSE - jsonb_agg(retain_val ORDER BY result_number) - END - ) - END retain_val -FROM - rx -GROUP BY - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,regex_function - ,target_item_number - ,result_key_name - ,target_json_path - ,map_key - ,retain_key -) - ---SELECT * FROM agg_to_target_items LIMIT 100 - - -, agg_to_target AS ( -SELECT - srce - ,id - ,rec - ,target - ,seq - ,map_intention - ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val - ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val -FROM - agg_to_target_items -GROUP BY - srce - ,id - ,rec - ,target - ,seq - ,map_intention -) - - -, agg_to_ret AS ( -SELECT - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val - ,count(*) "count" - ,jsonb_agg(rec) rec -FROM - agg_to_target -GROUP BY - srce - ,target - ,seq - ,map_intention - ,map_val - ,retain_val -) - -, link_map AS ( -SELECT - a.srce - ,a.target - ,a.seq - ,a.map_intention - ,a.map_val - ,a."count" - ,a.rec - ,a.retain_val - ,v.map mapped_val -FROM - agg_to_ret a - LEFT OUTER JOIN tps.map_rv v ON - v.srce = a.srce AND - v.target = a.target AND - v.retval = a.map_val -) -SELECT - l.srce - ,l.target - ,l.map_val - ,l."count" - ,l.rec -FROM - link_map l -WHERE - l.mapped_val IS NULL -ORDER BY - l.srce - ,l.target - ,l."count" desc; -END; -$f$ \ No newline at end of file diff --git a/package.json b/package.json deleted file mode 100644 index 38f783d..0000000 --- a/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "tps_etl", - "version": "1.0.0", - "description": "third party source data transformation", - "main": "index.js", - "scripts": { - "test": "uh" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/fleetside72/tps_etl.git" - }, - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/fleetside72/tps_etl/issues" - }, - "homepage": "https://github.com/fleetside72/tps_etl#readme" -}