vault backup: 2024-09-28 17:13:57

.gitignore

@@ -2,4 +2,5 @@ node_modules/*
 .env
 *.log
 *.pem
-.vscode/
+.vscode/
+*.swp

database/deploy/reload/dcard/curl_dcard.cmd

@@ -0,0 +1,4 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost/source
+curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost/regex
+curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost/mapping
+curl -v -F upload=@./d.csv http://localhost/import?srce=dcard

database/deploy/reload/dcard/d.csv

@@ -0,0 +1,139 @@
+Trans. Date,Post Date,Description,Amount,Category
+01/02/2018,01/02/2018,"GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV",4.26,"Services"
+01/02/2018,01/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
+01/03/2018,01/03/2018,"CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3",200.00,"Medical Services"
+01/04/2018,01/04/2018,"AT&T *PAYMENT 800-288-2020 TX",57.14,"Services"
+01/04/2018,01/07/2018,"WWW.KOHLS.COM #0873 MIDDLETOWN OH",-7.90,"Payments and Credits"
+01/05/2018,01/07/2018,"PIZZA HUT 007946 STOW OH",9.24,"Restaurants"
+01/05/2018,01/07/2018,"SUBWAY 00044289255 STOW OH",10.25,"Restaurants"
+01/06/2018,01/07/2018,"ACME NO. 17 STOW OH",103.98,"Supermarkets"
+01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",1.69,"Merchandise"
+01/06/2018,01/07/2018,"DISCOUNT DRUG MART 32 STOW OH",2.19,"Merchandise"
+01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00947R",3.94,"Gasoline"
+01/09/2018,01/09/2018,"CIRCLE K 05416 STOW OH00915R",52.99,"Gasoline"
+01/13/2018,01/13/2018,"AUTOZONE #0722 STOW OH",85.36,"Automotive"
+01/13/2018,01/13/2018,"DISCOUNT DRUG MART 32 STOW OH",26.68,"Merchandise"
+01/13/2018,01/13/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
+01/13/2018,01/13/2018,"TARGET STOW OH",197.90,"Merchandise"
+01/14/2018,01/14/2018,"DISCOUNT DRUG MART 32 STOW OH",13.48,"Merchandise"
+01/15/2018,01/15/2018,"TARGET.COM * 800-591-3869 MN",22.41,"Merchandise"
+01/16/2018,01/16/2018,"BUFFALO WILD WINGS KENT KENT OH",63.22,"Restaurants"
+01/16/2018,01/16/2018,"PARTA - KCG KENT OH",4.00,"Government Services"
+01/16/2018,01/16/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
+01/16/2018,01/16/2018,"TARGET.COM * 800-591-3869 MN",44.81,"Merchandise"
+01/16/2018,01/16/2018,"TREE CITY COFFEE & PASTR KENT OH",17.75,"Restaurants"
+01/17/2018,01/17/2018,"BESTBUYCOM805526794885 888-BESTBUY MN",343.72,"Merchandise"
+01/19/2018,01/19/2018,"DISCOUNT DRUG MART 32 STOW OH",5.98,"Merchandise"
+01/19/2018,01/19/2018,"U-HAUL OF KENT-STOW KENT OH",15.88,"Travel/ Entertainment"
+01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",5.99,"Supermarkets"
+01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",17.16,"Supermarkets"
+01/19/2018,01/19/2018,"WALMART GROCERY 800-966-6546 AR",500.97,"Supermarkets"
+01/20/2018,01/20/2018,"GOOGLE *GOOGLE PLAY G.CO/HELPPAY#CAP0HFFS7W",2.12,"Services"
+01/20/2018,01/20/2018,"LOWE'S OF STOW, OH. STOW OH",256.48,"Home Improvement"
+01/22/2018,01/22/2018,"HOBBY LOBBY #405 STOW OHITEM TRANSFERRED FROM PREV ACCOUNT",38.49,"Merchandise"
+01/23/2018,01/23/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-32.20,"Awards and Rebate Credits"
+01/23/2018,01/23/2018,"INTERNET PAYMENT - THANK YOU",-2394.51,"Payments and Credits"
+01/27/2018,01/27/2018,"GIANT-EAGLE #4096 STOW OH",67.81,"Supermarkets"
+01/27/2018,01/27/2018,"OFFICEMAX/OFFICE DEPOT63 STOW OH",21.06,"Merchandise"
+01/27/2018,01/27/2018,"TARGET STOW OH",71.00,"Merchandise"
+01/29/2018,01/29/2018,"NETFLIX.COM NETFLIX.COM CA19899514437",14.93,"Services"
+01/30/2018,01/30/2018,"PARTA - KCG KENT OH",1.00,"Government Services"
+01/30/2018,01/30/2018,"SPEEDWAY 09303 KEN KENT OH",46.57,"Gasoline"
+01/30/2018,01/30/2018,"SQ *TWISTED MELTZ KENT OH0002305843011416898511",16.87,"Restaurants"
+01/30/2018,01/30/2018,"TARGET STOW OH",49.37,"Merchandise"
+01/31/2018,01/31/2018,"TARGET STOW OH",4.14,"Merchandise"
+01/31/2018,01/31/2018,"TARGET STREETSBORO OH",14.28,"Merchandise"
+01/31/2018,02/01/2018,"TARGET STOW OH",-21.34,"Payments and Credits"
+01/31/2018,02/01/2018,"TARGET STREETSBORO OH",-9.60,"Payments and Credits"
+02/01/2018,02/01/2018,"EL CAMPESINO STOW OH",42.24,"Restaurants"
+02/02/2018,02/02/2018,"CASH ADVANCE FEE",5.00,"Fees"
+02/02/2018,02/02/2018,"GOOGLE *ASCIIFLOW.COM G.CO/HELPPAY#CAP0HQTYN5",5.00,"Cash Advances"
+02/03/2018,02/03/2018,"TARGET STREETSBORO OH",71.69,"Merchandise"
+02/03/2018,02/07/2018,"SAMS CLUB - #4750 CUYAHOGA FALLOH",371.90,"Warehouse Clubs"
+02/04/2018,02/04/2018,"ACME NO. 17 STOW OH",8.98,"Supermarkets"
+02/04/2018,02/04/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
+02/06/2018,02/06/2018,"MINIMUM INTEREST CHARGE FEE",0.50,"Fees"
+02/06/2018,02/07/2018,"BP#954778736210 7-ELEVEN STOW OH",52.80,"Gasoline"
+02/06/2018,02/07/2018,"CVS/PHARMACY #08932 TWINSBURG OH",13.87,"Merchandise"
+02/07/2018,02/07/2018,"AT&T *PAYMENT 800-288-2020 TXX51Z5QX7SMT2U01",57.14,"Services"
+02/07/2018,02/07/2018,"TOYS R US #9203 CUYAHOGA FALLOH",193.32,"Merchandise"
+02/08/2018,02/08/2018,"GIANT-EAGLE #4096 STOW OH",66.13,"Supermarkets"
+02/08/2018,02/08/2018,"TARGET STOW OH",121.32,"Merchandise"
+02/09/2018,02/09/2018,"GUIDOS ORIGINAL PIZZA KENT OH",11.75,"Restaurants"
+02/09/2018,02/09/2018,"MARATHON PETRO73601 TWINSBURG OH",44.30,"Gasoline"
+02/10/2018,02/10/2018,"RSVP NO. 36 STOW OH",14.43,"Supermarkets"
+02/10/2018,02/10/2018,"TARGET STOW OH",77.90,"Merchandise"
+02/11/2018,02/11/2018,"SUBWAY 00044289255 STOW OH",21.00,"Restaurants"
+02/13/2018,02/13/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",12.79,"Restaurants"
+02/13/2018,02/13/2018,"IN *MR. BULKY'S FOODS AKRON OHAJ16V8Q6",3.39,"Supermarkets"
+02/13/2018,02/13/2018,"TARGET CUYAHOGA FALLOH",5.33,"Supermarkets"
+02/14/2018,02/14/2018,"DISCOUNT DRUG MART 32 STOW OH",4.29,"Merchandise"
+02/14/2018,02/14/2018,"HANDELS ICE CREAM STOW STOW OH",7.95,"Supermarkets"
+02/15/2018,02/15/2018,"BATH&BODY STOW OH",47.19,"Merchandise"
+02/15/2018,02/15/2018,"TARGET STOW OH",76.35,"Merchandise"
+02/17/2018,02/17/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
+02/17/2018,02/17/2018,"WALMART GROCERY 800-966-6546 AR",461.36,"Supermarkets"
+02/18/2018,02/18/2018,"ACME NO. 17 STOW OH",32.68,"Supermarkets"
+02/18/2018,02/18/2018,"CHIPOTLE ONLINE 303-595-4000 CO",20.75,"Restaurants"
+02/19/2018,02/19/2018,"GIANT EAGLE #5863 STREETSBORO OH",25.00,"Supermarkets"
+02/20/2018,02/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
+02/21/2018,02/21/2018,"BP#954635936241 7-ELEVEN STOW OH",30.04,"Gasoline"
+02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",3.19,"Restaurants"
+02/22/2018,02/22/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",18.22,"Restaurants"
+02/22/2018,02/22/2018,"PET SUPPLIES PLUS #68 STOW OH",45.88,"Merchandise"
+02/22/2018,02/22/2018,"TOYS R US #9203 CUYAHOGA FALLOH",21.31,"Merchandise"
+02/23/2018,02/23/2018,"SUMMIT CO PARKING GAR AKRON OH",6.00,"Services"
+02/24/2018,02/24/2018,"GET GO #3396 STOW OH",26.46,"Gasoline"
+02/25/2018,02/25/2018,"DISCOUNT DRUG MART 32 STOW OH",19.70,"Merchandise"
+02/25/2018,02/25/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
+02/25/2018,02/25/2018,"SQ *CORNER CUP COFFEEH STOW OH0001152921507942036274",2.30,"Supermarkets"
+02/25/2018,02/25/2018,"TARGET STOW OH",18.49,"Merchandise"
+02/28/2018,02/28/2018,"NETFLIX.COM NETFLIX.COM CA20475539512",14.93,"Services"
+03/01/2018,03/01/2018,"GIANT-EAGLE #4032 STOW OH",2.99,"Supermarkets"
+03/01/2018,03/01/2018,"TARGET STOW OH",72.46,"Merchandise"
+03/02/2018,03/02/2018,"LATE FEE",27.00,"Fees"
+03/02/2018,03/02/2018,"MICROSOFT *ONEDRIVE 800-642-7676 WA",4.26,"Services"
+03/02/2018,03/02/2018,"PIZZA HUT 007946 STOW OH",9.89,"Restaurants"
+03/03/2018,03/03/2018,"CASHBACK BONUS REDEMPTION PYMT/STMT CRDT",-23.28,"Awards and Rebate Credits"
+03/03/2018,03/03/2018,"INTERNET PAYMENT - THANK YOU",-2451.43,"Payments and Credits"
+03/03/2018,03/03/2018,"LOWE'S OF STOW, OH. STOW OH",6.93,"Home Improvement"
+03/03/2018,03/07/2018,"WAL-MART SC - #2323 STOW OH",150.32,"Merchandise"
+03/04/2018,03/04/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",13.66,"Merchandise"
+03/06/2018,03/07/2018,"MFW BOOKS LLC 5732022000 MO",86.90,"Education"
+03/06/2018,03/07/2018,"OLD NAVY ON-LINE 800-OLDNAVY OH",127.46,"Merchandise"
+03/08/2018,03/08/2018,"ACME NO. 17 STOW OH",8.58,"Supermarkets"
+03/08/2018,03/08/2018,"CHICK-FIL-A #02197 CUYAHOGA FLS OH",2.12,"Restaurants"
+03/08/2018,03/08/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
+03/08/2018,03/08/2018,"SPEEDWAY 03686 496 STOW OH",45.24,"Gasoline"
+03/08/2018,03/08/2018,"SWENSONS STOW KENT STOW OH",8.30,"Restaurants"
+03/08/2018,03/08/2018,"TOYS R US #9203 CUYAHOGA FALLOH",5.33,"Merchandise"
+03/09/2018,03/09/2018,"SPEEDWAY 03686 496 STOW OH",48.29,"Gasoline"
+03/10/2018,03/10/2018,"WALMART GROCERY 800-966-6546 AR",522.22,"Supermarkets"
+03/11/2018,03/11/2018,"AT&T *PAYMENT 800-288-2020 TXQ8F55RY7SMT2N04",57.14,"Services"
+03/11/2018,03/11/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011470140810",2.30,"Supermarkets"
+03/12/2018,03/12/2018,"MICROSOFT *STORE 800-642-7676 WA",1.06,"Services"
+03/15/2018,03/15/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011475075512",2.30,"Supermarkets"
+03/16/2018,03/16/2018,"ACME NO. 17 STOW OH",15.85,"Supermarkets"
+03/16/2018,03/16/2018,"CHIPOTLE 1152 STOW OH",3.85,"Restaurants"
+03/16/2018,03/16/2018,"EL CAMPESINO STOW OH",6.50,"Restaurants"
+03/16/2018,03/16/2018,"PIZZA HUT 007946 STOW OH",13.98,"Restaurants"
+03/17/2018,03/17/2018,"CHIPOTLE ONLINE 303-595-4000 CO",15.75,"Restaurants"
+03/17/2018,03/17/2018,"DISCOUNT DRUG MART 32 STOW OH",9.89,"Merchandise"
+03/17/2018,03/17/2018,"MFW BOOKS LLC 5732022000 MO",66.75,"Education"
+03/18/2018,03/18/2018,"ACME NO. 17 STOW OH",27.78,"Supermarkets"
+03/18/2018,03/18/2018,"GIANT-EAGLE #4032 STOW OH",28.34,"Supermarkets"
+03/20/2018,03/20/2018,"REMEMBERNHU 402-935-7733 IA",60.00,"Services"
+03/20/2018,03/20/2018,"SONLIGHT CURRICULUM LTD 303-730-8193 CO",762.87,"Education"
+03/21/2018,03/21/2018,"BP#954635936241 7-ELEVEN STOW OH",8.87,"Gasoline"
+03/21/2018,03/21/2018,"DISCOUNT DRUG MART 32 STOW OH",18.07,"Merchandise"
+03/21/2018,03/21/2018,"SQ *CORNER CUP COFFEEH STOW OH0002305843011484061091",2.30,"Supermarkets"
+03/21/2018,03/21/2018,"TARGET STOW OH",1.95,"Merchandise"
+03/21/2018,03/21/2018,"TARGET STOW OH",224.85,"Merchandise"
+03/22/2018,03/22/2018,"JUSTICE #0639 STOW OH",16.01,"Merchandise"
+03/22/2018,03/22/2018,"SPEEDWAY 03686 496 STOW OH",32.54,"Gasoline"
+03/22/2018,03/22/2018,"SQ *TWISTED MELTZ KENT OH0002305843011486528725",6.74,"Restaurants"
+03/22/2018,03/22/2018,"TARGET STOW OH",6.60,"Merchandise"
+03/25/2018,03/25/2018,"ACME NO. 17 STOW OH",95.42,"Supermarkets"
+03/25/2018,03/25/2018,"ASIAN-GREEK CUISINES STOW OH",70.25,"Restaurants"
+03/25/2018,03/25/2018,"MARATHON PETRO73601 TWINSBURG OH",11.09,"Gasoline"
+03/25/2018,03/25/2018,"SPEEDWAY 09303 KEN KENT OH",53.28,"Gasoline"

database/deploy/reload/dcard/extract.sql

@@ -0,0 +1,24 @@
+--source
+SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'DCARD'
+
+--mapdef
+SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'DCARD') x
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'DCARD') X
+
+--records
+copy (
+    select 
+        to_char(r."Trans. Date",'mm/dd/yyyy') "Trans. Date"
+        ,to_char(r."Post Date",'mm/dd/yyyy') "Post Date"
+        ,r."Description"
+        ,r."Amount"
+        ,r."Category"
+    from
+        tps.trans 
+        join lateral jsonb_populate_record(null::tps.dcard, rec) r on true 
+    where 
+        srce = 'DCARD'
+) to 
+'C:\users\fleet\downloads\dcard.csv' with (format csv, header true);

database/deploy/reload/dcard/map.json

@@ -0,0 +1,25 @@
+[
+    {
+        "regex": {
+            "function": "extract",
+            "description": "pull first 20 characters from description for mapping",
+            "where": [
+                {}
+            ],
+            "defn": [
+                {
+                    "regex": ".{1,20}",
+                    "map": "y",
+                    "field": "f20",
+                    "flag": "",
+                    "key": "{Description}",
+                    "retain": "y"
+                }
+            ],
+            "name": "First 20"
+        },
+        "sequence": 2,
+        "name": "First 20",
+        "srce": "dcard"
+    }
+]

database/deploy/reload/dcard/srce.json

@@ -0,0 +1,76 @@
+{
+    "name": "dcard",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{Trans. Date}",
+        "{Post Date}",
+        "{Description}"
+    ],
+    "schemas": {
+        "default": [
+            {
+                "path": "{Trans. Date}",
+                "type": "date",
+                "column_name": "Trans. Date"
+            },
+            {
+                "path": "{Post Date}",
+                "type": "date",
+                "column_name": "Post Date"
+            },
+            {
+                "path": "{Description}",
+                "type": "text",
+                "column_name": "Description"
+            },
+            {
+                "path": "{Amount}",
+                "type": "numeric",
+                "column_name": "Amount"
+            },
+            {
+                "path": "{Category}",
+                "type": "text",
+                "column_name": "Category"
+            }
+        ],
+        "mapped": [
+            {
+                "path": "{Trans. Date}",
+                "type": "date",
+                "column_name": "Trans. Date"
+            },
+            {
+                "path": "{Post Date}",
+                "type": "date",
+                "column_name": "Post Date"
+            },
+            {
+                "path": "{Description}",
+                "type": "text",
+                "column_name": "Description"
+            },
+            {
+                "path": "{Amount}",
+                "type": "numeric",
+                "column_name": "Amount"
+            },
+            {
+                "path": "{Category}",
+                "type": "text",
+                "column_name": "Category"
+            },
+            {
+                "path": "{party}",
+                "type": "text",
+                "column_name": "Party"
+            },
+            {
+                "path": "{reason}",
+                "type": "text",
+                "column_name": "Reason"
+            }
+        ]
+    }
+}

database/deploy/reload/hunt/curl_hunt.cmd

@@ -0,0 +1,3 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set
+curl -v -F upload=@//mnt/c/Users/fleet/Downloads/hunt.csv http://localhost:81/import?srce=hunt

database/deploy/reload/hunt/extract.sql

@@ -0,0 +1,27 @@
+--source
+COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'HUNT') TO 'C:\users\fleet\documents\tps_etl\reload\hunt\srce.json' WITH (FORMAT TEXT, HEADER FALSE)
+
+--mapdef
+COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'HUNT') x) TO 'C:\users\fleet\documents\tps_etl\reload\hunt\map.json' WITH (FORMAT TEXT, HEADER FALSE)
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'HUNT') X
+
+--records
+copy (
+    select 
+        to_char(r."Date",'mm/dd/yy') "Date"
+        ,r."Reference Number"
+        ,r."Payee Name"
+        ,r."Memo"
+        ,r."Amount"
+		,r."Category Name"
+    from
+        tps.trans 
+        join lateral jsonb_populate_record(null::tps.hunt, rec) r on true 
+    where 
+        srce = 'HUNT'
+	order by 
+		r."Date" asc
+) to 
+'C:\users\fleet\downloads\hunt.csv' with (format csv, header true);

database/deploy/reload/hunt/map.json

@@ -0,0 +1,25 @@
+[
+    {
+        "name": "First 20",
+        "srce": "hunt",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Memo}",
+                    "map": "y",
+                    "flag": "",
+                    "field": "f20",
+                    "regex": ".{1,20}",
+                    "retain": "y"
+                }
+            ],
+            "name": "First 20",
+            "where": [
+                {}
+            ],
+            "function": "extract",
+            "description": "pull first 20 characters from description for mapping"
+        },
+        "sequence": 1
+    }
+]

database/deploy/reload/hunt/srce.json

@@ -0,0 +1,42 @@
+{
+    "name": "hunt",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{Date}"
+    ],
+    "schemas": {
+        "default": [
+            {
+                "path": "{Date}",
+                "type": "date",
+                "column_name": "Date"
+            },
+            {
+                "path": "{Reference Number}",
+                "type": "numeric",
+                "column_name": "Reference Number"
+            },
+            {
+                "path": "{Payee Name}",
+                "type": "text",
+                "column_name": "Payee Name"
+            },
+            {
+                "path": "{Memo}",
+                "type": "text",
+                "column_name": "Memo"
+            },
+            {
+                "path": "{Amount}",
+                "type": "numeric",
+                "column_name": "Amount"
+            },
+            {
+                "path": "{Category Name}",
+                "type": "text",
+                "column_name": "Cateogry Name"
+            }
+        ]
+    }
+}

database/deploy/reload/paycom/extract.sql

@@ -0,0 +1,58 @@
+--transactions with date in download format for constraint
+COPY
+(
+SELECT
+	r."perd_start",
+	r."perd_end",
+	r."check_date",
+	r."loc_code",
+	r."loc_descr",
+	r."loc_glseg",
+	r."loc_over",
+	r."dep_code",
+	r."dep_descr",
+	r."dep_nat",
+	r."dep_over",
+	r."di_code",
+	r."di_descr",
+	r."di_glseg",
+	r."di_over",
+	r."title_code",
+	r."title_descr",
+	r."title_glseg",
+	r."title_over",
+	r."ee_code",
+	r."ee_glseg",
+	r."ee_over",
+	r."acct_type_code",
+	r."hours",
+	r."nat_code",
+	r."nat_over",
+	r."gl_ref",
+	r."gl_group",
+	r."gl_descr",
+	r."gl_code",
+	r."gl_amount",
+	r."pp_code",
+	r."pp_descr",
+	r."pp_gl",
+	r."pp_over",
+	r."transaction"
+FROM
+	tps.trans
+	JOIN LATERAL jsonb_populate_record(NULL::tps.PAYCOM, rec) r ON TRUE
+WHERE
+	srce = 'PAYCOM'
+)
+TO 'C:\users\ptrowbridge\downloads\PAYCOM.csv' WITH (format csv, header TRUE)
+
+--source
+SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PAYCOM'
+
+--mapdef
+SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PAYCOM') x
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PAYCOM') X
+
+

database/deploy/reload/paycom/load.cmd

@@ -0,0 +1,4 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set
+curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set
+curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/PAYCOM.csv http://localhost:81/import?srce=PAYCOM

database/deploy/reload/paycom/map.json

@@ -0,0 +1,64 @@
+[
+    {
+        "name": "Extract month and code",
+        "srce": "PAYCOM",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{check_date}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "check_month",
+                    "regex": "(\\d{4})-(\\d{2})-\\d{2}",
+                    "retain": "y"
+                },
+                {
+                    "key": "{loc_code}",
+                    "map": "y",
+                    "flag": "",
+                    "field": "loc_code_rx",
+                    "regex": ".*",
+                    "retain": "n"
+                }
+            ],
+            "name": "Paycom",
+            "where": [
+                {}
+            ],
+            "function": "extract",
+            "description": "extract month and code"
+        },
+        "sequence": 1
+    },
+    {
+        "name": "SUBSEQUENT PAYOUT",
+        "srce": "PAYCOM",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{gl_group}",
+                    "map": "y",
+                    "flag": "",
+                    "field": "gl_group",
+                    "regex": ".*",
+                    "retain": "n"
+                },
+                {
+                    "key": "{acct_type_code}",
+                    "map": "y",
+                    "flag": "",
+                    "field": "acct_type_code",
+                    "regex": ".*",
+                    "retain": "n"
+                }
+            ],
+            "name": "Paycom",
+            "where": [
+                {}
+            ],
+            "function": "extract",
+            "description": "map payroll code and account group to subsequent payout"
+        },
+        "sequence": 2
+    }
+]

database/deploy/reload/paycom/srce.json

@@ -0,0 +1,392 @@
+{
+    "constraint": [
+        "{transaction}",
+        "{loc_code}",
+        "{perd_start}",
+        "{perd_end}"
+    ],
+    "name": "PAYCOM",
+    "source": "client_file",
+    "loading_function": "csv",
+    "schemas": {
+        "default": [
+            {
+                "path": "{perd_start}",
+                "type": "date",
+                "column_name": "perd_start"
+            },
+            {
+                "path": "{perd_end}",
+                "type": "date",
+                "column_name": "perd_end"
+            },
+            {
+                "path": "{check_date}",
+                "type": "date",
+                "column_name": "check_date"
+            },
+            {
+                "path": "{loc_code}",
+                "type": "text",
+                "column_name": "loc_code"
+            },
+            {
+                "path": "{loc_descr}",
+                "type": "text",
+                "column_name": "loc_descr"
+            },
+            {
+                "path": "{loc_glseg}",
+                "type": "text",
+                "column_name": "loc_glseg"
+            },
+            {
+                "path": "{loc_over}",
+                "type": "text",
+                "column_name": "loc_over"
+            },
+            {
+                "path": "{dep_code}",
+                "type": "text",
+                "column_name": "dep_code"
+            },
+            {
+                "path": "{dep_descr}",
+                "type": "text",
+                "column_name": "dep_descr"
+            },
+            {
+                "path": "{dep_nat}",
+                "type": "text",
+                "column_name": "dep_nat"
+            },
+            {
+                "path": "{dep_over}",
+                "type": "text",
+                "column_name": "dep_over"
+            },
+            {
+                "path": "{di_code}",
+                "type": "text",
+                "column_name": "di_code"
+            },
+            {
+                "path": "{di_descr}",
+                "type": "text",
+                "column_name": "di_descr"
+            },
+            {
+                "path": "{di_glseg}",
+                "type": "text",
+                "column_name": "di_glseg"
+            },
+            {
+                "path": "{di_over}",
+                "type": "text",
+                "column_name": "di_over"
+            },
+            {
+                "path": "{title_code}",
+                "type": "text",
+                "column_name": "title_code"
+            },
+            {
+                "path": "{title_descr}",
+                "type": "text",
+                "column_name": "title_descr"
+            },
+            {
+                "path": "{title_glseg}",
+                "type": "text",
+                "column_name": "title_glseg"
+            },
+            {
+                "path": "{title_over}",
+                "type": "text",
+                "column_name": "title_over"
+            },
+            {
+                "path": "{ee_code}",
+                "type": "text",
+                "column_name": "ee_code"
+            },
+            {
+                "path": "{ee_glseg}",
+                "type": "text",
+                "column_name": "ee_glseg"
+            },
+            {
+                "path": "{ee_over}",
+                "type": "text",
+                "column_name": "ee_over"
+            },
+            {
+                "path": "{acct_type_code}",
+                "type": "text",
+                "column_name": "acct_type_code"
+            },
+            {
+                "path": "{hours}",
+                "type": "numeric",
+                "column_name": "hours"
+            },
+            {
+                "path": "{nat_code}",
+                "type": "text",
+                "column_name": "nat_code"
+            },
+            {
+                "path": "{nat_over}",
+                "type": "text",
+                "column_name": "nat_over"
+            },
+            {
+                "path": "{gl_ref}",
+                "type": "text",
+                "column_name": "gl_ref"
+            },
+            {
+                "path": "{gl_group}",
+                "type": "text",
+                "column_name": "gl_group"
+            },
+            {
+                "path": "{gl_descr}",
+                "type": "text",
+                "column_name": "gl_descr"
+            },
+            {
+                "path": "{gl_code}",
+                "type": "text",
+                "column_name": "gl_code"
+            },
+            {
+                "path": "{gl_amount}",
+                "type": "numeric",
+                "column_name": "gl_amount"
+            },
+            {
+                "path": "{pp_code}",
+                "type": "text",
+                "column_name": "pp_code"
+            },
+            {
+                "path": "{pp_descr}",
+                "type": "text",
+                "column_name": "pp_descr"
+            },
+            {
+                "path": "{pp_gl}",
+                "type": "text",
+                "column_name": "pp_gl"
+            },
+            {
+                "path": "{pp_over}",
+                "type": "text",
+                "column_name": "pp_over"
+            },
+            {
+                "path": "{transaction}",
+                "type": "text",
+                "column_name": "transaction"
+            }
+        ],
+        "mapped":[
+            {
+                "path": "{perd_start}",
+                "type": "date",
+                "column_name": "perd_start"
+            },
+            {
+                "path": "{perd_end}",
+                "type": "date",
+                "column_name": "perd_end"
+            },
+            {
+                "path": "{check_date}",
+                "type": "date",
+                "column_name": "check_date"
+            },
+            {
+                "path": "{loc_code}",
+                "type": "text",
+                "column_name": "loc_code"
+            },
+            {
+                "path": "{loc_descr}",
+                "type": "text",
+                "column_name": "loc_descr"
+            },
+            {
+                "path": "{loc_glseg}",
+                "type": "text",
+                "column_name": "loc_glseg"
+            },
+            {
+                "path": "{loc_over}",
+                "type": "text",
+                "column_name": "loc_over"
+            },
+            {
+                "path": "{dep_code}",
+                "type": "text",
+                "column_name": "dep_code"
+            },
+            {
+                "path": "{dep_descr}",
+                "type": "text",
+                "column_name": "dep_descr"
+            },
+            {
+                "path": "{dep_nat}",
+                "type": "text",
+                "column_name": "dep_nat"
+            },
+            {
+                "path": "{dep_over}",
+                "type": "text",
+                "column_name": "dep_over"
+            },
+            {
+                "path": "{di_code}",
+                "type": "text",
+                "column_name": "di_code"
+            },
+            {
+                "path": "{di_descr}",
+                "type": "text",
+                "column_name": "di_descr"
+            },
+            {
+                "path": "{di_glseg}",
+                "type": "text",
+                "column_name": "di_glseg"
+            },
+            {
+                "path": "{di_over}",
+                "type": "text",
+                "column_name": "di_over"
+            },
+            {
+                "path": "{title_code}",
+                "type": "text",
+                "column_name": "title_code"
+            },
+            {
+                "path": "{title_descr}",
+                "type": "text",
+                "column_name": "title_descr"
+            },
+            {
+                "path": "{title_glseg}",
+                "type": "text",
+                "column_name": "title_glseg"
+            },
+            {
+                "path": "{title_over}",
+                "type": "text",
+                "column_name": "title_over"
+            },
+            {
+                "path": "{ee_code}",
+                "type": "text",
+                "column_name": "ee_code"
+            },
+            {
+                "path": "{ee_glseg}",
+                "type": "text",
+                "column_name": "ee_glseg"
+            },
+            {
+                "path": "{ee_over}",
+                "type": "text",
+                "column_name": "ee_over"
+            },
+            {
+                "path": "{acct_type_code}",
+                "type": "text",
+                "column_name": "acct_type_code"
+            },
+            {
+                "path": "{hours}",
+                "type": "numeric",
+                "column_name": "hours"
+            },
+            {
+                "path": "{nat_code}",
+                "type": "text",
+                "column_name": "nat_code"
+            },
+            {
+                "path": "{nat_over}",
+                "type": "text",
+                "column_name": "nat_over"
+            },
+            {
+                "path": "{gl_ref}",
+                "type": "text",
+                "column_name": "gl_ref"
+            },
+            {
+                "path": "{gl_group}",
+                "type": "text",
+                "column_name": "gl_group"
+            },
+            {
+                "path": "{gl_descr}",
+                "type": "text",
+                "column_name": "gl_descr"
+            },
+            {
+                "path": "{gl_code}",
+                "type": "text",
+                "column_name": "gl_code"
+            },
+            {
+                "path": "{gl_amount}",
+                "type": "numeric",
+                "column_name": "gl_amount"
+            },
+            {
+                "path": "{pp_code}",
+                "type": "text",
+                "column_name": "pp_code"
+            },
+            {
+                "path": "{pp_descr}",
+                "type": "text",
+                "column_name": "pp_descr"
+            },
+            {
+                "path": "{pp_gl}",
+                "type": "text",
+                "column_name": "pp_gl"
+            },
+            {
+                "path": "{pp_over}",
+                "type": "text",
+                "column_name": "pp_over"
+            },
+            {
+                "path": "{transaction}",
+                "type": "text",
+                "column_name": "transaction"
+            },
+            {
+                "path": "{trial_bal}",
+                "type": "text",
+                "column_name": "trial_bal"
+            },
+            {
+                "path": "{third_party}",
+                "type": "text",
+                "column_name": "third_party"
+            },
+            {
+                "path": "{check_month}",
+                "type": "jsonb",
+                "column_name": "check_month"
+            }
+        ]
+    }
+}

database/deploy/reload/paycom/vals.json

@@ -0,0 +1,139 @@
+[
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "65"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "SLS"
+        }
+    },
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "59"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "SEB"
+        }
+    },
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "93"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "MDF"
+        }
+    },
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "93"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "ELY"
+        }
+    },
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "93"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "TWN"
+        }
+    },
+    {
+        "map": "Extract month and code",
+        "mapped": {
+            "trial_bal": "88"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "loc_code_rx": "SPK"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "PRINCIPAL LIFE P"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "4.Liab",
+            "acct_type_code": "40P"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "PRINCIPAL LIFE P"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "4.Liab",
+            "acct_type_code": "CUF"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "PRINCIPAL LIFE P"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "4.Liab",
+            "acct_type_code": "K4L"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "PRINCIPAL LIFE P"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "5.Liab",
+            "acct_type_code": "40P"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "PRINCIPAL LIFE P"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "5.Liab",
+            "acct_type_code": "CUF"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "Basic NEO"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "4.Liab",
+            "acct_type_code": "FSA"
+        }
+    },
+    {
+        "map": "SUBSEQUENT PAYOUT",
+        "mapped": {
+            "third_party": "Basic NEO"
+        },
+        "source": "PAYCOM",
+        "ret_val": {
+            "gl_group": "4.Liab",
+            "acct_type_code": "FSD"
+        }
+    }
+]

database/deploy/reload/pncc/extract.sql

@@ -0,0 +1,32 @@
+--transactions with date in download format for constraint
+COPY
+(
+SELECT
+	to_char(r."AsOfDate",'mm/dd/yyyy') "AsOfDate"
+	,r."BankId"
+	,r."AccountNumber"
+	,r."AccountName"
+	,r."BaiControl"
+	,r."Currency"
+	,r."Transaction"
+	,r."Reference"
+	,r."Amount"
+	,r."Description"
+FROM
+	tps.trans
+	JOIN LATERAL jsonb_populate_record(NULL::tps.pncc, rec) r ON TRUE
+WHERE
+	srce = 'PNCC'
+)
+TO 'C:\users\ptrowbridge\downloads\pncc.csv' WITH (format csv, header TRUE)
+
+--source
+SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCC'
+
+--mapdef
+SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCC') x
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCC') X
+
+

database/deploy/reload/pncc/load.cmd

@@ -0,0 +1,4 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set
+curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set
+curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncc.csv http://localhost:81/import?srce=PNCC

database/deploy/reload/pncc/map.json

@@ -0,0 +1,455 @@
+[
+    {
+        "name": "Check Number",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "field": "checkn",
+                    "regex": "[^0-9]*([0-9]*)\\s|$",
+                    "retain": "y"
+                }
+            ],
+            "where": [
+                {
+                    "Transaction": "Checks Paid"
+                }
+            ],
+            "function": "extract"
+        },
+        "sequence": 2
+    },
+    {
+        "name": "Strip Amount Commas",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Amount}",
+                    "map": "n",
+                    "flag": "g",
+                    "field": "amount",
+                    "regex": ",",
+                    "retain": "y",
+                    "replace": ""
+                }
+            ],
+            "name": "Strip Amount Commas",
+            "where": [
+                {}
+            ],
+            "function": "replace",
+            "description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric"
+        },
+        "sequence": 1
+    },
+    {
+        "name": "Trans Type",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{AccountName}",
+                    "map": "y",
+                    "field": "acctn",
+                    "regex": "(.*)",
+                    "retain": "n"
+                },
+                {
+                    "key": "{Transaction}",
+                    "map": "y",
+                    "field": "trans",
+                    "regex": "(.*)",
+                    "retain": "n"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "y",
+                    "field": "ini",
+                    "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
+                    "retain": "y"
+                }
+            ],
+            "name": "Trans Type",
+            "where": [
+                {}
+            ],
+            "function": "extract",
+            "description": "extract intial description in conjunction with account name and transaction type for mapping"
+        },
+        "sequence": 1
+    },
+    {
+        "name": "Currency",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Description}",
+                    "map": "y",
+                    "field": "ini",
+                    "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "y",
+                    "field": "curr1",
+                    "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "y",
+                    "field": "curr2",
+                    "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*",
+                    "retain": "y"
+                }
+            ],
+            "name": "Currency",
+            "where": [
+                {
+                    "Transaction": "Miscellaneous Credits"
+                },
+                {
+                    "Transaction": "Miscellaneous Debits"
+                }
+            ],
+            "function": "extract",
+            "description": "pull out currency indicators from description of misc items and map"
+        },
+        "sequence": 2
+    },
+    {
+        "name": "Parse ACH Credits",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "beneficiary",
+                    "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Cust ID",
+                    "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Desc",
+                    "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "originator",
+                    "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Batch Discr",
+                    "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Comp ID",
+                    "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Addenda",
+                    "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "SETT",
+                    "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Date",
+                    "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Time",
+                    "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                }
+            ],
+            "name": "Parse ACH Credits",
+            "where": [
+                {
+                    "Transaction": "ACH Credits"
+                }
+            ],
+            "function": "extract",
+            "description": "parse select components of the description for ACH Credits Receieved"
+        },
+        "sequence": 2
+    },
+    {
+        "name": "Parse ACH Debits",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "originator",
+                    "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Cust ID",
+                    "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Desc",
+                    "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "beneficiary",
+                    "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Batch Discr",
+                    "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Comp ID",
+                    "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Addenda",
+                    "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "SETT",
+                    "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Date",
+                    "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "Time",
+                    "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
+                    "retain": "y"
+                }
+            ],
+            "name": "Parse ACH Debits",
+            "where": [
+                {
+                    "Transaction": "ACH Debits"
+                }
+            ],
+            "function": "extract",
+            "description": "parse select components of the description for ACH Credits Receieved"
+        },
+        "sequence": 2
+    },
+    {
+        "name": "Parse Wires",
+        "srce": "PNCC",
+        "regex": {
+            "defn": [
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "g",
+                    "field": "dparse",
+                    "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "beneficiary_components",
+                    "regex": "BENEFICIARY:(.*?)AC/([\\d-]*) (.*)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "originator_components",
+                    "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "beneficiary",
+                    "regex": "BENEFICIARY:(.*?)AC/[\\d-]* .*(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "originator",
+                    "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "OBI",
+                    "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "RFB",
+                    "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "ABA",
+                    "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "BBI",
+                    "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "BENEBNK",
+                    "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "IBK",
+                    "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "RATE",
+                    "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                },
+                {
+                    "key": "{Description}",
+                    "map": "n",
+                    "flag": "",
+                    "field": "RECVBNK",
+                    "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)",
+                    "retain": "y"
+                }
+            ],
+            "name": "Parse Wires",
+            "where": [
+                {
+                    "Transaction": "Money Transfer DB - Wire"
+                },
+                {
+                    "Transaction": "Money Transfer DB - Other"
+                },
+                {
+                    "Transaction": "Money Transfer CR-Wire"
+                },
+                {
+                    "Transaction": "Money Transfer CR-Other"
+                },
+                {
+                    "Transaction": "Intl Money Transfer Debits"
+                },
+                {
+                    "Transaction": "Intl Money Transfer Credits"
+                }
+            ],
+            "function": "extract",
+            "description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered"
+        },
+        "sequence": 2
+    }
+]

database/deploy/reload/pncc/srce.json

@@ -0,0 +1,67 @@
+{
+    "name": "PNCC",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{AsOfDate}"
+    ],
+    "schemas": {
+        "default": [
+            {
+                "path": "{AsOfDate}",
+                "type": "date",
+                "column_name": "AsOfDate"
+            },
+            {
+                "path": "{BankId}",
+                "type": "text",
+                "column_name": "BankId"
+            },
+            {
+                "path": "{AccountNumber}",
+                "type": "text",
+                "column_name": "AccountNumber"
+            },
+            {
+                "path": "{AccountName}",
+                "type": "text",
+                "column_name": "AccountName"
+            },
+            {
+                "path": "{BaiControl}",
+                "type": "text",
+                "column_name": "BaiControl"
+            },
+            {
+                "path": "{Currency}",
+                "type": "text",
+                "column_name": "Currency"
+            },
+            {
+                "path": "{Transaction}",
+                "type": "text",
+                "column_name": "Transaction"
+            },
+            {
+                "path": "{Reference}",
+                "type": "text",
+                "column_name": "Reference"
+            },
+            {
+                "path": "{Amount}",
+                "type": "text",
+                "column_name": "Amount"
+            },
+            {
+                "path":"{Description}",
+                "type": "text",
+                "column_name": "Description"
+            },
+            {
+                "path": "{AdditionalRemittance}",
+                "type": "text",
+                "column_name": "AdditionalRemittance"
+            }
+        ]
+    }
+}

database/deploy/reload/pncc/vals.json

@@ -0,0 +1,952 @@
+[
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "party": "PNC",
+            "ledger": "Manual",
+            "reason": "Bank Fees",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CANADA TAX",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Detail Debit Adjustments"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - ACH",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH DEBIT SETTLEMENT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ACH Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Returned Deposit RTM",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "RET DEP ITEM RTM",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Deposited Items Returned"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Returned Deposit STOP",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "RET DEP ITEM STOP",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Deposited Items Returned"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CREDIT ADJUSTMENT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Detail Credit Adjustments"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Returned Check",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "REFER TO MAKER OF CK RETURN CK",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Detail Credit Adjustments"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Payroll Adjustment",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEBIT ADJUSTMENT",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "Detail Debit Adjustments"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Detail Deposits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Wire",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "INTL WIRE OUT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Intl Money Transfer Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Wire",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "INTL WIRES OUT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Intl Money Transfer Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "WHLS LBX DEP",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Lockbox Deposits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "WHLS LBX DEP932855",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Lockbox Deposits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Revolver Advance",
+            "trantype": "Revolver Borrow"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ADVANCE",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Miscellaneous Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT:",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Misc Credit",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "MISC CREDIT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Miscellaneous Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Revolver Payment",
+            "trantype": "Revolver Borrow"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PAYMENT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Revolver Payment",
+            "trantype": "Revolver Payment"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PAYMENT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "INTTL WIRES IN",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Intl Money Transfer Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Revolver Advance",
+            "trantype": "Revolver Borrow"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PNC BANK- NJ LOAN PROCEEDS",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Money Transfer CR-Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Revolver Advance",
+            "trantype": "Revolver Borrow"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PNC BANK-PGH LOAN PROCEEDS",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Money Transfer CR-Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FED WIRE IN",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Money Transfer CR-Wire"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Returned Wires",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FED WIRE IN",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Money Transfer CR-Wire"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Returned Item",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "BOOK TRANSFER DEBIT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Money Transfer DB - Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Freight Wires",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FED WIRE OUT",
+            "acctn": "The HC Operating Company FREIG",
+            "trans": "Money Transfer DB - Wire"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Wire",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FED WIRE OUT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Money Transfer DB - Wire"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FED WIRE OUT",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "Money Transfer DB - Wire"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "ZBA Funding",
+            "trantype": "Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FUNDS TRANSFER FROM ACCT",
+            "acctn": "The HC Operating Company FREIG",
+            "trans": "ZBA Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "ZBA Funding",
+            "trantype": "Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FUNDS TRANSFER FROM ACCT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ZBA Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "ZBA Funding",
+            "trantype": "Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FUNDS TRANSFER FROM ACCT",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "ZBA Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "ZBA Funding",
+            "trantype": "Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FUNDS TRANSFER TO ACCT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ZBA Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "ZBA Funding",
+            "trantype": "Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "FUNDS TRANSFER TO ACCT",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "ZBA Debits"
+        }
+    },
+    {
+        "map": "Currency",
+        "mapped": {
+            "party": "The HC Canada Operating Company, Ltd.",
+            "ledger": "Manual",
+            "reason": "IC - Can to US Settlement",
+            "trantype": "Interco Collection"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT:",
+            "curr1": [
+                "CREDIT",
+                "USD"
+            ],
+            "curr2": [
+                "DEBIT",
+                "CAD"
+            ]
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "party": "PNC",
+            "ledger": "Manual",
+            "reason": "Bank Fees",
+            "trantype": "Fees"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CORPORATE ACCOUNT ANALYSIS CHARGE",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Miscellaneous Fees"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "party": "PNC",
+            "ledger": "Manual",
+            "reason": "Bank Fees",
+            "trantype": "Fees"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PNC MERCHANT FINCL ADJ",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Fees"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Revolver Payment",
+            "trantype": "Revolver Payment"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "PNC BANK- NJ LOAN PMTS",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "WITHDRAWAL:",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Miscellaneous Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Returned Item",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "BOOK TRANSFER CREDIT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Money Transfer CR-Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Returned Item",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "BOOK TRANSFER CREDIT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Money Transfer CR-Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Returned Item",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "BOOK TRANSFER CREDIT GHFTDD DDA CREDIT",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Money Transfer CR-Other"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Payroll Credits",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "19UDV",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "ACH Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH CREDIT RECEIVED",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "ACH Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "Payroll Credits",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH CREDIT RECEIVED",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "ACH Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "reason": "AP ACH Returned",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH CREDIT RETURN",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ACH Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Auto ACH Out",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH DEBIT RECEIVED",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "ACH Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Auto ACH Out",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH DEBIT RECEIVED",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ACH Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Auto ACH Out",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH DEBIT RECEIVED",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "ACH Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Check Run",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CASHED CHECK",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Payroll Checks",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CASHED CHECK",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Freight Checks",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CHECK",
+            "acctn": "The HC Operating Company FREIG",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Check Run",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CHECK",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Payroll Checks",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CHECK",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Freight Checks",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "SUBSTITUTE CHK",
+            "acctn": "The HC Operating Company FREIG",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "AP - Check Run",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "SUBSTITUTE CHK",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Returned Deposit NSF",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "RET DEP ITEM NSF UN",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Deposited Items Returned"
+        }
+    },
+    {
+        "map": "Currency",
+        "mapped": {
+            "party": "The HC Canada Operating Company, Ltd.",
+            "ledger": "Manual",
+            "reason": "IC - US to CAN Settlement",
+            "trantype": "Interco Funding"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "WITHDRAWAL:",
+            "curr1": [
+                "DEBIT",
+                "USD"
+            ],
+            "curr2": [
+                "CREDIT",
+                "CAD"
+            ]
+        }
+    },
+    {
+        "map": "Currency",
+        "mapped": {
+            "party": "The HC Canada Operating Company, Ltd.",
+            "ledger": "Manual",
+            "reason": "IC - round-trip settlement return",
+            "trantype": "Interco Collection"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT:",
+            "curr1": [
+                "CREDIT",
+                "USD"
+            ],
+            "curr2": [
+                "DEBIT",
+                "USD"
+            ]
+        }
+    },
+    {
+        "map": "Currency",
+        "mapped": {
+            "party": "The HC Canada Operating Company, Ltd.",
+            "ledger": "Manual",
+            "reason": "IC - Can to US Settlement",
+            "trantype": "Interco Collection"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT:",
+            "curr1": [
+                "DEBIT",
+                "USD"
+            ],
+            "curr2": [
+                "CREDIT",
+                "CAD"
+            ]
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "Manual",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "DEPOSIT:",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "Miscellaneous Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "reason": "Payroll Checks",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "SUBSTITUTE CHK",
+            "acctn": "The HC Operating Company PAYR",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "manual",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH DEBIT RETURN",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ACH Debits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "manual",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "ACH CREDIT SETTLEMENT",
+            "acctn": "The HC Operating Company OPERA",
+            "trans": "ACH Credits"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "-1",
+            "ledger": "Manual",
+            "trantype": "Disbursement"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "CHECK",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Checks Paid"
+        }
+    },
+    {
+        "map": "Trans Type",
+        "mapped": {
+            "sign": "1",
+            "ledger": "AR - Collections",
+            "trantype": "Collections"
+        },
+        "source": "PNCC",
+        "ret_val": {
+            "ini": "POSTING CORRECTION RETURN CK",
+            "acctn": "The HC Operating Company FBO P",
+            "trans": "Detail Credit Adjustments"
+        }
+    }
+]

database/deploy/reload/pncl/extract.sql

@@ -0,0 +1,30 @@
+--transactions with date in download format for constraint
+--transactions with date in download format for constraint
+COPY
+(
+SELECT
+	r."Schedule#"
+	,to_char(r."PostDate",'mm/dd/yyyy') "PostDate"
+	,r."Assn#"
+	,r."Coll#"
+	,COALESCE(r."AdvanceRate",0) "AdvanceRate"
+	,COALESCE(r."Sales",0) "Sales"
+	,COALESCE(r."Credits & Adjustments",0) "Credits & Adjustments"
+	,COALESCE(r."Gross Collections",0) "Gross Collections"
+	,COALESCE(r."CollateralBalance",0) "CollateralBalance"
+	,COALESCE(r."MaxEligible",0) "MaxEligible"
+	,COALESCE(r."Ineligible Amount",0) "Ineligible Amount"
+	,COALESCE(r."Reserve Amount",0) "Reserve Amount"
+FROM
+	tps.trans
+	JOIN LATERAL jsonb_populate_record(NULL::tps.pncl, rec) r ON TRUE
+WHERE
+	srce = 'PNCL'
+    --and case when rec->>'Credits & Adjustments' is null then 'null' else '' end <> 'null'
+)
+TO 'C:\users\ptrowbridge\downloads\pncl.csv' WITH (format csv, header TRUE)
+
+--source
+SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCL'
+
+

database/deploy/reload/pncl/load.cmd

@@ -0,0 +1,2 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncl.csv http://localhost:81/import?srce=PNCL

database/deploy/reload/pncl/srce.json

@@ -0,0 +1,73 @@
+{
+    "constraint": [
+        "{PostDate}",
+        "{Schedule#}"
+    ],
+    "source": "client_file",
+    "loading_function": "csv",
+    "name": "PNCL",
+    "schemas": {
+        "default": [
+            {
+                "path": "{Schedule#}",
+                "type": "text",
+                "column_name": "Schedule#"
+            },
+            {
+                "type": "date",
+                "column_name": "PostDate",
+                "path": "{PostDate}"
+            },
+            {
+                "type": "text",
+                "column_name": "Assn#",
+                "path": "{Assn#}"
+            },
+            {
+                "type": "text",
+                "column_name": "Coll#",
+                "path": "{Coll#}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "AdvanceRate",
+                "path": "{AdvanceRate}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "Sales",
+                "path": "{Sales}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "Credits & Adjustments",
+                "path": "{Credits & Adjustments}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "Gross Collections",
+                "path": "{Gross Collections}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "CollateralBalance",
+                "path": "{CollateralBalance}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "MaxEligible",
+                "path": "{MaxEligible}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "Ineligible Amount",
+                "path": "{Ineligible Amount}"
+            },
+            {
+                "type": "numeric",
+                "column_name": "Reserve Amount",
+                "path": "{Reserve Amount}"
+            }
+        ]
+    }
+}

database/deploy/reload/pnco/extract.sql

@@ -0,0 +1,32 @@
+--transactions with date in download format for constraint
+--transactions with date in download format for constraint
+COPY
+(
+SELECT
+	r."Loan#"
+	,to_char(r."Post Date",'mm/dd/yyyy') "Post Date"
+	,to_char(r."Effective Date",'mm/dd/yyyy') "Effective Date"
+	,r."Reference #"
+	,r."Description"
+	,r."Advances"
+	,r."Adjustments"
+	,r."Payments"
+	,r."Loan Balance"
+FROM
+	tps.trans
+	JOIN LATERAL jsonb_populate_record(NULL::tps.pnco, rec) r ON TRUE
+WHERE
+	srce = 'PNCO'
+)
+TO 'C:\users\ptrowbridge\downloads\pnco.csv' WITH (format csv, header TRUE)
+
+--source
+SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCO'
+
+--mapdef
+SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCO') x
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCO') X
+
+

database/deploy/reload/pnco/load.cmd

@@ -0,0 +1,2 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pnco.csv http://localhost:81/import?srce=PNCO

database/deploy/reload/pnco/srce.json

@@ -0,0 +1,60 @@
+{
+    "name": "PNCO",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{Post Date}",
+        "{Effective Date}",
+        "{Loan#}",
+        "{Reference #}"
+    ],
+    "schemas": {
+        "default": [
+            {
+                "path": "{Loan#}",
+                "type": "text",
+                "column_name":"Loan#"
+            },
+            {
+                "path": "{Post Date}",
+                "type": "date",
+                "column_name":"Post Date"
+            },
+            {
+                "path": "{Effective Date}",
+                "type": "date",
+                "column_name":"Effective Date"
+            },
+            {
+                "path": "{Reference #}",
+                "type": "text",
+                "column_name":"Reference #"
+            },
+            {
+                "path": "{Description}",
+                "type": "text",
+                "column_name":"Description"
+            },
+            {
+                "path": "{Advances}",
+                "type": "numeric",
+                "column_name":"Advances"
+            },
+            {
+                "path": "{Adjustments}",
+                "type": "numeric",
+                "column_name":"Adjustments"
+            },
+            {
+                "path": "{Payments}",
+                "type": "numeric",
+                "column_name":"Payments"
+            },
+            {
+                "path": "{Loan Balance}",
+                "type": "numeric",
+                "column_name":"Loan Balance"
+            }
+        ]
+    }
+}

database/deploy/reload/wmpd/curl_hunt.cmd

@@ -0,0 +1,3 @@
+curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
+curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set
+curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/WMPD.csv http://localhost:81/import?srce=WMPD

database/deploy/reload/wmpd/extract.sql

@@ -0,0 +1,63 @@
+--source
+COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'WMPD') TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\srce.json' WITH (FORMAT TEXT, HEADER FALSE)
+
+--mapdef
+COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'WMPD') x) TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\map.json' WITH (FORMAT TEXT, HEADER FALSE)
+
+--map values
+SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'WMPD') X
+
+--records
+copy (
+    select 
+        r."Carrier",
+        r."SCAC",
+        r."Mode",
+        r."Pro #",
+        r."B/L",
+        r."Pd Amt",
+        r."Loc#",
+        r."Pcs",
+        r."Wgt",
+        r."Chk#",
+        r."Pay Dt",
+        r."Acct #",
+        r."I/O",
+        r."Sh Nm",
+        r."Sh City",
+        r."Sh St",
+        r."Sh Zip",
+        r."Cons Nm",
+        r."D City ",
+        r."D St",
+        r."D Zip",
+        r."Sh Dt",
+        r."Inv Dt",
+        r."Customs Entry#",
+        r."Miles",
+        r."Frt Class",
+        r."Master B/L"
+    from
+        tps.trans 
+        join lateral jsonb_populate_record(null::tps.WMPD, rec) r on true 
+    where 
+        srce = 'WMPD'
+	order by 
+		r."Pay Dt" asc
+) to 
+'C:\users\ptrowbridge\downloads\WMPD.csv' with (format csv, header true);
+
+--rebuild source def to include PATH
+SELECT
+	ae.r
+	||jsonb_build_object(
+		'path',
+		(
+			'{'||(ae.r->>'column_name')||'}'
+		)
+	)
+FROM
+	tps.srce
+	JOIN LATERAL jsonb_array_elements(defn->'schemas'->'default') ae(r) ON TRUE
+WHERE
+	srce = 'WMPD'

database/deploy/reload/wmpd/srce.json

@@ -0,0 +1,148 @@
+{
+    "name": "WMPD",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{Pay Dt}",
+        "{Carrier}"
+    ],
+    "schemas": {
+        "default": [
+            {
+                "path": "{Carrier}",
+                "type": "text",
+                "column_name": "Carrier"
+            },
+            {
+                "path": "{SCAC}",
+                "type": "text",
+                "column_name": "SCAC"
+            },
+            {
+                "path": "{Mode}",
+                "type": "text",
+                "column_name": "Mode"
+            },
+            {
+                "path": "{Pro #}",
+                "type": "text",
+                "column_name": "Pro #"
+            },
+            {
+                "path": "{B/L}",
+                "type": "text",
+                "column_name": "B/L"
+            },
+            {
+                "path": "{Pd Amt}",
+                "type": "numeric",
+                "column_name": "Pd Amt"
+            },
+            {
+                "path": "{Loc#}",
+                "type": "text",
+                "column_name": "Loc#"
+            },
+            {
+                "path": "{Pcs}",
+                "type": "numeric",
+                "column_name": "Pcs"
+            },
+            {
+                "path": "{Wgt}",
+                "type": "numeric",
+                "column_name": "Wgt"
+            },
+            {
+                "path": "{Chk#}",
+                "type": "numeric",
+                "column_name": "Chk#"
+            },
+            {
+                "path": "{Pay Dt}",
+                "type": "date",
+                "column_name": "Pay Dt"
+            },
+            {
+                "path": "{Acct #}",
+                "type": "text",
+                "column_name": "Acct #"
+            },
+            {
+                "path": "{I/O}",
+                "type": "text",
+                "column_name": "I/O"
+            },
+            {
+                "path": "{Sh Nm}",
+                "type": "text",
+                "column_name": "Sh Nm"
+            },
+            {
+                "path": "{Sh City}",
+                "type": "text",
+                "column_name": "Sh City"
+            },
+            {
+                "path": "{Sh St}",
+                "type": "text",
+                "column_name": "Sh St"
+            },
+            {
+                "path": "{Sh Zip}",
+                "type": "text",
+                "column_name": "Sh Zip"
+            },
+            {
+                "path": "{Cons Nm}",
+                "type": "text",
+                "column_name": "Cons Nm"
+            },
+            {
+                "path": "{D City }",
+                "type": "text",
+                "column_name": "D City "
+            },
+            {
+                "path": "{D St}",
+                "type": "text",
+                "column_name": "D St"
+            },
+            {
+                "path": "{D Zip}",
+                "type": "text",
+                "column_name": "D Zip"
+            },
+            {
+                "path": "{Sh Dt}",
+                "type": "date",
+                "column_name": "Sh Dt"
+            },
+            {
+                "path": "{Inv Dt}",
+                "type": "date",
+                "column_name": "Inv Dt"
+            },
+            {
+                "path": "{Customs Entry#}",
+                "type": "text",
+                "column_name": "Customs Entry#"
+            },
+            {
+                "path": "{Miles}",
+                "type": "numeric",
+                "column_name": "Miles"
+            },
+            {
+                "path": "{Frt Class}",
+                "type": "text",
+                "column_name": "Frt Class"
+            },
+            {
+                "path": "{Master B/L}",
+                "type": "text",
+                "column_name": "Master B/L"
+            }
+        ]
+    }
+}

database/interface/import/map_trigger.sql

@@ -0,0 +1,257 @@
+CREATE OR REPLACE FUNCTION tps.trans_insert_map() RETURNS TRIGGER 
+AS 
+$f$
+    DECLARE
+        _cnt INTEGER;
+
+    BEGIN
+        IF (TG_OP = 'INSERT') THEN
+
+
+            WITH
+            --------------------apply regex operations to transactions-----------------------------------------------------------------------------------
+
+            rx AS (
+                SELECT 
+                    t.srce,
+                    t.id,
+                    t.rec,
+                    m.target,
+                    m.seq,
+                    regex->'regex'->>'function' regex_function,
+                    e.v ->> 'field' result_key_name,
+                    e.v ->> 'key' target_json_path,
+                    e.v ->> 'flag' regex_options_flag,
+                    e.v->>'map' map_intention,
+                    e.v->>'retain' retain_result,
+                    e.v->>'regex' regex_expression,
+                    e.rn target_item_number,
+                    COALESCE(mt.rn,rp.rn,1) result_number,
+                    mt.mt rx_match,
+                    rp.rp rx_replace,
+                    CASE e.v->>'map'
+                        WHEN 'y' THEN
+                            e.v->>'field'
+                        ELSE
+                            null
+                    END map_key,
+                    CASE e.v->>'map'
+                        WHEN 'y' THEN
+                            CASE regex->'regex'->>'function'
+                                WHEN 'extract' THEN
+                                    CASE WHEN array_upper(mt.mt,1)=1 
+                                        THEN to_json(mt.mt[1])
+                                        ELSE array_to_json(mt.mt)
+                                    END::jsonb
+                                WHEN 'replace' THEN
+                                    to_jsonb(rp.rp)
+                                ELSE
+                                    '{}'::jsonb
+                            END
+                        ELSE
+                            NULL
+                    END map_val,
+                    CASE e.v->>'retain'
+                        WHEN 'y' THEN
+                            e.v->>'field'
+                        ELSE
+                            NULL
+                    END retain_key,
+                    CASE e.v->>'retain'
+                        WHEN 'y' THEN
+                            CASE regex->'regex'->>'function'
+                                WHEN 'extract' THEN
+                                    CASE WHEN array_upper(mt.mt,1)=1 
+                                        THEN to_json(trim(mt.mt[1]))
+                                        ELSE array_to_json(mt.mt)
+                                    END::jsonb
+                                WHEN 'replace' THEN
+                                    to_jsonb(rtrim(rp.rp))
+                                ELSE
+                                    '{}'::jsonb
+                            END
+                        ELSE
+                            NULL
+                    END retain_val
+                FROM 
+                    --------------------------start with all regex maps------------------------------------------------------------------------------------
+                    tps.map_rm m
+                    --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+                    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+                    --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+                    INNER JOIN new_table t ON 
+                        t.srce = m.srce AND
+                        t.rec @> w.v
+                    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+                    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+                    --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+                    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+                        m.regex->'regex'->>'function' = 'extract'
+                    --------------------------same as above but for a replacement type function------------------------------------------------------------
+                    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+                        m.regex->'regex'->>'function' = 'replace'
+                ORDER BY 
+                    t.id DESC,
+                    m.target,
+                    e.rn,
+                    COALESCE(mt.rn,rp.rn,1)
+            )
+
+            --SELECT count(*) FROM rx LIMIT 100
+
+
+            , agg_to_target_items AS (
+            SELECT 
+                srce
+                ,id
+                ,target
+                ,seq
+                ,map_intention
+                ,regex_function
+                ,target_item_number
+                ,result_key_name
+                ,target_json_path
+                ,CASE WHEN map_key IS NULL 
+                    THEN    
+                        NULL 
+                    ELSE 
+                        jsonb_build_object(
+                            map_key,
+                            CASE WHEN max(result_number) = 1
+                                THEN
+                                    jsonb_agg(map_val ORDER BY result_number) -> 0
+                                ELSE
+                                    jsonb_agg(map_val ORDER BY result_number)
+                            END
+                        ) 
+                END map_val
+                ,CASE WHEN retain_key IS NULL 
+                    THEN 
+                        NULL 
+                    ELSE 
+                        jsonb_build_object(
+                            retain_key,
+                            CASE WHEN max(result_number) = 1
+                                THEN
+                                    jsonb_agg(retain_val ORDER BY result_number) -> 0
+                                ELSE
+                                    jsonb_agg(retain_val ORDER BY result_number)
+                            END
+                        ) 
+                END retain_val
+            FROM 
+                rx
+            GROUP BY
+                srce
+                ,id
+                ,target
+                ,seq
+                ,map_intention
+                ,regex_function
+                ,target_item_number
+                ,result_key_name
+                ,target_json_path
+                ,map_key
+                ,retain_key
+            )
+
+            --SELECT * FROM agg_to_target_items LIMIT 100
+
+
+            , agg_to_target AS (
+            SELECT
+                srce
+                ,id
+                ,target
+                ,seq
+                ,map_intention
+                ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+                ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+            FROM
+                agg_to_target_items
+            GROUP BY
+                srce
+                ,id
+                ,target
+                ,seq
+                ,map_intention
+            ORDER BY
+                id
+            )
+
+
+            --SELECT * FROM agg_to_target
+
+
+            , link_map AS (
+            SELECT
+                a.srce
+                ,a.id
+                ,a.target
+                ,a.seq
+                ,a.map_intention
+                ,a.map_val
+                ,a.retain_val retain_value
+                ,v.map
+            FROM
+                agg_to_target a
+                LEFT OUTER JOIN tps.map_rv v ON
+                    v.srce = a.srce AND
+                    v.target = a.target AND
+                    v.retval = a.map_val
+            )
+
+            --SELECT * FROM link_map
+
+            , agg_to_id AS (
+            SELECT
+                srce
+                ,id
+                ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val
+                ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map
+            FROM
+                link_map
+            GROUP BY
+                srce
+                ,id
+            )
+
+            --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100
+            
+            --create a complete list of all new inserts assuming some do not have maps (left join)
+            ,join_all AS (
+                SELECT
+                    n.srce
+                    ,n.id
+                    ,n.rec
+                    ,a.retain_val parse
+                    ,a.map
+                    ,n.rec||COALESCE(a.map||a.retain_val,'{}'::jsonb) allj
+                FROM
+                    new_table n
+                    LEFT OUTER JOIN agg_to_id a ON
+                        a.id = n.id
+            )
+
+            --update trans with join_all recs
+            UPDATE
+                tps.trans t
+            SET
+                parse = a.parse
+                ,map = a.map
+                ,allj = a.allj
+            FROM
+               join_all a
+            WHERE
+                t.id = a.id;
+                
+
+        END IF;
+        RETURN NULL;
+    END;
+$f$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trans_insert
+    AFTER INSERT ON tps.trans
+    REFERENCING NEW TABLE AS new_table
+    FOR EACH STATEMENT EXECUTE PROCEDURE tps.trans_insert_map();

database/interface/import/srce_import_do.sql

@@ -0,0 +1,184 @@
+DO 
+$f$
+DECLARE
+    _t text;
+    _c text;
+    _log_info jsonb;
+    _log_id text;
+    _cnt numeric;
+    _message jsonb;
+    _recs jsonb;
+    _srce text;
+    _defn jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+
+BEGIN
+
+    _srce := 'DMAPI';
+    _recs:= $${"id":1,"doc":{"rows":[{"elements":[{"status":"OK","distance":{"text":"225 mi","value":361940},"duration":{"text":"3 hours 50 mins","value":13812}}]}],"status":"OK","origin_addresses":["Washington, DC, USA"],"destination_addresses":["New York, NY, USA"]}}$$::jsonb;
+
+----------------------------------------------------test if source exists----------------------------------------------------------------------------------
+
+    SELECT
+        defn
+    INTO
+        _defn
+    FROM
+        tps.srce    
+    WHERE
+        srce = _srce;
+
+    IF _defn IS NULL THEN
+        _message:= 
+        format(
+            $$
+                {
+                    "status":"fail",
+                    "message":"source %L does not exists"
+                }
+            $$,
+            _srce
+        )::jsonb;
+        RAISE NOTICE '%s', _message;
+    END IF;
+
+    -------------unwrap the json record and apply the path(s) of the constraint to build a constraint key per record-----------------------------------------------------------------------------------
+
+    WITH
+    pending_list AS (
+        SELECT
+            _srce srce
+            ,j.rec
+            ,j.id
+            --aggregate back to the record since multiple paths may be listed in the constraint
+            --it is unclear why the "->>0" is required to correctly extract the text array from the jsonb
+            ,tps.jsonb_concat_obj(
+                jsonb_build_object(
+                    --the new json key is the path itself
+                    cons.path->>0
+                    ,j.rec#>((cons.path->>0)::text[])
+                ) 
+            ) json_key
+        FROM
+            jsonb_array_elements(_recs) WITH ORDINALITY j(rec,id)
+            JOIN LATERAL jsonb_array_elements(_defn->'constraint') WITH ORDINALITY cons(path, seq)  ON TRUE
+        GROUP BY
+            j.rec
+            ,j.id
+    )
+
+    -----------create a unique list of keys from staged rows------------------------------------------------------------------------------------------
+
+    , pending_keys AS (
+        SELECT DISTINCT
+            json_key
+        FROM 
+            pending_list
+    )
+
+    -----------list of keys already loaded to tps-----------------------------------------------------------------------------------------------------
+
+    , matched_keys AS (
+        SELECT DISTINCT
+            k.json_key
+        FROM
+            pending_keys k
+            INNER JOIN tps.trans t ON
+                t.ic = k.json_key
+    )
+
+    -----------return unique keys that are not already in tps.trans-----------------------------------------------------------------------------------
+
+    , unmatched_keys AS (
+    SELECT
+        json_key
+    FROM
+        pending_keys
+
+    EXCEPT
+
+    SELECT
+        json_key
+    FROM
+        matched_keys
+    )
+
+    --------build log record-------------------+------------------------------------------------------------------------------------------------
+
+    , logged AS (
+    INSERT INTO
+        tps.trans_log (info)
+    SELECT
+        JSONB_BUILD_OBJECT('time_stamp',CURRENT_TIMESTAMP)
+        ||JSONB_BUILD_OBJECT('srce',_srce)
+        --||JSONB_BUILD_OBJECT('path',_path)
+        ||JSONB_BUILD_OBJECT('not_inserted',
+            (
+                SELECT 
+                    jsonb_agg(json_key)
+                FROM
+                    matched_keys
+            )
+        )
+        ||JSONB_BUILD_OBJECT('inserted',
+            (
+                SELECT 
+                    jsonb_agg(json_key)
+                FROM
+                    unmatched_keys
+            )
+        )
+    RETURNING *
+    )
+
+    -----------insert pending rows that have key with no trans match-----------------------------------------------------------------------------------
+    --need to look into mapping the transactions prior to loading
+
+    , inserted AS (
+        INSERT INTO
+            tps.trans (srce, rec, ic, logid)
+        SELECT
+            pl.srce
+            ,pl.rec
+            ,pl.json_key
+            ,logged.id
+        FROM 
+            pending_list pl
+            INNER JOIN unmatched_keys u ON
+                u.json_key = pl.json_key
+            CROSS JOIN logged
+        ORDER BY
+            pl.id ASC
+        ----this conflict is only if an exact duplicate rec json happens, which will be rejected
+        ----therefore, records may not be inserted due to ay matches with certain json fields, or if the entire json is a duplicate, reason is not specified
+        RETURNING *
+    )
+
+    SELECT
+        id
+        ,info
+    INTO
+        _log_id
+        ,_log_info
+    FROM
+        logged;
+
+    --RAISE NOTICE 'import logged under id# %, info: %', _log_id, _log_info;
+
+    _message:= 
+    (
+        $$
+            {
+            "status":"complete"
+            }
+        $$::jsonb
+    )||jsonb_build_object('details',_log_info);
+
+    RAISE NOTICE '%s', _message;
+
+END;
+$f$
+LANGUAGE plpgsql
+

database/interface/map_def/srce_map_def_set.sql

@@ -0,0 +1,110 @@
+CREATE OR REPLACE FUNCTION tps.srce_map_def_set(_defn jsonb) RETURNS jsonb
+AS
+$f$
+
+DECLARE
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+
+BEGIN
+
+    WITH 
+    ------------------------------------------stage rows to insert-----------------------------------------------------
+    stg AS (
+        SELECT
+            --data source
+            ae.r->>'srce' srce
+            --map name
+            ,ae.r->>'name' target
+            --map definition
+            ,ae.r regex
+            --map aggregation sequence
+            ,(ae.r->>'sequence')::INTEGER seq
+            --history definition
+            ,jsonb_build_object(
+                'hist_defn',ae.r
+                ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+            ) || '[]'::jsonb hist
+            --determine if the rows are new or match
+            ,(m.regex->>'regex' = ae.r->>'regex')::BOOLEAN rebuild
+        FROM
+            jsonb_array_elements(_defn) ae(r)
+            LEFT OUTER JOIN tps.map_rm m ON
+                m.srce = ae.r->>'srce'
+                AND m.target = ae.t->>'name'
+    )
+    ---------------------------------------do the upsert-------------------------------------------------------------------
+    ,ins AS (
+        INSERT INTO
+            tps.map_rm (srce, target, regex, seq, hist)
+        SELECT
+            srce
+            ,target
+            ,regex
+            ,seq
+            ,hist
+        FROM
+            stg
+        ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET
+            srce = excluded.srce
+            ,target = excluded.target
+            ,regex = excluded.regex
+            ,seq = excluded.seq
+            ,hist = 
+                    --the new definition going to position -0-
+                    jsonb_build_object(
+                        'hist_defn',excluded.regex
+                        ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+                    ) 
+                    --the previous definition, set upper bound of effective range which was previously null
+                    || jsonb_set(
+                        map_rm.hist
+                        ,'{0,effective,1}'::text[]
+                        ,to_jsonb(CURRENT_TIMESTAMP)
+                    )
+    )
+    ---------------------------get list of sources that had maps change--------------------------------------------------------
+    , to_update AS (
+    SELECT DISTINCT
+        srce
+    FROM
+        ins
+    WHERE
+        rebuild = TRUE
+    )
+    --------------------------call the map overwrite for each source and return all the messages into message----------------
+    /*the whole source must be overwritten because if an element is no longer returned it shoudl be wiped from the data*/
+    SELECT
+        jsonb_agg(x.message)
+    INTO
+        _message
+    FROM
+        to_update
+        JOIN LATERAL tps.srce_map_overwrite(to_update.srce) AS x(message) ON TRUE;
+
+    _message:= jsonb_build_object('status','complete','message','definition has been set');
+    return _message;
+
+
+    GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error setting definition"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+        return _message;
+
+
+
+END;
+$f$
+language plpgsql

database/interface/map_def/srce_map_def_set_single.sql

@@ -0,0 +1,98 @@
+CREATE OR REPLACE FUNCTION tps.srce_map_def_set_single(_defn jsonb, _rebuild BOOLEAN) RETURNS jsonb
+AS
+$f$
+
+DECLARE
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+
+BEGIN
+
+    ---------test if anythign is changing--------------------------------------------------------------------------------------------
+
+    IF _defn->'regex' = (SELECT regex->'regex' FROM tps.map_rm WHERE srce = _defn->>'srce' and target = _defn->>'name') THEN
+         _message:= 
+        (
+            $$
+                {
+                "status":"complete",
+                "message":"map was not different no action taken"
+                }
+            $$::jsonb
+        );
+        RETURN _message;
+    END IF;
+
+    ---------do the rebuild-----------------------------------------------------------------------------------------------------------
+
+    INSERT INTO
+        tps.map_rm (srce, target, regex, seq, hist)
+    SELECT
+        --data source
+        _defn->>'srce'
+        --map name
+        ,_defn->>'name'
+        --map definition
+        ,_defn
+        --map aggregation sequence
+        ,(_defn->>'sequence')::INTEGER
+        --history definition
+        ,jsonb_build_object(
+            'hist_defn',_defn
+            ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+        ) || '[]'::jsonb
+    ON CONFLICT ON CONSTRAINT map_rm_pk DO UPDATE SET
+        srce = excluded.srce
+        ,target = excluded.target
+        ,regex = excluded.regex
+        ,seq = excluded.seq
+        ,hist = 
+                --the new definition going to position -0-
+                jsonb_build_object(
+                    'hist_defn',excluded.regex
+                    ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+                ) 
+                --the previous definition, set upper bound of effective range which was previously null
+                || jsonb_set(
+                    map_rm.hist
+                    ,'{0,effective,1}'::text[]
+                    ,to_jsonb(CURRENT_TIMESTAMP)
+                );
+
+    --------------if rebuild was flag call the rebuild--------------------------------------------------------------------------------
+
+    IF _rebuild THEN
+        SELECT
+            x.message||'{"step":"overwrite maps in tps.trans"}'::jsonb
+        INTO
+            _message
+        FROM
+            tps.srce_map_overwrite(_defn->>'srce') as X(message);
+    END IF;
+
+    return _message;
+
+    EXCEPTION WHEN OTHERS THEN
+    GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+
+    _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error setting definition"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    return _message;
+
+
+
+END;
+$f$
+language plpgsql

database/interface/map_def/test_regex.sql

@@ -0,0 +1,222 @@
+DROP FUNCTION IF EXISTS tps.test_regex(jsonb);
+CREATE FUNCTION tps.test_regex(_defn jsonb) RETURNS jsonb
+LANGUAGE plpgsql
+AS
+$f$
+DECLARE
+    _rslt jsonb;
+BEGIN
+
+    WITH
+
+    --------------------apply regex operations to transactions---------------------------------------------------------------------------------
+
+    rx AS (
+        SELECT 
+            t.srce,
+            t.id,
+            t.rec,
+            m.target,
+            m.seq,
+            regex->'regex'->>'function' regex_function,
+            e.v ->> 'field' result_key_name,
+            e.v ->> 'key' target_json_path,
+            e.v ->> 'flag' regex_options_flag,
+            e.v->>'map' map_intention,
+            e.v->>'retain' retain_result,
+            e.v->>'regex' regex_expression,
+            e.rn target_item_number,
+            COALESCE(mt.rn,rp.rn,1) result_number,
+            mt.mt rx_match,
+            rp.rp rx_replace,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    null
+            END map_key,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(mt.mt[1])
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rp.rp)
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END map_val,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    NULL
+            END retain_key,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(trim(mt.mt[1]))
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rtrim(rp.rp))
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END retain_val
+        FROM 
+            --------------------------start with all regex maps------------------------------------------------------------------------------------
+            (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
+            --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+            --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+            --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+            INNER JOIN tps.trans t ON 
+                t.srce = m.srce AND
+                t.rec @> w.v
+            --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+            LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+                m.regex->'regex'->>'function' = 'extract'
+            --------------------------same as above but for a replacement type function------------------------------------------------------------
+            LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+                m.regex->'regex'->>'function' = 'replace'
+        ORDER BY 
+            t.id DESC,
+            m.target,
+            e.rn,
+            COALESCE(mt.rn,rp.rn,1)
+    )
+
+    --SELECT * FROM rx LIMIT 100
+
+
+    , agg_to_target_items AS (
+    SELECT 
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,regex_function
+        ,target_item_number
+        ,result_key_name
+        ,target_json_path
+        ,CASE WHEN map_key IS NULL 
+            THEN    
+                NULL 
+            ELSE 
+                jsonb_build_object(
+                    map_key,
+                    CASE WHEN max(result_number) = 1
+                        THEN
+                            jsonb_agg(map_val ORDER BY result_number) -> 0
+                        ELSE
+                            jsonb_agg(map_val ORDER BY result_number)
+                    END
+                ) 
+        END map_val
+        ,CASE WHEN retain_key IS NULL 
+            THEN 
+                NULL 
+            ELSE 
+                jsonb_build_object(
+                    retain_key,
+                    CASE WHEN max(result_number) = 1
+                        THEN
+                            jsonb_agg(retain_val ORDER BY result_number) -> 0
+                        ELSE
+                            jsonb_agg(retain_val ORDER BY result_number)
+                    END
+                ) 
+        END retain_val
+    FROM 
+        rx
+    GROUP BY
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,regex_function
+        ,target_item_number
+        ,result_key_name
+        ,target_json_path
+        ,map_key
+        ,retain_key
+    )
+
+    --SELECT * FROM agg_to_target_items LIMIT 100
+
+
+    , agg_to_target AS (
+    SELECT
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+        ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+    FROM
+        agg_to_target_items
+    GROUP BY
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+    )
+
+
+    , agg_to_ret AS (
+    SELECT
+        srce
+        ,target
+        ,seq
+        ,map_intention
+        ,map_val
+        ,retain_val
+        ,count(*) "count"
+    FROM 
+        agg_to_target
+    GROUP BY
+        srce
+        ,target
+        ,seq
+        ,map_intention
+        ,map_val
+        ,retain_val
+    )
+    ,agg_to_id AS (
+    SELECT
+        l.srce
+        ,l.target
+        ,l.map_val
+        ,l."count"
+    FROM
+        agg_to_ret l
+    ORDER BY
+        l.srce
+        ,l.target
+        ,l."count" desc
+    )
+    SELECT
+        jsonb_agg(row_to_json(agg_to_id)::jsonb)
+    INTO
+        _rslt
+    FROM
+        agg_to_id;
+
+    RETURN _rslt;
+END;
+$f$;

database/interface/map_def/test_regex_recs.sql

@@ -0,0 +1,211 @@
+DROP FUNCTION IF EXISTS tps.test_regex_rec(jsonb);
+CREATE FUNCTION tps.test_regex_recs(_defn jsonb) RETURNS jsonb
+LANGUAGE plpgsql
+AS
+$f$
+DECLARE
+    _rslt jsonb;
+BEGIN
+
+WITH
+
+--------------------apply regex operations to transactions---------------------------------------------------------------------------------
+
+rx AS (
+        SELECT 
+            t.srce,
+            t.id,
+            t.rec,
+            m.target,
+            m.seq,
+            regex->'regex'->>'function' regex_function,
+            e.v ->> 'field' result_key_name,
+            e.v ->> 'key' target_json_path,
+            e.v ->> 'flag' regex_options_flag,
+            e.v->>'map' map_intention,
+            e.v->>'retain' retain_result,
+            e.v->>'regex' regex_expression,
+            e.rn target_item_number,
+            COALESCE(mt.rn,rp.rn,1) result_number,
+            mt.mt rx_match,
+            rp.rp rx_replace,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    null
+            END map_key,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(mt.mt[1])
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rp.rp)
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END map_val,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    NULL
+            END retain_key,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(trim(mt.mt[1]))
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rtrim(rp.rp))
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END retain_val
+        FROM 
+            --------------------------start with all regex maps------------------------------------------------------------------------------------
+            (SELECT _defn->>'srce' srce, _defn->>'name' target, _defn regex, (_defn->>'sequence')::numeric seq) m
+            --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+            --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+            --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+            INNER JOIN tps.trans t ON 
+                t.srce = m.srce AND
+                t.rec @> w.v
+            --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+            LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+                m.regex->'regex'->>'function' = 'extract'
+            --------------------------same as above but for a replacement type function------------------------------------------------------------
+            LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+                m.regex->'regex'->>'function' = 'replace'
+        ORDER BY 
+            t.id DESC,
+            m.target,
+            e.rn,
+            COALESCE(mt.rn,rp.rn,1)
+)
+
+--SELECT * FROM rx LIMIT 100
+
+
+, agg_to_target_items AS (
+SELECT 
+    srce
+    ,id
+    ,rec
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,CASE WHEN map_key IS NULL 
+        THEN    
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                map_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(map_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(map_val ORDER BY result_number)
+                END
+            ) 
+    END map_val
+    ,CASE WHEN retain_key IS NULL 
+        THEN 
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                retain_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                            jsonb_agg(retain_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(retain_val ORDER BY result_number)
+                END
+            ) 
+    END retain_val
+FROM 
+    rx
+GROUP BY
+    srce
+    ,id
+    ,rec
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,map_key
+    ,retain_key
+)
+
+--SELECT * FROM agg_to_target_items LIMIT 100
+
+
+, agg_to_target AS (
+SELECT
+    srce
+    ,id
+    ,rec
+    ,target
+    ,seq
+    ,map_intention
+    ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+    ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+FROM
+    agg_to_target_items
+GROUP BY
+    srce
+    ,id
+    ,rec
+    ,target
+    ,seq
+    ,map_intention
+)
+, agg_to_ret AS (
+SELECT
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+	,count(*) "count"
+    ,jsonb_agg(rec) rec
+FROM 
+	agg_to_target
+GROUP BY
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+)
+SELECT
+    jsonb_agg(row_to_json(agg_to_ret)::jsonb)
+INTO
+    _rslt
+FROM
+    agg_to_ret;
+RETURN _rslt;
+END;
+$f$;

database/interface/map_values/map_rv_set.sql

@@ -0,0 +1,64 @@
+DROP FUNCTION IF EXISTS tps.map_rv_set;
+CREATE OR REPLACE FUNCTION tps.map_rv_set(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+BEGIN
+    INSERT INTO
+        tps.map_rv (srce, target, retval, map, hist)
+    SELECT 
+        r.source
+        ,r.map
+        ,r.ret_val
+        ,r.mapped
+        ,jsonb_build_object(
+                'hist_defn',mapped
+                ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+            ) || '[]'::jsonb
+    FROM
+        JSONB_ARRAY_ELEMENTS(_defn) WITH ORDINALITY ae(r,s)
+        JOIN LATERAL jsonb_to_record(ae.r) r(source TEXT,map TEXT, ret_val jsonb, mapped jsonb) ON TRUE
+    ON CONFLICT ON CONSTRAINT map_rv_pk DO UPDATE
+        SET
+            map = excluded.map
+            ,hist = 
+                --the new definition going to position -0-
+                jsonb_build_object(
+                    'hist_defn',excluded.map
+                    ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+                ) 
+                --the previous definition, set upper bound of effective range which was previously null
+                || jsonb_set(
+                    map_rv.hist
+                    ,'{0,effective,1}'::text[]
+                    ,to_jsonb(CURRENT_TIMESTAMP)
+                );
+
+    -------return message--------------------------------------------------------------------------------------------------
+    _message:= jsonb_build_object('status','complete');
+    RETURN _message;
+
+EXCEPTION WHEN OTHERS THEN
+
+    GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+    _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error setting map value"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+
+        RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql;

database/interface/map_values/report_unmapped.sql

@@ -0,0 +1,249 @@
+
+DROP FUNCTION IF EXISTS tps.report_unmapped;
+CREATE FUNCTION tps.report_unmapped(_srce text) RETURNS TABLE 
+(
+    source text, 
+    map text,
+    ret_val jsonb,
+    "count" bigint
+)
+LANGUAGE plpgsql
+AS
+$f$
+BEGIN
+
+/*
+first get distinct target json values
+then apply regex
+*/
+
+RETURN QUERY
+WITH
+
+--------------------apply regex operations to transactions---------------------------------------------------------------------------------
+
+rx AS (
+    SELECT 
+        t.srce,
+        t.id,
+        t.rec,
+        m.target,
+        m.seq,
+        regex->'regex'->>'function' regex_function,
+        e.v ->> 'field' result_key_name,
+        e.v ->> 'key' target_json_path,
+        e.v ->> 'flag' regex_options_flag,
+        e.v->>'map' map_intention,
+        e.v->>'retain' retain_result,
+        e.v->>'regex' regex_expression,
+        e.rn target_item_number,
+        COALESCE(mt.rn,rp.rn,1) result_number,
+        mt.mt rx_match,
+        rp.rp rx_replace,
+        CASE e.v->>'map'
+            WHEN 'y' THEN
+                e.v->>'field'
+            ELSE
+                null
+        END map_key,
+        CASE e.v->>'map'
+            WHEN 'y' THEN
+                CASE regex->'regex'->>'function'
+                    WHEN 'extract' THEN
+                        CASE WHEN array_upper(mt.mt,1)=1 
+                            THEN to_json(mt.mt[1])
+                            ELSE array_to_json(mt.mt)
+                        END::jsonb
+                    WHEN 'replace' THEN
+                        to_jsonb(rp.rp)
+                    ELSE
+                        '{}'::jsonb
+                END
+            ELSE
+                NULL
+        END map_val,
+        CASE e.v->>'retain'
+            WHEN 'y' THEN
+                e.v->>'field'
+            ELSE
+                NULL
+        END retain_key,
+        CASE e.v->>'retain'
+            WHEN 'y' THEN
+                CASE regex->'regex'->>'function'
+                    WHEN 'extract' THEN
+                        CASE WHEN array_upper(mt.mt,1)=1 
+                            THEN to_json(trim(mt.mt[1]))
+                            ELSE array_to_json(mt.mt)
+                        END::jsonb
+                    WHEN 'replace' THEN
+                        to_jsonb(rtrim(rp.rp))
+                    ELSE
+                        '{}'::jsonb
+                END
+            ELSE
+                NULL
+        END retain_val
+    FROM 
+        --------------------------start with all regex maps------------------------------------------------------------------------------------
+        tps.map_rm m
+        --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+        LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+        --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+        INNER JOIN tps.trans t ON 
+            t.srce = m.srce AND
+            t.rec @> w.v
+        --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+        LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+        --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+        LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+            m.regex->'regex'->>'function' = 'extract'
+        --------------------------same as above but for a replacement type function------------------------------------------------------------
+        LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+            m.regex->'regex'->>'function' = 'replace'
+    WHERE
+    --t.allj IS NULL
+    t.srce = _srce AND
+    e.v @> '{"map":"y"}'::jsonb
+    --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+    --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
+    ORDER BY 
+        t.id DESC,
+        m.target,
+        e.rn,
+        COALESCE(mt.rn,rp.rn,1)
+)
+
+--SELECT * FROM rx LIMIT 100
+
+
+, agg_to_target_items AS (
+SELECT 
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,CASE WHEN map_key IS NULL 
+        THEN    
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                map_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(map_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(map_val ORDER BY result_number)
+                END
+            ) 
+    END map_val
+    ,CASE WHEN retain_key IS NULL 
+        THEN 
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                retain_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(retain_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(retain_val ORDER BY result_number)
+                END
+            ) 
+    END retain_val
+FROM 
+    rx
+GROUP BY
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,map_key
+    ,retain_key
+)
+
+--SELECT * FROM agg_to_target_items LIMIT 100
+
+
+, agg_to_target AS (
+SELECT
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+    ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+FROM
+    agg_to_target_items
+GROUP BY
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+)
+
+
+, agg_to_ret AS (
+SELECT
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+	,count(*) "count"
+FROM 
+	agg_to_target
+GROUP BY
+	srce
+	,target
+	,seq
+	,map_intention
+	,map_val
+	,retain_val
+)
+
+, link_map AS (
+SELECT
+    a.srce
+    ,a.target
+    ,a.seq
+    ,a.map_intention
+    ,a.map_val
+    ,a."count"
+    ,a.retain_val
+    ,v.map mapped_val
+FROM
+    agg_to_ret a
+    LEFT OUTER JOIN tps.map_rv v ON
+        v.srce = a.srce AND
+        v.target = a.target AND
+        v.retval = a.map_val
+)
+SELECT
+    l.srce
+    ,l.target
+    ,l.map_val
+    ,l."count"
+FROM
+    link_map l
+WHERE
+    l.mapped_val IS NULL
+ORDER BY
+    l.srce
+    ,l.target
+    ,l."count" desc;
+END;
+$f$

database/interface/map_values/report_unmapped_recs.sql

@@ -0,0 +1,258 @@
+DROP FUNCTION IF EXISTS tps.report_unmapped_recs;
+CREATE FUNCTION tps.report_unmapped_recs(_srce text) RETURNS TABLE 
+(
+    srce        text, 
+    target      text,
+    retval      jsonb,
+    map         jsonb,
+    hist        jsonb,
+    cnt         bigint,
+    recs        jsonb
+)
+LANGUAGE plpgsql
+AS
+$f$ 
+BEGIN
+
+/*
+first get distinct target json values
+then apply regex
+*/
+
+RETURN QUERY
+WITH
+
+--------------------apply regex operations to transactions---------------------------------------------------------------------------------
+
+rx AS (
+    SELECT 
+            t.srce,
+            t.id,
+            t.rec,
+            m.target,
+            m.seq,
+            regex->'regex'->>'function' regex_function,
+            e.v ->> 'field' result_key_name,
+            e.v ->> 'key' target_json_path,
+            e.v ->> 'flag' regex_options_flag,
+            e.v->>'map' map_intention,
+            e.v->>'retain' retain_result,
+            e.v->>'regex' regex_expression,
+            e.rn target_item_number,
+            COALESCE(mt.rn,rp.rn,1) result_number,
+            mt.mt rx_match,
+            rp.rp rx_replace,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    null
+            END map_key,
+            CASE e.v->>'map'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(mt.mt[1])
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rp.rp)
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END map_val,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    e.v->>'field'
+                ELSE
+                    NULL
+            END retain_key,
+            CASE e.v->>'retain'
+                WHEN 'y' THEN
+                    CASE regex->'regex'->>'function'
+                        WHEN 'extract' THEN
+                            CASE WHEN array_upper(mt.mt,1)=1 
+                                THEN to_json(trim(mt.mt[1]))
+                                ELSE array_to_json(mt.mt)
+                            END::jsonb
+                        WHEN 'replace' THEN
+                            to_jsonb(rtrim(rp.rp))
+                        ELSE
+                            '{}'::jsonb
+                    END
+                ELSE
+                    NULL
+            END retain_val
+    FROM 
+            --------------------------start with all regex maps------------------------------------------------------------------------------------
+            tps.map_rm m
+            --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+            --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+            INNER JOIN tps.trans t ON 
+                t.srce = m.srce AND
+                t.rec @> w.v
+            --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+            LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+            --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+            LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+                m.regex->'regex'->>'function' = 'extract'
+            --------------------------same as above but for a replacement type function------------------------------------------------------------
+            LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+                m.regex->'regex'->>'function' = 'replace'
+    WHERE
+        --t.allj IS NULL
+        t.srce = _srce AND
+        e.v @> '{"map":"y"}'::jsonb
+        --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+        --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
+    ORDER BY 
+            t.id DESC,
+            m.target,
+            e.rn,
+            COALESCE(mt.rn,rp.rn,1)
+)
+
+--SELECT * FROM rx LIMIT 100
+
+
+, agg_to_target_items AS (
+SELECT 
+     rx.srce
+    ,rx.id
+    ,rx.rec
+    ,rx.target
+    ,rx.seq
+    ,rx.map_intention
+    ,rx.regex_function
+    ,rx.target_item_number
+    ,rx.result_key_name
+    ,rx.target_json_path
+    ,CASE WHEN map_key IS NULL 
+        THEN    
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                map_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(map_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(map_val ORDER BY result_number)
+                END
+            ) 
+    END map_val
+    ,CASE WHEN retain_key IS NULL 
+        THEN 
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                retain_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                            jsonb_agg(retain_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(retain_val ORDER BY result_number)
+                END
+            ) 
+    END retain_val
+FROM 
+    rx
+GROUP BY
+     rx.srce
+    ,rx.id
+    ,rx.rec
+    ,rx.target
+    ,rx.seq
+    ,rx.map_intention
+    ,rx.regex_function
+    ,rx.target_item_number
+    ,rx.result_key_name
+    ,rx.target_json_path
+    ,rx.map_key
+    ,rx.retain_key
+)
+
+--SELECT * FROM agg_to_target_items LIMIT 100
+
+
+, agg_to_target AS (
+SELECT
+     a.srce
+    ,a.id
+    ,a.rec
+    ,a.target
+    ,a.seq
+    ,a.map_intention
+    ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+    ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+FROM
+    agg_to_target_items a
+GROUP BY
+     a.srce
+    ,a.id
+    ,a.rec
+    ,a.target
+    ,a.seq
+    ,a.map_intention
+)
+, agg_to_ret AS (
+SELECT
+	 a.srce
+	,a.target
+	,a.seq
+	,a.map_intention
+	,a.map_val
+	,a.retain_val
+	,count(*) "count"
+    ,jsonb_agg(rec) rec
+FROM 
+	agg_to_target a
+GROUP BY
+	 a.srce
+	,a.target
+	,a.seq
+	,a.map_intention
+	,a.map_val
+	,a.retain_val
+)
+
+, link_map AS (
+SELECT
+    a.srce
+    ,a.target
+    ,a.seq
+    ,a.map_intention
+    ,a.map_val
+    ,a."count"
+    ,a.rec
+    ,a.retain_val
+    ,v.map mapped_val
+FROM
+    agg_to_ret a
+    LEFT OUTER JOIN tps.map_rv v ON
+        v.srce = a.srce AND
+        v.target = a.target AND
+        v.retval = a.map_val
+)
+SELECT
+    l.srce       srce
+    ,l.target    target
+    ,l.map_val   retval
+    ,null::jsonb map
+    ,null::jsonb hist
+    ,l."count"   cnt
+    ,l.rec       recs
+FROM
+    link_map l
+WHERE
+    l.mapped_val IS NULL
+ORDER BY
+    l.srce
+    ,l.target
+    ,l."count" desc;
+END;
+$f$

database/interface/map_values/srce_map_overwrite.sql

@@ -0,0 +1,261 @@
+CREATE OR REPLACE FUNCTION tps.srce_map_overwrite(_srce text) RETURNS jsonb
+AS
+$f$
+DECLARE
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+
+BEGIN
+    WITH
+    --------------------apply regex operations to transactions-----------------------------------------------------------------------------------
+
+    rx AS (
+    SELECT 
+        t.srce,
+        t.id,
+        t.rec,
+        m.target,
+        m.seq,
+        regex->'regex'->>'function' regex_function,
+        e.v ->> 'field' result_key_name,
+        e.v ->> 'key' target_json_path,
+        e.v ->> 'flag' regex_options_flag,
+        e.v->>'map' map_intention,
+        e.v->>'retain' retain_result,
+        e.v->>'regex' regex_expression,
+        e.rn target_item_number,
+        COALESCE(mt.rn,rp.rn,1) result_number,
+        mt.mt rx_match,
+        rp.rp rx_replace,
+        CASE e.v->>'map'
+            WHEN 'y' THEN
+                e.v->>'field'
+            ELSE
+                null
+        END map_key,
+        CASE e.v->>'map'
+            WHEN 'y' THEN
+                CASE regex->'regex'->>'function'
+                    WHEN 'extract' THEN
+                        CASE WHEN array_upper(mt.mt,1)=1 
+                            THEN to_json(mt.mt[1])
+                            ELSE array_to_json(mt.mt)
+                        END::jsonb
+                    WHEN 'replace' THEN
+                        to_jsonb(rp.rp)
+                    ELSE
+                        '{}'::jsonb
+                END
+            ELSE
+                NULL
+        END map_val,
+        CASE e.v->>'retain'
+            WHEN 'y' THEN
+                e.v->>'field'
+            ELSE
+                NULL
+        END retain_key,
+        CASE e.v->>'retain'
+            WHEN 'y' THEN
+                CASE regex->'regex'->>'function'
+                    WHEN 'extract' THEN
+                        CASE WHEN array_upper(mt.mt,1)=1 
+                            THEN to_json(trim(mt.mt[1]))
+                            ELSE array_to_json(mt.mt)
+                        END::jsonb
+                    WHEN 'replace' THEN
+                        to_jsonb(rtrim(rp.rp))
+                    ELSE
+                        '{}'::jsonb
+                END
+            ELSE
+                NULL
+        END retain_val
+    FROM 
+        --------------------------start with all regex maps------------------------------------------------------------------------------------
+        tps.map_rm m
+        --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+        LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') w(v) ON TRUE
+        --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+        INNER JOIN tps.trans t ON 
+            t.srce = m.srce AND
+            t.rec @> w.v
+        --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+        LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY e(v, rn) ON true
+        --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+        LEFT JOIN LATERAL regexp_matches(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text,COALESCE(e.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+            m.regex->'regex'->>'function' = 'extract'
+        --------------------------same as above but for a replacement type function------------------------------------------------------------
+        LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+            m.regex->'regex'->>'function' = 'replace'
+    WHERE
+        --t.allj IS NULL
+        t.srce = _srce
+        --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+        --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
+    ORDER BY 
+        t.id DESC,
+        m.target,
+        e.rn,
+        COALESCE(mt.rn,rp.rn,1)
+    )
+
+    --SELECT count(*) FROM rx LIMIT 100
+
+
+    , agg_to_target_items AS (
+    SELECT 
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,regex_function
+        ,target_item_number
+        ,result_key_name
+        ,target_json_path
+        ,CASE WHEN map_key IS NULL 
+            THEN    
+                NULL 
+            ELSE 
+                jsonb_build_object(
+                    map_key,
+                    CASE WHEN max(result_number) = 1
+                        THEN
+                            jsonb_agg(map_val ORDER BY result_number) -> 0
+                        ELSE
+                            jsonb_agg(map_val ORDER BY result_number)
+                    END
+                ) 
+        END map_val
+        ,CASE WHEN retain_key IS NULL 
+            THEN 
+                NULL 
+            ELSE 
+                jsonb_build_object(
+                    retain_key,
+                    CASE WHEN max(result_number) = 1
+                        THEN
+                            jsonb_agg(retain_val ORDER BY result_number) -> 0
+                        ELSE
+                            jsonb_agg(retain_val ORDER BY result_number)
+                    END
+                ) 
+        END retain_val
+    FROM 
+        rx
+    GROUP BY
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,regex_function
+        ,target_item_number
+        ,result_key_name
+        ,target_json_path
+        ,map_key
+        ,retain_key
+    )
+
+    --SELECT * FROM agg_to_target_items LIMIT 100
+
+
+    , agg_to_target AS (
+    SELECT
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+        ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+        ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+    FROM
+        agg_to_target_items
+    GROUP BY
+        srce
+        ,id
+        ,target
+        ,seq
+        ,map_intention
+    ORDER BY
+        id
+    )
+
+
+    --SELECT * FROM agg_to_target
+
+
+    , link_map AS (
+    SELECT
+        a.srce
+        ,a.id
+        ,a.target
+        ,a.seq
+        ,a.map_intention
+        ,a.map_val
+        ,a.retain_val retain_value
+        ,v.map
+    FROM
+        agg_to_target a
+        LEFT OUTER JOIN tps.map_rv v ON
+            v.srce = a.srce AND
+            v.target = a.target AND
+            v.retval = a.map_val
+    )
+
+    --SELECT * FROM link_map
+
+    , agg_to_id AS (
+    SELECT
+        srce
+        ,id
+        ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val
+        ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map
+    FROM
+        link_map
+    GROUP BY
+        srce
+        ,id
+    )
+
+    --SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100
+
+
+
+    UPDATE
+        tps.trans t
+    SET
+        map = o.map,
+        parse = o.retain_val,
+        allj = t.rec||o.map||o.retain_val
+    FROM
+        agg_to_id o
+    WHERE
+        o.id = t.id;
+
+    _message:= jsonb_build_object('status','complete');
+    RETURN _message;
+
+EXCEPTION WHEN OTHERS THEN
+
+    GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+    _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error setting map value"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+
+    RETURN _message;
+END;
+$f$
+language plpgsql

database/interface/source_maint/srce_build_view.sql

@@ -0,0 +1,33 @@
+DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text);
+CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT
+AS
+$f$
+DECLARE	
+	--_schema text;
+	--_srce text;
+	_sql text;
+BEGIN
+    --_schema:= 'default';
+	--_srce:= 'dcard';
+SELECT
+   'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';'
+INTO
+	_sql
+FROM
+    tps.srce s
+    JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE
+    JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE
+    JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE 
+WHERE
+    srce = _srce
+    AND list.e->>'name' = _schema
+GROUP BY
+    s.srce
+    ,list.e;
+
+RETURN _sql;
+RAISE NOTICE '%',_sql;
+
+END
+$f$
+LANGUAGE plpgsql;

database/interface/source_maint/srce_delete.sql

@@ -0,0 +1,52 @@
+--setup function to delete a single source
+DROP FUNCTION IF EXISTS tps.srce_delete(jsonb);
+CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE 
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+    _rebuild BOOLEAN;
+BEGIN
+
+    -------------------------------do delete---------------------------------
+
+    DELETE FROM tps.srce WHERE srce = _defn->>'name';
+    --could move this record to a "recycle bin" table for a certain period of time
+    --need to handle cascading record deletes
+    
+    ---------------------------set message-----------------------------------
+    _message:= 
+    (
+        $$
+            {
+            "status":"complete",
+            "message":"source was permanently deleted"
+            }
+        $$::jsonb
+    );
+
+    RETURN _message;
+
+    
+
+    EXCEPTION WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error dropping the source"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql

database/interface/source_maint/srce_overwrite_all.sql

@@ -0,0 +1,108 @@
+/*
+This function takes and array of definition object where "name" object is the primary key
+It will force the entire body of sources to match what is received
+*/
+DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb);
+CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE 
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+    _rebuild BOOLEAN;
+    _list text;
+BEGIN
+
+    WITH
+    --retain the results of the update by srce
+    _set AS (
+        SELECT
+            j.rn rn
+            ,j.e->>'name' srce
+            ,j.e defn
+        FROM
+            jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn)
+    )
+    --full join
+    ,_full AS (
+        SELECT
+            COALESCE(_srce.srce,_set.srce) srce
+            ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn
+            ,COALESCE(_set.defn,_srce.defn) defn
+        FROM
+            tps.srce _srce
+            FULL OUTER JOIN _set ON
+                _set.srce = _srce.srce
+    )
+    --call functions from list
+    ,_do_set AS (
+        SELECT 
+            f.srce
+            ,f.actn
+            ,setd.message
+        FROM 
+            _full f
+            JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET'
+            --dual left joins for functions that touch the same table causes the first left join actions to be undone
+            --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
+    )
+    ,_do_del AS (
+        SELECT 
+            f.srce
+            ,f.actn
+            ,deld.message
+        FROM 
+            _full f
+            JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
+    )
+    --aggregate all the messages into one message
+    ----
+    ----    should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this.
+    ----
+    SELECT
+        jsonb_agg(m)
+    INTO
+        _message
+    FROM
+        (
+            SELECT
+                jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
+            FROM    
+                _do_set
+            UNION ALL
+            SELECT
+                jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
+            FROM
+                _do_del
+        ) x;
+
+    SELECT string_agg(srce,',') INTO _list FROM tps.srce;
+    RAISE NOTICE 'multi source list: %', _list;
+
+    RETURN _message;
+
+    SELECT string_agg(srce,',') INTO _list FROM tps.srce;
+    RAISE NOTICE 'after return: %', _list;
+
+    
+
+    EXCEPTION WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error updating sources"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql

database/interface/source_maint/srce_set.sql

@@ -0,0 +1,144 @@
+DROP FUNCTION IF EXISTS tps.srce_set(jsonb);
+CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE 
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+    _rebuild BOOLEAN;
+BEGIN
+
+    ---------test if anythign is changing--------------------------------------------------------------------------------------------
+
+    IF _defn = (SELECT defn FROM tps.srce WHERE srce = _defn->>'name') THEN
+         _message:= 
+        (
+            $$
+                {
+                "status":"complete",
+                "message":"source was not different no action taken"
+                }
+            $$::jsonb
+        );
+        RETURN _message;
+    END IF;
+
+    ---------if the constraint definition is changing, rebuild for existing records---------------------------------------------------
+
+    SELECT 
+        NOT (_defn->'constraint' = (SELECT defn->'constraint' FROM tps.srce WHERE srce = _defn->>'name'))
+    INTO
+        _rebuild;
+
+    RAISE NOTICE '%',_rebuild::text;
+
+    ---------do merge-----------------------------------------------------------------------------------------------------------------
+
+    INSERT INTO
+        tps.srce (srce, defn, hist)
+    SELECT
+        --extract name from defintion
+        _defn->>'name'
+        --add current timestamp to defintions
+        ,_defn
+        --add definition
+        ,jsonb_build_object(
+                'hist_defn',_defn
+                ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+        ) || '[]'::jsonb
+    ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE
+        SET
+            defn = _defn
+            ,hist = 
+                    --the new definition going to position -0-
+                    jsonb_build_object(
+                        'hist_defn',_defn
+                        ,'effective',jsonb_build_array(CURRENT_TIMESTAMP,null::timestamptz)
+                    ) 
+                    --the previous definition, set upper bound of effective range which was previously null
+                    || jsonb_set(
+                        srce.hist
+                        ,'{0,effective,1}'::text[]
+                        ,to_jsonb(CURRENT_TIMESTAMP)
+                    );
+    --rebuild constraint key if necessary---------------------------------------------------------------------------------------
+
+    IF _rebuild THEN
+        WITH
+        rebuild AS (
+            SELECT
+                j.srce
+                ,j.rec
+                ,j.id
+                --aggregate back to the record since multiple paths may be listed in the constraint
+                ,tps.jsonb_concat_obj(
+                    jsonb_build_object(
+                        --the new json key is the path itself
+                        cons.path->>0
+                        ,j.rec#>((cons.path->>0)::text[])
+                    ) 
+                ) json_key
+            FROM
+                tps.trans j
+                INNER JOIN tps.srce s ON
+                    s.srce = j.srce
+                JOIN LATERAL jsonb_array_elements(s.defn->'constraint') WITH ORDINALITY cons(path, seq)  ON TRUE
+            WHERE
+                s.srce = _defn->>'name'
+            GROUP BY
+                j.rec
+                ,j.id
+        )
+        UPDATE
+            tps.trans t
+        SET
+            ic = r.json_key
+        FROM
+            rebuild r
+        WHERE
+            t.id = r.id;
+         _message:= 
+        (
+            $$
+                {
+                "status":"complete",
+                "message":"source set and constraint rebuilt on existing records"
+                }
+            $$::jsonb
+        );
+    ELSE
+        _message:= 
+        (
+            $$
+                {
+                "status":"complete",
+                "message":"source set"
+                }
+            $$::jsonb
+        );
+    END IF;
+
+    RETURN _message;
+
+    
+
+    EXCEPTION WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error importing data"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql

database/readme.md

@@ -0,0 +1,128 @@
+Generic Data Transformation Tool
+=======================================================
+
+The goal is to:
+1. house external data and prevent duplication on insert
+2. facilitate regular exression operations to extract meaningful data
+3. be able to reference it from outside sources (no action required) and maintain reference to original data
+
+
+It is well suited for data from outside systems that 
+* requires complex transformation (parsing and mapping)
+* original data is retained for reference
+* don't feel like writing a map-reduce
+
+use cases:
+* on-going bank feeds
+* jumbled product lists
+* storing api results
+
+
+The data is converted to json by the importing program and inserted to the database.
+Regex expressions are applied to specified json components and the results can be mapped to other values.
+
+
+Major Interactions
+------------------------
+
+* Source Definitions (Maint/Inquire)
+* Regex Instructions (Maint/Inquire)
+* Cross Reference List (Maint/Inquire)
+* Run Import (Run Job)
+
+
+
+### Interaction Details
+* _Source Definitions (Maint/Inquire)_
+
+    * display a list of existing sources with display detials/edit options
+    * create new option
+    * underlying function is `tps.srce_set(_name text, _defn jsonb)`
+
+    * the current definition of a source includes data based on bad presumptions:
+        * how to load from a csv file using `COPY`
+        * setup a Postgres type to reflect the associated columns (if applicable)
+        
+
+* _Regex Instructions (Maint/Inquire)_
+
+    * display a list of existing instruction sets with display details/edit options
+    * create new option
+    * underlying function is `tps.srce_map_def_set(_srce text, _map text, _defn jsonb, _seq int)` which takes a source "code" and a json
+
+* _Cross Reference List (Maint/Inquire)_
+
+    * first step is to populate a list of values returned from the instructions (choose all or unmapped) `tps.report_unmapped(_srce text)`
+    * the list of rows facilitates additional named column(s) to be added which are used to assign values anytime the result occurs
+    * function to set the values of the cross reference `tps.srce_map_val_set_multi(_maps jsonb)`
+
+* _Run Import_
+
+    * underlying function is `tps.srce_import(_path text, _srce text)`
+
+
+
+source definition
+----------------------------------------------------------------------
+
+* **load data**
+    * the brwosers role is to extract the contents of a file and send them as a post body to the backend for processing under target function `based on srce defintion`
+        * the backend builds a json array of all the rows to be added and sends as an argument to a database insert function
+            * build constraint key `based on srce definition`
+            * handle violations
+            * increment global key list (this may not be possible depending on if a json with variable length arrays can be traversed)
+            * build an import log
+            * run maps (as opposed to relying on trigger)
+* **read data**
+    * the `schema` key contains either a text element or a text array in curly braces
+        * forcing everything to extract via `#>{}` would be cleaner but may be more expensive than `jsonb_populate_record`
+        * it took 5.5 seconds to parse 1,000,000 rows of an identicle google distance matrix json to a 5 column temp table
+    * top level key to table based on `jsonb_populate_record` extracting from `tps.type` developed from `srce.defn->schema`
+    * custom function parsing contents based on #> operator and extracting from `srce.defn->schema`
+    * view that `uses the source definiton` to extrapolate a table?
+    * a materialized table is built `based on the source definition` and any addtional regex?
+        * add regex = alter table add column with historic updates?
+        * no primary key?
+        * every document must work out to one row
+
+```
+{
+    "name":"dcard",
+    "source":"client_file",
+    "loading_function":"csv"
+    "constraint":[
+        "{Trans. Date}",
+        "{Post Date}"
+    ],
+    "schemas":{
+        "default":[
+            {
+                "path":"{doc,origin_addresses,0}",
+                "type":"text",
+                "column_name":"origin_address"
+            },
+            {
+                "path":"{doc,destination_addresses,0}",
+                "type":"text",
+                "column_name":"origin_address"
+            },
+            {
+                "path":"{doc,status}",
+                "type":"text",
+                "column_name":"status"
+            }
+            {
+                "path":"{doc,rows,0,elements,0,distance,value}",
+                "type":"numeric",
+                "column_name":"distance"
+            }
+            {
+                "path":"{doc,rows,0,elements,0,duration,value}",
+                "type":"numeric",
+                "column_name":"duration"
+            }
+        ],
+        "version2":[]
+    }
+}
+```

database/reports/all_map_return_values.sql

@@ -0,0 +1,189 @@
+WITH
+--------------------apply regex operations to transactions-----------------------------------------------------------------------------------
+rx AS (
+SELECT 
+    t.srce,
+    t.id,
+    t.rec,
+    m.target,
+    m.seq,
+    regex->'regex'->>'function' regex_function,
+    defn.v ->> 'field'          result_key_name,
+    defn.v ->> 'key'            target_json_path,
+    defn.v ->> 'flag'           regex_options_flag,
+    defn.v->>'map'              map_intention,
+    defn.v->>'retain'           retain_result,
+    defn.v->>'regex'            regex_expression,
+    defn.rn                     target_item_number,
+    COALESCE(mt.rn,rp.rn,1)     result_number,
+    mt.mt                       rx_match,
+    rp.rp                       rx_replace,
+    CASE defn.v->>'map'
+        WHEN 'y' THEN
+            defn.v->>'field'
+        ELSE
+            null
+    END                         map_key,
+    CASE defn.v->>'map'
+        WHEN 'y' THEN
+            CASE regex->'regex'->>'function'
+                WHEN 'extract' THEN
+                    CASE WHEN array_upper(mt.mt,1)=1 
+                        THEN to_json(mt.mt[1])
+                        ELSE array_to_json(mt.mt)
+                    END::jsonb
+                WHEN 'replace' THEN
+                    to_jsonb(rp.rp)
+                ELSE
+                    '{}'::jsonb
+            END
+        ELSE
+            NULL
+    END                         map_val,
+    CASE defn.v->>'retain'
+        WHEN 'y' THEN
+            defn.v->>'field'
+        ELSE
+            NULL
+    END                         retain_key,
+    CASE defn.v->>'retain'
+        WHEN 'y' THEN
+            CASE regex->'regex'->>'function'
+                WHEN 'extract' THEN
+                    CASE WHEN array_upper(mt.mt,1)=1 
+                        THEN to_json(trim(mt.mt[1]))
+                        ELSE array_to_json(mt.mt)
+                    END::jsonb
+                WHEN 'replace' THEN
+                    to_jsonb(rtrim(rp.rp))
+                ELSE
+                    '{}'::jsonb
+            END
+        ELSE
+            NULL
+    END                         retain_val
+FROM 
+    --------------------------start with all regex maps------------------------------------------------------------------------------------
+    tps.map_rm m
+    --------------------------isolate matching basis to limit map to only look at certain json---------------------------------------------
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') wh(v) ON TRUE
+    --------------------------join to main transaction table but only certain key/values are included--------------------------------------
+    INNER JOIN tps.trans t ON 
+        t.srce = m.srce AND
+        t.rec @> wh.v
+    --------------------------break out array of regluar expressions in the map------------------------------------------------------------
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
+    --------------------------each regex references a path to the target value, extract the target from the reference and do regex---------
+    LEFT JOIN LATERAL regexp_matches(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text,COALESCE(defn.v ->> 'flag','')) WITH ORDINALITY mt(mt, rn) ON
+        m.regex->'regex'->>'function' = 'extract'
+    --------------------------same as above but for a replacement type function------------------------------------------------------------
+    LEFT JOIN LATERAL regexp_replace(t.rec #>> ((defn.v ->> 'key')::text[]), defn.v ->> 'regex'::text, defn.v ->> 'replace'::text,defn.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON
+        m.regex->'regex'->>'function' = 'replace'
+WHERE
+    --t.allj IS NULL
+    t.srce = 'dcard'
+    --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}'
+    --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb
+ORDER BY 
+    t.id DESC,
+    m.target,
+    defn.rn,
+    COALESCE(mt.rn,rp.rn,1)
+)
+--SELECT * FROM rx
+--SELECT count(*) FROM rx LIMIT 100
+, agg_to_target_items AS (
+SELECT 
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,CASE WHEN map_key IS NULL 
+        THEN    
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                map_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(map_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(map_val ORDER BY result_number)
+                END
+            ) 
+    END map_val
+    ,CASE WHEN retain_key IS NULL 
+        THEN 
+            NULL 
+        ELSE 
+            jsonb_build_object(
+                retain_key,
+                CASE WHEN max(result_number) = 1
+                    THEN
+                        jsonb_agg(retain_val ORDER BY result_number) -> 0
+                    ELSE
+                        jsonb_agg(retain_val ORDER BY result_number)
+                END
+            ) 
+    END retain_val
+FROM 
+    rx
+GROUP BY
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,regex_function
+    ,target_item_number
+    ,result_key_name
+    ,target_json_path
+    ,map_key
+    ,retain_key
+)
+, agg_to_target AS (
+SELECT
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+    ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val
+    ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val
+FROM
+    agg_to_target_items
+GROUP BY
+    srce
+    ,id
+    ,target
+    ,seq
+    ,map_intention
+ORDER BY
+    id
+)
+, link_map AS (
+SELECT
+    a.srce
+    ,a.id
+    ,a.target
+    ,a.seq
+    ,a.map_intention
+    ,a.map_val
+    ,a.retain_val retain_value
+    ,v.map
+FROM
+    agg_to_target a
+    LEFT OUTER JOIN tps.map_rv v ON
+        v.srce = a.srce AND
+        v.target = a.target AND
+        v.retval = a.map_val
+)
+--SELECT * FROM rx
+--SELECT * FROM agg_to_target_items
+--SELECT * FROM agg_to_target
+SELECT * FROM link_map

database/reports/chase_bal.sql

@@ -0,0 +1,14 @@
+\timing
+SELECT 
+    r.id
+    ,r.logid
+    ,r."Type"
+    ,r."Trans Date"
+    ,r."Post Date"
+    ,r."Description"
+    ,-r."Amount" "Amount"
+    ,SUM(-r."Amount") OVER (ORDER BY r."Post Date" asc , r.id) + 374.23 balance
+FROM 
+    tpsv.chase_default r
+ORDER BY 
+    r."Post Date" desc

database/reports/colateral_balalance.sql

@@ -0,0 +1,19 @@
+SELECT 
+    r.*
+    ,CASE "Schedule#"
+        WHEN '02IN Raw Material' THEN 13097563.42
+        WHEN '03IN Finished Goods' THEN 35790696.52
+        ELSE 0
+    END + SUM("Sales"+"Credits & Adjustments"-"Gross Collections") OVER (PARTITION BY "Schedule#" ORDER BY "Schedule#" ASC, "PostDate" ASC) running_bal
+    ,(LEAST("CollateralBalance" - "Ineligible Amount","MaxEligible")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral
+    ,(("CollateralBalance" - "Ineligible Amount")*("AdvanceRate"/100))::NUMERIC(20,2) qualified_collateral_nl
+FROM
+    tpsv.pncl_default r
+WHERE
+    "Schedule#" = '01AR'
+    --"Schedule#" = '02IN Raw Material'
+    --"Schedule#" = '03IN Finished Goods'
+ORDER BY 
+    "Schedule#" asc
+    ,r."PostDate" asc
+    ,id

database/reports/dcard_bal.sql

@@ -0,0 +1,8 @@
+\timing
+SELECT 
+    r.*
+    ,SUM(r."Amount") OVER (ORDER BY r."Post Date" asc , r."Description") + 1083.50 balance
+FROM 
+    tpsv.dcard_mapped r
+ORDER BY 
+    r."Post Date" desc

database/reports/hunt_bal.sql

@@ -0,0 +1,8 @@
+\timing
+SELECT 
+    r.*
+    ,SUM(r."Amount") OVER (ORDER BY r."Date" asc , r.id) + 29909.75 balance
+FROM 
+    tpsv.hunt_mapped r
+ORDER BY 
+    r."Date" desc

database/reports/key_list.sql

@@ -0,0 +1,34 @@
+\timing
+
+/*--------------------------------------------------
+maintain statment level triggers to update a master log of keys
+* table based listing
+* composite type maintenance
+
+potential updates sources/events
+* tps.trans insert
+* tps.trans re-map
+--------------------------------------------------*/
+
+WITH ok AS (
+    SELECT
+        srce,
+        ok.k,
+        jsonb_typeof(allj->ok.k) typeof,
+        COUNT(*)
+    FROM
+        tps.trans
+        JOIN LATERAL jsonb_object_keys(allj) ok(k) ON TRUE
+    GROUP BY   
+        srce,
+        ok.k,
+        jsonb_typeof(allj->ok.k)
+    ORDER BY 
+        srce
+)
+SELECT
+    srce
+    ,k
+    ,typeof
+FROM
+    ok

database/reports/list_maps.sql

@@ -0,0 +1,21 @@
+SELECT 
+     m.srce
+    ,m.target
+    ,m.regex->'regex'->>'function' func_name
+    ,m.regex->'regex'->>'description' descr
+    ,filter.v  filters
+    ,defn.v->> 'field' result_key_name
+    ,defn.v->> 'key' target_json_path
+    ,defn.v->>'regex' regex_expression
+    ,defn.v->> 'flag' regex_options_flag
+    ,defn.v->>'map' map_intention
+    ,defn.v->>'retain' retain_result
+    ,defn.rn target_item_number
+FROM 
+    tps.map_rm m
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'defn') WITH ORDINALITY defn(v, rn) ON true
+    LEFT JOIN LATERAL jsonb_array_elements(m.regex->'regex'->'where') WITH ORDINALITY filter(v, rn) ON true
+ORDER BY 
+     m.srce
+    ,m.target
+    ,defn.rn

database/reports/loan_balance.sql

@@ -0,0 +1,12 @@
+\timing
+SELECT 
+    r.*,
+    SUM(r."Advances"+r."Adjustments"-r."Payments") OVER (PARTITION BY "Loan#" ORDER BY r."Post Date" asc, r."Reference #" asc)
+FROM 
+    tpsv.pnco_default r
+WHERE 
+    "Loan#" = '606780191'
+ORDER BY 
+    r."Loan#"
+    ,r."Post Date" ASC
+    ,r."Reference #" ASC

database/reports/stacked.sql

@@ -0,0 +1,86 @@
+with 
+dcard AS (
+    SELECT 
+        'dcard'                                                                           srce
+        ,r.id                                                                             id
+        ,r.logid                                                                          logid
+        --,r."Trans. Date"
+        ,r."Post Date"                                                                    pdate
+        ,r."Description"                                                                  descr
+        ,-r."Amount"                                                                       amount
+        ,r."Category"                                                                     category
+        ,r."Party"                                                                        party
+        ,r."Reason"                                                                       reason
+        ,-(SUM(r."Amount") OVER (ORDER BY r."Post Date" asc , r."Description") + 1083.50) balance
+    FROM 
+        tpsv.dcard_mapped r
+    ORDER BY 
+        r."Post Date" desc
+)
+,hunt as (
+    SELECT 
+        'hunt' srce
+        ,r.id
+        ,r.logid
+        --,r."Reference Number"
+        --,r."Payee Name"
+        ,r."Date"
+        ,r."Memo"
+        ,r."Amount"
+        ,r."Cateogry Name"
+        ,r."Party"
+        ,r."Reason"
+        ,SUM(r."Amount") OVER (ORDER BY r."Date" asc , r.id) + 29909.75 balance
+    FROM 
+        tpsv.hunt_mapped r
+    ORDER BY 
+        r."Date" desc
+)
+,chase AS (
+    SELECT 
+        'chase' srce
+        ,r.id
+        ,r.logid
+        --,r."Trans Date"
+        ,r."Post Date"
+        ,r."Description"
+        ,r."Amount" "Amount"
+        ,r."Type"
+        ,null::text "Party"
+        ,null::text "Reason"
+        ,-(SUM(-r."Amount") OVER (ORDER BY r."Post Date" asc , r.id) + 374.23) balance
+    FROM 
+        tpsv.chase_default r
+    ORDER BY 
+        r."Post Date" desc
+)
+,stacked AS (
+    SELECT * FROM dcard
+    UNION ALL
+    SELECT * FROM hunt
+    UNION ALL
+    SELECT * FROM chase 
+)
+,format AS (
+SELECT 
+     s.srce
+    ,s.id
+    ,s.logid
+    ,s.pdate
+    ,s.descr
+    ,s.amount
+    --,s.category
+    ,s.party
+    ,s.reason
+    --,s.balance
+    ,CASE WHEN s.srce = 'hunt' THEN s.balance ELSE null END hunt
+    ,CASE WHEN s.srce = 'dcard' THEN s.balance ELSE null END dcard
+    ,CASE WHEN s.srce = 'chase' THEN s.balance ELSE null END chase
+    ,sum(s.amount) over (ORDER BY s.pdate asc, s.id) + 28452.02 netbalance
+FROM 
+    stacked s
+ORDER BY 
+    s.pdate desc
+    ,s.id desc
+)
+SELECT * FROM format

database/templates/regex.json

@@ -0,0 +1,25 @@
+[
+    {
+        "regex": {
+            "function": "extract",
+            "description": "pull first 20 characters from description for mapping",
+            "where": [
+                {}
+            ],
+            "defn": [
+                {
+                    "regex": ".{1,20}",
+                    "map": "y",
+                    "field": "f20",
+                    "flag": "",
+                    "key": "{Description}",
+                    "retain": "y"
+                }
+            ],
+            "name": "First 20"
+        },
+        "sequence": 2,
+        "name": "First 20",
+        "srce": "dcard"
+    }
+]

database/templates/srce.json

@@ -0,0 +1,82 @@
+{
+    "name": "dcard",
+    "source": "client_file",
+    "loading_function": "csv",
+    "constraint": [
+        "{Trans. Date}",
+        "{Post Date}",
+        "{Description}"
+    ],
+    "schemas": [
+        {
+            "name": "default",
+            "columns": [
+                {
+                    "path": "{Trans. Date}",
+                    "type": "date",
+                    "column_name": "Trans. Date"
+                },
+                {
+                    "path": "{Post Date}",
+                    "type": "date",
+                    "column_name": "Post Date"
+                },
+                {
+                    "path": "{Description}",
+                    "type": "text",
+                    "column_name": "Description"
+                },
+                {
+                    "path": "{Amount}",
+                    "type": "numeric",
+                    "column_name": "Amount"
+                },
+                {
+                    "path": "{Category}",
+                    "type": "text",
+                    "column_name": "Category"
+                }
+            ]
+        },
+        {
+            "name": "mapped",
+            "columns": [
+                {
+                    "path": "{Trans. Date}",
+                    "type": "date",
+                    "column_name": "Trans. Date"
+                },
+                {
+                    "path": "{Post Date}",
+                    "type": "date",
+                    "column_name": "Post Date"
+                },
+                {
+                    "path": "{Description}",
+                    "type": "text",
+                    "column_name": "Description"
+                },
+                {
+                    "path": "{Amount}",
+                    "type": "numeric",
+                    "column_name": "Amount"
+                },
+                {
+                    "path": "{Category}",
+                    "type": "text",
+                    "column_name": "Category"
+                },
+                {
+                    "path": "{party}",
+                    "type": "text",
+                    "column_name": "Party"
+                },
+                {
+                    "path": "{reason}",
+                    "type": "text",
+                    "column_name": "Reason"
+                }
+            ]
+        }
+    ]
+}

database/templates/strip_commas.jsonc

@@ -0,0 +1,24 @@
+{
+    "name": "Strip Amount Commas",  //the name here currently also serves as the primary key in the database
+    "srce": "PNCC",                 //name of the target source
+    "sequence": 1 ,                 //only for edge cases where the instructions returns two keys of the same name, this determines priority. pretty much 1.
+    "regex": {                      //instruction set
+        "where": [                  //only apply this regex to these specified key value pairs, if none then use empty object {}
+            {
+                "example_key":"example_value"
+            }
+        ],
+        "function": "replace",      //even though there is an array of definitions they all have to operate under the same premise(extract or replace)
+        "defn": [                   //there is an array of instructions
+            {
+                "key": "{Amount}",  //key= the path to the json key/value pair to operate on. path woudl be a better term.
+                "map": "n",         //y or n to indicate if the returned value will be used to search a lookup table
+                "flag": "g",        //g indicates find all values, null or empty would be the other option I guess
+                "field": "amount",  //the key name to give the value that comes out of this instruction
+                "regex": ",",       //the reg expression itself
+                "retain": "y",      //flag to indicate if the returned value should be retained and included with the data
+                "replace": ""       //this key is only evaluated if the function is defined as replace
+            }
+        ]
+    }
+}

database/templates/transaction_type.jsonc

@@ -0,0 +1,45 @@
+{
+    "name": "Trans Type",
+    "srce": "PNCC",
+    "regex": {
+        "function": "extract",
+        "defn": [
+            {
+                "key": "{AccountName}",
+                "map": "y",
+                "field": "acctn",
+                "regex": "(.*)",
+                "retain": "n"
+            },
+            {
+                "key": "{Transaction}",
+                "map": "y",
+                "field": "trans",
+                "regex": "(.*)",
+                "retain": "n"
+            },
+            {
+                "key": "{Description}",
+                "map": "y",
+                "field": "ini",
+                "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
+                "retain": "y"
+            }
+        ],
+        "where": [
+            {}
+        ]
+    },
+    "sequence": 1
+}
+
+/*
+target      |                                                              retval                                                              |                                                                           map
+------------+----------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------- 
+ Trans Type | {"ini": "01346", "acctn": "The HC Operating Company OPERA", "trans": "Miscellaneous Fees"}                                       | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"}
+ Trans Type | {"ini": "CANADA TAX", "acctn": "The HC Operating Company OPERA", "trans": "Detail Debit Adjustments"}                            | {"sign": "-1", "party": "PNC", "ledger": "Manual", "reason": "Bank Fees", "trantype": "Disbursement"} 
+ Trans Type | {"ini": "ACH DEBIT SETTLEMENT", "acctn": "The HC Operating Company OPERA", "trans": "ACH Debits"}                                | {"sign": "-1", "ledger": "AP - ACH", "trantype": "Disbursement"} 
+ Trans Type | {"ini": "RET DEP ITEM RTM", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"}                      | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit RTM", "trantype": "Collections"} 
+ Trans Type | {"ini": "RET DEP ITEM STOP", "acctn": "The HC Operating Company FBO P", "trans": "Deposited Items Returned"}                     | {"sign": "-1", "ledger": "Manual", "reason": "Returned Deposit STOP", "trantype": "Collections"} 
+ Trans Type | {"ini": "CREDIT ADJUSTMENT", "acctn": "The HC Operating Company FBO P", "trans": "Detail Credit Adjustments"}                    | {"sign": "1", "ledger": "AR - Collections", "trantype": "Collections"}
+ */

database/upgrade_scripts/01.source_front_end_build/001.extract_schemas.sql

@@ -0,0 +1,30 @@
+WITH
+mod AS (
+SELECT 
+    srce
+    ,jsonb_pretty(defn) orig
+    ,(defn - 'schemas')||
+    --rebuild the schemas key value from below
+    jsonb_build_object(
+        'schemas'
+        --aggregate all the new key values for a single soure
+        ,jsonb_agg(
+            --combine a new key 'name' with the columns for that name
+            jsonb_build_object('name',k)||jsonb_build_object('columns',v)
+        )
+    ) rebuild
+FROM 
+    tps.srce
+    LEFT JOIN LATERAL jsonb_each(defn->'schemas') WITH ORDINALITY je(k,v, rn) ON TRUE
+GROUP BY
+    srce
+    ,defn
+)
+UPDATE
+    tps.srce s
+SET
+    defn = rebuild
+FROM
+    mod
+WHERE
+    mod.srce = s.srce

database/upgrade_scripts/01.source_front_end_build/002.schema_build_sql

@@ -0,0 +1,33 @@
+DROP FUNCTION IF EXISTS tps.build_srce_view_sql(text, text);
+CREATE OR REPLACE FUNCTION tps.build_srce_view_sql(_srce text, _schema text) RETURNS TEXT
+AS
+$f$
+DECLARE	
+	--_schema text;
+	--_srce text;
+	_sql text;
+BEGIN
+    --_schema:= 'default';
+	--_srce:= 'dcard';
+SELECT
+   'DROP VIEW IF EXISTS tpsv.'||s.srce||'_'||(list.e->>'name')||'; CREATE VIEW tpsv.'||s.srce||'_'||(list.e->>'name')||' AS SELECT id, logid, allj, '||string_agg('(allj#>>'''||rec.PATH::text||''')::'||rec.type||' AS "'||rec.column_name||'"',', ')||' FROM tps.trans WHERE srce = '''||s.srce||''';'
+INTO
+	_sql
+FROM
+    tps.srce s
+    JOIN LATERAL jsonb_array_elements(s.defn->'schemas') list (e) ON TRUE
+    JOIN LATERAL jsonb_array_elements(list.e->'columns') as cols(e) ON TRUE
+    JOIN LATERAL jsonb_to_record (cols.e) AS rec( PATH text[], "type" text, column_name text) ON TRUE 
+WHERE
+    srce = _srce
+    AND list.e->>'name' = _schema
+GROUP BY
+    s.srce
+    ,list.e;
+
+RETURN _sql;
+RAISE NOTICE '%',_sql;
+
+END
+$f$
+LANGUAGE plpgsql;

database/upgrade_scripts/01.source_front_end_build/003.srce_delete.sql

@@ -0,0 +1,52 @@
+--setup function to delete a single source
+DROP FUNCTION IF EXISTS tps.srce_delete(jsonb);
+CREATE FUNCTION tps.srce_delete(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE 
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+    _rebuild BOOLEAN;
+BEGIN
+
+    -------------------------------do delete---------------------------------
+
+    DELETE FROM tps.srce WHERE srce = _defn->>'name';
+    --could move this record to a "recycle bin" table for a certain period of time
+    --need to handle cascading record deletes
+    
+    ---------------------------set message-----------------------------------
+    _message:= 
+    (
+        $$
+            {
+            "status":"complete",
+            "message":"source was permanently deleted"
+            }
+        $$::jsonb
+    );
+
+    RETURN _message;
+
+    
+
+    EXCEPTION WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error dropping the source"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql

database/upgrade_scripts/01.source_front_end_build/004.source_overwrite.sql

@@ -0,0 +1,108 @@
+/*
+This function takes and array of definition object where "name" object is the primary key
+It will force the entire body of sources to match what is received
+*/
+DROP FUNCTION IF EXISTS tps.srce_overwrite_all(jsonb);
+CREATE FUNCTION tps.srce_overwrite_all(_defn jsonb) RETURNS jsonb
+AS
+$f$
+DECLARE 
+    _message jsonb;
+    _MESSAGE_TEXT text;
+    _PG_EXCEPTION_DETAIL text;
+    _PG_EXCEPTION_HINT text;
+    _rebuild BOOLEAN;
+    _list text;
+BEGIN
+
+    WITH
+    --retain the results of the update by srce
+    _set AS (
+        SELECT
+            j.rn rn
+            ,j.e->>'name' srce
+            ,j.e defn
+        FROM
+            jsonb_array_elements(_defn) WITH ORDINALITY j(e, rn)
+    )
+    --full join
+    ,_full AS (
+        SELECT
+            COALESCE(_srce.srce,_set.srce) srce
+            ,CASE COALESCE(_set.srce,'DELETE') WHEN 'DELETE' THEN 'DELETE' ELSE 'SET' END actn
+            ,COALESCE(_set.defn,_srce.defn) defn
+        FROM
+            tps.srce _srce
+            FULL OUTER JOIN _set ON
+                _set.srce = _srce.srce
+    )
+    --call functions from list
+    ,_do_set AS (
+        SELECT 
+            f.srce
+            ,f.actn
+            ,setd.message
+        FROM 
+            _full f
+            JOIN LATERAL tps.srce_set(defn) setd(message) ON f.actn = 'SET'
+            --dual left joins for functions that touch the same table causes the first left join actions to be undone
+            --LEFT JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
+    )
+    ,_do_del AS (
+        SELECT 
+            f.srce
+            ,f.actn
+            ,deld.message
+        FROM 
+            _full f
+            JOIN LATERAL tps.srce_delete(defn) deld(message) ON f.actn = 'DELETE'
+    )
+    --aggregate all the messages into one message
+    ----
+    ----    should look at rolling back the whole thing if one of the function returns a fail. stored proc could do this.
+    ----
+    SELECT
+        jsonb_agg(m)
+    INTO
+        _message
+    FROM
+        (
+            SELECT
+                jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
+            FROM    
+                _do_set
+            UNION ALL
+            SELECT
+                jsonb_build_object('source',srce,'status',message->>'status','message',message->>'message') m
+            FROM
+                _do_del
+        ) x;
+
+    SELECT string_agg(srce,',') INTO _list FROM tps.srce;
+    RAISE NOTICE 'multi source list: %', _list;
+
+    RETURN _message;
+
+    SELECT string_agg(srce,',') INTO _list FROM tps.srce;
+    RAISE NOTICE 'after return: %', _list;
+
+    
+
+    EXCEPTION WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS 
+            _MESSAGE_TEXT = MESSAGE_TEXT,
+            _PG_EXCEPTION_DETAIL = PG_EXCEPTION_DETAIL,
+            _PG_EXCEPTION_HINT = PG_EXCEPTION_HINT;
+        _message:= 
+        ($$
+            {
+                "status":"fail",
+                "message":"error updating sources"
+            }
+        $$::jsonb)
+        ||jsonb_build_object('message_text',_MESSAGE_TEXT)
+        ||jsonb_build_object('pg_exception_detail',_PG_EXCEPTION_DETAIL);
+    RETURN _message;
+END;
+$f$
+LANGUAGE plpgsql