diff --git a/build_json.xlsx b/build_json.xlsx index 66976d7..2f2b882 100644 Binary files a/build_json.xlsx and b/build_json.xlsx differ diff --git a/do_map_g_option.pgsql b/do_map_g_option.pgsql index 31ddc3b..eb3c65d 100644 --- a/do_map_g_option.pgsql +++ b/do_map_g_option.pgsql @@ -77,7 +77,8 @@ FROM m.regex->>'function' = 'replace' WHERE --t.srce = 'PNCC' - rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' + t.map IS NULL --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb ORDER BY t.id DESC, @@ -200,7 +201,7 @@ GROUP BY SELECT srce, id, jsonb_pretty(retain_val), jsonb_pretty(map) FROM agg_to_id -/* + UPDATE tps.trans t SET @@ -210,5 +211,4 @@ SET FROM agg_to_id o WHERE - o.id = t.id; -*/ \ No newline at end of file + o.id = t.id; \ No newline at end of file diff --git a/evt_log_gl_extract.pgsql b/evt_log_gl_extract.pgsql new file mode 100644 index 0000000..d172e08 --- /dev/null +++ b/evt_log_gl_extract.pgsql @@ -0,0 +1,22 @@ + +WITH j AS ( +SELECT + id, + post_stmp, + rec as r +FROM + evt.log +) + +--this is a dynamic approach that dumps all keys into the json except several that are required which it extracts +SELECT + id, + ARRAY['GL',rn::text] json_path, + post_stmp, + a.i->>'amt' amount, + a.i->>'account' account, + a.i->>'date' tran_date, + a.i - '{amt,account,date}'::text[] as therest +FROM + j + LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(j.r->'GL') WITH ORDINALITY a(i, rn) ON TRUE \ No newline at end of file diff --git a/map_rm_template.pgsql b/map_rm_template.pgsql index 34dae15..d61a036 100644 --- a/map_rm_template.pgsql +++ b/map_rm_template.pgsql @@ -150,7 +150,7 @@ FROM $j$::jsonb , 2) ) x; -*/ + DELETE FROM tps.map_rm where target = 'Parse ACH'; INSERT INTO @@ -249,3 +249,31 @@ FROM $j$::jsonb , 2) ) x; +*/ + +UPDATE + tps.map_rm +SET + regex = + $j$ + { + "name":"First 20", + "description":"pull first 20 characters from description for mapping", + "defn": [ + { + "key": "{Memo}", + "field":"f20", + "regex": ".{1,20}", + "flag":"", + "retain":"y" + } + ], + "function":"extract", + "map":"yes", + "where": [ + { + } + ] + } + $j$::jsonb +WHERE srce = 'HUNT' \ No newline at end of file diff --git a/new_format.pgsql b/new_format.pgsql deleted file mode 100644 index 83d5927..0000000 --- a/new_format.pgsql +++ /dev/null @@ -1,22 +0,0 @@ - -WITH j AS ( -SELECT - $${"vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card","module":"MJE_O","offset":"dcard","total":111.41,"vend item":[{"vend item":"GE CHIPS","amt":2.5,"account":"food","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":2.5,"account":"food","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":2.5,"account":"food","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":-2.19,"account":"food","item":"discount","reason":"food","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":-2.5,"account":"dcard","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":-2.5,"account":"dcard","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":-2.5,"account":"dcard","item":"chips","reason":"food","qty":1,"uom":"bag","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"},{"vend item":"GE CHIPS","amt":2.19,"account":"dcard","item":"discount","reason":"food","vendor":"Giant Eagle","date":"10/1/2017","instrument":"Discover Card"}]}$$::JSONB as r -) -select - a.i->>'amt', - a.i->>'account', - a.i->>'date', - a.i - '{amt,account,date}'::text[] as therest -FROM - j - LEFT JOIN LATERAL JSONB_ARRAY_ELEMENTS(j.r->'vend item') a(i) ON TRUE - -/* -SELECT -jsonb_pretty( -$$ -{"vendor":"Acme","date":"9/19/2017","instrument":"Discover Card","module":"MJE_O","offset":"dcard","total":127.66,"item":[{"item":"sun chips","amt":3.29,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cheetos paws","amt":3.69,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif","amt":4.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif whips","amt":3.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"jif","amt":4.99,"account":"food","reason":"peanut butter","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"goya chipotle pepp","amt":2.29,"account":"food","reason":"chili peppers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"quick oats","amt":4.69,"account":"food","reason":"oats","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"quick oats","amt":4.69,"account":"food","reason":"oats","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"sara lee artesano","amt":2.99,"account":"food","reason":"bread","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"dan lgt yog","amt":3.29,"account":"food","reason":"yogurt","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"bananas","amt":3.7,"account":"food","reason":"bananas","qty":7.11,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"yellow onions","amt":2.99,"account":"food","reason":"onions","qty":3,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"orange peppers","amt":4.11,"account":"food","reason":"peppers","qty":1.03,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"asparagus","amt":5.14,"account":"food","reason":"asparagus","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"broccoli","amt":2.99,"account":"food","reason":"broccoli","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"hn fat free frank","amt":5.49,"account":"food","reason":"hot dogs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"hn fat free frank","amt":5.49,"account":"food","reason":"hot dogs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"butterscotch morsels","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"chocolate chips","amt":3.49,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"seedless grapes","amt":11.45,"account":"food","reason":"grapes","qty":2.87,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"honecrisp apples","amt":14.76,"account":"food","reason":"apples","qty":3.7,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"garlic expressions","amt":4.99,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"olivegarden dressing","amt":3.49,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"garlic expressions","amt":4.99,"account":"food","reason":"salad dressing","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"thomas cinamon bagels","amt":4.69,"account":"food","reason":"bagels","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"stawberries","amt":1.25,"account":"food","reason":"strawberries","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"stawberries","amt":1.25,"account":"food","reason":"strawberries","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"provolone cheese","amt":6.99,"account":"food","reason":"cheese","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"lemon juice","amt":1.99,"account":"food","reason":"lemon juice","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cucumbers","amt":1.99,"account":"food","reason":"cucumbers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"cucumbers","amt":1.99,"account":"food","reason":"cucumbers","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"distilled water","amt":"0.89","account":"food","reason":"water","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.7,"account":"food","reason":"chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-0.99,"account":"food","reason":"broccoli","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-1.5,"account":"food","reason":"chocolate chips","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-8.03,"account":"food","reason":"grapes","qty":2.87,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"},{"item":"discount","amt":-5.55,"account":"food","reason":"apples","qty":3.7,"uom":"lbs","vendor":"Acme","date":"9/19/2017","instrument":"Discover Card"}]} -$$::jsonb -) -*/ \ No newline at end of file diff --git a/ubm_data.sql b/ubm_data.sql index caccd40..5c8c8c7 100644 --- a/ubm_data.sql +++ b/ubm_data.sql @@ -20,6 +20,7 @@ SET search_path = evt, pg_catalog; -- Data for Name: log; Type: TABLE DATA; Schema: evt; Owner: - -- +INSERT INTO log (id, rec, post_stmp) VALUES (1, '{"GL": [{"amt": 2.89, "date": "10/5/2017", "item": "oats", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "VT QUICK OATS", "instrument": "Discover Card"}, {"amt": 4.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "GARLIC EXPRESSIONS", "instrument": "Discover Card"}, {"amt": 4.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "GARLIC EXPRESSIONS", "instrument": "Discover Card"}, {"amt": 3.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "OLVGRDN SIG ITAL", "instrument": "Discover Card"}, {"amt": -1, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "OLIVE GRD DRESSING", "instrument": "Discover Card"}, {"amt": 3.39, "date": "10/5/2017", "item": "salt", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "MORTON SEA SALT AP", "instrument": "Discover Card"}, {"amt": 1.99, "date": "10/5/2017", "item": "ice", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "ACME ICE", "instrument": "Discover Card"}, {"amt": "0.1", "date": "10/5/2017", "item": "fuel rewards", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "FUEL REWARDS", "instrument": "Discover Card"}, {"amt": 10.64, "date": "10/5/2017", "item": "ham", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "ACME 1/4 HAM", "instrument": "Discover Card"}, {"amt": 9.69, "date": "10/5/2017", "item": "coffee", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "STARBUCKS WB SUMAT", "instrument": "Discover Card"}, {"amt": 9.69, "date": "10/5/2017", "item": "coffee", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "STARBUCKS WB SUMAT", "instrument": "Discover Card"}, {"amt": 3.29, "date": "10/5/2017", "item": "oregano", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "MC OREGANO LEAVES", "instrument": "Discover Card"}, {"amt": 1.99, "date": "10/5/2017", "item": "cucumber", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "CUCUMBERS", "instrument": "Discover Card"}, {"amt": 1.99, "date": "10/5/2017", "item": "cucumber", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "CUCUMBERS", "instrument": "Discover Card"}, {"amt": 2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": 2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": 2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": 2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "food", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": -2.89, "date": "10/5/2017", "item": "oats", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "VT QUICK OATS", "instrument": "Discover Card"}, {"amt": -4.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "GARLIC EXPRESSIONS", "instrument": "Discover Card"}, {"amt": -4.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "GARLIC EXPRESSIONS", "instrument": "Discover Card"}, {"amt": -3.99, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "OLVGRDN SIG ITAL", "instrument": "Discover Card"}, {"amt": 1, "date": "10/5/2017", "item": "salad dressing", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "OLIVE GRD DRESSING", "instrument": "Discover Card"}, {"amt": -3.39, "date": "10/5/2017", "item": "salt", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "MORTON SEA SALT AP", "instrument": "Discover Card"}, {"amt": -1.99, "date": "10/5/2017", "item": "ice", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "ACME ICE", "instrument": "Discover Card"}, {"amt": -0.1, "date": "10/5/2017", "item": "fuel rewards", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "FUEL REWARDS", "instrument": "Discover Card"}, {"amt": -10.64, "date": "10/5/2017", "item": "ham", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "ACME 1/4 HAM", "instrument": "Discover Card"}, {"amt": -9.69, "date": "10/5/2017", "item": "coffee", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "STARBUCKS WB SUMAT", "instrument": "Discover Card"}, {"amt": -9.69, "date": "10/5/2017", "item": "coffee", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "STARBUCKS WB SUMAT", "instrument": "Discover Card"}, {"amt": -3.29, "date": "10/5/2017", "item": "oregano", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "MC OREGANO LEAVES", "instrument": "Discover Card"}, {"amt": -1.99, "date": "10/5/2017", "item": "cucumber", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "CUCUMBERS", "instrument": "Discover Card"}, {"amt": -1.99, "date": "10/5/2017", "item": "cucumber", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "CUCUMBERS", "instrument": "Discover Card"}, {"amt": -2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": -2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": -2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "LIVING BASIL", "instrument": "Discover Card"}, {"amt": -2.5, "date": "10/5/2017", "item": "basil plant", "reason": "food", "vendor": "Acme", "account": "dcard", "vend item": "LIVING BASIL", "instrument": "Discover Card"}], "date": "10/5/2017", "total": 68.53, "module": "MJE", "offset": "dcard", "vendor": "Acme", "instrument": "Discover Card"}', '2017-10-26 23:21:22.904309-04'); SET search_path = tps, pg_catalog; @@ -3093,7 +3094,7 @@ SET search_path = evt, pg_catalog; -- Name: log_id_seq; Type: SEQUENCE SET; Schema: evt; Owner: - -- -SELECT pg_catalog.setval('log_id_seq', 1, false); +SELECT pg_catalog.setval('log_id_seq', 1, true); SET search_path = tps, pg_catalog; diff --git a/ubm_schema.sql b/ubm_schema.sql index 66d687c..26bc575 100644 --- a/ubm_schema.sql +++ b/ubm_schema.sql @@ -187,7 +187,8 @@ SET default_with_oids = false; CREATE TABLE log ( id integer NOT NULL, - rec jsonb + rec jsonb, + post_stmp timestamp with time zone DEFAULT now() );