From 175b7a38c5484f343a4e8e6a3e6bc5ed0545ad77 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 4 Nov 2017 09:33:16 -0400 Subject: [PATCH] merge wk branch concept of placing the mapping instructions in the regex items themselves --- do_map_g_option.pgsql | 29 ++- map_rm_template.pgsql | 571 +++++++++++++++++++++++++++++------------- 2 files changed, 420 insertions(+), 180 deletions(-) diff --git a/do_map_g_option.pgsql b/do_map_g_option.pgsql index eb3c65d..4d1efee 100644 --- a/do_map_g_option.pgsql +++ b/do_map_g_option.pgsql @@ -9,25 +9,26 @@ SELECT t.id, t.rec, m.target, - regex->>'map' map_intention, + m.seq, regex->>'function' regex_function, e.v ->> 'field' result_key_name, e.v ->> 'key' target_json_path, e.v ->> 'flag' regex_options_flag, + e.v->>'map' map_intention, e.v->>'retain' retain_result, e.v->>'regex' regex_expression, e.rn target_item_number, COALESCE(mt.rn,rp.rn,1) result_number, mt.mt rx_match, rp.rp rx_replace, - CASE regex->>'map' - WHEN 'yes' THEN + CASE e.v->>'map' + WHEN 'y' THEN e.v->>'field' ELSE null END map_key, - CASE regex->>'map' - WHEN 'yes' THEN + CASE e.v->>'map' + WHEN 'y' THEN CASE regex->>'function' WHEN 'extract' THEN CASE WHEN array_upper(mt.mt,1)=1 @@ -76,9 +77,9 @@ FROM LEFT JOIN LATERAL regexp_replace(t.rec #>> ((e.v ->> 'key')::text[]), e.v ->> 'regex'::text, e.v ->> 'replace'::text,e.v ->> 'flag') WITH ORDINALITY rp(rp, rn) ON m.regex->>'function' = 'replace' WHERE + t.allj IS NULL --t.srce = 'PNCC' --rec @> '{"Transaction":"ACH Credits","Transaction":"ACH Debits"}' - t.map IS NULL --rec @> '{"Description":"CHECK 93013270 086129935"}'::jsonb ORDER BY t.id DESC, @@ -87,13 +88,15 @@ ORDER BY COALESCE(mt.rn,rp.rn,1) ) ---SELECT * FROM rx +--SELECT count(*) FROM rx LIMIT 100 + , agg_to_target_items AS ( SELECT srce ,id ,target + ,seq ,map_intention ,regex_function ,target_item_number @@ -133,6 +136,7 @@ GROUP BY srce ,id ,target + ,seq ,map_intention ,regex_function ,target_item_number @@ -142,13 +146,15 @@ GROUP BY ,retain_key ) ---SELECT * FROM agg_to_target_items +--SELECT * FROM agg_to_target_items LIMIT 100 + , agg_to_target AS ( SELECT srce ,id ,target + ,seq ,map_intention ,tps.jsonb_concat_obj(COALESCE(map_val,'{}'::JSONB)) map_val ,jsonb_strip_nulls(tps.jsonb_concat_obj(COALESCE(retain_val,'{}'::JSONB))) retain_val @@ -158,6 +164,7 @@ GROUP BY srce ,id ,target + ,seq ,map_intention ORDER BY id @@ -172,6 +179,7 @@ SELECT a.srce ,a.id ,a.target + ,a.seq ,a.map_intention ,a.map_val ,a.retain_val retain_value @@ -190,7 +198,7 @@ FROM SELECT srce ,id - ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb)) retain_val + ,tps.jsonb_concat_obj(COALESCE(retain_value,'{}'::jsonb) ORDER BY seq DESC) retain_val ,tps.jsonb_concat_obj(COALESCE(map,'{}'::jsonb)) map FROM link_map @@ -199,7 +207,8 @@ GROUP BY ,id ) -SELECT srce, id, jsonb_pretty(retain_val), jsonb_pretty(map) FROM agg_to_id +--SELECT agg_to_id.srce, agg_to_id.id, jsonb_pretty(agg_to_id.retain_val) , jsonb_pretty(agg_to_id.map) FROM agg_to_id ORDER BY id desc LIMIT 100 + UPDATE diff --git a/map_rm_template.pgsql b/map_rm_template.pgsql index d61a036..bb90722 100644 --- a/map_rm_template.pgsql +++ b/map_rm_template.pgsql @@ -1,11 +1,6 @@ -/* -DELETE FROM tps.map_rm where target = 'Strip Amount Commas'; -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('PNCC', 'Strip Amount Commas', + +UPDATE tps.map_rm +SET regex = $j$ { "name":"Strip Amount Commas", @@ -17,263 +12,499 @@ FROM "regex": ",", "replace":"", "flag":"g", - "retain":"y" + "retain":"y", + "map":"n" } ], "function":"replace", - "map":"no", "where": [ { } ] } $j$::jsonb - , 1) -) x; +WHERE + target = 'Strip Amount Commas'; + -DELETE FROM tps.map_rm where target = 'Parse Descr'; -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('PNCC', 'Parse Descr', +UPDATE tps.map_rm +SET regex = $j$ { - "name":"Parse Descr", - "description":"parse the description based on at least three capital letters followed by a comma until another set of at lesat 3 capital letters and a comma is encountered", - "defn": [ - { - "key": "{Description}", - "field": "dparse", - "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", - "flag":"g", - "retain":"y" - } - ], - "function":"extract", - "map":"no", - "where": [ - { - } - ] - } - $j$::jsonb - , 2) -) x; - - -DELETE FROM tps.map_rm where target = 'Extract OBI'; -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('PNCC', 'Extract OBI', - $j$ - { - "name":"Extract OBI", - "description":"pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", - "defn": [ - { - "key": "{Description}", - "field": "obi", - "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y" - } - ], - "function":"extract", - "map":"no", - "where": [ - { - "Transaction":"Money Transfer DB - Wire" - }, - { - "Transaction":"Money Transfer CR-Other" - }, - { - "Transaction":"Intl Money Transfer Debits" - }, - { - "Transaction":"Money Transfer DB - Other" - }, - { - "Transaction":"Money Transfer CR-Wire" - } - ] - } - $j$::jsonb - , 2) -) x; - -DELETE FROM tps.map_rm where target = 'Extract RFB'; -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('PNCC', 'Extract RFB', - $j$ - { - "name":"Extract RFB", - "description":"pull out whatever follows RFB in the description until atleast 3 capital letters followed by a colon are encountered", - "defn": [ - { - "key": "{Description}", - "field": "rfb", - "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y" - } - ], - "function":"extract", - "map":"no", - "where": [ - { - "Transaction":"Money Transfer DB - Wire" - }, - { - "Transaction":"Money Transfer CR-Other" - }, - { - "Transaction":"Intl Money Transfer Debits" - }, - { - "Transaction":"Money Transfer DB - Other" - }, - { - "Transaction":"Money Transfer CR-Wire" - } - ] - } - $j$::jsonb - , 2) -) x; - -DELETE FROM tps.map_rm where target = 'Parse ACH'; - -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('PNCC', 'Parse ACH', - $j$ - { - "name":"Parse ACH", + "name":"Parse ACH Credits", "description":"parse select components of the description for ACH Credits Receieved", "defn": [ { "key": "{Description}", - "field":"Comp Name", - "regex": "Comp Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "field":"beneficiary", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Cust ID", "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Desc", "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", - "field":"Cust Name", + "field":"originator", "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Batch Discr", "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Comp ID", "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Addenda", "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"SETT", "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Date", "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" }, { "key": "{Description}", "field":"Time", "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" } ], "function":"extract", - "map":"no", "where": [ { "Transaction":"ACH Credits" + } + ] + } + $j$::jsonb +WHERE target = 'Parse ACH Credits'; + + +UPDATE tps.map_rm +SET regex = + $j$ + { + "name":"Parse ACH Debits", + "description":"parse select components of the description for ACH Credits Receieved", + "defn": [ + { + "key": "{Description}", + "field":"originator", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" }, + { + "key": "{Description}", + "field":"Cust ID", + "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Desc", + "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"beneficiary", + "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Batch Discr", + "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Comp ID", + "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Addenda", + "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"SETT", + "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Date", + "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field":"Time", + "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag":"", + "retain":"y", + "map":"n" + } + ], + "function":"extract", + "where": [ { "Transaction":"ACH Debits" } ] } $j$::jsonb - , 2) -) x; -*/ +WHERE target = 'Parse ACH Debits'; -UPDATE - tps.map_rm -SET - regex = +UPDATE tps.map_rm +SET regex = $j$ { - "name":"First 20", - "description":"pull first 20 characters from description for mapping", + "name":"Parse Wires", + "description":"pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", "defn": [ { - "key": "{Memo}", - "field":"f20", - "regex": ".{1,20}", + "key": "{Description}", + "field": "dparse", + "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", + "flag":"g", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "beneficiary_components", + "regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", "flag":"", - "retain":"y" + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "originator_components", + "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "beneficiary", + "regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "originator", + "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "OBI", + "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "RFB", + "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "ABA", + "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "BBI", + "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "BENEBNK", + "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "IBK", + "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "RATE", + "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + }, + { + "key": "{Description}", + "field": "RECVBNK", + "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag":"", + "retain":"y", + "map":"n" + } + ], + "function":"extract", + "where": [ + { + "Transaction":"Money Transfer DB - Wire" + }, + { + "Transaction":"Money Transfer DB - Other" + }, + { + "Transaction":"Money Transfer CR-Wire" + }, + { + "Transaction":"Money Transfer CR-Other" + }, + { + "Transaction":"Intl Money Transfer Debits" + }, + { + "Transaction":"Intl Money Transfer Credits" + } + ] + } + $j$::jsonb + WHERE target = 'Parse Wires'; + + +UPDATE tps.map_rm +SET regex = + $j$ + { + "name":"Trans Type", + "description":"extract intial description in conjunction with account name and transaction type for mapping", + "defn": [ + { + "key": "{AccountName}", + "field": "acctn", + "regex": "(.*)", + "retain": "n", + "map":"y" + }, + { + "key": "{Transaction}", + "field": "trans", + "regex": "(.*)", + "retain": "n", + "map":"y" + }, + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y", + "map":"y" + } + ], + "where": [ + { + } + ], + "function": "extract" + } + $j$::jsonb + WHERE target = 'Trans Type'; + + +UPDATE tps.map_rm +SET regex = + $j$ + { + "name":"Currency", + "description":"pull out currency indicators from description of misc items and map", + "defn": [ + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y", + "map":"y" + }, + { + "key": "{Description}", + "field": "curr1", + "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", + "retain": "y", + "map":"y" + }, + { + "key": "{Description}", + "field": "curr2", + "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", + "retain": "y", + "map":"y" + } + ], + "where": [ + { + "Transaction": "Miscellaneous Credits" + }, + { + "Transaction": "Miscellaneous Debits" + } + ], + "function": "extract" + } + $j$::jsonb + WHERE target = 'Currency'; + +UPDATE tps.map_rm +SET regex = + $j$ + { + "defn": [ + { + "key": "{Description}", + "field": "checkn", + "regex": "[^0-9]*([0-9]*)\\s|$", + "retain": "y", + "map":"n" + } + ], + "where": [ + { + "Transaction": "Checks Paid" + } + ], + "function": "extract" + } + $j$::jsonb + WHERE target = 'Check Number'; + + +UPDATE tps.map_rm +SET regex = + $j$ + { + "name":"ADP Codes", + "description":"link to adp code definitions", + "defn": [ + { + "key": "{gl_descr}", + "field": "gl_descr", + "regex": ".*", + "flag":"", + "retain":"n", + "map":"y" + }, + { + "key": "{prim_offset}", + "field": "prim_offset", + "regex": ".*", + "flag":"", + "retain":"n", + "map":"y" + }, + { + "key": "{pay_date}", + "field": "pay_month", + "regex": ".{1,4}", + "flag":"", + "retain":"y", + "map":"n" } ], "function":"extract", - "map":"yes", "where": [ { } ] } $j$::jsonb -WHERE srce = 'HUNT' \ No newline at end of file +WHERE + target = 'ADP Codes'; \ No newline at end of file