From 0992b00e92ed0b74e857f05ca1f8d3cfa76c3c69 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Tue, 6 Mar 2018 17:48:28 -0500 Subject: [PATCH 1/6] rename and change to json, remove sql --- templates/map_rm.pgsql | 43 -- templates/map_rm_template.json | 462 ++++++++++++++++ templates/map_rm_template.pgsql | 510 ------------------ ...srce_template.pgsql => srce_template.json} | 9 +- 4 files changed, 463 insertions(+), 561 deletions(-) delete mode 100644 templates/map_rm.pgsql create mode 100644 templates/map_rm_template.json delete mode 100644 templates/map_rm_template.pgsql rename templates/{srce_template.pgsql => srce_template.json} (97%) diff --git a/templates/map_rm.pgsql b/templates/map_rm.pgsql deleted file mode 100644 index 99a3900..0000000 --- a/templates/map_rm.pgsql +++ /dev/null @@ -1,43 +0,0 @@ -DELETE FROM tps.map_rm; -INSERT INTO -tps.map_rm -SELECT * -FROM -(VALUES - ('DCARD', 'First 20', - $j$ - { - "defn": [ - { - "key": "{Description}", - "field": "f20", - "regex": ".{1,20}" - ,"retain":"y" - } - ], - "where": [ - { - } - ] - } - $j$::jsonb - , 2) - ,('HUNT', 'First 20', - $j$ - { - "defn": [ - { - "key": "{Description}", - "field": "f20", - "regex": ".{1,20}" - ,"retain":"y" - } - ], - "where": [ - { - } - ] - } - $j$::jsonb - , 1) -) x \ No newline at end of file diff --git a/templates/map_rm_template.json b/templates/map_rm_template.json new file mode 100644 index 0000000..0d78fd6 --- /dev/null +++ b/templates/map_rm_template.json @@ -0,0 +1,462 @@ +{ + "strip commas": + { + "name": "Strip Amount Commas", + "description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric", + "defn": [ + { + "key": "{Amount}", + "field": "amount", + "regex": ",", + "replace": "", + "flag": "g", + "retain": "y", + "map": "n" + } + ], + "function": "replace", + "where": [ + {} + ] + }, + "Parse ACH Credits": + { + "name": "Parse ACH Credits", + "description": "parse select components of the description for ACH Credits Receieved", + "defn": [ + { + "key": "{Description}", + "field": "beneficiary", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Cust ID", + "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Desc", + "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "originator", + "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Batch Discr", + "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Comp ID", + "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Addenda", + "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "SETT", + "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Date", + "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Time", + "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + } + ], + "function": "extract", + "where": [ + { + "Transaction": "ACH Credits" + } + ] + }, + "Parse ACH Debits": + { + "name": "Parse ACH Debits", + "description": "parse select components of the description for ACH Credits Receieved", + "defn": [ + { + "key": "{Description}", + "field": "originator", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Cust ID", + "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Desc", + "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "beneficiary", + "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Batch Discr", + "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Comp ID", + "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Addenda", + "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "SETT", + "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Date", + "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "Time", + "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "flag": "", + "retain": "y", + "map": "n" + } + ], + "function": "extract", + "where": [ + { + "Transaction": "ACH Debits" + } + ] + }, + "Parse Wires": + { + "name": "Parse Wires", + "description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", + "defn": [ + { + "key": "{Description}", + "field": "dparse", + "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", + "flag": "g", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "beneficiary_components", + "regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "originator_components", + "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "beneficiary", + "regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "originator", + "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "OBI", + "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "RFB", + "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "ABA", + "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "BBI", + "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "BENEBNK", + "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "IBK", + "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "RATE", + "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + }, + { + "key": "{Description}", + "field": "RECVBNK", + "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "flag": "", + "retain": "y", + "map": "n" + } + ], + "function": "extract", + "where": [ + { + "Transaction": "Money Transfer DB - Wire" + }, + { + "Transaction": "Money Transfer DB - Other" + }, + { + "Transaction": "Money Transfer CR-Wire" + }, + { + "Transaction": "Money Transfer CR-Other" + }, + { + "Transaction": "Intl Money Transfer Debits" + }, + { + "Transaction": "Intl Money Transfer Credits" + } + ] + }, + "Trans Type": + { + "name": "Trans Type", + "description": "extract intial description in conjunction with account name and transaction type for mapping", + "defn": [ + { + "key": "{AccountName}", + "field": "acctn", + "regex": "(.*)", + "retain": "n", + "map": "y" + }, + { + "key": "{Transaction}", + "field": "trans", + "regex": "(.*)", + "retain": "n", + "map": "y" + }, + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y", + "map": "y" + } + ], + "where": [ + {} + ], + "function": "extract" + }, + "Currency": + { + "name": "Currency", + "description": "pull out currency indicators from description of misc items and map", + "defn": [ + { + "key": "{Description}", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y", + "map": "y" + }, + { + "key": "{Description}", + "field": "curr1", + "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", + "retain": "y", + "map": "y" + }, + { + "key": "{Description}", + "field": "curr2", + "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", + "retain": "y", + "map": "y" + } + ], + "where": [ + { + "Transaction": "Miscellaneous Credits" + }, + { + "Transaction": "Miscellaneous Debits" + } + ], + "function": "extract" + }, + "check number": + { + "defn": [ + { + "key": "{Description}", + "field": "checkn", + "regex": "[^0-9]*([0-9]*)\\s|$", + "retain": "y", + "map": "n" + } + ], + "where": [ + { + "Transaction": "Checks Paid" + } + ], + "function": "extract" + }, + "ADP Codes": + { + "name": "ADP Codes", + "description": "link to adp code definitions", + "defn": [ + { + "key": "{gl_descr}", + "field": "gl_descr", + "regex": ".*", + "flag": "", + "retain": "n", + "map": "y" + }, + { + "key": "{prim_offset}", + "field": "prim_offset", + "regex": ".*", + "flag": "", + "retain": "n", + "map": "y" + }, + { + "key": "{pay_date}", + "field": "pay_month", + "regex": ".{1,4}", + "flag": "", + "retain": "y", + "map": "n" + } + ], + "function": "extract", + "where": [ + {} + ] + } +} \ No newline at end of file diff --git a/templates/map_rm_template.pgsql b/templates/map_rm_template.pgsql deleted file mode 100644 index bb90722..0000000 --- a/templates/map_rm_template.pgsql +++ /dev/null @@ -1,510 +0,0 @@ - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Strip Amount Commas", - "description":"the Amount field come from PNC with commas embeded so it cannot be cast to numeric", - "defn": [ - { - "key": "{Amount}", - "field": "amount", - "regex": ",", - "replace":"", - "flag":"g", - "retain":"y", - "map":"n" - } - ], - "function":"replace", - "where": [ - { - } - ] - } - $j$::jsonb -WHERE - target = 'Strip Amount Commas'; - - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Parse ACH Credits", - "description":"parse select components of the description for ACH Credits Receieved", - "defn": [ - { - "key": "{Description}", - "field":"beneficiary", - "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Cust ID", - "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Desc", - "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"originator", - "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Batch Discr", - "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Comp ID", - "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Addenda", - "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"SETT", - "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Date", - "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Time", - "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - } - ], - "function":"extract", - "where": [ - { - "Transaction":"ACH Credits" - } - ] - } - $j$::jsonb -WHERE target = 'Parse ACH Credits'; - - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Parse ACH Debits", - "description":"parse select components of the description for ACH Credits Receieved", - "defn": [ - { - "key": "{Description}", - "field":"originator", - "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Cust ID", - "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Desc", - "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"beneficiary", - "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Batch Discr", - "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Comp ID", - "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Addenda", - "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"SETT", - "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Date", - "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field":"Time", - "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag":"", - "retain":"y", - "map":"n" - } - ], - "function":"extract", - "where": [ - { - "Transaction":"ACH Debits" - } - ] - } - $j$::jsonb -WHERE target = 'Parse ACH Debits'; - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Parse Wires", - "description":"pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", - "defn": [ - { - "key": "{Description}", - "field": "dparse", - "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", - "flag":"g", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "beneficiary_components", - "regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "originator_components", - "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "beneficiary", - "regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "originator", - "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "OBI", - "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "RFB", - "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "ABA", - "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "BBI", - "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "BENEBNK", - "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "IBK", - "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "RATE", - "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - }, - { - "key": "{Description}", - "field": "RECVBNK", - "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag":"", - "retain":"y", - "map":"n" - } - ], - "function":"extract", - "where": [ - { - "Transaction":"Money Transfer DB - Wire" - }, - { - "Transaction":"Money Transfer DB - Other" - }, - { - "Transaction":"Money Transfer CR-Wire" - }, - { - "Transaction":"Money Transfer CR-Other" - }, - { - "Transaction":"Intl Money Transfer Debits" - }, - { - "Transaction":"Intl Money Transfer Credits" - } - ] - } - $j$::jsonb - WHERE target = 'Parse Wires'; - - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Trans Type", - "description":"extract intial description in conjunction with account name and transaction type for mapping", - "defn": [ - { - "key": "{AccountName}", - "field": "acctn", - "regex": "(.*)", - "retain": "n", - "map":"y" - }, - { - "key": "{Transaction}", - "field": "trans", - "regex": "(.*)", - "retain": "n", - "map":"y" - }, - { - "key": "{Description}", - "field": "ini", - "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", - "retain": "y", - "map":"y" - } - ], - "where": [ - { - } - ], - "function": "extract" - } - $j$::jsonb - WHERE target = 'Trans Type'; - - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"Currency", - "description":"pull out currency indicators from description of misc items and map", - "defn": [ - { - "key": "{Description}", - "field": "ini", - "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", - "retain": "y", - "map":"y" - }, - { - "key": "{Description}", - "field": "curr1", - "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", - "retain": "y", - "map":"y" - }, - { - "key": "{Description}", - "field": "curr2", - "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", - "retain": "y", - "map":"y" - } - ], - "where": [ - { - "Transaction": "Miscellaneous Credits" - }, - { - "Transaction": "Miscellaneous Debits" - } - ], - "function": "extract" - } - $j$::jsonb - WHERE target = 'Currency'; - -UPDATE tps.map_rm -SET regex = - $j$ - { - "defn": [ - { - "key": "{Description}", - "field": "checkn", - "regex": "[^0-9]*([0-9]*)\\s|$", - "retain": "y", - "map":"n" - } - ], - "where": [ - { - "Transaction": "Checks Paid" - } - ], - "function": "extract" - } - $j$::jsonb - WHERE target = 'Check Number'; - - -UPDATE tps.map_rm -SET regex = - $j$ - { - "name":"ADP Codes", - "description":"link to adp code definitions", - "defn": [ - { - "key": "{gl_descr}", - "field": "gl_descr", - "regex": ".*", - "flag":"", - "retain":"n", - "map":"y" - }, - { - "key": "{prim_offset}", - "field": "prim_offset", - "regex": ".*", - "flag":"", - "retain":"n", - "map":"y" - }, - { - "key": "{pay_date}", - "field": "pay_month", - "regex": ".{1,4}", - "flag":"", - "retain":"y", - "map":"n" - } - ], - "function":"extract", - "where": [ - { - } - ] - } - $j$::jsonb -WHERE - target = 'ADP Codes'; \ No newline at end of file diff --git a/templates/srce_template.pgsql b/templates/srce_template.json similarity index 97% rename from templates/srce_template.pgsql rename to templates/srce_template.json index c12c125..06c6e92 100644 --- a/templates/srce_template.pgsql +++ b/templates/srce_template.json @@ -1,7 +1,3 @@ -UPDATE tps.SRCE - -SET DEFN = -$$ { "name": "WMPD", "descr": "Williams Paid File", @@ -122,7 +118,4 @@ $$ "{Carrier}" ] } -} -$$::JSONB -WHERE -SRCE = 'WMPD' \ No newline at end of file +} \ No newline at end of file From cce644740def5b33290d8de486e7f943e9322f7d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 7 Mar 2018 00:44:19 -0500 Subject: [PATCH 2/6] update srce_set to take a singluar json as argument --- deploy/ubm_schema.sql | 74 +++++++++++++++++++++++++----------------- functions/srce_set.sql | 18 +++++----- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/deploy/ubm_schema.sql b/deploy/ubm_schema.sql index bc31963..20ca8f1 100644 --- a/deploy/ubm_schema.sql +++ b/deploy/ubm_schema.sql @@ -49,25 +49,6 @@ CREATE TYPE tps."DCARD" AS ( COMMENT ON TYPE tps."DCARD" IS 'Discover Card'; --- --- Name: DCARD2; Type: TYPE; Schema: tps; Owner: - --- - -CREATE TYPE tps."DCARD2" AS ( - "Trans. Date" numeric, - "Post Date" numeric, - "Description" text, - "Amount" numeric, - "Category" text -); - - --- --- Name: TYPE "DCARD2"; Type: COMMENT; Schema: tps; Owner: - --- - -COMMENT ON TYPE tps."DCARD2" IS 'Discover Card'; - -- -- Name: DMAPI; Type: TYPE; Schema: tps; Owner: - @@ -78,6 +59,41 @@ CREATE TYPE tps."DMAPI" AS ( ); +-- +-- Name: WMPD; Type: TYPE; Schema: tps; Owner: - +-- + +CREATE TYPE tps."WMPD" AS ( + "Carrier" text, + "SCAC" text, + "Mode" text, + "Pro #" text, + "B/L" text, + "Pd Amt" numeric, + "Loc#" text, + "Pcs" numeric, + "Wgt" numeric, + "Chk#" numeric, + "Pay Dt" date, + "Acct #" text, + "I/O" text, + "Sh Nm" text, + "Sh City" text, + "Sh St" text, + "Sh Zip" text, + "Cons Nm" text, + "D City " text, + "D St" text, + "D Zip" text, + "Sh Dt" date, + "Inv Dt" date, + "Customs Entry#" text, + "Miles" numeric, + "Frt Class" text, + "Master B/L" text +); + + -- -- Name: dcard; Type: TYPE; Schema: tps; Owner: - -- @@ -1085,10 +1101,10 @@ $_$; -- --- Name: srce_set(text, jsonb); Type: FUNCTION; Schema: tps; Owner: - +-- Name: srce_set(jsonb); Type: FUNCTION; Schema: tps; Owner: - -- -CREATE FUNCTION tps.srce_set(_name text, _defn jsonb) RETURNS jsonb +CREATE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb LANGUAGE plpgsql AS $_$ @@ -1115,7 +1131,7 @@ BEGIN FROM tps.srce WHERE - srce = _name; + srce = _defn->>'name'; -------check for transctions already existing under this source----------- SELECT @@ -1125,7 +1141,7 @@ BEGIN FROM tps.trans WHERE - srce = _name; + srce = _defn->>'name'; --if there are transaction already and the schema is different stop-------- IF _cnt > 0 THEN @@ -1151,14 +1167,14 @@ BEGIN INSERT INTO tps.srce SELECT - _name, _defn + _defn->>'name', _defn ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE SET defn = _defn; ------------------drop existing type----------------------------------------- - EXECUTE format('DROP TYPE IF EXISTS tps.%I',_name); + EXECUTE format('DROP TYPE IF EXISTS tps.%I',_defn->>'name'); ------------------create new type-------------------------------------------- @@ -1171,15 +1187,15 @@ BEGIN --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = _name + srce = _defn->>'name' GROUP BY srce; - RAISE NOTICE 'CREATE TYPE tps.% AS (%)',_name,_sql; + RAISE NOTICE 'CREATE TYPE tps.% AS (%)',_defn->>'name',_sql; - EXECUTE format('CREATE TYPE tps.%I AS (%s)',_name,_sql); + EXECUTE format('CREATE TYPE tps.%I AS (%s)',_defn->>'name',_sql); - EXECUTE format('COMMENT ON TYPE tps.%I IS %L',_name,(_defn->>'description')); + EXECUTE format('COMMENT ON TYPE tps.%I IS %L',_defn->>'name',(_defn->>'description')); ----------------set message----------------------------------------------------- diff --git a/functions/srce_set.sql b/functions/srce_set.sql index db5ec32..8dccc52 100644 --- a/functions/srce_set.sql +++ b/functions/srce_set.sql @@ -1,5 +1,5 @@ -CREATE OR REPLACE FUNCTION tps.srce_set(_name text, _defn jsonb) RETURNS jsonb +CREATE OR REPLACE FUNCTION tps.srce_set(_defn jsonb) RETURNS jsonb AS $f$ DECLARE @@ -25,7 +25,7 @@ BEGIN FROM tps.srce WHERE - srce = _name; + srce = _defn->>'name'; -------check for transctions already existing under this source----------- SELECT @@ -35,7 +35,7 @@ BEGIN FROM tps.trans WHERE - srce = _name; + srce = _defn->>'name'; --if there are transaction already and the schema is different stop-------- IF _cnt > 0 THEN @@ -61,14 +61,14 @@ BEGIN INSERT INTO tps.srce SELECT - _name, _defn + _defn->>'name', _defn ON CONFLICT ON CONSTRAINT srce_pkey DO UPDATE SET defn = _defn; ------------------drop existing type----------------------------------------- - EXECUTE format('DROP TYPE IF EXISTS tps.%I',_name); + EXECUTE format('DROP TYPE IF EXISTS tps.%I',_defn->>'name'); ------------------create new type-------------------------------------------- @@ -81,15 +81,15 @@ BEGIN --unwrap the schema definition array LEFT JOIN LATERAL jsonb_populate_recordset(null::tps.srce_defn_schema, defn->'schema') prs ON TRUE WHERE - srce = _name + srce = _defn->>'name' GROUP BY srce; - RAISE NOTICE 'CREATE TYPE tps.% AS (%)',_name,_sql; + RAISE NOTICE 'CREATE TYPE tps.% AS (%)',_defn->>'name',_sql; - EXECUTE format('CREATE TYPE tps.%I AS (%s)',_name,_sql); + EXECUTE format('CREATE TYPE tps.%I AS (%s)',_defn->>'name',_sql); - EXECUTE format('COMMENT ON TYPE tps.%I IS %L',_name,(_defn->>'description')); + EXECUTE format('COMMENT ON TYPE tps.%I IS %L',_defn->>'name',(_defn->>'description')); ----------------set message----------------------------------------------------- From f07f47ca3256227137565eff5e7fa27392f04bf8 Mon Sep 17 00:00:00 2001 From: fleetside72 Date: Fri, 9 Mar 2018 11:34:30 -0500 Subject: [PATCH 3/6] Update readme.md --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 3577ff5..5b10424 100644 --- a/readme.md +++ b/readme.md @@ -10,6 +10,7 @@ The goal is to: It is well suited for data from outside systems that * requires complex transformation (parsing and mapping) * original data is retained for reference +* don't feel like writing a map-reduce use cases: * on-going bank feeds From 3ea353a58852aa5feedf7e7b1e95dcae55da5f42 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 10 Mar 2018 02:02:38 -0500 Subject: [PATCH 4/6] get rid of dcard2 --- deploy/ubm_schema.sql | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/deploy/ubm_schema.sql b/deploy/ubm_schema.sql index bc31963..8a617fa 100644 --- a/deploy/ubm_schema.sql +++ b/deploy/ubm_schema.sql @@ -49,24 +49,6 @@ CREATE TYPE tps."DCARD" AS ( COMMENT ON TYPE tps."DCARD" IS 'Discover Card'; --- --- Name: DCARD2; Type: TYPE; Schema: tps; Owner: - --- - -CREATE TYPE tps."DCARD2" AS ( - "Trans. Date" numeric, - "Post Date" numeric, - "Description" text, - "Amount" numeric, - "Category" text -); - - --- --- Name: TYPE "DCARD2"; Type: COMMENT; Schema: tps; Owner: - --- - -COMMENT ON TYPE tps."DCARD2" IS 'Discover Card'; -- From e00e682f0254b6a3e7bdcddf369a5b6502f59955 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 10 Mar 2018 02:20:55 -0500 Subject: [PATCH 5/6] get rid of wmpd --- deploy/ubm_schema.sql | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/deploy/ubm_schema.sql b/deploy/ubm_schema.sql index 20ca8f1..9ede6d1 100644 --- a/deploy/ubm_schema.sql +++ b/deploy/ubm_schema.sql @@ -59,40 +59,6 @@ CREATE TYPE tps."DMAPI" AS ( ); --- --- Name: WMPD; Type: TYPE; Schema: tps; Owner: - --- - -CREATE TYPE tps."WMPD" AS ( - "Carrier" text, - "SCAC" text, - "Mode" text, - "Pro #" text, - "B/L" text, - "Pd Amt" numeric, - "Loc#" text, - "Pcs" numeric, - "Wgt" numeric, - "Chk#" numeric, - "Pay Dt" date, - "Acct #" text, - "I/O" text, - "Sh Nm" text, - "Sh City" text, - "Sh St" text, - "Sh Zip" text, - "Cons Nm" text, - "D City " text, - "D St" text, - "D Zip" text, - "Sh Dt" date, - "Inv Dt" date, - "Customs Entry#" text, - "Miles" numeric, - "Frt Class" text, - "Master B/L" text -); - -- -- Name: dcard; Type: TYPE; Schema: tps; Owner: - From 3e970c65e2a602ea325f8bbcb1ce0b0da0068ac2 Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Sat, 10 Mar 2018 02:27:19 -0500 Subject: [PATCH 6/6] 2 lines --- deploy/ubm_schema.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/ubm_schema.sql b/deploy/ubm_schema.sql index 9ede6d1..27f3e5b 100644 --- a/deploy/ubm_schema.sql +++ b/deploy/ubm_schema.sql @@ -49,7 +49,6 @@ CREATE TYPE tps."DCARD" AS ( COMMENT ON TYPE tps."DCARD" IS 'Discover Card'; - -- -- Name: DMAPI; Type: TYPE; Schema: tps; Owner: - -- @@ -59,7 +58,6 @@ CREATE TYPE tps."DMAPI" AS ( ); - -- -- Name: dcard; Type: TYPE; Schema: tps; Owner: - --