From 3ffb8d4293118e3313462e26163763cb8c58cb0d Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Wed, 30 May 2018 09:48:30 -0400 Subject: [PATCH] pncc reload --- reload/pncc/extract.sql | 32 +++ reload/pncc/load.cmd | 5 +- reload/pncc/map.json | 475 ++++++++++++++++++++++++++++++++++++++-- reload/pncc/srce.json | 24 +- 4 files changed, 502 insertions(+), 34 deletions(-) create mode 100644 reload/pncc/extract.sql diff --git a/reload/pncc/extract.sql b/reload/pncc/extract.sql new file mode 100644 index 0000000..2edb86b --- /dev/null +++ b/reload/pncc/extract.sql @@ -0,0 +1,32 @@ +--transactions with date in download format for constraint +COPY +( +SELECT + to_char(r."AsOfDate",'mm/dd/yyyy') "AsOfDate" + ,r."BankId" + ,r."AccountNumber" + ,r."AccountName" + ,r."BaiControl" + ,r."Currency" + ,r."Transaction" + ,r."Reference" + ,r."Amount" + ,r."Description" +FROM + tps.trans + JOIN LATERAL jsonb_populate_record(NULL::tps.pncc, rec) r ON TRUE +WHERE + srce = 'PNCC' +) +TO 'C:\users\ptrowbridge\downloads\pncc.csv' WITH (format csv, header TRUE) + +--source +SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'PNCC' + +--mapdef +SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'PNCC') x + +--map values +SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'PNCC') X + + diff --git a/reload/pncc/load.cmd b/reload/pncc/load.cmd index caed4f2..fa087a8 100644 --- a/reload/pncc/load.cmd +++ b/reload/pncc/load.cmd @@ -1 +1,4 @@ -curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set \ No newline at end of file +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set +curl -H "Content-Type: application/json" -X POST -d@./map.json http://localhost:81/mapdef_set +curl -H "Content-Type: application/json" -X POST -d@./vals.json http://localhost:81/mapval_set +curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/pncc.csv http://localhost:81/import?srce=PNCC \ No newline at end of file diff --git a/reload/pncc/map.json b/reload/pncc/map.json index 461a018..f910106 100644 --- a/reload/pncc/map.json +++ b/reload/pncc/map.json @@ -1,22 +1,455 @@ -{ - "srce": "PNCC", - "name": "Check Number", - "regex": { - "defn": [ - { - "key": "{Description}", - "map": "n", - "field": "checkn", - "regex": "[^0-9]*([0-9]*)\\s|$", - "retain": "y" - } - ], - "where": [ - { - "Transaction": "Checks Paid" - } - ], - "function": "extract" +[ + { + "name": "Check Number", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Description}", + "map": "n", + "field": "checkn", + "regex": "[^0-9]*([0-9]*)\\s|$", + "retain": "y" + } + ], + "where": [ + { + "Transaction": "Checks Paid" + } + ], + "function": "extract" + }, + "sequence": 2 }, - "sequence": 2 -} \ No newline at end of file + { + "name": "Strip Amount Commas", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Amount}", + "map": "n", + "flag": "g", + "field": "amount", + "regex": ",", + "retain": "y", + "replace": "" + } + ], + "name": "Strip Amount Commas", + "where": [ + {} + ], + "function": "replace", + "description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric" + }, + "sequence": 1 + }, + { + "name": "Trans Type", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{AccountName}", + "map": "y", + "field": "acctn", + "regex": "(.*)", + "retain": "n" + }, + { + "key": "{Transaction}", + "map": "y", + "field": "trans", + "regex": "(.*)", + "retain": "n" + }, + { + "key": "{Description}", + "map": "y", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y" + } + ], + "name": "Trans Type", + "where": [ + {} + ], + "function": "extract", + "description": "extract intial description in conjunction with account name and transaction type for mapping" + }, + "sequence": 1 + }, + { + "name": "Currency", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Description}", + "map": "y", + "field": "ini", + "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "y", + "field": "curr1", + "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", + "retain": "y" + }, + { + "key": "{Description}", + "map": "y", + "field": "curr2", + "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", + "retain": "y" + } + ], + "name": "Currency", + "where": [ + { + "Transaction": "Miscellaneous Credits" + }, + { + "Transaction": "Miscellaneous Debits" + } + ], + "function": "extract", + "description": "pull out currency indicators from description of misc items and map" + }, + "sequence": 2 + }, + { + "name": "Parse ACH Credits", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "beneficiary", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Cust ID", + "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Desc", + "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "originator", + "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Batch Discr", + "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Comp ID", + "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Addenda", + "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "SETT", + "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Date", + "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Time", + "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + } + ], + "name": "Parse ACH Credits", + "where": [ + { + "Transaction": "ACH Credits" + } + ], + "function": "extract", + "description": "parse select components of the description for ACH Credits Receieved" + }, + "sequence": 2 + }, + { + "name": "Parse ACH Debits", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "originator", + "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Cust ID", + "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Desc", + "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "beneficiary", + "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Batch Discr", + "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Comp ID", + "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Addenda", + "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "SETT", + "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Date", + "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "Time", + "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", + "retain": "y" + } + ], + "name": "Parse ACH Debits", + "where": [ + { + "Transaction": "ACH Debits" + } + ], + "function": "extract", + "description": "parse select components of the description for ACH Credits Receieved" + }, + "sequence": 2 + }, + { + "name": "Parse Wires", + "srce": "PNCC", + "regex": { + "defn": [ + { + "key": "{Description}", + "map": "n", + "flag": "g", + "field": "dparse", + "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "beneficiary_components", + "regex": "BENEFICIARY:(.*?)AC/([\\d-]*) (.*)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "originator_components", + "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "beneficiary", + "regex": "BENEFICIARY:(.*?)AC/[\\d-]* .*(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "originator", + "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "OBI", + "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "RFB", + "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "ABA", + "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "BBI", + "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "BENEBNK", + "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "IBK", + "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "RATE", + "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + }, + { + "key": "{Description}", + "map": "n", + "flag": "", + "field": "RECVBNK", + "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", + "retain": "y" + } + ], + "name": "Parse Wires", + "where": [ + { + "Transaction": "Money Transfer DB - Wire" + }, + { + "Transaction": "Money Transfer DB - Other" + }, + { + "Transaction": "Money Transfer CR-Wire" + }, + { + "Transaction": "Money Transfer CR-Other" + }, + { + "Transaction": "Intl Money Transfer Debits" + }, + { + "Transaction": "Intl Money Transfer Credits" + } + ], + "function": "extract", + "description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered" + }, + "sequence": 2 + } +] \ No newline at end of file diff --git a/reload/pncc/srce.json b/reload/pncc/srce.json index 6a4e78f..3564fa6 100644 --- a/reload/pncc/srce.json +++ b/reload/pncc/srce.json @@ -1,5 +1,5 @@ { - "name": "pncc", + "name": "PNCC", "source": "client_file", "loading_function": "csv", "constraint": [ @@ -8,57 +8,57 @@ "schemas": { "default": [ { - "path": "AsOfDate", + "path": "{AsOfDate}", "type": "date", "column_name": "AsOfDate" }, { - "path": "BankId", + "path": "{BankId}", "type": "text", "column_name": "BankId" }, { - "path": "AccountNumber", + "path": "{AccountNumber}", "type": "text", "column_name": "AccountNumber" }, { - "path": "AccountName", + "path": "{AccountName}", "type": "text", "column_name": "AccountName" }, { - "path": "BaiControl", + "path": "{BaiControl}", "type": "text", "column_name": "BaiControl" }, { - "path": "Currency", + "path": "{Currency}", "type": "text", "column_name": "Currency" }, { - "path": "Transaction", + "path": "{Transaction}", "type": "text", "column_name": "Transaction" }, { - "path": "Reference", + "path": "{Reference}", "type": "text", "column_name": "Reference" }, { - "path": "Amount", + "path": "{Amount}", "type": "text", "column_name": "Amount" }, { - "path": "Description", + "path":"{Description}", "type": "text", "column_name": "Description" }, { - "path": "AdditionalRemittance", + "path": "{AdditionalRemittance}", "type": "text", "column_name": "AdditionalRemittance" }