From eb11380798d0c804a2407845bdba6203d305004a Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 31 May 2018 00:44:46 -0400 Subject: [PATCH] get rid of unnecessary files --- perf_test/parse_jsonb_with_path.sql | 44 -- perf_test/parse_jsonb_with_path2.sql | 37 -- sample_google_api/data.json | 31 -- sample_google_api/def.json | 31 -- sample_google_api/test/import.sql | 37 -- sample_google_api/test/srce.sql | 39 -- sample_google_api/test_scipt.sql | 89 ---- templates/insert_constraint.json | 6 - templates/map_rm_template.json | 471 -------------------- templates/regex.json | 25 ++ templates/{srce_template.json => srce.json} | 0 11 files changed, 25 insertions(+), 785 deletions(-) delete mode 100644 perf_test/parse_jsonb_with_path.sql delete mode 100644 perf_test/parse_jsonb_with_path2.sql delete mode 100644 sample_google_api/data.json delete mode 100644 sample_google_api/def.json delete mode 100644 sample_google_api/test/import.sql delete mode 100644 sample_google_api/test/srce.sql delete mode 100644 sample_google_api/test_scipt.sql delete mode 100644 templates/insert_constraint.json delete mode 100644 templates/map_rm_template.json create mode 100644 templates/regex.json rename templates/{srce_template.json => srce.json} (100%) diff --git a/perf_test/parse_jsonb_with_path.sql b/perf_test/parse_jsonb_with_path.sql deleted file mode 100644 index 31c26c8..0000000 --- a/perf_test/parse_jsonb_with_path.sql +++ /dev/null @@ -1,44 +0,0 @@ -create temp table x as ( -select - t.rec -from - generate_series(1,1000000,1) s - inner join tps.trans t on - srce = 'DMAPI' -) with data; - - -create temp table x2 as ( -select - ( - rec #>>( - '{doc,origin_addresses,0}'::text[] - ) - )::text as origin_address, - ( - rec #>>( - '{doc,destination_addresses,0}'::text[] - ) - )::text as desatination_address, - ( - rec #>>( - '{doc,status}'::text[] - ) - )::text as status, - ( - rec #>>( - '{doc,rows,0,elements,0,distance,value}'::text[] - ) - )::numeric as distance, - ( - rec #>>( - '{doc,rows,0,elements,0,duration,value}'::text[] - ) - )::numeric as duration -from - x -) with data; - - -drop table x; -drop table x2; \ No newline at end of file diff --git a/perf_test/parse_jsonb_with_path2.sql b/perf_test/parse_jsonb_with_path2.sql deleted file mode 100644 index 2ab1fde..0000000 --- a/perf_test/parse_jsonb_with_path2.sql +++ /dev/null @@ -1,37 +0,0 @@ -create temp table x as ( -select - (rec #>>('{batch}'::text[]))::text as batch - ,(rec #>>('{week}'::text[]))::text as week - ,(rec #>>('{period_end}'::text[]))::text as period_end - ,(rec #>>('{pay_date}'::text[]))::text as pay_date - ,(rec #>>('{adp_comp}'::text[]))::text as adp_comp - ,(rec #>>('{hours_reg}'::text[]))::numeric as hours_reg - ,(rec #>>('{hours_ot}'::text[]))::numeric as hours_ot - ,(rec #>>('{adp_dep_home}'::text[]))::text as adp_dep_home - ,(rec #>>('{adp_dep}'::text[]))::text as adp_dep - ,(rec #>>('{gl_dep}'::text[]))::text as gl_dep - ,(rec #>>('{checkn}'::text[]))::text as checkn - ,(rec #>>('{employee}'::text[]))::text as employee - ,(rec #>>('{title}'::text[]))::text as title - ,(rec #>>('{prim_offset}'::text[]))::text as prim_offset - ,(rec #>>('{cms_tb}'::text[]))::text as cms_tb - ,(rec #>>('{cms_acct}'::text[]))::text as cms_acct - ,(rec #>>('{gl_descr}'::text[]))::text as gl_descr - ,(rec #>>('{amount}'::text[]))::numeric as amount -FROM - tps.trans -WHERE - srce = 'ADPRP' - ) with data - --- SELECT 1603392 Query returned successfully in 13 secs 604 msec. - - -/* -build to table --> 13 sec -run an aggregate on the table --> 1.5 sec --versus- -run a basic aggregate on the json data live --> 7 sec --versus- -run a basic aggregate on the json data with jsonb_popualte_record --> 8 sec -*/ \ No newline at end of file diff --git a/sample_google_api/data.json b/sample_google_api/data.json deleted file mode 100644 index f8afdad..0000000 --- a/sample_google_api/data.json +++ /dev/null @@ -1,31 +0,0 @@ -[ - { - "id": 1, - "doc": { - "rows": [ - { - "elements": [ - { - "status": "OK", - "distance": { - "text": "225 mi", - "value": 361940 - }, - "duration": { - "text": "3 hours 50 mins", - "value": 13812 - } - } - ] - } - ], - "status": "OK", - "origin_addresses": [ - "Washington, DC, USA" - ], - "destination_addresses": [ - "New York, NY, USA" - ] - } - } -] \ No newline at end of file diff --git a/sample_google_api/def.json b/sample_google_api/def.json deleted file mode 100644 index 3c190ef..0000000 --- a/sample_google_api/def.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "DMAPI", - "type": "csv", - "schemas": { - "default": [ - { - "path": "{doc,origin_addresses,0}", - "type": "text", - "column_name": "origin_address" - }, - { - "path": "{doc,destination_addresses,0}", - "type": "text", - "column_name": "destination_address" - }, - { - "path": "{doc,rows,0,elements,0,distance,value}", - "type": "numeric", - "column_name": "distince" - }, - { - "path": "{doc,rows,0,elements,0,duration,value}", - "type": "numeric", - "column_name": "duration" - } - ], - "constraint": [ - "{doc}" - ] - } -} \ No newline at end of file diff --git a/sample_google_api/test/import.sql b/sample_google_api/test/import.sql deleted file mode 100644 index 21013ed..0000000 --- a/sample_google_api/test/import.sql +++ /dev/null @@ -1,37 +0,0 @@ -SELECT - * -FROM - tps.srce_import( - 'DMAPI' - ,$$ - [{ - "id": 1, - "doc": { - "rows": [ - { - "elements": [ - { - "status": "OK", - "distance": { - "text": "225 mi", - "value": 361940 - }, - "duration": { - "text": "3 hours 50 mins", - "value": 13812 - } - } - ] - } - ], - "status": "OK", - "origin_addresses": [ - "Washington, DC, USA" - ], - "destination_addresses": [ - "New York, NY, USA" - ] - } - }] - $$::JSONB - ) \ No newline at end of file diff --git a/sample_google_api/test/srce.sql b/sample_google_api/test/srce.sql deleted file mode 100644 index d357f42..0000000 --- a/sample_google_api/test/srce.sql +++ /dev/null @@ -1,39 +0,0 @@ -SELECT - jsonb_pretty(r.x) -FROM - tps.srce_set( - $$ -{ - "name": "DMAPI", - "type": "csv", - "schemas": { - "default": [ - { - "path": "{doc,origin_addresses,0}", - "type": "text", - "column_name": "origin_address" - }, - { - "path": "{doc,destination_addresses,0}", - "type": "text", - "column_name": "destination_address" - }, - { - "path": "{doc,rows,0,elements,0,distance,value}", - "type": "numeric", - "column_name": "distince" - }, - { - "path": "{doc,rows,0,elements,0,duration,value}", - "type": "numeric", - "column_name": "duration" - } - ] - }, - "constraint": [ - "{doc,origin_addresses}", - "{doc,destination_addresses}" - ] -} - $$ -) r(x); \ No newline at end of file diff --git a/sample_google_api/test_scipt.sql b/sample_google_api/test_scipt.sql deleted file mode 100644 index cb8ff9b..0000000 --- a/sample_google_api/test_scipt.sql +++ /dev/null @@ -1,89 +0,0 @@ -delete from tps.trans where srce = 'DMAPI'; - -----------------------set definition----------------- -SELECT - jsonb_pretty(r.x) -FROM - tps.srce_set( - $$ -{ - "name": "DMAPI", - "type": "csv", - "schemas": { - "default": [ - { - "path": "{doc,origin_addresses,0}", - "type": "text", - "column_name": "origin_address" - }, - { - "path": "{doc,destination_addresses,0}", - "type": "text", - "column_name": "destination_address" - }, - { - "path": "{doc,rows,0,elements,0,distance,value}", - "type": "numeric", - "column_name": "distince" - }, - { - "path": "{doc,rows,0,elements,0,duration,value}", - "type": "numeric", - "column_name": "duration" - } - ], - "constraint": [ - "{doc}" - ] - } -} - $$ -) r(x); ---------------------------build a csv file--------------------- - -copy -( -select -$$ -{ - "id": 1, - "doc": { - "rows": [ - { - "elements": [ - { - "status": "OK", - "distance": { - "text": "225 mi", - "value": 361940 - }, - "duration": { - "text": "3 hours 50 mins", - "value": 13812 - } - } - ] - } - ], - "status": "OK", - "origin_addresses": [ - "Washington, DC, USA" - ], - "destination_addresses": [ - "New York, NY, USA" - ] - } -} -$$::JSONB DOC -) -to 'C:\users\fleet\downloads\testj.csv' with (FORMAT CSV, QUOTE '"', HEADER true); - ----------------------------------insert rows---------------------------------------- - -SELECT - * -FROM - tps.srce_import('C:\users\fleet\downloads\testj.csv','DMAPI') x(message); - - -select id, srce, jsonb_pretty(rec) from tps.trans where srce = 'DMAPI'; \ No newline at end of file diff --git a/templates/insert_constraint.json b/templates/insert_constraint.json deleted file mode 100644 index 438d296..0000000 --- a/templates/insert_constraint.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "constraint": [ - "{doc,origin_addresses}", - "{doc,destination_addresses}" - ] -} \ No newline at end of file diff --git a/templates/map_rm_template.json b/templates/map_rm_template.json deleted file mode 100644 index a06fb07..0000000 --- a/templates/map_rm_template.json +++ /dev/null @@ -1,471 +0,0 @@ -[ - { - "srce": "dcard", - "sequence": 1, - "defn": [ - { - "key": "{Description}", - "map": "y", - "flag": "", - "field": "f20", - "regex": ".{1,20}", - "retain": "y" - } - ], - "name": "First 20", - "where": [ - {} - ], - "function": "extract", - "description": "pull first 20 characters from description for mapping" - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Strip Amount Commas", - "description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric", - "defn": [ - { - "key": "{Amount}", - "field": "amount", - "regex": ",", - "replace": "", - "flag": "g", - "retain": "y", - "map": "n" - } - ], - "function": "replace", - "where": [ - {} - ] - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Parse ACH Credits", - "description": "parse select components of the description for ACH Credits Receieved", - "defn": [ - { - "key": "{Description}", - "field": "beneficiary", - "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Cust ID", - "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Desc", - "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "originator", - "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Batch Discr", - "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Comp ID", - "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Addenda", - "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "SETT", - "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Date", - "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Time", - "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - } - ], - "function": "extract", - "where": [ - { - "Transaction": "ACH Credits" - } - ] - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Parse ACH Debits", - "description": "parse select components of the description for ACH Credits Receieved", - "defn": [ - { - "key": "{Description}", - "field": "originator", - "regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Cust ID", - "regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Desc", - "regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "beneficiary", - "regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Batch Discr", - "regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Comp ID", - "regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Addenda", - "regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "SETT", - "regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Date", - "regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "Time", - "regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)", - "flag": "", - "retain": "y", - "map": "n" - } - ], - "function": "extract", - "where": [ - { - "Transaction": "ACH Debits" - } - ] - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Parse Wires", - "description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered", - "defn": [ - { - "key": "{Description}", - "field": "dparse", - "regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)", - "flag": "g", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "beneficiary_components", - "regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "originator_components", - "regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "beneficiary", - "regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "originator", - "regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "OBI", - "regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "RFB", - "regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "ABA", - "regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "BBI", - "regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "BENEBNK", - "regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "IBK", - "regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "RATE", - "regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - }, - { - "key": "{Description}", - "field": "RECVBNK", - "regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)", - "flag": "", - "retain": "y", - "map": "n" - } - ], - "function": "extract", - "where": [ - { - "Transaction": "Money Transfer DB - Wire" - }, - { - "Transaction": "Money Transfer DB - Other" - }, - { - "Transaction": "Money Transfer CR-Wire" - }, - { - "Transaction": "Money Transfer CR-Other" - }, - { - "Transaction": "Intl Money Transfer Debits" - }, - { - "Transaction": "Intl Money Transfer Credits" - } - ] - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Trans Type", - "description": "extract intial description in conjunction with account name and transaction type for mapping", - "defn": [ - { - "key": "{AccountName}", - "field": "acctn", - "regex": "(.*)", - "retain": "n", - "map": "y" - }, - { - "key": "{Transaction}", - "field": "trans", - "regex": "(.*)", - "retain": "n", - "map": "y" - }, - { - "key": "{Description}", - "field": "ini", - "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", - "retain": "y", - "map": "y" - } - ], - "where": [ - {} - ], - "function": "extract" - }, - { - "srce": "pncc", - "sequence": 1, - "name": "Currency", - "description": "pull out currency indicators from description of misc items and map", - "defn": [ - { - "key": "{Description}", - "field": "ini", - "regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)", - "retain": "y", - "map": "y" - }, - { - "key": "{Description}", - "field": "curr1", - "regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*", - "retain": "y", - "map": "y" - }, - { - "key": "{Description}", - "field": "curr2", - "regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*", - "retain": "y", - "map": "y" - } - ], - "where": [ - { - "Transaction": "Miscellaneous Credits" - }, - { - "Transaction": "Miscellaneous Debits" - } - ], - "function": "extract" - }, - { - "srce": "adprp", - "sequence": 1, - "name": "ADP Codes", - "description": "link to adp code definitions", - "defn": [ - { - "key": "{gl_descr}", - "field": "gl_descr", - "regex": ".*", - "flag": "", - "retain": "n", - "map": "y" - }, - { - "key": "{prim_offset}", - "field": "prim_offset", - "regex": ".*", - "flag": "", - "retain": "n", - "map": "y" - }, - { - "key": "{pay_date}", - "field": "pay_month", - "regex": ".{1,4}", - "flag": "", - "retain": "y", - "map": "n" - } - ], - "function": "extract", - "where": [ - {} - ] - } -] \ No newline at end of file diff --git a/templates/regex.json b/templates/regex.json new file mode 100644 index 0000000..a40517b --- /dev/null +++ b/templates/regex.json @@ -0,0 +1,25 @@ +[ + { + "regex": { + "function": "extract", + "description": "pull first 20 characters from description for mapping", + "where": [ + {} + ], + "defn": [ + { + "regex": ".{1,20}", + "map": "y", + "field": "f20", + "flag": "", + "key": "{Description}", + "retain": "y" + } + ], + "name": "First 20" + }, + "sequence": 2, + "name": "First 20", + "srce": "dcard" + } +] \ No newline at end of file diff --git a/templates/srce_template.json b/templates/srce.json similarity index 100% rename from templates/srce_template.json rename to templates/srce.json