pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

get rid of unnecessary files

Browse Source
This commit is contained in:
Paul Trowbridge 2018-05-31 00:44:46 -04:00
parent 26ecccda00
commit eb11380798
11 changed files with 25 additions and 785 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
perf_test/parse_jsonb_with_path.sql
View File

@ -1,44 +0,0 @@
create temp table x as (
select
t.rec
from
generate_series(1,1000000,1) s
inner join tps.trans t on
srce = 'DMAPI'
) with data;
create temp table x2 as (
select
(
rec #>>(
'{doc,origin_addresses,0}'::text[]
)
)::text as origin_address,
(
rec #>>(
'{doc,destination_addresses,0}'::text[]
)
)::text as desatination_address,
(
rec #>>(
'{doc,status}'::text[]
)
)::text as status,
(
rec #>>(
'{doc,rows,0,elements,0,distance,value}'::text[]
)
)::numeric as distance,
(
rec #>>(
'{doc,rows,0,elements,0,duration,value}'::text[]
)
)::numeric as duration
from
x
) with data;
drop table x;
drop table x2;

37
perf_test/parse_jsonb_with_path2.sql
View File

@ -1,37 +0,0 @@
create temp table x as (
select
(rec #>>('{batch}'::text[]))::text as batch
,(rec #>>('{week}'::text[]))::text as week
,(rec #>>('{period_end}'::text[]))::text as period_end
,(rec #>>('{pay_date}'::text[]))::text as pay_date
,(rec #>>('{adp_comp}'::text[]))::text as adp_comp
,(rec #>>('{hours_reg}'::text[]))::numeric as hours_reg
,(rec #>>('{hours_ot}'::text[]))::numeric as hours_ot
,(rec #>>('{adp_dep_home}'::text[]))::text as adp_dep_home
,(rec #>>('{adp_dep}'::text[]))::text as adp_dep
,(rec #>>('{gl_dep}'::text[]))::text as gl_dep
,(rec #>>('{checkn}'::text[]))::text as checkn
,(rec #>>('{employee}'::text[]))::text as employee
,(rec #>>('{title}'::text[]))::text as title
,(rec #>>('{prim_offset}'::text[]))::text as prim_offset
,(rec #>>('{cms_tb}'::text[]))::text as cms_tb
,(rec #>>('{cms_acct}'::text[]))::text as cms_acct
,(rec #>>('{gl_descr}'::text[]))::text as gl_descr
,(rec #>>('{amount}'::text[]))::numeric as amount
FROM
tps.trans
WHERE
srce = 'ADPRP'
) with data
-- SELECT 1603392 Query returned successfully in 13 secs 604 msec.
/*
build to table --> 13 sec
run an aggregate on the table --> 1.5 sec
-versus-
run a basic aggregate on the json data live --> 7 sec
-versus-
run a basic aggregate on the json data with jsonb_popualte_record --> 8 sec
*/

31
sample_google_api/data.json
View File

@ -1,31 +0,0 @@
[
{
"id": 1,
"doc": {
"rows": [
{
"elements": [
{
"status": "OK",
"distance": {
"text": "225 mi",
"value": 361940
},
"duration": {
"text": "3 hours 50 mins",
"value": 13812
}
}
]
}
],
"status": "OK",
"origin_addresses": [
"Washington, DC, USA"
],
"destination_addresses": [
"New York, NY, USA"
]
}
}
]

31
sample_google_api/def.json
View File

@ -1,31 +0,0 @@
{
"name": "DMAPI",
"type": "csv",
"schemas": {
"default": [
{
"path": "{doc,origin_addresses,0}",
"type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
}
],
"constraint": [
"{doc}"
]
}
}

37
sample_google_api/test/import.sql
View File

@ -1,37 +0,0 @@
SELECT
*
FROM
tps.srce_import(
'DMAPI'
,$$
[{
"id": 1,
"doc": {
"rows": [
{
"elements": [
{
"status": "OK",
"distance": {
"text": "225 mi",
"value": 361940
},
"duration": {
"text": "3 hours 50 mins",
"value": 13812
}
}
]
}
],
"status": "OK",
"origin_addresses": [
"Washington, DC, USA"
],
"destination_addresses": [
"New York, NY, USA"
]
}
}]
$$::JSONB
)

39
sample_google_api/test/srce.sql
View File

@ -1,39 +0,0 @@
SELECT
jsonb_pretty(r.x)
FROM
tps.srce_set(
$$
{
"name": "DMAPI",
"type": "csv",
"schemas": {
"default": [
{
"path": "{doc,origin_addresses,0}",
"type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
}
]
},
"constraint": [
"{doc,origin_addresses}",
"{doc,destination_addresses}"
]
}
$$
) r(x);

89
sample_google_api/test_scipt.sql
View File

@ -1,89 +0,0 @@
delete from tps.trans where srce = 'DMAPI';
----------------------set definition-----------------
SELECT
jsonb_pretty(r.x)
FROM
tps.srce_set(
$$
{
"name": "DMAPI",
"type": "csv",
"schemas": {
"default": [
{
"path": "{doc,origin_addresses,0}",
"type": "text",
"column_name": "origin_address"
},
{
"path": "{doc,destination_addresses,0}",
"type": "text",
"column_name": "destination_address"
},
{
"path": "{doc,rows,0,elements,0,distance,value}",
"type": "numeric",
"column_name": "distince"
},
{
"path": "{doc,rows,0,elements,0,duration,value}",
"type": "numeric",
"column_name": "duration"
}
],
"constraint": [
"{doc}"
]
}
}
$$
) r(x);
--------------------------build a csv file---------------------
copy
(
select
$$
{
"id": 1,
"doc": {
"rows": [
{
"elements": [
{
"status": "OK",
"distance": {
"text": "225 mi",
"value": 361940
},
"duration": {
"text": "3 hours 50 mins",
"value": 13812
}
}
]
}
],
"status": "OK",
"origin_addresses": [
"Washington, DC, USA"
],
"destination_addresses": [
"New York, NY, USA"
]
}
}
$$::JSONB DOC
)
to 'C:\users\fleet\downloads\testj.csv' with (FORMAT CSV, QUOTE '"', HEADER true);
---------------------------------insert rows----------------------------------------
SELECT
*
FROM
tps.srce_import('C:\users\fleet\downloads\testj.csv','DMAPI') x(message);
select id, srce, jsonb_pretty(rec) from tps.trans where srce = 'DMAPI';

6
templates/insert_constraint.json
View File

@ -1,6 +0,0 @@
{
"constraint": [
"{doc,origin_addresses}",
"{doc,destination_addresses}"
]
}

471
templates/map_rm_template.json
View File

@ -1,471 +0,0 @@
[
{
"srce": "dcard",
"sequence": 1,
"defn": [
{
"key": "{Description}",
"map": "y",
"flag": "",
"field": "f20",
"regex": ".{1,20}",
"retain": "y"
}
],
"name": "First 20",
"where": [
{}
],
"function": "extract",
"description": "pull first 20 characters from description for mapping"
},
{
"srce": "pncc",
"sequence": 1,
"name": "Strip Amount Commas",
"description": "the Amount field come from PNC with commas embeded so it cannot be cast to numeric",
"defn": [
{
"key": "{Amount}",
"field": "amount",
"regex": ",",
"replace": "",
"flag": "g",
"retain": "y",
"map": "n"
}
],
"function": "replace",
"where": [
{}
]
},
{
"srce": "pncc",
"sequence": 1,
"name": "Parse ACH Credits",
"description": "parse select components of the description for ACH Credits Receieved",
"defn": [
{
"key": "{Description}",
"field": "beneficiary",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Cust ID",
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Desc",
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "originator",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Batch Discr",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Comp ID",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Addenda",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "SETT",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Date",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Time",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
}
],
"function": "extract",
"where": [
{
"Transaction": "ACH Credits"
}
]
},
{
"srce": "pncc",
"sequence": 1,
"name": "Parse ACH Debits",
"description": "parse select components of the description for ACH Credits Receieved",
"defn": [
{
"key": "{Description}",
"field": "originator",
"regex": "Comp Name:(.+?)(?=\\d{6} Com|SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Cust ID",
"regex": "Cust ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Desc",
"regex": "Desc:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "beneficiary",
"regex": "Cust Name:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Batch Discr",
"regex": "Batch Discr:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Comp ID",
"regex": "Comp ID:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Addenda",
"regex": "Addenda:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "SETT",
"regex": "SETT:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Date",
"regex": "Date:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "Time",
"regex": "Time:(.+?)(?=SEC:|Cust ID:|Desc:|Comp Name:|Comp ID:|Batch Discr:|Cust Name:|Addenda:|SETT:|Date:|Time:|$)",
"flag": "",
"retain": "y",
"map": "n"
}
],
"function": "extract",
"where": [
{
"Transaction": "ACH Debits"
}
]
},
{
"srce": "pncc",
"sequence": 1,
"name": "Parse Wires",
"description": "pull out whatever follows OBI in the description until atleast 3 capital letters followed by a colon are encountered",
"defn": [
{
"key": "{Description}",
"field": "dparse",
"regex": "([A-Z]{3,}?:)(.*)(?=[A-Z]{3,}?:|$)",
"flag": "g",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "beneficiary_components",
"regex": "BENEFICIARY:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "originator_components",
"regex": "ORIGINATOR:(.*?)AC/(\\d*) (.*)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "beneficiary",
"regex": "BENEFICIARY:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "originator",
"regex": "ORIGINATOR:(.*?)AC/\\d* .*(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "OBI",
"regex": "OBI:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "RFB",
"regex": "RFB:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "ABA",
"regex": "ABA:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "BBI",
"regex": "BBI:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "BENEBNK",
"regex": "BENEBNK:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "IBK",
"regex": "IBK:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "RATE",
"regex": "RATE:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
},
{
"key": "{Description}",
"field": "RECVBNK",
"regex": "RECVBNK:(.*?)(?=[A-Z]{3,}?:|$)",
"flag": "",
"retain": "y",
"map": "n"
}
],
"function": "extract",
"where": [
{
"Transaction": "Money Transfer DB - Wire"
},
{
"Transaction": "Money Transfer DB - Other"
},
{
"Transaction": "Money Transfer CR-Wire"
},
{
"Transaction": "Money Transfer CR-Other"
},
{
"Transaction": "Intl Money Transfer Debits"
},
{
"Transaction": "Intl Money Transfer Credits"
}
]
},
{
"srce": "pncc",
"sequence": 1,
"name": "Trans Type",
"description": "extract intial description in conjunction with account name and transaction type for mapping",
"defn": [
{
"key": "{AccountName}",
"field": "acctn",
"regex": "(.*)",
"retain": "n",
"map": "y"
},
{
"key": "{Transaction}",
"field": "trans",
"regex": "(.*)",
"retain": "n",
"map": "y"
},
{
"key": "{Description}",
"field": "ini",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y",
"map": "y"
}
],
"where": [
{}
],
"function": "extract"
},
{
"srce": "pncc",
"sequence": 1,
"name": "Currency",
"description": "pull out currency indicators from description of misc items and map",
"defn": [
{
"key": "{Description}",
"field": "ini",
"regex": "([\\w].*?)(?=$| -|\\s[0-9].*?|\\s[\\w/]+?:)",
"retain": "y",
"map": "y"
},
{
"key": "{Description}",
"field": "curr1",
"regex": ".*(DEBIT|CREDIT).*(USD|CAD).*(?=DEBIT|CREDIT).*(?=USD|CAD).*",
"retain": "y",
"map": "y"
},
{
"key": "{Description}",
"field": "curr2",
"regex": ".*(?=DEBIT|CREDIT).*(?=USD|CAD).*(DEBIT|CREDIT).*(USD|CAD).*",
"retain": "y",
"map": "y"
}
],
"where": [
{
"Transaction": "Miscellaneous Credits"
},
{
"Transaction": "Miscellaneous Debits"
}
],
"function": "extract"
},
{
"srce": "adprp",
"sequence": 1,
"name": "ADP Codes",
"description": "link to adp code definitions",
"defn": [
{
"key": "{gl_descr}",
"field": "gl_descr",
"regex": ".*",
"flag": "",
"retain": "n",
"map": "y"
},
{
"key": "{prim_offset}",
"field": "prim_offset",
"regex": ".*",
"flag": "",
"retain": "n",
"map": "y"
},
{
"key": "{pay_date}",
"field": "pay_month",
"regex": ".{1,4}",
"flag": "",
"retain": "y",
"map": "n"
}
],
"function": "extract",
"where": [
{}
]
}
]

25
templates/regex.json Normal file
View File

@ -0,0 +1,25 @@
[
{
"regex": {
"function": "extract",
"description": "pull first 20 characters from description for mapping",
"where": [
{}
],
"defn": [
{
"regex": ".{1,20}",
"map": "y",
"field": "f20",
"flag": "",
"key": "{Description}",
"retain": "y"
}
],
"name": "First 20"
},
"sequence": 2,
"name": "First 20",
"srce": "dcard"
}
]

0
templates/srce_template.json → templates/srce.json
View File