pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

move files to reload and get rid of the rest

Browse Source
This commit is contained in:
Paul Trowbridge 2018-05-30 22:41:50 -04:00
parent 494c0b13de
commit 56fe1df074
16 changed files with 0 additions and 566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
sample_discovercard/curl_import.cmd → reload/dcard/curl_dcard.cmd
View File

0
sample_discovercard/data.csv → reload/dcard/data.csv
View File

0
sample_discovercard/data.json → reload/dcard/data.json
View File

0
sample_discovercard/mapdef.json → reload/dcard/mapdef.json
View File

0
sample_discovercard/srce.json → reload/dcard/srce.json
View File

0
sample_discovercard/vals.json → reload/dcard/vals.json
View File

1
sample_discovercard/curl_mapdef.cmd
View File

@ -1 +0,0 @@
curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set

1
sample_discovercard/curl_mapvals.cmd
View File

@ -1 +0,0 @@
curl -H "Content-Type: application/json" -X POST -d@//mnt/c/Users/fleet/Documents/tps_etl/reload/mapval.json http://localhost:81/mapval_set

1
sample_discovercard/curl_srcedef.cmd
View File

@ -1 +0,0 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set

157
sample_discovercard/definition.md
View File

@ -1,157 +0,0 @@
sample data
------------------------------------
| Trans. Date | Post Date | Description | Amount | Category |
| ----------- | --------- | ---------------------------------------------- | ------ | -------------------- |
| 1/2/2018 | 1/2/2018 | GOOGLE *YOUTUBE VIDEOS G.CO/HELPPAY#CAP0H07TXV | 4.26 | Services |
| 1/2/2018 | 1/2/2018 | MICROSOFT *ONEDRIVE 800-642-7676 WA | 4.26 | Services |
| 1/3/2018 | 1/3/2018 | CLE CLINIC PT PMTS 216-445-6249 OHAK2C57F2F0B3 | 200 | Medical Services |
| 1/4/2018 | 1/4/2018 | AT&T *PAYMENT 800-288-2020 TX | 57.14 | Services |
| 1/4/2018 | 1/7/2018 | WWW.KOHLS.COM #0873 MIDDLETOWN OH | -7.9 | Payments and Credits |
| 1/5/2018 | 1/7/2018 | PIZZA HUT 007946 STOW OH | 9.24 | Restaurants |
| 1/5/2018 | 1/7/2018 | SUBWAY 00044289255 STOW OH | 10.25 | Restaurants |
| 1/6/2018 | 1/7/2018 | ACME NO. 17 STOW OH | 103.98 | Supermarkets |
| 1/6/2018 | 1/7/2018 | DISCOUNT DRUG MART 32 STOW OH | 1.69 | Merchandise |
| 1/6/2018 | 1/7/2018 | DISCOUNT DRUG MART 32 STOW OH | 2.19 | Merchandise |
| 1/9/2018 | 1/9/2018 | CIRCLE K 05416 STOW OH00947R | 3.94 | Gasoline |
| 1/9/2018 | 1/9/2018 | CIRCLE K 05416 STOW OH00915R | 52.99 | Gasoline |
screen
------------------------------------
```
+---------------+
Name:|DCARD |
+---------------+
+---------------+
Desc:|Discover Card |
+---------------+
Col Name Data Type Unique Constraint Flag
+-----------------------------------------------------------------------+
+-----------------+ +-------------------+ +---+
|Trans. Date | |date |\/| | X |
+-----------------+ +-------------------+ +---+
+-----------------+ +-------------------+ +---+
|Post Date | |date |\/| | X |
+-----------------+ +-------------------+ +---+
+-----------------+ +-------------------+ +---+
|Description | |text |\/| | X |
+-----------------+ +-------------------+ +---+
+-----------------+ +-------------------+ +---+
|Amount | |numeric |\/| | |
+-----------------+ +-------------------+ +---+
+-----------------+ +-------------------+ +---+
|Category | |text |\/| | |
+-----------------+ +-------------------+ +---+
Somehow be able to add more
```
screen builds json
--------------------------------------
{
"name": "DCARD",
"description":"Discover Card",
"type": "csv",
"schema": [
{
"key": "Trans. Date",
"type": "date"
},
{
"key": "Post Date",
"type": "date"
},
{
"key": "Description",
"type": "text"
},
{
"key": "Amount",
"type": "numeric"
},
{
"key": "Category",
"type": "text"
}
],
"unique_constraint": {
"type": "key",
"fields": [
"{Post Date}",
"{Trans. Date}",
"{Description}"
]
}
}
SQL
---------------------------------------
SELECT
jsonb_pretty(r.x)
FROM
tps.srce_set(
'DCARD',
$$
{
"name": "DCARD",
"description":"Discover Card",
"type": "csv",
"schema": [
{
"key": "Trans. Date",
"type": "date"
},
{
"key": "Post Date",
"type": "date"
},
{
"key": "Description",
"type": "text"
},
{
"key": "Amount",
"type": "numeric"
},
{
"key": "Category",
"type": "text"
}
],
"unique_constraint": {
"type": "key",
"fields": [
"{Post Date}",
"{Trans. Date}",
"{Description}"
]
}
}
$$
) r(x)
backend handles SQL
-----------------------------------
`sql = "SELECT tps.srce_set(_name, _json)"`
`json_return_value = connection.execute(sql)`
handle json_return_value
* insert: notify and clear? update list of sources on screen?
* could not insert: print reason from json
import the data
------------------------------------
SELECT
*
FROM
tps.srce_import('C:\users\fleet\documents\tps_etl\sample_discovercard\data.csv','DCARD') x(message)

169
sample_discovercard/mapping.md
View File

@ -1,169 +0,0 @@
regular expression screen
---------------------------------------------
```
+------------------------+----+
source | DCARD | \/ |
+-----------------------------+
+-----------------------------+
new name |First 20 |
+-----------------------------+
+-----------------------------+
function | extract(replace) | \/ |
+------------------------+----+
(each block is a regex, all blocks are concatenated into an array and linked to mapped ^alues)
first 20 characters
+----------------------------------+
+----------------------------------------------------------------------------------------------------+
+ | +-----------------------------+ |
| | Field Name| Description | \/| expressed as jsonb path "{Description}") |
+-------+ | +-----------------------------+ |
| | +-------------------------+ |
+ |label of return ^alue |f20 | |
| +-------------------------+ |
| +-------------------------+ |
| regular expression |.{1,20} | supply test run ^alues |
| +-------------------------+ |
| +-------------------------+ |
| replace wit | | |
| +-------------------------+ |
| +---+ |
| Map Results|Y/N| |
| +---+ |
| +---+ |
| Find All Matches| g | |
| +---+ |
| filter(s) |
| +-------------------------------------------+ |
| +----------------+ +----------------+ |
| |Category | |Restaurantes | |
| +----------------+ +----------------+ |
| +----------------+ +----------------+ |
| |Category | |Services | |
| +----------------+ +----------------+ |
| |
+----------------------------------------------------------------------------------------------------+
```
map definition
----------------------------------------------------------
{
"defn": [
{
"key": "{Description}",
"map": "y",
"flag": "g",
"field": "f20",
"regex": ".{1,20}",
"retain": "y"
}
],
"name": "First 20",
"where": [
{"Category":"Restaurantes"},
{"Category":"Services"}
],
"function": "extract",
"description": "pull first 20 characters from description for mapping"
}
SQL
---------------------------------------------
SELECT
x.x
FROM
tps.srce_map_def_set(
'DCARD'::text,
'First 20'::text,
$$ {
"defn": [
{
"key": "{Description}",
"map": "y",
"flag": "",
"field": "f20",
"regex": ".{1,20}",
"retain": "y"
}
],
"name": "First 20",
"where": [
{}
],
"function": "extract",
"description": "pull first 20 characters from description for mapping"
} $$::jsonb,
1::int
) x(x)
assign new key/values to the results of the regular expression, and then back to the underlying row it came from
-----------------------------------------------------------------------------------------------------------------
| returned from expression | party | reason | city | provice | |
| ------------------------------- | ----------------- | ------------- | ------ | ------- | --- |
| {"f20": "BIG LOTS #00453 STOW"} | Big Lots | Home Supplies | Stow | Ohio | |
| {"f20": "1794MOTHERHOOD #1794"} | Motherhood | Clothes | | | |
| {"f20": "3 PALMS HUDSON OH"} | 3 Palms | Restaurantes | Hudson | Ohio | |
| {"f20": "36241 7-ELEVEN STOW "} | 7-Eleven | Gasoline | Stow | Ohio | |
| {"f20": "7-ELEVEN 36241 STOW "} | 7-Eleven | Gasoline | Stow | Ohio | |
| {"f20": "98626 - 200 PUBLIC S"} | Public Sq Parking | Recreation | | | |
| {"f20": "ACE HARDWARE HUDSON "} | Ace Hardware | Home Maint | Hudson | Ohio | |
| {"f20": "ACH CAFE AND STARBUC"} | Starbucks | Restaurantes | | | |
function call to setup a a new map value
---------------------------------------------
SELECT
x.message
FROM
tps.srce_map_val_set(
'DCARD'
,'First 20'
,'{"f20": "DISCOUNT DRUG MART 3"}'::JSONB
,'{"party":"Discount Drug Mart","reason":"groceries"}'::JSONB
) x(message);
function call to re-run all the maps for a source
----------------------------------------------------
SELECT
x.message
FROM
tps.srce_map_overwrite('DCARD') x(message);
mass insert
------------------------------------------------------
INSERT INTO
tps.map_rv
SELECT
*
FROM
(
VALUES
('DCARD','First 20','{"f20": "DISCOUNT DRUG MART 3"}','{"party":"Discount Drug Mart","reason":"groceries"}'),
('DCARD','First 20','{"f20": "TARGET STOW OH"}','{"party":"Target","reason":"groceries"}'),
('DCARD','First 20','{"f20": "WALMART GROCERY 800-"}','{"party":"Walmart","reason":"groceries"}'),
('DCARD','First 20','{"f20": "CIRCLE K 05416 STOW "}','{"party":"Circle K","reason":"gasoline"}'),
('DCARD','First 20','{"f20": "TARGET.COM * 800-591"}','{"party":"Target","reason":"home supplies"}'),
('DCARD','First 20','{"f20": "ACME NO. 17 STOW OH"}','{"party":"Acme","reason":"groceries"}'),
('DCARD','First 20','{"f20": "AT&T *PAYMENT 800-28"}','{"party":"AT&T","reason":"internet"}'),
('DCARD','First 20','{"f20": "AUTOZONE #0722 STOW "}','{"party":"Autozone","reason":"auto maint"}'),
('DCARD','First 20','{"f20": "BESTBUYCOM8055267948"}','{"party":"BestBuy","reason":"home supplies"}'),
('DCARD','First 20','{"f20": "BUFFALO WILD WINGS K"}','{"party":"Buffalo Wild Wings","reason":"restaurante"}'),
('DCARD','First 20','{"f20": "CASHBACK BONUS REDEM"}','{"party":"Discover Card","reason":"financing"}'),
('DCARD','First 20','{"f20": "CLE CLINIC PT PMTS 2"}','{"party":"Cleveland Clinic","reason":"medical"}')
) x

4
sample_discovercard/test_scripts/srce_set_test.sql
View File

@ -1,4 +0,0 @@
SELECT * FROM TPS.SRCE_SET(
$$
{"name":"dcard","source":"client_file","loading_function":"csv","constraint":["{Trans. Date}","{Post Date}","{Description}"],"schemas":{"default":[{"path":"{Trans. Date}","type":"date","column_name":"Trans. Date"},{"path":"{Post Date}","type":"date","column_name":"Post Date"},{"path":"{Description}","type":"text","column_name":"Description"},{"path":"{Amount}","type":"numeric","column_name":"Amount"},{"path":"{Category}","type":"text","column_name":"Category"}],"mapped":[{"path":"{Trans. Date}","type":"date","column_name":"Trans. Date"},{"path":"{Post Date}","type":"date","column_name":"Post Date"},{"path":"{Description}","type":"text","column_name":"Description"},{"path":"{Amount}","type":"numeric","column_name":"Amount"},{"path":"{Category}","type":"text","column_name":"Category"},{"path":"{party}","type":"text","column_name":"Party"},{"path":"{reason}","type":"text","column_name":"Reason"}]}}
$$::JSONB)

27
sample_discovercard/test_scripts/test_map_set.sql
View File

@ -1,27 +0,0 @@
SELECT
*
FROM
tps.srce_map_def_set(
$$
{
"srce":"dcard",
"sequence":1,
"defn": [
{
"key": "{Description}",
"map": "y",
"flag": "",
"field": "f20",
"regex": ".{1,20}",
"retain": "y"
}
],
"name": "First 20",
"where": [
{}
],
"function": "extract",
"description": "pull first 20 characters from description for mapping"
}
$$
)

6
sample_discovercard/test_scripts/test_map_val_set.sql
View File

@ -1,6 +0,0 @@
SELECT
*
FROM
tps.map_rv_set(
$$[{"source":"dcard","map":"First 20","ret_val":{"f20": "DISCOUNT DRUG MART 3"},"mapped":{"party":"Discount Drug Mart","reason":"groceries"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "TARGET STOW OH"},"mapped":{"party":"Target","reason":"groceries"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "WALMART GROCERY 800-"},"mapped":{"party":"Walmart","reason":"groceries"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "CIRCLE K 05416 STOW "},"mapped":{"party":"Circle K","reason":"gasoline"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "TARGET.COM * 800-591"},"mapped":{"party":"Target","reason":"home supplies"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "ACME NO. 17 STOW OH"},"mapped":{"party":"Acme","reason":"groceries"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "AT&T *PAYMENT 800-28"},"mapped":{"party":"AT&T","reason":"internet"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "AUTOZONE #0722 STOW "},"mapped":{"party":"Autozone","reason":"auto maint"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "BESTBUYCOM8055267948"},"mapped":{"party":"BestBuy","reason":"home supplies"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "BUFFALO WILD WINGS K"},"mapped":{"party":"Buffalo Wild Wings","reason":"restaurante"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "CASHBACK BONUS REDEM"},"mapped":{"party":"Discover Card","reason":"financing"}},{"source":"dcard","map":"First 20","ret_val":{"f20": "CLE CLINIC PT PMTS 2"},"mapped":{"party":"Cleveland Clinic","reason":"medical"}}]$$::jsonb
)

7
sample_discovercard/test_scripts/test_srce_import.sql
View File

File diff suppressed because one or more lines are too long

193
sample_paycom/srce.json
View File

@ -1,193 +0,0 @@
{
"name": "paycom",
"source": "client_file",
"loading_function": "csv",
"unique_constraint": {
"type": "key",
"fields": ["{transaction}", "{loc_code}", "{perd_start}", "{perd_end}"]
},
"schemas": {
"default": [
{
"key": "perd_start",
"type": "date",
"column_name": "perd_start"
},
{
"key": "perd_end",
"type": "date",
"column_name": "perd_end"
},
{
"key": "check_date",
"type": "date",
"column_name": "check_date"
},
{
"key": "loc_code",
"type": "text",
"column_name": "loc_code"
},
{
"key": "loc_descr",
"type": "text",
"column_name": "loc_descr"
},
{
"key": "loc_glseg",
"type": "text",
"column_name": "loc_glseg"
},
{
"key": "loc_over",
"type": "text",
"column_name": "loc_over"
},
{
"key": "dep_code",
"type": "text",
"column_name": "dep_code"
},
{
"key": "dep_descr",
"type": "text",
"column_name": "dep_descr"
},
{
"key": "dep_nat",
"type": "text",
"column_name": "dep_nat"
},
{
"key": "dep_over",
"type": "text",
"column_name": "dep_over"
},
{
"key": "di_code",
"type": "text",
"column_name": "di_code"
},
{
"key": "di_descr",
"type": "text",
"column_name": "di_descr"
},
{
"key": "di_glseg",
"type": "text",
"column_name": "di_glseg"
},
{
"key": "di_over",
"type": "text",
"column_name": "di_over"
},
{
"key": "title_code",
"type": "text",
"column_name": "titel_code"
},
{
"key": "title_descr",
"type": "text",
"column_name": "title_descr"
},
{
"key": "title_glseg",
"type": "text",
"column_name": "title_glseg"
},
{
"key": "title_over",
"type": "text",
"column_name": "title_over"
},
{
"key": "ee_code",
"type": "text",
"column_name": "ee_code"
},
{
"key": "ee_glseg",
"type": "text",
"column_name": "ee_glseg"
},
{
"key": "ee_over",
"type": "text",
"column_name": "ee_over"
},
{
"key": "acct_type_code",
"type": "text",
"column_name": "acct_type_code"
},
{
"key": "hours",
"type": "numeric",
"column_name": "hours"
},
{
"key": "nat_code",
"type": "text",
"column_name": "nat_code"
},
{
"key": "nat_over",
"type": "text",
"column_name": "nat_over"
},
{
"key": "gl_ref",
"type": "text",
"column_name": "gl_ref"
},
{
"key": "gl_group",
"type": "text",
"column_name": "gl_group"
},
{
"key": "gl_descr",
"type": "text",
"column_name": "gl_descr"
},
{
"key": "gl_code",
"type": "text",
"column_name": "gl_code"
},
{
"key": "gl_amount",
"type": "numeric",
"column_name": "gl_amount"
},
{
"key": "pp_code",
"type": "text",
"column_name": "pp_code"
},
{
"key": "pp_descr",
"type": "text",
"column_name": "pp_descr"
},
{
"key": "pp_gl",
"type": "text",
"column_name": "pp_gl"
},
{
"key": "pp_over",
"type": "text",
"column_name": "pp_over"
},
{
"key": "transaction",
"type": "text",
"column_name": "transaction"
}
]
}
}