pt/tps
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reload williams paid data

Browse Source
This commit is contained in:
Paul Trowbridge 2018-05-31 14:57:04 -04:00
parent da9e9b08f1
commit 7c28ea8384
3 changed files with 214 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deploy/reload/wmpd/curl_hunt.cmd Normal file
View File

@ -0,0 +1,3 @@
curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set
curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set
curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/WMPD.csv http://localhost:81/import?srce=WMPD

63
deploy/reload/wmpd/extract.sql Normal file
View File

@ -0,0 +1,63 @@
--source
COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'WMPD') TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\srce.json' WITH (FORMAT TEXT, HEADER FALSE)
--mapdef
COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'WMPD') x) TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\map.json' WITH (FORMAT TEXT, HEADER FALSE)
--map values
SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'WMPD') X
--records
copy (
select
r."Carrier",
r."SCAC",
r."Mode",
r."Pro #",
r."B/L",
r."Pd Amt",
r."Loc#",
r."Pcs",
r."Wgt",
r."Chk#",
r."Pay Dt",
r."Acct #",
r."I/O",
r."Sh Nm",
r."Sh City",
r."Sh St",
r."Sh Zip",
r."Cons Nm",
r."D City ",
r."D St",
r."D Zip",
r."Sh Dt",
r."Inv Dt",
r."Customs Entry#",
r."Miles",
r."Frt Class",
r."Master B/L"
from
tps.trans
join lateral jsonb_populate_record(null::tps.WMPD, rec) r on true
where
srce = 'WMPD'
order by
r."Pay Dt" asc
) to
'C:\users\ptrowbridge\downloads\WMPD.csv' with (format csv, header true);
--rebuild source def to include PATH
SELECT
ae.r
||jsonb_build_object(
'path',
(
'{'||(ae.r->>'column_name')||'}'
)
)
FROM
tps.srce
JOIN LATERAL jsonb_array_elements(defn->'schemas'->'default') ae(r) ON TRUE
WHERE
srce = 'WMPD'

148
deploy/reload/wmpd/srce.json Normal file
View File

@ -0,0 +1,148 @@
{
"name": "WMPD",
"source": "client_file",
"loading_function": "csv",
"constraint": [
"{Pay Dt}",
"{Carrier}"
],
"schemas": {
"default": [
{
"path": "{Carrier}",
"type": "text",
"column_name": "Carrier"
},
{
"path": "{SCAC}",
"type": "text",
"column_name": "SCAC"
},
{
"path": "{Mode}",
"type": "text",
"column_name": "Mode"
},
{
"path": "{Pro #}",
"type": "text",
"column_name": "Pro #"
},
{
"path": "{B/L}",
"type": "text",
"column_name": "B/L"
},
{
"path": "{Pd Amt}",
"type": "numeric",
"column_name": "Pd Amt"
},
{
"path": "{Loc#}",
"type": "text",
"column_name": "Loc#"
},
{
"path": "{Pcs}",
"type": "numeric",
"column_name": "Pcs"
},
{
"path": "{Wgt}",
"type": "numeric",
"column_name": "Wgt"
},
{
"path": "{Chk#}",
"type": "numeric",
"column_name": "Chk#"
},
{
"path": "{Pay Dt}",
"type": "date",
"column_name": "Pay Dt"
},
{
"path": "{Acct #}",
"type": "text",
"column_name": "Acct #"
},
{
"path": "{I/O}",
"type": "text",
"column_name": "I/O"
},
{
"path": "{Sh Nm}",
"type": "text",
"column_name": "Sh Nm"
},
{
"path": "{Sh City}",
"type": "text",
"column_name": "Sh City"
},
{
"path": "{Sh St}",
"type": "text",
"column_name": "Sh St"
},
{
"path": "{Sh Zip}",
"type": "text",
"column_name": "Sh Zip"
},
{
"path": "{Cons Nm}",
"type": "text",
"column_name": "Cons Nm"
},
{
"path": "{D City }",
"type": "text",
"column_name": "D City "
},
{
"path": "{D St}",
"type": "text",
"column_name": "D St"
},
{
"path": "{D Zip}",
"type": "text",
"column_name": "D Zip"
},
{
"path": "{Sh Dt}",
"type": "date",
"column_name": "Sh Dt"
},
{
"path": "{Inv Dt}",
"type": "date",
"column_name": "Inv Dt"
},
{
"path": "{Customs Entry#}",
"type": "text",
"column_name": "Customs Entry#"
},
{
"path": "{Miles}",
"type": "numeric",
"column_name": "Miles"
},
{
"path": "{Frt Class}",
"type": "text",
"column_name": "Frt Class"
},
{
"path": "{Master B/L}",
"type": "text",
"column_name": "Master B/L"
}
]
}
}