From 7c28ea83849c81977943c281cd27ff52f534e9ef Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 31 May 2018 14:57:04 -0400 Subject: [PATCH] reload williams paid data --- deploy/reload/wmpd/curl_hunt.cmd | 3 + deploy/reload/wmpd/extract.sql | 63 +++++++++++++ deploy/reload/wmpd/srce.json | 148 +++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 deploy/reload/wmpd/curl_hunt.cmd create mode 100644 deploy/reload/wmpd/extract.sql create mode 100644 deploy/reload/wmpd/srce.json diff --git a/deploy/reload/wmpd/curl_hunt.cmd b/deploy/reload/wmpd/curl_hunt.cmd new file mode 100644 index 0000000..e14ed84 --- /dev/null +++ b/deploy/reload/wmpd/curl_hunt.cmd @@ -0,0 +1,3 @@ +curl -H "Content-Type: application/json" -X POST -d@./srce.json http://localhost:81/srce_set +curl -H "Content-Type: application/json" -X POST -d@./mapdef.json http://localhost:81/mapdef_set + curl -v -F upload=@//mnt/c/Users/ptrowbridge/Downloads/WMPD.csv http://localhost:81/import?srce=WMPD \ No newline at end of file diff --git a/deploy/reload/wmpd/extract.sql b/deploy/reload/wmpd/extract.sql new file mode 100644 index 0000000..9113bc6 --- /dev/null +++ b/deploy/reload/wmpd/extract.sql @@ -0,0 +1,63 @@ +--source +COPY (SELECT DEFN FROM TPS.SRCE WHERE SRCE = 'WMPD') TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\srce.json' WITH (FORMAT TEXT, HEADER FALSE) + +--mapdef +COPY (SELECT jsonb_agg(row_to_json(x)::jsonb) FROM (SELECT srce, target "name", regex, seq "sequence" FROM tps.map_rm WHERE srce = 'WMPD') x) TO 'C:\users\ptrowbridge\documents\tps_etl\deploy\reload\wmpd\map.json' WITH (FORMAT TEXT, HEADER FALSE) + +--map values +SELECT jsonb_agg(row_to_JSON(x)::jsonb) FROM (SELECT srce "source", target "map", retval ret_val, "map" mapped FROM tps.map_rv WHERE srce = 'WMPD') X + +--records +copy ( + select + r."Carrier", + r."SCAC", + r."Mode", + r."Pro #", + r."B/L", + r."Pd Amt", + r."Loc#", + r."Pcs", + r."Wgt", + r."Chk#", + r."Pay Dt", + r."Acct #", + r."I/O", + r."Sh Nm", + r."Sh City", + r."Sh St", + r."Sh Zip", + r."Cons Nm", + r."D City ", + r."D St", + r."D Zip", + r."Sh Dt", + r."Inv Dt", + r."Customs Entry#", + r."Miles", + r."Frt Class", + r."Master B/L" + from + tps.trans + join lateral jsonb_populate_record(null::tps.WMPD, rec) r on true + where + srce = 'WMPD' + order by + r."Pay Dt" asc +) to +'C:\users\ptrowbridge\downloads\WMPD.csv' with (format csv, header true); + +--rebuild source def to include PATH +SELECT + ae.r + ||jsonb_build_object( + 'path', + ( + '{'||(ae.r->>'column_name')||'}' + ) + ) +FROM + tps.srce + JOIN LATERAL jsonb_array_elements(defn->'schemas'->'default') ae(r) ON TRUE +WHERE + srce = 'WMPD' diff --git a/deploy/reload/wmpd/srce.json b/deploy/reload/wmpd/srce.json new file mode 100644 index 0000000..eaef776 --- /dev/null +++ b/deploy/reload/wmpd/srce.json @@ -0,0 +1,148 @@ +{ + "name": "WMPD", + "source": "client_file", + "loading_function": "csv", + "constraint": [ + "{Pay Dt}", + "{Carrier}" + ], + "schemas": { + "default": [ + { + "path": "{Carrier}", + "type": "text", + "column_name": "Carrier" + }, + { + "path": "{SCAC}", + "type": "text", + "column_name": "SCAC" + }, + { + "path": "{Mode}", + "type": "text", + "column_name": "Mode" + }, + { + "path": "{Pro #}", + "type": "text", + "column_name": "Pro #" + }, + { + "path": "{B/L}", + "type": "text", + "column_name": "B/L" + }, + { + "path": "{Pd Amt}", + "type": "numeric", + "column_name": "Pd Amt" + }, + { + "path": "{Loc#}", + "type": "text", + "column_name": "Loc#" + }, + { + "path": "{Pcs}", + "type": "numeric", + "column_name": "Pcs" + }, + { + "path": "{Wgt}", + "type": "numeric", + "column_name": "Wgt" + }, + { + "path": "{Chk#}", + "type": "numeric", + "column_name": "Chk#" + }, + { + "path": "{Pay Dt}", + "type": "date", + "column_name": "Pay Dt" + }, + { + "path": "{Acct #}", + "type": "text", + "column_name": "Acct #" + }, + { + "path": "{I/O}", + "type": "text", + "column_name": "I/O" + }, + { + "path": "{Sh Nm}", + "type": "text", + "column_name": "Sh Nm" + }, + { + "path": "{Sh City}", + "type": "text", + "column_name": "Sh City" + }, + { + "path": "{Sh St}", + "type": "text", + "column_name": "Sh St" + }, + { + "path": "{Sh Zip}", + "type": "text", + "column_name": "Sh Zip" + }, + { + "path": "{Cons Nm}", + "type": "text", + "column_name": "Cons Nm" + }, + { + "path": "{D City }", + "type": "text", + "column_name": "D City " + }, + { + "path": "{D St}", + "type": "text", + "column_name": "D St" + }, + { + "path": "{D Zip}", + "type": "text", + "column_name": "D Zip" + }, + { + "path": "{Sh Dt}", + "type": "date", + "column_name": "Sh Dt" + }, + { + "path": "{Inv Dt}", + "type": "date", + "column_name": "Inv Dt" + }, + { + "path": "{Customs Entry#}", + "type": "text", + "column_name": "Customs Entry#" + }, + { + "path": "{Miles}", + "type": "numeric", + "column_name": "Miles" + }, + { + "path": "{Frt Class}", + "type": "text", + "column_name": "Frt Class" + }, + { + "path": "{Master B/L}", + "type": "text", + "column_name": "Master B/L" + } + ] + } +} \ No newline at end of file