From aa797b7fa87d9eea4c4f94cd5e60cb1356c16a7f Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 16 Aug 2018 02:42:48 -0400 Subject: [PATCH] build insert --- runner.cs | 86 +++++++++++++++----- stkmm.sql | 236 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 stkmm.sql diff --git a/runner.cs b/runner.cs index 5806f8b..409901b 100644 --- a/runner.cs +++ b/runner.cs @@ -10,7 +10,7 @@ namespace test static void Main(string[] args) { - var i = new Int32(); + //var i = new Int32(); //-------------------------------------------establis connections------------------------------------------------- //var ibmc = new System.Data.Odbc.OdbcConnection("Driver={iSeries Access ODBC Driver};System=TEST400;Uid=tstdillen;Pwd=tstdillen"); @@ -42,41 +42,87 @@ namespace test //---------------------------------------------setup adapters--------------------------------------------------------- //var ibmds = new System.Data.DataSet(); //var ibmda = new System.Data.Odbc.OdbcDataAdapter(ibmcmd); - Console.Write(DateTime.Now); //ibmda.Fill(ibmds); var pgds = new System.Data.DataSet(); var pgda = new NpgsqlDataAdapter(pgcmd); pgda.Fill(pgds); + //pgda.UpdateBatchSize = 100; + + Console.Write("etl start:" + DateTime.Now.ToString()); + Console.Write(Environment.NewLine); //--------------------------------------------move to target-------------------------------------------------------- var ibmdr = ibmcmd.ExecuteReader(); - while (ibmdr.Read()) { - var pgr = pgds.Tables[0].NewRow(); - ibmdr.GetValues(pgr.ItemArray); - pgds.Tables[0].Rows.Add(pgr); - i=i+1; - if (i> 500){ - new NpgsqlCommandBuilder(pgda); - pgda.Update(pgds); - i=0; + var getv = new object[ibmdr.FieldCount]; + int r = 0; + string sql = ""; + string nr = ""; + string nc = ""; + var pgcom = pgc.CreateCommand(); + while (ibmdr.Read()) { + r = r + 1; + nr = ""; + /* + for ( int i = 0 ; i < ibmdr.GetValues(getv);i++) { + if (getv[i] != null) { + switch (ibmdr.GetDataTypeName(i)){ + case "VARCHAR": + nc = "'" + getv[i].ToString().Replace("'","''") + "'"; + break; + case "CHAR": + nc = "'" + getv[i].ToString().Replace("'","''") + "'"; + break; + case "DATE": + if (getv[i].ToString() != "1/1/0001 12:00:00 AM") { + nc = "'" + getv[i].ToString() + "'"; + } + else { + nc = "NULL"; + } + break; + case "TIME": + nc = "'" + getv[i].ToString() + "'"; + break; + case "TIMESTAMP": + nc = "'" + getv[i].ToString() + "'"; + break; + default: + nc = getv[i].ToString(); + break; + } + } + else { + nc = "NULL"; + } + if (i!=0) { + nr = nr + ","; + } + nr = nr + nc; } + if (sql!="") { + sql = sql + ","; + } + sql = sql + "(" + nr + ")"; + if (r == 500) { + r = 0; + pgcom.CommandText = "INSERT INTO lgdat.stkmm VALUES " + sql; + //pgcom.ExecuteNonQuery(); + sql = ""; + } + */ } - new NpgsqlCommandBuilder(pgda); - try { - pgda.Update(pgds); + if (r != 0) { + pgcom.CommandText = "INSERT INTO lgdat.stkmm VALUES " + sql; + //pgcom.ExecuteNonQuery(); + sql = ""; } - catch (Exception e) { - //Console.WriteLine("{0} Exception caught.", e); - Console.WriteLine(e.Message); - } - ibmc.Close(); pgc.Close(); - Console.Write(DateTime.Now); + Console.Write("etl end:" + DateTime.Now.ToString()); } } diff --git a/stkmm.sql b/stkmm.sql new file mode 100644 index 0000000..0cfc515 --- /dev/null +++ b/stkmm.sql @@ -0,0 +1,236 @@ +SELECT + REPLACE( avpart, CHR(0), '' ), + REPLACE( avdes1, CHR(0), '' ), + REPLACE( avdes2, CHR(0), '' ), + REPLACE( avdes3, CHR(0), '' ), + REPLACE( avunti, CHR(0), '' ), + REPLACE( avmajg, CHR(0), '' ), + REPLACE( avming, CHR(0), '' ), + REPLACE( avmajs, CHR(0), '' ), + REPLACE( avmins, CHR(0), '' ), + REPLACE( avglcd, CHR(0), '' ), + REPLACE( avgled, CHR(0), '' ), + REPLACE( avharm, CHR(0), '' ), + REPLACE( avclss, CHR(0), '' ), + REPLACE( avfut3, CHR(0), '' ), + REPLACE( avfut4, CHR(0), '' ), + REPLACE( avfut5, CHR(0), '' ), + REPLACE( avfutc, CHR(0), '' ), + REPLACE( avfutd, CHR(0), '' ), + REPLACE( avupcc, CHR(0), '' ), + avgtin, + avnwht, + REPLACE( avnwun, CHR(0), '' ), + avsvol, + REPLACE( avsvun, CHR(0), '' ), + REPLACE( avengc, CHR(0), '' ), + REPLACE( avrevl, CHR(0), '' ), + avrdat, + REPLACE( avcoil, CHR(0), '' ), + REPLACE( avfut9, CHR(0), '' ), + REPLACE( avmspc, CHR(0), '' ), + avslif, + REPLACE( avdplt, CHR(0), '' ), + REPLACE( avuver, CHR(0), '' ), + REPLACE( avcata, CHR(0), '' ), + REPLACE( avfedc, CHR(0), '' ), + REPLACE( avtaxr, CHR(0), '' ), + REPLACE( avcust, CHR(0), '' ), + REPLACE( avcpt#, CHR(0), '' ), + REPLACE( avrev#, CHR(0), '' ), + REPLACE( avfut7, CHR(0), '' ), + REPLACE( avfut8, CHR(0), '' ), + REPLACE( avfutb, CHR(0), '' ), + avldat, + REPLACE( avtype, CHR(0), '' ), + REPLACE( avfutk, CHR(0), '' ), + REPLACE( avfutn, CHR(0), '' ), + REPLACE( avppap, CHR(0), '' ), + REPLACE( avcusr, CHR(0), '' ), + avcdat, + avctme, + REPLACE( avuusr, CHR(0), '' ), + avudat, + avutme, + REPLACE( avsizc, CHR(0), '' ), + REPLACE( avstlc, CHR(0), '' ), + REPLACE( avcolc, CHR(0), '' ), + REPLACE( avassc, CHR(0), '' ), + REPLACE( avsasc, CHR(0), '' ), + REPLACE( avlcoil, CHR(0), '' ), + REPLACE( avgdfl, CHR(0), '' ), + REPLACE( avvdfl, CHR(0), '' ), + REPLACE( avwoqf, CHR(0), '' ), + REPLACE( avnmfc, CHR(0), '' ), + avfuth, + avfuti, + REPLACE( avfutl, CHR(0), '' ), + REPLACE( avfutm, CHR(0), '' ), + REPLACE( avfutp, CHR(0), '' ), + avfutq, + avfutr, + avprun, + avtotp, + REPLACE( avdisc, CHR(0), '' ), + REPLACE( avinvt, CHR(0), '' ), + REPLACE( avbk01, CHR(0), '' ), + REPLACE( avbill, CHR(0), '' ), + REPLACE( avbk04, CHR(0), '' ), + REPLACE( avbk06, CHR(0), '' ), + REPLACE( avoldd, CHR(0), '' ), + REPLACE( avoldd2, CHR(0), '' ), + REPLACE( avlotf, CHR(0), '' ), + REPLACE( avserf, CHR(0), '' ), + REPLACE( avorig, CHR(0), '' ), + REPLACE( avplan, CHR(0), '' ), + avminq, + avmaxq, + avlead, + REPLACE( avscdt, CHR(0), '' ), + avestv, + REPLACE( avcntr, CHR(0), '' ), + REPLACE( avuntp, CHR(0), '' ), + avlvl, + avmoqt, + avpack, + REPLACE( avpacu, CHR(0), '' ), + avprpt, + REPLACE( avkitc, CHR(0), '' ), + REPLACE( avproc, CHR(0), '' ), + REPLACE( avlrsp, CHR(0), '' ), + avlrsc, + REPLACE( avlrsf, CHR(0), '' ), + REPLACE( avlrmp, CHR(0), '' ), + avlrmc, + REPLACE( avlrmf, CHR(0), '' ), + REPLACE( avlwsp, CHR(0), '' ), + avlwsc, + REPLACE( avlwsf, CHR(0), '' ), + REPLACE( avlcsp, CHR(0), '' ), + avlcsc, + REPLACE( avlcsf, CHR(0), '' ), + REPLACE( avlcmp, CHR(0), '' ), + avlcmc, + REPLACE( avlcmf, CHR(0), '' ), + avlssc, + REPLACE( avlssf, CHR(0), '' ), + REPLACE( avlsmp, CHR(0), '' ), + avlsmc, + REPLACE( avlsmf, CHR(0), '' ), + avmpck, + REPLACE( avmpku, CHR(0), '' ), + avaltf, + REPLACE( avdbuy, CHR(0), '' ), + REPLACE( avfut1, CHR(0), '' ), + REPLACE( avfut2, CHR(0), '' ), + avfut6, + REPLACE( avfuta, CHR(0), '' ), + avfutg, + REPLACE( avrecu, CHR(0), '' ), + avftmf, + REPLACE( avrecc, CHR(0), '' ), + REPLACE( avfutj, CHR(0), '' ), + REPLACE( avabcc, CHR(0), '' ), + REPLACE( avcorg, CHR(0), '' ), + avperc, + avoptr, + avmult, + avminr, + avshrk, + REPLACE( avstat, CHR(0), '' ), + REPLACE( avreas, CHR(0), '' ), + avspdt, + avbbdt, + REPLACE( avpsor, CHR(0), '' ), + REPLACE( avfuto, CHR(0), '' ), + REPLACE( avdrws, CHR(0), '' ), + REPLACE( avdrwl, CHR(0), '' ), + REPLACE( avdenc, CHR(0), '' ), + REPLACE( avdrel, CHR(0), '' ), + avddat, + REPLACE( avfute, CHR(0), '' ), + REPLACE( avdrws2, CHR(0), '' ), + REPLACE( avdrwl2, CHR(0), '' ), + REPLACE( avdenc2, CHR(0), '' ), + REPLACE( avdrel2, CHR(0), '' ), + avddat2, + REPLACE( avfutf, CHR(0), '' ), + REPLACE( avstcl, CHR(0), '' ), + REPLACE( avvlcd, CHR(0), '' ), + avsppp, + REPLACE( avsppc, CHR(0), '' ), + avmppp, + REPLACE( avmppc, CHR(0), '' ), + REPLACE( avflg01, CHR(0), '' ), + REPLACE( avflg02, CHR(0), '' ), + REPLACE( avflg03, CHR(0), '' ), + REPLACE( avflg04, CHR(0), '' ), + REPLACE( avflg05, CHR(0), '' ), + REPLACE( avflg06, CHR(0), '' ), + REPLACE( avflg07, CHR(0), '' ), + REPLACE( avflg08, CHR(0), '' ), + REPLACE( avflg09, CHR(0), '' ), + REPLACE( avflg10, CHR(0), '' ), + REPLACE( avfut01, CHR(0), '' ), + REPLACE( avfut02, CHR(0), '' ), + REPLACE( avfut03, CHR(0), '' ), + REPLACE( avfut04, CHR(0), '' ), + REPLACE( avfut05, CHR(0), '' ), + REPLACE( avfut06, CHR(0), '' ), + REPLACE( avfut07, CHR(0), '' ), + REPLACE( avfut08, CHR(0), '' ), + REPLACE( avfut09, CHR(0), '' ), + REPLACE( avfut10, CHR(0), '' ), + REPLACE( avfut11, CHR(0), '' ), + REPLACE( avfut12, CHR(0), '' ), + REPLACE( avfut13, CHR(0), '' ), + REPLACE( avfut14, CHR(0), '' ), + REPLACE( avfut15, CHR(0), '' ), + REPLACE( avfut16, CHR(0), '' ), + REPLACE( avfut17, CHR(0), '' ), + REPLACE( avfut18, CHR(0), '' ), + REPLACE( avfut19, CHR(0), '' ), + REPLACE( avfut20, CHR(0), '' ), + REPLACE( avfut21, CHR(0), '' ), + REPLACE( avfut22, CHR(0), '' ), + REPLACE( avfut23, CHR(0), '' ), + REPLACE( avfut24, CHR(0), '' ), + REPLACE( avfut25, CHR(0), '' ), + avfut26, + avfut27, + avfut28, + avfut29, + avfut30, + avfut31, + avfut32, + avfut33, + avfut34, + avfut35, + avfut36, + avfut37, + avfut38, + avfut39, + avfut40, + avfut41, + avfut42, + avfut43, + avfut44, + avfut45, + REPLACE( avudft, CHR(0), '' ), + REPLACE( avfrml, CHR(0), '' ), + REPLACE( avhaza, CHR(0), '' ), + avlled, + REPLACE( avdbac, CHR(0), '' ), + REPLACE( avscdpt, CHR(0), '' ), + REPLACE( avfifo, CHR(0), '' ), + REPLACE( avantid, CHR(0), '' ), + REPLACE( avsubt, CHR(0), '' ), + REPLACE( avprcl, CHR(0), '' ), + REPLACE( avotp, CHR(0), '' ), + REPLACE( avcbyl, CHR(0), '' ), + REPLACE( avsrvc, CHR(0), '' ), + REPLACE( avspal, CHR(0), '' ), + REPLACE( avpgrp, CHR(0), '' ) +FROM + LGDAT.STKMM +FETCH FIRST 10000 ROWS ONLY \ No newline at end of file