pt/dotnet_runner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

huh

Browse Source
This commit is contained in:
Paul Trowbridge 2020-01-11 00:32:48 -05:00
parent 7cae86a78f
commit cfab73c961
3 changed files with 366 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -1,4 +1,4 @@
/bin /bin
/obj /obj
launch.json launch.json
.gitignore .gitignore

692
runner.cs
View File

@ -1,346 +1,346 @@
using System; using System;
using Npgsql; using Npgsql;
namespace test namespace test
{ {
class Program class Program
{ {
private private
static void Main(string[] args) static void Main(string[] args)
{ {
string scs = ""; string scs = "";
string dcs = ""; string dcs = "";
string sdt = ""; string sdt = "";
string ddt = ""; string ddt = "";
string sq = ""; string sq = "";
string dt = ""; string dt = "";
Boolean trim = false; Boolean trim = false;
int r = 0; int r = 0;
int t = 0; int t = 0;
string sql = ""; string sql = "";
string nr = ""; string nr = "";
string nc = ""; string nc = "";
string nl = Environment.NewLine; string nl = Environment.NewLine;
//declaring these objects globally will asume these libraries exist, they may not //declaring these objects globally will asume these libraries exist, they may not
System.Data.Odbc.OdbcConnection s_odbc = new System.Data.Odbc.OdbcConnection(); System.Data.Odbc.OdbcConnection s_odbc = new System.Data.Odbc.OdbcConnection();
System.Data.Odbc.OdbcConnection d_odbc = new System.Data.Odbc.OdbcConnection(); System.Data.Odbc.OdbcConnection d_odbc = new System.Data.Odbc.OdbcConnection();
NpgsqlConnection s_npgsql = new NpgsqlConnection(); NpgsqlConnection s_npgsql = new NpgsqlConnection();
NpgsqlConnection d_npgsql = new NpgsqlConnection(); NpgsqlConnection d_npgsql = new NpgsqlConnection();
System.Data.Common.DbConnection sc; //source connection System.Data.Common.DbConnection sc; //source connection
System.Data.Common.DbConnection dc; //destination connection System.Data.Common.DbConnection dc; //destination connection
string msg = "Help:"; string msg = "Help:";
msg = msg + nl + "version 0.25"; msg = msg + nl + "version 0.25";
msg = msg + nl; msg = msg + nl;
msg = msg + nl + "options:"; msg = msg + nl + "options:";
msg = msg + nl + "-sdt sourc driver type"; msg = msg + nl + "-sdt sourc driver type";
msg = msg + nl + "-scs source connection string"; msg = msg + nl + "-scs source connection string";
msg = msg + nl + "-ddt destination driver type"; msg = msg + nl + "-ddt destination driver type";
msg = msg + nl + "-dcs destination connection string"; msg = msg + nl + "-dcs destination connection string";
msg = msg + nl + "-sq path to source query"; msg = msg + nl + "-sq path to source query";
msg = msg + nl + "-dt fully qualified name of destination table"; msg = msg + nl + "-dt fully qualified name of destination table";
msg = msg + nl + "-t trim text"; msg = msg + nl + "-t trim text";
msg = msg + nl + "--help info"; msg = msg + nl + "--help info";
msg = msg + nl; msg = msg + nl;
msg = msg + nl + "available driver types:"; msg = msg + nl + "available driver types:";
msg = msg + nl + "-------------------------"; msg = msg + nl + "-------------------------";
msg = msg + nl + "* odbc"; msg = msg + nl + "* odbc";
msg = msg + nl + "* npgsql"; msg = msg + nl + "* npgsql";
//---------------------------------------parse args into variables------------------------------------------------- //---------------------------------------parse args into variables-------------------------------------------------
for (int i = 0; i < args.Length; i = i +1 ){ for (int i = 0; i < args.Length; i = i +1 ){
switch (args[i]) { switch (args[i]) {
//sourc driver type //sourc driver type
case "-sdt": case "-sdt":
sdt = args[i+1]; sdt = args[i+1];
break; break;
//source connection string //source connection string
case "-scs": case "-scs":
scs = args[i+1]; scs = args[i+1];
break; break;
//destination driver type //destination driver type
case "-ddt": case "-ddt":
ddt = args[i+1]; ddt = args[i+1];
break; break;
//destination connection string //destination connection string
case "-dcs": case "-dcs":
dcs = args[i+1]; dcs = args[i+1];
break; break;
//source query path //source query path
case "-sq": case "-sq":
try { try {
sq = System.IO.File.ReadAllText(args[i+1]); sq = System.IO.File.ReadAllText(args[i+1]);
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl + "error reasing source sql file: " + e.Message); Console.Write(nl + "error reasing source sql file: " + e.Message);
return; return;
} }
break; break;
//destination table name //destination table name
case "-dt": case "-dt":
dt = "INSERT INTO " + args[i+1] + " VALUES "; dt = "INSERT INTO " + args[i+1] + " VALUES ";
break; break;
case "-t": case "-t":
trim = true; trim = true;
break; break;
case "--help": case "--help":
Console.Write(nl); Console.Write(nl);
Console.Write(msg); Console.Write(msg);
return; return;
case "-help": case "-help":
Console.Write(nl); Console.Write(nl);
Console.Write(msg); Console.Write(msg);
return; return;
case "-h": case "-h":
Console.Write(nl); Console.Write(nl);
Console.Write(msg); Console.Write(msg);
return; return;
case @"\?": case @"\?":
Console.Write(nl); Console.Write(nl);
Console.Write(msg); Console.Write(msg);
return; return;
default: default:
break; break;
} }
} }
Console.Write(nl); Console.Write(nl);
Console.Write(sdt); Console.Write(sdt);
Console.Write(scs); Console.Write(scs);
Console.Write(nl); Console.Write(nl);
Console.Write(ddt); Console.Write(ddt);
Console.Write(dcs); Console.Write(dcs);
Console.Write(nl); Console.Write(nl);
Console.Write(sq); Console.Write(sq);
Console.Write(nl); Console.Write(nl);
Console.Write(dt); Console.Write(dt);
//return; //return;
//-------------------------------------------establish connections------------------------------------------------- //-------------------------------------------establish connections-------------------------------------------------
//setup source //setup source
switch (sdt){ switch (sdt){
case "odbc": case "odbc":
try { try {
s_odbc.ConnectionString = scs; s_odbc.ConnectionString = scs;
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl + "bad source connection string: " + e.Message); Console.Write(nl + "bad source connection string: " + e.Message);
return; return;
} }
sc = s_odbc; sc = s_odbc;
return; return;
default: default:
break; break;
} }
s_odbc.ConnectionString = scs; s_odbc.ConnectionString = scs;
sc = s_odbc; sc = s_odbc;
//setup destination //setup destination
switch (ddt) { switch (ddt) {
case "npgsql": case "npgsql":
try { try {
d_npgsql.ConnectionString = dcs; d_npgsql.ConnectionString = dcs;
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl + "bad source connection string: " + e.Message); Console.Write(nl + "bad source connection string: " + e.Message);
return; return;
} }
dc = d_npgsql; dc = d_npgsql;
return; return;
default: default:
break; break;
} }
d_npgsql.ConnectionString = dcs; d_npgsql.ConnectionString = dcs;
dc = d_npgsql; dc = d_npgsql;
/* /*
//polymorph open //polymorph open
try { try {
sc.Open(); sc.Open();
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl + "issue connection to source: " + e.Message); Console.Write(nl + "issue connection to source: " + e.Message);
return; return;
} }
try { try {
dc.Open(); dc.Open();
} }
catch (Exception e) { catch (Exception e) {
sc.Close(); sc.Close();
Console.Write(nl + "issue connecting to destination: "+ e.Message); Console.Write(nl + "issue connecting to destination: "+ e.Message);
} }
//----------------------------------------------setup commands--------------------------------------------------- //----------------------------------------------setup commands---------------------------------------------------
var s_cmd = new System.Data.Odbc.OdbcCommand(); var s_cmd = new System.Data.Odbc.OdbcCommand();
var pgcom = d_npgsql.CreateCommand(); var pgcom = d_npgsql.CreateCommand();
//---------------------------------------------begin transaction--------------------------------------------------------- //---------------------------------------------begin transaction---------------------------------------------------------
Console.Write(nl); Console.Write(nl);
Console.Write("etl start:" + DateTime.Now.ToString()); Console.Write("etl start:" + DateTime.Now.ToString());
/* /*
NpgsqlTransaction pgt = pgc.BeginTransaction(); NpgsqlTransaction pgt = pgc.BeginTransaction();
s_odbcommandTimeout = 6000; s_odbcommandTimeout = 6000;
s_odbcommandTimeout = 6000; s_odbcommandTimeout = 6000;
system.Data.Odbc.OdbcDataReader ibmdr; system.Data.Odbc.OdbcDataReader ibmdr;
try { try {
ibmdr = s_odbcxecuteReader(); ibmdr = s_odbcxecuteReader();
ibmdr = s_odbcxecuteReader(); ibmdr = s_odbcxecuteReader();
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl); Console.Write(nl);
Console.Write("error on source sql:"); Console.Write("error on source sql:");
Console.Write(nl); Console.Write(nl);
Console.Write(e.Message); Console.Write(e.Message);
s_odbcse(); s_odbcse();
s_odbcse(); s_odbcse();
pgc.Cldse(); pgc.Cldse();
return; return;
} }
//setup getv object array dimensioned to number of columns for scenario //setup getv object array dimensioned to number of columns for scenario
//getv hold an array of datareader row //getv hold an array of datareader row
var getv = new object[ibmdr.FieldCount]; var getv = new object[ibmdr.FieldCount];
int cols = ibmdr.FieldCount; int cols = ibmdr.FieldCount;
//dtn holds list of data types per column //dtn holds list of data types per column
var dtn = new string[ibmdr.FieldCount]; var dtn = new string[ibmdr.FieldCount];
while (ibmdr.Read()) { while (ibmdr.Read()) {
r = r + 1; r = r + 1;
t = t +1 ; t = t +1 ;
nr = ""; nr = "";
//populate all the data type names into a string array instead of calling against ibmdr in every iteration //populate all the data type names into a string array instead of calling against ibmdr in every iteration
if (t == 1 ) { if (t == 1 ) {
for (int i = 0; i < cols; i++){ for (int i = 0; i < cols; i++){
dtn[i] = ibmdr.GetDataTypeName(i); dtn[i] = ibmdr.GetDataTypeName(i);
} }
} }
for (int i = 0 ; i < cols;i++) { for (int i = 0 ; i < cols;i++) {
//Console.Write(nl); //Console.Write(nl);
//Console.Write(DBNull.Value.Equals(ibmdr.GetValue(i)).ToString()); //Console.Write(DBNull.Value.Equals(ibmdr.GetValue(i)).ToString());
//Console.Write(ibmdr.GetValue(i).ToString()); //Console.Write(ibmdr.GetValue(i).ToString());
Boolean dnull = false; Boolean dnull = false;
if (dtn[i] == "BIGINT") { if (dtn[i] == "BIGINT") {
dnull = DBNull.Value.Equals(ibmdr.GetInt64(i)); dnull = DBNull.Value.Equals(ibmdr.GetInt64(i));
} }
else { else {
dnull = DBNull.Value.Equals(ibmdr.GetValue(i)); dnull = DBNull.Value.Equals(ibmdr.GetValue(i));
} }
if (! dnull) { if (! dnull) {
switch (dtn[i]){ switch (dtn[i]){
case "VARCHAR": case "VARCHAR":
if (trim) { if (trim) {
nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''").TrimEnd() + "'"; nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''").TrimEnd() + "'";
} }
else { else {
nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''") + "'"; nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''") + "'";
} }
break; break;
case "CHAR": case "CHAR":
if (trim) { if (trim) {
nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''").TrimEnd() + "'"; nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''").TrimEnd() + "'";
} }
else { else {
nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''") + "'"; nc = "'" + ibmdr.GetValue(i).ToString().Replace("'","''") + "'";
} }
break; break;
case "DATE": case "DATE":
if (ibmdr.GetValue(i).ToString() == "1/1/0001 12:00:00 AM" || ibmdr.GetValue(i).ToString() == "") { if (ibmdr.GetValue(i).ToString() == "1/1/0001 12:00:00 AM" || ibmdr.GetValue(i).ToString() == "") {
nc = "NULL"; nc = "NULL";
} }
else { else {
nc = "'" + ibmdr.GetValue(i).ToString() + "'"; nc = "'" + ibmdr.GetValue(i).ToString() + "'";
} }
break; break;
case "TIME": case "TIME":
nc = "'" + ibmdr.GetValue(i).ToString() + "'"; nc = "'" + ibmdr.GetValue(i).ToString() + "'";
break; break;
case "TIMESTAMP": case "TIMESTAMP":
nc = "'" + ibmdr.GetDateTime(i).ToString() + "'"; nc = "'" + ibmdr.GetDateTime(i).ToString() + "'";
break; break;
case "BIGINT": case "BIGINT":
nc = ibmdr.GetInt64(i).ToString(); nc = ibmdr.GetInt64(i).ToString();
break; break;
default: default:
if (ibmdr.GetValue(i).ToString() != "") { if (ibmdr.GetValue(i).ToString() != "") {
nc = ibmdr.GetValue(i).ToString(); nc = ibmdr.GetValue(i).ToString();
} }
else { else {
nc = "NULL"; nc = "NULL";
} }
break; break;
} }
} }
else { else {
nc = "NULL"; nc = "NULL";
} }
if (i!=0) { if (i!=0) {
nr = nr + ","; nr = nr + ",";
} }
nr = nr + nc; nr = nr + nc;
} }
if (sql!="") { if (sql!="") {
sql = sql + ","; sql = sql + ",";
} }
sql = sql + "(" + nr + ")"; sql = sql + "(" + nr + ")";
if (r == 250) { if (r == 250) {
r = 0; r = 0;
sql = dt + sql; sql = dt + sql;
pgcom.CommandText = sql; pgcom.CommandText = sql;
try { try {
pgcom.ExecuteNonQuery(); pgcom.ExecuteNonQuery();
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl); Console.Write(nl);
Console.Write(e.Message); Console.Write(e.Message);
System.IO.File.WriteAllText(@"C:\Users\ptrowbridge\Downloads\runner_error.sql",sql); System.IO.File.WriteAllText(@"C:\Users\ptrowbridge\Downloads\runner_error.sql",sql);
s_odbcse(); s_odbcse();
s_odbcse(); s_odbcse();
pgt.Rollbdck(); pgt.Rollbdck();
pgc.Close(); pgc.Close();
return; return;
} }
sql = ""; sql = "";
Console.Write(nl + t.ToString()); Console.Write(nl + t.ToString());
} }
} }
if (r != 0) { if (r != 0) {
sql = dt + sql; sql = dt + sql;
pgcom.CommandText = sql; pgcom.CommandText = sql;
try { try {
pgcom.ExecuteNonQuery(); pgcom.ExecuteNonQuery();
} }
catch (Exception e) { catch (Exception e) {
Console.Write(nl); Console.Write(nl);
Console.Write(e.Message); Console.Write(e.Message);
System.IO.File.WriteAllText(@"C:\Users\ptrowbridge\Downloads\runner_error.sql",sql); System.IO.File.WriteAllText(@"C:\Users\ptrowbridge\Downloads\runner_error.sql",sql);
//s_odbcse(); //s_odbcse();
//s_odbcse(); //s_odbcse();
pgt.Rodlback(); pgt.Rodlback();
pgc.Close(); pgc.Close();
return; return;
} }
sql = ""; sql = "";
Console.Write(nl + t.ToString()); Console.Write(nl + t.ToString());
} }
pgt.Commit(); pgt.Commit();
s_odbcse(); s_odbcse();
s_odbcse(); s_odbcse();
pgc.Cldse(); pgc.Cldse();
Console.Write(nl); Console.Write(nl);
Console.Write("etl end:" + DateTime.Now.ToString()); Console.Write("etl end:" + DateTime.Now.ToString());
*/ */
} }
} }
} }

34
runner.csproj
View File

@ -1,17 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework> <TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Npgsql" Version="4.0.0" /> <PackageReference Include="Npgsql" Version="4.0.0" />
<PackageReference Include="System.Data.Odbc" Version="4.5.0" /> <PackageReference Include="System.Data.Odbc" Version="4.5.0" />
</ItemGroup> </ItemGroup>
<PropertyGroup> <PropertyGroup>
<RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers> <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
</PropertyGroup> </PropertyGroup>
</Project> </Project>