in RowsetImportEngine/TextRowsetImporter.cs [524:756]
private void ProcessFile ()
{
bool InRowset = false;
string line = ""; // current line
string CurrentRowText = "";// text of current row (usually == line except in case of multi-line rows, a la DBCC INPUTBUFFER)
string lineprev1 = null; // previous line
string lineprev2 = null; // line prior to the previous line
int PercentComplete = 0;
try
{
this.TotalLinesProcessed = 0;
while ((line != null) && (line=this.sr.ReadLine()) != null)
{
//fixing a bug where indexes script dup rowcnt column
line = line.Replace("objid rowcnt", "objid rowcnt2");
line = line.Replace("client_interface_name total_elapsed_time", "client_interface_name total_elapsed_tim2");
//apparently, bulkcopy object thinks row_mods is a key word and skip importing the rows if NULL is present
line = line.Replace("row_mods", "row_mod2");
this.CurrentPosition += line.Length;
this.TotalLinesProcessed++;
if (!InRowset)
{
// Check for a "non-tabular" rowset (one without traditional column headers)
InRowset = CheckForRowsetStart(line, lineprev1, lineprev2, true);
}
if (InRowset)
{
// If we are in the middle of processing a rowset, see whether we've reached the end.
if (CurrentRowset.IsEndOfRowset(line, CurrentRowText))
{
line="";
CurrentRowText="";
CurrentRowset.Clear();
this.CurrentRowset = null;
InRowset = false;
if ((TotalRowsInserted - TotalRowsInsertedAtLastFlushCheck) > BATCH_COMMIT_ROWCOUNT)
{
// Periodically, flush the BCP batch for all active rowsets so that we don't
// suffer from excessive log autogrowth.
FlushRowsets();
TotalRowsInsertedAtLastFlushCheck = TotalRowsInserted;
}
}
else // If we're in a rowset and not at the end of the rowset, this must be a row.
{
CurrentRowText = line;
// Handle multi-line rows by reading in subsequent rows until we come to end-of-row marker.
if (!CurrentRowset.IsEndOfRow(line))
{
//performance optimization
//using string concat will lead to CPU hang when 'losing track'
StringBuilder Builder = new StringBuilder();
while (((line = this.sr.ReadLine()) != null) && (!CurrentRowset.IsEndOfRow(line)))
{
Builder.AppendFormat(" {0}", line);
//CurrentRowText += " " + line;
}
CurrentRowText += Builder.ToString();
}
// ParseRow will break out row data on column boundaries and populate the data in the Columns collection.
this.CurrentRowset.ParseRow(CurrentRowText);
// Copy token data to/from any columns that are linked to tokens in this rowset
if (this.CurrentRowset.UsesTokens)
{
foreach (Column c in this.CurrentRowset.Columns)
{
if (""!=c.DefineToken)
{
if (this.DefinedTokens.Contains(c.DefineToken))
this.DefinedTokens[c.DefineToken] = c.Data;
else
this.DefinedTokens.Add (c.DefineToken, c.Data);
}
if (""!=c.ValueToken)
{
if (this.DefinedTokens.Contains(c.ValueToken))
c.Data = this.DefinedTokens[c.ValueToken];
else
{
// Handle built-in application-provided tokens
switch (c.ValueToken)
{
case "INPUTFILENAME":
c.Data = this.filename;
break;
case "USERNAME":
c.Data = Environment.UserDomainName + @"\" + Environment.UserName;
break;
case "IMPORTDATE":
c.Data = this.ImportStartDate.ToString("yyyy-mm-dd hh:mm:ss");
break;
case "IMPORTDATEUTC":
c.Data = this.ImportStartDate.ToUniversalTime().ToString("yyyy-mm-dd hh:mm:ss");
break;
case "CURRENTDATE":
c.Data = DateTime.Now.ToString("yyyy-mm-dd hh:mm:ss");
break;
case "CURRENTDATEUTC":
c.Data = DateTime.Now.ToUniversalTime().ToString("yyyy-mm-dd hh:mm:ss");
break;
case "ROWNUMBER":
c.Data = this.CurrentRowset.RowsInserted;
break;
default:
c.Data = null;
break;
}
}
}
}
}
// Insert current row into SQL.
InsertRow();
// Save the last line read (the inputbuffer special rowset uses the same marker for end-of-row and end-of-rowset)
// TODO: clean this up (same soln as in SimpleMessageRowset)
CurrentRowText = line;
// If we are in the middle of processing a rowset, see whether we've reached the end.
if (CurrentRowset.IsEndOfRowset(line, CurrentRowText))
{
CurrentRowText="";
CurrentRowset.Clear();
this.CurrentRowset = null;
InRowset = false;
}
}
}
if (!InRowset)
{
// See if we're reached a possible start of a new rowset.
// Check for both "-----" (osql) and "~~~~~" (VMSTAT) as a column header indicator.
// HACK: In order to handle the blocker script's non-tabular DBCC OPENTRAN output we
// also check for the string "DBCC OPENTRAN FOR".
// TODO: Modify blocker script to use WITH TABLE_RESULTS for DBCC OPENTRAN so that
// it can be imported like every other rowset without this hack. Remove this once that
// change has been made.
if ((line != null) && (line.Trim().Length > 3) && (
(line.TrimStart().Substring(0, 2).Equals("--"))
|| (line.TrimStart().Substring(0, 2).Equals("~~"))
//|| ((line.Length > 17) && (line.Substring(0, 17).Equals("DBCC OPENTRAN FOR")))
))
InRowset = CheckForRowsetStart(line, lineprev1, lineprev2, false);
// Keep track of the previous two lines -- we'll need them when we encounter a new rowset.
lineprev2=lineprev1;
lineprev1=line;
}
// Make sure the host hasn't asked us to stop.
if (this.Cancelled)
{
this.State = ImportState.Canceling;
break;
}
// If our parent provided us with a progress update delegate, notify him of % complete status
bool ShouldUpdate = false;
if (this.TotalLinesProcessed > 100000)
{
if (0 == (this.TotalLinesProcessed % 10000))
{
ShouldUpdate = true;
}
}
else if (this.TotalLinesProcessed > 30000)
{
if (0 == (this.TotalLinesProcessed % 5000))
{
ShouldUpdate = true;
}
}
else if (this.TotalLinesProcessed > 10000)
{
if (0 == (this.TotalLinesProcessed % 3000))
{
ShouldUpdate = true;
}
}
else
{
if (0 == (this.TotalLinesProcessed % 1000))
{
ShouldUpdate = true;
}
}
//now update progress of rows processed
if (ShouldUpdate == true)
{
if (0 == this.CurrentPosition)
PercentComplete = 0;
else if (0 == this.FileSize)
PercentComplete = 100;
else
PercentComplete = Convert.ToInt32(Convert.ToInt64(100) * this.CurrentPosition / this.FileSize);
if (!(null == m_ProgressUpdateFunction))
{
m_ProgressUpdateFunction(PercentComplete);
}
} //end of if totallinesprocessed
} //end of while
return;
}
catch (Exception e)
{
ErrorDialog ed = new ErrorDialog(e, false, this.logger);
ed.Handle();
}
finally
{
FlushRowsets();
if (this.CurrentRowset != null) CurrentRowset.Clear();
this.CurrentRowset = null;
}
return;
}