in SamplesV1/ParameterizedPipelinesForAzureML/DeployDataFactory/DeployDataFactory/RetrainingPipeline.cs [156:285]
private static void CreateInputOutputDatasets(
string resourceGroupName,
string dataFactoryName,
DataFactoryManagementClient client,
out string inputDataset,
out IList<string> outputModelDatasets,
out IList<string> outputPlaceholderDatasets,
string[] parameters,
IList<UpdateResourceEndpoint> endpoints)
{
// create input and output tables
Console.WriteLine("Creating input and output tables");
inputDataset = "InputDatasetScoring";
outputModelDatasets = new List<string>();
outputPlaceholderDatasets = new List<string>();
client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
new DatasetCreateOrUpdateParameters()
{
Dataset = new Dataset()
{
Name = inputDataset,
Properties = new DatasetProperties()
{
LinkedServiceName = "LinkedService-AzureStorage",
TypeProperties = new AzureBlobDataset()
{
FolderPath = "inputdatascoring/",
FileName = "input.csv",
Format = new TextFormat()
{
ColumnDelimiter = ","
}
},
External = true,
Availability = new Availability()
{
Frequency = SchedulePeriod.Minute,
Interval = 15,
},
Policy = new Policy()
{
Validation = new ValidationPolicy()
{
MinimumRows = 1
}
}
}
}
});
foreach (string parameter in parameters)
{
string[] parameterList = parameter.Split(',');
string region = parameterList[0];
string outputModelDataset = String.Format("outputModel_{0}", region);
client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
new DatasetCreateOrUpdateParameters()
{
Dataset = new Dataset()
{
Name = outputModelDataset,
Properties = new DatasetProperties()
{
LinkedServiceName = "LinkedService-AzureStorage",
TypeProperties = new AzureBlobDataset()
{
FolderPath = String.Format("outputmodel/{0}/", region) + "{Slice}",
FileName = "model.ilearner",
Format = new TextFormat()
{
},
PartitionedBy = new Collection<Partition>()
{
new Partition()
{
Name = "Slice",
Value = new DateTimePartitionValue()
{
Date = "SliceStart",
Format = "yyyyMMdd-HHmmss"
}
}
}
},
Availability = new Availability()
{
Frequency = SchedulePeriod.Minute,
Interval = DataFactoryConfig.PipelineFrequencyInMinutes,
},
}
}
});
outputModelDatasets.Add(outputModelDataset);
string outputPlaceholderDataset = String.Format("outputplaceholder_{0}", region);
client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
new DatasetCreateOrUpdateParameters()
{
Dataset = new Dataset()
{
Name = outputPlaceholderDataset,
Properties = new DatasetProperties()
{
LinkedServiceName = "LinkedService-AzureStorage",
TypeProperties = new AzureBlobDataset()
{
FolderPath = "any",
Format = new TextFormat()
{
},
},
Availability = new Availability()
{
Frequency = SchedulePeriod.Minute,
Interval = 15,
},
}
}
});
outputPlaceholderDatasets.Add(outputPlaceholderDataset);
}
}