private static void CreateInputOutputDatasets()

in SamplesV1/ParameterizedPipelinesForAzureML/DeployDataFactory/DeployDataFactory/RetrainingPipeline.cs [156:285]


        private static void CreateInputOutputDatasets(
            string resourceGroupName, 
            string dataFactoryName, 
            DataFactoryManagementClient client, 
            out string inputDataset, 
            out IList<string> outputModelDatasets, 
            out IList<string> outputPlaceholderDatasets, 
            string[] parameters, 
            IList<UpdateResourceEndpoint> endpoints)
        {
            // create input and output tables
            Console.WriteLine("Creating input and output tables");
            inputDataset = "InputDatasetScoring";
            outputModelDatasets = new List<string>();
            outputPlaceholderDatasets = new List<string>();

            client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
                new DatasetCreateOrUpdateParameters()
                {
                    Dataset = new Dataset()
                    {
                        Name = inputDataset,
                        Properties = new DatasetProperties()
                        {
                            LinkedServiceName = "LinkedService-AzureStorage",
                            TypeProperties = new AzureBlobDataset()
                            {
                                FolderPath = "inputdatascoring/",
                                FileName = "input.csv",
                                Format = new TextFormat()
                                {
                                    ColumnDelimiter = ","
                                }
                            },
                            External = true,
                            Availability = new Availability()
                            {
                                Frequency = SchedulePeriod.Minute,
                                Interval = 15,
                            },

                            Policy = new Policy()
                            {
                                Validation = new ValidationPolicy()
                                {
                                    MinimumRows = 1
                                }
                            }
                        }
                    }
                });

            foreach (string parameter in parameters)
            {
                string[] parameterList = parameter.Split(',');
                string region = parameterList[0];
                
                string outputModelDataset = String.Format("outputModel_{0}", region);

                client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
                    new DatasetCreateOrUpdateParameters()
                    {
                        Dataset = new Dataset()
                        {
                            Name = outputModelDataset,
                            Properties = new DatasetProperties()
                            {
                                LinkedServiceName = "LinkedService-AzureStorage",
                                TypeProperties = new AzureBlobDataset()
                                {
                                    FolderPath = String.Format("outputmodel/{0}/", region) + "{Slice}",
                                    FileName = "model.ilearner",
                                    Format = new TextFormat()
                                    {                                        
                                    },
                                    PartitionedBy = new Collection<Partition>()
                                    {
                                        new Partition()
                                        {
                                            Name = "Slice",
                                            Value = new DateTimePartitionValue()
                                            {
                                                Date = "SliceStart",
                                                Format = "yyyyMMdd-HHmmss"
                                            }
                                        }
                                    }
                                },

                                Availability = new Availability()
                                {
                                    Frequency = SchedulePeriod.Minute,
                                    Interval = DataFactoryConfig.PipelineFrequencyInMinutes,
                                },
                            }
                        }
                    });

                outputModelDatasets.Add(outputModelDataset);

                string outputPlaceholderDataset = String.Format("outputplaceholder_{0}", region);
                client.Datasets.CreateOrUpdate(resourceGroupName, dataFactoryName,
                    new DatasetCreateOrUpdateParameters()
                    {
                        Dataset = new Dataset()
                        {
                            Name = outputPlaceholderDataset,
                            Properties = new DatasetProperties()
                            {
                                LinkedServiceName = "LinkedService-AzureStorage",
                                TypeProperties = new AzureBlobDataset()
                                {
                                    FolderPath = "any",                                   
                                    Format = new TextFormat()
                                    {
                                    },
                                },

                                Availability = new Availability()
                                {
                                    Frequency = SchedulePeriod.Minute,
                                    Interval = 15,
                                },
                            }
                        }
                    });

                outputPlaceholderDatasets.Add(outputPlaceholderDataset);
            }
        }