src/extensions/speech_extension/commands/custom_speech_recognition_command.cs (960 lines of code) (raw):
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CognitiveServices.Speech;
using Microsoft.CognitiveServices.Speech.Audio;
using System.Collections.Generic;
using System.Net;
using System.Text.Json;
namespace Azure.AI.Details.Common.CLI
{
public class CustomSpeechRecognitionCommand : Command
{
public CustomSpeechRecognitionCommand(ICommandValues values) : base(values)
{
_quiet = _values.GetOrDefault("x.quiet", false);
_verbose = _values.GetOrDefault("x.verbose", true);
}
public bool RunCommand()
{
try
{
RunCustomSpeechCommand();
}
catch (WebException ex)
{
FileHelpers.LogException(_values, ex);
ConsoleHelpers.WriteLineError($"\n ERROR: {ex.Message}");
JsonHelpers.PrintJson(HttpHelpers.ReadWriteJson(ex.Response, _values, "csr"));
}
return _values.GetOrDefault("passed", true);
}
private bool RunCustomSpeechCommand()
{
DoCommand(_values.GetCommand());
return _values.GetOrDefault("passed", true);
}
private void DoCommand(string command)
{
CheckPath();
switch (command.Replace("speech.", ""))
{
case "csr.list": DoList(); break;
case "csr.download": DoDownload(); break;
case "csr.project.create": DoCreateProject(); break;
case "csr.project.update": DoUpdateProject(); break;
case "csr.project.delete": DoDeleteProject(); break;
case "csr.project.status": DoProjectStatus(); break;
case "csr.project.list": DoList("projects"); break;
case "csr.dataset.download": DoDownload(); break;
case "csr.dataset.create": DoCreateDataset(); break;
case "csr.dataset.upload": DoUploadDataset(); break;
case "csr.dataset.update": DoUpdateDataset(); break;
case "csr.dataset.delete": DoDeleteDataset(); break;
case "csr.dataset.status": DoDatasetStatus(); break;
case "csr.dataset.list": DoList("datasets"); break;
case "csr.model.download": DoDownload(); break;
case "csr.model.create": DoCreateModel(); break;
case "csr.model.update": DoUpdateModel(); break;
case "csr.model.delete": DoDeleteModel(); break;
case "csr.model.status": DoModelStatus(); break;
case "csr.model.list": DoList("models"); break;
case "csr.model.copy": DoModelCopy(); break;
case "csr.evaluation.create": DoCreateEvaluation(); break;
case "csr.evaluation.update": DoUpdateEvaluation(); break;
case "csr.evaluation.delete": DoDeleteEvaluation(); break;
case "csr.evaluation.status": DoEvaluationStatus(); break;
case "csr.evaluation.list": DoList("evaluations"); break;
case "csr.endpoint.download": DoDownload(); break;
case "csr.endpoint.create": DoCreateEndpoint(); break;
case "csr.endpoint.update": DoUpdateEndpoint(); break;
case "csr.endpoint.delete": DoDeleteEndpoint(); break;
case "csr.endpoint.status": DoEndpointStatus(); break;
case "csr.endpoint.list": DoList("endpoints"); break;
default:
_values.AddThrowError("WARNING:", $"'{command.Replace('.', ' ')}' NOT YET IMPLEMENTED!!");
break;
}
}
private void DoList(string kind = "")
{
GetListParameters(kind, out string path, out string message, out string query);
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, path, null, query);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response, true);
var parsed = JsonDocument.Parse(json);
var nextLink = parsed.GetPropertyStringOrNull("@nextLink");
if (!string.IsNullOrEmpty(nextLink) && !query.Contains("top="))
{
var allPages = new List<JsonElement>();
allPages.Add(parsed.RootElement);
for (; ;)
{
message = $"Following @nextLink {nextLink} ...";
if (!_quiet) Console.WriteLine(message);
var requestNext = CreateWebRequest(WebRequestMethods.Http.Get, nextLink, null, null);
var payloadNext = CheckWriteOutputRequest(requestNext, null, true);
var responseNext = HttpHelpers.GetWebResponse(requestNext, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var jsonNext = ReadWritePrintJson(responseNext, true);
var parsedNext = JsonDocument.Parse(jsonNext);
var thisPage = parsedNext.RootElement;
allPages.Add(thisPage);
nextLink = thisPage.GetPropertyStringOrNull("@nextLink");
if (string.IsNullOrEmpty(nextLink)) break;
}
json = JsonHelpers.MergeJsonObjects(allPages);
}
FileHelpers.CheckOutputJson(json, _values, "csr");
UrlHelpers.CheckWriteOutputUrlsOrIds(json, "values", "self", _values, "csr");
}
private string? DoDownload()
{
var url = _values.GetOrEmpty("csr.download.url");
var urlOk = !string.IsNullOrEmpty(url);
if (urlOk) return DownloadUrl(url);
GetDownloadParameters(out string path, out string query, out string message);
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, path, null, query);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
if (!response.ContentType.Contains("application/json"))
{
_values.AddThrowError(
"WARNING:", $"Service did not return valid json payload!",
"TRY:", $"{Program.Name} csr download --url URL");
}
var json = ReadWritePrintJson(response);
var parsed = JsonDocument.Parse(json);
var fileUrl = parsed.GetPropertyElementOrNull("links")?.GetPropertyStringOrNull("contentUrl") ?? string.Empty;
var fileName = parsed.GetPropertyStringOrNull("name");
return DownloadUrl(fileUrl, fileName);
}
private void DoCreateProject()
{
var name = _values.GetOrEmpty("csr.project.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot create project; project name not found!",
"USE:", $"{Program.Name} csr project create --name NAME [...]");
}
var message = $"Creating project '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, "projects");
var payload = CheckWriteOutputRequest(request, GetCreateProjectPostJson(name));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");
}
private void DoUpdateProject()
{
var id = _values.GetOrEmpty("csr.project.id");
var message = $"Updating project '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("PATCH", "projects", id);
var payload = CheckWriteOutputRequest(request, GetUpdateProjectPostJson());
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDeleteProject()
{
var id = _values.GetOrEmpty("csr.project.id");
var message = $"Deleting project '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("DELETE", "projects", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoProjectStatus()
{
var id = _values.GetOrEmpty("csr.project.id");
var message = $"Getting status for project {id} ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, "projects", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
}
private void DoCreateDataset()
{
var name = _values.GetOrEmpty("csr.dataset.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot create dataset; dataset name not found!",
"USE:", $"{Program.Name} csr dataset create --name NAME [...]");
}
var message = $"Start creating dataset '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, "datasets");
var payload = CheckWriteOutputRequest(request, GetCreateDatasetPostJson(name));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
var url = UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
var id = UrlHelpers.IdFromUrl(url);
CheckWaitForComplete(response, null, "csr", "dataset", id);
}
private void DoUploadDataset()
{
var name = _values.GetOrEmpty("csr.dataset.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot upload dataset; dataset name not found!",
"USE:", $"{Program.Name} csr dataset upload --name NAME [...]");
}
var message = $"Start uploading dataset '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var boundary = "----WebKitFormBoundaryAbzpBAzkUAgYKeM8";
var contentType = $"multipart/form-data; boundary={boundary}";
var request = CreateWebRequest(WebRequestMethods.Http.Post, "datasets/upload", null, null, contentType);
var formData = GetUploadDatasetPostFormData(name, boundary);
CheckWriteOutputRequest(request, Encoding.UTF8.GetString(formData));
var response = HttpHelpers.GetWebResponse(request, formData);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
var url = UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
var id = UrlHelpers.IdFromUrl(url);
CheckWaitForComplete(response, null, "csr", "dataset", id);
}
private void DoUpdateDataset()
{
var id = _values.GetOrEmpty("csr.dataset.id");
var message = $"Updating dataset '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("PATCH", "datasets", id);
var payload = CheckWriteOutputRequest(request, GetUpdateDatasetPostJson());
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDeleteDataset()
{
var id = _values.GetOrEmpty("csr.dataset.id");
var message = $"Deleting dataset {id} ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("DELETE", "datasets", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDatasetStatus()
{
var id = _values.GetOrEmpty("csr.dataset.id");
var projectId = _values.GetOrEmpty("csr.project.id");
if (!string.IsNullOrEmpty(projectId))
{
_values.AddThrowError(
"WARNING:", $"Cannot check dataset status for projectid!",
"USE:", $"{Program.Name} csr dataset status --id ID [...]");
}
var message = $"Getting status for dataset {id}...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, "datasets", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
CheckWaitForComplete(null, json, "csr", "dataset", id);
}
private void DoCreateModel()
{
var name = _values.GetOrEmpty("csr.model.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot create model; model name not found!",
"USE:", $"{Program.Name} csr model create --name NAME [...]");
}
var message = $"Start creating model '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, "models");
var payload = CheckWriteOutputRequest(request, GetCreateModelPostJson(name));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
var url = UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
var id = UrlHelpers.IdFromUrl(url);
CheckWaitForComplete(response, null, "csr", "model", id);
}
private void DoUpdateModel()
{
var id = _values.GetOrEmpty("csr.model.id");
var message = $"Updating model '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("PATCH", "models", id);
var payload = CheckWriteOutputRequest(request, GetUpdateModelPostJson());
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDeleteModel()
{
var id = _values.GetOrEmpty("csr.model.id");
var message = $"Deleting model {id} ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("DELETE", "models", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoModelStatus()
{
var id = _values.GetOrEmpty("csr.model.id");
var projectId = _values.GetOrEmpty("csr.project.id");
if (!string.IsNullOrEmpty(projectId))
{
_values.AddThrowError(
"WARNING:", $"Cannot check model status for projectid!",
"USE:", $"{Program.Name} csr model status --id ID [...]");
}
var message = $"Getting status for model {id}...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, "models", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
CheckWaitForComplete(null, json, "csr", "model", id);
}
private void DoModelCopy()
{
var id = _values.GetOrEmpty("csr.model.id");
var projectId = _values.GetOrEmpty("csr.project.id");
var targetKey = _values.GetOrEmpty("csr.model.copy.target.key");
if (!string.IsNullOrEmpty(projectId))
{
_values.AddThrowError(
"WARNING:", $"Cannot copy model for projectid!",
"USE:", $"{Program.Name} csr model copy --id ID [...]");
}
else if (string.IsNullOrEmpty(targetKey))
{
_values.AddThrowError(
"WARNING:", $"Cannot copy model; requires valid target subscription key.",
"USE:", $"{Program.Name} csr model copy --target key KEY [...]");
}
var message = $"Copying model {id}...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, $"models/{id}/copyto");
var payload = CheckWriteOutputRequest(request, GetCopyModelPostJson(targetKey));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
}
private void DoCreateEvaluation()
{
var name = _values.GetOrEmpty("csr.evaluation.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot create evaluation; evaluation name not found!",
"USE:", $"{Program.Name} csr evaluation create --name NAME [...]");
}
var message = $"Start creating evaluation '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, "evaluations");
var payload = CheckWriteOutputRequest(request, GetCreateEvaluationPostJson(name));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
var url = UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
var id = UrlHelpers.IdFromUrl(url);
CheckWaitForComplete(response, null, "csr", "evaluation", id);
}
private void DoUpdateEvaluation()
{
var id = _values.GetOrEmpty("csr.evaluation.id");
var message = $"Updating evaluation '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("PATCH", "evaluations", id);
var payload = CheckWriteOutputRequest(request, GetUpdateEvaluationPostJson());
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDeleteEvaluation()
{
var id = _values.GetOrEmpty("csr.evaluation.id");
var message = $"Deleting evaluation {id} ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("DELETE", "evaluations", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoEvaluationStatus()
{
var id = _values.GetOrEmpty("csr.evaluation.id");
var projectId = _values.GetOrEmpty("csr.project.id");
if (!string.IsNullOrEmpty(projectId))
{
_values.AddThrowError(
"WARNING:", $"Cannot check evaluation status for projectid!",
"USE:", $"{Program.Name} csr evaluation status --id ID [...]");
}
var message = $"Getting status for evaluation {id}...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, "evaluations", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
CheckWaitForComplete(null, json, "csr", "evaluation", id);
}
private void DoCreateEndpoint()
{
var name = _values.GetOrEmpty("csr.endpoint.name");
if (string.IsNullOrEmpty(name))
{
_values.AddThrowError(
"WARNING:", $"Cannot create endpoint; endpoint name not found!",
"USE:", $"{Program.Name} csr endpoint create --name NAME [...]");
}
var message = $"Start creating endpoint '{name}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Post, "endpoints");
var payload = CheckWriteOutputRequest(request, GetCreateEndpointPostJson(name));
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
var url = UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
var id = UrlHelpers.IdFromUrl(url);
CheckWaitForComplete(response, null, "csr", "endpoint", id);
}
private void DoUpdateEndpoint()
{
var id = _values.GetOrEmpty("csr.endpoint.id");
var message = $"Updating endpoint '{id}' ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("PATCH", "endpoints", id);
var payload = CheckWriteOutputRequest(request, GetUpdateEndpointPostJson());
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoDeleteEndpoint()
{
var id = _values.GetOrEmpty("csr.endpoint.id");
var message = $"Deleting endpoint {id} ...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest("DELETE", "endpoints", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
ReadWritePrintJson(response);
}
private void DoEndpointStatus()
{
var id = _values.GetOrEmpty("csr.endpoint.id");
var projectId = _values.GetOrEmpty("csr.project.id");
if (!string.IsNullOrEmpty(projectId))
{
_values.AddThrowError(
"WARNING:", $"Cannot check endpoint status for projectid!",
"USE:", $"{Program.Name} csr endpoint status --id ID [...]");
}
var message = $"Getting status for endpoint {id}...";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, "endpoints", id);
var payload = CheckWriteOutputRequest(request);
var response = HttpHelpers.GetWebResponse(request, payload);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
var json = ReadWritePrintJson(response);
UrlHelpers.CheckWriteOutputUrlOrId(json, "self", _values, "csr");;
CheckWaitForComplete(null, json, "csr", "endpoint", id);
}
private void GetListParameters(string kind, out string path, out string message, out string query)
{
kind = _values.GetOrDefault("csr.list.kind", kind);
var listLanguages = _values.GetOrDefault("csr.list.languages", false);
var languageKind = _values.GetOrDefault("csr.list.languages.kind", listLanguages ? kind.TrimEnd('s') : "");
var list = _values.GetOrEmpty("csr.list.id");
var listOk = list.StartsWith("http");
var listId = listOk ? "" : list;
var dataset = _values.GetOrEmpty("csr.dataset.id");
var datasetOk = !string.IsNullOrEmpty(dataset) && dataset.StartsWith("http");
var kindIsDatasets = kind == "datasets";
var datasetId = IdHelpers.GetIdFromNamedValue(_values, "csr.dataset.id", kindIsDatasets ? listId : "");
var datasetIdOk = !string.IsNullOrEmpty(datasetId) && !datasetId.StartsWith("http");
var datasetFiles = _values.GetOrDefault("csr.list.dataset.files", datasetOk || datasetIdOk);
if (datasetFiles && !datasetOk && !datasetIdOk)
{
_values.AddThrowError(
"WARNING:", $"Cannot list dataset files without a valid dataset ID/URL!",
"USE:", $"{Program.Name} csr dataset list --dataset ID/URL --files");
}
var evaluation = _values.GetOrEmpty("csr.evaluation.id");
var evaluationOk = !string.IsNullOrEmpty(evaluation) && evaluation.StartsWith("http");
var kindIsEvaluations = kind == "evaluations";
var evaluationId = IdHelpers.GetIdFromNamedValue(_values, "csr.evaluation.id", kindIsEvaluations ? listId : "");
var evaluationIdOk = !string.IsNullOrEmpty(evaluationId) && !evaluationId.StartsWith("http");
var evaluationFiles = _values.GetOrDefault("csr.list.evaluation.files", evaluationOk || evaluationIdOk);
if (evaluationFiles && !evaluationIdOk && !evaluationIdOk)
{
_values.AddThrowError(
"WARNING:", $"Cannot list evaluation files without a valid evaluation ID/URL!",
"USE:", $"{Program.Name} csr evaluation list --evaluation ID/URL --files");
}
var endpoint = _values.GetOrEmpty("csr.endpoint.id");
var endpointOk = !string.IsNullOrEmpty(endpoint) && endpoint.StartsWith("http");
var kindIsEndpoints = kind == "endpoints";
var endpointId = IdHelpers.GetIdFromNamedValue(_values, "csr.endpoint.id", kindIsEndpoints ? listId : "");
var endpointIdOk = !string.IsNullOrEmpty(endpointId) && !endpointId.StartsWith("http");
var endpointLogs = _values.GetOrDefault("csr.list.endpoint.logs", endpointOk || endpointIdOk);
if (endpointLogs && !endpointOk && !endpointIdOk)
{
_values.AddThrowError(
"WARNING:", $"Cannot list endpoint logs without a valid endpoint ID/URL!",
"USE:", $"{Program.Name} csr endpoint list --endpoint ID/URL --logs");
}
path = "";
message = "";
if (datasetOk)
{
path = $"{dataset}/files";
message = "Listing dataset files ...";
}
else if (datasetIdOk)
{
path = $"datasets/{datasetId}/files";
message = "Listing dataset files ...";
}
else if (evaluationOk)
{
path = evaluation.EndsWith("/files") ? evaluation : $"{evaluation}/files";
message = "Listing evaluation files ...";
}
else if (evaluationIdOk)
{
path = $"evaluations/{evaluationId}/files";
message = "Listing evaluation files ...";
}
else if (endpointOk)
{
path = endpoint.EndsWith("/files/logs") ? endpoint : $"{endpoint}/files/logs";
message = "Listing endpoint logs";
}
else if (endpointIdOk)
{
path = $"endpoints/{endpointId}/files/logs";
message = "Listing endpoint logs ...";
}
else if (listOk && list.Contains("/datasets/"))
{
path = list.EndsWith("/files") ? list : $"{list}/files";
message = "Listing dataset files ...";
}
else if (listOk && list.Contains("/evaluations/"))
{
path = list.EndsWith("/files") ? list : $"{list}/files";
message = "Listing evaluation files ...";
}
else if (listOk && list.Contains("/endpoints/"))
{
path = list.EndsWith("/files") ? list : $"{list}/files";
message = "Listing endpoint files ...";
}
else if (!string.IsNullOrEmpty(languageKind))
{
path = $"{languageKind}s/locales";
message = $"Listing {languageKind} languages ...";
}
else if (!string.IsNullOrEmpty(kind))
{
var projectId = IdHelpers.GetIdFromNamedValue(_values, "csr.project.id");
var projectOk = !string.IsNullOrEmpty(projectId);
path = projectOk ? $"projects/{projectId}/{kind}" : kind;
message = $"Listing {kind} ...";
}
else
{
_values.AddThrowError(
"WARNING:", $"Couldn't find resource type to list!",
"SEE:", $"{Program.Name} help csr");
}
var top = _values.GetOrEmpty("csr.top");
var skip = _values.GetOrEmpty("csr.skip");
query = "";
if (!string.IsNullOrEmpty(skip)) query += $"&skip={skip}";
if (!string.IsNullOrEmpty(top)) query += $"&top={top}";
query = query.Trim('&');
}
private void GetDownloadParameters(out string path, out string query, out string message)
{
path = query = message = "";
CheckDownloadFile(ref path, ref message);
CheckDownloadDatasetFile(ref path, ref message);
CheckDownloadEndpointLog(ref path, ref message);
CheckDownloadEvaluationFile(ref path, ref message);
if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(message))
{
var command = _values.GetCommandForDisplay();
_values.AddThrowError(
"WARNING:", $"Couldn't determine what to download!",
"SEE:", $"{Program.Name} help {command}");
}
}
private void CheckDownloadFile(ref string path, ref string message)
{
var file = _values.GetOrEmpty("csr.download.file");
var fileOk = !string.IsNullOrEmpty(file) && file.StartsWith("http");
if (fileOk)
{
path = file;
message = file.Substring(0, Math.Min(100, file.Length));
message = file.Length > 100
? $"Locating file {message}... "
: $"Locating file {message} ... ";
}
}
private void CheckDownloadDatasetFile(ref string path, ref string message)
{
var datasetFile = _values.GetOrEmpty("csr.dataset.file.id");
var datasetFileOk = !string.IsNullOrEmpty(datasetFile) && datasetFile.StartsWith("http");
var downloadId = _values.GetOrEmpty("csr.download.id");
var datasetId = IdHelpers.GetIdFromNamedValue(_values, "csr.dataset.id", downloadId);
var datasetIdOk = !string.IsNullOrEmpty(datasetId);
var datasetFileId = IdHelpers.IdFromString(datasetFile);
var datasetFileIdOk = !!string.IsNullOrEmpty(datasetFileId);
if (datasetFileOk)
{
path = datasetFile;
message = $"Locating dataset file {datasetFile} ...";
}
else if (datasetIdOk && datasetFileIdOk)
{
path = $"datasets/{datasetId}/files/{datasetFileId}";
message = $"Locating dataset file {datasetFileId} ...";
}
}
private void CheckDownloadEvaluationFile(ref string path, ref string message)
{
var evaluationFile = _values.GetOrEmpty("csr.evaluation.file.id");
var evaluationFileOk = !string.IsNullOrEmpty(evaluationFile) && evaluationFile.StartsWith("http");
var downloadId = _values.GetOrEmpty("csr.download.id");
var evaluationId = IdHelpers.GetIdFromNamedValue(_values, "csr.evaluation.id", downloadId);
var evaluationIdOk = !string.IsNullOrEmpty(evaluationId);
var evaluationFileId = IdHelpers.IdFromString(evaluationFile);
var evaluationFileIdOk = !!string.IsNullOrEmpty(evaluationFileId);
if (evaluationFileOk)
{
path = evaluationFile;
message = $"Locating evaluation file {evaluationFile} ...";
}
else if (evaluationIdOk && evaluationFileIdOk)
{
path = $"evaluation/{evaluationId}/files/{evaluationFileId}";
message = $"Locating evaluation file {evaluationFileId} ...";
}
}
private void CheckDownloadEndpointLog(ref string path, ref string message)
{
var endpointLog = _values.GetOrEmpty("csr.endpoint.log.id");
var endpointLogOk = !string.IsNullOrEmpty(endpointLog) && endpointLog.StartsWith("http");
var downloadId = _values.GetOrEmpty("csr.download.id");
var endpointId = IdHelpers.GetIdFromNamedValue(_values, "csr.endpoint.id", downloadId);
var endpointIdOk = !string.IsNullOrEmpty(endpointId);
var endpointLogId = IdHelpers.GetIdFromNamedValue(_values, "csr.endpoint.log.id");
var endpointLogIdOk = !string.IsNullOrEmpty(endpointLogId);
if (endpointLogOk)
{
path = endpointLog;
message = $"Locating endpoint log {endpointLog} ...";
}
else if (endpointIdOk && endpointLogIdOk)
{
path = $"endpoints/{endpointId}/files/logs/{endpointLogId}";
message = $"Locating endpoint log {endpointLogId} ...";
}
}
private string GetCreateProjectPostJson(string name)
{
var language = _values.GetOrDefault("csr.project.language", "en-US");
var description = _values.GetOrEmpty("csr.project.description");
return $"{{ \"locale\": \"{language}\", \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetUpdateProjectPostJson()
{
var name = _values.GetOrEmpty("csr.project.name");
var description = _values.GetOrEmpty("csr.project.description");
var region = _values.GetOrEmpty("service.config.region");
var projectId = _values.GetOrEmpty("csr.project.id");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
return $"{{ {projectRef} \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetCreateModelPostJson(string name)
{
var projectId = _values.GetOrEmpty("csr.project.id");
var region = _values.GetOrEmpty("service.config.region");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
var baseId = _values.GetOrEmpty("csr.model.create.base.model.id");
var baseUrl = GetCustomSpeechUrl(region, "models/base", baseId);
var baseRef = !string.IsNullOrEmpty(baseId) ? $"\"baseModel\": {{ \"self\": \"{baseUrl}\" }}," : "";
StringBuilder sb = new StringBuilder();
var datasetIds = _values.GetOrDefault("csr.model.create.dataset.ids", _values.GetOrEmpty("csr.model.create.dataset.id"));
foreach (var datasetId in datasetIds.Split(";\r\n".ToCharArray()))
{
if (!string.IsNullOrEmpty(datasetId))
{
var datasetUrl = GetCustomSpeechUrl(region, "datasets", datasetId);
sb.Append($"{{ \"self\": \"{datasetUrl}\" }}, ");
}
}
var datasets = sb.ToString().Trim(',', ' ');
var datasetRefs = datasets.Length > 0 ? $"\"datasets\": [ {datasets} ]," : "";
var text = _values.GetOrEmpty("csr.model.create.text");
var textRef = !string.IsNullOrEmpty(text) ? $"\"text\": \"{text}\"," : "";
var language = _values.GetOrDefault("csr.model.language", "en-US");
var description = _values.GetOrEmpty("csr.model.description");
return $"{{ {projectRef} {baseRef} {datasetRefs} {textRef} \"locale\": \"{language}\", \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetUpdateModelPostJson()
{
var name = _values.GetOrEmpty("csr.model.name");
var description = _values.GetOrEmpty("csr.model.description");
var region = _values.GetOrEmpty("service.config.region");
var projectId = _values.GetOrEmpty("csr.project.id");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
return $"{{ {projectRef} \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetCopyModelPostJson(string targetKey)
{
return $"{{ \"targetSubscriptionKey\": \"{targetKey}\" }}\n";
}
private string GetCreateEvaluationPostJson(string name)
{
var projectId = _values.GetOrEmpty("csr.project.id");
var region = _values.GetOrEmpty("service.config.region");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
var modelId1 = _values.GetOrEmpty("csr.evaluation.create.model1.id");
var modelUrl1 = GetCustomSpeechUrl(region, "models", modelId1);
var modelRef1 = !string.IsNullOrEmpty(modelId1) ? $"\"model1\": {{ \"self\": \"{modelUrl1}\" }}," : "";
var modelId2 = _values.GetOrEmpty("csr.evaluation.create.model2.id");
var modelUrl2 = GetCustomSpeechUrl(region, "models", modelId2);
var modelRef2 = !string.IsNullOrEmpty(modelId2) ? $"\"model2\": {{ \"self\": \"{modelUrl2}\" }}," : "";
var datasetId = _values.GetOrEmpty("csr.evaluation.create.dataset.id");
var datasetUrl = GetCustomSpeechUrl(region, "datasets", datasetId);
var datasetRef = !string.IsNullOrEmpty(modelId1) ? $"\"dataset\": {{ \"self\": \"{datasetUrl}\" }}," : "";
var language = _values.GetOrDefault("csr.evaluation.language", "en-US");
var description = _values.GetOrEmpty("csr.evaluation.description");
return $"{{ {projectRef} {modelRef1} {modelRef2} {datasetRef} \"locale\": \"{language}\", \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetUpdateEvaluationPostJson()
{
var name = _values.GetOrEmpty("csr.evaluation.name");
var description = _values.GetOrEmpty("csr.evaluation.description");
var region = _values.GetOrEmpty("service.config.region");
var projectId = _values.GetOrEmpty("csr.project.id");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
return $"{{ {projectRef} \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetCreateDatasetPostJson(string name)
{
var contentUrl = _values.GetOrEmpty("csr.dataset.create.content.url");
var projectId = _values.GetOrEmpty("csr.project.id");
var region = _values.GetOrEmpty("service.config.region");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
var kind = _values.GetOrEmpty("csr.dataset.kind");
var language = _values.GetOrDefault("csr.dataset.language", "en-US");
var description = _values.GetOrEmpty("csr.dataset.description");
return $"{{ {projectRef} \"kind\": \"{kind}\", \"contentUrl\": \"{contentUrl}\", \"locale\": \"{language}\", \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private byte[] GetUploadDatasetPostFormData(string name, string boundary)
{
var dataFile = _values.GetOrEmpty("csr.dataset.upload.data.file");
if (string.IsNullOrEmpty(dataFile))
{
_values.AddThrowError(
"WARNING:", $"Cannot upload data; missing data file!",
"USE:", $"{Program.Name} csr dataset upload --data FILENAME [...]");
}
var zipFile = FileHelpers.FindFileInDataPath(dataFile, _values);
if (string.IsNullOrEmpty(zipFile))
{
_values.AddThrowError("ERROR:", $"Cannot find data file: \"{dataFile}\"");
}
var projectId = _values.GetOrEmpty("csr.project.id");
var region = _values.GetOrEmpty("service.config.region");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var kind = _values.GetOrEmpty("csr.dataset.kind");
var language = _values.GetOrDefault("csr.dataset.language", "en-US");
var description = _values.GetOrEmpty("csr.dataset.description");
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(projectId)) sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"project\"\r\n\r\n{projectUrl}\r\n");
sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"kind\"\r\n\r\n{kind}\r\n");
sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"locale\"\r\n\r\n{language}\r\n");
sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"displayName\"\r\n\r\n{name}\r\n");
sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"description\"\r\n\r\n{description}\r\n");
var contentType = "text/plain";
switch (kind.ToLower())
{
case "audio": contentType = "application/x-zip-compressed"; break;
case "acoustic": contentType = "application/zip"; break;
}
sb.Append($"--{boundary}\r\nContent-Disposition: form-data; name=\"data\"; filename=\"{dataFile}\"\r\nContent-Type: {contentType}\r\n\r\n");
var bytes1 = Encoding.UTF8.GetBytes(sb.ToString());
var bytes2 = FileHelpers.ReadAllBytes(zipFile!);
var bytes3 = Encoding.UTF8.GetBytes($"\r\n--{boundary}--\r\n");
var bytes = new byte[bytes1.Length + bytes2.Length + bytes3.Length];
bytes1.CopyTo(bytes, 0);
bytes2.CopyTo(bytes, bytes1.Length);
bytes3.CopyTo(bytes, bytes1.Length + bytes2.Length);
return bytes;
}
private string GetUpdateDatasetPostJson()
{
var name = _values.GetOrEmpty("csr.dataset.name");
var description = _values.GetOrEmpty("csr.dataset.description");
var region = _values.GetOrEmpty("service.config.region");
var projectId = _values.GetOrEmpty("csr.project.id");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
return $"{{ {projectRef} \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetCreateEndpointPostJson(string name)
{
var projectId = _values.GetOrEmpty("csr.project.id");
var region = _values.GetOrEmpty("service.config.region");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
var modelId = _values.GetOrEmpty("csr.endpoint.create.model.id");
var modelUrl = GetCustomSpeechUrl(region, "models", modelId);
var modelRef = !string.IsNullOrEmpty(modelId) ? $"\"model\": {{ \"self\": \"{modelUrl}\" }}, " : "";
var text = _values.GetOrEmpty("csr.endpoint.create.text");
var textRef = !string.IsNullOrEmpty(text) ? $"\"text\": \"{text}\"," : "";
var language = _values.GetOrDefault("csr.endpoint.language", "en-US");
var description = _values.GetOrEmpty("csr.endpoint.description");
var properties = GetProperties("endpoint");
return $"{{ {projectRef} {modelRef} {textRef} {properties} \"locale\": \"{language}\", \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetUpdateEndpointPostJson()
{
var name = _values.GetOrEmpty("csr.endpoint.name");
var description = _values.GetOrEmpty("csr.endpoint.description");
var properties = GetProperties("endpoint");
var region = _values.GetOrEmpty("service.config.region");
var projectId = _values.GetOrEmpty("csr.project.id");
var projectUrl = GetCustomSpeechUrl(region, "projects", projectId);
var projectRef = !string.IsNullOrEmpty(projectId) ? $"\"project\": {{ \"self\": \"{projectUrl}\" }}," : "";
return $"{{ {projectRef} {properties} \"displayName\": \"{name}\", \"description\": \"{description}\" }}";
}
private string GetProperties(string thing)
{
var loggingEnabled = _values.GetOrDefault("service.config.content.logging.enabled", "false");
var loggingPart = thing == "endpoint" ? $"\"contentLoggingEnabled\": {loggingEnabled}" : "";
return $"\"properties\": {{ {loggingPart} }},";
}
private HttpWebRequest CreateWebRequest(string method, string path, string? id = null, string? query = null, string? contentType = null)
{
var key = _values.GetOrEmpty("service.config.key");
var region = _values.GetOrEmpty("service.config.region");
var timeout = _values.GetOrDefault("csr.wait.timeout", 100000);
if (string.IsNullOrEmpty(region) || string.IsNullOrEmpty(key))
{
_values.AddThrowError("ERROR:", $"Creating request ({path}); requires valid region and key.");
}
string url = GetCustomSpeechUrl(region, path, id, query);
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Accept = "application/json";
request.Headers.Add("Ocp-Apim-Subscription-Key", key);
request.Timeout = timeout;
if (!string.IsNullOrEmpty(contentType))
{
request.ContentType = contentType;
}
else if (method == WebRequestMethods.Http.Post || method == "PATCH")
{
request.ContentType = "application/json";
}
return request;
}
private string GetCustomSpeechUrl(string region, string path, string? id = null, string? query = null)
{
if (path.StartsWith("http")) return path;
var idIsUrl = id != null && id.StartsWith("http");
if (idIsUrl && id.Contains(path)) return id;
var version = _values.GetOrDefault("csr.api.version", "v3.1");
var pathPart = string.IsNullOrEmpty(id) ? path : $"{path}/{id}";
var queryPart = string.IsNullOrEmpty(query) ? "" : $"?{query}";
var endpoint = $"https://{region}.api.cognitive.microsoft.com/speechtotext/{version}";
endpoint = _values.GetOrDefault("csr.api.endpoint", endpoint);
return $"{endpoint}/{pathPart}{queryPart}";
}
private string CheckWriteOutputRequest(HttpWebRequest request, string payload = null, bool append = false)
{
var output = _values.GetOrEmpty("csr.output.request.file");
if (!string.IsNullOrEmpty(output))
{
var fileName = FileHelpers.GetOutputDataFileName(output, _values);
payload = HttpHelpers.WriteOutputRequest(request, fileName, payload, append);
}
return payload;
}
private bool CheckWaitForComplete(HttpWebResponse created, string statusJson, string domain, string thing, string id)
{
string message = $"Awaiting {thing} {id} success or failure ...";
HttpWebRequest createWebRequest() => CreateWebRequest(WebRequestMethods.Http.Get, $"{thing}s", id);
return HttpHelpers.CheckWaitForComplete(created, statusJson, createWebRequest, _values, domain, message, _quiet, _verbose);
}
private string? DownloadUrl(string url, string defaultFileName = null)
{
var message = url.Substring(0, Math.Min(100, url.Length));
message = url.Length > 100
? $"Downloading {message}... "
: $"Downloading {message} ... ";
if (!_quiet) Console.WriteLine(message);
var request = CreateWebRequest(WebRequestMethods.Http.Get, url);
var response = HttpHelpers.GetWebResponse(request);
if (!_quiet) Console.WriteLine($"{message} Done!\n");
return ReadWritePrintResponse(response, defaultFileName);
}
private string? ReadWritePrintResponse(HttpWebResponse response, string defaultFileName = null)
{
var saveAs = HttpHelpers.GetOutputDataFileName(defaultFileName, response, _values, "csr", out _, out bool isJson);
var message = !_quiet ? "Saving as" : null;
var printJson = !_quiet && _verbose && isJson;
var downloaded = HttpHelpers.ReadWriteResponse(response, saveAs, message, printJson);
if (printJson) JsonHelpers.PrintJson(downloaded);
return downloaded;
}
private string ReadWritePrintJson(HttpWebResponse response, bool skipWrite = false)
{
var json = HttpHelpers.ReadWriteJson(response, _values, "csr", skipWrite);
if (!_quiet && _verbose) JsonHelpers.PrintJson(json);
return json;
}
private bool _quiet = false;
private bool _verbose = false;
}
}