in src/ApiForFhirMigrationTool.Function/ExportOrchestrator.cs [255:447]
private string GetQueryStringForExport(ILogger logger)
{
logger?.LogInformation("Started retrieving query for the export operation.");
var since = string.Empty;
var till = string.Empty;
var since_new = default(DateTimeOffset);
var updateSinceDate = default(DateTimeOffset);
logger?.LogInformation("Creating table clients");
TableClient chunktableClient = _azureTableClientFactory.Create(_options.ChunkTableName);
TableClient exportTableClient = _azureTableClientFactory.Create(_options.ExportTableName);
logger?.LogInformation("Table clients created successfully.");
TableEntity qEntity = _azureTableMetadataStore.GetEntity(chunktableClient, _options.PartitionKey, _options.RowKey);
since = _options.IsParallel == true ? (string)qEntity["since"] : (string)qEntity["globalSinceExportType"];
var duration = _options.ExportChunkDuration;
if (_options.StartDate == DateTime.MinValue && string.IsNullOrEmpty(since))
{
var sinceDate = SinceDate();
since_new = sinceDate.Result;
}
else if (string.IsNullOrEmpty(since))
{
since_new = _options.StartDate;
}
else
{
since_new = DateTimeOffset.Parse(since);
}
if (duration != null)
{
if (duration == "Days")
{
updateSinceDate = _options.IsParallel == true ? since_new.AddDays(_options.ExportChunkTime) : since_new.AddDays(_options.ResourceExportChunkTime);
}
else if (duration == "Hours")
{
updateSinceDate = _options.IsParallel == true ? since_new.AddHours(_options.ExportChunkTime) : since_new.AddHours(_options.ResourceExportChunkTime);
}
else
{
updateSinceDate = _options.IsParallel == true ? since_new.AddMinutes(_options.ExportChunkTime) : since_new.AddMinutes(_options.ResourceExportChunkTime);
}
}
if (updateSinceDate > DateTimeOffset.UtcNow)
{
if (_options.IsParallel == true)
{
updateSinceDate = DateTimeOffset.UtcNow;
}
else
{
updateSinceDate = qEntity["globalTillExportType"]?.ToString() != ""
? DateTimeOffset.Parse((string)qEntity["globalTillExportType"])
: DateTimeOffset.UtcNow;
}
}
since = since_new.ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
if (_options.SpecificRun)
{
if (updateSinceDate > _options.EndDate) { updateSinceDate = _options.EndDate; }
}
till = updateSinceDate.ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
logger?.LogInformation("Creating export URL and query.");
//need to check count whether getting data from gen1
string? resourceType = string.Empty;
string setUrl = string.Empty;
List<string>? completedResourceTypeList = new List<string>();
if (_options.IsParallel == true)
{
logger?.LogInformation("Processing parallel export.");
var checkValidRequest = CheckResourceCount(since, till, _options.ExportChunkTime, _options.ExportChunkDuration);
till = checkValidRequest.Result.ToString();
if (_options.IsExportDeidentified == true)
{
string configFileName = Path.GetFileNameWithoutExtension(_options.ConfigFile);
setUrl = $"/$export?_isParallel={_options.IsParallel.ToString().ToLower()}&_container=anonymization&_anonymizationConfig={configFileName}.json";
}
else
{
setUrl = $"/$export?_isParallel={_options.IsParallel.ToString().ToLower()}";
}
}
else
{
logger?.LogInformation("Processing non-parallel export.");
bool IsLastRun = false;
int tot = 0;
TableEntity qEntityGetResourceIndex = _azureTableMetadataStore.GetEntity(chunktableClient, _options.PartitionKey, _options.RowKey);
int index = (int)qEntityGetResourceIndex["resourceTypeIndex"]; // get index from DB
// mark global since and till
if (index == 0) //
{
qEntityGetResourceIndex["globalSinceExportType"] = since;
qEntityGetResourceIndex["globalTillExportType"] = till;
_azureTableMetadataStore.UpdateEntity(chunktableClient, qEntityGetResourceIndex);
}
do
{
resourceType = _options.ResourceTypes?[index];
if (qEntityGetResourceIndex?["multiExport"].ToString() == "Running")
{
since = qEntityGetResourceIndex["subSinceExportType"].ToString();
till = qEntityGetResourceIndex["subTillExportType"].ToString();
}
logger?.LogInformation($"Checking resource count for type '{resourceType}'.");
var response = CheckResourceTypeCount(since!, till!, resourceType!, _options.ResourceExportChunkTime, _options.ExportChunkDuration);
tot = response.Result;
qEntityGetResourceIndex = _azureTableMetadataStore.GetEntity(chunktableClient, _options.PartitionKey, _options.RowKey);
if (tot > 0 && index != (int)qEntityGetResourceIndex["resourceTypeIndex"])
{
qEntityGetResourceIndex["resourceTypeIndex"] = index; //
_azureTableMetadataStore.UpdateEntity(chunktableClient, qEntityGetResourceIndex);
}
IsLastRun = CheckLastCount(index);
if (tot == 0 && qEntityGetResourceIndex?["multiExport"].ToString() == "Running")
{
// multiexport run and no data to export then assigining till to since and global till to sub till
qEntityGetResourceIndex["subSinceExportType"] = qEntityGetResourceIndex["subTillExportType"];
qEntityGetResourceIndex["subTillExportType"] = qEntityGetResourceIndex["globalTillExportType"];
_azureTableMetadataStore.UpdateEntity(chunktableClient, qEntityGetResourceIndex);
}
if (qEntityGetResourceIndex?["multiExport"].ToString() != "Running" && tot == 0 && index < _options.ResourceTypes?.Count() - 1)
{
index++;
}
} while (tot == 0 && IsLastRun == false);
if (qEntityGetResourceIndex?["multiExport"].ToString() == "Running")
{
qEntityGetResourceIndex = _azureTableMetadataStore.GetEntity(chunktableClient, _options.PartitionKey, _options.RowKey);
since = qEntityGetResourceIndex["subSinceExportType"].ToString(); // assigning till to since as next round trip withing subexport
till = qEntityGetResourceIndex["subTillExportType"].ToString();
}
if (tot == 0 && IsLastRun == true)
{
// logger?.LogInformation(" Resetting table entity.");
TableEntity qEntitynew = _azureTableMetadataStore.GetEntity(chunktableClient, _options.PartitionKey, _options.RowKey);
qEntitynew["globalSinceExportType"] = qEntitynew["globalTillExportType"];
qEntitynew["globalTillExportType"] = "";
qEntitynew["resourceTypeIndex"] = 0; // all the import will done so will reset index
qEntitynew["multiExport"] = "";
qEntitynew["subSinceExportType"] = "";
qEntitynew["subTillExportType"] = "";
_azureTableMetadataStore.UpdateEntity(chunktableClient, qEntitynew);
logger?.LogInformation("Updating logs in Application Insights.");
_telemetryClient.TrackEvent(
"ImportTill",
new Dictionary<string, string>()
{
{ "Till", qEntitynew["globalTillExportType"].ToString() }
});
logger?.LogInformation("Logs updated successfully in Application Insights.");
return "";
}
else
{
setUrl = $"/$export?_type={resourceType}";
}
}
string query = string.Empty;
if (_options.ExportWithHistory == true && _options.ExportWithDelete == true)
query = string.Format("includeAssociatedData=_history,_deleted&_since={0}&_till={1}", since, till);
else if (_options.ExportWithHistory == true)
query = string.Format("includeAssociatedData=_history&_since={0}&_till={1}", since, till);
else if (_options.ExportWithDelete == true)
query = string.Format("includeAssociatedData=_deleted&_since={0}&_till={1}", since, till);
else
query = string.Format("_since={0}&_till={1}", since, till);
logger?.LogInformation("Query for the export operation retrieved successfully.");
if (_options.MaxCount == true)
{
var maxValue = _options.MaxCountValue == 0 ? 10000 : _options.MaxCountValue;
return $"{setUrl}&{query}&_maxCount={maxValue.ToString()}";
}
else
return $"{setUrl}&{query}";
}