in azure-slurm/slurmcc/cost.py [0:0]
def process_jobs(self, azcost: azurecost, jobsfp, out_fmt_t):
_job_rec_file = self.fetch_job_records()
running = self.process_queue()
fp = open(_job_rec_file, newline='')
reader = csv.reader(fp, delimiter='|')
writer = csv.writer(jobsfp, delimiter=',')
for row in map(self.in_fmt_t._make, reader):
self.stats.jobs += 1
if row.state == 'RUNNING' and int(row.jobid) in running:
admincomment = running[int(row.jobid)]
self.stats.running_jobs += 1
else:
admincomment = row.admincomment
try:
comment_d = self.parse_admincomment(admincomment)[0]
sku_name = comment_d['vm_size']
cpupernode = comment_d['pcpu_count']
region = comment_d['location']
spot = comment_d['spot']
except (json.JSONDecodeError,IndexError) as e:
log.debug(f"Cannot parse admincomment job={row.jobid} cluster={row.cluster} admincomment={admincomment}")
self.stats.admincomment_err += 1
self.stats.unprocessed += 1
continue
except KeyError as e:
log.debug(f"Key: {e.args[0]} not found in admincomment, job={row.jobid}, cluster={row.cluster}")
self.stats.admincomment_err +=1
self.stats.unprocessed += 1
continue
charge_factor = float(row.ncpus) / float(cpupernode)
az_fmt = azcost.get_job(sku_name, region, spot)
charged_cost = ((az_fmt.rate/3600) * float(row.elapsedraw)) * charge_factor
c_fmt = self.c_fmt_t(cost=charged_cost)
if (region,sku_name) not in self.stats.cost_per_sku:
self.stats.cost_per_sku[(region,sku_name)] = 0
self.stats.cost_per_sku[(region,sku_name)] += charged_cost
out_row = []
for f in out_fmt_t._fields:
if f in self.in_fmt_t._fields:
out_row.append(row._asdict()[f])
elif f in az_fmt._fields:
out_row.append(az_fmt._asdict()[f])
elif f in self.c_fmt_t._fields:
out_row.append(c_fmt._asdict()[f])
else:
log.error(f"encountered an unexpected field {f}")
writer.writerow(out_row)
self.stats.processed += 1
fp.close()