int qi_submit_sync()

in intel/dmar.c [1340:1458]

• 81 lines of code
• 20 McCabe index (conditional complexity)

int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
		   unsigned int count, unsigned long options)
{
	struct q_inval *qi = iommu->qi;
	s64 devtlb_start_ktime = 0;
	s64 iotlb_start_ktime = 0;
	s64 iec_start_ktime = 0;
	struct qi_desc wait_desc;
	int wait_index, index;
	unsigned long flags;
	int offset, shift;
	int rc, i;
	u64 type;

	if (!qi)
		return 0;

	type = desc->qw0 & GENMASK_ULL(3, 0);

	if ((type == QI_IOTLB_TYPE || type == QI_EIOTLB_TYPE) &&
	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))
		iotlb_start_ktime = ktime_to_ns(ktime_get());

	if ((type == QI_DIOTLB_TYPE || type == QI_DEIOTLB_TYPE) &&
	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))
		devtlb_start_ktime = ktime_to_ns(ktime_get());

	if (type == QI_IEC_TYPE &&
	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))
		iec_start_ktime = ktime_to_ns(ktime_get());

restart:
	rc = 0;

	raw_spin_lock_irqsave(&qi->q_lock, flags);
	/*
	 * Check if we have enough empty slots in the queue to submit,
	 * the calculation is based on:
	 * # of desc + 1 wait desc + 1 space between head and tail
	 */
	while (qi->free_cnt < count + 2) {
		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
		cpu_relax();
		raw_spin_lock_irqsave(&qi->q_lock, flags);
	}

	index = qi->free_head;
	wait_index = (index + count) % QI_LENGTH;
	shift = qi_shift(iommu);

	for (i = 0; i < count; i++) {
		offset = ((index + i) % QI_LENGTH) << shift;
		memcpy(qi->desc + offset, &desc[i], 1 << shift);
		qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
		trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,
				desc[i].qw2, desc[i].qw3);
	}
	qi->desc_status[wait_index] = QI_IN_USE;

	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
	if (options & QI_OPT_WAIT_DRAIN)
		wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
	wait_desc.qw2 = 0;
	wait_desc.qw3 = 0;

	offset = wait_index << shift;
	memcpy(qi->desc + offset, &wait_desc, 1 << shift);

	qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
	qi->free_cnt -= count + 1;

	/*
	 * update the HW tail register indicating the presence of
	 * new descriptors.
	 */
	writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);

	while (qi->desc_status[wait_index] != QI_DONE) {
		/*
		 * We will leave the interrupts disabled, to prevent interrupt
		 * context to queue another cmd while a cmd is already submitted
		 * and waiting for completion on this cpu. This is to avoid
		 * a deadlock where the interrupt context can wait indefinitely
		 * for free slots in the queue.
		 */
		rc = qi_check_fault(iommu, index, wait_index);
		if (rc)
			break;

		raw_spin_unlock(&qi->q_lock);
		cpu_relax();
		raw_spin_lock(&qi->q_lock);
	}

	for (i = 0; i < count; i++)
		qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;

	reclaim_free_desc(qi);
	raw_spin_unlock_irqrestore(&qi->q_lock, flags);

	if (rc == -EAGAIN)
		goto restart;

	if (iotlb_start_ktime)
		dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,
				ktime_to_ns(ktime_get()) - iotlb_start_ktime);

	if (devtlb_start_ktime)
		dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,
				ktime_to_ns(ktime_get()) - devtlb_start_ktime);

	if (iec_start_ktime)
		dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,
				ktime_to_ns(ktime_get()) - iec_start_ktime);

	return rc;
}