int hl_hw_queue_schedule_cs()

in habanalabs/common/hw_queue.c [624:787]

• 135 lines of code
• 32 McCabe index (conditional complexity)

int hl_hw_queue_schedule_cs(struct hl_cs *cs)
{
	enum hl_device_status status;
	struct hl_cs_counters_atomic *cntr;
	struct hl_ctx *ctx = cs->ctx;
	struct hl_device *hdev = ctx->hdev;
	struct hl_cs_job *job, *tmp;
	struct hl_hw_queue *q;
	int rc = 0, i, cq_cnt;
	bool first_entry;
	u32 max_queues;

	cntr = &hdev->aggregated_cs_counters;

	hdev->asic_funcs->hw_queues_lock(hdev);

	if (!hl_device_operational(hdev, &status)) {
		atomic64_inc(&cntr->device_in_reset_drop_cnt);
		atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt);
		dev_err(hdev->dev,
			"device is %s, CS rejected!\n", hdev->status[status]);
		rc = -EPERM;
		goto out;
	}

	max_queues = hdev->asic_prop.max_queues;

	q = &hdev->kernel_queues[0];
	for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {
		if (cs->jobs_in_queue_cnt[i]) {
			switch (q->queue_type) {
			case QUEUE_TYPE_EXT:
				rc = ext_queue_sanity_checks(hdev, q,
						cs->jobs_in_queue_cnt[i],
						cs_needs_completion(cs) ?
								true : false);
				break;
			case QUEUE_TYPE_INT:
				rc = int_queue_sanity_checks(hdev, q,
						cs->jobs_in_queue_cnt[i]);
				break;
			case QUEUE_TYPE_HW:
				rc = hw_queue_sanity_checks(hdev, q,
						cs->jobs_in_queue_cnt[i]);
				break;
			default:
				dev_err(hdev->dev, "Queue type %d is invalid\n",
					q->queue_type);
				rc = -EINVAL;
				break;
			}

			if (rc) {
				atomic64_inc(
					&ctx->cs_counters.queue_full_drop_cnt);
				atomic64_inc(&cntr->queue_full_drop_cnt);
				goto unroll_cq_resv;
			}

			if (q->queue_type == QUEUE_TYPE_EXT)
				cq_cnt++;
		}
	}

	if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {
		rc = init_signal_wait_cs(cs);
		if (rc)
			goto unroll_cq_resv;
	} else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {
		rc = hdev->asic_funcs->collective_wait_init_cs(cs);
		if (rc)
			goto unroll_cq_resv;
	}


	if (cs->encaps_signals && cs->staged_first) {
		rc = encaps_sig_first_staged_cs_handler(hdev, cs);
		if (rc)
			goto unroll_cq_resv;
	}

	spin_lock(&hdev->cs_mirror_lock);

	/* Verify staged CS exists and add to the staged list */
	if (cs->staged_cs && !cs->staged_first) {
		struct hl_cs *staged_cs;

		staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
		if (!staged_cs) {
			dev_err(hdev->dev,
				"Cannot find staged submission sequence %llu",
				cs->staged_sequence);
			rc = -EINVAL;
			goto unlock_cs_mirror;
		}

		if (is_staged_cs_last_exists(hdev, staged_cs)) {
			dev_err(hdev->dev,
				"Staged submission sequence %llu already submitted",
				cs->staged_sequence);
			rc = -EINVAL;
			goto unlock_cs_mirror;
		}

		list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);

		/* update stream map of the first CS */
		if (hdev->supports_wait_for_multi_cs)
			staged_cs->fence->stream_master_qid_map |=
					cs->fence->stream_master_qid_map;
	}

	list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);

	/* Queue TDR if the CS is the first entry and if timeout is wanted */
	first_entry = list_first_entry(&hdev->cs_mirror_list,
					struct hl_cs, mirror_node) == cs;
	if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
				first_entry && cs_needs_timeout(cs)) {
		cs->tdr_active = true;
		schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);

	}

	spin_unlock(&hdev->cs_mirror_lock);

	list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
		switch (job->queue_type) {
		case QUEUE_TYPE_EXT:
			ext_queue_schedule_job(job);
			break;
		case QUEUE_TYPE_INT:
			int_queue_schedule_job(job);
			break;
		case QUEUE_TYPE_HW:
			hw_queue_schedule_job(job);
			break;
		default:
			break;
		}

	cs->submitted = true;

	goto out;

unlock_cs_mirror:
	spin_unlock(&hdev->cs_mirror_lock);
unroll_cq_resv:
	q = &hdev->kernel_queues[0];
	for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {
		if ((q->queue_type == QUEUE_TYPE_EXT) &&
						(cs->jobs_in_queue_cnt[i])) {
			atomic_t *free_slots =
				&hdev->completion_queue[i].free_slots_cnt;
			atomic_add(cs->jobs_in_queue_cnt[i], free_slots);
			cq_cnt--;
		}
	}

out:
	hdev->asic_funcs->hw_queues_unlock(hdev);

	return rc;
}