in arm/arm-smmu/arm-smmu.c [628:837]
static int arm_smmu_init_domain_context(struct iommu_domain *domain,
struct arm_smmu_device *smmu,
struct device *dev)
{
int irq, start, ret = 0;
unsigned long ias, oas;
struct io_pgtable_ops *pgtbl_ops;
struct io_pgtable_cfg pgtbl_cfg;
enum io_pgtable_fmt fmt;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
irqreturn_t (*context_fault)(int irq, void *dev);
mutex_lock(&smmu_domain->init_mutex);
if (smmu_domain->smmu)
goto out_unlock;
if (domain->type == IOMMU_DOMAIN_IDENTITY) {
smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
smmu_domain->smmu = smmu;
goto out_unlock;
}
/*
* Mapping the requested stage onto what we support is surprisingly
* complicated, mainly because the spec allows S1+S2 SMMUs without
* support for nested translation. That means we end up with the
* following table:
*
* Requested Supported Actual
* S1 N S1
* S1 S1+S2 S1
* S1 S2 S2
* S1 S1 S1
* N N N
* N S1+S2 S2
* N S2 S2
* N S1 S1
*
* Note that you can't actually request stage-2 mappings.
*/
if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
/*
* Choosing a suitable context format is even more fiddly. Until we
* grow some way for the caller to express a preference, and/or move
* the decision into the io-pgtable code where it arguably belongs,
* just aim for the closest thing to the rest of the system, and hope
* that the hardware isn't esoteric enough that we can't assume AArch64
* support to be a superset of AArch32 support...
*/
if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
!IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
(smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
(smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
(smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
ARM_SMMU_FEAT_FMT_AARCH64_16K |
ARM_SMMU_FEAT_FMT_AARCH64_4K)))
cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
ret = -EINVAL;
goto out_unlock;
}
switch (smmu_domain->stage) {
case ARM_SMMU_DOMAIN_S1:
cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
start = smmu->num_s2_context_banks;
ias = smmu->va_size;
oas = smmu->ipa_size;
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
fmt = ARM_64_LPAE_S1;
} else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
fmt = ARM_32_LPAE_S1;
ias = min(ias, 32UL);
oas = min(oas, 40UL);
} else {
fmt = ARM_V7S;
ias = min(ias, 32UL);
oas = min(oas, 32UL);
}
smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
break;
case ARM_SMMU_DOMAIN_NESTED:
/*
* We will likely want to change this if/when KVM gets
* involved.
*/
case ARM_SMMU_DOMAIN_S2:
cfg->cbar = CBAR_TYPE_S2_TRANS;
start = 0;
ias = smmu->ipa_size;
oas = smmu->pa_size;
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
fmt = ARM_64_LPAE_S2;
} else {
fmt = ARM_32_LPAE_S2;
ias = min(ias, 40UL);
oas = min(oas, 40UL);
}
if (smmu->version == ARM_SMMU_V2)
smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
else
smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
break;
default:
ret = -EINVAL;
goto out_unlock;
}
ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
if (ret < 0) {
goto out_unlock;
}
smmu_domain->smmu = smmu;
cfg->cbndx = ret;
if (smmu->version < ARM_SMMU_V2) {
cfg->irptndx = atomic_inc_return(&smmu->irptndx);
cfg->irptndx %= smmu->num_context_irqs;
} else {
cfg->irptndx = cfg->cbndx;
}
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
cfg->vmid = cfg->cbndx + 1;
else
cfg->asid = cfg->cbndx;
pgtbl_cfg = (struct io_pgtable_cfg) {
.pgsize_bitmap = smmu->pgsize_bitmap,
.ias = ias,
.oas = oas,
.coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
.tlb = smmu_domain->flush_ops,
.iommu_dev = smmu->dev,
};
if (smmu->impl && smmu->impl->init_context) {
ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
if (ret)
goto out_clear_smmu;
}
if (smmu_domain->pgtbl_quirks)
pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
if (!pgtbl_ops) {
ret = -ENOMEM;
goto out_clear_smmu;
}
/* Update the domain's page sizes to reflect the page table format */
domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
domain->geometry.aperture_start = ~0UL << ias;
domain->geometry.aperture_end = ~0UL;
} else {
domain->geometry.aperture_end = (1UL << ias) - 1;
}
domain->geometry.force_aperture = true;
/* Initialise the context bank with our page table cfg */
arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
arm_smmu_write_context_bank(smmu, cfg->cbndx);
/*
* Request context fault interrupt. Do this last to avoid the
* handler seeing a half-initialised domain state.
*/
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
if (smmu->impl && smmu->impl->context_fault)
context_fault = smmu->impl->context_fault;
else
context_fault = arm_smmu_context_fault;
ret = devm_request_irq(smmu->dev, irq, context_fault,
IRQF_SHARED, "arm-smmu-context-fault", domain);
if (ret < 0) {
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
cfg->irptndx, irq);
cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
}
mutex_unlock(&smmu_domain->init_mutex);
/* Publish page table ops for map/unmap */
smmu_domain->pgtbl_ops = pgtbl_ops;
return 0;
out_clear_smmu:
__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
smmu_domain->smmu = NULL;
out_unlock:
mutex_unlock(&smmu_domain->init_mutex);
return ret;
}