diff options
Diffstat (limited to 'hw/arm/smmu-common.c')
| -rw-r--r-- | hw/arm/smmu-common.c | 354 |
1 files changed, 310 insertions, 44 deletions
diff --git a/hw/arm/smmu-common.c b/hw/arm/smmu-common.c index 1ce706b..f39b99e 100644 --- a/hw/arm/smmu-common.c +++ b/hw/arm/smmu-common.c @@ -57,7 +57,7 @@ static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) (k1->vmid == k2->vmid); } -SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint16_t vmid, uint64_t iova, +SMMUIOTLBKey smmu_get_iotlb_key(int asid, int vmid, uint64_t iova, uint8_t tg, uint8_t level) { SMMUIOTLBKey key = {.asid = asid, .vmid = vmid, .iova = iova, @@ -66,8 +66,10 @@ SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint16_t vmid, uint64_t iova, return key; } -SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, - SMMUTransTableInfo *tt, hwaddr iova) +static SMMUTLBEntry *smmu_iotlb_lookup_all_levels(SMMUState *bs, + SMMUTransCfg *cfg, + SMMUTransTableInfo *tt, + hwaddr iova) { uint8_t tg = (tt->granule_sz - 10) / 2; uint8_t inputsize = 64 - tt->tsz; @@ -88,6 +90,36 @@ SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, } level++; } + return entry; +} + +/** + * smmu_iotlb_lookup - Look up for a TLB entry. + * @bs: SMMU state which includes the TLB instance + * @cfg: Configuration of the translation + * @tt: Translation table info (granule and tsz) + * @iova: IOVA address to lookup + * + * returns a valid entry on success, otherwise NULL. + * In case of nested translation, tt can be updated to include + * the granule of the found entry as it might different from + * the IOVA granule. + */ +SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, + SMMUTransTableInfo *tt, hwaddr iova) +{ + SMMUTLBEntry *entry = NULL; + + entry = smmu_iotlb_lookup_all_levels(bs, cfg, tt, iova); + /* + * For nested translation also try the s2 granule, as the TLB will insert + * it if the size of s2 tlb entry was smaller. + */ + if (!entry && (cfg->stage == SMMU_NESTED) && + (cfg->s2cfg.granule_sz != tt->granule_sz)) { + tt->granule_sz = cfg->s2cfg.granule_sz; + entry = smmu_iotlb_lookup_all_levels(bs, cfg, tt, iova); + } if (entry) { cfg->iotlb_hits++; @@ -127,24 +159,35 @@ void smmu_iotlb_inv_all(SMMUState *s) g_hash_table_remove_all(s->iotlb); } -static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, - gpointer user_data) +static gboolean smmu_hash_remove_by_asid_vmid(gpointer key, gpointer value, + gpointer user_data) { - uint16_t asid = *(uint16_t *)user_data; + SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data; SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; - return SMMU_IOTLB_ASID(*iotlb_key) == asid; + return (SMMU_IOTLB_ASID(*iotlb_key) == info->asid) && + (SMMU_IOTLB_VMID(*iotlb_key) == info->vmid); } static gboolean smmu_hash_remove_by_vmid(gpointer key, gpointer value, gpointer user_data) { - uint16_t vmid = *(uint16_t *)user_data; + int vmid = *(int *)user_data; SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; return SMMU_IOTLB_VMID(*iotlb_key) == vmid; } +static gboolean smmu_hash_remove_by_vmid_s1(gpointer key, gpointer value, + gpointer user_data) +{ + int vmid = *(int *)user_data; + SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; + + return (SMMU_IOTLB_VMID(*iotlb_key) == vmid) && + (SMMU_IOTLB_ASID(*iotlb_key) >= 0); +} + static gboolean smmu_hash_remove_by_asid_vmid_iova(gpointer key, gpointer value, gpointer user_data) { @@ -163,6 +206,46 @@ static gboolean smmu_hash_remove_by_asid_vmid_iova(gpointer key, gpointer value, ((entry->iova & ~info->mask) == info->iova); } +static gboolean smmu_hash_remove_by_vmid_ipa(gpointer key, gpointer value, + gpointer user_data) +{ + SMMUTLBEntry *iter = (SMMUTLBEntry *)value; + IOMMUTLBEntry *entry = &iter->entry; + SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data; + SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key; + + if (SMMU_IOTLB_ASID(iotlb_key) >= 0) { + /* This is a stage-1 address. */ + return false; + } + if (info->vmid != SMMU_IOTLB_VMID(iotlb_key)) { + return false; + } + return ((info->iova & ~entry->addr_mask) == entry->iova) || + ((entry->iova & ~info->mask) == info->iova); +} + +static gboolean +smmu_hash_remove_by_sid_range(gpointer key, gpointer value, gpointer user_data) +{ + SMMUDevice *sdev = (SMMUDevice *)key; + uint32_t sid = smmu_get_sid(sdev); + SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data; + + if (sid < sid_range->start || sid > sid_range->end) { + return false; + } + trace_smmu_config_cache_inv(sid); + return true; +} + +void smmu_configs_inv_sid_range(SMMUState *s, SMMUSIDRange sid_range) +{ + trace_smmu_configs_inv_sid_range(sid_range.start, sid_range.end); + g_hash_table_foreach_remove(s->configs, smmu_hash_remove_by_sid_range, + &sid_range); +} + void smmu_iotlb_inv_iova(SMMUState *s, int asid, int vmid, dma_addr_t iova, uint8_t tg, uint64_t num_pages, uint8_t ttl) { @@ -191,18 +274,57 @@ void smmu_iotlb_inv_iova(SMMUState *s, int asid, int vmid, dma_addr_t iova, &info); } -void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) +/* + * Similar to smmu_iotlb_inv_iova(), but for Stage-2, ASID is always -1, + * in Stage-1 invalidation ASID = -1, means don't care. + */ +void smmu_iotlb_inv_ipa(SMMUState *s, int vmid, dma_addr_t ipa, uint8_t tg, + uint64_t num_pages, uint8_t ttl) { - trace_smmu_iotlb_inv_asid(asid); - g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); + uint8_t granule = tg ? tg * 2 + 10 : 12; + int asid = -1; + + if (ttl && (num_pages == 1)) { + SMMUIOTLBKey key = smmu_get_iotlb_key(asid, vmid, ipa, tg, ttl); + + if (g_hash_table_remove(s->iotlb, &key)) { + return; + } + } + + SMMUIOTLBPageInvInfo info = { + .iova = ipa, + .vmid = vmid, + .mask = (num_pages << granule) - 1}; + + g_hash_table_foreach_remove(s->iotlb, + smmu_hash_remove_by_vmid_ipa, + &info); } -void smmu_iotlb_inv_vmid(SMMUState *s, uint16_t vmid) +void smmu_iotlb_inv_asid_vmid(SMMUState *s, int asid, int vmid) +{ + SMMUIOTLBPageInvInfo info = { + .asid = asid, + .vmid = vmid, + }; + + trace_smmu_iotlb_inv_asid_vmid(asid, vmid); + g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid_vmid, &info); +} + +void smmu_iotlb_inv_vmid(SMMUState *s, int vmid) { trace_smmu_iotlb_inv_vmid(vmid); g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_vmid, &vmid); } +inline void smmu_iotlb_inv_vmid_s1(SMMUState *s, int vmid) +{ + trace_smmu_iotlb_inv_vmid_s1(vmid); + g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_vmid_s1, &vmid); +} + /* VMSAv8-64 Translation */ /** @@ -286,8 +408,41 @@ SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) return NULL; } +/* Translate stage-1 table address using stage-2 page table. */ +static inline int translate_table_addr_ipa(SMMUState *bs, + dma_addr_t *table_addr, + SMMUTransCfg *cfg, + SMMUPTWEventInfo *info) +{ + dma_addr_t addr = *table_addr; + SMMUTLBEntry *cached_entry; + int asid; + + /* + * The translation table walks performed from TTB0 or TTB1 are always + * performed in IPA space if stage 2 translations are enabled. + */ + asid = cfg->asid; + cfg->stage = SMMU_STAGE_2; + cfg->asid = -1; + cached_entry = smmu_translate(bs, cfg, addr, IOMMU_RO, info); + cfg->asid = asid; + cfg->stage = SMMU_NESTED; + + if (cached_entry) { + *table_addr = CACHED_ENTRY_TO_ADDR(cached_entry, addr); + return 0; + } + + info->stage = SMMU_STAGE_2; + info->addr = addr; + info->is_ipa_descriptor = true; + return -EINVAL; +} + /** * smmu_ptw_64_s1 - VMSAv8-64 Walk of the page tables for a given IOVA + * @bs: smmu state which includes TLB instance * @cfg: translation config * @iova: iova to translate * @perm: access type @@ -299,12 +454,12 @@ SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) * Upon success, @tlbe is filled with translated_addr and entry * permission rights. */ -static int smmu_ptw_64_s1(SMMUTransCfg *cfg, +static int smmu_ptw_64_s1(SMMUState *bs, SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) { dma_addr_t baseaddr, indexmask; - int stage = cfg->stage; + SMMUStage stage = cfg->stage; SMMUTransTableInfo *tt = select_tt(cfg, iova); uint8_t level, granule_sz, inputsize, stride; @@ -318,7 +473,8 @@ static int smmu_ptw_64_s1(SMMUTransCfg *cfg, inputsize = 64 - tt->tsz; level = 4 - (inputsize - 4) / stride; indexmask = VMSA_IDXMSK(inputsize, stride, level); - baseaddr = extract64(tt->ttb, 0, 48); + + baseaddr = extract64(tt->ttb, 0, cfg->oas); baseaddr &= ~indexmask; while (level < VMSA_LEVELS) { @@ -349,6 +505,11 @@ static int smmu_ptw_64_s1(SMMUTransCfg *cfg, goto error; } baseaddr = get_table_pte_address(pte, granule_sz); + if (cfg->stage == SMMU_NESTED) { + if (translate_table_addr_ipa(bs, &baseaddr, cfg, info)) { + goto error; + } + } level++; continue; } else if (is_page_pte(pte, level)) { @@ -381,10 +542,21 @@ static int smmu_ptw_64_s1(SMMUTransCfg *cfg, goto error; } + /* + * The address output from the translation causes a stage 1 Address + * Size fault if it exceeds the range of the effective IPA size for + * the given CD. + */ + if (gpa >= (1ULL << cfg->oas)) { + info->type = SMMU_PTW_ERR_ADDR_SIZE; + goto error; + } + tlbe->entry.translated_addr = gpa; tlbe->entry.iova = iova & ~mask; tlbe->entry.addr_mask = mask; - tlbe->entry.perm = PTE_AP_TO_PERM(ap); + tlbe->parent_perm = PTE_AP_TO_PERM(ap); + tlbe->entry.perm = tlbe->parent_perm; tlbe->level = level; tlbe->granule = granule_sz; return 0; @@ -392,7 +564,7 @@ static int smmu_ptw_64_s1(SMMUTransCfg *cfg, info->type = SMMU_PTW_ERR_TRANSLATION; error: - info->stage = 1; + info->stage = SMMU_STAGE_1; tlbe->entry.perm = IOMMU_NONE; return -EINVAL; } @@ -415,7 +587,7 @@ static int smmu_ptw_64_s2(SMMUTransCfg *cfg, dma_addr_t ipa, IOMMUAccessFlags perm, SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) { - const int stage = 2; + const SMMUStage stage = SMMU_STAGE_2; int granule_sz = cfg->s2cfg.granule_sz; /* ARM DDI0487I.a: Table D8-7. */ int inputsize = 64 - cfg->s2cfg.tsz; @@ -426,8 +598,8 @@ static int smmu_ptw_64_s2(SMMUTransCfg *cfg, * Get the ttb from concatenated structure. * The offset is the idx * size of each ttb(number of ptes * (sizeof(pte)) */ - uint64_t baseaddr = extract64(cfg->s2cfg.vttb, 0, 48) + (1 << stride) * - idx * sizeof(uint64_t); + uint64_t baseaddr = extract64(cfg->s2cfg.vttb, 0, cfg->s2cfg.eff_ps) + + (1 << stride) * idx * sizeof(uint64_t); dma_addr_t indexmask = VMSA_IDXMSK(inputsize, stride, level); baseaddr &= ~indexmask; @@ -438,7 +610,7 @@ static int smmu_ptw_64_s2(SMMUTransCfg *cfg, */ if (ipa >= (1ULL << inputsize)) { info->type = SMMU_PTW_ERR_TRANSLATION; - goto error; + goto error_ipa; } while (level < VMSA_LEVELS) { @@ -484,13 +656,13 @@ static int smmu_ptw_64_s2(SMMUTransCfg *cfg, */ if (!PTE_AF(pte) && !cfg->s2cfg.affd) { info->type = SMMU_PTW_ERR_ACCESS; - goto error; + goto error_ipa; } s2ap = PTE_AP(pte); if (is_permission_fault_s2(s2ap, perm)) { info->type = SMMU_PTW_ERR_PERMISSION; - goto error; + goto error_ipa; } /* @@ -499,28 +671,53 @@ static int smmu_ptw_64_s2(SMMUTransCfg *cfg, */ if (gpa >= (1ULL << cfg->s2cfg.eff_ps)) { info->type = SMMU_PTW_ERR_ADDR_SIZE; - goto error; + goto error_ipa; } tlbe->entry.translated_addr = gpa; tlbe->entry.iova = ipa & ~mask; tlbe->entry.addr_mask = mask; - tlbe->entry.perm = s2ap; + tlbe->parent_perm = s2ap; + tlbe->entry.perm = tlbe->parent_perm; tlbe->level = level; tlbe->granule = granule_sz; return 0; } info->type = SMMU_PTW_ERR_TRANSLATION; +error_ipa: + info->addr = ipa; error: - info->stage = 2; + info->stage = SMMU_STAGE_2; tlbe->entry.perm = IOMMU_NONE; return -EINVAL; } +/* + * combine S1 and S2 TLB entries into a single entry. + * As a result the S1 entry is overridden with combined data. + */ +static void combine_tlb(SMMUTLBEntry *tlbe, SMMUTLBEntry *tlbe_s2, + dma_addr_t iova, SMMUTransCfg *cfg) +{ + if (tlbe_s2->entry.addr_mask < tlbe->entry.addr_mask) { + tlbe->entry.addr_mask = tlbe_s2->entry.addr_mask; + tlbe->granule = tlbe_s2->granule; + tlbe->level = tlbe_s2->level; + } + + tlbe->entry.translated_addr = CACHED_ENTRY_TO_ADDR(tlbe_s2, + tlbe->entry.translated_addr); + + tlbe->entry.iova = iova & ~tlbe->entry.addr_mask; + /* parent_perm has s2 perm while perm keeps s1 perm. */ + tlbe->parent_perm = tlbe_s2->entry.perm; +} + /** * smmu_ptw - Walk the page tables for an IOVA, according to @cfg * + * @bs: smmu state which includes TLB instance * @cfg: translation configuration * @iova: iova to translate * @perm: tentative access type @@ -529,12 +726,16 @@ error: * * return 0 on success */ -int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, - SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) +int smmu_ptw(SMMUState *bs, SMMUTransCfg *cfg, dma_addr_t iova, + IOMMUAccessFlags perm, SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) { - if (cfg->stage == 1) { - return smmu_ptw_64_s1(cfg, iova, perm, tlbe, info); - } else if (cfg->stage == 2) { + int ret; + SMMUTLBEntry tlbe_s2; + dma_addr_t ipa; + + if (cfg->stage == SMMU_STAGE_1) { + return smmu_ptw_64_s1(bs, cfg, iova, perm, tlbe, info); + } else if (cfg->stage == SMMU_STAGE_2) { /* * If bypassing stage 1(or unimplemented), the input address is passed * directly to stage 2 as IPA. If the input address of a transaction @@ -543,7 +744,7 @@ int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, */ if (iova >= (1ULL << cfg->oas)) { info->type = SMMU_PTW_ERR_ADDR_SIZE; - info->stage = 1; + info->stage = SMMU_STAGE_1; tlbe->entry.perm = IOMMU_NONE; return -EINVAL; } @@ -551,7 +752,72 @@ int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, return smmu_ptw_64_s2(cfg, iova, perm, tlbe, info); } - g_assert_not_reached(); + /* SMMU_NESTED. */ + ret = smmu_ptw_64_s1(bs, cfg, iova, perm, tlbe, info); + if (ret) { + return ret; + } + + ipa = CACHED_ENTRY_TO_ADDR(tlbe, iova); + ret = smmu_ptw_64_s2(cfg, ipa, perm, &tlbe_s2, info); + if (ret) { + return ret; + } + + combine_tlb(tlbe, &tlbe_s2, iova, cfg); + return 0; +} + +SMMUTLBEntry *smmu_translate(SMMUState *bs, SMMUTransCfg *cfg, dma_addr_t addr, + IOMMUAccessFlags flag, SMMUPTWEventInfo *info) +{ + SMMUTLBEntry *cached_entry = NULL; + SMMUTransTableInfo *tt; + int status; + + /* + * Combined attributes used for TLB lookup, holds the attributes for + * the input stage. + */ + SMMUTransTableInfo tt_combined; + + if (cfg->stage == SMMU_STAGE_2) { + /* Stage2. */ + tt_combined.granule_sz = cfg->s2cfg.granule_sz; + tt_combined.tsz = cfg->s2cfg.tsz; + } else { + /* Select stage1 translation table. */ + tt = select_tt(cfg, addr); + if (!tt) { + info->type = SMMU_PTW_ERR_TRANSLATION; + info->stage = SMMU_STAGE_1; + return NULL; + } + tt_combined.granule_sz = tt->granule_sz; + tt_combined.tsz = tt->tsz; + } + + cached_entry = smmu_iotlb_lookup(bs, cfg, &tt_combined, addr); + if (cached_entry) { + if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & + cached_entry->parent_perm & IOMMU_WO)) { + info->type = SMMU_PTW_ERR_PERMISSION; + info->stage = !(cached_entry->entry.perm & IOMMU_WO) ? + SMMU_STAGE_1 : + SMMU_STAGE_2; + return NULL; + } + return cached_entry; + } + + cached_entry = g_new0(SMMUTLBEntry, 1); + status = smmu_ptw(bs, cfg, addr, flag, cached_entry, info); + if (status) { + g_free(cached_entry); + return NULL; + } + smmu_iotlb_insert(bs, cfg, cached_entry); + return cached_entry; } /** @@ -620,20 +886,16 @@ static const PCIIOMMUOps smmu_ops = { .get_address_space = smmu_find_add_as, }; -IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) +SMMUDevice *smmu_find_sdev(SMMUState *s, uint32_t sid) { uint8_t bus_n, devfn; SMMUPciBus *smmu_bus; - SMMUDevice *smmu; bus_n = PCI_BUS_NUM(sid); smmu_bus = smmu_find_smmu_pcibus(s, bus_n); if (smmu_bus) { devfn = SMMU_PCI_DEVFN(sid); - smmu = smmu_bus->pbdev[devfn]; - if (smmu) { - return &smmu->iommu; - } + return smmu_bus->pbdev[devfn]; } return NULL; } @@ -682,7 +944,12 @@ static void smmu_base_realize(DeviceState *dev, Error **errp) } } -static void smmu_base_reset_hold(Object *obj, ResetType type) +/* + * Make sure the IOMMU is reset in 'exit' phase after + * all outstanding DMA requests have been quiesced during + * the 'enter' or 'hold' reset phases + */ +static void smmu_base_reset_exit(Object *obj, ResetType type) { SMMUState *s = ARM_SMMU(obj); @@ -692,14 +959,13 @@ static void smmu_base_reset_hold(Object *obj, ResetType type) g_hash_table_remove_all(s->iotlb); } -static Property smmu_dev_properties[] = { +static const Property smmu_dev_properties[] = { DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, TYPE_PCI_BUS, PCIBus *), - DEFINE_PROP_END_OF_LIST(), }; -static void smmu_base_class_init(ObjectClass *klass, void *data) +static void smmu_base_class_init(ObjectClass *klass, const void *data) { DeviceClass *dc = DEVICE_CLASS(klass); ResettableClass *rc = RESETTABLE_CLASS(klass); @@ -708,7 +974,7 @@ static void smmu_base_class_init(ObjectClass *klass, void *data) device_class_set_props(dc, smmu_dev_properties); device_class_set_parent_realize(dc, smmu_base_realize, &sbc->parent_realize); - rc->phases.hold = smmu_base_reset_hold; + rc->phases.exit = smmu_base_reset_exit; } static const TypeInfo smmu_base_info = { |