Project import generated by Copybara.
GitOrigin-RevId: d6eb47ca943538580bd75ebe6090596d359fbd28
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index fd75d9e..f7ecb30 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1,7 +1,19 @@
-// SPDX-License-Identifier: GPL-2.0-only
/*
* IOMMU API for ARM architected SMMU implementations.
*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
* Copyright (C) 2013 ARM Limited
*
* Author: Will Deacon <will.deacon@arm.com>
@@ -12,20 +24,19 @@
* - v7/v8 long-descriptor format
* - Non-secure access to the SMMU
* - Context fault reporting
- * - Extended Stream ID (16 bit)
*/
#define pr_fmt(fmt) "arm-smmu: " fmt
-#include <linux/acpi.h>
-#include <linux/acpi_iort.h>
-#include <linux/bitfield.h>
+#include <linux/atomic.h>
#include <linux/delay.h>
#include <linux/dma-iommu.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/of.h>
@@ -34,48 +45,277 @@
#include <linux/of_iommu.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
#include <linux/slab.h>
+#include <linux/spinlock.h>
#include <linux/amba/bus.h>
-#include <linux/fsl/mc.h>
-#include "arm-smmu.h"
+#include "io-pgtable.h"
+
+/* Maximum number of context banks per SMMU */
+#define ARM_SMMU_MAX_CBS 128
+
+/* SMMU global address space */
+#define ARM_SMMU_GR0(smmu) ((smmu)->base)
+#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
/*
- * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
- * global register space are still, in fact, using a hypervisor to mediate it
- * by trapping and emulating register accesses. Sadly, some deployed versions
- * of said trapping code have bugs wherein they go horribly wrong for stores
- * using r31 (i.e. XZR/WZR) as the source register.
+ * SMMU global address space with conditional offset to access secure
+ * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
+ * nsGFSYNR0: 0x450)
*/
-#define QCOM_DUMMY_VAL -1
+#define ARM_SMMU_GR0_NS(smmu) \
+ ((smmu)->base + \
+ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
+ ? 0x400 : 0))
+/*
+ * Some 64-bit registers only make sense to write atomically, but in such
+ * cases all the data relevant to AArch32 formats lies within the lower word,
+ * therefore this actually makes more sense than it might first appear.
+ */
+#ifdef CONFIG_64BIT
+#define smmu_write_atomic_lq writeq_relaxed
+#else
+#define smmu_write_atomic_lq writel_relaxed
+#endif
+
+/* Configuration registers */
+#define ARM_SMMU_GR0_sCR0 0x0
+#define sCR0_CLIENTPD (1 << 0)
+#define sCR0_GFRE (1 << 1)
+#define sCR0_GFIE (1 << 2)
+#define sCR0_GCFGFRE (1 << 4)
+#define sCR0_GCFGFIE (1 << 5)
+#define sCR0_USFCFG (1 << 10)
+#define sCR0_VMIDPNE (1 << 11)
+#define sCR0_PTM (1 << 12)
+#define sCR0_FB (1 << 13)
+#define sCR0_VMID16EN (1 << 31)
+#define sCR0_BSU_SHIFT 14
+#define sCR0_BSU_MASK 0x3
+
+/* Auxiliary Configuration register */
+#define ARM_SMMU_GR0_sACR 0x10
+
+/* Identification registers */
+#define ARM_SMMU_GR0_ID0 0x20
+#define ARM_SMMU_GR0_ID1 0x24
+#define ARM_SMMU_GR0_ID2 0x28
+#define ARM_SMMU_GR0_ID3 0x2c
+#define ARM_SMMU_GR0_ID4 0x30
+#define ARM_SMMU_GR0_ID5 0x34
+#define ARM_SMMU_GR0_ID6 0x38
+#define ARM_SMMU_GR0_ID7 0x3c
+#define ARM_SMMU_GR0_sGFSR 0x48
+#define ARM_SMMU_GR0_sGFSYNR0 0x50
+#define ARM_SMMU_GR0_sGFSYNR1 0x54
+#define ARM_SMMU_GR0_sGFSYNR2 0x58
+
+#define ID0_S1TS (1 << 30)
+#define ID0_S2TS (1 << 29)
+#define ID0_NTS (1 << 28)
+#define ID0_SMS (1 << 27)
+#define ID0_ATOSNS (1 << 26)
+#define ID0_PTFS_NO_AARCH32 (1 << 25)
+#define ID0_PTFS_NO_AARCH32S (1 << 24)
+#define ID0_CTTW (1 << 14)
+#define ID0_NUMIRPT_SHIFT 16
+#define ID0_NUMIRPT_MASK 0xff
+#define ID0_NUMSIDB_SHIFT 9
+#define ID0_NUMSIDB_MASK 0xf
+#define ID0_NUMSMRG_SHIFT 0
+#define ID0_NUMSMRG_MASK 0xff
+
+#define ID1_PAGESIZE (1 << 31)
+#define ID1_NUMPAGENDXB_SHIFT 28
+#define ID1_NUMPAGENDXB_MASK 7
+#define ID1_NUMS2CB_SHIFT 16
+#define ID1_NUMS2CB_MASK 0xff
+#define ID1_NUMCB_SHIFT 0
+#define ID1_NUMCB_MASK 0xff
+
+#define ID2_OAS_SHIFT 4
+#define ID2_OAS_MASK 0xf
+#define ID2_IAS_SHIFT 0
+#define ID2_IAS_MASK 0xf
+#define ID2_UBS_SHIFT 8
+#define ID2_UBS_MASK 0xf
+#define ID2_PTFS_4K (1 << 12)
+#define ID2_PTFS_16K (1 << 13)
+#define ID2_PTFS_64K (1 << 14)
+#define ID2_VMID16 (1 << 15)
+
+#define ID7_MAJOR_SHIFT 4
+#define ID7_MAJOR_MASK 0xf
+
+/* Global TLB invalidation */
+#define ARM_SMMU_GR0_TLBIVMID 0x64
+#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
+#define ARM_SMMU_GR0_TLBIALLH 0x6c
+#define ARM_SMMU_GR0_sTLBGSYNC 0x70
+#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
+#define sTLBGSTATUS_GSACTIVE (1 << 0)
#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
-#define TLB_SPIN_COUNT 10
-#define MSI_IOVA_BASE 0x8000000
-#define MSI_IOVA_LENGTH 0x100000
+/* Stream mapping registers */
+#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
+#define SMR_VALID (1 << 31)
+#define SMR_MASK_SHIFT 16
+#define SMR_ID_SHIFT 0
+
+#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
+#define S2CR_CBNDX_SHIFT 0
+#define S2CR_CBNDX_MASK 0xff
+#define S2CR_TYPE_SHIFT 16
+#define S2CR_TYPE_MASK 0x3
+enum arm_smmu_s2cr_type {
+ S2CR_TYPE_TRANS,
+ S2CR_TYPE_BYPASS,
+ S2CR_TYPE_FAULT,
+};
+
+#define S2CR_PRIVCFG_SHIFT 24
+#define S2CR_PRIVCFG_MASK 0x3
+enum arm_smmu_s2cr_privcfg {
+ S2CR_PRIVCFG_DEFAULT,
+ S2CR_PRIVCFG_DIPAN,
+ S2CR_PRIVCFG_UNPRIV,
+ S2CR_PRIVCFG_PRIV,
+};
+
+/* Context bank attribute registers */
+#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
+#define CBAR_VMID_SHIFT 0
+#define CBAR_VMID_MASK 0xff
+#define CBAR_S1_BPSHCFG_SHIFT 8
+#define CBAR_S1_BPSHCFG_MASK 3
+#define CBAR_S1_BPSHCFG_NSH 3
+#define CBAR_S1_MEMATTR_SHIFT 12
+#define CBAR_S1_MEMATTR_MASK 0xf
+#define CBAR_S1_MEMATTR_WB 0xf
+#define CBAR_TYPE_SHIFT 16
+#define CBAR_TYPE_MASK 0x3
+#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
+#define CBAR_IRPTNDX_SHIFT 24
+#define CBAR_IRPTNDX_MASK 0xff
+
+#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
+#define CBA2R_RW64_32BIT (0 << 0)
+#define CBA2R_RW64_64BIT (1 << 0)
+#define CBA2R_VMID_SHIFT 16
+#define CBA2R_VMID_MASK 0xffff
+
+/* Translation context bank */
+#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
+#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
+
+#define ARM_SMMU_CB_SCTLR 0x0
+#define ARM_SMMU_CB_ACTLR 0x4
+#define ARM_SMMU_CB_RESUME 0x8
+#define ARM_SMMU_CB_TTBCR2 0x10
+#define ARM_SMMU_CB_TTBR0 0x20
+#define ARM_SMMU_CB_TTBR1 0x28
+#define ARM_SMMU_CB_TTBCR 0x30
+#define ARM_SMMU_CB_CONTEXTIDR 0x34
+#define ARM_SMMU_CB_S1_MAIR0 0x38
+#define ARM_SMMU_CB_S1_MAIR1 0x3c
+#define ARM_SMMU_CB_PAR 0x50
+#define ARM_SMMU_CB_FSR 0x58
+#define ARM_SMMU_CB_FAR 0x60
+#define ARM_SMMU_CB_FSYNR0 0x68
+#define ARM_SMMU_CB_S1_TLBIVA 0x600
+#define ARM_SMMU_CB_S1_TLBIASID 0x610
+#define ARM_SMMU_CB_S1_TLBIVAL 0x620
+#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
+#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
+#define ARM_SMMU_CB_ATS1PR 0x800
+#define ARM_SMMU_CB_ATSR 0x8f0
+
+#define SCTLR_S1_ASIDPNE (1 << 12)
+#define SCTLR_CFCFG (1 << 7)
+#define SCTLR_CFIE (1 << 6)
+#define SCTLR_CFRE (1 << 5)
+#define SCTLR_E (1 << 4)
+#define SCTLR_AFE (1 << 2)
+#define SCTLR_TRE (1 << 1)
+#define SCTLR_M (1 << 0)
+
+#define ARM_MMU500_ACTLR_CPRE (1 << 1)
+
+#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
+
+#define CB_PAR_F (1 << 0)
+
+#define ATSR_ACTIVE (1 << 0)
+
+#define RESUME_RETRY (0 << 0)
+#define RESUME_TERMINATE (1 << 0)
+
+#define TTBCR2_SEP_SHIFT 15
+#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
+
+#define TTBRn_ASID_SHIFT 48
+
+#define FSR_MULTI (1 << 31)
+#define FSR_SS (1 << 30)
+#define FSR_UUT (1 << 8)
+#define FSR_ASF (1 << 7)
+#define FSR_TLBLKF (1 << 6)
+#define FSR_TLBMCF (1 << 5)
+#define FSR_EF (1 << 4)
+#define FSR_PF (1 << 3)
+#define FSR_AFF (1 << 2)
+#define FSR_TF (1 << 1)
+
+#define FSR_IGN (FSR_AFF | FSR_ASF | \
+ FSR_TLBMCF | FSR_TLBLKF)
+#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
+ FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
+
+#define FSYNR0_WNR (1 << 4)
static int force_stage;
module_param(force_stage, int, S_IRUGO);
MODULE_PARM_DESC(force_stage,
"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
-static bool disable_bypass =
- IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
+static bool disable_bypass;
module_param(disable_bypass, bool, S_IRUGO);
MODULE_PARM_DESC(disable_bypass,
"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
+enum arm_smmu_arch_version {
+ ARM_SMMU_V1,
+ ARM_SMMU_V1_64K,
+ ARM_SMMU_V2,
+};
+
+enum arm_smmu_implementation {
+ GENERIC_SMMU,
+ ARM_MMU500,
+ CAVIUM_SMMUV2,
+ QCOM_SMMUV2,
+};
+
+struct arm_smmu_s2cr {
+ struct iommu_group *group;
+ int count;
+ enum arm_smmu_s2cr_type type;
+ enum arm_smmu_s2cr_privcfg privcfg;
+ u8 cbndx;
+};
+
#define s2cr_init_val (struct arm_smmu_s2cr){ \
.type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
}
-struct arm_smmu_cb {
- u64 ttbr[2];
- u32 tcr[2];
- u32 mair[2];
- struct arm_smmu_cfg *cfg;
+struct arm_smmu_smr {
+ u16 mask;
+ u16 id;
+ bool valid;
};
struct arm_smmu_master_cfg {
@@ -90,32 +330,122 @@ struct arm_smmu_master_cfg {
#define for_each_cfg_sme(fw, i, idx) \
for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
+struct arm_smmu_device {
+ struct device *dev;
+
+ void __iomem *base;
+ unsigned long size;
+ unsigned long pgshift;
+
+#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
+#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
+#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
+#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
+#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
+#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
+#define ARM_SMMU_FEAT_VMID16 (1 << 6)
+#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
+#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
+#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
+#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
+#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
+ u32 features;
+
+#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
+ u32 options;
+ enum arm_smmu_arch_version version;
+ enum arm_smmu_implementation model;
+
+ u32 num_context_banks;
+ u32 num_s2_context_banks;
+ DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
+ atomic_t irptndx;
+
+ u32 num_mapping_groups;
+ u16 streamid_mask;
+ u16 smr_mask_mask;
+ struct arm_smmu_smr *smrs;
+ struct arm_smmu_s2cr *s2crs;
+ struct mutex stream_map_mutex;
+
+ unsigned long va_size;
+ unsigned long ipa_size;
+ unsigned long pa_size;
+ unsigned long pgsize_bitmap;
+
+ u32 num_global_irqs;
+ u32 num_context_irqs;
+ unsigned int *irqs;
+
+ u32 cavium_id_base; /* Specific to Cavium */
+};
+
+enum arm_smmu_context_fmt {
+ ARM_SMMU_CTX_FMT_NONE,
+ ARM_SMMU_CTX_FMT_AARCH64,
+ ARM_SMMU_CTX_FMT_AARCH32_L,
+ ARM_SMMU_CTX_FMT_AARCH32_S,
+};
+
+struct arm_smmu_cfg {
+ u8 cbndx;
+ u8 irptndx;
+ u32 cbar;
+ enum arm_smmu_context_fmt fmt;
+};
+#define INVALID_IRPTNDX 0xff
+
+#define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
+#define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
+
+enum arm_smmu_domain_stage {
+ ARM_SMMU_DOMAIN_S1 = 0,
+ ARM_SMMU_DOMAIN_S2,
+ ARM_SMMU_DOMAIN_NESTED,
+};
+
+struct arm_smmu_domain {
+ struct arm_smmu_device *smmu;
+ struct io_pgtable_ops *pgtbl_ops;
+ spinlock_t pgtbl_lock;
+ struct arm_smmu_cfg cfg;
+ enum arm_smmu_domain_stage stage;
+ struct mutex init_mutex; /* Protects smmu pointer */
+ struct iommu_domain domain;
+};
+
+struct arm_smmu_option_prop {
+ u32 opt;
+ const char *prop;
+};
+
+static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
+
static bool using_legacy_binding, using_generic_binding;
-static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
-{
- if (pm_runtime_enabled(smmu->dev))
- return pm_runtime_resume_and_get(smmu->dev);
-
- return 0;
-}
-
-static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
-{
- if (pm_runtime_enabled(smmu->dev))
- pm_runtime_put(smmu->dev);
-}
+static struct arm_smmu_option_prop arm_smmu_options[] = {
+ { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
+ { 0, NULL},
+};
static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
{
return container_of(dom, struct arm_smmu_domain, domain);
}
-static struct platform_driver arm_smmu_driver;
-static struct iommu_ops arm_smmu_ops;
+static void parse_driver_options(struct arm_smmu_device *smmu)
+{
+ int i = 0;
-#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
-static int arm_smmu_bus_init(struct iommu_ops *ops);
+ do {
+ if (of_property_read_bool(smmu->dev->of_node,
+ arm_smmu_options[i].prop)) {
+ smmu->options |= arm_smmu_options[i].opt;
+ dev_notice(smmu->dev, "option %s\n",
+ arm_smmu_options[i].prop);
+ }
+ } while (arm_smmu_options[++i].opt);
+}
static struct device_node *dev_get_dev_node(struct device *dev)
{
@@ -143,7 +473,7 @@ static int __find_legacy_master_phandle(struct device *dev, void *data)
int err;
of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
- "#stream-id-cells", -1)
+ "#stream-id-cells", 0)
if (it->node == np) {
*(void **)data = dev;
return 1;
@@ -152,6 +482,9 @@ static int __find_legacy_master_phandle(struct device *dev, void *data)
return err == -ENOENT ? 0 : err;
}
+static struct platform_driver arm_smmu_driver;
+static struct iommu_ops arm_smmu_ops;
+
static int arm_smmu_register_legacy_master(struct device *dev,
struct arm_smmu_device **smmu)
{
@@ -203,40 +536,9 @@ static int arm_smmu_register_legacy_master(struct device *dev,
return err;
}
-/*
- * With the legacy DT binding in play, we have no guarantees about
- * probe order, but then we're also not doing default domains, so we can
- * delay setting bus ops until we're sure every possible SMMU is ready,
- * and that way ensure that no add_device() calls get missed.
- */
-static int arm_smmu_legacy_bus_init(void)
+static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
{
- if (using_legacy_binding)
- return arm_smmu_bus_init(&arm_smmu_ops);
- return 0;
-}
-device_initcall_sync(arm_smmu_legacy_bus_init);
-#else
-static int arm_smmu_register_legacy_master(struct device *dev,
- struct arm_smmu_device **smmu)
-{
- return -ENODEV;
-}
-#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
-
-static int __arm_smmu_alloc_cb(struct arm_smmu_device *smmu, int start,
- struct device *dev)
-{
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
- unsigned long *map = smmu->context_map;
- int end = smmu->num_context_banks;
int idx;
- int i;
-
- for_each_cfg_sme(fwspec, i, idx) {
- if (smmu->s2crs[idx].pinned)
- return smmu->s2crs[idx].cbndx;
- }
do {
idx = find_next_zero_bit(map, end, start);
@@ -253,233 +555,124 @@ static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
}
/* Wait for any pending TLB invalidations to complete */
-static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
- int sync, int status)
+static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
{
- unsigned int spin_cnt, delay;
- u32 reg;
+ int count = 0;
+ void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
- arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
- for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
- for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
- reg = arm_smmu_readl(smmu, page, status);
- if (!(reg & sTLBGSTATUS_GSACTIVE))
- return;
- cpu_relax();
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
+ while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
+ & sTLBGSTATUS_GSACTIVE) {
+ cpu_relax();
+ if (++count == TLB_LOOP_TIMEOUT) {
+ dev_err_ratelimited(smmu->dev,
+ "TLB sync timed out -- SMMU may be deadlocked\n");
+ return;
}
- udelay(delay);
+ udelay(1);
}
- dev_err_ratelimited(smmu->dev,
- "TLB sync timed out -- SMMU may be deadlocked\n");
}
-static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&smmu->global_sync_lock, flags);
- __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
- ARM_SMMU_GR0_sTLBGSTATUS);
- spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
-}
-
-static void arm_smmu_tlb_sync_context(void *cookie)
+static void arm_smmu_tlb_sync(void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
- unsigned long flags;
-
- spin_lock_irqsave(&smmu_domain->cb_lock, flags);
- __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
- ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
- spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
+ __arm_smmu_tlb_sync(smmu_domain->smmu);
}
-static void arm_smmu_tlb_sync_vmid(void *cookie)
+static void arm_smmu_tlb_inv_context(void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
-
- arm_smmu_tlb_sync_global(smmu_domain->smmu);
-}
-
-static void arm_smmu_tlb_inv_context_s1(void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- /*
- * The TLBI write may be relaxed, so ensure that PTEs cleared by the
- * current CPU are visible beforehand.
- */
- wmb();
- arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
- ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
- arm_smmu_tlb_sync_context(cookie);
-}
-
-static void arm_smmu_tlb_inv_context_s2(void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
-
- /* See above */
- wmb();
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
- arm_smmu_tlb_sync_global(smmu);
-}
-
-static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
- size_t granule, bool leaf, void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- int reg, idx = cfg->cbndx;
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+ void __iomem *base;
- if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
- wmb();
-
- reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
-
- if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
- iova = (iova >> 12) << 12;
- iova |= cfg->asid;
- do {
- arm_smmu_cb_write(smmu, idx, reg, iova);
- iova += granule;
- } while (size -= granule);
+ if (stage1) {
+ base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
+ base + ARM_SMMU_CB_S1_TLBIASID);
} else {
+ base = ARM_SMMU_GR0(smmu);
+ writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
+ base + ARM_SMMU_GR0_TLBIVMID);
+ }
+
+ __arm_smmu_tlb_sync(smmu);
+}
+
+static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
+ size_t granule, bool leaf, void *cookie)
+{
+ struct arm_smmu_domain *smmu_domain = cookie;
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+ void __iomem *reg;
+
+ if (stage1) {
+ reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
+
+ if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
+ iova &= ~12UL;
+ iova |= ARM_SMMU_CB_ASID(smmu, cfg);
+ do {
+ writel_relaxed(iova, reg);
+ iova += granule;
+ } while (size -= granule);
+ } else {
+ iova >>= 12;
+ iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
+ do {
+ writeq_relaxed(iova, reg);
+ iova += granule >> 12;
+ } while (size -= granule);
+ }
+ } else if (smmu->version == ARM_SMMU_V2) {
+ reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
+ ARM_SMMU_CB_S2_TLBIIPAS2;
iova >>= 12;
- iova |= (u64)cfg->asid << 48;
do {
- arm_smmu_cb_writeq(smmu, idx, reg, iova);
+ smmu_write_atomic_lq(iova, reg);
iova += granule >> 12;
} while (size -= granule);
+ } else {
+ reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
+ writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
}
}
-static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
- size_t granule, bool leaf, void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
- int reg, idx = smmu_domain->cfg.cbndx;
-
- if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
- wmb();
-
- reg = leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
- iova >>= 12;
- do {
- if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
- arm_smmu_cb_writeq(smmu, idx, reg, iova);
- else
- arm_smmu_cb_write(smmu, idx, reg, iova);
- iova += granule >> 12;
- } while (size -= granule);
-}
-
-/*
- * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
- * almost negligible, but the benefit of getting the first one in as far ahead
- * of the sync as possible is significant, hence we don't just make this a
- * no-op and set .tlb_sync to arm_smmu_tlb_inv_context_s2() as you might think.
- */
-static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
- size_t granule, bool leaf, void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
-
- if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
- wmb();
-
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
-}
-
-static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
- size_t granule, void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
-
- ops->tlb_inv_range(iova, size, granule, false, cookie);
- ops->tlb_sync(cookie);
-}
-
-static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
- size_t granule, void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
-
- ops->tlb_inv_range(iova, size, granule, true, cookie);
- ops->tlb_sync(cookie);
-}
-
-static void arm_smmu_tlb_add_page(struct iommu_iotlb_gather *gather,
- unsigned long iova, size_t granule,
- void *cookie)
-{
- struct arm_smmu_domain *smmu_domain = cookie;
- const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
-
- ops->tlb_inv_range(iova, granule, granule, true, cookie);
-}
-
-static const struct arm_smmu_flush_ops arm_smmu_s1_tlb_ops = {
- .tlb = {
- .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
- .tlb_flush_walk = arm_smmu_tlb_inv_walk,
- .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
- .tlb_add_page = arm_smmu_tlb_add_page,
- },
- .tlb_inv_range = arm_smmu_tlb_inv_range_s1,
- .tlb_sync = arm_smmu_tlb_sync_context,
-};
-
-static const struct arm_smmu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
- .tlb = {
- .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
- .tlb_flush_walk = arm_smmu_tlb_inv_walk,
- .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
- .tlb_add_page = arm_smmu_tlb_add_page,
- },
- .tlb_inv_range = arm_smmu_tlb_inv_range_s2,
- .tlb_sync = arm_smmu_tlb_sync_context,
-};
-
-static const struct arm_smmu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
- .tlb = {
- .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
- .tlb_flush_walk = arm_smmu_tlb_inv_walk,
- .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
- .tlb_add_page = arm_smmu_tlb_add_page,
- },
- .tlb_inv_range = arm_smmu_tlb_inv_vmid_nosync,
- .tlb_sync = arm_smmu_tlb_sync_vmid,
+static struct iommu_gather_ops arm_smmu_gather_ops = {
+ .tlb_flush_all = arm_smmu_tlb_inv_context,
+ .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
+ .tlb_sync = arm_smmu_tlb_sync,
};
static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
{
- u32 fsr, fsynr, cbfrsynra;
+ u32 fsr, fsynr;
unsigned long iova;
struct iommu_domain *domain = dev;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct arm_smmu_device *smmu = smmu_domain->smmu;
- int idx = smmu_domain->cfg.cbndx;
+ void __iomem *cb_base;
- fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
+
if (!(fsr & FSR_FAULT))
return IRQ_NONE;
- fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
- iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
- cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
+ fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
+ iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
dev_err_ratelimited(smmu->dev,
- "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
- fsr, iova, fsynr, cbfrsynra, idx);
+ "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
+ fsr, iova, fsynr, cfg->cbndx);
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
+ writel(fsr, cb_base + ARM_SMMU_CB_FSR);
return IRQ_HANDLED;
}
@@ -487,11 +680,12 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
{
u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
struct arm_smmu_device *smmu = dev;
+ void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
- gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
- gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
- gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
- gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
+ gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
+ gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
+ gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
+ gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
if (!gfsr)
return IRQ_NONE;
@@ -502,147 +696,119 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
"\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
gfsr, gfsynr0, gfsynr1, gfsynr2);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
+ writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
return IRQ_HANDLED;
}
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
struct io_pgtable_cfg *pgtbl_cfg)
{
- struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
- bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
-
- cb->cfg = cfg;
-
- /* TCR */
- if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
- } else {
- cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
- cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
- cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
- cb->tcr[1] |= TCR2_AS;
- }
- } else {
- cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
- }
-
- /* TTBRs */
- if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
- cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
- } else {
- cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
- cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
- cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
- cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
- }
- } else {
- cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
- }
-
- /* MAIRs (stage-1 only) */
- if (stage1) {
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
- cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
- } else {
- cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
- cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
- }
- }
-}
-
-static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
-{
- u32 reg;
+ u32 reg, reg2;
+ u64 reg64;
bool stage1;
- struct arm_smmu_cb *cb = &smmu->cbs[idx];
- struct arm_smmu_cfg *cfg = cb->cfg;
+ struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ void __iomem *cb_base, *gr1_base;
- /* Unassigned context banks only need disabling */
- if (!cfg) {
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
- return;
- }
-
+ gr1_base = ARM_SMMU_GR1(smmu);
stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
- /* CBA2R */
if (smmu->version > ARM_SMMU_V1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
- reg = CBA2R_VA64;
+ reg = CBA2R_RW64_64BIT;
else
- reg = 0;
+ reg = CBA2R_RW64_32BIT;
/* 16-bit VMIDs live in CBA2R */
if (smmu->features & ARM_SMMU_FEAT_VMID16)
- reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
+ reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
- arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
+ writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
}
/* CBAR */
- reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
+ reg = cfg->cbar;
if (smmu->version < ARM_SMMU_V2)
- reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
+ reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
/*
* Use the weakest shareability/memory types, so they are
* overridden by the ttbcr/pte.
*/
if (stage1) {
- reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
- FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
+ reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
+ (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
/* 8-bit VMIDs live in CBAR */
- reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
+ reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
}
- arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
-
- /*
- * TCR
- * We must write this before the TTBRs, since it determines the
- * access behaviour of some fields (in particular, ASID[15:8]).
- */
- if (stage1 && smmu->version > ARM_SMMU_V1)
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
+ writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
/* TTBRs */
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
+ if (stage1) {
+ u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
+
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
+ reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
+ writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
+ } else {
+ reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+ reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
+ writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
+ reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
+ reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
+ writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
+ }
} else {
- arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
- if (stage1)
- arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
- cb->ttbr[1]);
+ reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
+ writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
}
+ /* TTBCR */
+ if (stage1) {
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ reg = pgtbl_cfg->arm_v7s_cfg.tcr;
+ reg2 = 0;
+ } else {
+ reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
+ reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
+ reg2 |= TTBCR2_SEP_UPSTREAM;
+ }
+ if (smmu->version > ARM_SMMU_V1)
+ writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
+ } else {
+ reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
+ }
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
+
/* MAIRs (stage-1 only) */
if (stage1) {
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
+ if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+ reg = pgtbl_cfg->arm_v7s_cfg.prrr;
+ reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
+ } else {
+ reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
+ reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
+ }
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
+ writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
}
/* SCTLR */
reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
if (stage1)
reg |= SCTLR_S1_ASIDPNE;
- if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
- reg |= SCTLR_E;
-
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
+#ifdef __BIG_ENDIAN
+ reg |= SCTLR_E;
+#endif
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
}
static int arm_smmu_init_domain_context(struct iommu_domain *domain,
- struct arm_smmu_device *smmu,
- struct device *dev)
+ struct arm_smmu_device *smmu)
{
int irq, start, ret = 0;
unsigned long ias, oas;
@@ -656,12 +822,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
if (smmu_domain->smmu)
goto out_unlock;
- if (domain->type == IOMMU_DOMAIN_IDENTITY) {
- smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
- smmu_domain->smmu = smmu;
- goto out_unlock;
- }
-
/*
* Mapping the requested stage onto what we support is surprisingly
* complicated, mainly because the spec allows S1+S2 SMMUs without
@@ -728,7 +888,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
ias = min(ias, 32UL);
oas = min(oas, 32UL);
}
- smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
break;
case ARM_SMMU_DOMAIN_NESTED:
/*
@@ -747,16 +906,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
ias = min(ias, 40UL);
oas = min(oas, 40UL);
}
- if (smmu->version == ARM_SMMU_V2)
- smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
- else
- smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
break;
default:
ret = -EINVAL;
goto out_unlock;
}
- ret = __arm_smmu_alloc_cb(smmu, start, dev);
+
+ ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
+ smmu->num_context_banks);
if (ret < 0)
goto out_unlock;
@@ -768,30 +925,15 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
cfg->irptndx = cfg->cbndx;
}
- if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
- cfg->vmid = cfg->cbndx + 1;
- else
- cfg->asid = cfg->cbndx;
-
- smmu_domain->smmu = smmu;
- if (smmu->impl && smmu->impl->init_context) {
- ret = smmu->impl->init_context(smmu_domain);
- if (ret)
- goto out_unlock;
- }
-
pgtbl_cfg = (struct io_pgtable_cfg) {
.pgsize_bitmap = smmu->pgsize_bitmap,
.ias = ias,
.oas = oas,
- .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
- .tlb = &smmu_domain->flush_ops->tlb,
+ .tlb = &arm_smmu_gather_ops,
.iommu_dev = smmu->dev,
};
- if (smmu_domain->non_strict)
- pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
-
+ smmu_domain->smmu = smmu;
pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
if (!pgtbl_ops) {
ret = -ENOMEM;
@@ -805,7 +947,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
/* Initialise the context bank with our page table cfg */
arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
- arm_smmu_write_context_bank(smmu, cfg->cbndx);
/*
* Request context fault interrupt. Do this last to avoid the
@@ -827,7 +968,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
return 0;
out_clear_smmu:
- __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
smmu_domain->smmu = NULL;
out_unlock:
mutex_unlock(&smmu_domain->init_mutex);
@@ -839,21 +979,18 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
- int ret, irq;
+ void __iomem *cb_base;
+ int irq;
- if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
- return;
-
- ret = arm_smmu_rpm_get(smmu);
- if (ret < 0)
+ if (!smmu)
return;
/*
* Disable the context bank and free the page tables before freeing
* it.
*/
- smmu->cbs[cfg->cbndx].cfg = NULL;
- arm_smmu_write_context_bank(smmu, cfg->cbndx);
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
if (cfg->irptndx != INVALID_IRPTNDX) {
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
@@ -862,17 +999,13 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
free_io_pgtable_ops(smmu_domain->pgtbl_ops);
__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
-
- arm_smmu_rpm_put(smmu);
}
static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
{
struct arm_smmu_domain *smmu_domain;
- if (type != IOMMU_DOMAIN_UNMANAGED &&
- type != IOMMU_DOMAIN_DMA &&
- type != IOMMU_DOMAIN_IDENTITY)
+ if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
return NULL;
/*
* Allocate the domain and initialise some of its data structures.
@@ -890,7 +1023,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
}
mutex_init(&smmu_domain->init_mutex);
- spin_lock_init(&smmu_domain->cb_lock);
+ spin_lock_init(&smmu_domain->pgtbl_lock);
return &smmu_domain->domain;
}
@@ -911,24 +1044,21 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
{
struct arm_smmu_smr *smr = smmu->smrs + idx;
- u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
+ u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
- if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
+ if (smr->valid)
reg |= SMR_VALID;
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
+ writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
}
static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
{
struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
- u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
- FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
- FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
+ u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
+ (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
+ (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
- if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
- smmu->smrs[idx].valid)
- reg |= S2CR_EXIDVALID;
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
+ writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
}
static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
@@ -938,52 +1068,6 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
arm_smmu_write_smr(smmu, idx);
}
-/*
- * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
- * should be called after sCR0 is written.
- */
-static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
-{
- u32 s2cr;
- u32 smr;
- int idx;
-
- if (!smmu->smrs)
- return;
-
- for (idx = 0; idx < smmu->num_mapping_groups; idx++) {
- if (smmu->features & ARM_SMMU_FEAT_EXIDS) {
- s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(idx));
- if (!FIELD_GET(S2CR_EXIDVALID, s2cr))
- break;
- } else {
- smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(idx));
- if (!FIELD_GET(SMR_VALID, smr))
- break;
- }
- }
-
- if (idx == smmu->num_mapping_groups) {
- dev_err(smmu->dev, "Unable to compute streamid_mask\n");
- return;
- }
-
- /*
- * SMR.ID bits may not be preserved if the corresponding MASK
- * bits are set, so check each one separately. We can reject
- * masters later if they try to claim IDs outside these masks.
- */
- smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), smr);
- smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(idx));
- smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
-
- smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), smr);
- smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(idx));
- smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
-}
-
static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
{
struct arm_smmu_smr *smrs = smmu->smrs;
@@ -1028,26 +1112,19 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
{
- bool pinned = smmu->s2crs[idx].pinned;
- u8 cbndx = smmu->s2crs[idx].cbndx;;
-
if (--smmu->s2crs[idx].count)
return false;
smmu->s2crs[idx] = s2cr_init_val;
- if (pinned) {
- smmu->s2crs[idx].pinned = true;
- smmu->s2crs[idx].cbndx = cbndx;
- } else if (smmu->smrs) {
+ if (smmu->smrs)
smmu->smrs[idx].valid = false;
- }
return true;
}
static int arm_smmu_master_alloc_smes(struct device *dev)
{
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
struct arm_smmu_device *smmu = cfg->smmu;
struct arm_smmu_smr *smrs = smmu->smrs;
@@ -1057,8 +1134,8 @@ static int arm_smmu_master_alloc_smes(struct device *dev)
mutex_lock(&smmu->stream_map_mutex);
/* Figure out a viable stream map entry allocation */
for_each_cfg_sme(fwspec, i, idx) {
- u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
- u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
+ u16 sid = fwspec->ids[i];
+ u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
if (idx != INVALID_SMENDX) {
ret = -EEXIST;
@@ -1126,15 +1203,10 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
{
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_s2cr *s2cr = smmu->s2crs;
+ enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
u8 cbndx = smmu_domain->cfg.cbndx;
- enum arm_smmu_s2cr_type type;
int i, idx;
- if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
- type = S2CR_TYPE_BYPASS;
- else
- type = S2CR_TYPE_TRANS;
-
for_each_cfg_sme(fwspec, i, idx) {
if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
continue;
@@ -1150,7 +1222,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
int ret;
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct arm_smmu_device *smmu;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -1170,16 +1242,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return -ENODEV;
smmu = fwspec_smmu(fwspec);
-
- ret = arm_smmu_rpm_get(smmu);
+ /* Ensure that the domain is finalised */
+ ret = arm_smmu_init_domain_context(domain, smmu);
if (ret < 0)
return ret;
- /* Ensure that the domain is finalised */
- ret = arm_smmu_init_domain_context(domain, smmu, dev);
- if (ret < 0)
- goto rpm_put;
-
/*
* Sanity check the domain. We don't support domains across
* different SMMUs.
@@ -1188,77 +1255,47 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
dev_err(dev,
"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
- ret = -EINVAL;
- goto rpm_put;
+ return -EINVAL;
}
/* Looks ok, so add the device to the domain */
- ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
-
-rpm_put:
- arm_smmu_rpm_put(smmu);
- return ret;
+ return arm_smmu_domain_add_master(smmu_domain, fwspec);
}
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
+ phys_addr_t paddr, size_t size, int prot)
{
- struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
- struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
int ret;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
if (!ops)
return -ENODEV;
- arm_smmu_rpm_get(smmu);
+ spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
ret = ops->map(ops, iova, paddr, size, prot);
- arm_smmu_rpm_put(smmu);
-
+ spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
return ret;
}
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
+ size_t size)
{
- struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
- struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
size_t ret;
+ unsigned long flags;
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+ struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
if (!ops)
return 0;
- arm_smmu_rpm_get(smmu);
- ret = ops->unmap(ops, iova, size, gather);
- arm_smmu_rpm_put(smmu);
-
+ spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
+ ret = ops->unmap(ops, iova, size);
+ spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
return ret;
}
-static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
-{
- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- struct arm_smmu_device *smmu = smmu_domain->smmu;
-
- if (smmu_domain->flush_ops) {
- arm_smmu_rpm_get(smmu);
- smmu_domain->flush_ops->tlb.tlb_flush_all(smmu_domain);
- arm_smmu_rpm_put(smmu);
- }
-}
-
-static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
- struct iommu_iotlb_gather *gather)
-{
- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- struct arm_smmu_device *smmu = smmu_domain->smmu;
-
- if (smmu_domain->flush_ops) {
- arm_smmu_rpm_get(smmu);
- smmu_domain->flush_ops->tlb_sync(smmu_domain);
- arm_smmu_rpm_put(smmu);
- }
-}
-
static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
dma_addr_t iova)
{
@@ -1267,66 +1304,60 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
struct device *dev = smmu->dev;
- void __iomem *reg;
+ void __iomem *cb_base;
u32 tmp;
u64 phys;
- unsigned long va, flags;
- int ret, idx = cfg->cbndx;
- phys_addr_t addr = 0;
+ unsigned long va;
- ret = arm_smmu_rpm_get(smmu);
- if (ret < 0)
- return 0;
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
- spin_lock_irqsave(&smmu_domain->cb_lock, flags);
+ /* ATS1 registers can only be written atomically */
va = iova & ~0xfffUL;
- if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
- arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
- else
- arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
+ if (smmu->version == ARM_SMMU_V2)
+ smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
+ else /* Register is only 32-bit in v1 */
+ writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
- reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
- if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
- spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
+ if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
+ !(tmp & ATSR_ACTIVE), 5, 50)) {
dev_err(dev,
"iova to phys timed out on %pad. Falling back to software table walk.\n",
&iova);
- arm_smmu_rpm_put(smmu);
return ops->iova_to_phys(ops, iova);
}
- phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
- spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
+ phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
if (phys & CB_PAR_F) {
dev_err(dev, "translation fault!\n");
dev_err(dev, "PAR = 0x%llx\n", phys);
- goto out;
+ return 0;
}
- addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
-out:
- arm_smmu_rpm_put(smmu);
-
- return addr;
+ return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
}
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
+ phys_addr_t ret;
+ unsigned long flags;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
-
- if (domain->type == IOMMU_DOMAIN_IDENTITY)
- return iova;
+ struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
if (!ops)
return 0;
+ spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
- smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
- return arm_smmu_iova_to_phys_hard(domain, iova);
+ smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+ ret = arm_smmu_iova_to_phys_hard(domain, iova);
+ } else {
+ ret = ops->iova_to_phys(ops, iova);
+ }
- return ops->iova_to_phys(ops, iova);
+ spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
+
+ return ret;
}
static bool arm_smmu_capable(enum iommu_cap cap)
@@ -1338,6 +1369,8 @@ static bool arm_smmu_capable(enum iommu_cap cap)
* requests.
*/
return true;
+ case IOMMU_CAP_INTR_REMAP:
+ return true; /* MSIs are just memory writes */
case IOMMU_CAP_NOEXEC:
return true;
default:
@@ -1345,11 +1378,15 @@ static bool arm_smmu_capable(enum iommu_cap cap)
}
}
-static
-struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
+static int arm_smmu_match_node(struct device *dev, void *data)
{
- struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
- fwnode);
+ return dev->of_node == data;
+}
+
+static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
+{
+ struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
+ np, arm_smmu_match_node);
put_device(dev);
return dev ? dev_get_drvdata(dev) : NULL;
}
@@ -1358,30 +1395,24 @@ static int arm_smmu_add_device(struct device *dev)
{
struct arm_smmu_device *smmu;
struct arm_smmu_master_cfg *cfg;
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
int i, ret;
if (using_legacy_binding) {
ret = arm_smmu_register_legacy_master(dev, &smmu);
-
- /*
- * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
- * will allocate/initialise a new one. Thus we need to update fwspec for
- * later use.
- */
- fwspec = dev_iommu_fwspec_get(dev);
+ fwspec = dev->iommu_fwspec;
if (ret)
goto out_free;
} else if (fwspec && fwspec->ops == &arm_smmu_ops) {
- smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
+ smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
} else {
return -ENODEV;
}
ret = -EINVAL;
for (i = 0; i < fwspec->num_ids; i++) {
- u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
- u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
+ u16 sid = fwspec->ids[i];
+ u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
if (sid & ~smmu->streamid_mask) {
dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
@@ -1390,7 +1421,7 @@ static int arm_smmu_add_device(struct device *dev)
}
if (mask & ~smmu->smr_mask_mask) {
dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
- mask, smmu->smr_mask_mask);
+ sid, smmu->smr_mask_mask);
goto out_free;
}
}
@@ -1406,52 +1437,27 @@ static int arm_smmu_add_device(struct device *dev)
while (i--)
cfg->smendx[i] = INVALID_SMENDX;
- ret = arm_smmu_rpm_get(smmu);
- if (ret < 0)
- goto out_cfg_free;
-
ret = arm_smmu_master_alloc_smes(dev);
- arm_smmu_rpm_put(smmu);
-
if (ret)
- goto out_cfg_free;
-
- iommu_device_link(&smmu->iommu, dev);
-
- device_link_add(dev, smmu->dev,
- DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
+ goto out_free;
return 0;
-out_cfg_free:
- kfree(cfg);
out_free:
+ if (fwspec)
+ kfree(fwspec->iommu_priv);
iommu_fwspec_free(dev);
return ret;
}
static void arm_smmu_remove_device(struct device *dev)
{
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
- struct arm_smmu_master_cfg *cfg;
- struct arm_smmu_device *smmu;
- int ret;
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
if (!fwspec || fwspec->ops != &arm_smmu_ops)
return;
- cfg = fwspec->iommu_priv;
- smmu = cfg->smmu;
-
- ret = arm_smmu_rpm_get(smmu);
- if (ret < 0)
- return;
-
- iommu_device_unlink(&smmu->iommu, dev);
arm_smmu_master_free_smes(fwspec);
-
- arm_smmu_rpm_put(smmu);
-
iommu_group_remove_device(dev);
kfree(fwspec->iommu_priv);
iommu_fwspec_free(dev);
@@ -1459,7 +1465,7 @@ static void arm_smmu_remove_device(struct device *dev)
static struct iommu_group *arm_smmu_device_group(struct device *dev)
{
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
struct iommu_group *group = NULL;
int i, idx;
@@ -1473,12 +1479,10 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
}
if (group)
- return iommu_group_ref_get(group);
+ return group;
if (dev_is_pci(dev))
group = pci_device_group(dev);
- else if (dev_is_fsl_mc(dev))
- group = fsl_mc_device_group(dev);
else
group = generic_device_group(dev);
@@ -1490,27 +1494,12 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- switch(domain->type) {
- case IOMMU_DOMAIN_UNMANAGED:
- switch (attr) {
- case DOMAIN_ATTR_NESTING:
- *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
- return 0;
- default:
- return -ENODEV;
- }
- break;
- case IOMMU_DOMAIN_DMA:
- switch (attr) {
- case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
- *(int *)data = smmu_domain->non_strict;
- return 0;
- default:
- return -ENODEV;
- }
- break;
+ switch (attr) {
+ case DOMAIN_ATTR_NESTING:
+ *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
+ return 0;
default:
- return -EINVAL;
+ return -ENODEV;
}
}
@@ -1522,36 +1511,23 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
mutex_lock(&smmu_domain->init_mutex);
- switch(domain->type) {
- case IOMMU_DOMAIN_UNMANAGED:
- switch (attr) {
- case DOMAIN_ATTR_NESTING:
- if (smmu_domain->smmu) {
- ret = -EPERM;
- goto out_unlock;
- }
+ switch (attr) {
+ case DOMAIN_ATTR_NESTING:
+ if (smmu_domain->smmu) {
+ ret = -EPERM;
+ goto out_unlock;
+ }
- if (*(int *)data)
- smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
- else
- smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
- break;
- default:
- ret = -ENODEV;
- }
- break;
- case IOMMU_DOMAIN_DMA:
- switch (attr) {
- case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
- smmu_domain->non_strict = *(int *)data;
- break;
- default:
- ret = -ENODEV;
- }
+ if (*(int *)data)
+ smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
+ else
+ smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
+
break;
default:
- ret = -EINVAL;
+ ret = -ENODEV;
}
+
out_unlock:
mutex_unlock(&smmu_domain->init_mutex);
return ret;
@@ -1559,44 +1535,17 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
{
- u32 mask, fwid = 0;
+ u32 fwid = 0;
if (args->args_count > 0)
- fwid |= FIELD_PREP(SMR_ID, args->args[0]);
+ fwid |= (u16)args->args[0];
if (args->args_count > 1)
- fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
- else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
- fwid |= FIELD_PREP(SMR_MASK, mask);
+ fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
return iommu_fwspec_add_ids(dev, &fwid, 1);
}
-static void arm_smmu_get_resv_regions(struct device *dev,
- struct list_head *head)
-{
- struct iommu_resv_region *region;
- int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
-
- region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
- prot, IOMMU_RESV_SW_MSI);
- if (!region)
- return;
-
- list_add_tail(®ion->list, head);
-
- iommu_dma_get_resv_regions(dev, head);
-}
-
-static void arm_smmu_put_resv_regions(struct device *dev,
- struct list_head *head)
-{
- struct iommu_resv_region *entry, *next;
-
- list_for_each_entry_safe(entry, next, head, list)
- kfree(entry);
-}
-
static struct iommu_ops arm_smmu_ops = {
.capable = arm_smmu_capable,
.domain_alloc = arm_smmu_domain_alloc,
@@ -1604,8 +1553,7 @@ static struct iommu_ops arm_smmu_ops = {
.attach_dev = arm_smmu_attach_dev,
.map = arm_smmu_map,
.unmap = arm_smmu_unmap,
- .flush_iotlb_all = arm_smmu_flush_iotlb_all,
- .iotlb_sync = arm_smmu_iotlb_sync,
+ .map_sg = default_iommu_map_sg,
.iova_to_phys = arm_smmu_iova_to_phys,
.add_device = arm_smmu_add_device,
.remove_device = arm_smmu_remove_device,
@@ -1613,19 +1561,19 @@ static struct iommu_ops arm_smmu_ops = {
.domain_get_attr = arm_smmu_domain_get_attr,
.domain_set_attr = arm_smmu_domain_set_attr,
.of_xlate = arm_smmu_of_xlate,
- .get_resv_regions = arm_smmu_get_resv_regions,
- .put_resv_regions = arm_smmu_put_resv_regions,
.pgsize_bitmap = -1UL, /* Restricted during device attach */
};
static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
+ void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+ void __iomem *cb_base;
int i;
- u32 reg;
+ u32 reg, major;
/* clear global FSR */
- reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
+ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
+ writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
/*
* Reset stream mapping groups: Initial values mark all SMRn as
@@ -1634,17 +1582,40 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
for (i = 0; i < smmu->num_mapping_groups; ++i)
arm_smmu_write_sme(smmu, i);
+ /*
+ * Before clearing ARM_MMU500_ACTLR_CPRE, need to
+ * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
+ * bit is only present in MMU-500r2 onwards.
+ */
+ reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
+ major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
+ if ((smmu->model == ARM_MMU500) && (major >= 2)) {
+ reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
+ reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
+ writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
+ }
+
/* Make sure all context banks are disabled and clear CB_FSR */
for (i = 0; i < smmu->num_context_banks; ++i) {
- arm_smmu_write_context_bank(smmu, i);
- arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
+ /*
+ * Disable MMU-500's not-particularly-beneficial next-page
+ * prefetcher for the sake of errata #841119 and #826419.
+ */
+ if (smmu->model == ARM_MMU500) {
+ reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
+ reg &= ~ARM_MMU500_ACTLR_CPRE;
+ writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
+ }
}
/* Invalidate the TLB, just in case */
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
- reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
+ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
/* Enable fault reporting */
reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
@@ -1663,20 +1634,14 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
reg &= ~sCR0_FB;
/* Don't upgrade barriers */
- reg &= ~(sCR0_BSU);
+ reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
if (smmu->features & ARM_SMMU_FEAT_VMID16)
reg |= sCR0_VMID16EN;
- if (smmu->features & ARM_SMMU_FEAT_EXIDS)
- reg |= sCR0_EXIDENABLE;
-
- if (smmu->impl && smmu->impl->reset)
- smmu->impl->reset(smmu);
-
/* Push the button */
- arm_smmu_tlb_sync_global(smmu);
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
+ __arm_smmu_tlb_sync(smmu);
+ writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
}
static int arm_smmu_id_size_to_bits(int size)
@@ -1700,9 +1665,10 @@ static int arm_smmu_id_size_to_bits(int size)
static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
{
- unsigned int size;
+ unsigned long size;
+ void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
u32 id;
- bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
+ bool cttw_dt, cttw_reg;
int i;
dev_notice(smmu->dev, "probing hardware configuration...\n");
@@ -1710,7 +1676,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
smmu->version == ARM_SMMU_V2 ? 2 : 1);
/* ID0 */
- id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
+ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
/* Restrict available stages based on module parameter */
if (force_stage == 1)
@@ -1747,35 +1713,50 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
/*
* In order for DMA API calls to work properly, we must defer to what
- * the FW says about coherency, regardless of what the hardware claims.
+ * the DT says about coherency, regardless of what the hardware claims.
* Fortunately, this also opens up a workaround for systems where the
* ID register value has ended up configured incorrectly.
*/
+ cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
cttw_reg = !!(id & ID0_CTTW);
- if (cttw_fw || cttw_reg)
+ if (cttw_dt)
+ smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
+ if (cttw_dt || cttw_reg)
dev_notice(smmu->dev, "\t%scoherent table walk\n",
- cttw_fw ? "" : "non-");
- if (cttw_fw != cttw_reg)
+ cttw_dt ? "" : "non-");
+ if (cttw_dt != cttw_reg)
dev_notice(smmu->dev,
- "\t(IDR0.CTTW overridden by FW configuration)\n");
+ "\t(IDR0.CTTW overridden by dma-coherent property)\n");
/* Max. number of entries we have for stream matching/indexing */
- if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
- smmu->features |= ARM_SMMU_FEAT_EXIDS;
- size = 1 << 16;
- } else {
- size = 1 << FIELD_GET(ID0_NUMSIDB, id);
- }
+ size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
smmu->streamid_mask = size - 1;
if (id & ID0_SMS) {
+ u32 smr;
+
smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
- size = FIELD_GET(ID0_NUMSMRG, id);
+ size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
if (size == 0) {
dev_err(smmu->dev,
"stream-matching supported, but no SMRs present!\n");
return -ENODEV;
}
+ /*
+ * SMR.ID bits may not be preserved if the corresponding MASK
+ * bits are set, so check each one separately. We can reject
+ * masters later if they try to claim IDs outside these masks.
+ */
+ smr = smmu->streamid_mask << SMR_ID_SHIFT;
+ writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
+ smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
+ smmu->streamid_mask = smr >> SMR_ID_SHIFT;
+
+ smr = smmu->streamid_mask << SMR_MASK_SHIFT;
+ writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
+ smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
+ smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
+
/* Zero-initialised to mark as invalid */
smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
GFP_KERNEL);
@@ -1783,7 +1764,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
return -ENOMEM;
dev_notice(smmu->dev,
- "\tstream matching with %u register groups", size);
+ "\tstream matching with %lu register groups, mask 0x%x",
+ size, smmu->smr_mask_mask);
}
/* s2cr->type == 0 means translation, so initialise explicitly */
smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
@@ -1795,7 +1777,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
smmu->num_mapping_groups = size;
mutex_init(&smmu->stream_map_mutex);
- spin_lock_init(&smmu->global_sync_lock);
if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
@@ -1804,38 +1785,44 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
}
/* ID1 */
- id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
+ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
/* Check for size mismatch of SMMU address space from mapped region */
- size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
- if (smmu->numpage != 2 * size << smmu->pgshift)
+ size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
+ size *= 2 << smmu->pgshift;
+ if (smmu->size != size)
dev_warn(smmu->dev,
- "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
- 2 * size << smmu->pgshift, smmu->numpage);
- /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
- smmu->numpage = size;
+ "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
+ size, smmu->size);
- smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
- smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
+ smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
+ smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
if (smmu->num_s2_context_banks > smmu->num_context_banks) {
dev_err(smmu->dev, "impossible number of S2 context banks!\n");
return -ENODEV;
}
dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
smmu->num_context_banks, smmu->num_s2_context_banks);
- smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
- sizeof(*smmu->cbs), GFP_KERNEL);
- if (!smmu->cbs)
- return -ENOMEM;
+ /*
+ * Cavium CN88xx erratum #27704.
+ * Ensure ASID and VMID allocation is unique across all SMMUs in
+ * the system.
+ */
+ if (smmu->model == CAVIUM_SMMUV2) {
+ smmu->cavium_id_base =
+ atomic_add_return(smmu->num_context_banks,
+ &cavium_smmu_context_count);
+ smmu->cavium_id_base -= smmu->num_context_banks;
+ }
/* ID2 */
- id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
- size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
+ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
+ size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
smmu->ipa_size = size;
/* The output mask is also applied for bypass */
- size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
+ size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
smmu->pa_size = size;
if (id & ID2_VMID16)
@@ -1855,7 +1842,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
if (smmu->version == ARM_SMMU_V1_64K)
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
} else {
- size = FIELD_GET(ID2_UBS, id);
+ size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
smmu->va_size = arm_smmu_id_size_to_bits(size);
if (id & ID2_PTFS_4K)
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
@@ -1892,9 +1879,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
smmu->ipa_size, smmu->pa_size);
- if (smmu->impl && smmu->impl->cfg_probe)
- return smmu->impl->cfg_probe(smmu);
-
return 0;
}
@@ -1904,7 +1888,7 @@ struct arm_smmu_match_data {
};
#define ARM_SMMU_MATCH_DATA(name, ver, imp) \
-static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
+static struct arm_smmu_match_data name = { .version = ver, .model = imp }
ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
@@ -1925,95 +1909,19 @@ static const struct of_device_id arm_smmu_of_match[] = {
};
MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
-#ifdef CONFIG_ACPI
-static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
-{
- int ret = 0;
-
- switch (model) {
- case ACPI_IORT_SMMU_V1:
- case ACPI_IORT_SMMU_CORELINK_MMU400:
- smmu->version = ARM_SMMU_V1;
- smmu->model = GENERIC_SMMU;
- break;
- case ACPI_IORT_SMMU_CORELINK_MMU401:
- smmu->version = ARM_SMMU_V1_64K;
- smmu->model = GENERIC_SMMU;
- break;
- case ACPI_IORT_SMMU_V2:
- smmu->version = ARM_SMMU_V2;
- smmu->model = GENERIC_SMMU;
- break;
- case ACPI_IORT_SMMU_CORELINK_MMU500:
- smmu->version = ARM_SMMU_V2;
- smmu->model = ARM_MMU500;
- break;
- case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
- smmu->version = ARM_SMMU_V2;
- smmu->model = CAVIUM_SMMUV2;
- break;
- default:
- ret = -ENODEV;
- }
-
- return ret;
-}
-
-static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
- struct arm_smmu_device *smmu)
-{
- struct device *dev = smmu->dev;
- struct acpi_iort_node *node =
- *(struct acpi_iort_node **)dev_get_platdata(dev);
- struct acpi_iort_smmu *iort_smmu;
- int ret;
-
- /* Retrieve SMMU1/2 specific data */
- iort_smmu = (struct acpi_iort_smmu *)node->node_data;
-
- ret = acpi_smmu_get_data(iort_smmu->model, smmu);
- if (ret < 0)
- return ret;
-
- /* Ignore the configuration access interrupt */
- smmu->num_global_irqs = 1;
-
- if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
- smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
-
- return 0;
-}
-#else
-static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
- struct arm_smmu_device *smmu)
-{
- return -ENODEV;
-}
-#endif
-
-static int arm_smmu_device_dt_probe(struct platform_device *pdev,
- struct arm_smmu_device *smmu)
+static int arm_smmu_device_dt_probe(struct platform_device *pdev)
{
const struct arm_smmu_match_data *data;
+ struct resource *res;
+ struct arm_smmu_device *smmu;
struct device *dev = &pdev->dev;
+ int num_irqs, i, err;
bool legacy_binding;
- if (of_property_read_u32(dev->of_node, "#global-interrupts",
- &smmu->num_global_irqs)) {
- dev_err(dev, "missing #global-interrupts property\n");
- return -ENODEV;
- }
-
- data = of_device_get_match_data(dev);
- smmu->version = data->version;
- smmu->model = data->model;
-
legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
if (legacy_binding && !using_generic_binding) {
- if (!using_legacy_binding) {
- pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
- IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
- }
+ if (!using_legacy_binding)
+ pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
using_legacy_binding = true;
} else if (!legacy_binding && !using_legacy_binding) {
using_generic_binding = true;
@@ -2022,66 +1930,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev,
return -ENODEV;
}
- if (of_dma_is_coherent(dev->of_node))
- smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
-
- return 0;
-}
-
-static int arm_smmu_bus_init(struct iommu_ops *ops)
-{
- int err;
-
- /* Oh, for a proper bus abstraction */
- if (!iommu_present(&platform_bus_type)) {
- err = bus_set_iommu(&platform_bus_type, ops);
- if (err)
- return err;
- }
-#ifdef CONFIG_ARM_AMBA
- if (!iommu_present(&amba_bustype)) {
- err = bus_set_iommu(&amba_bustype, ops);
- if (err)
- goto err_reset_platform_ops;
- }
-#endif
-#ifdef CONFIG_PCI
- if (!iommu_present(&pci_bus_type)) {
- err = bus_set_iommu(&pci_bus_type, ops);
- if (err)
- goto err_reset_amba_ops;
- }
-#endif
-#ifdef CONFIG_FSL_MC_BUS
- if (!iommu_present(&fsl_mc_bus_type)) {
- err = bus_set_iommu(&fsl_mc_bus_type, ops);
- if (err)
- goto err_reset_pci_ops;
- }
-#endif
- return 0;
-
-err_reset_pci_ops: __maybe_unused;
-#ifdef CONFIG_PCI
- bus_set_iommu(&pci_bus_type, NULL);
-#endif
-err_reset_amba_ops: __maybe_unused;
-#ifdef CONFIG_ARM_AMBA
- bus_set_iommu(&amba_bustype, NULL);
-#endif
-err_reset_platform_ops: __maybe_unused;
- bus_set_iommu(&platform_bus_type, NULL);
- return err;
-}
-
-static int arm_smmu_device_probe(struct platform_device *pdev)
-{
- struct resource *res;
- resource_size_t ioaddr;
- struct arm_smmu_device *smmu;
- struct device *dev = &pdev->dev;
- int num_irqs, i, err;
-
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
if (!smmu) {
dev_err(dev, "failed to allocate arm_smmu_device\n");
@@ -2089,28 +1937,21 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
}
smmu->dev = dev;
- if (dev->of_node)
- err = arm_smmu_device_dt_probe(pdev, smmu);
- else
- err = arm_smmu_device_acpi_probe(pdev, smmu);
-
- if (err)
- return err;
-
- smmu = arm_smmu_impl_init(smmu);
- if (IS_ERR(smmu))
- return PTR_ERR(smmu);
+ data = of_device_get_match_data(dev);
+ smmu->version = data->version;
+ smmu->model = data->model;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- ioaddr = res->start;
smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
- /*
- * The resource size should effectively match the value of SMMU_TOP;
- * stash that temporarily until we know PAGESIZE to validate it with.
- */
- smmu->numpage = resource_size(res);
+ smmu->size = resource_size(res);
+
+ if (of_property_read_u32(dev->of_node, "#global-interrupts",
+ &smmu->num_global_irqs)) {
+ dev_err(dev, "missing #global-interrupts property\n");
+ return -ENODEV;
+ }
num_irqs = 0;
while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
@@ -2125,7 +1966,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
return -ENODEV;
}
- smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
+ smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
GFP_KERNEL);
if (!smmu->irqs) {
dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
@@ -2142,31 +1983,18 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
smmu->irqs[i] = irq;
}
- err = devm_clk_bulk_get_all(dev, &smmu->clks);
- if (err < 0) {
- dev_err(dev, "failed to get clocks %d\n", err);
- return err;
- }
- smmu->num_clks = err;
-
- err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
- if (err)
- return err;
-
err = arm_smmu_device_cfg_probe(smmu);
if (err)
return err;
- if (smmu->version == ARM_SMMU_V2) {
- if (smmu->num_context_banks > smmu->num_context_irqs) {
- dev_err(dev,
- "found only %d context irq(s) but %d required\n",
- smmu->num_context_irqs, smmu->num_context_banks);
- return -ENODEV;
- }
+ parse_driver_options(smmu);
- /* Ignore superfluous interrupts */
- smmu->num_context_irqs = smmu->num_context_banks;
+ if (smmu->version == ARM_SMMU_V2 &&
+ smmu->num_context_banks != smmu->num_context_irqs) {
+ dev_err(dev,
+ "found only %d context interrupt(s) but %d required\n",
+ smmu->num_context_irqs, smmu->num_context_banks);
+ return -ENODEV;
}
for (i = 0; i < smmu->num_global_irqs; ++i) {
@@ -2182,45 +2010,23 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
}
}
- err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
- "smmu.%pa", &ioaddr);
- if (err) {
- dev_err(dev, "Failed to register iommu in sysfs\n");
- return err;
- }
-
- iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
- iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
-
- err = iommu_device_register(&smmu->iommu);
- if (err) {
- dev_err(dev, "Failed to register iommu\n");
- return err;
- }
-
+ of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
platform_set_drvdata(pdev, smmu);
arm_smmu_device_reset(smmu);
- arm_smmu_test_smr_masks(smmu);
- /*
- * We want to avoid touching dev->power.lock in fastpaths unless
- * it's really going to do something useful - pm_runtime_enabled()
- * can serve as an ideal proxy for that decision. So, conditionally
- * enable pm_runtime.
- */
- if (dev->pm_domain) {
- pm_runtime_set_active(dev);
- pm_runtime_enable(dev);
+ /* Oh, for a proper bus abstraction */
+ if (!iommu_present(&platform_bus_type))
+ bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
+#ifdef CONFIG_ARM_AMBA
+ if (!iommu_present(&amba_bustype))
+ bus_set_iommu(&amba_bustype, &arm_smmu_ops);
+#endif
+#ifdef CONFIG_PCI
+ if (!iommu_present(&pci_bus_type)) {
+ pci_request_acs();
+ bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
}
-
- /*
- * For ACPI and generic DT bindings, an SMMU will be probed before
- * any device which might need it, so we want the bus ops in place
- * ready to handle default domain setup as soon as any SMMU exists.
- */
- if (!using_legacy_binding)
- return arm_smmu_bus_init(&arm_smmu_ops);
-
+#endif
return 0;
}
@@ -2234,87 +2040,59 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
dev_err(&pdev->dev, "removing device with active domains!\n");
- arm_smmu_bus_init(NULL);
- iommu_device_unregister(&smmu->iommu);
- iommu_device_sysfs_remove(&smmu->iommu);
-
- arm_smmu_rpm_get(smmu);
/* Turn the thing off */
- arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
- arm_smmu_rpm_put(smmu);
-
- if (pm_runtime_enabled(smmu->dev))
- pm_runtime_force_suspend(smmu->dev);
- else
- clk_bulk_disable(smmu->num_clks, smmu->clks);
-
- clk_bulk_unprepare(smmu->num_clks, smmu->clks);
+ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
return 0;
}
-static void arm_smmu_device_shutdown(struct platform_device *pdev)
-{
- arm_smmu_device_remove(pdev);
-}
-
-static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
-{
- struct arm_smmu_device *smmu = dev_get_drvdata(dev);
- int ret;
-
- ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
- if (ret)
- return ret;
-
- arm_smmu_device_reset(smmu);
-
- return 0;
-}
-
-static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
-{
- struct arm_smmu_device *smmu = dev_get_drvdata(dev);
-
- clk_bulk_disable(smmu->num_clks, smmu->clks);
-
- return 0;
-}
-
-static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
-{
- if (pm_runtime_suspended(dev))
- return 0;
-
- return arm_smmu_runtime_resume(dev);
-}
-
-static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
-{
- if (pm_runtime_suspended(dev))
- return 0;
-
- return arm_smmu_runtime_suspend(dev);
-}
-
-static const struct dev_pm_ops arm_smmu_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
- SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
- arm_smmu_runtime_resume, NULL)
-};
-
static struct platform_driver arm_smmu_driver = {
.driver = {
- .name = "arm-smmu",
- .of_match_table = of_match_ptr(arm_smmu_of_match),
- .pm = &arm_smmu_pm_ops,
- .suppress_bind_attrs = true,
+ .name = "arm-smmu",
+ .of_match_table = of_match_ptr(arm_smmu_of_match),
},
- .probe = arm_smmu_device_probe,
+ .probe = arm_smmu_device_dt_probe,
.remove = arm_smmu_device_remove,
- .shutdown = arm_smmu_device_shutdown,
};
-module_platform_driver(arm_smmu_driver);
+
+static int __init arm_smmu_init(void)
+{
+ static bool registered;
+ int ret = 0;
+
+ if (!registered) {
+ ret = platform_driver_register(&arm_smmu_driver);
+ registered = !ret;
+ }
+ return ret;
+}
+
+static void __exit arm_smmu_exit(void)
+{
+ return platform_driver_unregister(&arm_smmu_driver);
+}
+
+subsys_initcall(arm_smmu_init);
+module_exit(arm_smmu_exit);
+
+static int __init arm_smmu_of_init(struct device_node *np)
+{
+ int ret = arm_smmu_init();
+
+ if (ret)
+ return ret;
+
+ if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
+ return -ENODEV;
+
+ return 0;
+}
+IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
+IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
-MODULE_AUTHOR("Will Deacon <will@kernel.org>");
+MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
MODULE_LICENSE("GPL v2");
