[PATCH 11/12] swiotlb: merge swiotlb-xen initialization into swiotlb

Stefano Stabellini sstabellini at kernel.org
Tue Mar 1 18:55:47 PST 2022


On Tue, 1 Mar 2022, Christoph Hellwig wrote:
> Allow to pass a remap argument to the swiotlb initialization functions
> to handle the Xen/x86 remap case.  ARM/ARM64 never did any remapping
> from xen_swiotlb_fixup, so we don't even need that quirk.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  arch/arm/xen/mm.c               |  23 +++---
>  arch/x86/include/asm/xen/page.h |   5 --
>  arch/x86/kernel/pci-dma.c       |  19 +++--
>  arch/x86/pci/sta2x11-fixup.c    |   2 +-
>  drivers/xen/swiotlb-xen.c       | 128 +-------------------------------
>  include/linux/swiotlb.h         |   7 +-
>  include/xen/arm/page.h          |   1 -
>  include/xen/swiotlb-xen.h       |   8 +-
>  kernel/dma/swiotlb.c            | 120 +++++++++++++++---------------
>  9 files changed, 96 insertions(+), 217 deletions(-)
> 
> diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c
> index a7e54a087b802..58b40f87617d3 100644
> --- a/arch/arm/xen/mm.c
> +++ b/arch/arm/xen/mm.c
> @@ -23,22 +23,20 @@
>  #include <asm/xen/hypercall.h>
>  #include <asm/xen/interface.h>
>  
> -unsigned long xen_get_swiotlb_free_pages(unsigned int order)
> +static gfp_t xen_swiotlb_gfp(void)
>  {
>  	phys_addr_t base;
> -	gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM;
>  	u64 i;
>  
>  	for_each_mem_range(i, &base, NULL) {
>  		if (base < (phys_addr_t)0xffffffff) {
>  			if (IS_ENABLED(CONFIG_ZONE_DMA32))
> -				flags |= __GFP_DMA32;
> -			else
> -				flags |= __GFP_DMA;
> -			break;
> +				return __GFP_DMA32;
> +			return __GFP_DMA;
>  		}
>  	}
> -	return __get_free_pages(flags, order);
> +
> +	return GFP_KERNEL;
>  }

Unrelated to this specific patch series: now that I think about it, if
io_tlb_default_mem.nslabs is already allocated by the time xen_mm_init
is called, wouldn't we potentially have an issue with the GFP flags used
for the earlier allocation (e.g. GFP_DMA32 not used)? Maybe something
for another day.


>  static bool hypercall_cflush = false;
> @@ -143,10 +141,15 @@ static int __init xen_mm_init(void)
>  	if (!xen_swiotlb_detect())
>  		return 0;
>  
> -	rc = xen_swiotlb_init();
>  	/* we can work with the default swiotlb */
> -	if (rc < 0 && rc != -EEXIST)
> -		return rc;
> +	if (!io_tlb_default_mem.nslabs) {
> +		if (!xen_initial_domain())
> +			return -EINVAL;

I don't think we need this xen_initial_domain() check. It is all
already sorted out by the xen_swiotlb_detect() check above.

Aside from that the rest looks OK. Also, you can add my:

Tested-by: Stefano Stabellini <sstabellini at kernel.org>


> +		rc = swiotlb_init_late(swiotlb_size_or_default(),
> +				       xen_swiotlb_gfp(), NULL);
> +		if (rc < 0)
> +			return rc;
> +	}
>  
>  	cflush.op = 0;
>  	cflush.a.dev_bus_addr = 0;
> diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
> index e989bc2269f54..1fc67df500145 100644
> --- a/arch/x86/include/asm/xen/page.h
> +++ b/arch/x86/include/asm/xen/page.h
> @@ -357,9 +357,4 @@ static inline bool xen_arch_need_swiotlb(struct device *dev,
>  	return false;
>  }
>  
> -static inline unsigned long xen_get_swiotlb_free_pages(unsigned int order)
> -{
> -	return __get_free_pages(__GFP_NOWARN, order);
> -}
> -
>  #endif /* _ASM_X86_XEN_PAGE_H */
> diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
> index e0def4b1c3181..2f2c468acb955 100644
> --- a/arch/x86/kernel/pci-dma.c
> +++ b/arch/x86/kernel/pci-dma.c
> @@ -71,15 +71,12 @@ static inline void __init pci_swiotlb_detect(void)
>  #endif /* CONFIG_SWIOTLB */
>  
>  #ifdef CONFIG_SWIOTLB_XEN
> -static bool xen_swiotlb;
> -
>  static void __init pci_xen_swiotlb_init(void)
>  {
>  	if (!xen_initial_domain() && !x86_swiotlb_enable)
>  		return;
>  	x86_swiotlb_enable = false;
> -	xen_swiotlb = true;
> -	xen_swiotlb_init_early();
> +	swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup);
>  	dma_ops = &xen_swiotlb_dma_ops;
>  	if (IS_ENABLED(CONFIG_PCI))
>  		pci_request_acs();
> @@ -87,14 +84,16 @@ static void __init pci_xen_swiotlb_init(void)
>  
>  int pci_xen_swiotlb_init_late(void)
>  {
> -	int rc;
> -
> -	if (xen_swiotlb)
> +	if (dma_ops == &xen_swiotlb_dma_ops)
>  		return 0;
>  
> -	rc = xen_swiotlb_init();
> -	if (rc)
> -		return rc;
> +	/* we can work with the default swiotlb */
> +	if (!io_tlb_default_mem.nslabs) {
> +		int rc = swiotlb_init_late(swiotlb_size_or_default(),
> +					   GFP_KERNEL, xen_swiotlb_fixup);
> +		if (rc < 0)
> +			return rc;
> +	}
>  
>  	/* XXX: this switches the dma ops under live devices! */
>  	dma_ops = &xen_swiotlb_dma_ops;
> diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
> index c2da3eb4826e8..df8085b50df10 100644
> --- a/drivers/xen/swiotlb-xen.c
> +++ b/drivers/xen/swiotlb-xen.c
> @@ -104,7 +104,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
>  	return 0;
>  }
>  
> -static int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
> +int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
>  {
>  	int rc;
>  	unsigned int order = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT);
> @@ -130,132 +130,6 @@ static int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
>  	return 0;
>  }
>  
> -enum xen_swiotlb_err {
> -	XEN_SWIOTLB_UNKNOWN = 0,
> -	XEN_SWIOTLB_ENOMEM,
> -	XEN_SWIOTLB_EFIXUP
> -};
> -
> -static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
> -{
> -	switch (err) {
> -	case XEN_SWIOTLB_ENOMEM:
> -		return "Cannot allocate Xen-SWIOTLB buffer\n";
> -	case XEN_SWIOTLB_EFIXUP:
> -		return "Failed to get contiguous memory for DMA from Xen!\n"\
> -		    "You either: don't have the permissions, do not have"\
> -		    " enough free memory under 4GB, or the hypervisor memory"\
> -		    " is too fragmented!";
> -	default:
> -		break;
> -	}
> -	return "";
> -}
> -
> -int xen_swiotlb_init(void)
> -{
> -	enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
> -	unsigned long bytes = swiotlb_size_or_default();
> -	unsigned long nslabs = bytes >> IO_TLB_SHIFT;
> -	unsigned int order, repeat = 3;
> -	int rc = -ENOMEM;
> -	char *start;
> -
> -	if (io_tlb_default_mem.nslabs) {
> -		pr_warn("swiotlb buffer already initialized\n");
> -		return -EEXIST;
> -	}
> -
> -retry:
> -	m_ret = XEN_SWIOTLB_ENOMEM;
> -	order = get_order(bytes);
> -
> -	/*
> -	 * Get IO TLB memory from any location.
> -	 */
> -#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
> -#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
> -	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
> -		start = (void *)xen_get_swiotlb_free_pages(order);
> -		if (start)
> -			break;
> -		order--;
> -	}
> -	if (!start)
> -		goto exit;
> -	if (order != get_order(bytes)) {
> -		pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n",
> -			(PAGE_SIZE << order) >> 20);
> -		nslabs = SLABS_PER_PAGE << order;
> -		bytes = nslabs << IO_TLB_SHIFT;
> -	}
> -
> -	/*
> -	 * And replace that memory with pages under 4GB.
> -	 */
> -	rc = xen_swiotlb_fixup(start, nslabs);
> -	if (rc) {
> -		free_pages((unsigned long)start, order);
> -		m_ret = XEN_SWIOTLB_EFIXUP;
> -		goto error;
> -	}
> -	rc = swiotlb_late_init_with_tbl(start, nslabs);
> -	if (rc)
> -		return rc;
> -	return 0;
> -error:
> -	if (nslabs > 1024 && repeat--) {
> -		/* Min is 2MB */
> -		nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
> -		bytes = nslabs << IO_TLB_SHIFT;
> -		pr_info("Lowering to %luMB\n", bytes >> 20);
> -		goto retry;
> -	}
> -exit:
> -	pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc);
> -	return rc;
> -}
> -
> -#ifdef CONFIG_X86
> -void __init xen_swiotlb_init_early(void)
> -{
> -	unsigned long bytes = swiotlb_size_or_default();
> -	unsigned long nslabs = bytes >> IO_TLB_SHIFT;
> -	unsigned int repeat = 3;
> -	char *start;
> -	int rc;
> -
> -retry:
> -	/*
> -	 * Get IO TLB memory from any location.
> -	 */
> -	start = memblock_alloc(PAGE_ALIGN(bytes),
> -			       IO_TLB_SEGSIZE << IO_TLB_SHIFT);
> -	if (!start)
> -		panic("%s: Failed to allocate %lu bytes\n",
> -		      __func__, PAGE_ALIGN(bytes));
> -
> -	/*
> -	 * And replace that memory with pages under 4GB.
> -	 */
> -	rc = xen_swiotlb_fixup(start, nslabs);
> -	if (rc) {
> -		memblock_free(start, PAGE_ALIGN(bytes));
> -		if (nslabs > 1024 && repeat--) {
> -			/* Min is 2MB */
> -			nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
> -			bytes = nslabs << IO_TLB_SHIFT;
> -			pr_info("Lowering to %luMB\n", bytes >> 20);
> -			goto retry;
> -		}
> -		panic("%s (rc:%d)", xen_swiotlb_error(XEN_SWIOTLB_EFIXUP), rc);
> -	}
> -
> -	if (swiotlb_init_with_tbl(start, nslabs, SWIOTLB_VERBOSE))
> -		panic("Cannot allocate SWIOTLB buffer");
> -}
> -#endif /* CONFIG_X86 */
> -
>  static void *
>  xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
>  			   dma_addr_t *dma_handle, gfp_t flags,
> diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
> index ee655f2e4d28b..919cf82ed978e 100644
> --- a/include/linux/swiotlb.h
> +++ b/include/linux/swiotlb.h
> @@ -34,10 +34,11 @@ struct scatterlist;
>  /* default to 64MB */
>  #define IO_TLB_DEFAULT_SIZE (64UL<<20)
>  
> -int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, unsigned int flags);
>  unsigned long swiotlb_size_or_default(void);
> -extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
> -int swiotlb_init_late(size_t size, gfp_t gfp_mask);
> +int swiotlb_init_late(size_t size, gfp_t gfp_mask,
> +		int (*remap)(void *tlb, unsigned long nslabs));
> +void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
> +		int (*remap)(void *tlb, unsigned long nslabs));
>  extern void __init swiotlb_update_mem_attributes(void);
>  
>  phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
> diff --git a/include/xen/arm/page.h b/include/xen/arm/page.h
> index ac1b654705631..7e199c6656b90 100644
> --- a/include/xen/arm/page.h
> +++ b/include/xen/arm/page.h
> @@ -115,6 +115,5 @@ static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
>  bool xen_arch_need_swiotlb(struct device *dev,
>  			   phys_addr_t phys,
>  			   dma_addr_t dev_addr);
> -unsigned long xen_get_swiotlb_free_pages(unsigned int order);
>  #endif /* _ASM_ARM_XEN_PAGE_H */
> diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
> index b3e647f86e3e2..590ceb923f0c8 100644
> --- a/include/xen/swiotlb-xen.h
> +++ b/include/xen/swiotlb-xen.h
> @@ -10,8 +10,12 @@ void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle,
>  void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle,
>  			     size_t size, enum dma_data_direction dir);
>  
> -int xen_swiotlb_init(void);
> -void __init xen_swiotlb_init_early(void);
> +#ifdef CONFIG_SWIOTLB_XEN
> +int xen_swiotlb_fixup(void *buf, unsigned long nslabs);
> +#else
> +#define xen_swiotlb_fixup NULL
> +#endif
> +
>  extern const struct dma_map_ops xen_swiotlb_dma_ops;
>  
>  #endif /* __LINUX_SWIOTLB_XEN_H */
> diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
> index 77cf73dc20a78..128363dc9b5bb 100644
> --- a/kernel/dma/swiotlb.c
> +++ b/kernel/dma/swiotlb.c
> @@ -234,40 +234,17 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
>  	return;
>  }
>  
> -int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs,
> -		unsigned int flags)
> -{
> -	struct io_tlb_mem *mem = &io_tlb_default_mem;
> -	size_t alloc_size;
> -
> -	if (swiotlb_force_disable)
> -		return 0;
> -
> -	/* protect against double initialization */
> -	if (WARN_ON_ONCE(mem->nslabs))
> -		return -ENOMEM;
> -
> -	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
> -	mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
> -	if (!mem->slots)
> -		panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
> -		      __func__, alloc_size, PAGE_SIZE);
> -
> -	swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, false);
> -	mem->force_bounce = flags & SWIOTLB_FORCE;
> -
> -	if (flags & SWIOTLB_VERBOSE)
> -		swiotlb_print_info();
> -	return 0;
> -}
> -
>  /*
>   * Statically reserve bounce buffer space and initialize bounce buffer data
>   * structures for the software IO TLB used to implement the DMA API.
>   */
> -void __init swiotlb_init(bool addressing_limit, unsigned int flags)
> +void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
> +		int (*remap)(void *tlb, unsigned long nslabs))
>  {
> -	size_t bytes = PAGE_ALIGN(default_nslabs << IO_TLB_SHIFT);
> +	struct io_tlb_mem *mem = &io_tlb_default_mem;
> +	unsigned long nslabs = default_nslabs;
> +	size_t alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
> +	size_t bytes;
>  	void *tlb;
>  
>  	if (!addressing_limit && !swiotlb_force_bounce)
> @@ -275,23 +252,48 @@ void __init swiotlb_init(bool addressing_limit, unsigned int flags)
>  	if (swiotlb_force_disable)
>  		return;
>  
> +	/* protect against double initialization */
> +	if (WARN_ON_ONCE(mem->nslabs))
> +		return;
> +
>  	/*
>  	 * By default allocate the bonuce buffer memory from low memory.
>  	 */
> +retry:
> +	bytes = PAGE_ALIGN(default_nslabs << IO_TLB_SHIFT);
>  	if (flags & SWIOTLB_ANY)
>  		tlb = memblock_alloc(bytes, PAGE_SIZE);
>  	else
>  		tlb = memblock_alloc_low(bytes, PAGE_SIZE);
>  	if (!tlb)
> -		goto fail;
> -	if (swiotlb_init_with_tbl(tlb, default_nslabs, flags))
> -		goto fail_free_mem;
> -	return;
> +		panic("%s: failed to allocate tlb structure\n", __func__);
> +
> +	if (remap && remap(tlb, nslabs) < 0) {
> +		memblock_free(tlb, PAGE_ALIGN(bytes));
> +
> +		/* Min is 2MB */
> +		if (nslabs <= 1024)
> +			panic("%s: Failed to remap %zu bytes\n",
> +			      __func__, bytes);
> +		nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
> +		goto retry;
> +	}
> +
> +	mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
> +	if (!mem->slots)
> +		panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
> +		      __func__, alloc_size, PAGE_SIZE);
>  
> -fail_free_mem:
> -	memblock_free(tlb, bytes);
> -fail:
> -	pr_warn("Cannot allocate buffer");
> +	swiotlb_init_io_tlb_mem(mem, __pa(tlb), default_nslabs, false);
> +	mem->force_bounce = flags & SWIOTLB_FORCE;
> +
> +	if (flags & SWIOTLB_VERBOSE)
> +		swiotlb_print_info();
> +}
> +
> +void __init swiotlb_init(bool addressing_limit, unsigned int flags)
> +{
> +	return swiotlb_init_remap(addressing_limit, flags, NULL);
>  }
>  
>  /*
> @@ -299,8 +301,10 @@ void __init swiotlb_init(bool addressing_limit, unsigned int flags)
>   * initialize the swiotlb later using the slab allocator if needed.
>   * This should be just like above, but with some error catching.
>   */
> -int swiotlb_init_late(size_t size, gfp_t gfp_mask)
> +int swiotlb_init_late(size_t size, gfp_t gfp_mask,
> +		int (*remap)(void *tlb, unsigned long nslabs))
>  {
> +	struct io_tlb_mem *mem = &io_tlb_default_mem;
>  	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
>  	unsigned long bytes;
>  	unsigned char *vstart = NULL;
> @@ -310,9 +314,14 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask)
>  	if (swiotlb_force_disable)
>  		return 0;
>  
> +	/* protect against double initialization */
> +	if (WARN_ON_ONCE(mem->nslabs))
> +		return -ENOMEM;
> +
>  	/*
>  	 * Get IO TLB memory from the low pages
>  	 */
> +retry:
>  	order = get_order(nslabs << IO_TLB_SHIFT);
>  	nslabs = SLABS_PER_PAGE << order;
>  	bytes = nslabs << IO_TLB_SHIFT;
> @@ -333,33 +342,28 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask)
>  			(PAGE_SIZE << order) >> 20);
>  		nslabs = SLABS_PER_PAGE << order;
>  	}
> -	rc = swiotlb_late_init_with_tbl(vstart, nslabs);
> -	if (rc)
> -		free_pages((unsigned long)vstart, order);
> -
> -	return rc;
> -}
> -
> -int
> -swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
> -{
> -	struct io_tlb_mem *mem = &io_tlb_default_mem;
> -	unsigned long bytes = nslabs << IO_TLB_SHIFT;
>  
> -	if (swiotlb_force_disable)
> -		return 0;
> +	if (remap)
> +		rc = remap(vstart, nslabs);
> +	if (rc) {
> +		free_pages((unsigned long)vstart, order);
>  
> -	/* protect against double initialization */
> -	if (WARN_ON_ONCE(mem->nslabs))
> -		return -ENOMEM;
> +		/* Min is 2MB */
> +		if (nslabs <= 1024)
> +			return rc;
> +		nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
> +		goto retry;
> +	}
>  
>  	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
>  		get_order(array_size(sizeof(*mem->slots), nslabs)));
> -	if (!mem->slots)
> +	if (!mem->slots) {
> +		free_pages((unsigned long)vstart, order);
>  		return -ENOMEM;
> +	}
>  
> -	set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
> -	swiotlb_init_io_tlb_mem(mem, virt_to_phys(tlb), nslabs, true);
> +	set_memory_decrypted((unsigned long)vstart, bytes >> PAGE_SHIFT);
> +	swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, true);
>  
>  	swiotlb_print_info();
>  	return 0;
> -- 
> 2.30.2
> 
> 
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
> 



More information about the linux-riscv mailing list