[PATCH v2 1/3] arm64: KVM: Fix stage-2 PGD allocation to have per-page refcounting
Christoffer Dall
christoffer.dall at linaro.org
Wed Mar 11 06:20:56 PDT 2015
On Wed, Mar 11, 2015 at 2:13 PM, Marc Zyngier <marc.zyngier at arm.com> wrote:
> Hi Christoffer,
>
> On 11/03/15 12:16, Christoffer Dall wrote:
>> Hi Marc,
>>
>> On Tue, Mar 10, 2015 at 07:06:59PM +0000, Marc Zyngier wrote:
>>> We're using __get_free_pages with to allocate the guest's stage-2
>>> PGD. The standard behaviour of this function is to return a set of
>>> pages where only the head page has a valid refcount.
>>>
>>> This behaviour gets us into trouble when we're trying to increment
>>> the refcount on a non-head page:
>>>
>>> page:ffff7c00cfb693c0 count:0 mapcount:0 mapping: (null) index:0x0
>>> flags: 0x4000000000000000()
>>> page dumped because: VM_BUG_ON_PAGE((*({ __attribute__((unused)) typeof((&page->_count)->counter) __var = ( typeof((&page->_count)->counter)) 0; (volatile typeof((&page->_count)->counter) *)&((&page->_count)->counter); })) <= 0)
>>> BUG: failure at include/linux/mm.h:548/get_page()!
>>> Kernel panic - not syncing: BUG!
>>> CPU: 1 PID: 1695 Comm: kvm-vcpu-0 Not tainted 4.0.0-rc1+ #3825
>>> Hardware name: APM X-Gene Mustang board (DT)
>>> Call trace:
>>> [<ffff80000008a09c>] dump_backtrace+0x0/0x13c
>>> [<ffff80000008a1e8>] show_stack+0x10/0x1c
>>> [<ffff800000691da8>] dump_stack+0x74/0x94
>>> [<ffff800000690d78>] panic+0x100/0x240
>>> [<ffff8000000a0bc4>] stage2_get_pmd+0x17c/0x2bc
>>> [<ffff8000000a1dc4>] kvm_handle_guest_abort+0x4b4/0x6b0
>>> [<ffff8000000a420c>] handle_exit+0x58/0x180
>>> [<ffff80000009e7a4>] kvm_arch_vcpu_ioctl_run+0x114/0x45c
>>> [<ffff800000099df4>] kvm_vcpu_ioctl+0x2e0/0x754
>>> [<ffff8000001c0a18>] do_vfs_ioctl+0x424/0x5c8
>>> [<ffff8000001c0bfc>] SyS_ioctl+0x40/0x78
>>> CPU0: stopping
>>>
>>> A possible approach for this is to split the compound page using
>>> split_page() at allocation time, and change the teardown path to
>>> free one page at a time.
>>>
>>> While we're at it, the PGD allocation code is reworked to reduce
>>> duplication.
>>>
>>> This has been tested on an X-Gene platform with a 4kB/48bit-VA host
>>> kernel, and kvmtool hacked to place memory in the second page of
>>> the hardware PGD (PUD for the host kernel). Also regression-tested
>>> on a Cubietruck (Cortex-A7).
>>>
>>> Reported-by: Mark Rutland <mark.rutland at arm.com>
>>> Signed-off-by: Marc Zyngier <marc.zyngier at arm.com>
>>> ---
>>> arch/arm/include/asm/kvm_mmu.h | 9 ++---
>>> arch/arm/kvm/mmu.c | 77 +++++++++++++++++++++++++++++++---------
>>> arch/arm64/include/asm/kvm_mmu.h | 46 +++---------------------
>>> 3 files changed, 66 insertions(+), 66 deletions(-)
>>>
>>> diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
>>> index 0187606..ff56f91 100644
>>> --- a/arch/arm/include/asm/kvm_mmu.h
>>> +++ b/arch/arm/include/asm/kvm_mmu.h
>>> @@ -162,18 +162,13 @@ static inline bool kvm_page_empty(void *ptr)
>>>
>>> #define KVM_PREALLOC_LEVEL 0
>>>
>>> -static inline int kvm_prealloc_hwpgd(struct kvm *kvm, pgd_t *pgd)
>>> -{
>>> - return 0;
>>> -}
>>> -
>>> -static inline void kvm_free_hwpgd(struct kvm *kvm) { }
>>> -
>>> static inline void *kvm_get_hwpgd(struct kvm *kvm)
>>> {
>>> return kvm->arch.pgd;
>>> }
>>>
>>> +static inline unsigned int kvm_get_hwpgd_order(void) { return S2_PGD_ORDER; }
>>> +
>>> struct kvm;
>>>
>>> #define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))
>>> diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
>>> index 69c2b4c..0a5457c 100644
>>> --- a/arch/arm/kvm/mmu.c
>>> +++ b/arch/arm/kvm/mmu.c
>>> @@ -634,6 +634,31 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
>>> __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
>>> }
>>>
>>> +/* Free the HW pgd, one page at a time */
>>> +static void kvm_free_hwpgd(unsigned long hwpgd)
>>> +{
>>> + int i;
>>> +
>>> + for (i = 0; i < (1 << kvm_get_hwpgd_order()); i += PAGE_SIZE)
>>> + free_page(hwpgd + i);
>>> +}
>>> +
>>> +/* Allocate the HW PGD, making sure that each page gets its own refcount */
>>> +static int kvm_alloc_hwpgd(unsigned long *hwpgdp)
>>
>> I think this can be simplified somewhat by just returning an unsigned
>> long that can be 0 in the error case which will make the caller look
>> a little nicer too.
>
> Sure.
>
>>> +{
>>> + unsigned long hwpgd;
>>> + unsigned int order = kvm_get_hwpgd_order();
>>> +
>>> + hwpgd = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
>>> + if (!hwpgd)
>>> + return -ENOMEM;
>>> +
>>> + split_page(virt_to_page((void *)hwpgd), order);
>>
>> nit: alloc_pages_exact() and free_pages_exact() seem to do this for us,
>> is it worth using those instead?
>
> Ah, I didn't know about these.
>
>> It would look something like this on top of your changes:
>>
>>
>> arch/arm/include/asm/kvm_mmu.h | 5 ++++-
>> arch/arm/kvm/mmu.c | 32 ++++++++++----------------------
>> arch/arm64/include/asm/kvm_mmu.h | 6 +++---
>> 3 files changed, 17 insertions(+), 26 deletions(-)
>>
>> diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
>> index fc05ba8..4cf48c3 100644
>> --- a/arch/arm/include/asm/kvm_mmu.h
>> +++ b/arch/arm/include/asm/kvm_mmu.h
>> @@ -168,7 +168,10 @@ static inline void *kvm_get_hwpgd(struct kvm *kvm)
>> return kvm->arch.pgd;
>> }
>>
>> -static inline unsigned int kvm_get_hwpgd_order(void) { return S2_PGD_ORDER; }
>> +static inline unsigned int kvm_get_hwpgd_size(void)
>> +{
>> + return PTRS_PER_S2_PGD * sizeof(pgd_t);
>> +}
>>
>> struct kvm;
>>
>> diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
>> index 8e91bea3..5656d79 100644
>> --- a/arch/arm/kvm/mmu.c
>> +++ b/arch/arm/kvm/mmu.c
>> @@ -633,28 +633,17 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
>> }
>>
>> /* Free the HW pgd, one page at a time */
>> -static void kvm_free_hwpgd(unsigned long hwpgd)
>> +static void kvm_free_hwpgd(void *hwpgd)
>> {
>> - int i;
>> -
>> - for (i = 0; i < (1 << kvm_get_hwpgd_order()); i += PAGE_SIZE)
>> - free_page(hwpgd + i);
>> + free_pages_exact(hwpgd, kvm_get_hwpgd_size());
>> }
>>
>> /* Allocate the HW PGD, making sure that each page gets its own refcount */
>> -static int kvm_alloc_hwpgd(unsigned long *hwpgdp)
>> +static void *kvm_alloc_hwpgd(void)
>> {
>> - unsigned long hwpgd;
>> - unsigned int order = kvm_get_hwpgd_order();
>> -
>> - hwpgd = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
>> - if (!hwpgd)
>> - return -ENOMEM;
>> -
>> - split_page(virt_to_page((void *)hwpgd), order);
>> + unsigned int size = kvm_get_hwpgd_size();
>>
>> - *hwpgdp = hwpgd;
>> - return 0;
>> + return alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
>> }
>>
>> /**
>> @@ -670,18 +659,17 @@ static int kvm_alloc_hwpgd(unsigned long *hwpgdp)
>> */
>> int kvm_alloc_stage2_pgd(struct kvm *kvm)
>> {
>> - int ret;
>> pgd_t *pgd;
>> - unsigned long hwpgd;
>> + void *hwpgd;
>>
>> if (kvm->arch.pgd != NULL) {
>> kvm_err("kvm_arch already initialized?\n");
>> return -EINVAL;
>> }
>>
>> - ret = kvm_alloc_hwpgd(&hwpgd);
>> - if (ret)
>> - return ret;
>> + hwpgd = kvm_alloc_hwpgd();
>> + if (!hwpgd)
>> + return -ENOMEM;
>>
>> /* When the kernel uses more levels of page tables than the
>> * guest, we allocate a fake PGD and pre-populate it to point
>> @@ -829,7 +817,7 @@ void kvm_free_stage2_pgd(struct kvm *kvm)
>> return;
>>
>> unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
>> - kvm_free_hwpgd((unsigned long)kvm_get_hwpgd(kvm));
>> + kvm_free_hwpgd(kvm_get_hwpgd(kvm));
>> if (KVM_PREALLOC_LEVEL > 0)
>> kfree(kvm->arch.pgd);
>>
>> diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
>> index 3668110..bbfb600 100644
>> --- a/arch/arm64/include/asm/kvm_mmu.h
>> +++ b/arch/arm64/include/asm/kvm_mmu.h
>> @@ -189,11 +189,11 @@ static inline void *kvm_get_hwpgd(struct kvm *kvm)
>> return pmd_offset(pud, 0);
>> }
>>
>> -static inline unsigned int kvm_get_hwpgd_order(void)
>> +static inline unsigned int kvm_get_hwpgd_size(void)
>> {
>> if (KVM_PREALLOC_LEVEL > 0)
>> - return PTRS_PER_S2_PGD_SHIFT;
>> - return S2_PGD_ORDER;
>> + return PTRS_PER_S2_PGD * PAGE_SIZE;
>> + return PTRS_PER_S2_PGD * sizeof(pgd_t);
>> }
>>
>> static inline bool kvm_page_empty(void *ptr)
>
> Looks good to me. I'll fold that into the patch, amend the commit
> message and resend.
>
I've done that here:
https://git.linaro.org/people/christoffer.dall/linux-kvm-arm.git/shortlog/refs/heads/pgd-fixes-alt
(except for the commit message, which I can also adjust).
I gave this a quick spin on Juno and TC2 (but couldn't test 64K pages
because I don't have a platform for that handy this very second)...
-Christoffer
More information about the linux-arm-kernel
mailing list