[PATCH v12 3/7] crash: add generic infrastructure for crash hotplug support
Sourabh Jain
sourabhjain at linux.ibm.com
Sun Oct 16 23:45:35 PDT 2022
On 08/10/22 00:44, Eric DeVolder wrote:
>
>
> On 10/3/22 12:51, Sourabh Jain wrote:
>> Hello Eric,
>>
>> On 10/09/22 02:35, Eric DeVolder wrote:
>>> CPU and memory change notifications are received in order to
>>> regenerate the elfcorehdr.
>>>
>>> To support cpu hotplug, a callback is registered to capture the
>>> CPUHP_AP_ONLINE_DYN online and offline events via
>>> cpuhp_setup_state_nocalls().
>>>
>>> To support memory hotplug, a notifier is registered to capture the
>>> MEM_ONLINE and MEM_OFFLINE events via register_memory_notifier().
>>>
>>> The cpu callback and memory notifiers call handle_hotplug_event()
>>> which performs needed tasks and then dispatches the event to the
>>> architecture specific arch_crash_handle_hotplug_event(). During the
>>> process, the kexec_mutex is held.
>>>
>>> Signed-off-by: Eric DeVolder <eric.devolder at oracle.com>
>>> Acked-by: Baoquan He <bhe at redhat.com>
>>> ---
>>> include/linux/crash_core.h | 8 +++
>>> include/linux/kexec.h | 26 +++++++
>>> kernel/crash_core.c | 134
>>> +++++++++++++++++++++++++++++++++++++
>>> 3 files changed, 168 insertions(+)
>>>
>>> diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
>>> index de62a722431e..a270f8660538 100644
>>> --- a/include/linux/crash_core.h
>>> +++ b/include/linux/crash_core.h
>>> @@ -84,4 +84,12 @@ int parse_crashkernel_high(char *cmdline,
>>> unsigned long long system_ram,
>>> int parse_crashkernel_low(char *cmdline, unsigned long long
>>> system_ram,
>>> unsigned long long *crash_size, unsigned long long
>>> *crash_base);
>>> +#define KEXEC_CRASH_HP_REMOVE_CPU 0
>>> +#define KEXEC_CRASH_HP_ADD_CPU 1
>>> +#define KEXEC_CRASH_HP_REMOVE_MEMORY 2
>>> +#define KEXEC_CRASH_HP_ADD_MEMORY 3
>>> +#define KEXEC_CRASH_HP_INVALID_CPU -1U
>>> +
>>> +struct kimage;
>>> +
>>> #endif /* LINUX_CRASH_CORE_H */
>>> diff --git a/include/linux/kexec.h b/include/linux/kexec.h
>>> index 4eefa631e0ae..9597b41136ec 100644
>>> --- a/include/linux/kexec.h
>>> +++ b/include/linux/kexec.h
>>> @@ -374,6 +374,13 @@ struct kimage {
>>> struct purgatory_info purgatory_info;
>>> #endif
>>> +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_MEMORY_HOTPLUG)
>>> + bool hotplug_event;
>>> + unsigned int offlinecpu;
>>> + bool elfcorehdr_index_valid;
>>> + int elfcorehdr_index;
>>> +#endif
>>> +
>>> #ifdef CONFIG_IMA_KEXEC
>>> /* Virtual address of IMA measurement buffer for kexec syscall */
>>> void *ima_buffer;
>>> @@ -503,6 +510,25 @@ static inline int
>>> arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, g
>>> static inline void arch_kexec_pre_free_pages(void *vaddr, unsigned
>>> int pages) { }
>>> #endif
>>> +#ifndef arch_map_crash_pages
>>> +static inline void *arch_map_crash_pages(unsigned long paddr,
>>> + unsigned long size)
>>> +{
>>> + return NULL;
>>> +}
>>> +#endif
>>> +
>>> +#ifndef arch_unmap_crash_pages
>>> +static inline void arch_unmap_crash_pages(void **ptr) { }
>>> +#endif
>>> +
>>> +#ifndef arch_crash_handle_hotplug_event
>>> +static inline void arch_crash_handle_hotplug_event(struct kimage
>>> *image,
>>> + unsigned int hp_action)
>>> +{
>>> +}
>>> +#endif
>>> +
>>> #else /* !CONFIG_KEXEC_CORE */
>>> struct pt_regs;
>>> struct task_struct;
>>> diff --git a/kernel/crash_core.c b/kernel/crash_core.c
>>> index 8c648fd5897a..4b15d91f0b21 100644
>>> --- a/kernel/crash_core.c
>>> +++ b/kernel/crash_core.c
>>> @@ -11,6 +11,8 @@
>>> #include <linux/vmalloc.h>
>>> #include <linux/sizes.h>
>>> #include <linux/kexec.h>
>>> +#include <linux/memory.h>
>>> +#include <linux/cpuhotplug.h>
>>> #include <asm/page.h>
>>> #include <asm/sections.h>
>>> @@ -18,6 +20,7 @@
>>> #include <crypto/sha1.h>
>>> #include "kallsyms_internal.h"
>>> +#include "kexec_internal.h"
>>> /* vmcoreinfo stuff */
>>> unsigned char *vmcoreinfo_data;
>>> @@ -612,3 +615,134 @@ static int __init
>>> crash_save_vmcoreinfo_init(void)
>>> }
>>> subsys_initcall(crash_save_vmcoreinfo_init);
>>> +
>>> +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_MEMORY_HOTPLUG)
>>> +/*
>>> + * To accurately reflect hot un/plug changes, the elfcorehdr (which
>>> + * is passed to the crash kernel via the elfcorehdr= parameter)
>>> + * must be updated with the new list of CPUs and memories.
>>> + *
>>> + * In order to make changes to elfcorehdr, two conditions are needed:
>>> + * First, the segment containing the elfcorehdr must be large enough
>>> + * to permit a growing number of resources. The elfcorehdr memory is
>>> + * typically based on CONFIG_NR_CPUS and
>>> CONFIG_CRASH_MAX_MEMORY_RANGES.
>>> + * Second, purgatory must explicitly exclude the elfcorehdr from the
>>> + * list of segments it checks (since the elfcorehdr changes and thus
>>> + * would require an update to purgatory itself to update the digest).
>>> + */
>>> +static void handle_hotplug_event(unsigned int hp_action, unsigned
>>> int cpu)
>>> +{
>>> + /* Obtain lock while changing crash information */
>>> + mutex_lock(&kexec_mutex);
>>> +
>>> + /* Check kdump is loaded */
>>> + if (kexec_crash_image) {
>>> + struct kimage *image = kexec_crash_image;
>>> +
>>> + if (hp_action == KEXEC_CRASH_HP_ADD_CPU ||
>>> + hp_action == KEXEC_CRASH_HP_REMOVE_CPU)
>>> + pr_debug("crash hp: hp_action %u, cpu %u\n", hp_action,
>>> cpu);
>>> + else
>>> + pr_debug("crash hp: hp_action %u\n", hp_action);
>>> +
>>> + /*
>>> + * When the struct kimage is allocated, it is wiped to
>>> zero, so
>>> + * the elfcorehdr_index_valid defaults to false. Find the
>>> + * segment containing the elfcorehdr, if not already found.
>>> + * This works for both the kexec_load and kexec_file_load
>>> paths.
>>> + */
>>> + if (!image->elfcorehdr_index_valid) {
>>> + unsigned char *ptr;
>>> + unsigned long mem, memsz;
>>> + unsigned int n;
>>> +
>>> + for (n = 0; n < image->nr_segments; n++) {
>>> + mem = image->segment[n].mem;
>>> + memsz = image->segment[n].memsz;
>>> + ptr = arch_map_crash_pages(mem, memsz);
>>> + if (ptr) {
>>> + /* The segment containing elfcorehdr */
>>> + if (memcmp(ptr, ELFMAG, SELFMAG) == 0) {
>>> + image->elfcorehdr_index = (int)n;
>>> + image->elfcorehdr_index_valid = true;
>>> + }
>>> + }
>>> + arch_unmap_crash_pages((void **)&ptr);
>>> + }
>>> + }
>>> +
>>> + if (!image->elfcorehdr_index_valid) {
>>> + pr_err("crash hp: unable to locate elfcorehdr segment");
>>> + goto out;
>>> + }
>>> +
>>> + /* Needed in order for the segments to be updated */
>>> + arch_kexec_unprotect_crashkres();
>>> +
>>> + /* Flag to differentiate between normal load and hotplug */
>>> + image->hotplug_event = true;
>>> +
>>> + /* Now invoke arch-specific update handler */
>>> + arch_crash_handle_hotplug_event(image, hp_action);
>>> +
>>> + /* No longer handling a hotplug event */
>>> + image->hotplug_event = false;
>>> +
>>> + /* Change back to read-only */
>>> + arch_kexec_protect_crashkres();
>>> + }
>>> +
>>> +out:
>>> + /* Release lock now that update complete */
>>> + mutex_unlock(&kexec_mutex);
>>> +}
>>> +
>>> +static int crash_memhp_notifier(struct notifier_block *nb, unsigned
>>> long val, void *v)
>>> +{
>>> + switch (val) {
>>> + case MEM_ONLINE:
>>> + handle_hotplug_event(KEXEC_CRASH_HP_ADD_MEMORY, 0);
>>> + break;
>>> +
>>> + case MEM_OFFLINE:
>>> + handle_hotplug_event(KEXEC_CRASH_HP_REMOVE_MEMORY, 0);
>>> + break;
>>> + }
>>> + return NOTIFY_OK;
>>
>> Can we pass v (memory_notify) argument to
>> arch_crash_handle_hotplug_event function
>> via handle_hotplug_event?
>>
>> Because the way memory hotplug is handled on PowerPC, it is hard to
>> update the elfcorehdr
>> without memory_notify args.
>>
>> On PowePC memblock data structure is used to prepare elfcorehdr for
>> kdump. Since the notifier
>> used for memory hotplug crash handler get initiated before the
>> memblock data structure update
>> happens (as depicted below), the newly prepared elfcorehdr still
>> holds the old memory regions.
>> So if the system crash with obsolete elfcorehdr, makedumpfile failed
>> to collect vmcore.
>>
>> Sequence of actions done on PowerPC to server the memory hotplug:
>>
>> Initiate memory hot remove
>> |
>> v
>> offline pages
>> |
>> v
>> initiate memory notify call chain
>> for MEM_OFFLINE event.
>> (same is used for crash update)
>> |
>> v
>> prepare new elfcorehdr for kdump using
>> memblock data structure
>> |
>> v
>> update memblock data structure
>>
>> How passing memory_notify to arch crash hotplug handler will help?
>>
>> memory_notify holds the start PFN and page count, with that we can get
>> the base address and size of hot unplugged memory and can use the same
>> to avoid hot unplugged memeory region to get added in the elfcorehdr..
>>
>> Thanks,
>> Sourabh Jain
>>
>
> Sourabh, let's see what Baoquan thinks.
>
> Baoquan, are you OK with this request? I once had these parameters to the
> crash hotplug handler and since they were unused at the time, you asked
> that I remove them, which I did.
>
> To accommodate this, how about this:
>
> static void handle_hotplug_event(unsigned int hp_action, unsigned int
> cpu,
> unsigned long mem_start, unsigned long mem_size)
>
> For CPU events, I would just pass zeros for mem_start/size. For memory
> events,
> I would pass KEXEC_CRASH_HP_INVALID_CPU.
How about passing memory_notify struct as is and let architecture handle
the rest?
Thanks,
Sourabh Jain
More information about the kexec
mailing list