From 6a53211ca4653f65c0bc6293a4e33ec54e9e2e38 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 30 Apr 2026 08:25:55 +0800 Subject: [PATCH 01/85] perf/x86/intel: Always reprogram ACR events to prevent stale masks Members of an ACR group are logically linked via a bitmask of their hardware counter indices. If some members of the group are assigned new hardware counters during rescheduling, even events that keep their original counter index must be updated with a new mask. Without this, an event will continue to use a stale acr_mask that references the old indices of its group peers. Ensure all ACR events are reprogrammed during the scheduling path to maintain consistency across the group. Fixes: ec980e4facef ("perf/x86/intel: Support auto counter reload") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260430002558.712334-3-dapeng1.mi@linux.intel.com (cherry picked from commit 8ba0b706a485b1e607594cf4210786d517ad1611) Signed-off-by: Wentao Guan --- arch/x86/events/core.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d5b3b19b87195..498e4114c677e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1276,13 +1276,16 @@ int x86_perf_rdpmc_index(struct perf_event *event) return event->hw.event_base_rdpmc; } -static inline int match_prev_assignment(struct hw_perf_event *hwc, +static inline int match_prev_assignment(struct perf_event *event, struct cpu_hw_events *cpuc, int i) { + struct hw_perf_event *hwc = &event->hw; + return hwc->idx == cpuc->assign[i] && - hwc->last_cpu == smp_processor_id() && - hwc->last_tag == cpuc->tags[i]; + hwc->last_cpu == smp_processor_id() && + hwc->last_tag == cpuc->tags[i] && + !is_acr_event_group(event); } static void x86_pmu_start(struct perf_event *event, int flags); @@ -1328,7 +1331,7 @@ static void x86_pmu_enable(struct pmu *pmu) * - no other event has used the counter since */ if (hwc->idx == -1 || - match_prev_assignment(hwc, cpuc, i)) + match_prev_assignment(event, cpuc, i)) continue; /* @@ -1348,7 +1351,7 @@ static void x86_pmu_enable(struct pmu *pmu) event = cpuc->event_list[i]; hwc = &event->hw; - if (!match_prev_assignment(hwc, cpuc, i)) + if (!match_prev_assignment(event, cpuc, i)) x86_assign_hw_event(event, cpuc, i); else if (i < n_running) continue; From 1b4d0a45f1cb3aac93c79c564f8fa7fbbe9d6212 Mon Sep 17 00:00:00 2001 From: Zongyao Chen Date: Fri, 22 May 2026 10:21:50 -0700 Subject: [PATCH 02/85] KVM: selftests: Test guest_memfd binding overlap without GPA overlap The guest_memfd binding overlap test recreates the deleted slot with GPA ranges that overlap the still-live slot. KVM rejects those attempts from the generic memslot overlap check before reaching kvm_gmem_bind(), so the test can pass even if guest_memfd binding overlap detection is broken. Recreate the slot at its original, non-overlapping GPA and use guest_memfd offsets that overlap the front and back halves of the other slot's binding. Expand the guest_memfd so the back-half case remains within the file size. Fixes: 2feabb855df8 ("KVM: selftests: Expand set_memory_region_test to validate guest_memfd()") Signed-off-by: Zongyao Chen Reviewed-by: Ackerley Tng Tested-by: Ackerley Tng [sean: keep the existing GPA overlap testcases] Link: https://patch.msgid.link/20260522172151.3530267-3-seanjc@google.com Signed-off-by: Sean Christopherson (cherry picked from commit 772989854acd17e6cc9b87c54595aba4ff002130) Signed-off-by: Wentao Guan --- .../selftests/kvm/set_memory_region_test.c | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index a78394faee7c6..b7177600a7a74 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -442,7 +442,7 @@ static void test_add_overlapping_private_memory_regions(void) vm = vm_create_barebones_protected_vm(); - memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 4, 0); + memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 5, 0); vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA, MEM_REGION_SIZE * 2, 0, memfd, 0); @@ -458,7 +458,30 @@ static void test_add_overlapping_private_memory_regions(void) vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA, 0, NULL, -1, 0); - /* Overlap the front half of the other slot. */ + /* + * Verify that overlap in the guest_memfd bindings (i.e. in guest_memfd + * file offsets), but _not_ in the GPA space, fails with -EEXIST. + */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, + MEM_REGION_SIZE * 2, + 0, memfd, MEM_REGION_SIZE); + TEST_ASSERT(r == -1 && errno == EEXIST, + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + /* And now the back half of the other slot's guest_memfd binding. */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, + MEM_REGION_SIZE * 2, + 0, memfd, MEM_REGION_SIZE * 3); + TEST_ASSERT(r == -1 && errno == EEXIST, + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + /* + * Repeat the overlap tests, but this time with overlap in the memslots + * GPA space. Regardless of where there is overlap, KVM should return + * -EEXIST. + */ r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA * 2 - MEM_REGION_SIZE, MEM_REGION_SIZE * 2, From 060682bf3a868150346011f1494cb23dd687cd94 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 22 May 2026 10:15:34 -0700 Subject: [PATCH 03/85] KVM: selftests: Cast guest_memfd fd to a signed int when checking for >= 0 When conditionally closing a memory region's guest_memfd file descriptor, cast the field to a signed it so that negative values are correctly detected. Because selftests reuse "struct kvm_userspace_memory_region2" instead of providing custom storage, they pick up the kernel uAPI's __u32 definition of the file descriptor, not the more common "int" definition, e.g. that's used for userspace_mem_region.fd. Fixes: bb2968ad6c33 ("KVM: selftests: Add support for creating private memslots") Reported-by: Bibo Mao Closes: https://lore.kernel.org/all/20260508015013.4108345-1-maobibo@loongson.cn Reviewed-by: Bibo Mao Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260522171535.3525890-2-seanjc@google.com Signed-off-by: Sean Christopherson (cherry picked from commit 849d65e27bd62bca9a9383f7dfa14da711f4bebc) Signed-off-by: Wentao Guan --- tools/testing/selftests/kvm/lib/kvm_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 8b0690914994f..02843dc7d2dad 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -673,7 +673,7 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret)); close(region->fd); } - if (region->region.guest_memfd >= 0) + if ((int)region->region.guest_memfd >= 0) close(region->region.guest_memfd); free(region); From 6290074491fbcb33586582a9dee197d5dc51bd96 Mon Sep 17 00:00:00 2001 From: Deepanshu Kartikey Date: Sun, 31 May 2026 06:47:21 +0530 Subject: [PATCH 04/85] wifi: mac80211: limit injected antenna index in ieee80211_parse_tx_radiotap When parsing the radiotap header of an injected frame, ieee80211_parse_tx_radiotap() uses the IEEE80211_RADIOTAP_ANTENNA value directly as a shift count: info->control.antennas |= BIT(*iterator.this_arg); *iterator.this_arg is an 8-bit value taken straight from the frame supplied by userspace, so BIT() can be asked to shift by up to 255. That is undefined behaviour on the unsigned long and is reported by UBSAN: UBSAN: shift-out-of-bounds in net/mac80211/tx.c:2174:30 shift exponent 235 is too large for 64-bit type 'unsigned long' Call Trace: ieee80211_parse_tx_radiotap+0xadb/0x1950 net/mac80211/tx.c:2174 ieee80211_monitor_start_xmit+0xb1f/0x1250 net/mac80211/tx.c:2451 ... packet_sendmsg+0x3eb6/0x50f0 net/packet/af_packet.c:3109 info->control.antennas is a 2-bit bitmap (u8 antennas:2), so only antenna indices 0 and 1 can ever be represented. Ignore any larger value instead of shifting out of bounds. Reported-by: syzbot+8e0622f6d9446420271f@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=8e0622f6d9446420271f Fixes: ef246a1480cc ("wifi: mac80211: support antenna control in injection") Signed-off-by: Deepanshu Kartikey Link: https://patch.msgid.link/20260531011721.102941-1-kartikey406@gmail.com Signed-off-by: Johannes Berg (cherry picked from commit 6c0cf89f36ac0c0fd8687a4ccdce2efb23a9c663) Signed-off-by: Wentao Guan --- net/mac80211/tx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 0cfa211b04b63..46c47bee776fa 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2183,7 +2183,9 @@ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, case IEEE80211_RADIOTAP_ANTENNA: /* this can appear multiple times, keep a bitmap */ - info->control.antennas |= BIT(*iterator.this_arg); + /* control.antennas is only a 2-bit bitmap */ + if (*iterator.this_arg < 2) + info->control.antennas |= BIT(*iterator.this_arg); break; case IEEE80211_RADIOTAP_DATA_RETRIES: From fd1ad32fb02ee88257097145cdee56db08b8b43a Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Tue, 2 Jun 2026 07:49:02 -0700 Subject: [PATCH 05/85] perf/x86/intel/uncore: Fix discovery unit lookup for multi-die systems In uncore_find_add_unit(), PMON units with the same unit ID may be added to the uncore discovery RB-tree for different dies. These units are distinguished by node->die. However, intel_generic_uncore_box_ctl() uses a fixed die ID of -1 when looking up the discovery unit, which may retrieve the wrong node on multi-die systems. Use box->dieid instead so the correct discovery unit is selected. No functional issue has been observed so far because currently supported platforms happen to use the same unit control register for such units. Remove WARN_ON_ONCE() because with the above change a NULL unit can be expected, e.g. when a CPU die is offline during uncore enumeration and the unit is not added to the RB-tree. In this case, intel_uncore_find_discovery_unit() returns NULL once the die becomes online, and it is expected that the PMU box is not functional for that die. Fixes: b1d9ea2e1ca4 ("perf/x86/uncore: Apply the unit control RB tree to MSR uncore units") Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260602144908.263680-2-zide.chen@intel.com (cherry picked from commit 63f48abd55d0417996bca86022925c853a2b436b) Signed-off-by: Wentao Guan --- arch/x86/events/intel/uncore_discovery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 6765f31479c5d..5a7a65aec263d 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -479,8 +479,8 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) struct intel_uncore_discovery_unit *unit; unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes, - -1, box->pmu->pmu_idx); - if (WARN_ON_ONCE(!unit)) + box->dieid, box->pmu->pmu_idx); + if (!unit) return 0; return unit->addr; From b933a683b3f88416f06b3e468e33c4383e767f3c Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Fri, 29 May 2026 18:23:26 +0100 Subject: [PATCH 06/85] fs/proc/task_mmu: use huge_page_size() in pagemap_scan_hugetlb_entry() The partial-page check compares against HPAGE_SIZE (PMD_SIZE), which is wrong for gigantic hugetlb hstates (e.g. 1G). The walker hands the callback a huge_page_size()-sized range, never start + HPAGE_SIZE, so the comparison always declares it partial and aborts the WP. Compare against the actual hstate's page size. Link: https://lore.kernel.org/20260529172331.356655-3-kas@kernel.org Fixes: 52526ca7fdb9 ("fs/proc/task_mmu: implement IOCTL to get and optionally clear info about PTEs") Signed-off-by: Kiryl Shutsemau Reported-by: Sashiko AI review Reviewed-by: Lorenzo Stoakes Reviewed-by: Dev Jain Cc: David Hildenbrand Cc: Michal Hocko Cc: Mike Rapoport Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Balbir Singh Cc: Signed-off-by: Andrew Morton (cherry picked from commit 1b074e3270e1c061c829150c742eb83bad4dddd1) Signed-off-by: Wentao Guan --- fs/proc/task_mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 01ba7ff218b7e..fdd34c8341d68 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2297,7 +2297,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, if (~categories & PAGE_IS_WRITTEN) goto out_unlock; - if (end != start + HPAGE_SIZE) { + if (end != start + huge_page_size(hstate_vma(vma))) { /* Partial HugeTLB page WP isn't possible. */ pagemap_scan_backout_range(p, start, end); p->arg.walk_end = start; From 80de64dd1b4057ba07a40bbe1c6fb1ba03c5a61e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Fri, 29 May 2026 18:23:27 +0100 Subject: [PATCH 07/85] fs/proc/task_mmu: fix hugetlb self-deadlock in pagemap_scan_pte_hole() A PAGEMAP_SCAN ioctl requesting PM_SCAN_WP_MATCHING on a hugetlb VMA hangs the calling thread, unkillably, as soon as the scan reaches an unpopulated part of the range: do_pagemap_scan() walk_page_range() walk_hugetlb_range() hugetlb_vma_lock_read() # take the vma lock for read ... pagemap_scan_pte_hole() # ... ->pte_hole() for a hole uffd_wp_range() change_protection() hugetlb_change_protection() hugetlb_vma_lock_write() # ... and block taking it for write walk_hugetlb_range() holds the hugetlb vma lock for read across the whole walk. A present entry goes to ->hugetlb_entry(); an unpopulated one goes to ->pte_hole(), i.e. pagemap_scan_pte_hole(). To write-protect the hole that handler calls uffd_wp_range(), which on a hugetlb VMA reaches hugetlb_change_protection() and takes the same vma lock for write. The thread then blocks in down_write() waiting for the read lock it is itself holding. The populated path avoids this: pagemap_scan_hugetlb_entry() write-protects the entry inline under the page-table lock and never enters hugetlb_change_protection(). Do the same for holes. Fault in the page table and install the uffd-wp marker directly with make_uffd_wp_huge_pte() under the page-table lock, rather than routing through uffd_wp_range(). That is the same sequence hugetlb_change_protection() runs for an unpopulated entry, minus the vma write lock -- which is safe to skip because PMD sharing is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() runs at registration), leaving nothing for that lock to serialise against. Link: https://lore.kernel.org/20260529172331.356655-4-kas@kernel.org Fixes: 52526ca7fdb9 ("fs/proc/task_mmu: implement IOCTL to get and optionally clear info about PTEs") Signed-off-by: Kiryl Shutsemau Reported-by: Sashiko AI review Assisted-by: Claude:claude-opus-4-8 Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Balbir Singh Cc: Signed-off-by: Andrew Morton (cherry picked from commit e92d92bbafb264dc0518d52b846a3c07ed8d523f) [a31d14125b281 ("fs/proc/task_mmu: fix hugetlb self-deadlock in pagemap_scan_pte_hole()", upstream e92d92bbafb264) carries the three-argument huge_ptep_get(mm, addr, ptep) from newer kernels, but this tree's huge_ptep_get() takes a single ptep argument, like every other call site in this file. Fix the build.] Signed-off-by: Wentao Guan --- fs/proc/task_mmu.c | 59 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index fdd34c8341d68..f120d73bf11b9 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2314,8 +2314,62 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, return ret; } + +/* + * Write-protect the unpopulated hugetlb entries covering [addr, end) by + * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() + * does for populated entries. + * + * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the + * loop normally runs a single iteration; it is written to cover the full range + * in case the walker ever coalesces adjacent holes. + * + * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- + * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table + * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the + * same VMA, so the scanning thread would deadlock against itself. PMD sharing + * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so + * the vma lock guards nothing that matters for these entries anyway. + */ +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct hstate *h = hstate_vma(vma); + unsigned long psize = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pte_t *ptep; + pte_t pte; + + for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { + ptep = huge_pte_alloc(mm, vma, addr, psize); + if (!ptep) + return -ENOMEM; + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(ptep); + make_uffd_wp_huge_pte(vma, addr, ptep, pte); + /* + * A none entry has no cached translation, so installing the + * marker needs no TLB flush. Flush only if a fault populated + * the entry between huge_pte_alloc() and the page table lock. + */ + if (!huge_pte_none(pte)) + flush_hugetlb_tlb_range(vma, addr, addr + psize); + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + return 0; +} #else #define pagemap_scan_hugetlb_entry NULL +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + return 0; +} #endif static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, @@ -2335,7 +2389,10 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, if (~p->arg.flags & PM_SCAN_WP_MATCHING) return ret; - err = uffd_wp_range(vma, addr, end - addr, true); + if (is_vm_hugetlb_page(vma)) + err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); + else + err = uffd_wp_range(vma, addr, end - addr, true); if (err < 0) ret = err; From e13e09b63e51479f6e15d949dfb8b2f8445d2362 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Fri, 5 Jun 2026 04:33:06 -0700 Subject: [PATCH 08/85] wifi: mt76: mt7996: fix potential tx_retries underflow When FIELD_GET returns 0 for the retry count, subtracting 1 causes an unsigned integer underflow, resulting in tx_retries becoming a very large value (0xFFFFFFFF for u32). Fix by checking if count is non-zero before subtracting 1. Fixes: 2461599f835e ("wifi: mt76: mt7996: get tx_retries and tx_failed from txfree") Signed-off-by: Ryder Lee Link: https://patch.msgid.link/20260605113306.3485554-4-ryder.lee@mediatek.com Signed-off-by: Felix Fietkau (cherry picked from commit 4d8bba99d645bcb46a442b18eb42402610cba03a) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7996/mac.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 096a4f48b2898..eca6bfe45007a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -1146,13 +1146,13 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) spin_unlock_bh(&mdev->sta_poll_lock); continue; } else if (info & MT_TXFREE_INFO_HEADER) { - u32 tx_retries = 0, tx_failed = 0; + u32 tx_retries = 0, tx_failed = 0, count; if (!wcid) continue; - tx_retries = - FIELD_GET(MT_TXFREE_INFO_COUNT, info) - 1; + count = FIELD_GET(MT_TXFREE_INFO_COUNT, info); + tx_retries = count ? count - 1 : 0; tx_failed = tx_retries + !!FIELD_GET(MT_TXFREE_INFO_STAT, info); From 3fcb235483b36423166be885457e31d4bfe81bc5 Mon Sep 17 00:00:00 2001 From: HyeongJun An Date: Sat, 6 Jun 2026 02:49:05 +0900 Subject: [PATCH 09/85] platform/x86: intel-hid: Protect ACPI notify handler against recursion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit e2ffcda16290 ("ACPI: OSL: Allow Notify () handlers to run on all CPUs") ACPI notify handlers like the intel-hid notify_handler() may run on multiple CPU cores racing with themselves. On convertibles and detachables (matched by DMI chassis-type 31 and 32 in dmi_auto_add_switch[]) the SW_TABLET_MODE input device is registered lazily from notify_handler() on the first tablet-mode event, via intel_hid_switches_setup(). When two such events race on different CPUs both can pass the !priv->switches check and register the priv->switches input device twice, resulting in a duplicate sysfs entry and a subsequent NULL pointer dereference. This is the same class of bug fixed by commit e075c3b13a0a ("platform/x86: intel-vbtn: Protect ACPI notify handler against recursion") for the sibling intel-vbtn driver. Protect intel-hid notify_handler() from racing with itself with a mutex to fix this. Fixes: e2ffcda16290 ("ACPI: OSL: Allow Notify () handlers to run on all CPUs") Cc: stable@vger.kernel.org Signed-off-by: HyeongJun An Link: https://patch.msgid.link/20260605174905.131095-1-sammiee5311@gmail.com Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen (cherry picked from commit c085d82613d5618814b84406c8b2d64f1bc305e7) Signed-off-by: Wentao Guan --- drivers/platform/x86/intel/hid.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 346fbad34e24a..faf501466c91d 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -7,11 +7,13 @@ */ #include +#include #include #include #include #include #include +#include #include #include #include "../dual_accel_detect.h" @@ -210,6 +212,7 @@ static const struct dmi_system_id dmi_auto_add_switch[] = { }; struct intel_hid_priv { + struct mutex mutex; /* Avoid notify_handler() racing with itself */ struct input_dev *input_dev; struct input_dev *array; struct input_dev *switches; @@ -538,6 +541,8 @@ static void notify_handler(acpi_handle handle, u32 event, void *context) unsigned long long ev_index; int err; + guard(mutex)(&priv->mutex); + /* * Some convertible have unreliable VGBS return which could cause incorrect * SW_TABLET_MODE report, in these cases we enable support when receiving @@ -689,6 +694,10 @@ static int intel_hid_probe(struct platform_device *device) return -ENOMEM; dev_set_drvdata(&device->dev, priv); + err = devm_mutex_init(&device->dev, &priv->mutex); + if (err) + return err; + /* See dual_accel_detect.h for more info on the dual_accel check. */ if (enable_sw_tablet_mode == TABLET_SW_AUTO) { if (dmi_check_system(dmi_vgbs_allow_list)) From 4828281a27bfb54e8149c8ec280b2392fbc2662a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 8 Jun 2026 16:12:57 +0100 Subject: [PATCH 10/85] vfio/qat: fix f_pos race in qat_vf_resume_write() qat_vf_resume_write() checks filp->f_pos before taking migf->lock, but copies into the migration-state buffer after taking the lock and re-reading the shared file position. Two concurrent writers could therefore pass the bounds check with the old offset, then have the second writer copy after the first advanced f_pos, writing past the end of the migration-state buffer. Take migf->lock before doing the boundary checks. Fixes: bb208810b1ab ("vfio/qat: Add vfio_pci driver for Intel QAT SR-IOV VF devices") Reviewed-by: Ahsan Atta Signed-off-by: Giovanni Cabiddu Link: https://lore.kernel.org/r/20260608151317.136613-1-giovanni.cabiddu@intel.com Signed-off-by: Alex Williamson (cherry picked from commit 4ec5e932e636896e97e4c6a8205b0ac76d52421a) Signed-off-by: Wentao Guan --- drivers/vfio/pci/qat/main.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/vfio/pci/qat/main.c b/drivers/vfio/pci/qat/main.c index 1e3563fe7cab2..1841ed3c93db1 100644 --- a/drivers/vfio/pci/qat/main.c +++ b/drivers/vfio/pci/qat/main.c @@ -304,14 +304,18 @@ static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf, return -ESPIPE; offs = &filp->f_pos; - if (*offs < 0 || - check_add_overflow(len, *offs, &end)) - return -EOVERFLOW; + mutex_lock(&migf->lock); - if (end > mig_dev->state_size) - return -ENOMEM; + if (*offs < 0 || check_add_overflow(len, *offs, &end)) { + done = -EOVERFLOW; + goto out_unlock; + } + + if (end > mig_dev->state_size) { + done = -ENOMEM; + goto out_unlock; + } - mutex_lock(&migf->lock); if (migf->disabled) { done = -ENODEV; goto out_unlock; From 3aa0be6349560f00000d9596baf8d8ad16d91c89 Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Fri, 12 Jun 2026 14:45:01 +0800 Subject: [PATCH 11/85] net: ethernet: mtk_wed: debugfs: correct index in wed_amsdu_show() WED_MON_AMSDU_ENG_CNT point to different entry by 'base+n*offset' mode, correct the wed amsdu entry number in wed_amsdu_show(). Fixes: 3f3de094e8342 ("net: ethernet: mtk_wed: debugfs: add WED 3.0 debugfs entries") Signed-off-by: Wentao Guan Link: https://patch.msgid.link/20260612064501.203058-1-guanwentao@uniontech.com Signed-off-by: Jakub Kicinski (cherry picked from commit 14a8bc41ce9edae42d56466063a7f2c84a16c45c) Signed-off-by: Wentao Guan --- drivers/net/ethernet/mediatek/mtk_wed_debugfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c index 781c691473e14..519c364e87d19 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c @@ -310,9 +310,9 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_MAX_QGPP_CNT), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_CUR_ENTRY), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG2 INFO"), @@ -414,7 +414,7 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_CUR_ENTRY), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(4), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG8 INFO"), From d96eb28991424c0847381407de6bdc46b3fb9a5d Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Sun, 28 Jun 2026 18:11:18 +0800 Subject: [PATCH 12/85] selftests/mm: pagemap_ioctl: use the correct page size for transact_test() There are several places in transact_test() where we use the hardcoded 0x1000 (4k) as page size, which is not always correct for architectures supporting multiple page sizes. Switch to use the correct page size. Otherwise ./ksft_pagemap.sh on a 16k-page-size arm64 box fails with $ ./ksft_pagemap.sh [...] # ok 96 mprotect_tests Both pages written after remap and mprotect # ok 97 mprotect_tests Clear and make the pages written # Bail out! ioctl failed # # Planned tests != run tests (117 != 97) # # Totals: pass:97 fail:0 xfail:0 xpass:0 skip:0 error:0 # [FAIL] not ok 1 pagemap_ioctl # exit=1 # SUMMARY: PASS=0 SKIP=0 FAIL=1 1..1 Link: https://lore.kernel.org/20260628101118.35861-1-zenghui.yu@linux.dev Fixes: 46fd75d4a3c9 ("selftests: mm: add pagemap ioctl tests") Signed-off-by: Zenghui Yu Cc: Muhammad Usama Anjum Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zenghui Yu Cc: Signed-off-by: Andrew Morton (cherry picked from commit dccf636bf1e68c3fda92f0c9e1018ab7e0ac8b2c) Signed-off-by: Wentao Guan --- tools/testing/selftests/mm/pagemap_ioctl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 6d9b7ec9a1ba0..fb8599b5f1bbb 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -1360,7 +1360,7 @@ void *thread_proc(void *mem) ksft_exit_fail_msg("pthread_barrier_wait\n"); for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); ret = pthread_barrier_wait(&end_barrier); if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) @@ -1394,15 +1394,15 @@ static void transact_test(int page_size) if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) ksft_exit_fail_msg("pthread_barrier_init\n"); - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, + mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); + wp_init(mem, page_size * nthreads * pages_per_thread); + wp_addr_range(mem, page_size * nthreads * pages_per_thread); - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); + memset(mem, 0, page_size * nthreads * pages_per_thread); count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); ksft_test_result(count > 0, "%s count %u\n", __func__, count); @@ -1411,7 +1411,7 @@ static void transact_test(int page_size) finish = 0; for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); + pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); extra_pages = 0; for (i = 0; i < iter_count; ++i) { From 3a421542414c1aa92c2a80aac3ff6b8249515e63 Mon Sep 17 00:00:00 2001 From: Zong-Zhe Yang Date: Thu, 25 Jun 2026 14:15:42 +0800 Subject: [PATCH 13/85] wifi: rtw89: fw: fix link ID filling for LPS MLO common info The link ID field in H2C command of LPS MLO common info is incorrectly filled with the PHY index. Fix it with the target link ID. Fixes: 20380a039ddd ("wifi: rtw89: phy: add H2C command to send detail RX gain and link parameters for PS mode") Signed-off-by: Zong-Zhe Yang Signed-off-by: Ping-Ke Shih Link: https://patch.msgid.link/20260625061545.44808-8-pkshih@realtek.com (cherry picked from commit c1eabaaa088ddbb1b937cd339adfa4c18e93c93d) Signed-off-by: Wentao Guan --- drivers/net/wireless/realtek/rtw89/fw.c | 4 ++-- drivers/net/wireless/realtek/rtw89/fw.h | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 516e48b6b8932..2a35b171a5469 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -3393,7 +3393,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, h2c->rfe_type = efuse->rfe_type; h2c->rssi_main = U8_MAX; - memset(h2c->link_id, 0xfe, RTW89_BB_PS_LINK_BUF_MAX); + memset(h2c->link_id, RTW89_BB_PS_LINK_ID_SKIP, RTW89_BB_PS_LINK_BUF_MAX); rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) { u8 phy_idx = rtwvif_link->phy_idx; @@ -3401,7 +3401,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, bb = rtw89_get_bb_ctx(rtwdev, phy_idx); chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); - h2c->link_id[phy_idx] = phy_idx; + h2c->link_id[phy_idx] = link_id; h2c->central_ch[phy_idx] = chan->channel; h2c->pri_ch[phy_idx] = chan->primary_channel; h2c->band[phy_idx] = chan->band_type; diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 62cb42e8ec234..cfff8c99d6964 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -2031,6 +2031,8 @@ enum rtw89_bb_link_rx_gain_table_type { RTW89_BB_PS_LINK_RX_GAIN_TAB_MAX, }; +#define RTW89_BB_PS_LINK_ID_SKIP 0xfe + enum rtw89_bb_ps_link_buf_id { RTW89_BB_PS_LINK_BUF_0 = 0x00, RTW89_BB_PS_LINK_BUF_1 = 0x01, From 4503ee7589534e57228a21cf4ba5744c08f3016f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 8 Jun 2026 15:59:40 +0100 Subject: [PATCH 14/85] crypto: qat - cancel work on re-enable SR-IOV timeout The QAT reset worker queues SR-IOV reenable work using a work_struct and completion embedded in an on-stack adf_sriov_dev_data. If the completion wait times out, the reset worker can return while device_sriov_wq still holds or executes the stack-backed work item. Cancel the work on the device_sriov_wq on timeout before the stack frame unwinds. Fixes: 4469f9b23468 ("crypto: qat - re-enable sriov after pf reset") Signed-off-by: Giovanni Cabiddu Reviewed-by: Ahsan Atta Signed-off-by: Herbert Xu (cherry picked from commit 455b0f3ac9e254edab9f5a873d337abe5e6e3604) Signed-off-by: Wentao Guan --- drivers/crypto/intel/qat/qat_common/adf_aer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 179d62cb891d1..6d3a268f57aa9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -140,6 +140,8 @@ static void adf_device_reset_worker(struct work_struct *work) queue_work(device_sriov_wq, &sriov_data.sriov_work); if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); From 0b7e19c0636af307f5c53e25072c2ed52c33ccc8 Mon Sep 17 00:00:00 2001 From: Sumeet Pawnikar Date: Thu, 23 Jul 2026 22:53:20 +0530 Subject: [PATCH 15/85] powercap: intel_rapl_tpmi: Handle PMU registration failure during probe intel_rapl_tpmi_probe() invokes rapl_package_add_pmu() but ignores its return value, so a PMU registration failure would leave the driver reporting probe success despite the PMU being absent, with no log trace. Since PMU registration is an optional auxiliary feature for perf energy counters, its failure should not break the primary powercap functionality. Check the return value and log a warning to ensure graceful degradation. Fixes: 963a9ad3c589 ("powercap: intel_rapl_tpmi: Enable PMU support") Signed-off-by: Sumeet Pawnikar [ rjw: Changed the log level of the new message to "info" ] Link: https://patch.msgid.link/20260723172321.5960-1-sumeet4linux@gmail.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 9229916d59918ec9d3639e7263e1e97be638e361) Signed-off-by: Wentao Guan --- drivers/powercap/intel_rapl_tpmi.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 1618138c5cac1..1fae66a609298 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -313,7 +313,10 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, goto err; } - rapl_package_add_pmu(trp->rp); + ret = rapl_package_add_pmu(trp->rp); + if (ret) + dev_info(&auxdev->dev, "Failed to add RAPL PMU for Package%d, %d\n", + info->package_id, ret); auxiliary_set_drvdata(auxdev, trp); From c37115b769056c854c9705a88ceadb3e3ccaeef1 Mon Sep 17 00:00:00 2001 From: Eason Lai Date: Wed, 1 Jul 2026 09:06:54 +0800 Subject: [PATCH 16/85] wifi: mt76: mt792x: fix use-after-free in mt76_rx_poll_complete A use-after-free issue occurs in mt76_rx_poll_complete due to a race condition. The STA has already been removed, but the rx_status still had a pointer to the wcid in the STA. Set the links' wcid pointers to be NULL for a MLD in mt7925_sta_pre_rcu_remove() BUG: KASAN: invalid-access in mt76_rx_poll_complete+0x280/0x470 Call trace: dump_backtrace+0xec/0x128 show_stack+0x18/0x28 dump_stack_lvl+0x40/0xc8 print_report+0x1b8/0x710 kasan_report+0xe0/0x144 do_bad_area+0x120/0x260 do_tag_check_fault+0x20/0x34 do_mem_abort+0x54/0xa8 el1_abort+0x3c/0x5c el1h_64_sync_handler+0x40/0xcc el1h_64_sync+0x7c/0x80 mt76_rx_poll_complete+0x280/0x470 mt76_dma_rx_poll+0x114/0x51c mt792x_poll_rx+0x60/0xf8 napi_threaded_poll_loop+0xe0/0x450 napi_threaded_poll+0x80/0x9c kthread+0x11c/0x158 ret_from_fork+0x10/0x20 Fixes: c948b5da6bbe ("wifi: mt76: mt7925: add Mediatek Wi-Fi7 driver for mt7925 chips") Signed-off-by: Eason Lai Link: https://patch.msgid.link/20260701010654.956863-1-eason.lai@mediatek.com Signed-off-by: Felix Fietkau (cherry picked from commit 217f9e7bb02558759be9d9ecfe532e9708741c50) Signed-off-by: Wentao Guan --- .../net/wireless/mediatek/mt76/mt7925/main.c | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index bc35ff376f384..262e18f68042d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -2208,6 +2208,40 @@ static void mt7925_unassign_vif_chanctx(struct ieee80211_hw *hw, mutex_unlock(&dev->mt76.mutex); } +static void mt7925_sta_pre_rcu_remove(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt76_phy *phy = hw->priv; + struct mt76_dev *dev = phy->dev; + struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; + + mutex_lock(&dev->mutex); + spin_lock_bh(&dev->status_lock); + + if (ieee80211_vif_is_mld(vif)) { + struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + unsigned long valid = mvif->valid_links; + struct mt792x_link_sta *mlink; + unsigned int link_id; + + for_each_set_bit(link_id, &valid, IEEE80211_MLD_MAX_NUM_LINKS) { + mlink = mt792x_sta_to_link(msta, link_id); + if (!mlink || !mlink->wcid.sta) + continue; + if (mlink->wcid.idx < ARRAY_SIZE(dev->wcid)) + rcu_assign_pointer(dev->wcid[mlink->wcid.idx], + NULL); + } + } else { + rcu_assign_pointer(dev->wcid[wcid->idx], NULL); + } + + spin_unlock_bh(&dev->status_lock); + mutex_unlock(&dev->mutex); +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, @@ -2220,7 +2254,7 @@ const struct ieee80211_ops mt7925_ops = { .start_ap = mt7925_start_ap, .stop_ap = mt7925_stop_ap, .sta_state = mt76_sta_state, - .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, + .sta_pre_rcu_remove = mt7925_sta_pre_rcu_remove, .set_key = mt7925_set_key, .sta_set_decap_offload = mt7925_sta_set_decap_offload, #if IS_ENABLED(CONFIG_IPV6) From 5b8a1e5558f547fa2c9dd313eeabdface816b573 Mon Sep 17 00:00:00 2001 From: Devin Wittmayer Date: Tue, 14 Jul 2026 19:33:48 -0700 Subject: [PATCH 17/85] wifi: mt76: mt7925: ensure tx headroom in usb_sdio_tx_prepare_skb mt7925_usb_sdio_tx_prepare_skb() pushes a TX descriptor and a USB header onto every skb and assumes the headroom for them is already there. That holds for locally generated traffic, where mac80211 reserves hw->extra_tx_headroom, but forwarded frames are sent through ieee80211_8023_xmit(), which does not reserve it. Bridge a wired interface to an mt7925u AP and the first forwarded frame that arrives short panics the kernel: skbuff: skb_under_panic: len:415 put:4 tail:0x19b end:0x640 dev:wlan1 kernel BUG at net/core/skbuff.c:212! Call trace: skb_panic+0x58/0x60 (P) skb_push+0x58/0x60 mt7925_usb_sdio_tx_prepare_skb+0xf8/0x1b8 [mt7925_common] mt76u_tx_queue_skb+0xa0/0x1f8 [mt76_usb] __mt76_tx_queue_skb+0x54/0xe8 [mt76] mt76_txq_schedule.part.0+0x204/0x478 [mt76] mt76_txq_schedule_all+0x50/0x80 [mt76] mt792x_tx_worker+0x68/0x100 [mt792x_lib] __mt76_worker_fn+0x84/0x150 [mt76] Whether a given setup hits it depends on how much headroom the ingress netdev leaves in its rx skbs. Reproduced on a Raspberry Pi 5 bridging onboard ethernet to a Netgear A9000; originally reported on an MT7986 router running OpenWrt. Nick Morrow's testing on a Pi 4 (bcmgenet), which leaves more headroom, helped narrow the trigger to the ingress path. The same bug was fixed on mt7921 by commit 98c4d0abf5c4 ("mt76: mt7921: don't assume adequate headroom for SDIO headers"), but mt7925 was copied from mt7921 without the fix. Add the same guard here. Fixes: c948b5da6bbe ("wifi: mt76: mt7925: add Mediatek Wi-Fi7 driver for mt7925 chips") Cc: stable@vger.kernel.org Link: https://github.com/morrownr/mt76/issues/52 Signed-off-by: Devin Wittmayer Link: https://patch.msgid.link/20260715023348.59506-1-lucid_duck@justthetip.ca Signed-off-by: Felix Fietkau (cherry picked from commit ef3e34874d2332d0f63e72c2c35ce5c93568c125) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7925/mac.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 58b46be0aaed7..a8fbf3b50f3d5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -1432,6 +1432,10 @@ int mt7925_usb_sdio_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (!wcid) wcid = &dev->mt76.global_wcid; + err = skb_cow_head(skb, MT_SDIO_TXD_SIZE + MT_SDIO_HDR_SIZE); + if (err) + return err; + if (sta) { struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; From fa7d80e375198d26e54ca35750d2ba34a9013efc Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Tue, 30 Jun 2026 17:02:18 +0800 Subject: [PATCH 18/85] wifi: mt76: mt7925: cancel pending mlo_pm_work If the device is reset, suspended or unregistered within that window, the pending work can still run and access vif/bss data that may already be freed, or send MCU commands while the firmware is not available. Add cancel_delayed_work_sync(&dev->mlo_pm_work) in all relevant teardown and suspend paths: - mt7925_mac_reset_work() (chip reset recovery) - mt7925e_unregister_device() (PCIe unbind) - mt7925_pci_suspend() (PCIe bus suspend) - mt7925_suspend() (mac80211 suspend) - mt7925u_suspend() (USB bus / runtime suspend) This ensures the work is stopped before the device state becomes invalid. Assisted-by: kimi-cli:kimi-k2.7 code Assisted-by: atomcode:glm-5.2 #Reported-by Fixes: 276a568832577 ("wifi: mt76: mt7925: update the power-saving flow") Cc: stable@vger.kernel.org Signed-off-by: Wentao Guan Link: https://patch.msgid.link/20260630090218.3202029-1-guanwentao@uniontech.com Signed-off-by: Felix Fietkau (cherry picked from commit 2889e84282dda147f10b10d94cf0efd90a349c53) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7925/mac.c | 1 + drivers/net/wireless/mediatek/mt76/mt7925/main.c | 1 + drivers/net/wireless/mediatek/mt76/mt7925/pci.c | 2 ++ drivers/net/wireless/mediatek/mt76/mt7925/usb.c | 1 + 4 files changed, 5 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index a8fbf3b50f3d5..afed2c31dc800 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -1323,6 +1323,7 @@ void mt7925_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&dev->mphy.mac_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 262e18f68042d..05ea185ea7085 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -1543,6 +1543,7 @@ static int mt7925_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&phy->mt76->mac_work); cancel_delayed_work_sync(&dev->pm.ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); mt792x_mutex_acquire(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index a927198a299eb..76999ad0ed638 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -37,6 +37,7 @@ static void mt7925e_unregister_device(struct mt792x_dev *dev) mt76_for_each_q_rx(&dev->mt76, i) napi_disable(&dev->mt76.napi[i]); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); cancel_work_sync(&dev->reset_work); @@ -449,6 +450,7 @@ static int mt7925_pci_suspend(struct device *device) pm->suspended = true; flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); mt7925_roc_abort_sync(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c index 487d1e853e101..0e7520270ade6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c @@ -250,6 +250,7 @@ static int mt7925u_suspend(struct usb_interface *intf, pm_message_t state) pm->suspended = true; flush_work(&dev->reset_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, true); if (err) From 7b629673802367453867497f354509805d8d2f80 Mon Sep 17 00:00:00 2001 From: "Jared.Huang" Date: Wed, 17 Jun 2026 15:13:20 +0800 Subject: [PATCH 19/85] wifi: mt76: mt7925: fix msg len mismatch between driver and firmware The mt7925_tx_power_limit_tlv struct begins with a 4-byte rsv[] field that acts as a UNI command header prefix. The firmware dispatcher did not use the 4-byte rsv[] and will only check the payloads after the 4-byte rsv[] As a result, the total message length minus the 4-byte prefix. Fix this by setting len to msg_len - 4. Fixes: ccb186326bb6 ("wifi: mt76: mt7925: fix incorrect length field in txpower command") Signed-off-by: Jared.Huang Link: https://patch.msgid.link/20260617071320.1808499-1-jb.tsai@mediatek.com Signed-off-by: Felix Fietkau (cherry picked from commit 9ddb7487aa7cccb6e1880b4151ab5895109eb8d6) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7925/mcu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index 7f1aa94c79866..e732ba68d587f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -3453,7 +3453,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(msg_len); + tx_power_tlv->len = cpu_to_le16(msg_len - 4); switch (band) { case NL80211_BAND_2GHZ: From 0955bbb59b01547af14f6d57c2b780b2e70326da Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Wed, 22 Jul 2026 08:26:03 +0000 Subject: [PATCH 20/85] wifi: mt76: mt7915: write RX header translation bit to the correct register MT_MDP_DCR0_RX_HDR_TRANS_EN is a field of MT_MDP_DCR0, but monitor-mode handling applied it to the per-band MT_DMA_DCR0 register instead. As a result RX header translation was never disabled in the MDP when entering monitor mode, and an undocumented bit of MT_DMA_DCR0 was toggled. Target MT_MDP_DCR0, matching the mt7996 driver. Fixes: b2491018587a ("wifi: mt76: mt7915: fix monitor mode issues") Link: https://patch.msgid.link/20260722082610.2699628-11-nbd@nbd.name Signed-off-by: Felix Fietkau (cherry picked from commit 236145737480c4c0c515e09061d0d5f77cf52d3f) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7915/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index e696d19811406..305c3dcaaeef0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -495,7 +495,7 @@ static int mt7915_config(struct ieee80211_hw *hw, u32 changed) mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, enabled); - mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_MDP_DCR0_RX_HDR_TRANS_EN, + mt76_rmw_field(dev, MT_MDP_DCR0, MT_MDP_DCR0_RX_HDR_TRANS_EN, !dev->monitor_mask); mt76_testmode_reset(phy->mt76, true); mt76_wr(dev, MT_WF_RFCR(band), rxfilter); From ff47ae3056d4e0408ba4ae27a1785135e6ee93f3 Mon Sep 17 00:00:00 2001 From: Rex Lu Date: Wed, 22 Jul 2026 08:25:54 +0000 Subject: [PATCH 21/85] wifi: mt76: check txfree done event on the WED hw path Check the txfree done event DW1 bit 15 when WED is enabled, to avoid the driver reading a txfree done event before WED has finished reading it. No need to check this flag on WED v2, otherwise SER will occur. The bit position was previously defined as MT_DMA_CTL_BURST, which is unused; rename it to match its function on the txfree ring. Fixes: 83eafc9251d6 ("wifi: mt76: mt7996: add wed tx support") Signed-off-by: Rex Lu Signed-off-by: Shayne Chen Link: https://patch.msgid.link/20260722082610.2699628-2-nbd@nbd.name Signed-off-by: Felix Fietkau (cherry picked from commit 3310e71a74b176d3613dfb42b6bc630d99e90cbb) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/dma.c | 9 +++++++++ drivers/net/wireless/mediatek/mt76/dma.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index d592f55139427..7de5ddf3748a4 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -490,6 +490,15 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* on WED v3 the M_DONE bit signals that WED is done reading + * the txfree descriptor; WED v2 does not set it + */ + else if (dev->mmio.wed.version > 2 && + mt76_queue_is_wed_tx_free(q) && + !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) + return NULL; +#endif } q->tail = (q->tail + 1) % q->ndesc; diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 1de5a2b20f747..1e633b2f96d1a 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -11,7 +11,7 @@ #define MT_DMA_CTL_SD_LEN1 GENMASK(13, 0) #define MT_DMA_CTL_LAST_SEC1 BIT(14) -#define MT_DMA_CTL_BURST BIT(15) +#define MT_DMA_CTL_M_DONE BIT(15) #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) From 16a524d4ac0c0558548a3992364a6e2f9db55ed0 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Fri, 7 Aug 2026 09:44:36 +0800 Subject: [PATCH 22/85] tcp: fix icsk_ack.ato bitfield overflow On cross-region connections we observed delayed ACKs suddenly turning into immediate ACKs plus a TCP_MAX_QUICKACKS burst, as if the connection had just received its first data segment. Commit 95b9a87c6a6b ("tcp: record last received ipv6 flowlabel") squeezed icsk_ack.ato into 8 bits, sized for TCP_DELACK_MAX. But both writers still bound ato by icsk_rto, which can be well above 255 jiffies, so the bitfield assignment silently wraps mod 256: repeated delack timer misses double ato up to icsk_rto, storing 320 as 64 and 256 as 0, and ato == 0 is the "first data packet" sentinel in tcp_event_data_recv(). Clamp both writers to TCP_DELACK_MAX, which the static_assert already guarantees to fit and tcp_send_delayed_ack() effectively caps ato at anyway. Fixes: 95b9a87c6a6b ("tcp: record last received ipv6 flowlabel") Signed-off-by: Jiayuan Chen Reviewed-by: Neal Cardwell Link: https://patch.msgid.link/20260807014437.36687-1-jiayuan.chen@linux.dev Signed-off-by: Jakub Kicinski (cherry picked from commit 60837e4b840a9c3f7ec826e3584df0bc6542a2c2) Signed-off-by: Wentao Guan --- net/ipv4/tcp_input.c | 6 +++--- net/ipv4/tcp_timer.c | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index eeb944374f892..ee114e082dcbf 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -845,9 +845,9 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* The fastest case is the first. */ icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; } else if (m < icsk->icsk_ack.ato) { - icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; - if (icsk->icsk_ack.ato > icsk->icsk_rto) - icsk->icsk_ack.ato = icsk->icsk_rto; + icsk->icsk_ack.ato = min3((icsk->icsk_ack.ato >> 1) + (u32)m, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else if (m > icsk->icsk_rto) { /* Too long gap. Apparently sender failed to * restart window, so that we send ACKs quickly. diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index be40e424f27cd..4db663ba6c3d6 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -318,7 +318,9 @@ void tcp_delack_timer_handler(struct sock *sk) if (inet_csk_ack_scheduled(sk)) { if (!inet_csk_in_pingpong_mode(sk)) { /* Delayed ACK missed: inflate ATO. */ - icsk->icsk_ack.ato = min_t(u32, icsk->icsk_ack.ato << 1, icsk->icsk_rto); + icsk->icsk_ack.ato = min3((u32)icsk->icsk_ack.ato << 1, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else { /* Delayed ACK missed: leave pingpong mode and * deflate ATO. From 0f6a100585002e49330da502a97f8ed7b4bba77d Mon Sep 17 00:00:00 2001 From: Oleksij Rempel Date: Thu, 6 Aug 2026 15:47:16 +0200 Subject: [PATCH 23/85] net: phy: realtek: fix EEE advertisement write on the internal PHY MMD path In rtlgen_write_mmd(), the MDIO_AN_EEE_ADV case swaps the arguments to rtlgen_write_vend2(): it passes the MMD register number as the OCP address and the OCP address constant as the value. The caller's value is discarded and the write lands on the wrong register, so the EEE advertisement cannot be configured on the affected PHYs. Mirror rtlgen_read_mmd() and write the value to RTL_MDIO_AN_EEE_ADV. Fixes: da681ed73fb9 ("net: phy: realtek: improve mmd register access for internal PHY's") Signed-off-by: Oleksij Rempel Reviewed-by: Andrew Lunn Reviewed-by: Nicolai Buchwitz Link: https://patch.msgid.link/20260806134716.3511821-1-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski (cherry picked from commit 202fef9bbbf5784487eec27581389c6fb97c350d) Signed-off-by: Wentao Guan --- drivers/net/phy/realtek/realtek_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index 6ffa2ca14318a..71b3730de25bb 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -1171,7 +1171,7 @@ static int rtlgen_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, if (devnum == MDIO_MMD_VEND2) ret = rtlgen_write_vend2(phydev, regnum, val); else if (devnum == MDIO_MMD_AN && regnum == MDIO_AN_EEE_ADV) - ret = rtlgen_write_vend2(phydev, regnum, RTL_MDIO_AN_EEE_ADV); + ret = rtlgen_write_vend2(phydev, RTL_MDIO_AN_EEE_ADV, val); else ret = -EOPNOTSUPP; From 908fce167d0fc93b46c9d30163ecc9104a1c6ec4 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 13 Aug 2026 14:17:16 +0100 Subject: [PATCH 24/85] KVM: arm64: Avoid mismatched accesses to 'struct kvm_nvhe_init_params' When running with hVHE enabled, ___kvm_hyp_init() calls __kvm_init_el2_state() on the CPU initialisation path during onlining and resume from suspend. In order to avoid clobbering the link register across this call, it is stashed away in the 'tmp' member of 'struct kvm_nvhe_init_params', however this save/restore operation is performed with the stage-1 MMU disabled at EL2 and therefore gives rise to coherency problems because the field is not aligned or padded to the CWG. For example, a cacheable write to a physically-adjacent structure sharing the same cacheline could lead to an eviction and subsequent write-back, overwriting the saved LR while the incoming CPU is executing __kvm_init_el2_state(). Save the lr in far_el2 and remove the 'tmp' member from 'struct kvm_nvhe_init_params' altogether. Cc: Oliver Upton Cc: Marc Zyngier Fixes: afa9b48f327c ("KVM: arm64: Shave a few bytes from the EL2 idmap code") Signed-off-by: Will Deacon Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260813131717.5885-1-will@kernel.org Signed-off-by: Oliver Upton (cherry picked from commit 028d8df0a1e376c6a87409302d9b6131ea4374f6) Signed-off-by: Wentao Guan --- arch/arm64/include/asm/kvm_asm.h | 1 - arch/arm64/kernel/asm-offsets.c | 1 - arch/arm64/kvm/hyp/nvhe/hyp-init.S | 11 +++++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 443e30284753c..24b5e6b23417b 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -179,7 +179,6 @@ struct kvm_nvhe_init_params { unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; - unsigned long tmp; }; /* diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index aead3d38e570e..ea2d740db81c5 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -148,7 +148,6 @@ int main(void) DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); - DEFINE(NVHE_INIT_TMP, offsetof(struct kvm_nvhe_init_params, tmp)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 62c39a470eeea..11e16f98c67b4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -101,16 +101,19 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) and x2, x1, x2 cbz x2, 1f - // hVHE: Replay the EL2 setup to account for the E2H bit - // TPIDR_EL2 is used to preserve x0 across the macro maze... + /* + * hVHE: Replay the EL2 setup to account for the E2H bit + * TPIDR_EL2 and FAR_EL2 are used to preserve x0 and LR across + * the macro maze... + */ isb msr tpidr_el2, x0 - str lr, [x0, #NVHE_INIT_TMP] + msr far_el2, lr bl __kvm_init_el2_state + mrs lr, far_el2 mrs x0, tpidr_el2 - ldr lr, [x0, #NVHE_INIT_TMP] 1: ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] From 3c3b487e1052fa0257486d62acbd5bb847b12b25 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 27 Jun 2024 12:01:01 +0100 Subject: [PATCH 25/85] iommufd/iova_bitmap: Cache mapped length in iova_bitmap_map struct The amount of IOVA mapped will be used more often in iova_bitmap_set() in preparation to dynamically iterate the bitmap. Cache said length to avoid having to calculate it all the time. Link: https://lore.kernel.org/r/20240627110105.62325-8-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Kevin Tian Tested-by: Matt Ochs Signed-off-by: Jason Gunthorpe (cherry picked from commit a84c690e10ae03f1cddec908ac7f5068ceed67a8) Signed-off-by: Wentao Guan --- drivers/iommu/iommufd/iova_bitmap.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index 5e2a22a3f034e..1babeae211527 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -35,6 +35,9 @@ struct iova_bitmap_map { /* base IOVA representing bit 0 of the first page */ unsigned long iova; + /* mapped length */ + unsigned long length; + /* page size order that each bit granules to */ unsigned long pgshift; @@ -155,6 +158,8 @@ static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip); } +static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap); + /* * Pins the bitmap user pages for the current range window. * This is internal to IOVA bitmap and called when advancing the @@ -205,6 +210,7 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) * aligned. */ mapped->pgoff = offset_in_page(addr); + mapped->length = iova_bitmap_mapped_length(bitmap); return 0; } From 2461200cbb47055ca9363bbaa4607315cc278007 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 27 Jun 2024 12:01:02 +0100 Subject: [PATCH 26/85] iommufd/iova_bitmap: Move initial pinning to iova_bitmap_for_each() The pinned pages are only relevant when it starts iterating the bitmap so defer that into iova_bitmap_for_each(). Link: https://lore.kernel.org/r/20240627110105.62325-9-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Kevin Tian Tested-by: Matt Ochs Signed-off-by: Jason Gunthorpe (cherry picked from commit 781bc08797a2146400332acf2d7706793b51e20f) Signed-off-by: Wentao Guan --- drivers/iommu/iommufd/iova_bitmap.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index 1babeae211527..c0588f71e2148 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -268,9 +268,6 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, goto err; } - rc = iova_bitmap_get(bitmap); - if (rc) - goto err; return bitmap; err: @@ -424,6 +421,10 @@ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, { int ret = 0; + ret = iova_bitmap_get(bitmap); + if (ret) + return ret; + for (; !iova_bitmap_done(bitmap) && !ret; ret = iova_bitmap_advance(bitmap)) { ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap), From f884dc58d731583ed8fa5d1fa6615726e8a67d3a Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 27 Jun 2024 12:01:03 +0100 Subject: [PATCH 27/85] iommufd/iova_bitmap: Consolidate iova_bitmap_set exit conditionals There's no need to have two conditionals when they are closely tied together. Move the setting of bitmap::set_ahead_length after it checks for ::pages array out of bounds access. Link: https://lore.kernel.org/r/20240627110105.62325-10-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Kevin Tian Tested-by: Matt Ochs Signed-off-by: Jason Gunthorpe (cherry picked from commit 00fa1a89917fbc319909231e648439b26e8857af) Signed-off-by: Wentao Guan --- drivers/iommu/iommufd/iova_bitmap.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index c0588f71e2148..1947191e269aa 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -464,18 +464,18 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, last_bit - cur_bit + 1); void *kaddr; - if (unlikely(page_idx > last_page_idx)) + if (unlikely(page_idx > last_page_idx)) { + unsigned long left = + ((last_bit - cur_bit + 1) << mapped->pgshift); + + bitmap->set_ahead_length = left; break; + } kaddr = kmap_local_page(mapped->pages[page_idx]); bitmap_set(kaddr, offset, nbits); kunmap_local(kaddr); cur_bit += nbits; } while (cur_bit <= last_bit); - - if (unlikely(cur_bit <= last_bit)) { - bitmap->set_ahead_length = - ((last_bit - cur_bit + 1) << bitmap->mapped.pgshift); - } } EXPORT_SYMBOL_NS_GPL(iova_bitmap_set, IOMMUFD); From 7ac59d52f53a3f4615beac95db140368f06c570e Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 27 Jun 2024 12:01:04 +0100 Subject: [PATCH 28/85] iommufd/iova_bitmap: Dynamic pinning on iova_bitmap_set() Today zerocopy iova bitmaps use a static iteration scheme where it walks the bitmap data in a max iteration size of 2M of bitmap of data at a time. That translates to a fixed window of IOVA space that can span up to 64G (e.g. base pages, x86). Here 'window' refers to the IOVA space represented by the bitmap data it is iterating. This static scheme is the ideal one where the reported page-size is the same as the one behind the dirty tracker. However, problems start to appear when the dirty tracker may dirty in many PTE sizes beyond or unaligned at the boundaries of the iteration window. Such is the case for the IOMMU and commit 2780025e01e2 ("iommufd/iova_bitmap: Handle recording beyond the mapped pages") tried to fix the problem by handling the PTEs that get dirty which surprass the end of the iteration. But the fix was incomplete and it didn't handle all the data structure issues namely: 1) when there's nothing to dirty but the end of the iteration IOVA range is a IOMMU hugepage PTE that crosses iterations, when it goes to the next iteration it finds the other end of the said hugepage but don't account that it had checked for that IOPTE already. iommu driver then walk the IOVA space as if it is a new page without accounting that it is past the start of a bigger page which ends up setting (future) dirty bits slightly offset-ed. Note that the partial ranges here are self induced due as a result of the fixed 'window' scheme being unaligned to this hugepage IOPTE. 2) on the same line of thinking between pinning pages of different iterations it could allow DMA to mark PTEs as dirty on the second part of this previously mentioned partial hugepage. This leads to marking part of the hugepage as dirty but still clearing IOPTE leading to missed dirty data. So to fix these problems more fundamentally and avoid future ones: instead of iterating the whole bitmap in fixed chunks, instead only pin the bitmap pages when it has dirty bits to set. The logic is simple in iova_bitmap_set(): check where the current iova range to be marked as dirty is pinned and pin the bitmap pages where to-be-recorded @iova starts if it's not. If it's partially mapped out of the whole set, continue pinning it and set bits until the whole dirty-size is covered. The latter is more relevant with AMD iommu pgtable v1 format where you can have up 64G/128G/256G page sizes and thus you can set 64G at a time. Code also gets simpler and easier to follow. Fixing this without changing this iteration scheme means changing iommu drivers to ignore any partial pages and not clear dirty bits, which is a bit hacky. Though getting to walk only part of a IOMMU hugepage is a self-induced due to this iteration scheme as it doesn't (and can't) align the iteration boundary to the huge IOPTE at the end. Thus it can't know what the hugepage size the iteration should align to until it walks the begin/end. Dynamically pinning adds some comparisons inside iova_bitmap_set() to check if something needs to be pinned if the IOVA range is out of range. Though it has the benefit that non-dirty IOVA ranges only walk page tables without needing to pin any bitmap pages. This dynamic scheme should be better for IOMMUs where upper layers don't need or know what PTE sizes IOVAs map into (and there could be more than one PTE size[*]) until they walk the IOMMU page tables. A follow-up change will remove the iteration logic. [*] Specially on AMD v1 iommu pgtable format where most powers of two are supported as page-size. Link: https://lore.kernel.org/linux-iommu/6b90f949-48da-4cb3-ad9a-ed54f1351a9a@oracle.com/ Fixes: 2780025e01e2 ("iommufd/iova_bitmap: Handle recording beyond the mapped pages") Link: https://lore.kernel.org/r/20240627110105.62325-11-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Kevin Tian Tested-by: Matt Ochs Signed-off-by: Jason Gunthorpe (cherry picked from commit 7a7bba16244a5c55861d8fefea72cdbb8b05323e) Signed-off-by: Wentao Guan --- drivers/iommu/iommufd/iova_bitmap.c | 73 ++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index 1947191e269aa..e652861356e16 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -119,6 +119,9 @@ struct iova_bitmap { /* length of the IOVA range set ahead the pinned pages */ unsigned long set_ahead_length; + + /* true if it using the iterator otherwise it pins dynamically */ + bool iterator; }; /* @@ -339,6 +342,17 @@ static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) return remaining; } +/* + * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range. + */ +static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped, + unsigned long iova, size_t length) +{ + return mapped->npages && + (iova >= mapped->iova && + (iova + length - 1) <= (mapped->iova + mapped->length - 1)); +} + /* * Returns true if there's not more data to iterate. */ @@ -373,6 +387,27 @@ static int iova_bitmap_set_ahead(struct iova_bitmap *bitmap, return ret; } +/* + * Advances to a selected range, releases the current pinned + * pages and pins the next set of bitmap pages. + * Returns 0 on success or otherwise errno. + */ +static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, + unsigned long iova) +{ + unsigned long index; + + index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); + if (index >= bitmap->mapped_total_index) + return -EINVAL; + bitmap->mapped_base_index = index; + + iova_bitmap_put(bitmap); + + /* Pin the next set of bitmap pages */ + return iova_bitmap_get(bitmap); +} + /* * Advances to the next range, releases the current pinned * pages and pins the next set of bitmap pages. @@ -425,6 +460,7 @@ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, if (ret) return ret; + bitmap->iterator = true; for (; !iova_bitmap_done(bitmap) && !ret; ret = iova_bitmap_advance(bitmap)) { ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap), @@ -432,6 +468,7 @@ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, if (ret) break; } + bitmap->iterator = false; return ret; } @@ -451,11 +488,28 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long iova, size_t length) { struct iova_bitmap_map *mapped = &bitmap->mapped; - unsigned long cur_bit = ((iova - mapped->iova) >> - mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; - unsigned long last_bit = (((iova + length - 1) - mapped->iova) >> - mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; - unsigned long last_page_idx = mapped->npages - 1; + unsigned long cur_bit, last_bit, last_page_idx; + +update_indexes: + if (unlikely(!bitmap->iterator && + !iova_bitmap_mapped_range(mapped, iova, length))) { + + /* + * The attempt to advance the base index to @iova + * may fail if it's out of bounds, or pinning the pages + * returns an error. + * + * It is a no-op if within a iova_bitmap_for_each() closure. + */ + if (iova_bitmap_advance_to(bitmap, iova)) + return; + } + + last_page_idx = mapped->npages - 1; + cur_bit = ((iova - mapped->iova) >> + mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; + last_bit = (((iova + length - 1) - mapped->iova) >> + mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; do { unsigned int page_idx = cur_bit / BITS_PER_PAGE; @@ -468,8 +522,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long left = ((last_bit - cur_bit + 1) << mapped->pgshift); - bitmap->set_ahead_length = left; - break; + if (bitmap->iterator) { + bitmap->set_ahead_length = left; + return; + } + iova += (length - left); + length = left; + goto update_indexes; } kaddr = kmap_local_page(mapped->pages[page_idx]); From ad28ed503cd99ef127b96b053221fd7ba3cd05b2 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 27 Jun 2024 12:01:05 +0100 Subject: [PATCH 29/85] iommufd/iova_bitmap: Remove iterator logic The newly introduced dynamic pinning/windowing greatly simplifies the code and there's no obvious performance advantage that has been identified that justifies maintinaing both schemes. Remove the iterator logic and have iova_bitmap_for_each() just invoke the callback with the total iova/length. Fixes: 2780025e01e2 ("iommufd/iova_bitmap: Handle recording beyond the mapped pages") Link: https://lore.kernel.org/r/20240627110105.62325-12-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Kevin Tian Tested-by: Matt Ochs Signed-off-by: Jason Gunthorpe (cherry picked from commit 53e6b65693b68519dcfd384280bfc3d34c7398e2) Signed-off-by: Wentao Guan --- drivers/iommu/iommufd/iova_bitmap.c | 97 +---------------------------- 1 file changed, 2 insertions(+), 95 deletions(-) diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index e652861356e16..4156c9b44a512 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -116,12 +116,6 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; - - /* length of the IOVA range set ahead the pinned pages */ - unsigned long set_ahead_length; - - /* true if it using the iterator otherwise it pins dynamically */ - bool iterator; }; /* @@ -353,40 +347,6 @@ static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped, (iova + length - 1) <= (mapped->iova + mapped->length - 1)); } -/* - * Returns true if there's not more data to iterate. - */ -static bool iova_bitmap_done(struct iova_bitmap *bitmap) -{ - return bitmap->mapped_base_index >= bitmap->mapped_total_index; -} - -static int iova_bitmap_set_ahead(struct iova_bitmap *bitmap, - size_t set_ahead_length) -{ - int ret = 0; - - while (set_ahead_length > 0 && !iova_bitmap_done(bitmap)) { - unsigned long length = iova_bitmap_mapped_length(bitmap); - unsigned long iova = iova_bitmap_mapped_iova(bitmap); - - ret = iova_bitmap_get(bitmap); - if (ret) - break; - - length = min(length, set_ahead_length); - iova_bitmap_set(bitmap, iova, length); - - set_ahead_length -= length; - bitmap->mapped_base_index += - iova_bitmap_offset_to_index(bitmap, length - 1) + 1; - iova_bitmap_put(bitmap); - } - - bitmap->set_ahead_length = 0; - return ret; -} - /* * Advances to a selected range, releases the current pinned * pages and pins the next set of bitmap pages. @@ -408,36 +368,6 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, return iova_bitmap_get(bitmap); } -/* - * Advances to the next range, releases the current pinned - * pages and pins the next set of bitmap pages. - * Returns 0 on success or otherwise errno. - */ -static int iova_bitmap_advance(struct iova_bitmap *bitmap) -{ - unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1; - unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1; - - bitmap->mapped_base_index += count; - - iova_bitmap_put(bitmap); - - /* Iterate, set and skip any bits requested for next iteration */ - if (bitmap->set_ahead_length) { - int ret; - - ret = iova_bitmap_set_ahead(bitmap, bitmap->set_ahead_length); - if (ret) - return ret; - } - - if (iova_bitmap_done(bitmap)) - return 0; - - /* When advancing the index we pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); -} - /** * iova_bitmap_for_each() - Iterates over the bitmap * @bitmap: IOVA bitmap to iterate @@ -454,23 +384,7 @@ static int iova_bitmap_advance(struct iova_bitmap *bitmap) int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - int ret = 0; - - ret = iova_bitmap_get(bitmap); - if (ret) - return ret; - - bitmap->iterator = true; - for (; !iova_bitmap_done(bitmap) && !ret; - ret = iova_bitmap_advance(bitmap)) { - ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap), - iova_bitmap_mapped_length(bitmap), opaque); - if (ret) - break; - } - bitmap->iterator = false; - - return ret; + return fn(bitmap, bitmap->iova, bitmap->length, opaque); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, IOMMUFD); @@ -491,15 +405,12 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long cur_bit, last_bit, last_page_idx; update_indexes: - if (unlikely(!bitmap->iterator && - !iova_bitmap_mapped_range(mapped, iova, length))) { + if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* * The attempt to advance the base index to @iova * may fail if it's out of bounds, or pinning the pages * returns an error. - * - * It is a no-op if within a iova_bitmap_for_each() closure. */ if (iova_bitmap_advance_to(bitmap, iova)) return; @@ -522,10 +433,6 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long left = ((last_bit - cur_bit + 1) << mapped->pgshift); - if (bitmap->iterator) { - bitmap->set_ahead_length = left; - return; - } iova += (length - left); length = left; goto update_indexes; From f28bfdd40482d645446b85afc83729bc7636abdb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 21 Feb 2024 17:26:40 -0800 Subject: [PATCH 30/85] KVM: x86/mmu: Retry fault before acquiring mmu_lock if mapping is changing Retry page faults without acquiring mmu_lock, and without even faulting the page into the primary MMU, if the resolved gfn is covered by an active invalidation. Contending for mmu_lock is especially problematic on preemptible kernels as the mmu_notifier invalidation task will yield mmu_lock (see rwlock_needbreak()), delay the in-progress invalidation, and ultimately increase the latency of resolving the page fault. And in the worst case scenario, yielding will be accompanied by a remote TLB flush, e.g. if the invalidation covers a large range of memory and vCPUs are accessing addresses that were already zapped. Faulting the page into the primary MMU is similarly problematic, as doing so may acquire locks that need to be taken for the invalidation to complete (the primary MMU has finer grained locks than KVM's MMU), and/or may cause unnecessary churn (getting/putting pages, marking them accessed, etc). Alternatively, the yielding issue could be mitigated by teaching KVM's MMU iterators to perform more work before yielding, but that wouldn't solve the lock contention and would negatively affect scenarios where a vCPU is trying to fault in an address that is NOT covered by the in-progress invalidation. Add a dedicated lockess version of the range-based retry check to avoid false positives on the sanity check on start+end WARN, and so that it's super obvious that checking for a racing invalidation without holding mmu_lock is unsafe (though obviously useful). Wrap mmu_invalidate_in_progress in READ_ONCE() to ensure that pre-checking invalidation in a loop won't put KVM into an infinite loop, e.g. due to caching the in-progress flag and never seeing it go to '0'. Force a load of mmu_invalidate_seq as well, even though it isn't strictly necessary to avoid an infinite loop, as doing so improves the probability that KVM will detect an invalidation that already completed before acquiring mmu_lock and bailing anyways. Do the pre-check even for non-preemptible kernels, as waiting to detect the invalidation until mmu_lock is held guarantees the vCPU will observe the worst case latency in terms of handling the fault, and can generate even more mmu_lock contention. E.g. the vCPU will acquire mmu_lock, detect retry, drop mmu_lock, re-enter the guest, retake the fault, and eventually re-acquire mmu_lock. This behavior is also why there are no new starvation issues due to losing the fairness guarantees provided by rwlocks: if the vCPU needs to retry, it _must_ drop mmu_lock, i.e. waiting on mmu_lock doesn't guarantee forward progress in the face of _another_ mmu_notifier invalidation event. Note, adding READ_ONCE() isn't entirely free, e.g. on x86, the READ_ONCE() may generate a load into a register instead of doing a direct comparison (MOV+TEST+Jcc instead of CMP+Jcc), but practically speaking the added cost is a few bytes of code and maaaaybe a cycle or three. Reported-by: Yan Zhao Closes: https://lore.kernel.org/all/ZNnPF4W26ZbAyGto@yzhao56-desk.sh.intel.com Reported-by: Friedrich Weber Cc: Kai Huang Cc: Yan Zhao Cc: Yuan Yao Cc: Xu Yilun Acked-by: Kai Huang Reviewed-by: Yan Zhao Link: https://lore.kernel.org/r/20240222012640.2820927-1-seanjc@google.com Signed-off-by: Sean Christopherson (cherry picked from commit d02c357e5bfa7dfd618b7b3015624beb71f58f1f) Signed-off-by: Wentao Guan --- arch/x86/kvm/mmu/mmu.c | 42 ++++++++++++++++++++++++++++++++++++++++ include/linux/kvm_host.h | 26 +++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 740cce3b61fee..b80024c080f7f 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4401,6 +4401,31 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq; smp_rmb(); + /* + * Check for a relevant mmu_notifier invalidation event before getting + * the pfn from the primary MMU, and before acquiring mmu_lock. + * + * For mmu_lock, if there is an in-progress invalidation and the kernel + * allows preemption, the invalidation task may drop mmu_lock and yield + * in response to mmu_lock being contended, which is *very* counter- + * productive as this vCPU can't actually make forward progress until + * the invalidation completes. + * + * Retrying now can also avoid unnessary lock contention in the primary + * MMU, as the primary MMU doesn't necessarily hold a single lock for + * the duration of the invalidation, i.e. faulting in a conflicting pfn + * can cause the invalidation to take longer by holding locks that are + * needed to complete the invalidation. + * + * Do the pre-check even for non-preemtible kernels, i.e. even if KVM + * will never yield mmu_lock in response to contention, as this vCPU is + * *guaranteed* to need to retry, i.e. waiting until mmu_lock is held + * to detect retry guarantees the worst case latency for the vCPU. + */ + if (fault->slot && + mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn)) + return RET_PF_RETRY; + ret = __kvm_faultin_pfn(vcpu, fault); if (ret != RET_PF_CONTINUE) return ret; @@ -4411,6 +4436,18 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, if (unlikely(!fault->slot)) return kvm_handle_noslot_fault(vcpu, fault, access); + /* + * Check again for a relevant mmu_notifier invalidation event purely to + * avoid contending mmu_lock. Most invalidations will be detected by + * the previous check, but checking is extremely cheap relative to the + * overall cost of failing to detect the invalidation until after + * mmu_lock is acquired. + */ + if (mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn)) { + kvm_release_pfn_clean(fault->pfn); + return RET_PF_RETRY; + } + return RET_PF_CONTINUE; } @@ -4438,6 +4475,11 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu, if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) return true; + /* + * Check for a relevant mmu_notifier invalidation event one last time + * now that mmu_lock is held, as the "unsafe" checks performed without + * holding mmu_lock can get false negatives. + */ return fault->slot && mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn); } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 2c463def87550..468877b5d8489 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2080,6 +2080,32 @@ static inline int mmu_invalidate_retry_gfn(struct kvm *kvm, return 1; return 0; } + +/* + * This lockless version of the range-based retry check *must* be paired with a + * call to the locked version after acquiring mmu_lock, i.e. this is safe to + * use only as a pre-check to avoid contending mmu_lock. This version *will* + * get false negatives and false positives. + */ +static inline bool mmu_invalidate_retry_gfn_unsafe(struct kvm *kvm, + unsigned long mmu_seq, + gfn_t gfn) +{ + /* + * Use READ_ONCE() to ensure the in-progress flag and sequence counter + * are always read from memory, e.g. so that checking for retry in a + * loop won't result in an infinite retry loop. Don't force loads for + * start+end, as the key to avoiding infinite retry loops is observing + * the 1=>0 transition of in-progress, i.e. getting false negatives + * due to stale start+end values is acceptable. + */ + if (unlikely(READ_ONCE(kvm->mmu_invalidate_in_progress)) && + gfn >= kvm->mmu_invalidate_range_start && + gfn < kvm->mmu_invalidate_range_end) + return true; + + return READ_ONCE(kvm->mmu_invalidate_seq) != mmu_seq; +} #endif #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING From e0a1b14a50cb0bd40a25acd33d87b22d845aa685 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 27 Feb 2024 18:41:40 -0800 Subject: [PATCH 31/85] KVM: x86/mmu: Move private vs. shared check above slot validity checks Prioritize private vs. shared gfn attribute checks above slot validity checks to ensure a consistent userspace ABI. E.g. as is, KVM will exit to userspace if there is no memslot, but emulate accesses to the APIC access page even if the attributes mismatch. Fixes: 8dd2eee9d526 ("KVM: x86/mmu: Handle page fault for private memory") Cc: Yu Zhang Cc: Chao Peng Cc: Fuad Tabba Cc: Michael Roth Cc: Isaku Yamahata Signed-off-by: Sean Christopherson Reviewed-by: Kai Huang Message-ID: <20240228024147.41573-10-seanjc@google.com> Signed-off-by: Paolo Bonzini (cherry picked from commit 44f42ef37deb49682abf0108bf9ede88d4478a20) Signed-off-by: Wentao Guan --- arch/x86/kvm/mmu/mmu.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b80024c080f7f..896953eef124e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4356,11 +4356,6 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault return RET_PF_EMULATE; } - if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) { - kvm_mmu_prepare_memory_fault_exit(vcpu, fault); - return -EFAULT; - } - if (fault->is_private) return kvm_faultin_pfn_private(vcpu, fault); @@ -4398,9 +4393,24 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, { int ret; + /* + * Note that the mmu_invalidate_seq also serves to detect a concurrent + * change in attributes. is_page_fault_stale() will detect an + * invalidation relate to fault->fn and resume the guest without + * installing a mapping in the page tables. + */ fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq; smp_rmb(); + /* + * Now that we have a snapshot of mmu_invalidate_seq we can check for a + * private vs. shared mismatch. + */ + if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) { + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); + return -EFAULT; + } + /* * Check for a relevant mmu_notifier invalidation event before getting * the pfn from the primary MMU, and before acquiring mmu_lock. From 37b181549a915a60b16f038b1fea0a75bb1cbe36 Mon Sep 17 00:00:00 2001 From: Isaku Yamahata Date: Thu, 18 Jul 2024 14:12:14 -0700 Subject: [PATCH 32/85] KVM: Add member to struct kvm_gfn_range to indicate private/shared Add new members to strut kvm_gfn_range to indicate which mapping (private-vs-shared) to operate on: enum kvm_gfn_range_filter attr_filter. Update the core zapping operations to set them appropriately. TDX utilizes two GPA aliases for the same memslots, one for memory that is for private memory and one that is for shared. For private memory, KVM cannot always perform the same operations it does on memory for default VMs, such as zapping pages and having them be faulted back in, as this requires guest coordination. However, some operations such as guest driven conversion of memory between private and shared should zap private memory. Internally to the MMU, private and shared mappings are tracked on separate roots. Mapping and zapping operations will operate on the respective GFN alias for each root (private or shared). So zapping operations will by default zap both aliases. Add fields in struct kvm_gfn_range to allow callers to specify which aliases so they can only target the aliases appropriate for their specific operation. There was feedback that target aliases should be specified such that the default value (0) is to operate on both aliases. Several options were considered. Several variations of having separate bools defined such that the default behavior was to process both aliases. They either allowed nonsensical configurations, or were confusing for the caller. A simple enum was also explored and was close, but was hard to process in the caller. Instead, use an enum with the default value (0) reserved as a disallowed value. Catch ranges that didn't have the target aliases specified by looking for that specific value. Set target alias with enum appropriately for these MMU operations: - For KVM's mmu notifier callbacks, zap shared pages only because private pages won't have a userspace mapping - For setting memory attributes, kvm_arch_pre_set_memory_attributes() chooses the aliases based on the attribute. - For guest_memfd invalidations, zap private only. Link: https://lore.kernel.org/kvm/ZivIF9vjKcuGie3s@google.com/ Signed-off-by: Isaku Yamahata Co-developed-by: Rick Edgecombe Signed-off-by: Rick Edgecombe Message-ID: <20240718211230.1492011-3-rick.p.edgecombe@intel.com> Signed-off-by: Paolo Bonzini (cherry picked from commit dca6c88532322830d5d92486467fcc91b67a9ad8) Signed-off-by: Wentao Guan --- arch/x86/kvm/mmu/mmu.c | 6 ++++++ include/linux/kvm_host.h | 6 ++++++ virt/kvm/guest_memfd.c | 2 ++ virt/kvm/kvm_main.c | 14 ++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 896953eef124e..8935a4ad75e5e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7333,6 +7333,12 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm))) return false; + /* Unmap the old attribute page. */ + if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE) + range->attr_filter = KVM_FILTER_SHARED; + else + range->attr_filter = KVM_FILTER_PRIVATE; + return kvm_unmap_gfn_range(kvm, range); } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 468877b5d8489..231d25e5a210e 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -260,11 +260,17 @@ union kvm_mmu_notifier_arg { unsigned long attributes; }; +enum kvm_gfn_range_filter { + KVM_FILTER_SHARED = BIT(0), + KVM_FILTER_PRIVATE = BIT(1), +}; + struct kvm_gfn_range { struct kvm_memory_slot *slot; gfn_t start; gfn_t end; union kvm_mmu_notifier_arg arg; + enum kvm_gfn_range_filter attr_filter; bool may_block; }; bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index e846282eeb32e..9bcbbcbf55545 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -64,6 +64,8 @@ static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, .end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff, .slot = slot, .may_block = true, + /* guest memfd is relevant to only private mappings. */ + .attr_filter = KVM_FILTER_PRIVATE, }; if (!found_memslot) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2f3775b290c72..9c42cdf677c82 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -641,6 +641,11 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm, */ gfn_range.arg = range->arg; gfn_range.may_block = range->may_block; + /* + * HVA-based notifications aren't relevant to private + * mappings as they don't have a userspace mapping. + */ + gfn_range.attr_filter = KVM_FILTER_SHARED; /* * {gfn(page) | page intersects with [hva_start, hva_end)} = @@ -2504,6 +2509,14 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, gfn_range.arg = range->arg; gfn_range.may_block = range->may_block; + /* + * If/when KVM supports more attributes beyond private .vs shared, this + * _could_ set KVM_FILTER_{SHARED,PRIVATE} appropriately if the entire target + * range already has the desired private vs. shared state (it's unclear + * if that is a net win). For now, KVM reaches this point if and only + * if the private flag is being toggled, i.e. all mappings are in play. + */ + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { slots = __kvm_memslots(kvm, i); @@ -2560,6 +2573,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, + .arg.attributes = attributes, .handler = kvm_pre_set_memory_attributes, .on_lock = kvm_mmu_invalidate_begin, .flush_on_ret = true, From 71b558dbfcadb1755316727e200c0f77672a1fbe Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Apr 2025 15:09:54 -0700 Subject: [PATCH 33/85] KVM: x86/mmu: Prevent installing hugepages when mem attributes are changing When changing memory attributes on a subset of a potential hugepage, add the hugepage to the invalidation range tracking to prevent installing a hugepage until the attributes are fully updated. Like the actual hugepage tracking updates in kvm_arch_post_set_memory_attributes(), process only the head and tail pages, as any potential hugepages that are entirely covered by the range will already be tracked. Note, only hugepage chunks whose current attributes are NOT mixed need to be added to the invalidation set, as mixed attributes already prevent installing a hugepage, and it's perfectly safe to install a smaller mapping for a gfn whose attributes aren't changing. Fixes: 8dd2eee9d526 ("KVM: x86/mmu: Handle page fault for private memory") Cc: stable@vger.kernel.org Reported-by: Michael Roth Tested-by: Michael Roth Link: https://lore.kernel.org/r/20250430220954.522672-1-seanjc@google.com Signed-off-by: Sean Christopherson (cherry picked from commit 9129633d568edd36aa22bf703b12835153cec985) Signed-off-by: Wentao Guan --- arch/x86/kvm/mmu/mmu.c | 69 ++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8935a4ad75e5e..f906213ab8056 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7316,9 +7316,30 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) } #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; +} + bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range) { + struct kvm_memory_slot *slot = range->slot; + int level; + /* * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM @@ -7333,6 +7354,38 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm))) return false; + if (WARN_ON_ONCE(range->end <= range->start)) + return false; + + /* + * If the head and tail pages of the range currently allow a hugepage, + * i.e. reside fully in the slot and don't have mixed attributes, then + * add each corresponding hugepage range to the ongoing invalidation, + * e.g. to prevent KVM from creating a hugepage in response to a fault + * for a gfn whose attributes aren't changing. Note, only the range + * of gfns whose attributes are being modified needs to be explicitly + * unmapped, as that will unmap any existing hugepages. + */ + for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { + gfn_t start = gfn_round_for_level(range->start, level); + gfn_t end = gfn_round_for_level(range->end - 1, level); + gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level); + + if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed(slot, start, level)) + kvm_mmu_invalidate_range_add(kvm, start, start + nr_pages); + + if (end == start) + continue; + + if ((end + nr_pages) > range->end && + (end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed(slot, end, level)) + kvm_mmu_invalidate_range_add(kvm, end, end + nr_pages); + } + /* Unmap the old attribute page. */ if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE) range->attr_filter = KVM_FILTER_SHARED; @@ -7342,23 +7395,7 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, return kvm_unmap_gfn_range(kvm, range); } -static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG; -} - -static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG; -} -static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; -} static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, int level, unsigned long attrs) From 4f500f596bad81394cc145a6c84f393090d0fd10 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 30 Oct 2023 17:20:49 -0700 Subject: [PATCH 34/85] KVM: selftests: Add a memory region subtest to validate invalid flags Add a subtest to set_memory_region_test to verify that KVM rejects invalid flags and combinations with -EINVAL. KVM might or might not fail with EINVAL anyways, but we can at least try. Signed-off-by: Sean Christopherson Message-Id: <20231031002049.3915752-1-seanjc@google.com> Signed-off-by: Paolo Bonzini (cherry picked from commit 5d74316466f4aabdd2ee1e33b45e4933c9bc3ea1) Signed-off-by: Wentao Guan --- .../testing/selftests/kvm/guest_memfd_test.c | 9 +--- .../selftests/kvm/set_memory_region_test.c | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index d7f504e7f917f..f6d63a3137222 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -136,20 +136,13 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) size); } - for (flag = 1; flag; flag <<= 1) { + for (flag = 0; flag; flag <<= 1) { uint64_t bit; fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", flag); - - for_each_set_bit(bit, &valid_flags, 64) { - fd = __vm_create_guest_memfd(vm, page_size, flag | BIT_ULL(bit)); - TEST_ASSERT(fd == -1 && errno == EINVAL, - "guest_memfd() with flags '0x%llx' should fail with EINVAL", - flag | BIT_ULL(bit)); - } } } diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index b7177600a7a74..069cb05b42aea 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -326,6 +326,53 @@ static void test_zero_memory_regions(void) } #endif /* __x86_64__ */ +static void test_invalid_memory_region_flags(void) +{ + uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES; + const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD; + struct kvm_vm *vm; + int r, i; + +#ifdef __x86_64__ + supported_flags |= KVM_MEM_READONLY; + + if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)) + vm = vm_create_barebones_protected_vm(); + else +#endif + vm = vm_create_barebones(); + + if (kvm_check_cap(KVM_CAP_MEMORY_ATTRIBUTES) & KVM_MEMORY_ATTRIBUTE_PRIVATE) + supported_flags |= KVM_MEM_GUEST_MEMFD; + + for (i = 0; i < 32; i++) { + if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i))) + continue; + + r = __vm_set_user_memory_region(vm, MEM_REGION_SLOT, BIT(i), + MEM_REGION_GPA, MEM_REGION_SIZE, NULL); + + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i)); + + if (supported_flags & BIT(i)) + continue; + + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, BIT(i), + MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i)); + } + + if (supported_flags & KVM_MEM_GUEST_MEMFD) { + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, + KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); + } +} + /* * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any * tentative to add further slots should fail. @@ -514,6 +561,8 @@ int main(int argc, char *argv[]) test_zero_memory_regions(); #endif + test_invalid_memory_region_flags(); + test_add_max_memory_regions(); #ifdef __x86_64__ From 6755d5563d74cc8cf0a95bddd652f5507cb8b24c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 21 Nov 2023 11:24:08 -0500 Subject: [PATCH 35/85] selftests/kvm: fix compilation on non-x86_64 platforms MEM_REGION_SLOT and MEM_REGION_GPA are not really needed in test_invalid_memory_region_flags; the VM never runs and there are no other slots, so it is okay to use slot 0 and place it at address zero. This fixes compilation on architectures that do not define them. Fixes: 5d74316466f4 ("KVM: selftests: Add a memory region subtest to validate invalid flags") Signed-off-by: Paolo Bonzini (cherry picked from commit e9e60c82fe391d04db55a91c733df4a017c28b2f) Signed-off-by: Wentao Guan --- tools/testing/selftests/kvm/guest_memfd_test.c | 2 -- tools/testing/selftests/kvm/set_memory_region_test.c | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index f6d63a3137222..92eae206baa62 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -137,8 +137,6 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) } for (flag = 0; flag; flag <<= 1) { - uint64_t bit; - fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 069cb05b42aea..265bee8d06dc1 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -349,8 +349,8 @@ static void test_invalid_memory_region_flags(void) if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i))) continue; - r = __vm_set_user_memory_region(vm, MEM_REGION_SLOT, BIT(i), - MEM_REGION_GPA, MEM_REGION_SIZE, NULL); + r = __vm_set_user_memory_region(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i)); @@ -358,16 +358,16 @@ static void test_invalid_memory_region_flags(void) if (supported_flags & BIT(i)) continue; - r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, BIT(i), - MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + r = __vm_set_user_memory_region2(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL, 0, 0); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i)); } if (supported_flags & KVM_MEM_GUEST_MEMFD) { - r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, + r = __vm_set_user_memory_region2(vm, 0, KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, - MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + 0, MEM_REGION_SIZE, NULL, 0, 0); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); } From a9810f5fc71ba2e2b8e1246e43a858942e0aab84 Mon Sep 17 00:00:00 2001 From: Patrick Roy Date: Thu, 24 Oct 2024 10:59:53 +0100 Subject: [PATCH 36/85] KVM: selftests: fix unintentional noop test in guest_memfd_test.c The loop in test_create_guest_memfd_invalid() that is supposed to test that nothing is accepted as a valid flag to KVM_CREATE_GUEST_MEMFD was initializing `flag` as 0 instead of BIT(0). This caused the loop to immediately exit instead of iterating over BIT(0), BIT(1), ... . Fixes: 8a89efd43423 ("KVM: selftests: Add basic selftest for guest_memfd()") Signed-off-by: Patrick Roy Reviewed-by: James Gowans Reviewed-by: Muhammad Usama Anjum Link: https://lore.kernel.org/r/20241024095956.3668818-1-roypat@amazon.co.uk Signed-off-by: Sean Christopherson (cherry picked from commit 945bdae20be5a13f1fcdcb14ec356dcbeee35839) Signed-off-by: Wentao Guan --- tools/testing/selftests/kvm/guest_memfd_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 92eae206baa62..2f3379b7e67dd 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -136,7 +136,7 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) size); } - for (flag = 0; flag; flag <<= 1) { + for (flag = BIT(0); flag; flag <<= 1) { fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", From 012c31382ea5c0f15e191158c9c25be779b31a33 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Mon, 29 Jan 2024 13:45:51 +0800 Subject: [PATCH 37/85] mm/khugepaged: bypassing unnecessary scans with MMF_DISABLE_THP check khugepaged scans the entire address space in the background for each given mm, looking for opportunities to merge sequences of basic pages into huge pages. However, when an mm is inserted to the mm_slots list, and the MMF_DISABLE_THP flag is set later, this scanning process becomes unnecessary for that mm and can be skipped to avoid redundant operations, especially in scenarios with a large address space. On an Intel Core i5 CPU, the time taken by khugepaged to scan the address space of the process, which has been set with the MMF_DISABLE_THP flag after being added to the mm_slots list, is as follows (shorter is better): VMA Count | Old | New | Change --------------------------------------- 50 | 23us | 9us | -60.9% 100 | 32us | 9us | -71.9% 200 | 44us | 9us | -79.5% 400 | 75us | 9us | -88.0% 800 | 98us | 9us | -90.8% Once the count of VMAs for the process exceeds page_to_scan, khugepaged needs to wait for scan_sleep_millisecs ms before scanning the next process. IMO, unnecessary scans could actually be skipped with a very inexpensive mm->flags check in this case. This commit introduces a check before each scanning process to test the MMF_DISABLE_THP flag for the given mm; if the flag is set, the scanning process is bypassed, thereby improving the efficiency of khugepaged. This optimization is not a correctness issue but rather an enhancement to save expensive checks on each VMA when userspace cannot prctl itself before spawning into the new process. On some servers within our company, we deploy a daemon responsible for monitoring and updating local applications. Some applications prefer not to use THP, so the daemon calls prctl to disable THP before fork/exec. Conversely, for other applications, the daemon calls prctl to enable THP before fork/exec. Ideally, the daemon should invoke prctl after the fork, but its current implementation follows the described approach. In the Go standard library, there is no direct encapsulation of the fork system call; instead, fork and execve are combined into one through syscall.ForkExec. Link: https://lkml.kernel.org/r/20240129054551.57728-1-ioworker0@gmail.com Signed-off-by: Lance Yang Acked-by: David Hildenbrand Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Zach O'Keefe Signed-off-by: Andrew Morton (cherry picked from commit 879c6000e191b61b97e17bce44c4564ee42eb612) Signed-off-by: Wentao Guan --- mm/khugepaged.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 89b7e415d9a11..4d394109a1c86 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -410,6 +410,12 @@ static inline int hpage_collapse_test_exit(struct mm_struct *mm) return atomic_read(&mm->mm_users) == 0; } +static inline int hpage_collapse_test_exit_or_disable(struct mm_struct *mm) +{ + return hpage_collapse_test_exit(mm) || + test_bit(MMF_DISABLE_THP, &mm->flags); +} + void __khugepaged_enter(struct mm_struct *mm) { struct khugepaged_mm_slot *mm_slot; @@ -1409,7 +1415,7 @@ static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) lockdep_assert_held(&khugepaged_mm_lock); - if (hpage_collapse_test_exit(mm)) { + if (hpage_collapse_test_exit_or_disable(mm)) { /* free mm_slot */ hash_del(&slot->hash); list_del(&slot->mm_node); @@ -2349,7 +2355,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, goto breakouterloop_mmap_lock; progress++; - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) goto breakouterloop; vma_iter_init(&vmi, mm, khugepaged_scan.address); @@ -2357,7 +2363,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, unsigned long hstart, hend; cond_resched(); - if (unlikely(hpage_collapse_test_exit(mm))) { + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) { progress++; break; } @@ -2379,7 +2385,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, bool mmap_locked = true; cond_resched(); - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) goto breakouterloop; VM_BUG_ON(khugepaged_scan.address < hstart || @@ -2397,7 +2403,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, fput(file); if (*result == SCAN_PTE_MAPPED_HUGEPAGE) { mmap_read_lock(mm); - if (hpage_collapse_test_exit(mm)) + if (hpage_collapse_test_exit_or_disable(mm)) goto breakouterloop; *result = collapse_pte_mapped_thp(mm, khugepaged_scan.address, false); @@ -2439,7 +2445,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, * Release the current mm_slot if this mm is about to die, or * if we scanned all vmas of this mm. */ - if (hpage_collapse_test_exit(mm) || !vma) { + if (hpage_collapse_test_exit_or_disable(mm) || !vma) { /* * Make sure that if mm_users is reaching zero while * khugepaged runs here, khugepaged_exit will find From abc0a72b0215f0804f0e1cc28cb8f6e845a919ba Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 25 Apr 2024 05:00:55 +0100 Subject: [PATCH 38/85] mm: simplify thp_vma_allowable_order Combine the three boolean arguments into one flags argument for readability. Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Kefeng Wang Cc: Ryan Roberts Signed-off-by: Andrew Morton (cherry picked from commit e0ffb29bc54d86b9ab10ebafc66eb1b7229e0cd7) Signed-off-by: Wentao Guan --- fs/proc/task_mmu.c | 4 ++-- include/linux/huge_mm.h | 29 +++++++++++++++-------------- mm/huge_memory.c | 7 +++++-- mm/khugepaged.c | 16 +++++++--------- mm/memory.c | 10 ++++++---- 5 files changed, 35 insertions(+), 31 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index f120d73bf11b9..fdfaef8d3c339 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -867,8 +867,8 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %8u\n", - !!thp_vma_allowable_orders(vma, vma->vm_flags, true, false, - true, THP_ORDERS_ALL)); + !!thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_SMAPS | TVA_ENFORCE_SYSFS, THP_ORDERS_ALL)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 09179237dac4b..7a9fc015bf25e 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -84,8 +84,12 @@ extern struct kobj_attribute shmem_enabled_attr; */ #define THP_ORDERS_ALL (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE) -#define thp_vma_allowable_order(vma, vm_flags, smaps, in_pf, enforce_sysfs, order) \ - (!!thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, enforce_sysfs, BIT(order))) +#define TVA_SMAPS (1 << 0) /* Will be used for procfs */ +#define TVA_IN_PF (1 << 1) /* Page fault handler */ +#define TVA_ENFORCE_SYSFS (1 << 2) /* Obey sysfs configuration */ + +#define thp_vma_allowable_order(vma, vm_flags, tva_flags, order) \ + (!!thp_vma_allowable_orders(vma, vm_flags, tva_flags, BIT(order))) #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define HPAGE_PMD_SHIFT PMD_SHIFT @@ -211,17 +215,15 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) } unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders); /** * thp_vma_allowable_orders - determine hugepage orders that are allowed for vma * @vma: the vm area to check * @vm_flags: use these vm_flags instead of vma->vm_flags - * @smaps: whether answer will be used for smaps file - * @in_pf: whether answer will be used by page fault handler - * @enforce_sysfs: whether sysfs config should be taken into account + * @tva_flags: Which TVA flags to honour * @orders: bitfield of all orders to consider * * Calculates the intersection of the requested hugepage orders and the allowed @@ -234,12 +236,12 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { /* Optimization to check if required orders are enabled early. */ - if (enforce_sysfs && vma_is_anonymous(vma)) { + if ((tva_flags & TVA_ENFORCE_SYSFS) && vma_is_anonymous(vma)) { unsigned long mask = READ_ONCE(huge_anon_orders_always); if (vm_flags & VM_HUGEPAGE) @@ -253,8 +255,7 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, return 0; } - return __thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, - enforce_sysfs, orders); + return __thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders); } #define transparent_hugepage_use_zero_page() \ @@ -422,8 +423,8 @@ static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, } static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { return 0; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 34635ebc7bc84..05d131650490b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -82,10 +82,13 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { + bool smaps = tva_flags & TVA_SMAPS; + bool in_pf = tva_flags & TVA_IN_PF; + bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; /* Check the intersection of requested and supported orders. */ orders &= vma_is_anonymous(vma) ? THP_ORDERS_ALL_ANON : THP_ORDERS_ALL_FILE; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 4d394109a1c86..f0c7413816e62 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -453,7 +453,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && hugepage_flags_enabled()) { - if (thp_vma_allowable_order(vma, vm_flags, false, false, true, + if (thp_vma_allowable_order(vma, vm_flags, TVA_ENFORCE_SYSFS, PMD_ORDER)) __khugepaged_enter(vma->vm_mm); } @@ -905,6 +905,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, struct collapse_control *cc) { struct vm_area_struct *vma; + unsigned long tva_flags = cc->is_khugepaged ? TVA_ENFORCE_SYSFS : 0; if (unlikely(hpage_collapse_test_exit(mm))) return SCAN_ANY_PROCESS; @@ -915,8 +916,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!thp_vma_suitable_order(vma, address, PMD_ORDER)) return SCAN_ADDRESS_RANGE; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - cc->is_khugepaged, PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, tva_flags, PMD_ORDER)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -1499,8 +1499,7 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, * and map it by a PMD, regardless of sysfs THP settings. As such, let's * analogously elide sysfs THP settings here. */ - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return SCAN_VMA_CHECK; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ @@ -2367,8 +2366,8 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, progress++; break; } - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - true, PMD_ORDER)) { + if (!thp_vma_allowable_order(vma, vma->vm_flags, + TVA_ENFORCE_SYSFS, PMD_ORDER)) { skip: progress++; continue; @@ -2705,8 +2704,7 @@ int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, *prev = vma; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return -EINVAL; cc = kmalloc(sizeof(*cc), GFP_KERNEL); diff --git a/mm/memory.c b/mm/memory.c index eaf07a4d1e0d5..181164a70e924 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4412,8 +4412,8 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) * for this vma. Then filter out the orders that can't be allocated over * the faulting address and still be fully contained in the vma. */ - orders = thp_vma_allowable_orders(vma, vma->vm_flags, false, true, true, - BIT(PMD_ORDER) - 1); + orders = thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1); orders = thp_vma_suitable_orders(vma, vmf->address, orders); if (!orders) @@ -5492,7 +5492,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, return VM_FAULT_OOM; retry_pud: if (pud_none(*vmf.pud) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PUD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5526,7 +5527,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, goto retry_pud; if (pmd_none(*vmf.pmd) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PMD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) { ret = create_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; From 30e0683b1ae72937d04dd1d2eebc1cf2c27f476c Mon Sep 17 00:00:00 2001 From: Ran Xiaokai Date: Wed, 15 May 2024 10:47:54 +0800 Subject: [PATCH 39/85] mm/huge_memory: mark racy access onhuge_anon_orders_always huge_anon_orders_always is accessed lockless, it is better to use the READ_ONCE() wrapper. This is not fixing any visible bug, hopefully this can cease some KCSAN complains in the future. Also do that for huge_anon_orders_madvise. Link: https://lkml.kernel.org/r/20240515104754889HqrahFPePOIE1UlANHVAh@zte.com.cn Signed-off-by: Ran Xiaokai Acked-by: David Hildenbrand Reviewed-by: Lu Zhongjun Reviewed-by: xu xin Cc: Yang Yang Cc: Matthew Wilcox (Oracle) Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton (cherry picked from commit 7f83bf14603ef41a44dc907594d749a283e22c37) Signed-off-by: Wentao Guan --- include/linux/huge_mm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 7a9fc015bf25e..06eb978c549f9 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -126,8 +126,8 @@ static inline bool hugepage_flags_enabled(void) * So we don't need to look at huge_anon_orders_inherit. */ return hugepage_global_enabled() || - huge_anon_orders_always || - huge_anon_orders_madvise; + READ_ONCE(huge_anon_orders_always) || + READ_ONCE(huge_anon_orders_madvise); } static inline int highest_order(unsigned long orders) From b795bea902af31093c5fe7cdca91a92d2a4e11f5 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 4 Jul 2024 10:10:50 +0100 Subject: [PATCH 40/85] mm: fix khugepaged activation policy Since the introduction of mTHP, the docuementation has stated that khugepaged would be enabled when any mTHP size is enabled, and disabled when all mTHP sizes are disabled. There are 2 problems with this; 1. this is not what was implemented by the code and 2. this is not the desirable behavior. Desirable behavior is for khugepaged to be enabled when any PMD-sized THP is enabled, anon or file. (Note that file THP is still controlled by the top-level control so we must always consider that, as well as the PMD-size mTHP control for anon). khugepaged only supports collapsing to PMD-sized THP so there is no value in enabling it when PMD-sized THP is disabled. So let's change the code and documentation to reflect this policy. Further, per-size enabled control modification events were not previously forwarded to khugepaged to give it an opportunity to start or stop. Consequently the following was resulting in khugepaged eroneously not being activated: echo never > /sys/kernel/mm/transparent_hugepage/enabled echo always > /sys/kernel/mm/transparent_hugepage/hugepages-2048kB/enabled [ryan.roberts@arm.com: v3] Link: https://lkml.kernel.org/r/20240705102849.2479686-1-ryan.roberts@arm.com Link: https://lkml.kernel.org/r/20240705102849.2479686-1-ryan.roberts@arm.com Link: https://lkml.kernel.org/r/20240704091051.2411934-1-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Fixes: 3485b88390b0 ("mm: thp: introduce multi-size THP sysfs interface") Closes: https://lore.kernel.org/linux-mm/7a0bbe69-1e3d-4263-b206-da007791a5c4@redhat.com/ Acked-by: David Hildenbrand Cc: Baolin Wang Cc: Barry Song Cc: Jonathan Corbet Cc: Lance Yang Cc: Yang Shi Cc: Signed-off-by: Andrew Morton (cherry picked from commit 00f58104202c472e487f0866fbd38832523fd4f9) Signed-off-by: Wentao Guan --- Documentation/admin-guide/mm/transhuge.rst | 11 ++++---- include/linux/huge_mm.h | 12 -------- mm/huge_memory.c | 7 +++++ mm/khugepaged.c | 33 +++++++++++++++++----- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 04eb45a2f9406..6bc9b2114a3dd 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -202,12 +202,11 @@ PMD-mappable transparent hugepage:: cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size -khugepaged will be automatically started when one or more hugepage -sizes are enabled (either by directly setting "always" or "madvise", -or by setting "inherit" while the top-level enabled is set to "always" -or "madvise"), and it'll be automatically shutdown when the last -hugepage size is disabled (either by directly setting "never", or by -setting "inherit" while the top-level enabled is set to "never"). +khugepaged will be automatically started when PMD-sized THP is enabled +(either of the per-size anon control or the top-level control are set +to "always" or "madvise"), and it'll be automatically shutdown when +PMD-sized THP is disabled (when both the per-size anon control and the +top-level control are "never") Khugepaged controls ------------------- diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 06eb978c549f9..88ba8780f6e4d 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -118,18 +118,6 @@ static inline bool hugepage_global_always(void) (1< 0) { + int err; + + err = start_stop_khugepaged(); + if (err) + ret = err; + } return ret; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index f0c7413816e62..7ee1fbb47bfc2 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -416,6 +416,26 @@ static inline int hpage_collapse_test_exit_or_disable(struct mm_struct *mm) test_bit(MMF_DISABLE_THP, &mm->flags); } +static bool hugepage_pmd_enabled(void) +{ + /* + * We cover both the anon and the file-backed case here; file-backed + * hugepages, when configured in, are determined by the global control. + * Anon pmd-sized hugepages are determined by the pmd-size control. + */ + if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && + hugepage_global_enabled()) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_always)) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_madvise)) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) && + hugepage_global_enabled()) + return true; + return false; +} + void __khugepaged_enter(struct mm_struct *mm) { struct khugepaged_mm_slot *mm_slot; @@ -452,7 +472,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - hugepage_flags_enabled()) { + hugepage_pmd_enabled()) { if (thp_vma_allowable_order(vma, vm_flags, TVA_ENFORCE_SYSFS, PMD_ORDER)) __khugepaged_enter(vma->vm_mm); @@ -2469,8 +2489,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, static int khugepaged_has_work(void) { - return !list_empty(&khugepaged_scan.mm_head) && - hugepage_flags_enabled(); + return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled(); } static int khugepaged_wait_event(void) @@ -2543,7 +2562,7 @@ static void khugepaged_wait_work(void) return; } - if (hugepage_flags_enabled()) + if (hugepage_pmd_enabled()) wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); } @@ -2574,7 +2593,7 @@ static void set_recommended_min_free_kbytes(void) int nr_zones = 0; unsigned long recommended_min; - if (!hugepage_flags_enabled()) { + if (!hugepage_pmd_enabled()) { calculate_min_free_kbytes(); goto update_wmarks; } @@ -2624,7 +2643,7 @@ int start_stop_khugepaged(void) int err = 0; mutex_lock(&khugepaged_mutex); - if (hugepage_flags_enabled()) { + if (hugepage_pmd_enabled()) { if (!khugepaged_thread) khugepaged_thread = kthread_run(khugepaged, NULL, "khugepaged"); @@ -2650,7 +2669,7 @@ int start_stop_khugepaged(void) void khugepaged_min_free_kbytes_update(void) { mutex_lock(&khugepaged_mutex); - if (hugepage_flags_enabled() && khugepaged_thread) + if (hugepage_pmd_enabled() && khugepaged_thread) set_recommended_min_free_kbytes(); mutex_unlock(&khugepaged_mutex); } From 789e319862c08e735a1626f483bf5ab1852e386d Mon Sep 17 00:00:00 2001 From: Chen Ni Date: Wed, 4 Sep 2024 15:52:13 +0800 Subject: [PATCH 41/85] wifi: mt76: mt7925: convert comma to semicolon Replace comma between expressions with semicolons. Using a ',' in place of a ';' can have unintended side effects. Although that is not the case here, it is seems best to use ';' unless ',' is intended. Found by inspection. No functional change intended. Compile tested only. Signed-off-by: Chen Ni Reviewed-by: Matthias Brugger Link: https://patch.msgid.link/20240904075213.1352976-1-nichen@iscas.ac.cn Signed-off-by: Felix Fietkau (cherry picked from commit df6b08670f76b77f7025ab543686ee4ef45a5724) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7925/mcu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index e732ba68d587f..43b550612732e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -2171,12 +2171,12 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_RLM, sizeof(*req)); req = (struct bss_rlm_tlv *)tlv; - req->control_channel = chandef->chan->hw_value, - req->center_chan = ieee80211_frequency_to_channel(freq1), - req->center_chan2 = ieee80211_frequency_to_channel(freq2), - req->tx_streams = hweight8(phy->antenna_mask), - req->ht_op_info = 4, /* set HT 40M allowed */ - req->rx_streams = hweight8(phy->antenna_mask), + req->control_channel = chandef->chan->hw_value; + req->center_chan = ieee80211_frequency_to_channel(freq1); + req->center_chan2 = ieee80211_frequency_to_channel(freq2); + req->tx_streams = hweight8(phy->antenna_mask); + req->ht_op_info = 4; /* set HT 40M allowed */ + req->rx_streams = hweight8(phy->antenna_mask); req->band = band; switch (chandef->width) { From 2c63873db36dc897a059f382ff018eb98ea8b3eb Mon Sep 17 00:00:00 2001 From: Ming Yen Hsieh Date: Tue, 10 Dec 2024 17:19:15 -0800 Subject: [PATCH 42/85] wifi: mt76: mt7925: fix wrong parameter for related cmd of chan info Fix incorrect parameters for the related channel information command. Fixes: 86c051f2c418 ("wifi: mt76: mt7925: enabling MLO when the firmware supports it") Signed-off-by: Ming Yen Hsieh Signed-off-by: Sean Wang Link: https://patch.msgid.link/20241211011926.5002-6-sean.wang@kernel.org Signed-off-by: Felix Fietkau (cherry picked from commit 3f0d2178aaf1ed1c017e61cde9ce8a4432c804d1) Signed-off-by: Wentao Guan --- .../net/wireless/mediatek/mt76/mt7925/mcu.c | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index 43b550612732e..a64d357b1ba68 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1208,6 +1208,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_phy *phy, struct mt792x_bss_conf *mconf req.roc[i].bw_from_ap = CMD_CBW_20MHZ; req.roc[i].center_chan = center_ch; req.roc[i].center_chan_from_ap = center_ch; + req.roc[i].center_chan2 = 0; + req.roc[i].center_chan2_from_ap = 0; /* STR : 0xfe indicates BAND_ALL with enabling DBDC * EMLSR : 0xff indicates (BAND_AUTO) without DBDC @@ -2173,11 +2175,27 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, req = (struct bss_rlm_tlv *)tlv; req->control_channel = chandef->chan->hw_value; req->center_chan = ieee80211_frequency_to_channel(freq1); - req->center_chan2 = ieee80211_frequency_to_channel(freq2); + req->center_chan2 = 0; req->tx_streams = hweight8(phy->antenna_mask); req->ht_op_info = 4; /* set HT 40M allowed */ req->rx_streams = hweight8(phy->antenna_mask); - req->band = band; + req->center_chan2 = 0; + req->sco = 0; + req->band = 1; + + switch (band) { + case NL80211_BAND_2GHZ: + req->band = 1; + break; + case NL80211_BAND_5GHZ: + req->band = 2; + break; + case NL80211_BAND_6GHZ: + req->band = 3; + break; + default: + break; + } switch (chandef->width) { case NL80211_CHAN_WIDTH_40: @@ -2188,6 +2206,7 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, break; case NL80211_CHAN_WIDTH_80P80: req->bw = CMD_CBW_8080MHZ; + req->center_chan2 = ieee80211_frequency_to_channel(freq2); break; case NL80211_CHAN_WIDTH_160: req->bw = CMD_CBW_160MHZ; From 547826b8aac0cc2fa93522c0007679d37db57b8c Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Sat, 25 Apr 2026 14:49:55 -0500 Subject: [PATCH 43/85] wifi: mt76: mt7925: advertise EHT 320MHz capabilities for 6GHz band MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mt7925_init_eht_caps() only populates EHT MCS/NSS maps for BW <= 80 and BW = 160, but never sets BW = 320. This means iw phy shows no 320MHz MCS map entries even though the hardware supports 320MHz operation in the 6GHz band. Add the missing 320MHz capability bits for 6GHz: - PHY_CAP0: IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ - PHY_CAP1: beamformee SS for 320MHz - PHY_CAP2: sounding dimensions for 320MHz - PHY_CAP6: MCS15 support for 320MHz width - MCS/NSS: populate bw._320 maps for 6GHz band Introduce is_320mhz_supported() to gate 320MHz on MT7927 only, since MT7925 does not support 320MHz operation. Tested-by: Marcin FM Tested-by: Cristian-Florin Radoi Tested-by: George Salukvadze Tested-by: Evgeny Kapusta <3193631@gmail.com> Tested-by: Samu Toljamo Tested-by: Ariel Rosenfeld Tested-by: Chapuis Dario Tested-by: Thibaut François Tested-by: 张旭涵 Reviewed-by: Sean Wang Signed-off-by: Javier Tia Link: https://patch.msgid.link/20260425195011.790265-6-sean.wang@kernel.org Signed-off-by: Felix Fietkau (cherry picked from commit 77833c57a33450c0409e4b90d7721d255ea23a9e) Signed-off-by: Wentao Guan --- .../net/wireless/mediatek/mt76/mt76_connac.h | 5 +++++ .../net/wireless/mediatek/mt76/mt7925/main.c | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 445d0f0ab7795..deb3c27ddc317 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -177,6 +177,11 @@ static inline bool is_mt7925(struct mt76_dev *dev) return mt76_chip(dev) == 0x7925; } +static inline bool is_320mhz_supported(struct mt76_dev *dev) +{ + return mt76_chip(dev) == 0x7927; +} + static inline bool is_mt7920(struct mt76_dev *dev) { return mt76_chip(dev) == 0x7920; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 05ea185ea7085..96f29123bfd1b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -178,6 +178,10 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[0] |= + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + eht_cap_elem->phy_cap_info[0] |= u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); @@ -188,10 +192,20 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[1] |= + u8_encode_bits(sts - 1, + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); + eht_cap_elem->phy_cap_info[2] = u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK) | u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[2] |= + u8_encode_bits(sts - 1, + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK); + eht_cap_elem->phy_cap_info[3] = IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | @@ -212,7 +226,8 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(0x11, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_160 ? 0x7 : + val = width == NL80211_CHAN_WIDTH_320 ? 0xf : + width == NL80211_CHAN_WIDTH_160 ? 0x7 : width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; eht_cap_elem->phy_cap_info[6] = u8_encode_bits(u8_get_bits(0x11, GENMASK(4, 2)), @@ -234,6 +249,11 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_nss->bw._160.rx_tx_mcs9_max_nss = val; eht_nss->bw._160.rx_tx_mcs11_max_nss = val; eht_nss->bw._160.rx_tx_mcs13_max_nss = val; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) { + eht_nss->bw._320.rx_tx_mcs9_max_nss = val; + eht_nss->bw._320.rx_tx_mcs11_max_nss = val; + eht_nss->bw._320.rx_tx_mcs13_max_nss = val; + } } int mt7925_init_mlo_caps(struct mt792x_phy *phy) From 93c78844a9e628a58c79ba21d5cda508353e53ee Mon Sep 17 00:00:00 2001 From: "shengwei.lu" Date: Thu, 23 Jul 2026 11:11:08 +0800 Subject: [PATCH 44/85] wifi: mt76: mt7925: Fix EHT Beamformee SS subfields to meet 802.11be minimum Per IEEE 802.11be, the Beamformee SS <= 80/160/320 MHz 3-bit subfields in the EHT PHY Capabilities are encoded as (Nss - 1) and are required to be >= 3 (i.e. at least 4 SS receive capability) whenever SU Beamformee is advertised. MT7925 is a 2x2 STA (sts = 2), so directly filling (sts - 1) = 1 violates the spec minimum. Clamp the encoded value to 3 when sts <= 3, otherwise use (sts - 1). This is applied consistently to the BEAMFORMEE_SS <= 80 MHz (split across phy_cap_info[0]/[1]), <= 160 MHz and <= 320 MHz (6 GHz only) subfields. Fixes: c948b5da6bbe ("wifi: mt76: mt7925: add Mediatek Wi-Fi7 driver for mt7925 chips") Signed-off-by: shengwei.lu Link: https://patch.msgid.link/20260723031108.2017653-1-jb.tsai@mediatek.com Signed-off-by: Felix Fietkau (cherry picked from commit 404c4e564f6b1eeffd10bf2b2d3b86620f5794c3) Signed-off-by: Wentao Guan --- drivers/net/wireless/mediatek/mt76/mt7925/main.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 96f29123bfd1b..f78911f6d8aa6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -182,19 +182,21 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_cap_elem->phy_cap_info[0] |= IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + val = (sts > 3) ? sts - 1 : 3; + eht_cap_elem->phy_cap_info[0] |= - u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), + u8_encode_bits(u8_get_bits(val, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); eht_cap_elem->phy_cap_info[1] = - u8_encode_bits(u8_get_bits(sts - 1, GENMASK(2, 1)), + u8_encode_bits(u8_get_bits(val, GENMASK(2, 1)), IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK) | - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) eht_cap_elem->phy_cap_info[1] |= - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); eht_cap_elem->phy_cap_info[2] = From 057aa8c127b3c8e27a6ea79697fe8cfd89f5bdb7 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 24 Oct 2023 00:01:30 +0200 Subject: [PATCH 45/85] net: ethernet: mtk_wed: remove wo pointer in wo_r32/wo_w32 signature wo pointer is no longer used in wo_r32 and wo_w32 routines so get rid of it. Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/530537db0872f7523deff21f0a5dfdd9b75fdc9d.1698098459.git.lorenzo@kernel.org Signed-off-by: Jakub Kicinski (cherry picked from commit 42c815c545a8b5202a0525c5d2cf1a87c98bdbea) Signed-off-by: Wentao Guan --- drivers/net/ethernet/mediatek/mtk_wed_mcu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c index fee9c9d3a92fe..ea0884186d764 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c @@ -32,12 +32,12 @@ static struct mtk_wed_wo_memory_region mem_region[] = { }, }; -static u32 wo_r32(struct mtk_wed_wo *wo, u32 reg) +static u32 wo_r32(u32 reg) { return readl(mem_region[MTK_WED_WO_REGION_BOOT].addr + reg); } -static void wo_w32(struct mtk_wed_wo *wo, u32 reg, u32 val) +static void wo_w32(u32 reg, u32 val) { writel(val, mem_region[MTK_WED_WO_REGION_BOOT].addr + reg); } @@ -373,13 +373,13 @@ mtk_wed_mcu_load_firmware(struct mtk_wed_wo *wo) boot_cr = MTK_WO_MCU_CFG_LS_WA_BOOT_ADDR_ADDR; else boot_cr = MTK_WO_MCU_CFG_LS_WM_BOOT_ADDR_ADDR; - wo_w32(wo, boot_cr, mem_region[MTK_WED_WO_REGION_EMI].phy_addr >> 16); + wo_w32(boot_cr, mem_region[MTK_WED_WO_REGION_EMI].phy_addr >> 16); /* wo firmware reset */ - wo_w32(wo, MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); + wo_w32(MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); - val = wo_r32(wo, MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR) | + val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR) | MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; - wo_w32(wo, MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); + wo_w32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); out: release_firmware(fw); From 6b33abe3070febaa4178088997acc24ceb09c6b0 Mon Sep 17 00:00:00 2001 From: Zhi-Jun You Date: Thu, 11 Jun 2026 23:00:51 +0800 Subject: [PATCH 46/85] net: ethernet: mtk_wed: fix loading WO firmware for MT7986 MT7986 requires a different mask for second WO firmware. Without this, WO would timeout after loading FW. The correct mask was removed when adding WED for MT7988. Add it back and add a WED version check to fix it. This can be reproduced with a MT7986 + MT7916 board. Fixes: e2f64db13aa1 ("net: ethernet: mtk_wed: introduce WED support for MT7988") Signed-off-by: Zhi-Jun You Link: https://patch.msgid.link/20260611150051.586-1-hujy652@gmail.com Signed-off-by: Jakub Kicinski (cherry picked from commit 9192a18f6de2f5e3eb3813ecd2895ac0f5c008a9) Signed-off-by: Wentao Guan --- drivers/net/ethernet/mediatek/mtk_wed_mcu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c index ea0884186d764..f565ad628b19a 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c @@ -377,8 +377,12 @@ mtk_wed_mcu_load_firmware(struct mtk_wed_wo *wo) /* wo firmware reset */ wo_w32(MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); - val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR) | - MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; + val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR); + + if (!mtk_wed_is_v3_or_greater(wo->hw) && wo->hw->index) + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WA_CPU_RSTB_MASK; + else + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; wo_w32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); out: release_firmware(fw); From cb79dae965c1293d6908556fdc5f7a863ce0fd15 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Sat, 11 May 2024 07:01:31 +1000 Subject: [PATCH 47/85] Revert "drm/nouveau/firmware: Fix SG_DEBUG error with nvkm_firmware_ctor()" This reverts commit 52a6947bf576b97ff8e14bb0a31c5eaf2d0d96e2. This causes loading failures in [ 0.367379] nouveau 0000:01:00.0: NVIDIA GP104 (134000a1) [ 0.474499] nouveau 0000:01:00.0: bios: version 86.04.50.80.13 [ 0.474620] nouveau 0000:01:00.0: pmu: firmware unavailable [ 0.474977] nouveau 0000:01:00.0: fb: 8192 MiB GDDR5 [ 0.484371] nouveau 0000:01:00.0: sec2(acr): mbox 00000001 00000000 [ 0.484377] nouveau 0000:01:00.0: sec2(acr):load: boot failed: -5 [ 0.484379] nouveau 0000:01:00.0: acr: init failed, -5 [ 0.484466] nouveau 0000:01:00.0: init failed with -5 [ 0.484468] nouveau: DRM-master:00000000:00000080: init failed with -5 [ 0.484470] nouveau 0000:01:00.0: DRM-master: Device allocation failed: -5 [ 0.485078] nouveau 0000:01:00.0: probe with driver nouveau failed with error -50 I tried tracking it down but ran out of time this week, will revisit next week. Reported-by: Dan Moulding Cc: stable@vger.kernel.org Signed-off-by: Dave Airlie (cherry picked from commit a222a6470d7eea91193946e8162066fa88da64c2) Signed-off-by: Wentao Guan --- drivers/gpu/drm/nouveau/nvkm/core/firmware.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index e0d35050fef75..91fb494d40093 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -187,9 +187,7 @@ nvkm_firmware_dtor(struct nvkm_firmware *fw) break; case NVKM_FIRMWARE_IMG_DMA: nvkm_memory_unref(&memory); - dma_unmap_single(fw->device->dev, fw->phys, sg_dma_len(&fw->mem.sgl), - DMA_TO_DEVICE); - kfree(fw->img); + dma_free_coherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), fw->img, fw->phys); break; default: WARN_ON(1); @@ -213,17 +211,14 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, fw->img = kmemdup(src, fw->len, GFP_KERNEL); break; case NVKM_FIRMWARE_IMG_DMA: { - len = ALIGN(fw->len, PAGE_SIZE); + dma_addr_t addr; - fw->img = kmalloc(len, GFP_KERNEL); - if (!fw->img) - return -ENOMEM; + len = ALIGN(fw->len, PAGE_SIZE); - memcpy(fw->img, src, fw->len); - fw->phys = dma_map_single(fw->device->dev, fw->img, len, DMA_TO_DEVICE); - if (dma_mapping_error(fw->device->dev, fw->phys)) { - kfree(fw->img); - return -EFAULT; + fw->img = dma_alloc_coherent(fw->device->dev, len, &addr, GFP_KERNEL); + if (fw->img) { + memcpy(fw->img, src, fw->len); + fw->phys = addr; } sg_init_one(&fw->mem.sgl, fw->img, len); From 17949610e6f0c2308215ab8c17dd5a9692314714 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Fri, 16 Aug 2024 06:19:23 +1000 Subject: [PATCH 48/85] nouveau/firmware: use dma non-coherent allocator commit 9b340aeb26d50e9a9ec99599e2a39b035fac978e upstream. Currently, enabling SG_DEBUG in the kernel will cause nouveau to hit a BUG() on startup, when the iommu is enabled: kernel BUG at include/linux/scatterlist.h:187! invalid opcode: 0000 [#1] PREEMPT SMP NOPTI CPU: 7 PID: 930 Comm: (udev-worker) Not tainted 6.9.0-rc3Lyude-Test+ #30 Hardware name: MSI MS-7A39/A320M GAMING PRO (MS-7A39), BIOS 1.I0 01/22/2019 RIP: 0010:sg_init_one+0x85/0xa0 Code: 69 88 32 01 83 e1 03 f6 c3 03 75 20 a8 01 75 1e 48 09 cb 41 89 54 24 08 49 89 1c 24 41 89 6c 24 0c 5b 5d 41 5c e9 7b b9 88 00 <0f> 0b 0f 0b 0f 0b 48 8b 05 5e 46 9a 01 eb b2 66 66 2e 0f 1f 84 00 RSP: 0018:ffffa776017bf6a0 EFLAGS: 00010246 RAX: 0000000000000000 RBX: ffffa77600d87000 RCX: 000000000000002b RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffffa77680d87000 RBP: 000000000000e000 R08: 0000000000000000 R09: 0000000000000000 R10: ffff98f4c46aa508 R11: 0000000000000000 R12: ffff98f4c46aa508 R13: ffff98f4c46aa008 R14: ffffa77600d4a000 R15: ffffa77600d4a018 FS: 00007feeb5aae980(0000) GS:ffff98f5c4dc0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f22cb9a4520 CR3: 00000001043ba000 CR4: 00000000003506f0 Call Trace: ? die+0x36/0x90 ? do_trap+0xdd/0x100 ? sg_init_one+0x85/0xa0 ? do_error_trap+0x65/0x80 ? sg_init_one+0x85/0xa0 ? exc_invalid_op+0x50/0x70 ? sg_init_one+0x85/0xa0 ? asm_exc_invalid_op+0x1a/0x20 ? sg_init_one+0x85/0xa0 nvkm_firmware_ctor+0x14a/0x250 [nouveau] nvkm_falcon_fw_ctor+0x42/0x70 [nouveau] ga102_gsp_booter_ctor+0xb4/0x1a0 [nouveau] r535_gsp_oneinit+0xb3/0x15f0 [nouveau] ? srso_return_thunk+0x5/0x5f ? srso_return_thunk+0x5/0x5f ? nvkm_udevice_new+0x95/0x140 [nouveau] ? srso_return_thunk+0x5/0x5f ? srso_return_thunk+0x5/0x5f ? ktime_get+0x47/0xb0 Fix this by using the non-coherent allocator instead, I think there might be a better answer to this, but it involve ripping up some of APIs using sg lists. Cc: stable@vger.kernel.org Fixes: 2541626cfb79 ("drm/nouveau/acr: use common falcon HS FW code for ACR FWs") Signed-off-by: Dave Airlie Signed-off-by: Danilo Krummrich Link: https://patchwork.freedesktop.org/patch/msgid/20240815201923.632803-1-airlied@gmail.com Signed-off-by: Greg Kroah-Hartman (cherry picked from commit cc29c5546c6a373648363ac49781f1d74b530707) Signed-off-by: Wentao Guan --- drivers/gpu/drm/nouveau/nvkm/core/firmware.c | 9 ++++++--- drivers/gpu/drm/nouveau/nvkm/falcon/fw.c | 6 ++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index 91fb494d40093..afc8d4e3e16f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -187,7 +187,8 @@ nvkm_firmware_dtor(struct nvkm_firmware *fw) break; case NVKM_FIRMWARE_IMG_DMA: nvkm_memory_unref(&memory); - dma_free_coherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), fw->img, fw->phys); + dma_free_noncoherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), + fw->img, fw->phys, DMA_TO_DEVICE); break; default: WARN_ON(1); @@ -212,10 +213,12 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, break; case NVKM_FIRMWARE_IMG_DMA: { dma_addr_t addr; - len = ALIGN(fw->len, PAGE_SIZE); - fw->img = dma_alloc_coherent(fw->device->dev, len, &addr, GFP_KERNEL); + fw->img = dma_alloc_noncoherent(fw->device->dev, + len, &addr, + DMA_TO_DEVICE, + GFP_KERNEL); if (fw->img) { memcpy(fw->img, src, fw->len); fw->phys = addr; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index 1a4ff5ed6dd58..9d782c84a1944 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -89,6 +89,12 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, nvkm_falcon_fw_dtor_sigs(fw); } + /* after last write to the img, sync dma mappings */ + dma_sync_single_for_device(fw->fw.device->dev, + fw->fw.phys, + sg_dma_len(&fw->fw.mem.sgl), + DMA_TO_DEVICE); + FLCNFW_DBG(fw, "resetting"); fw->func->reset(fw); From 3095b14b4b1bcca001d200374a82cd34ad77f01c Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 12 Mar 2025 11:39:38 +0000 Subject: [PATCH 49/85] crypto: qat - introduce fuse array Change the representation of fuses in the accelerator device structure from a single value to an array. This allows the structure to accommodate additional fuses that are required for future generations of QAT hardware. This does not introduce any functional changes. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu (cherry picked from commit f3bda3b9b69cb193968ba781446ff18c734f0276) Signed-off-by: Wentao Guan --- .../crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c | 2 +- drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 2 +- drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 2 +- drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 2 +- .../crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 4 ++-- drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 2 +- drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c | 4 ++-- drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 4 ++-- .../crypto/intel/qat/qat_common/adf_accel_devices.h | 12 +++++++++++- .../crypto/intel/qat/qat_common/adf_gen2_hw_data.c | 2 +- .../intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 6 +++--- drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c | 2 +- 12 files changed, 27 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index ef5f03be41906..0e7d0db475d5d 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -98,7 +98,7 @@ static struct adf_hw_device_class adf_420xx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_420XX_ACCELENGINES_MASK; } diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 2a3598409eeb5..696d6ea05ff78 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -79,7 +79,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index bbd92c017c28e..a6d253ff20888 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -101,7 +101,7 @@ static struct adf_hw_device_class adf_4xxx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index d26564cebdec4..1e281fdeb1c59 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -81,7 +81,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 201f9412c5823..e78f7bfd30b85 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c3xxx_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index 956a4c85609a9..a984ebe0fe28d 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxx(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 6b5b0cf9c7c74..32ebe09477a8d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c62x_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index ad0ca43849985..20b4bb3ccf0cb 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62x(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); @@ -169,7 +169,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ - i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; + i = (hw_data->fuses[ADF_FUSECTL0] & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index ad36afe7d2cf7..f5b4cf2b515c5 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -52,6 +52,16 @@ enum adf_accel_capabilities { ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 }; +enum adf_fuses { + ADF_FUSECTL0, + ADF_FUSECTL1, + ADF_FUSECTL2, + ADF_FUSECTL3, + ADF_FUSECTL4, + ADF_FUSECTL5, + ADF_MAX_FUSES +}; + struct adf_bar { resource_size_t base_addr; void __iomem *virt_addr; @@ -343,7 +353,7 @@ struct adf_hw_device_data { struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; - u32 fuses; + u32 fuses[ADF_MAX_FUSES]; u32 straps; u32 accel_capabilities_mask; u32 extended_dc_capabilities; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 1f64bf49b221c..2b263442c8565 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -115,8 +115,8 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fuses = hw_data->fuses[ADF_FUSECTL0]; u32 straps = hw_data->straps; - u32 fuses = hw_data->fuses; u32 legfuses; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c0661ff5e9292..e48bcf1818cd1 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -29,7 +29,7 @@ static struct adf_hw_device_class dh895xcc_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & ADF_DH895XCC_ACCELERATORS_MASK; @@ -37,7 +37,7 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } @@ -99,7 +99,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) + int sku = (self->fuses[ADF_FUSECTL0] & ADF_DH895XCC_FUSECTL_SKU_MASK) >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; switch (sku) { diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 9d24754a30eae..14ce565b3663c 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -128,7 +128,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcc(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); From 493e4a70e0333926abb409032a5cb4bacbfa6cb3 Mon Sep 17 00:00:00 2001 From: Ahsan Atta Date: Tue, 24 Mar 2026 11:12:34 +0000 Subject: [PATCH 50/85] crypto: qat - disable 420xx AE cluster when lead engine is fused off The get_ae_mask() function only disables individual engines based on the fuse register, but engines are organized in clusters of 4. If the lead engine of a cluster is fused off, the entire cluster must be disabled. Replace the single bitmask inversion with explicit test_bit() checks on the lead engine of each group, disabling the full ADF_AE_GROUP when the lead bit is set. Signed-off-by: Ahsan Atta Reviewed-by: Giovanni Cabiddu Fixes: fcf60f4bcf54 ("crypto: qat - add support for 420xx devices") Signed-off-by: Herbert Xu (cherry picked from commit f216e0f2d1787e662bb6662c9c522185aa3b855a) Signed-off-by: Wentao Guan --- .../intel/qat/qat_420xx/adf_420xx_hw_data.c | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 0e7d0db475d5d..5d32f2958da87 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -98,9 +98,25 @@ static struct adf_hw_device_class adf_420xx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses[ADF_FUSECTL4]; + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_420XX_ACCELENGINES_MASK; - return ~me_disable & ADF_420XX_ACCELENGINES_MASK; + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + if (test_bit(12, &fuses)) + mask &= ~ADF_AE_GROUP_3; + + if (test_bit(16, &fuses)) + mask &= ~ADF_AE_GROUP_4; + + return mask; } static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) From dcdc1d35b00a1ed6bcebb8e2b5677597cd9fdf75 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:24 +0800 Subject: [PATCH 51/85] crypto: iaa - Remove dst_null support Remove the unused dst_null support. Signed-off-by: Herbert Xu (cherry picked from commit 02c974294c740bfb747ec64933e12148eb3d99e1) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 136 +-------------------- 1 file changed, 6 insertions(+), 130 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index ae9b00b0ac486..7652135e4b548 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1126,8 +1126,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc, - bool disable_async) + u32 *compression_crc) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1170,7 +1169,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src2_size = sizeof(struct aecs_comp_table_record); desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1183,7 +1182,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) + } else if (ctx->async_mode) req->base.data = idxd_desc; dev_dbg(dev, "%s: compression mode %s," @@ -1204,7 +1203,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_comp_calls(); update_wq_comp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1224,7 +1223,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, *compression_crc = idxd_desc->iax_completion->crc; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); out: return ret; @@ -1490,13 +1489,11 @@ static int iaa_comp_acompress(struct acomp_req *req) struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - bool disable_async = false; int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; u32 compression_crc; struct idxd_wq *wq; struct device *dev; - int order = -1; compression_ctx = crypto_tfm_ctx(tfm); @@ -1526,21 +1523,6 @@ static int iaa_comp_acompress(struct acomp_req *req) iaa_wq = idxd_wq_get_private(wq); - if (!req->dst) { - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - - /* incompressible data will always be < 2 * slen */ - req->dlen = 2 * req->slen; - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - disable_async = true; - } - dev = &wq->idxd->pdev->dev; nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); @@ -1570,7 +1552,7 @@ static int iaa_comp_acompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc, disable_async); + &req->dlen, &compression_crc); if (ret == -EINPROGRESS) return ret; @@ -1601,100 +1583,6 @@ static int iaa_comp_acompress(struct acomp_req *req) out: iaa_wq_put(wq); - if (order >= 0) - sgl_free_order(req->dst, order); - - return ret; -} - -static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) -{ - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? - GFP_KERNEL : GFP_ATOMIC; - struct crypto_tfm *tfm = req->base.tfm; - dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; - struct iaa_wq *iaa_wq; - struct device *dev; - struct idxd_wq *wq; - int order = -1; - - cpu = get_cpu(); - wq = wq_table_next_wq(cpu); - put_cpu(); - if (!wq) { - pr_debug("no wq configured for cpu=%d\n", cpu); - return -ENODEV; - } - - ret = iaa_wq_get(wq); - if (ret) { - pr_debug("no wq available for cpu=%d\n", cpu); - return -ENODEV; - } - - iaa_wq = idxd_wq_get_private(wq); - - dev = &wq->idxd->pdev->dev; - - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; - } - src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, - req->src, req->slen, sg_dma_len(req->src)); - - req->dlen = 4 * req->slen; /* start with ~avg comp rato */ -alloc_dest: - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; - } - - dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, - req->dst, req->dlen, sg_dma_len(req->dst)); - ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, true); - if (ret == -EOVERFLOW) { - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - req->dlen *= 2; - if (req->dlen > CRYPTO_ACOMP_DST_MAX) - goto err_map_dst; - goto alloc_dest; - } - - if (ret != 0) - dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: - iaa_wq_put(wq); - - if (order >= 0) - sgl_free_order(req->dst, order); - return ret; } @@ -1717,9 +1605,6 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -EINVAL; } - if (!req->dst) - return iaa_comp_adecompress_alloc_dest(req); - cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1800,19 +1685,10 @@ static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) return 0; } -static void dst_free(struct scatterlist *sgl) -{ - /* - * Called for req->dst = NULL cases but we free elsewhere - * using sgl_free_order(). - */ -} - static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, - .dst_free = dst_free, .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", From 7ae5b9b7daf8b8708b9e9c2fb8e12f2242dcb9bb Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Mar 2025 11:57:32 +0800 Subject: [PATCH 52/85] crypto: iaa - Move compression CRC into request object Rather than passing around a CRC between the functions, embed it into the acomp_request context. Signed-off-by: Herbert Xu (cherry picked from commit 184e56e77c06a7eef68a021e9d4b11a11a8ab096) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 7652135e4b548..a84400d5f7fbd 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1015,8 +1015,7 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc); + dma_addr_t dst_addr, unsigned int *dlen); static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1082,10 +1081,10 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } if (ctx->compress && compression_ctx->verify_compress) { + u32 *compression_crc = acomp_request_ctx(ctx->req); dma_addr_t src_addr, dst_addr; - u32 compression_crc; - compression_crc = idxd_desc->iax_completion->crc; + *compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1095,8 +1094,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, - ctx->req->slen, dst_addr, &ctx->req->dlen, - compression_crc); + ctx->req->slen, dst_addr, &ctx->req->dlen); if (ret) { dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); err = -EIO; @@ -1125,11 +1123,11 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1277,11 +1275,11 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *compression_crc = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1341,10 +1339,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (compression_crc != idxd_desc->iax_completion->crc) { + if (*compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", compression_crc, + " comp=0x%x, decomp=0x%x\n", *compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1491,7 +1489,6 @@ static int iaa_comp_acompress(struct acomp_req *req) dma_addr_t src_addr, dst_addr; int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; - u32 compression_crc; struct idxd_wq *wq; struct device *dev; @@ -1552,7 +1549,7 @@ static int iaa_comp_acompress(struct acomp_req *req) req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc); + &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1564,7 +1561,7 @@ static int iaa_comp_acompress(struct acomp_req *req) } ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, compression_crc); + dst_addr, &req->dlen); if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); @@ -1689,6 +1686,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, + .reqsize = sizeof(u32), .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", From bae555c80445aa823d1beaeafbb9fc0fc4f39a24 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 24 Mar 2025 12:04:18 +0800 Subject: [PATCH 53/85] crypto: iaa - Do not clobber req->base.data The req->base.data field is for the user and must not be touched by the driver, unless you save it first. The iaa driver doesn't seem to be using the req->base.data value so just remove the assignment. Fixes: 09646c98d0bf ("crypto: iaa - Add irq support for the crypto async interface") Signed-off-by: Herbert Xu (cherry picked from commit cc98d8ce934b99789d30421957fd6a20fffb1c22) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index a84400d5f7fbd..0819b33abef12 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1180,8 +1180,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: compression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1418,8 +1417,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: decompression mode %s," " desc->src1_addr %llx, desc->src1_size %d," From 7d3d998127be4bd1eb2a556a5c4b9bcd6bb8bb5d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:21:06 +0800 Subject: [PATCH 54/85] crypto: iaa - Use cra_reqsize for acomp Use the common reqsize field for acomp algorithms. Signed-off-by: Herbert Xu (cherry picked from commit 47b5b6f9eb736b1868b0f9c1a1575b5922451cc6) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 0819b33abef12..d9939ffa34c46 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1684,12 +1684,12 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, - .reqsize = sizeof(u32), .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(u32), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } From a110b95b135fffc114fdd654f28263d1201c0358 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 5 Aug 2026 14:19:23 -0700 Subject: [PATCH 55/85] crypto: iaa - fall back to software for multi-entry scatterlists IAA cannot process source or destination scatterlists with more than one entry directly. Instead of failing these requests, route them through a separate deflate acomp transform and keep the request alive in software. The IAA driver has never handled multi-entry scatterlists, but the limitation was latent until commit e2c3b6b21c77 ("mm: zswap: use SG list decompression APIs from zsmalloc") made zswap pass the raw zsmalloc SG list directly to crypto drivers, so objects spanning multiple pages now reach IAA as multi-entry sources and would otherwise fail decompression. Fallback to the generic DEFLATE implementation for scatterlists with more than one entry. After the multi-entry cases fall back early, simplify the DMA mapping path to a single scatterlist entry and fall back on mapping failure as well. Add counters to track the number of requests processed by the software implementation on the compression direction. Fixes: 2ec6761df889 ("crypto: iaa - Add support for deflate-iaa compression algorithm") Fixes: e2c3b6b21c77 ("mm: zswap: use SG list decompression APIs from zsmalloc") Cc: stable@vger.kernel.org Signed-off-by: Giovanni Cabiddu Signed-off-by: Vinicius Costa Gomes Signed-off-by: Herbert Xu (cherry picked from commit c7fdfd2bee1cf1448e5244da1a734e680f634b02) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 111 +++++++++++--------- drivers/crypto/intel/iaa/iaa_crypto_stats.c | 9 ++ drivers/crypto/intel/iaa/iaa_crypto_stats.h | 2 + 3 files changed, 71 insertions(+), 51 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index d9939ffa34c46..d8d6da81d2887 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ #include +#include #include #include #include @@ -1008,6 +1009,19 @@ static int deflate_generic_decompress(struct acomp_req *req) return ret; } +static int deflate_generic_compress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_compress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_comp_calls(); + + return ret; +} + static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, struct acomp_req *req, dma_addr_t *src_addr, dma_addr_t *dst_addr); @@ -1485,7 +1499,7 @@ static int iaa_comp_acompress(struct acomp_req *req) struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct idxd_wq *wq; struct device *dev; @@ -1497,11 +1511,15 @@ static int iaa_comp_acompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not compressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not compressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_compress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1520,30 +1538,25 @@ static int iaa_comp_acompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_compress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_compress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, @@ -1563,8 +1576,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_FROM_DEVICE); goto out; } @@ -1572,9 +1585,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); out: iaa_wq_put(wq); @@ -1585,7 +1597,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) { struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; @@ -1595,11 +1607,15 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not decompressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not decompressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_decompress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1618,30 +1634,25 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, @@ -1652,10 +1663,8 @@ static int iaa_comp_adecompress(struct acomp_req *req) if (ret != 0) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); iaa_wq_put(wq); return ret; diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c index f5cc3d29ca19e..2f2ed88c88120 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -19,6 +19,7 @@ static atomic64_t total_comp_calls; static atomic64_t total_decomp_calls; +static atomic64_t total_sw_comp_calls; static atomic64_t total_sw_decomp_calls; static atomic64_t total_comp_bytes_out; static atomic64_t total_decomp_bytes_in; @@ -43,6 +44,11 @@ void update_total_decomp_calls(void) atomic64_inc(&total_decomp_calls); } +void update_total_sw_comp_calls(void) +{ + atomic64_inc(&total_sw_comp_calls); +} + void update_total_sw_decomp_calls(void) { atomic64_inc(&total_sw_decomp_calls); @@ -104,6 +110,7 @@ static void reset_iaa_crypto_stats(void) { atomic64_set(&total_comp_calls, 0); atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_comp_calls, 0); atomic64_set(&total_sw_decomp_calls, 0); atomic64_set(&total_comp_bytes_out, 0); atomic64_set(&total_decomp_bytes_in, 0); @@ -174,6 +181,8 @@ static int global_stats_show(struct seq_file *m, void *v) atomic64_read(&total_comp_calls)); seq_printf(m, " total_decomp_calls: %llu\n", atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_comp_calls: %llu\n", + atomic64_read(&total_sw_comp_calls)); seq_printf(m, " total_sw_decomp_calls: %llu\n", atomic64_read(&total_sw_decomp_calls)); seq_printf(m, " total_comp_bytes_out: %llu\n", diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h index 3787a5f507eb2..6e0c6f9939bfa 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.h +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -11,6 +11,7 @@ void iaa_crypto_debugfs_cleanup(void); void update_total_comp_calls(void); void update_total_comp_bytes_out(int n); void update_total_decomp_calls(void); +void update_total_sw_comp_calls(void); void update_total_sw_decomp_calls(void); void update_total_decomp_bytes_in(int n); void update_completion_einval_errs(void); @@ -29,6 +30,7 @@ static inline void iaa_crypto_debugfs_cleanup(void) {} static inline void update_total_comp_calls(void) {} static inline void update_total_comp_bytes_out(int n) {} static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_comp_calls(void) {} static inline void update_total_sw_decomp_calls(void) {} static inline void update_total_decomp_bytes_in(int n) {} static inline void update_completion_einval_errs(void) {} From b833841bc06737f8d31aa110ce7210b41bafdaf0 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 5 Aug 2026 14:19:25 -0700 Subject: [PATCH 56/85] crypto: iaa - use bounce buffer for multi-sg decompress input Since commit e2c3b6b21c77 ("mm: zswap: use SG list decompression APIs from zsmalloc"), zswap passes the raw zsmalloc SG list directly to crypto drivers, so a compressed object spanning multiple pages reaches IAA as a multi-entry source. Such requests currently fall back to software decompression. As IAA hardware requires a single DMA source buffer, linearize small multi-entry sources into a pre-allocated bounce page and submit that to the hardware instead of falling back to software. Keep the software fallback only for multi-entry destinations. This recovers most of the performance lost by using the software fallback. Store the bounce-page state in the acomp request context alongside the existing compression CRC, free it through a shared source-unmap helper, and back the pages with a small module-wide mempool so the path remains available in reclaim-driven callers. Signed-off-by: Giovanni Cabiddu Signed-off-by: Vinicius Costa Gomes Signed-off-by: Herbert Xu (cherry picked from commit a229e50741de3d78d7acdd952b70509ab971215d) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 128 +++++++++++++++++---- 1 file changed, 106 insertions(+), 22 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index d8d6da81d2887..4c52b0bf15cb5 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -158,6 +159,16 @@ static bool async_mode; /* Use interrupts */ static bool use_irq; +struct iaa_req_ctx { + u32 compression_crc; + struct page *bounce_src; + dma_addr_t bounce_src_dma; + unsigned int bounce_src_len; +}; + +static mempool_t *iaa_bounce_pool; +#define IAA_BOUNCE_POOL_SIZE 128 + /** * set_iaa_sync_mode - Set IAA sync mode * @name: The name of the sync mode @@ -990,6 +1001,23 @@ static inline int check_completion(struct device *dev, return ret; } +static void iaa_unmap_src(struct device *dev, struct acomp_req *req) +{ + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + + if (req_ctx->bounce_src) { + dma_unmap_page(dev, req_ctx->bounce_src_dma, + req_ctx->bounce_src_len, DMA_TO_DEVICE); + mempool_free(req_ctx->bounce_src, iaa_bounce_pool); + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + return; + } + + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); +} + static int deflate_generic_decompress(struct acomp_req *req) { void *src, *dst; @@ -1039,6 +1067,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *compression_ctx; struct crypto_ctx *ctx = __ctx; + struct iaa_req_ctx *req_ctx = acomp_request_ctx(ctx->req); struct iaa_device *iaa_device; struct idxd_device *idxd; struct iaa_wq *iaa_wq; @@ -1095,10 +1124,9 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } if (ctx->compress && compression_ctx->verify_compress) { - u32 *compression_crc = acomp_request_ctx(ctx->req); dma_addr_t src_addr, dst_addr; - *compression_crc = idxd_desc->iax_completion->crc; + req_ctx->compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1121,7 +1149,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } err: dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE); - dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE); + iaa_unmap_src(dev, ctx->req); out: if (ret != 0) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); @@ -1141,7 +1169,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *compression_crc = acomp_request_ctx(req); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1232,7 +1260,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_comp_bytes_out(*dlen); update_wq_comp_bytes(wq, *dlen); - *compression_crc = idxd_desc->iax_completion->crc; + req_ctx->compression_crc = idxd_desc->iax_completion->crc; if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); @@ -1292,7 +1320,7 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *compression_crc = acomp_request_ctx(req); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1352,10 +1380,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (*compression_crc != idxd_desc->iax_completion->crc) { + if (req_ctx->compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; - dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", *compression_crc, + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch: comp=0x%x, decomp=0x%x\n", + req_ctx->compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1496,6 +1524,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, static int iaa_comp_acompress(struct acomp_req *req) { + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; @@ -1504,6 +1533,10 @@ static int iaa_comp_acompress(struct acomp_req *req) struct idxd_wq *wq; struct device *dev; + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + compression_ctx = crypto_tfm_ctx(tfm); if (!iaa_crypto_enabled) { @@ -1595,12 +1628,19 @@ static int iaa_comp_acompress(struct acomp_req *req) static int iaa_comp_adecompress(struct acomp_req *req) { + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; + bool use_bounce_src = false; int cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; + struct page *page; + + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; if (!iaa_crypto_enabled) { pr_debug("iaa_crypto disabled, not decompressing\n"); @@ -1612,10 +1652,16 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -EINVAL; } - /* Fall back to software if src or dst has multiple sg entries */ - if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + /* Fall back to software if dst has multiple sg entries */ + if (sg_nents(req->dst) > 1) return deflate_generic_decompress(req); + if (sg_nents(req->src) > 1) { + if (req->slen > PAGE_SIZE) + return deflate_generic_decompress(req); + use_bounce_src = true; + } + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1634,20 +1680,47 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { - dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", - iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); - iaa_wq_put(wq); - return deflate_generic_decompress(req); + if (unlikely(use_bounce_src)) { + page = mempool_alloc(iaa_bounce_pool, GFP_ATOMIC); + if (!page) { + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + if (sg_copy_to_buffer(req->src, sg_nents(req->src), + page_address(page), req->slen) != req->slen) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = dma_map_page(dev, page, 0, req->slen, DMA_TO_DEVICE); + if (dma_mapping_error(dev, src_addr)) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + req_ctx->bounce_src = page; + req_ctx->bounce_src_dma = src_addr; + req_ctx->bounce_src_len = req->slen; + } else { + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, + req->src, req->slen, sg_dma_len(req->src)); } - src_addr = sg_dma_address(req->src); - dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, - req->src, req->slen, sg_dma_len(req->src)); if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); - dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_unmap_src(dev, req); iaa_wq_put(wq); return deflate_generic_decompress(req); } @@ -1664,7 +1737,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); - dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_unmap_src(dev, req); iaa_wq_put(wq); return ret; @@ -1698,7 +1771,7 @@ static struct acomp_alg iaa_acomp_fixed_deflate = { .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), - .cra_reqsize = sizeof(u32), + .cra_reqsize = sizeof(struct iaa_req_ctx), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } @@ -1908,6 +1981,12 @@ static int __init iaa_crypto_init_module(void) goto err_aecs_init; } + iaa_bounce_pool = mempool_create_page_pool(IAA_BOUNCE_POOL_SIZE, 0); + if (!iaa_bounce_pool) { + ret = -ENOMEM; + goto err_bounce_pool; + } + ret = idxd_driver_register(&iaa_crypto_driver); if (ret) { pr_debug("IAA wq sub-driver registration failed\n"); @@ -1941,6 +2020,9 @@ static int __init iaa_crypto_init_module(void) err_verify_attr_create: idxd_driver_unregister(&iaa_crypto_driver); err_driver_reg: + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; +err_bounce_pool: iaa_aecs_cleanup_fixed(); err_aecs_init: crypto_free_comp(deflate_generic_tfm); @@ -1959,6 +2041,8 @@ static void __exit iaa_crypto_cleanup_module(void) driver_remove_file(&iaa_crypto_driver.drv, &driver_attr_verify_compress); idxd_driver_unregister(&iaa_crypto_driver); + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; iaa_aecs_cleanup_fixed(); crypto_free_comp(deflate_generic_tfm); From eb73b84cb8b854c0e3e0f57d82603863a7d7a133 Mon Sep 17 00:00:00 2001 From: Vinicius Costa Gomes Date: Wed, 5 Aug 2026 14:19:26 -0700 Subject: [PATCH 57/85] crypto: iaa - unmap dst before software fallback on decompress On a hardware analytics error, decompress retries through the software fallback, which writes req->dst with the CPU while it is still mapped DMA_FROM_DEVICE. With SWIOTLB active the later dma_unmap_sg() copies the stale bounce buffer over req->dst, corrupting the result. Unmap before the fallback runs. The async path unmaps inline; the sync path signals the retry with -EAGAIN so iaa_comp_adecompress() runs the fallback after unmapping. Fixes: 2ec6761df889 ("crypto: iaa - Add support for deflate-iaa compression algorithm") Cc: stable@vger.kernel.org Signed-off-by: Vinicius Costa Gomes Signed-off-by: Herbert Xu (cherry picked from commit 94a25930477113730372e0fa2985da4c5ac95c9a) Signed-off-by: Wentao Guan --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 35 +++++++++++----------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 4c52b0bf15cb5..93e0f071e43b0 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1001,6 +1001,11 @@ static inline int check_completion(struct device *dev, return ret; } +static bool iaa_error_should_retry(struct idxd_desc *idxd_desc) +{ + return idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR; +} + static void iaa_unmap_src(struct device *dev, struct acomp_req *req) { struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); @@ -1094,18 +1099,21 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, ctx->compress, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (!ctx->compress && - idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { + if (!ctx->compress && iaa_error_should_retry(idxd_desc)) { pr_warn("%s: falling back to deflate-generic decompress, " "analytics error code %x\n", __func__, idxd_desc->iax_completion->error_code); + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), + DMA_FROM_DEVICE); + iaa_unmap_src(dev, ctx->req); + ret = deflate_generic_decompress(ctx->req); if (ret) { dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", __func__, ret); err = -EIO; - goto err; } + goto out; } else { err = -EIO; goto err; @@ -1488,19 +1496,9 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, ret = check_completion(dev, idxd_desc->iax_completion, false, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { - pr_warn("%s: falling back to deflate-generic decompress, " - "analytics error code %x\n", __func__, - idxd_desc->iax_completion->error_code); - ret = deflate_generic_decompress(req); - if (ret) { - dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", - __func__, ret); - goto err; - } - } else { - goto err; - } + if (iaa_error_should_retry(idxd_desc)) + ret = -EAGAIN; + goto err; } else { req->dlen = idxd_desc->iax_completion->output_size; } @@ -1733,13 +1731,16 @@ static int iaa_comp_adecompress(struct acomp_req *req) if (ret == -EINPROGRESS) return ret; - if (ret != 0) + if (ret != 0 && ret != -EAGAIN) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); iaa_unmap_src(dev, req); iaa_wq_put(wq); + if (ret == -EAGAIN) + ret = deflate_generic_decompress(req); + return ret; } From 287217d5a6f092dbe39de88aea4bbca7773e9f19 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Mon, 9 Oct 2023 13:54:36 +0200 Subject: [PATCH 58/85] BACKPORT: ASoC: SOF: Intel: Move binding to display driver outside of deferred probe Now that we can use -EPROBE_DEFER, it's no longer required to spin off the snd_hdac_i915_init into a workqueue. Use the -EPROBE_DEFER mechanism instead, which must be returned in the probe function. The previously added probe_early can be used for this, and we also use the newly added remove_late for unbinding afterwards. Signed-off-by: Maarten Lankhorst Cc: Pierre-Louis Bossart Link: https://lore.kernel.org/r/20231009115437.99976-13-maarten.lankhorst@linux.intel.com Signed-off-by: Takashi Iwai (cherry picked from commit 3d1a0558136150448c91f9dd915dec843de8eacf) Conflicts: sound/soc/sof/intel/hda.h (Changes related to this file is present in merge commit "39cd06e3f7b75b629f2987aa51ab26fb820d167b hence changes are made locally.) BUG=b:326869955 TEST=Test Audio use cases. Change-Id: I9181066767e5c239308892d82a2828d2f51634cd Signed-off-by: Debi sahoo Signed-off-by: Hubert Mazur (cherry picked from commit f4cd343129caabaa648f8a5d8c0c24b8706fb8f4) [(BACKPORT of upstream 3d1a055813615) left a 'return 0;' at the end of hda_dsp_remove(), which already returns void in this tree (e4d09de3919bb equivalent). Fix the build.] Signed-off-by: Wentao Guan --- sound/soc/sof/intel/hda-common-ops.c | 1 + sound/soc/sof/intel/hda.c | 20 +++++++------------- sound/soc/sof/intel/hda.h | 1 + 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/sound/soc/sof/intel/hda-common-ops.c b/sound/soc/sof/intel/hda-common-ops.c index 0620536235f4d..d71bb66b99911 100644 --- a/sound/soc/sof/intel/hda-common-ops.c +++ b/sound/soc/sof/intel/hda-common-ops.c @@ -19,6 +19,7 @@ struct snd_sof_dsp_ops sof_hda_common_ops = { .probe_early = hda_dsp_probe_early, .probe = hda_dsp_probe, .remove = hda_dsp_remove, + .remove_late = hda_dsp_remove_late, /* Register IO uses direct mmio */ diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index ed4180b08469d..d2b1b43ed991b 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -1213,6 +1213,7 @@ int hda_dsp_probe_early(struct snd_sof_dev *sdev) return -ENOMEM; sdev->pdata->hw_pdata = hdev; hdev->desc = chip; + ret = hda_init(sdev); err: return ret; @@ -1222,7 +1223,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) { struct pci_dev *pci = to_pci_dev(sdev->dev); struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; - struct hdac_bus *bus; int ret = 0; hdev->dmic_dev = platform_device_register_data(sdev->dev, "dmic-codec", @@ -1246,12 +1246,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) if (sdev->dspless_mode_selected) hdev->no_ipc_position = 1; - /* set up HDA base */ - bus = sof_to_bus(sdev); - ret = hda_init(sdev); - if (ret < 0) - goto hdac_bus_unmap; - if (sdev->dspless_mode_selected) goto skip_dsp_setup; @@ -1360,8 +1354,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) iounmap(sdev->bar[HDA_DSP_BAR]); hdac_bus_unmap: platform_device_unregister(hdev->dmic_dev); - iounmap(bus->remap_addr); - hda_codec_i915_exit(sdev); return ret; } @@ -1370,7 +1362,6 @@ void hda_dsp_remove(struct snd_sof_dev *sdev) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; const struct sof_intel_dsp_desc *chip = hda->desc; - struct hdac_bus *bus = sof_to_bus(sdev); struct pci_dev *pci = to_pci_dev(sdev->dev); struct nhlt_acpi_table *nhlt = hda->nhlt; @@ -1420,12 +1411,15 @@ void hda_dsp_remove(struct snd_sof_dev *sdev) if (!sdev->dspless_mode_selected) iounmap(sdev->bar[HDA_DSP_BAR]); +} - iounmap(bus->remap_addr); - +int hda_dsp_remove_late(struct snd_sof_dev *sdev) +{ + iounmap(sof_to_bus(sdev)->remap_addr); sof_hda_bus_exit(sdev); - hda_codec_i915_exit(sdev); + + return 0; } int hda_power_down_dsp(struct snd_sof_dev *sdev) diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 94bef8762965b..8427e7c81f7a9 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -583,6 +583,7 @@ bool hda_is_chain_dma_supported(struct snd_sof_dev *sdev, u32 dai_type); int hda_dsp_probe_early(struct snd_sof_dev *sdev); int hda_dsp_probe(struct snd_sof_dev *sdev); void hda_dsp_remove(struct snd_sof_dev *sdev); +int hda_dsp_remove_late(struct snd_sof_dev *sdev); int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask); From 9e316c0207829b5b63e587483e3f9eec5119bbca Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Mon, 23 Oct 2023 17:36:05 +0200 Subject: [PATCH 59/85] ASoC: SOF: Make return of remove_late void, too Like the change we've done for remove callback, the newly introduced remove_late callback should be changed to void return, too. Fixes: 17baaa1f950b ("ASoC: SOF: core: Add probe_early and remove_late callbacks") Acked-by: Mark Brown Link: https://lore.kernel.org/r/20231023153605.863-1-tiwai@suse.de Signed-off-by: Takashi Iwai (cherry picked from commit 264aeb994ea8db4a39a393d91b1f551b42309759) Signed-off-by: Wentao Guan --- sound/soc/sof/intel/hda.c | 4 +--- sound/soc/sof/intel/hda.h | 2 +- sound/soc/sof/ops.h | 6 ++---- sound/soc/sof/sof-priv.h | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index d2b1b43ed991b..619ee3e528160 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -1413,13 +1413,11 @@ void hda_dsp_remove(struct snd_sof_dev *sdev) iounmap(sdev->bar[HDA_DSP_BAR]); } -int hda_dsp_remove_late(struct snd_sof_dev *sdev) +void hda_dsp_remove_late(struct snd_sof_dev *sdev) { iounmap(sof_to_bus(sdev)->remap_addr); sof_hda_bus_exit(sdev); hda_codec_i915_exit(sdev); - - return 0; } int hda_power_down_dsp(struct snd_sof_dev *sdev) diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 8427e7c81f7a9..97bdabf795b2d 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -583,7 +583,7 @@ bool hda_is_chain_dma_supported(struct snd_sof_dev *sdev, u32 dai_type); int hda_dsp_probe_early(struct snd_sof_dev *sdev); int hda_dsp_probe(struct snd_sof_dev *sdev); void hda_dsp_remove(struct snd_sof_dev *sdev); -int hda_dsp_remove_late(struct snd_sof_dev *sdev); +void hda_dsp_remove_late(struct snd_sof_dev *sdev); int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask); diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h index 5e96748cd33c1..99e041c4b58c2 100644 --- a/sound/soc/sof/ops.h +++ b/sound/soc/sof/ops.h @@ -57,12 +57,10 @@ static inline void snd_sof_remove(struct snd_sof_dev *sdev) sof_ops(sdev)->remove(sdev); } -static inline int snd_sof_remove_late(struct snd_sof_dev *sdev) +static inline void snd_sof_remove_late(struct snd_sof_dev *sdev) { if (sof_ops(sdev)->remove_late) - return sof_ops(sdev)->remove_late(sdev); - - return 0; + sof_ops(sdev)->remove_late(sdev); } static inline int snd_sof_shutdown(struct snd_sof_dev *sdev) diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index f6ce275b7fe38..4378dd2461570 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -177,7 +177,7 @@ struct snd_sof_dsp_ops { int (*probe_early)(struct snd_sof_dev *sof_dev); /* optional */ int (*probe)(struct snd_sof_dev *sof_dev); /* mandatory */ void (*remove)(struct snd_sof_dev *sof_dev); /* optional */ - int (*remove_late)(struct snd_sof_dev *sof_dev); /* optional */ + void (*remove_late)(struct snd_sof_dev *sof_dev); /* optional */ int (*shutdown)(struct snd_sof_dev *sof_dev); /* optional */ /* DSP core boot / reset */ From 7f0c58f8e8f1397c79d318f9fed0dd4d1e03fac6 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Thu, 19 Jun 2025 13:28:48 +0300 Subject: [PATCH 60/85] ASoC: SOF: ipc4-pcm: Enable delay reporting for ChainDMA streams All streams (currently) which is configured to use ChainDMA can only work on Link/host DMA pairs where the link side position can be access via host registers (like HDA on CAVS 2.5 platforms). Since the firmware does not provide time_info for ChainDMA, unlike for HDA stream, the kernel should calculate the start and end offsets that is needed for the delay calculation. With this small change we can report accurate delays when the stream is configured to use ChainDMA. Signed-off-by: Peter Ujfalusi Reviewed-by: Bard Liao Reviewed-by: Liam Girdwood Reviewed-by: Ranjani Sridharan Link: https://patch.msgid.link/20250619102848.12389-1-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown (cherry picked from commit a1d203d390e04798ccc1c3c06019cd4411885d6d) Signed-off-by: Wentao Guan --- sound/soc/sof/ipc4-pcm.c | 49 ++++++++++++++++++++++++++++++++--- sound/soc/sof/ipc4-topology.c | 6 ++--- sound/soc/sof/ipc4-topology.h | 1 + 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index 80dc26f28ade9..ae6b062587de1 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -409,9 +409,33 @@ static int sof_ipc4_trigger_pipelines(struct snd_soc_component *component, * If use_chain_dma attribute is set we proceed to chained DMA * trigger function that handles the rest for the substream. */ - if (pipeline->use_chain_dma) - return sof_ipc4_chain_dma_trigger(sdev, spcm, substream->stream, - pipeline_list, state, cmd); + if (pipeline->use_chain_dma) { + struct sof_ipc4_timestamp_info *time_info; + + time_info = sof_ipc4_sps_to_time_info(&spcm->stream[substream->stream]); + + ret = sof_ipc4_chain_dma_trigger(sdev, spcm, substream->stream, + pipeline_list, state, cmd); + if (ret || !time_info) + return ret; + + if (state == SOF_IPC4_PIPE_PAUSED) { + /* + * Record the DAI position for delay reporting + * To handle multiple pause/resume/xrun we need to add + * the positions to simulate how the firmware behaves + */ + u64 pos = snd_sof_pcm_get_dai_frame_counter(sdev, component, + substream); + + time_info->stream_end_offset += pos; + } else if (state == SOF_IPC4_PIPE_RESET) { + /* Reset the end offset as the stream is stopped */ + time_info->stream_end_offset = 0; + } + + return 0; + } /* allocate memory for the pipeline data */ trigger_list = kzalloc(struct_size(trigger_list, pipeline_instance_ids, @@ -939,8 +963,24 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, if (!host_copier || !dai_copier) return -EINVAL; - if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_INVALID_NODE_ID) + if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_INVALID_NODE_ID) { return -EINVAL; + } else if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_CHAIN_DMA_NODE_ID) { + /* + * While the firmware does not supports time_info reporting for + * streams using ChainDMA, it is granted that ChainDMA can only + * be used on Host+Link pairs where the link position is + * accessible from the host side. + * + * Enable delay calculation in case of ChainDMA via host + * accessible registers. + * + * The ChainDMA uses 2x 1ms ping-pong buffer, dai side starts + * when 1ms data is available + */ + time_info->stream_start_offset = substream->runtime->rate / MSEC_PER_SEC; + goto out; + } node_index = SOF_IPC4_NODE_INDEX(host_copier->data.gtw_cfg.node_id); offset = offsetof(struct sof_ipc4_fw_registers, pipeline_regs) + node_index * sizeof(ppl_reg); @@ -958,6 +998,7 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, time_info->stream_end_offset = ppl_reg.stream_end_offset; do_div(time_info->stream_end_offset, dai_sample_size); +out: /* * Calculate the wrap boundary need to be used for delay calculation * The host counter is in bytes, it will wrap earlier than the frames diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 16dc4514fef0b..e068e84222928 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -1712,10 +1712,10 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget, pipeline->msg.extension |= SOF_IPC4_GLB_EXT_CHAIN_DMA_FIFO_SIZE(fifo_size); /* - * Chain DMA does not support stream timestamping, set node_id to invalid - * to skip the code in sof_ipc4_get_stream_start_offset(). + * Chain DMA does not support stream timestamping, but it + * can use the host side registers for delay calculation. */ - copier_data->gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID; + copier_data->gtw_cfg.node_id = SOF_IPC4_CHAIN_DMA_NODE_ID; return 0; } diff --git a/sound/soc/sof/ipc4-topology.h b/sound/soc/sof/ipc4-topology.h index 42b4559500924..5743f66ceb59a 100644 --- a/sound/soc/sof/ipc4-topology.h +++ b/sound/soc/sof/ipc4-topology.h @@ -57,6 +57,7 @@ #define SOF_IPC4_DMA_DEVICE_MAX_COUNT 16 +#define SOF_IPC4_CHAIN_DMA_NODE_ID 0x7fffffff #define SOF_IPC4_INVALID_NODE_ID 0xffffffff /* FW requires minimum 4ms DMA buffer size */ From 7e1c586db63c47e4b3aacbed50014dcdb302eea1 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Oct 2025 10:47:15 +0300 Subject: [PATCH 61/85] ASoC: SOF: ipc4-pcm: fix delay calculation when DSP resamples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the sampling rates going in (host) and out (dai) from the DSP are different, the IPC4 delay reporting does not work correctly. Add support for this case by scaling the all raw position values to a common timebase before calculating real-time delay for the PCM. Cc: stable@vger.kernel.org Fixes: 0ea06680dfcb ("ASoC: SOF: ipc4-pcm: Correct the delay calculation") Signed-off-by: Kai Vehmanen Reviewed-by: Péter Ujfalusi Reviewed-by: Bard Liao Signed-off-by: Peter Ujfalusi Link: https://patch.msgid.link/20251002074719.2084-2-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown (cherry picked from commit bcd1383516bb5a6f72b2d1e7f7ad42c4a14837d1) Signed-off-by: Wentao Guan --- sound/soc/sof/ipc4-pcm.c | 83 ++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index ae6b062587de1..fa9d61b7c8790 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -19,12 +19,14 @@ * struct sof_ipc4_timestamp_info - IPC4 timestamp info * @host_copier: the host copier of the pcm stream * @dai_copier: the dai copier of the pcm stream - * @stream_start_offset: reported by fw in memory window (converted to frames) - * @stream_end_offset: reported by fw in memory window (converted to frames) + * @stream_start_offset: reported by fw in memory window (converted to + * frames at host_copier sampling rate) + * @stream_end_offset: reported by fw in memory window (converted to + * frames at host_copier sampling rate) * @llp_offset: llp offset in memory window - * @boundary: wrap boundary should be used for the LLP frame counter * @delay: Calculated and stored in pointer callback. The stored value is - * returned in the delay callback. + * returned in the delay callback. Expressed in frames at host copier + * sampling rate. */ struct sof_ipc4_timestamp_info { struct sof_ipc4_copier *host_copier; @@ -33,7 +35,6 @@ struct sof_ipc4_timestamp_info { u64 stream_end_offset; u32 llp_offset; - u64 boundary; snd_pcm_sframes_t delay; }; @@ -48,6 +49,16 @@ struct sof_ipc4_pcm_stream_priv { bool chain_dma_allocated; }; +/* + * Modulus to use to compare host and link position counters. The sampling + * rates may be different, so the raw hardware counters will wrap + * around at different times. To calculate differences, use + * DELAY_BOUNDARY as a common modulus. This value must be smaller than + * the wrap-around point of any hardware counter, and larger than any + * valid delay measurement. + */ +#define DELAY_BOUNDARY U32_MAX + static inline struct sof_ipc4_timestamp_info * sof_ipc4_sps_to_time_info(struct snd_sof_pcm_stream *sps) { @@ -948,6 +959,35 @@ static int sof_ipc4_pcm_hw_params(struct snd_soc_component *component, return 0; } +static u64 sof_ipc4_frames_dai_to_host(struct sof_ipc4_timestamp_info *time_info, u64 value) +{ + u64 dai_rate, host_rate; + + if (!time_info->dai_copier || !time_info->host_copier) + return value; + + /* + * copiers do not change sampling rate, so we can use the + * out_format independently of stream direction + */ + dai_rate = time_info->dai_copier->data.out_format.sampling_frequency; + host_rate = time_info->host_copier->data.out_format.sampling_frequency; + + if (!dai_rate || !host_rate || dai_rate == host_rate) + return value; + + /* take care not to overflow u64, rates can be up to 768000 */ + if (value > U32_MAX) { + value = div64_u64(value, dai_rate); + value *= host_rate; + } else { + value *= host_rate; + value = div64_u64(value, dai_rate); + } + + return value; +} + static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, struct snd_sof_pcm_stream *sps, @@ -998,14 +1038,13 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, time_info->stream_end_offset = ppl_reg.stream_end_offset; do_div(time_info->stream_end_offset, dai_sample_size); + /* convert to host frame time */ + time_info->stream_start_offset = + sof_ipc4_frames_dai_to_host(time_info, time_info->stream_start_offset); + time_info->stream_end_offset = + sof_ipc4_frames_dai_to_host(time_info, time_info->stream_end_offset); + out: - /* - * Calculate the wrap boundary need to be used for delay calculation - * The host counter is in bytes, it will wrap earlier than the frames - * based link counter. - */ - time_info->boundary = div64_u64(~((u64)0), - frames_to_bytes(substream->runtime, 1)); /* Initialize the delay value to 0 (no delay) */ time_info->delay = 0; @@ -1048,6 +1087,8 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, /* For delay calculation we need the host counter */ host_cnt = snd_sof_pcm_get_host_byte_counter(sdev, component, substream); + + /* Store the original value to host_ptr */ host_ptr = host_cnt; /* convert the host_cnt to frames */ @@ -1066,6 +1107,8 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, sof_mailbox_read(sdev, time_info->llp_offset, &llp, sizeof(llp)); dai_cnt = ((u64)llp.reading.llp_u << 32) | llp.reading.llp_l; } + + dai_cnt = sof_ipc4_frames_dai_to_host(time_info, dai_cnt); dai_cnt += time_info->stream_end_offset; /* In two cases dai dma counter is not accurate @@ -1099,8 +1142,9 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, dai_cnt -= time_info->stream_start_offset; } - /* Wrap the dai counter at the boundary where the host counter wraps */ - div64_u64_rem(dai_cnt, time_info->boundary, &dai_cnt); + /* Convert to a common base before comparisons */ + dai_cnt &= DELAY_BOUNDARY; + host_cnt &= DELAY_BOUNDARY; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { head_cnt = host_cnt; @@ -1110,14 +1154,11 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, tail_cnt = host_cnt; } - if (head_cnt < tail_cnt) { - time_info->delay = time_info->boundary - tail_cnt + head_cnt; - goto out; - } - - time_info->delay = head_cnt - tail_cnt; + if (unlikely(head_cnt < tail_cnt)) + time_info->delay = DELAY_BOUNDARY - tail_cnt + head_cnt; + else + time_info->delay = head_cnt - tail_cnt; -out: /* * Convert the host byte counter to PCM pointer which wraps in buffer * and it is in frames From c8518a75f5ea99a9f5973f055c872958f106472d Mon Sep 17 00:00:00 2001 From: Denis Arefev Date: Tue, 7 Oct 2025 10:38:31 +0300 Subject: [PATCH 62/85] ALSA: hda: cs35l41: Fix NULL pointer dereference in cs35l41_get_acpi_mute_state() [ Upstream commit 8527bbb33936340525a3504a00932b2f8fd75754 ] Return value of a function acpi_evaluate_dsm() is dereferenced without checking for NULL, but it is usually checked for this function. acpi_evaluate_dsm() may return NULL, when acpi_evaluate_object() returns acpi_status other than ACPI_SUCCESS, so add a check to prevent the crach. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: 447106e92a0c ("ALSA: hda: cs35l41: Support mute notifications for CS35L41 HDA") Cc: stable@vger.kernel.org Signed-off-by: Denis Arefev Signed-off-by: Takashi Iwai Stable-dep-of: 3b597d24dc04 ("ALSA: hda: cs35l41: validate and free ACPI mute object") Signed-off-by: Sasha Levin (cherry picked from commit 19129a365d2bd019fb60662b36b2655931997f12) Signed-off-by: Wentao Guan --- sound/pci/hda/cs35l41_hda.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c index 7ccd95b6ccd15..20c12200841b4 100644 --- a/sound/pci/hda/cs35l41_hda.c +++ b/sound/pci/hda/cs35l41_hda.c @@ -1185,6 +1185,8 @@ static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) { ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL); + if (!ret) + return -EINVAL; mute = *ret->buffer.pointer; dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute); } From ff0ca60aa65c8beda85adcda9245b02722a39a7a Mon Sep 17 00:00:00 2001 From: Guangshuo Li Date: Wed, 8 Jul 2026 19:36:25 +0800 Subject: [PATCH 63/85] ALSA: hda: cs35l41: validate and free ACPI mute object [ Upstream commit 3b597d24dc0455ae926f1053f97c2725038fc3cd ] cs35l41_get_acpi_mute_state() evaluates a _DSM method to get the ACPI mute state and reads the first byte from the returned object. However, the returned ACPI object is owned by the caller and is never freed after use, so each successful query leaks the _DSM result object. The code also assumes that the returned object is a buffer with at least one byte. A malformed firmware response can return a different object type or an empty buffer, and the direct ret->buffer.pointer dereference can then access an invalid pointer. Use the typed _DSM helper, validate that the returned buffer contains at least one byte, and free the ACPI object after reading it. Fixes: 447106e92a0c ("ALSA: hda: cs35l41: Support mute notifications for CS35L41 HDA") Signed-off-by: Guangshuo Li Link: https://patch.msgid.link/20260708113625.752913-1-lgs201920130244@gmail.com Signed-off-by: Takashi Iwai Signed-off-by: Sasha Levin (cherry picked from commit 7fea0c89ed39a13d9a31163a74f8c62de30a4ffc) Signed-off-by: Wentao Guan --- sound/pci/hda/cs35l41_hda.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c index 20c12200841b4..f765c30bc73d5 100644 --- a/sound/pci/hda/cs35l41_hda.c +++ b/sound/pci/hda/cs35l41_hda.c @@ -1184,10 +1184,19 @@ static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle guid_parse(CS35L41_UUID, &guid); if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) { - ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL); + ret = acpi_evaluate_dsm_typed(handle, &guid, 0, + CS35L41_DSM_GET_MUTE, NULL, + ACPI_TYPE_BUFFER); + if (!ret) return -EINVAL; + if (!ret->buffer.length || !ret->buffer.pointer) { + ACPI_FREE(ret); + return -EINVAL; + } + mute = *ret->buffer.pointer; + ACPI_FREE(ret); dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute); } From de98f3ca9beea6b4125d3f0cbc5bc23ff7270e5e Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 16 Oct 2024 13:59:51 +0300 Subject: [PATCH 64/85] platform/x86: intel: Add 'intel' prefix to the modules automatically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework Makefile to add 'intel' prefix to the modules automatically. This removes a lot of boilerplate code in it and also makes robust against mistypos in the prefix. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20241016105950.785820-2-andriy.shevchenko@linux.intel.com Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen (cherry picked from commit df7f9acd8646bfad565e6d68a293ec0d8e3f2108) Signed-off-by: Wentao Guan --- drivers/platform/x86/intel/Makefile | 68 ++++++++----------- .../intel/{intel_plr_tpmi.c => plr_tpmi.c} | 0 .../x86/intel/{tpmi.c => vsec_tpmi.c} | 2 +- 3 files changed, 30 insertions(+), 40 deletions(-) rename drivers/platform/x86/intel/{intel_plr_tpmi.c => plr_tpmi.c} (100%) rename drivers/platform/x86/intel/{tpmi.c => vsec_tpmi.c} (99%) diff --git a/drivers/platform/x86/intel/Makefile b/drivers/platform/x86/intel/Makefile index 74db065c82d61..78acb414e1540 100644 --- a/drivers/platform/x86/intel/Makefile +++ b/drivers/platform/x86/intel/Makefile @@ -17,50 +17,40 @@ obj-$(CONFIG_INTEL_UNCORE_FREQ_CONTROL) += uncore-frequency/ # Intel input drivers -intel-hid-y := hid.o -obj-$(CONFIG_INTEL_HID_EVENT) += intel-hid.o -intel-vbtn-y := vbtn.o -obj-$(CONFIG_INTEL_VBTN) += intel-vbtn.o +intel-target-$(CONFIG_INTEL_HID_EVENT) += hid.o +intel-target-$(CONFIG_INTEL_VBTN) += vbtn.o # Intel miscellaneous drivers -obj-$(CONFIG_INTEL_ISHTP_ECLITE) += ishtp_eclite.o -intel_int0002_vgpio-y := int0002_vgpio.o -obj-$(CONFIG_INTEL_INT0002_VGPIO) += intel_int0002_vgpio.o -intel_oaktrail-y := oaktrail.o -obj-$(CONFIG_INTEL_OAKTRAIL) += intel_oaktrail.o -intel_sdsi-y := sdsi.o -obj-$(CONFIG_INTEL_SDSI) += intel_sdsi.o -intel_vsec-y := vsec.o -obj-$(CONFIG_INTEL_VSEC) += intel_vsec.o +intel-target-$(CONFIG_INTEL_INT0002_VGPIO) += int0002_vgpio.o +intel-target-$(CONFIG_INTEL_ISHTP_ECLITE) += ishtp_eclite.o +intel-target-$(CONFIG_INTEL_OAKTRAIL) += oaktrail.o +intel-target-$(CONFIG_INTEL_SDSI) += sdsi.o +intel-target-$(CONFIG_INTEL_VSEC) += vsec.o # Intel PMIC / PMC / P-Unit drivers -intel_bxtwc_tmu-y := bxtwc_tmu.o -obj-$(CONFIG_INTEL_BXTWC_PMIC_TMU) += intel_bxtwc_tmu.o -intel_crystal_cove_charger-y := crystal_cove_charger.o -obj-$(CONFIG_X86_ANDROID_TABLETS) += intel_crystal_cove_charger.o -intel_bytcrc_pwrsrc-y := bytcrc_pwrsrc.o -obj-$(CONFIG_INTEL_BYTCRC_PWRSRC) += intel_bytcrc_pwrsrc.o -intel_chtdc_ti_pwrbtn-y := chtdc_ti_pwrbtn.o -obj-$(CONFIG_INTEL_CHTDC_TI_PWRBTN) += intel_chtdc_ti_pwrbtn.o -intel_chtwc_int33fe-y := chtwc_int33fe.o -obj-$(CONFIG_INTEL_CHTWC_INT33FE) += intel_chtwc_int33fe.o -intel_mrfld_pwrbtn-y := mrfld_pwrbtn.o -obj-$(CONFIG_INTEL_MRFLD_PWRBTN) += intel_mrfld_pwrbtn.o -intel_punit_ipc-y := punit_ipc.o -obj-$(CONFIG_INTEL_PUNIT_IPC) += intel_punit_ipc.o +intel-target-$(CONFIG_INTEL_BYTCRC_PWRSRC) += bytcrc_pwrsrc.o +intel-target-$(CONFIG_INTEL_BXTWC_PMIC_TMU) += bxtwc_tmu.o +intel-target-$(CONFIG_INTEL_CHTDC_TI_PWRBTN) += chtdc_ti_pwrbtn.o +intel-target-$(CONFIG_INTEL_CHTWC_INT33FE) += chtwc_int33fe.o +intel-target-$(CONFIG_X86_ANDROID_TABLETS) += crystal_cove_charger.o +intel-target-$(CONFIG_INTEL_MRFLD_PWRBTN) += mrfld_pwrbtn.o +intel-target-$(CONFIG_INTEL_PUNIT_IPC) += punit_ipc.o # TPMI drivers -intel_vsec_tpmi-y := tpmi.o -obj-$(CONFIG_INTEL_TPMI) += intel_vsec_tpmi.o -obj-$(CONFIG_INTEL_PLR_TPMI) += intel_plr_tpmi.o - -intel_tpmi_power_domains-y := tpmi_power_domains.o -obj-$(CONFIG_INTEL_TPMI_POWER_DOMAINS) += intel_tpmi_power_domains.o +intel-target-$(CONFIG_INTEL_PLR_TPMI) += plr_tpmi.o +intel-target-$(CONFIG_INTEL_TPMI_POWER_DOMAINS) += tpmi_power_domains.o +intel-target-$(CONFIG_INTEL_TPMI) += vsec_tpmi.o # Intel Uncore drivers -intel-rst-y := rst.o -obj-$(CONFIG_INTEL_RST) += intel-rst.o -intel-smartconnect-y := smartconnect.o -obj-$(CONFIG_INTEL_SMARTCONNECT) += intel-smartconnect.o -intel_turbo_max_3-y := turbo_max_3.o -obj-$(CONFIG_INTEL_TURBO_MAX_3) += intel_turbo_max_3.o +intel-target-$(CONFIG_INTEL_RST) += rst.o +intel-target-$(CONFIG_INTEL_SMARTCONNECT) += smartconnect.o +intel-target-$(CONFIG_INTEL_TURBO_MAX_3) += turbo_max_3.o + +# Add 'intel' prefix to each module listed in intel-target-* +define INTEL_OBJ_TARGET +intel-$(1)-y := $(1).o +obj-$(2) += intel-$(1).o +endef + +$(foreach target, $(basename $(intel-target-y)), $(eval $(call INTEL_OBJ_TARGET,$(target),y))) +$(foreach target, $(basename $(intel-target-m)), $(eval $(call INTEL_OBJ_TARGET,$(target),m))) diff --git a/drivers/platform/x86/intel/intel_plr_tpmi.c b/drivers/platform/x86/intel/plr_tpmi.c similarity index 100% rename from drivers/platform/x86/intel/intel_plr_tpmi.c rename to drivers/platform/x86/intel/plr_tpmi.c diff --git a/drivers/platform/x86/intel/tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c similarity index 99% rename from drivers/platform/x86/intel/tpmi.c rename to drivers/platform/x86/intel/vsec_tpmi.c index a3430abeb9869..3ba2282636fa9 100644 --- a/drivers/platform/x86/intel/tpmi.c +++ b/drivers/platform/x86/intel/vsec_tpmi.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * intel-tpmi : Driver to enumerate TPMI features and create devices + * Driver to enumerate TPMI features and create devices * * Copyright (c) 2023, Intel Corporation. * All Rights Reserved. From 077dd3689cc90ae67b7a4fc4f876f1ea2e07a7c8 Mon Sep 17 00:00:00 2001 From: Ricardo Neri Date: Tue, 27 Jan 2026 15:45:40 -0800 Subject: [PATCH 65/85] platform/x86/intel/tpmi/plr: Make the file domain/status writeable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file sys/kernel/debug/tpmi-/plr/domain/status has store and show callbacks. Make it writeable. Fixes: 811f67c51636d ("platform/x86/intel/tpmi: Add new auxiliary driver for performance limits") Signed-off-by: Ricardo Neri Link: https://patch.msgid.link/20260127-plr-debugfs-write-v1-1-1fffbc370b1e@linux.intel.com Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen (cherry picked from commit 008bec8ffe6e7746588d1e12c5b3865fa478fc91) Signed-off-by: Wentao Guan --- drivers/platform/x86/intel/plr_tpmi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/x86/intel/plr_tpmi.c b/drivers/platform/x86/intel/plr_tpmi.c index 69ace6a629bc7..ffb2f7ffc7b51 100644 --- a/drivers/platform/x86/intel/plr_tpmi.c +++ b/drivers/platform/x86/intel/plr_tpmi.c @@ -315,7 +315,7 @@ static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxilia snprintf(name, sizeof(name), "domain%d", i); dentry = debugfs_create_dir(name, plr->dbgfs_dir); - debugfs_create_file("status", 0444, dentry, &plr->die_info[i], + debugfs_create_file("status", 0644, dentry, &plr->die_info[i], &plr_status_fops); } From 5193f6aa0029e47affbbf839f444a4b4487d56a4 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Thu, 30 Apr 2026 08:11:01 -0700 Subject: [PATCH 66/85] platform/x86: intel: Move debugfs register before creating devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is possible that the driver handling device is enumerated before registering debugfs. If the driver wants to access debugfs by calling tpmi_get_debugfs_dir(), this will return error in this case. Hence register debugfs before creating devices. Fixes: 811f67c51636 ("platform/x86/intel/tpmi: Add new auxiliary driver for performance limits") Signed-off-by: Srinivas Pandruvada Cc: Stable@vger.kernel.org Link: https://patch.msgid.link/20260430151103.1549733-2-srinivas.pandruvada@linux.intel.com Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen (cherry picked from commit ad3bff944c0f4f2e913298a9664391af32f87491) Signed-off-by: Wentao Guan --- drivers/platform/x86/intel/vsec_tpmi.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c index 3ba2282636fa9..838c3587873b2 100644 --- a/drivers/platform/x86/intel/vsec_tpmi.c +++ b/drivers/platform/x86/intel/vsec_tpmi.c @@ -809,10 +809,6 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) auxiliary_set_drvdata(auxdev, tpmi_info); - ret = tpmi_create_devices(tpmi_info); - if (ret) - return ret; - /* * Allow debugfs when security policy allows. Everything this debugfs * interface provides, can also be done via /dev/mem access. If @@ -822,6 +818,12 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) if (!security_locked_down(LOCKDOWN_DEV_MEM) && capable(CAP_SYS_RAWIO)) tpmi_dbgfs_register(tpmi_info); + ret = tpmi_create_devices(tpmi_info); + if (ret) { + debugfs_remove_recursive(tpmi_info->dbgfs_dir); + return ret; + } + return 0; } From 08c5f45e24bca2d5f09fc57a843b1a40d4d268f8 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 28 Jun 2024 11:17:57 +0800 Subject: [PATCH 67/85] perf/x86/intel/cstate: Add Arrowlake support Like Alderlake, Arrowlake supports CC1/CC6/CC7 and PC2/PC3/PC6/PC8/PC10. Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Link: https://lore.kernel.org/r/20240628031758.43103-3-rui.zhang@intel.com (cherry picked from commit a31000753d41305d2fb7faa8cc80a8edaeb7b56b) Signed-off-by: Wentao Guan --- arch/x86/events/intel/cstate.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 2e52ca87b4a8f..41021391ef833 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -41,7 +41,7 @@ * MSR_CORE_C1_RES: CORE C1 Residency Counter * perf code: 0x00 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL - * MTL,SRF,GRR + * MTL,SRF,GRR,ARL * Scope: Core (each processor core has a MSR) * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter * perf code: 0x01 @@ -53,30 +53,31 @@ * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, - * GRR + * GRR,ARL * Scope: Core * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter * perf code: 0x03 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML, - * ICL,TGL,RKL,ADL,RPL,MTL + * ICL,TGL,RKL,ADL,RPL,MTL,ARL * Scope: Core * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. * perf code: 0x00 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL, - * RPL,SPR,MTL + * RPL,SPR,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. * perf code: 0x01 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, - * ADL,RPL,MTL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, - * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF + * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, + * ARL * Scope: Package (physical package) * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. * perf code: 0x03 @@ -86,7 +87,7 @@ * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter. * perf code: 0x04 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL, - * ADL,RPL,MTL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter. * perf code: 0x05 @@ -95,7 +96,7 @@ * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter. * perf code: 0x06 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL, - * TNT,RKL,ADL,RPL,MTL + * TNT,RKL,ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter. * perf code: 0x00 @@ -762,6 +763,9 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &adl_cstates), X86_MATCH_VFM(INTEL_METEORLAKE, &adl_cstates), X86_MATCH_VFM(INTEL_METEORLAKE_L, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE_H, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE_U, &adl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); From 5ba3feac9827cdc17d34692b0f88efe3ab281f7b Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 28 Jun 2024 11:17:58 +0800 Subject: [PATCH 68/85] perf/x86/intel/cstate: Add Lunarlake support Compared with previous client platforms, PC8 is removed from Lunarlake. It supports CC1/CC6/CC7 and PC2/PC3/PC6/PC10 residency counters. Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Link: https://lore.kernel.org/r/20240628031758.43103-4-rui.zhang@intel.com (cherry picked from commit 26579860fbd5129e18de9d6fa0751a48420b26b7) Signed-off-by: Wentao Guan --- arch/x86/events/intel/cstate.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 41021391ef833..fb6c8ddbeb5ba 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -41,7 +41,7 @@ * MSR_CORE_C1_RES: CORE C1 Residency Counter * perf code: 0x00 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL - * MTL,SRF,GRR,ARL + * MTL,SRF,GRR,ARL,LNL * Scope: Core (each processor core has a MSR) * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter * perf code: 0x01 @@ -53,31 +53,31 @@ * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, - * GRR,ARL + * GRR,ARL,LNL * Scope: Core * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter * perf code: 0x03 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML, - * ICL,TGL,RKL,ADL,RPL,MTL,ARL + * ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL * Scope: Core * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. * perf code: 0x00 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL, - * RPL,SPR,MTL,ARL + * RPL,SPR,MTL,ARL,LNL * Scope: Package (physical package) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. * perf code: 0x01 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, - * ADL,RPL,MTL,ARL + * ADL,RPL,MTL,ARL,LNL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, - * ARL + * ARL,LNL * Scope: Package (physical package) * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. * perf code: 0x03 @@ -96,7 +96,7 @@ * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter. * perf code: 0x06 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL, - * TNT,RKL,ADL,RPL,MTL,ARL + * TNT,RKL,ADL,RPL,MTL,ARL,LNL * Scope: Package (physical package) * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter. * perf code: 0x00 @@ -643,6 +643,17 @@ static const struct cstate_model adl_cstates __initconst = { BIT(PERF_CSTATE_PKG_C10_RES), }; +static const struct cstate_model lnl_cstates __initconst = { + .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | + BIT(PERF_CSTATE_CORE_C6_RES) | + BIT(PERF_CSTATE_CORE_C7_RES), + + .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | + BIT(PERF_CSTATE_PKG_C3_RES) | + BIT(PERF_CSTATE_PKG_C6_RES) | + BIT(PERF_CSTATE_PKG_C10_RES), +}; + static const struct cstate_model slm_cstates __initconst = { .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | BIT(PERF_CSTATE_CORE_C6_RES), @@ -766,6 +777,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_ARROWLAKE, &adl_cstates), X86_MATCH_VFM(INTEL_ARROWLAKE_H, &adl_cstates), X86_MATCH_VFM(INTEL_ARROWLAKE_U, &adl_cstates), + X86_MATCH_VFM(INTEL_LUNARLAKE_M, &lnl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); From ed8ba91c9d86a4125b58e293fdcedcd3fd9201db Mon Sep 17 00:00:00 2001 From: Zhenyu Wang Date: Wed, 17 Jul 2024 11:16:09 +0800 Subject: [PATCH 69/85] perf/x86/intel/cstate: Add pkg C2 residency counter for Sierra Forest Package C2 residency counter is also available on Sierra Forest. So add it support in srf_cstates. Fixes: 3877d55a0db2 ("perf/x86/intel/cstate: Add Sierra Forest support") Signed-off-by: Zhenyu Wang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Tested-by: Wendy Wang Link: https://lore.kernel.org/r/20240717031609.74513-1-zhenyuw@linux.intel.com (cherry picked from commit b1d0e15c8725d21a73c22c099418a63940261041) Signed-off-by: Wentao Guan --- arch/x86/events/intel/cstate.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index fb6c8ddbeb5ba..27089c906ab0e 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -64,7 +64,7 @@ * perf code: 0x00 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL, - * RPL,SPR,MTL,ARL,LNL + * RPL,SPR,MTL,ARL,LNL,SRF * Scope: Package (physical package) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. * perf code: 0x01 @@ -695,7 +695,8 @@ static const struct cstate_model srf_cstates __initconst = { .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | BIT(PERF_CSTATE_CORE_C6_RES), - .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES), + .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | + BIT(PERF_CSTATE_PKG_C6_RES), .module_events = BIT(PERF_CSTATE_MODULE_C6_RES), }; From 1fa6de14035ca9c4fff202885d123f687b97f298 Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Thu, 11 Jun 2026 09:00:27 -0700 Subject: [PATCH 70/85] perf/x86/intel/uncore: Fix refcnt and other cleanups Fix typo UNCORE_BOX_FLAG_INITIATED to UNCORE_BOX_FLAG_INITIALIZED. Rename the 'id' parameter in uncore_box_{ref,unref}() to 'die' to reflect its actual meaning and be consistent with other functions. box->refcnt is incremented in the PCI PMU register path but has never been checked or decremented. Although for PCI PMUs box->refcnt effectively tracks only a single user, add atomic_dec_return() in the PCI PMU unregister path to make the reference counting complete and consistent. Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260611160033.66760-3-zide.chen@intel.com (cherry picked from commit 7d3a9ff98898b3521eb5d7a3daf703b383f7935a) Signed-off-by: Wentao Guan --- arch/x86/events/intel/uncore.c | 16 +++++++++------- arch/x86/events/intel/uncore.h | 6 +++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 425e88a06a483..ae699579b6ab8 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1239,8 +1239,10 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1505,7 +1507,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1516,7 +1518,7 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) uncore_box_exit(box); } @@ -1591,14 +1593,14 @@ static int allocate_boxes(struct intel_uncore_type **types, } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; int i, ret; - ret = allocate_boxes(types, id, cpu); + ret = allocate_boxes(types, die, cpu); if (ret) return ret; @@ -1606,7 +1608,7 @@ static int uncore_box_ref(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) uncore_box_init(box); } diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index 9ba2305c0d9ae..85a30bc6a6b65 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -164,7 +164,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -537,7 +537,7 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, static inline void uncore_box_init(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->init_box) box->pmu->type->ops->init_box(box); } @@ -545,7 +545,7 @@ static inline void uncore_box_init(struct intel_uncore_box *box) static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } From 7d4db737f1720c4035c3628b23cd932b1a09c46e Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Thu, 11 Jun 2026 09:00:26 -0700 Subject: [PATCH 71/85] perf/x86/intel/uncore: Fix PCI PMU cleanup on setup failure When uncore_pci_pmu_register() fails, pmu->boxes[die] is set to NULL before returning. In the uncore_pci_remove() path, this causes uncore_pci_pmu_unregister() to be skipped entirely, leaking pmu->activeboxes. In the uncore_bus_notify() path, uncore_pci_pmu_unregister() may still be called and must exit early when pmu->boxes[die] is NULL to avoid a NULL pointer dereference, and to ensure activeboxes is only decremented for a previously active box. Additionally, since pci_get_drvdata() returns NULL on registration failure, uncore_pci_remove() can no longer treat NULL drvdata as an indicator of an auxiliary PCI device. Remove the associated WARN_ON_ONCE(). Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://sashiko.dev/#/patchset/20260512233048.9577-1-zide.chen@intel.com?part=1 Link: https://patch.msgid.link/20260611160033.66760-2-zide.chen@intel.com (cherry picked from commit 003267cb94e21d762eb72d6977d84f44f1705bb7) Signed-off-by: Wentao Guan --- arch/x86/events/intel/uncore.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index ae699579b6ab8..d7d9ddf1c921e 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1171,6 +1171,7 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, /* First active box registers the pmu */ ret = uncore_pmu_register(pmu); if (ret) { + atomic_dec(&pmu->activeboxes); pmu->boxes[die] = NULL; uncore_box_exit(box); kfree(box); @@ -1236,6 +1237,9 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); @@ -1262,7 +1266,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } From 11970dc792c228de8480d4983506fdb0e2212a99 Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Thu, 11 Jun 2026 09:00:29 -0700 Subject: [PATCH 72/85] perf/x86/intel/uncore: Keep PCI PMUs working when MMIO/MSR setup fails uncore_event_cpu_online() returns -ENOMEM early when both the MSR and MMIO box allocations fail. This also aborts PCI uncore setup, even though PCI PMUs are independent of the MSR/MMIO paths. Remove the early return so PCI uncore setup always runs regardless of whether MSR or MMIO box allocation succeeds. Fixes: 3da04b8a00dd ("perf/x86/intel/uncore: Support MMIO type uncore blocks") Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ian Rogers Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260611160033.66760-5-zide.chen@intel.com (cherry picked from commit 3012af7df3430788eddd30b3c6654d0a0a5f06c6) Signed-off-by: Wentao Guan --- arch/x86/events/intel/uncore.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index d7d9ddf1c921e..06ae1f7bdf22f 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1626,8 +1626,6 @@ static int uncore_event_cpu_online(unsigned int cpu) die = topology_logical_die_id(cpu); msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; /* * Check if there is an online cpu in the package From 2eea539af273bd1f1340cb6cbad7e4c4ce59fe36 Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Thu, 11 Jun 2026 09:00:30 -0700 Subject: [PATCH 73/85] perf/x86/intel/uncore: Factor out box setup code The PCI uncore PMU path already implements a lazy registration model: the PMU is registered when the first active box appears and unregistered when the last active box is removed. Factor this registration management into a shared helper, so the same code can be reused by the MSR and MMIO paths in later changes. No functional change intended. Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ian Rogers Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260611160033.66760-6-zide.chen@intel.com (cherry picked from commit ae7ca8796ddac708db592c5a68555414c451afcc) Signed-off-by: Wentao Guan --- arch/x86/events/intel/uncore.c | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 06ae1f7bdf22f..442ca37a5463c 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1136,6 +1136,29 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + uncore_box_init(box); + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1162,20 +1185,13 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - atomic_dec(&pmu->activeboxes); - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } From 2193a891eb630a4390cc3c92a2e60dc00be9241b Mon Sep 17 00:00:00 2001 From: Zide Chen Date: Thu, 11 Jun 2026 09:00:31 -0700 Subject: [PATCH 74/85] perf/x86/intel/uncore: Introduce PMU flags and broken state Replace the boolean 'registered' field in intel_uncore_pmu with an unsigned long 'flags' field, and add a PMU_BROKEN flag to track box setup failures. The broken flag is sticky, meaning it is cleared only by a module reload or system reboot. Broken PMUs are skipped in the CPU hotplug and box allocation paths. When any box fails to initialize, the PMU is marked broken. Broken PMUs reject new event assignments and skip future box setup attempts. If the PMU was already registered, it remains so to avoid disrupting in-flight events on other boxes. Signed-off-by: Zide Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260611160033.66760-7-zide.chen@intel.com (cherry picked from commit 30c0a1095652275768a5de67188ff888d1f5d190) Signed-off-by: Wentao Guan [backport note: upstream uses the int-returning uncore_box_init() from cbbc25209ce34 ("perf/x86/intel/uncore: Let init_box() callback report failures"), which is not backported to this tree; this tree still has the older void uncore_box_init() that cannot fail, so the impossible error check on uncore_box_init() is dropped here to fix the build.] --- arch/x86/events/intel/uncore.c | 40 ++++++++++++++++++++++-------- arch/x86/events/intel/uncore.h | 13 +++++++++- arch/x86/events/intel/uncore_snb.c | 2 +- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 442ca37a5463c..61c9d5e8d9814 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -744,7 +744,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -940,16 +940,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1141,6 +1143,9 @@ static int uncore_box_setup(struct intel_uncore_pmu *pmu, { int ret; + if (uncore_pmu_broken(pmu)) + return -ENODEV; + uncore_box_init(box); /* First active box registers the pmu. */ @@ -1155,6 +1160,16 @@ static int uncore_box_setup(struct intel_uncore_pmu *pmu, return 0; err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); uncore_box_exit(box); return ret; } @@ -1178,8 +1193,10 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; @@ -1497,7 +1514,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1505,12 +1523,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1586,7 +1606,7 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); if (!box) diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index 85a30bc6a6b65..a797dd6172ea0 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -125,13 +125,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 47ceade21f562..9985937046268 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -874,7 +874,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ From a5e91975ef1d4689963f3f7ada7d2cdb6568044e Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Sun, 25 Aug 2024 20:27:45 -0700 Subject: [PATCH 75/85] binfmt_elf_fdpic: fix AUXV size calculation when ELF_HWCAP2 is defined create_elf_fdpic_tables() does not correctly account the space for the AUX vector when an architecture has ELF_HWCAP2 defined. Prior to the commit 10e29251be0e ("binfmt_elf_fdpic: fix /proc//auxv") it resulted in the last entry of the AUX vector being set to zero, but with that change it results in a kernel BUG. Fix that by adding one to the number of AUXV entries (nitems) when ELF_HWCAP2 is defined. Fixes: 10e29251be0e ("binfmt_elf_fdpic: fix /proc//auxv") Cc: stable@vger.kernel.org Reported-by: Greg Ungerer Closes: https://lore.kernel.org/lkml/5b51975f-6d0b-413c-8b38-39a6a45e8821@westnet.com.au/ Signed-off-by: Max Filippov Tested-by: Greg Ungerer Link: https://lore.kernel.org/r/20240826032745.3423812-1-jcmvbkbc@gmail.com Signed-off-by: Kees Cook (cherry picked from commit c6a09e342f8e6d3cac7f7c5c14085236aca284b9) Signed-off-by: Wentao Guan --- fs/binfmt_elf_fdpic.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index 84c29ddd86717..fb1f4cc6f2634 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -595,6 +595,9 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, if (bprm->have_execfd) nitems++; +#ifdef ELF_HWCAP2 + nitems++; +#endif csp = sp; sp -= nitems * 2 * sizeof(unsigned long); From a09e8fe65a2cbca2123c546ab4cc719ec0a15685 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 17 Feb 2026 18:01:05 +0000 Subject: [PATCH 76/85] binfmt_elf_fdpic: fix AUXV size calculation for ELF_HWCAP3 and ELF_HWCAP4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 4e6e8c2b757f ("binfmt_elf: Wire up AT_HWCAP3 at AT_HWCAP4") added support for AT_HWCAP3 and AT_HWCAP4, but it missed updating the AUX vector size calculation in create_elf_fdpic_tables() and AT_VECTOR_SIZE_BASE in include/linux/auxvec.h. Similar to the fix for AT_HWCAP2 in commit c6a09e342f8e ("binfmt_elf_fdpic: fix AUXV size calculation when ELF_HWCAP2 is defined"), this omission leads to a mismatch between the reserved space and the actual number of AUX entries, eventually triggering a kernel BUG_ON(csp != sp). Fix this by incrementing nitems when ELF_HWCAP3 or ELF_HWCAP4 are defined and updating AT_VECTOR_SIZE_BASE. Cc: Mark Brown Cc: Max Filippov Reviewed-by: Michal Koutný Reviewed-by: Mark Brown Reviewed-by: Cyrill Gorcunov Reviewed-by: Alexander Mikhalitsyn Fixes: 4e6e8c2b757f ("binfmt_elf: Wire up AT_HWCAP3 at AT_HWCAP4") Signed-off-by: Andrei Vagin Link: https://patch.msgid.link/20260217180108.1420024-2-avagin@google.com Signed-off-by: Kees Cook (cherry picked from commit 4ced4cf5c9d172d91f181df3accdf949d3761aab) Signed-off-by: Wentao Guan --- fs/binfmt_elf_fdpic.c | 6 ++++++ include/linux/auxvec.h | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index fb1f4cc6f2634..7ac12a9d44fc7 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -598,6 +598,12 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, #ifdef ELF_HWCAP2 nitems++; #endif +#ifdef ELF_HWCAP3 + nitems++; +#endif +#ifdef ELF_HWCAP4 + nitems++; +#endif csp = sp; sp -= nitems * 2 * sizeof(unsigned long); diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h index 407f7005e6d60..8bcb9b7262628 100644 --- a/include/linux/auxvec.h +++ b/include/linux/auxvec.h @@ -4,6 +4,6 @@ #include -#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */ +#define AT_VECTOR_SIZE_BASE 24 /* NEW_AUX_ENT entries in auxiliary table */ /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */ #endif /* _LINUX_AUXVEC_H */ From b9c67642eab04d79c0c62e650be5f725c07d7dd6 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 23 May 2026 20:51:22 +0200 Subject: [PATCH 77/85] lockdep/selftests: Restore migrate_disable() state on PREEMPT_RT The lockdep selftests deliberately run unbalanced locking patterns. dotest() restores the task state they leave behind before running the next testcase. On PREEMPT_RT, spin_lock() uses migrate_disable() instead of disabling preemption. dotest() cleans up the resulting migration-disabled state, but that cleanup is still guarded by CONFIG_SMP. That used to match the scheduler data model, where migration_disabled was also CONFIG_SMP-only. The commit referenced below made SMP scheduler state unconditional, so CONFIG_SMP=n PREEMPT_RT kernels with CONFIG_DEBUG_LOCKING_API_SELFTESTS=y report success from the selftests and then trip over stale current->migration_disabled state: releasing a pinned lock bad: scheduling from the idle thread! Kernel panic - not syncing: Fatal exception Save and restore current->migration_disabled for every PREEMPT_RT build. Fixes: cac5cefbade9 ("sched/smp: Make SMP unconditional") Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260523185123.17482-2-kmehltretter@gmail.com (cherry picked from commit d8c897b20bf4d4cbb1e935a8ceb666bcc0f82580) Signed-off-by: Wentao Guan --- lib/locking-selftest.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 6f6a5fc85b42f..14f351ca04b80 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1431,9 +1431,7 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { int saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; #endif @@ -1471,10 +1469,8 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); From d19f6eaa75f39a8750f51f34dd9ab32b26d685f7 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 23 May 2026 20:51:23 +0200 Subject: [PATCH 78/85] lockdep/selftests: Restore sched_rt_mutex state on PREEMPT_RT The WW-mutex selftests deliberately exercise failing lock paths. On PREEMPT_RT, some of those paths enter the RT-mutex scheduler helpers. The change referenced by the Fixes tag made those helpers track RT-mutex scheduling state in current->sched_rt_mutex. The bit is normally cleared by the matching post-schedule helper, but some WW-mutex selftests disable the runtime debug_locks flag before that happens. With debug_locks cleared, lockdep_assert() does not evaluate the expression that clears the bit, leaving stale state for the next testcase. With CONFIG_PREEMPT_RT=y and CONFIG_DEBUG_LOCKING_API_SELFTESTS=y, that stale state produces warnings such as: WARNING: kernel/sched/core.c:7557 at rt_mutex_pre_schedule+0x26/0x2d RIP: 0010:rt_mutex_pre_schedule+0x26/0x2d Save and restore current->sched_rt_mutex around each testcase, matching the existing PREEMPT_RT cleanup for task-local migration and RCU state. Fixes: d14f9e930b90 ("locking/rtmutex: Use rt_mutex specific scheduler helpers") Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260523185123.17482-3-kmehltretter@gmail.com (cherry picked from commit 06961d60a0e410bf8df69ccff7eb1bd824912b8f) Signed-off-by: Wentao Guan --- lib/locking-selftest.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 14f351ca04b80..9fe0f4f895d73 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1433,6 +1433,7 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) #ifdef CONFIG_PREEMPT_RT int saved_mgd_count = current->migration_disabled; int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1469,6 +1470,8 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); From 14c4ef7bae236e318dcd45581ca05393c8380b1a Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 6 Nov 2024 23:01:02 +0100 Subject: [PATCH 79/85] mei: vsc: Do not re-enable interrupt from vsc_tp_reset() The only 2 callers of vsc_tp_reset() are: 1. mei_vsc_hw_reset(), which immediataly calls vsc_tp_intr_disable() afterwards. 2. vsc_tp_shutdown() which immediately calls free_irq() afterwards. So neither actually wants the interrupt to be enabled after resetting the chip and having the interrupt enabled for a short time afer the reset is undesirable. Drop the enable_irq() call from vsc_tp_reset(), so that the interrupt is left disabled after vsc_tp_reset(). Link: https://github.com/intel/ivsc-driver/issues/51 Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20241106220102.40549-1-hdegoede@redhat.com Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 49988a7975420eb206c783f8a384458aae85d938) Signed-off-by: Wentao Guan --- drivers/misc/mei/platform-vsc.c | 2 -- drivers/misc/mei/vsc-tp.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/drivers/misc/mei/platform-vsc.c b/drivers/misc/mei/platform-vsc.c index c51ef6a922116..f67490e7b2911 100644 --- a/drivers/misc/mei/platform-vsc.c +++ b/drivers/misc/mei/platform-vsc.c @@ -256,8 +256,6 @@ static int mei_vsc_hw_reset(struct mei_device *mei_dev, bool intr_enable) vsc_tp_reset(hw->tp); - vsc_tp_intr_disable(hw->tp); - return vsc_tp_init(hw->tp, mei_dev->dev); } diff --git a/drivers/misc/mei/vsc-tp.c b/drivers/misc/mei/vsc-tp.c index c50bf6097b290..79c7a890f67b4 100644 --- a/drivers/misc/mei/vsc-tp.c +++ b/drivers/misc/mei/vsc-tp.c @@ -369,8 +369,6 @@ void vsc_tp_reset(struct vsc_tp *tp) gpiod_set_value_cansleep(tp->wakeupfw, 1); atomic_set(&tp->assert_cnt, 0); - - enable_irq(tp->spi->irq); } EXPORT_SYMBOL_NS_GPL(vsc_tp_reset, VSC_TP); From c42c9e27ea6fbe0ff70cd910608bd3ac7142d116 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Mon, 9 Oct 2023 13:54:28 +0200 Subject: [PATCH 80/85] ASoC: SOF: Intel: Fix error handling in hda_init() The hda_codec_i915_init() errors are ignored in hda_init() so it can never return -EPROBE_DEFER. Fix this before we move the call to hda_init() from the deferred probe to early probe. While at it, also fix error handling when hda_dsp_ctrl_get_caps fails. Suggested-by: Kai Vehmanen Signed-off-by: Maarten Lankhorst Reviewed-by: Kai Vehmanen Reviewed-by: Peter Ujfalusi Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20231009115437.99976-5-maarten.lankhorst@linux.intel.com Signed-off-by: Takashi Iwai (cherry picked from commit 03448e5df586c79f9c88d8cbe29014820cc2904f) Signed-off-by: Wentao Guan --- sound/soc/sof/intel/hda.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 619ee3e528160..22f2dd7303382 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -901,13 +901,21 @@ static int hda_init(struct snd_sof_dev *sdev) /* init i915 and HDMI codecs */ ret = hda_codec_i915_init(sdev); - if (ret < 0) - dev_warn(sdev->dev, "init of i915 and HDMI codec failed\n"); + if (ret < 0 && ret != -ENODEV) { + dev_err_probe(sdev->dev, ret, "init of i915 and HDMI codec failed\n"); + goto out; + } /* get controller capabilities */ ret = hda_dsp_ctrl_get_caps(sdev); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "error: get caps error\n"); + hda_codec_i915_exit(sdev); + } + +out: + if (ret < 0) + iounmap(sof_to_bus(sdev)->remap_addr); return ret; } From 7582a2cbd68d93f12b8dcaf02044c440b34a5633 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Tue, 27 Feb 2024 11:51:35 +0800 Subject: [PATCH 81/85] mm/khugepaged: keep mm in mm_slot without MMF_DISABLE_THP check Previously, we removed the mm from mm_slot and dropped mm_count if the MMF_THP_DISABLE flag was set. However, we didn't re-add the mm back after clearing the MMF_THP_DISABLE flag. Additionally, We add a check for the MMF_THP_DISABLE flag in hugepage_vma_revalidate(). Link: https://lkml.kernel.org/r/20240227035135.54593-1-ioworker0@gmail.com Fixes: 879c6000e191 ("mm/khugepaged: bypassing unnecessary scans with MMF_DISABLE_THP check") Signed-off-by: Lance Yang Suggested-by: Yang Shi Reviewed-by: Yang Shi Reviewed-by: David Hildenbrand Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Zach O'Keefe Signed-off-by: Andrew Morton (cherry picked from commit 5dad604809c5acc546ec74057498db1623f1c408) Signed-off-by: Wentao Guan --- mm/khugepaged.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 7ee1fbb47bfc2..45381731fd756 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -927,7 +927,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, struct vm_area_struct *vma; unsigned long tva_flags = cc->is_khugepaged ? TVA_ENFORCE_SYSFS : 0; - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) return SCAN_ANY_PROCESS; *vmap = vma = find_vma(mm, address); @@ -1435,7 +1435,7 @@ static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) lockdep_assert_held(&khugepaged_mm_lock); - if (hpage_collapse_test_exit_or_disable(mm)) { + if (hpage_collapse_test_exit(mm)) { /* free mm_slot */ hash_del(&slot->hash); list_del(&slot->mm_node); @@ -2464,7 +2464,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, * Release the current mm_slot if this mm is about to die, or * if we scanned all vmas of this mm. */ - if (hpage_collapse_test_exit_or_disable(mm) || !vma) { + if (hpage_collapse_test_exit(mm) || !vma) { /* * Make sure that if mm_users is reaching zero while * khugepaged runs here, khugepaged_exit will find From 34faa1a7e5cd7775d7fbbd4b6b1966deb6b69d9c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 22 Feb 2024 11:06:11 -0800 Subject: [PATCH 82/85] KVM: selftests: Create GUEST_MEMFD for relevant invalid flags testcases Actually create a GUEST_MEMFD instance and pass it to KVM when doing negative tests for KVM_SET_USER_MEMORY_REGION2 + KVM_MEM_GUEST_MEMFD. Without a valid GUEST_MEMFD file descriptor, KVM_SET_USER_MEMORY_REGION2 will always fail with -EINVAL, resulting in false passes for any and all tests of illegal combinations of KVM_MEM_GUEST_MEMFD and other flags. Fixes: 5d74316466f4 ("KVM: selftests: Add a memory region subtest to validate invalid flags") Link: https://lore.kernel.org/r/20240222190612.2942589-5-seanjc@google.com Signed-off-by: Sean Christopherson (cherry picked from commit 63e5c5a10559077bb5f32edf783084e7164af9c3) Signed-off-by: Wentao Guan --- tools/testing/selftests/kvm/set_memory_region_test.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 265bee8d06dc1..197eb1f7e9562 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -365,11 +365,15 @@ static void test_invalid_memory_region_flags(void) } if (supported_flags & KVM_MEM_GUEST_MEMFD) { + int guest_memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0); + r = __vm_set_user_memory_region2(vm, 0, KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, - 0, MEM_REGION_SIZE, NULL, 0, 0); + 0, MEM_REGION_SIZE, NULL, guest_memfd, 0); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); + + close(guest_memfd); } } From f41cafb3f97c484a76a071933cf6ff9c5808faad Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Wed, 13 Nov 2024 05:57:03 +1000 Subject: [PATCH 83/85] nouveau: fw: sync dma after setup is called. When this code moved to non-coherent allocator the sync was put too early for some firmwares which called the setup function, move the sync down after the setup function. Reported-by: Diogo Ivo Tested-by: Diogo Ivo Reviewed-by: Lyude Paul Fixes: 9b340aeb26d5 ("nouveau/firmware: use dma non-coherent allocator") Cc: stable@vger.kernel.org Signed-off-by: Dave Airlie Link: https://patchwork.freedesktop.org/patch/msgid/20241114004603.3095485-1-airlied@gmail.com (cherry picked from commit 21ec425eaf2cb7c0371f7683f81ad7d9679b6eb5) Signed-off-by: Wentao Guan --- drivers/gpu/drm/nouveau/nvkm/falcon/fw.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index 9d782c84a1944..4e8b3f1c7e25d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -89,11 +89,6 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, nvkm_falcon_fw_dtor_sigs(fw); } - /* after last write to the img, sync dma mappings */ - dma_sync_single_for_device(fw->fw.device->dev, - fw->fw.phys, - sg_dma_len(&fw->fw.mem.sgl), - DMA_TO_DEVICE); FLCNFW_DBG(fw, "resetting"); fw->func->reset(fw); @@ -105,6 +100,12 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, goto done; } + /* after last write to the img, sync dma mappings */ + dma_sync_single_for_device(fw->fw.device->dev, + fw->fw.phys, + sg_dma_len(&fw->fw.mem.sgl), + DMA_TO_DEVICE); + ret = fw->func->load(fw); if (ret) goto done; From 57285a9b2f59aac3258d200d642d42fefac6bfe6 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Oct 2025 10:47:16 +0300 Subject: [PATCH 84/85] ASoC: SOF: ipc4-pcm: fix start offset calculation for chain DMA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assumption that chain DMA module starts the link DMA when 1ms of data is available from host is not correct. Instead the firmware chain DMA module fills the link DMA with initial buffer of zeroes and the host and link DMAs are started at the same time. This results in a small error in delay calculation. This can become a more severe problem if host DMA has delays that exceed 1ms. This results in negative delay to be calculated and bogus values reported to applications. This can confuse some applications like alsa_conformance_test. Fix the issue by correctly calculating the firmware chain DMA preamble size and initializing the start offset to this value. Cc: stable@vger.kernel.org Fixes: a1d203d390e0 ("ASoC: SOF: ipc4-pcm: Enable delay reporting for ChainDMA streams") Signed-off-by: Kai Vehmanen Reviewed-by: Péter Ujfalusi Reviewed-by: Bard Liao Signed-off-by: Peter Ujfalusi Link: https://patch.msgid.link/20251002074719.2084-3-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown (cherry picked from commit bace10b59624e6bd8d68bc9304357f292f1b3dcf) Signed-off-by: Wentao Guan --- sound/soc/sof/ipc4-pcm.c | 14 ++++++++++---- sound/soc/sof/ipc4-topology.c | 1 - sound/soc/sof/ipc4-topology.h | 2 ++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index fa9d61b7c8790..fed6c6a8d257b 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -1007,7 +1007,7 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, return -EINVAL; } else if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_CHAIN_DMA_NODE_ID) { /* - * While the firmware does not supports time_info reporting for + * While the firmware does not support time_info reporting for * streams using ChainDMA, it is granted that ChainDMA can only * be used on Host+Link pairs where the link position is * accessible from the host side. @@ -1015,10 +1015,16 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, * Enable delay calculation in case of ChainDMA via host * accessible registers. * - * The ChainDMA uses 2x 1ms ping-pong buffer, dai side starts - * when 1ms data is available + * The ChainDMA prefills the link DMA with a preamble + * of zero samples. Set the stream start offset based + * on size of the preamble (driver provided fifo size + * multiplied by 2.5). We add 1ms of margin as the FW + * will align the buffer size to DMA hardware + * alignment that is not known to host. */ - time_info->stream_start_offset = substream->runtime->rate / MSEC_PER_SEC; + int pre_ms = SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS * 5 / 2 + 1; + + time_info->stream_start_offset = pre_ms * substream->runtime->rate / MSEC_PER_SEC; goto out; } diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index e068e84222928..71442ca58f7fe 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -32,7 +32,6 @@ MODULE_PARM_DESC(ipc4_ignore_cpc, #define SOF_IPC4_GAIN_PARAM_ID 0 #define SOF_IPC4_TPLG_ABI_SIZE 6 -#define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2 static DEFINE_IDA(alh_group_ida); static DEFINE_IDA(pipeline_ida); diff --git a/sound/soc/sof/ipc4-topology.h b/sound/soc/sof/ipc4-topology.h index 5743f66ceb59a..afafb99435245 100644 --- a/sound/soc/sof/ipc4-topology.h +++ b/sound/soc/sof/ipc4-topology.h @@ -249,6 +249,8 @@ struct sof_ipc4_dma_stream_ch_map { #define SOF_IPC4_DMA_METHOD_HDA 1 #define SOF_IPC4_DMA_METHOD_GPDMA 2 /* defined for consistency but not used */ +#define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2 + /** * struct sof_ipc4_dma_config: DMA configuration * @dma_method: HDAudio or GPDMA From 984fa640c0f634d6fa074385eaa2cbedfdeed896 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Thu, 23 Oct 2025 15:37:52 -0700 Subject: [PATCH 85/85] perf/x86/intel/cstate: Remove PC3 support from LunarLake LunarLake doesn't support Package C3. Remove the PC3 residency counter support from LunarLake. Fixes: 26579860fbd5 ("perf/x86/intel/cstate: Add Lunarlake support") Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20251023223754.1743928-3-zide.chen@intel.com (cherry picked from commit 4ba45f041abe60337fdeeb68553b9ee1217d544e) Signed-off-by: Wentao Guan --- arch/x86/events/intel/cstate.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 27089c906ab0e..83f40f5bce359 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -70,7 +70,7 @@ * perf code: 0x01 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, - * ADL,RPL,MTL,ARL,LNL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 @@ -649,7 +649,6 @@ static const struct cstate_model lnl_cstates __initconst = { BIT(PERF_CSTATE_CORE_C7_RES), .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | - BIT(PERF_CSTATE_PKG_C3_RES) | BIT(PERF_CSTATE_PKG_C6_RES) | BIT(PERF_CSTATE_PKG_C10_RES), };