diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 04eb45a2f9406..6bc9b2114a3dd 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -202,12 +202,11 @@ PMD-mappable transparent hugepage:: cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size -khugepaged will be automatically started when one or more hugepage -sizes are enabled (either by directly setting "always" or "madvise", -or by setting "inherit" while the top-level enabled is set to "always" -or "madvise"), and it'll be automatically shutdown when the last -hugepage size is disabled (either by directly setting "never", or by -setting "inherit" while the top-level enabled is set to "never"). +khugepaged will be automatically started when PMD-sized THP is enabled +(either of the per-size anon control or the top-level control are set +to "always" or "madvise"), and it'll be automatically shutdown when +PMD-sized THP is disabled (when both the per-size anon control and the +top-level control are "never") Khugepaged controls ------------------- diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 443e30284753c..24b5e6b23417b 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -179,7 +179,6 @@ struct kvm_nvhe_init_params { unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; - unsigned long tmp; }; /* diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index aead3d38e570e..ea2d740db81c5 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -148,7 +148,6 @@ int main(void) DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); - DEFINE(NVHE_INIT_TMP, offsetof(struct kvm_nvhe_init_params, tmp)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 62c39a470eeea..11e16f98c67b4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -101,16 +101,19 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) and x2, x1, x2 cbz x2, 1f - // hVHE: Replay the EL2 setup to account for the E2H bit - // TPIDR_EL2 is used to preserve x0 across the macro maze... + /* + * hVHE: Replay the EL2 setup to account for the E2H bit + * TPIDR_EL2 and FAR_EL2 are used to preserve x0 and LR across + * the macro maze... + */ isb msr tpidr_el2, x0 - str lr, [x0, #NVHE_INIT_TMP] + msr far_el2, lr bl __kvm_init_el2_state + mrs lr, far_el2 mrs x0, tpidr_el2 - ldr lr, [x0, #NVHE_INIT_TMP] 1: ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d5b3b19b87195..498e4114c677e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1276,13 +1276,16 @@ int x86_perf_rdpmc_index(struct perf_event *event) return event->hw.event_base_rdpmc; } -static inline int match_prev_assignment(struct hw_perf_event *hwc, +static inline int match_prev_assignment(struct perf_event *event, struct cpu_hw_events *cpuc, int i) { + struct hw_perf_event *hwc = &event->hw; + return hwc->idx == cpuc->assign[i] && - hwc->last_cpu == smp_processor_id() && - hwc->last_tag == cpuc->tags[i]; + hwc->last_cpu == smp_processor_id() && + hwc->last_tag == cpuc->tags[i] && + !is_acr_event_group(event); } static void x86_pmu_start(struct perf_event *event, int flags); @@ -1328,7 +1331,7 @@ static void x86_pmu_enable(struct pmu *pmu) * - no other event has used the counter since */ if (hwc->idx == -1 || - match_prev_assignment(hwc, cpuc, i)) + match_prev_assignment(event, cpuc, i)) continue; /* @@ -1348,7 +1351,7 @@ static void x86_pmu_enable(struct pmu *pmu) event = cpuc->event_list[i]; hwc = &event->hw; - if (!match_prev_assignment(hwc, cpuc, i)) + if (!match_prev_assignment(event, cpuc, i)) x86_assign_hw_event(event, cpuc, i); else if (i < n_running) continue; diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 2e52ca87b4a8f..83f40f5bce359 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -41,7 +41,7 @@ * MSR_CORE_C1_RES: CORE C1 Residency Counter * perf code: 0x00 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL - * MTL,SRF,GRR + * MTL,SRF,GRR,ARL,LNL * Scope: Core (each processor core has a MSR) * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter * perf code: 0x01 @@ -53,30 +53,31 @@ * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, - * GRR + * GRR,ARL,LNL * Scope: Core * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter * perf code: 0x03 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML, - * ICL,TGL,RKL,ADL,RPL,MTL + * ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL * Scope: Core * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. * perf code: 0x00 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL, - * RPL,SPR,MTL + * RPL,SPR,MTL,ARL,LNL,SRF * Scope: Package (physical package) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. * perf code: 0x01 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, - * ADL,RPL,MTL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, - * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF + * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF, + * ARL,LNL * Scope: Package (physical package) * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. * perf code: 0x03 @@ -86,7 +87,7 @@ * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter. * perf code: 0x04 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL, - * ADL,RPL,MTL + * ADL,RPL,MTL,ARL * Scope: Package (physical package) * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter. * perf code: 0x05 @@ -95,7 +96,7 @@ * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter. * perf code: 0x06 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL, - * TNT,RKL,ADL,RPL,MTL + * TNT,RKL,ADL,RPL,MTL,ARL,LNL * Scope: Package (physical package) * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter. * perf code: 0x00 @@ -642,6 +643,16 @@ static const struct cstate_model adl_cstates __initconst = { BIT(PERF_CSTATE_PKG_C10_RES), }; +static const struct cstate_model lnl_cstates __initconst = { + .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | + BIT(PERF_CSTATE_CORE_C6_RES) | + BIT(PERF_CSTATE_CORE_C7_RES), + + .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | + BIT(PERF_CSTATE_PKG_C6_RES) | + BIT(PERF_CSTATE_PKG_C10_RES), +}; + static const struct cstate_model slm_cstates __initconst = { .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | BIT(PERF_CSTATE_CORE_C6_RES), @@ -683,7 +694,8 @@ static const struct cstate_model srf_cstates __initconst = { .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | BIT(PERF_CSTATE_CORE_C6_RES), - .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES), + .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | + BIT(PERF_CSTATE_PKG_C6_RES), .module_events = BIT(PERF_CSTATE_MODULE_C6_RES), }; @@ -762,6 +774,10 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &adl_cstates), X86_MATCH_VFM(INTEL_METEORLAKE, &adl_cstates), X86_MATCH_VFM(INTEL_METEORLAKE_L, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE_H, &adl_cstates), + X86_MATCH_VFM(INTEL_ARROWLAKE_U, &adl_cstates), + X86_MATCH_VFM(INTEL_LUNARLAKE_M, &lnl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 425e88a06a483..61c9d5e8d9814 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -744,7 +744,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -940,16 +940,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1136,6 +1138,42 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + if (uncore_pmu_broken(pmu)) + return -ENODEV; + + uncore_box_init(box); + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1155,26 +1193,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } @@ -1236,11 +1270,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1260,7 +1299,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } @@ -1476,7 +1514,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1484,12 +1523,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1505,7 +1546,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1516,7 +1557,7 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) uncore_box_exit(box); } @@ -1565,7 +1606,7 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); if (!box) @@ -1591,14 +1632,14 @@ static int allocate_boxes(struct intel_uncore_type **types, } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; int i, ret; - ret = allocate_boxes(types, id, cpu); + ret = allocate_boxes(types, die, cpu); if (ret) return ret; @@ -1606,7 +1647,7 @@ static int uncore_box_ref(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) uncore_box_init(box); } @@ -1621,8 +1662,6 @@ static int uncore_event_cpu_online(unsigned int cpu) die = topology_logical_die_id(cpu); msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; /* * Check if there is an online cpu in the package diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index 9ba2305c0d9ae..a797dd6172ea0 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -125,13 +125,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; @@ -164,7 +175,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -537,7 +548,7 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, static inline void uncore_box_init(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->init_box) box->pmu->type->ops->init_box(box); } @@ -545,7 +556,7 @@ static inline void uncore_box_init(struct intel_uncore_box *box) static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 6765f31479c5d..5a7a65aec263d 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -479,8 +479,8 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) struct intel_uncore_discovery_unit *unit; unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes, - -1, box->pmu->pmu_idx); - if (WARN_ON_ONCE(!unit)) + box->dieid, box->pmu->pmu_idx); + if (!unit) return 0; return unit->addr; diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 47ceade21f562..9985937046268 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -874,7 +874,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 740cce3b61fee..f906213ab8056 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4356,11 +4356,6 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault return RET_PF_EMULATE; } - if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) { - kvm_mmu_prepare_memory_fault_exit(vcpu, fault); - return -EFAULT; - } - if (fault->is_private) return kvm_faultin_pfn_private(vcpu, fault); @@ -4398,9 +4393,49 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, { int ret; + /* + * Note that the mmu_invalidate_seq also serves to detect a concurrent + * change in attributes. is_page_fault_stale() will detect an + * invalidation relate to fault->fn and resume the guest without + * installing a mapping in the page tables. + */ fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq; smp_rmb(); + /* + * Now that we have a snapshot of mmu_invalidate_seq we can check for a + * private vs. shared mismatch. + */ + if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) { + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); + return -EFAULT; + } + + /* + * Check for a relevant mmu_notifier invalidation event before getting + * the pfn from the primary MMU, and before acquiring mmu_lock. + * + * For mmu_lock, if there is an in-progress invalidation and the kernel + * allows preemption, the invalidation task may drop mmu_lock and yield + * in response to mmu_lock being contended, which is *very* counter- + * productive as this vCPU can't actually make forward progress until + * the invalidation completes. + * + * Retrying now can also avoid unnessary lock contention in the primary + * MMU, as the primary MMU doesn't necessarily hold a single lock for + * the duration of the invalidation, i.e. faulting in a conflicting pfn + * can cause the invalidation to take longer by holding locks that are + * needed to complete the invalidation. + * + * Do the pre-check even for non-preemtible kernels, i.e. even if KVM + * will never yield mmu_lock in response to contention, as this vCPU is + * *guaranteed* to need to retry, i.e. waiting until mmu_lock is held + * to detect retry guarantees the worst case latency for the vCPU. + */ + if (fault->slot && + mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn)) + return RET_PF_RETRY; + ret = __kvm_faultin_pfn(vcpu, fault); if (ret != RET_PF_CONTINUE) return ret; @@ -4411,6 +4446,18 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, if (unlikely(!fault->slot)) return kvm_handle_noslot_fault(vcpu, fault, access); + /* + * Check again for a relevant mmu_notifier invalidation event purely to + * avoid contending mmu_lock. Most invalidations will be detected by + * the previous check, but checking is extremely cheap relative to the + * overall cost of failing to detect the invalidation until after + * mmu_lock is acquired. + */ + if (mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn)) { + kvm_release_pfn_clean(fault->pfn); + return RET_PF_RETRY; + } + return RET_PF_CONTINUE; } @@ -4438,6 +4485,11 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu, if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) return true; + /* + * Check for a relevant mmu_notifier invalidation event one last time + * now that mmu_lock is held, as the "unsafe" checks performed without + * holding mmu_lock can get false negatives. + */ return fault->slot && mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn); } @@ -7264,9 +7316,30 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) } #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; +} + bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range) { + struct kvm_memory_slot *slot = range->slot; + int level; + /* * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM @@ -7281,27 +7354,49 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm))) return false; - return kvm_unmap_gfn_range(kvm, range); -} + if (WARN_ON_ONCE(range->end <= range->start)) + return false; -static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG; -} + /* + * If the head and tail pages of the range currently allow a hugepage, + * i.e. reside fully in the slot and don't have mixed attributes, then + * add each corresponding hugepage range to the ongoing invalidation, + * e.g. to prevent KVM from creating a hugepage in response to a fault + * for a gfn whose attributes aren't changing. Note, only the range + * of gfns whose attributes are being modified needs to be explicitly + * unmapped, as that will unmap any existing hugepages. + */ + for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { + gfn_t start = gfn_round_for_level(range->start, level); + gfn_t end = gfn_round_for_level(range->end - 1, level); + gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level); -static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG; -} + if ((start != range->start || start + nr_pages > range->end) && + start >= slot->base_gfn && + start + nr_pages <= slot->base_gfn + slot->npages && + !hugepage_test_mixed(slot, start, level)) + kvm_mmu_invalidate_range_add(kvm, start, start + nr_pages); -static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, - int level) -{ - lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; + if (end == start) + continue; + + if ((end + nr_pages) > range->end && + (end + nr_pages) <= (slot->base_gfn + slot->npages) && + !hugepage_test_mixed(slot, end, level)) + kvm_mmu_invalidate_range_add(kvm, end, end + nr_pages); + } + + /* Unmap the old attribute page. */ + if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE) + range->attr_filter = KVM_FILTER_SHARED; + else + range->attr_filter = KVM_FILTER_PRIVATE; + + return kvm_unmap_gfn_range(kvm, range); } + + static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, int level, unsigned long attrs) { diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index ae9b00b0ac486..93e0f071e43b0 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -2,11 +2,13 @@ /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -157,6 +159,16 @@ static bool async_mode; /* Use interrupts */ static bool use_irq; +struct iaa_req_ctx { + u32 compression_crc; + struct page *bounce_src; + dma_addr_t bounce_src_dma; + unsigned int bounce_src_len; +}; + +static mempool_t *iaa_bounce_pool; +#define IAA_BOUNCE_POOL_SIZE 128 + /** * set_iaa_sync_mode - Set IAA sync mode * @name: The name of the sync mode @@ -989,6 +1001,28 @@ static inline int check_completion(struct device *dev, return ret; } +static bool iaa_error_should_retry(struct idxd_desc *idxd_desc) +{ + return idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR; +} + +static void iaa_unmap_src(struct device *dev, struct acomp_req *req) +{ + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + + if (req_ctx->bounce_src) { + dma_unmap_page(dev, req_ctx->bounce_src_dma, + req_ctx->bounce_src_len, DMA_TO_DEVICE); + mempool_free(req_ctx->bounce_src, iaa_bounce_pool); + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + return; + } + + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); +} + static int deflate_generic_decompress(struct acomp_req *req) { void *src, *dst; @@ -1008,6 +1042,19 @@ static int deflate_generic_decompress(struct acomp_req *req) return ret; } +static int deflate_generic_compress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_compress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_comp_calls(); + + return ret; +} + static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, struct acomp_req *req, dma_addr_t *src_addr, dma_addr_t *dst_addr); @@ -1015,8 +1062,7 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc); + dma_addr_t dst_addr, unsigned int *dlen); static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1026,6 +1072,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *compression_ctx; struct crypto_ctx *ctx = __ctx; + struct iaa_req_ctx *req_ctx = acomp_request_ctx(ctx->req); struct iaa_device *iaa_device; struct idxd_device *idxd; struct iaa_wq *iaa_wq; @@ -1052,18 +1099,21 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, ctx->compress, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (!ctx->compress && - idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { + if (!ctx->compress && iaa_error_should_retry(idxd_desc)) { pr_warn("%s: falling back to deflate-generic decompress, " "analytics error code %x\n", __func__, idxd_desc->iax_completion->error_code); + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), + DMA_FROM_DEVICE); + iaa_unmap_src(dev, ctx->req); + ret = deflate_generic_decompress(ctx->req); if (ret) { dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", __func__, ret); err = -EIO; - goto err; } + goto out; } else { err = -EIO; goto err; @@ -1083,9 +1133,8 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, if (ctx->compress && compression_ctx->verify_compress) { dma_addr_t src_addr, dst_addr; - u32 compression_crc; - compression_crc = idxd_desc->iax_completion->crc; + req_ctx->compression_crc = idxd_desc->iax_completion->crc; ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); if (ret) { @@ -1095,8 +1144,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, - ctx->req->slen, dst_addr, &ctx->req->dlen, - compression_crc); + ctx->req->slen, dst_addr, &ctx->req->dlen); if (ret) { dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); err = -EIO; @@ -1109,7 +1157,7 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, } err: dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE); - dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE); + iaa_unmap_src(dev, ctx->req); out: if (ret != 0) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); @@ -1125,12 +1173,11 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 *compression_crc, - bool disable_async) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1170,7 +1217,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, desc->src2_size = sizeof(struct aecs_comp_table_record); desc->completion_addr = idxd_desc->compl_dma; - if (ctx->use_irq && !disable_async) { + if (ctx->use_irq) { desc->flags |= IDXD_OP_FLAG_RCI; idxd_desc->crypto.req = req; @@ -1183,8 +1230,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: compression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1204,7 +1250,7 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_comp_calls(); update_wq_comp_calls(wq); - if (ctx->async_mode && !disable_async) { + if (ctx->async_mode) { ret = -EINPROGRESS; dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); goto out; @@ -1222,9 +1268,9 @@ static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, update_total_comp_bytes_out(*dlen); update_wq_comp_bytes(wq, *dlen); - *compression_crc = idxd_desc->iax_completion->crc; + req_ctx->compression_crc = idxd_desc->iax_completion->crc; - if (!ctx->async_mode || disable_async) + if (!ctx->async_mode) idxd_free_desc(wq, idxd_desc); out: return ret; @@ -1278,11 +1324,11 @@ static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen, - u32 compression_crc) + dma_addr_t dst_addr, unsigned int *dlen) { struct iaa_device_compression_mode *active_compression_mode; struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_device *iaa_device; struct idxd_desc *idxd_desc; struct iax_hw_desc *desc; @@ -1342,10 +1388,10 @@ static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, goto err; } - if (compression_crc != idxd_desc->iax_completion->crc) { + if (req_ctx->compression_crc != idxd_desc->iax_completion->crc) { ret = -EINVAL; - dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=0x%x, decomp=0x%x\n", compression_crc, + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch: comp=0x%x, decomp=0x%x\n", + req_ctx->compression_crc, idxd_desc->iax_completion->crc); print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, idxd_desc->iax_completion, 64, 0); @@ -1421,8 +1467,7 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, " src_addr %llx, dst_addr %llx\n", __func__, active_compression_mode->name, src_addr, dst_addr); - } else if (ctx->async_mode && !disable_async) - req->base.data = idxd_desc; + } dev_dbg(dev, "%s: decompression mode %s," " desc->src1_addr %llx, desc->src1_size %d," @@ -1451,19 +1496,9 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, ret = check_completion(dev, idxd_desc->iax_completion, false, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { - pr_warn("%s: falling back to deflate-generic decompress, " - "analytics error code %x\n", __func__, - idxd_desc->iax_completion->error_code); - ret = deflate_generic_decompress(req); - if (ret) { - dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", - __func__, ret); - goto err; - } - } else { - goto err; - } + if (iaa_error_should_retry(idxd_desc)) + ret = -EAGAIN; + goto err; } else { req->dlen = idxd_desc->iax_completion->output_size; } @@ -1487,16 +1522,18 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, static int iaa_comp_acompress(struct acomp_req *req) { + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - bool disable_async = false; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; - u32 compression_crc; struct idxd_wq *wq; struct device *dev; - int order = -1; + + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; compression_ctx = crypto_tfm_ctx(tfm); @@ -1505,11 +1542,15 @@ static int iaa_comp_acompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not compressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not compressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_compress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1526,51 +1567,31 @@ static int iaa_comp_acompress(struct acomp_req *req) iaa_wq = idxd_wq_get_private(wq); - if (!req->dst) { - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - - /* incompressible data will always be < 2 * slen */ - req->dlen = 2 * req->slen; - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - disable_async = true; - } - dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_compress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_compress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, - &req->dlen, &compression_crc, disable_async); + &req->dlen); if (ret == -EINPROGRESS) return ret; @@ -1582,12 +1603,12 @@ static int iaa_comp_acompress(struct acomp_req *req) } ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, compression_crc); + dst_addr, &req->dlen); if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_FROM_DEVICE); goto out; } @@ -1595,130 +1616,49 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: - iaa_wq_put(wq); - - if (order >= 0) - sgl_free_order(req->dst, order); - - return ret; -} - -static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) -{ - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? - GFP_KERNEL : GFP_ATOMIC; - struct crypto_tfm *tfm = req->base.tfm; - dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; - struct iaa_wq *iaa_wq; - struct device *dev; - struct idxd_wq *wq; - int order = -1; - - cpu = get_cpu(); - wq = wq_table_next_wq(cpu); - put_cpu(); - if (!wq) { - pr_debug("no wq configured for cpu=%d\n", cpu); - return -ENODEV; - } - - ret = iaa_wq_get(wq); - if (ret) { - pr_debug("no wq available for cpu=%d\n", cpu); - return -ENODEV; - } - - iaa_wq = idxd_wq_get_private(wq); - - dev = &wq->idxd->pdev->dev; - - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; - } - src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, - req->src, req->slen, sg_dma_len(req->src)); - - req->dlen = 4 * req->slen; /* start with ~avg comp rato */ -alloc_dest: - order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); - req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); - if (!req->dst) { - ret = -ENOMEM; - order = -1; - goto out; - } - - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; - } - - dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, - req->dst, req->dlen, sg_dma_len(req->dst)); - ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, - dst_addr, &req->dlen, true); - if (ret == -EOVERFLOW) { - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - req->dlen *= 2; - if (req->dlen > CRYPTO_ACOMP_DST_MAX) - goto err_map_dst; - goto alloc_dest; - } - - if (ret != 0) - dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); out: iaa_wq_put(wq); - if (order >= 0) - sgl_free_order(req->dst, order); - return ret; } static int iaa_comp_adecompress(struct acomp_req *req) { + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + bool use_bounce_src = false; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; + struct page *page; + + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; if (!iaa_crypto_enabled) { pr_debug("iaa_crypto disabled, not decompressing\n"); return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not decompressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not decompressing\n"); return -EINVAL; } - if (!req->dst) - return iaa_comp_adecompress_alloc_dest(req); + /* Fall back to software if dst has multiple sg entries */ + if (sg_nents(req->dst) > 1) + return deflate_generic_decompress(req); + + if (sg_nents(req->src) > 1) { + if (req->slen > PAGE_SIZE) + return deflate_generic_decompress(req); + use_bounce_src = true; + } cpu = get_cpu(); wq = wq_table_next_wq(cpu); @@ -1738,30 +1678,52 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (unlikely(use_bounce_src)) { + page = mempool_alloc(iaa_bounce_pool, GFP_ATOMIC); + if (!page) { + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + if (sg_copy_to_buffer(req->src, sg_nents(req->src), + page_address(page), req->slen) != req->slen) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = dma_map_page(dev, page, 0, req->slen, DMA_TO_DEVICE); + if (dma_mapping_error(dev, src_addr)) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + req_ctx->bounce_src = page; + req_ctx->bounce_src_dma = src_addr; + req_ctx->bounce_src_len = req->slen; + } else { + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, + req->src, req->slen, sg_dma_len(req->src)); } - src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, - req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_unmap_src(dev, req); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, @@ -1769,15 +1731,16 @@ static int iaa_comp_adecompress(struct acomp_req *req) if (ret == -EINPROGRESS) return ret; - if (ret != 0) + if (ret != 0 && ret != -EAGAIN) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + iaa_unmap_src(dev, req); iaa_wq_put(wq); + if (ret == -EAGAIN) + ret = deflate_generic_decompress(req); + return ret; } @@ -1800,24 +1763,16 @@ static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) return 0; } -static void dst_free(struct scatterlist *sgl) -{ - /* - * Called for req->dst = NULL cases but we free elsewhere - * using sgl_free_order(). - */ -} - static struct acomp_alg iaa_acomp_fixed_deflate = { .init = iaa_comp_init_fixed, .compress = iaa_comp_acompress, .decompress = iaa_comp_adecompress, - .dst_free = dst_free, .base = { .cra_name = "deflate", .cra_driver_name = "deflate-iaa", .cra_flags = CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(struct iaa_req_ctx), .cra_module = THIS_MODULE, .cra_priority = IAA_ALG_PRIORITY, } @@ -2027,6 +1982,12 @@ static int __init iaa_crypto_init_module(void) goto err_aecs_init; } + iaa_bounce_pool = mempool_create_page_pool(IAA_BOUNCE_POOL_SIZE, 0); + if (!iaa_bounce_pool) { + ret = -ENOMEM; + goto err_bounce_pool; + } + ret = idxd_driver_register(&iaa_crypto_driver); if (ret) { pr_debug("IAA wq sub-driver registration failed\n"); @@ -2060,6 +2021,9 @@ static int __init iaa_crypto_init_module(void) err_verify_attr_create: idxd_driver_unregister(&iaa_crypto_driver); err_driver_reg: + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; +err_bounce_pool: iaa_aecs_cleanup_fixed(); err_aecs_init: crypto_free_comp(deflate_generic_tfm); @@ -2078,6 +2042,8 @@ static void __exit iaa_crypto_cleanup_module(void) driver_remove_file(&iaa_crypto_driver.drv, &driver_attr_verify_compress); idxd_driver_unregister(&iaa_crypto_driver); + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; iaa_aecs_cleanup_fixed(); crypto_free_comp(deflate_generic_tfm); diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c index f5cc3d29ca19e..2f2ed88c88120 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -19,6 +19,7 @@ static atomic64_t total_comp_calls; static atomic64_t total_decomp_calls; +static atomic64_t total_sw_comp_calls; static atomic64_t total_sw_decomp_calls; static atomic64_t total_comp_bytes_out; static atomic64_t total_decomp_bytes_in; @@ -43,6 +44,11 @@ void update_total_decomp_calls(void) atomic64_inc(&total_decomp_calls); } +void update_total_sw_comp_calls(void) +{ + atomic64_inc(&total_sw_comp_calls); +} + void update_total_sw_decomp_calls(void) { atomic64_inc(&total_sw_decomp_calls); @@ -104,6 +110,7 @@ static void reset_iaa_crypto_stats(void) { atomic64_set(&total_comp_calls, 0); atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_comp_calls, 0); atomic64_set(&total_sw_decomp_calls, 0); atomic64_set(&total_comp_bytes_out, 0); atomic64_set(&total_decomp_bytes_in, 0); @@ -174,6 +181,8 @@ static int global_stats_show(struct seq_file *m, void *v) atomic64_read(&total_comp_calls)); seq_printf(m, " total_decomp_calls: %llu\n", atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_comp_calls: %llu\n", + atomic64_read(&total_sw_comp_calls)); seq_printf(m, " total_sw_decomp_calls: %llu\n", atomic64_read(&total_sw_decomp_calls)); seq_printf(m, " total_comp_bytes_out: %llu\n", diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h index 3787a5f507eb2..6e0c6f9939bfa 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.h +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -11,6 +11,7 @@ void iaa_crypto_debugfs_cleanup(void); void update_total_comp_calls(void); void update_total_comp_bytes_out(int n); void update_total_decomp_calls(void); +void update_total_sw_comp_calls(void); void update_total_sw_decomp_calls(void); void update_total_decomp_bytes_in(int n); void update_completion_einval_errs(void); @@ -29,6 +30,7 @@ static inline void iaa_crypto_debugfs_cleanup(void) {} static inline void update_total_comp_calls(void) {} static inline void update_total_comp_bytes_out(int n) {} static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_comp_calls(void) {} static inline void update_total_sw_decomp_calls(void) {} static inline void update_total_decomp_bytes_in(int n) {} static inline void update_completion_einval_errs(void) {} diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index ef5f03be41906..5d32f2958da87 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -98,9 +98,25 @@ static struct adf_hw_device_class adf_420xx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_420XX_ACCELENGINES_MASK; - return ~me_disable & ADF_420XX_ACCELENGINES_MASK; + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + if (test_bit(12, &fuses)) + mask &= ~ADF_AE_GROUP_3; + + if (test_bit(16, &fuses)) + mask &= ~ADF_AE_GROUP_4; + + return mask; } static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 2a3598409eeb5..696d6ea05ff78 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -79,7 +79,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index bbd92c017c28e..a6d253ff20888 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -101,7 +101,7 @@ static struct adf_hw_device_class adf_4xxx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index d26564cebdec4..1e281fdeb1c59 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -81,7 +81,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 201f9412c5823..e78f7bfd30b85 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c3xxx_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index 956a4c85609a9..a984ebe0fe28d 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxx(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 6b5b0cf9c7c74..32ebe09477a8d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c62x_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index ad0ca43849985..20b4bb3ccf0cb 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62x(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); @@ -169,7 +169,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ - i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; + i = (hw_data->fuses[ADF_FUSECTL0] & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index ad36afe7d2cf7..f5b4cf2b515c5 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -52,6 +52,16 @@ enum adf_accel_capabilities { ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 }; +enum adf_fuses { + ADF_FUSECTL0, + ADF_FUSECTL1, + ADF_FUSECTL2, + ADF_FUSECTL3, + ADF_FUSECTL4, + ADF_FUSECTL5, + ADF_MAX_FUSES +}; + struct adf_bar { resource_size_t base_addr; void __iomem *virt_addr; @@ -343,7 +353,7 @@ struct adf_hw_device_data { struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; - u32 fuses; + u32 fuses[ADF_MAX_FUSES]; u32 straps; u32 accel_capabilities_mask; u32 extended_dc_capabilities; diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 179d62cb891d1..6d3a268f57aa9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -140,6 +140,8 @@ static void adf_device_reset_worker(struct work_struct *work) queue_work(device_sriov_wq, &sriov_data.sriov_work); if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 1f64bf49b221c..2b263442c8565 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -115,8 +115,8 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fuses = hw_data->fuses[ADF_FUSECTL0]; u32 straps = hw_data->straps; - u32 fuses = hw_data->fuses; u32 legfuses; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c0661ff5e9292..e48bcf1818cd1 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -29,7 +29,7 @@ static struct adf_hw_device_class dh895xcc_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & ADF_DH895XCC_ACCELERATORS_MASK; @@ -37,7 +37,7 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } @@ -99,7 +99,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) + int sku = (self->fuses[ADF_FUSECTL0] & ADF_DH895XCC_FUSECTL_SKU_MASK) >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; switch (sku) { diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 9d24754a30eae..14ce565b3663c 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -128,7 +128,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcc(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index e0d35050fef75..afc8d4e3e16f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -187,9 +187,8 @@ nvkm_firmware_dtor(struct nvkm_firmware *fw) break; case NVKM_FIRMWARE_IMG_DMA: nvkm_memory_unref(&memory); - dma_unmap_single(fw->device->dev, fw->phys, sg_dma_len(&fw->mem.sgl), - DMA_TO_DEVICE); - kfree(fw->img); + dma_free_noncoherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), + fw->img, fw->phys, DMA_TO_DEVICE); break; default: WARN_ON(1); @@ -213,17 +212,16 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, fw->img = kmemdup(src, fw->len, GFP_KERNEL); break; case NVKM_FIRMWARE_IMG_DMA: { + dma_addr_t addr; len = ALIGN(fw->len, PAGE_SIZE); - fw->img = kmalloc(len, GFP_KERNEL); - if (!fw->img) - return -ENOMEM; - - memcpy(fw->img, src, fw->len); - fw->phys = dma_map_single(fw->device->dev, fw->img, len, DMA_TO_DEVICE); - if (dma_mapping_error(fw->device->dev, fw->phys)) { - kfree(fw->img); - return -EFAULT; + fw->img = dma_alloc_noncoherent(fw->device->dev, + len, &addr, + DMA_TO_DEVICE, + GFP_KERNEL); + if (fw->img) { + memcpy(fw->img, src, fw->len); + fw->phys = addr; } sg_init_one(&fw->mem.sgl, fw->img, len); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index 1a4ff5ed6dd58..4e8b3f1c7e25d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -89,6 +89,7 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, nvkm_falcon_fw_dtor_sigs(fw); } + FLCNFW_DBG(fw, "resetting"); fw->func->reset(fw); @@ -99,6 +100,12 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, goto done; } + /* after last write to the img, sync dma mappings */ + dma_sync_single_for_device(fw->fw.device->dev, + fw->fw.phys, + sg_dma_len(&fw->fw.mem.sgl), + DMA_TO_DEVICE); + ret = fw->func->load(fw); if (ret) goto done; diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index 5e2a22a3f034e..4156c9b44a512 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -35,6 +35,9 @@ struct iova_bitmap_map { /* base IOVA representing bit 0 of the first page */ unsigned long iova; + /* mapped length */ + unsigned long length; + /* page size order that each bit granules to */ unsigned long pgshift; @@ -113,9 +116,6 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; - - /* length of the IOVA range set ahead the pinned pages */ - unsigned long set_ahead_length; }; /* @@ -155,6 +155,8 @@ static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip); } +static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap); + /* * Pins the bitmap user pages for the current range window. * This is internal to IOVA bitmap and called when advancing the @@ -205,6 +207,7 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) * aligned. */ mapped->pgoff = offset_in_page(addr); + mapped->length = iova_bitmap_mapped_length(bitmap); return 0; } @@ -262,9 +265,6 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, goto err; } - rc = iova_bitmap_get(bitmap); - if (rc) - goto err; return bitmap; err: @@ -337,66 +337,34 @@ static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) } /* - * Returns true if there's not more data to iterate. + * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range. */ -static bool iova_bitmap_done(struct iova_bitmap *bitmap) -{ - return bitmap->mapped_base_index >= bitmap->mapped_total_index; -} - -static int iova_bitmap_set_ahead(struct iova_bitmap *bitmap, - size_t set_ahead_length) +static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped, + unsigned long iova, size_t length) { - int ret = 0; - - while (set_ahead_length > 0 && !iova_bitmap_done(bitmap)) { - unsigned long length = iova_bitmap_mapped_length(bitmap); - unsigned long iova = iova_bitmap_mapped_iova(bitmap); - - ret = iova_bitmap_get(bitmap); - if (ret) - break; - - length = min(length, set_ahead_length); - iova_bitmap_set(bitmap, iova, length); - - set_ahead_length -= length; - bitmap->mapped_base_index += - iova_bitmap_offset_to_index(bitmap, length - 1) + 1; - iova_bitmap_put(bitmap); - } - - bitmap->set_ahead_length = 0; - return ret; + return mapped->npages && + (iova >= mapped->iova && + (iova + length - 1) <= (mapped->iova + mapped->length - 1)); } /* - * Advances to the next range, releases the current pinned + * Advances to a selected range, releases the current pinned * pages and pins the next set of bitmap pages. * Returns 0 on success or otherwise errno. */ -static int iova_bitmap_advance(struct iova_bitmap *bitmap) +static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, + unsigned long iova) { - unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1; - unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1; + unsigned long index; - bitmap->mapped_base_index += count; + index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); + if (index >= bitmap->mapped_total_index) + return -EINVAL; + bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); - /* Iterate, set and skip any bits requested for next iteration */ - if (bitmap->set_ahead_length) { - int ret; - - ret = iova_bitmap_set_ahead(bitmap, bitmap->set_ahead_length); - if (ret) - return ret; - } - - if (iova_bitmap_done(bitmap)) - return 0; - - /* When advancing the index we pin the next set of bitmap pages */ + /* Pin the next set of bitmap pages */ return iova_bitmap_get(bitmap); } @@ -416,17 +384,7 @@ static int iova_bitmap_advance(struct iova_bitmap *bitmap) int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - int ret = 0; - - for (; !iova_bitmap_done(bitmap) && !ret; - ret = iova_bitmap_advance(bitmap)) { - ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap), - iova_bitmap_mapped_length(bitmap), opaque); - if (ret) - break; - } - - return ret; + return fn(bitmap, bitmap->iova, bitmap->length, opaque); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, IOMMUFD); @@ -444,11 +402,25 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long iova, size_t length) { struct iova_bitmap_map *mapped = &bitmap->mapped; - unsigned long cur_bit = ((iova - mapped->iova) >> - mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; - unsigned long last_bit = (((iova + length - 1) - mapped->iova) >> - mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; - unsigned long last_page_idx = mapped->npages - 1; + unsigned long cur_bit, last_bit, last_page_idx; + +update_indexes: + if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { + + /* + * The attempt to advance the base index to @iova + * may fail if it's out of bounds, or pinning the pages + * returns an error. + */ + if (iova_bitmap_advance_to(bitmap, iova)) + return; + } + + last_page_idx = mapped->npages - 1; + cur_bit = ((iova - mapped->iova) >> + mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; + last_bit = (((iova + length - 1) - mapped->iova) >> + mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE; do { unsigned int page_idx = cur_bit / BITS_PER_PAGE; @@ -457,18 +429,19 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, last_bit - cur_bit + 1); void *kaddr; - if (unlikely(page_idx > last_page_idx)) - break; + if (unlikely(page_idx > last_page_idx)) { + unsigned long left = + ((last_bit - cur_bit + 1) << mapped->pgshift); + + iova += (length - left); + length = left; + goto update_indexes; + } kaddr = kmap_local_page(mapped->pages[page_idx]); bitmap_set(kaddr, offset, nbits); kunmap_local(kaddr); cur_bit += nbits; } while (cur_bit <= last_bit); - - if (unlikely(cur_bit <= last_bit)) { - bitmap->set_ahead_length = - ((last_bit - cur_bit + 1) << bitmap->mapped.pgshift); - } } EXPORT_SYMBOL_NS_GPL(iova_bitmap_set, IOMMUFD); diff --git a/drivers/misc/mei/platform-vsc.c b/drivers/misc/mei/platform-vsc.c index c51ef6a922116..f67490e7b2911 100644 --- a/drivers/misc/mei/platform-vsc.c +++ b/drivers/misc/mei/platform-vsc.c @@ -256,8 +256,6 @@ static int mei_vsc_hw_reset(struct mei_device *mei_dev, bool intr_enable) vsc_tp_reset(hw->tp); - vsc_tp_intr_disable(hw->tp); - return vsc_tp_init(hw->tp, mei_dev->dev); } diff --git a/drivers/misc/mei/vsc-tp.c b/drivers/misc/mei/vsc-tp.c index c50bf6097b290..79c7a890f67b4 100644 --- a/drivers/misc/mei/vsc-tp.c +++ b/drivers/misc/mei/vsc-tp.c @@ -369,8 +369,6 @@ void vsc_tp_reset(struct vsc_tp *tp) gpiod_set_value_cansleep(tp->wakeupfw, 1); atomic_set(&tp->assert_cnt, 0); - - enable_irq(tp->spi->irq); } EXPORT_SYMBOL_NS_GPL(vsc_tp_reset, VSC_TP); diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c index 781c691473e14..519c364e87d19 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c @@ -310,9 +310,9 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_MAX_QGPP_CNT), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_CUR_ENTRY), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(2), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(1), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG2 INFO"), @@ -414,7 +414,7 @@ wed_amsdu_show(struct seq_file *s, void *data) WED_AMSDU_ENG_CUR_ENTRY), DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_BUF_MERGED), - DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(4), + DUMP_WED_MASK(WED_MON_AMSDU_ENG_CNT9(7), WED_AMSDU_ENG_MAX_MSDU_MERGED), DUMP_STR("WED AMDSU ENG8 INFO"), diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c index fee9c9d3a92fe..f565ad628b19a 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c @@ -32,12 +32,12 @@ static struct mtk_wed_wo_memory_region mem_region[] = { }, }; -static u32 wo_r32(struct mtk_wed_wo *wo, u32 reg) +static u32 wo_r32(u32 reg) { return readl(mem_region[MTK_WED_WO_REGION_BOOT].addr + reg); } -static void wo_w32(struct mtk_wed_wo *wo, u32 reg, u32 val) +static void wo_w32(u32 reg, u32 val) { writel(val, mem_region[MTK_WED_WO_REGION_BOOT].addr + reg); } @@ -373,13 +373,17 @@ mtk_wed_mcu_load_firmware(struct mtk_wed_wo *wo) boot_cr = MTK_WO_MCU_CFG_LS_WA_BOOT_ADDR_ADDR; else boot_cr = MTK_WO_MCU_CFG_LS_WM_BOOT_ADDR_ADDR; - wo_w32(wo, boot_cr, mem_region[MTK_WED_WO_REGION_EMI].phy_addr >> 16); + wo_w32(boot_cr, mem_region[MTK_WED_WO_REGION_EMI].phy_addr >> 16); /* wo firmware reset */ - wo_w32(wo, MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); + wo_w32(MTK_WO_MCU_CFG_LS_WF_MCCR_CLR_ADDR, 0xc00); - val = wo_r32(wo, MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR) | - MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; - wo_w32(wo, MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); + val = wo_r32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR); + + if (!mtk_wed_is_v3_or_greater(wo->hw) && wo->hw->index) + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WA_CPU_RSTB_MASK; + else + val |= MTK_WO_MCU_CFG_LS_WF_WM_WA_WM_CPU_RSTB_MASK; + wo_w32(MTK_WO_MCU_CFG_LS_WF_MCU_CFG_WM_WA_ADDR, val); out: release_firmware(fw); diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index 6ffa2ca14318a..71b3730de25bb 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -1171,7 +1171,7 @@ static int rtlgen_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, if (devnum == MDIO_MMD_VEND2) ret = rtlgen_write_vend2(phydev, regnum, val); else if (devnum == MDIO_MMD_AN && regnum == MDIO_AN_EEE_ADV) - ret = rtlgen_write_vend2(phydev, regnum, RTL_MDIO_AN_EEE_ADV); + ret = rtlgen_write_vend2(phydev, RTL_MDIO_AN_EEE_ADV, val); else ret = -EOPNOTSUPP; diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index d592f55139427..7de5ddf3748a4 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -490,6 +490,15 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* on WED v3 the M_DONE bit signals that WED is done reading + * the txfree descriptor; WED v2 does not set it + */ + else if (dev->mmio.wed.version > 2 && + mt76_queue_is_wed_tx_free(q) && + !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) + return NULL; +#endif } q->tail = (q->tail + 1) % q->ndesc; diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 1de5a2b20f747..1e633b2f96d1a 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -11,7 +11,7 @@ #define MT_DMA_CTL_SD_LEN1 GENMASK(13, 0) #define MT_DMA_CTL_LAST_SEC1 BIT(14) -#define MT_DMA_CTL_BURST BIT(15) +#define MT_DMA_CTL_M_DONE BIT(15) #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 445d0f0ab7795..deb3c27ddc317 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -177,6 +177,11 @@ static inline bool is_mt7925(struct mt76_dev *dev) return mt76_chip(dev) == 0x7925; } +static inline bool is_320mhz_supported(struct mt76_dev *dev) +{ + return mt76_chip(dev) == 0x7927; +} + static inline bool is_mt7920(struct mt76_dev *dev) { return mt76_chip(dev) == 0x7920; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index e696d19811406..305c3dcaaeef0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -495,7 +495,7 @@ static int mt7915_config(struct ieee80211_hw *hw, u32 changed) mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, enabled); - mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_MDP_DCR0_RX_HDR_TRANS_EN, + mt76_rmw_field(dev, MT_MDP_DCR0, MT_MDP_DCR0_RX_HDR_TRANS_EN, !dev->monitor_mask); mt76_testmode_reset(phy->mt76, true); mt76_wr(dev, MT_WF_RFCR(band), rxfilter); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 58b46be0aaed7..afed2c31dc800 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -1323,6 +1323,7 @@ void mt7925_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&dev->mphy.mac_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { @@ -1432,6 +1433,10 @@ int mt7925_usb_sdio_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (!wcid) wcid = &dev->mt76.global_wcid; + err = skb_cow_head(skb, MT_SDIO_TXD_SIZE + MT_SDIO_HDR_SIZE); + if (err) + return err; + if (sta) { struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index bc35ff376f384..f78911f6d8aa6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -178,20 +178,36 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[0] |= + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + + val = (sts > 3) ? sts - 1 : 3; + eht_cap_elem->phy_cap_info[0] |= - u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), + u8_encode_bits(u8_get_bits(val, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); eht_cap_elem->phy_cap_info[1] = - u8_encode_bits(u8_get_bits(sts - 1, GENMASK(2, 1)), + u8_encode_bits(u8_get_bits(val, GENMASK(2, 1)), IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK) | - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[1] |= + u8_encode_bits(val, + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); + eht_cap_elem->phy_cap_info[2] = u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK) | u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[2] |= + u8_encode_bits(sts - 1, + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK); + eht_cap_elem->phy_cap_info[3] = IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | @@ -212,7 +228,8 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(0x11, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_160 ? 0x7 : + val = width == NL80211_CHAN_WIDTH_320 ? 0xf : + width == NL80211_CHAN_WIDTH_160 ? 0x7 : width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; eht_cap_elem->phy_cap_info[6] = u8_encode_bits(u8_get_bits(0x11, GENMASK(4, 2)), @@ -234,6 +251,11 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_nss->bw._160.rx_tx_mcs9_max_nss = val; eht_nss->bw._160.rx_tx_mcs11_max_nss = val; eht_nss->bw._160.rx_tx_mcs13_max_nss = val; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) { + eht_nss->bw._320.rx_tx_mcs9_max_nss = val; + eht_nss->bw._320.rx_tx_mcs11_max_nss = val; + eht_nss->bw._320.rx_tx_mcs13_max_nss = val; + } } int mt7925_init_mlo_caps(struct mt792x_phy *phy) @@ -1543,6 +1565,7 @@ static int mt7925_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&phy->mt76->mac_work); cancel_delayed_work_sync(&dev->pm.ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); mt792x_mutex_acquire(dev); @@ -2208,6 +2231,40 @@ static void mt7925_unassign_vif_chanctx(struct ieee80211_hw *hw, mutex_unlock(&dev->mt76.mutex); } +static void mt7925_sta_pre_rcu_remove(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt76_phy *phy = hw->priv; + struct mt76_dev *dev = phy->dev; + struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; + + mutex_lock(&dev->mutex); + spin_lock_bh(&dev->status_lock); + + if (ieee80211_vif_is_mld(vif)) { + struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + unsigned long valid = mvif->valid_links; + struct mt792x_link_sta *mlink; + unsigned int link_id; + + for_each_set_bit(link_id, &valid, IEEE80211_MLD_MAX_NUM_LINKS) { + mlink = mt792x_sta_to_link(msta, link_id); + if (!mlink || !mlink->wcid.sta) + continue; + if (mlink->wcid.idx < ARRAY_SIZE(dev->wcid)) + rcu_assign_pointer(dev->wcid[mlink->wcid.idx], + NULL); + } + } else { + rcu_assign_pointer(dev->wcid[wcid->idx], NULL); + } + + spin_unlock_bh(&dev->status_lock); + mutex_unlock(&dev->mutex); +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, @@ -2220,7 +2277,7 @@ const struct ieee80211_ops mt7925_ops = { .start_ap = mt7925_start_ap, .stop_ap = mt7925_stop_ap, .sta_state = mt76_sta_state, - .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, + .sta_pre_rcu_remove = mt7925_sta_pre_rcu_remove, .set_key = mt7925_set_key, .sta_set_decap_offload = mt7925_sta_set_decap_offload, #if IS_ENABLED(CONFIG_IPV6) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index 7f1aa94c79866..a64d357b1ba68 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1208,6 +1208,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_phy *phy, struct mt792x_bss_conf *mconf req.roc[i].bw_from_ap = CMD_CBW_20MHZ; req.roc[i].center_chan = center_ch; req.roc[i].center_chan_from_ap = center_ch; + req.roc[i].center_chan2 = 0; + req.roc[i].center_chan2_from_ap = 0; /* STR : 0xfe indicates BAND_ALL with enabling DBDC * EMLSR : 0xff indicates (BAND_AUTO) without DBDC @@ -2171,13 +2173,29 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_RLM, sizeof(*req)); req = (struct bss_rlm_tlv *)tlv; - req->control_channel = chandef->chan->hw_value, - req->center_chan = ieee80211_frequency_to_channel(freq1), - req->center_chan2 = ieee80211_frequency_to_channel(freq2), - req->tx_streams = hweight8(phy->antenna_mask), - req->ht_op_info = 4, /* set HT 40M allowed */ - req->rx_streams = hweight8(phy->antenna_mask), - req->band = band; + req->control_channel = chandef->chan->hw_value; + req->center_chan = ieee80211_frequency_to_channel(freq1); + req->center_chan2 = 0; + req->tx_streams = hweight8(phy->antenna_mask); + req->ht_op_info = 4; /* set HT 40M allowed */ + req->rx_streams = hweight8(phy->antenna_mask); + req->center_chan2 = 0; + req->sco = 0; + req->band = 1; + + switch (band) { + case NL80211_BAND_2GHZ: + req->band = 1; + break; + case NL80211_BAND_5GHZ: + req->band = 2; + break; + case NL80211_BAND_6GHZ: + req->band = 3; + break; + default: + break; + } switch (chandef->width) { case NL80211_CHAN_WIDTH_40: @@ -2188,6 +2206,7 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, break; case NL80211_CHAN_WIDTH_80P80: req->bw = CMD_CBW_8080MHZ; + req->center_chan2 = ieee80211_frequency_to_channel(freq2); break; case NL80211_CHAN_WIDTH_160: req->bw = CMD_CBW_160MHZ; @@ -3453,7 +3472,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(msg_len); + tx_power_tlv->len = cpu_to_le16(msg_len - 4); switch (band) { case NL80211_BAND_2GHZ: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index a927198a299eb..76999ad0ed638 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -37,6 +37,7 @@ static void mt7925e_unregister_device(struct mt792x_dev *dev) mt76_for_each_q_rx(&dev->mt76, i) napi_disable(&dev->mt76.napi[i]); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); cancel_work_sync(&dev->reset_work); @@ -449,6 +450,7 @@ static int mt7925_pci_suspend(struct device *device) pm->suspended = true; flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); mt7925_roc_abort_sync(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c index 487d1e853e101..0e7520270ade6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c @@ -250,6 +250,7 @@ static int mt7925u_suspend(struct usb_interface *intf, pm_message_t state) pm->suspended = true; flush_work(&dev->reset_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, true); if (err) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 096a4f48b2898..eca6bfe45007a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -1146,13 +1146,13 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) spin_unlock_bh(&mdev->sta_poll_lock); continue; } else if (info & MT_TXFREE_INFO_HEADER) { - u32 tx_retries = 0, tx_failed = 0; + u32 tx_retries = 0, tx_failed = 0, count; if (!wcid) continue; - tx_retries = - FIELD_GET(MT_TXFREE_INFO_COUNT, info) - 1; + count = FIELD_GET(MT_TXFREE_INFO_COUNT, info); + tx_retries = count ? count - 1 : 0; tx_failed = tx_retries + !!FIELD_GET(MT_TXFREE_INFO_STAT, info); diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 516e48b6b8932..2a35b171a5469 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -3393,7 +3393,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, h2c->rfe_type = efuse->rfe_type; h2c->rssi_main = U8_MAX; - memset(h2c->link_id, 0xfe, RTW89_BB_PS_LINK_BUF_MAX); + memset(h2c->link_id, RTW89_BB_PS_LINK_ID_SKIP, RTW89_BB_PS_LINK_BUF_MAX); rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) { u8 phy_idx = rtwvif_link->phy_idx; @@ -3401,7 +3401,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, bb = rtw89_get_bb_ctx(rtwdev, phy_idx); chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); - h2c->link_id[phy_idx] = phy_idx; + h2c->link_id[phy_idx] = link_id; h2c->central_ch[phy_idx] = chan->channel; h2c->pri_ch[phy_idx] = chan->primary_channel; h2c->band[phy_idx] = chan->band_type; diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 62cb42e8ec234..cfff8c99d6964 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -2031,6 +2031,8 @@ enum rtw89_bb_link_rx_gain_table_type { RTW89_BB_PS_LINK_RX_GAIN_TAB_MAX, }; +#define RTW89_BB_PS_LINK_ID_SKIP 0xfe + enum rtw89_bb_ps_link_buf_id { RTW89_BB_PS_LINK_BUF_0 = 0x00, RTW89_BB_PS_LINK_BUF_1 = 0x01, diff --git a/drivers/platform/x86/intel/Makefile b/drivers/platform/x86/intel/Makefile index 74db065c82d61..78acb414e1540 100644 --- a/drivers/platform/x86/intel/Makefile +++ b/drivers/platform/x86/intel/Makefile @@ -17,50 +17,40 @@ obj-$(CONFIG_INTEL_UNCORE_FREQ_CONTROL) += uncore-frequency/ # Intel input drivers -intel-hid-y := hid.o -obj-$(CONFIG_INTEL_HID_EVENT) += intel-hid.o -intel-vbtn-y := vbtn.o -obj-$(CONFIG_INTEL_VBTN) += intel-vbtn.o +intel-target-$(CONFIG_INTEL_HID_EVENT) += hid.o +intel-target-$(CONFIG_INTEL_VBTN) += vbtn.o # Intel miscellaneous drivers -obj-$(CONFIG_INTEL_ISHTP_ECLITE) += ishtp_eclite.o -intel_int0002_vgpio-y := int0002_vgpio.o -obj-$(CONFIG_INTEL_INT0002_VGPIO) += intel_int0002_vgpio.o -intel_oaktrail-y := oaktrail.o -obj-$(CONFIG_INTEL_OAKTRAIL) += intel_oaktrail.o -intel_sdsi-y := sdsi.o -obj-$(CONFIG_INTEL_SDSI) += intel_sdsi.o -intel_vsec-y := vsec.o -obj-$(CONFIG_INTEL_VSEC) += intel_vsec.o +intel-target-$(CONFIG_INTEL_INT0002_VGPIO) += int0002_vgpio.o +intel-target-$(CONFIG_INTEL_ISHTP_ECLITE) += ishtp_eclite.o +intel-target-$(CONFIG_INTEL_OAKTRAIL) += oaktrail.o +intel-target-$(CONFIG_INTEL_SDSI) += sdsi.o +intel-target-$(CONFIG_INTEL_VSEC) += vsec.o # Intel PMIC / PMC / P-Unit drivers -intel_bxtwc_tmu-y := bxtwc_tmu.o -obj-$(CONFIG_INTEL_BXTWC_PMIC_TMU) += intel_bxtwc_tmu.o -intel_crystal_cove_charger-y := crystal_cove_charger.o -obj-$(CONFIG_X86_ANDROID_TABLETS) += intel_crystal_cove_charger.o -intel_bytcrc_pwrsrc-y := bytcrc_pwrsrc.o -obj-$(CONFIG_INTEL_BYTCRC_PWRSRC) += intel_bytcrc_pwrsrc.o -intel_chtdc_ti_pwrbtn-y := chtdc_ti_pwrbtn.o -obj-$(CONFIG_INTEL_CHTDC_TI_PWRBTN) += intel_chtdc_ti_pwrbtn.o -intel_chtwc_int33fe-y := chtwc_int33fe.o -obj-$(CONFIG_INTEL_CHTWC_INT33FE) += intel_chtwc_int33fe.o -intel_mrfld_pwrbtn-y := mrfld_pwrbtn.o -obj-$(CONFIG_INTEL_MRFLD_PWRBTN) += intel_mrfld_pwrbtn.o -intel_punit_ipc-y := punit_ipc.o -obj-$(CONFIG_INTEL_PUNIT_IPC) += intel_punit_ipc.o +intel-target-$(CONFIG_INTEL_BYTCRC_PWRSRC) += bytcrc_pwrsrc.o +intel-target-$(CONFIG_INTEL_BXTWC_PMIC_TMU) += bxtwc_tmu.o +intel-target-$(CONFIG_INTEL_CHTDC_TI_PWRBTN) += chtdc_ti_pwrbtn.o +intel-target-$(CONFIG_INTEL_CHTWC_INT33FE) += chtwc_int33fe.o +intel-target-$(CONFIG_X86_ANDROID_TABLETS) += crystal_cove_charger.o +intel-target-$(CONFIG_INTEL_MRFLD_PWRBTN) += mrfld_pwrbtn.o +intel-target-$(CONFIG_INTEL_PUNIT_IPC) += punit_ipc.o # TPMI drivers -intel_vsec_tpmi-y := tpmi.o -obj-$(CONFIG_INTEL_TPMI) += intel_vsec_tpmi.o -obj-$(CONFIG_INTEL_PLR_TPMI) += intel_plr_tpmi.o - -intel_tpmi_power_domains-y := tpmi_power_domains.o -obj-$(CONFIG_INTEL_TPMI_POWER_DOMAINS) += intel_tpmi_power_domains.o +intel-target-$(CONFIG_INTEL_PLR_TPMI) += plr_tpmi.o +intel-target-$(CONFIG_INTEL_TPMI_POWER_DOMAINS) += tpmi_power_domains.o +intel-target-$(CONFIG_INTEL_TPMI) += vsec_tpmi.o # Intel Uncore drivers -intel-rst-y := rst.o -obj-$(CONFIG_INTEL_RST) += intel-rst.o -intel-smartconnect-y := smartconnect.o -obj-$(CONFIG_INTEL_SMARTCONNECT) += intel-smartconnect.o -intel_turbo_max_3-y := turbo_max_3.o -obj-$(CONFIG_INTEL_TURBO_MAX_3) += intel_turbo_max_3.o +intel-target-$(CONFIG_INTEL_RST) += rst.o +intel-target-$(CONFIG_INTEL_SMARTCONNECT) += smartconnect.o +intel-target-$(CONFIG_INTEL_TURBO_MAX_3) += turbo_max_3.o + +# Add 'intel' prefix to each module listed in intel-target-* +define INTEL_OBJ_TARGET +intel-$(1)-y := $(1).o +obj-$(2) += intel-$(1).o +endef + +$(foreach target, $(basename $(intel-target-y)), $(eval $(call INTEL_OBJ_TARGET,$(target),y))) +$(foreach target, $(basename $(intel-target-m)), $(eval $(call INTEL_OBJ_TARGET,$(target),m))) diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 346fbad34e24a..faf501466c91d 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -7,11 +7,13 @@ */ #include +#include #include #include #include #include #include +#include #include #include #include "../dual_accel_detect.h" @@ -210,6 +212,7 @@ static const struct dmi_system_id dmi_auto_add_switch[] = { }; struct intel_hid_priv { + struct mutex mutex; /* Avoid notify_handler() racing with itself */ struct input_dev *input_dev; struct input_dev *array; struct input_dev *switches; @@ -538,6 +541,8 @@ static void notify_handler(acpi_handle handle, u32 event, void *context) unsigned long long ev_index; int err; + guard(mutex)(&priv->mutex); + /* * Some convertible have unreliable VGBS return which could cause incorrect * SW_TABLET_MODE report, in these cases we enable support when receiving @@ -689,6 +694,10 @@ static int intel_hid_probe(struct platform_device *device) return -ENOMEM; dev_set_drvdata(&device->dev, priv); + err = devm_mutex_init(&device->dev, &priv->mutex); + if (err) + return err; + /* See dual_accel_detect.h for more info on the dual_accel check. */ if (enable_sw_tablet_mode == TABLET_SW_AUTO) { if (dmi_check_system(dmi_vgbs_allow_list)) diff --git a/drivers/platform/x86/intel/intel_plr_tpmi.c b/drivers/platform/x86/intel/plr_tpmi.c similarity index 99% rename from drivers/platform/x86/intel/intel_plr_tpmi.c rename to drivers/platform/x86/intel/plr_tpmi.c index 69ace6a629bc7..ffb2f7ffc7b51 100644 --- a/drivers/platform/x86/intel/intel_plr_tpmi.c +++ b/drivers/platform/x86/intel/plr_tpmi.c @@ -315,7 +315,7 @@ static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxilia snprintf(name, sizeof(name), "domain%d", i); dentry = debugfs_create_dir(name, plr->dbgfs_dir); - debugfs_create_file("status", 0444, dentry, &plr->die_info[i], + debugfs_create_file("status", 0644, dentry, &plr->die_info[i], &plr_status_fops); } diff --git a/drivers/platform/x86/intel/tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c similarity index 99% rename from drivers/platform/x86/intel/tpmi.c rename to drivers/platform/x86/intel/vsec_tpmi.c index a3430abeb9869..838c3587873b2 100644 --- a/drivers/platform/x86/intel/tpmi.c +++ b/drivers/platform/x86/intel/vsec_tpmi.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * intel-tpmi : Driver to enumerate TPMI features and create devices + * Driver to enumerate TPMI features and create devices * * Copyright (c) 2023, Intel Corporation. * All Rights Reserved. @@ -809,10 +809,6 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) auxiliary_set_drvdata(auxdev, tpmi_info); - ret = tpmi_create_devices(tpmi_info); - if (ret) - return ret; - /* * Allow debugfs when security policy allows. Everything this debugfs * interface provides, can also be done via /dev/mem access. If @@ -822,6 +818,12 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) if (!security_locked_down(LOCKDOWN_DEV_MEM) && capable(CAP_SYS_RAWIO)) tpmi_dbgfs_register(tpmi_info); + ret = tpmi_create_devices(tpmi_info); + if (ret) { + debugfs_remove_recursive(tpmi_info->dbgfs_dir); + return ret; + } + return 0; } diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 1618138c5cac1..1fae66a609298 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -313,7 +313,10 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, goto err; } - rapl_package_add_pmu(trp->rp); + ret = rapl_package_add_pmu(trp->rp); + if (ret) + dev_info(&auxdev->dev, "Failed to add RAPL PMU for Package%d, %d\n", + info->package_id, ret); auxiliary_set_drvdata(auxdev, trp); diff --git a/drivers/vfio/pci/qat/main.c b/drivers/vfio/pci/qat/main.c index 1e3563fe7cab2..1841ed3c93db1 100644 --- a/drivers/vfio/pci/qat/main.c +++ b/drivers/vfio/pci/qat/main.c @@ -304,14 +304,18 @@ static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf, return -ESPIPE; offs = &filp->f_pos; - if (*offs < 0 || - check_add_overflow(len, *offs, &end)) - return -EOVERFLOW; + mutex_lock(&migf->lock); - if (end > mig_dev->state_size) - return -ENOMEM; + if (*offs < 0 || check_add_overflow(len, *offs, &end)) { + done = -EOVERFLOW; + goto out_unlock; + } + + if (end > mig_dev->state_size) { + done = -ENOMEM; + goto out_unlock; + } - mutex_lock(&migf->lock); if (migf->disabled) { done = -ENODEV; goto out_unlock; diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index 84c29ddd86717..7ac12a9d44fc7 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -595,6 +595,15 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, if (bprm->have_execfd) nitems++; +#ifdef ELF_HWCAP2 + nitems++; +#endif +#ifdef ELF_HWCAP3 + nitems++; +#endif +#ifdef ELF_HWCAP4 + nitems++; +#endif csp = sp; sp -= nitems * 2 * sizeof(unsigned long); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 01ba7ff218b7e..fdfaef8d3c339 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -867,8 +867,8 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %8u\n", - !!thp_vma_allowable_orders(vma, vma->vm_flags, true, false, - true, THP_ORDERS_ALL)); + !!thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_SMAPS | TVA_ENFORCE_SYSFS, THP_ORDERS_ALL)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); @@ -2297,7 +2297,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, if (~categories & PAGE_IS_WRITTEN) goto out_unlock; - if (end != start + HPAGE_SIZE) { + if (end != start + huge_page_size(hstate_vma(vma))) { /* Partial HugeTLB page WP isn't possible. */ pagemap_scan_backout_range(p, start, end); p->arg.walk_end = start; @@ -2314,8 +2314,62 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, return ret; } + +/* + * Write-protect the unpopulated hugetlb entries covering [addr, end) by + * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() + * does for populated entries. + * + * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the + * loop normally runs a single iteration; it is written to cover the full range + * in case the walker ever coalesces adjacent holes. + * + * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- + * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table + * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the + * same VMA, so the scanning thread would deadlock against itself. PMD sharing + * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so + * the vma lock guards nothing that matters for these entries anyway. + */ +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct hstate *h = hstate_vma(vma); + unsigned long psize = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pte_t *ptep; + pte_t pte; + + for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { + ptep = huge_pte_alloc(mm, vma, addr, psize); + if (!ptep) + return -ENOMEM; + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(ptep); + make_uffd_wp_huge_pte(vma, addr, ptep, pte); + /* + * A none entry has no cached translation, so installing the + * marker needs no TLB flush. Flush only if a fault populated + * the entry between huge_pte_alloc() and the page table lock. + */ + if (!huge_pte_none(pte)) + flush_hugetlb_tlb_range(vma, addr, addr + psize); + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + return 0; +} #else #define pagemap_scan_hugetlb_entry NULL +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + return 0; +} #endif static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, @@ -2335,7 +2389,10 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, if (~p->arg.flags & PM_SCAN_WP_MATCHING) return ret; - err = uffd_wp_range(vma, addr, end - addr, true); + if (is_vm_hugetlb_page(vma)) + err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); + else + err = uffd_wp_range(vma, addr, end - addr, true); if (err < 0) ret = err; diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h index 407f7005e6d60..8bcb9b7262628 100644 --- a/include/linux/auxvec.h +++ b/include/linux/auxvec.h @@ -4,6 +4,6 @@ #include -#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */ +#define AT_VECTOR_SIZE_BASE 24 /* NEW_AUX_ENT entries in auxiliary table */ /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */ #endif /* _LINUX_AUXVEC_H */ diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 09179237dac4b..88ba8780f6e4d 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -84,8 +84,12 @@ extern struct kobj_attribute shmem_enabled_attr; */ #define THP_ORDERS_ALL (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE) -#define thp_vma_allowable_order(vma, vm_flags, smaps, in_pf, enforce_sysfs, order) \ - (!!thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, enforce_sysfs, BIT(order))) +#define TVA_SMAPS (1 << 0) /* Will be used for procfs */ +#define TVA_IN_PF (1 << 1) /* Page fault handler */ +#define TVA_ENFORCE_SYSFS (1 << 2) /* Obey sysfs configuration */ + +#define thp_vma_allowable_order(vma, vm_flags, tva_flags, order) \ + (!!thp_vma_allowable_orders(vma, vm_flags, tva_flags, BIT(order))) #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define HPAGE_PMD_SHIFT PMD_SHIFT @@ -114,18 +118,6 @@ static inline bool hugepage_global_always(void) (1<vm_flags - * @smaps: whether answer will be used for smaps file - * @in_pf: whether answer will be used by page fault handler - * @enforce_sysfs: whether sysfs config should be taken into account + * @tva_flags: Which TVA flags to honour * @orders: bitfield of all orders to consider * * Calculates the intersection of the requested hugepage orders and the allowed @@ -234,12 +224,12 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { /* Optimization to check if required orders are enabled early. */ - if (enforce_sysfs && vma_is_anonymous(vma)) { + if ((tva_flags & TVA_ENFORCE_SYSFS) && vma_is_anonymous(vma)) { unsigned long mask = READ_ONCE(huge_anon_orders_always); if (vm_flags & VM_HUGEPAGE) @@ -253,8 +243,7 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, return 0; } - return __thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, - enforce_sysfs, orders); + return __thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders); } #define transparent_hugepage_use_zero_page() \ @@ -422,8 +411,8 @@ static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, } static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { return 0; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 2c463def87550..231d25e5a210e 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -260,11 +260,17 @@ union kvm_mmu_notifier_arg { unsigned long attributes; }; +enum kvm_gfn_range_filter { + KVM_FILTER_SHARED = BIT(0), + KVM_FILTER_PRIVATE = BIT(1), +}; + struct kvm_gfn_range { struct kvm_memory_slot *slot; gfn_t start; gfn_t end; union kvm_mmu_notifier_arg arg; + enum kvm_gfn_range_filter attr_filter; bool may_block; }; bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); @@ -2080,6 +2086,32 @@ static inline int mmu_invalidate_retry_gfn(struct kvm *kvm, return 1; return 0; } + +/* + * This lockless version of the range-based retry check *must* be paired with a + * call to the locked version after acquiring mmu_lock, i.e. this is safe to + * use only as a pre-check to avoid contending mmu_lock. This version *will* + * get false negatives and false positives. + */ +static inline bool mmu_invalidate_retry_gfn_unsafe(struct kvm *kvm, + unsigned long mmu_seq, + gfn_t gfn) +{ + /* + * Use READ_ONCE() to ensure the in-progress flag and sequence counter + * are always read from memory, e.g. so that checking for retry in a + * loop won't result in an infinite retry loop. Don't force loads for + * start+end, as the key to avoiding infinite retry loops is observing + * the 1=>0 transition of in-progress, i.e. getting false negatives + * due to stale start+end values is acceptable. + */ + if (unlikely(READ_ONCE(kvm->mmu_invalidate_in_progress)) && + gfn >= kvm->mmu_invalidate_range_start && + gfn < kvm->mmu_invalidate_range_end) + return true; + + return READ_ONCE(kvm->mmu_invalidate_seq) != mmu_seq; +} #endif #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 6f6a5fc85b42f..9fe0f4f895d73 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1431,10 +1431,9 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { int saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1471,10 +1470,10 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 34635ebc7bc84..2f3447174488a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -82,10 +82,13 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { + bool smaps = tva_flags & TVA_SMAPS; + bool in_pf = tva_flags & TVA_IN_PF; + bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; /* Check the intersection of requested and supported orders. */ orders &= vma_is_anonymous(vma) ? THP_ORDERS_ALL_ANON : THP_ORDERS_ALL_FILE; @@ -509,6 +512,13 @@ static ssize_t thpsize_enabled_store(struct kobject *kobj, } else ret = -EINVAL; + if (ret > 0) { + int err; + + err = start_stop_khugepaged(); + if (err) + ret = err; + } return ret; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 89b7e415d9a11..45381731fd756 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -410,6 +410,32 @@ static inline int hpage_collapse_test_exit(struct mm_struct *mm) return atomic_read(&mm->mm_users) == 0; } +static inline int hpage_collapse_test_exit_or_disable(struct mm_struct *mm) +{ + return hpage_collapse_test_exit(mm) || + test_bit(MMF_DISABLE_THP, &mm->flags); +} + +static bool hugepage_pmd_enabled(void) +{ + /* + * We cover both the anon and the file-backed case here; file-backed + * hugepages, when configured in, are determined by the global control. + * Anon pmd-sized hugepages are determined by the pmd-size control. + */ + if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && + hugepage_global_enabled()) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_always)) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_madvise)) + return true; + if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) && + hugepage_global_enabled()) + return true; + return false; +} + void __khugepaged_enter(struct mm_struct *mm) { struct khugepaged_mm_slot *mm_slot; @@ -446,8 +472,8 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - hugepage_flags_enabled()) { - if (thp_vma_allowable_order(vma, vm_flags, false, false, true, + hugepage_pmd_enabled()) { + if (thp_vma_allowable_order(vma, vm_flags, TVA_ENFORCE_SYSFS, PMD_ORDER)) __khugepaged_enter(vma->vm_mm); } @@ -899,8 +925,9 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, struct collapse_control *cc) { struct vm_area_struct *vma; + unsigned long tva_flags = cc->is_khugepaged ? TVA_ENFORCE_SYSFS : 0; - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) return SCAN_ANY_PROCESS; *vmap = vma = find_vma(mm, address); @@ -909,8 +936,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!thp_vma_suitable_order(vma, address, PMD_ORDER)) return SCAN_ADDRESS_RANGE; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - cc->is_khugepaged, PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, tva_flags, PMD_ORDER)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -1493,8 +1519,7 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, * and map it by a PMD, regardless of sysfs THP settings. As such, let's * analogously elide sysfs THP settings here. */ - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return SCAN_VMA_CHECK; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ @@ -2349,7 +2374,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, goto breakouterloop_mmap_lock; progress++; - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) goto breakouterloop; vma_iter_init(&vmi, mm, khugepaged_scan.address); @@ -2357,12 +2382,12 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, unsigned long hstart, hend; cond_resched(); - if (unlikely(hpage_collapse_test_exit(mm))) { + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) { progress++; break; } - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - true, PMD_ORDER)) { + if (!thp_vma_allowable_order(vma, vma->vm_flags, + TVA_ENFORCE_SYSFS, PMD_ORDER)) { skip: progress++; continue; @@ -2379,7 +2404,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, bool mmap_locked = true; cond_resched(); - if (unlikely(hpage_collapse_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit_or_disable(mm))) goto breakouterloop; VM_BUG_ON(khugepaged_scan.address < hstart || @@ -2397,7 +2422,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, fput(file); if (*result == SCAN_PTE_MAPPED_HUGEPAGE) { mmap_read_lock(mm); - if (hpage_collapse_test_exit(mm)) + if (hpage_collapse_test_exit_or_disable(mm)) goto breakouterloop; *result = collapse_pte_mapped_thp(mm, khugepaged_scan.address, false); @@ -2464,8 +2489,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, static int khugepaged_has_work(void) { - return !list_empty(&khugepaged_scan.mm_head) && - hugepage_flags_enabled(); + return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled(); } static int khugepaged_wait_event(void) @@ -2538,7 +2562,7 @@ static void khugepaged_wait_work(void) return; } - if (hugepage_flags_enabled()) + if (hugepage_pmd_enabled()) wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); } @@ -2569,7 +2593,7 @@ static void set_recommended_min_free_kbytes(void) int nr_zones = 0; unsigned long recommended_min; - if (!hugepage_flags_enabled()) { + if (!hugepage_pmd_enabled()) { calculate_min_free_kbytes(); goto update_wmarks; } @@ -2619,7 +2643,7 @@ int start_stop_khugepaged(void) int err = 0; mutex_lock(&khugepaged_mutex); - if (hugepage_flags_enabled()) { + if (hugepage_pmd_enabled()) { if (!khugepaged_thread) khugepaged_thread = kthread_run(khugepaged, NULL, "khugepaged"); @@ -2645,7 +2669,7 @@ int start_stop_khugepaged(void) void khugepaged_min_free_kbytes_update(void) { mutex_lock(&khugepaged_mutex); - if (hugepage_flags_enabled() && khugepaged_thread) + if (hugepage_pmd_enabled() && khugepaged_thread) set_recommended_min_free_kbytes(); mutex_unlock(&khugepaged_mutex); } @@ -2699,8 +2723,7 @@ int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, *prev = vma; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return -EINVAL; cc = kmalloc(sizeof(*cc), GFP_KERNEL); diff --git a/mm/memory.c b/mm/memory.c index eaf07a4d1e0d5..181164a70e924 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4412,8 +4412,8 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) * for this vma. Then filter out the orders that can't be allocated over * the faulting address and still be fully contained in the vma. */ - orders = thp_vma_allowable_orders(vma, vma->vm_flags, false, true, true, - BIT(PMD_ORDER) - 1); + orders = thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1); orders = thp_vma_suitable_orders(vma, vmf->address, orders); if (!orders) @@ -5492,7 +5492,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, return VM_FAULT_OOM; retry_pud: if (pud_none(*vmf.pud) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PUD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5526,7 +5527,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, goto retry_pud; if (pmd_none(*vmf.pmd) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PMD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) { ret = create_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index eeb944374f892..ee114e082dcbf 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -845,9 +845,9 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* The fastest case is the first. */ icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; } else if (m < icsk->icsk_ack.ato) { - icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; - if (icsk->icsk_ack.ato > icsk->icsk_rto) - icsk->icsk_ack.ato = icsk->icsk_rto; + icsk->icsk_ack.ato = min3((icsk->icsk_ack.ato >> 1) + (u32)m, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else if (m > icsk->icsk_rto) { /* Too long gap. Apparently sender failed to * restart window, so that we send ACKs quickly. diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index be40e424f27cd..4db663ba6c3d6 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -318,7 +318,9 @@ void tcp_delack_timer_handler(struct sock *sk) if (inet_csk_ack_scheduled(sk)) { if (!inet_csk_in_pingpong_mode(sk)) { /* Delayed ACK missed: inflate ATO. */ - icsk->icsk_ack.ato = min_t(u32, icsk->icsk_ack.ato << 1, icsk->icsk_rto); + icsk->icsk_ack.ato = min3((u32)icsk->icsk_ack.ato << 1, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else { /* Delayed ACK missed: leave pingpong mode and * deflate ATO. diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 0cfa211b04b63..46c47bee776fa 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2183,7 +2183,9 @@ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, case IEEE80211_RADIOTAP_ANTENNA: /* this can appear multiple times, keep a bitmap */ - info->control.antennas |= BIT(*iterator.this_arg); + /* control.antennas is only a 2-bit bitmap */ + if (*iterator.this_arg < 2) + info->control.antennas |= BIT(*iterator.this_arg); break; case IEEE80211_RADIOTAP_DATA_RETRIES: diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c index 7ccd95b6ccd15..f765c30bc73d5 100644 --- a/sound/pci/hda/cs35l41_hda.c +++ b/sound/pci/hda/cs35l41_hda.c @@ -1184,8 +1184,19 @@ static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle guid_parse(CS35L41_UUID, &guid); if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) { - ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL); + ret = acpi_evaluate_dsm_typed(handle, &guid, 0, + CS35L41_DSM_GET_MUTE, NULL, + ACPI_TYPE_BUFFER); + + if (!ret) + return -EINVAL; + if (!ret->buffer.length || !ret->buffer.pointer) { + ACPI_FREE(ret); + return -EINVAL; + } + mute = *ret->buffer.pointer; + ACPI_FREE(ret); dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute); } diff --git a/sound/soc/sof/intel/hda-common-ops.c b/sound/soc/sof/intel/hda-common-ops.c index 0620536235f4d..d71bb66b99911 100644 --- a/sound/soc/sof/intel/hda-common-ops.c +++ b/sound/soc/sof/intel/hda-common-ops.c @@ -19,6 +19,7 @@ struct snd_sof_dsp_ops sof_hda_common_ops = { .probe_early = hda_dsp_probe_early, .probe = hda_dsp_probe, .remove = hda_dsp_remove, + .remove_late = hda_dsp_remove_late, /* Register IO uses direct mmio */ diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index ed4180b08469d..22f2dd7303382 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -901,13 +901,21 @@ static int hda_init(struct snd_sof_dev *sdev) /* init i915 and HDMI codecs */ ret = hda_codec_i915_init(sdev); - if (ret < 0) - dev_warn(sdev->dev, "init of i915 and HDMI codec failed\n"); + if (ret < 0 && ret != -ENODEV) { + dev_err_probe(sdev->dev, ret, "init of i915 and HDMI codec failed\n"); + goto out; + } /* get controller capabilities */ ret = hda_dsp_ctrl_get_caps(sdev); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "error: get caps error\n"); + hda_codec_i915_exit(sdev); + } + +out: + if (ret < 0) + iounmap(sof_to_bus(sdev)->remap_addr); return ret; } @@ -1213,6 +1221,7 @@ int hda_dsp_probe_early(struct snd_sof_dev *sdev) return -ENOMEM; sdev->pdata->hw_pdata = hdev; hdev->desc = chip; + ret = hda_init(sdev); err: return ret; @@ -1222,7 +1231,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) { struct pci_dev *pci = to_pci_dev(sdev->dev); struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; - struct hdac_bus *bus; int ret = 0; hdev->dmic_dev = platform_device_register_data(sdev->dev, "dmic-codec", @@ -1246,12 +1254,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) if (sdev->dspless_mode_selected) hdev->no_ipc_position = 1; - /* set up HDA base */ - bus = sof_to_bus(sdev); - ret = hda_init(sdev); - if (ret < 0) - goto hdac_bus_unmap; - if (sdev->dspless_mode_selected) goto skip_dsp_setup; @@ -1360,8 +1362,6 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) iounmap(sdev->bar[HDA_DSP_BAR]); hdac_bus_unmap: platform_device_unregister(hdev->dmic_dev); - iounmap(bus->remap_addr); - hda_codec_i915_exit(sdev); return ret; } @@ -1370,7 +1370,6 @@ void hda_dsp_remove(struct snd_sof_dev *sdev) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; const struct sof_intel_dsp_desc *chip = hda->desc; - struct hdac_bus *bus = sof_to_bus(sdev); struct pci_dev *pci = to_pci_dev(sdev->dev); struct nhlt_acpi_table *nhlt = hda->nhlt; @@ -1420,11 +1419,12 @@ void hda_dsp_remove(struct snd_sof_dev *sdev) if (!sdev->dspless_mode_selected) iounmap(sdev->bar[HDA_DSP_BAR]); +} - iounmap(bus->remap_addr); - +void hda_dsp_remove_late(struct snd_sof_dev *sdev) +{ + iounmap(sof_to_bus(sdev)->remap_addr); sof_hda_bus_exit(sdev); - hda_codec_i915_exit(sdev); } diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 94bef8762965b..97bdabf795b2d 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -583,6 +583,7 @@ bool hda_is_chain_dma_supported(struct snd_sof_dev *sdev, u32 dai_type); int hda_dsp_probe_early(struct snd_sof_dev *sdev); int hda_dsp_probe(struct snd_sof_dev *sdev); void hda_dsp_remove(struct snd_sof_dev *sdev); +void hda_dsp_remove_late(struct snd_sof_dev *sdev); int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask); diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index 80dc26f28ade9..fed6c6a8d257b 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -19,12 +19,14 @@ * struct sof_ipc4_timestamp_info - IPC4 timestamp info * @host_copier: the host copier of the pcm stream * @dai_copier: the dai copier of the pcm stream - * @stream_start_offset: reported by fw in memory window (converted to frames) - * @stream_end_offset: reported by fw in memory window (converted to frames) + * @stream_start_offset: reported by fw in memory window (converted to + * frames at host_copier sampling rate) + * @stream_end_offset: reported by fw in memory window (converted to + * frames at host_copier sampling rate) * @llp_offset: llp offset in memory window - * @boundary: wrap boundary should be used for the LLP frame counter * @delay: Calculated and stored in pointer callback. The stored value is - * returned in the delay callback. + * returned in the delay callback. Expressed in frames at host copier + * sampling rate. */ struct sof_ipc4_timestamp_info { struct sof_ipc4_copier *host_copier; @@ -33,7 +35,6 @@ struct sof_ipc4_timestamp_info { u64 stream_end_offset; u32 llp_offset; - u64 boundary; snd_pcm_sframes_t delay; }; @@ -48,6 +49,16 @@ struct sof_ipc4_pcm_stream_priv { bool chain_dma_allocated; }; +/* + * Modulus to use to compare host and link position counters. The sampling + * rates may be different, so the raw hardware counters will wrap + * around at different times. To calculate differences, use + * DELAY_BOUNDARY as a common modulus. This value must be smaller than + * the wrap-around point of any hardware counter, and larger than any + * valid delay measurement. + */ +#define DELAY_BOUNDARY U32_MAX + static inline struct sof_ipc4_timestamp_info * sof_ipc4_sps_to_time_info(struct snd_sof_pcm_stream *sps) { @@ -409,9 +420,33 @@ static int sof_ipc4_trigger_pipelines(struct snd_soc_component *component, * If use_chain_dma attribute is set we proceed to chained DMA * trigger function that handles the rest for the substream. */ - if (pipeline->use_chain_dma) - return sof_ipc4_chain_dma_trigger(sdev, spcm, substream->stream, - pipeline_list, state, cmd); + if (pipeline->use_chain_dma) { + struct sof_ipc4_timestamp_info *time_info; + + time_info = sof_ipc4_sps_to_time_info(&spcm->stream[substream->stream]); + + ret = sof_ipc4_chain_dma_trigger(sdev, spcm, substream->stream, + pipeline_list, state, cmd); + if (ret || !time_info) + return ret; + + if (state == SOF_IPC4_PIPE_PAUSED) { + /* + * Record the DAI position for delay reporting + * To handle multiple pause/resume/xrun we need to add + * the positions to simulate how the firmware behaves + */ + u64 pos = snd_sof_pcm_get_dai_frame_counter(sdev, component, + substream); + + time_info->stream_end_offset += pos; + } else if (state == SOF_IPC4_PIPE_RESET) { + /* Reset the end offset as the stream is stopped */ + time_info->stream_end_offset = 0; + } + + return 0; + } /* allocate memory for the pipeline data */ trigger_list = kzalloc(struct_size(trigger_list, pipeline_instance_ids, @@ -924,6 +959,35 @@ static int sof_ipc4_pcm_hw_params(struct snd_soc_component *component, return 0; } +static u64 sof_ipc4_frames_dai_to_host(struct sof_ipc4_timestamp_info *time_info, u64 value) +{ + u64 dai_rate, host_rate; + + if (!time_info->dai_copier || !time_info->host_copier) + return value; + + /* + * copiers do not change sampling rate, so we can use the + * out_format independently of stream direction + */ + dai_rate = time_info->dai_copier->data.out_format.sampling_frequency; + host_rate = time_info->host_copier->data.out_format.sampling_frequency; + + if (!dai_rate || !host_rate || dai_rate == host_rate) + return value; + + /* take care not to overflow u64, rates can be up to 768000 */ + if (value > U32_MAX) { + value = div64_u64(value, dai_rate); + value *= host_rate; + } else { + value *= host_rate; + value = div64_u64(value, dai_rate); + } + + return value; +} + static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, struct snd_sof_pcm_stream *sps, @@ -939,8 +1003,30 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, if (!host_copier || !dai_copier) return -EINVAL; - if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_INVALID_NODE_ID) + if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_INVALID_NODE_ID) { return -EINVAL; + } else if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_CHAIN_DMA_NODE_ID) { + /* + * While the firmware does not support time_info reporting for + * streams using ChainDMA, it is granted that ChainDMA can only + * be used on Host+Link pairs where the link position is + * accessible from the host side. + * + * Enable delay calculation in case of ChainDMA via host + * accessible registers. + * + * The ChainDMA prefills the link DMA with a preamble + * of zero samples. Set the stream start offset based + * on size of the preamble (driver provided fifo size + * multiplied by 2.5). We add 1ms of margin as the FW + * will align the buffer size to DMA hardware + * alignment that is not known to host. + */ + int pre_ms = SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS * 5 / 2 + 1; + + time_info->stream_start_offset = pre_ms * substream->runtime->rate / MSEC_PER_SEC; + goto out; + } node_index = SOF_IPC4_NODE_INDEX(host_copier->data.gtw_cfg.node_id); offset = offsetof(struct sof_ipc4_fw_registers, pipeline_regs) + node_index * sizeof(ppl_reg); @@ -958,13 +1044,13 @@ static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev, time_info->stream_end_offset = ppl_reg.stream_end_offset; do_div(time_info->stream_end_offset, dai_sample_size); - /* - * Calculate the wrap boundary need to be used for delay calculation - * The host counter is in bytes, it will wrap earlier than the frames - * based link counter. - */ - time_info->boundary = div64_u64(~((u64)0), - frames_to_bytes(substream->runtime, 1)); + /* convert to host frame time */ + time_info->stream_start_offset = + sof_ipc4_frames_dai_to_host(time_info, time_info->stream_start_offset); + time_info->stream_end_offset = + sof_ipc4_frames_dai_to_host(time_info, time_info->stream_end_offset); + +out: /* Initialize the delay value to 0 (no delay) */ time_info->delay = 0; @@ -1007,6 +1093,8 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, /* For delay calculation we need the host counter */ host_cnt = snd_sof_pcm_get_host_byte_counter(sdev, component, substream); + + /* Store the original value to host_ptr */ host_ptr = host_cnt; /* convert the host_cnt to frames */ @@ -1025,6 +1113,8 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, sof_mailbox_read(sdev, time_info->llp_offset, &llp, sizeof(llp)); dai_cnt = ((u64)llp.reading.llp_u << 32) | llp.reading.llp_l; } + + dai_cnt = sof_ipc4_frames_dai_to_host(time_info, dai_cnt); dai_cnt += time_info->stream_end_offset; /* In two cases dai dma counter is not accurate @@ -1058,8 +1148,9 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, dai_cnt -= time_info->stream_start_offset; } - /* Wrap the dai counter at the boundary where the host counter wraps */ - div64_u64_rem(dai_cnt, time_info->boundary, &dai_cnt); + /* Convert to a common base before comparisons */ + dai_cnt &= DELAY_BOUNDARY; + host_cnt &= DELAY_BOUNDARY; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { head_cnt = host_cnt; @@ -1069,14 +1160,11 @@ static int sof_ipc4_pcm_pointer(struct snd_soc_component *component, tail_cnt = host_cnt; } - if (head_cnt < tail_cnt) { - time_info->delay = time_info->boundary - tail_cnt + head_cnt; - goto out; - } - - time_info->delay = head_cnt - tail_cnt; + if (unlikely(head_cnt < tail_cnt)) + time_info->delay = DELAY_BOUNDARY - tail_cnt + head_cnt; + else + time_info->delay = head_cnt - tail_cnt; -out: /* * Convert the host byte counter to PCM pointer which wraps in buffer * and it is in frames diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 16dc4514fef0b..71442ca58f7fe 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -32,7 +32,6 @@ MODULE_PARM_DESC(ipc4_ignore_cpc, #define SOF_IPC4_GAIN_PARAM_ID 0 #define SOF_IPC4_TPLG_ABI_SIZE 6 -#define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2 static DEFINE_IDA(alh_group_ida); static DEFINE_IDA(pipeline_ida); @@ -1712,10 +1711,10 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget, pipeline->msg.extension |= SOF_IPC4_GLB_EXT_CHAIN_DMA_FIFO_SIZE(fifo_size); /* - * Chain DMA does not support stream timestamping, set node_id to invalid - * to skip the code in sof_ipc4_get_stream_start_offset(). + * Chain DMA does not support stream timestamping, but it + * can use the host side registers for delay calculation. */ - copier_data->gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID; + copier_data->gtw_cfg.node_id = SOF_IPC4_CHAIN_DMA_NODE_ID; return 0; } diff --git a/sound/soc/sof/ipc4-topology.h b/sound/soc/sof/ipc4-topology.h index 42b4559500924..afafb99435245 100644 --- a/sound/soc/sof/ipc4-topology.h +++ b/sound/soc/sof/ipc4-topology.h @@ -57,6 +57,7 @@ #define SOF_IPC4_DMA_DEVICE_MAX_COUNT 16 +#define SOF_IPC4_CHAIN_DMA_NODE_ID 0x7fffffff #define SOF_IPC4_INVALID_NODE_ID 0xffffffff /* FW requires minimum 4ms DMA buffer size */ @@ -248,6 +249,8 @@ struct sof_ipc4_dma_stream_ch_map { #define SOF_IPC4_DMA_METHOD_HDA 1 #define SOF_IPC4_DMA_METHOD_GPDMA 2 /* defined for consistency but not used */ +#define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2 + /** * struct sof_ipc4_dma_config: DMA configuration * @dma_method: HDAudio or GPDMA diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h index 5e96748cd33c1..99e041c4b58c2 100644 --- a/sound/soc/sof/ops.h +++ b/sound/soc/sof/ops.h @@ -57,12 +57,10 @@ static inline void snd_sof_remove(struct snd_sof_dev *sdev) sof_ops(sdev)->remove(sdev); } -static inline int snd_sof_remove_late(struct snd_sof_dev *sdev) +static inline void snd_sof_remove_late(struct snd_sof_dev *sdev) { if (sof_ops(sdev)->remove_late) - return sof_ops(sdev)->remove_late(sdev); - - return 0; + sof_ops(sdev)->remove_late(sdev); } static inline int snd_sof_shutdown(struct snd_sof_dev *sdev) diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index f6ce275b7fe38..4378dd2461570 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -177,7 +177,7 @@ struct snd_sof_dsp_ops { int (*probe_early)(struct snd_sof_dev *sof_dev); /* optional */ int (*probe)(struct snd_sof_dev *sof_dev); /* mandatory */ void (*remove)(struct snd_sof_dev *sof_dev); /* optional */ - int (*remove_late)(struct snd_sof_dev *sof_dev); /* optional */ + void (*remove_late)(struct snd_sof_dev *sof_dev); /* optional */ int (*shutdown)(struct snd_sof_dev *sof_dev); /* optional */ /* DSP core boot / reset */ diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index d7f504e7f917f..2f3379b7e67dd 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -136,20 +136,11 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) size); } - for (flag = 1; flag; flag <<= 1) { - uint64_t bit; - + for (flag = BIT(0); flag; flag <<= 1) { fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", flag); - - for_each_set_bit(bit, &valid_flags, 64) { - fd = __vm_create_guest_memfd(vm, page_size, flag | BIT_ULL(bit)); - TEST_ASSERT(fd == -1 && errno == EINVAL, - "guest_memfd() with flags '0x%llx' should fail with EINVAL", - flag | BIT_ULL(bit)); - } } } diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 8b0690914994f..02843dc7d2dad 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -673,7 +673,7 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret)); close(region->fd); } - if (region->region.guest_memfd >= 0) + if ((int)region->region.guest_memfd >= 0) close(region->region.guest_memfd); free(region); diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index a78394faee7c6..197eb1f7e9562 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -326,6 +326,57 @@ static void test_zero_memory_regions(void) } #endif /* __x86_64__ */ +static void test_invalid_memory_region_flags(void) +{ + uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES; + const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD; + struct kvm_vm *vm; + int r, i; + +#ifdef __x86_64__ + supported_flags |= KVM_MEM_READONLY; + + if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)) + vm = vm_create_barebones_protected_vm(); + else +#endif + vm = vm_create_barebones(); + + if (kvm_check_cap(KVM_CAP_MEMORY_ATTRIBUTES) & KVM_MEMORY_ATTRIBUTE_PRIVATE) + supported_flags |= KVM_MEM_GUEST_MEMFD; + + for (i = 0; i < 32; i++) { + if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i))) + continue; + + r = __vm_set_user_memory_region(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL); + + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i)); + + if (supported_flags & BIT(i)) + continue; + + r = __vm_set_user_memory_region2(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL, 0, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i)); + } + + if (supported_flags & KVM_MEM_GUEST_MEMFD) { + int guest_memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0); + + r = __vm_set_user_memory_region2(vm, 0, + KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, + 0, MEM_REGION_SIZE, NULL, guest_memfd, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); + + close(guest_memfd); + } +} + /* * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any * tentative to add further slots should fail. @@ -442,7 +493,7 @@ static void test_add_overlapping_private_memory_regions(void) vm = vm_create_barebones_protected_vm(); - memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 4, 0); + memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 5, 0); vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA, MEM_REGION_SIZE * 2, 0, memfd, 0); @@ -458,7 +509,30 @@ static void test_add_overlapping_private_memory_regions(void) vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA, 0, NULL, -1, 0); - /* Overlap the front half of the other slot. */ + /* + * Verify that overlap in the guest_memfd bindings (i.e. in guest_memfd + * file offsets), but _not_ in the GPA space, fails with -EEXIST. + */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, + MEM_REGION_SIZE * 2, + 0, memfd, MEM_REGION_SIZE); + TEST_ASSERT(r == -1 && errno == EEXIST, + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + /* And now the back half of the other slot's guest_memfd binding. */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, + MEM_REGION_SIZE * 2, + 0, memfd, MEM_REGION_SIZE * 3); + TEST_ASSERT(r == -1 && errno == EEXIST, + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + /* + * Repeat the overlap tests, but this time with overlap in the memslots + * GPA space. Regardless of where there is overlap, KVM should return + * -EEXIST. + */ r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, MEM_REGION_GPA * 2 - MEM_REGION_SIZE, MEM_REGION_SIZE * 2, @@ -491,6 +565,8 @@ int main(int argc, char *argv[]) test_zero_memory_regions(); #endif + test_invalid_memory_region_flags(); + test_add_max_memory_regions(); #ifdef __x86_64__ diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 6d9b7ec9a1ba0..fb8599b5f1bbb 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -1360,7 +1360,7 @@ void *thread_proc(void *mem) ksft_exit_fail_msg("pthread_barrier_wait\n"); for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); ret = pthread_barrier_wait(&end_barrier); if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) @@ -1394,15 +1394,15 @@ static void transact_test(int page_size) if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) ksft_exit_fail_msg("pthread_barrier_init\n"); - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, + mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); + wp_init(mem, page_size * nthreads * pages_per_thread); + wp_addr_range(mem, page_size * nthreads * pages_per_thread); - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); + memset(mem, 0, page_size * nthreads * pages_per_thread); count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); ksft_test_result(count > 0, "%s count %u\n", __func__, count); @@ -1411,7 +1411,7 @@ static void transact_test(int page_size) finish = 0; for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); + pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); extra_pages = 0; for (i = 0; i < iter_count; ++i) { diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index e846282eeb32e..9bcbbcbf55545 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -64,6 +64,8 @@ static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, .end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff, .slot = slot, .may_block = true, + /* guest memfd is relevant to only private mappings. */ + .attr_filter = KVM_FILTER_PRIVATE, }; if (!found_memslot) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2f3775b290c72..9c42cdf677c82 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -641,6 +641,11 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm, */ gfn_range.arg = range->arg; gfn_range.may_block = range->may_block; + /* + * HVA-based notifications aren't relevant to private + * mappings as they don't have a userspace mapping. + */ + gfn_range.attr_filter = KVM_FILTER_SHARED; /* * {gfn(page) | page intersects with [hva_start, hva_end)} = @@ -2504,6 +2509,14 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, gfn_range.arg = range->arg; gfn_range.may_block = range->may_block; + /* + * If/when KVM supports more attributes beyond private .vs shared, this + * _could_ set KVM_FILTER_{SHARED,PRIVATE} appropriately if the entire target + * range already has the desired private vs. shared state (it's unclear + * if that is a net win). For now, KVM reaches this point if and only + * if the private flag is being toggled, i.e. all mappings are in play. + */ + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { slots = __kvm_memslots(kvm, i); @@ -2560,6 +2573,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, + .arg.attributes = attributes, .handler = kvm_pre_set_memory_attributes, .on_lock = kvm_mmu_invalidate_begin, .flush_on_ret = true,