/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2026, Netflix, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Support for the AMD IBS */ #include #include #include #include #include #include #include #include #include #include #include #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_IBS #include #include #include #include #include #define IBS_STOP_ITER 50 /* Stopping iterations */ #define IBS_NMI_CREDIT_MAX 2 /* Max owed IBS NMIs per CPU */ /* AMD IBS PMCs */ struct ibs_descr { struct pmc_descr pm_descr; /* "base class" */ }; /* * Globals */ static uint64_t ibs_features; static uint64_t ibs_fetch_allowed_mask; static uint64_t ibs_op_allowed_mask; static uint64_t ibs_fetch_ctl2_allowed_mask; static uint64_t ibs_op_ctl2_allowed_mask; static bool ibs_fetch_ctl2_supported; static bool ibs_op_ctl2_supported; static uint64_t ibs_fetch_extra_mask; static uint64_t ibs_fetch_ctl2_extra_mask; static uint64_t ibs_op_extra_mask; static uint64_t ibs_op_ctl2_extra_mask; SYSCTL_DECL(_kern_hwpmc); SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_extra_mask, CTLFLAG_RDTUN, &ibs_fetch_extra_mask, 0, "Extra allowed bits in the IBS fetch control MSR (override; default 0)"); SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_ctl2_extra_mask, CTLFLAG_RDTUN, &ibs_fetch_ctl2_extra_mask, 0, "Extra allowed bits in the IBS fetch control 2 MSR (override; default 0)"); SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_extra_mask, CTLFLAG_RDTUN, &ibs_op_extra_mask, 0, "Extra allowed bits in the IBS op control MSR (override; default 0)"); SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_ctl2_extra_mask, CTLFLAG_RDTUN, &ibs_op_ctl2_extra_mask, 0, "Extra allowed bits in the IBS op control 2 MSR (override; default 0)"); /* * Per-processor information */ #define IBS_CPU_RUNNING 1 #define IBS_CPU_STOPPING 2 #define IBS_CPU_STOPPED 3 struct ibs_cpu { int pc_status; int pc_nmi_credit; /* owed NMIs already serviced */ struct pmc_hw pc_ibspmcs[IBS_NPMCS]; }; static struct ibs_cpu **ibs_pcpu; static void ibs_init_policy(void) { ibs_fetch_allowed_mask = IBS_FETCH_ALLOWED_MASK_BASE; ibs_fetch_ctl2_allowed_mask = 0; ibs_op_allowed_mask = IBS_OP_CTL_MAXCNTBASEMASK; ibs_op_ctl2_allowed_mask = 0; if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0) ibs_fetch_allowed_mask |= IBS_FETCH_CTL_L3MISSONLY; if ((ibs_features & CPUID_IBSID_OPCNT) != 0) ibs_op_allowed_mask |= IBS_OP_CTL_COUNTERCONTROL; if ((ibs_features & CPUID_IBSID_OPCNTEXT) != 0) ibs_op_allowed_mask |= IBS_OP_CTL_MAXCNTEXTMASK; if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0) ibs_op_allowed_mask |= IBS_OP_CTL_L3MISSONLY; if ((ibs_features & CPUID_IBSID_FETCHLATFILTERING) != 0) ibs_fetch_ctl2_allowed_mask |= IBS_FETCH_CTL2_LATFILTERMASK; if ((ibs_features & CPUID_IBSID_STRMSTANDRMTSOCKET) != 0) ibs_op_ctl2_allowed_mask |= IBS_OP_CTL2_STRMSTFILTER; if ((ibs_features & CPUID_IBSID_IBSDIS) != 0) { ibs_fetch_ctl2_supported = true; ibs_op_ctl2_supported = true; } /* * ctl2 MSRs only exist on Zen 6; writing them on older silicon * would #GP. */ if (!ibs_fetch_ctl2_supported) ibs_fetch_ctl2_supported = (ibs_fetch_ctl2_allowed_mask != 0); if (!ibs_op_ctl2_supported) ibs_op_ctl2_supported = (ibs_op_ctl2_allowed_mask != 0); } static int ibs_validate_fetch_config(uint64_t config) { uint64_t allowed_mask; allowed_mask = ibs_fetch_allowed_mask | ibs_fetch_extra_mask; if ((config & ~allowed_mask) != 0) { return (EXTERROR(EINVAL, "IBS fetch ctl config 0x%jx has bits outside allowed mask" " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); } return (0); } static int ibs_validate_op_config(uint64_t config) { uint64_t allowed_mask; allowed_mask = ibs_op_allowed_mask; if ((config & IBS_OP_CTL_LATFLTEN) != 0) { if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0) return (EXTERROR(EINVAL, "IBS op load-latency filter is not supported")); /* * Zen 6 decouples L3MISSONLY from load-latency filtering * (AMD pub 69205); enforce the pairing only on older parts. */ if ((ibs_features & CPUID_IBSID_IBSDIS) == 0 && (config & IBS_OP_CTL_L3MISSONLY) == 0) return (EXTERROR(EINVAL, "IBS op load-latency filter requires l3miss")); allowed_mask |= IBS_OP_CTL_LDLATMASK | IBS_OP_CTL_L3MISSONLY; } allowed_mask |= ibs_op_extra_mask; if ((config & ~allowed_mask) != 0) { return (EXTERROR(EINVAL, "IBS op ctl config 0x%jx has bits outside allowed mask" " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); } return (0); } static int ibs_validate_fetch_ctl2_config(uint64_t config) { uint64_t allowed_mask; if (config == 0) return (0); if (!ibs_fetch_ctl2_supported) return (EXTERROR(EINVAL, "IBS fetch ctl2 features are not supported on this CPU")); allowed_mask = ibs_fetch_ctl2_allowed_mask | ibs_fetch_ctl2_extra_mask; if ((config & ~allowed_mask) != 0) { return (EXTERROR(EINVAL, "IBS fetch ctl2 config 0x%jx has bits outside allowed" " mask 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); } return (0); } static int ibs_validate_op_ctl2_config(uint64_t config) { uint64_t allowed_mask; if (config == 0) return (0); if (!ibs_op_ctl2_supported) return (EXTERROR(EINVAL, "IBS op ctl2 features are not supported on this CPU")); allowed_mask = ibs_op_ctl2_allowed_mask | ibs_op_ctl2_extra_mask; if ((config & ~allowed_mask) != 0) { return (EXTERROR(EINVAL, "IBS op ctl2 config 0x%jx has bits outside allowed mask" " 0x%jx", (uintmax_t)config, (uintmax_t)allowed_mask)); } return (0); } static int ibs_validate_pmc_config(int ri, uint64_t config, uint64_t config2) { int error; switch (ri) { case IBS_PMC_FETCH: error = ibs_validate_fetch_config(config); if (error != 0) return (error); return (ibs_validate_fetch_ctl2_config(config2)); case IBS_PMC_OP: error = ibs_validate_op_config(config); if (error != 0) return (error); return (ibs_validate_op_ctl2_config(config2)); default: return (EXTERROR(EINVAL, "Unsupported IBS PMC type %ju", (uintmax_t)ri)); } } /* * Read a PMC value from the MSR. */ static int ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); KASSERT(ibs_pcpu[cpu], ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu)); /* read the IBS count */ switch (ri) { case IBS_PMC_FETCH: *v = IBS_FETCH_CTL_TO_COUNT(rdmsr(IBS_FETCH_CTL)); break; case IBS_PMC_OP: *v = IBS_OP_CTL_TO_COUNT(rdmsr(IBS_OP_CTL)); break; } PMCDBG2(MDP, REA, 2, "ibs-read id=%d -> %jd", ri, *v); return (0); } /* * Write a PMC MSR. */ static int ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) { pmc_value_t m; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); /* write the IBS count */ switch (ri) { case IBS_PMC_FETCH: m = rdmsr(IBS_FETCH_CTL) & ~IBS_FETCH_CTL_CURCNTMASK; /* Setting a count greater than interval is undefined. */ if (IBS_FETCH_CTL_TO_INTERVAL(m) > v) m |= IBS_FETCH_COUNT_TO_CTL(v); wrmsr(IBS_FETCH_CTL, m); break; case IBS_PMC_OP: m = rdmsr(IBS_OP_CTL) & ~IBS_OP_CTL_CURCNTMASK; /* Setting a count greater than interval is undefined */ if (IBS_OP_CTL_TO_INTERVAL(m) > v) m |= IBS_OP_COUNT_TO_CTL(v); wrmsr(IBS_OP_CTL, m); break; } PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v); return (0); } /* * Configure hardware PMC according to the configuration recorded in 'pm'. */ static int ibs_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[ibs,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } /* * Retrieve a configured PMC pointer from hardware state. */ static int ibs_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = ibs_pcpu[cpu]->pc_ibspmcs[ri].phw_pmc; return (0); } /* * Check if a given PMC allocation is feasible. */ static int ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint64_t caps, config, config2; int error; KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row index %d", __LINE__, ri)); /* Row discriminators; the allocation loop probes every row. */ if (a->pm_class != PMC_CLASS_IBS) return (EINVAL); if (a->pm_md.pm_ibs.ibs_type != ri) return (EINVAL); caps = pm->pm_caps; PMCDBG2(MDP, ALL, 1, "ibs-allocate ri=%d caps=0x%x", ri, caps); if ((ibs_features & CPUID_IBSID_ADDRBIT63FILTERING) != 0) { if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) return (EXTERROR(EINVAL, "IBS requires at least USER or SYSTEM capability")); } else { if ((caps & PMC_CAP_SYSTEM) == 0) return (EXTERROR(EINVAL, "IBS requires SYSTEM capability")); if ((caps & PMC_CAP_USER) != 0) return (EXTERROR(EINVAL, "IBS USER filtering requires Zen 6 addr63 support")); } if (!PMC_IS_SAMPLING_MODE(a->pm_mode)) return (EXTERROR(EINVAL, "IBS only supports sampling mode")); config = a->pm_md.pm_ibs.ibs_ctl; config2 = a->pm_md.pm_ibs.ibs_ctl2; error = ibs_validate_pmc_config(ri, config, config2); if (error != 0) return (error); pm->pm_md.pm_ibs.ibs_ctl = config; pm->pm_md.pm_ibs.ibs_ctl2 = config2; if ((ibs_features & CPUID_IBSID_ADDRBIT63FILTERING) != 0) { if ((caps & PMC_CAP_USER) != 0 && (caps & PMC_CAP_SYSTEM) == 0) { if (ri == IBS_PMC_FETCH) pm->pm_md.pm_ibs.ibs_ctl2 |= IBS_FETCH_CTL2_EXCLADDR63EQ1; else pm->pm_md.pm_ibs.ibs_ctl2 |= IBS_OP_CTL2_EXCLRIP63EQ1; } else if ((caps & PMC_CAP_SYSTEM) != 0 && (caps & PMC_CAP_USER) == 0) { if (ri == IBS_PMC_FETCH) pm->pm_md.pm_ibs.ibs_ctl2 |= IBS_FETCH_CTL2_EXCLADDR63EQ0; else pm->pm_md.pm_ibs.ibs_ctl2 |= IBS_OP_CTL2_EXCLRIP63EQ0; } } PMCDBG3(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%jx config2=0x%jx", ri, config, config2); return (0); } /* * Release machine dependent state associated with a PMC. This is a * no-op on this architecture. */ static int ibs_release_pmc(int cpu, int ri, struct pmc *pmc __unused) { struct pmc_hw *phw __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); PMCDBG1(MDP, ALL, 1, "ibs-release ri=%d", ri); phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[ibs,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } /* * Start a PMC. */ static int ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm) { uint64_t config; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP, STA, 1, "ibs-start cpu=%d ri=%d", cpu, ri); /* * This is used to handle spurious NMIs. All that matters is that it * is not in the stopping state. */ atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING); /* * Turn on the ENABLE bit. Zeroing out the control register eliminates * stale valid bits from spurious NMIs and it resets the counter. This * is safe here because the counter is not yet enabled; the NMI re-arm * path must not do the same (Family 10h erratum #420). */ switch (ri) { case IBS_PMC_FETCH: wrmsr(IBS_FETCH_CTL, 0); if (ibs_fetch_ctl2_supported) wrmsr(IBS_FETCH_CTL2, pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_FETCH_CTL2_DISABLE); config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE; wrmsr(IBS_FETCH_CTL, config); break; case IBS_PMC_OP: wrmsr(IBS_OP_CTL, 0); if (ibs_op_ctl2_supported) wrmsr(IBS_OP_CTL2, pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_OP_CTL2_DISABLE); config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE; wrmsr(IBS_OP_CTL, config); break; } return (0); } /* * Stop a PMC. */ static int ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) { int i; uint64_t config, config2; bool use_alt_disable; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] illegal row-index %d", __LINE__, ri)); PMCDBG1(MDP, STO, 1, "ibs-stop ri=%d", ri); /* * Turn off the ENABLE bit, but unfortunately there are a few quirks * that generate excess NMIs. Workaround #420 in the Revision Guide * for AMD Family 10h Processors 41322 Rev. 3.92 March 2012. requires * that we clear the max count before clearing enable. * * Even after clearing the counter spurious NMIs are still possible so * we use a per-CPU atomic variable to notify the interrupt handler we * are stopping and discard spurious NMIs. We then retry clearing the * control register for 50us. This gives us enough time and ensures * that the valid bit is not accidently stuck after a spurious NMI. * * On Zen 6 with the alternate disable bit (CPUID IbsDis), assert the * ctl2 DISABLE bit first. This avoids an RMW hazard in ctl1 that the * processor may update concurrently while sampling. */ config = pm->pm_md.pm_ibs.ibs_ctl; config2 = pm->pm_md.pm_ibs.ibs_ctl2; use_alt_disable = (ibs_features & CPUID_IBSID_IBSDIS) != 0; atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING); /* * On Zen 6, ctl2 DISABLE is the authoritative stop switch; skip * the legacy ctl1 RMW and clear it directly */ switch (ri) { case IBS_PMC_FETCH: if (use_alt_disable) { wrmsr(IBS_FETCH_CTL2, config2 | IBS_FETCH_CTL2_DISABLE); wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_ENABLE); } else { wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK); DELAY(1); config &= ~IBS_FETCH_CTL_ENABLE; wrmsr(IBS_FETCH_CTL, config); } break; case IBS_PMC_OP: if (use_alt_disable) { wrmsr(IBS_OP_CTL2, config2 | IBS_OP_CTL2_DISABLE); wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_ENABLE); } else { wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_MAXCNTMASK); DELAY(1); config &= ~IBS_OP_CTL_ENABLE; wrmsr(IBS_OP_CTL, config); } break; } for (i = 0; i < IBS_STOP_ITER; i++) { DELAY(1); switch (ri) { case IBS_PMC_FETCH: wrmsr(IBS_FETCH_CTL, 0); if (ibs_fetch_ctl2_supported) wrmsr(IBS_FETCH_CTL2, 0); break; case IBS_PMC_OP: wrmsr(IBS_OP_CTL, 0); if (ibs_op_ctl2_supported) wrmsr(IBS_OP_CTL2, 0); break; } } atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPED); return (0); } static void pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config) { struct pmc_multipart mpd; if (pm == NULL) return; if (pm->pm_state != PMC_STATE_RUNNING) return; memset(&mpd, 0, sizeof(mpd)); mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH; mpd.pl_length = PMC_MPIDX_FETCH_MAX; mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config; if ((ibs_features & CPUID_IBSID_IBSFETCHCTLEXTD) != 0) { mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL); } mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR); if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) { mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] = rdmsr(IBS_FETCH_PHYSADDR); } if (ibs_fetch_ctl2_supported) { mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL2] = rdmsr(IBS_FETCH_CTL2); } pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); wrmsr(IBS_FETCH_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE); } static void pmc_ibs_process_op(struct pmc *pm, struct trapframe *tf, uint64_t config) { struct pmc_multipart mpd; if (pm == NULL) return; if (pm->pm_state != PMC_STATE_RUNNING) return; memset(&mpd, 0, sizeof(mpd)); mpd.pl_type = PMC_CC_MULTIPART_IBS_OP; mpd.pl_length = PMC_MPIDX_OP_MAX; mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config; mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP); mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA); mpd.pl_mpdata[PMC_MPIDX_OP_DATA2] = rdmsr(IBS_OP_DATA2); mpd.pl_mpdata[PMC_MPIDX_OP_DATA3] = rdmsr(IBS_OP_DATA3); mpd.pl_mpdata[PMC_MPIDX_OP_DC_LINADDR] = rdmsr(IBS_OP_DC_LINADDR); mpd.pl_mpdata[PMC_MPIDX_OP_DC_PHYSADDR] = rdmsr(IBS_OP_DC_PHYSADDR); if ((ibs_features & CPUID_IBSID_BRNTRGT) != 0) { mpd.pl_mpdata[PMC_MPIDX_OP_TGT_RIP] = rdmsr(IBS_OP_TGT_RIP); } if ((ibs_features & CPUID_IBSID_IBSOPDATA4) != 0) { mpd.pl_mpdata[PMC_MPIDX_OP_DATA4] = rdmsr(IBS_OP_DATA4); } if (ibs_op_ctl2_supported) { mpd.pl_mpdata[PMC_MPIDX_OP_CTL2] = rdmsr(IBS_OP_CTL2); } pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE); } /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed * to sleep or do anything a 'fast' interrupt handler is not allowed * to do. */ int pmc_ibs_intr(struct trapframe *tf) { struct ibs_cpu *pac; struct pmc *pm; int retval, cpu; uint64_t config; if (ibs_pcpu == NULL) return (0); cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] out of range CPU %d", __LINE__, cpu)); retval = 0; pac = ibs_pcpu[cpu]; /* * Fetch and op share an NMI. Both valid bits may be set on entry, * so service each valid source or the skipped unit can stay frozen * with no NMI pending to re-arm it. */ config = rdmsr(IBS_FETCH_CTL); if ((config & IBS_FETCH_CTL_VALID) != 0) { pm = pac->pc_ibspmcs[IBS_PMC_FETCH].phw_pmc; retval++; pmc_ibs_process_fetch(pm, tf, config); } config = rdmsr(IBS_OP_CTL); if ((config & IBS_OP_CTL_VALID) != 0) { pm = pac->pc_ibspmcs[IBS_PMC_OP].phw_pmc; retval++; pmc_ibs_process_op(pm, tf, config); } /* * When both units were serviced, the second unit's NMI is still * owed and will arrive with no valid bit set. It is not always * the next NMI: on Zen 6 one or more NMIs carrying a single new * sample can arrive first, and those may carry both samples again. * Count the owed NMIs and let each NMI with no valid bit consume one, * so they are not reported as unknown. Cap the count, since every * leftover credit would hide a genuine unknown NMI later. */ if (retval == 2) { if (pac->pc_nmi_credit < IBS_NMI_CREDIT_MAX) pac->pc_nmi_credit++; } else if (retval == 0 && pac->pc_nmi_credit > 0) { pac->pc_nmi_credit--; retval = 1; } if (retval > 1) retval = 1; if (retval == 0) { // Lets check for a stray NMI when stopping if (atomic_load_int(&pac->pc_status) == IBS_CPU_STOPPING) { return (1); } } /* The caller updates the interrupt statistics. */ return (retval); } /* * Describe a PMC. */ static int ibs_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < IBS_NPMCS, ("[ibs,%d] row-index %d out of range", __LINE__, ri)); phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; if (ri == IBS_PMC_FETCH) { strlcpy(pi->pm_name, "IBS-FETCH", sizeof(pi->pm_name)); pi->pm_class = PMC_CLASS_IBS; pi->pm_enabled = true; *ppmc = phw->phw_pmc; } else { strlcpy(pi->pm_name, "IBS-OP", sizeof(pi->pm_name)); pi->pm_class = PMC_CLASS_IBS; pi->pm_enabled = true; *ppmc = phw->phw_pmc; } return (0); } /* * Processor-dependent initialization. */ static int ibs_pcpu_init(struct pmc_mdep *md, int cpu) { struct ibs_cpu *pac; struct pmc_cpu *pc; struct pmc_hw *phw; int first_ri, n; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "ibs-init cpu=%d", cpu); ibs_pcpu[cpu] = pac = malloc(sizeof(struct ibs_cpu), M_PMC, M_WAITOK | M_ZERO); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu pointer", __LINE__)); for (n = 0, phw = pac->pc_ibspmcs; n < IBS_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * Processor-dependent cleanup prior to the KLD being unloaded. */ static int ibs_pcpu_fini(struct pmc_mdep *md, int cpu) { struct ibs_cpu *pac; struct pmc_cpu *pc; int first_ri, i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "ibs-cleanup cpu=%d", cpu); /* * Turn off IBS. */ wrmsr(IBS_FETCH_CTL, 0); wrmsr(IBS_OP_CTL, 0); if (ibs_fetch_ctl2_supported) wrmsr(IBS_FETCH_CTL2, 0); if (ibs_op_ctl2_supported) wrmsr(IBS_OP_CTL2, 0); /* * Free up allocated space. */ if ((pac = ibs_pcpu[cpu]) == NULL) return (0); ibs_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu state", __LINE__)); first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; /* * Reset pointers in the MI 'per-cpu' state. */ for (i = 0; i < IBS_NPMCS; i++) pc->pc_hwpmcs[i + first_ri] = NULL; free(pac, M_PMC); return (0); } /* * Initialize ourselves. */ int pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus) { u_int regs[4]; struct pmc_classdep *pcd; /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ ibs_pcpu = malloc(sizeof(struct ibs_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Initialize AMD IBS handling. */ pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS]; pcd->pcd_caps = IBS_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IBS; pcd->pcd_num = IBS_NPMCS; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 0; pcd->pcd_allocate_pmc = ibs_allocate_pmc; pcd->pcd_config_pmc = ibs_config_pmc; pcd->pcd_describe = ibs_describe; pcd->pcd_get_config = ibs_get_config; pcd->pcd_pcpu_fini = ibs_pcpu_fini; pcd->pcd_pcpu_init = ibs_pcpu_init; pcd->pcd_release_pmc = ibs_release_pmc; pcd->pcd_start_pmc = ibs_start_pmc; pcd->pcd_stop_pmc = ibs_stop_pmc; pcd->pcd_read_pmc = ibs_read_pmc; pcd->pcd_write_pmc = ibs_write_pmc; pmc_mdep->pmd_npmc += IBS_NPMCS; if (cpu_exthigh >= CPUID_IBSID) { do_cpuid(CPUID_IBSID, regs); ibs_features = regs[0]; if ((ibs_features & CPUID_IBSID_IBSFFV) == 0) ibs_features = 0; } else { ibs_features = 0; } ibs_init_policy(); PMCDBG0(MDP, INI, 0, "ibs-initialize"); return (0); } /* * Finalization code for AMD CPUs. */ void pmc_ibs_finalize(struct pmc_mdep *md) { PMCDBG0(MDP, INI, 1, "ibs-finalize"); /* Safe even if the IBS class was skipped at initialize time. */ if (ibs_pcpu == NULL) return; for (int i = 0; i < pmc_cpu_max(); i++) KASSERT(ibs_pcpu[i] == NULL, ("[ibs,%d] non-null pcpu cpu %d", __LINE__, i)); free(ibs_pcpu, M_PMC); ibs_pcpu = NULL; }