/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2026, Netflix, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _DEV_HWPMC_IBS_H_ #define _DEV_HWPMC_IBS_H_ 1 #define IBS_NPMCS 2 #define IBS_PMC_FETCH 0 #define IBS_PMC_OP 1 /* * All of the CPUID definitions come from AMD PPR Vol 1 for AMD Family 1Ah * Model 02h C1 (57238) 2024-09-29 Revision 0.24. * Zen 6 CPUID bits (IBSDIS, FETCHLATFILTERING, ADDRBIT63FILTERING) come from * AMD64 Architecture Programmer's Manual Volume 2: System Programming (24593) * 2025-07-02 Version 3.43. */ #define CPUID_IBSID 0x8000001B #define CPUID_IBSID_IBSFFV 0x00000001 /* IBS Feature Flags Valid */ #define CPUID_IBSID_FETCHSAM 0x00000002 /* IBS Fetch Sampling */ #define CPUID_IBSID_OPSAM 0x00000004 /* IBS Execution Sampling */ #define CPUID_IBSID_RDWROPCNT 0x00000008 /* RdWr Operationg Counter */ #define CPUID_IBSID_OPCNT 0x00000010 /* Operation Counter */ #define CPUID_IBSID_BRNTRGT 0x00000020 /* Branch Target Address */ #define CPUID_IBSID_OPCNTEXT 0x00000040 /* Extend Counter */ #define CPUID_IBSID_RIPINVALIDCHK 0x00000080 /* Invalid RIP Indication */ #define CPUID_IBSID_OPFUSE 0x00000100 /* Fused Branch Operation */ #define CPUID_IBSID_IBSFETCHCTLEXTD 0x00000200 /* IBS Fetch Control Ext */ #define CPUID_IBSID_IBSOPDATA4 0x00000400 /* IBS OP DATA4 */ #define CPUID_IBSID_ZEN4IBSEXTENSIONS 0x00000800 /* IBS Zen 4 Extensions */ #define CPUID_IBSID_IBSLOADLATENCYFILT 0x00001000 /* Load Latency Filtering */ #define CPUID_IBSID_IBSDIS 0x00002000 /* Alternate IBS Disable */ #define CPUID_IBSID_FETCHLATFILTERING 0x00004000 /* Fetch Latency Filter */ #define CPUID_IBSID_ADDRBIT63FILTERING 0x00008000 /* Addr Bit 63 Filter */ #define CPUID_IBSID_STRMSTANDRMTSOCKET 0x00010000 /* StrmSt + RmtSocket */ #define CPUID_IBSID_BUFFERV1 0x00020000 /* IBS Buffering V1 */ #define CPUID_IBSID_MEMPROFILERV1 0x00040000 /* IBS Memory Profiler V1 */ #define CPUID_IBSID_IBSUPDTDDTLBSTATS 0x00080000 /* Simplified DTLB Stats */ /* * All of these definitions here come from AMD64 Architecture Programmer's * Manual Volume 2: System Programming (24593) 2025-07-02 Version 3.43. with * the following exceptions: * * OpData4 and fields come from the BKDG for AMD Family 15h Model 70-7Fh * (55072) 2018-06-20 Revision 3.09. */ /* IBS MSRs */ #define IBS_CTL 0xC001103A /* IBS Control */ #define IBS_CTL_LVTOFFSETVALID (1ULL << 8) #define IBS_CTL_LVTOFFSETMASK 0x0000000F /* * The minimum sampling rate was selected to match the default used by other * counters that was also found to be experimentally stable by providing enough * time between consecutive NMIs. The maximum sample rate is determined by * setting all available counter bits, i.e., all available bits except the * bottom four that are zero extended. */ #define IBS_FETCH_MIN_RATE 65536 #define IBS_FETCH_MAX_RATE 1048560 #define IBS_OP_MIN_RATE 65536 #define IBS_OP_MAX_RATE 134217712 #define IBS_OP_MAX_RATE_PREEXT 1048560 /* IBS Fetch Control */ #define IBS_FETCH_CTL 0xC0011030 /* IBS Fetch Control */ #define IBS_FETCH_CTL_L3MISS (1ULL << 61) /* L3 Cache Miss */ #define IBS_FETCH_CTL_OPCACHEMISS (1ULL << 60) /* Op Cache Miss */ #define IBS_FETCH_CTL_L3MISSONLY (1ULL << 59) /* L3 Miss Filtering */ #define IBS_FETCH_CTL_L2MISS (1ULL << 58) #define IBS_FETCH_CTL_RANDOMIZE (1ULL << 57) /* Randomized Tagging */ #define IBS_FETCH_CTL_L2TLBMISS (1ULL << 56) #define IBS_FETCH_CTL_L1TLBMISS (1ULL << 55) /* L1 TLB Miss */ #define IBS_FETCH_CTL_TO_PGSZ(_d) (((_d) >> 53) & 0x3) #define IBS_FETCH_CTL_PHYSADDRVALID (1ULL << 52) /* PHYSADDR Valid */ #define IBS_FETCH_CTL_ICMISS (1ULL << 51) /* Inst. Cache Miss */ #define IBS_FETCH_CTL_COMPLETE (1ULL << 50) /* Complete */ #define IBS_FETCH_CTL_VALID (1ULL << 49) /* Valid */ #define IBS_FETCH_CTL_ENABLE (1ULL << 48) /* Enable */ #define IBS_FETCH_CTL_MAXCNTMASK 0x0000FFFFULL #define IBS_FETCH_CTL_CURCNTMASK 0xFFFF0000ULL #define IBS_FETCH_INTERVAL_TO_CTL(_c) (((_c) >> 4) & 0x0000FFFF) #define IBS_FETCH_CTL_TO_INTERVAL(_c) (((_c) & IBS_FETCH_CTL_MAXCNTMASK) << 4) #define IBS_FETCH_CTL_TO_LAT(_c) (((_c) >> 32) & 0x0000FFFF) #define IBS_FETCH_COUNT_TO_CTL(_c) (((_c) << 12) & IBS_FETCH_CTL_CURCNTMASK) #define IBS_FETCH_CTL_TO_COUNT(_c) (((_c) & IBS_FETCH_CTL_CURCNTMASK) >> 12) #define IBS_FETCH_ALLOWED_MASK_BASE (IBS_FETCH_CTL_MAXCNTMASK | \ IBS_FETCH_CTL_RANDOMIZE) #define IBS_FETCH_LINADDR 0xC0011031 /* Fetch Linear Address */ #define IBS_FETCH_PHYSADDR 0xC0011032 /* Fetch Physical Address */ #define IBS_FETCH_EXTCTL 0xC001103C /* Fetch Control Extended */ /* IBS Fetch Control 2 (Zen 6) */ #define IBS_FETCH_CTL2 0xC001103F /* IBS Fetch Control 2 */ #define IBS_FETCH_CTL2_DISABLE (1ULL << 0) /* IBS Fetch Disable */ #define IBS_FETCH_CTL2_LATFILTERMASK (0xFULL << 1) /* Fetch Latency Filter */ #define IBS_FETCH_CTL2_EXCLADDR63EQ1 (1ULL << 5) /* Exclude addr bit63=1 */ #define IBS_FETCH_CTL2_EXCLADDR63EQ0 (1ULL << 6) /* Exclude addr bit63=0 */ #define IBS_FETCH_CTL2_ADDR63MASK (IBS_FETCH_CTL2_EXCLADDR63EQ0 | \ IBS_FETCH_CTL2_EXCLADDR63EQ1) #define IBS_FETCH_CTL2_LAT_MIN 128 #define IBS_FETCH_CTL2_LAT_MAX 1920 #define IBS_FETCH_CTL2_LAT_STEP 128 #define IBS_FETCH_CTL2_LAT_TO_CTL(_l) ((((_l) >> 7) & 0xFULL) << 1) #define IBS_FETCH_CTL2_CTL_TO_LAT(_c) ((((_c) >> 1) & 0xFULL) << 7) #define PMC_MPIDX_FETCH_CTL 0 #define PMC_MPIDX_FETCH_EXTCTL 1 #define PMC_MPIDX_FETCH_LINADDR 2 #define PMC_MPIDX_FETCH_PHYSADDR 3 #define PMC_MPIDX_FETCH_CTL2 4 #define PMC_MPIDX_FETCH_MAX (PMC_MPIDX_FETCH_CTL2 + 1) /* IBS Execution Control */ #define IBS_OP_CTL 0xC0011033 /* IBS Execution Control */ #define IBS_OP_CTL_LATFLTEN (1ULL << 63) /* Load Latency Filtering */ #define IBS_OP_CTL_COUNTERCONTROL (1ULL << 19) /* Counter Control */ #define IBS_OP_CTL_VALID (1ULL << 18) /* Valid */ #define IBS_OP_CTL_ENABLE (1ULL << 17) /* Enable */ #define IBS_OP_CTL_L3MISSONLY (1ULL << 16) /* L3 Miss Filtering */ #define IBS_OP_CTL_MAXCNTMASK 0x07F0FFFFULL /* Max Count */ #define IBS_OP_CTL_MAXCNTEXTMASK 0x07F00000ULL /* Max Count Extended */ #define IBS_OP_CTL_MAXCNTBASEMASK (IBS_OP_CTL_MAXCNTMASK & \ ~IBS_OP_CTL_MAXCNTEXTMASK) /* Max Count Base */ #define IBS_OP_CTL_CURCNTMASK 0x07FFFFFF00000000ULL #define IBS_OP_CTL_LDLATTRSHMASK (0xFULL << 59) /* Load Lat Threshold */ #define IBS_OP_CTL_LDLATMASK (IBS_OP_CTL_LATFLTEN | \ IBS_OP_CTL_LDLATTRSHMASK) /* Load Lat Combined */ #define IBS_OP_CTL_LDLAT_TO_CTL(_c) (((((_c) >> 7) - 1) & 0xFULL) << 59) #define IBS_OP_INTERVAL_TO_CTL(_c) \ ((((_c) >> 4) & IBS_OP_CTL_MAXCNTBASEMASK) | \ ((_c) & IBS_OP_CTL_MAXCNTEXTMASK)) #define IBS_OP_CTL_TO_INTERVAL(_c) \ ((((_c) & IBS_OP_CTL_MAXCNTBASEMASK) << 4) | \ ((_c) & IBS_OP_CTL_MAXCNTEXTMASK)) #define IBS_OP_COUNT_TO_CTL(_c) (((_c) << 32) & IBS_OP_CTL_CURCNTMASK) #define IBS_OP_CTL_TO_COUNT(_c) (((_c) & IBS_OP_CTL_CURCNTMASK) >> 32) #define IBS_OP_RIP 0xC0011034 /* IBS Op RIP */ #define IBS_OP_DATA 0xC0011035 /* IBS Op Data */ #define IBS_OP_DATA_RIPINVALID (1ULL << 38) /* RIP Invalid */ #define IBS_OP_DATA_BRANCHRETIRED (1ULL << 37) /* Branch Retired */ #define IBS_OP_DATA_BRANCHMISPREDICTED (1ULL << 36) /* Branch Mispredicted */ #define IBS_OP_DATA_BRANCHTAKEN (1ULL << 35) /* Branch Taken */ #define IBS_OP_DATA_RETURN (1ULL << 34) /* Return */ #define IBS_OP_DATA_TO_TAGTORET(_d) (((_d) >> 16) & 0xffff) #define IBS_OP_DATA_TO_COMPTORET(_d) ((_d) & 0xffff) /* * Datasrc reserves 5 bits but only uses 4 up to Zen 5. * From PPR for AMD Family 1Ah Model 70h A0 */ #define IBS_OP_DATA2 0xC0011036 /* IBS Op Data 2 */ #define IBS_OP_DATA2_RMTSOCKET (1ULL << 9) /* Remote Socket */ #define IBS_OP_DATA2_STRMST (1ULL << 8) /* Streaming Store */ #define IBS_OP_DATA2_HITO (1 << 5) #define IBS_OP_DATA2_DATASRC(_d) (((_d) & 0x7) | (((_d) >> 3) & 0x8)) #define IBS_DATASRC_LOCALCCX 0x1 #define IBS_DATASRC_NEARFARCACHE_NEAR 0x2 #define IBS_DATASRC_DRAMIO_NEAR 0x3 #define IBS_DATASRC_NEARFARCACHE_FAR 0x5 #define IBS_DATASRC_LONGLAT_NEARFAR 0x6 #define IBS_DATASRC_DRAMIO_FAR 0x7 #define IBS_DATASRC_EXT_NEARFAR 0x8 #define IBS_DATASRC_PEER_NEARFAR 0xC /* * Memwidth reserves 4 bits but only uses 3 up to Zen 5. * From PPR for AMD Family 1Ah Model 70h A0 */ #define IBS_OP_DATA3 0xC0011037 /* IBS Op Data 3 */ #define IBS_OP_DATA3_PREFETCH (1ULL << 21) #define IBS_OP_DATA3_L2MISS (1ULL << 20) #define IBS_OP_DATA3_DCPHYADDRVALID (1ULL << 18) /* DC Physical Address */ #define IBS_OP_DATA3_DCLINADDRVALID (1ULL << 17) /* DC Linear Address */ #define IBS_OP_DATA3_LOCKEDOP (1ULL << 15) /* DC Locked Op */ #define IBS_OP_DATA3_UCMEMACCESS (1ULL << 14) /* DC UC Memory Access */ #define IBS_OP_DATA3_WCMEMACCESS (1ULL << 13) /* DC WC Memory Access */ #define IBS_OP_DATA3_DCMISALIGN (1ULL << 8) /* DC Misaligned Access */ #define IBS_OP_DATA3_DCMISS (1ULL << 7) /* DC Miss */ #define IBS_OP_DATA3_DCL1TLBHIT1G (1ULL << 5) /* DC L1 TLB Hit 1-GB */ #define IBS_OP_DATA3_DCL1TLBHIT2M (1ULL << 4) /* DC L1 TLB Hit 2-MB */ #define IBS_OP_DATA3_DCL1TLBMISS (1ULL << 2) /* DC L1 TLB Miss */ #define IBS_OP_DATA3_STORE (1ULL << 1) /* Store */ #define IBS_OP_DATA3_LOAD (1ULL << 0) /* Load */ #define IBS_OP_DATA3_TO_DCLAT(_c) ((_c >> 32) & 0x0000FFFF) #define IBS_OP_DATA3_MEMWIDTH(_d) (((_d) >> 22) & 0x7) #define IBS_OP_DATA3_PGSZ(_d) (((_d) >> 4) & 0x3) #define IBS_OP_DATA3_TO_TLBREFILLLAT(_c) (((_c) >> 48) & 0x0000ffff) #define IBS_OP_DATA3_TO_OPENMEMREQS(_c) (((_c) >> 26) & 0x003f) #define IBSOPDATA2_VALIDMASK (IBS_OP_DATA3_LOAD | IBS_OP_DATA3_DCMISS | IBS_OP_DATA3_L2MISS) #define IBS_OP_DC_LINADDR 0xC0011038 /* IBS DC Linear Address */ #define IBS_OP_DC_PHYSADDR 0xC0011039 /* IBS DC Physical Address */ #define IBS_OP_TGT_RIP 0xC001103B /* IBS Branch Target */ #define IBS_OP_DATA4 0xC001103D /* IBS Op Data 4 */ #define IBS_OP_DATA4_LDRESYNC (1ULL << 0) /* Load Resync */ /* IBS Execution Control 2 (Zen 6) */ #define IBS_OP_CTL2 0xC001103E /* IBS Execution Control 2 */ #define IBS_OP_CTL2_DISABLE (1ULL << 0) /* IBS Execution Disable */ #define IBS_OP_CTL2_EXCLRIP63EQ0 (1ULL << 1) /* Exclude RIP bit63=0 */ #define IBS_OP_CTL2_EXCLRIP63EQ1 (1ULL << 2) /* Exclude RIP bit63=1 */ #define IBS_OP_CTL2_STRMSTFILTER (1ULL << 3) /* Streaming Store Filter */ #define IBS_OP_CTL2_RIP63MASK (IBS_OP_CTL2_EXCLRIP63EQ0 | \ IBS_OP_CTL2_EXCLRIP63EQ1) #define PMC_MPIDX_OP_CTL 0 #define PMC_MPIDX_OP_RIP 1 #define PMC_MPIDX_OP_DATA 2 #define PMC_MPIDX_OP_DATA2 3 #define PMC_MPIDX_OP_DATA3 4 #define PMC_MPIDX_OP_DC_LINADDR 5 #define PMC_MPIDX_OP_DC_PHYSADDR 6 #define PMC_MPIDX_OP_TGT_RIP 7 #define PMC_MPIDX_OP_DATA4 8 #define PMC_MPIDX_OP_CTL2 9 #define PMC_MPIDX_OP_MAX (PMC_MPIDX_OP_CTL2 + 1) /* * IBS data is encoded as using the multipart flag in the existing callchain * structure. The PMC ID number tells you if the sample contains a fetch or an * op sample. The available payload will be encoded in the MSR order with a * variable length. */ struct pmc_md_ibs_op_pmcallocate { uint32_t ibs_flag; uint32_t ibs_type; uint64_t ibs_ctl; uint64_t ibs_ctl2; }; #ifdef _KERNEL /* MD extension for 'struct pmc' */ struct pmc_md_ibs_pmc { uint32_t ibs_flag; uint32_t ibs_type; uint64_t ibs_ctl; uint64_t ibs_ctl2; }; #define IBS_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) int pmc_ibs_initialize(struct pmc_mdep *md, int ncpu); void pmc_ibs_finalize(struct pmc_mdep *md); int pmc_ibs_intr(struct trapframe *tf); #endif /* _KERNEL */ #endif /* _DEV_HWPMC_IBS_H_ */