/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2018 Intel Corporation * Copyright (c) 2026 The FreeBSD Foundation * * Portions of this software were developed by ShengYi Hung * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted providing that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi_if.h" #include "cpufreq_if.h" extern uint64_t tsc_freq; static int intel_hwpstate_probe(device_t dev); static int intel_hwpstate_attach(device_t dev); static int intel_hwpstate_detach(device_t dev); static int intel_hwpstate_suspend(device_t dev); static int intel_hwpstate_resume(device_t dev); static int intel_hwpstate_get(device_t dev, struct cf_setting *cf); static int intel_hwpstate_type(device_t dev, int *type); static device_method_t intel_hwpstate_methods[] = { /* Device interface */ DEVMETHOD(device_identify, intel_hwpstate_identify), DEVMETHOD(device_probe, intel_hwpstate_probe), DEVMETHOD(device_attach, intel_hwpstate_attach), DEVMETHOD(device_detach, intel_hwpstate_detach), DEVMETHOD(device_suspend, intel_hwpstate_suspend), DEVMETHOD(device_resume, intel_hwpstate_resume), /* cpufreq interface */ DEVMETHOD(cpufreq_drv_get, intel_hwpstate_get), DEVMETHOD(cpufreq_drv_type, intel_hwpstate_type), DEVMETHOD_END }; #define RDMSR_ON_CPU(sc, msr, val) \ (x86_msr_op(msr, \ MSR_OP_RENDEZVOUS_ONE | MSR_OP_READ | \ MSR_OP_CPUID(sc->cpuid), \ 0, val)); #define WRMSR_ON_CPU(sc, msr, val) \ x86_msr_op(msr, \ MSR_OP_RENDEZVOUS_ONE | MSR_OP_WRITE | \ MSR_OP_CPUID(sc->cpuid), \ val, NULL) struct hwp_softc { device_t dev; u_int cpuid; bool hwp_notifications; bool hwp_activity_window; bool hwp_pref_ctrl; bool hwp_pkg_ctrl; bool hwp_pkg_ctrl_en; bool hwp_perf_bias; bool hwp_perf_bias_cached; /* Cached copy of HWP_REQUEST/HWP_REQUEST_PKG. */ uint64_t req; /* Cache PERF_BIAS. */ uint64_t hwp_energy_perf_bias; uint8_t high; uint8_t guaranteed; uint8_t efficient; uint8_t low; }; static driver_t hwpstate_intel_driver = { "hwpstate_intel", intel_hwpstate_methods, sizeof(struct hwp_softc), }; DRIVER_MODULE(hwpstate_intel, cpu, hwpstate_intel_driver, NULL, NULL); MODULE_VERSION(hwpstate_intel, 1); /* * Internal errors conveyed by code executing on another CPU. */ #define HWP_ERROR_CPPC_ENABLE (1 << 0) #define HWP_ERROR_CPPC_CAPS (1 << 1) #define HWP_ERROR_CPPC_REQUEST (1 << 2) #define HWP_ERROR_CPPC_REQUEST_WRITE (1 << 3) #define HWP_ERROR_CPPC_REQUEST_PKG (1 << 4) #define HWP_ERROR_CPPC_EPP_WRITE (1 << 5) static inline bool hwp_has_error(u_int res, u_int err) { return ((res & err) != 0); } struct get_cppc_regs_data { /* Inputs */ const struct hwp_softc *sc; /* Outputs */ uint64_t enabled; uint64_t caps; uint64_t request; uint64_t request_pkg; /* HWP_ERROR_CPPC_* except HWP_ERROR_*_WRITE */ u_int res; }; static void get_cppc_regs_cb(void *args) { struct get_cppc_regs_data *const data = args; int error; data->res = 0; error = rdmsr_safe(MSR_IA32_PM_ENABLE, &data->enabled); if (error != 0) data->res |= HWP_ERROR_CPPC_ENABLE; error = rdmsr_safe(MSR_IA32_HWP_CAPABILITIES, &data->caps); if (error != 0) data->res |= HWP_ERROR_CPPC_CAPS; error = rdmsr_safe(MSR_IA32_HWP_REQUEST, &data->request); if (error != 0) data->res |= HWP_ERROR_CPPC_REQUEST; if (data->sc->hwp_pkg_ctrl) { error = rdmsr_safe(MSR_IA32_HWP_REQUEST_PKG, &data->request_pkg); if (error != 0) data->res |= HWP_ERROR_CPPC_REQUEST_PKG; } } static inline void get_cppc_regs_one(const struct hwp_softc *const sc, struct get_cppc_regs_data *const req) { req->sc = sc; smp_rendezvous_cpu(sc->cpuid, smp_no_rendezvous_barrier, get_cppc_regs_cb, smp_no_rendezvous_barrier, req); } #define MSR_NOT_READ_MSG "Not read (fault or previous errors)" static int intel_hwp_dump_sysctl_handler(SYSCTL_HANDLER_ARGS) { const struct hwp_softc *const sc = arg1; const u_int cpuid = sc->cpuid; struct sbuf *sb; struct get_cppc_regs_data data; int ret = 0; get_cppc_regs_one(sc, &data); sb = sbuf_new(NULL, NULL, 1024, SBUF_FIXEDLEN | SBUF_INCLUDENUL); sbuf_putc(sb, '\n'); if (hwp_has_error(data.res, HWP_ERROR_CPPC_ENABLE)) sbuf_printf(sb, "CPU%u: IA32_PM_ENABLE: " MSR_NOT_READ_MSG "\n", cpuid); else sbuf_printf(sb, "CPU%d: HWP %sabled\n", cpuid, ((data.enabled & 1) ? "En" : "Dis")); if (data.enabled == 0) goto out; if (hwp_has_error(data.res, HWP_ERROR_CPPC_CAPS)) { sbuf_printf(sb, "IA32_HWP_CAPABILITIES: " MSR_NOT_READ_MSG "\n"); } else { sbuf_printf(sb, "\tHighest Performance: %03ju\n", data.caps & 0xff); sbuf_printf(sb, "\tGuaranteed Performance: %03ju\n", (data.caps >> 8) & 0xff); sbuf_printf(sb, "\tEfficient Performance: %03ju\n", (data.caps >> 16) & 0xff); sbuf_printf(sb, "\tLowest Performance: %03ju\n", (data.caps >> 24) & 0xff); sbuf_putc(sb, '\n'); } #define pkg_print(x, name, offset) do { \ if (!sc->hwp_pkg_ctrl || (data.request & x) != 0) \ sbuf_printf(sb, "\t%s: %03u\n", name, \ (unsigned)(data.request >> offset) & 0xff); \ else \ sbuf_printf(sb, "\t%s: %03u\n", name, \ (unsigned)(data.request_pkg >> offset) & 0xff); \ } while (0) if (hwp_has_error(data.res, HWP_ERROR_CPPC_REQUEST)) { sbuf_printf(sb, "IA32_HWP_REQUEST: " MSR_NOT_READ_MSG "\n"); } else if (hwp_has_error(data.res, HWP_ERROR_CPPC_REQUEST_PKG)) { sbuf_printf(sb, "IA32_HWP_REQUEST_PKG: " MSR_NOT_READ_MSG "\n"); } else { pkg_print(IA32_HWP_REQUEST_EPP_VALID, "Requested Efficiency Performance Preference", 24); pkg_print(IA32_HWP_REQUEST_DESIRED_VALID, "Requested Desired Performance", 16); pkg_print(IA32_HWP_REQUEST_MAXIMUM_VALID, "Requested Maximum Performance", 8); pkg_print(IA32_HWP_REQUEST_MINIMUM_VALID, "Requested Minimum Performance", 0); } #undef pkg_print sbuf_putc(sb, '\n'); out: ret = sbuf_finish(sb); if (ret == 0) ret = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb)); sbuf_delete(sb); return (ret); } /* * Multiplying by 17 here allows to send [0;15] to the full [0;255] range (so * 255 is reported when EPB is set to 15, instead of 240). */ #define EPB_TO_EPP(x) ((x) * 17) #define EPP_TO_EPB(x) ((x) >> 4) static int sysctl_epp_select(SYSCTL_HANDLER_ARGS) { const device_t dev = arg1; struct hwp_softc *const sc = device_get_softc(dev); uint64_t epb; uint32_t val; int ret; if (!sc->hwp_pref_ctrl && !sc->hwp_perf_bias) return (ENODEV); if (sc->hwp_pref_ctrl) { val = (sc->req & IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE) >> 24; } else { /* * If cpuid indicates EPP is not supported, the HWP controller * uses MSR_IA32_ENERGY_PERF_BIAS instead (Intel SDM §14.4.4). * This register is per-core (but not HT). */ if (!sc->hwp_perf_bias_cached) { ret = RDMSR_ON_CPU(sc, MSR_IA32_ENERGY_PERF_BIAS, &epb); if (ret) goto out; sc->hwp_energy_perf_bias = epb; sc->hwp_perf_bias_cached = true; } val = sc->hwp_energy_perf_bias & IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK; val = EPB_TO_EPP(val); } MPASS(val >= 0 && val <= 255); ret = sysctl_handle_int(oidp, &val, 0, req); if (ret || req->newptr == NULL) goto out; if (val > 255 || val < 0) { ret = EINVAL; goto out; } if (sc->hwp_pref_ctrl) { const uint64_t req_cached = sc->req = (sc->req & ~IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE) | (val << 24u); if (sc->hwp_pkg_ctrl_en) ret = WRMSR_ON_CPU(sc, MSR_IA32_HWP_REQUEST_PKG, req_cached); else ret = WRMSR_ON_CPU(sc, MSR_IA32_HWP_REQUEST, req_cached); } else { val = EPP_TO_EPB(val); MPASS((val & ~IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK) == 0); sc->hwp_energy_perf_bias = ((sc->hwp_energy_perf_bias & ~IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK) | val); ret = WRMSR_ON_CPU(sc, MSR_IA32_ENERGY_PERF_BIAS, sc->hwp_energy_perf_bias); } out: return (ret); } void intel_hwpstate_identify(driver_t *driver, device_t parent) { int i; if (device_find_child(parent, "hwpstate_intel", DEVICE_UNIT_ANY) != NULL) return; if (cpu_vendor_id != CPU_VENDOR_INTEL) return; if (resource_disabled("hwpstate_intel", 0)) return; /* * Intel SDM 14.4.1 (HWP Programming Interfaces): * Availability of HWP baseline resource and capability, * CPUID.06H:EAX[bit 7]: If this bit is set, HWP provides several new * architectural MSRs: IA32_PM_ENABLE, IA32_HWP_CAPABILITIES, * IA32_HWP_REQUEST, IA32_HWP_STATUS. */ if ((cpu_power_eax & CPUTPM1_HWP) == 0) return; /* * On hybrid-core systems, package-level control cannot be used. * It may cause all cores to run at the E-core frequency because * the resulting package frequency depends on the last core that * sets the frequency. */ CPU_FOREACH(i) { if (pcpu_find(i)->pc_small_core) { hwpstate_pkg_ctrl_enable = false; break; } } if (BUS_ADD_CHILD(parent, 10, "hwpstate_intel", device_get_unit(parent)) == NULL) device_printf(parent, "hwpstate_intel: add child failed\n"); } static int intel_hwpstate_probe(device_t dev) { device_set_desc(dev, "Intel Speed Shift"); return (BUS_PROBE_NOWILDCARD); } struct set_autonomous_hwp_cb { struct hwp_softc *sc; uint32_t flag; }; static void set_autonomous_hwp_cb(void *arg) { int ret; uint64_t caps, req; struct set_autonomous_hwp_cb *data = arg; struct hwp_softc *sc = data->sc; data->flag = 0; /* XXX: Many MSRs aren't readable until feature is enabled */ ret = wrmsr_safe(MSR_IA32_PM_ENABLE, 1); if (ret) { /* * This is actually a package-level MSR, and only the first * write is not ignored. So it is harmless to enable it across * all devices, and this allows us not to care especially in * which order cores (and packages) are probed. This error * condition should not happen given we gate on the HWP CPUID * feature flag, if the Intel SDM is correct. */ data->flag |= HWP_ERROR_CPPC_ENABLE; return; } ret = rdmsr_safe(MSR_IA32_HWP_CAPABILITIES, &caps); if (ret) { data->flag |= HWP_ERROR_CPPC_CAPS; } ret = rdmsr_safe(MSR_IA32_HWP_REQUEST, &req); if (ret) { data->flag |= HWP_ERROR_CPPC_REQUEST; } /* * High and low are static; "guaranteed" is dynamic; and efficient is * also dynamic. */ sc->high = IA32_HWP_CAPABILITIES_HIGHEST_PERFORMANCE(caps); sc->guaranteed = IA32_HWP_CAPABILITIES_GUARANTEED_PERFORMANCE(caps); sc->efficient = IA32_HWP_CAPABILITIES_EFFICIENT_PERFORMANCE(caps); sc->low = IA32_HWP_CAPABILITIES_LOWEST_PERFORMANCE(caps); /* hardware autonomous selection determines the performance target */ req &= ~IA32_HWP_DESIRED_PERFORMANCE; /* enable HW dynamic selection of window size */ req &= ~IA32_HWP_ACTIVITY_WINDOW; /* IA32_HWP_REQUEST.Minimum_Performance = * IA32_HWP_CAPABILITIES.Lowest_Performance */ req &= ~IA32_HWP_MINIMUM_PERFORMANCE; req |= sc->low; /* IA32_HWP_REQUEST.Maximum_Performance = * IA32_HWP_CAPABILITIES.Highest_Performance. */ req &= ~IA32_HWP_REQUEST_MAXIMUM_PERFORMANCE; req |= sc->high << 8; sc->req = req; /* If supported, request package-level control for this CPU. */ if (sc->hwp_pkg_ctrl_en) ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req | IA32_HWP_REQUEST_PACKAGE_CONTROL); else ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req); if (ret) data->flag |= HWP_ERROR_CPPC_REQUEST_WRITE; /* If supported, write the PKG-wide control MSR. */ if (sc->hwp_pkg_ctrl_en) { /* * "The structure of the IA32_HWP_REQUEST_PKG MSR * (package-level) is identical to the IA32_HWP_REQUEST MSR * with the exception of the Package Control field, which does * not exist." (Intel SDM §14.4.4) */ ret = wrmsr_safe(MSR_IA32_HWP_REQUEST_PKG, sc->req); if (ret) data->flag |= HWP_ERROR_CPPC_REQUEST_PKG; } } static inline void set_autonomous_hwp_send_one(struct hwp_softc *sc, struct set_autonomous_hwp_cb *data) { data->sc = sc; smp_rendezvous_cpu(sc->cpuid, smp_no_rendezvous_barrier, set_autonomous_hwp_cb, smp_no_rendezvous_barrier, data); } static int set_autonomous_hwp(struct hwp_softc *const sc) { const device_t dev = sc->dev; const u_int cpuid = sc->cpuid; struct set_autonomous_hwp_cb data; set_autonomous_hwp_send_one(sc, &data); if (hwp_has_error(data.flag, HWP_ERROR_CPPC_ENABLE)) { device_printf(dev, "Failed to enable HWP for cpu%d (%d)\n", cpuid, EFAULT); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_REQUEST)) { device_printf(dev, "Failed to read HWP request MSR for cpu%d (%d)\n", cpuid, EFAULT); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_CAPS)) { device_printf(dev, "Failed to read HWP capabilities MSR for cpu%d (%d)\n", cpuid, EFAULT); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_REQUEST_WRITE)) { device_printf(dev, "Failed to setup%s autonomous HWP for cpu%d\n", sc->hwp_pkg_ctrl_en ? " PKG" : "", cpuid); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_REQUEST_PKG)) device_printf(dev, "Failed to set autonomous HWP for package\n"); out: if (data.flag) return (EFAULT); return (0); } static int intel_hwpstate_attach(device_t dev) { struct hwp_softc *sc; sc = device_get_softc(dev); sc->dev = dev; if (cpu_get_pcpu(dev) == NULL) { device_printf(dev, "Parent bus does not provide a per-CPU structure!"); return (ENXIO); } sc->cpuid = cpu_get_pcpuid(dev); /* eax */ if (cpu_power_eax & CPUTPM1_HWP_NOTIFICATION) sc->hwp_notifications = true; if (cpu_power_eax & CPUTPM1_HWP_ACTIVITY_WINDOW) sc->hwp_activity_window = true; if (cpu_power_eax & CPUTPM1_HWP_PERF_PREF) sc->hwp_pref_ctrl = true; if (cpu_power_eax & CPUTPM1_HWP_PKG) sc->hwp_pkg_ctrl = true; /* Allow administrators to disable pkg-level control. */ sc->hwp_pkg_ctrl_en = (sc->hwp_pkg_ctrl && hwpstate_pkg_ctrl_enable); /* ecx */ if (cpu_power_ecx & CPUID_PERF_BIAS) sc->hwp_perf_bias = true; set_autonomous_hwp(sc); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_STATIC_CHILDREN(_debug), OID_AUTO, device_get_nameunit(dev), CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, sc, 0, intel_hwp_dump_sysctl_handler, "A", ""); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "epp", CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, dev, 0, sysctl_epp_select, "I", "Efficiency/Performance Preference " "(range from 0, most performant, through 255, most efficient)"); return (cpufreq_register(dev)); } static int intel_hwpstate_detach(device_t dev) { return (cpufreq_unregister(dev)); } static int intel_hwpstate_get(device_t dev, struct cf_setting *set) { const struct hwp_softc *const sc = device_get_softc(dev); uint64_t rate; int ret; if (set == NULL) return (EINVAL); memset(set, CPUFREQ_VAL_UNKNOWN, sizeof(*set)); set->dev = dev; ret = cpu_est_clockrate(sc->cpuid, &rate); if (ret == 0) set->freq = rate / 1000000; set->volts = CPUFREQ_VAL_UNKNOWN; set->power = CPUFREQ_VAL_UNKNOWN; set->lat = CPUFREQ_VAL_UNKNOWN; return (0); } static int intel_hwpstate_type(device_t dev, int *type) { if (type == NULL) return (EINVAL); *type = CPUFREQ_TYPE_ABSOLUTE | CPUFREQ_FLAG_INFO_ONLY | CPUFREQ_FLAG_UNCACHED; return (0); } static int intel_hwpstate_suspend(device_t dev) { return (0); } struct hwpstate_resume_cb { const struct hwp_softc *sc; uint32_t flag; }; static void hwpstate_resume_cb(void *arg) { struct hwpstate_resume_cb *const data = arg; const struct hwp_softc *const sc = data->sc; int ret; data->flag = 0; ret = wrmsr_safe(MSR_IA32_PM_ENABLE, 1); if (ret) { data->flag |= HWP_ERROR_CPPC_ENABLE; return; } if (sc->hwp_pkg_ctrl_en) ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req | IA32_HWP_REQUEST_PACKAGE_CONTROL); else ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req); if (ret) { data->flag |= HWP_ERROR_CPPC_REQUEST_WRITE; return; } if (sc->hwp_pkg_ctrl_en) { ret = wrmsr_safe(MSR_IA32_HWP_REQUEST_PKG, sc->req); if (ret) { data->flag |= HWP_ERROR_CPPC_REQUEST_PKG; return; } } if (!sc->hwp_pref_ctrl && sc->hwp_perf_bias_cached) { ret = wrmsr_safe(MSR_IA32_ENERGY_PERF_BIAS, sc->hwp_energy_perf_bias); if (ret) { data->flag |= HWP_ERROR_CPPC_EPP_WRITE; } } } static inline void hwpstate_resume_send_one(const struct hwp_softc *sc, struct hwpstate_resume_cb *req) { req->sc = sc; smp_rendezvous_cpu(sc->cpuid, smp_no_rendezvous_barrier, hwpstate_resume_cb, smp_no_rendezvous_barrier, req); } /* * Redo a subset of set_autonomous_hwp on resume; untested. Without this, * testers observed that on resume MSR_IA32_HWP_REQUEST was bogus. */ static int intel_hwpstate_resume(device_t dev) { const struct hwp_softc *const sc = device_get_softc(dev); const u_int cpuid = sc->cpuid; struct hwpstate_resume_cb data; hwpstate_resume_send_one(sc, &data); if (hwp_has_error(data.flag, HWP_ERROR_CPPC_ENABLE)) { device_printf(dev, "Failed to enable HWP for cpu%d after suspend (%d)\n", cpuid, EFAULT); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_REQUEST_WRITE)) { device_printf(dev, "Failed to set%s autonomous HWP for cpu%d after suspend\n", sc->hwp_pkg_ctrl_en ? " PKG" : "", cpuid); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_REQUEST_PKG)) { device_printf(dev, "Failed to set autonomous HWP for package after " "suspend\n"); goto out; } if (hwp_has_error(data.flag, HWP_ERROR_CPPC_EPP_WRITE)) { device_printf(dev, "Failed to set energy perf bias for cpu%d after " "suspend\n", cpuid); } out: if (data.flag) return (EFAULT); return (0); }