corefreqk.c

/*
 * CoreFreq
 * Copyright (C) 2015-2021 CYRIL INGENIERIE
 * Licenses: GPL2
 */

#include <linux/version.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/pci.h>
#ifdef CONFIG_DMI
#include <linux/dmi.h>
#endif /* CONFIG_DMI */
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/percpu.h>
#include <linux/utsname.h>
#ifdef CONFIG_CPU_IDLE
#include <linux/cpuidle.h>
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
#include <linux/cpufreq.h>
#endif /* CONFIG_CPU_FREQ */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
#include <linux/sched/signal.h>
#endif /* KERNEL_VERSION(4, 11, 0) */
#include <linux/clocksource.h>
#include <asm/msr.h>
#include <asm/nmi.h>
#ifdef CONFIG_XEN
#include <xen/xen.h>
#endif /* CONFIG_XEN */
#include <asm/mwait.h>

#include "bitasm.h"
#include "amdmsr.h"
#include "intelmsr.h"
#include "coretypes.h"
#include "corefreq-api.h"
#include "corefreqk.h"

#ifdef CONFIG_AMD_NB
#include <asm/amd_nb.h>
#endif

MODULE_AUTHOR ("CYRIL INGENIERIE <labs[at]cyring[dot]fr>");
MODULE_DESCRIPTION ("CoreFreq Processor Driver");
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)
MODULE_SUPPORTED_DEVICE ("Intel,AMD,HYGON");
#endif
MODULE_LICENSE ("GPL");
MODULE_VERSION (COREFREQ_VERSION);

static signed int ArchID = -1;
module_param(ArchID, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ArchID, "Force an architecture (ID)");

static signed int AutoClock = 0b11;
module_param(AutoClock, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(AutoClock, "Estimate Clock Frequency 0:Spec; 1:Once; 2:Auto");

static unsigned int SleepInterval = 0;
module_param(SleepInterval, uint, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(SleepInterval, "Timer interval (ms)");

static unsigned int TickInterval = 0;
module_param(TickInterval, uint, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(TickInterval, "System requested interval (ms)");

static signed int Experimental = 0;
module_param(Experimental, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Experimental, "Enable features under development");

static signed int CPU_Count = -1;
module_param(CPU_Count, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CPU_Count, "-1:Kernel(default); 0:Hardware; >0: User value");

static signed short Target_Ratio_Unlock = -1;
module_param(Target_Ratio_Unlock, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Target_Ratio_Unlock, "1:Target Ratio Unlock; 0:Lock");

static signed short Clock_Ratio_Unlock = -1;
module_param(Clock_Ratio_Unlock, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Clock_Ratio_Unlock, "1:MinRatio; 2:MaxRatio; 3:Both Unlock");

static signed short Turbo_Ratio_Unlock = -1;
module_param(Turbo_Ratio_Unlock, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Turbo_Ratio_Unlock, "1:Turbo Ratio Unlock; 0:Lock");

static signed short Uncore_Ratio_Unlock = -1;
module_param(Uncore_Ratio_Unlock, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Uncore_Ratio_Unlock, "1:Uncore Ratio Unlock; 0:Lock");

static signed int ServiceProcessor = -1; /* -1=ANY ; 0=BSP */
module_param(ServiceProcessor, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ServiceProcessor, "Select a CPU to run services with");

static SERVICE_PROC DefaultSMT = {.Proc = -1};

static unsigned short RDPMC_Enable = 0;
module_param(RDPMC_Enable, ushort, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(RDPMC_Enable, "Enable RDPMC bit in CR4 register");

static unsigned short NMI_Disable = 1;
module_param(NMI_Disable, ushort, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(NMI_Disable, "Disable the NMI Handler");

static int Override_SubCstate_Depth = 0;
static unsigned short Override_SubCstate[CPUIDLE_STATE_MAX] = {
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
module_param_array(	Override_SubCstate,ushort, &Override_SubCstate_Depth, \
			S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Override_SubCstate, "Override Sub C-States");

static signed short PkgCStateLimit = -1;
module_param(PkgCStateLimit, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PkgCStateLimit, "Package C-State Limit");

static signed short IOMWAIT_Enable = -1;
module_param(IOMWAIT_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(IOMWAIT_Enable, "I/O MWAIT Redirection Enable");

static signed short CStateIORedir = -1;
module_param(CStateIORedir, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CStateIORedir, "Power Mgmt IO Redirection C-State");

static signed short Config_TDP_Level = -1;
module_param(Config_TDP_Level, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Config_TDP_Level, "Config TDP Control Level");

static unsigned int Custom_TDP_Count = 0;
static signed short Custom_TDP_Offset[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
module_param_array(Custom_TDP_Offset, short, &Custom_TDP_Count ,	\
					S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Custom_TDP_Offset, "TDP Limit Offset (watt)");

static unsigned int TDP_Limiting_Count = 0;
static signed short Activate_TDP_Limit[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
			-1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
module_param_array(Activate_TDP_Limit, short, &TDP_Limiting_Count,	\
					S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDP_Limit, "Activate TDP Limiting");

static unsigned int TDP_Clamping_Count = 0;
static signed short Activate_TDP_Clamp[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
			-1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
module_param_array(Activate_TDP_Clamp, short, &TDP_Clamping_Count,	\
					S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDP_Clamp, "Activate TDP Clamping");

static unsigned short Custom_TDC_Offset = 0;
module_param(Custom_TDC_Offset, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Custom_TDC_Offset, "TDC Limit Offset (amp)");

static signed short Activate_TDC_Limit = -1;
module_param(Activate_TDC_Limit, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDC_Limit, "Activate TDC Limiting");

static signed short L1_HW_PREFETCH_Disable = -1;
module_param(L1_HW_PREFETCH_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_HW_PREFETCH_Disable, "Disable L1 HW Prefetcher");

static signed short L1_HW_IP_PREFETCH_Disable = -1;
module_param(L1_HW_IP_PREFETCH_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_HW_IP_PREFETCH_Disable, "Disable L1 HW IP Prefetcher");

static signed short L2_HW_PREFETCH_Disable = -1;
module_param(L2_HW_PREFETCH_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_HW_PREFETCH_Disable, "Disable L2 HW Prefetcher");

static signed short L2_HW_CL_PREFETCH_Disable = -1;
module_param(L2_HW_CL_PREFETCH_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_HW_CL_PREFETCH_Disable, "Disable L2 HW CL Prefetcher");

static signed short SpeedStep_Enable = -1;
module_param(SpeedStep_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(SpeedStep_Enable, "Enable SpeedStep");

static signed short C1E_Enable = -1;
module_param(C1E_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1E_Enable, "Enable SpeedStep C1E");

static signed short TurboBoost_Enable = -1;
module_param(TurboBoost_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(TurboBoost_Enable, "Enable Turbo Boost");

static signed short C3A_Enable = -1;
module_param(C3A_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C3A_Enable, "Enable C3 Auto Demotion");

static signed short C1A_Enable = -1;
module_param(C1A_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1A_Enable, "Enable C3 Auto Demotion");

static signed short C3U_Enable = -1;
module_param(C3U_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C3U_Enable, "Enable C3 UnDemotion");

static signed short C1U_Enable = -1;
module_param(C1U_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1U_Enable, "Enable C1 UnDemotion");

static signed short CC6_Enable = -1;
module_param(CC6_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CC6_Enable, "Enable Core C6 State");

static signed short PC6_Enable = -1;
module_param(PC6_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PC6_Enable, "Enable Package C6 State");

static signed short ODCM_Enable = -1;
module_param(ODCM_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ODCM_Enable, "Enable On-Demand Clock Modulation");

static signed short ODCM_DutyCycle = -1;
module_param(ODCM_DutyCycle, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ODCM_DutyCycle, "ODCM DutyCycle [0-7] | [0-14]");

static signed short PowerMGMT_Unlock = -1;
module_param(PowerMGMT_Unlock, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerMGMT_Unlock, "Unlock Power Management");

static signed short PowerPolicy = -1;
module_param(PowerPolicy, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerPolicy, "Power Policy Preference [0-15]");

static signed short Turbo_Activation_Ratio = -1;
module_param(Turbo_Activation_Ratio, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Turbo_Activation_Ratio, "Turbo Activation Ratio");

static signed int PState_FID = -1;
module_param(PState_FID, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PState_FID, "P-State Frequency Id");

static signed int PState_VID = -1;
module_param(PState_VID, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PState_VID, "P-State Voltage Id");

static signed short HWP_Enable = -1;
module_param(HWP_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HWP_Enable, "Hardware-Controlled Performance States");

static signed short HWP_EPP = -1;
module_param(HWP_EPP, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HWP_EPP, "Energy Performance Preference");

static signed short HDC_Enable = -1;
module_param(HDC_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HDC_Enable, "Hardware Duty Cycling");

static signed short EEO_Disable = -1;
module_param(EEO_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(EEO_Disable, "Disable Energy Efficiency Optimization");

static signed short R2H_Disable = -1;
module_param(R2H_Disable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(R2H_Disable, "Disable Race to Halt");

static unsigned int Clear_Events = 0;
module_param(Clear_Events, uint, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Clear_Events, "Clear Thermal and Power Events");

static int ThermalScope = -1;
module_param(ThermalScope, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ThermalScope, "[0:None; 1:SMT; 2:Core; 3:Package]");

static int VoltageScope = -1;
module_param(VoltageScope, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(VoltageScope, "[0:None; 1:SMT; 2:Core; 3:Package]");

static int PowerScope = -1;
module_param(PowerScope, int, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerScope, "[0:None; 1:SMT; 2:Core; 3:Package]");

static signed short Register_CPU_Idle = -1;
module_param(Register_CPU_Idle, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_CPU_Idle, "Register the Kernel cpuidle driver");

static signed short Register_CPU_Freq = -1;
module_param(Register_CPU_Freq, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_CPU_Freq, "Register the Kernel cpufreq driver");

static signed short Register_Governor = -1;
module_param(Register_Governor, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_Governor, "Register the Kernel governor");

static signed short Register_ClockSource = -1;
module_param(Register_ClockSource, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_ClockSource, "Register Clock Source driver");

static signed short Idle_Route = -1;
module_param(Idle_Route, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Idle_Route, "[0:Default; 1:I/O; 2:HALT; 3:MWAIT]");

static signed short Mech_IBRS = -1;
module_param(Mech_IBRS, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_IBRS, "Mitigation Mechanism IBRS");

static signed short Mech_STIBP = -1;
module_param(Mech_STIBP, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_STIBP, "Mitigation Mechanism STIBP");

static signed short Mech_SSBD = -1;
module_param(Mech_SSBD, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_SSBD, "Mitigation Mechanism SSBD");

static signed short Mech_IBPB = -1;
module_param(Mech_IBPB, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_IBPB, "Mitigation Mechanism IBPB");

static signed short Mech_L1D_FLUSH = -1;
module_param(Mech_L1D_FLUSH, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_L1D_FLUSH, "Mitigation Mechanism Cache L1D Flush");

static signed short Mech_PSFD = -1;
module_param(Mech_PSFD, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_PSFD, "Mitigation Mechanism PSFD");

static signed short WDT_Enable = -1;
module_param(WDT_Enable, short, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(WDT_Enable, "Watchdog Hardware Timer");

static struct {
	signed int		Major;
	struct cdev		*kcdev;
	dev_t			nmdev, mkdev;
	struct class		*clsdev;
#ifdef CONFIG_CPU_IDLE
	struct cpuidle_device __percpu *IdleDevice;
	struct cpuidle_driver	IdleDriver;
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
	struct cpufreq_driver	FreqDriver;
	struct cpufreq_governor FreqGovernor;
#endif /* CONFIG_CPU_FREQ */
} CoreFreqK = {
#ifdef CONFIG_CPU_IDLE
	.IdleDriver = {
			.name	= "corefreqk-idle",
			.owner	= THIS_MODULE
	},
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
	.FreqDriver = {
			.name	= "corefreqk-perf",
			.flags	= CPUFREQ_CONST_LOOPS,
			.exit	= CoreFreqK_Policy_Exit,
	/*MANDATORY*/	.init	= CoreFreqK_Policy_Init,
	/*MANDATORY*/	.verify = CoreFreqK_Policy_Verify,
	/*MANDATORY*/	.setpolicy = CoreFreqK_SetPolicy,
			.bios_limit= CoreFreqK_Bios_Limit,
			.set_boost = CoreFreqK_SetBoost
	},
	.FreqGovernor = {
			.name	= "corefreq-policy",
			.owner	= THIS_MODULE,
			.show_setspeed	= CoreFreqK_Show_SetSpeed,
			.store_setspeed = CoreFreqK_Store_SetSpeed
	}
#endif /* CONFIG_CPU_FREQ */
};

static KPUBLIC *KPublic = NULL;
static KPRIVATE *KPrivate = NULL;
static ktime_t RearmTheTimer;

#define AT( _loc_ )		[ _loc_ ]
#define OF( _ptr_ , ...)	-> _ptr_ __VA_ARGS__
#define RO( _ptr_ , ...)	OF( _ptr_##_RO , __VA_ARGS__ )
#define RW( _ptr_ , ...)	OF( _ptr_##_RW , __VA_ARGS__ )
#define ADDR( _head_ , _mbr_ )	( _head_ _mbr_ )
#define PUBLIC(...)		ADDR( KPublic , __VA_ARGS__ )
#define PRIVATE(...)		ADDR( KPrivate, __VA_ARGS__ )

#define RESET_ARRAY(_array, _cnt, _val, ... )				\
({									\
	unsigned int rst;						\
	for (rst = 0; rst < _cnt; rst++) {				\
		_array[rst] __VA_ARGS__ = _val;				\
	}								\
})

static long CoreFreqK_Thermal_Scope(int scope)
{
    if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
    {
	PUBLIC(RO(Proc))->thermalFormula = \
		(KIND_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula) << 8)|scope;

	return RC_SUCCESS;
    } else {
	return -EINVAL;
    }
}

static long CoreFreqK_Voltage_Scope(int scope)
{
    if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
    {
	PUBLIC(RO(Proc))->voltageFormula = \
		(KIND_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula) << 8)|scope;

	return RC_SUCCESS;
    } else {
	return -EINVAL;
    }
}

static long CoreFreqK_Power_Scope(int scope)
{
    if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
    {
	PUBLIC(RO(Proc))->powerFormula = \
		(KIND_OF_FORMULA(PUBLIC(RO(Proc))->powerFormula) << 8)|scope;

	return RC_SUCCESS;
    } else {
	return -EINVAL;
    }
}

unsigned int FixMissingRatioAndFrequency(unsigned int r32, CLOCK *pClock)
{
	unsigned long long r64 = r32;
  if (PUBLIC(RO(Proc))->Features.Factory.Freq != 0)
  {
   if ((r32 == 0) && (pClock->Q > 0))
   {	/*	Fix missing ratio.					*/
      r64=DIV_ROUND_CLOSEST(PUBLIC(RO(Proc))->Features.Factory.Freq, pClock->Q);
      PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)]=\
		(unsigned int) r64;
   }
  }
  else if (r32 > 0)
  {	/*	Fix the Factory frequency (unit: MHz)			*/
	r64 = pClock->Hz * r32;
	r64 = r64 / 1000000LLU;
	PUBLIC(RO(Proc))->Features.Factory.Freq = (unsigned int) r64;
  }
	PUBLIC(RO(Proc))->Features.Factory.Clock.Q  = pClock->Q;
	PUBLIC(RO(Proc))->Features.Factory.Clock.R  = pClock->R;
	PUBLIC(RO(Proc))->Features.Factory.Clock.Hz = pClock->Hz;

  if (PUBLIC(RO(Proc))->Features.Factory.Clock.Hz > 0)
  {
    r64 = PUBLIC(RO(Proc))->Features.Factory.Freq * 1000000LLU;
    r64 = DIV_ROUND_CLOSEST(r64, PUBLIC(RO(Proc))->Features.Factory.Clock.Hz);
    PUBLIC(RO(Proc))->Features.Factory.Ratio = (unsigned int) r64;
  }
	return (unsigned int) r64;
}

unsigned long long CoreFreqK_Read_CS_From_Invariant_TSC(struct clocksource *cs)
{
	unsigned long long TSC __attribute__ ((aligned (8)));
	UNUSED(cs);
	RDTSCP64(TSC);
	return TSC;
}

unsigned long long CoreFreqK_Read_CS_From_Variant_TSC(struct clocksource *cs)
{
	unsigned long long TSC __attribute__ ((aligned (8)));
	UNUSED(cs);
	RDTSC64(TSC);
	return TSC;
}

static struct clocksource CoreFreqK_CS = {
	.name	= "corefreq",
	.rating = 250,
	.mask	= CLOCKSOURCE_MASK(64),
	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
};

static long CoreFreqK_UnRegister_ClockSource(void)
{
	long rc = -EINVAL;
    if (PUBLIC(RO(Proc))->Registration.Driver.CS & REGISTRATION_ENABLE)
    {
	int rx = clocksource_unregister(&CoreFreqK_CS);
	switch ( rx ) {
	case 0:
		PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
		rc = RC_SUCCESS;
		break;
	default:
		rc = (long) rx;
		break;
	}
    }
	return rc;
}

static long CoreFreqK_Register_ClockSource(unsigned int cpu)
{
	long rc = -EINVAL;
    if (Register_ClockSource == 1)
    {
	unsigned long long Freq_Hz;
	unsigned int Freq_KHz;

	if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
	||  (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
	{
		CoreFreqK_CS.read = CoreFreqK_Read_CS_From_Invariant_TSC;
	}
	else
	{
		CoreFreqK_CS.read = CoreFreqK_Read_CS_From_Variant_TSC;
	}

	Freq_Hz = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
		* PUBLIC(RO(Core, AT(cpu)))->Clock.Hz;
	Freq_KHz = Freq_Hz / 1000U;
	if (Freq_KHz != 0)
	{
		int rx = clocksource_register_khz(&CoreFreqK_CS, Freq_KHz);
	    switch (rx) {
	    default:
		fallthrough;
	    case -EBUSY:
		PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
		rc = (long) rx;
		break;
	    case 0:
		PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_ENABLE;
		rc = RC_SUCCESS;

		pr_debug("%s: Freq_KHz[%u] Kernel CPU_KHZ[%u] TSC_KHZ[%u]\n" \
			"LPJ[%lu] mask[%llx] mult[%u] shift[%u]\n",
		CoreFreqK_CS.name, Freq_KHz, cpu_khz, tsc_khz, loops_per_jiffy,
		CoreFreqK_CS.mask, CoreFreqK_CS.mult, CoreFreqK_CS.shift);

		break;
	    }
	}
    } else {
		PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
    }
	return rc;
}

void VendorFromCPUID(	char *pVendorID, unsigned int *pLargestFunc,
			unsigned int *pCRC, enum HYPERVISOR *pHypervisor,
			unsigned long leaf, unsigned long subLeaf )
{
    struct {
		char			*vendorID;
		size_t			vendorLen;
		enum CRC_MANUFACTURER	mfrCRC;
		enum HYPERVISOR 	hypervisor;
    } mfrTbl[] = {
	{VENDOR_INTEL ,__builtin_strlen(VENDOR_INTEL) ,CRC_INTEL ,  BARE_METAL},
	{VENDOR_AMD   ,__builtin_strlen(VENDOR_AMD)   ,CRC_AMD   ,  BARE_METAL},
	{VENDOR_HYGON ,__builtin_strlen(VENDOR_HYGON) ,CRC_HYGON ,  BARE_METAL},
	{VENDOR_KVM   ,__builtin_strlen(VENDOR_KVM)   ,CRC_KVM   ,  HYPERV_KVM},
	{VENDOR_VBOX  ,__builtin_strlen(VENDOR_VBOX)  ,CRC_VBOX  , HYPERV_VBOX},
	{VENDOR_KBOX  ,__builtin_strlen(VENDOR_KBOX)  ,CRC_KBOX  , HYPERV_KBOX},
      {VENDOR_VMWARE ,__builtin_strlen(VENDOR_VMWARE),CRC_VMWARE,HYPERV_VMWARE},
      {VENDOR_HYPERV ,__builtin_strlen(VENDOR_HYPERV),CRC_HYPERV,HYPERV_HYPERV}
    };
	unsigned int eax = 0x0, ebx = 0x0, ecx = 0x0, edx = 0x0; /*DWORD Only!*/

	__asm__ volatile
	(
		"movq	%4, %%rax	\n\t"
		"movq	%5, %%rcx	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (eax),
		  "=r" (ebx),
		  "=r" (ecx),
		  "=r" (edx)
		: "ir" (leaf),
		  "ir" (subLeaf)
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
	pVendorID[ 0] = ebx;
	pVendorID[ 1] = (ebx >> 8);
	pVendorID[ 2] = (ebx >> 16);
	pVendorID[ 3] = (ebx >> 24);
	pVendorID[ 4] = edx;
	pVendorID[ 5] = (edx >> 8);
	pVendorID[ 6] = (edx >> 16);
	pVendorID[ 7] = (edx >> 24);
	pVendorID[ 8] = ecx;
	pVendorID[ 9] = (ecx >> 8);
	pVendorID[10] = (ecx >> 16);
	pVendorID[11] = (ecx >> 24);
	pVendorID[12] = '\0';

	(*pLargestFunc) = eax;

    for (eax = 0; eax < sizeof(mfrTbl) / sizeof(mfrTbl[0]); eax++) {
	if (!strncmp(pVendorID, mfrTbl[eax].vendorID, mfrTbl[eax].vendorLen))
	{
		(*pCRC) = mfrTbl[eax].mfrCRC;
		(*pHypervisor) = mfrTbl[eax].hypervisor;

		return;
	}
    }
}

signed int SearchArchitectureID(void)
{
	signed int id;
    for (id = ARCHITECTURES - 1; id > 0; id--)
    {	/* Search for an architecture signature. */
	if ( (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily \
		== Arch[id].Signature.ExtFamily)
	&& (PUBLIC(RO(Proc))->Features.Std.EAX.Family \
		== Arch[id].Signature.Family)
	&& ( ( (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel \
			==  Arch[id].Signature.ExtModel)
		&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model \
			==  Arch[id].Signature.Model) )
		|| (!Arch[id].Signature.ExtModel \
		&& !Arch[id].Signature.Model) ) )
	{
		break;
	}
    }
	return id;
}

void BrandCleanup(char *pBrand, char inOrder[])
{
	unsigned long ix, jx;
	for (jx = 0; jx < BRAND_LENGTH; jx++) {
		if (inOrder[jx] != 0x20) {
			break;
		}
	}
	for (ix = 0; jx < BRAND_LENGTH; jx++) {
		if (!(inOrder[jx] == 0x20 && inOrder[jx + 1] == 0x20)) {
			pBrand[ix++] = inOrder[jx];
		}
	}
}

void BrandFromCPUID(char *buffer)
{
	BRAND Brand;
	unsigned long ix;
	unsigned int jx , px = 0;

	for (ix = 0; ix < 3; ix++)
	{	__asm__ volatile
		(
			"movq	%4,    %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r"  (Brand.AX),
			  "=r"  (Brand.BX),
			  "=r"  (Brand.CX),
			  "=r"  (Brand.DX)
			: "r"   (0x80000002LU + ix)
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		for (jx = 0; jx < 4; jx++, px++) {
			buffer[px     ] = Brand.AX.Chr[jx];
			buffer[px +  4] = Brand.BX.Chr[jx];
			buffer[px +  8] = Brand.CX.Chr[jx];
			buffer[px + 12] = Brand.DX.Chr[jx];
		}
		px += BRAND_PART;
	}
}

unsigned int Intel_Brand(char *pBrand, char buffer[])
{
	unsigned int ix, frequency = 0, multiplier = 0;

	BrandFromCPUID(buffer);

	for (ix = 0; ix < (BRAND_LENGTH - 2); ix++)
	{
		if ((buffer[ix + 1] == 'H') && (buffer[ix + 2] == 'z')) {
			switch (buffer[ix]) {
			case 'M':
					multiplier = 1;
				break;
			case 'G':
					multiplier = 1000;
				break;
			case 'T':
					multiplier = 1000000;
				break;
			}
			break;
		}
	}
	if (multiplier > 0)
	{
	    if (buffer[ix - 3] == '.') {
		frequency  = (int) (buffer[ix - 4] - '0') * multiplier;
		frequency += (int) (buffer[ix - 2] - '0') * (multiplier / 10);
		frequency += (int) (buffer[ix - 1] - '0') * (multiplier / 100);
	    } else {
		frequency  = (int) (buffer[ix - 4] - '0') * 1000;
		frequency += (int) (buffer[ix - 3] - '0') * 100;
		frequency += (int) (buffer[ix - 2] - '0') * 10;
		frequency += (int) (buffer[ix - 1] - '0');
		frequency *= frequency;
	    }
	}

	BrandCleanup(pBrand, buffer);

	return frequency;
}

/* Retreive the Processor(BSP) features. */
static void Query_Features(void *pArg)
{	/* Must have x86 CPUID 0x0, 0x1, and Intel CPUID 0x4 */
	INIT_ARG *iArg = (INIT_ARG *) pArg;
	unsigned int eax = 0x0, ebx = 0x0, ecx = 0x0, edx = 0x0; /*DWORD Only!*/
	enum HYPERVISOR hypervisor = HYPERV_NONE;

	VendorFromCPUID(iArg->Features->Info.Vendor.ID,
			&iArg->Features->Info.LargestStdFunc,
			&iArg->Features->Info.Vendor.CRC,
			&hypervisor,
			0x0LU, 0x0LU);

	if (hypervisor != BARE_METAL) {
		iArg->rc = -ENXIO;
		return;
	}

	__asm__ volatile
	(
		"movq	$0x1,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->Std.EAX),
		  "=r" (iArg->Features->Std.EBX),
		  "=r" (iArg->Features->Std.ECX),
		  "=r" (iArg->Features->Std.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);

    if (iArg->Features->Info.LargestStdFunc >= 0x5)
    {
	__asm__ volatile
	(
		"movq	$0x5,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->MWait.EAX),
		  "=r" (iArg->Features->MWait.EBX),
		  "=r" (iArg->Features->MWait.ECX),
		  "=r" (iArg->Features->MWait.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
      switch (Override_SubCstate_Depth) {
      case 8:
	iArg->Features->MWait.EDX.SubCstate_MWAIT7 = Override_SubCstate[7];
	fallthrough;
      case 7:
	iArg->Features->MWait.EDX.SubCstate_MWAIT6 = Override_SubCstate[6];
	fallthrough;
      case 6:
	iArg->Features->MWait.EDX.SubCstate_MWAIT5 = Override_SubCstate[5];
	fallthrough;
      case 5:
	iArg->Features->MWait.EDX.SubCstate_MWAIT4 = Override_SubCstate[4];
	fallthrough;
      case 4:
	iArg->Features->MWait.EDX.SubCstate_MWAIT3 = Override_SubCstate[3];
	fallthrough;
      case 3:
	iArg->Features->MWait.EDX.SubCstate_MWAIT2 = Override_SubCstate[2];
	fallthrough;
      case 2:
	iArg->Features->MWait.EDX.SubCstate_MWAIT1 = Override_SubCstate[1];
	fallthrough;
      case 1:
	iArg->Features->MWait.EDX.SubCstate_MWAIT0 = Override_SubCstate[0];
	break;
      };
    }
    if (iArg->Features->Info.LargestStdFunc >= 0x6)
    {
	__asm__ volatile
	(
		"movq	$0x6,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->Power.EAX),
		  "=r" (iArg->Features->Power.EBX),
		  "=r" (iArg->Features->Power.ECX),
		  "=r" (iArg->Features->Power.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
    }
    if (iArg->Features->Info.LargestStdFunc >= 0x7)
    {
	__asm__ volatile
	(
		"movq	$0x7,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx    \n\t"
		"xorq	%%rcx, %%rcx    \n\t"
		"xorq	%%rdx, %%rdx    \n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->ExtFeature.EAX),
		  "=r" (iArg->Features->ExtFeature.EBX),
		  "=r" (iArg->Features->ExtFeature.ECX),
		  "=r" (iArg->Features->ExtFeature.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
	if (iArg->Features->ExtFeature.EAX.MaxSubLeaf >= 1)
	{
		__asm__ volatile
		(
			"movq	$0x7,  %%rax	\n\t"
			"movq	$0x1,  %%rcx    \n\t"
			"xorq	%%rbx, %%rbx    \n\t"
			"xorq	%%rdx, %%rdx    \n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (iArg->Features->ExtFeature_Leaf1.EAX),
			  "=r" (iArg->Features->ExtFeature_Leaf1.EBX),
			  "=r" (iArg->Features->ExtFeature_Leaf1.ECX),
			  "=r" (iArg->Features->ExtFeature_Leaf1.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
	}
    }
	/* Must have 0x80000000,0x80000001,0x80000002,0x80000003,0x80000004 */
	__asm__ volatile
	(
		"movq	$0x80000000, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%eax, %0		\n\t"
		"mov	%%ebx, %1		\n\t"
		"mov	%%ecx, %2		\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->Info.LargestExtFunc),
		  "=r" (ebx),
		  "=r" (ecx),
		  "=r" (edx)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
	__asm__ volatile
	(
		"movq	$0x80000001, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%eax, %0		\n\t"
		"mov	%%ebx, %1		\n\t"
		"mov	%%ecx, %2		\n\t"
		"mov	%%edx, %3"
		: "=r" (eax),
		  "=r" (ebx),
		  "=r" (iArg->Features->ExtInfo.ECX),
		  "=r" (iArg->Features->ExtInfo.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
    if (iArg->Features->Info.LargestExtFunc >= 0x80000007)
    {
	__asm__ volatile
	(
		"movq	$0x80000007, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%eax, %0		\n\t"
		"mov	%%ebx, %1		\n\t"
		"mov	%%ecx, %2		\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->AdvPower.EAX),
		  "=r" (iArg->Features->AdvPower.EBX),
		  "=r" (iArg->Features->AdvPower.ECX),
		  "=r" (iArg->Features->AdvPower.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
    }
    if (iArg->Features->Info.LargestExtFunc >= 0x80000008)
    {
	__asm__ volatile
	(
		"movq	$0x80000008, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%eax, %0		\n\t"
		"mov	%%ebx, %1		\n\t"
		"mov	%%ecx, %2		\n\t"
		"mov	%%edx, %3"
		: "=r" (iArg->Features->leaf80000008.EAX),
		  "=r" (iArg->Features->leaf80000008.EBX),
		  "=r" (iArg->Features->leaf80000008.ECX),
		  "=r" (iArg->Features->leaf80000008.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
    }
	/* Reset the performance features bits (present is zero) */
	iArg->Features->PerfMon.EBX.CoreCycles    = 1;
	iArg->Features->PerfMon.EBX.InstrRetired  = 1;
	iArg->Features->PerfMon.EBX.RefCycles     = 1;
	iArg->Features->PerfMon.EBX.LLC_Ref       = 1;
	iArg->Features->PerfMon.EBX.LLC_Misses    = 1;
	iArg->Features->PerfMon.EBX.BranchRetired = 1;
	iArg->Features->PerfMon.EBX.BranchMispred = 1;
	iArg->Features->PerfMon.EBX.TopdownSlots  = 1;

	/* Per Vendor features */
    if (iArg->Features->Info.Vendor.CRC == CRC_INTEL)
    {
	__asm__ volatile
	(
		"movq	$0x4,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (eax),
		  "=r" (ebx),
		  "=r" (ecx),
		  "=r" (edx)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
	iArg->SMT_Count = (eax >> 26) & 0x3f;
	iArg->SMT_Count++;

	if (iArg->Features->Info.LargestStdFunc >= 0xa)
	{
		__asm__ volatile
		(
			"movq	$0xa,  %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (iArg->Features->PerfMon.EAX),
			  "=r" (iArg->Features->PerfMon.EBX),
			  "=r" (iArg->Features->PerfMon.ECX),
			  "=r" (iArg->Features->PerfMon.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
	}
	/*	Extract the factory frequency from the brand string.	*/
	iArg->Features->Factory.Freq = Intel_Brand( iArg->Features->Info.Brand,
							iArg->Brand );

    } else if ( (iArg->Features->Info.Vendor.CRC == CRC_AMD)
	||	(iArg->Features->Info.Vendor.CRC == CRC_HYGON) )
    {	/*	Specified as Core Performance 64 bits General Counters. */
	iArg->Features->PerfMon.EAX.MonWidth = 64;

	if (iArg->Features->ExtInfo.ECX.PerfCore)
	{
		iArg->Features->PerfMon.EAX.MonCtrs = 6;
	} else {
		iArg->Features->PerfMon.EAX.MonCtrs = 4;
	}
	/* Fix the Performance Counters. Use Intel bits as AMD placeholder */
	iArg->Features->PerfMon.EDX.FixWidth = 64;

	if	( iArg->Features->Power.ECX.HCF_Cap
		| iArg->Features->AdvPower.EDX.EffFrqRO )
	{
		iArg->Features->PerfMon.EBX.CoreCycles = 0;
		iArg->Features->PerfMon.EBX.RefCycles  = 0;
		iArg->Features->PerfMon.EDX.FixCtrs += 2;
	}
	if (iArg->Features->ExtInfo.ECX.PerfLLC)
	{ /* PerfCtrExtLLC: Last Level Cache performance counter extensions */
		iArg->Features->PerfMon.EBX.LLC_Ref = 0;
	}
	if (iArg->Features->Info.LargestExtFunc >= 0x80000008)
	{
		iArg->SMT_Count = iArg->Features->leaf80000008.ECX.NC + 1;
		/* Add the Retired Instructions Perf Counter to the Fixed set */
		if (iArg->Features->leaf80000008.EBX.IRPerf)
		{
			iArg->Features->PerfMon.EBX.InstrRetired = 0;
			iArg->Features->PerfMon.EDX.FixCtrs++;
		}
	}
	else if (iArg->Features->Std.EDX.HTT)
	{
		iArg->SMT_Count = iArg->Features->Std.EBX.Max_SMT_ID;
	} else {
		iArg->SMT_Count = 1;
	}
	BrandFromCPUID(iArg->Brand);
	BrandCleanup(iArg->Features->Info.Brand, iArg->Brand);
    }
}

void Compute_Interval(void)
{
	if ( (SleepInterval >= LOOP_MIN_MS)
	  && (SleepInterval <= LOOP_MAX_MS))
	{
		PUBLIC(RO(Proc))->SleepInterval = SleepInterval;
	} else {
		PUBLIC(RO(Proc))->SleepInterval = LOOP_DEF_MS;
	}
	/*		Compute the tick steps .			*/
	PUBLIC(RO(Proc))->tickReset = \
		(  (TickInterval >= PUBLIC(RO(Proc))->SleepInterval)
		&& (TickInterval <= LOOP_MAX_MS) ) ?
			TickInterval
		:	KMAX(TICK_DEF_MS, PUBLIC(RO(Proc))->SleepInterval);

	PUBLIC(RO(Proc))->tickReset /= PUBLIC(RO(Proc))->SleepInterval;
	PUBLIC(RO(Proc))->tickStep = PUBLIC(RO(Proc))->tickReset;

	RearmTheTimer = ktime_set( 0,	PUBLIC(RO(Proc))->SleepInterval
					* 1000000LU );
}

#ifdef CONFIG_SMP
	#define THIS_LPJ	this_cpu_read(cpu_info.loops_per_jiffy)
#else
	#define THIS_LPJ	loops_per_jiffy
#endif

#define COMPUTE_LPJ(BCLK_Hz, COF)	( (BCLK_Hz * COF) / HZ )

#if defined(DELAY_TSC) && (DELAY_TSC == 1)
FEAT_MSG("udelay() built with TSC implementation")
#define CLOCK_TSC( CYCLES, _TIMER, CTR )				\
({									\
	__asm__ volatile						\
	(								\
		ASM_RD##_TIMER(r14)					\
		"addq	%[cycles], %%rax"		"\n\t"		\
		"movq	%%rax	, %%r15"		"\n\t"		\
	"1:"						"\n\t"		\
		ASM_RD##_TIMER(r13)					\
		"cmpq	%%r15	, %%r13"		"\n\t"		\
		"jc	1b"				"\n\t"		\
		"movq	%%r14	, %[ctr0]"		"\n\t"		\
		"movq	%%r13	, %[ctr1]"				\
		: [ctr0] "=m"	(CTR[0]),				\
		  [ctr1] "=m"	(CTR[1])				\
		: [cycles] "ir" (CYCLES)				\
		: "%rax", "%rbx", "%rcx", "%rdx",			\
		  "%r13", "%r14", "%r15",				\
		  "cc", "memory"					\
	);								\
})

#define CLOCK2CYCLE(INTERVAL_NS) ((INTERVAL_NS * THIS_LPJ * HZ) / 1000000LLU)

#define CLOCK_DELAY(INTERVAL_NS, _TIMER, CTR)				\
({									\
	CLOCK_TSC( CLOCK2CYCLE(INTERVAL_NS), _TIMER, CTR );		\
})

#define CLOCK_OVERHEAD(_TIMER, CTR)	CLOCK_TSC( 1LLU, _TIMER, CTR )

#else

#define CLOCK_DELAY(INTERVAL_NS, _TIMER, CTR)				\
({									\
	RD##_TIMER##64(CTR[0]);						\
	udelay(INTERVAL_NS);						\
	RD##_TIMER##64(CTR[1]);						\
})

#define CLOCK_OVERHEAD(_TIMER, CTR)	CLOCK_DELAY( 0LLU, _TIMER, CTR )

#endif

static void ComputeWithSerializedTSC(COMPUTE_ARG *pCompute)
{
	unsigned int loop;
	/*		Writeback and Invalidate Caches.		*/
	WBINVD();
	/*		Warm-up & Overhead				*/
	for (loop = 0; loop < OCCURRENCES; loop++)
	{
		CLOCK_OVERHEAD(TSCP, pCompute->TSC[0][loop].V);
	}
	/*		Estimation					*/
	for (loop = 0; loop < OCCURRENCES; loop++)
	{
		CLOCK_DELAY(1000LLU, TSCP, pCompute->TSC[1][loop].V);
	}
}

static void ComputeWithUnSerializedTSC(COMPUTE_ARG *pCompute)
{
	unsigned int loop;
	/*		Writeback and Invalidate Caches.		*/
	WBINVD();
	/*		Warm-up & Overhead				*/
	for (loop = 0; loop < OCCURRENCES; loop++)
	{
		CLOCK_OVERHEAD(TSC, pCompute->TSC[0][loop].V);
	}
	/*		Estimation					*/
	for (loop = 0; loop < OCCURRENCES; loop++)
	{
		CLOCK_DELAY(1000LLU, TSC, pCompute->TSC[1][loop].V);
	}
}

static void Compute_TSC(void *arg)
{
	COMPUTE_ARG *pCompute = (COMPUTE_ARG *) arg;
	unsigned long long D[2][OCCURRENCES];
	unsigned int ratio = (unsigned int) pCompute->Clock.Q;
	unsigned int loop = 0, what = 0, best[2] = {0, 0}, top[2] = {0, 0};
/*
	TSC[0] stores the overhead
	TSC[1] stores the estimation
*/
	/*	Is the TSC invariant or can serialize  ?		*/
	if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
	||  (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
	{
		ComputeWithSerializedTSC(pCompute);
	} else {
		ComputeWithUnSerializedTSC(pCompute);
	}
	/*		Select the best clock.				*/
	memset(D, 0, 2 * OCCURRENCES);
	for (loop = 0; loop < OCCURRENCES; loop++)
		for (what = 0; what < 2; what++) {
			D[what][loop]	= pCompute->TSC[what][loop].V[1]
					- pCompute->TSC[what][loop].V[0];
		}
	for (loop = 0; loop < OCCURRENCES; loop++) {
		unsigned int inner = 0, count[2] = {0, 0};
		for (inner = loop; inner < OCCURRENCES; inner++) {
			for (what = 0; what < 2; what++) {
				if (D[what][loop] == D[what][inner])
					count[what]++;
			}
		}
		for (what = 0; what < 2; what++) {
		    if ((count[what] > top[what])
			|| ((count[what] == top[what])
			&& (D[what][loop] < D[what][best[what]]))) {

				top[what]  = count[what];
				best[what] = loop;

		    }
		}
	}
	/*		Substract the overhead .			*/
	D[1][best[1]] -= D[0][best[0]];
	/*		Compute the Base Clock .			*/
	REL_BCLK(pCompute->Clock, ratio, D[1][best[1]], 1LLU);
}

CLOCK Compute_Clock(unsigned int cpu, COMPUTE_ARG *pCompute)
{
/*	Synchronously call the Base Clock estimation on a pinned CPU.
 * 1/ Preemption is disabled by smp_call_function_single() > get_cpu()
 * 2/ IRQ are suspended by generic_exec_single(func) > local_irq_save()
 * 3/ Function 'func' is executed
 * 4/ IRQ are resumed > local_irq_restore()
 * 5/ Preemption is enabled > put_cpu()
 */
	smp_call_function_single(cpu, Compute_TSC, pCompute, 1);

	return pCompute->Clock;
}

void ClockToHz(CLOCK *clock)
{
	clock->Hz  = clock->Q * 1000000L;
	clock->Hz += clock->R * PRECISION;
}

/* [Genuine Intel] */
CLOCK BaseClock_GenuineIntel(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};

	if ((PUBLIC(RO(Proc))->Features.Factory.Freq > 0) && (ratio > 0))
	{
		clock.Hz=(PUBLIC(RO(Proc))->Features.Factory.Freq * 1000000L)
			/ ratio;

		clock.Q = PUBLIC(RO(Proc))->Features.Factory.Freq / ratio;

		clock.R = (PUBLIC(RO(Proc))->Features.Factory.Freq % ratio)
			* PRECISION;
	}
	return clock;
};

/* [Authentic AMD] */
CLOCK BaseClock_AuthenticAMD(unsigned int ratio)
{	/* For AMD Families 0Fh, 10h up to 16h */
	CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};
	UNUSED(ratio);
	return clock;
};

/* [Core] */
CLOCK BaseClock_Core(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};
	FSB_FREQ FSB = {.value = 0};

	RDMSR(FSB, MSR_FSB_FREQ);
	switch (FSB.Bus_Speed) {
	case 0b101: {
		clock.Q = 100;
		clock.R = 0;
		};
		break;
	case 0b001: {
		clock.Q = 133;
		clock.R = 3333;
		}
		break;
	case 0b011: {
		clock.Q = 166;
		clock.R = 6666;
		}
		break;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Core2] */
CLOCK BaseClock_Core2(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};
	FSB_FREQ FSB = {.value = 0};

	RDMSR(FSB, MSR_FSB_FREQ);
	switch (FSB.Bus_Speed) {
	case 0b101: {
		clock.Q = 100;
		clock.R = 0;
		}
		break;
	case 0b001: {
		clock.Q = 133;
		clock.R = 3333;
		}
		break;
	case 0b011: {
		clock.Q = 166;
		clock.R = 6666;
		}
		break;
	case 0b010: {
		clock.Q = 200;
		clock.R = 0;
		}
		break;
	case 0b000: {
		clock.Q = 266;
		clock.R = 6666;
		}
		break;
	case 0b100: {
		clock.Q = 333;
		clock.R = 3333;
		}
		break;
	case 0b110: {
		clock.Q = 400;
		clock.R = 0;
		}
		break;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Atom] */
CLOCK BaseClock_Atom(unsigned int ratio)
{
	CLOCK clock = {.Q = 83, .R = 0};
	FSB_FREQ FSB = {.value = 0};

	RDMSR(FSB, MSR_FSB_FREQ);
	switch (FSB.Bus_Speed) {
	case 0b111: {
		clock.Q = 83;
		clock.R = 2000;
		}
		break;
	case 0b101: {
		clock.Q = 99;
		clock.R = 8400;
		}
		break;
	case 0b001: {
		clock.Q = 133;
		clock.R = 2000;
		}
		break;
	case 0b011: {
		clock.Q = 166;
		clock.R = 4000;
		}
		break;
	default: {
		clock.Q = 83;
		clock.R = 2000;
		}
		break;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Airmont] */
CLOCK BaseClock_Airmont(unsigned int ratio)
{
	CLOCK clock = {.Q = 87, .R = 5};
	FSB_FREQ FSB = {.value = 0};

	RDMSR(FSB, MSR_FSB_FREQ);
	switch (FSB.Bus_Speed) {
	case 0b000: {
		clock.Q = 83;
		clock.R = 3333;
		}
		break;
	case 0b001: {
		clock.Q = 100;
		clock.R = 0000;
		}
		break;
	case 0b010: {
		clock.Q = 133;
		clock.R = 3333;
		}
		break;
	case 0b011: {
		clock.Q = 116;
		clock.R = 6666;
		}
		break;
	case 0b100: {
		clock.Q = 80;
		clock.R = 0000;
		}
		break;
	case 0b101: {
		clock.Q = 93;
		clock.R = 3333;
		}
		break;
	case 0b110: {
		clock.Q = 90;
		clock.R = 0000;
		}
		break;
	case 0b111: {
		clock.Q = 88;
		clock.R = 9000;
		}
		break;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Silvermont] */
CLOCK BaseClock_Silvermont(unsigned int ratio)
{
	CLOCK clock = {.Q = 83, .R = 3};
	FSB_FREQ FSB = {.value = 0};

	RDMSR(FSB, MSR_FSB_FREQ);
	switch (FSB.Bus_Speed)
	{
	case 0b100: {
			clock.Q = 80;
			clock.R = 0;
		}
		break;
	case 0b000: {
			clock.Q = 83;
			clock.R = 3000;
		}
		break;
	case 0b001: {
			clock.Q = 100;
			clock.R = 0;
		}
		break;
	case 0b010: {
			clock.Q = 133;
			clock.R = 3333;
		}
		break;
	case 0b011: {
			clock.Q = 116;
			clock.R = 7000;
		}
		break;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Nehalem] */
CLOCK BaseClock_Nehalem(unsigned int ratio)
{
	CLOCK clock = {.Q = 133, .R = 3333};
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Westmere] */
CLOCK BaseClock_Westmere(unsigned int ratio)
{
	CLOCK clock = {.Q = 133, .R = 3333};
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [SandyBridge] */
CLOCK BaseClock_SandyBridge(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [IvyBridge] */
CLOCK BaseClock_IvyBridge(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Haswell] */
CLOCK BaseClock_Haswell(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

/* [Skylake] */
CLOCK BaseClock_Skylake(unsigned int ratio)
{
	CLOCK clock = {.Q = 100, .R = 0};

	if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x16) {
		unsigned int eax = 0x0, ebx = 0x0, edx = 0x0, fsb = 0;
		__asm__ volatile
		(
			"movq	$0x16, %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (eax),
			  "=r" (ebx),
			  "=r" (fsb),
			  "=r" (edx)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		if (fsb > 0)
			clock.Q = fsb;
		else
			clock.Q = 100;
	}
	ClockToHz(&clock);
	clock.R *= ratio;
	return clock;
};

CLOCK BaseClock_AMD_Family_17h(unsigned int ratio)
{	/* Source: AMD PPR Family 17h § 1.4/ Table 11: REFCLK = 100 MHz */
	CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};
	UNUSED(ratio);
	return clock;
};

void Define_CPUID(CORE_RO *Core, const CPUID_STRUCT CpuIDforVendor[])
{	/*	Per vendor, define a CPUID dump table to query .	*/
	enum CPUID_ENUM i;
	for (i = 0; i < CPUID_MAX_FUNC; i++) {
		Core->CpuID[i].func = CpuIDforVendor[i].func;
		Core->CpuID[i].sub  = CpuIDforVendor[i].sub;
	}
}

void Cache_Topology(CORE_RO *Core)
{
	unsigned long level = 0x0;
	if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
	    for (level = 0; level < CACHE_MAX_LEVEL; level++) {
		__asm__ volatile
		(
			"movq	$0x4,  %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"movq	%4,    %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (Core->T.Cache[level].AX),
			  "=r" (Core->T.Cache[level].BX),
			  "=r" (Core->T.Cache[level].Set),
			  "=r" (Core->T.Cache[level].DX)
			: "r" (level)
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		if (!Core->T.Cache[level].Type)
			break;
	    }
	}
	else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
	{
	    struct CACHE_INFO CacheInfo; /* Employ same Intel algorithm. */

	    if (PUBLIC(RO(Proc))->Features.Info.LargestExtFunc >= 0x80000005)
	    {
		Core->T.Cache[0].Level = 1;
		Core->T.Cache[0].Type  = 2;		/* Inst. */
		Core->T.Cache[1].Level = 1;
		Core->T.Cache[1].Type  = 1;		/* Data */

		/*	Fn8000_0005 L1 Data and Inst. caches.		*/
		__asm__ volatile
		(
			"movq	$0x80000005, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"xorq	%%rcx, %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (CacheInfo.AX),
			  "=r" (CacheInfo.BX),
			  "=r" (CacheInfo.CX),
			  "=r" (CacheInfo.DX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		/* L1 Inst. */
		Core->T.Cache[0].Way  = CacheInfo.CPUID_0x80000005_L1I.Assoc;
		Core->T.Cache[0].Size = CacheInfo.CPUID_0x80000005_L1I.Size;
		/* L1 Data */
		Core->T.Cache[1].Way  = CacheInfo.CPUID_0x80000005_L1D.Assoc;
		Core->T.Cache[1].Size = CacheInfo.CPUID_0x80000005_L1D.Size;
	    }
	    if (PUBLIC(RO(Proc))->Features.Info.LargestExtFunc >= 0x80000006)
	    {
		Core->T.Cache[2].Level = 2;
		Core->T.Cache[2].Type  = 3;		/* Unified! */
		Core->T.Cache[3].Level = 3;
		Core->T.Cache[3].Type  = 3;

		/*	Fn8000_0006 L2 and L3 caches.			*/
		__asm__ volatile
		(
			"movq	$0x80000006, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"xorq	%%rcx, %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (CacheInfo.AX),
			  "=r" (CacheInfo.BX),
			  "=r" (CacheInfo.CX),
			  "=r" (CacheInfo.DX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		/* L2 */
		Core->T.Cache[2].Way  = CacheInfo.CPUID_0x80000006_L2.Assoc;
		Core->T.Cache[2].Size = CacheInfo.CPUID_0x80000006_L2.Size;
		/* L3 */
		Core->T.Cache[3].Way  = CacheInfo.CPUID_0x80000006_L3.Assoc;
		Core->T.Cache[3].Size = CacheInfo.CPUID_0x80000006_L3.Size;
	    }
	}
}

unsigned int L3_SubCache_AMD_Piledriver(unsigned int bits)
{	/* Return the AMD Piledriver L3 Sub-Cache size in unit of 512 KB  */
	switch (bits) {
	case 0xc:
		return 4;
	case 0xd:
	case 0xe:
		return 2;
	default:
		return 0;
	}
}

static void Map_AMD_Topology(void *arg)
{
    if (arg != NULL)
    {
	CORE_RO *Core = (CORE_RO *) arg;

	struct CPUID_0x00000001_EBX leaf1_ebx = {0};

	CPUID_0x80000008 leaf80000008 = {
		.EAX = {0}, .EBX = {0}, .ECX = {0}, .EDX = {0}
	};

	Cache_Topology(Core);

	RDMSR(Core->T.Base, MSR_IA32_APICBASE);

	__asm__ volatile
	(
		"movq	$0x1,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%ebx, %0"
		: "=r" (leaf1_ebx)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);

	__asm__ volatile
	(
		"movq	$0x80000008, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%ecx, %0"
		: "=r" (leaf80000008.ECX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);

	switch (PUBLIC(RO(Proc))->ArchID) {
	default:
	case AMD_Family_0Fh:	/* Legacy processor. */
		Core->T.ApicID    = leaf1_ebx.Init_APIC_ID;
		Core->T.CoreID    = leaf1_ebx.Init_APIC_ID;
		Core->T.PackageID = 0;
		break;
	case AMD_Family_10h:
	    {
		CPUID_0x80000001 leaf80000001 = {
			.EAX = {0}, .EBX = {0}, .ECX = {{0}}, .EDX = {{0}}
		};

		__asm__ volatile
		(
			"movq	$0x80000001, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"xorq	%%rcx, %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (leaf80000001.EAX),
			  "=r" (leaf80000001.EBX),
			  "=r" (leaf80000001.ECX),
			  "=r" (leaf80000001.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		Core->T.Cluster.Node = leaf80000001.ECX.NodeId;
	    }
		fallthrough;
	case AMD_Family_11h:
	case AMD_Family_12h:
	case AMD_Family_14h:
		Core->T.ApicID    = leaf1_ebx.Init_APIC_ID;
		Core->T.CoreID    = leaf1_ebx.Init_APIC_ID;
		Core->T.PackageID = leaf1_ebx.Init_APIC_ID
				  >> leaf80000008.ECX.ApicIdCoreIdSize;
		break;
	case AMD_Family_15h:
	    if ((PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel == 0x0)
	     && (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
	     && (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf))
	    {
		L3_CACHE_PARAMETER L3;
		RDPCI(L3, PCI_AMD_L3_CACHE_PARAMETER);

	    Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize0);
	    Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize1);
	    Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize2);
	    Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize3);
	    }
		fallthrough;
	case AMD_Family_16h:
		Core->T.ApicID    = leaf1_ebx.Init_APIC_ID;
		Core->T.PackageID = leaf1_ebx.Init_APIC_ID
				  >> leaf80000008.ECX.ApicIdCoreIdSize;
		Core->T.CoreID    = leaf1_ebx.Init_APIC_ID
				  - (Core->T.PackageID
					<< leaf80000008.ECX.ApicIdCoreIdSize);

	    if (PUBLIC(RO(Proc))->Features.ExtInfo.ECX.ExtApicId == 1)
	    {
		CPUID_0x8000001e leaf8000001e;

		__asm__ volatile
		(
			"movq	$0x8000001e, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"xorq	%%rcx, %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (leaf8000001e.EAX),
			  "=r" (leaf8000001e.EBX),
			  "=r" (leaf8000001e.ECX),
			  "=r" (leaf8000001e.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		/*	TODO(Case of leaf8000001e.EAX.ExtApicId)	*/
		Core->T.Cluster.Node= leaf8000001e.ECX.NodeId;
		Core->T.Cluster.CMP = leaf8000001e.EBX.CompUnitId;
	    }
	    break;
	case AMD_Zen:
	case AMD_Zen_APU:
	case AMD_ZenPlus:
	case AMD_ZenPlus_APU:
	case AMD_Zen_Dali:
	case AMD_EPYC_Rome_CPK:
	case AMD_Zen2_Renoir:
	case AMD_Zen2_LCN:
	case AMD_Zen2_MTS:
	case AMD_Zen2_Ariel:
	case AMD_Zen3_VMR:
	case AMD_Zen3_CZN:
	case AMD_EPYC_Milan:
	case AMD_Zen3_Chagall:
	case AMD_Family_17h:
	case AMD_Family_18h:
	case AMD_Family_19h:
	    if (PUBLIC(RO(Proc))->Features.ExtInfo.ECX.ExtApicId == 1)
	    {
		struct CACHE_INFO CacheInfo = {
			.AX = 0, .BX = 0, .CX = 0, .DX = 0, .Size = 0
		};
		CPUID_0x8000001e leaf8000001e = {
			.EAX = {0}, .EBX = {{0}}, .ECX = {0}, .EDX = {0}
		};
		/*	Fn8000_001D Cache Properties.			*/
		unsigned long idx, level[CACHE_MAX_LEVEL] = {1, 0, 2, 3};
		/*	Skip one CDD on two with Threadripper.		*/
		unsigned int factor = 0;

		for (idx = 0; idx < CACHE_MAX_LEVEL; idx++ ) {
		    __asm__ volatile
		    (
			"movq	$0x8000001d, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"movq	%4,    %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (CacheInfo.AX),
			  "=r" (CacheInfo.BX),
			  "=r" (CacheInfo.CX),
			  "=r" (CacheInfo.DX)
			: "r" (idx)
			: "%rax", "%rbx", "%rcx", "%rdx"
		    );
			Core->T.Cache[level[idx]].WrBack = CacheInfo.WrBack;
			Core->T.Cache[level[idx]].Inclus = CacheInfo.Inclus;
		}
		/*	Fn8000_001E {ExtApic, Core, Node} Identifiers.	*/
		__asm__ volatile
		(
			"movq	$0x8000001e, %%rax	\n\t"
			"xorq	%%rbx, %%rbx		\n\t"
			"xorq	%%rcx, %%rcx		\n\t"
			"xorq	%%rdx, %%rdx		\n\t"
			"cpuid				\n\t"
			"mov	%%eax, %0		\n\t"
			"mov	%%ebx, %1		\n\t"
			"mov	%%ecx, %2		\n\t"
			"mov	%%edx, %3"
			: "=r" (leaf8000001e.EAX),
			  "=r" (leaf8000001e.EBX),
			  "=r" (leaf8000001e.ECX),
			  "=r" (leaf8000001e.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		Core->T.ApicID    = leaf8000001e.EAX.ExtApicId;
		Core->T.CoreID    = leaf8000001e.EBX.CoreId;
		Core->T.PackageID = leaf8000001e.ECX.NodeId;

	      if (leaf8000001e.EBX.ThreadsPerCore > 0)
	      { 	/*		SMT is enabled .		*/
			Core->T.ThreadID  = leaf8000001e.EAX.ExtApicId & 1;

		/* CCD factor for [x24 ... x128] SMT Core Threadripper	*/
		factor	=  (leaf80000008.ECX.NC == 0x7f)
			|| (leaf80000008.ECX.NC == 0x3f)
			|| (leaf80000008.ECX.NC == 0x2f)

			|| ((leaf80000008.ECX.NC == 0x1f)
			 && ((PUBLIC(RO(Proc))->ArchID == AMD_EPYC_Rome_CPK)))

			|| ((leaf80000008.ECX.NC == 0x17)
			 && ((PUBLIC(RO(Proc))->ArchID == AMD_EPYC_Rome_CPK)
			  || (PUBLIC(RO(Proc))->ArchID == AMD_Zen3_Chagall)));
	      }
	      else
	      { 	/*		SMT is disabled.		*/
			Core->T.ThreadID  = 0;

		/* CCD factor for [x12 ... x64] physical Core Threadripper */
		factor	=  (leaf80000008.ECX.NC == 0x3f)
			|| (leaf80000008.ECX.NC == 0x1f)
			|| (leaf80000008.ECX.NC == 0x17)

			|| ((leaf80000008.ECX.NC == 0x0f)
			 && ((PUBLIC(RO(Proc))->ArchID == AMD_EPYC_Rome_CPK)))

			|| ((leaf80000008.ECX.NC == 0x0b)
			 && ((PUBLIC(RO(Proc))->ArchID == AMD_EPYC_Rome_CPK)
			  || (PUBLIC(RO(Proc))->ArchID == AMD_Zen3_Chagall)));
	      }

		Core->T.Cluster.Node=leaf8000001e.ECX.NodeId;
		Core->T.Cluster.CCD = (Core->T.CoreID >> 3) << factor;
		Core->T.Cluster.CCX = Core->T.CoreID >> 2;
	    } else {	/*	Fallback algorithm.			*/
		Core->T.ApicID    = leaf1_ebx.Init_APIC_ID;
		Core->T.PackageID = leaf1_ebx.Init_APIC_ID
				  >> leaf80000008.ECX.ApicIdCoreIdSize;
		Core->T.CoreID    = leaf1_ebx.Init_APIC_ID
				  - (Core->T.PackageID
					<< leaf80000008.ECX.ApicIdCoreIdSize);
		Core->T.ThreadID  = 0;
	    }
	    break;
	}
    }
}

/*
 Enumerate the topology of Processors, Cores and Threads
 Remark: Early single-core processors are not processed.
 Sources: Intel Software Developer's Manual vol 3A §8.9 /
	  Intel whitepaper: Detecting Hyper-Threading Technology /
*/
unsigned short FindMaskWidth(unsigned short maxCount)
{
	unsigned short maskWidth = 0, count = (maxCount - 1);

	if (BITBSR(count, maskWidth) == 0)
		maskWidth++;

	return maskWidth;
}

static void Map_Intel_Topology(void *arg)
{
    if (arg != NULL)
    {
	CORE_RO *Core = (CORE_RO *) arg;
	unsigned short	SMT_Mask_Width, CORE_Mask_Width,
			SMT_Select_Mask, CORE_Select_Mask, PKG_Select_Mask;

	struct CPUID_0x00000001_EBX leaf1_ebx;

	struct
	{	/* CPUID 4 */
		unsigned int
		Type		:  5-0,
		Level		:  8-5,
		Init		:  9-8,
		Assoc		: 10-9,
		Unused		: 14-10,
		Cache_SMT_ID	: 26-14,
		Max_Core_ID	: 32-26;
	} leaf4_eax;

	RDMSR(Core->T.Base, MSR_IA32_APICBASE);

	__asm__ volatile
	(
		"movq	$0x1,  %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%ebx, %0"
		: "=r" (leaf1_ebx)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);

	if (PUBLIC(RO(Proc))->Features.Std.EDX.HTT) {
		SMT_Mask_Width = leaf1_ebx.Max_SMT_ID;

		__asm__ volatile
		(
			"movq	$0x4,  %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0"
			: "=r" (leaf4_eax)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);

		CORE_Mask_Width = leaf4_eax.Max_Core_ID + 1;
	} else {
		SMT_Mask_Width = 0;
		CORE_Mask_Width = 1;
	}

	if (CORE_Mask_Width != 0)
	   SMT_Mask_Width = FindMaskWidth(SMT_Mask_Width) / CORE_Mask_Width;

	SMT_Select_Mask   = ~((-1) << SMT_Mask_Width);

	CORE_Select_Mask  = (~((-1) << (CORE_Mask_Width + SMT_Mask_Width)))
			  ^ SMT_Select_Mask;

	PKG_Select_Mask   = (-1) << (CORE_Mask_Width + SMT_Mask_Width);

	Core->T.ThreadID  = leaf1_ebx.Init_APIC_ID & SMT_Select_Mask;

	Core->T.CoreID    = (leaf1_ebx.Init_APIC_ID & CORE_Select_Mask)
			  >> SMT_Mask_Width;

	Core->T.PackageID = (leaf1_ebx.Init_APIC_ID & PKG_Select_Mask)
			  >> (CORE_Mask_Width + SMT_Mask_Width);

	Core->T.ApicID    = leaf1_ebx.Init_APIC_ID;

	Cache_Topology(Core);
    }
}

static void Map_Intel_Extended_Topology(void *arg)
{
    if (arg != NULL) {
	CORE_RO *Core = (CORE_RO *) arg;

	long	InputLevel = 0;
	int	NoMoreLevels = 0,
		SMT_Mask_Width = 0, SMT_Select_Mask = 0,
		CorePlus_Mask_Width = 0, CoreOnly_Select_Mask = 0,
		Package_Select_Mask = 0;

	CPUID_TOPOLOGY_LEAF ExtTopology = {
		.AX.Register = 0,
		.BX.Register = 0,
		.CX.Register = 0,
		.DX.Register = 0
	};

	RDMSR(Core->T.Base, MSR_IA32_APICBASE);

	do {
		__asm__ volatile
		(
			"movq	$0xb,  %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"movq	%4,    %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (ExtTopology.AX),
			  "=r" (ExtTopology.BX),
			  "=r" (ExtTopology.CX),
			  "=r" (ExtTopology.DX)
			: "r" (InputLevel)
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
		/*	Exit from the loop if the BX register equals 0 or
			if the requested level exceeds the level of a Core. */
		if (!ExtTopology.BX.Register || (InputLevel > LEVEL_CORE))
			NoMoreLevels = 1;
		else {
		    switch (ExtTopology.CX.Type) {
		    case LEVEL_THREAD: {
			SMT_Mask_Width   = ExtTopology.AX.SHRbits;

			SMT_Select_Mask  = ~((-1) << SMT_Mask_Width);

			Core->T.ThreadID = ExtTopology.DX.x2ApicID
					 & SMT_Select_Mask;
			}
			break;
		    case LEVEL_CORE: {
			CorePlus_Mask_Width  = ExtTopology.AX.SHRbits;

			CoreOnly_Select_Mask = (~((-1) << CorePlus_Mask_Width))
					     ^ SMT_Select_Mask;

			Core->T.CoreID	= (ExtTopology.DX.x2ApicID
						& CoreOnly_Select_Mask)
					>> SMT_Mask_Width;

			Package_Select_Mask = (-1) << CorePlus_Mask_Width;

			Core->T.PackageID = (ExtTopology.DX.x2ApicID
						& Package_Select_Mask)
					  >> CorePlus_Mask_Width;
			}
			break;
		    }
		}
		InputLevel++;
	} while (!NoMoreLevels);

	Core->T.ApicID = ExtTopology.DX.x2ApicID;

	Cache_Topology(Core);
    }
}

int Core_Topology(unsigned int cpu)
{
	int rc = smp_call_function_single(cpu,
		( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) ) ?
			Map_AMD_Topology
		: (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0xb) ?
			Map_Intel_Extended_Topology : Map_Intel_Topology,
		PUBLIC(RO(Core, AT(cpu))), 1); /* Synchronous call. */

	if (	!rc
		&& (PUBLIC(RO(Proc))->Features.HTT_Enable == 0)
		&& (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0) )
	{
			PUBLIC(RO(Proc))->Features.HTT_Enable = 1;
	}
	return rc;
}

unsigned int Proc_Topology(void)
{
	unsigned int cpu, CountEnabledCPU = 0;

    for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
	PUBLIC(RO(Core, AT(cpu)))->T.Base.value = 0;
	PUBLIC(RO(Core, AT(cpu)))->T.ApicID     = -1;
	PUBLIC(RO(Core, AT(cpu)))->T.CoreID     = -1;
	PUBLIC(RO(Core, AT(cpu)))->T.ThreadID   = -1;
	PUBLIC(RO(Core, AT(cpu)))->T.PackageID  = -1;
	PUBLIC(RO(Core, AT(cpu)))->T.Cluster.ID = 0;

	BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
	BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);

	if (cpu_present(cpu)) { /*	CPU state probed by the OS.	*/
	    if (Core_Topology(cpu) == 0) {
		/* CPU state based on the hardware. */
		if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID >= 0)
		{
			BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine,HW);

			CountEnabledCPU++;
		}
		BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);
	    }
	}
    }
	return CountEnabledCPU;
}

#define HyperThreading_Technology()					\
(									\
	PUBLIC(RO(Proc))->CPU.OnLine = Proc_Topology()			\
)

void Package_Init_Reset(void)
{
	PUBLIC(RO(Proc))->Features.TgtRatio_Unlock = 1;
	PUBLIC(RO(Proc))->Features.ClkRatio_Unlock = 0;
	PUBLIC(RO(Proc))->Features.TDP_Unlock = 0;
	PUBLIC(RO(Proc))->Features.Turbo_Unlock = 0;
	PUBLIC(RO(Proc))->Features.TurboActiv_Lock = 1;
	PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock = 1;
	PUBLIC(RO(Proc))->Features.Uncore_Unlock = 0;
}

void Default_Unlock_Reset(void)
{
    switch (Target_Ratio_Unlock) {
    case COREFREQ_TOGGLE_OFF:
    case COREFREQ_TOGGLE_ON:
	PUBLIC(RO(Proc))->Features.TgtRatio_Unlock = Target_Ratio_Unlock;
	break;
    }
    switch (Clock_Ratio_Unlock) {
    case COREFREQ_TOGGLE_OFF:
    case 0b01:
    case 0b10:
    case 0b11:
	PUBLIC(RO(Proc))->Features.ClkRatio_Unlock = Clock_Ratio_Unlock;
	break;
    }
    switch (Turbo_Ratio_Unlock) {
    case COREFREQ_TOGGLE_OFF:
    case COREFREQ_TOGGLE_ON:
	PUBLIC(RO(Proc))->Features.Turbo_Unlock = Turbo_Ratio_Unlock;
	break;
    }
    switch (Uncore_Ratio_Unlock) {
    case COREFREQ_TOGGLE_OFF:
    case COREFREQ_TOGGLE_ON:
	PUBLIC(RO(Proc))->Features.Uncore_Unlock = Uncore_Ratio_Unlock;
	break;
    }
}

void OverrideCodeNameString(PROCESSOR_SPECIFIC *pSpecific)
{
	StrCopy(PUBLIC(RO(Proc))->Architecture,
		Arch[PUBLIC(RO(Proc))->ArchID].Architecture[
							pSpecific->CodeNameIdx
						].CodeName,
		CODENAME_LEN);
}

void OverrideUnlockCapability(PROCESSOR_SPECIFIC *pSpecific)
{
    if (pSpecific->Latch & LATCH_TGT_RATIO_UNLOCK) {
	PUBLIC(RO(Proc))->Features.TgtRatio_Unlock=pSpecific->TgtRatioUnlocked;
    }
    if (pSpecific->Latch & LATCH_CLK_RATIO_UNLOCK) {
	PUBLIC(RO(Proc))->Features.ClkRatio_Unlock=pSpecific->ClkRatioUnlocked;
    }
    if (pSpecific->Latch & LATCH_TURBO_UNLOCK) {
	PUBLIC(RO(Proc))->Features.Turbo_Unlock = pSpecific->TurboUnlocked;
    }
    if (pSpecific->Latch & LATCH_UNCORE_UNLOCK) {
	PUBLIC(RO(Proc))->Features.Uncore_Unlock = pSpecific->UncoreUnlocked;
    }
    if (pSpecific->Latch & LATCH_HSMP_CAPABLE) {
	PUBLIC(RO(Proc))->Features.HSMP_Capable = pSpecific->HSMP_Capable;
    }
}

PROCESSOR_SPECIFIC *LookupProcessor(void)
{
	PROCESSOR_SPECIFIC *pSpecific;
    for (pSpecific = Arch[PUBLIC(RO(Proc))->ArchID].Specific;
		(pSpecific != NULL) && (pSpecific->Brand != NULL);
			pSpecific++)
    {
	char **brands, *brand;
	for (brands = pSpecific->Brand, brand = *brands;
		brand != NULL;
			brands++, brand = *brands)
	{
		if (strstr(PUBLIC(RO(Proc))->Features.Info.Brand, brand)) {
			return pSpecific;
		}
	}
    }
	return NULL;
}

void Solve_CodeNameString_From_Arch_Stepping(signed int architectureID)
{
	switch (architectureID) {
	case AMD_Zen:
		switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
		case 0x1:	/* [Zen/Summit Ridge] 8F_01h Stepping 1 */
			StrCopy(PUBLIC(RO(Proc))->Architecture,
				Arch_AMD_Zen[CN_SUMMIT_RIDGE].CodeName,
				CODENAME_LEN);
			break;
		case 0x2:	/* [EPYC/Naples] 8F_01h Stepping 2	*/
			StrCopy(PUBLIC(RO(Proc))->Architecture,
				Arch_AMD_Zen[CN_NAPLES].CodeName,
				CODENAME_LEN);
			break;
		}
		break;
	case AMD_Zen_APU:
		switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
		case 0x0:	/* [Zen/Raven Ridge] 8F_11h Stepping 0	*/
			StrCopy(PUBLIC(RO(Proc))->Architecture,
				Arch_AMD_Zen_APU[CN_RAVEN_RIDGE].CodeName,
				CODENAME_LEN);
			break;
		case 0x2:	/* [Zen/Snowy Owl] 8F_11h Stepping 2	*/
			StrCopy(PUBLIC(RO(Proc))->Architecture,
				Arch_AMD_Zen_APU[CN_SNOWY_OWL].CodeName,
				CODENAME_LEN);
			break;
		}
		break;
	}
}

int Intel_MaxBusRatio(PLATFORM_ID *PfID)
{
	struct SIGNATURE whiteList[] = {
		_Core_Conroe,		/* 06_0F */
		_Core_Penryn,		/* 06_17 */
		_Atom_Bonnell,		/* 06_1C */
		_Atom_Silvermont,	/* 06_26 */
		_Atom_Lincroft, 	/* 06_27 */
		_Atom_Clover_Trail,	/* 06_35 */
		_Atom_Saltwell, 	/* 06_36 */
		_Silvermont_Bay_Trail,	/* 06_37 */
	};
	int id, ids = sizeof(whiteList) / sizeof(whiteList[0]);
	for (id = 0; id < ids; id++) {
		if ((whiteList[id].ExtFamily \
			== PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
		 && (whiteList[id].Family \
			== PUBLIC(RO(Proc))->Features.Std.EAX.Family)
		 && (whiteList[id].ExtModel \
			== PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
		 && (whiteList[id].Model \
			== PUBLIC(RO(Proc))->Features.Std.EAX.Model))
		{
			RDMSR((*PfID), MSR_IA32_PLATFORM_ID);
			return 0;
		}
	}
	return -1;
}

void Intel_Core_Platform_Info(unsigned int cpu)
{
	PLATFORM_ID PfID = {.value = 0};
	PLATFORM_INFO PfInfo = {.value = 0};
	PERF_STATUS PerfStatus = {.value = 0};
	unsigned int ratio0 = 10, ratio1 = 10; /*Arbitrary values*/

	RDMSR(PfInfo, MSR_PLATFORM_INFO);
	if (PfInfo.value != 0) {
		ratio0 = PfInfo.MaxNonTurboRatio;
	}

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	if (PerfStatus.value != 0) {				/* §18.18.3.4 */
		if (PerfStatus.CORE.XE_Enable) {
			ratio1 = PerfStatus.CORE.MaxBusRatio;
		} else {
			if (Intel_MaxBusRatio(&PfID) == 0) {
				if (PfID.value != 0)
				{
					ratio1 = PfID.MaxBusRatio;
				}
			}
		}
	} else {
			if (Intel_MaxBusRatio(&PfID) == 0) {
				if (PfID.value != 0)
				{
					ratio1 = PfID.MaxBusRatio;
				}
			}
	}

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] =	KMIN(ratio0, ratio1);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] =	KMAX(ratio0, ratio1);
}

void Compute_Intel_Core_Burst(unsigned int cpu)
{
	PERF_STATUS PerfStatus = {.value = 0};
	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);

	if (PRIVATE(OF(Specific)) != NULL)
	{
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
				+ PRIVATE(OF(Specific))->Boost[0]
				+ PRIVATE(OF(Specific))->Boost[1];

	    if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] > 0) {
		PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
	    }
	} else {	/* Is Processor half ratio or Burst capable ?	*/
	    if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
			PerfStatus.value != 0 ? PerfStatus.CORE.MaxBusRatio
			: 2 + PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];

		PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
	    } else {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = 0;
		PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
	    }
	}
}

PLATFORM_INFO Intel_Platform_Info(unsigned int cpu)
{
	PLATFORM_INFO PfInfo = {.value = 0};
	RDMSR(PfInfo, MSR_PLATFORM_INFO);

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;

	return PfInfo;
}

typedef union {
	unsigned long long	value;
} TURBO_RATIO_CFG_MSR;

typedef union {
	TURBO_RATIO_CFG_MSR	MSR;
	TURBO_RATIO_CONFIG0	Cfg0;
	TURBO_RATIO_CONFIG1	Cfg1;
	TURBO_RATIO_CONFIG2	Cfg2;
} TURBO_CONFIG;

typedef struct {
	CLOCK_ARG	*pClockMod;
	TURBO_CONFIG	Config;
	long		rc;
} CLOCK_TURBO_ARG;

void Assign_8C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
	pBoost[BOOST(8C)] = pConfig->Cfg0.MaxRatio_8C;
	pBoost[BOOST(7C)] = pConfig->Cfg0.MaxRatio_7C;
	pBoost[BOOST(6C)] = pConfig->Cfg0.MaxRatio_6C;
	pBoost[BOOST(5C)] = pConfig->Cfg0.MaxRatio_5C;
	pBoost[BOOST(4C)] = pConfig->Cfg0.MaxRatio_4C;
	pBoost[BOOST(3C)] = pConfig->Cfg0.MaxRatio_3C;
	pBoost[BOOST(2C)] = pConfig->Cfg0.MaxRatio_2C;
	pBoost[BOOST(1C)] = pConfig->Cfg0.MaxRatio_1C;
}

void Assign_15C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
	pBoost[BOOST(15C)] = pConfig->Cfg1.IVB_EP.MaxRatio_15C;
	pBoost[BOOST(14C)] = pConfig->Cfg1.IVB_EP.MaxRatio_14C;
	pBoost[BOOST(13C)] = pConfig->Cfg1.IVB_EP.MaxRatio_13C;
	pBoost[BOOST(12C)] = pConfig->Cfg1.IVB_EP.MaxRatio_12C;
	pBoost[BOOST(11C)] = pConfig->Cfg1.IVB_EP.MaxRatio_11C;
	pBoost[BOOST(10C)] = pConfig->Cfg1.IVB_EP.MaxRatio_10C;
	pBoost[BOOST(9C) ] = pConfig->Cfg1.IVB_EP.MaxRatio_9C;
}

void Assign_16C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
	pBoost[BOOST(16C)] = pConfig->Cfg1.HSW_EP.MaxRatio_16C;
	pBoost[BOOST(15C)] = pConfig->Cfg1.HSW_EP.MaxRatio_15C;
	pBoost[BOOST(14C)] = pConfig->Cfg1.HSW_EP.MaxRatio_14C;
	pBoost[BOOST(13C)] = pConfig->Cfg1.HSW_EP.MaxRatio_13C;
	pBoost[BOOST(12C)] = pConfig->Cfg1.HSW_EP.MaxRatio_12C;
	pBoost[BOOST(11C)] = pConfig->Cfg1.HSW_EP.MaxRatio_11C;
	pBoost[BOOST(10C)] = pConfig->Cfg1.HSW_EP.MaxRatio_10C;
	pBoost[BOOST(9C) ] = pConfig->Cfg1.HSW_EP.MaxRatio_9C;
}

void Assign_18C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
	pBoost[BOOST(18C)] = pConfig->Cfg2.MaxRatio_18C;
	pBoost[BOOST(17C)] = pConfig->Cfg2.MaxRatio_17C;
}

void Assign_SKL_X_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
	pBoost[BOOST(16C)] = pConfig->Cfg1.SKL_X.NUMCORE_7;
	pBoost[BOOST(15C)] = pConfig->Cfg1.SKL_X.NUMCORE_6;
	pBoost[BOOST(14C)] = pConfig->Cfg1.SKL_X.NUMCORE_5;
	pBoost[BOOST(13C)] = pConfig->Cfg1.SKL_X.NUMCORE_4;
	pBoost[BOOST(12C)] = pConfig->Cfg1.SKL_X.NUMCORE_3;
	pBoost[BOOST(11C)] = pConfig->Cfg1.SKL_X.NUMCORE_2;
	pBoost[BOOST(10C)] = pConfig->Cfg1.SKL_X.NUMCORE_1;
	pBoost[BOOST(9C) ] = pConfig->Cfg1.SKL_X.NUMCORE_0;
}

long For_All_Turbo_Clock(CLOCK_ARG *pClockMod, void (*ConfigFunc)(void *))
{
	long rc = RC_SUCCESS;
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
    do {
	cpu--;	/* From last AP to BSP */

	if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
	&& ((pClockMod->cpu == -1) || (pClockMod->cpu == cpu)))
	{
		CLOCK_TURBO_ARG ClockTurbo = {
			.pClockMod = pClockMod,
			.Config = {.MSR = {.value = 0}},
			.rc = RC_SUCCESS
		};

		smp_call_function_single(cpu, ConfigFunc, &ClockTurbo, 1);

		rc = ClockTurbo.rc;
	}
    } while ((cpu != 0) && (rc >= RC_SUCCESS)) ;

	return rc;
}

static void Intel_Turbo_Cfg8C_PerCore(void *arg)
{
	CLOCK_TURBO_ARG *pClockCfg8C = (CLOCK_TURBO_ARG *) arg;

	RDMSR(pClockCfg8C->Config.Cfg0, MSR_TURBO_RATIO_LIMIT);

  if (pClockCfg8C->pClockMod != NULL)	/* Read-Only function called ?	*/
  {
	pClockCfg8C->rc = RC_SUCCESS;
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
    {
	unsigned short WrRd8C = 0;
    switch (pClockCfg8C->pClockMod->NC) {
    case 1:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_1C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_1C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 2:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_2C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_2C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 3:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_3C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_3C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 4:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_4C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_4C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 5:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_5C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_5C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 6:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_6C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_6C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 7:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_7C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_7C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    case 8:
      if (pClockCfg8C->pClockMod->cpu == -1) {
	pClockCfg8C->Config.Cfg0.MaxRatio_8C = pClockCfg8C->pClockMod->Ratio;
      } else {
	pClockCfg8C->Config.Cfg0.MaxRatio_8C += pClockCfg8C->pClockMod->Offset;
      }
	WrRd8C = 1;
	break;
    default:
	WrRd8C = 0;
	break;
    }
      if (WrRd8C) {
	WRMSR(pClockCfg8C->Config.Cfg0, MSR_TURBO_RATIO_LIMIT);
	RDMSR(pClockCfg8C->Config.Cfg0, MSR_TURBO_RATIO_LIMIT);
	pClockCfg8C->rc = RC_OK_COMPUTE;
      }
    } else {
	pClockCfg8C->rc = -RC_TURBO_PREREQ;
    }
  }
}

long Intel_Turbo_Config8C(CLOCK_ARG *pClockMod)
{
	long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg8C_PerCore);

	return rc;
}

static void Intel_Turbo_Cfg15C_PerCore(void *arg)
{
	CLOCK_TURBO_ARG *pClockCfg15C = (CLOCK_TURBO_ARG *) arg;

	RDMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);

  if (pClockCfg15C->pClockMod != NULL)
  {
	pClockCfg15C->rc = RC_SUCCESS;
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
    {
	unsigned short WrRd15C = 0;
    switch (pClockCfg15C->pClockMod->NC) {
    case 9:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_9C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_9C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 10:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_10C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_10C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 11:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_11C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_11C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 12:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_12C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_12C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 13:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_13C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_13C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 14:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_14C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_14C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    case 15:
      if (pClockCfg15C->pClockMod->cpu == -1) {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_15C = \
						pClockCfg15C->pClockMod->Ratio;
      } else {
	pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_15C += \
						pClockCfg15C->pClockMod->Offset;
      }
	WrRd15C = 1;
	break;
    default:
	WrRd15C = 0;
	break;
    }
      if (WrRd15C) {
	WRMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	RDMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	pClockCfg15C->rc = RC_OK_COMPUTE;
      }
    } else {
	pClockCfg15C->rc = -RC_TURBO_PREREQ;
    }
  }
}

long Intel_Turbo_Config15C(CLOCK_ARG *pClockMod)
{
	long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg15C_PerCore);

	return rc;
}

static void Intel_Turbo_Cfg16C_PerCore(void *arg)
{
	CLOCK_TURBO_ARG *pClockCfg16C = (CLOCK_TURBO_ARG *) arg;

	RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);

  if (pClockCfg16C->pClockMod != NULL)
  {
	pClockCfg16C->rc = RC_SUCCESS;
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
    {
	unsigned short WrRd16C = 0;
    switch (pClockCfg16C->pClockMod->NC) {
    case 9:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_9C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_9C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 10:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_10C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_10C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 11:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_11C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_11C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 12:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_12C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_12C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 13:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_13C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_13C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 14:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_14C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_14C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 15:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_15C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_15C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 16:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_16C = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_16C += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    default:
	WrRd16C = 0;
	break;
    }
      if (WrRd16C) {
	WRMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	pClockCfg16C->rc = RC_OK_COMPUTE;
      }
    } else {
	pClockCfg16C->rc = -RC_TURBO_PREREQ;
    }
  }
}

long Intel_Turbo_Config16C(CLOCK_ARG *pClockMod)
{
	long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg16C_PerCore);

	return rc;
}

static void Intel_Turbo_Cfg18C_PerCore(void *arg)
{
	CLOCK_TURBO_ARG *pClockCfg18C = (CLOCK_TURBO_ARG *) arg;

	RDMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);

  if (pClockCfg18C->pClockMod != NULL)
  {
	pClockCfg18C->rc = RC_SUCCESS;
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
    {
	unsigned short WrRd18C = 0;
    switch (pClockCfg18C->pClockMod->NC) {
    case 17:
      if (pClockCfg18C->pClockMod->cpu == -1) {
	pClockCfg18C->Config.Cfg2.MaxRatio_17C = \
						pClockCfg18C->pClockMod->Ratio;
      } else {
	pClockCfg18C->Config.Cfg2.MaxRatio_17C += \
						pClockCfg18C->pClockMod->Offset;
      }
	WrRd18C = 1;
	break;
    case 18:
      if (pClockCfg18C->pClockMod->cpu == -1) {
	pClockCfg18C->Config.Cfg2.MaxRatio_18C = \
						pClockCfg18C->pClockMod->Ratio;
      } else {
	pClockCfg18C->Config.Cfg2.MaxRatio_18C += \
						pClockCfg18C->pClockMod->Offset;
      }
	WrRd18C = 1;
	break;
    default:
	WrRd18C = 0;
	break;
    }
      if (WrRd18C) {
	WRMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);
	RDMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);
	pClockCfg18C->rc = RC_OK_COMPUTE;
      }
    } else {
	pClockCfg18C->rc = -RC_TURBO_PREREQ;
    }
  }
}

long Intel_Turbo_Config18C(CLOCK_ARG *pClockMod)
{
	long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg18C_PerCore);

	return rc;
}

static void Intel_Turbo_Cfg_SKL_X_PerCore(void *arg)
{
	CLOCK_TURBO_ARG *pClockCfg16C = (CLOCK_TURBO_ARG *) arg;

	RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);

  if (pClockCfg16C->pClockMod != NULL)
  {
	pClockCfg16C->rc = RC_SUCCESS;
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
    {
	unsigned short WrRd16C = 0;
    switch (pClockCfg16C->pClockMod->NC) {
    case 9:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_0 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_0 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 10:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_1 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_1 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 11:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_2 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_2 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 12:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_3 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_3 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 13:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_4 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_4 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 14:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_5 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_5 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 15:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_6 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_6 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    case 16:
      if (pClockCfg16C->pClockMod->cpu == -1) {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_7 = \
						pClockCfg16C->pClockMod->Ratio;
      } else {
	pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_7 += \
						pClockCfg16C->pClockMod->Offset;
      }
	WrRd16C = 1;
	break;
    default:
	WrRd16C = 0;
	break;
    }
      if (WrRd16C) {
	WRMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
	pClockCfg16C->rc = RC_OK_COMPUTE;
      }
    } else {
	pClockCfg16C->rc = -RC_TURBO_PREREQ;
    }
  }
}

long Skylake_X_Turbo_Config16C(CLOCK_ARG *pClockMod)
{
	long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg_SKL_X_PerCore);

	return rc;
}

long TurboClock_IvyBridge_EP(CLOCK_ARG *pClockMod)
{
	long rc = Intel_Turbo_Config8C(pClockMod);
	if (rc >= RC_SUCCESS)
	{
		rc = Intel_Turbo_Config15C(pClockMod);
		if (rc >= RC_SUCCESS)
		{
			TURBO_RATIO_CONFIG1 Cfg1;
			RDMSR(Cfg1, MSR_TURBO_RATIO_LIMIT1);
			Cfg1.IVB_EP.Semaphore = 1;
			WRMSR(Cfg1, MSR_TURBO_RATIO_LIMIT1);
		}
	}
	return rc;
}

long TurboClock_Haswell_EP(CLOCK_ARG *pClockMod)
{
	long rc = Intel_Turbo_Config8C(pClockMod);
	if (rc >= RC_SUCCESS) {
		rc = Intel_Turbo_Config16C(pClockMod);
	}
	if (rc >= RC_SUCCESS) {
		rc = Intel_Turbo_Config18C(pClockMod);

		if (rc >= RC_SUCCESS)
		{
			TURBO_RATIO_CONFIG2 Cfg2;
			RDMSR(Cfg2, MSR_TURBO_RATIO_LIMIT2);
			Cfg2.Semaphore = 1;
			WRMSR(Cfg2, MSR_TURBO_RATIO_LIMIT2);
		}
	}
	return rc;
}

long TurboClock_Broadwell_EP(CLOCK_ARG *pClockMod)
{
	long rc = Intel_Turbo_Config8C(pClockMod);
	if (rc >= RC_SUCCESS) {
		rc = Intel_Turbo_Config16C(pClockMod);
	}
	if (rc >= RC_SUCCESS) {
		rc = Intel_Turbo_Config18C(pClockMod);

		if (rc >= RC_SUCCESS)
		{
			TURBO_RATIO_CONFIG3 Cfg3;
			RDMSR(Cfg3, MSR_TURBO_RATIO_LIMIT3);
			Cfg3.Semaphore = 1;
			WRMSR(Cfg3, MSR_TURBO_RATIO_LIMIT3);
		}
	}
	return rc;
}

long TurboClock_Skylake_X(CLOCK_ARG *pClockMod)
{
	long rc = Intel_Turbo_Config8C(pClockMod);
	if (rc >= RC_SUCCESS) {
		rc = Skylake_X_Turbo_Config16C(pClockMod);
	}
	return rc;
}

static void PerCore_Intel_HWP_Notification(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

    if ((arg != NULL) && PUBLIC(RO(Proc))->Features.Power.EAX.HWP_Int) {
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->HWP_Mask, Core->Bind);
	/* HWP Notifications are fully disabled.			*/
	Core->PowerThermal.HWP_Interrupt.value = 0;
	WRMSR(Core->PowerThermal.HWP_Interrupt, MSR_HWP_INTERRUPT);

	RDMSR(Core->PowerThermal.HWP_Interrupt, MSR_HWP_INTERRUPT);
	if (Core->PowerThermal.HWP_Interrupt.value == 0) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->HWP, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->HWP, Core->Bind);
	}
    }
}

static void PerCore_Intel_HWP_Ignition(void *arg)
{
    if ((arg != NULL) && PUBLIC(RO(Proc))->Features.Power.EAX.HWP_EPP) {
	CORE_RO *Core = (CORE_RO *) arg;

	RDMSR(Core->PowerThermal.HWP_Capabilities, MSR_IA32_HWP_CAPABILITIES);
	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);

	Core->PowerThermal.HWP_Request.Minimum_Perf =
				Core->PowerThermal.HWP_Capabilities.Lowest;

	Core->PowerThermal.HWP_Request.Maximum_Perf =
				Core->PowerThermal.HWP_Capabilities.Highest;

	Core->PowerThermal.HWP_Request.Desired_Perf =
				Core->PowerThermal.HWP_Capabilities.Guaranteed;

	if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff)) {
		Core->PowerThermal.HWP_Request.Energy_Pref = HWP_EPP;
	}
	WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
    }
}

void Intel_Hardware_Performance(void)
{
    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
	PM_ENABLE PM_Enable = {.value = 0};
	HDC_CONTROL HDC_Control = {.value = 0};

	if (PUBLIC(RO(Proc))->Features.Power.EAX.HWP_Reg)
	{
		/*	MSR_IA32_PM_ENABLE is a Package register.	*/
		RDMSR(PM_Enable, MSR_IA32_PM_ENABLE);
		/*	Is the HWP requested and its current state is off ? */
	  if ((HWP_Enable == 1) && (PM_Enable.HWP_Enable == 0))
	  {
		unsigned int cpu;
		/*	From last AP to BSP				*/
			cpu = PUBLIC(RO(Proc))->CPU.Count;
		do {
			cpu--;

		    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
			/*	Synchronous call.			*/
			smp_call_function_single(cpu,
						PerCore_Intel_HWP_Notification,
						&PUBLIC(RO(Core, AT(cpu))), 1);
		    }
		  } while (cpu != 0) ;

	    if (BITCMP_CC(LOCKLESS, \
			PUBLIC(RW(Proc))->HWP, PUBLIC(RO(Proc))->HWP_Mask) )
	    {
		/*	Enable the Hardware-controlled Performance States. */
		PM_Enable.HWP_Enable = 1;
		WRMSR(PM_Enable, MSR_IA32_PM_ENABLE);
		RDMSR(PM_Enable, MSR_IA32_PM_ENABLE);

		if (PM_Enable.HWP_Enable)
		{
			cpu = PUBLIC(RO(Proc))->CPU.Count;
		do {
			cpu--;

		    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
			/*	Asynchronous call.			*/
			smp_call_function_single(cpu,
						PerCore_Intel_HWP_Ignition,
						&PUBLIC(RO(Core, AT(cpu))), 0);
		    }
		  } while (cpu != 0) ;
		}
	    }
	  }
	}
	PUBLIC(RO(Proc))->Features.HWP_Enable = PM_Enable.HWP_Enable;
	/*		Hardware Duty Cycling				*/
	if (PUBLIC(RO(Proc))->Features.Power.EAX.HDC_Reg)
	{
		RDMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
		switch (HDC_Enable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			HDC_Control.HDC_Enable = HDC_Enable;
			WRMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
			RDMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
			break;
		}
	}
	PUBLIC(RO(Proc))->Features.HDC_Enable = HDC_Control.HDC_Enable;
    }
}

void Skylake_PowerControl(void)
{
	unsigned short WrRdMSR = 0;

	POWER_CONTROL PowerCtrl = {.value = 0};
	RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);

	switch (EEO_Disable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerCtrl.EEO_Disable = EEO_Disable;
			WrRdMSR = 1;
		break;
	}
	switch (R2H_Disable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerCtrl.R2H_Disable = R2H_Disable;
			WrRdMSR = 1;
		break;
	}
	if (WrRdMSR) {
		WRMSR(PowerCtrl, MSR_IA32_POWER_CTL);
		RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
	}
	PUBLIC(RO(Proc))->Features.EEO_Enable = !PowerCtrl.EEO_Disable;
	PUBLIC(RO(Proc))->Features.R2H_Enable = !PowerCtrl.R2H_Disable;
}

void SandyBridge_Uncore_Ratio(unsigned int cpu)
{
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
}

long Haswell_Uncore_Ratio(CLOCK_ARG *pClockMod)
{
	long rc = RC_SUCCESS;
	UNCORE_RATIO_LIMIT UncoreRatio = {.value = 0};
	RDMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);

	if (pClockMod != NULL) {	/* Called as Read-Only ?	*/
	    if (PUBLIC(RO(Proc))->Features.Uncore_Unlock)
	    {
		unsigned short WrRdMSR;
		switch (pClockMod->NC) {
		case CLOCK_MOD_MAX:
		    if (pClockMod->cpu == -1) {
			UncoreRatio.MaxRatio = pClockMod->Ratio;
		    } else {
			UncoreRatio.MaxRatio += pClockMod->Offset;
		    }
			WrRdMSR = 1;
			break;
		case CLOCK_MOD_MIN:
		    if (pClockMod->cpu == -1) {
			UncoreRatio.MinRatio = pClockMod->Ratio;
		    } else {
			UncoreRatio.MinRatio += pClockMod->Offset;
		    }
			WrRdMSR = 1;
			break;
		default:
			WrRdMSR = 0;
			rc = -RC_UNIMPLEMENTED;
			break;
		}
		if (WrRdMSR) {
			WRMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);
			RDMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);
			rc = RC_OK_COMPUTE;
		}
	    } else {
		rc = -RC_UNCORE_PREREQ;
	    }
	}

	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)]=UncoreRatio.MinRatio;
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)]=UncoreRatio.MaxRatio;

	return rc;
}

void Nehalem_PowerLimit(void)
{
	unsigned short WrRdMSR = 0;

	NEHALEM_POWER_LIMIT PowerLimit = {.value = 0};
	RDMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);

    if (Custom_TDP_Count > 0) {
	if (Custom_TDP_Offset[PWR_DOMAIN(PKG)] != 0)
	{	/*	Register is an 8 watt multiplier	*/
		signed short	TDP_Limit = PowerLimit.TDP_Limit >> 3;
				TDP_Limit += Custom_TDP_Offset[PWR_DOMAIN(PKG)];
	    if (TDP_Limit > 0)
	    {
		PowerLimit.TDP_Limit = TDP_Limit << 3;
		WrRdMSR = 1;
	    }
	}
    }
    if (TDP_Limiting_Count > 0) {
	switch (Activate_TDP_Limit[PWR_DOMAIN(PKG)]) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		PowerLimit.TDP_Override = Activate_TDP_Limit[PWR_DOMAIN(PKG)];
		WrRdMSR = 1;
		break;
	}
    }
    if (Custom_TDC_Offset != 0) {
	signed short	TDC_Limit = PowerLimit.TDC_Limit >> 3;
			TDC_Limit += Custom_TDC_Offset;
	if (TDC_Limit > 0)
	{
		PowerLimit.TDC_Limit = TDC_Limit << 3;
		WrRdMSR = 1;
	}
    }
	switch (Activate_TDC_Limit) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		PowerLimit.TDC_Override = Activate_TDC_Limit;
		WrRdMSR = 1;
		break;
	}
	if (WrRdMSR) {
		if (PUBLIC(RO(Proc))->Features.TDP_Unlock) {
			WRMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);
			RDMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);
		}
	}
	PUBLIC(RO(Proc))->PowerThermal.PowerLimit[PWR_DOMAIN(PKG)].Domain_Limit1
		= PowerLimit.TDP_Limit;

	PUBLIC(RO(Proc))->PowerThermal.PowerLimit[PWR_DOMAIN(PKG)].Enable_Limit1
		= PowerLimit.TDP_Override;
	/*	TDP: 1/(2 << (3-1)) = 1/8 watt	*/
	PUBLIC(RO(Proc))->PowerThermal.Unit.PU = 3;
	/*	TDC: 1/8 amp			*/
	PUBLIC(RO(Proc))->PowerThermal.TDC = PowerLimit.TDC_Limit >> 3;
	PUBLIC(RO(Proc))->PowerThermal.Enable_Limit.TDC=PowerLimit.TDC_Override;
}

void Intel_PowerInterface(void)
{
	RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_RAPL_POWER_UNIT);
	RDMSR(PUBLIC(RO(Proc))->PowerThermal.PowerInfo, MSR_PKG_POWER_INFO);
}

void Intel_DomainPowerLimit(	unsigned int MSR_DOMAIN_POWER_LIMIT,
				unsigned long long PowerLimitLockMask,
				enum PWR_DOMAIN pw )
{
	unsigned short WrRdMSR = 0;
	const unsigned int lt = PWR_LIMIT_SIZE * pw, rt = 1 + lt;

	DOMAIN_POWER_LIMIT PowerLimit = {.value = 0};
	RDMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);

    if ((PowerLimit.value & PowerLimitLockMask) == 0
     && (PUBLIC(RO(Proc))->PowerThermal.Unit.PU > 0))
    {
	unsigned short	pwrUnits = PUBLIC(RO(Proc))->PowerThermal.Unit.PU - 1;
			pwrUnits = 2 << pwrUnits;

	if (Custom_TDP_Count > lt) {
	    if (Custom_TDP_Offset[lt] != 0)
	    {
		signed short	TDP_Limit = PowerLimit.Domain_Limit1 / pwrUnits;
				TDP_Limit = TDP_Limit + Custom_TDP_Offset[lt];
		if (TDP_Limit > 0) {
			PowerLimit.Domain_Limit1 = pwrUnits * TDP_Limit;
			WrRdMSR = 1;
		}
	    }
	}
	if (PowerLimitLockMask == PKG_POWER_LIMIT_LOCK_MASK) {
	  if (Custom_TDP_Count > rt) {
	    if (Custom_TDP_Offset[rt] != 0)
	    {
		signed short	TDP_Limit = PowerLimit.Domain_Limit2 / pwrUnits;
				TDP_Limit = TDP_Limit + Custom_TDP_Offset[rt];
		if (TDP_Limit > 0) {
			PowerLimit.Domain_Limit2 = pwrUnits * TDP_Limit;
			WrRdMSR = 1;
		}
	    }
	  }
	}
    }
	if (TDP_Limiting_Count > lt) {
		switch (Activate_TDP_Limit[lt]) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerLimit.Enable_Limit1 = Activate_TDP_Limit[lt];
			WrRdMSR = 1;
			break;
		}
	}
	if (TDP_Clamping_Count > lt) {
		switch (Activate_TDP_Clamp[lt]) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerLimit.Clamping1 = Activate_TDP_Clamp[lt];
			WrRdMSR = 1;
			break;
		}
	}
	if (PowerLimitLockMask == PKG_POWER_LIMIT_LOCK_MASK) {
	    if (TDP_Limiting_Count > rt) {
		switch (Activate_TDP_Limit[rt]) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerLimit.Enable_Limit2 = Activate_TDP_Limit[rt];
			WrRdMSR = 1;
			break;
		}
	    }
	    if (TDP_Clamping_Count > rt) {
		switch (Activate_TDP_Clamp[rt]) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PowerLimit.Clamping2 = Activate_TDP_Clamp[rt];
			WrRdMSR = 1;
			break;
		}
	    }
	}
	if (WrRdMSR) {
		if (PUBLIC(RO(Proc))->Features.TDP_Unlock) {
			WRMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);
			RDMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);
		}
	}
	PUBLIC(RO(Proc))->PowerThermal.PowerLimit[pw] = PowerLimit;
}

void Intel_Processor_PIN(bool capable)
{
	if (capable) {
		INTEL_PPIN_CTL PPinCtl = {.value = 0};
		RDMSR(PPinCtl, MSR_PPIN_CTL);
		if (PPinCtl.Enable) {
			INTEL_PPIN_NUM PPinNum = {.value = 0};
			RDMSR(PPinNum, MSR_PPIN);
			PUBLIC(RO(Proc))->Features.Factory.PPIN = PPinNum.value;
		}
	}
}

void AMD_Processor_PIN(bool capable)
{
	if (capable) {
		AMD_PPIN_CTL PPinCtl = {.value = 0};
		RDMSR(PPinCtl, MSR_AMD_PPIN_CTL);
		if (PPinCtl.Enable) {
			AMD_PPIN_NUM PPinNum = {.value = 0};
			RDMSR(PPinNum, MSR_AMD_PPIN);
			PUBLIC(RO(Proc))->Features.Factory.PPIN = PPinNum.value;
		}
	}
}

void Query_Same_Platform_Features(unsigned int cpu)
{
	PLATFORM_INFO PfInfo;

	PfInfo = Intel_Platform_Info(cpu);
	PUBLIC(RO(Proc))->Features.TDP_Unlock = PfInfo.ProgrammableTDP;
	PUBLIC(RO(Proc))->Features.TDP_Levels = PfInfo.ConfigTDPlevels;
	PUBLIC(RO(Proc))->Features.Turbo_Unlock = PfInfo.ProgrammableTurbo;

	if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL) {
		OverrideCodeNameString(PRIVATE(OF(Specific)));
		OverrideUnlockCapability(PRIVATE(OF(Specific)));
	}
	Default_Unlock_Reset();

	Intel_Processor_PIN(PfInfo.PPIN_CAP);

	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
}

void Nehalem_Platform_Info(unsigned int cpu)
{
	Query_Same_Platform_Features(cpu);

	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	8C	*/
}

void IvyBridge_EP_Platform_Info(unsigned int cpu)
{
	const unsigned int NC = \
	PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;

	Query_Same_Platform_Features(cpu);

	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	8C	*/
    if (NC > 8) {
	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 7; /*	15C	*/
    }
}

void Haswell_EP_Platform_Info(unsigned int cpu)
{
	const unsigned int NC = \
	PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;

	Query_Same_Platform_Features(cpu);

	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	8C	*/
    if (NC > 8) {
	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	16C	*/
    }
    if (NC > 16) {
	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 2; /*	18C	*/
    }
}

void Skylake_X_Platform_Info(unsigned int cpu)
{
	const unsigned int NC = \
	PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;

    switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
    case 11:
    case 10:
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	  Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_COOPERLAKE_X].CodeName,
		CODENAME_LEN);
	break;
    case 7:
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	 Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_CASCADELAKE_X].CodeName,
		CODENAME_LEN);
	break;
    case 4:
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	  Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_SKYLAKE_X].CodeName,
		CODENAME_LEN);
	break;
    }

	Query_Same_Platform_Features(cpu);

	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	8C	*/
    if (NC > 8) {
	PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /*	16C	*/
    }
}

unsigned int AMD_HSMP_Exec(	enum HSMP_FUNC MSG_FUNC,
				HSMP_ARG MSG_ARG[],
				unsigned int HSMP_CmdRegister,
				unsigned int HSMP_ArgRegister,
				unsigned int HSMP_RspRegister,
				unsigned int SMU_IndexRegister,
				unsigned int SMU_DataRegister )
{
	return(AMD_HSMP_Mailbox(MSG_FUNC,
				MSG_ARG,
				HSMP_CmdRegister,
				HSMP_ArgRegister,
				HSMP_RspRegister,
				SMU_IndexRegister,
				SMU_DataRegister));
}


typedef void (*ROUTER)(void __iomem *mchmap, unsigned short mc);

PCI_CALLBACK Router(struct pci_dev *dev, unsigned int offset,
		unsigned int bsize, unsigned long long wsize,
		ROUTER route, unsigned short mc)
{
	void __iomem *mchmap;
	union {
		unsigned long long addr;
		struct {
			unsigned int low;
			unsigned int high;
		};
	} mchbar;
	unsigned long long wmask = BITCPL(wsize);
	unsigned char mchbarEnable = 0;

	switch (bsize) {
	case 32:
		pci_read_config_dword(dev, offset    , &mchbar.low);
		mchbar.high = 0;
		break;
	case 64:
		pci_read_config_dword(dev, offset    , &mchbar.low);
		pci_read_config_dword(dev, offset + 4, &mchbar.high);
		break;
	}
	mchbarEnable = BITVAL(mchbar, 0);
	if (mchbarEnable) {
		mchbar.addr &= wmask;
		mchbar.addr += wsize * mc;
		mchmap = ioremap(mchbar.addr, wsize);
		if (mchmap != NULL) {
			route(mchmap, mc);

			iounmap(mchmap);

			return 0;
		} else
			return (PCI_CALLBACK) -ENOMEM;
	} else
		return (PCI_CALLBACK) -ENOMEM;
}

void Query_P945(void __iomem *mchmap, unsigned short mc)
{	/* Source: Mobile Intel 945 Express Chipset Family.		*/
	unsigned short cha;

	PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].P945.DCC.value = readl(mchmap + 0x200);

	switch (PUBLIC(RO(Proc))->Uncore.MC[mc].P945.DCC.DAMC) {
	case 0b00:
	case 0b11:
		PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 1;
		break;
	case 0b01:
	case 0b10:
		PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
		break;
	}

	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	unsigned short rank, rankCount;

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT0.value = \
					readl(mchmap + 0x110 + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT1.value = \
					readw(mchmap + 0x114 + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT2.value = \
					readl(mchmap + 0x118 + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.BANK.value = \
					readw(mchmap + 0x10e + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value = \
					readw(mchmap + 0x40c + 0x80 * cha);
      if (cha == 0) {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value &= \
								0b11111111;
	rankCount = 4;
      } else {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value &= 0b1111;
	rankCount = 2;
      }
      for (rank = 0; rank < rankCount; rank++)
      {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRB[rank].value = \
				readb(mchmap + 0x100 + rank + 0x80 * cha);
      }
    }
}

void Query_P955(void __iomem *mchmap, unsigned short mc)
{	/* Source: Intel 82955X Memory Controller Hub (MCH)		*/
	unsigned short cha;

	PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].P955.DCC.value = readl(mchmap + 0x200);

	switch (PUBLIC(RO(Proc))->Uncore.MC[mc].P955.DCC.DAMC) {
	case 0b00:
	case 0b11:
		PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 1;
		break;
	case 0b01:
	case 0b10:
		PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
		break;
	}

	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	unsigned short rank;

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.DRT1.value = \
					readw(mchmap + 0x114 + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.BANK.value = \
					readw(mchmap + 0x10e + 0x80 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.WIDTH.value = \
					readw(mchmap + 0x40c + 0x80 * cha);

      for (rank = 0; rank < 4; rank++)
      {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.DRB[rank].value = \
				readb(mchmap + 0x100 + rank + 0x80 * cha);
      }
    }
}

void Query_P965(void __iomem *mchmap, unsigned short mc)
{
	unsigned short cha;

	PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.value = readl(mchmap + 0x260);
	PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.value = readl(mchmap + 0x660);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
		  (PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.RankPop0 != 0)
		+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.RankPop0 != 0);

	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
	  (PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.SingleDimmPop ? 1 : 2)
	+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.SingleDimmPop ? 1 : 2);

	for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
	{
		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT0.value = \
					readl(mchmap + 0x29c + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT1.value = \
					readw(mchmap + 0x250 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT2.value = \
					readl(mchmap + 0x252 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT3.value = \
					readw(mchmap + 0x256 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT4.value = \
					readl(mchmap + 0x258 + 0x400 * cha);
	}
}

void Query_G965(void __iomem *mchmap, unsigned short mc)
{	/* Source: Mobile Intel 965 Express Chipset Family.		*/
	unsigned short cha, slot;

	PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB0.value = readl(mchmap+0x1200);
	PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB1.value = readl(mchmap+0x1300);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
		  (PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB0.Rank1Addr != 0)
		+ (PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB1.Rank1Addr != 0);

	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
			PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 1 ? 1:2;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT0.value = \
					readl(mchmap + 0x1210 + 0x100 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT1.value = \
					readl(mchmap + 0x1214 + 0x100 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT2.value = \
					readl(mchmap + 0x1218 + 0x100 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT3.value = \
					readl(mchmap + 0x121c + 0x100 * cha);

      for (slot = 0; slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount; slot++)
      {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].DRA.value = \
					readl(mchmap + 0x1208 + 0x100 * cha);
      }
    }
}

void Query_P35(void __iomem *mchmap, unsigned short mc)
{	/* Source: Intel® 3 Series Express Chipset Family. */
	unsigned short cha;

	PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.value = readl(mchmap + 0x260);
	PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.value = readl(mchmap + 0x660);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
		  (PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.RankPop0 != 0)
		+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.RankPop0 != 0);

	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
	  (PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.SingleDimmPop ? 1 : 2)
	+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.SingleDimmPop ? 1 : 2);

	for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
	{
		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT0.value = \
					readw(mchmap + 0x265 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT1.value = \
					readw(mchmap + 0x250 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT2.value = \
					readl(mchmap + 0x252 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT3.value = \
					readl(mchmap + 0x256 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT4.value = \
					readl(mchmap + 0x258 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT5.value = \
					readw(mchmap + 0x25d + 0x400 * cha);
	}
}

kernel_ulong_t Query_NHM_Timing(struct pci_dev *pdev,
				unsigned short mc,
				unsigned short cha)
{	/*Source:Micron Technical Note DDR3 Power-Up, Initialization, & Reset*/
	struct pci_dev *dev = pci_get_domain_bus_and_slot(
						pci_domain_nr(pdev->bus),
						pdev->bus->number,
						PCI_DEVFN(4, 0)
				);
    if (dev != NULL)
    {
	pci_read_config_dword(dev, 0x70,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.MR0_1.value);

	pci_read_config_dword(dev, 0x74,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.MR2_3.value);

	pci_read_config_dword(dev ,0x80,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Rank_A.value);

	pci_read_config_dword(dev ,0x84,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Rank_B.value);

	pci_read_config_dword(dev ,0x88,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Bank.value);

	pci_read_config_dword(dev ,0x8c,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Refresh.value);

	pci_read_config_dword(dev ,0x90,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.CKE_Timing.value);

	pci_read_config_dword(dev, 0xb8,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Params.value);

	pci_dev_put(dev);
	return 0;
    } else
	return -ENODEV;
}

kernel_ulong_t Query_NHM_DIMM(struct pci_dev *pdev,
				unsigned short mc,
				unsigned short cha)
{
	struct pci_dev *dev = pci_get_domain_bus_and_slot(
						pci_domain_nr(pdev->bus),
						pdev->bus->number,
						PCI_DEVFN(4, 1)
				);
    if (dev != NULL)
    {
	unsigned short slot;
	for (slot = 0; slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount; slot++)
	{
	pci_read_config_dword(dev, 0x48 + 4 * slot,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].DOD.value);
	}
	pci_dev_put(dev);
	return 0;
    } else {
	return -ENODEV;
    }
}

void Query_NHM_MaxDIMMs(struct pci_dev *dev, unsigned short mc)
{
	pci_read_config_dword(dev, 0x64,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].MaxDIMMs.NHM.DOD.value);

	switch (PUBLIC(RO(Proc))->Uncore.MC[mc].MaxDIMMs.NHM.DOD.MAXNUMDIMMS) {
	case 0b00:
		PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
		break;
	case 0b01:
		PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;
		break;
	case 0b10:
		PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 3;
		break;
	default:
		PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
		break;
	}
}

kernel_ulong_t Query_NHM_IMC(struct pci_dev *dev, unsigned short mc)
{
	kernel_ulong_t rc = 0;
	unsigned short cha;

	Query_NHM_MaxDIMMs(dev, mc);

	pci_read_config_dword(dev, 0x48,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.value);

	pci_read_config_dword(dev, 0x4c,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.STATUS.value);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
	  (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL0_ACTIVE != 0)
	+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL1_ACTIVE != 0)
	+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL2_ACTIVE != 0);

    for (cha = 0;
	(cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount) && !rc; cha++)
    {
	rc = Query_NHM_Timing(dev, mc, cha) & Query_NHM_DIMM(dev, mc, cha);
    }
	return rc;
}

kernel_ulong_t Query_Lynnfield_IMC(struct pci_dev *dev, unsigned short mc)
{
	kernel_ulong_t rc = 0;
	unsigned short cha;

	Query_NHM_MaxDIMMs(dev, mc);

	pci_read_config_dword(dev, 0x48,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.value);

	pci_read_config_dword(dev, 0x4c,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.STATUS.value);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
	  (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL0_ACTIVE != 0)
	+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL1_ACTIVE != 0);

    for (cha = 0;
	(cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount) && !rc; cha++)
    {
	rc = Query_NHM_Timing(dev, mc, cha) & Query_NHM_DIMM(dev, mc, cha);
    }
	return rc;
}

void Query_SNB_IMC(void __iomem *mchmap, unsigned short mc)
{	/* Sources:	2nd & 3rd Generation Intel® Core™ Processor Family
			Intel® Xeon Processor E3-1200 Family		*/
	unsigned short cha, dimmCount[2];

	PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.value = readl(mchmap + 0x5004);
	PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.value = readl(mchmap + 0x5008);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;

	dimmCount[0]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_A_Size > 0)
		    +(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_B_Size > 0);
	dimmCount[1]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_A_Size > 0)
		    +(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_B_Size > 0);

    for (cha = 0; cha < 2; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount += (dimmCount[cha] > 0);
    }
    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.DBP.value = \
					readl(mchmap + 0x4000 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RAP.value = \
					readl(mchmap + 0x4004 + 0x400 * cha);

		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RFTP.value = \
					readl(mchmap + 0x4298 + 0x400 * cha);
    }
	/*		Is Dual DIMM Per Channel Disable ?		*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
			(PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.DDPCD == 1) ?
			1 : PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
}

void Query_Turbo_TDP_Config(void __iomem *mchmap)
{
	TURBO_ACTIVATION TurboActivation = {.value = 0};
	CONFIG_TDP_NOMINAL NominalTDP = {.value = 0};
	CONFIG_TDP_CONTROL ControlTDP = {.value = 0};
	CONFIG_TDP_LEVEL ConfigTDP[2] = {{.value = 0}, {.value = 0}};
	unsigned int cpu, local = get_cpu();	/* TODO(preempt_disable) */

	NominalTDP.value = readl(mchmap + 0x5f3c);
	PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP)] = NominalTDP.Ratio;

	ConfigTDP[0].value = readq(mchmap + 0x5f40);
	PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP1)] = ConfigTDP[0].Ratio;

	ConfigTDP[1].value = readq(mchmap + 0x5f48);
	PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP2)] = ConfigTDP[1].Ratio;

	ControlTDP.value = readl(mchmap + 0x5f50);
	PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock  = ControlTDP.Lock;
	PUBLIC(RO(Proc))->Features.TDP_Cfg_Level = ControlTDP.Level;

	TurboActivation.value = readl(mchmap + 0x5f54);
	PUBLIC(RO(Core, AT(local)))->Boost[BOOST(ACT)]=TurboActivation.MaxRatio;
	PUBLIC(RO(Proc))->Features.TurboActiv_Lock = TurboActivation.Ratio_Lock;

	put_cpu();	/* TODO(preempt_enable) */

    switch (PUBLIC(RO(Proc))->Features.TDP_Cfg_Level) {
    case 2:
	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
							ConfigTDP[1].PkgPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
							ConfigTDP[1].MinPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
							ConfigTDP[1].MaxPower;
	break;
    case 1:
	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
							ConfigTDP[0].PkgPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
							ConfigTDP[0].MinPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
							ConfigTDP[0].MaxPower;
	break;
    case 0:
	/*TODO(Unknown CSR for Nominal {Pkg, Min, Max} Power settings ?) */
	break;
    }

	PUBLIC(RO(Proc))->Features.TDP_Levels = 3;

	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
	    if (cpu != local)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP)] = \
				PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP)];

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP1)] = \
				PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP1)];

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP2)] = \
				PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP2)];

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(ACT)] = \
				PUBLIC(RO(Core, AT(local)))->Boost[BOOST(ACT)];
	    }
	}
}

void Query_HSW_IMC(void __iomem *mchmap, unsigned short mc)
{	/*Source: Desktop 4th & 5th Generation Intel® Core™ Processor Family.*/
	unsigned short cha, dimmCount[2];

	PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.value = readl(mchmap + 0x5004);
	PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.value = readl(mchmap + 0x5008);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;

	dimmCount[0]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_A_Size > 0)
		    +(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_B_Size > 0);
	dimmCount[1]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_A_Size > 0)
		    +(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_B_Size > 0);

    for (cha = 0; cha < 2; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount += (dimmCount[cha] > 0);
    }
    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
/*TODO( Unsolved: What is the channel #1 'X' factor of Haswell registers ? )*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.REG4C00.value = \
							readl(mchmap + 0x4c00);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Timing.value = \
							readl(mchmap + 0x4c04);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank_A.value = \
							readl(mchmap + 0x4c08);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank_B.value = \
							readl(mchmap + 0x4c0c);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank.value = \
							readl(mchmap + 0x4c14);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Refresh.value = \
							readl(mchmap + 0x4e98);
    }
	/*		Is Dual DIMM Per Channel Disable ?		*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
			(PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.DDPCD == 1) ?
			1 : PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
    if (mc == 0) {
	Query_Turbo_TDP_Config(mchmap);
    }
}

void Query_SKL_IMC(void __iomem *mchmap, unsigned short mc)
{	/*Source: 6th & 7th Generation Intel® Processor for S-Platforms Vol 2*/
	unsigned short cha;
	/*		Intra channel configuration			*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADCH.value = readl(mchmap+0x5000);
    if (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADCH.CH_L_MAP)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC0.value = readl(mchmap+0x5008);
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC1.value = readl(mchmap+0x5004);
    } else {
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC0.value = readl(mchmap+0x5004);
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC1.value = readl(mchmap+0x5008);
    }
	/*		DIMM parameters					*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.value = readl(mchmap+0x500c);
	PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.value = readl(mchmap+0x5010);
	/*		Sum up any present DIMM per channel.		*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_S_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_S_Size != 0));
	/*		Count of populated DIMMs L and DIMMs S	TODO
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_L_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_S_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_S_Size != 0));
	*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Timing.value = \
					readl(mchmap + 0x4000 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.ACT.value = \
					readl(mchmap + 0x4004 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.RDRD.value = \
					readl(mchmap + 0x400c + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.RDWR.value = \
					readl(mchmap + 0x4010 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.WRRD.value = \
					readl(mchmap + 0x4014 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.WRWR.value = \
					readl(mchmap + 0x4018 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Sched.value = \
					readl(mchmap + 0x401c + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.ODT.value = \
					readl(mchmap + 0x4070 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Refresh.value = \
					readl(mchmap + 0x423c + 0x400 * cha);
    }
    if (mc == 0) {
	Query_Turbo_TDP_Config(mchmap);
    }
}

void Query_RKL_IMC(void __iomem *mchmap, unsigned short mc)
{	/*Source: 11th Generation Intel® Core Processor Datasheet Vol 2*/
	unsigned short cha;
	/*		Intra channel configuration			*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADCH.value = readl(mchmap+0x5000);
    if (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADCH.CH_L_MAP)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC0.value = readl(mchmap+0x5008);
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC1.value = readl(mchmap+0x5004);
    } else {
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC0.value = readl(mchmap+0x5004);
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC1.value = readl(mchmap+0x5008);
    }
	/*		DIMM parameters					*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.value = readl(mchmap+0x500c);
	PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.value = readl(mchmap+0x5010);
	/*		Sum up any present DIMM per channel.		*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_S_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_S_Size != 0));
	/*		Count of populated DIMMs L and DIMMs S	TODO
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_L_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_S_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_S_Size != 0));
	*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Timing.value = \
					readl(mchmap + 0x4000 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.ACT.value = \
					readl(mchmap + 0x4004 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.RDRD.value = \
					readl(mchmap + 0x400c + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.RDWR.value = \
					readl(mchmap + 0x4010 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.WRRD.value = \
					readl(mchmap + 0x4014 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.WRWR.value = \
					readl(mchmap + 0x4018 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Sched.value = \
					readq(mchmap + 0x4088 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.PWDEN.value = \
					readq(mchmap + 0x4050 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.ODT.value = \
					readq(mchmap + 0x4070 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Refresh.value = \
					readl(mchmap + 0x423c + 0x400 * cha);
    }
    if (mc == 0) {
	Query_Turbo_TDP_Config(mchmap);
    }
}

void Query_TGL_IMC(void __iomem *mchmap, unsigned short mc)
{	/*Source: 11th Generation Intel® Core Processor Datasheet Vol 2*/
	unsigned short cha;

	/*		Intra channel configuration			*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADCH.value = readl(mchmap+0x5000);
    if (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADCH.CH_L_MAP)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC0.value = readl(mchmap+0x5008);
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC1.value = readl(mchmap+0x5004);
    } else {
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC0.value = readl(mchmap+0x5004);
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC1.value = readl(mchmap+0x5008);
    }
	/*		DIMM parameters					*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.value = readl(mchmap+0x500c);
	PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.value = readl(mchmap+0x5010);
	/*		Sum up any present DIMM per channel.		*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_S_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_S_Size != 0));
	/*		TODO(Count of populated DIMMs L and DIMMs S)
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
	  ((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_L_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_L_Size != 0))
	+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_S_Size != 0)
	|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_S_Size != 0));
	*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Timing.value = \
					readl(mchmap + 0x4000 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.ACT.value = \
					readl(mchmap + 0x4008 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.RDRD.value = \
					readl(mchmap + 0x400c + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.RDWR.value = \
					readl(mchmap + 0x4010 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.WRRD.value = \
					readl(mchmap + 0x4014 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.WRWR.value = \
					readl(mchmap + 0x4018 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Sched.value = \
					readq(mchmap + 0x4088 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.PWDEN.value = \
					readq(mchmap + 0x4050 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.ODT.value = \
					readq(mchmap + 0x4070 + 0x400 * cha);

	PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Refresh.value = \
					readl(mchmap + 0x423c + 0x400 * cha);
    }
    if (mc == 0) {
	Query_Turbo_TDP_Config(mchmap);
    }
}

static PCI_CALLBACK P945(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x44, 32, 0x4000, Query_P945, 0);
}

static PCI_CALLBACK P955(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x44, 32, 0x4000, Query_P955, 0);
}

static PCI_CALLBACK P965(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x4000, Query_P965, 0);
}

static PCI_CALLBACK G965(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x4000, Query_G965, 0);
}

static PCI_CALLBACK P35(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x4000, Query_P35, 0);
}

static PCI_CALLBACK ICH_TCO(struct pci_dev *dev)
{
	Intel_TCO1_CNT TCO1_CNT = {.value = 0};

	pci_read_config_word(dev, 0x40 + 8, &TCO1_CNT.value);

	switch (WDT_Enable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		TCO1_CNT.TCO_TMR_HALT = !WDT_Enable;
		pci_write_config_word(dev, 0x40 + 8, TCO1_CNT.value);
		pci_read_config_word(dev, 0x40 + 8, &TCO1_CNT.value);
		break;
	}
	if (TCO1_CNT.TCO_TMR_HALT) {
		BITCLR_CC( LOCKLESS,	PUBLIC(RW(Proc))->WDT,
					PUBLIC(RO(Proc))->Service.Core );
	} else {
		BITSET_CC( LOCKLESS,	PUBLIC(RW(Proc))->WDT,
					PUBLIC(RO(Proc))->Service.Core );
	}
	BITSET_CC( LOCKLESS,	PUBLIC(RO(Proc))->WDT_Mask,
				PUBLIC(RO(Proc))->Service.Core );

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK SoC_SLM(struct pci_dev *dev)
{/* DRP */
	PCI_MCR MsgCtrlReg = {
	.MBZ = 0, .Bytes = 0, .Offset = 0x0, .Port = 0x1, .OpCode = 0x10
	};

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.value);
/* DTR0 */
	MsgCtrlReg.Offset = 0x1;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR0.value);
/* DTR1 */
	MsgCtrlReg.Offset = 0x2;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR1.value);
/* DTR2 */
	MsgCtrlReg.Offset = 0x3;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR2.value);
/* DTR3 */
	MsgCtrlReg.Offset = 0x4;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR3.value);
/* DRFC */
	MsgCtrlReg.Offset = 0x8;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRFC.value);
/* BIOS_CFG */
	MsgCtrlReg.Port = 0x4;
	MsgCtrlReg.Offset = 0x6;

	pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);

	pci_read_config_dword(dev, 0xd4,
		&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.value);

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 1
		+ ( PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.EFF_DUAL_CH_EN
		| !PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.DUAL_CH_DIS );

	PUBLIC(RO(Proc))->Uncore.MC[0].SlotCount = (
			PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN0
		|	PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN1
	) + (
			PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN2
		|	PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKNE3
	);
	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK Lynnfield_IMC(struct pci_dev *dev)
{	/*		Clarksfield; Lynnfield				*/
	kernel_ulong_t rc = 0;
	unsigned short mc;

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
	for (mc = 0; (mc < PUBLIC(RO(Proc))->Uncore.CtrlCount) && !rc; mc++) {
		rc = Query_Lynnfield_IMC(dev, 0);
	}
	return (PCI_CALLBACK) rc;
}

static PCI_CALLBACK Jasper_Forest_IMC(struct pci_dev *pdev)
{
	kernel_ulong_t rc = 0;
	unsigned int cpu = 0, SocketID = 0, bus_number = pdev->bus->number;
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;

    while ((cpu < PUBLIC(RO(Proc))->CPU.Count) && !rc)
    {
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	if (Core->T.PackageID == SocketID)
	{
		struct pci_dev *dev = pci_get_domain_bus_and_slot(
						pci_domain_nr(pdev->bus),
						bus_number,
						PCI_DEVFN(3, 0) );
	    if (dev != NULL)
	    {
		const unsigned short mc = ((unsigned short) SocketID)
					% MC_MAX_CTRL;
		rc = Query_NHM_IMC(dev, mc);
		pci_dev_put(dev);
	    }
		PUBLIC(RO(Proc))->Uncore.CtrlCount++;

		if (bus_number < 0xff) {
			bus_number++;
		}
		SocketID++;
	}
	cpu++;
    }
	return (PCI_CALLBACK) rc;
}

static PCI_CALLBACK Nehalem_IMC(struct pci_dev *dev)
{ /* Arrandale; Beckton; Bloomfield; Clarkdale; Eagleton; Gainestown; Gulftown*/
	kernel_ulong_t rc = 0;
	unsigned short mc;

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
	for (mc = 0; (mc < PUBLIC(RO(Proc))->Uncore.CtrlCount) && !rc; mc++) {
		rc = Query_NHM_IMC(dev, mc);
	}
	return (PCI_CALLBACK) rc;
}

static PCI_CALLBACK NHM_IMC_TR(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0x50,
				&PUBLIC(RO(Proc))->Uncore.Bus.DimmClock.value);

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK NHM_NON_CORE(struct pci_dev *dev)
{
	NHM_CURRENT_UCLK_RATIO UncoreClock = {.value = 0};

	pci_read_config_dword(dev, 0xc0, &UncoreClock.value);

	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)]=UncoreClock.UCLK;
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)]=UncoreClock.MinRatio;

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK X58_QPI(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0xd0,
				&PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.value);

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK X58_VTD(struct pci_dev *dev)
{
	kernel_ulong_t rc = 0;
	unsigned int VTBAR = 0;

	pci_read_config_dword(dev, 0x180, &VTBAR);
  if (BITVAL(VTBAR, 0) == 1)
  {
	void __iomem *VT_d_MMIO;
	const unsigned int VT_d_Bar = VTBAR & 0xffffe000;

	VT_d_MMIO = ioremap(VT_d_Bar, 0x1000);
    if (VT_d_MMIO != NULL)
    {
	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Ver.value = readl(VT_d_MMIO + 0x0);
	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Cap.value = readq(VT_d_MMIO + 0x8);

	iounmap(VT_d_MMIO);
    }
	PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.X58.VT_d = 0;
  } else {
	PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.X58.VT_d = 1;
  }
	return (PCI_CALLBACK) rc;
}

static PCI_CALLBACK SNB_IMC(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0xe4,
				&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x8000, Query_SNB_IMC, 0);
}

static PCI_CALLBACK IVB_IMC(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0xe4,
				&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);

	pci_read_config_dword(dev, 0xe8,
				&PUBLIC(RO(Proc))->Uncore.Bus.IVB_Cap.value);

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x8000, Query_SNB_IMC, 0);
}

static PCI_CALLBACK SNB_EP_HB(struct pci_dev *dev)
{
	UNUSED(dev);
/*TODO(Harware missing! for testings)					*/
	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK SNB_EP_CAP(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0x84,
			&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap0.value);

	pci_read_config_dword(dev, 0x88,
			&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap1.value);

	pci_read_config_dword(dev, 0x8c,
			&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap2.value);

	pci_read_config_dword(dev, 0x90,
			&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap3.value);

	pci_read_config_dword(dev, 0x94,
			&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap4.value);

	return (PCI_CALLBACK) 0;
}

kernel_ulong_t SNB_EP_CTRL(struct pci_dev *dev, unsigned short mc)
{
	pci_read_config_dword(dev, 0x7c,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].SNB_EP.TECH.value);

	pci_read_config_dword(dev, 0x80,
			&PUBLIC(RO(Proc))->Uncore.MC[mc].SNB_EP.TAD.value);

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
			PUBLIC(RO(Proc))->Uncore.MC[mc].SNB_EP.TAD.CH_WAY;

	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount++;
/*TODO(Specs missing)*/
	PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;

	return 0;
}

static PCI_CALLBACK SNB_EP_CTRL0(struct pci_dev *dev)
{
	if (PUBLIC(RO(Proc))->Uncore.CtrlCount < 1) {
		PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
	}
	SNB_EP_CTRL(dev, 0);

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK SNB_EP_CTRL1(struct pci_dev *dev)
{
	if (PUBLIC(RO(Proc))->Uncore.CtrlCount < 2) {
		PUBLIC(RO(Proc))->Uncore.CtrlCount = 2;
	}
	SNB_EP_CTRL(dev, 1);

	return (PCI_CALLBACK) 0;
}

kernel_ulong_t SNB_EP_IMC(struct pci_dev *dev ,unsigned short mc,
						unsigned short cha)
{
	pci_read_config_dword(dev, 0x200,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.DBP.value);

	pci_read_config_dword(dev, 0x204,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RAP.value);

	pci_read_config_dword(dev, 0x208,
		&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RWP.value);

	pci_read_config_dword(dev, 0x214,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RFTP.value);

	return 0;
}

static PCI_CALLBACK SNB_EP_IMC_CTRL0_CHA0(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 0, 0);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL0_CHA1(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 0, 1);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL0_CHA2(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 0, 2);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL0_CHA3(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 0, 3);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL1_CHA0(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 1, 0);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL1_CHA1(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 1, 1);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL1_CHA2(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 1, 2);
}

static PCI_CALLBACK SNB_EP_IMC_CTRL1_CHA3(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_IMC(dev, 1, 3);
}

kernel_ulong_t SNB_EP_TAD(struct pci_dev *dev,	unsigned short mc,
						unsigned short cha)
{
	pci_read_config_dword(dev, 0x80,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[0].MTR.value);

	pci_read_config_dword(dev, 0x84,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[1].MTR.value);

	pci_read_config_dword(dev, 0x88,
	    &PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[2].MTR.value);
	return 0;
}

static PCI_CALLBACK SNB_EP_TAD_CTRL0_CHA0(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 0, 0);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL0_CHA1(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 0, 1);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL0_CHA2(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 0, 2);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL0_CHA3(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 0, 3);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL1_CHA0(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 1, 0);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL1_CHA1(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 1, 1);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL1_CHA2(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 1, 2);
}

static PCI_CALLBACK SNB_EP_TAD_CTRL1_CHA3(struct pci_dev *dev)
{
	return (PCI_CALLBACK) SNB_EP_TAD(dev, 1, 3);
}

static PCI_CALLBACK SNB_EP_QPI(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0xd4,
				&PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.value);

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK HSW_IMC(struct pci_dev *dev)
{
	pci_read_config_dword(dev, 0xe4,
				&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);

	pci_read_config_dword(dev, 0xe8,
				&PUBLIC(RO(Proc))->Uncore.Bus.IVB_Cap.value);

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return Router(dev, 0x48, 64, 0x8000, Query_HSW_IMC, 0);
}

void SoC_SKL_VTD(void)
{
  if (PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_A.VT_d == 0)
  {
	void __iomem *VT_d_MMIO;
	const unsigned int VTBAR = 0xfed90000;

	VT_d_MMIO = ioremap(VTBAR, 0x1000);
    if (VT_d_MMIO != NULL)
    {
	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Ver.value = readl(VT_d_MMIO + 0x0);
	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Cap.value = readq(VT_d_MMIO + 0x8);

	iounmap(VT_d_MMIO);
    }
  }
}

static PCI_CALLBACK SKL_HOST(	struct pci_dev *dev,
				ROUTER Query,
				unsigned long long wsize,
				unsigned short mc )
{
	pci_read_config_dword(dev, 0xe4,
				&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_A.value);

	pci_read_config_dword(dev, 0xe8,
				&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_B.value);

	pci_read_config_dword(dev, 0xec,
				&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_C.value);

	SoC_SKL_VTD();

	return Router(dev, 0x48, 64, wsize, Query, mc);
}

static PCI_CALLBACK SKL_IMC(struct pci_dev *dev)
{
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return SKL_HOST(dev, Query_SKL_IMC, 0x8000, 0);
}

/* TODO(Hardware missing)
static PCI_CALLBACK SKL_SA(struct pci_dev *dev)
{
	SKL_SA_PLL_RATIOS PllRatios = {.value = 0};

	pci_read_config_dword(dev, 0x5918, &PllRatios.value);

	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)] = PllRatios.UCLK;
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = 0;

	pci_read_config_dword(dev, 0xe4,
				&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_A.value);

	pci_read_config_dword(dev, 0xe8,
				&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_B.value);

	return Router(dev, 0x48, 64, 0x8000, Query_SKL_IMC, 0);

	return 0;
}
*/
static PCI_CALLBACK CML_PCH(struct pci_dev *dev)
{
	UNUSED(dev);
	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK RKL_IMC(struct pci_dev *dev)
{	/* Same address offsets as used in Skylake			*/
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

	return SKL_HOST(dev, Query_RKL_IMC, 0x8000, 0);
}

static PCI_CALLBACK TGL_IMC(struct pci_dev *dev)
{	/* Source: drivers/edac/igen6_edac.c				*/
	PCI_CALLBACK rc = 0;
	unsigned short mc;

	PUBLIC(RO(Proc))->Uncore.CtrlCount = 2;
	/*	Controller #0 is not necessary activated but enabled.	*/
	for (mc = 0; mc < PUBLIC(RO(Proc))->Uncore.CtrlCount; mc++) {
		rc = SKL_HOST(dev, Query_TGL_IMC, 0x10000, mc);
	}
	pci_read_config_dword(dev, 0xf0,
				&PUBLIC(RO(Proc))->Uncore.Bus.TGL_Cap_E.value);
	return rc;
}

static PCI_CALLBACK AMD_0Fh_MCH(struct pci_dev *dev)
{	/* Source: BKDG for AMD NPT Family 0Fh Processors.		*/
	unsigned short cha, slot, chip;
	/*		As defined by specifications.			*/
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
	/*		DRAM Configuration low register.		*/
	pci_read_config_dword(dev, 0x90,
			&PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRL.value);
	/*		DRAM Configuration high register.		*/
	pci_read_config_dword(dev, 0x94,
			&PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRH.value);
	/*	One channel if 64 bits / two channels if 128 bits width. */
	PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = \
			PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRL.Width128 + 1;
	/*		DIMM Geometry.					*/
    for (chip = 0; chip < 8; chip++)
    {
	cha = chip >> 2;
	slot = chip % 4;
	pci_read_config_dword(dev, 0x40 + 4 * chip,
	    &PUBLIC(RO(Proc))->Uncore.MC[0].Channel[cha].DIMM[slot].MBA.value);

	PUBLIC(RO(Proc))->Uncore.MC[0].SlotCount += \
	  PUBLIC(RO(Proc))->Uncore.MC[0].Channel[cha].DIMM[slot].MBA.CSEnable;
    }
	/* DIMM Size. */
	pci_read_config_dword(dev, 0x80,
		&PUBLIC(RO(Proc))->Uncore.MC[0].MaxDIMMs.AMD0Fh.CS.value);
	/* DRAM Timings. */
	pci_read_config_dword(dev, 0x88,
		&PUBLIC(RO(Proc))->Uncore.MC[0].Channel[0].AMD0Fh.DTRL.value);
	/* Assume same timings for both channels. */
	PUBLIC(RO(Proc))->Uncore.MC[0].Channel[1].AMD0Fh.DTRL.value = \
		PUBLIC(RO(Proc))->Uncore.MC[0].Channel[0].AMD0Fh.DTRL.value;

	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK AMD_0Fh_HTT(struct pci_dev *dev)
{
	unsigned int link;

	pci_read_config_dword(dev, 0x60,
				&PUBLIC(RO(Proc))->Uncore.Bus.NodeID.value);

    switch ( PCI_SLOT(dev->devfn) ) {
    case 24:
	PUBLIC(RO(Core, AT(0)))->T.Cluster.Node = \
				PUBLIC(RO(Proc))->Uncore.Bus.NodeID.Node;
	break;
    case 25:
      if ( ( (1 + PUBLIC(RO(Proc))->Uncore.Bus.NodeID.CPUCnt)
		>=PUBLIC(RO(Proc))->CPU.Count )
	&& (PUBLIC(RO(Proc))->CPU.Count >= 2) )
      {
	PUBLIC(RO(Core, AT(1)))->T.Cluster.Node = \
				PUBLIC(RO(Proc))->Uncore.Bus.NodeID.Node;
      }
	break;
    }

	pci_read_config_dword(dev, 0x64,
				&PUBLIC(RO(Proc))->Uncore.Bus.UnitID.value);

      for (link = 0; link < 3; link++)
      {
	pci_read_config_dword(dev, 0x88 + 0x20 * link,
			&PUBLIC(RO(Proc))->Uncore.Bus.LDTi_Freq[link].value);
      };

	return (PCI_CALLBACK) 0;
}

#ifdef CONFIG_AMD_NB
static PCI_CALLBACK AMD_17h_ZenIF(struct pci_dev *dev)
{
	if (PRIVATE(OF(ZenIF_dev)) == NULL) {
		PRIVATE(OF(ZenIF_dev)) = dev;
	}
	return (PCI_CALLBACK) 0;
}
#endif /* CONFIG_AMD_NB */

static PCI_CALLBACK AMD_Zen_IOMMU(struct pci_dev *dev)
{
/* Sources:
*	AMD I/O Virtualization Technology (IOMMU) Specification Jan. 2020
*	coreboot/src/soc/amd/picasso/agesa_acpi.c
*/
	AMD_IOMMU_CAP_BAR IOMMU_Cap_Bar;
	unsigned char IOMMU_UnLock = 0;

	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_CR.value = 0x0;

	pci_read_config_dword(	dev, 0x40,
				&PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_HDR.value );

	pci_read_config_dword(dev, 0x44, &IOMMU_Cap_Bar.low);
	pci_read_config_dword(dev, 0x48, &IOMMU_Cap_Bar.high);

	IOMMU_UnLock = IOMMU_Cap_Bar.addr & 0x1;
	IOMMU_Cap_Bar.addr = IOMMU_Cap_Bar.addr & 0xffffc000;
    if ((IOMMU_Cap_Bar.addr != 0x0) && IOMMU_UnLock)
    {
	void __iomem *IOMMU_MMIO;
	const size_t bsize = PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_HDR.EFRSup ?
				0x80000 : 0x4000;

      if ((IOMMU_MMIO = ioremap(IOMMU_Cap_Bar.addr, bsize)) != NULL)
      {
	PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_CR.value = readq(IOMMU_MMIO + 0x18);

	iounmap(IOMMU_MMIO);
      }
      else
      {
	return (PCI_CALLBACK) -ENOMEM;
      }
    }
	return (PCI_CALLBACK) 0;
}

static PCI_CALLBACK AMD_17h_UMC(struct pci_dev *dev)
{
	AMD_17_UMC_SDP_CTRL SDP_CTRL;
	unsigned short mc, cha, chip, sec;
	UNUSED(dev);
/*TODO( Query the number of UMC )					*/
	PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;

  for (mc = 0; mc < PUBLIC(RO(Proc))->Uncore.CtrlCount; mc++)
  {
	unsigned int UMC_BAR[MC_MAX_CHA] = { 0,0,0,0,0,0,0,0 };

	unsigned short count = 0;
    for (cha = 0; cha < MC_MAX_CHA; cha++)
    {
	SDP_CTRL.value = 0;

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.ECC,
			(SMU_AMD_UMC_BASE_CHA_F17H(cha) + 0xdf4),
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

	Core_AMD_SMN_Read(	SDP_CTRL,
				(SMU_AMD_UMC_BASE_CHA_F17H(cha) + 0x104),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	if ((SDP_CTRL.value != 0xffffffff) && (SDP_CTRL.INIT))
	{
		UMC_BAR[count++] = SMU_AMD_UMC_BASE_CHA_F17H(cha);

		Core_AMD_SMN_Read(
		    PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[0].DAC,
			(SMU_AMD_UMC_BASE_CHA_F17H(0) + 0x30),
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

		Core_AMD_SMN_Read(
		    PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[1].DAC,
			(SMU_AMD_UMC_BASE_CHA_F17H(0) + 0x34),
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );
	}
    }
	PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = count;

    for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
    {
	unsigned int CHIP_BAR[2][2];

	CHIP_BAR[0][0] = UMC_BAR[cha] + 0x0;

	CHIP_BAR[0][1] = UMC_BAR[cha] + 0x20;

	CHIP_BAR[1][0] = UMC_BAR[cha] + 0x10;

	CHIP_BAR[1][1] = UMC_BAR[cha] + 0x28;

	for (chip = 0; chip < 4; chip++)
	{
	  for (sec = 0; sec < 2; sec++)
	  {
		unsigned int addr[2], ranks = 0;

		addr[1] = CHIP_BAR[sec][1] + 4 * (chip >> 1);

		Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
				.AMD17h.CHIP[chip][sec].Mask,
				addr[1],
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	    if ( (ranks == 0)
		&& (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
			.AMD17h.CHIP[chip][sec].Mask.value != 0) )
	    {
		ranks = BITVAL(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
				.AMD17h.CHIP[chip][sec].Mask.value, 9) ? 1 : 2;
	    }
	    if (ranks == 2) {
		addr[0] = CHIP_BAR[sec][0] + 4 * chip;
	    } else {
		addr[0] = CHIP_BAR[sec][0] + 4 * (chip - (chip > 2));
	    }
		Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
				.AMD17h.CHIP[chip][sec].Chip,
				addr[0],
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	    if (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.Ranks == 0)
	    {
		PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.Ranks=ranks;
	    }
	  }
	}
    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.SPAZ,
			UMC_BAR[cha] + 0x12c,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.MISC,
			UMC_BAR[cha] + 0x200,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR1,
			UMC_BAR[cha] + 0x204,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR2,
			UMC_BAR[cha] + 0x208,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR3,
			UMC_BAR[cha] + 0x20c,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR4,
			UMC_BAR[cha] + 0x210,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR5,
			UMC_BAR[cha] + 0x214,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR6,
			UMC_BAR[cha] + 0x218,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR7,
			UMC_BAR[cha] + 0x21c,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR8,
			UMC_BAR[cha] + 0x220,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR9,
			UMC_BAR[cha] + 0x224,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR10,
			UMC_BAR[cha] + 0x228,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR12,
			UMC_BAR[cha] + 0x230,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR13,
			UMC_BAR[cha] + 0x234,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR20,
			UMC_BAR[cha] + 0x250,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR21,
			UMC_BAR[cha] + 0x254,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR22,
			UMC_BAR[cha] + 0x258,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTRFC,
			UMC_BAR[cha] + 0x260,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR35,
			UMC_BAR[cha] + 0x28c,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

    Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.BGS,
			UMC_BAR[cha] + 0x58,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );

  Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.BGS_ALT,
			UMC_BAR[cha] + 0xd0,
			SMU_AMD_INDEX_REGISTER_F17H,
			SMU_AMD_DATA_REGISTER_F17H );
    }
  }
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)] = \
	PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = \
		PUBLIC(RO(Proc))->Uncore.MC[0].Channel[0].AMD17h.MISC.MEMCLK/3;

	return (PCI_CALLBACK) 0;
}

static void CoreFreqK_ResetChip(struct pci_dev *dev)
{
	UNUSED(dev);

	memset( PUBLIC(RO(Proc))->Uncore.Chip, 0,
		CHIP_MAX_PCI*sizeof(struct CHIP_ST) );
}

static void CoreFreqK_AppendChip(struct pci_dev *dev)
{
	unsigned int idx;
	for (idx = 0; idx < CHIP_MAX_PCI; idx++)
	{
		if (PUBLIC(RO(Proc))->Uncore.Chip[idx].VID == 0)
		{
			PUBLIC(RO(Proc))->Uncore.Chip[idx].VID = dev->vendor;
			PUBLIC(RO(Proc))->Uncore.Chip[idx].DID = dev->device;

			break;
		}
	}
}

static int CoreFreqK_ProbePCI(	struct pci_device_id PCI_ids[],
				void (*PreProbe)(struct pci_dev*),
				void (*PostProbe)(struct pci_dev*) )
{
	struct pci_device_id *id = PCI_ids;
	struct pci_dev *dev = NULL;
	int rc = -ENODEV;

	if (PreProbe != NULL) {
		PreProbe(dev);
	}
	while (id->vendor || id->subvendor || id->class_mask)
	{
		dev = pci_get_device(id->vendor, id->device, NULL);
	  if (dev != NULL) {
	    if (!pci_enable_device(dev))
	    {
		PCI_CALLBACK Callback = (PCI_CALLBACK) id->driver_data;

		if ((rc = (int) Callback(dev)) == 0)
		{
			if (PostProbe != NULL) {
				PostProbe(dev);
			}
		}
		pci_disable_device(dev);
	    }
	    pci_dev_put(dev);
	  }
		id++;
	}
	return rc;
}

void Query_Same_Genuine_Features(void)
{
	if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
	{
		OverrideCodeNameString(PRIVATE(OF(Specific)));
		OverrideUnlockCapability(PRIVATE(OF(Specific)));
	}
	Default_Unlock_Reset();

	if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
	{
		PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
	} else {
		PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
	}
}

void Query_GenuineIntel(unsigned int cpu)
{
	Query_Same_Genuine_Features();
	Intel_Core_Platform_Info(cpu);
	HyperThreading_Technology();
}

void Query_Core2(unsigned int cpu)
{
	Query_Same_Genuine_Features();
	Intel_Core_Platform_Info(cpu);
	HyperThreading_Technology();
}

void Query_Silvermont(unsigned int cpu) /* Tables 2-6, 2-7, 2-8(BT), 2-9(BT) */
{	/*	Query the Min and Max frequency ratios			*/
	PLATFORM_INFO PfInfo = {.value = 0};
	RDMSR(PfInfo, MSR_PLATFORM_INFO);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;
	/*	Assume a count of Boost ratios equals to the CPU count	*/
	PUBLIC(RO(Proc))->Features.SpecTurboRatio=PUBLIC(RO(Proc))->CPU.Count;
	/*	But can be overridden following the specifications	*/
	if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
	{
		OverrideCodeNameString(PRIVATE(OF(Specific)));
		OverrideUnlockCapability(PRIVATE(OF(Specific)));
	}
	Default_Unlock_Reset();

	HyperThreading_Technology();
	/*	The architecture is gifted of Power and Energy registers */
	RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_RAPL_POWER_UNIT);
}

void Query_Goldmont(unsigned int cpu)	/* Tables 2-6, 2-12		*/
{
	PLATFORM_INFO PfInfo = {.value = 0};
	RDMSR(PfInfo, MSR_PLATFORM_INFO);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;
	PUBLIC(RO(Proc))->Features.SpecTurboRatio=PUBLIC(RO(Proc))->CPU.Count;
	if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
	{
		OverrideCodeNameString(PRIVATE(OF(Specific)));
		OverrideUnlockCapability(PRIVATE(OF(Specific)));
	}
	Default_Unlock_Reset();

	HyperThreading_Technology();
	Intel_PowerInterface();
}

void Query_Airmont(unsigned int cpu)	/* Tables 2-6, 2-7, 2-8, 2-11	*/
{
	Query_Silvermont(cpu);
}

void Query_Nehalem(unsigned int cpu)	/* Table 2-15			*/
{
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();
}

void Query_Nehalem_EX(unsigned int cpu) /* Tables 2-15, 2-17		*/
{
	Query_Same_Genuine_Features();
	Intel_Core_Platform_Info(cpu);
	HyperThreading_Technology();
}

void Query_Avoton(unsigned int cpu)	/* Table 2-10			*/
{
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();

	Intel_PowerInterface();
}

void Query_SandyBridge(unsigned int cpu)
{
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();
	SandyBridge_Uncore_Ratio(cpu);
	Intel_PowerInterface();
}

void Query_SandyBridge_EP(unsigned int cpu)
{
	Query_SandyBridge(cpu);
}

void Query_IvyBridge(unsigned int cpu)
{
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();
	SandyBridge_Uncore_Ratio(cpu);
	Intel_PowerInterface();
}

void Query_IvyBridge_EP(unsigned int cpu)
{
	IvyBridge_EP_Platform_Info(cpu);
	HyperThreading_Technology();
	SandyBridge_Uncore_Ratio(cpu);
	Intel_PowerInterface();
}

void Query_Haswell(unsigned int cpu)
{
	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
	{
		PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
	}
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();
	SandyBridge_Uncore_Ratio(cpu);
	Intel_PowerInterface();
}

void Query_Haswell_EP(unsigned int cpu)
{
	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
	{
		PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
	}
	Haswell_EP_Platform_Info(cpu);
	HyperThreading_Technology();
	Haswell_Uncore_Ratio(NULL);
	Intel_PowerInterface();
}

void Query_Haswell_ULT(unsigned int cpu)
{
	Query_IvyBridge(cpu);
}

void Query_Haswell_ULX(unsigned int cpu)
{
	Query_IvyBridge(cpu);
}

void Query_Broadwell(unsigned int cpu)
{
	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
	{
		PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
	}
	Nehalem_Platform_Info(cpu);
	HyperThreading_Technology();
	Haswell_Uncore_Ratio(NULL);
	Intel_PowerInterface();
	Intel_Hardware_Performance();
}

void Query_Broadwell_EP(unsigned int cpu)
{
	Query_Haswell_EP(cpu);
	Intel_Hardware_Performance();
}

void Query_Skylake(unsigned int cpu)
{
	Query_Broadwell(cpu);

	PUBLIC(RO(Proc))->Features.EEO_Capable = 1;
	PUBLIC(RO(Proc))->Features.R2H_Capable = 1;
	Skylake_PowerControl();
}

void Query_Kaby_Lake(unsigned int cpu)
{
	Power_ACCU_Skylake = Power_ACCU_SKL_PLATFORM;
	Query_Skylake(cpu);
}

void Query_Skylake_X(unsigned int cpu)
{
	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
	{
		PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
	}
	Skylake_X_Platform_Info(cpu);
	HyperThreading_Technology();
	Haswell_Uncore_Ratio(NULL);
	Intel_PowerInterface();
	Intel_Hardware_Performance();
}

unsigned short Compute_AuthenticAMD_Boost(unsigned int cpu)
{
	unsigned short SpecTurboRatio = 0;
	/* Lowest frequency according to BKDG				*/
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 8;

  if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate == 1)
  {
	COFVID CofVid = {.value = 0};

    switch (Arch[PUBLIC(RO(Proc))->ArchID].Signature.ExtFamily) {
    case 0x1:
    case 0x6:
    case 0x7:
	RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]=CofVid.Arch_COF.MaxCpuCof;
	break;
    case 0x2:
	RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
						CofVid.Arch_Pll.MainPllOpFidMax;
      if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
      {
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] += 0x8;
      }
	break;
    case 0x3:
    case 0x5:
	RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
						CofVid.Arch_Pll.MainPllOpFidMax;
      if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
      {
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] += 0x10;
      }
	break;
    }
  } else {
	/* TODO(Get PLL for non HwPstate processor)			*/
  }
  if (PRIVATE(OF(Specific)) != NULL)
  {
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
				+ PRIVATE(OF(Specific))->Boost[0];

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(2C)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)]
				+ PRIVATE(OF(Specific))->Boost[1];
	SpecTurboRatio = 2;
  } else {
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
	SpecTurboRatio = 0;
  }
	return SpecTurboRatio;
}

void Query_VirtualMachine(unsigned int cpu)
{
	Query_Same_Genuine_Features();
	/* Reset Max ratio to call the clock estimation in Controller_Init() */
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = 0;
    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
    {
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 10;
    }
    else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
	||  (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
    {	/* Lowest frequency according to BKDG				*/
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 8;

	Solve_CodeNameString_From_Arch_Stepping(SearchArchitectureID());

      if (PRIVATE(OF(Specific)) != NULL) {
	/*	Save the thermal parameters if specified		*/
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
      } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
      }
   }
	HyperThreading_Technology();
}

void Query_AuthenticAMD(unsigned int cpu)
{	/*	Fallback algorithm for unspecified AMD architectures.	*/
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL) {
	/*	Save thermal parameters for later use in the Daemon	*/
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	/*	Override Processor CodeName & Locking capabilities	*/
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	PUBLIC(RO(Proc))->Features.SpecTurboRatio = \
					Compute_AuthenticAMD_Boost(cpu);
	HyperThreading_Technology();
}

unsigned short Compute_AMD_Family_0Fh_Boost(unsigned int cpu)
{	/* Source: BKDG for AMD NPT Family 0Fh: §13.8			*/
	unsigned short SpecTurboRatio = 0;

    if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.FID == 1)
    {	/*		Processor supports FID changes .		*/
	FIDVID_STATUS FidVidStatus = {.value = 0};
	RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = \
						VCO[FidVidStatus.StartFID].MCF;

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = 8 + FidVidStatus.MaxFID;

	if (FidVidStatus.StartFID < 0b1000)
	{
		unsigned int t;
	    for (t = 0; t < 5; t++)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(SIZE)-5+t] = \
					VCO[FidVidStatus.StartFID].PCF[t];
	    }
		SpecTurboRatio = 5;
	} else {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
							8 + FidVidStatus.MaxFID;
		SpecTurboRatio = 1;
	}
    } else {
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] =	8
					+ HwCfgRegister.Family_0Fh.StartFID;

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C) ] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];
	SpecTurboRatio = 1;
    }
	return SpecTurboRatio;
}

void Query_AMD_Family_0Fh(unsigned int cpu)
{
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	PUBLIC(RO(Proc))->Features.SpecTurboRatio = \
					Compute_AMD_Family_0Fh_Boost(cpu);
	HyperThreading_Technology();
}

void Compute_AMD_Family_10h_Boost(unsigned int cpu)
{
	unsigned int pstate, sort[5] = {
		BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C), BOOST(MIN)
	};
	for (pstate = 0; pstate <= 4; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = \
					(PstateDef.Family_10h.CpuFid + 0x10)
					/ (1 << PstateDef.Family_10h.CpuDid);
	}
}

void Query_AMD_Family_10h(unsigned int cpu)
{
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	Compute_AMD_Family_10h_Boost(cpu);
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 3;

	HyperThreading_Technology();
}

void Compute_AMD_Family_11h_Boost(unsigned int cpu)
{
	unsigned int pstate, sort[8] = {
		BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
		BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
	};
	for (pstate = 0; pstate <= 7; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = \
					(PstateDef.Family_10h.CpuFid + 0x8)
					/ (1 << PstateDef.Family_10h.CpuDid);
	}
}

void Query_AMD_Family_11h(unsigned int cpu)
{
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	Compute_AMD_Family_11h_Boost(cpu);
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;

	HyperThreading_Technology();
}

void Compute_AMD_Family_12h_Boost(unsigned int cpu)
{
	unsigned int pstate, sort[8] = {
		BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
		BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
	};
	for (pstate = 0; pstate <= 7; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = \
					(PstateDef.Family_12h.CpuFid + 0x10)
					/  PstateDef.Family_12h.CpuDid;
	}
}

void Query_AMD_Family_12h(unsigned int cpu)
{
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	Compute_AMD_Family_12h_Boost(cpu);
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;

	HyperThreading_Technology();
}

void Compute_AMD_Family_14h_Boost(unsigned int cpu)
{
	COFVID CofVid = {.value = 0};
	unsigned int MaxFreq = 100, ClockDiv;
	unsigned int pstate, sort[8] = {
		BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
		BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
	};
	RDMSR(CofVid, MSR_AMD_COFVID_STATUS);

	if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
		MaxFreq *= (CofVid.Arch_Pll.MainPllOpFidMax + 0x10);

	for (pstate = 0; pstate <= 7; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));

		ClockDiv = (PstateDef.Family_14h.CpuDidMSD + 1) * 4;
		ClockDiv += PstateDef.Family_14h.CpuDidLSD;

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = (MaxFreq * 4)
							/ ClockDiv;
	}	/*	Frequency @ MainPllOpFidMax (MHz)		*/
}

void Query_AMD_Family_14h(unsigned int cpu)
{
	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	Compute_AMD_Family_14h_Boost(cpu);
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;

	HyperThreading_Technology();
}

inline unsigned int AMD_F15h_CoreCOF(unsigned int FID, unsigned int DID)
{/*	CoreCOF (MHz) = 100 * (CpuFid[5:0] + 10h) / (2 ^ CpuDid)	*/
	unsigned int COF = (FID + 0x10) / (1 << DID);

	return COF;
}

inline unsigned int AMD_F15h_CoreFID(unsigned int COF, unsigned int DID)
{
	unsigned int FID = (COF * (1 << DID)) - 0x10;

	return FID;
}

void Compute_AMD_Family_15h_Boost(unsigned int cpu)
{
    if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
    {
	unsigned int pstate, sort[8] = {
		BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
		BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
	};
	for (pstate = 0; pstate <= 7; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		unsigned int COF;

		RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));

		COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
					PstateDef.Family_15h.CpuDid );

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = COF;
	}
    }
}

void Query_AMD_Family_15h(unsigned int cpu)
{
	Compute_AMD_Family_15h_Boost(cpu);
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;

	HyperThreading_Technology();

  /* Find micro-architecture based on the CPUID model. Bulldozer initialized */
	PRIVATE(OF(Specific)) = LookupProcessor();
    switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
    case 0x0:
      if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
	&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
      {
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	  Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_PILEDRIVER].CodeName,
		CODENAME_LEN);
      }
	break;
    case 0x1:
      if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
	&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
      {
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	  Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_PILEDRIVER].CodeName,
		CODENAME_LEN);
      }
	break;
    case 0x3:
      if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
	&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
      {
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	  Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_STEAMROLLER].CodeName,
			CODENAME_LEN);
      }
	break;
    case 0x6:
    case 0x7:
      if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
	&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
      {
	StrCopy(PUBLIC(RO(Proc))->Architecture,
	    Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_EXCAVATOR].CodeName,
		CODENAME_LEN);
	/*	One thermal sensor through the SMU interface		*/
	PUBLIC(RO(Proc))->thermalFormula = \
				CoreFreqK_Thermal_Scope(FORMULA_SCOPE_PKG);
      }
	break;
    }
    if (PRIVATE(OF(Specific)) != NULL)
    {
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();
}

inline COF_ST AMD_Zen_CoreCOF(unsigned int FID, unsigned int DID)
{/* Source: PPR for AMD Family 17h Model 01h, Revision B1 Processors
    CoreCOF = (PStateDef[CpuFid[7:0]] / PStateDef[CpuDfsId]) * 200	*/
	COF_ST COF;
	if (DID != 0) {
		COF.Q = (FID << 1) / DID;
		COF.R = (UNIT_KHz(1) * (FID - ((COF.Q * DID) >> 1))) >> 2;
	} else {
		COF.Q = FID >> 2;
		COF.R = 0;
	}
	return COF;
}

unsigned short AMD_Zen_Compute_FID_DID( unsigned int COF,
					unsigned int *FID,
					unsigned int *DID )
{
	unsigned int tmp = (*FID);
	if ((*DID) != 0) {
		tmp = (COF * (*DID)) >> 1;
	} else {
		tmp = COF << 2;
	}
	if (tmp < (1 << 8)) {
		(*FID) = tmp;
		return 0;
	} else {
		return 1;
	}
}

inline unsigned short AMD_Zen_CoreFID(	unsigned int COF,
					unsigned int *FID,
					unsigned int *DID )
{
	unsigned short ret = AMD_Zen_Compute_FID_DID(COF, FID, DID);
	while ((ret != 0) && ((*DID) > 0))
	{
		(*DID) = (*DID) >> 1;
		ret = AMD_Zen_Compute_FID_DID(COF, FID, DID);
	}
	return ret;
}

bool Compute_AMD_Zen_Boost(unsigned int cpu)
{
	COF_ST COF = {.Q = 0, .R = 0};
	unsigned int pstate, sort[8] = { /* P[0..7]-States */
		BOOST(MAX), BOOST(1C), BOOST(2C), BOOST(3C),
		BOOST(4C) , BOOST(5C), BOOST(6C), BOOST(7C)
	};
	HWCR HwCfgRegister = {.value = 0};
	PSTATEDEF PstateDef = {.value = 0};
	PSTATECTRL PstateCtrl = {.value = 0};

    for (pstate = BOOST(MIN); pstate < BOOST(SIZE); pstate++) {
		PUBLIC(RO(Core, AT(cpu)))->Boost[pstate] = 0;
    }
    if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
    {
	PSTATELIMIT PstateLimit = {.value = 0};

	RDMSR(PstateLimit, MSR_AMD_PSTATE_CURRENT_LIMIT);

	RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE
			+ PstateLimit.Family_17h.PstateMaxVal));

	COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
				PstateDef.Family_17h.CpuDfsId );

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = COF.Q;
    } else {
	/*Core & L3 frequencies < 400MHz are not supported by the architecture*/
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 4;
    }
	/* Loop over all frequency ids. */
    for (pstate = 0; pstate <= 7; pstate++)
    {
	RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
	/* Handle only valid P-States. */
	if (PstateDef.Family_17h.PstateEn)
	{
		COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId );

		PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = COF.Q;
	}
    }
	PUBLIC(RO(Proc))->Features.SpecTurboRatio = pstate;

	/*		Read the Target P-State				*/
	RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
	RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);

	COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
				PstateDef.Family_17h.CpuDfsId );

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = COF.Q;

	/*	If CPB is enabled then add Boost + XFR to the P0 ratio. */
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (HwCfgRegister.Family_17h.CpbDis == 0)
    {
	AMD_17_ZEN2_COF XtraCOF = {.value = 0};

	switch (PUBLIC(RO(Proc))->ArchID) {
	case AMD_Zen3_VMR:
	case AMD_Zen2_MTS:
	case AMD_Zen2_Ariel:
		Core_AMD_SMN_Read(XtraCOF,
				SMU_AMD_F17H_MATISSE_COF,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H);
		break;
	case AMD_EPYC_Rome_CPK:
		Core_AMD_SMN_Read(XtraCOF,
				SMU_AMD_F17H_ZEN2_MCM_COF,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H);
		break;
	}
	if (XtraCOF.value != 0)
	{
		unsigned int	CPB = XtraCOF.BoostRatio >> 2,
				XFR = !!(XtraCOF.BoostRatio & 0b11);

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] = CPB;
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] = CPB + XFR;
	}
	if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] == 0)
	{
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];

	    if (PRIVATE(OF(Specific)) != NULL)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] += \
						PRIVATE(OF(Specific))->Boost[0];
	    }
	}
	if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] == 0)
	{
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] = \
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];

	    if (PRIVATE(OF(Specific)) != NULL)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] += \
						PRIVATE(OF(Specific))->Boost[0];

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] += \
						PRIVATE(OF(Specific))->Boost[1];
	    }
	}
	return true;	/* Have CPB: inform to add the Xtra Boost ratios */
    } else {
	return false;	/* CPB is disabled				*/
    }
}

typedef struct {
	CLOCK_ARG *pClockMod;
	long	rc;
	unsigned long long PstateAddr;
	unsigned int BoostIndex;
} CLOCK_ZEN_ARG;

static void TargetClock_AMD_Zen_PerCore(void *arg)
{
	CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
	unsigned int cpu = smp_processor_id();
	COF_ST COF = {.Q = 0, .R = 0};
	unsigned int pstate,
		target	= (pClockZen->pClockMod->cpu == -1) ?
			pClockZen->pClockMod->Ratio
		:
			PUBLIC(RO(Core, AT(cpu)))->Boost[pClockZen->BoostIndex]
			+ pClockZen->pClockMod->Offset;

	unsigned short RdWrMSR = 0;

    if (target == 0) {
	pstate = 0;	/*	AUTO Frequency is requested by User	*/
	RdWrMSR = 1;
    } else {
    /* Look-up for the first enabled P-State with the same target frequency */
	for (pstate = 0; pstate <= 7; pstate++)
	{
		PSTATEDEF PstateDef = {.value = 0};
		RDMSR(PstateDef, pClockZen->PstateAddr + pstate);

	    if (PstateDef.Family_17h.PstateEn)
	    {
		COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId );
		if (COF.Q == target) {
			RdWrMSR = 1;
			break;
		}
	    }
	}
    }
    if (RdWrMSR == 1)
    {
	PSTATECTRL PstateCtrl = {.value = 0};
	/*		Command a new target P-state			*/
	RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
	PstateCtrl.PstateCmd = pstate;
	WRMSR(PstateCtrl, MSR_AMD_PERF_CTL);
	pClockZen->rc = RC_OK_COMPUTE;
    } else {
	pClockZen->rc = -RC_PSTATE_NOT_FOUND;
    }
}

static void TurboClock_AMD_Zen_PerCore(void *arg)
{
	CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
	PSTATEDEF PstateDef = {.value = 0};
	HWCR HwCfgRegister = {.value = 0};
	COF_ST COF = {.Q = 0, .R = 0};
	unsigned int FID, DID;
	/*	Make sure the Core Performance Boost is disabled.	*/
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
  if (HwCfgRegister.Family_17h.CpbDis)
  {
	/*	Apply if and only if the P-State is enabled ?		*/
	RDMSR(PstateDef, pClockZen->PstateAddr);
    if (PstateDef.Family_17h.PstateEn)
    {
	if (pClockZen->pClockMod->cpu == -1) { /* Request an absolute COF */
		COF.Q = pClockZen->pClockMod->Ratio;
	} else { /* Compute the requested COF based on a frequency offset */
		COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId );

		COF.Q = COF.Q + pClockZen->pClockMod->Offset;
	}
	FID = PstateDef.Family_17h.CpuFid;
	DID = PstateDef.Family_17h.CpuDfsId;
	/*	Attempt to write the P-State MSR with new FID and DID	*/
	if (AMD_Zen_CoreFID(COF.Q, &FID, &DID) == 0)
	{
		PstateDef.Family_17h.CpuFid = FID;
		PstateDef.Family_17h.CpuDfsId = DID;
		WRMSR(PstateDef, pClockZen->PstateAddr);

		pClockZen->rc = RC_OK_COMPUTE;
	} else {
		pClockZen->rc = -RC_PSTATE_NOT_FOUND;
	}
    } else {
	pClockZen->rc = -ENODEV;
    }
  } else {
	pClockZen->rc = -RC_TURBO_PREREQ;
  }
}

static void BaseClock_AMD_Zen_PerCore(void *arg)
{
	CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
	PSTATEDEF PstateDef = {.value = 0};
	COMPUTE_ARG Compute = { .TSC = { NULL, NULL } };

	Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL);
    if (Compute.TSC[0] == NULL) {
	goto OutOfMemory;
    }
	Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL);
    if (Compute.TSC[1] == NULL) {
	goto OutOfMemory;
    }
	TurboClock_AMD_Zen_PerCore(arg);

    if (pClockZen->rc == RC_OK_COMPUTE)
    {
	const unsigned int cpu = smp_processor_id();
	/*			Calibration Phase One			*/
	RDMSR(PstateDef, pClockZen->PstateAddr);

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
			AMD_Zen_CoreCOF(PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId).Q;

	cpu_data(cpu).loops_per_jiffy = \
		COMPUTE_LPJ(	PUBLIC(RO(Core, AT(cpu)))->Clock.Hz,
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] );
	/*			Calibration Phase Two			*/
	Compute.Clock.Q = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
	Compute.Clock.R = 0;
	Compute.Clock.Hz = 0;

	Compute_TSC(&Compute);

	PUBLIC(RO(Core, AT(cpu)))->Clock = Compute.Clock;
	/*			Calibration Phase Three 		*/
	RDMSR(PstateDef, pClockZen->PstateAddr);

	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
			AMD_Zen_CoreCOF(PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId).Q;

	cpu_data(cpu).loops_per_jiffy = \
		COMPUTE_LPJ(	PUBLIC(RO(Core, AT(cpu)))->Clock.Hz,
				PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] );
    }
OutOfMemory:
    if (Compute.TSC[1] != NULL) {
	kfree(Compute.TSC[1]);
    } else {
	pClockZen->rc = -ENOMEM;
    }
    if (Compute.TSC[0] != NULL) {
	kfree(Compute.TSC[0]);
    } else {
	pClockZen->rc = -ENOMEM;
    }
}

long For_All_AMD_Zen_Clock(CLOCK_ZEN_ARG *pClockZen, void (*PerCore)(void *))
{
	long rc = RC_SUCCESS;
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;

  do {
	cpu--;	/* From last AP to BSP */

    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
    && ((pClockZen->pClockMod->cpu == -1)||(pClockZen->pClockMod->cpu == cpu)))
    {
	smp_call_function_single(cpu, PerCore, pClockZen, 1);
	rc = pClockZen->rc;
    }
  } while ((cpu != 0) && (rc >= RC_SUCCESS)) ;

	return rc;
}

long For_All_AMD_Zen_BaseClock(CLOCK_ZEN_ARG *pClockZen, void (*PerCore)(void*))
{
	long rc;
	unsigned int cpu = PUBLIC(RO(Proc))->Service.Core;

	CoreFreqK_UnRegister_ClockSource();

	rc = For_All_AMD_Zen_Clock(pClockZen, PerCore);

    if (rc == RC_OK_COMPUTE)
    {
	PUBLIC(RO(Proc))->Features.Factory.Clock = (CLOCK) {.Q=0, .R=0, .Hz=0};
	PUBLIC(RO(Proc))->Features.Factory.Ratio = 0;
	PUBLIC(RO(Proc))->Features.Factory.Freq = 0;

	FixMissingRatioAndFrequency(PUBLIC(RO(Core,AT(cpu)))->Boost[BOOST(MAX)],
					&PUBLIC(RO(Core, AT(cpu)))->Clock);

	loops_per_jiffy = cpu_data(cpu).loops_per_jiffy;

	cpu_khz = tsc_khz = (unsigned int) ((loops_per_jiffy * HZ) / 1000LU);
    }
	CoreFreqK_Register_ClockSource(cpu);

	return rc;
}

long TurboClock_AMD_Zen(CLOCK_ARG *pClockMod)
{
  if (pClockMod != NULL) {
    if ((pClockMod->NC >= 1) && (pClockMod->NC <= 7))
    {
	CLOCK_ZEN_ARG ClockZen = {	/* P[1..7]-States allowed	*/
		.pClockMod  = pClockMod,
		.PstateAddr = MSR_AMD_PSTATE_DEF_BASE + pClockMod->NC,
		.BoostIndex = BOOST(SIZE) - pClockMod->NC,
		.rc = RC_SUCCESS
	};
	return For_All_AMD_Zen_Clock(&ClockZen, TurboClock_AMD_Zen_PerCore);
    } else {
	return -RC_UNIMPLEMENTED;
    }
  } else {
	return -EINVAL;
  }
}

long ClockMod_AMD_Zen(CLOCK_ARG *pClockMod)
{
  if (pClockMod != NULL) {
    switch (pClockMod->NC) {
    case CLOCK_MOD_MAX:
      {
	CLOCK_ZEN_ARG ClockZen = {	/* P[0]:Max non-boosted P-State */
		.pClockMod  = pClockMod,
		.PstateAddr = MSR_AMD_PSTATE_DEF_BASE,
		.BoostIndex = BOOST(MAX),
		.rc = RC_SUCCESS
	};
	return(For_All_AMD_Zen_BaseClock(&ClockZen, BaseClock_AMD_Zen_PerCore));
      }
    case CLOCK_MOD_TGT:
      {
		CLOCK_ZEN_ARG ClockZen = {	/* Target non-boosted P-State*/
			.pClockMod  = pClockMod,
			.PstateAddr = MSR_AMD_PSTATE_DEF_BASE,
			.BoostIndex = BOOST(TGT),
			.rc = RC_SUCCESS
		};
	return For_All_AMD_Zen_Clock(&ClockZen, TargetClock_AMD_Zen_PerCore);
      }
    default:
	return -RC_UNIMPLEMENTED;
    }
  } else {
	return -EINVAL;
  }
}

void Query_AMD_F17h_Power_Limits(CORE_RO *Core)
{
	UNUSED(Core);
	/*		Package Power Tracking				*/
	Core_AMD_SMN_Read( PUBLIC(RO(Proc))->PowerThermal.Zen.PWR,
				SMU_AMD_F17H_ZEN2_MCM_PWR,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );
	/*		Junction Temperature				*/
	PUBLIC(RO(Proc))->PowerThermal.Param.Offset[0] = \
				PUBLIC(RO(Proc))->PowerThermal.Zen.PWR.TjMax;
	/*		Thermal Design Power				*/
	Core_AMD_SMN_Read( PUBLIC(RO(Proc))->PowerThermal.Zen.TDP,
				SMU_AMD_F17H_ZEN2_MCM_TDP,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );
	/*		Electric Design Current				*/
	Core_AMD_SMN_Read( PUBLIC(RO(Proc))->PowerThermal.Zen.EDC,
				SMU_AMD_F17H_ZEN2_MCM_EDC,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );
}

void Query_AMD_Family_17h(unsigned int cpu)
{
	unsigned int rx;
	HSMP_ARG arg[8];

	PRIVATE(OF(Specific)) = LookupProcessor();
    if (PRIVATE(OF(Specific)) != NULL)
    {
	/*	Save thermal parameters for later use in the Daemon	*/
	PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
	/*	Override Processor CodeName & Locking capabilities	*/
	OverrideCodeNameString(PRIVATE(OF(Specific)));
	OverrideUnlockCapability(PRIVATE(OF(Specific)));
    } else {
	PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
    }
	Default_Unlock_Reset();

	Query_AMD_F17h_Power_Limits( PUBLIC(RO(Core, AT(cpu))) );

	if (Compute_AMD_Zen_Boost(cpu) == true)
	{	/*	Count the Xtra Boost ratios			*/
		PUBLIC(RO(Proc))->Features.XtraCOF = 2;
	}
	else {	/*	Disabled CPB: Hide ratios			*/
		PUBLIC(RO(Proc))->Features.XtraCOF = 0;
	}
	/*	Apply same register bit fields as Intel RAPL_POWER_UNIT */
	RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_AMD_RAPL_POWER_UNIT);

	HyperThreading_Technology();

	AMD_Processor_PIN(PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PPIN);

	if (PUBLIC(RO(Proc))->Features.HSMP_Capable)
	{ /* Mark the SMU as Enable if the reachability test is successful */
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_TEST_MSG, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 1;
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	    }
	} else {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	}
	if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
	{
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_RD_SMU_VER, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->Features.Factory.SMU.Version = arg[0].value;
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	    }
	}
	if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
	{
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_RD_VERSION, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->Features.Factory.SMU.Interface = arg[0].value;
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	    }
	}
	if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
	{
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_RD_PKG_PL1, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
			PWR_DOMAIN(PKG)
		].Domain_Limit1 = arg[0].value / 1000;

		PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
			PWR_DOMAIN(PKG)
		].Enable_Limit1 = PUBLIC(RO(Proc))->Features.TDP_Unlock = 1;
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable =	\
		PUBLIC(RO(Proc))->Features.TDP_Unlock = 0;
	    }
	}
	if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
	{
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_RD_MAX_PPT, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
			PWR_DOMAIN(PKG)
		].Domain_Limit2 = arg[0].value / 1000;

		PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
			PWR_DOMAIN(PKG)
		].Enable_Limit2 = 1;

		PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
			PWR_DOMAIN(PKG)
		].Clamping2 = 1;
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	    }
	}
}

void Query_AMD_F17h_PerSocket(unsigned int cpu)
{
	Core_AMD_Family_17h_Temp = CTL_AMD_Family_17h_Temp;
	Query_AMD_Family_17h(cpu);
}

void Query_AMD_F17h_PerCluster(unsigned int cpu)
{
	Core_AMD_Family_17h_Temp = CCD_AMD_Family_17h_Zen2_Temp;
	Query_AMD_Family_17h(cpu);
}

void Dump_CPUID(CORE_RO *Core)
{
	unsigned int i;

	__asm__ volatile
	(
		"xorq	%%rax, %%rax	\n\t"
		"xorq	%%rbx, %%rbx	\n\t"
		"xorq	%%rcx, %%rcx	\n\t"
		"xorq	%%rdx, %%rdx	\n\t"
		"cpuid			\n\t"
		"mov	%%eax, %0	\n\t"
		"mov	%%ebx, %1	\n\t"
		"mov	%%ecx, %2	\n\t"
		"mov	%%edx, %3"
		: "=r" (Core->Query.StdFunc.LargestStdFunc),
		  "=r" (Core->Query.StdFunc.BX),
		  "=r" (Core->Query.StdFunc.CX),
		  "=r" (Core->Query.StdFunc.DX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
	__asm__ volatile
	(
		"movq	$0x80000000, %%rax	\n\t"
		"xorq	%%rbx, %%rbx		\n\t"
		"xorq	%%rcx, %%rcx		\n\t"
		"xorq	%%rdx, %%rdx		\n\t"
		"cpuid				\n\t"
		"mov	%%eax, %0		\n\t"
		"mov	%%ebx, %1		\n\t"
		"mov	%%ecx, %2		\n\t"
		"mov	%%edx, %3"
		: "=r" (Core->Query.ExtFunc.LargestExtFunc),
		  "=r" (Core->Query.ExtFunc.EBX),
		  "=r" (Core->Query.ExtFunc.ECX),
		  "=r" (Core->Query.ExtFunc.EDX)
		:
		: "%rax", "%rbx", "%rcx", "%rdx"
	);
/* Intel: CPUID INSTRUCTION
   If a value is entered for CPUID.EAX is invalid for a particular processor,
   the data for the highest basic information leaf is returned.
*/
	for (i = 0; i < CPUID_MAX_FUNC; i++)
	{	__asm__ volatile
		(
			"xorq	%%rax, %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"mov	%4,    %%eax	\n\t"
			"mov	%5,    %%ecx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (Core->CpuID[i].reg[0]),
			  "=r" (Core->CpuID[i].reg[1]),
			  "=r" (Core->CpuID[i].reg[2]),
			  "=r" (Core->CpuID[i].reg[3])
			:  "r" (Core->CpuID[i].func),
			   "r" (Core->CpuID[i].sub)
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
	}
}

void AMD_F17h_DCU_Technology(CORE_RO *Core)			/* Per SMT[?] */
{
	AMD_DC_CFG DC_Cfg1 = {.value = 0};
	AMD_CU_CFG3 CU_Cfg3 = {.value = 0};

	RDMSR(DC_Cfg1, MSR_AMD_DC_CFG);
	switch (L1_HW_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		DC_Cfg1.L1_HW_Prefetch = L1_HW_PREFETCH_Disable;
		WRMSR(DC_Cfg1, MSR_AMD_DC_CFG);
		RDMSR(DC_Cfg1, MSR_AMD_DC_CFG);
		break;
	}

	RDMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
	switch (L2_HW_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		CU_Cfg3.L2_HW_Prefetch = !L2_HW_PREFETCH_Disable;
		WRMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
		RDMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
		break;
	}

    if (DC_Cfg1.L1_HW_Prefetch)
    {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
    }
    if (CU_Cfg3.L2_HW_Prefetch)
    {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
    } else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
    }
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->DCU_Mask, Core->Bind);
}

void Intel_DCU_Technology(CORE_RO *Core)			/*Per Core */
{ /* Avoton[06_4D], GDM[06_5C], NHM[06_1A, 06_1E, 06_1F, 06_2E], SNB+, Phi */
  if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
  {
	int ToggleFeature = 0;
	MISC_FEATURE_CONTROL MiscFeatCtrl = {.value = 0};
	RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);

	switch (L2_HW_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		MiscFeatCtrl.L2_HW_Prefetch = L2_HW_PREFETCH_Disable;
		ToggleFeature = 1;
		break;
	}
	switch (L2_HW_CL_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		MiscFeatCtrl.L2_HW_CL_Prefetch = L2_HW_CL_PREFETCH_Disable;
		ToggleFeature = 1;
		break;
	}
	switch (L1_HW_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		MiscFeatCtrl.L1_HW_Prefetch = L1_HW_PREFETCH_Disable;
		ToggleFeature = 1;
		break;
	}
	switch (L1_HW_IP_PREFETCH_Disable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		MiscFeatCtrl.L1_HW_IP_Prefetch = L1_HW_IP_PREFETCH_Disable;
		ToggleFeature = 1;
		break;
	}
    if (ToggleFeature == 1)
    {
	WRMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
	RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
    }
    if (MiscFeatCtrl.L2_HW_Prefetch == 1) {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
    }
    if (MiscFeatCtrl.L2_HW_CL_Prefetch == 1) {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch, Core->Bind);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch, Core->Bind);
    }
    if (MiscFeatCtrl.L1_HW_Prefetch == 1) {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
    }
    if (MiscFeatCtrl.L1_HW_IP_Prefetch == 1) {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
    }
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->DCU_Mask, Core->Bind);
  }
}

void SpeedStep_Technology(CORE_RO *Core)			/*Per Package*/
{
  if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
    if (PUBLIC(RO(Proc))->Features.Std.ECX.EIST == 1)
    {
	MISC_PROC_FEATURES MiscFeatures = {.value = 0};
	RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);

	switch (SpeedStep_Enable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		MiscFeatures.EIST = SpeedStep_Enable;
		WRMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
		RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
		break;
	}
	if (MiscFeatures.EIST)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
	}
    } else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
    }
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
  }
}

void Intel_Turbo_Config(CORE_RO *Core, void (*ConfigFunc)(void*),/*Per Package*/
			void (*AssignFunc)(unsigned int*, TURBO_CONFIG*))
{
	CLOCK_TURBO_ARG ClockTurbo = {
		.pClockMod = NULL,	/*	Read-Only Operation	*/
		.Config = {.MSR = {.value = 0}},
		.rc = RC_SUCCESS
	};

	ConfigFunc(&ClockTurbo);
	AssignFunc(Core->Boost, &ClockTurbo.Config);
}

typedef void (*SET_TARGET)(CORE_RO*, unsigned int);

static void Set_Core2_Target(CORE_RO *Core, unsigned int ratio)
{
	Core->PowerThermal.PerfControl.CORE.TargetRatio = ratio;
}

static void Set_Nehalem_Target(CORE_RO *Core, unsigned int ratio)
{
	Core->PowerThermal.PerfControl.NHM.TargetRatio = ratio & 0xff;
}

static void Set_SandyBridge_Target(CORE_RO *Core, unsigned int ratio)
{
	Core->PowerThermal.PerfControl.SNB.TargetRatio = ratio;
}

#define GET_CORE2_TARGET(Core)						\
(									\
	Core->PowerThermal.PerfControl.CORE.TargetRatio 		\
)

typedef unsigned int (*GET_TARGET)(CORE_RO*);

static unsigned int Get_Core2_Target(CORE_RO *Core)
{
	return GET_CORE2_TARGET(Core);
}

#define GET_NEHALEM_TARGET(Core)					\
(									\
	Core->PowerThermal.PerfControl.NHM.TargetRatio & 0xff		\
)

static unsigned int Get_Nehalem_Target(CORE_RO *Core)
{
	return GET_NEHALEM_TARGET(Core);
}

#define GET_SANDYBRIDGE_TARGET(Core)					\
(									\
	Core->PowerThermal.PerfControl.SNB.TargetRatio			\
)

static unsigned int Get_SandyBridge_Target(CORE_RO *Core)
{
	return GET_SANDYBRIDGE_TARGET(Core);
}

typedef int (*CMP_TARGET)(CORE_RO*, unsigned int);

static int Cmp_Core2_Target(CORE_RO *Core, unsigned int ratio)
{
	return (Core->PowerThermal.PerfControl.CORE.TargetRatio == ratio);
}

static int Cmp_Nehalem_Target(CORE_RO *Core, unsigned int ratio)
{
	return (Core->PowerThermal.PerfControl.NHM.TargetRatio == ratio);
}

static int Cmp_SandyBridge_Target(CORE_RO *Core, unsigned int ratio)
{
	return (Core->PowerThermal.PerfControl.SNB.TargetRatio > ratio);
}

bool IsPerformanceControlCapable(void)
{
	struct SIGNATURE blackList[] = {
		_Silvermont_Bay_Trail,	/* 06_37 */
		_Atom_Merrifield,	/* 06_4A */
		_Atom_Avoton,		/* 06_4D */
		_Atom_Moorefield,	/* 06_5A */
		_Atom_Sofia		/* 06_5D */
	};
	const int ids = sizeof(blackList) / sizeof(blackList[0]);
	int id;
     for (id = 0; id < ids; id++) {
      if((blackList[id].ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
      && (blackList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
      && (blackList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
      && (blackList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
      {
	return (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq == 1);
      }
     }
	return true;
}

bool WritePerformanceControl(PERF_CONTROL *pPerfControl)
{
	bool isCapable = IsPerformanceControlCapable();
	if (isCapable == true)
	{
		WRMSR((*pPerfControl), MSR_IA32_PERF_CTL);
	}
	return isCapable;
}

void TurboBoost_Technology(CORE_RO *Core,	SET_TARGET SetTarget,
						GET_TARGET GetTarget,
						CMP_TARGET CmpTarget,
						unsigned int TurboRatio,
						unsigned int ValidRatio)
{								/* Per SMT */
	int ToggleFeature;
	MISC_PROC_FEATURES MiscFeatures = {.value = 0};
	RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
	UNUSED(GetTarget);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

  if ( (MiscFeatures.Turbo_IDA == 0)
	&& (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) )
  {
    switch (TurboBoost_Enable) {
    case COREFREQ_TOGGLE_OFF:	/*	Restore the nominal P-state	*/
	Core->PowerThermal.PerfControl.Turbo_IDA = 1;
	SetTarget(Core, Core->Boost[BOOST(MAX)]);
	ToggleFeature = 1;
	break;
    case COREFREQ_TOGGLE_ON:	/*	Request the Turbo P-state	*/
	Core->PowerThermal.PerfControl.Turbo_IDA = 0;
	SetTarget(Core, TurboRatio);
	ToggleFeature = 1;
	break;
    default:
	ToggleFeature = 0;
	break;
    }
    if (ToggleFeature == 1)
    {
	WritePerformanceControl(&Core->PowerThermal.PerfControl);
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
    }
    if (Core->PowerThermal.PerfControl.Turbo_IDA == 0)
    {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
    } else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
    }
  } else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
  }

  if (PUBLIC(RO(Proc))->Features.HWP_Enable)
  {
	RDMSR(Core->PowerThermal.HWP_Capabilities, MSR_IA32_HWP_CAPABILITIES);
	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);

    if (PUBLIC(RO(Proc))->Features.Power.EAX.HWP_EPP)
    {		/*		EPP mode				*/
	if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff))
	{
		Core->PowerThermal.HWP_Request.Energy_Pref = HWP_EPP;
		WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
		RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
	}
    } else {	/*		EPB fallback mode			*/
	if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq
			& (REGISTRATION_ENABLE | REGISTRATION_FULLCTRL) )
	{
		/*	Turbo is a function of the Target P-state	*/
	    if (!CmpTarget(Core, ValidRatio))
	    {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    }
	}
    }
	Core->Boost[BOOST(HWP_MIN)]=Core->PowerThermal.HWP_Request.Minimum_Perf;
	Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
	Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
  } else {	/*			EPB mode			*/
	if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq
			& (REGISTRATION_ENABLE | REGISTRATION_FULLCTRL) )
	{
	    if (!CmpTarget(Core, ValidRatio))
	    {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    }
	}
  }
}

void DynamicAcceleration(CORE_RO *Core) 			/* Unique */
{
  if (IsPerformanceControlCapable() == true)
  {
    if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
    {
	TurboBoost_Technology(	Core,
				Set_Core2_Target,
				Get_Core2_Target,
				Cmp_Core2_Target,
				1 + Core->Boost[BOOST(MAX)],
				1 + Core->Boost[BOOST(MAX)] );
    } else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
    }
  } else {
	int ToggleFeature;

	MISC_PROC_FEATURES MiscFeatures = {.value = 0};
	RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);

	switch (TurboBoost_Enable) {
	case COREFREQ_TOGGLE_OFF:
		MiscFeatures.Turbo_IDA = 1;
		ToggleFeature = 1;
		break;
	case COREFREQ_TOGGLE_ON:
		MiscFeatures.Turbo_IDA = 0;
		ToggleFeature = 1;
		break;
	default:
		ToggleFeature = 0;
		break;
	}
	if (ToggleFeature == 1)
	{
		WRMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
		RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
		/*	Refresh the Turbo capability in the leaf.	*/
	    if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x6) {
		__asm__ volatile
		(
			"movq	$0x6,  %%rax	\n\t"
			"xorq	%%rbx, %%rbx	\n\t"
			"xorq	%%rcx, %%rcx	\n\t"
			"xorq	%%rdx, %%rdx	\n\t"
			"cpuid			\n\t"
			"mov	%%eax, %0	\n\t"
			"mov	%%ebx, %1	\n\t"
			"mov	%%ecx, %2	\n\t"
			"mov	%%edx, %3"
			: "=r" (PUBLIC(RO(Proc))->Features.Power.EAX),
			  "=r" (PUBLIC(RO(Proc))->Features.Power.EBX),
			  "=r" (PUBLIC(RO(Proc))->Features.Power.ECX),
			  "=r" (PUBLIC(RO(Proc))->Features.Power.EDX)
			:
			: "%rax", "%rbx", "%rcx", "%rdx"
		);
	    }
	}
	if (MiscFeatures.Turbo_IDA == 0)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
  }
}

void SoC_Turbo_Override(CORE_RO *Core)
{
	int ToggleFeature;

	PKG_TURBO_CONFIG TurboCfg = {.value = 0};
	RDMSR(TurboCfg, MSR_PKG_TURBO_CFG);

	switch (TurboBoost_Enable) {
	case COREFREQ_TOGGLE_OFF:
		TurboCfg.TjMax_Turbo = 0x0;
		ToggleFeature = 1;
		break;
	case COREFREQ_TOGGLE_ON:
		TurboCfg.TjMax_Turbo = 0x2;
		ToggleFeature = 1;
		break;
	default:
		ToggleFeature = 0;
		break;
	}
	if ((ToggleFeature == 1)
	 && (Core->Bind == PUBLIC(RO(Proc))->Service.Core))	/* Package */
	{
		WRMSR(TurboCfg, MSR_PKG_TURBO_CFG);
		RDMSR(TurboCfg, MSR_PKG_TURBO_CFG);
	}
	if (TurboCfg.TjMax_Turbo == 0x2)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
}

typedef struct {
	CLOCK_ARG *pClockMod;
	SET_TARGET SetTarget;
	GET_TARGET GetTarget;
	long	rc;
} CLOCK_PPC_ARG;

static void ClockMod_PPC_PerCore(void *arg)
{
	CLOCK_PPC_ARG *pClockPPC;
	CORE_RO *Core;
	unsigned int cpu;

	pClockPPC = (CLOCK_PPC_ARG *) arg;
	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

	pClockPPC->SetTarget(Core, (pClockPPC->pClockMod->cpu == -1) ?
		pClockPPC->pClockMod->Ratio
	:	pClockPPC->GetTarget(Core) + pClockPPC->pClockMod->Offset);

	if (WritePerformanceControl(&Core->PowerThermal.PerfControl) == true) {
		pClockPPC->rc = RC_OK_COMPUTE;
	} else {
		pClockPPC->rc = -EINTR;
	}
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

	Core->Boost[BOOST(TGT)] = pClockPPC->GetTarget(Core);
}

long For_All_PPC_Clock(CLOCK_PPC_ARG *pClockPPC)
{
	long rc = RC_SUCCESS;
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
  do {
	cpu--;	/*		From last AP to BSP			*/

    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
    && ((pClockPPC->pClockMod->cpu == -1)||(pClockPPC->pClockMod->cpu == cpu)))
    {
	smp_call_function_single(cpu, ClockMod_PPC_PerCore, pClockPPC, 1);
	rc = pClockPPC->rc;
    }
  } while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
	return rc;
}

long ClockMod_Core2_PPC(CLOCK_ARG *pClockMod)
{
	if (pClockMod != NULL) {
		if (pClockMod->NC == CLOCK_MOD_TGT)
		{
			CLOCK_PPC_ARG ClockPPC = {
				.pClockMod = pClockMod,
				.SetTarget = Set_Core2_Target,
				.GetTarget = Get_Core2_Target,
				.rc = RC_SUCCESS
			};
			return For_All_PPC_Clock(&ClockPPC);
		} else {
			return -RC_UNIMPLEMENTED;
		}
	} else {
		return -EINVAL;
	}
}

long ClockMod_Nehalem_PPC(CLOCK_ARG *pClockMod)
{
	if (pClockMod != NULL) {
		if (pClockMod->NC == CLOCK_MOD_TGT)
		{
			CLOCK_PPC_ARG ClockPPC = {
				.pClockMod = pClockMod,
				.SetTarget = Set_Nehalem_Target,
				.GetTarget = Get_Nehalem_Target,
				.rc = RC_SUCCESS
			};
			return For_All_PPC_Clock(&ClockPPC);
		} else {
			return -RC_UNIMPLEMENTED;
		}
	} else {
		return -EINVAL;
	}
}

long ClockMod_SandyBridge_PPC(CLOCK_ARG *pClockMod)
{
	if (pClockMod != NULL) {
		if (pClockMod->NC == CLOCK_MOD_TGT)
		{
			CLOCK_PPC_ARG ClockPPC = {
				.pClockMod = pClockMod,
				.SetTarget = Set_SandyBridge_Target,
				.GetTarget = Get_SandyBridge_Target,
				.rc = RC_SUCCESS
			};
			return For_All_PPC_Clock(&ClockPPC);
		} else {
			return -RC_UNIMPLEMENTED;
		}
	} else {
		return -EINVAL;
	}
}

typedef struct {
	CLOCK_ARG *pClockMod;
	long	rc;
} CLOCK_HWP_ARG;

static void ClockMod_HWP_PerCore(void *arg)
{
	CLOCK_HWP_ARG *pClockHWP;
	CORE_RO *Core;
	unsigned int cpu;
	unsigned short WrRdHWP;

	pClockHWP = (CLOCK_HWP_ARG *) arg;
	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);

  switch (pClockHWP->pClockMod->NC) {
  case CLOCK_MOD_HWP_MIN:
    if (pClockHWP->pClockMod->cpu == -1) {
    Core->PowerThermal.HWP_Request.Minimum_Perf = pClockHWP->pClockMod->Ratio;
    } else {
    Core->PowerThermal.HWP_Request.Minimum_Perf += pClockHWP->pClockMod->Offset;
    }
	WrRdHWP = 1;
	break;
  case CLOCK_MOD_HWP_MAX:
    if (pClockHWP->pClockMod->cpu == -1) {
    Core->PowerThermal.HWP_Request.Maximum_Perf = pClockHWP->pClockMod->Ratio;
    } else {
    Core->PowerThermal.HWP_Request.Maximum_Perf += pClockHWP->pClockMod->Offset;
    }
	WrRdHWP = 1;
	break;
  case CLOCK_MOD_HWP_TGT:
    if (pClockHWP->pClockMod->cpu == -1) {
    Core->PowerThermal.HWP_Request.Desired_Perf = pClockHWP->pClockMod->Ratio;
    } else {
    Core->PowerThermal.HWP_Request.Desired_Perf += pClockHWP->pClockMod->Offset;
    }
	WrRdHWP = 1;
	break;
  default:
	WrRdHWP = 0;
	pClockHWP->rc = -RC_UNIMPLEMENTED;
	break;
  }
  if (WrRdHWP == 1)
  {
	WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
	pClockHWP->rc = RC_OK_COMPUTE;
  }
}

long For_All_HWP_Clock(CLOCK_HWP_ARG *pClockHWP)
{
	long rc = RC_SUCCESS;
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
  do {
	cpu--;	/*	From last AP to BSP				*/

    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
    && ((pClockHWP->pClockMod->cpu == -1)||(pClockHWP->pClockMod->cpu == cpu)))
	{
	smp_call_function_single(cpu, ClockMod_HWP_PerCore, pClockHWP, 1);
	rc = pClockHWP->rc;
	}
  } while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
	return rc;
}

long ClockMod_Intel_HWP(CLOCK_ARG *pClockMod)
{
	if (PUBLIC(RO(Proc))->Features.HWP_Enable) {
		if (pClockMod != NULL) {
			switch (pClockMod->NC) {
			case CLOCK_MOD_HWP_MIN:
			case CLOCK_MOD_HWP_MAX:
			case CLOCK_MOD_HWP_TGT:
			    {
				CLOCK_HWP_ARG ClockHWP = {
					.pClockMod = pClockMod,
					.rc = RC_SUCCESS
				};
				return For_All_HWP_Clock(&ClockHWP);
			    }
			case CLOCK_MOD_TGT:
				return ClockMod_SandyBridge_PPC(pClockMod);
			default:
				return -RC_UNIMPLEMENTED;
			}
		} else {
			return -EINVAL;
		}
	} else {
		return ClockMod_SandyBridge_PPC(pClockMod);
	}
}


void AMD_Watchdog(CORE_RO *Core)
{	/*		CPU Watchdog Timer.				*/
	if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
	{
		AMD_CPU_WDT_CFG CPU_WDT_CFG = {.value = 0};
		RDMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);

		switch (WDT_Enable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			CPU_WDT_CFG.TmrCfgEn = WDT_Enable;
			WRMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);
			RDMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);
			break;
		}
		if (CPU_WDT_CFG.TmrCfgEn) {
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->WDT, Core->Bind);
		} else {
			BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->WDT, Core->Bind);
		}
		BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
	}
}

void PerCore_AMD_CState_BAR(CORE_RO *Core)
{	/* Families: 10h, 12h, 14h, 15h, 16h, 17h: I/O C-State Base Address. */
	CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
	RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
	Core->Query.CStateBaseAddr = CStateBaseAddr.IOaddr;
}

void PerCore_Query_AMD_Zen_Features(CORE_RO *Core)		/* Per SMT */
{
	ZEN_CSTATE_CONFIG CStateCfg = {.value = 0};
	int ToggleFeature;

	/*		Read The Hardware Configuration Register.	*/
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);

	/* Query the SMM. */
	if (HwCfgRegister.Family_17h.SmmLock) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SMM, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SMM, Core->Bind);
	}
	/*		Enable or Disable the Core Performance Boost.	*/
	switch (TurboBoost_Enable) {
	case COREFREQ_TOGGLE_OFF:
		HwCfgRegister.Family_17h.CpbDis = 1;
		ToggleFeature = 1;
		break;
	case COREFREQ_TOGGLE_ON:
		HwCfgRegister.Family_17h.CpbDis = 0;
		ToggleFeature = 1;
		break;
	default:
		ToggleFeature = 0;
		break;
	}
	if (ToggleFeature == 1)
	{
		WRMSR(HwCfgRegister, MSR_K7_HWCR);
		RDMSR(HwCfgRegister, MSR_K7_HWCR);
	}
	if (HwCfgRegister.Family_17h.CpbDis == 0)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);

	/*	Enable or Disable the Core C6 State. Bit[22,14,16]	*/
	RDMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);

	switch (CC6_Enable) {
	case COREFREQ_TOGGLE_OFF:
		CStateCfg.CCR2_CC6EN = 0;
		CStateCfg.CCR1_CC6EN = 0;
		CStateCfg.CCR0_CC6EN = 0;
		ToggleFeature = 1;
		break;
	case COREFREQ_TOGGLE_ON:
		CStateCfg.CCR2_CC6EN = 1;
		CStateCfg.CCR1_CC6EN = 1;
		CStateCfg.CCR0_CC6EN = 1;
		ToggleFeature = 1;
		break;
	default:
		ToggleFeature = 0;
		break;
	}
	if (ToggleFeature == 1)
	{
		WRMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);
		RDMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);
	}
	if (CStateCfg.CCR2_CC6EN && CStateCfg.CCR1_CC6EN && CStateCfg.CCR0_CC6EN
		&& HwCfgRegister.Family_17h.INVDWBINVD)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->CC6, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->CC6, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	/*	Enable or Disable the Package C6 State . Bit[32]	*/
    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	ZEN_PMGT_MISC PmgtMisc = {.value = 0};
	RDMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);

	switch (PC6_Enable) {
	case COREFREQ_TOGGLE_OFF:
		PmgtMisc.PC6En = 0;
		ToggleFeature = 1;
		break;
	case COREFREQ_TOGGLE_ON:
		PmgtMisc.PC6En = 1;
		ToggleFeature = 1;
		break;
	default:
		ToggleFeature = 0;
		break;
	}
	if (ToggleFeature == 1) {
		WRMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);
		RDMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);
	}
	if (PmgtMisc.PC6En == 1)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->PC6, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PC6, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PC6_Mask, Core->Bind);

	if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
	{
		unsigned int rx;
		HSMP_ARG arg[8];
		RESET_ARRAY(arg, 8, 0, .value);
		rx = AMD_HSMP_Exec(	HSMP_RD_PROCHOT, arg, SMU_HSMP_F19H,
					SMU_AMD_INDEX_REGISTER_F17H,
					SMU_AMD_DATA_REGISTER_F17H );
	    if (rx == HSMP_RESULT_OK)
	    {
		PUBLIC(RO(Proc))->PowerThermal.Events=((arg[0].value & 0x1)<<1);
	    }
	    else if (IS_HSMP_OOO(rx))
	    {
		PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
	    }
	}
    }
	/*		SMT C-State Base Address.			*/
	PerCore_AMD_CState_BAR(Core);
	/*		Package C-State: Configuration Control .	*/
	Core->Query.CfgLock = 1;
	/*		Package C-State: I/O MWAIT Redirection .	*/
	Core->Query.IORedir = 0;

	AMD_Watchdog(Core);
}

void Intel_Watchdog(CORE_RO *Core)
{
	struct pci_device_id PCI_WDT_ids[] = {
		{
		PCI_VDEVICE(INTEL, DID_INTEL_ICH10_LPC),
		.driver_data = (kernel_ulong_t) ICH_TCO
		},
		{0, }
	};
	UNUSED(Core);

	CoreFreqK_ProbePCI(PCI_WDT_ids, NULL, NULL);
}

void Intel_Turbo_Activation_Ratio(CORE_RO *Core)
{
	TURBO_ACTIVATION TurboActivation = {.value = 0};

	RDMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);

    if (!TurboActivation.Ratio_Lock && (Turbo_Activation_Ratio >= 0)
     && (Turbo_Activation_Ratio != TurboActivation.MaxRatio))
    {
	const short MaxRatio = \
	MAXCLOCK_TO_RATIO(short, PUBLIC(RO(Core, AT(Core->Bind))->Clock.Hz));

	if (Turbo_Activation_Ratio <=  MaxRatio)
	{
		TurboActivation.MaxRatio = Turbo_Activation_Ratio;
		WRMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);
		RDMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);
	}
    }
	Core->Boost[BOOST(ACT)] = TurboActivation.MaxRatio;

    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	PUBLIC(RO(Proc))->Features.TurboActiv_Lock = TurboActivation.Ratio_Lock;
    }
}

void Intel_Turbo_TDP_Config(CORE_RO *Core)
{
	CONFIG_TDP_NOMINAL NominalTDP = {.value = 0};
	CONFIG_TDP_CONTROL ControlTDP = {.value = 0};
	CONFIG_TDP_LEVEL ConfigTDP[2] = {{.value = 0}, {.value = 0}};

	RDMSR(NominalTDP, MSR_CONFIG_TDP_NOMINAL);
	Core->Boost[BOOST(TDP)] = NominalTDP.Ratio;

	RDMSR(ConfigTDP[1], MSR_CONFIG_TDP_LEVEL_2);
	Core->Boost[BOOST(TDP2)] = ConfigTDP[1].Ratio;

	RDMSR(ConfigTDP[0], MSR_CONFIG_TDP_LEVEL_1);
	Core->Boost[BOOST(TDP1)] = ConfigTDP[0].Ratio;

  if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
  {
	RDMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);

	if ((ControlTDP.Lock == 0) && (Config_TDP_Level >= 0))
	{
		ControlTDP.Level = Config_TDP_Level;
		WRMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);
		RDMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);
	}
	PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock  = ControlTDP.Lock;
	PUBLIC(RO(Proc))->Features.TDP_Cfg_Level = ControlTDP.Level;

    switch (PUBLIC(RO(Proc))->Features.TDP_Cfg_Level) {
    case 2:
	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
							ConfigTDP[1].PkgPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
							ConfigTDP[1].MinPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
							ConfigTDP[1].MaxPower;
	break;
    case 1:
	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
							ConfigTDP[0].PkgPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
							ConfigTDP[0].MinPower;

	PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
							ConfigTDP[0].MaxPower;
	break;
    case 0:
	/*			MSR_PKG_POWER_INFO			*/
	break;
    }
  }
}

void Query_Intel_C1E(CORE_RO *Core)				/*Per Package*/
{
	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		POWER_CONTROL PowerCtrl = {.value = 0};
		RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);

		switch (C1E_Enable) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				PowerCtrl.C1E = C1E_Enable;
				WRMSR(PowerCtrl, MSR_IA32_POWER_CTL);
				RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
			break;
		}
		if (PowerCtrl.C1E)
		{
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1E, Core->Bind);
		} else {
			BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1E, Core->Bind);
		}
		BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
	}
}

void Query_AMD_Family_0Fh_C1E(CORE_RO *Core)			/* Per Core */
{
	INT_PENDING_MSG IntPendingMsg = {.value = 0};

	RDMSR(IntPendingMsg, MSR_K8_INT_PENDING_MSG);

	if (IntPendingMsg.C1eOnCmpHalt & !IntPendingMsg.SmiOnCmpHalt)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1E, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1E, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
}

void ThermalMonitor2_Set(CORE_RO *Core, MISC_PROC_FEATURES MiscFeatures)
{	/* Intel Core Solo Duo. */
	struct SIGNATURE whiteList[] = {
		_Core_Yonah,		/* 06_0E */
		_Core_Conroe,		/* 06_0F */
		_Core_Penryn,		/* 06_17 */
		_Atom_Bonnell,		/* 06_1C */
		_Atom_Silvermont,	/* 06_26 */
		_Atom_Lincroft,		/* 06_27 */
		_Atom_Clover_Trail,	/* 06_35 */
		_Atom_Saltwell,		/* 06_36 */
	};
	int id, ids = sizeof(whiteList) / sizeof(whiteList[0]);
  for (id = 0; id < ids; id++)
  {
    if((whiteList[id].ExtFamily == PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
    && (whiteList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
    && (whiteList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
    && (whiteList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
    {
	if (MiscFeatures.TCC)
	{
		THERM2_CONTROL Therm2Control = {.value = 0};

		RDMSR(Therm2Control, MSR_THERM2_CTL);

		if (Therm2Control.TM_SELECT)
		{
			BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1, Core->Bind);
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
		} else {
			BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
		}
	}
	break;
    }
  }
}

void ThermalMonitor_Set(CORE_RO *Core)
{
	TJMAX TjMax = {.value = 0};
	MISC_PROC_FEATURES MiscFeatures = {.value = 0};
	THERM_STATUS ThermStatus = {.value = 0};
	PLATFORM_INFO PfInfo = {.value = 0};
	int ClearBit;
	/* Silvermont + Xeon[06_57] + Nehalem + Sandy Bridge & superior arch. */
	RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);

	Core->PowerThermal.Param.Offset[0] = TjMax.Target;
	if (Core->PowerThermal.Param.Offset[0] == 0)
	{
		Core->PowerThermal.Param.Offset[0] = 100;
	}
	/*		Query the TM1 and TM2 features state.		*/
	RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
	if (MiscFeatures.TCC) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1, Core->Bind);
	}
	if (MiscFeatures.TM2_Enable) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
	}
	ThermalMonitor2_Set(Core, MiscFeatures);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask, Core->Bind);
	/*	Clear Thermal Events if requested by User.		*/
	ClearBit = 0;
	RDMSR(ThermStatus, MSR_IA32_THERM_STATUS);

	if (Clear_Events & EVENT_THERM_SENSOR) {
		ThermStatus.StatusLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_PROCHOT) {
		ThermStatus.PROCHOTLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_CRIT) {
		ThermStatus.CriticalTempLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_THOLD) {
		ThermStatus.Threshold1Log = 0;
		ThermStatus.Threshold2Log = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_POWER_LIMIT) {
		ThermStatus.PwrLimitLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_CURRENT_LIMIT) {
		ThermStatus.CurLimitLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_CROSS_DOMAIN) {
		ThermStatus.XDomLimitLog = 0;
		ClearBit = 1;
	}
	if (ClearBit)
	{
		THERM_INTERRUPT ThermInterrupt = {.value = 0};
		RDMSR(ThermInterrupt, MSR_IA32_THERM_INTERRUPT);
		if (!(ThermInterrupt.High_Temp_Int|ThermInterrupt.Low_Temp_Int))
		{
			WRMSR(ThermStatus, MSR_IA32_THERM_STATUS);
			RDMSR(ThermStatus, MSR_IA32_THERM_STATUS);
		}
	}
	Core->PowerThermal.Events = \
		( (ThermStatus.StatusBit | ThermStatus.StatusLog ) << 0 )
		| (ThermStatus.PROCHOTLog << 1)
		| (ThermStatus.CriticalTempLog << 2)
		| ((ThermStatus.Threshold1Log | ThermStatus.Threshold2Log) << 3)
		| (ThermStatus.PwrLimitLog << 4)
		| (ThermStatus.CurLimitLog << 5)
		| (ThermStatus.XDomLimitLog << 6);

    if (PUBLIC(RO(Proc))->Features.Power.EAX.PTM
    && (Core->Bind == PUBLIC(RO(Proc))->Service.Core))
    {
	ClearBit = 0;
	ThermStatus.value = 0;
	RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);

	if (Clear_Events & EVENT_THERM_SENSOR) {
		ThermStatus.StatusLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_PROCHOT) {
		ThermStatus.PROCHOTLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_CRIT) {
		ThermStatus.CriticalTempLog = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_THERM_THOLD) {
		ThermStatus.Threshold1Log = 0;
		ThermStatus.Threshold2Log = 0;
		ClearBit = 1;
	}
	if (Clear_Events & EVENT_POWER_LIMIT) {
		ThermStatus.PwrLimitLog = 0;
		ClearBit = 1;
	}
	if (ClearBit)
	{
		THERM_INTERRUPT ThermInterrupt = {.value = 0};
		RDMSR(ThermInterrupt, MSR_IA32_PACKAGE_THERM_INTERRUPT);
		if (!(ThermInterrupt.High_Temp_Int|ThermInterrupt.Low_Temp_Int))
		{
			WRMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);
			RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);
		}
	}
	PUBLIC(RO(Proc))->PowerThermal.Events = \
		( (ThermStatus.StatusBit | ThermStatus.StatusLog) << 0 )
		| (ThermStatus.PROCHOTLog << 1)
		| (ThermStatus.CriticalTempLog << 2)
		| ((ThermStatus.Threshold1Log | ThermStatus.Threshold2Log) << 3)
		| (ThermStatus.PwrLimitLog << 4);
    }

	RDMSR(PfInfo, MSR_PLATFORM_INFO);

	if (PfInfo.ProgrammableTj)
	{
		switch (PUBLIC(RO(Proc))->ArchID) {
		case Atom_Goldmont:
		case Xeon_Phi:
	/*TODO( case 06_85h: )	*/
			Core->PowerThermal.Param.Offset[1] = TjMax.Atom.Offset;
			break;
		case IvyBridge_EP:
		case Broadwell_EP:
		case Broadwell_D:
		case Skylake_X:
			Core->PowerThermal.Param.Offset[1] = TjMax.EP.Offset;
			break;
		default:
			Core->PowerThermal.Param.Offset[1] = TjMax.Offset;
			break;
		case Core_Yonah ... Core_Dunnington:
			break;
		}
	}
}

void PowerThermal(CORE_RO *Core)
{
	struct {
		struct SIGNATURE Arch;
		unsigned short	grantPWR_MGMT	:  1-0,
				grantODCM	:  2-1,
				experimental	:  3-2,
				freeToUse	: 16-3;
	} whiteList[] = {
		{_Core_Yonah,		0, 1, 1, 0},
		{_Core_Conroe,		0, 1, 0, 0},
		{_Core_Kentsfield,	0, 1, 1, 0},
		{_Core_Conroe_616,	0, 1, 1, 0},
		{_Core_Penryn,		0, 1, 0, 0},
		{_Core_Dunnington,	0, 1, 1, 0},

		{_Atom_Bonnell ,	0, 1, 1, 0},	/* 06_1C */
		{_Atom_Silvermont,	0, 1, 1, 0},	/* 06_26 */
		{_Atom_Lincroft,	0, 1, 1, 0},	/* 06_27 */
		{_Atom_Clover_Trail,	0, 1, 1, 0},	/* 06_35 */
		{_Atom_Saltwell,	0, 1, 1, 0},	/* 06_36 */

		{_Silvermont_Bay_Trail, 0, 1, 0, 0},	/* 06_37 */

		{_Atom_Avoton,		0, 1, 1, 0},	/* 06_4D */
		{_Atom_Airmont ,	0, 0, 1, 0},	/* 06_4C */
		{_Atom_Goldmont,	1, 0, 1, 0},	/* 06_5C */
		{_Atom_Sofia,		0, 1, 1, 0},	/* 06_5D */
		{_Atom_Merrifield,	0, 1, 1, 0},	/* 06_4A */
		{_Atom_Moorefield,	0, 1, 1, 0},	/* 06_5A */

		{_Nehalem_Bloomfield,	1, 1, 0, 0},	/* 06_1A */
		{_Nehalem_Lynnfield,	1, 1, 0, 0},	/* 06_1E */
		{_Nehalem_MB,		1, 1, 0, 0},	/* 06_1F */
		{_Nehalem_EX,		1, 1, 0, 0},	/* 06_2E */

		{_Westmere,		1, 1, 0, 0},	/* 06_25 */
		{_Westmere_EP,		1, 1, 0, 0},	/* 06_2C */
		{_Westmere_EX,		1, 1, 0, 0},	/* 06_2F */

		{_SandyBridge,		1, 1, 0, 0},	/* 06_2A */
		{_SandyBridge_EP,	1, 1, 0, 0},	/* 06_2D */

		{_IvyBridge,		1, 0, 1, 0},	/* 06_3A */
		{_IvyBridge_EP ,	1, 1, 0, 0},	/* 06_3E */

		{_Haswell_DT,		1, 1, 0, 0},	/* 06_3C */
		{_Haswell_EP,		1, 1, 1, 0},	/* 06_3F */
		{_Haswell_ULT,		1, 1, 0, 0},	/* 06_45 */
		{_Haswell_ULX,		1, 1, 1, 0},	/* 06_46 */

		{_Broadwell,		1, 1, 1, 0},	/* 06_3D */
		{_Broadwell_D,		1, 1, 1, 0},	/* 06_56 */
		{_Broadwell_H,		1, 1, 1, 0},	/* 06_47 */
		{_Broadwell_EP ,	1, 1, 1, 0},	/* 06_4F */

		{_Skylake_UY,		1, 1, 1, 0},	/* 06_4E */
		{_Skylake_S,		1, 0, 0, 0},	/* 06_5E */
		{_Skylake_X,		1, 1, 1, 0},	/* 06_55 */

		{_Xeon_Phi,		0, 1, 1, 0},	/* 06_57 */

		{_Kabylake,		1, 1, 0, 0},	/* 06_9E */
		{_Kabylake_UY,		1, 1, 1, 0},	/* 06_8E */

		{_Cannonlake,		1, 1, 1, 0},	/* 06_66 */
		{_Geminilake,		1, 0, 1, 0},	/* 06_7A */
		{_Icelake_UY,		1, 1, 1, 0},	/* 06_7E */

		{_Icelake_X,		1, 1, 1, 0},
		{_Icelake_D,		1, 1, 1, 0},
		{_Sunny_Cove,		1, 1, 1, 0},
		{_Tigerlake,		1, 1, 1, 0},
		{_Tigerlake_U,		1, 1, 0, 0},	/* 06_8C */
		{_Cometlake,		1, 1, 1, 0},
		{_Cometlake_UY,		1, 1, 1, 0},
		{_Atom_Denverton,	1, 1, 1, 0},
		{_Tremont_Jacobsville,	1, 1, 1, 0},
		{_Tremont_Lakefield,	1, 1, 1, 0},
		{_Tremont_Elkhartlake,	1, 1, 1, 0},
		{_Tremont_Jasperlake,	1, 1, 1, 0},
		{_Sapphire_Rapids,	1, 1, 1, 0},
		{_Rocketlake,		1, 1, 1, 0},
		{_Rocketlake_U,		1, 1, 1, 0}
	};
	unsigned int id, ids = sizeof(whiteList) / sizeof(whiteList[0]);
 for (id = 0; id < ids; id++)
 {
 if((whiteList[id].Arch.ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
  && (whiteList[id].Arch.Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
  && (whiteList[id].Arch.ExtModel==PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
  && (whiteList[id].Arch.Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
  {
	break;
  }
 }
  if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x6)
  {
    struct THERMAL_POWER_LEAF Power = {
	.EAX = {0}, .EBX = {0}, .ECX = {{0}}, .EDX = {0}
    };

    __asm__ volatile
    (
	"movq	$0x6,  %%rax	\n\t"
	"xorq	%%rbx, %%rbx	\n\t"
	"xorq	%%rcx, %%rcx	\n\t"
	"xorq	%%rdx, %%rdx	\n\t"
	"cpuid			\n\t"
	"mov	%%eax, %0	\n\t"
	"mov	%%ebx, %1	\n\t"
	"mov	%%ecx, %2	\n\t"
	"mov	%%edx, %3"
	: "=r" (Power.EAX),
	  "=r" (Power.EBX),
	  "=r" (Power.ECX),
	  "=r" (Power.EDX)
	:
	: "%rax", "%rbx", "%rcx", "%rdx"
    );
    if (Power.ECX.SETBH == 1)
    {
      if ((id < ids) && (whiteList[id].grantPWR_MGMT == 1))
      {
	RDMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
	RDMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);

	switch (PowerMGMT_Unlock) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
	  if (!whiteList[id].experimental
	   || (whiteList[id].experimental
	   && PUBLIC(RO(Proc))->Registration.Experimental))
	  {
	    Core->PowerThermal.PwrManagement.Perf_BIAS_Enable=PowerMGMT_Unlock;
	    WRMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);
	    RDMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);
	  }
	     break;
	}
	if (Core->PowerThermal.PwrManagement.Perf_BIAS_Enable)
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
	else
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
      } else
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);

      if ((PowerPolicy >= 0) && (PowerPolicy <= 15))
      {
	if (!whiteList[id].experimental
	 || (whiteList[id].experimental
	 && PUBLIC(RO(Proc))->Registration.Experimental))
	{
	Core->PowerThermal.PerfEnergyBias.PowerPolicy = PowerPolicy;
	WRMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
	RDMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
	}
      }
    }
    else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
    }
    if ((PUBLIC(RO(Proc))->Features.Std.EDX.ACPI == 1)
     && (id < ids) && (whiteList[id].grantODCM == 1))
    {
	CLOCK_MODULATION ClockModulation = {.value = 0};
	int ToggleFeature = 0;

	RDMSR(Core->PowerThermal.ClockModulation, MSR_IA32_THERM_CONTROL);
	ClockModulation = Core->PowerThermal.ClockModulation;

	switch (ODCM_Enable) {
	case COREFREQ_TOGGLE_OFF:
	case COREFREQ_TOGGLE_ON:
		ClockModulation.ODCM_Enable = ODCM_Enable;
		ToggleFeature = 1;
		break;
	}
	if ((ODCM_DutyCycle >= 0)
	 && (ODCM_DutyCycle <= (7 << Power.EAX.ECMD)))
	{
	    ClockModulation.DutyCycle = ODCM_DutyCycle << !Power.EAX.ECMD;
	    ToggleFeature = 1;
	}
	if (ToggleFeature == 1)
	{
	  if (!whiteList[id].experimental
	   || (whiteList[id].experimental
	   && PUBLIC(RO(Proc))->Registration.Experimental))
	  {
	    WRMSR(ClockModulation, MSR_IA32_THERM_CONTROL);
	    RDMSR(Core->PowerThermal.ClockModulation, MSR_IA32_THERM_CONTROL);
	  }
	}
	Core->PowerThermal.ClockModulation.ECMD = Power.EAX.ECMD;
	if (Core->PowerThermal.ClockModulation.ODCM_Enable)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->ODCM, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->ODCM, Core->Bind);
	}
    }
    else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->ODCM, Core->Bind);
    }
  }
  else {
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->ODCM, Core->Bind);
  }
  BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask, Core->Bind);
  BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
}

#define UNSPEC 0b11111111

struct CSTATES_ENCODING_ST {
	enum CSTATES_ENCODING	enc;
	unsigned short int	dec;
} Limit_CSTATES_NHM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0001 },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0010 }, /*Cannot be used to limit package C-State*/
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },
	{ _C7S	, UNSPEC },
	{  _C8	, UNSPEC },
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, IORedir_CSTATES_NHM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, UNSPEC },
	{  _C1	, UNSPEC },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0000 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0001 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0010 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0011 },	/* TODO(Undefined!)	*/
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, Limit_CSTATES_SNB[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0000 },
	{  _C2	, 0b0001 },
	{  _C3	, 0b0010 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },
	{ _C7S	, 0b0101 },
	{  _C8	, 0b0110 },
	{  _C9	, 0b0111 },
	{ _C10	, 0b1000 }
}, IORedir_CSTATES_SNB[CSTATES_ENCODING_COUNT] = {
	{  _C0	, UNSPEC },
	{  _C1	, UNSPEC },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0000 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0001 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0010 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0011 },	/* TODO(Untested?)	*/
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, Limit_CSTATES_ULT[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0000 },
	{  _C2	, 0b0001 },
	{  _C3	, 0b0010 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0110 },
	{  _C9	, 0b0111 },
	{ _C10	, 0b1000 }
}, IORedir_CSTATES_ULT[CSTATES_ENCODING_COUNT] = {
	{  _C0	, UNSPEC },
	{  _C1	, UNSPEC },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0000 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0001 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0010 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0011 },	/* TODO(Untested?)	*/
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, Limit_CSTATES_SKL[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0000 },
	{  _C2	, 0b0001 },
	{  _C3	, 0b0010 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0110 },
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, IORedir_CSTATES_SKL[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0001 },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0010 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0101 },
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, Limit_CSTATES_SOC_SLM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },	/* Silvermont, Airmont	*/
	{  _C1	, 0b0001 },	/* Silvermont, Airmont	*/
	{  _C2	, 0b0010 },	/* Airmont		*/
	{  _C3	, UNSPEC },
	{  _C4	, 0b0100 },	/* Silvermont		*/
	{  _C6	, 0b0110 },	/* Silvermont, Airmont	*/
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0111 },	/* Silvermont, Airmont	*/
	{ _C7S	, UNSPEC },
	{  _C8	, UNSPEC },
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, IORedir_CSTATES_SOC_SLM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, UNSPEC },
	{  _C1	, UNSPEC },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0000 },	/* Airmont		*/
	{  _C4	, 0b0100 },	/* Silvermont: 0b0100	*/
	{  _C6	, 0b0001 },	/* Silvermont: 0b0110, Atom Deep Power Down */
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0010 },	/* Silvermont: 0b0111, Airmont: 0b0010	*/
	{ _C7S	, UNSPEC },
	{  _C8	, UNSPEC },
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
}, Limit_CSTATES_SOC_GDM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, 0b0000 },
	{  _C1	, 0b0001 },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0010 },	/* Goldmont, Tremont	*/
	{  _C4	, UNSPEC },
	{  _C6	, 0b0011 },	/* Goldmont, Tremont	*/
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0100 },	/* Goldmont, Tremont	*/
	{ _C7S	, 0b0101 },	/* Goldmont, Tremont	*/
	{  _C8	, 0b0110 },	/* Goldmont, Tremont	*/
	{  _C9	, 0b0111 },	/* Goldmont, Tremont	*/
	{ _C10	, 0b1000 }
}, IORedir_CSTATES_SOC_GDM[CSTATES_ENCODING_COUNT] = {
	{  _C0	, UNSPEC },
	{  _C1	, UNSPEC },
	{  _C2	, UNSPEC },
	{  _C3	, 0b0000 },
	{  _C4	, UNSPEC },
	{  _C6	, 0b0001 },
	{ _C6R	, UNSPEC },
	{  _C7	, 0b0010 },
	{ _C7S	, UNSPEC },
	{  _C8	, 0b0011 },	/* TODO(Untested?)	*/
	{  _C9	, UNSPEC },
	{ _C10	, UNSPEC }
};

#define MAKE_TOGGLE_CSTATE_FUNC( _type, _feature, _parameter )		\
inline unsigned int Toggle_CState_##_feature( _type *pConfigRegister,	\
					typeof(_parameter) _parameter)  \
{									\
	switch ( _parameter )						\
	{								\
	case COREFREQ_TOGGLE_OFF:					\
	case COREFREQ_TOGGLE_ON:					\
		pConfigRegister->_feature = _parameter;			\
		return 1;						\
	}								\
	return 0;							\
}

MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C3autoDemotion, C3A_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C1autoDemotion, C1A_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C3undemotion, C3U_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C1undemotion, C1U_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, IO_MWAIT_Redir, IOMWAIT_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CC6_CONFIG, CC6demotion, CC6_Enable)
MAKE_TOGGLE_CSTATE_FUNC(MC6_CONFIG, MC6demotion, PC6_Enable)

#undef MAKE_TOGGLE_CSTATE

#define Toggle_CState_Feature( _config, _feature, _parameter )		\
(									\
	Toggle_CState_##_feature( _config, _parameter ) 		\
)

#define For_All_Encodings(	loopCondition, breakStatement,		\
				bodyStatement, closure )		\
({									\
	unsigned int idx, ret = 0;					\
    for (idx = 0; idx < CSTATES_ENCODING_COUNT && (loopCondition); idx++)\
    {									\
	if (breakStatement)						\
	{								\
		ret = 1;						\
		bodyStatement						\
		break;							\
	}								\
    }									\
	closure								\
	ret;								\
})

void Control_IO_MWAIT(	struct CSTATES_ENCODING_ST IORedir[],
			CORE_RO *Core )
{
	CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
	RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*		SMT C-State Base Address.			*/
	Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;

    if (Core->Query.IORedir)
    {
	For_All_Encodings(
			/* loopCondition: */
			(CStateIORedir >= 0),
			/* breakStatement: */
			( (IORedir[idx].dec != UNSPEC)
			&& (CStateIORedir == IORedir[idx].enc) ),
			/* bodyStatement: */
			{
			CState_IO_MWAIT.CStateRange = IORedir[idx].dec;
			WRMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
			RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
			},
			/* closure: */
			{}
	);
    }
	For_All_Encodings(
		/* loopCondition: */ (1),
		/* breakStatement: */
		((CState_IO_MWAIT.CStateRange & 0b111) == IORedir[idx].dec),
		/* bodyStatement: */
		{
		Core->Query.CStateInclude = IORedir[idx].enc;
		},
		/* closure: */
		if (!ret) {
			Core->Query.CStateInclude = _UNSPEC;
		}
	);
}

void Control_CSTATES_NHM(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_1A, 06_1E, 06_1F, 06_25, 06_2C, 06_2E		*/
	CSTATE_CONFIG CStateConfig = {.value = 0};
	unsigned int toggleFeature = 0;

	RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C3autoDemotion,
						C3A_Enable );

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C1autoDemotion,
						C1A_Enable );
    if (CStateConfig.CFG_Lock == 0)
    {
	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						IO_MWAIT_Redir,
						IOMWAIT_Enable);

	toggleFeature |= For_All_Encodings(
				/* loopCondition: */
				(PkgCStateLimit >= 0),
				/* breakStatement: */
				( (Limit[idx].dec != UNSPEC)
				&& (PkgCStateLimit == Limit[idx].enc) ),
				/* bodyStatement: */
				{
				CStateConfig.Pkg_CStateLimit = Limit[idx].dec
				| (0b1000 & CStateConfig.Pkg_CStateLimit);
				},
				/* closure: */
				{}
	);
    }
	if (toggleFeature == 1) {
		WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
		RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
	}

	if (CStateConfig.C3autoDemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C3A, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3A, Core->Bind);
	}
	if (CStateConfig.C1autoDemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1A, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1A, Core->Bind);
	}

	Core->Query.CfgLock = CStateConfig.CFG_Lock;
	Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;

	For_All_Encodings(
		/* loopCondition: */ (1),
		/* breakStatement: */
		((CStateConfig.Pkg_CStateLimit & 0b0111) == Limit[idx].dec),
		/* bodyStatement: */
		{
			Core->Query.CStateLimit = Limit[idx].enc;
		},
		/* closure: */
		if (!ret) {
			Core->Query.CStateLimit = _UNSPEC;
		}
	);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);

	Control_IO_MWAIT(IORedir, Core);
}

void Control_CSTATES_COMMON(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core,
				const unsigned short bitMask,
				const unsigned short unMask )
{
	CSTATE_CONFIG CStateConfig = {.value = 0};
	unsigned int toggleFeature = 0;

	RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C3autoDemotion,
						C3A_Enable );

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C1autoDemotion,
						C1A_Enable );

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C3undemotion,
						C3U_Enable );

	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						C1undemotion,
						C1U_Enable );
    if (CStateConfig.CFG_Lock == 0)
    {
	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						IO_MWAIT_Redir,
						IOMWAIT_Enable);

	toggleFeature |= For_All_Encodings(
				/* loopCondition: */
				(PkgCStateLimit >= 0),
				/* breakStatement: */
				( (Limit[idx].dec != UNSPEC)
				&& (PkgCStateLimit == Limit[idx].enc) ),
				/* bodyStatement: */
				{
				CStateConfig.Pkg_CStateLimit = Limit[idx].dec
				| (unMask & CStateConfig.Pkg_CStateLimit);
				},
				/* closure: */
				{}
	);
    }
	if (toggleFeature == 1) {
		WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
		RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
	}

	if (CStateConfig.C3autoDemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C3A, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3A, Core->Bind);
	}
	if (CStateConfig.C1autoDemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1A, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1A, Core->Bind);
	}
	if (CStateConfig.C3undemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C3U, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3U, Core->Bind);
	}
	if (CStateConfig.C1undemotion)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1U, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1U, Core->Bind);
	}

	Core->Query.CfgLock = CStateConfig.CFG_Lock;
	Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;

	For_All_Encodings(
		/* loopCondition: */ (1),
		/* breakStatement: */
		((CStateConfig.Pkg_CStateLimit & bitMask) == Limit[idx].dec),
		/* bodyStatement: */
		{
			Core->Query.CStateLimit = Limit[idx].enc;
		},
		/* closure: */
		if (!ret) {
			Core->Query.CStateLimit = _UNSPEC;
		}
	);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);

	Control_IO_MWAIT(IORedir, Core);
}

void Control_CSTATES_SNB(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_2A, 06_3A, 06_3E, 06_3F, 06_4F, 06_56, 06_57, 06_85 */
	Control_CSTATES_COMMON(Limit, IORedir, Core, 0b0111, 0b1000);
}

void Control_CSTATES_ULT(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_3C, 06_3D, 06_45, 06_46, 06_47, 06_86		*/
	Control_CSTATES_COMMON(Limit, IORedir, Core, 0b1111, 0b0000);
}

void Control_CSTATES_SKL(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_4E, 06_5E, 06_55, 06_66, 06_7D, 06_7E, 06_8E, 06_9E */
	Control_CSTATES_COMMON(Limit, IORedir, Core, 0b0111, 0b1000);
}

void Control_CSTATES_SOC_ATOM(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core,
				const unsigned short bitMask,
				const unsigned short unMask )
{
	CSTATE_CONFIG CStateConfig = {.value = 0};
	unsigned int toggleFeature = 0;

	RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);

    if (CStateConfig.CFG_Lock == 0)
    {
	toggleFeature |= Toggle_CState_Feature(	&CStateConfig,
						IO_MWAIT_Redir,
						IOMWAIT_Enable);

	toggleFeature |= For_All_Encodings(
				/* loopCondition: */
				(PkgCStateLimit >= 0),
				/* breakStatement: */
				( (Limit[idx].dec != UNSPEC)
				&& (PkgCStateLimit == Limit[idx].enc) ),
				/* bodyStatement: */
				{
				CStateConfig.Pkg_CStateLimit = Limit[idx].dec
				| (unMask & CStateConfig.Pkg_CStateLimit);
				},
				/* closure: */
				{}
	);
    }
	if (toggleFeature == 1) {
		WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
		RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
	}

	Core->Query.CfgLock = CStateConfig.CFG_Lock;
	Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;

	For_All_Encodings(
		/* loopCondition: */ (1),
		/* breakStatement: */
		((CStateConfig.Pkg_CStateLimit & bitMask) == Limit[idx].dec),
		/* bodyStatement: */
		{
			Core->Query.CStateLimit = Limit[idx].enc;
		},
		/* closure: */
		if (!ret) {
			Core->Query.CStateLimit = _UNSPEC;
		}
	);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);

	Control_IO_MWAIT(IORedir, Core);
}

void Control_CSTATES_SOC_SLM(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_37, 06_4A, 06_4C, 06_4D, 06_5A			*/
	CC6_CONFIG CC6_Config = {.value = 0};
	MC6_CONFIG MC6_Config = {.value = 0};

	Control_CSTATES_SOC_ATOM(Limit, IORedir, Core, 0b0111, 0b1000);

	RDMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
    if (Toggle_CState_Feature(&CC6_Config, CC6demotion, CC6_Enable))
    {
	WRMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
	RDMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
    }

	RDMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
    if (Toggle_CState_Feature(&MC6_Config, MC6demotion, PC6_Enable))
    {
	WRMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
	RDMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
    }

    if (CC6_Config.CC6demotion)
    {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->CC6, Core->Bind);
    } else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->CC6, Core->Bind);
    }
    if (MC6_Config.MC6demotion)
    {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->PC6, Core->Bind);
    } else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PC6, Core->Bind);
    }
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PC6_Mask, Core->Bind);
}

void Control_CSTATES_SOC_GDM(	struct CSTATES_ENCODING_ST Limit[],
				struct CSTATES_ENCODING_ST IORedir[],
				CORE_RO *Core )
{	/* Family: 06_5CH						*/
	Control_CSTATES_SOC_ATOM(Limit, IORedir, Core, 0b1111, 0b0000);
}

#undef UNSPEC
#undef For_All_Encodings
#undef Toggle_CState_Feature

#define Intel_CStatesConfiguration( _CLASS, Core )			\
({									\
	Control_##_CLASS(Limit_##_CLASS, IORedir_##_CLASS, Core);	\
})

void PerCore_AMD_Family_0Fh_PStates(CORE_RO *Core)
{
	FIDVID_STATUS FidVidStatus = {.value = 0};
	FIDVID_CONTROL FidVidControl = {.value = 0};
	int NewFID = -1, NewVID = -1, loop = 100;
	UNUSED(Core);

	RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);

	NewFID	= ((PState_FID >= FidVidStatus.StartFID)
		&& (PState_FID <= FidVidStatus.MaxFID)) ?
		PState_FID : FidVidStatus.CurrFID,

	NewVID	= ((PState_VID <= FidVidStatus.StartVID)
		&& (PState_VID >= FidVidStatus.MaxVID)) ?
		PState_VID : FidVidStatus.CurrVID;
	do {
		if (FidVidStatus.FidVidPending == 0) {
			RDMSR(FidVidControl, MSR_K7_FID_VID_CTL);

			FidVidControl.InitFidVid = 1;
			FidVidControl.StpGntTOCnt = 400;
			FidVidControl.NewFID = NewFID;
			FidVidControl.NewVID = NewVID;

			WRMSR(FidVidControl, MSR_K7_FID_VID_CTL);
			loop = 0;
		} else {
			RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);
		}
		if (loop == 0) {
			break;
		} else {
			loop-- ;
		}
	} while (FidVidStatus.FidVidPending == 1) ;
}

void SystemRegisters(CORE_RO *Core)
{
	if (RDPMC_Enable) {
		__asm__ volatile
		(
			"movq	%%cr4, %%rax"	"\n\t"
			"btsq	%0,    %%rax"	"\n\t"
			"movq	%%rax, %%cr4"	"\n\t"
			"wbinvd"
			:
			: "i" (CR4_PCE)
			: "%rax"
		);
	}
	__asm__ volatile
	(
		"# RFLAGS"		"\n\t"
		"pushfq"		"\n\t"
		"popq	%0"		"\n\t"
		"movq	%%cr0, %1"	"\n\t"
		"movq	%%cr3, %2"	"\n\t"
		"movq	%%cr4, %3"	"\n\t"
		"movq	%%cr8, %4"	"\n\t"
		"# EFER"		"\n\t"
		"xorq	%%rax, %%rax"	"\n\t"
		"xorq	%%rdx, %%rdx"	"\n\t"
		"movq	%6,%%rcx"	"\n\t"
		"rdmsr"			"\n\t"
		"shlq	$32, %%rdx"	"\n\t"
		"orq	%%rdx, %%rax"	"\n\t"
		"movq	%%rax, %5"
		: "=r" (Core->SystemRegister.RFLAGS),
		  "=r" (Core->SystemRegister.CR0),
		  "=r" (Core->SystemRegister.CR3),
		  "=r" (Core->SystemRegister.CR4),
		  "=r" (Core->SystemRegister.CR8),
		  "=r" (Core->SystemRegister.EFER)
		: "i" (MSR_EFER)
		: "%rax", "%rcx", "%rdx"
	);
	if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
		__asm__ volatile
		(
			"# EFCR"		"\n\t"
			"xorq	%%rax, %%rax"	"\n\t"
			"xorq	%%rdx, %%rdx"	"\n\t"
			"movq	%1,%%rcx"	"\n\t"
			"rdmsr"			"\n\t"
			"shlq	$32, %%rdx"	"\n\t"
			"orq	%%rdx, %%rax"	"\n\t"
			"movq	%%rax, %0"
			: "=r" (Core->SystemRegister.EFCR)
			: "i" (MSR_IA32_FEAT_CTL)
			: "%rax", "%rcx", "%rdx"
		);
		/*		Virtualization Technology.		*/
		if (BITVAL(Core->SystemRegister.EFCR, EXFCR_VMX_IN_SMX)
		  | BITVAL(Core->SystemRegister.EFCR, EXFCR_VMXOUT_SMX))
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->VM, Core->Bind);
	}
	else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
	{
		RDMSR(Core->SystemRegister.VMCR, MSR_VM_CR);
		/*		Secure Virtual Machine .		*/
		if (!Core->SystemRegister.VMCR.SVME_Disable
		  && Core->SystemRegister.VMCR.SVM_Lock)
		{
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->VM, Core->Bind);
		}
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CR_Mask, Core->Bind);
}

void Intel_Mitigation_Mechanisms(CORE_RO *Core)
{
	SPEC_CTRL Spec_Ctrl = {.value = 0};
	PRED_CMD  Pred_Cmd  = {.value = 0};
	FLUSH_CMD Flush_Cmd = {.value = 0};
	unsigned short WrRdMSR = 0;

	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
	 || PUBLIC(RO(Proc))->Features.ExtFeature.EDX.STIBP_Cap
	 || PUBLIC(RO(Proc))->Features.ExtFeature.EDX.SSBD_Cap)
	{
		RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
	}
	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
	&& ((Mech_IBRS == COREFREQ_TOGGLE_OFF)
	 || (Mech_IBRS == COREFREQ_TOGGLE_ON)))
	{
		Spec_Ctrl.IBRS = Mech_IBRS;
		WrRdMSR = 1;
	}
	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.STIBP_Cap
	&& ((Mech_STIBP == COREFREQ_TOGGLE_OFF)
	 || (Mech_STIBP == COREFREQ_TOGGLE_ON)))
	{
		Spec_Ctrl.STIBP = Mech_STIBP;
		WrRdMSR = 1;
	}
	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.SSBD_Cap
	&& ((Mech_SSBD == COREFREQ_TOGGLE_OFF)
	 || (Mech_SSBD == COREFREQ_TOGGLE_ON)))
	{
		Spec_Ctrl.SSBD = Mech_SSBD;
		WrRdMSR = 1;
	}
	if (WrRdMSR == 1)
	{
	    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 11)
		x86_spec_ctrl_base = Spec_Ctrl.value;
	    #endif
		WRMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
		RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
	}
	if (Spec_Ctrl.IBRS) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS, Core->Bind);
	}
	if (Spec_Ctrl.STIBP) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->STIBP, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->STIBP, Core->Bind);
	}
	if (Spec_Ctrl.SSBD) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SSBD, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SSBD, Core->Bind);
	}
	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
	&& ((Mech_IBPB == COREFREQ_TOGGLE_OFF)
	 || (Mech_IBPB == COREFREQ_TOGGLE_ON)))
	{
		Pred_Cmd.IBPB = Mech_IBPB;
		WRMSR(Pred_Cmd, MSR_IA32_PRED_CMD);
	}
	if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.L1D_FLUSH_Cap
	&& ((Mech_L1D_FLUSH == COREFREQ_TOGGLE_OFF)
	 || (Mech_L1D_FLUSH == COREFREQ_TOGGLE_ON)))
	{
		Flush_Cmd.L1D_FLUSH_CMD = Mech_L1D_FLUSH;
		WRMSR(Flush_Cmd, MSR_IA32_FLUSH_CMD);
	}
    if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IA32_ARCH_CAP)
    {
		ARCH_CAPABILITIES Arch_Cap = {.value = 0};

		RDMSR(Arch_Cap, MSR_IA32_ARCH_CAPABILITIES);

	if (Arch_Cap.RDCL_NO) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->RDCL_NO, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RDCL_NO, Core->Bind);
	}
	if (Arch_Cap.IBRS_ALL) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS_ALL, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS_ALL, Core->Bind);
	}
	if (Arch_Cap.RSBA) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->RSBA, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RSBA, Core->Bind);
	}
	if (Arch_Cap.L1DFL_VMENTRY_NO) {
	    BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
	} else {
	    BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
	}
	if (Arch_Cap.SSB_NO) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SSB_NO, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SSB_NO, Core->Bind);
	}
	if (Arch_Cap.MDS_NO) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->MDS_NO, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->MDS_NO, Core->Bind);
	}
	if (Arch_Cap.PSCHANGE_MC_NO) {
	    BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
	} else {
	    BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
	}
	if (Arch_Cap.TAA_NO) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TAA_NO, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TAA_NO, Core->Bind);
	}
    }
    if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IA32_CORE_CAP)
    {
	CORE_CAPABILITIES Core_Cap = {.value = 0};

	RDMSR(Core_Cap, MSR_IA32_CORE_CAPABILITIES);

	if (Core_Cap.STLB_SUPPORTED) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->STLB, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->STLB, Core->Bind);
	}
	if (Core_Cap.FUSA_SUPPORTED) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->FUSA, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->FUSA, Core->Bind);
	}
	if (Core_Cap.RSM_IN_CPL0_ONLY) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->RSM_CPL0, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RSM_CPL0, Core->Bind);
	}
	if (Core_Cap.SPLA_EXCEPTION) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SPLA, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SPLA, Core->Bind);
	}
	if (Core_Cap.SNOOP_FILTER_SUP) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);
	}
    } else {
	/* Source: arch/x86/kernel/cpu/intel.c				*/
	struct SIGNATURE whiteList[] = {
		_Icelake_UY,		/* 06_7E */
		_Icelake_X,		/* 06_6A */
		_Tremont_Jacobsville,	/* 06_86 */
		_Tremont_Elkhartlake,	/* 06_96 */
		_Tremont_Jasperlake	/* 06_9C */
	};
	const int ids = sizeof(whiteList) / sizeof(whiteList[0]);
	int id;
     for (id = 0; id < ids; id++) {
      if((whiteList[id].ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
      && (whiteList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
      && (whiteList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
      && (whiteList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
      {
	BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SPLA, Core->Bind);
	break;
      }
     }
    }
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask, Core->Bind);
}

void AMD_Mitigation_Mechanisms(CORE_RO *Core)
{
	AMD_SPEC_CTRL Spec_Ctrl = {.value = 0};
	AMD_PRED_CMD  Pred_Cmd  = {.value = 0};
	unsigned short WrRdMSR = 0;

    if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.IBRS
     || PUBLIC(RO(Proc))->Features.leaf80000008.EBX.STIBP
     || PUBLIC(RO(Proc))->Features.leaf80000008.EBX.SSBD
     || PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PSFD)
    {
	RDMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);

	if ((Mech_IBRS == COREFREQ_TOGGLE_OFF)
	 || (Mech_IBRS == COREFREQ_TOGGLE_ON))
	{
		Spec_Ctrl.IBRS = Mech_IBRS;
		WrRdMSR = 1;
	}
	if ((Mech_STIBP == COREFREQ_TOGGLE_OFF)
	 || (Mech_STIBP == COREFREQ_TOGGLE_ON))
	{
		Spec_Ctrl.STIBP = Mech_STIBP;
		WrRdMSR = 1;
	}
	if ((Mech_SSBD == COREFREQ_TOGGLE_OFF)
	 || (Mech_SSBD == COREFREQ_TOGGLE_ON))
	{
		Spec_Ctrl.SSBD = Mech_SSBD;
		WrRdMSR = 1;
	}
	if ((Mech_PSFD == COREFREQ_TOGGLE_OFF)
	 || (Mech_PSFD == COREFREQ_TOGGLE_ON))
	{
	    if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PSFD) {
		Spec_Ctrl.PSFD = Mech_PSFD;
		WrRdMSR = 1;
	    }
	}
	if (WrRdMSR == 1)
	{
	    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 11)
		x86_spec_ctrl_base = Spec_Ctrl.value;
	    #endif
		WRMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);
		RDMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);
	}
	if (Spec_Ctrl.IBRS) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS, Core->Bind);
	}
	if (Spec_Ctrl.STIBP) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->STIBP, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->STIBP, Core->Bind);
	}
	if (Spec_Ctrl.SSBD) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SSBD, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SSBD, Core->Bind);
	}
	if (Spec_Ctrl.PSFD) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->PSFD, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PSFD, Core->Bind);
	}
    }
	if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.IBPB
	&& ((Mech_IBPB == COREFREQ_TOGGLE_OFF)
	 || (Mech_IBPB == COREFREQ_TOGGLE_ON)))
	{
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
	    {
		Pred_Cmd.IBPB = Mech_IBPB;
		WRMSR(Pred_Cmd, MSR_AMD_PRED_CMD);
	    }
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask, Core->Bind);
}

void Intel_VirtualMachine(CORE_RO *Core)
{
	if (PUBLIC(RO(Proc))->Features.Std.ECX.VMX) {
		VMX_BASIC VMX_Basic = {.value = 0};
		/*		Basic VMX Information.			*/
		RDMSR(VMX_Basic, MSR_IA32_VMX_BASIC);

		if (VMX_Basic.SMM_DualMon)
		{
			BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->SMM, Core->Bind);
		} else {
			BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SMM, Core->Bind);
		}
	}
}

void Intel_Microcode(CORE_RO *Core)
{
	MICROCODE_ID Microcode = {.value = 0};

	RDMSR(Microcode, MSR_IA32_UCODE_REV);
	Core->Query.Microcode = Microcode.Signature;
}

void AMD_Microcode(CORE_RO *Core)
{
	unsigned long long value = 0;
	RDMSR64(value, MSR_AMD64_PATCH_LEVEL);
	Core->Query.Microcode = (unsigned int) value;
}

void PerCore_Reset(CORE_RO *Core)
{
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->DCU_Mask  , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->HWP_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->PC6_Mask	, Core->Bind);

	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->CR_Mask	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RO(Proc))->WDT_Mask	, Core->Bind);

	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->ODCM	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_Prefetch	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_Prefetch	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PowerMgmt , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SpeedStep , Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->HWP	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1E	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3A	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1A	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3U	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1U	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->CC6	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PC6	, Core->Bind);

	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->STIBP 	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SSBD	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PSFD	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RDCL_NO	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->IBRS_ALL	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RSBA	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SSB_NO	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->MDS_NO	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TAA_NO	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->STLB	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->FUSA	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->RSM_CPL0	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SPLA	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);

	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->SMM	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->VM	, Core->Bind);
	BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->WDT	, Core->Bind);
}

static void PerCore_VirtualMachine(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
	{
		PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)] = 10;

		Intel_VirtualMachine(Core);

		Intel_Microcode(Core);

		SpeedStep_Technology(Core);
	}
	else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
	{
		PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)] = 8;

		if (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily >= 1) {
			AMD_Microcode(Core);
		}

		BITSET_CC(LOCKLESS,PUBLIC(RO(Proc))->SpeedStep_Mask,Core->Bind);
		BITSET_CC(LOCKLESS,PUBLIC(RO(Proc))->ODCM_Mask, Core->Bind);
		BITSET_CC(LOCKLESS,PUBLIC(RO(Proc))->PowerMgmt_Mask,Core->Bind);
	}

	SystemRegisters(Core);

	Dump_CPUID(Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->WDT_Mask,
				PUBLIC(RO(Proc))->Service.Core);
}

static void PerCore_Intel_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Core_Platform_Info(Core->Bind);

	SystemRegisters(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	SpeedStep_Technology(Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->WDT_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);
}

static void PerCore_AuthenticAMD_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AuthenticAMD_Boost(Core->Bind);

	SystemRegisters(Core);

	if (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily >= 1) {
		AMD_Microcode(Core);
	}
	Dump_CPUID(Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}

static void PerCore_Core2_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Core_Platform_Info(Core->Bind);

	SystemRegisters(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	SpeedStep_Technology(Core);
	DynamicAcceleration(Core);				/* Unique */
	Compute_Intel_Core_Burst(Core->Bind);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->WDT_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);				/* Shared | Unique */

	ThermalMonitor_Set(Core);
}

void Compute_Intel_Silvermont_Burst(CORE_RO *Core)
{
	enum RATIO_BOOST boost;
	unsigned int burstRatio;
	PLATFORM_ID PfID = {.value = 0};
	PERF_STATUS PerfStatus = {.value = 0};
	bool initialize = false;

	burstRatio = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)];
    if (Intel_MaxBusRatio(&PfID) == 0) {
	burstRatio = KMAX(burstRatio, PfID.MaxBusRatio);
    }
	PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(ACT)] = burstRatio;

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	burstRatio = KMAX(burstRatio, PerfStatus.CORE.MaxBusRatio);

    if (PRIVATE(OF(Specific)) != NULL)
    {
	unsigned int XtraBoost	= PRIVATE(OF(Specific))->Boost[0]
				+ PRIVATE(OF(Specific))->Boost[1];
      if (XtraBoost > 0) {
	burstRatio = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)];
	burstRatio = burstRatio + XtraBoost;
      }
    }
    if (PUBLIC(RO(Proc))->Features.Turbo_Unlock == 1)
    {	/*	Read the Turbo Boost register from any programmed ratio */
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	boost = BOOST(SIZE) - PUBLIC(RO(Proc))->Features.SpecTurboRatio;
      do
      {
	if (PUBLIC(RO(Core, AT(Core->Bind)))->Boost[boost] == 0)
	{
		PUBLIC(RO(Core, AT(Core->Bind)))->Boost[boost] = burstRatio;

		initialize = true;
	}
      } while ( ++boost < BOOST(SIZE) );

      if (initialize == true)
      { /*	Re-program the register if at least one value was zero	*/
	TURBO_RATIO_CONFIG0 Cfg0 = {
	.MaxRatio_1C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(1C)],
	.MaxRatio_2C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(2C)],
	.MaxRatio_3C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(3C)],
	.MaxRatio_4C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(4C)],
	.MaxRatio_5C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(5C)],
	.MaxRatio_6C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(6C)],
	.MaxRatio_7C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(7C)],
	.MaxRatio_8C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(8C)]
	};
	WRMSR(Cfg0, MSR_TURBO_RATIO_LIMIT);
      }
    }
}

static void PerCore_Silvermont_Query(void *arg)
{	/*	Query the Min and Max frequency ratios per CPU		*/
	CORE_RO *Core = (CORE_RO *) arg;

	PLATFORM_INFO PfInfo = {.value = 0};
	RDMSR(PfInfo, MSR_PLATFORM_INFO);

	PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)]=PfInfo.MinimumRatio;
    PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)]=PfInfo.MaxNonTurboRatio;

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	SpeedStep_Technology(Core);
	DynamicAcceleration(Core);				/* Unique */
	SoC_Turbo_Override(Core);
	Compute_Intel_Silvermont_Burst(Core);

	Query_Intel_C1E(Core);

	Intel_CStatesConfiguration(CSTATES_SOC_SLM, Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	PowerThermal(Core);				/* Shared | Unique */

	ThermalMonitor_Set(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,	/* Table 2-8 */
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_Airmont_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_Silvermont_Query(arg);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,	/* Table 2-11 */
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );
	}
}

static void PerCore_Goldmont_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_SOC_GDM, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	Intel_Turbo_Activation_Ratio(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,	/* Table 2-12 */
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,	/* Table 2-12 */
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(RAM) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_Nehalem_Same_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_Nehalem_Target,
				Get_Nehalem_Target,
				Cmp_Nehalem_Target,
				1 + Core->Boost[BOOST(MAX)],
				1 + Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_NHM, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);
}

static void PerCore_Nehalem_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
	PerCore_Nehalem_Same_Query(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Nehalem_PowerLimit();		/* Table 2-15	*/

		Intel_Watchdog(Core);
	}
}

static void PerCore_Nehalem_EX_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Core_Platform_Info(Core->Bind);
	PerCore_Nehalem_Same_Query(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Nehalem_PowerLimit();		/* Table 2-15	*/

		Intel_Watchdog(Core);
	}
}

static void PerCore_Avoton_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,	/* Table 2-10 */
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );
	}
}

static void PerCore_SandyBridge_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_SNB, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_SandyBridge_EP_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_SandyBridge_Query(arg);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(RAM) );
	}
}

static void PerCore_IvyBridge_Query(void *arg)
{
	PerCore_SandyBridge_Query(arg);
	Intel_Turbo_Activation_Ratio( (CORE_RO*) arg );
	Intel_Turbo_TDP_Config( (CORE_RO*) arg );
}

static void PerCore_IvyBridge_EP_Query(void *arg)
{
	Intel_Turbo_Config(	(CORE_RO*) arg,
				Intel_Turbo_Cfg15C_PerCore,
				Assign_15C_Boost );

	PerCore_SandyBridge_EP_Query(arg);
}

static void PerCore_Haswell_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_SandyBridge_Query(arg);
	Intel_Turbo_Activation_Ratio(Core);
	Intel_Turbo_TDP_Config(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(UNCORE) );
	}
}

static void PerCore_Haswell_EP_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);

	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	Intel_Turbo_Config(	(CORE_RO*) arg,
				Intel_Turbo_Cfg16C_PerCore,
				Assign_16C_Boost );

	Intel_Turbo_Config(	(CORE_RO*) arg,
				Intel_Turbo_Cfg18C_PerCore,
				Assign_18C_Boost );

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

    if (PUBLIC(RO(Proc))->Registration.Experimental) {
	Intel_Microcode(Core);
    }
	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_SNB, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	Intel_Turbo_Activation_Ratio(Core);
	Intel_Turbo_TDP_Config(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );

		Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(RAM) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_Haswell_ULT_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_ULT, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );

		Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(UNCORE) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_Haswell_ULX(void *arg)
{
	PerCore_IvyBridge_Query(arg);
}

static void PerCore_Broadwell_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_SandyBridge_Query(arg);
	Intel_Turbo_Activation_Ratio(Core);
	Intel_Turbo_TDP_Config(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(UNCORE) );
	}
}

static void PerCore_Skylake_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_SKL, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	Intel_Turbo_Activation_Ratio(Core);
	Intel_Turbo_TDP_Config(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );

		Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(UNCORE) );

		Intel_DomainPowerLimit( MSR_PLATFORM_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PLATFORM) );

		Intel_Watchdog(Core);
	}
}

static void PerCore_Skylake_X_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Intel_Turbo_Config(	Core,
				Intel_Turbo_Cfg_SKL_X_PerCore,
				Assign_SKL_X_Boost );

	Intel_Platform_Info(Core->Bind);
	Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);

	SystemRegisters(Core);

	Intel_Mitigation_Mechanisms(Core);

	Intel_VirtualMachine(Core);

	Intel_Microcode(Core);

	Dump_CPUID(Core);

	Intel_DCU_Technology(Core);

	SpeedStep_Technology(Core);

	TurboBoost_Technology(	Core,
				Set_SandyBridge_Target,
				Get_SandyBridge_Target,
				Cmp_SandyBridge_Target,
				Core->Boost[BOOST(1C)],
				Core->Boost[BOOST(MAX)] );

	Query_Intel_C1E(Core);

	if (Core->T.ThreadID == 0) {				/* Per Core */
		Intel_CStatesConfiguration(CSTATES_SKL, Core);
	} else {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
	/*	Store the C-State Base Address used by I/O-MWAIT	*/
		Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	PowerThermal(Core);

	ThermalMonitor_Set(Core);

	Intel_Turbo_Activation_Ratio(Core);
	Intel_Turbo_TDP_Config(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PKG) );

		Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(CORES) );
/*TODO(Unsupported MSR and CSR registers)
		Intel_DomainPowerLimit( MSR_PLATFORM_POWER_LIMIT,
					PKG_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(PLATFORM) );

		Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(RAM) );

		Intel_Watchdog(Core);
*/
	}
}

static void PerCore_Kaby_Lake_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_Skylake_Query(arg);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
					PPn_POWER_LIMIT_LOCK_MASK,
					PWR_DOMAIN(RAM) );
	}
}

static void PerCore_AMD_Family_0Fh_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	PerCore_AMD_Family_0Fh_PStates(Core);		/* Alter P-States */
	Compute_AMD_Family_0Fh_Boost(Core->Bind);	/* Query P-States */

	SystemRegisters(Core);

	Dump_CPUID(Core);

	Query_AMD_Family_0Fh_C1E(Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}

static void PerCore_AMD_Family_Same_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	SystemRegisters(Core);

	AMD_Microcode(Core);

	Dump_CPUID(Core);

    if (PUBLIC(RO(Proc))->Registration.Experimental) {
	Query_AMD_Family_0Fh_C1E(Core);
    } else {
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
    }

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->CC6_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS,	PUBLIC(RO(Proc))->PC6_Mask,
				PUBLIC(RO(Proc))->Service.Core);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}

static void PerCore_AMD_Family_10h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_10h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	/*TODO(Errata 438) PerCore_AMD_CState_BAR(Core);		*/
	AMD_Watchdog(Core);
}

static void PerCore_AMD_Family_11h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_11h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	/*TODO(Unspecified) PerCore_AMD_CState_BAR(Core);		*/
	/*TODO(Watchdog): D18F3x{40,44} MCA NB Registers - Hardware needed */
}

static void PerCore_AMD_Family_12h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_12h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	PerCore_AMD_CState_BAR(Core);
	AMD_Watchdog(Core);
}

static void PerCore_AMD_Family_14h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_14h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	PerCore_AMD_CState_BAR(Core);
	AMD_Watchdog(Core);
}

static void PerCore_AMD_Family_15h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_15h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	PerCore_AMD_CState_BAR(Core);

	switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
	case 0x0:
	case 0x1:
	/*TODO(Watchdog): D18F3x{40,44} MCA NB Registers - Hardware needed */
		break;
	case 0x3:
	case 0x6:
	case 0x7:
		if ((PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
		 && (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf))
		{
			AMD_Watchdog(Core);
		}
		break;
	}
}

static void PerCore_AMD_Family_16h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;

	Compute_AMD_Family_15h_Boost(Core->Bind);
	PerCore_AMD_Family_Same_Query(Core);
	PerCore_AMD_CState_BAR(Core);
	AMD_Watchdog(Core);
}

static void PerCore_AMD_Family_17h_Query(void *arg)
{
	CORE_RO *Core = (CORE_RO *) arg;
	PM16 PM = {.value = 0};
	int ToggleFeature = 0;
	bool CPB_State;

	/*	Query the Min, Max, Target & Turbo P-States		*/
	PerCore_Query_AMD_Zen_Features(Core);
	CPB_State = Compute_AMD_Zen_Boost(Core->Bind);
	/*	Query and set features per Package			*/
    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	TCTL_THERM_TRIP ThermTrip = {.value = 0};
	TCTL_HTC HTC = {.value = 0};
	unsigned int rx;
	HSMP_ARG arg[8];
	/*	Query the HTC and THERM_TRIP features from SMUTHM	*/
	Core_AMD_SMN_Read(	HTC,
				SMU_AMD_THM_TCTL_REGISTER_F17H + 0x4,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );
	if (HTC.HTC_EN) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM2, Core->Bind);
	}
	Core_AMD_SMN_Read(	ThermTrip,
				SMU_AMD_THM_TCTL_REGISTER_F17H + 0x8,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );
	if (ThermTrip.THERM_TP_EN) {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TM1, Core->Bind);
	}
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->TM_Mask, Core->Bind);
	/*		Count CPB and XFR ratios			*/
	if (CPB_State == true)
	{
		PUBLIC(RO(Proc))->Features.XtraCOF = 2;
	}
	else {
		PUBLIC(RO(Proc))->Features.XtraCOF = 0;
	}
	#define _lt (PWR_LIMIT_SIZE * PWR_DOMAIN(PKG))
	if (Custom_TDP_Count > _lt) {
	    if (Custom_TDP_Offset[_lt] != 0)
	    {
		signed int TDP_Limit;
		TDP_Limit = PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
					PWR_DOMAIN(PKG)
			].Domain_Limit1;

		TDP_Limit = TDP_Limit + Custom_TDP_Offset[_lt];

		if (PUBLIC(RO(Proc))->Features.HSMP_Enable && (TDP_Limit > 0))
		{
			RESET_ARRAY(arg, 8, 0, .value);
			arg[0].value = 1000 * TDP_Limit;
			rx = AMD_HSMP_Exec(HSMP_WR_PKG_PL1, arg, SMU_HSMP_F19H,
						SMU_AMD_INDEX_REGISTER_F17H,
						SMU_AMD_DATA_REGISTER_F17H);
		    if (rx == HSMP_RESULT_OK)
		    {
			RESET_ARRAY(arg, 8, 0, .value);
			rx = AMD_HSMP_Exec(HSMP_RD_PKG_PL1, arg, SMU_HSMP_F19H,
						SMU_AMD_INDEX_REGISTER_F17H,
						SMU_AMD_DATA_REGISTER_F17H);
		    }
		    if (rx == HSMP_RESULT_OK)
		    {
			PUBLIC(RO(Proc))->PowerThermal.PowerLimit[
				PWR_DOMAIN(PKG)
			].Domain_Limit1 = arg[0].value / 1000;
		    }
		    else if (IS_HSMP_OOO(rx))
		    {
			PUBLIC(RO(Proc))->Features.HSMP_Enable =	\
			PUBLIC(RO(Proc))->Features.TDP_Unlock = 0;
		    }
		}
	    }
	}
	#undef _lt
    }
	SystemRegisters(Core);

	AMD_Microcode(Core);

	Dump_CPUID(Core);

	AMD_F17h_DCU_Technology(Core);

	AMD_Mitigation_Mechanisms(Core);

	/*	Query the FCH for various registers			*/
	AMD_FCH_PM_Read16(AMD_FCH_PM_CSTATE_EN, PM);
	switch (C3U_Enable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PM.CStateEn.C1eToC3En = C3U_Enable;
			ToggleFeature = 1;
		break;
	}
	switch (C1U_Enable) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			PM.CStateEn.C1eToC2En = C1U_Enable;
			ToggleFeature = 1;
		break;
	}
	if (ToggleFeature == 1) {
		AMD_FCH_PM_Write16(AMD_FCH_PM_CSTATE_EN, PM);
		AMD_FCH_PM_Read16(AMD_FCH_PM_CSTATE_EN, PM);
	}
	if (PM.CStateEn.C1eToC2En)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C1U, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C1U, Core->Bind);
	}
	if (PM.CStateEn.C1eToC3En)
	{
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->C3U, Core->Bind);
	} else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->C3U, Core->Bind);
	}

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1E_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1A_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C3U_Mask	, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->C1U_Mask	, Core->Bind);

	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
	BITSET_CC(LOCKLESS, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);

	Core->PowerThermal.Param = PUBLIC(RO(Proc))->PowerThermal.Param;
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 56)
void Sys_DumpTask(SYSGATE_RO *SysGate)
{
	SysGate->taskCount = 0;
}
#else /* KERNEL_VERSION(3, 10, 56) */
void Sys_DumpTask(SYSGATE_RO *SysGate)
{	/* Source: /include/linux/sched.h */
	struct task_struct *process, *thread;
	int cnt = 0;

	rcu_read_lock();
	for_each_process_thread(process, thread) {
#if defined(CONFIG_SCHED_MUQSS) \
 || defined(CONFIG_SCHED_BMQ) \
 || defined(CONFIG_SCHED_PDS)
		SysGate->taskList[cnt].runtime  = tsk_seruntime(thread);
#else
		SysGate->taskList[cnt].runtime  = thread->se.sum_exec_runtime;
#endif /* CONFIG_SCHED_*	*/
		SysGate->taskList[cnt].usertime = thread->utime;
		SysGate->taskList[cnt].systime  = thread->stime;
		SysGate->taskList[cnt].pid      = thread->pid;
		SysGate->taskList[cnt].tgid     = thread->tgid;
		SysGate->taskList[cnt].ppid     = thread->parent->pid;
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
		SysGate->taskList[cnt].state    = (short int) thread->state;
#else
		SysGate->taskList[cnt].state    = (short int) thread->__state;
#endif
#if defined(CONFIG_SCHED_BMQ) \
 || defined(CONFIG_SCHED_PDS)
		SysGate->taskList[cnt].wake_cpu = (short int) thread->cpu;
#else
		SysGate->taskList[cnt].wake_cpu = (short int) thread->wake_cpu;
#endif /* CONFIG_SCHED_BMQ	*/
		memcpy(SysGate->taskList[cnt].comm, thread->comm,TASK_COMM_LEN);

		if (cnt < TASK_LIMIT) {
			cnt++;
		}
	}
	rcu_read_unlock();
	SysGate->taskCount = cnt;
}
#endif /* KERNEL_VERSION(3, 10, 56) */

void Sys_MemInfo(SYSGATE_RO *SysGate)
{	/* Source: /include/uapi/linux/sysinfo.h */
	struct sysinfo info;
	si_meminfo(&info);

	SysGate->memInfo.totalram  = info.totalram  << (PAGE_SHIFT - 10);
	SysGate->memInfo.sharedram = info.sharedram << (PAGE_SHIFT - 10);
	SysGate->memInfo.freeram   = info.freeram   << (PAGE_SHIFT - 10);
	SysGate->memInfo.bufferram = info.bufferram << (PAGE_SHIFT - 10);
	SysGate->memInfo.totalhigh = info.totalhigh << (PAGE_SHIFT - 10);
	SysGate->memInfo.freehigh  = info.freehigh  << (PAGE_SHIFT - 10);
}

#define Sys_Tick(Pkg, ...)					\
({								\
	if (PUBLIC(OF(Gate)) != NULL)				\
	{							\
		Pkg->tickStep--;				\
		if (!Pkg->tickStep) {				\
			Pkg->tickStep = Pkg->tickReset ;	\
			Sys_DumpTask( PUBLIC(OF(Gate)) );	\
			Sys_MemInfo( PUBLIC(OF(Gate)) );	\
			__VA_ARGS__				\
		}						\
	}							\
})

static void InitTimer(void *Cycle_Function)
{
	unsigned int cpu = smp_processor_id();

	if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED) == 0)
	{
		hrtimer_init(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
				CLOCK_MONOTONIC,
				HRTIMER_MODE_REL_PINNED);

		PRIVATE(OF(Join, AT(cpu)))->Timer.function = Cycle_Function;
		BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED);
	}
}

void Controller_Init(void)
{
	CLOCK sClock = {.Q = 0, .R = 0, .Hz = 0};
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count, ratio = 0;

	Package_Init_Reset();

    if (Arch[PUBLIC(RO(Proc))->ArchID].Query != NULL)
    {
	Arch[PUBLIC(RO(Proc))->ArchID].Query(PUBLIC(RO(Proc))->Service.Core);
    }

   ratio=PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)];

    if (Arch[PUBLIC(RO(Proc))->ArchID].BaseClock != NULL)
    {
	sClock = Arch[PUBLIC(RO(Proc))->ArchID].BaseClock(ratio);
    }
    if (sClock.Hz == 0) {	/*	Fallback to 100 MHz		*/
	sClock.Q = 100;
	sClock.R = 0;
	sClock.Hz = 100000000LLU;
    }
	ratio = FixMissingRatioAndFrequency(ratio, &sClock);

  if ((AutoClock & 0b01) || PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv)
  {
	CLOCK vClock = {.Q = 0, .R =0, .Hz = 0};
	COMPUTE_ARG Compute;
	struct kmem_cache *hwCache = NULL;
	/* Allocate Cache aligned resources. */
	hwCache = kmem_cache_create(	"CoreFreqCache",
					STRUCT_SIZE, 0,
					SLAB_HWCACHE_ALIGN, NULL);
    if (hwCache != NULL)
    {
      do {	/*		from last AP to BSP			*/
		cpu--;

	if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS))
	{
		Compute.TSC[0] = kmem_cache_alloc(hwCache, GFP_ATOMIC);
	  if (Compute.TSC[0] != NULL)
	  {
		Compute.TSC[1] = kmem_cache_alloc(hwCache, GFP_ATOMIC);
	    if (Compute.TSC[1] != NULL)
	    {
		if (ratio != 0)
		{
		Compute.Clock.Q = ratio;
		Compute.Clock.R = 0;
		Compute.Clock.Hz = 0;

		PUBLIC(RO(Core, AT(cpu)))->Clock = Compute_Clock(cpu,&Compute);

		    if (PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv)
		    {
			PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = ratio;
		    }
		}
		else
		{
		vClock.Q = 0; vClock.R = 0; vClock.Hz = 0;
		Compute.Clock.Q = sClock.Q;
		Compute.Clock.R = 0;
		Compute.Clock.Hz = 0;

		vClock = Compute_Clock(cpu, &Compute);

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = vClock.Q;
		}
		/*		Release memory resources.		*/
		kmem_cache_free(hwCache, Compute.TSC[1]);
	    }
		kmem_cache_free(hwCache, Compute.TSC[0]);
	  }
	}
      } while (cpu != 0) ;

	kmem_cache_destroy(hwCache);
    }
	if ((ratio == 0) && (vClock.Hz != 0)) {
		ratio = FixMissingRatioAndFrequency(vClock.Q, &sClock);
	}
  }
	/*	Launch a high resolution timer per online CPU.		*/
	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
	{
		if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
			if (!PUBLIC(RO(Core, AT(cpu)))->Clock.Hz)
			{
				PUBLIC(RO(Core, AT(cpu)))->Clock = sClock;
			}
			if (Arch[PUBLIC(RO(Proc))->ArchID].Timer != NULL)
			{
				Arch[PUBLIC(RO(Proc))->ArchID].Timer(cpu);
			}
		}
	}
}

void Controller_Start(int wait)
{
    if (Arch[PUBLIC(RO(Proc))->ArchID].Start != NULL)
    {
	unsigned int cpu;
      for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
      {
	if ((BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED) == 1)
	 && (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED) == 0))
	{
		smp_call_function_single(cpu,
					Arch[PUBLIC(RO(Proc))->ArchID].Start,
					NULL, wait);
	}
      }
    }
}

void Controller_Stop(int wait)
{
    if (Arch[PUBLIC(RO(Proc))->ArchID].Stop != NULL)
    {
	unsigned int cpu;
	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
	    if ((BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED) == 1)
	     && (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED) == 1))
	    {
		smp_call_function_single(cpu,
					Arch[PUBLIC(RO(Proc))->ArchID].Stop,
					NULL, wait);
	    }
    }
}

void Controller_Exit(void)
{
	unsigned int cpu;

	if (Arch[PUBLIC(RO(Proc))->ArchID].Exit != NULL)
	{
		Arch[PUBLIC(RO(Proc))->ArchID].Exit();
	}
	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
	{
		BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED);
	}
}

void Intel_Core_Counters_Set(CORE_RO *Core)
{
    if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 2) {
	CORE_GLOBAL_PERF_CONTROL	Core_GlobalPerfControl = {.value = 0};
	CORE_FIXED_PERF_CONTROL 	Core_FixedPerfControl = {.value = 0};
	CORE_GLOBAL_PERF_STATUS 	Core_PerfOverflow = {.value = 0};
	CORE_GLOBAL_PERF_OVF_CTRL	Core_PerfOvfControl = {.value = 0};

	RDMSR(Core_GlobalPerfControl, MSR_CORE_PERF_GLOBAL_CTRL);
	Core->SaveArea.Core_GlobalPerfControl = Core_GlobalPerfControl;
	Core_GlobalPerfControl.EN_FIXED_CTR0  = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
	Core_GlobalPerfControl.EN_FIXED_CTR1  = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
	Core_GlobalPerfControl.EN_FIXED_CTR2  = 1;
#endif
	WRMSR(Core_GlobalPerfControl, MSR_CORE_PERF_GLOBAL_CTRL);

	RDMSR(Core_FixedPerfControl, MSR_CORE_PERF_FIXED_CTR_CTRL);
	Core->SaveArea.Core_FixedPerfControl = Core_FixedPerfControl;
	Core_FixedPerfControl.EN0_OS = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
	Core_FixedPerfControl.EN1_OS = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
	Core_FixedPerfControl.EN2_OS = 1;
#endif
	Core_FixedPerfControl.EN0_Usr = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
	Core_FixedPerfControl.EN1_Usr = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
	Core_FixedPerfControl.EN2_Usr = 1;
#endif

	if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3) {
		if (!PUBLIC(RO(Proc))->Features.HTT_Enable) {
			Core_FixedPerfControl.AnyThread_EN0 = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
			Core_FixedPerfControl.AnyThread_EN1 = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
			Core_FixedPerfControl.AnyThread_EN2 = 1;
#endif
		} else {
			/* Per Thread */
			Core_FixedPerfControl.AnyThread_EN0 = 0;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
			Core_FixedPerfControl.AnyThread_EN1 = 0;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
			Core_FixedPerfControl.AnyThread_EN2 = 0;
#endif
		}
	}
	WRMSR(Core_FixedPerfControl, MSR_CORE_PERF_FIXED_CTR_CTRL);

	RDMSR(Core_PerfOverflow, MSR_CORE_PERF_GLOBAL_STATUS);
	RDMSR(Core_PerfOvfControl, MSR_CORE_PERF_GLOBAL_OVF_CTRL);
	if (Core_PerfOverflow.Overflow_CTR0)
		Core_PerfOvfControl.Clear_Ovf_CTR0 = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
	if (Core_PerfOverflow.Overflow_CTR1)
		Core_PerfOvfControl.Clear_Ovf_CTR1 = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
	if (Core_PerfOverflow.Overflow_CTR2)
		Core_PerfOvfControl.Clear_Ovf_CTR2 = 1;
#endif
	if (Core_PerfOverflow.Overflow_CTR0
	  | Core_PerfOverflow.Overflow_CTR1
	  | Core_PerfOverflow.Overflow_CTR2)
		WRMSR(Core_PerfOvfControl, MSR_CORE_PERF_GLOBAL_OVF_CTRL);
    }
}

#define AMD_Core_Counters_Set(Core, PMU)				\
({									\
    if (PUBLIC(RO(Proc))->Features.PerfMon.EBX.InstrRetired == 0)	\
    {									\
	HWCR HwCfgRegister = {.value = 0};				\
									\
	RDMSR(HwCfgRegister, MSR_K7_HWCR);				\
	Core->SaveArea.Core_HardwareConfiguration = HwCfgRegister;	\
	HwCfgRegister.PMU.IRPerfEn = 1;					\
	WRMSR(HwCfgRegister, MSR_K7_HWCR);				\
    }									\
})

#define Uncore_Counters_Set(PMU)					\
({									\
  if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3)		\
  {									\
	UNCORE_GLOBAL_PERF_CONTROL  Uncore_GlobalPerfControl;		\
	UNCORE_FIXED_PERF_CONTROL   Uncore_FixedPerfControl;		\
	UNCORE_GLOBAL_PERF_STATUS   Uncore_PerfOverflow = {.value = 0}; \
	UNCORE_GLOBAL_PERF_OVF_CTRL Uncore_PerfOvfControl = {.value = 0};\
									\
	RDMSR(	Uncore_GlobalPerfControl,				\
		MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL );			\
									\
	PUBLIC(RO(Proc))->SaveArea.Uncore_GlobalPerfControl =		\
						Uncore_GlobalPerfControl;\
									\
	Uncore_GlobalPerfControl.PMU.EN_FIXED_CTR0  = 1;		\
	WRMSR(	Uncore_GlobalPerfControl,				\
		MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL );			\
									\
	RDMSR(	Uncore_FixedPerfControl,				\
		MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL );		\
									\
	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl =		\
						Uncore_FixedPerfControl;\
									\
	Uncore_FixedPerfControl.PMU.EN_CTR0 = 1;			\
	WRMSR(	Uncore_FixedPerfControl,				\
		MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL );		\
									\
	RDMSR(Uncore_PerfOverflow, MSR_##PMU##_UNCORE_PERF_GLOBAL_STATUS);\
									\
    if (Uncore_PerfOverflow.PMU.Overflow_CTR0) {			\
	RDMSR(Uncore_PerfOvfControl, MSR_UNCORE_PERF_GLOBAL_OVF_CTRL);	\
	Uncore_PerfOvfControl.Clear_Ovf_CTR0 = 1;			\
	WRMSR(Uncore_PerfOvfControl, MSR_UNCORE_PERF_GLOBAL_OVF_CTRL);	\
    }									\
  }									\
})

void Intel_Core_Counters_Clear(CORE_RO *Core)
{
    if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 2)
    {
	WRMSR(Core->SaveArea.Core_FixedPerfControl,
					MSR_CORE_PERF_FIXED_CTR_CTRL);

	WRMSR(Core->SaveArea.Core_GlobalPerfControl,
					MSR_CORE_PERF_GLOBAL_CTRL);
    }
}

void AMD_Core_Counters_Clear(CORE_RO *Core)
{
	if (PUBLIC(RO(Proc))->Features.PerfMon.EBX.InstrRetired == 0)
	{
		WRMSR(Core->SaveArea.Core_HardwareConfiguration, MSR_K7_HWCR);
	}
}

#define Uncore_Counters_Clear(PMU)					\
({									\
    if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3)		\
    {									\
	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl,	\
		MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL );		\
									\
	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_GlobalPerfControl,	\
		MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL );			\
    }									\
})

#define Counters_VirtualMachine(Core, T)				\
({									\
	if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) { 	\
		RDTSC64(Core->Counter[T].TSC);				\
	} else {							\
		RDTSCP64(Core->Counter[T].TSC);				\
	}								\
	/* HV_X64_MSR_VP_RUNTIME: vcpu runtime in 100ns units	*/	\
	RDCOUNTER(Core->Counter[T].C0.URC, 0x40000010);			\
									\
	Core->Counter[T].C0.URC = Core->Counter[T].C0.URC		\
	        * PUBLIC(RO(Proc))->Features.Factory.Ratio * 10;	\
									\
	Core->Counter[T].C0.UCC = Core->Counter[T].C0.URC;		\
	/* Derive C1: */						\
	Core->Counter[T].C1 =						\
	    (Core->Counter[T].TSC > Core->Counter[T].C0.URC) ?		\
		Core->Counter[T].TSC - Core->Counter[T].C0.URC : 0;	\
})

#define Counters_Generic(Core, T)					\
({									\
	RDTSC_COUNTERx2(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC);	\
	/* Derive C1: */						\
	Core->Counter[T].C1 =						\
	  (Core->Counter[T].TSC > Core->Counter[T].C0.URC) ?		\
	    Core->Counter[T].TSC - Core->Counter[T].C0.URC		\
	    : 0;							\
})

#define Counters_Core2(Core, T)						\
({									\
    if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC)		\
    {									\
	RDTSC_COUNTERx3(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
    } else {								\
	RDTSCP_COUNTERx3(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
    }									\
	/* Derive C1: */						\
	Core->Counter[T].C1 =						\
	  (Core->Counter[T].TSC > Core->Counter[T].C0.URC) ?		\
	    Core->Counter[T].TSC - Core->Counter[T].C0.URC		\
	    : 0;							\
})

#define Counters_SLM(Core, T)						\
({									\
	RDTSC_COUNTERx6(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST, \
			MSR_CORE_C1_RESIDENCY, Core->Counter[T].C1,	\
			MSR_CORE_C3_RESIDENCY, Core->Counter[T].C3,	\
			MSR_CORE_C6_RESIDENCY, Core->Counter[T].C6);	\
})

#define Counters_Goldmont(Core, T) Counters_SLM(Core, T)

#define SMT_Counters_Nehalem(Core, T)					\
({									\
	register unsigned long long Cx;					\
									\
	RDTSCP_COUNTERx5(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_CORE_C3_RESIDENCY,Core->Counter[T].C3,	\
			MSR_CORE_C6_RESIDENCY,Core->Counter[T].C6,	\
			MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
	/* Derive C1: */						\
	Cx =	Core->Counter[T].C6					\
		+ Core->Counter[T].C3					\
		+ Core->Counter[T].C0.URC;				\
									\
	Core->Counter[T].C1 =						\
		(Core->Counter[T].TSC > Cx) ?				\
			Core->Counter[T].TSC - Cx			\
			: 0;						\
})

#define SMT_Counters_SandyBridge(Core, T)				\
({									\
	register unsigned long long Cx;					\
									\
	RDTSCP_COUNTERx6(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_CORE_C3_RESIDENCY,Core->Counter[T].C3,	\
			MSR_CORE_C6_RESIDENCY,Core->Counter[T].C6,	\
			MSR_CORE_C7_RESIDENCY,Core->Counter[T].C7,	\
			MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
	/* Derive C1: */						\
	Cx =	Core->Counter[T].C7					\
		+ Core->Counter[T].C6					\
		+ Core->Counter[T].C3					\
		+ Core->Counter[T].C0.URC;				\
									\
	Core->Counter[T].C1 =						\
		(Core->Counter[T].TSC > Cx) ?				\
			Core->Counter[T].TSC - Cx			\
			: 0;						\
})

#define SMT_Counters_AMD_Family_17h(Core, T)				\
({									\
	RDTSCP_COUNTERx3(Core->Counter[T].TSC,				\
			MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC,	\
			MSR_CORE_PERF_URC, Core->Counter[T].C0.URC,	\
			MSR_AMD_F17H_IRPERF, Core->Counter[T].INST);	\
									\
    if(PUBLIC(RO(Proc))->Registration.Driver.CPUidle == REGISTRATION_ENABLE)\
    {	/* Read Virtual PMC and cumulative store: */			\
	Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C1, Core->VPMC.C1); \
	Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C3, Core->VPMC.C2); \
	Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C3, Core->VPMC.C3); \
	Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C6, Core->VPMC.C4); \
	Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C6, Core->VPMC.C5); \
	Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C6, Core->VPMC.C6); \
    }									\
    else								\
    {									\
	Core->Counter[T].C1 =						\
		(Core->Counter[T].TSC > Core->Counter[T].C0.URC) ?	\
			Core->Counter[T].TSC - Core->Counter[T].C0.URC	\
			: 0;						\
    }									\
})

#define Mark_OVH(Core)							\
({									\
	RDTSCP64(Core->Overhead.TSC);					\
})

#define Core_OVH(Core)							\
({									\
	Core->Delta.TSC -= (Core->Counter[1].TSC - Core->Overhead.TSC); \
})

#define Delta_TSC(Core) 						\
({									\
	Core->Delta.TSC = Core->Counter[1].TSC				\
			- Core->Counter[0].TSC; 			\
})

#define Delta_TSC_OVH(Core)						\
({									\
	Delta_TSC(Core);						\
									\
	if (AutoClock & 0b10) { 					\
		Core_OVH(Core); 					\
									\
		REL_BCLK(Core->Clock,					\
			Core->Boost[BOOST(MAX)],			\
			Core->Delta.TSC,				\
			PUBLIC(RO(Proc))->SleepInterval);		\
	}								\
})

#define Delta_C0(Core)							\
({	/* Absolute Delta of Unhalted (Core & Ref) C0 Counter. */	\
	Core->Delta.C0.UCC =						\
		(Core->Counter[0].C0.UCC >				\
		Core->Counter[1].C0.UCC) ?				\
			Core->Counter[0].C0.UCC				\
			- Core->Counter[1].C0.UCC			\
			: Core->Counter[1].C0.UCC			\
			- Core->Counter[0].C0.UCC;			\
									\
	Core->Delta.C0.URC = Core->Counter[1].C0.URC			\
			   - Core->Counter[0].C0.URC;			\
})

#define Delta_C1(Core)							\
({									\
	Core->Delta.C1 =						\
		(Core->Counter[0].C1 >					\
		 Core->Counter[1].C1) ?					\
			Core->Counter[0].C1				\
			- Core->Counter[1].C1				\
			: Core->Counter[1].C1				\
			- Core->Counter[0].C1;				\
})

#define Delta_C3(Core)							\
({									\
	Core->Delta.C3  = Core->Counter[1].C3				\
			- Core->Counter[0].C3;				\
})

#define Delta_C6(Core)							\
({									\
	Core->Delta.C6  = Core->Counter[1].C6				\
			- Core->Counter[0].C6;				\
})

#define Delta_C7(Core)							\
({									\
	Core->Delta.C7  = Core->Counter[1].C7				\
			- Core->Counter[0].C7;				\
})

#define Delta_INST(Core)						\
({	/* Delta of Retired Instructions */				\
	Core->Delta.INST = Core->Counter[1].INST			\
			 - Core->Counter[0].INST;			\
})

#define PKG_Counters_VirtualMachine(Core, T)				\
({									\
	PUBLIC(RO(Proc))->Counter[T].PTSC = Core->Counter[T].TSC;	\
})

#define PKG_Counters_Generic(Core, T)					\
({									\
	PUBLIC(RO(Proc))->Counter[T].PTSC = Core->Counter[T].TSC;	\
})

#define PKG_Counters_SLM(Core, T)					\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
	MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,	\
	MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,	\
	MSR_ATOM_PKG_C4_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC04,	\
	MSR_ATOM_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,	\
	MSR_ATOM_MC6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].MC6);	\
})

#define PKG_Counters_Nehalem(Core, T)					\
({									\
    RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_NHM_UNCORE_PERF_FIXED_CTR0 ,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

#define PKG_Counters_SandyBridge(Core, T)				\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_SNB_UNCORE_PERF_FIXED_CTR0 ,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

#define PKG_COUNTERS_SANDYBRIDGE_EP(Core, T)				\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_SNB_EP_UNCORE_PERF_FIXED_CTR0,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

static void PKG_Counters_SandyBridge_EP(CORE_RO *Core, unsigned int T)
{
	UNUSED(Core);

	PKG_COUNTERS_SANDYBRIDGE_EP(Core, T);
}

#define PKG_COUNTERS_IVYBRIDGE_EP(Core, T)				\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_SNB_EP_UNCORE_PERF_FIXED_CTR0,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

static void PKG_Counters_IvyBridge_EP(CORE_RO *Core, unsigned int T)
{
	UNUSED(Core);

	PKG_COUNTERS_IVYBRIDGE_EP(Core, T);
}

#define PKG_Counters_Haswell_EP(Core, T)				\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC ,		\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_HSW_EP_UNCORE_PERF_FIXED_CTR0,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

#define PKG_Counters_Haswell_ULT(Core, T)				\
({									\
    RDTSCP_COUNTERx7(PUBLIC(RO(Proc))->Counter[T].PTSC,			\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_PKG_C8_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC08,\
		MSR_PKG_C9_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC09,\
		MSR_PKG_C10_RESIDENCY,PUBLIC(RO(Proc))->Counter[T].PC10);\
									\
	RDCOUNTER	(PUBLIC(RO(Proc))->Counter[T].Uncore.FC0,	\
			MSR_SNB_UNCORE_PERF_FIXED_CTR0 );		\
})

#define PKG_Counters_Goldmont(Core, T)					\
({									\
    RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PTSC,			\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C10_RESIDENCY,PUBLIC(RO(Proc))->Counter[T].PC10);\
})

#define PKG_Counters_Skylake(Core, T)					\
({									\
    RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PTSC,			\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
		MSR_SKL_UNCORE_PERF_FIXED_CTR0 ,			\
			PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);	\
})

#define PKG_Counters_Skylake_X(Core, T) 				\
({									\
    RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PTSC,			\
		MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
		MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
		MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
		MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07);\
})

#define Pkg_OVH(Pkg, Core)						\
({									\
	Pkg->Delta.PTSC -= (Pkg->Counter[1].PTSC - Core->Overhead.TSC); \
})

#define Delta_PTSC(Pkg) 						\
({									\
	Pkg->Delta.PTSC = Pkg->Counter[1].PTSC				\
			- Pkg->Counter[0].PTSC; 			\
})

#define Delta_PTSC_OVH(Pkg, Core)					\
({									\
	Delta_PTSC(Pkg);						\
									\
	if (AutoClock & 0b10)						\
		Pkg_OVH(Pkg, Core);					\
})

#define Delta_PC02(Pkg) 						\
({									\
	Pkg->Delta.PC02 = Pkg->Counter[1].PC02				\
			- Pkg->Counter[0].PC02;				\
})

#define Delta_PC03(Pkg) 						\
({									\
	Pkg->Delta.PC03 = Pkg->Counter[1].PC03				\
			- Pkg->Counter[0].PC03;				\
})

#define Delta_PC04(Pkg) 						\
({									\
	Pkg->Delta.PC04 = Pkg->Counter[1].PC04				\
			- Pkg->Counter[0].PC04;				\
})

#define Delta_PC06(Pkg) 						\
({									\
	Pkg->Delta.PC06 = Pkg->Counter[1].PC06				\
			- Pkg->Counter[0].PC06;				\
})

#define Delta_PC07(Pkg) 						\
({									\
	Pkg->Delta.PC07 = Pkg->Counter[1].PC07				\
			- Pkg->Counter[0].PC07;				\
})

#define Delta_PC08(Pkg) 						\
({									\
	Pkg->Delta.PC08 = Pkg->Counter[1].PC08				\
			- Pkg->Counter[0].PC08;				\
})

#define Delta_PC09(Pkg) 						\
({									\
	Pkg->Delta.PC09 = Pkg->Counter[1].PC09				\
			- Pkg->Counter[0].PC09;				\
})

#define Delta_PC10(Pkg) 						\
({									\
	Pkg->Delta.PC10 = Pkg->Counter[1].PC10				\
			- Pkg->Counter[0].PC10;				\
})

#define Delta_MC6(Pkg)							\
({									\
	Pkg->Delta.MC6	= Pkg->Counter[1].MC6				\
			- Pkg->Counter[0].MC6;				\
})

#define Delta_UNCORE_FC0(Pkg)						\
({									\
	Pkg->Delta.Uncore.FC0 =						\
		(Pkg->Counter[0].Uncore.FC0 >				\
		Pkg->Counter[1].Uncore.FC0) ?				\
			Pkg->Counter[0].Uncore.FC0			\
			- Pkg->Counter[1].Uncore.FC0			\
			: Pkg->Counter[1].Uncore.FC0			\
			- Pkg->Counter[0].Uncore.FC0;			\
})

#define Save_TSC(Core)							\
({	/* Save Time Stamp Counter. */					\
	Core->Counter[0].TSC = Core->Counter[1].TSC;			\
})

#define Save_C0(Core)							\
({	/* Save the Unhalted Core & Reference Counter */		\
	Core->Counter[0].C0.UCC = Core->Counter[1].C0.UCC;		\
	Core->Counter[0].C0.URC = Core->Counter[1].C0.URC;		\
})

#define Save_C1(Core)							\
({									\
	Core->Counter[0].C1 = Core->Counter[1].C1;			\
})

#define Save_C3(Core)							\
({									\
	Core->Counter[0].C3 = Core->Counter[1].C3;			\
})

#define Save_C6(Core)							\
({									\
	Core->Counter[0].C6 = Core->Counter[1].C6;			\
})

#define Save_C7(Core)							\
({									\
	Core->Counter[0].C7 = Core->Counter[1].C7;			\
})

#define Save_INST(Core) 						\
({	/* Save the Instructions counter. */				\
	Core->Counter[0].INST = Core->Counter[1].INST;			\
})

#define Save_PTSC(Pkg)							\
({									\
	Pkg->Counter[0].PTSC = Pkg->Counter[1].PTSC;			\
})

#define Save_PC02(Pkg)							\
({									\
	Pkg->Counter[0].PC02 = Pkg->Counter[1].PC02;			\
})

#define Save_PC03(Pkg)							\
({									\
	Pkg->Counter[0].PC03 = Pkg->Counter[1].PC03;			\
})

#define Save_PC04(Pkg)							\
({									\
	Pkg->Counter[0].PC04 = Pkg->Counter[1].PC04;			\
})

#define Save_PC06(Pkg)							\
({									\
	Pkg->Counter[0].PC06 = Pkg->Counter[1].PC06;			\
})

#define Save_PC07(Pkg)							\
({									\
	Pkg->Counter[0].PC07 = Pkg->Counter[1].PC07;			\
})

#define Save_PC08(Pkg)							\
({									\
	Pkg->Counter[0].PC08 = Pkg->Counter[1].PC08;			\
})

#define Save_PC09(Pkg)							\
({									\
	Pkg->Counter[0].PC09 = Pkg->Counter[1].PC09;			\
})

#define Save_PC10(Pkg)							\
({									\
	Pkg->Counter[0].PC10 = Pkg->Counter[1].PC10;			\
})

#define Save_MC6(Pkg)							\
({									\
	Pkg->Counter[0].MC6 = Pkg->Counter[1].MC6;			\
})

#define Save_UNCORE_FC0(Pkg)						\
({									\
	Pkg->Counter[0].Uncore.FC0 = Pkg->Counter[1].Uncore.FC0;	\
})

#define PWR_ACCU_Goldmont(Pkg, T)					\
({									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)],		\
						MSR_PKG_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)],	\
						MSR_PP0_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)],	\
						MSR_PP1_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)],		\
						MSR_DRAM_ENERGY_STATUS);\
})

#define PWR_ACCU_SandyBridge(Pkg, T)					\
({									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)],		\
						MSR_PKG_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)],	\
						MSR_PP0_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)],	\
						MSR_PP1_ENERGY_STATUS); \
})

#define PWR_ACCU_SandyBridge_EP(Pkg, T) 				\
({									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)],		\
						MSR_PKG_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)],	\
						MSR_PP0_ENERGY_STATUS); \
									\
	RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)],		\
						MSR_DRAM_ENERGY_STATUS);\
})

#define PWR_ACCU_Skylake(Pkg, T)					\
({									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)],		\
						MSR_PKG_ENERGY_STATUS); \
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)],	\
						MSR_PP0_ENERGY_STATUS); \
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)],	\
						MSR_PP1_ENERGY_STATUS); \
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)],		\
						MSR_DRAM_ENERGY_STATUS);\
})

static void Power_ACCU_SKL_DEFAULT(PROC_RO *Pkg, unsigned int T)
{
	PWR_ACCU_Skylake(Pkg, T);
}

#define PWR_ACCU_SKL_PLATFORM(Pkg, T)					\
({									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)],		\
						MSR_PKG_ENERGY_STATUS); \
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)],	\
						MSR_PP0_ENERGY_STATUS); \
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PLATFORM)],	\
					MSR_PLATFORM_ENERGY_STATUS);	\
									\
        RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)],		\
						MSR_DRAM_ENERGY_STATUS);\
})

static void Power_ACCU_SKL_PLATFORM(PROC_RO *Pkg, unsigned int T)
{
	PWR_ACCU_SKL_PLATFORM(Pkg, T);
}

#define Delta_PWR_ACCU(Pkg, PwrDomain)					\
({									\
	PUBLIC(RW(Pkg))->Delta.Power.ACCU[PWR_DOMAIN(PwrDomain)] =	\
	(PUBLIC(RO(Pkg))->Counter[1].Power.ACCU[PWR_DOMAIN(PwrDomain)]	\
	- PUBLIC(RO(Pkg))->Counter[0].Power.ACCU[PWR_DOMAIN(PwrDomain)])\
	& 0x7fffffffU;							\
})

#define Save_PWR_ACCU(Pkg, PwrDomain)					\
({									\
	Pkg->Counter[0].Power.ACCU[PWR_DOMAIN(PwrDomain)] =		\
		Pkg->Counter[1].Power.ACCU[PWR_DOMAIN(PwrDomain)];	\
})

void Core_Intel_Temp(CORE_RO *Core)
{
	THERM_STATUS ThermStatus = {.value = 0};
	RDMSR(ThermStatus, MSR_IA32_THERM_STATUS);	/*All Intel families.*/

	Core->PowerThermal.Sensor = ThermStatus.DTS;
	Core->PowerThermal.Events = (	(ThermStatus.StatusBit
					|ThermStatus.StatusLog ) << 0)
				  | (ThermStatus.PROCHOTLog << 1)
				  | (ThermStatus.CriticalTempLog << 2)
				  | (	(ThermStatus.Threshold1Log
					|ThermStatus.Threshold2Log ) << 3)
				  | (ThermStatus.PwrLimitLog << 4)
				  | (ThermStatus.CurLimitLog << 5)
				  | (ThermStatus.XDomLimitLog << 6);
}

#define Pkg_Intel_Temp(Pkg)						\
({									\
    if (Pkg->Features.Power.EAX.PTM)					\
    {									\
	THERM_STATUS ThermStatus = {.value = 0};			\
	RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);		\
									\
	Pkg->PowerThermal.Sensor = ThermStatus.DTS;			\
	Pkg->PowerThermal.Events = (	(ThermStatus.StatusBit		\
					|ThermStatus.StatusLog ) << 0)	\
				 | (ThermStatus.PROCHOTLog << 1)	\
				 | (ThermStatus.CriticalTempLog << 2)	\
				 | (	(ThermStatus.Threshold1Log	\
					|ThermStatus.Threshold2Log )<<3)\
				 | (ThermStatus.PwrLimitLog << 4);	\
    }									\
})

void Core_AMD_Family_0Fh_Temp(CORE_RO *Core)
{
	if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
		THERMTRIP_STATUS ThermTrip = {0};

		RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);

		/* Select Core to read sensor from: */
		ThermTrip.SensorCoreSelect = Core->Bind;

		WRPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);
		RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);

		/* Formula is " CurTmp - (TjOffset * 2) - 49 " */
		Core->PowerThermal.Param.Target = ThermTrip.TjOffset;
		Core->PowerThermal.Sensor = ThermTrip.CurrentTemp;

		Core->PowerThermal.Events = ThermTrip.SensorTrip << 0;
	}
}

void Core_AMD_Family_15h_00h_Temp(CORE_RO *Core)
{
	TCTL_REGISTER TctlSensor = {.value = 0};

	RDPCI(TctlSensor, PCI_AMD_TEMPERATURE_TCTL);
	Core->PowerThermal.Sensor = TctlSensor.CurTmp;

	if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
		THERMTRIP_STATUS ThermTrip = {0};

		RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);

		Core->PowerThermal.Events = ThermTrip.SensorTrip << 0;
	}
}

/*	Bulldozer/Excavator through SMU interface			*/
void Core_AMD_Family_15_60h_Temp(CORE_RO *Core)
{
	TCTL_REGISTER TctlSensor = {.value = 0};

	Core_AMD_SMN_Read(	TctlSensor,
				SMU_AMD_THM_TCTL_REGISTER_F15H,
				SMU_AMD_INDEX_REGISTER_F15H,
				SMU_AMD_DATA_REGISTER_F15H );

	Core->PowerThermal.Sensor = TctlSensor.CurTmp;

	if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
		THERMTRIP_STATUS ThermTrip = {0};

		WRPCI(	SMU_AMD_THM_TRIP_REGISTER_F15H,
			SMU_AMD_INDEX_REGISTER_F15H);
		RDPCI(ThermTrip, SMU_AMD_DATA_REGISTER_F15H);

		Core->PowerThermal.Events = ThermTrip.SensorTrip << 0;
	}
}

void Core_AMD_Family_15h_Temp(CORE_RO *Core)
{
	switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
	case 0x6:
	case 0x7:
		if ((PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
		 && (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf)) {
			Core_AMD_Family_15_60h_Temp(Core);
		} else {
			Core_AMD_Family_15h_00h_Temp(Core);
		}
		break;
	default:
		Core_AMD_Family_15h_00h_Temp(Core);
		break;
	}
}

static void CTL_AMD_Family_17h_Temp(CORE_RO *Core)
{
	TCTL_REGISTER TctlSensor = {.value = 0};
	TCTL_THERM_TRIP ThermTrip = {.value = 0};

	Core_AMD_SMN_Read(	TctlSensor,
				SMU_AMD_THM_TCTL_REGISTER_F17H,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	Core->PowerThermal.Sensor = TctlSensor.CurTmp;

	if (TctlSensor.CurTempRangeSel == 1)
	{
	/* Register: SMU::THM::THM_TCON_CUR_TMP - Bit 19: CUR_TEMP_RANGE_SEL
		0 = Report on 0C to 225C scale range.
		1 = Report on -49C to 206C scale range.
	*/
		Core->PowerThermal.Param.Offset[1] = 49;
	} else {
		Core->PowerThermal.Param.Offset[1] = 0;
	}

	Core_AMD_SMN_Read(	ThermTrip,
				SMU_AMD_THM_TCTL_REGISTER_F17H + 0x8,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	if (ThermTrip.THERM_TP_EN) {
		Core->PowerThermal.Events = ThermTrip.THERM_TP << 0;
	}
}

static void CCD_AMD_Family_17h_Zen2_Temp(CORE_RO *Core)
{
	TCCD_REGISTER TccdSensor = {.value = 0};
	TCTL_THERM_TRIP ThermTrip = {.value = 0};

	Core_AMD_SMN_Read(	TccdSensor,
				(SMU_AMD_THM_TCTL_CCD_REGISTER_F17H
				+ (Core->T.Cluster.CCD << 2)),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	Core->PowerThermal.Sensor = TccdSensor.CurTmp;

	if (TccdSensor.CurTempRangeSel == 1)
	{
		Core->PowerThermal.Param.Offset[1] = 49;
	} else {
		Core->PowerThermal.Param.Offset[1] = 0;
	}

	Core_AMD_SMN_Read(	ThermTrip,
				SMU_AMD_THM_TCTL_REGISTER_F17H + 0x8,
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	if (ThermTrip.THERM_TP_EN) {
		Core->PowerThermal.Events = ThermTrip.THERM_TP << 0;
	}
}

#define Pkg_AMD_Family_17h_Temp(Pkg, Core)				\
({									\
	Core_AMD_Family_17h_Temp(Core);					\
									\
	Pkg->PowerThermal.Sensor = Core->PowerThermal.Sensor;		\
})

#define Pkg_AMD_Family_19h_PROCHOT(Pkg) 				\
({									\
    if (Pkg->Features.HSMP_Enable)					\
    {									\
	unsigned int rx;						\
	HSMP_ARG arg[8];						\
	RESET_ARRAY(arg, 8, 0, .value);					\
	rx = AMD_HSMP_Exec(	HSMP_RD_PROCHOT, arg, SMU_HSMP_F19H,	\
				SMU_AMD_INDEX_REGISTER_F17H,		\
				SMU_AMD_DATA_REGISTER_F17H );		\
	if (rx == HSMP_RESULT_OK)					\
	{								\
		Pkg->PowerThermal.Events = ((arg[0].value & 0x1) << 1); \
	}								\
	else if (IS_HSMP_OOO(rx))					\
	{								\
		Pkg->Features.HSMP_Enable = 0;				\
	}								\
    }									\
})


static enum hrtimer_restart Cycle_VirtualMachine(struct hrtimer *pTimer)
{
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
		RDTSC64(Core->Overhead.TSC);
	} else {
		RDTSCP64(Core->Overhead.TSC);
	}
	if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
	{
		hrtimer_forward(pTimer,
				hrtimer_cb_get_time(pTimer),
				RearmTheTimer);

		Counters_VirtualMachine(Core, 1);

		if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
		{
			PKG_Counters_VirtualMachine(Core, 1);

			Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

			Save_PTSC(PUBLIC(RO(Proc)));

			Sys_Tick(PUBLIC(RO(Proc)));
		}

		Delta_C0(Core);

		Delta_TSC_OVH(Core);

		Delta_C1(Core);

		Save_TSC(Core);

		Save_C0(Core);

		Save_C1(Core);

		BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

void InitTimer_VirtualMachine(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_VirtualMachine, 1);
}

static void Start_VirtualMachine(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_VirtualMachine(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_VirtualMachine(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_VirtualMachine(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_GenuineIntel(struct hrtimer *pTimer)
{
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
		RDTSC64(Core->Overhead.TSC);
	} else {
		RDTSCP64(Core->Overhead.TSC);
	}
	if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
	{
		hrtimer_forward(pTimer,
				hrtimer_cb_get_time(pTimer),
				RearmTheTimer);

		Counters_Generic(Core, 1);

		if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
		{
			PKG_Counters_Generic(Core, 1);

			Pkg_Intel_Temp(PUBLIC(RO(Proc)));

		    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
		    {
		    case FORMULA_SCOPE_PKG:
			Core_Intel_Temp(Core);
			break;
		    }

			Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

			Save_PTSC(PUBLIC(RO(Proc)));

			Sys_Tick(PUBLIC(RO(Proc)));
		}

		switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
		case FORMULA_SCOPE_CORE:
		    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
			Core_Intel_Temp(Core);
		    }
			break;
		case FORMULA_SCOPE_SMT:
			Core_Intel_Temp(Core);
			break;
		}

		Delta_C0(Core);

		Delta_TSC_OVH(Core);

		Delta_C1(Core);

		Save_TSC(Core);

		Save_C0(Core);

		Save_C1(Core);

		BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

void InitTimer_GenuineIntel(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_GenuineIntel, 1);
}

static void Start_GenuineIntel(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_GenuineIntel(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_AuthenticAMD(struct hrtimer *pTimer)
{
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
		RDTSC64(Core->Overhead.TSC);
	} else {
		RDTSCP64(Core->Overhead.TSC);
	}
	if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
	{
		hrtimer_forward(pTimer,
				hrtimer_cb_get_time(pTimer),
				RearmTheTimer);

		Counters_Generic(Core, 1);

		if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
		{
			PKG_Counters_Generic(Core, 1);

			Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

			Save_PTSC(PUBLIC(RO(Proc)));

			Sys_Tick(PUBLIC(RO(Proc)));
		}

		Delta_C0(Core);

		Delta_TSC_OVH(Core);

		Delta_C1(Core);

		Save_TSC(Core);

		Save_C0(Core);

		Save_C1(Core);

		BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

void InitTimer_AuthenticAMD(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AuthenticAMD, 1);
}

static void Start_AuthenticAMD(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_AuthenticAMD(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_Core2(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

    if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
	RDTSC64(Core->Overhead.TSC);
    } else {
	RDTSCP64(Core->Overhead.TSC);
    }
    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	Counters_Core2(Core, 1);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.CORE.CurrFID;

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_CORE2_TARGET(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Generic(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

		switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
		{
		case FORMULA_SCOPE_PKG:
			Core_Intel_Temp(Core);
			break;
		}

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PerfStatus.CORE.CurrVID;

		switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
		{
		case FORMULA_SCOPE_PKG:
			Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
			break;
		}

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Save_PTSC(PUBLIC(RO(Proc)));

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
		break;
	}

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Core2(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Core2, 1);
}

static void Start_Core2(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	Counters_Core2(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Core2(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_Silvermont(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

  if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
  {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	Counters_SLM(Core, 1);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.CORE.CurrFID;

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_CORE2_TARGET(Core);

    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	PKG_Counters_SLM(Core, 1);

	Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	{
	case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	}

	PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PerfStatus.CORE.CurrVID;

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	{
	case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
		break;
	}
	RDCOUNTER( PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(PKG)],
			MSR_PKG_ENERGY_STATUS );

	RDCOUNTER( PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(CORES)],
			MSR_PP0_ENERGY_STATUS );

	Delta_PC02(PUBLIC(RO(Proc)));

	Delta_PC03(PUBLIC(RO(Proc)));

	Delta_PC04(PUBLIC(RO(Proc)));

	Delta_PC06(PUBLIC(RO(Proc)));

	Delta_MC6(PUBLIC(RO(Proc)));

	Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

	Delta_PWR_ACCU(Proc, PKG);

	Delta_PWR_ACCU(Proc, CORES);

	Save_PC02(PUBLIC(RO(Proc)));

	Save_PC03(PUBLIC(RO(Proc)));

	Save_PC04(PUBLIC(RO(Proc)));

	Save_PC06(PUBLIC(RO(Proc)));

	Save_MC6(PUBLIC(RO(Proc)));

	Save_PTSC(PUBLIC(RO(Proc)));

	Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

	Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

	Sys_Tick(PUBLIC(RO(Proc)));
    } else {
	Core->PowerThermal.VID = 0;
    }

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C1(Core);

	Save_C3(Core);

	Save_C6(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
  } else
	return HRTIMER_NORESTART;
}

void InitTimer_Silvermont(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Silvermont, 1);
}

static void Start_Silvermont(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	Counters_SLM(Core, 0);

    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
	}
	PKG_Counters_SLM(Core, 0);

	RDCOUNTER( PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(PKG)],
			MSR_PKG_ENERGY_STATUS );

	RDCOUNTER( PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(CORES)],
			MSR_PP0_ENERGY_STATUS );
    }
	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Silvermont(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_Nehalem(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_Nehalem(Core, 1);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Nehalem(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    #if defined(HWM_CHIPSET) \
	    && ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
		RDSIO(	PUBLIC(RO(Proc))->PowerThermal.VID.CPU,
			HWM_SIO_CPUVCORE,
			HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
	    #endif

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
	    #if defined(HWM_CHIPSET) \
	    && ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
		Core->PowerThermal.VID = PUBLIC(RO(Proc))->PowerThermal.VID.CPU;
	    #endif
		break;
	    }

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
	    #if defined(HWM_CHIPSET) \
	    && ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
		Core->PowerThermal.VID = 0;
	    #endif
	}

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_NEHALEM_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.NHM.CurrentRatio;

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
	    #if defined(HWM_CHIPSET) \
	    && ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
		RDSIO(	Core->PowerThermal.VID, HWM_SIO_CPUVCORE,
			HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
	    #endif
	    }
		break;
	case FORMULA_SCOPE_SMT:
	    #if defined(HWM_CHIPSET) \
	    && ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
		RDSIO(	Core->PowerThermal.VID, HWM_SIO_CPUVCORE,
			HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
	    #endif
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Nehalem(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Nehalem, 1);
}

static void Start_Nehalem(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_Nehalem(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Nehalem(Core, 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Nehalem(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_Nehalem(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Set(NHM);
}

static void Stop_Uncore_Nehalem(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Clear(NHM);
}


static enum hrtimer_restart Cycle_SandyBridge(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_SandyBridge(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, UNCORE);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_SandyBridge(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_SandyBridge, 1);
}

static void Start_SandyBridge(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_SandyBridge(Core, 0);
		PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_SandyBridge(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_SandyBridge(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Set(SNB);
}

static void Stop_Uncore_SandyBridge(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Clear(SNB);
}


static enum hrtimer_restart Cycle_Intel_Xeon_EP(struct hrtimer *pTimer,
			void (*PKG_Counters_Intel_EP)(CORE_RO*, unsigned int))
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Intel_EP(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, RAM);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

static enum hrtimer_restart Cycle_SandyBridge_EP(struct hrtimer *pTimer)
{
	return Cycle_Intel_Xeon_EP(pTimer, PKG_Counters_SandyBridge_EP);
}

void InitTimer_SandyBridge_EP(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_SandyBridge_EP, 1);
}

static void Entry_Intel_Xeon_EP(void *arg,
			void (*PKG_Counters_Intel_EP)(CORE_RO*, unsigned int))
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Intel_EP(Core, 0);
		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_SandyBridge_EP(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_SandyBridge_EP(void *arg)
{
	Entry_Intel_Xeon_EP(arg, PKG_Counters_SandyBridge_EP);
}

/*	SandyBridge/EP Uncore PMU MSR for Xeon family 06_2Dh		*/
static void Start_Uncore_SandyBridge_EP(void *arg)
{
	UNUSED(arg);

    if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version > 3)
    {
	/*	Tables 2-20 , 2-23 , 2-24				*/
	UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
	UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;

	RDMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl = \
						Uncore_FixedPerfControl;

	Uncore_FixedPerfControl.SNB.EN_CTR0 = 1;

	WRMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	RDMSR(Uncore_PMonGlobalControl, MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl = \
						Uncore_PMonGlobalControl;

	Uncore_PMonGlobalControl.Unfreeze_All = 1;

	WRMSR(Uncore_PMonGlobalControl, MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);
    }
}

static void Stop_Uncore_SandyBridge_EP(void *arg)
{
	UNUSED(arg);

    if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version > 3)
    {
      if (PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl.SNB.EN_CTR0 == 0)
      {
	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl.Freeze_All = 1;
      }
	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl,
		MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);

	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl,
		MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
    }
}

static enum hrtimer_restart Cycle_IvyBridge_EP(struct hrtimer *pTimer)
{
	return Cycle_Intel_Xeon_EP(pTimer, PKG_Counters_IvyBridge_EP);
}

void InitTimer_IvyBridge_EP(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_IvyBridge_EP, 1);
}

static void Start_IvyBridge_EP(void *arg)
{
	Entry_Intel_Xeon_EP(arg, PKG_Counters_IvyBridge_EP);
}

/*	IvyBridge/EP Uncore PMU MSR for Xeon family 06_3E		*/
static void Start_Uncore_IvyBridge_EP(void *arg)
{	/*	Tables 2-20 , 2-24 , 2-26 , 2-27 , 2-28 		*/
	UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
	UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;
	UNUSED(arg);

	RDMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl = \
						Uncore_FixedPerfControl;

	Uncore_FixedPerfControl.SNB.EN_CTR0 = 1;

	WRMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	RDMSR(Uncore_PMonGlobalControl, MSR_IVB_EP_PMON_GLOBAL_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl = \
						Uncore_PMonGlobalControl;

	Uncore_PMonGlobalControl.Unfreeze_All = 1;

	WRMSR(Uncore_PMonGlobalControl, MSR_IVB_EP_PMON_GLOBAL_CTRL);
}

static void Stop_Uncore_IvyBridge_EP(void *arg)
{
	UNUSED(arg);
	/* If fixed counter was disable at entry, force freezing	*/
    if (PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl.SNB.EN_CTR0 == 0)
    {
	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl.Freeze_All = 1;
    }
	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl,
		MSR_IVB_EP_PMON_GLOBAL_CTRL);

	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl,
		MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
}


static enum hrtimer_restart Cycle_Haswell_ULT(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Haswell_ULT(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PC08(PUBLIC(RO(Proc)));

		Delta_PC09(PUBLIC(RO(Proc)));

		Delta_PC10(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, UNCORE);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PC08(PUBLIC(RO(Proc)));

		Save_PC09(PUBLIC(RO(Proc)));

		Save_PC10(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Haswell_ULT(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Haswell_ULT, 1);
}

static void Start_Haswell_ULT(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Haswell_ULT(Core, 0);
		PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Haswell_ULT(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_Haswell_ULT(void *arg)
{
	UNUSED(arg);

    if (PUBLIC(RO(Proc))->Registration.Experimental) {
	Uncore_Counters_Set(SNB);
    }
}

static void Stop_Uncore_Haswell_ULT(void *arg)
{
	UNUSED(arg);

    if (PUBLIC(RO(Proc))->Registration.Experimental) {
	Uncore_Counters_Clear(SNB);
    }
}


static enum hrtimer_restart Cycle_Goldmont(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	Counters_Goldmont(Core, 1);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Goldmont(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_Goldmont(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC10(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, UNCORE);

		Delta_PWR_ACCU(Proc, RAM);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC10(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C1(Core);

	Save_C3(Core);

	Save_C6(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Goldmont(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Goldmont, 1);
}

static void Start_Goldmont(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	Counters_Goldmont(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Goldmont(Core, 0);
		PWR_ACCU_Goldmont(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Goldmont(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}


static enum hrtimer_restart Cycle_Haswell_EP(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Haswell_EP(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, RAM);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Haswell_EP(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Haswell_EP, 1);
}

static void Start_Haswell_EP(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Haswell_EP(Core, 0);
		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Haswell_EP(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_Haswell_EP(void *arg)
{
	UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
	UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;
	UNUSED(arg);

	RDMSR(Uncore_FixedPerfControl, MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl = \
						Uncore_FixedPerfControl;

	Uncore_FixedPerfControl.HSW_EP.EN_CTR0 = 1;

	WRMSR(Uncore_FixedPerfControl, MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL);

	RDMSR(Uncore_PMonGlobalControl, MSR_HSW_EP_PMON_GLOBAL_CTRL);

	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl = \
						Uncore_PMonGlobalControl;

	Uncore_PMonGlobalControl.Unfreeze_All = 1;

	WRMSR(Uncore_PMonGlobalControl, MSR_HSW_EP_PMON_GLOBAL_CTRL);
}

static void Stop_Uncore_Haswell_EP(void *arg)
{
	UNUSED(arg);

    if (PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl.HSW_EP.EN_CTR0 == 0)
    {
	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl.Freeze_All = 1;
    }
	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_PMonGlobalControl,
		MSR_HSW_EP_PMON_GLOBAL_CTRL );

	WRMSR(	PUBLIC(RO(Proc))->SaveArea.Uncore_FixedPerfControl,
		MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL );
}


static enum hrtimer_restart Cycle_Skylake(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Skylake(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		Power_ACCU_Skylake(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, UNCORE);

		Delta_PWR_ACCU(Proc, RAM);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Skylake(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Skylake, 1);
}

static void Start_Skylake(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Skylake(Core, 0);

		Power_ACCU_Skylake(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Skylake(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_Skylake(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Set(SKL);
}

static void Stop_Uncore_Skylake(void *arg)
{
	UNUSED(arg);

	Uncore_Counters_Clear(SKL);
}

static enum hrtimer_restart Cycle_Skylake_X(struct hrtimer *pTimer)
{
	PERF_STATUS PerfStatus = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	SMT_Counters_SandyBridge(Core, 1);

	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);

	RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
	Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Skylake_X(Core, 1);

		Pkg_Intel_Temp(PUBLIC(RO(Proc)));

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_Intel_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	    }

		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);

		Delta_PC02(PUBLIC(RO(Proc)));

		Delta_PC03(PUBLIC(RO(Proc)));

		Delta_PC06(PUBLIC(RO(Proc)));

		Delta_PC07(PUBLIC(RO(Proc)));

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_UNCORE_FC0(PUBLIC(RO(Proc)));

		Delta_PWR_ACCU(Proc, PKG);

		Delta_PWR_ACCU(Proc, CORES);

		Delta_PWR_ACCU(Proc, RAM);

		Save_PC02(PUBLIC(RO(Proc)));

		Save_PC03(PUBLIC(RO(Proc)));

		Save_PC06(PUBLIC(RO(Proc)));

		Save_PC07(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_UNCORE_FC0(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);

		Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_Intel_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_Intel_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
		break;
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_C7(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C7(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_Skylake_X(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_Skylake_X, 1);
}

static void Start_Skylake_X(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Intel_Core_Counters_Set(Core);
	SMT_Counters_SandyBridge(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Skylake_X(Core, 0);
		PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
	}

	RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_Skylake_X(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	Intel_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_Uncore_Skylake_X(void *arg)
{
	UNUSED(arg);
/*TODO(Hardware needed)
	Uncore_Counters_Set(SKL_X);
*/
}

static void Stop_Uncore_Skylake_X(void *arg)
{
	UNUSED(arg);
/*TODO(Hardware needed)
	Uncore_Counters_Clear(SKL_X);
*/
}

static enum hrtimer_restart Cycle_AMD_Family_0Fh(struct hrtimer *pTimer)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	FIDVID_CONTROL FidVidControl = {.value = 0};
	FIDVID_STATUS FidVidStatus = {.value = 0};

	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	RDMSR(FidVidControl, MSR_K7_FID_VID_CTL);
	PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = FidVidControl.NewFID;

	RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);

	Core->PowerThermal.VID	= FidVidStatus.CurrVID;
	Core->Ratio.Perf	= 8 + FidVidStatus.CurrFID;

	/* P-States */
	Core->Counter[1].C0.UCC = Core->Counter[0].C0.UCC
				+ Core->Ratio.Perf
				* Core->Clock.Hz;

	Core->Counter[1].C0.URC = Core->Counter[1].C0.UCC;

	Core->Counter[1].TSC	= Core->Counter[0].TSC
				+ (Core->Boost[BOOST(MAX)]
					* Core->Clock.Hz);

	/* Derive C1 */
	Core->Counter[1].C1 =
	  (Core->Counter[1].TSC > Core->Counter[1].C0.URC) ?
	    Core->Counter[1].TSC - Core->Counter[1].C0.URC
	    : 0;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Generic(Core, 1);

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_AMD_Family_0Fh_Temp(Core);
		break;
	    }

		PUBLIC(RO(Proc))->PowerThermal.VID.CPU = FidVidStatus.CurrVID;

		Delta_PTSC(PUBLIC(RO(Proc)));

		Save_PTSC(PUBLIC(RO(Proc)));

		Sys_Tick(PUBLIC(RO(Proc)));
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core_AMD_Family_0Fh_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_AMD_Family_0Fh_Temp(Core);
		break;
	}

	Delta_C0(Core);

	Delta_TSC(Core);

	Delta_C1(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C1(Core);

	if (AutoClock & 0b10)
	{
		REL_BCLK(Core->Clock,
			Core->Boost[BOOST(MAX)],
			Core->Delta.TSC,
			PUBLIC(RO(Proc))->SleepInterval);
	}
	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_AMD_Family_0Fh(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_Family_0Fh, 1);
}

static void Start_AMD_Family_0Fh(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_AMD_Family_0Fh(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_AMD_Family_10h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_AMD_Family_10h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_AMD_Family_11h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_AMD_Family_12h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Start_AMD_Family_14h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static enum hrtimer_restart Cycle_AMD_Family_15h(struct hrtimer *pTimer)
{
	PSTATECTRL PstateCtrl = {.value = 0};
	PSTATESTAT PstateStat = {.value = 0};
	PSTATEDEF PstateDef = {.value = 0};
	CORE_RO *Core;
	unsigned int cpu;

	cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

    if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
    {
	hrtimer_forward(pTimer,
			hrtimer_cb_get_time(pTimer),
			RearmTheTimer);

	Counters_Generic(Core, 1);

	if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
	{
		unsigned int pstate, COF;
		/*	Read the Target & Status P-State.		*/
		RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
		RDMSR(PstateDef,MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);

		COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
					PstateDef.Family_15h.CpuDid );

		PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = COF;

		/*	Read the current P-State number.		*/
		RDMSR(PstateStat, MSR_AMD_PERF_STATUS);
		/*	Offset the P-State base register.		*/
		pstate = MSR_AMD_PSTATE_DEF_BASE + PstateStat.Current;
		/*	Read the voltage ID at the offset		*/
		RDMSR(PstateDef, pstate);

		COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
					PstateDef.Family_15h.CpuDid );
		Core->Ratio.Perf = COF;
	}
	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Generic(Core, 1);

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core_AMD_Family_15h_Temp(Core);
		break;
	    }

	    PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PstateDef.Family_15h.CpuVid;

	    switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
	    {
	    case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
		break;
	    }

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Save_PTSC(PUBLIC(RO(Proc)));

		Sys_Tick(PUBLIC(RO(Proc)));
	} else {
		Core->PowerThermal.VID = 0;
	}
	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if (Core->T.CoreID == 0) {
		Core_AMD_Family_15h_Temp(Core);
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core_AMD_Family_15h_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
		Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
	    }
		break;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
		break;
	}

	Delta_C0(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

	return HRTIMER_RESTART;
    } else
	return HRTIMER_NORESTART;
}

void InitTimer_AMD_Family_15h(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_Family_15h, 1);
}

static void Start_AMD_Family_15h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	Counters_Generic(Core, 0);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
		}
		PKG_Counters_Generic(Core, 0);
	}

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

void Cycle_AMD_Family_17h(CORE_RO *Core,
			void (*Call_SMU)(const unsigned int, const unsigned int,
					const unsigned long long),
			const unsigned int plane0, const unsigned int plane1,
			const unsigned long long factor)
{
	PSTATEDEF PstateDef;
	PSTATECTRL PstateCtrl;
	COF_ST COF;

	SMT_Counters_AMD_Family_17h(Core, 1);

	/*		Read the Target P-State.			*/
	RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
	RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);

	COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
				PstateDef.Family_17h.CpuDfsId );

	PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(TGT)] = COF.Q;

	/*	Read the Boosted Frequency and voltage VID.		*/
	RDMSR(PstateDef, MSR_AMD_F17H_HW_PSTATE_STATUS);
	COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
				PstateDef.Family_17h.CpuDfsId );
	Core->Ratio.COF = COF;

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
	{
		PKG_Counters_Generic(Core, 1);

		Pkg_AMD_Family_17h_Temp(PUBLIC(RO(Proc)), Core);

	  switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	  case FORMULA_SCOPE_PKG:
		Core_AMD_Family_17h_Temp(Core);
		break;
	  }

	  switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	  case FORMULA_SCOPE_PKG:
		Core->PowerThermal.VID = PstateDef.Family_17h.CpuVid;
		break;
	  }

		Call_SMU(plane0, plane1, factor);

	    RDCOUNTER(PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(PKG)],
			MSR_AMD_PKG_ENERGY_STATUS);

		Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);

		Delta_PWR_ACCU(Proc, PKG);

		Save_PTSC(PUBLIC(RO(Proc)));

		Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);

		Sys_Tick(PUBLIC(RO(Proc)),
			{
				Pkg_AMD_Family_19h_PROCHOT(PUBLIC(RO(Proc)));
			});
	} else {
		Core->PowerThermal.VID = 0;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
	case FORMULA_SCOPE_CORE:
	    if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
	    {
		Core_AMD_Family_17h_Temp(Core);
		break;
	    }
		fallthrough;
	case FORMULA_SCOPE_SMT:
		Core_AMD_Family_17h_Temp(Core);
		break;
	}

	switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
	case FORMULA_SCOPE_CORE:
	    if (!((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)))
	    {
		break;
	    }
		fallthrough;
	case FORMULA_SCOPE_SMT:
		Core->PowerThermal.VID = PstateDef.Family_17h.CpuVid;
		break;
	}

	/*		Read the Physical Core RAPL counter.		*/
    if (Core->T.ThreadID == 0)
    {
	RDCOUNTER(Core->Counter[1].Power.ACCU,MSR_AMD_PP0_ENERGY_STATUS);
	Core->Counter[1].Power.ACCU &= 0xffffffff;

	Core->Delta.Power.ACCU  = Core->Counter[1].Power.ACCU
				- Core->Counter[0].Power.ACCU;

	Core->Delta.Power.ACCU &= 0xffffffff;

	Core->Counter[0].Power.ACCU = Core->Counter[1].Power.ACCU;
    }

	Delta_INST(Core);

	Delta_C0(Core);

	Delta_C3(Core);

	Delta_C6(Core);

	Delta_TSC_OVH(Core);

	Delta_C1(Core);

	Save_INST(Core);

	Save_TSC(Core);

	Save_C0(Core);

	Save_C3(Core);

	Save_C6(Core);

	Save_C1(Core);

	BITSET(LOCKLESS, PUBLIC(RW(Core, AT(Core->Bind)))->Sync.V, NTFY);
}

inline void SoC_RAPL(AMD_17_SVI SVI, const unsigned long long factor)
{
	unsigned long long VCC, ICC, ACCU;
	/*	PLATFORM RAPL workaround to provide the SoC power	*/
	VCC = 155000LLU - (625LLU * SVI.VID);
	ICC = SVI.IDD * factor;
	ACCU = VCC * ICC;
	ACCU = ACCU << PUBLIC(RO(Proc))->PowerThermal.Unit.ESU;
	ACCU = ACCU / (100000LLU * 1000000LLU);
	ACCU = (PUBLIC(RO(Proc))->SleepInterval * ACCU) / 1000LLU;
	PUBLIC(RW(Proc))->Delta.Power.ACCU[PWR_DOMAIN(UNCORE)] = ACCU;
}

void Call_SVI(	const unsigned int plane0, const unsigned int plane1,
		const unsigned long long factor )
{
	AMD_17_SVI SVI = {.value = 0};

	Core_AMD_SMN_Read(	SVI,
				SMU_AMD_F17H_SVI(plane0),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	PUBLIC(RO(Proc))->PowerThermal.VID.CPU = SVI.VID;

	Core_AMD_SMN_Read(	SVI,
				SMU_AMD_F17H_SVI(plane1),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.VID;

	SoC_RAPL(SVI, factor);
}

void Call_SVI_APU(	const unsigned int plane0, const unsigned int plane1,
			const unsigned long long factor )
{
	AMD_17_SVI SVI = {.value = 0};

	Core_AMD_SMN_Read(	SVI,
				SMU_AMD_F17_60H_SVI(plane0),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	PUBLIC(RO(Proc))->PowerThermal.VID.CPU = SVI.VID;

	Core_AMD_SMN_Read(	SVI,
				SMU_AMD_F17_60H_SVI(plane1),
				SMU_AMD_INDEX_REGISTER_F17H,
				SMU_AMD_DATA_REGISTER_F17H );

	PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.VID;

	SoC_RAPL(SVI, factor);
}

void Call_DFLT( const unsigned int plane0, const unsigned int plane1,
		const unsigned long long factor )
{
	UNUSED(plane0);
	UNUSED(plane1);
	UNUSED(factor);

	PUBLIC(RO(Proc))->PowerThermal.VID.CPU = \
	PUBLIC(RO(Core,AT( PUBLIC(RO(Proc))->Service.Core )))->PowerThermal.VID;
}

static enum hrtimer_restart Entry_AMD_F17h(struct hrtimer *pTimer,
		void (*Call_SMU)(const unsigned int, const unsigned int,
				const unsigned long long),
		const unsigned int plane0, const unsigned int plane1,
		const unsigned long long factor)
{
	CORE_RO *Core;
	unsigned int cpu = smp_processor_id();
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	Mark_OVH(Core);

	if (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD) == 1)
	{
		hrtimer_forward(pTimer,
				hrtimer_cb_get_time(pTimer),
				RearmTheTimer);

		Cycle_AMD_Family_17h(Core, Call_SMU, plane0, plane1, factor);

		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

static enum hrtimer_restart Cycle_AMD_F17h_Zen(struct hrtimer *pTimer)
{
	return Entry_AMD_F17h(pTimer, Call_SVI, 0, 1, 360772LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_SP(struct hrtimer *pTimer)
{
	return Entry_AMD_F17h(pTimer, Call_SVI, 1, 0, 294300LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_MP(struct hrtimer *pTimer)
{
	return Entry_AMD_F17h(pTimer, Call_SVI, 2, 1, 294300LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_APU(struct hrtimer *pTimer)
{
	return Entry_AMD_F17h(pTimer, Call_SVI_APU, 0, 1, 294300LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h(struct hrtimer *pTimer)
{
	return Entry_AMD_F17h(pTimer, Call_DFLT, 0, 0, 0LLU);
}

void InitTimer_AMD_Family_17h(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h, 1);
}

void InitTimer_AMD_F17h_Zen(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen, 1);
}

void InitTimer_AMD_F17h_Zen2_SP(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_SP, 1);
}

void InitTimer_AMD_F17h_Zen2_MP(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_MP, 1);
}

void InitTimer_AMD_F17h_Zen2_APU(unsigned int cpu)
{
	smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_APU, 1);
}

static void Start_AMD_Family_17h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
	}

	AMD_Core_Counters_Set(Core, Family_17h);
	SMT_Counters_AMD_Family_17h(Core, 0);

    if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
    {
	if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
		Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
	}
	PKG_Counters_Generic(Core, 0);

	RDCOUNTER(PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(PKG)],
			MSR_AMD_PKG_ENERGY_STATUS );
    }
    if (Core->T.ThreadID == 0)
    {
	RDCOUNTER(Core->Counter[0].Power.ACCU,MSR_AMD_PP0_ENERGY_STATUS);
	Core->Counter[0].Power.ACCU &= 0xffffffff;
    }

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_start(	&PRIVATE(OF(Join, AT(cpu)))->Timer,
			RearmTheTimer,
			HRTIMER_MODE_REL_PINNED);

	BITSET(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

static void Stop_AMD_Family_17h(void *arg)
{
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	UNUSED(arg);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, MUSTFWD);

	hrtimer_cancel(&PRIVATE(OF(Join, AT(cpu)))->Timer);

	AMD_Core_Counters_Clear(Core);

	if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
		if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
			Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
		}
	}
	PerCore_Reset(Core);

	BITCLR(LOCKLESS, PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED);
}

long Sys_OS_Driver_Query(void)
{
	int rc = RC_SUCCESS;
#ifdef CONFIG_CPU_FREQ
	const char *pFreqDriver;
	struct cpufreq_policy freqPolicy;
#endif /* CONFIG_CPU_FREQ */
#ifdef CONFIG_CPU_IDLE
	struct cpuidle_driver *idleDriver;
#endif /* CONFIG_CPU_IDLE */
	memset(&PUBLIC(RO(Proc))->OS, 0, sizeof(OS_DRIVER));
#ifdef CONFIG_CPU_IDLE
    if ((idleDriver = cpuidle_get_driver()) != NULL)
    {
	int idx;

	StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.Name,
		idleDriver->name,
		CPUIDLE_NAME_LEN);

      if (idleDriver->state_count < CPUIDLE_STATE_MAX) {
	PUBLIC(RO(Proc))->OS.IdleDriver.stateCount = idleDriver->state_count;
      } else {
	PUBLIC(RO(Proc))->OS.IdleDriver.stateCount = CPUIDLE_STATE_MAX;
      }
	PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = idleDriver->state_count;

      for (idx = 0; idx < PUBLIC(RO(Proc))->OS.IdleDriver.stateCount; idx++)
      {
	StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].Name,
		idleDriver->states[idx].name, CPUIDLE_NAME_LEN);

	StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].Desc,
		idleDriver->states[idx].desc, CPUIDLE_NAME_LEN);

	PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].exitLatency = \
					idleDriver->states[idx].exit_latency;

	PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].powerUsage = \
					idleDriver->states[idx].power_usage;

	PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].targetResidency = \
				idleDriver->states[idx].target_residency;
      }
      if(PUBLIC(RO(Proc))->Registration.Driver.CPUidle == REGISTRATION_ENABLE)
      {
	for (idx = 0; idx < CoreFreqK.IdleDriver.state_count; idx++)
	{
	    if (CoreFreqK.IdleDriver.states[idx].flags & CPUIDLE_FLAG_UNUSABLE)
	    {
		PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = idx;
		break;
	    }
	}
      }
    }
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
	if ((pFreqDriver = cpufreq_get_current_driver()) != NULL) {
		StrCopy(PUBLIC(RO(Proc))->OS.FreqDriver.Name,
			pFreqDriver, CPUFREQ_NAME_LEN);
	}
	memset(&freqPolicy, 0, sizeof(freqPolicy));
    if((rc=cpufreq_get_policy(&freqPolicy, PUBLIC(RO(Proc))->Service.Core))==0)
    {
	struct cpufreq_governor *pGovernor = freqPolicy.governor;
	if (pGovernor != NULL) {
		StrCopy(PUBLIC(RO(Proc))->OS.FreqDriver.Governor,
			pGovernor->name, CPUFREQ_NAME_LEN);
	} else {
		PUBLIC(RO(Proc))->OS.FreqDriver.Governor[0] = '\0';
	}
    } else {
	PUBLIC(RO(Proc))->OS.FreqDriver.Governor[0] = '\0';
    }
#endif /* CONFIG_CPU_FREQ */
	return rc;
}

long Sys_Kernel(SYSGATE_RO *SysGate)
{	/* Sources:	/include/generated/uapi/linux/version.h
			/include/uapi/linux/utsname.h			*/
	if (SysGate != NULL) {
		SysGate->kernelVersionNumber = LINUX_VERSION_CODE;
		memcpy(SysGate->sysname, utsname()->sysname, MAX_UTS_LEN);
		memcpy(SysGate->release, utsname()->release, MAX_UTS_LEN);
		memcpy(SysGate->version, utsname()->version, MAX_UTS_LEN);
		memcpy(SysGate->machine, utsname()->machine, MAX_UTS_LEN);

		return RC_OK_SYSGATE;
	} else {
		return -ENXIO;
	}
}

long SysGate_OnDemand(void)
{
	long rc = -1;
    if (PUBLIC(OF(Gate)) == NULL)
    {	/*			On-demand allocation.			*/
	PUBLIC(OF(Gate)) = alloc_pages_exact(PUBLIC(RO(Proc))->Gate.ReqMem.Size,
						GFP_KERNEL);
	if (PUBLIC(OF(Gate)) != NULL)
	{
		const size_t
		allocPages = PAGE_SIZE << PUBLIC(RO(Proc))->Gate.ReqMem.Order;
		memset(PUBLIC(OF(Gate)), 0, allocPages);
		rc = 0;
	}
    } else {					/* Already allocated	*/
		rc = 1;
    }
	return rc;
}

#define Atomic_Write_VPMC( _Core, cycles, _lvl)				\
{									\
	switch (_lvl) {							\
	case 0:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C1, cycles);	\
		break;							\
	case 1:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C2, cycles);	\
		break;							\
	case 2:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C3, cycles);	\
		break;							\
	case 3:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C4, cycles);	\
		break;							\
	case 4:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C5, cycles);	\
		break;							\
	case 5:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C6, cycles);	\
		break;							\
	case 6:								\
		Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C7, cycles);	\
		break;							\
	};								\
}

#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
	/*			MWAIT Idle methods			*/
static int CoreFreqK_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{/*	Source: /drivers/cpuidle/cpuidle.c				*/
	unsigned long MWAIT=(CoreFreqK.IdleDriver.states[index].flags>>24)&0xff;
	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);

	mwait_idle_with_hints(MWAIT, 1UL);

	return index;
}

static int CoreFreqK_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{/* Avoid kernel idle loop to be stuck if the MWAIT opcode has been disabled */
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
	unsigned long MWAIT=(CoreFreqK.IdleDriver.states[index].flags>>24)&0xff;
	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);

	mwait_idle_with_hints(MWAIT, 1UL);

#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
	return index;
#endif /* 5.9.0 */
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	CoreFreqK_S2_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
    }
}
#else
static int CoreFreqK_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_S2_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}
#endif /* 5.9.0 */
	/*			HALT Idle methods			*/
static int CoreFreqK_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);
/*	Source: /arch/x86/include/asm/irqflags.h: native_safe_halt();	*/
	__asm__ volatile
	(
		"sti"		"\n\t"
		"hlt"		"\n\t"
		"# RFLAGS"	"\n\t"
		"pushfq"	"\n\t"
		"popq	%0"
		: "=m" (Core->SystemRegister.RFLAGS)
		:
		: "cc", "memory"
	);
	return index;
}

static int CoreFreqK_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_HALT_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);

	__asm__ volatile
	(
		"sti"		"\n\t"
		"hlt"		"\n\t"
		"# RFLAGS"	"\n\t"
		"pushfq"	"\n\t"
		"popq	%0"
		: "=m" (Core->SystemRegister.RFLAGS)
		:
		: "cc", "memory"
	);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
	return index;
#endif /* 5.9.0 */
}


#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	CoreFreqK_S2_HALT_Handler(pIdleDevice, pIdleDriver, index);
    }
}
#else
static int CoreFreqK_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_S2_HALT_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}
#endif /* 5.9.0 */
	/*			I/O Idle methods			*/
static int CoreFreqK_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);

	__asm__ volatile
	(
		"xorw	%%ax,	%%ax"	"\n\t"
		"movw	%1,	%%dx"	"\n\t"
		"inb	%%dx,	%%al"	"\n\t"
		"# RFLAGS"		"\n\t"
		"pushfq"		"\n\t"
		"popq	%0"
		: "=m" (Core->SystemRegister.RFLAGS)
		: "ir" (Core->Query.CStateBaseAddr)
		: "%ax", "%dx", "cc", "memory"
	);
	return index;
}

static int CoreFreqK_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_IO_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	UNUSED(pIdleDevice);
	UNUSED(pIdleDriver);

	__asm__ volatile
	(
		"xorw	%%ax,	%%ax"	"\n\t"
		"movw	%1,	%%dx"	"\n\t"
		"inb	%%dx,	%%al"	"\n\t"
		"# RFLAGS"		"\n\t"
		"pushfq"		"\n\t"
		"popq	%0"
		: "=m" (Core->SystemRegister.RFLAGS)
		: "ir" (Core->Query.CStateBaseAddr)
		: "%ax", "%dx", "cc", "memory"
	);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
	return index;
#endif /* 5.9.0 */
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	CoreFreqK_S2_IO_Handler(pIdleDevice, pIdleDriver, index);
    }
}
#else
static int CoreFreqK_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	HWCR HwCfgRegister = {.value = 0};
	RDMSR(HwCfgRegister, MSR_K7_HWCR);
    if (BITVAL(HwCfgRegister.value, 9) == 0)
    {
	return CoreFreqK_S2_IO_Handler(pIdleDevice, pIdleDriver, index);
    } else {
	return index;
    }
}
#endif /* 5.9.0 */
	/*		Idle Cycles callback functions			*/
static int Alternative_Computation_Of_Cycles(
	int (*Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
			struct cpuidle_device *pIdleDevice,
			struct cpuidle_driver *pIdleDriver, int index
)
{
	unsigned long long TSC[3] __attribute__ ((aligned (8)));
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	const unsigned short lvl = \
			(CoreFreqK.IdleDriver.states[index].flags >> 28) & 0xf;

	if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
	||  (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
	{
		RDTSCP64(TSC[0]);

		RDTSCP64(TSC[1]);

		Handler(pIdleDevice, pIdleDriver, index);

		RDTSCP64(TSC[2]);
	}
	else
	{
		RDTSC64(TSC[0]);

		RDTSCP64(TSC[1]);

		Handler(pIdleDevice, pIdleDriver, index);

		RDTSC64(TSC[2]);
	}
	TSC[2]	= TSC[2] - TSC[1]
		- (TSC[1] > TSC[0] ? TSC[1] - TSC[0] : TSC[0] - TSC[1]);

	Atomic_Write_VPMC(Core, TSC[2], lvl);

	return index;
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void Alternative_Computation_Of_Cycles_S2(
	void (*S2_Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
				struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index
)
#else
static int Alternative_Computation_Of_Cycles_S2(
	int (*S2_Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
			struct cpuidle_device *pIdleDevice,
			struct cpuidle_driver *pIdleDriver, int index
)
#endif /* 5.9.0 */
{
	unsigned long long TSC[3] __attribute__ ((aligned (8)));
	const unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
	const unsigned short lvl = \
			(CoreFreqK.IdleDriver.states[index].flags >> 28) & 0xf;

	if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
	||  (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
	{
		RDTSCP64(TSC[0]);

		RDTSCP64(TSC[1]);

		S2_Handler(pIdleDevice, pIdleDriver, index);

		RDTSCP64(TSC[2]);
	}
	else
	{
		RDTSC64(TSC[0]);

		RDTSC64(TSC[1]);

		S2_Handler(pIdleDevice, pIdleDriver, index);

		RDTSCP64(TSC[2]);
	}
	TSC[2]	= TSC[2] - TSC[1]
		- (TSC[1] > TSC[0] ? TSC[1] - TSC[0] : TSC[0] - TSC[1]);

	Atomic_Write_VPMC(Core, TSC[2], lvl);

#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
	return index;
#endif /* 5.9.0 */
}
#undef Atomic_Write_VPMC

	/*		Alternative Idle methods			*/
static int CoreFreqK_Alt_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_MWAIT_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static int CoreFreqK_Alt_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_HALT_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static int CoreFreqK_Alt_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_IO_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static int CoreFreqK_Alt_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_MWAIT_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static int CoreFreqK_Alt_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_HALT_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static int CoreFreqK_Alt_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	return Alternative_Computation_Of_Cycles( CoreFreqK_IO_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
 || ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_Alt_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static void CoreFreqK_Alt_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static void CoreFreqK_Alt_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}
static void CoreFreqK_Alt_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static void CoreFreqK_Alt_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}

static void CoreFreqK_Alt_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_AMD_Handler,
							pIdleDevice,
							pIdleDriver,
							index );
}
#else
static int CoreFreqK_Alt_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}

static int CoreFreqK_Alt_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}

static int CoreFreqK_Alt_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}

static int CoreFreqK_Alt_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_AMD_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}

static int CoreFreqK_Alt_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_AMD_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}

static int CoreFreqK_Alt_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
				struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
	int rx = index;
#else
	int rx =
#endif
	Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_AMD_Handler,
						pIdleDevice,
						pIdleDriver,
						index );
	return rx;
}
#endif /* 5.9.0 */
#endif /* CONFIG_CPU_IDLE and 4.14.0 */

static void CoreFreqK_IdleDriver_UnInit(void)
{
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
	struct cpuidle_device *device;
	unsigned int cpu;

	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
	    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW)) {
		if ((device=per_cpu_ptr(CoreFreqK.IdleDevice, cpu)) != NULL)
		{
			cpuidle_unregister_device(device);
		}
	    }
	}
	cpuidle_unregister_driver(&CoreFreqK.IdleDriver);
	free_percpu(CoreFreqK.IdleDevice);
#endif /* CONFIG_CPU_IDLE and 4.14.0 */
}

static int CoreFreqK_IdleDriver_Init(void)
{
	int rc = -RC_UNIMPLEMENTED;
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
  if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
  {
	IDLE_STATE *pIdleState;
	pIdleState = Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState;
   if ((pIdleState != NULL) && PUBLIC(RO(Proc))->Features.Std.ECX.MONITOR)
   {
    if ((CoreFreqK.IdleDevice=alloc_percpu(struct cpuidle_device)) == NULL)
    {
	rc = -ENOMEM;
    }
    else
    {
	struct cpuidle_device *device;
	unsigned int cpu, enroll = 0;
	unsigned int subState[] = {
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT0,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT1,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT1, /* C1E */
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT2,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT3,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT4,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT5,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT6,
	PUBLIC(RO(Proc))->Features.MWait.EDX.SubCstate_MWAIT7
	};
	const unsigned int subStateCount = sizeof(subState)
					 / sizeof(subState[0]);
	/*		Kernel polling loop			*/
	cpuidle_poll_state_init(&CoreFreqK.IdleDriver);

	CoreFreqK.IdleDriver.state_count = 1;
	/*		Idle States				*/
     while (pIdleState->Name != NULL)
     {
	if (CoreFreqK.IdleDriver.state_count < subStateCount)
	{
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].flags = pIdleState->flags;

	    if (subState[CoreFreqK.IdleDriver.state_count] == 0)
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].flags |= CPUIDLE_FLAG_UNUSABLE;
	    }
		StrCopy(CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].name, pIdleState->Name, CPUIDLE_NAME_LEN);

		StrCopy(CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].desc, pIdleState->Desc, CPUIDLE_NAME_LEN);

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].exit_latency = pIdleState->Latency;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].target_residency = pIdleState->Residency;

	  switch (Idle_Route) {
	  case ROUTE_MWAIT:
	    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter = CoreFreqK_Alt_MWAIT_Handler;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter_s2idle = CoreFreqK_Alt_S2_MWAIT_Handler;
	    }
	    else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter = CoreFreqK_Alt_MWAIT_AMD_Handler;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter_s2idle = CoreFreqK_Alt_S2_MWAIT_AMD_Handler;
	    }
	    else {
		goto IDLE_DEFAULT;
	    }
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[0] = 'M';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[1] = 'W';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[2] = 'T';

		PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_MWAIT;
		break;

	  case ROUTE_HALT:
	    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter = CoreFreqK_Alt_HALT_Handler;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter_s2idle = CoreFreqK_Alt_S2_HALT_Handler;
	    }
	    else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter = CoreFreqK_Alt_HALT_AMD_Handler;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter_s2idle = CoreFreqK_Alt_S2_HALT_AMD_Handler;
	    }
	    else {
		goto IDLE_DEFAULT;
	    }
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[0] = 'H';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[1] = 'L';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[2] = 'T';

		PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_HALT;
		break;

	  case ROUTE_IO:
	  {
	    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
	    {
		CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
		RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
		if (CState_IO_MWAIT.LVL2_BaseAddr != 0x0)
		{
			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter = CoreFreqK_Alt_IO_Handler;

			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter_s2idle = CoreFreqK_Alt_S2_IO_Handler;
		} else {
			goto IDLE_DEFAULT;
		}
	    }
	    else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
	    {
		CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
		RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
		if (CStateBaseAddr.IOaddr != 0x0)
		{
			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter = CoreFreqK_Alt_IO_AMD_Handler;

			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter_s2idle=CoreFreqK_Alt_S2_IO_AMD_Handler;
		} else {
			goto IDLE_DEFAULT;
		}
	    }
	    else {
		goto IDLE_DEFAULT;
	    }
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[0] = 'I';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[1] = '/';

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].desc[2] = 'O';
	  }

		PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_IO;
		break;

	  case ROUTE_DEFAULT:
	  IDLE_DEFAULT:
	  default:
		PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_DEFAULT;

	    if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
	    {
		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter = CoreFreqK_MWAIT_Handler;

		CoreFreqK.IdleDriver.states[
			CoreFreqK.IdleDriver.state_count
		].enter_s2idle = CoreFreqK_S2_MWAIT_Handler;

		PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_MWAIT;
	    }
	    else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
		|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
	    {
		CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
		RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
		if (CStateBaseAddr.IOaddr != 0x0)
		{
			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter = CoreFreqK_IO_Handler;

			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter_s2idle = CoreFreqK_S2_IO_Handler;

			PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_IO;
		}
		else
		{
			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter = CoreFreqK_HALT_AMD_Handler;

			CoreFreqK.IdleDriver.states[
				CoreFreqK.IdleDriver.state_count
			].enter_s2idle = CoreFreqK_S2_HALT_AMD_Handler;

			PUBLIC(RO(Proc))->Registration.Driver.Route=ROUTE_HALT;
		}
	    }
	    else {
		pr_warn("CoreFreq: "					\
			"No Idle implementation for Vendor CRC 0x%x\n",
			PUBLIC(RO(Proc))->Features.Info.Vendor.CRC);
	    }
		break;
	  }
		CoreFreqK.IdleDriver.state_count++;
	}
	pIdleState++;
     }
     if ((rc = cpuidle_register_driver(&CoreFreqK.IdleDriver)) == 0)
     {
	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
	{
	    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW))
	    {
		device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
	      if (device != NULL)
	      {
		device->cpu = cpu;
		if ((rc = cpuidle_register_device(device)) == 0) {
			continue;
		}
	      }
		break;
	    }
	}
	enroll = cpu;
     }
     if (rc != 0)
     { /* Cancel the registration if the driver and/or a device failed */
	for (cpu = 0; cpu < enroll; cpu++)
	{
		device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
	    if (device != NULL)
	    {
		cpuidle_unregister_device(device);
	    }
	}
	cpuidle_unregister_driver(&CoreFreqK.IdleDriver);
	free_percpu(CoreFreqK.IdleDevice);
     }
    }
   }
  }
#endif /* CONFIG_CPU_IDLE and 4.14.0 */
	return rc;
}

#ifdef CONFIG_CPU_IDLE
static void CoreFreqK_Idle_State_Withdraw(int idx, bool disable)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) || (RHEL_MAJOR == 8)
	struct cpuidle_device *device;
	unsigned int cpu;
  for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
  {
    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW)
    && ((device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu)) != NULL))
    {
      if (disable) {
	device->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;
      } else {
	device->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER;
      }
    }
  }
#else
	CoreFreqK.IdleDriver.states[idx].disabled = disable;
#endif /* 5.5.0 */
}
#endif /* CONFIG_CPU_IDLE */

static long CoreFreqK_Limit_Idle(int target)
{
	long rc = -EINVAL;
#ifdef CONFIG_CPU_IDLE
	int idx, floor = -1;

    if ((target > 0) && (target <= CoreFreqK.IdleDriver.state_count))
    {
	for (idx = 0; idx < CoreFreqK.IdleDriver.state_count; idx++)
	{
	    if (idx < target)
	    {
		CoreFreqK_Idle_State_Withdraw(idx, false);

		floor = idx;
	    } else {
		CoreFreqK_Idle_State_Withdraw(idx, true);
	    }
	}
	rc = RC_SUCCESS;
    }
    else if (target == 0)
    {
	for (idx = 0; idx < CoreFreqK.IdleDriver.state_count; idx++)
	{
		CoreFreqK_Idle_State_Withdraw(idx, false);

		floor = idx;
	}
	rc = RC_SUCCESS;
    }
    if (floor != -1) {
	PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = 1 + floor;
    }
#endif /* CONFIG_CPU_IDLE */
	return rc;
}

#ifdef CONFIG_CPU_FREQ
static int CoreFreqK_Policy_Exit(struct cpufreq_policy *policy)
{
	UNUSED(policy);
	return 0;
}

static int CoreFreqK_Policy_Init(struct cpufreq_policy *policy)
{
    if (policy != NULL) {
	if (policy->cpu < PUBLIC(RO(Proc))->CPU.Count)
	{
		CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));

		policy->cpuinfo.min_freq =(Core->Boost[BOOST(MIN)]
					 * Core->Clock.Hz) / 1000LLU;

		policy->cpuinfo.max_freq =(Core->Boost[BOOST(MAX)]
					 * Core->Clock.Hz) / 1000LLU;

		/*		MANDATORY Per-CPU Initialization	*/
		policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
		policy->cur = policy->cpuinfo.max_freq;
		policy->min = policy->cpuinfo.min_freq;
		policy->max = policy->cpuinfo.max_freq;
		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
	    if (Register_Governor == 1) {
		policy->governor = &CoreFreqK.FreqGovernor;
	    } else {
		policy->governor = NULL;
	    }
	}
    }
	return 0;
}

#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 19))	\
  && (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 5, 0)))	\
  || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 3))	\
  || (RHEL_MAJOR == 8)
static int CoreFreqK_Policy_Verify(struct cpufreq_policy_data *policy)
#else
static int CoreFreqK_Policy_Verify(struct cpufreq_policy *policy)
#endif
{
	if (policy != NULL) {
		cpufreq_verify_within_cpu_limits(policy);
	}
	return 0;
}

static int CoreFreqK_SetPolicy(struct cpufreq_policy *policy)
{
	UNUSED(policy);
	return 0;
}

static int CoreFreqK_Bios_Limit(int cpu, unsigned int *limit)
{
    if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count) && (limit != NULL))
    {
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	(*limit) = (Core->Boost[BOOST(MAX)] * Core->Clock.Hz) / 1000LLU;
    }
	return 0;
}

void Policy_Aggregate_Turbo(void)
{
    if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq & REGISTRATION_ENABLE) {
	CoreFreqK.FreqDriver.boost_enabled = (
			BITWISEAND_CC(	LOCKLESS,
					PUBLIC(RW(Proc))->TurboBoost,
					PUBLIC(RO(Proc))->TurboBoost_Mask ) != 0
	);
    }
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)	\
  || ((RHEL_MAJOR == 8) && (RHEL_MINOR > 3))
static int CoreFreqK_SetBoost(struct cpufreq_policy *policy, int state)
{
	UNUSED(policy);
#else
static int CoreFreqK_SetBoost(int state)
{
#endif /* 5.8.0 */
	Controller_Stop(1);
	TurboBoost_Enable = (state != 0);
	Controller_Start(1);
	TurboBoost_Enable = -1;
	Policy_Aggregate_Turbo();
	BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY); /* Notify Daemon*/
	return 0;
}

static ssize_t CoreFreqK_Show_SetSpeed(struct cpufreq_policy *policy,char *buf)
{
  if (policy != NULL) {
	CORE_RO *Core;
    if (policy->cpu < PUBLIC(RO(Proc))->CPU.Count)
    {
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));
    } else {
	Core = (CORE_RO *) PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)));
    }
	return sprintf(buf, "%7llu\n",
			(Core->Boost[BOOST(TGT)] * Core->Clock.Hz) / 1000LLU);
  }
	return 0;
}

static int CoreFreqK_Store_SetSpeed(struct cpufreq_policy *policy,
					unsigned int freq)
{
    if (policy != NULL) {
	if ((policy->cpu < PUBLIC(RO(Proc))->CPU.Count)
	 && (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget != NULL))
	{
		CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));
		unsigned int ratio = (freq * 1000LLU) / Core->Clock.Hz;

	    if (ratio > 0) {
		if (smp_call_function_single(policy->cpu,
			Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget,
						&ratio, 1) == 0)
		{
			BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY);
		}
	    }
		return 0;
	}
    }
	return -EINVAL;
}
#endif /* CONFIG_CPU_FREQ */

static unsigned int Policy_GetFreq(unsigned int cpu)
{
	unsigned int CPU_Freq = PUBLIC(RO(Proc))->Features.Factory.Freq * 1000U;

    if (cpu < PUBLIC(RO(Proc))->CPU.Count)
    {
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	unsigned int Freq_MHz	= Core->Delta.C0.UCC
				/ PUBLIC(RO(Proc))->SleepInterval;

	if (Freq_MHz > 0) {	/* at least 1 interval must have been elapsed */
		CPU_Freq = Freq_MHz;
	}
    }
	return CPU_Freq;
}

static void Policy_Core2_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
	unsigned int *ratio = (unsigned int*) arg;
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

    if ((*ratio) <= Core->Boost[BOOST(1C)])
    {
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Set_Core2_Target(Core, (*ratio));
	WritePerformanceControl(&Core->PowerThermal.PerfControl);
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
	    if (Cmp_Core2_Target(Core, 1 + Core->Boost[BOOST(MAX)]))
	    {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    } else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    }
	}
	Core->Boost[BOOST(TGT)] = Get_Core2_Target(Core);
    }
#endif /* CONFIG_CPU_FREQ */
}

static void Policy_Nehalem_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
	unsigned int *ratio = (unsigned int*) arg;
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

    if ((*ratio) <= Core->Boost[BOOST(1C)])
    {
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Set_Nehalem_Target(Core, (*ratio));
	WritePerformanceControl(&Core->PowerThermal.PerfControl);
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
	    if (Cmp_Nehalem_Target(Core, 1 + Core->Boost[BOOST(MAX)]))
	    {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    } else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    }
	}
	Core->Boost[BOOST(TGT)] = Get_Nehalem_Target(Core);
    }
#endif /* CONFIG_CPU_FREQ */
}

static void Policy_SandyBridge_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
	unsigned int *ratio = (unsigned int*) arg;
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

    if ((*ratio) <= Core->Boost[BOOST(1C)])
    {
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
	Set_SandyBridge_Target(Core, (*ratio));
	WritePerformanceControl(&Core->PowerThermal.PerfControl);
	RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);

	if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
	    if (Cmp_SandyBridge_Target(Core, Core->Boost[BOOST(MAX)]))
	    {
		BITSET_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    } else {
		BITCLR_CC(LOCKLESS, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
	    }
	}
	Core->Boost[BOOST(TGT)] = Get_SandyBridge_Target(Core);
    }
#endif /* CONFIG_CPU_FREQ */
}

static void Policy_HWP_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
  if (PUBLIC(RO(Proc))->Features.HWP_Enable)
  {
	unsigned int *ratio = (unsigned int*) arg;
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
    if (((*ratio) >= Core->PowerThermal.HWP_Capabilities.Lowest)
     && ((*ratio) <= Core->PowerThermal.HWP_Capabilities.Highest))
    {
	Core->PowerThermal.HWP_Request.Maximum_Perf =	\
	Core->PowerThermal.HWP_Request.Desired_Perf = (*ratio);
	WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
	RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);

	Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
	Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
    }
  } else {
	Policy_SandyBridge_SetTarget(arg);
  }
#endif /* CONFIG_CPU_FREQ */
}

static void Policy_Zen_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
	unsigned int *ratio = (unsigned int*) arg;
	unsigned int cpu = smp_processor_id();
	CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));

	PSTATEDEF PstateDef;
	COF_ST COF = {.Q = 0, .R = 0};
	unsigned int pstate;
	unsigned short WrModRd = 0;
	/* Look-up for the first enabled P-State with the same target ratio */
	for (pstate = 0; pstate <= 7; pstate++)
	{
		PstateDef.value = 0;
		RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + pstate);

	    if (PstateDef.Family_17h.PstateEn)
	    {
		COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId );
		if (COF.Q == (*ratio)) {
			WrModRd = 1;
			break;
		}
	    }
	}
	if (WrModRd == 1)
	{
		PSTATECTRL PstateCtrl;
		/*	Write-Modify-Read the new target P-state	*/
		PstateCtrl.value = 0;
		RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
		PstateCtrl.PstateCmd = pstate;
		WRMSR(PstateCtrl, MSR_AMD_PERF_CTL);

		PstateCtrl.value = 0;
		RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
		PstateDef.value = 0;
		RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + pstate);

		COF = AMD_Zen_CoreCOF(	PstateDef.Family_17h.CpuFid,
					PstateDef.Family_17h.CpuDfsId );

		Core->Boost[BOOST(TGT)] = COF.Q;
	}
#endif /* CONFIG_CPU_FREQ */
}

static int CoreFreqK_FreqDriver_UnInit(void)
{
	int rc = -EINVAL;
#ifdef CONFIG_CPU_FREQ
	rc = cpufreq_unregister_driver(&CoreFreqK.FreqDriver);
#endif /* CONFIG_CPU_FREQ */
	return rc;
}

static int CoreFreqK_FreqDriver_Init(void)
{
	int rc = -RC_UNIMPLEMENTED;
#ifdef CONFIG_CPU_FREQ
 if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
 {
  if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.GetFreq != NULL)
  {
  CoreFreqK.FreqDriver.get=Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.GetFreq;

	rc = cpufreq_register_driver(&CoreFreqK.FreqDriver);
  }
 }
#endif /* CONFIG_CPU_FREQ */
	return rc;
}

static void CoreFreqK_Governor_UnInit(void)
{
#ifdef CONFIG_CPU_FREQ
	cpufreq_unregister_governor(&CoreFreqK.FreqGovernor);
#endif /* CONFIG_CPU_FREQ */
}

static int CoreFreqK_Governor_Init(void)
{
	int rc = -RC_UNIMPLEMENTED;
#ifdef CONFIG_CPU_FREQ
    if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
    {
	if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget != NULL)
	{
		rc = cpufreq_register_governor(&CoreFreqK.FreqGovernor);
	}
    }
#endif /* CONFIG_CPU_FREQ */
	return rc;
}

signed int Seek_Topology_Core_Peer(unsigned int cpu, signed int exclude)
{
	unsigned int seek;

    for (seek = 0; seek < PUBLIC(RO(Proc))->CPU.Count; seek++) {
	if ( ((exclude ^ cpu) > 0)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ApicID \
		!= PUBLIC(RO(Core, AT(cpu)))->T.ApicID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.CoreID \
		== PUBLIC(RO(Core, AT(cpu)))->T.CoreID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ThreadID \
		!= PUBLIC(RO(Core, AT(cpu)))->T.ThreadID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.PackageID \
		== PUBLIC(RO(Core, AT(cpu)))->T.PackageID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ThreadID == 0)
	  && !BITVAL(PUBLIC(RO(Core, AT(seek)))->OffLine, OS) )
	{
		return (signed int) seek;
	}
    }
	return -1;
}

signed int Seek_Topology_Thread_Peer(unsigned int cpu, signed int exclude)
{
	unsigned int seek;

    for (seek = 0; seek < PUBLIC(RO(Proc))->CPU.Count; seek++) {
	if ( ((exclude ^ cpu) > 0)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ApicID \
		!= PUBLIC(RO(Core, AT(cpu)))->T.ApicID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.CoreID \
		== PUBLIC(RO(Core, AT(cpu)))->T.CoreID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ThreadID \
		!= PUBLIC(RO(Core, AT(cpu)))->T.ThreadID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.PackageID \
		== PUBLIC(RO(Core, AT(cpu)))->T.PackageID)
	  && (PUBLIC(RO(Core, AT(seek)))->T.ThreadID > 0)
	  && !BITVAL(PUBLIC(RO(Core, AT(seek)))->OffLine, OS) )
	{
		return (signed int) seek;
	}
    }
	return -1;
}

void MatchCoreForService(SERVICE_PROC *pService,unsigned int cpi,signed int cpx)
{
	unsigned int cpu;

    for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
	if ( ((cpx ^ cpu) > 0)
	  && (PUBLIC(RO(Core, AT(cpu)))->T.PackageID \
		== PUBLIC(RO(Core, AT(cpi)))->T.PackageID)
	  && !BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS) )
	{
		pService->Core = cpu;
		pService->Thread = -1;
		break;
	}
    }
}

int MatchPeerForService(SERVICE_PROC *pService, unsigned int cpi,signed int cpx)
{
	unsigned int cpu = cpi, cpn = 0;
	signed int seek;
MATCH:
	if (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID == 0)
	{
		if ((seek = Seek_Topology_Thread_Peer(cpu, cpx)) != -1) {
			pService->Core = cpu;
			pService->Thread = seek;
			return 0;
		}
	}
	else if (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0)
	{
		if ((seek = Seek_Topology_Core_Peer(cpu, cpx)) != -1) {
			pService->Core = seek;
			pService->Thread = cpu;
			return 0;
		}
	}
	while (cpn < PUBLIC(RO(Proc))->CPU.Count) {
		cpu = cpn++;
		if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
			goto MATCH;
		}
	}
	return -1;
}

void MatchPeerForDefaultService(SERVICE_PROC *pService, unsigned int cpu)
{
	if (PUBLIC(RO(Proc))->Features.HTT_Enable) {
		if (MatchPeerForService(pService, cpu, -1) == -1)
		{
			MatchCoreForService(pService, cpu, -1);
		}
	} else {
		pService->Core = cpu;
		pService->Thread = -1;
	}
	if (ServiceProcessor != -1) {
		DefaultSMT.Proc = pService->Proc;
	}
}

void MatchPeerForUpService(SERVICE_PROC *pService, unsigned int cpu)
{	/* Try to restore the initial Service affinity or move to SMT peer. */
	SERVICE_PROC hService = {
		.Core = cpu,
		.Thread = -1,
	};
	if (PUBLIC(RO(Proc))->Features.HTT_Enable)
	{
		signed int seek;

		if ((PUBLIC(RO(Core, AT(cpu)))->T.ThreadID == 0)
		&& ((seek = Seek_Topology_Thread_Peer(cpu, -1)) != -1))
		{
			hService.Core = cpu;
			hService.Thread = seek;
		} else {
			if ((PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0)
			&& ((seek = Seek_Topology_Core_Peer(cpu, -1)) != -1))
			{
				hService.Core = seek;
				hService.Thread = cpu;
			}
		}
	}
	if (pService->Proc != DefaultSMT.Proc)
	{
		if (hService.Proc == DefaultSMT.Proc) {
			pService->Proc = hService.Proc;
		} else {
			if ((pService->Thread == -1) && (hService.Thread > 0))
			{
				pService->Proc = hService.Proc;
			}
		}
	}
}

void MatchPeerForDownService(SERVICE_PROC *pService, unsigned int cpu)
{
	int rc = -1;

	if (PUBLIC(RO(Proc))->Features.HTT_Enable) {
		rc = MatchPeerForService(pService, cpu, cpu);
	}
	if (rc == -1) {
		MatchCoreForService(pService, cpu, cpu);
	}
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
static int CoreFreqK_NMI_Handler(unsigned int type, struct pt_regs *pRegs)
{
	unsigned int cpu = smp_processor_id();
	UNUSED(pRegs);

	switch (type) {
	case NMI_LOCAL:
		PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.LOCAL++;
		break;
	case NMI_UNKNOWN:
		PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.UNKNOWN++;
		break;
	case NMI_SERR:
		PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.PCISERR++;
		break;
	case NMI_IO_CHECK:
		PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.IOCHECK++;
		break;
	}
	return NMI_DONE;
}

static long CoreFreqK_UnRegister_CPU_Idle(void)
{
	long rc = -EINVAL;
    if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
    {
	CoreFreqK_IdleDriver_UnInit();
	PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
	rc = RC_SUCCESS;
    }
	return rc;
}

static long CoreFreqK_Register_CPU_Idle(void)
{
	long rc = -EINVAL;
  if (Register_CPU_Idle == 1)
  {
	int rx = CoreFreqK_IdleDriver_Init();
    switch ( rx ) {
    default:
	fallthrough;
    case -ENODEV:
    case -ENOMEM:
	PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
	rc = (long) rx;
	break;
    case 0:	/*	Registration succeeded.				*/
	PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_ENABLE;
	rc = RC_SUCCESS;
	break;
    }
  } else {	/*	Nothing requested by User.			*/
	PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
  }
	return rc;
}

static long CoreFreqK_UnRegister_CPU_Freq(void)
{
	long rc = -EINVAL;
    if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq & REGISTRATION_ENABLE)
    {
	int rx = CoreFreqK_FreqDriver_UnInit();
	switch ( rx ) {
	case 0:
	    PUBLIC(RO(Proc))->Registration.Driver.CPUfreq=REGISTRATION_DISABLE;
		rc = RC_SUCCESS;
		break;
	default:
		rc = (long) rx;
		break;
	}
    }
	return rc;
}

static long CoreFreqK_Register_CPU_Freq(void)
{ /* Source: cpufreq_register_driver @ /drivers/cpufreq/cpufreq.c	*/
	long rc = -EINVAL;
  if (Register_CPU_Freq == 1)
  {
	int rx = CoreFreqK_FreqDriver_Init();
    switch ( rx ) {
    default:
	fallthrough;
    case -EEXIST:		/*	Another driver is in control.	*/
	PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_DISABLE;
	rc = (long) rx;
	break;
    case -ENODEV:		/*	Missing CPU-Freq or Interfaces.	*/
    case -EPROBE_DEFER:	/*	CPU probing failed			*/
    case -EINVAL:		/*	Missing CPU-Freq prerequisites. */
	PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_FULLCTRL;
	rc = (long) rx;
	break;
    case 0:		/*	Registration succeeded .		*/
	PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_ENABLE;
	rc = RC_SUCCESS;
	break;
    }
  } else {		/*	Invalid or no User request.		*/
#ifdef CONFIG_CPU_FREQ
	PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_DISABLE;
#else	/* No CPU-FREQ built in Kernel, presume we have the full control. */
	PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_FULLCTRL;
#endif /* CONFIG_CPU_FREQ */
  }
	return rc;
}

static long CoreFreqK_UnRegister_Governor(void)
{
	long rc = EINVAL;
    if (PUBLIC(RO(Proc))->Registration.Driver.Governor & REGISTRATION_ENABLE)
    {
	CoreFreqK_Governor_UnInit();
	PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
	rc = RC_SUCCESS;
    }
	return rc;
}

static long CoreFreqK_Register_Governor(void)
{
	long rc = -EINVAL;
  if (Register_Governor == 1)
  {
	int rx = CoreFreqK_Governor_Init();
    switch ( rx ) {
    default:
    case -ENODEV:
	PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
	rc = (long) rx;
	break;
    case 0:		/*	Registration succeeded .		*/
	PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_ENABLE;
	rc = RC_SUCCESS;
	break;
    }
  } else {	/* Nothing requested by User.				*/
	PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
  }
	return rc;
}

static void CoreFreqK_Register_NMI(void)
{
  if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL) == 0)
  {
    if(register_nmi_handler(NMI_LOCAL,
			CoreFreqK_NMI_Handler,
			0,
			"corefreqk") == 0)
    {
	BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
    } else {
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
    }
  }
  if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN) == 0)
  {
    if(register_nmi_handler(NMI_UNKNOWN,
			CoreFreqK_NMI_Handler,
			0,
			"corefreqk") == 0)
    {
	BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
    } else {
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
    }
  }
  if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR) == 0)
  {
    if(register_nmi_handler(NMI_SERR,
			CoreFreqK_NMI_Handler,
			0,
			"corefreqk") == 0)
    {
	BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
    } else {
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
    }
  }
  if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK) == 0)
  {
    if(register_nmi_handler(NMI_IO_CHECK,
			CoreFreqK_NMI_Handler,
			0,
			"corefreqk") == 0)
    {
	BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
    } else {
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
    }
  }
}

static void CoreFreqK_UnRegister_NMI(void)
{
    if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL) == 1)
    {
	unregister_nmi_handler(NMI_LOCAL, "corefreqk");
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
    }
    if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN) == 1)
    {
	unregister_nmi_handler(NMI_UNKNOWN, "corefreqk");
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
    }
    if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR) == 1)
    {
	unregister_nmi_handler(NMI_SERR, "corefreqk");
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
    }
    if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK) == 1)
    {
	unregister_nmi_handler(NMI_IO_CHECK, "corefreqk");
	BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
    }
}
#else
static void CoreFreqK_Register_NMI(void) {}
static void CoreFreqK_UnRegister_NMI(void) {}
#endif


static void For_All_CPU_Compute_Clock(void)
{
	unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
  do {
	CLOCK Clock = {
		.Q  = PUBLIC(RO(Proc))->Features.Factory.Clock.Q,
		.R  = PUBLIC(RO(Proc))->Features.Factory.Clock.R,
		.Hz = PUBLIC(RO(Proc))->Features.Factory.Clock.Hz
	};
	/* from last AP to BSP */
	cpu--;

    if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS))
    {
	COMPUTE_ARG Compute = {
		.TSC = {NULL, NULL},
		.Clock = {
			.Q = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)],
			.R = 0, .Hz = 0
		}
	};
      if ((Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
      {
	if ((Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
	{
		Clock = Compute_Clock(cpu, &Compute);

		kfree(Compute.TSC[1]);
	}
		kfree(Compute.TSC[0]);
      }
    }
	PUBLIC(RO(Core, AT(cpu)))->Clock.Q  = Clock.Q;
	PUBLIC(RO(Core, AT(cpu)))->Clock.R  = Clock.R;
	PUBLIC(RO(Core, AT(cpu)))->Clock.Hz = Clock.Hz;
  } while (cpu != 0) ;
}

#define SYSGATE_UPDATE(_rc)						\
({									\
	_rc = Sys_OS_Driver_Query();					\
	_rc = (_rc != -ENXIO) ? RC_OK_SYSGATE : _rc;			\
})

static long CoreFreqK_ioctl(	struct file *filp,
				unsigned int cmd,
				unsigned long arg )
{
	long rc = -EPERM;
	UNUSED(filp);

    switch ((enum COREFREQ_MAGIC_COMMAND) cmd)
    {
    case COREFREQ_IOCTL_SYSUPDT:
	Controller_Stop(1);
	SYSGATE_UPDATE(rc);
	Controller_Start(1);
	BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY);
    break;

    case COREFREQ_IOCTL_SYSONCE:
	rc = Sys_OS_Driver_Query();
	rc = (rc != -ENXIO) ? Sys_Kernel(PUBLIC(OF(Gate))) : rc;
    break;

    case COREFREQ_IOCTL_MACHINE:
    {
	RING_ARG_QWORD prm = {.arg = arg};

      switch (prm.dl.hi)
      {
      case MACHINE_CONTROLLER:
	switch (prm.dl.lo)
	{
	case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		rc = RC_SUCCESS;
		break;
	case COREFREQ_TOGGLE_ON:
		Controller_Start(1);
	#ifdef CONFIG_CPU_FREQ
		Policy_Aggregate_Turbo();
	#endif /* CONFIG_CPU_FREQ */
		rc = RC_OK_COMPUTE;
		break;
	}
	break;

      case MACHINE_INTERVAL:
	Controller_Stop(1);
	SleepInterval = prm.dl.lo;
	Compute_Interval();
	Controller_Start(1);
	rc = RC_SUCCESS;
	break;

      case MACHINE_AUTOCLOCK:
	switch (prm.dl.lo)
	{
	case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		For_All_CPU_Compute_Clock();
		BITCLR(LOCKLESS, AutoClock, 1);
		PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
		Controller_Start(1);
		rc = RC_SUCCESS;
		break;
	case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		BITSET(LOCKLESS, AutoClock, 1);
		PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
		Controller_Start(1);
		rc = RC_SUCCESS;
		break;
	}
	break;

      case MACHINE_EXPERIMENTAL:
	switch (prm.dl.lo) {
	    case COREFREQ_TOGGLE_OFF:
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		PUBLIC(RO(Proc))->Registration.Experimental = prm.dl.lo;
		Controller_Start(1);
	    if (PUBLIC(RO(Proc))->Registration.Experimental)
	    {
	      if ( !PUBLIC(RO(Proc))->Registration.PCI ) {
		PUBLIC(RO(Proc))->Registration.PCI = \
			CoreFreqK_ProbePCI(
				Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
				CoreFreqK_ResetChip, CoreFreqK_AppendChip
			) == 0;
		rc = RC_OK_COMPUTE;
	     } else {
		rc = RC_SUCCESS;
	     }
	    } else {
		rc = RC_SUCCESS;
	    }
		break;
	}
	break;

      case MACHINE_INTERRUPTS:
	switch (prm.dl.lo)
	{
	    case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		CoreFreqK_UnRegister_NMI();
		Controller_Start(1);
		rc = RC_SUCCESS;
		break;
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		CoreFreqK_Register_NMI();
		Controller_Start(1);
		rc = RC_SUCCESS;
		break;
	}
	break;

      case MACHINE_LIMIT_IDLE:
	if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
	{
		rc = CoreFreqK_Limit_Idle(prm.dl.lo);
	}
	break;

      case MACHINE_CPU_IDLE:
	switch (prm.dl.lo)
	{
	    case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		rc = CoreFreqK_UnRegister_CPU_Idle();
		Register_CPU_Idle = -1;
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		Register_CPU_Idle = 1;
		rc = CoreFreqK_Register_CPU_Idle();
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	}
	break;

      case MACHINE_IDLE_ROUTE:
	if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
	{
		Controller_Stop(1);
		rc = CoreFreqK_UnRegister_CPU_Idle();
		Register_CPU_Idle = -1;
		if (rc == RC_SUCCESS)
		{
			Register_CPU_Idle = 1;
			Idle_Route = prm.dl.lo;
			rc = CoreFreqK_Register_CPU_Idle();
			if (rc == RC_SUCCESS) {
				SYSGATE_UPDATE(rc);
			}
		}
		Controller_Start(1);
	}
	break;

      case MACHINE_CPU_FREQ:
	switch (prm.dl.lo)
	{
	    case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		rc = CoreFreqK_UnRegister_CPU_Freq();
		Register_CPU_Freq = -1;
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		Register_CPU_Freq = 1;
		rc = CoreFreqK_Register_CPU_Freq();
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	}
	break;

      case MACHINE_GOVERNOR:
	switch (prm.dl.lo)
	{
	    case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		rc = CoreFreqK_UnRegister_Governor();
		Register_Governor = -1;
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		Register_Governor = 1;
		rc = CoreFreqK_Register_Governor();
		if (rc == RC_SUCCESS) {
			SYSGATE_UPDATE(rc);
		}
		Controller_Start(1);
		break;
	}
	break;

      case MACHINE_CLOCK_SOURCE:
	switch (prm.dl.lo)
	{
	    case COREFREQ_TOGGLE_OFF:
		Controller_Stop(1);
		rc = CoreFreqK_UnRegister_ClockSource();
		Register_ClockSource = -1;
		Controller_Start(1);
		break;
	    case COREFREQ_TOGGLE_ON:
		Controller_Stop(1);
		Register_ClockSource = 1;
	      rc=CoreFreqK_Register_ClockSource(PUBLIC(RO(Proc))->Service.Core);
		Controller_Start(1);
		break;
	}
	break;

      case MACHINE_FORMULA_SCOPE:
	switch (prm.dl.lo)
	{
	    case 0:
		rc = CoreFreqK_Thermal_Scope(prm.dh.lo);
		break;
	    case 1:
		rc = CoreFreqK_Voltage_Scope(prm.dh.lo);
		break;
	    case 2:
		rc = CoreFreqK_Power_Scope(prm.dh.lo);
		break;
	}
	break;
      }
    }
    break;

    case COREFREQ_IOCTL_TECHNOLOGY:
    {
	RING_ARG_QWORD prm = {.arg = arg};

	switch (prm.dl.hi)
	{
	case TECHNOLOGY_L1_HW_PREFETCH:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			L1_HW_PREFETCH_Disable = !prm.dl.lo;
			Controller_Start(1);
			L1_HW_PREFETCH_Disable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_L1_HW_IP_PREFETCH:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			L1_HW_IP_PREFETCH_Disable = !prm.dl.lo;
			Controller_Start(1);
			L1_HW_IP_PREFETCH_Disable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_L2_HW_PREFETCH:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			L2_HW_PREFETCH_Disable = !prm.dl.lo;
			Controller_Start(1);
			L2_HW_PREFETCH_Disable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_L2_HW_CL_PREFETCH:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			L2_HW_CL_PREFETCH_Disable = !prm.dl.lo;
			Controller_Start(1);
			L2_HW_CL_PREFETCH_Disable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_EIST:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			SpeedStep_Enable = prm.dl.lo;
			Controller_Start(1);
			SpeedStep_Enable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_C1E:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			C1E_Enable = prm.dl.lo;
			Controller_Start(1);
			C1E_Enable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_TURBO:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				TurboBoost_Enable = prm.dl.lo;
				Controller_Start(1);
				TurboBoost_Enable = -1;
			#ifdef CONFIG_CPU_FREQ
				Policy_Aggregate_Turbo();
			#endif /* CONFIG_CPU_FREQ */
				rc = RC_OK_COMPUTE;
				break;
		}
		break;

	case TECHNOLOGY_C1A:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				C1A_Enable = prm.dl.lo;
				Controller_Start(1);
				C1A_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_C3A:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				C3A_Enable = prm.dl.lo;
				Controller_Start(1);
				C3A_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_C1U:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				C1U_Enable = prm.dl.lo;
				Controller_Start(1);
				C1U_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_C3U:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				C3U_Enable = prm.dl.lo;
				Controller_Start(1);
				C3U_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_CC6:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				CC6_Enable = prm.dl.lo;
				Controller_Start(1);
				CC6_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_PC6:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				PC6_Enable = prm.dl.lo;
				Controller_Start(1);
				PC6_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_PKG_CSTATE:
		Controller_Stop(1);
		PkgCStateLimit = prm.dl.lo;
		Controller_Start(1);
		PkgCStateLimit = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_IO_MWAIT:
		switch (prm.dl.lo) {
			case COREFREQ_TOGGLE_OFF:
			case COREFREQ_TOGGLE_ON:
				Controller_Stop(1);
				IOMWAIT_Enable = prm.dl.lo;
				Controller_Start(1);
				IOMWAIT_Enable = -1;
				rc = RC_SUCCESS;
				break;
		}
		break;

	case TECHNOLOGY_IO_MWAIT_REDIR:
		Controller_Stop(1);
		CStateIORedir = prm.dl.lo;
		Controller_Start(1);
		CStateIORedir = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_ODCM:
		Controller_Stop(1);
		ODCM_Enable = prm.dl.lo;
		Controller_Start(1);
		ODCM_Enable = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_ODCM_DUTYCYCLE:
		Controller_Stop(1);
		ODCM_DutyCycle = prm.dl.lo;
		Controller_Start(1);
		ODCM_DutyCycle = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_POWER_POLICY:
		Controller_Stop(1);
		PowerPolicy = prm.dl.lo;
		Controller_Start(1);
		PowerPolicy = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_HWP:
		Controller_Stop(1);
		HWP_Enable = prm.dl.lo;
		Intel_Hardware_Performance();
		Controller_Start(1);
		HWP_Enable = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_HWP_EPP:
		Controller_Stop(1);
		HWP_EPP = prm.dl.lo;
		Controller_Start(1);
		HWP_EPP = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_HDC:
		Controller_Stop(1);
		HDC_Enable = prm.dl.lo;
		Intel_Hardware_Performance();
		Controller_Start(1);
		HDC_Enable = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_EEO:
	    if (PUBLIC(RO(Proc))->Features.EEO_Capable)
	    {
		EEO_Disable = prm.dl.lo;
		Skylake_PowerControl();
		EEO_Disable = -1;
		rc = RC_SUCCESS;
	    } else {
		rc = -ENXIO;
	    }
		break;

	case TECHNOLOGY_R2H:
	    if (PUBLIC(RO(Proc))->Features.R2H_Capable)
	    {
		R2H_Disable = prm.dl.lo;
		Skylake_PowerControl();
		R2H_Disable = -1;
		rc = RC_SUCCESS;
	    } else {
		rc = -ENXIO;
	    }
		break;

	case TECHNOLOGY_CFG_TDP_LVL:
		Controller_Stop(1);
		Config_TDP_Level = prm.dl.lo;
		Controller_Start(1);
		Config_TDP_Level = -1;
		rc = RC_SUCCESS;
		break;

	case TECHNOLOGY_TDP_LIMITING:
	    {
		const enum PWR_DOMAIN	pw = prm.dh.lo;
		const enum PWR_LIMIT	pl = prm.dh.hi;

		const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
	      if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
	      {
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			TDP_Limiting_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
			RESET_ARRAY(Activate_TDP_Limit, TDP_Limiting_Count, -1);
			Activate_TDP_Limit[idx] = prm.dl.lo;

			Controller_Start(1);
			RESET_ARRAY(Activate_TDP_Limit, TDP_Limiting_Count, -1);
			TDP_Limiting_Count = 0;
			rc = RC_SUCCESS;
			break;
		}
	      }
	    }
		break;

	case TECHNOLOGY_TDP_CLAMPING:
	    {
		const enum PWR_DOMAIN	pw = prm.dh.lo;
		const enum PWR_LIMIT	pl = prm.dh.hi;

		const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
	      if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
	      {
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			TDP_Clamping_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
			RESET_ARRAY(Activate_TDP_Clamp, TDP_Clamping_Count, -1);
			Activate_TDP_Clamp[idx] = prm.dl.lo;

			Controller_Start(1);
			RESET_ARRAY(Activate_TDP_Clamp, TDP_Clamping_Count, -1);
			TDP_Clamping_Count = 0;
			rc = RC_SUCCESS;
			break;
		}
	      }
	    }
		break;

	case TECHNOLOGY_TDP_OFFSET:
	    {
		const enum PWR_DOMAIN	pw = prm.dh.lo;
		const enum PWR_LIMIT	pl = prm.dh.hi;
		/* Offset is capped within [ -127 , +128 ] watt */
		const signed short offset = (signed char) prm.dl.lo;

		const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
	      if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
	      {
		Controller_Stop(1);
		Custom_TDP_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
		RESET_ARRAY(Custom_TDP_Offset, Custom_TDP_Count, 0);
		Custom_TDP_Offset[idx] = offset;

		Controller_Start(1);
		RESET_ARRAY(Custom_TDP_Offset, Custom_TDP_Count, 0);
		Custom_TDP_Offset[idx] = 0;
		rc = RC_SUCCESS;
	      }
	    }
		break;

	case TECHNOLOGY_TDC_LIMITING:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			Activate_TDC_Limit = prm.dl.lo;
			Controller_Start(1);
			Activate_TDC_Limit = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;

	case TECHNOLOGY_TDC_OFFSET:
	    {
		const signed short offset = (signed char) prm.dl.lo;
		Controller_Stop(1);
		Custom_TDC_Offset = offset;
		Controller_Start(1);
		Custom_TDC_Offset = 0;
		rc = RC_SUCCESS;
	    }
		break;

	case TECHNOLOGY_WDT:
		switch (prm.dl.lo) {
		case COREFREQ_TOGGLE_OFF:
		case COREFREQ_TOGGLE_ON:
			Controller_Stop(1);
			WDT_Enable = prm.dl.lo;
			Controller_Start(1);
			WDT_Enable = -1;
			rc = RC_SUCCESS;
			break;
		}
		break;
	}
	break;
    }
    break;

    case COREFREQ_IOCTL_CPU_OFF:
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
	{
	unsigned int cpu = (unsigned int) arg;

	if (cpu < PUBLIC(RO(Proc))->CPU.Count) {
		if (!cpu_is_hotpluggable(cpu)) {
			rc = -EBUSY;
		} else {
	#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
			if ((rc = remove_cpu(cpu)) == 0) {
				rc = RC_OK_COMPUTE;
			}
	#else
			if ((rc = cpu_down(cpu)) == 0) {
				rc = RC_OK_COMPUTE;
			}
	#endif
		}
	    }
	}
    #else
	rc = -EINVAL;
    #endif
	break;

    case COREFREQ_IOCTL_CPU_ON:
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
	{
	unsigned int cpu = (unsigned int) arg;

	if (cpu < PUBLIC(RO(Proc))->CPU.Count) {
		if (!cpu_is_hotpluggable(cpu)) {
			rc = -EBUSY;
		} else {
	#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
			if ((rc = add_cpu(cpu)) == 0) {
				rc = RC_OK_COMPUTE;
			}
	#else
			if ((rc = cpu_up(cpu)) == 0) {
				rc = RC_OK_COMPUTE;
			}
	#endif
		}
	    }
	}
    #else
	rc = -EINVAL;
    #endif
	break;

    case COREFREQ_IOCTL_TURBO_CLOCK:
	if (Arch[PUBLIC(RO(Proc))->ArchID].TurboClock) {
		CLOCK_ARG clockMod = {.sllong = arg};
		Controller_Stop(1);
		rc = Arch[PUBLIC(RO(Proc))->ArchID].TurboClock(&clockMod);
		Controller_Start(1);
	} else {
		rc = -RC_UNIMPLEMENTED;
	}
	break;

    case COREFREQ_IOCTL_RATIO_CLOCK:
	if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
		CLOCK_ARG clockMod = {.sllong = arg};
		Controller_Stop(1);
		rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
		Controller_Start(1);
	#ifdef CONFIG_CPU_FREQ
		Policy_Aggregate_Turbo();
	#endif /* CONFIG_CPU_FREQ */
	} else {
		rc = -RC_UNIMPLEMENTED;
	}
	break;

    case COREFREQ_IOCTL_CONFIG_TDP: {
		CLOCK_ARG clockMod = {.sllong = arg};
		const short MaxRatio = MAXCLOCK_TO_RATIO(short, \
		PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core))->Clock.Hz));

		switch (clockMod.NC) {
		case CLOCK_MOD_ACT:
			if ((clockMod.Ratio >= 0)&&(clockMod.Ratio <= MaxRatio))
			{
				Controller_Stop(1);
				Turbo_Activation_Ratio = clockMod.Ratio;
				Controller_Start(1);
				Turbo_Activation_Ratio = -1;
				rc = RC_OK_COMPUTE;
			}
			break;
		}
	}
	break;

    case COREFREQ_IOCTL_UNCORE_CLOCK:
	if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.ClockMod) {
		CLOCK_ARG clockMod = {.sllong = arg};
		Controller_Stop(1);
		rc = Arch[PUBLIC(RO(Proc))->ArchID].Uncore.ClockMod(&clockMod);
		Controller_Start(1);
	} else {
		rc = -RC_UNIMPLEMENTED;
	}
	break;

    case COREFREQ_IOCTL_CLEAR_EVENTS:
	switch (arg) {
		case EVENT_THERM_SENSOR:
		case EVENT_THERM_PROCHOT:
		case EVENT_THERM_CRIT:
		case EVENT_THERM_THOLD:
		case EVENT_POWER_LIMIT:
		case EVENT_CURRENT_LIMIT:
		case EVENT_CROSS_DOMAIN:
			Controller_Stop(1);
			Clear_Events = arg;
			Controller_Start(1);
			Clear_Events = 0;
			rc = RC_OK_COMPUTE;
			break;
	}
	break;

    default:
	rc = -EINVAL;
	break;
    }
	return rc;
}

#undef SYSGATE_UPDATE

static int CoreFreqK_mmap(struct file *pfile, struct vm_area_struct *vma)
{
	unsigned long reqSize = vma->vm_end - vma->vm_start;
	int rc = -EIO;
	UNUSED(pfile);

  if (vma->vm_pgoff == ID_RO_VMA_PROC) {
    if (PUBLIC(RO(Proc)) != NULL)
    {
	const unsigned long secSize = ROUND_TO_PAGES(sizeof(PROC_RO));
	if (reqSize != secSize) {
		rc = -EAGAIN;
		goto EXIT_PAGE;
	}

	vma->vm_flags = VM_READ;
	vma->vm_page_prot = PAGE_READONLY;

	rc = remap_pfn_range(	vma,
				vma->vm_start,
			virt_to_phys((void *) PUBLIC(RO(Proc))) >> PAGE_SHIFT,
				reqSize,
				vma->vm_page_prot);
    }
  } else if (vma->vm_pgoff == ID_RW_VMA_PROC) {
    if (PUBLIC(RW(Proc)) != NULL)
    {
	const unsigned long secSize = ROUND_TO_PAGES(sizeof(PROC_RW));
	if (reqSize != secSize) {
		rc = -EAGAIN;
		goto EXIT_PAGE;
	}

	rc = remap_pfn_range(	vma,
				vma->vm_start,
			virt_to_phys((void *) PUBLIC(RW(Proc))) >> PAGE_SHIFT,
				reqSize,
				vma->vm_page_prot);
    }
  } else if (vma->vm_pgoff == ID_RO_VMA_GATE) {
    if (PUBLIC(RO(Proc)) != NULL)
    {
	switch (SysGate_OnDemand()) {
	default:
	case -1:
		break;
	case 1:
		fallthrough;
	case 0: {
		const unsigned long
		secSize = PAGE_SIZE << PUBLIC(RO(Proc))->Gate.ReqMem.Order;
		if (reqSize != secSize) {
			return -EAGAIN;
		}

		vma->vm_flags = VM_READ;
		vma->vm_page_prot = PAGE_READONLY;

		rc = remap_pfn_range(	vma,
					vma->vm_start,
			virt_to_phys((void *) PUBLIC(OF(Gate))) >> PAGE_SHIFT,
					reqSize,
					vma->vm_page_prot);
	    }
		break;
	}
    }
  } else if ((vma->vm_pgoff >= ID_RO_VMA_CORE)
	  && (vma->vm_pgoff < ID_RW_VMA_CORE))
  {
	signed int cpu = vma->vm_pgoff - ID_RO_VMA_CORE;

    if (PUBLIC(RO(Proc)) != NULL) {
      if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count)) {
	if (PUBLIC(RO(Core, AT(cpu))) != NULL)
	{
		const unsigned long secSize = ROUND_TO_PAGES(sizeof(CORE_RO));
		if (reqSize != secSize) {
			rc = -EAGAIN;
			goto EXIT_PAGE;
		}

		vma->vm_flags = VM_READ;
		vma->vm_page_prot = PAGE_READONLY;

		rc = remap_pfn_range(	vma,
					vma->vm_start,
		virt_to_phys((void *) PUBLIC(RO(Core, AT(cpu)))) >> PAGE_SHIFT,
					reqSize,
					vma->vm_page_prot);
	}
      }
    }
  } else if ((vma->vm_pgoff >= ID_RW_VMA_CORE)
	  && (vma->vm_pgoff < ID_ANY_VMA_JAIL))
  {
	signed int cpu = vma->vm_pgoff - ID_RW_VMA_CORE;

    if (PUBLIC(RO(Proc)) != NULL) {
      if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count)) {
	if (PUBLIC(RW(Core, AT(cpu))) != NULL)
	{
		const unsigned long secSize = ROUND_TO_PAGES(sizeof(CORE_RW));
		if (reqSize != secSize) {
			rc = -EAGAIN;
			goto EXIT_PAGE;
		}

		rc = remap_pfn_range(	vma,
					vma->vm_start,
		virt_to_phys((void *) PUBLIC(RW(Core, AT(cpu)))) >> PAGE_SHIFT,
					reqSize,
					vma->vm_page_prot);
	}
      }
    }
  }
EXIT_PAGE:
	return rc;
}

static DEFINE_MUTEX(CoreFreqK_mutex);		/* Only one driver instance. */

static int CoreFreqK_open(struct inode *inode, struct file *pfile)
{
	UNUSED(inode);
	UNUSED(pfile);

	if (!mutex_trylock(&CoreFreqK_mutex))
		return -EBUSY;
	else
		return 0;
}

static int CoreFreqK_release(struct inode *inode, struct file *pfile)
{
	UNUSED(inode);
	UNUSED(pfile);

	mutex_unlock(&CoreFreqK_mutex);
	return 0;
}

static struct file_operations CoreFreqK_fops = {
	.open		= CoreFreqK_open,
	.release	= CoreFreqK_release,
	.mmap		= CoreFreqK_mmap,
	.unlocked_ioctl = CoreFreqK_ioctl,
	.owner		= THIS_MODULE,
};

#ifdef CONFIG_PM_SLEEP
static int CoreFreqK_Suspend(struct device *dev)
{
	UNUSED(dev);

	Controller_Stop(1);

	pr_notice("CoreFreq: Suspend\n");

	return 0;
}

static int CoreFreqK_Resume(struct device *dev)
{	/*		Probe Processor again				*/
	UNUSED(dev);

    if (Arch[PUBLIC(RO(Proc))->ArchID].Query != NULL) {
	Arch[PUBLIC(RO(Proc))->ArchID].Query(PUBLIC(RO(Proc))->Service.Core);
    }
	/*		Probe PCI again 				*/
    if (PUBLIC(RO(Proc))->Registration.PCI) {
	PUBLIC(RO(Proc))->Registration.PCI = \
		CoreFreqK_ProbePCI(Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
				CoreFreqK_ResetChip, CoreFreqK_AppendChip) == 0;
    }
	Controller_Start(1);

#ifdef CONFIG_CPU_FREQ
	Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */

	BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY); /* Notify Daemon*/

	pr_notice("CoreFreq: Resume\n");

	return 0;
}

static SIMPLE_DEV_PM_OPS(CoreFreqK_pm_ops, CoreFreqK_Suspend, CoreFreqK_Resume);
#define COREFREQ_PM_OPS (&CoreFreqK_pm_ops)
#else /* CONFIG_PM_SLEEP */
#define COREFREQ_PM_OPS NULL
#endif /* CONFIG_PM_SLEEP */


#ifdef CONFIG_HOTPLUG_CPU
static int CoreFreqK_HotPlug_CPU_Online(unsigned int cpu)
{
  if (cpu < PUBLIC(RO(Proc))->CPU.Count)
  {
	/*	Is this the very first time the processor is online ?	*/
   if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID == -1)
   {
    if (Core_Topology(cpu) == 0)
    {
     if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID >= 0)
     {
	BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
	/*		Is the BCLK frequency missing  ?		*/
      if (PUBLIC(RO(Core, AT(cpu)))->Clock.Hz == 0)
      {
       if (AutoClock & 0b01)
       {
	COMPUTE_ARG Compute = {
		.TSC = {NULL, NULL},
		.Clock = {
			.Q = PUBLIC(RO(Core, \
			AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)],
			.R = 0, .Hz = 0
		}
	};
	if ((Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
	{
	    if ((Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
	    {
		PUBLIC(RO(Core, AT(cpu)))->Clock = Compute_Clock(cpu, &Compute);

		kfree(Compute.TSC[1]);
	    }
		kfree(Compute.TSC[0]);
	}
       } else {
	PUBLIC(RO(Core, AT(cpu)))->Clock = \
		PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)))->Clock;
       }
      }
     }
    } else {
	BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
    }
   }
	/*		Start the collect timer dedicated to this CPU.	*/
   if (Arch[PUBLIC(RO(Proc))->ArchID].Timer != NULL) {
	Arch[PUBLIC(RO(Proc))->ArchID].Timer(cpu);
   }
   if ((BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED) == 0)
    && (Arch[PUBLIC(RO(Proc))->ArchID].Start != NULL)) {
		smp_call_function_single(cpu,
					Arch[PUBLIC(RO(Proc))->ArchID].Start,
					NULL, 0);
   }
	PUBLIC(RO(Proc))->CPU.OnLine++;
	BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);

	MatchPeerForUpService(&PUBLIC(RO(Proc))->Service, cpu);

#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
   if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE) {
	struct cpuidle_device *device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
	if (device != NULL) {
		if (device->registered == 0) {
			device->cpu = cpu;
			cpuidle_register_device(device);
		}
	}
   }
#endif /* CONFIG_CPU_IDLE */

#ifdef CONFIG_CPU_FREQ
	Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */

	return 0;
  } else
	return -EINVAL;
}

static int CoreFreqK_HotPlug_CPU_Offline(unsigned int cpu)
{
    if (cpu < PUBLIC(RO(Proc))->CPU.Count) {
	/*		Stop the associated collect timer.		*/
	if((BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, CREATED) == 1)
	&& (BITVAL(PRIVATE(OF(Join, AT(cpu)))->TSM, STARTED) == 1)
	&& (Arch[PUBLIC(RO(Proc))->ArchID].Stop != NULL)) {
		smp_call_function_single(cpu,
					Arch[PUBLIC(RO(Proc))->ArchID].Stop,
					NULL, 1);
	}
	PUBLIC(RO(Proc))->CPU.OnLine--;
	BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);

	/*		Seek for an alternate Service Processor.	*/
	if ((cpu == PUBLIC(RO(Proc))->Service.Core)
	 || (cpu == PUBLIC(RO(Proc))->Service.Thread))
	{
		MatchPeerForDownService(&PUBLIC(RO(Proc))->Service, cpu);

	  if (PUBLIC(RO(Proc))->Service.Core != cpu)
	  {
	    if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL)
	    {
		smp_call_function_single(PUBLIC(RO(Proc))->Service.Core,
					Arch[PUBLIC(RO(Proc))->ArchID].Update,
			PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core))),1);
	    }
#if CONFIG_HAVE_PERF_EVENTS==1
		/*	Reinitialize the PMU Uncore counters.		*/
	    if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL)
	    {
		smp_call_function_single(PUBLIC(RO(Proc))->Service.Core,
				Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop,
					NULL, 1); /* Must wait! */
	    }
	    if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL)
	    {
		smp_call_function_single(PUBLIC(RO(Proc))->Service.Core,
				Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start,
					NULL, 0); /* Don't wait */
	    }
#endif /* CONFIG_HAVE_PERF_EVENTS */
	  }
	}
#ifdef CONFIG_CPU_FREQ
	Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
	return 0;
    } else
	return -EINVAL;
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
static int CoreFreqK_HotPlug(	struct notifier_block *nfb,
				unsigned long action,
				void *hcpu)
{
	unsigned int cpu = (unsigned long) hcpu, rc = 0;

	switch (action) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
/*TODO	case CPU_ONLINE_FROZEN: 					*/
		rc = CoreFreqK_HotPlug_CPU_Online(cpu);
		break;
	case CPU_DOWN_PREPARE:
/*TODO	case CPU_DOWN_PREPARE_FROZEN:					*/
		rc = CoreFreqK_HotPlug_CPU_Offline(cpu);
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block CoreFreqK_notifier_block = {
	.notifier_call = CoreFreqK_HotPlug,
};
#endif /* KERNEL_VERSION(4, 10, 0) */
#endif /* CONFIG_HOTPLUG_CPU */

void SMBIOS_Collect(void)
{
#ifdef CONFIG_DMI
	struct {
		enum dmi_field field;
		char *recipient;
	} dmi_collect[] = {
		{ DMI_BIOS_VENDOR,	PUBLIC(RO(Proc))->SMB.BIOS.Vendor    },
		{ DMI_BIOS_VERSION,	PUBLIC(RO(Proc))->SMB.BIOS.Version   },
		{ DMI_BIOS_DATE,	PUBLIC(RO(Proc))->SMB.BIOS.Release   },
		{ DMI_SYS_VENDOR,	PUBLIC(RO(Proc))->SMB.System.Vendor  },
		{ DMI_PRODUCT_NAME,	PUBLIC(RO(Proc))->SMB.Product.Name   },
		{ DMI_PRODUCT_VERSION,	PUBLIC(RO(Proc))->SMB.Product.Version},
		{ DMI_PRODUCT_SERIAL,	PUBLIC(RO(Proc))->SMB.Product.Serial },
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
		{ DMI_PRODUCT_SKU,	PUBLIC(RO(Proc))->SMB.Product.SKU    },
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
		{ DMI_PRODUCT_FAMILY,	PUBLIC(RO(Proc))->SMB.Product.Family },
#endif
		{ DMI_BOARD_VENDOR,	PUBLIC(RO(Proc))->SMB.Board.Vendor   },
		{ DMI_BOARD_NAME,	PUBLIC(RO(Proc))->SMB.Board.Name     },
		{ DMI_BOARD_VERSION,	PUBLIC(RO(Proc))->SMB.Board.Version  },
		{ DMI_BOARD_SERIAL,	PUBLIC(RO(Proc))->SMB.Board.Serial   }
	};
	size_t count = sizeof(dmi_collect) / sizeof(dmi_collect[0]), idx;

	for (idx = 0; idx < count; idx++) {
		const char *pInfo = dmi_get_system_info(dmi_collect[idx].field);
		if ((pInfo != NULL) && (strlen(pInfo) > 0)) {
			StrCopy(dmi_collect[idx].recipient, pInfo, MAX_UTS_LEN);
		}
	}
#endif /* CONFIG_DMI */
}

static char *CoreFreqK_DevNode(struct device *dev, umode_t *mode)
{
	UNUSED(dev);

	if (mode != NULL) {
		(*mode) = 0600 ; /*	Device access is crw------	*/
	}
	return NULL;
}

static void CoreFreqK_Empty_Func_Level_Down(void)
{
}

static void CoreFreqK_Alloc_Features_Level_Down(void)
{
	pr_notice("CoreFreq: Unload\n");
}

static int CoreFreqK_Alloc_Features_Level_Up(INIT_ARG *pArg)
{
	pArg->Features = kmalloc(sizeof(FEATURES), GFP_KERNEL);
	if (pArg->Features == NULL)
	{
		return -ENOMEM;
	} else {
		memset(pArg->Features, 0, sizeof(FEATURES));
	}
	pArg->Brand = kmalloc(BRAND_SIZE, GFP_KERNEL);
	if (pArg->Brand == NULL)
	{
		return -ENOMEM;
	} else {
		memset(pArg->Brand, 0x20, BRAND_SIZE);
	}
	return 0;
}

#define CoreFreqK_Query_Features_Level_Down CoreFreqK_Empty_Func_Level_Down

static int CoreFreqK_Query_Features_Level_Up(INIT_ARG *pArg)
{
	int rc = 0;
	if (ServiceProcessor == -1)
	{	/*	Query features on any processor.		*/
		pArg->localProcessor = get_cpu(); /* TODO(preempt_disable) */
		Query_Features(pArg);
		put_cpu();	/* TODO(preempt_enable) */
		rc = pArg->rc;
	} else { /*	Query features on User selected processor.	*/
		if (ServiceProcessor >= 0)
		{
			pArg->localProcessor = ServiceProcessor;
			if ((rc = smp_call_function_single(pArg->localProcessor,
							Query_Features,
							pArg, 1)) == 0)
			{
				rc = pArg->rc;
			}
		} else {
			rc = -ENXIO;
		}
	}
	if (rc == 0)
	{
		switch (CPU_Count) {
		case -1:
		    {	/* Rely on the operating system's cpu counting. */
			unsigned int OS_Count = num_present_cpus();
			if (pArg->SMT_Count != OS_Count) {
				pArg->SMT_Count = OS_Count;
			}
		    }
			break;
		default:
			if ((CPU_Count > 0) && (CPU_Count <= CORE_COUNT)
			 && (CPU_Count <= NR_CPUS))
			{ /* Hardware probing unless User override value */
				pArg->SMT_Count = (unsigned int) CPU_Count;
			}
			break;
		}
	} else {
		rc = -ENXIO;
	}
	return rc;
}

static void CoreFreqK_Alloc_Device_Level_Down(void)
{
	unregister_chrdev_region(CoreFreqK.mkdev, 1);
}

static int CoreFreqK_Alloc_Device_Level_Up(INIT_ARG *pArg)
{
	UNUSED(pArg);

	CoreFreqK.kcdev = cdev_alloc();
	CoreFreqK.kcdev->ops = &CoreFreqK_fops;
	CoreFreqK.kcdev->owner = THIS_MODULE;

	if (alloc_chrdev_region(&CoreFreqK.nmdev, 0, 1, DRV_FILENAME) >= 0)
	{
		return 0;
	} else {
		return -EBUSY;
	}
}

static void CoreFreqK_Make_Device_Level_Down(void)
{
	cdev_del(CoreFreqK.kcdev);
}

static int CoreFreqK_Make_Device_Level_Up(INIT_ARG *pArg)
{
	UNUSED(pArg);

	CoreFreqK.Major = MAJOR(CoreFreqK.nmdev);
	CoreFreqK.mkdev = MKDEV(CoreFreqK.Major, 0);

	if (cdev_add(CoreFreqK.kcdev, CoreFreqK.mkdev, 1) >= 0)
	{
		return 0;
	} else {
		return -EBUSY;
	}
}

static void CoreFreqK_Create_Device_Level_Down(void)
{
	device_destroy(CoreFreqK.clsdev, CoreFreqK.mkdev);
	class_destroy(CoreFreqK.clsdev);
}

static int CoreFreqK_Create_Device_Level_Up(INIT_ARG *pArg)
{
	struct device *tmpDev;
	UNUSED(pArg);

	CoreFreqK.clsdev = class_create(THIS_MODULE, DRV_DEVNAME);
	CoreFreqK.clsdev->pm = COREFREQ_PM_OPS;
	CoreFreqK.clsdev->devnode = CoreFreqK_DevNode;

	if ((tmpDev = device_create(	CoreFreqK.clsdev, NULL,
					CoreFreqK.mkdev, NULL,
					DRV_DEVNAME)) != NULL)
	{
		return 0;
	} else {
		return -EBUSY;
	}
}

static void CoreFreqK_Alloc_Public_Level_Down(void)
{
	if (PUBLIC() != NULL)
	{
		kfree(PUBLIC());
	}
}

static int CoreFreqK_Alloc_Public_Level_Up(INIT_ARG *pArg)
{
	const unsigned long publicSize	= sizeof(KPUBLIC)
					+ sizeof(CORE_RO*) * pArg->SMT_Count
					+ sizeof(CORE_RW*) * pArg->SMT_Count;

	if (((PUBLIC() = kmalloc(publicSize, GFP_KERNEL)) != NULL))
	{
		memset(PUBLIC(), 0, publicSize);

		PUBLIC(RO(Core))	= (CORE_RO**) &PUBLIC()
					+ sizeof(KPUBLIC);

		PUBLIC(RW(Core))	= (CORE_RW**) PUBLIC(RO(Core))
					+ sizeof(CORE_RO*) * pArg->SMT_Count;

		return 0;
	} else{
		return -ENOMEM;
	}
}

static void CoreFreqK_Alloc_Private_Level_Down(void)
{
	if (PRIVATE() != NULL)
	{
		kfree(PRIVATE());
	}
}

static int CoreFreqK_Alloc_Private_Level_Up(INIT_ARG *pArg)
{
	const unsigned long privateSize = sizeof(KPRIVATE)
					+ sizeof(JOIN*) * pArg->SMT_Count;

	if (((PRIVATE() = kmalloc(privateSize, GFP_KERNEL)) != NULL))
	{
		memset(PRIVATE(), 0, privateSize);

		return 0;
	} else {
		return -ENOMEM;
	}
}

static void CoreFreqK_Alloc_Processor_RO_Level_Down(void)
{
	if (PUBLIC(RO(Proc)) != NULL)
	{
		kfree(PUBLIC(RO(Proc)));
	}
}

static int CoreFreqK_Alloc_Processor_RO_Level_Up(INIT_ARG *pArg)
{
	const unsigned long procSize = ROUND_TO_PAGES(sizeof(PROC_RO));
	UNUSED(pArg);

	if ( (PUBLIC(RO(Proc)) = kmalloc(procSize, GFP_KERNEL)) != NULL)
	{
		memset(PUBLIC(RO(Proc)), 0, procSize);

		return 0;
	} else {
		return -ENOMEM;
	}
}

static void CoreFreqK_Alloc_Processor_RW_Level_Down(void)
{
	if (PUBLIC(RW(Proc)) != NULL)
	{
		kfree(PUBLIC(RW(Proc)));
	}
}

static int CoreFreqK_Alloc_Processor_RW_Level_Up(INIT_ARG *pArg)
{
	const unsigned long procSize = ROUND_TO_PAGES(sizeof(PROC_RW));
	UNUSED(pArg);

	if ( (PUBLIC(RW(Proc)) = kmalloc(procSize, GFP_KERNEL)) != NULL)
	{
		memset(PUBLIC(RW(Proc)), 0, procSize);

		return 0;
	} else {
		return -ENOMEM;
	}
}

#define CoreFreqK_Scale_And_Compute_Level_Down CoreFreqK_Empty_Func_Level_Down

static int CoreFreqK_Scale_And_Compute_Level_Up(INIT_ARG *pArg)
{
	SET_FOOTPRINT(PUBLIC(RO(Proc))->FootPrint,	\
					MAX_FREQ_HZ,	\
					CORE_COUNT,	\
					TASK_ORDER,	\
					COREFREQ_MAJOR, \
					COREFREQ_MINOR, \
					COREFREQ_REV	);

	PUBLIC(RO(Proc))->CPU.Count = pArg->SMT_Count;
	/* PreCompute SysGate memory allocation. */
	PUBLIC(RO(Proc))->Gate.ReqMem.Size = sizeof(SYSGATE_RO);

	PUBLIC(RO(Proc))->Gate.ReqMem.Order = \
				get_order(PUBLIC(RO(Proc))->Gate.ReqMem.Size);

	PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
	PUBLIC(RO(Proc))->Registration.Experimental = Experimental;

	Compute_Interval();

	memcpy(&PUBLIC(RO(Proc))->Features, pArg->Features, sizeof(FEATURES));

	/* Initialize default uArch's codename with the CPUID brand. */
	Arch[GenuineArch].Architecture->CodeName = \
				PUBLIC(RO(Proc))->Features.Info.Vendor.ID;
	return 0;
}

static void CoreFreqK_Alloc_Public_Cache_Level_Down(void)
{
	if (PUBLIC(OF(Cache)) != NULL)
	{
		kmem_cache_destroy(PUBLIC(OF(Cache)));
	}
}

static int CoreFreqK_Alloc_Public_Cache_Level_Up(INIT_ARG *pArg)
{
	const unsigned long cacheSize = KMAX( ROUND_TO_PAGES(sizeof(CORE_RO)),
					      ROUND_TO_PAGES(sizeof(CORE_RW)) );
	UNUSED(pArg);

	if ( (PUBLIC(OF(Cache)) = kmem_cache_create(	"corefreqk-pub",
							cacheSize, 0,
							SLAB_HWCACHE_ALIGN,
							NULL ) ) != NULL)
	{
		return 0;
	} else {
		return -ENOMEM;
	}
}

static void CoreFreqK_Alloc_Private_Cache_Level_Down(void)
{
	if (PRIVATE(OF(Cache)) != NULL)
	{
		kmem_cache_destroy(PRIVATE(OF(Cache)));
	}
}

static int CoreFreqK_Alloc_Private_Cache_Level_Up(INIT_ARG *pArg)
{
	const unsigned long joinSize = ROUND_TO_PAGES(sizeof(JOIN));
	UNUSED(pArg);

	if ( (PRIVATE(OF(Cache)) = kmem_cache_create(	"corefreqk-priv",
							joinSize, 0,
							SLAB_HWCACHE_ALIGN,
							NULL ) ) != NULL)
	{
		return 0;
	} else {
		return -ENOMEM;
	}
}

static void CoreFreqK_Alloc_Per_CPU_Level_Down(void)
{
	unsigned int cpu;
    for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
    {
	if (PUBLIC(OF(Cache)) != NULL)
	{
	    if (PUBLIC(RO(Core, AT(cpu))) != NULL) {
		kmem_cache_free(PUBLIC(OF(Cache)), PUBLIC(RO(Core, AT(cpu))));
	    }
	    if (PUBLIC(RW(Core, AT(cpu))) != NULL) {
		kmem_cache_free(PUBLIC(OF(Cache)), PUBLIC(RW(Core, AT(cpu))));
	    }
	}
	if (PRIVATE(OF(Cache)) != NULL)
	{
	    if (PRIVATE(OF(Join, AT(cpu))) != NULL) {
		kmem_cache_free(PRIVATE(OF(Cache)), PRIVATE(OF(Join, AT(cpu))));
	    }
	}
    }
}

static int CoreFreqK_Alloc_Per_CPU_Level_Up(INIT_ARG *pArg)
{
	const unsigned long cacheSize = KMAX( ROUND_TO_PAGES(sizeof(CORE_RO)),
					      ROUND_TO_PAGES(sizeof(CORE_RW)) );
	const unsigned long joinSize = ROUND_TO_PAGES(sizeof(JOIN));
	unsigned int cpu;
	int rc = 0;
	UNUSED(pArg);

	for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
	{
		void *kcache = NULL;

		kcache = kmem_cache_alloc(PUBLIC(OF(Cache)), GFP_KERNEL);
		if (kcache != NULL) {
			memset(kcache, 0, cacheSize);
			PUBLIC(RO(Core, AT(cpu))) = kcache;
		} else {
			rc = -ENOMEM;
			break;
		}
		kcache = kmem_cache_alloc(PUBLIC(OF(Cache)), GFP_KERNEL);
		if (kcache != NULL) {
			memset(kcache, 0, cacheSize);
			PUBLIC(RW(Core, AT(cpu))) = kcache;
		} else {
			rc = -ENOMEM;
			break;
		}
		kcache = kmem_cache_alloc(PRIVATE(OF(Cache)), GFP_KERNEL);
		if (kcache != NULL) {
			memset(kcache, 0, joinSize);
			PRIVATE(OF(Join, AT(cpu))) = kcache;
		} else {
			rc = -ENOMEM;
			break;
		}

		BITCLR(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);

		PUBLIC(RO(Core, AT(cpu)))->Bind = cpu;

		Define_CPUID(PUBLIC(RO(Core, AT(cpu))), CpuIDforVendor);
	}
	return rc;
}

static void CoreFreqK_Ignition_Level_Down(void)
{
	CoreFreqK_UnRegister_ClockSource();
	CoreFreqK_UnRegister_Governor();
	CoreFreqK_UnRegister_CPU_Freq();
	CoreFreqK_UnRegister_CPU_Idle();
	CoreFreqK_UnRegister_NMI();

#ifdef CONFIG_HOTPLUG_CPU
    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
	unregister_hotcpu_notifier(&CoreFreqK_notifier_block);
    #else /* KERNEL_VERSION(4, 10, 0) */
    if ( !(PUBLIC(RO(Proc))->Registration.HotPlug < 0) )
    {
	cpuhp_remove_state_nocalls(PUBLIC(RO(Proc))->Registration.HotPlug);
    }
    #endif /* KERNEL_VERSION(4, 10, 0) */
#endif /* CONFIG_HOTPLUG_CPU */

	Controller_Stop(1);
	Controller_Exit();

    if (PUBLIC(OF(Gate)) != NULL)
    {
	free_pages_exact(PUBLIC(OF(Gate)), PUBLIC(RO(Proc))->Gate.ReqMem.Size);
    }
}

static int CoreFreqK_Ignition_Level_Up(INIT_ARG *pArg)
{
	BIT_ATOM_INIT(PRIVATE(OF(AMD_SMN_LOCK)), ATOMIC_SEED);
	BIT_ATOM_INIT(PRIVATE(OF(AMD_FCH_LOCK)), ATOMIC_SEED);

	switch (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC) {
	case CRC_INTEL: {
		Arch[GenuineArch].Query = Query_GenuineIntel;
		Arch[GenuineArch].Update= PerCore_Intel_Query;
		Arch[GenuineArch].Start = Start_GenuineIntel;
		Arch[GenuineArch].Stop	= Stop_GenuineIntel;
		Arch[GenuineArch].Timer = InitTimer_GenuineIntel;
		Arch[GenuineArch].BaseClock = BaseClock_GenuineIntel;

		Arch[GenuineArch].thermalFormula=THERMAL_FORMULA_INTEL;

		Arch[GenuineArch].voltageFormula=VOLTAGE_FORMULA_INTEL;

		Arch[GenuineArch].powerFormula = POWER_FORMULA_INTEL;
		}
		break;
	case CRC_HYGON:
		fallthrough;
	case CRC_AMD: {
		Arch[GenuineArch].Query = Query_AuthenticAMD;
		Arch[GenuineArch].Update= PerCore_AuthenticAMD_Query;
		Arch[GenuineArch].Start = Start_AuthenticAMD;
		Arch[GenuineArch].Stop	= Stop_AuthenticAMD;
		Arch[GenuineArch].Timer = InitTimer_AuthenticAMD;
		Arch[GenuineArch].BaseClock = BaseClock_AuthenticAMD;

		Arch[GenuineArch].thermalFormula = THERMAL_FORMULA_AMD;

		Arch[GenuineArch].voltageFormula = VOLTAGE_FORMULA_AMD;

		Arch[GenuineArch].powerFormula = POWER_FORMULA_AMD;
		}
		break;
	case CRC_KVM:
	case CRC_VBOX:
	case CRC_KBOX:
	case CRC_VMWARE:
	case CRC_HYPERV:
		/* Unexpected */
		break;
	}
	/*	Is an architecture identifier requested by user ?	*/
	if ( (ArchID != -1) && (ArchID >= 0) && (ArchID < ARCHITECTURES) )
	{
		PUBLIC(RO(Proc))->ArchID = ArchID;
	} else {
		PUBLIC(RO(Proc))->ArchID = SearchArchitectureID();
	}
	/*	Set the uArch's name with the first found codename	*/
	StrCopy(PUBLIC(RO(Proc))->Architecture,
		Arch[PUBLIC(RO(Proc))->ArchID].Architecture[0].CodeName,
		CODENAME_LEN);

	/*	Check if the Processor is actually virtualized ?	*/
	#ifdef CONFIG_XEN
	if (xen_pv_domain() || xen_hvm_domain())
	{
		if (PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv == 0) {
			PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv = 1;
		}
		PUBLIC(RO(Proc))->HypervisorID = HYPERV_XEN;
	}
	#endif /* CONFIG_XEN */

	if ((PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv == 1) && (ArchID == -1))
	{
		VendorFromCPUID(PUBLIC(RO(Proc))->Features.Info.Hypervisor.ID,
				&PUBLIC(RO(Proc))->Features.Info.LargestHypFunc,
				&PUBLIC(RO(Proc))->Features.Info.Hypervisor.CRC,
				&PUBLIC(RO(Proc))->HypervisorID,
				0x40000000LU, 0x0);

		switch (PUBLIC(RO(Proc))->HypervisorID) {
		case HYPERV_NONE:
		case HYPERV_KVM:
		case HYPERV_VBOX:
		case HYPERV_KBOX:
		case HYPERV_VMWARE:
		case HYPERV_HYPERV:
			PUBLIC(RO(Proc))->ArchID = GenuineArch;
			Arch[GenuineArch].Query = Query_VirtualMachine;
			Arch[GenuineArch].Update= PerCore_VirtualMachine;
			Arch[GenuineArch].Start = Start_VirtualMachine;
			Arch[GenuineArch].Stop	= Stop_VirtualMachine;
			Arch[GenuineArch].Timer = InitTimer_VirtualMachine;

			Arch[GenuineArch].thermalFormula = THERMAL_FORMULA_NONE;
			Arch[GenuineArch].voltageFormula = VOLTAGE_FORMULA_NONE;
			Arch[GenuineArch].powerFormula = POWER_FORMULA_NONE;
			break;
		case BARE_METAL:
		case HYPERV_XEN:
		/*	Xen virtualizes better the MSR & PCI registers	*/
			break;
		}
	}
	PUBLIC(RO(Proc))->thermalFormula = \
				Arch[PUBLIC(RO(Proc))->ArchID].thermalFormula;

	PUBLIC(RO(Proc))->voltageFormula = \
				Arch[PUBLIC(RO(Proc))->ArchID].voltageFormula;

	PUBLIC(RO(Proc))->powerFormula = \
				Arch[PUBLIC(RO(Proc))->ArchID].powerFormula;

	CoreFreqK_Thermal_Scope(ThermalScope);
	CoreFreqK_Voltage_Scope(VoltageScope);
	CoreFreqK_Power_Scope(PowerScope);

	/*	Copy various SMBIOS data [version 3.2]			*/
	SMBIOS_Collect();

	/*	Initialize the CoreFreq controller			*/
	Controller_Init();

	MatchPeerForDefaultService(	&PUBLIC(RO(Proc))->Service,
					pArg->localProcessor );

	/*	Register the Idle & Frequency sub-drivers		*/
	CoreFreqK_Register_CPU_Idle();
	CoreFreqK_Register_CPU_Freq();
	CoreFreqK_Register_Governor();

	if (NMI_Disable == 0) {
		CoreFreqK_Register_NMI();
	}

	pr_info(KERN_INFO "CoreFreq(%u:%d):"	\
		" Processor [%2X%1X_%1X%1X]"	\
		" Architecture [%s] %3s [%u/%u]\n",
		PUBLIC(RO(Proc))->Service.Core,PUBLIC(RO(Proc))->Service.Thread,
		PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily,
		PUBLIC(RO(Proc))->Features.Std.EAX.Family,
		PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel,
		PUBLIC(RO(Proc))->Features.Std.EAX.Model,
		PUBLIC(RO(Proc))->Architecture,
		PUBLIC(RO(Proc))->Features.HTT_Enable ? "SMT" : "CPU",
		PUBLIC(RO(Proc))->CPU.OnLine,
		PUBLIC(RO(Proc))->CPU.Count);

	Controller_Start(0);

	PUBLIC(RO(Proc))->Registration.PCI = \
		CoreFreqK_ProbePCI(Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
				CoreFreqK_ResetChip, CoreFreqK_AppendChip) == 0;

#ifdef CONFIG_HOTPLUG_CPU
	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
	/*	Always returns zero (kernel/notifier.c)			*/
	PUBLIC(RO(Proc))->Registration.HotPlug = \
			register_hotcpu_notifier(&CoreFreqK_notifier_block);
	#else	/*	Continue with or without cpu hot-plugging.	*/
	PUBLIC(RO(Proc))->Registration.HotPlug = \
			cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
						"corefreqk/cpu:online",
						CoreFreqK_HotPlug_CPU_Online,
						CoreFreqK_HotPlug_CPU_Offline);
	#endif /* KERNEL_VERSION(4, 6, 0) */
#endif /* CONFIG_HOTPLUG_CPU */

	#ifdef CONFIG_CPU_FREQ
	Policy_Aggregate_Turbo();
	#endif /* CONFIG_CPU_FREQ */

	CoreFreqK_Register_ClockSource(pArg->localProcessor);

	return 0;
}

#define CoreFreqK_User_Ops_Level_Down CoreFreqK_Empty_Func_Level_Down

static int CoreFreqK_User_Ops_Level_Up(INIT_ARG *pArg)
{
	UNUSED(pArg);

  if (PUBLIC(RO(Proc))->ArchID != AMD_Family_0Fh)
  {
	const unsigned int cpu = PUBLIC(RO(Proc))->Service.Core;

	const signed int MinFID = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)],

	MaxFID = MAXCLOCK_TO_RATIO(	signed int,
					PUBLIC(RO(Core, AT(cpu))->Clock.Hz) );

    if (PState_FID != -1) {
      if ((PState_FID >= MinFID) && (PState_FID <= MaxFID)) {
	if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod != NULL)
	{
		long rc;
		CLOCK_ARG clockMod = {
			.NC = CLOCK_MOD_MAX,
			.Ratio = PState_FID,
			.cpu = -1
		};

		Controller_Stop(1);
		rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
		Controller_Start(0);

	  if (rc < RC_SUCCESS) {
		pr_warn("CoreFreq: "					\
			"'PState_FID' Execution failure code %ld\n", rc);
	  }
	} else {
		pr_warn("CoreFreq: "					\
			"Unsupported architecture #%d for 'PState_FID'\n",
			PUBLIC(RO(Proc))->ArchID);
	}
      } else {
	pr_warn("CoreFreq: "						\
		"'PState_FID' is out of range [%d, %d]\n", MinFID, MaxFID);
      }
	PState_FID = -1;
    }
  } /* else handled by function PerCore_AMD_Family_0Fh_PStates()	*/

	return 0;
}

enum RUN_LEVEL {
	Alloc_Features_Level,
	Query_Features_Level,
	Alloc_Device_Level,
	Make_Device_Level,
	Create_Device_Level,
	Alloc_Public_Level,
	Alloc_Private_Level,
	Alloc_Processor_RO_Level,
	Alloc_Processor_RW_Level,
	Scale_And_Compute_Level,
	Alloc_Public_Cache_Level,
	Alloc_Private_Cache_Level,
	Alloc_Per_CPU_Level,
	Ignition_Level,
	User_Ops_Level,
	Running_Level
};

static enum RUN_LEVEL RunLevel = Alloc_Features_Level;

#define COREFREQ_RUN( _level, _action )	CoreFreqK_##_level##_##_action

static void CoreFreqK_ShutDown(void)
{
	void (*LevelFunc[Running_Level])(void) = {
		COREFREQ_RUN(Alloc_Features_Level, Down),
		COREFREQ_RUN(Query_Features_Level, Down),
		COREFREQ_RUN(Alloc_Device_Level, Down),
		COREFREQ_RUN(Make_Device_Level, Down),
		COREFREQ_RUN(Create_Device_Level, Down),
		COREFREQ_RUN(Alloc_Public_Level, Down),
		COREFREQ_RUN(Alloc_Private_Level, Down),
		COREFREQ_RUN(Alloc_Processor_RO_Level, Down),
		COREFREQ_RUN(Alloc_Processor_RW_Level, Down),
		COREFREQ_RUN(Scale_And_Compute_Level, Down),
		COREFREQ_RUN(Alloc_Public_Cache_Level, Down),
		COREFREQ_RUN(Alloc_Private_Cache_Level, Down),
		COREFREQ_RUN(Alloc_Per_CPU_Level, Down),
		COREFREQ_RUN(Ignition_Level, Down),
		COREFREQ_RUN(User_Ops_Level, Down)
	};

	do
	{
		RunLevel--;
		LevelFunc[RunLevel]();
	} while (RunLevel != Alloc_Features_Level) ;
}

static int CoreFreqK_StartUp(void)
{
	int (*LevelFunc[Running_Level])(INIT_ARG *pArg) = {
		COREFREQ_RUN(Alloc_Features_Level, Up),
		COREFREQ_RUN(Query_Features_Level, Up),
		COREFREQ_RUN(Alloc_Device_Level, Up),
		COREFREQ_RUN(Make_Device_Level, Up),
		COREFREQ_RUN(Create_Device_Level, Up),
		COREFREQ_RUN(Alloc_Public_Level, Up),
		COREFREQ_RUN(Alloc_Private_Level, Up),
		COREFREQ_RUN(Alloc_Processor_RO_Level, Up),
		COREFREQ_RUN(Alloc_Processor_RW_Level, Up),
		COREFREQ_RUN(Scale_And_Compute_Level, Up),
		COREFREQ_RUN(Alloc_Public_Cache_Level, Up),
		COREFREQ_RUN(Alloc_Private_Cache_Level, Up),
		COREFREQ_RUN(Alloc_Per_CPU_Level, Up),
		COREFREQ_RUN(Ignition_Level, Up),
		COREFREQ_RUN(User_Ops_Level, Up)
	};
	INIT_ARG iArg = {
		.Features = NULL, .Brand = NULL,
		.SMT_Count = 0, .localProcessor = 0, .rc = 0
	};
	int rc = 0;

	do
	{
		rc = LevelFunc[RunLevel](&iArg);
		RunLevel++;
	} while (RunLevel != Running_Level && rc == 0) ;

	/*	Free any initialization memory allocation.		*/
	if (iArg.Features != NULL)
	{
		kfree(iArg.Features);
	}
	if (iArg.Brand != NULL)
	{
		kfree(iArg.Brand);
	}
	return rc;
}

#undef COREFREQ_RUN
#undef CoreFreqK_Scale_And_Compute_Level_Down
#undef CoreFreqK_Query_Features_Level_Down
#undef COREFREQ_PM_OPS

static int __init CoreFreqK_Init(void)
{
	int rc = CoreFreqK_StartUp();

	if (rc != 0) {
		CoreFreqK_ShutDown();
	}
	return rc;
}

static void __exit CoreFreqK_Exit(void)
{
	CoreFreqK_ShutDown();
}

module_init(CoreFreqK_Init)
module_exit(CoreFreqK_Exit)