Path: blob/master/modules/hekate_libsys_minerva/sys_sdrammtc.c
3694 views
/*1* Minerva Training Cell2* DRAM Training for Tegra X1 SoC. Supports LPDDR4.3*4* Copyright (c) 2018-2025 CTCaer <[email protected]>5*6* This program is free software; you can redistribute it and/or modify it7* under the terms and conditions of the GNU General Public License,8* version 2, as published by the Free Software Foundation.9*10* This program is distributed in the hope it will be useful, but WITHOUT11* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or12* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for13* more details.14*15* You should have received a copy of the GNU General Public License16* along with this program. If not, see <http://www.gnu.org/licenses/>.17*/1819#include <string.h>20#include <stdlib.h>21#include "mtc.h"22#include "mtc_mc_emc_regs.h"23#include "mtc_switch_tables.h"24#include "types.h"25#include <module.h>2627#define DVFS_T21X_CC_VERSION "Minerva Training Cell v0.2_T21X"28#define DVFS_T210_CC_VERSION "Minerva Training Cell v1.6_T210"2930#define EPRINTF(...)31#define EPRINTFARGS(...)3233bool train_ram_patterns;3435/*36* REF: PLL Input reference (OSC_FREQ).37* DIVN: PLL feedback divider.38* DIVM: PLL input divider.39* DIVP: PLL post divider.40* PLL_OUT = (REF / DIVM) * DIVN / DIVP41*42* DIVP | DIVP43* Encoded | Real44* ----------------------45* 0 | 1 (DIVP off)46* 1 | 247* 2 | 348* 3 | 449* 4 | 550* 5 | 651* 6 | 852* 7 | 1053* 8 | 1254* 9 | 1655* 10 | 1256* 11 | 1657* 12 | 2058* 13 | 2459* 14 | 3260*/61static pllm_clk_config_t pllm_clk_config_table[] =62{63// pll_osc_in, pll_out, pll_feedback_div, pll_input_div, pll_post_div.64// f_in, f_out, n, m, p.65// f_out = ((f_in / m) * n) / p. Example: 1600000 = (38400 / 2) * 97.66{38400, 297600, 93, 4, 2},67{38400, 400000, 125, 4, 2},68{38400, 408000, 85, 4, 1},69{38400, 532800, 111, 4, 1},70{38400, 665600, 104, 3, 1},71{38400, 800000, 125, 3, 1},72{38400, 931200, 97, 4, 0},73{38400, 1065600, 111, 4, 0},74{38400, 1200000, 125, 4, 0},75{38400, 1331200, 104, 3, 0},76{38400, 1459200, 76, 2, 0},77{38400, 1600000, 125, 3, 0},78{38400, 1728000, 90, 2, 0}, // Custom. Normalized 1733 MHz.79{38400, 1795200, 187, 4, 0}, // Custom. Normalized 1800 MHz.80{38400, 1862400, 97, 2, 0}, // JEDEC Standard. (T210 official max).81{38400, 1894400, 148, 3, 0}, // Custom. Normalized 1900 MHz.82{38400, 1932800, 151, 3, 0}, // Custom. Normalized 1933 MHz.83{38400, 1958400, 102, 2, 0}, // Custom. Normalized 1966 MHz.84{38400, 1996800, 104, 2, 0}, // Custom. Normalized 2000 MHz.85{38400, 2035200, 106, 2, 0}, // Custom. Normalized 2033 MHz.86{38400, 2064000, 215, 4, 0}, // Custom. Normalized 2066 MHz.87{38400, 2099200, 164, 3, 0}, // Custom. Normalized 2100 MHz.88{38400, 2131200, 111, 2, 0}, // JEDEC Standard. (T210B01 official max).89{38400, 2163200, 169, 3, 0}, // Custom. Normalized 2166 MHz.90{38400, 2188800, 114, 2, 0}, // Custom. Normalized 2200 MHz.91{38400, 2227200, 116, 2, 0}, // Custom. Normalized 2233 MHz.92{38400, 2265600, 118, 2, 0}, // Custom. Normalized 2266 MHz.93{38400, 2291200, 179, 3, 0}, // Custom. Normalized 2300 MHz.94{38400, 2329600, 182, 3, 0}, // Custom. Normalized 2333 MHz.95{38400, 2361600, 123, 2, 0}, // Custom. Normalized 2366 MHz.96{0, 0, 0, 0, 0}97};9899static const u32 burst_regs_emc_addr_table[221] = {100EMC_RC,101EMC_RFC,102EMC_RFCPB,103EMC_REFCTRL2,104EMC_RFC_SLR,105EMC_RAS,106EMC_RP,107EMC_R2W,108EMC_W2R,109EMC_R2P,110EMC_W2P,111EMC_R2R,112EMC_TPPD,113EMC_CCDMW,114EMC_RD_RCD,115EMC_WR_RCD,116EMC_RRD,117EMC_REXT,118EMC_WEXT,119EMC_WDV_CHK,120EMC_WDV,121EMC_WSV,122EMC_WEV,123EMC_WDV_MASK,124EMC_WS_DURATION,125EMC_WE_DURATION,126EMC_QUSE,127EMC_QUSE_WIDTH,128EMC_IBDLY,129EMC_OBDLY,130EMC_EINPUT,131EMC_MRW6,132EMC_EINPUT_DURATION,133EMC_PUTERM_EXTRA,134EMC_PUTERM_WIDTH,135EMC_QRST,136EMC_QSAFE,137EMC_RDV,138EMC_RDV_MASK,139EMC_RDV_EARLY,140EMC_RDV_EARLY_MASK,141EMC_REFRESH,142EMC_BURST_REFRESH_NUM,143EMC_PRE_REFRESH_REQ_CNT,144EMC_PDEX2WR,145EMC_PDEX2RD,146EMC_PCHG2PDEN,147EMC_ACT2PDEN,148EMC_AR2PDEN,149EMC_RW2PDEN,150EMC_CKE2PDEN,151EMC_PDEX2CKE,152EMC_PDEX2MRR,153EMC_TXSR,154EMC_TXSRDLL,155EMC_TCKE,156EMC_TCKESR,157EMC_TPD,158EMC_TFAW,159EMC_TRPAB,160EMC_TCLKSTABLE,161EMC_TCLKSTOP,162EMC_MRW7,163EMC_TREFBW,164EMC_ODT_WRITE,165EMC_FBIO_CFG5,166EMC_FBIO_CFG7,167EMC_CFG_DIG_DLL,168EMC_CFG_DIG_DLL_PERIOD,169EMC_PMACRO_IB_RXRT,170EMC_CFG_PIPE_1,171EMC_CFG_PIPE_2,172EMC_PMACRO_QUSE_DDLL_RANK0_4,173EMC_PMACRO_QUSE_DDLL_RANK0_5,174EMC_PMACRO_QUSE_DDLL_RANK1_4,175EMC_PMACRO_QUSE_DDLL_RANK1_5,176EMC_MRW8,177EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,178EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,179EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,180EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,181EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,182EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,183EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,184EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,185EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,186EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,187EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,188EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,189EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,190EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,191EMC_PMACRO_DDLL_LONG_CMD_0,192EMC_PMACRO_DDLL_LONG_CMD_1,193EMC_PMACRO_DDLL_LONG_CMD_2,194EMC_PMACRO_DDLL_LONG_CMD_3,195EMC_PMACRO_DDLL_LONG_CMD_4,196EMC_PMACRO_DDLL_SHORT_CMD_0,197EMC_PMACRO_DDLL_SHORT_CMD_1,198EMC_PMACRO_DDLL_SHORT_CMD_2,199EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,200EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,201EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,202EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,203EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,204EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,205EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,206EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,207EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,208EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,209EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,210EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,211EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,212EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,213EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,214EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,215EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,216EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,217EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,218EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,219EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,220EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,221EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,222EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,223EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,224EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,225EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,226EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,227EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,228EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,229EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,230EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,231EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,232EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,233EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,234EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,235EMC_TXDSRVTTGEN,236EMC_FDPD_CTRL_DQ,237EMC_FDPD_CTRL_CMD,238EMC_FBIO_SPARE,239EMC_ZCAL_INTERVAL,240EMC_ZCAL_WAIT_CNT,241EMC_MRS_WAIT_CNT,242EMC_MRS_WAIT_CNT2,243EMC_AUTO_CAL_CHANNEL,244EMC_DLL_CFG_0,245EMC_DLL_CFG_1,246EMC_PMACRO_AUTOCAL_CFG_COMMON,247EMC_PMACRO_ZCTRL,248EMC_CFG,249EMC_CFG_PIPE,250EMC_DYN_SELF_REF_CONTROL,251EMC_QPOP,252EMC_DQS_BRLSHFT_0,253EMC_DQS_BRLSHFT_1,254EMC_CMD_BRLSHFT_2,255EMC_CMD_BRLSHFT_3,256EMC_PMACRO_PAD_CFG_CTRL,257EMC_PMACRO_DATA_PAD_RX_CTRL,258EMC_PMACRO_CMD_PAD_RX_CTRL,259EMC_PMACRO_DATA_RX_TERM_MODE,260EMC_PMACRO_CMD_RX_TERM_MODE,261EMC_PMACRO_CMD_PAD_TX_CTRL,262EMC_PMACRO_DATA_PAD_TX_CTRL,263EMC_PMACRO_COMMON_PAD_TX_CTRL,264EMC_PMACRO_VTTGEN_CTRL_0,265EMC_PMACRO_VTTGEN_CTRL_1,266EMC_PMACRO_VTTGEN_CTRL_2,267EMC_PMACRO_BRICK_CTRL_RFU1,268EMC_PMACRO_CMD_BRICK_CTRL_FDPD,269EMC_PMACRO_BRICK_CTRL_RFU2,270EMC_PMACRO_DATA_BRICK_CTRL_FDPD,271EMC_PMACRO_BG_BIAS_CTRL_0,272EMC_CFG_3,273EMC_PMACRO_TX_PWRD_0,274EMC_PMACRO_TX_PWRD_1,275EMC_PMACRO_TX_PWRD_2,276EMC_PMACRO_TX_PWRD_3,277EMC_PMACRO_TX_PWRD_4,278EMC_PMACRO_TX_PWRD_5,279EMC_CONFIG_SAMPLE_DELAY,280EMC_PMACRO_TX_SEL_CLK_SRC_0,281EMC_PMACRO_TX_SEL_CLK_SRC_1,282EMC_PMACRO_TX_SEL_CLK_SRC_2,283EMC_PMACRO_TX_SEL_CLK_SRC_3,284EMC_PMACRO_TX_SEL_CLK_SRC_4,285EMC_PMACRO_TX_SEL_CLK_SRC_5,286EMC_PMACRO_DDLL_BYPASS,287EMC_PMACRO_DDLL_PWRD_0,288EMC_PMACRO_DDLL_PWRD_1,289EMC_PMACRO_DDLL_PWRD_2,290EMC_PMACRO_CMD_CTRL_0,291EMC_PMACRO_CMD_CTRL_1,292EMC_PMACRO_CMD_CTRL_2,293EMC_TR_TIMING_0,294EMC_TR_DVFS,295EMC_TR_CTRL_1,296EMC_TR_RDV,297EMC_TR_QPOP,298EMC_TR_RDV_MASK,299EMC_MRW14,300EMC_TR_QSAFE,301EMC_TR_QRST,302EMC_TRAINING_CTRL,303EMC_TRAINING_SETTLE,304EMC_TRAINING_VREF_SETTLE,305EMC_TRAINING_CA_FINE_CTRL,306EMC_TRAINING_CA_CTRL_MISC,307EMC_TRAINING_CA_CTRL_MISC1,308EMC_TRAINING_CA_VREF_CTRL,309EMC_TRAINING_QUSE_CORS_CTRL,310EMC_TRAINING_QUSE_FINE_CTRL,311EMC_TRAINING_QUSE_CTRL_MISC,312EMC_TRAINING_QUSE_VREF_CTRL,313EMC_TRAINING_READ_FINE_CTRL,314EMC_TRAINING_READ_CTRL_MISC,315EMC_TRAINING_READ_VREF_CTRL,316EMC_TRAINING_WRITE_FINE_CTRL,317EMC_TRAINING_WRITE_CTRL_MISC,318EMC_TRAINING_WRITE_VREF_CTRL,319EMC_TRAINING_MPC,320EMC_MRW15321};322323static const u32 burst_reg_per_ch_emc01_addr_table[8] = {324EMC0_MRW10,325EMC1_MRW10,326EMC0_MRW11,327EMC1_MRW11,328EMC0_MRW12,329EMC1_MRW12,330EMC0_MRW13,331EMC1_MRW13332};333334static const u32 vref_perch_regs_emc01_addr_table[4] = {335EMC0_TRAINING_OPT_DQS_IB_VREF_RANK0,336EMC1_TRAINING_OPT_DQS_IB_VREF_RANK0,337EMC0_TRAINING_OPT_DQS_IB_VREF_RANK1,338EMC1_TRAINING_OPT_DQS_IB_VREF_RANK1339};340341static const u32 training_mod_regs_emc01_addr_table[20] = {342EMC0_TRAINING_RW_OFFSET_IB_BYTE0,343EMC1_TRAINING_RW_OFFSET_IB_BYTE0,344EMC0_TRAINING_RW_OFFSET_IB_BYTE1,345EMC1_TRAINING_RW_OFFSET_IB_BYTE1,346EMC0_TRAINING_RW_OFFSET_IB_BYTE2,347EMC1_TRAINING_RW_OFFSET_IB_BYTE2,348EMC0_TRAINING_RW_OFFSET_IB_BYTE3,349EMC1_TRAINING_RW_OFFSET_IB_BYTE3,350EMC0_TRAINING_RW_OFFSET_IB_MISC,351EMC1_TRAINING_RW_OFFSET_IB_MISC,352EMC0_TRAINING_RW_OFFSET_OB_BYTE0,353EMC1_TRAINING_RW_OFFSET_OB_BYTE0,354EMC0_TRAINING_RW_OFFSET_OB_BYTE1,355EMC1_TRAINING_RW_OFFSET_OB_BYTE1,356EMC0_TRAINING_RW_OFFSET_OB_BYTE2,357EMC1_TRAINING_RW_OFFSET_OB_BYTE2,358EMC0_TRAINING_RW_OFFSET_OB_BYTE3,359EMC1_TRAINING_RW_OFFSET_OB_BYTE3,360EMC0_TRAINING_RW_OFFSET_OB_MISC,361EMC1_TRAINING_RW_OFFSET_OB_MISC362};363364static const u32 trim_regs_emc_addr_table[138] = {365EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,366EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,367EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,368EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,369EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,370EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,371EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,372EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,373EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,374EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,375EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,376EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,377EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,378EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,379EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,380EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,381EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,382EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,383EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,384EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,385EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,386EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,387EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,388EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,389EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,390EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,391EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,392EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,393EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,394EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,395EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,396EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,397EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,398EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,399EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,400EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,401EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,402EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,403EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,404EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,405EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,406EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,407EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,408EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,409EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,410EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,411EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,412EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,413EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,414EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,415EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,416EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,417EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,418EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,419EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,420EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,421EMC_PMACRO_IB_VREF_DQS_0,422EMC_PMACRO_IB_VREF_DQS_1,423EMC_PMACRO_IB_VREF_DQ_0,424EMC_PMACRO_IB_VREF_DQ_1,425EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,426EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,427EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,428EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,429EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,430EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,431EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,432EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,433EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,434EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,435EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,436EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,437EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,438EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,439EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,440EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,441EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,442EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,443EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,444EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,445EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,446EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,447EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,448EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,449EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,450EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,451EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,452EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,453EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,454EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,455EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,456EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,457EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,458EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,459EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,460EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,461EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,462EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,463EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,464EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,465EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,466EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,467EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,468EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,469EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,470EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,471EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,472EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,473EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,474EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,475EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,476EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,477EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,478EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,479EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,480EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,481EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,482EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,483EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,484EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,485EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,486EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,487EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,488EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,489EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,490EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,491EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,492EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,493EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,494EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,495EMC_PMACRO_QUSE_DDLL_RANK0_0,496EMC_PMACRO_QUSE_DDLL_RANK0_1,497EMC_PMACRO_QUSE_DDLL_RANK0_2,498EMC_PMACRO_QUSE_DDLL_RANK0_3,499EMC_PMACRO_QUSE_DDLL_RANK1_0,500EMC_PMACRO_QUSE_DDLL_RANK1_1,501EMC_PMACRO_QUSE_DDLL_RANK1_2,502EMC_PMACRO_QUSE_DDLL_RANK1_3503};504505static const u32 trim_perch_regs_emc01_addr_table[10] = {506EMC0_CMD_BRLSHFT_0,507EMC1_CMD_BRLSHFT_1,508EMC0_DATA_BRLSHFT_0,509EMC1_DATA_BRLSHFT_0,510EMC0_DATA_BRLSHFT_1,511EMC1_DATA_BRLSHFT_1,512EMC0_QUSE_BRLSHFT_0,513EMC1_QUSE_BRLSHFT_1,514EMC0_QUSE_BRLSHFT_2,515EMC1_QUSE_BRLSHFT_3516};517518static const u32 burst_mc_regs_addr_table[33] = {519MC_EMEM_ARB_CFG,520MC_EMEM_ARB_OUTSTANDING_REQ,521MC_EMEM_ARB_REFPB_HP_CTRL,522MC_EMEM_ARB_REFPB_BANK_CTRL,523MC_EMEM_ARB_TIMING_RCD,524MC_EMEM_ARB_TIMING_RP,525MC_EMEM_ARB_TIMING_RC,526MC_EMEM_ARB_TIMING_RAS,527MC_EMEM_ARB_TIMING_FAW,528MC_EMEM_ARB_TIMING_RRD,529MC_EMEM_ARB_TIMING_RAP2PRE,530MC_EMEM_ARB_TIMING_WAP2PRE,531MC_EMEM_ARB_TIMING_R2R,532MC_EMEM_ARB_TIMING_W2W,533MC_EMEM_ARB_TIMING_R2W,534MC_EMEM_ARB_TIMING_CCDMW,535MC_EMEM_ARB_TIMING_W2R,536MC_EMEM_ARB_TIMING_RFCPB,537MC_EMEM_ARB_DA_TURNS,538MC_EMEM_ARB_DA_COVERS,539MC_EMEM_ARB_MISC0,540MC_EMEM_ARB_MISC1,541MC_EMEM_ARB_MISC2,542MC_EMEM_ARB_RING1_THROTTLE,543MC_EMEM_ARB_DHYST_CTRL,544MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,545MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,546MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,547MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,548MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,549MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,550MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,551MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7552};553554static const u32 la_scale_regs_mc_addr_table[24] = {555MC_MLL_MPCORER_PTSA_RATE,556MC_FTOP_PTSA_RATE,557MC_PTSA_GRANT_DECREMENT,558MC_LATENCY_ALLOWANCE_XUSB_0,559MC_LATENCY_ALLOWANCE_XUSB_1,560MC_LATENCY_ALLOWANCE_TSEC_0,561MC_LATENCY_ALLOWANCE_SDMMCA_0,562MC_LATENCY_ALLOWANCE_SDMMCAA_0,563MC_LATENCY_ALLOWANCE_SDMMC_0,564MC_LATENCY_ALLOWANCE_SDMMCAB_0,565MC_LATENCY_ALLOWANCE_PPCS_0,566MC_LATENCY_ALLOWANCE_PPCS_1,567MC_LATENCY_ALLOWANCE_MPCORE_0,568MC_LATENCY_ALLOWANCE_HC_0,569MC_LATENCY_ALLOWANCE_HC_1,570MC_LATENCY_ALLOWANCE_AVPC_0,571MC_LATENCY_ALLOWANCE_GPU_0,572MC_LATENCY_ALLOWANCE_GPU2_0,573MC_LATENCY_ALLOWANCE_NVENC_0,574MC_LATENCY_ALLOWANCE_NVDEC_0,575MC_LATENCY_ALLOWANCE_VIC_0,576MC_LATENCY_ALLOWANCE_VI2_0,577MC_LATENCY_ALLOWANCE_ISP2_0,578MC_LATENCY_ALLOWANCE_ISP2_1579};580581static const u32 periodic_training_addr[10] =582{583EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,584EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,585EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,586EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,587EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,588EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,589EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,590EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,591EMC_DATA_BRLSHFT_0,592EMC_DATA_BRLSHFT_1593};594595static const u32 ram_pattern_dq_table[0x500] = {596/* DQ RAM Patterns Table 0 */5970x18181818, 0x61616161, 0x85858585, 0x14141414, 0x51515151,5980x47474747, 0x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494,5990x51515151, 0x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C,6000x71717171, 0xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E,6010xFBFBFBFB, 0xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848,6020x21212121, 0x85858585, 0x16161616, 0x59595959, 0x66666666,6030x9A9A9A9A, 0x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F,6040x3F3F3F3F, 0xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737,6050xDCDCDCDC, 0x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E,6060xFAFAFAFA, 0xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC,6070x31313131, 0xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F,6080x7F7F7F7F, 0xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242,6090x08080808, 0x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x18181818,6100x61616161, 0x85858585, 0x14141414, 0x51515151, 0x47474747,6110x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494, 0x51515151,6120x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C, 0x71717171,6130xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E, 0xFBFBFBFB,6140xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848, 0x21212121,6150x85858585, 0x16161616, 0x59595959, 0x66666666, 0x9A9A9A9A,6160x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F, 0x3F3F3F3F,6170xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737, 0xDCDCDCDC,6180x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E, 0xFAFAFAFA,6190xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC, 0x31313131,6200xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F, 0x7F7F7F7F,6210xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242, 0x08080808,6220x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x06060606, 0x18181818,6230x21212121, 0x05050505, 0x14141414, 0x11111111, 0x07070707,6240x1E1E1E1E, 0x39393939, 0x25252525, 0x14141414, 0x11111111,6250x06060606, 0x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131,6260x05050505, 0x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B,6270x2D2D2D2D, 0x34343434, 0x12121212, 0x08080808, 0x21212121,6280x05050505, 0x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A,6290x29292929, 0x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F,6300x3C3C3C3C, 0x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C,6310x30303030, 0x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A,6320x2B2B2B2B, 0x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131,6330x05050505, 0x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F,6340x3D3D3D3D, 0x34343434, 0x10101010, 0x02020202, 0x08080808,6350x23232323, 0x0F0F0F0F, 0x06060606, 0x18181818, 0x21212121,6360x05050505, 0x14141414, 0x11111111, 0x07070707, 0x1E1E1E1E,6370x39393939, 0x25252525, 0x14141414, 0x11111111, 0x06060606,6380x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131, 0x05050505,6390x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B, 0x2D2D2D2D,6400x34343434, 0x12121212, 0x08080808, 0x21212121, 0x05050505,6410x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A, 0x29292929,6420x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F, 0x3C3C3C3C,6430x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C, 0x30303030,6440x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A, 0x2B2B2B2B,6450x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131, 0x05050505,6460x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F, 0x3D3D3D3D,6470x34343434, 0x10101010, 0x02020202, 0x08080808, 0x23232323,6480x0F0F0F0F,649650/* DQ RAM Patterns Table 1 */6510x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,6520x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,6530x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,6540x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,6550xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6560x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,6570xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6580x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6590xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,6600xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,6610x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,6620x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6630x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,6640x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,6650x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6660x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000,6670xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,6680xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,6690xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,6700x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,6710xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,6720x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0xFFFFFFFF,6730xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,6740xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,6750xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,6760x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,6770x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x00000000,6780x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000,6790x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,6800x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,6810x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,6820x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,6830x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,6840x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,6850x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,6860x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,6870x00000000, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,6880x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,6890x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,6900x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,6910x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,6920x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,6930x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,6940x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,6950x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,6960x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,6970x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,6980x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,6990x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7000x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,7010x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,7020x00000000,703704/* DQ RAM Patterns Table 2 */7050x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7060xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7070x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7080xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7090x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7100xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7110x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7120xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7130x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7140xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7150x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7160xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7170x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7180xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7190x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7200xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7210x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7220xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7230x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7240xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7250x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7260xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7270x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7280xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,7290x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,7300xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x3F3F3F3F,7310x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7320x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7330x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7340x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7350x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7360x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7370x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7380x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7390x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7400x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7410x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7420x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7430x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7440x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7450x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7460x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7470x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7480x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7490x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7500x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7510x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7520x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7530x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7540x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,7550x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,7560x3F3F3F3F,757758/* DQ RAM Patterns Table 3 */7590x80808080, 0x00000000, 0x80808080, 0x00000000, 0x80808080,7600x00000000, 0x80808080, 0x40404040, 0x00000000, 0x40404040,7610x00000000, 0x40404040, 0x00000000, 0x40404040, 0x20202020,7620x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,7630x20202020, 0x10101010, 0x00000000, 0x10101010, 0x00000000,7640x10101010, 0x00000000, 0x10101010, 0x08080808, 0x00000000,7650x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,7660x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,7670x00000000, 0x04040404, 0x02020202, 0x00000000, 0x02020202,7680x00000000, 0x02020202, 0x00000000, 0x02020202, 0x01010101,7690x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,7700x01010101, 0x00000000, 0x00000000, 0x00000000, 0x00000000,7710x00000000, 0x00000000, 0x00000000, 0x00000000, 0x80808080,7720x00000000, 0x80808080, 0x00000000, 0x80808080, 0x00000000,7730x80808080, 0x40404040, 0x00000000, 0x40404040, 0x00000000,7740x40404040, 0x00000000, 0x40404040, 0x20202020, 0x00000000,7750x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,7760x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,7770x00000000, 0x10101010, 0x08080808, 0x00000000, 0x08080808,7780x00000000, 0x08080808, 0x00000000, 0x08080808, 0x04040404,7790x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,7800x04040404, 0x02020202, 0x00000000, 0x02020202, 0x00000000,7810x02020202, 0x00000000, 0x02020202, 0x01010101, 0x00000000,7820x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,7830x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,7840x00000000, 0x00000000, 0x00000000, 0x00000000, 0x20202020,7850x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,7860x20202020, 0x00000000, 0x20202020, 0x00000000, 0x10101010,7870x00000000, 0x10101010, 0x00000000, 0x10101010, 0x00000000,7880x10101010, 0x00000000, 0x10101010, 0x00000000, 0x08080808,7890x00000000, 0x08080808, 0x00000000, 0x08080808, 0x00000000,7900x08080808, 0x00000000, 0x08080808, 0x00000000, 0x04040404,7910x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,7920x04040404, 0x00000000, 0x04040404, 0x00000000, 0x02020202,7930x00000000, 0x02020202, 0x00000000, 0x02020202, 0x00000000,7940x02020202, 0x00000000, 0x02020202, 0x00000000, 0x01010101,7950x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,7960x01010101, 0x00000000, 0x01010101, 0x00000000, 0x00000000,7970x00000000, 0x00000000, 0x00000000, 0x20202020, 0x00000000,7980x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,7990x00000000, 0x20202020, 0x00000000, 0x10101010, 0x00000000,8000x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,8010x00000000, 0x10101010, 0x00000000, 0x08080808, 0x00000000,8020x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,8030x00000000, 0x08080808, 0x00000000, 0x04040404, 0x00000000,8040x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,8050x00000000, 0x04040404, 0x00000000, 0x02020202, 0x00000000,8060x02020202, 0x00000000, 0x02020202, 0x00000000, 0x02020202,8070x00000000, 0x02020202, 0x00000000, 0x01010101, 0x00000000,8080x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,8090x00000000, 0x01010101, 0x00000000, 0x00000000, 0x00000000,8100x00000000,811812/* DQ RAM Patterns Table 4 */8130xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8140x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8150xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8160x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8170xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8180x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8190xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8200x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8210xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8220x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8230xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8240x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8250xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8260x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8270xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8280x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8290xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8300x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8310xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8320x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8330xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8340x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8350xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8360x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8370xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8380x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8390xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8400x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8410xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8420x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8430xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8440x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8450xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8460x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8470xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8480x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8490xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8500x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8510xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8520x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8530xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8540x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8550xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8560x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8570xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8580x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8590xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8600x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,8610xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,8620x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,8630xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,8640x33333333865};866867static const u32 ram_pattern_dmi_table[0x500] = {868/* DMI RAM Patterns Table 0 */8690xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,8700x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,8710xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,8720xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,8730xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,8740x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,8750x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,8760x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,8770xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,8780x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,8790xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,8800xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,8810xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,8820x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,8830x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,8840x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,8850x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8860x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8870x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8880x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8890x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8900x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8910x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8920x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8930x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8940x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8950x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8960x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8970x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8980x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,8990x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9000x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,901902/* DMI RAM Patterns Table 1 */9030x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,9040xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,9050xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,9060x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,9070xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,9080xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,9090xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,9100xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,9110x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,9120xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,9130xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,9140x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,9150xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,9160xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,9170xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,9180xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,9190x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9200x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9210x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9220x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9230x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9240x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9250x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9260x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9270x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9280x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9290x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9300x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9310x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9320x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9330x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9340x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,935936/* DMI RAM Patterns Table 2 */9370x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9380x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9390x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9400x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9410x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9420x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9430x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9440x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9450x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9460x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9470x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9480x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9490x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9500x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9510x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9520x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,9530x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9540x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9550x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9560x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9570x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9580x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9590x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9600x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9610x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9620x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9630x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9640x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9650x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9660x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9670x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9680x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,969970/* DMI RAM Patterns Table 3 */9710x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9720x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9730x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9740x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9750x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9760x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9770x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9780xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,9790x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9800x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9810x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9820x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9830x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9840x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9850x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9860xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,9870x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9880x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9890x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9900x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9910x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9920x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9930x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9940x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9950x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9960x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9970x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9980x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,9990x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,10000x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,10010x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,10020x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,10031004/* DMI RAM Patterns Table 4 */10050xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10060xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10070xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10080xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10090xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10100xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10110xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10120xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10130xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10140xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10150xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10160xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10170xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10180xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10190xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10200xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10210xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10220xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10230xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10240xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10250xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10260xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10270xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10280xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10290xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10300xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10310xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10320xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10330xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10340xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10350xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,10360xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x31037};10381039static void _usleep(u32 microseconds)1040{1041u32 start = TMR(0x10);1042while ((u32)(TMR(0x10) - start) <= microseconds)1043;1044}10451046static u32 _div_o3(u32 a, u32 b)1047{1048u32 result = a / b;10491050if ((b * result) < a)1051return result + 1;1052else1053return result;1054}10551056static u32 _actual_osc_clocks(u32 in)1057{1058u32 actual_clock;10591060actual_clock = 16 * in;1061if (in > 63)1062{1063actual_clock = 2048;1064if (in > 127)1065{1066if (in >= 192)1067actual_clock = 8192;1068else1069actual_clock = 4096;1070}1071}10721073return actual_clock;1074}10751076static void _ccfifo_write(u32 addr, u32 data_val, u32 delay) //addr and delay are u161077{1078EMC(EMC_CCFIFO_DATA) = data_val;1079EMC(EMC_CCFIFO_ADDR) = BIT(31) | (addr & 0xffff) | ((delay & 0x7FFF) << 16);1080}10811082static bool _wait_emc_status(u32 reg_offset, u32 bit_mask, bool updated_state, u32 emc_channel)1083{1084bool err = true;10851086for (u32 i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++)1087{1088if (emc_channel)1089{1090if (emc_channel != 1)1091goto done;10921093if (((EMC_CH1(reg_offset) & bit_mask) != 0) == updated_state)1094{1095err = false;1096break;1097}1098}1099else if (((EMC(reg_offset) & bit_mask) != 0) == updated_state)1100{1101err = false;1102break;1103}1104_usleep(1);1105}11061107done:1108return err;1109}11101111static void _request_mmr_data(u32 data, bool dual_channel)1112{1113EMC(EMC_MRR) = data;1114_wait_emc_status(EMC_EMC_STATUS, MRR_DIVLD, true, EMC_CHANNEL0);1115if (dual_channel)1116_wait_emc_status(EMC_EMC_STATUS, MRR_DIVLD, true, EMC_CHANNEL1);1117}11181119static void _start_periodic_compensation()1120{1121EMC(EMC_MPC) = 0x4B;1122(void)EMC(EMC_MPC);1123}11241125static bool _timing_update(u32 dual_channel)1126{1127bool err = 0;11281129EMC(EMC_TIMING_CONTROL) = 1;1130err = _wait_emc_status(EMC_EMC_STATUS, TIMING_UPDATE_STALLED, false, EMC_CHANNEL0);1131if (dual_channel)1132err |= _wait_emc_status(EMC_EMC_STATUS, TIMING_UPDATE_STALLED, false, EMC_CHANNEL1);11331134return err;1135}11361137static u32 _get_dram_temperature()1138{1139u32 mr4_0 = 0;1140u32 mr4_1 = 0;11411142bool channel1_enabled = (EMC(EMC_FBIO_CFG7) >> 2) & 1;1143u32 emc_cfg_o = EMC(EMC_CFG);11441145_wait_emc_status(EMC_EMC_STATUS, MRR_DIVLD, false, EMC_CHANNEL0);11461147if (emc_cfg_o & 0x20000000)1148{1149EMC(EMC_CFG) = emc_cfg_o & 0xDFFFFFFF;1150_timing_update(channel1_enabled);1151}11521153_request_mmr_data(0x80040000, EMC_CHANNEL0);1154mr4_0 = EMC(EMC_MRR) & 0xFFFF;11551156if (mr4_0 < 0xF001)1157mr4_0 &= 0x7;1158else1159{1160mr4_0 = -1;1161goto out;1162}11631164if (channel1_enabled)1165{1166_request_mmr_data(0x40040000, EMC_CHANNEL1);1167mr4_1 = EMC(EMC_MRR) & 0xFFFF;11681169if (mr4_1 < 0xF001)1170mr4_1 &= 0x7;1171else1172goto out;11731174if (mr4_1 > mr4_0)1175mr4_0 = mr4_1;1176}11771178out:1179if (emc_cfg_o & 0x20000000)1180{1181EMC(EMC_CFG) = emc_cfg_o;1182_timing_update(channel1_enabled);1183}11841185return mr4_0;1186}11871188static u32 _pllm_clk_base_cfg(u32 rate_KHz, u32 clk_src_emc, bool new_src_is_PLLMB)1189{1190u32 dividers = 0;1191u32 i = 0;1192u32 pll_ref = 38400; // Only 38.4MHz crystal is supported for T210.11931194pllm_clk_config_t *pllm_clk_config = NULL;11951196for (i = 0; pllm_clk_config_table[i].pll_osc_in; i++)1197{1198if (pllm_clk_config_table[i].pll_osc_in == pll_ref && (pllm_clk_config_table[i].pll_out - 19200) <= rate_KHz)1199pllm_clk_config = &pllm_clk_config_table[i];1200}12011202if (pllm_clk_config && pllm_clk_config->pll_osc_in)1203{1204dividers = pllm_clk_config->pll_input_div | (pllm_clk_config->pll_feedback_div << 8) | ((pllm_clk_config->pll_post_div & 0x1F) << 20);1205if (new_src_is_PLLMB)1206{1207CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) = dividers;1208CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) |= PLLM_ENABLE;12091210if ((clk_src_emc >> EMC_2X_CLK_SRC_SHIFT) == PLLM_UD)1211clk_src_emc = (clk_src_emc & 0x1FFFFFFF) | (PLLMB_UD << EMC_2X_CLK_SRC_SHIFT);1212else if (!(clk_src_emc >> EMC_2X_CLK_SRC_SHIFT))1213clk_src_emc |= (PLLMB_OUT0 << EMC_2X_CLK_SRC_SHIFT);12141215while (!(CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) & PLLM_LOCK))1216;1217}1218else1219{1220CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) = dividers;1221CLOCK(CLK_RST_CONTROLLER_PLLM_MISC2) |= PLLM_EN_LCKDET;1222CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) |= PLLM_ENABLE;12231224if ((clk_src_emc >> EMC_2X_CLK_SRC_SHIFT) == PLLM_UD)1225clk_src_emc = (clk_src_emc & 0x1FFFFFFF) | (PLLM_UD << EMC_2X_CLK_SRC_SHIFT);1226while (!(CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) & PLLM_LOCK))1227;1228}1229}1230return clk_src_emc;1231}12321233static void _change_dll_src(emc_table_t *mtc_table_entry, u32 clk_src_emc)1234{1235u32 emc_2x_clk_src = clk_src_emc >> EMC_2X_CLK_SRC_SHIFT;12361237u32 dll_setting = ((((mtc_table_entry->dll_clk_src & 0x1FFFFFFF)1238| (emc_2x_clk_src << EMC_2X_CLK_SRC_SHIFT)) & 0xFFFFFF00)1239| (clk_src_emc & 0xFF)) & 0xFFFFF3FF;12401241if (emc_2x_clk_src == PLLMB_UD)1242dll_setting |= EMC_DLL_PLLM_VCOB;1243else if (emc_2x_clk_src != PLLM_UD)1244dll_setting |= EMC_DLL_SWITCH_OUT;12451246CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL) = dll_setting;12471248// Commit clock write.1249(void)CLOCK(CLK_RST_CONTROLLER_CLK_OUT_ENB_X);1250_usleep(2);12511252// Enable/Disable EMC DLL.1253if (mtc_table_entry->clk_out_enb_x_0_clk_enb_emc_dll)1254CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_SET) = BIT(14);1255else1256CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_CLR) = BIT(14);12571258// Commit clock write.1259(void)CLOCK(CLK_RST_CONTROLLER_CLK_OUT_ENB_X);1260_usleep(2);1261}12621263static u32 _digital_dll_prelock(emc_table_t *mtc_table_entry, bool in_training, u32 selected_clk_src_emc)1264{1265u32 dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1);12661267EMC(EMC_CFG_DIG_DLL) = (EMC(EMC_CFG_DIG_DLL) & 0xFFFFF824) | 0x3C8;12681269_timing_update(dual_channel);12701271while (EMC(EMC_CFG_DIG_DLL) & 1)1272;1273if (dual_channel)1274while (EMC_CH1(EMC_CFG_DIG_DLL) & 1)1275;12761277EMC(EMC_DLL_CFG_0) = mtc_table_entry->burst_regs.emc_dll_cfg_0;1278EMC(EMC_DLL_CFG_1) = mtc_table_entry->burst_regs.emc_dll_cfg_1;12791280_change_dll_src(mtc_table_entry, selected_clk_src_emc);12811282EMC(EMC_CFG_DIG_DLL) |= 1;12831284_timing_update(dual_channel);12851286while (!(EMC(EMC_CFG_DIG_DLL) & 1))1287;1288if (dual_channel)1289while (!(EMC_CH1(EMC_CFG_DIG_DLL) & 1))1290;12911292while ((((EMC(EMC_DIG_DLL_STATUS) >> 17) & 1) ^ 1) | (((EMC(EMC_DIG_DLL_STATUS) >> 15) & 1) ^ 1))1293;12941295if (in_training)1296{1297EMC(EMC_DBG) |= EMC_DBG_WRITE_MUX_ACTIVE;1298EMC(EMC_CFG_DIG_DLL) &= 0xFFFFFFFE; // Disable CFG_DLL_EN.1299EMC(EMC_DBG) &= ~EMC_DBG_WRITE_MUX_ACTIVE;13001301while (EMC(EMC_CFG_DIG_DLL) & 1)1302;1303if (dual_channel)1304while (EMC_CH1(EMC_CFG_DIG_DLL) & 1)1305;1306}13071308return EMC(EMC_DIG_DLL_STATUS) & 0x7FF;1309}13101311static void _digital_dll_disable()1312{1313bool dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1);13141315EMC(EMC_CFG_DIG_DLL) &= 0xFFFFFFFE;13161317_timing_update(dual_channel);13181319while (EMC(EMC_CFG_DIG_DLL) & 1)1320;1321if (dual_channel)1322while (EMC_CH1(EMC_CFG_DIG_DLL) & 1)1323;1324}13251326static void _digital_dll_enable_rs(u32 channel1_enabled)1327{1328EMC(EMC_CFG_DIG_DLL) = (EMC(EMC_CFG_DIG_DLL) & 0xFFFFFF24) | 0x89;13291330_timing_update(channel1_enabled);13311332while (!(EMC(EMC_CFG_DIG_DLL) & 1))1333;1334if (channel1_enabled)1335while (!(EMC_CH1(EMC_CFG_DIG_DLL) & 1))1336;1337}13381339static u32 _dvfs_power_ramp_down(bool flip_backward, emc_table_t *src_emc_table_entry, emc_table_t *dst_emc_table_entry, u32 src_clock_period)1340{1341u32 pmacro_cmd_pad;1342u32 pmacro_rfu1;1343u32 pmacro_cfg5;1344u32 pmacro_com_pad;1345u32 pmacro_dq_pad;13461347u32 src_clk_per_pc = (100000 / src_clock_period) + 1;13481349if (flip_backward)1350{1351pmacro_cmd_pad = dst_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl;1352pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl;1353pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1;1354pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5;1355pmacro_com_pad = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl;1356}1357else1358{1359pmacro_cmd_pad = src_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl;1360pmacro_dq_pad = (dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x101) | src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl;1361pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1;1362pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5;1363pmacro_com_pad = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl;1364}13651366u32 pmacro_cmd_pad_drvforceon = pmacro_cmd_pad | 0x4000000;13671368u32 ramp_down_wait = src_clock_period * 12;13691370_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon, 0);1371_ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 | 0x100, 12);13721373if (src_clock_period >= 1000) // Dvfs high speed threshold.1374{1375_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, (u32)(src_clk_per_pc + 19));1376ramp_down_wait += 100000 + (src_clock_period * 20);1377}1378else1379{1380if (src_clock_period >= 416) // Iobrick dcc threshold.1381_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, (u32)src_clk_per_pc);1382else1383{1384pmacro_dq_pad = (pmacro_dq_pad & 0xFEFEFDFD) | 0x10200;1385pmacro_cmd_pad_drvforceon = (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4010200;1386_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon, (u32)src_clk_per_pc);1387_ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);1388_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, 0);1389}1390ramp_down_wait += 300000;1391_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, (u32)src_clk_per_pc);13921393if (src_clock_period >= 416) // Iobrick dcc threshold.1394_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, (u32)src_clk_per_pc);1395else1396{1397_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon & 0xFEFEFDFD, (u32)src_clk_per_pc);1398_ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad & 0xFEFEFDFD, 0);1399_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, 0);1400}1401}14021403if (src_clock_period >= 1666) // Dvfs mid speed threshold.1404_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFF0, (u32)src_clk_per_pc);1405else1406{1407ramp_down_wait += 400000;1408_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFFA, (u32)src_clk_per_pc);1409_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFF0, (u32)src_clk_per_pc);1410_ccfifo_write(EMC_INTSTATUS, 0, (u32)src_clk_per_pc);1411}14121413return ramp_down_wait;1414}14151416static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_entry, emc_table_t *dst_emc_table_entry, u32 needs_training, u32 dst_clock_period)1417{1418u32 pmacro_cmd_pad;1419u32 pmacro_dq_pad;1420u32 pmacro_rfu1;1421u32 pmacro_cfg5;1422u32 pmacro_com_pad;1423u32 pmacro_cmd_pad_data;1424u32 ramp_up_wait = 0;14251426u32 dst_clk_per_pc = (100000 / dst_clock_period) + 1;14271428if (flip_backward)1429{1430pmacro_cmd_pad = src_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl;1431pmacro_dq_pad = src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl;1432pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1;1433pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5;1434pmacro_com_pad = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl;1435}1436else if (needs_training & NEEDS_TRAINING_CA_COMBO)1437{1438pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_cmd_pad_tx_ctrl;1439pmacro_dq_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_data_pad_tx_ctrl;1440pmacro_rfu1 = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_brick_ctrl_rfu1;1441pmacro_cfg5 = dst_emc_table_entry->shadow_regs_ca_train.emc_fbio_cfg5;1442pmacro_com_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_common_pad_tx_ctrl;1443}1444else if (needs_training & NEEDS_TRAINING_QUSE_COMBO)1445{1446pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_cmd_pad_tx_ctrl;1447pmacro_dq_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_data_pad_tx_ctrl;1448pmacro_rfu1 = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_brick_ctrl_rfu1;1449pmacro_cfg5 = dst_emc_table_entry->shadow_regs_quse_train.emc_fbio_cfg5;1450pmacro_com_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_common_pad_tx_ctrl;1451}1452else if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO))1453{1454pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_cmd_pad_tx_ctrl;1455pmacro_dq_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_data_pad_tx_ctrl;1456pmacro_rfu1 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_brick_ctrl_rfu1;1457pmacro_cfg5 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_fbio_cfg5;1458pmacro_com_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_common_pad_tx_ctrl;1459}1460else1461{1462pmacro_cmd_pad = dst_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl;1463pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl;1464pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1;1465pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5;1466pmacro_com_pad = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl;1467}14681469pmacro_cmd_pad_data = (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4000000;14701471if (dst_clock_period >= 1666) // Dvfs mid speed threshold.1472{1473_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad | 8, 0);14741475_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x600, 0);1476_ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & 0xFFFFFEFF, 12);14771478ramp_up_wait = (dst_clock_period * 12) + 0;1479}1480else1481{1482_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xA, 0);1483_ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xF, dst_clk_per_pc);14841485if (dst_clock_period < 1000) // Dvfs high speed threshold.1486{1487if (dst_clock_period >= 416) // Iobrick dcc threshold.1488_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, dst_clk_per_pc);1489else1490{1491_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4010200, dst_clk_per_pc);1492_ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, (pmacro_dq_pad & 0xFEFEFDFD) | 0x10200, 0);1493_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, 0);1494}14951496_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, dst_clk_per_pc);14971498if (dst_clock_period >= 416) // Iobrick dcc threshold.1499_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, dst_clk_per_pc);1500else1501{1502pmacro_cmd_pad_data = pmacro_cmd_pad | 0x5010202;1503_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_data, dst_clk_per_pc);1504_ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad | 0x1010202, 0);1505_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, 0);1506}15071508ramp_up_wait = 500000 + (dst_clock_period * 10);1509}1510else // 1000 > dst_clock_period < 1666.1511{1512_ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x6000600, dst_clk_per_pc);15131514ramp_up_wait = 200000 + (dst_clock_period * 10);1515}15161517_ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & 0xFFFFFEFF, dst_clk_per_pc + 9);1518}15191520_ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_data & 0xFBFFFFFF, 5);15211522return ramp_up_wait;1523}15241525static u32 _minerva_update_clock_tree_delay(emc_table_t *src_emc_entry, emc_table_t *dst_emc_entry, u32 dram_dev_num, u32 channel1_enabled, enum tree_update_mode_t update_type)1526{1527u32 cval = 0;1528u32 adelta = 0;1529s32 tdelta = 0;15301531u32 temp_ch0_0 = 0;1532u32 temp_ch0_1 = 0;1533u32 temp_ch1_0 = 0;1534u32 temp_ch1_1 = 0;15351536u32 upd_type_bits = BIT(update_type);1537u32 dst_rate_mhz = dst_emc_entry->rate_khz / 1000;1538u32 src_rate_mhz = src_emc_entry->rate_khz / 1000;15391540u32 tval = 1000000 * _actual_osc_clocks(src_emc_entry->run_clocks) / 2;15411542if (update_type > PERIODIC_TRAINING_UPDATE)1543return 0;15441545if (upd_type_bits & 0x5400)1546{1547_request_mmr_data(0x80130000, channel1_enabled); // Dev0 MRR 19.1548u32 mrr = EMC(EMC_MRR);1549temp_ch0_0 = (mrr & 0xFF) << 8;1550temp_ch0_1 = mrr & 0xFF00;1551if (channel1_enabled)1552{1553mrr = EMC_CH1(EMC_MRR);1554temp_ch1_0 = (mrr & 0xFF) << 8;1555temp_ch1_1 = mrr & 0xFF00;1556}15571558_request_mmr_data(0x80120000, channel1_enabled); // Dev0 MRR 18.1559mrr = EMC(EMC_MRR);1560temp_ch0_0 |= mrr & 0xFF;1561temp_ch0_1 |= (mrr & 0xFF00) >> 8;1562if (channel1_enabled)1563{1564mrr = EMC_CH1(EMC_MRR);1565temp_ch1_0 |= mrr & 0xFF;1566temp_ch1_1 |= (mrr & 0xFF00) >> 8;1567}1568}15691570cval = tval / (src_rate_mhz * temp_ch0_0);1571switch (update_type)1572{1573case DVFS_PT1:1574case TRAINING_PT1:1575dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 += 100 * cval;1576if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1577goto calc_td0_0;1578break;1579case DVFS_UPDATE:1580dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 =1581dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1582break;1583case TRAINING_UPDATE:1584dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 =1585dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 / dst_emc_entry->ptfv_list.ptfv_write_samples;1586break;1587case PERIODIC_TRAINING_UPDATE:1588dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 =1589(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0)1590/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1591break;1592default:1593if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1594goto calc_td0_0;1595break;1596}15971598tdelta = dst_emc_entry->current_dram_clktree_c0d0u0 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 / 100);1599if (tdelta < 0)1600tdelta *= -1;1601adelta = tdelta;1602if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1603dst_emc_entry->current_dram_clktree_c0d0u0 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 / 100;16041605calc_td0_0:1606cval = tval / (src_rate_mhz * temp_ch0_1);1607switch (update_type)1608{1609case DVFS_PT1:1610case TRAINING_PT1:1611dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 += 100 * cval;1612if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1613goto calc_td1_0;1614break;1615case DVFS_UPDATE:1616dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 =1617dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1618break;1619case TRAINING_UPDATE:1620dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 =1621dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 / dst_emc_entry->ptfv_list.ptfv_write_samples;1622break;1623case PERIODIC_TRAINING_UPDATE:1624dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 =1625(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1)1626/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1627break;1628default:1629if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1630goto calc_td1_0;1631break;1632}16331634tdelta = dst_emc_entry->current_dram_clktree_c0d0u1 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 / 100);1635if (tdelta < 0)1636tdelta *= -1;1637if ((u32)tdelta > adelta)1638adelta = tdelta;1639if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1640dst_emc_entry->current_dram_clktree_c0d0u1 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 / 100;16411642calc_td1_0:1643if (channel1_enabled)1644{1645cval = tval / (src_rate_mhz * temp_ch1_0);1646switch (update_type)1647{1648case DVFS_PT1:1649case TRAINING_PT1:1650dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 += 100 * cval;1651if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1652goto calc_td1_1;1653break;1654case DVFS_UPDATE:1655dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 =1656dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1657break;1658case TRAINING_UPDATE:1659dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 =1660dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 / dst_emc_entry->ptfv_list.ptfv_write_samples;1661break;1662case PERIODIC_TRAINING_UPDATE:1663dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 =1664(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0)1665/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1666break;1667default:1668if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1669goto calc_td1_1;1670break;1671}16721673tdelta = dst_emc_entry->current_dram_clktree_c1d0u0 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 / 100);1674if (tdelta < 0)1675tdelta *= -1;1676if ((u32)tdelta > adelta)1677adelta = tdelta;1678if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1679dst_emc_entry->current_dram_clktree_c1d0u0 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 / 100;16801681calc_td1_1:1682cval = tval / (src_rate_mhz * temp_ch1_1);1683switch (update_type)1684{1685case DVFS_PT1:1686case TRAINING_PT1:1687dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 += 100 * cval;1688if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1689goto calc_dev2;1690break;1691case DVFS_UPDATE:1692dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 =1693dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1694break;1695case TRAINING_UPDATE:1696dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 =1697dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 / dst_emc_entry->ptfv_list.ptfv_write_samples;1698break;1699case PERIODIC_TRAINING_UPDATE:1700dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 =1701(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1)1702/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1703break;1704default:1705if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1706goto calc_dev2;1707break;1708}17091710tdelta = dst_emc_entry->current_dram_clktree_c1d0u1 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 / 100);1711if (tdelta < 0)1712tdelta *= -1;1713if ((u32)tdelta > adelta)1714adelta = tdelta;1715if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1716dst_emc_entry->current_dram_clktree_c1d0u1 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 / 100;1717}17181719calc_dev2:1720if (dram_dev_num != TWO_RANK)1721goto out;17221723if (update_type <= PERIODIC_TRAINING_UPDATE && upd_type_bits & 0x5400)1724{1725_request_mmr_data(0x40130000, channel1_enabled); // Dev1 MRR 19.1726u32 mrr = EMC(EMC_MRR);1727temp_ch0_0 = (mrr & 0xFF) << 8;1728temp_ch0_1 = mrr & 0xFF00;1729if (channel1_enabled)1730{1731mrr = EMC_CH1(EMC_MRR);1732temp_ch1_0 = (mrr & 0xFF) << 8;1733temp_ch1_1 = mrr & 0xFF00;1734}17351736_request_mmr_data(0x40120000, channel1_enabled); // Dev1 MRR 181737mrr = EMC(EMC_MRR);1738temp_ch0_0 |= mrr & 0xFF;1739temp_ch0_1 |= (mrr & 0xFF00) >> 8;1740if (channel1_enabled)1741{1742mrr = EMC_CH1(EMC_MRR);1743temp_ch1_0 |= mrr & 0xFF;1744temp_ch1_1 |= (mrr & 0xFF00) >> 8;1745}1746}17471748cval = tval / (src_rate_mhz * temp_ch0_0);1749switch (update_type )1750{1751case DVFS_PT1:1752case TRAINING_PT1:1753dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 += 100 * cval;1754if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1755goto calc_tmp_td0_1;1756break;1757case DVFS_UPDATE:1758dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 =1759dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1760break;1761case TRAINING_UPDATE:1762dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 =1763dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 / dst_emc_entry->ptfv_list.ptfv_write_samples;1764break;1765case PERIODIC_TRAINING_UPDATE:1766dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 =1767(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0)1768/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1769break;1770default:1771if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1772goto calc_tmp_td0_1;1773break;1774}17751776tdelta = dst_emc_entry->current_dram_clktree_c0d1u0 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 / 100);1777if (tdelta < 0)1778tdelta *= -1;1779if ((u32)tdelta > adelta)1780adelta = tdelta;1781if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1782dst_emc_entry->current_dram_clktree_c0d1u0 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 / 100;17831784calc_tmp_td0_1:1785cval = tval / (src_rate_mhz * temp_ch0_1);1786switch (update_type)1787{1788case DVFS_PT1:1789case TRAINING_PT1:1790dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 += 100 * cval;1791if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1792goto calc_tmp_td1_0;1793break;1794case DVFS_UPDATE:1795dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 =1796dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1797break;1798case TRAINING_UPDATE:1799dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 =1800dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 / dst_emc_entry->ptfv_list.ptfv_write_samples;1801break;1802case PERIODIC_TRAINING_UPDATE:1803dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 =1804(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1)1805/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1806break;1807default:1808if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1809goto calc_tmp_td1_0;1810break;1811}18121813tdelta = dst_emc_entry->current_dram_clktree_c0d1u1 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 / 100);1814if (tdelta < 0)1815tdelta *= -1;1816if ((u32)tdelta > adelta)1817adelta = tdelta;1818if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1819dst_emc_entry->current_dram_clktree_c0d1u1 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 / 100;18201821calc_tmp_td1_0:1822if (channel1_enabled)1823{1824cval = tval / (src_rate_mhz * temp_ch1_0);1825switch (update_type)1826{1827case DVFS_PT1:1828case TRAINING_PT1:1829dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 += 100 * cval;1830if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1831goto calc_tmp_td1_1;1832break;1833case DVFS_UPDATE:1834dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 =1835dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1836break;1837case TRAINING_UPDATE:1838dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 =1839dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 / dst_emc_entry->ptfv_list.ptfv_write_samples;1840break;1841case PERIODIC_TRAINING_UPDATE:1842dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 =1843(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0)1844/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1845break;1846default:1847if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1848goto calc_tmp_td1_1;1849break;1850}18511852tdelta = dst_emc_entry->current_dram_clktree_c1d1u0 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 / 100);1853if (tdelta < 0)1854tdelta *= -1;1855if ((u32)tdelta > adelta)1856adelta = tdelta;1857if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1858dst_emc_entry->current_dram_clktree_c1d1u0 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 / 100;18591860calc_tmp_td1_1:1861cval = tval / (src_rate_mhz * temp_ch1_1);1862switch (update_type)1863{1864case DVFS_PT1:1865case TRAINING_PT1:1866dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 += 100 * cval;1867if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1868goto out;1869break;1870case DVFS_UPDATE:1871dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 =1872dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 / dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1873break;1874case TRAINING_UPDATE:1875dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 =1876dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 / dst_emc_entry->ptfv_list.ptfv_write_samples;1877break;1878case PERIODIC_TRAINING_UPDATE:1879dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 =1880(100 * cval + dst_emc_entry->ptfv_list.ptfv_movavg_weight * dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1)1881/ (dst_emc_entry->ptfv_list.ptfv_movavg_weight + 1);1882break;1883default:1884if (update_type > PERIODIC_TRAINING_UPDATE || !(upd_type_bits & 0x6800))1885goto out;1886break;1887}18881889tdelta = dst_emc_entry->current_dram_clktree_c1d1u1 - (dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 / 100);1890if (tdelta < 0)1891tdelta *= -1;1892if ((u32)tdelta > adelta)1893adelta = tdelta;1894if (update_type == TRAINING_UPDATE || ((dst_rate_mhz * tdelta * 128) / 1000000) > dst_emc_entry->tree_margin)1895dst_emc_entry->current_dram_clktree_c1d1u1 = dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 / 100;1896}18971898out:1899if (update_type == TRAINING_UPDATE)1900{1901dst_emc_entry->trained_dram_clktree_c0d0u0 = dst_emc_entry->current_dram_clktree_c0d0u0;1902dst_emc_entry->trained_dram_clktree_c0d0u1 = dst_emc_entry->current_dram_clktree_c0d0u1;1903dst_emc_entry->trained_dram_clktree_c0d1u0 = dst_emc_entry->current_dram_clktree_c0d1u0;1904dst_emc_entry->trained_dram_clktree_c0d1u1 = dst_emc_entry->current_dram_clktree_c0d1u1;1905dst_emc_entry->trained_dram_clktree_c1d0u0 = dst_emc_entry->current_dram_clktree_c1d0u0;1906dst_emc_entry->trained_dram_clktree_c1d0u1 = dst_emc_entry->current_dram_clktree_c1d0u1;1907dst_emc_entry->trained_dram_clktree_c1d1u0 = dst_emc_entry->current_dram_clktree_c1d1u0;1908dst_emc_entry->trained_dram_clktree_c1d1u1 = dst_emc_entry->current_dram_clktree_c1d1u1;1909}19101911return (u32)adelta;1912}19131914static u32 _minerva_periodic_compensation_handler(emc_table_t *src_emc_entry, emc_table_t *dst_emc_entry, u32 dram_dev_num, u32 channel1_enabled, enum comp_seq_t seq_type)1915{1916if (!dst_emc_entry->periodic_training)1917return 0;19181919u32 delay = 1000 * _actual_osc_clocks(src_emc_entry->run_clocks) / src_emc_entry->rate_khz + 2;19201921if (seq_type == DVFS_SEQUENCE)1922{1923if (src_emc_entry->periodic_training && dst_emc_entry->ptfv_list.ptfv_config_ctrl & 1)1924{1925u32 samples = dst_emc_entry->ptfv_list.ptfv_dvfs_samples;1926dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 * samples;1927dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 * samples;1928dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 * samples;1929dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 * samples;1930dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 * samples;1931dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 * samples;1932dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 * samples;1933dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 = src_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 * samples;1934}1935else1936{1937dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 = 0;1938dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 = 0;1939dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 = 0;1940dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 = 0;1941dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 = 0;1942dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 = 0;1943dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 = 0;1944dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 = 0;19451946for (u32 i = 0; i < dst_emc_entry->ptfv_list.ptfv_dvfs_samples; i++)1947{1948_start_periodic_compensation();1949_usleep(delay);1950_minerva_update_clock_tree_delay(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, DVFS_PT1);1951}1952}19531954return _minerva_update_clock_tree_delay(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, DVFS_UPDATE);1955}1956else if (seq_type == WRITE_TRAINING_SEQUENCE)1957{1958dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u0 = 0;1959dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d0u1 = 0;1960dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u0 = 0;1961dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d0u1 = 0;1962dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u0 = 0;1963dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c0d1u1 = 0;1964dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u0 = 0;1965dst_emc_entry->ptfv_list.ptfv_dqsosc_movavg_c1d1u1 = 0;19661967for (u32 i = 0; i < dst_emc_entry->ptfv_list.ptfv_write_samples; i++)1968{1969_start_periodic_compensation();1970_usleep(delay);1971_minerva_update_clock_tree_delay(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, TRAINING_PT1);1972}19731974return _minerva_update_clock_tree_delay(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, TRAINING_UPDATE);1975}1976else if (seq_type == PERIODIC_TRAINING_SEQUENCE)1977{1978_start_periodic_compensation();1979_usleep(delay);1980return _minerva_update_clock_tree_delay(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, PERIODIC_TRAINING_UPDATE);1981}19821983return seq_type;1984}19851986#define STORE_TRIM_VAL(chan, rank, reg, byte) \1987((mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank##rank##_##reg >> \1988EMC_PMACRO_OB_DDLL_LONG_DQ_BYTE##byte##_SHIFT) & 0x7FF) \1989+ \1990(((mtc_table_entry->trim_perch_regs.emc##chan##_data_brlshft_##rank >> \1991EMC_DATA_BRLSHFT_##rank##_RANK##rank##_BYTE##byte##_DATA_BRLSHFT_SHIFT) & 0x7) << 6)19921993static u32 _minerva_apply_periodic_compensation_trimmer(emc_table_t *mtc_table_entry, u32 trim_emc_reg_addr)1994{1995u32 trimmer = 0;1996s32 tree_delta[4] = {0};1997s32 tree_delta_taps[4] = {0};1998s32 new_trim[] = {1999// chan, rank, reg, byte.2000STORE_TRIM_VAL(0, 0, 0, 0),2001STORE_TRIM_VAL(0, 0, 0, 1),2002STORE_TRIM_VAL(0, 0, 1, 2),2003STORE_TRIM_VAL(0, 0, 1, 3),20042005STORE_TRIM_VAL(1, 0, 2, 4),2006STORE_TRIM_VAL(1, 0, 2, 5),2007STORE_TRIM_VAL(1, 0, 3, 6),2008STORE_TRIM_VAL(1, 0, 3, 7),20092010STORE_TRIM_VAL(0, 1, 0, 0),2011STORE_TRIM_VAL(0, 1, 0, 1),2012STORE_TRIM_VAL(0, 1, 1, 2),2013STORE_TRIM_VAL(0, 1, 1, 3),20142015STORE_TRIM_VAL(1, 1, 2, 4),2016STORE_TRIM_VAL(1, 1, 2, 5),2017STORE_TRIM_VAL(1, 1, 3, 6),2018STORE_TRIM_VAL(1, 1, 3, 7)2019};20202021u32 dst_rate_mhz = mtc_table_entry->rate_khz / 1000;20222023switch (trim_emc_reg_addr)2024{2025case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:2026case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:2027case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:2028case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:2029case EMC_DATA_BRLSHFT_0:2030tree_delta[0] = (mtc_table_entry->current_dram_clktree_c0d0u0 - mtc_table_entry->trained_dram_clktree_c0d0u0) * 128;2031tree_delta[1] = (mtc_table_entry->current_dram_clktree_c0d0u1 - mtc_table_entry->trained_dram_clktree_c0d0u1) * 128;2032tree_delta[2] = (mtc_table_entry->current_dram_clktree_c1d0u0 - mtc_table_entry->trained_dram_clktree_c1d0u0) * 128;2033tree_delta[3] = (mtc_table_entry->current_dram_clktree_c1d0u1 - mtc_table_entry->trained_dram_clktree_c1d0u1) * 128;2034tree_delta_taps[0] = (tree_delta[0] * (s32)dst_rate_mhz) / 1000000;2035tree_delta_taps[1] = (tree_delta[1] * (s32)dst_rate_mhz) / 1000000;2036tree_delta_taps[2] = (tree_delta[2] * (s32)dst_rate_mhz) / 1000000;2037tree_delta_taps[3] = (tree_delta[3] * (s32)dst_rate_mhz) / 1000000;2038for (u32 i = 0; i < 4; i++)2039{2040// Check if tap exceeds margins and apply it.2041if ((tree_delta_taps[i] > (s32)mtc_table_entry->tree_margin) || (tree_delta_taps[i] < (-1 * (s32)mtc_table_entry->tree_margin)))2042{2043new_trim[i * 2] += tree_delta_taps[i];2044new_trim[i * 2 + 1] += tree_delta_taps[i];2045}2046}2047if (trim_emc_reg_addr == EMC_DATA_BRLSHFT_0)2048{2049for (u32 i = 0; i < 8; i++)2050new_trim[i] /= 64;2051}2052else2053{2054for (u32 i = 0; i < 8; i++)2055new_trim[i] %= 64;2056}2057break;2058case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:2059case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:2060case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:2061case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:2062case EMC_DATA_BRLSHFT_1:2063tree_delta[0] = (mtc_table_entry->current_dram_clktree_c0d1u0 - mtc_table_entry->trained_dram_clktree_c0d1u0) * 128;2064tree_delta[1] = (mtc_table_entry->current_dram_clktree_c0d1u1 - mtc_table_entry->trained_dram_clktree_c0d1u1) * 128;2065tree_delta[2] = (mtc_table_entry->current_dram_clktree_c1d1u0 - mtc_table_entry->trained_dram_clktree_c1d1u0) * 128;2066tree_delta[3] = (mtc_table_entry->current_dram_clktree_c1d1u1 - mtc_table_entry->trained_dram_clktree_c1d1u1) * 128;2067tree_delta_taps[0] = (tree_delta[0] * (s32)dst_rate_mhz) / 1000000;2068tree_delta_taps[1] = (tree_delta[1] * (s32)dst_rate_mhz) / 1000000;2069tree_delta_taps[2] = (tree_delta[2] * (s32)dst_rate_mhz) / 1000000;2070tree_delta_taps[3] = (tree_delta[3] * (s32)dst_rate_mhz) / 1000000;2071for (u32 i = 0; i < 4; i++)2072{2073// Check if tap exceeds margins and apply it.2074if ((tree_delta_taps[i] > (s32)mtc_table_entry->tree_margin) || (tree_delta_taps[i] < (-1 * (s32)mtc_table_entry->tree_margin)))2075{2076new_trim[8 + i * 2] += tree_delta_taps[i];2077new_trim[8 + i * 2 + 1] += tree_delta_taps[i];2078}2079}2080if (trim_emc_reg_addr == EMC_DATA_BRLSHFT_1)2081{2082for (u32 i = 0; i < 8; i++)2083new_trim[i + 8] /= 64;2084}2085else2086{2087for (u32 i = 0; i < 8; i++)2088new_trim[i + 8] %= 64;2089}2090break;2091default:2092break;2093}20942095switch (trim_emc_reg_addr)2096{2097case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:2098trimmer = (new_trim[0] & 0x7FF) | ((new_trim[1] & 0x7FF) << 16);2099break;2100case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:2101trimmer = (new_trim[2] & 0x7FF) | ((new_trim[3] & 0x7FF) << 16);2102break;2103case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:2104trimmer = (new_trim[4] & 0x7FF) | ((new_trim[5] & 0x7FF) << 16);2105break;2106case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:2107trimmer = (new_trim[6] & 0x7FF) | ((new_trim[7] & 0x7FF) << 16);2108break;2109case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:2110trimmer = (new_trim[8] & 0x7FF) | ((new_trim[9] & 0x7FF) << 16);2111break;2112case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:2113trimmer = (new_trim[10] & 0x7FF) | ((new_trim[11] & 0x7FF) << 16);2114break;2115case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:2116trimmer = (new_trim[12] & 0x7FF) | ((new_trim[13] & 0x7FF) << 16);2117break;2118case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:2119trimmer = (new_trim[14] & 0x7FF) | ((new_trim[15] & 0x7FF) << 16);2120break;2121case EMC_DATA_BRLSHFT_0:2122trimmer = ((new_trim[0] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT)2123| ((new_trim[1] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT)2124| ((new_trim[2] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT)2125| ((new_trim[3] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT)2126| ((new_trim[4] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT)2127| ((new_trim[5] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT)2128| ((new_trim[6] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT)2129| ((new_trim[7] & 7) << EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT);2130break;2131case EMC_DATA_BRLSHFT_1:2132trimmer = ((new_trim[8] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT)2133| ((new_trim[9] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT)2134| ((new_trim[10] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT)2135| ((new_trim[11] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT)2136| ((new_trim[12] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT)2137| ((new_trim[13] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT)2138| ((new_trim[14] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT)2139| ((new_trim[15] & 7) << EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT);2140break;2141default:2142break;2143}21442145return trimmer;2146}21472148static bool _check_freq_changed(u32 dst_entry_rate_KHz, u32 dst_entry_clk_src_emc, u32 src_entry_rate_KHz, u32 src_entry_clk_src_emc)2149{2150u64 dst_div_clock;2151u64 src_div_clock;2152u32 src_end_div_clk_ratio;21532154u32 src_entry_emc_2X_clk_src = src_entry_clk_src_emc >> EMC_2X_CLK_SRC_SHIFT;2155u32 dst_entry_emc_2X_clk_src = dst_entry_clk_src_emc >> EMC_2X_CLK_SRC_SHIFT;2156u32 src_entry_emc_2X_clk_src_div = src_entry_clk_src_emc & 0xFF;2157u32 dst_entry_emc_2X_clk_src_div = dst_entry_clk_src_emc & 0xFF;2158u32 pll_post_divider = 0;21592160switch (CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC) >> EMC_2X_CLK_SRC_SHIFT)2161{2162case PLLM_OUT0:2163case PLLM_UD:2164pll_post_divider = (CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) >> 20) & 0x1F;2165break;2166case PLLMB_UD:2167case PLLMB_OUT0:2168pll_post_divider = (CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) >> 20) & 0x1F;2169break;2170default:2171break;2172}21732174// Hang if post div is wrong.2175if (pll_post_divider > 5)2176while (true)2177;21782179if (src_entry_emc_2X_clk_src <= PLLMB_UD)2180src_entry_emc_2X_clk_src_div = 0;2181if (dst_entry_emc_2X_clk_src <= PLLMB_UD)2182dst_entry_emc_2X_clk_src_div = 0;21832184if (dst_entry_emc_2X_clk_src != src_entry_emc_2X_clk_src2185&& (dst_entry_emc_2X_clk_src & 0xFFFFFFFB || src_entry_emc_2X_clk_src & 0xFFFFFFFB))2186return true;21872188dst_div_clock = dst_entry_rate_KHz * (pll_post_divider + 1)2189* ((dst_entry_emc_2X_clk_src_div >> 1) * 10 + (dst_entry_emc_2X_clk_src_div & 1) * 5 + 10) / 10; // Accounting for 7.1 div.2190src_div_clock = src_entry_rate_KHz * (pll_post_divider + 1)2191* ((src_entry_emc_2X_clk_src_div >> 1) * 10 + (src_entry_emc_2X_clk_src_div & 1) * 5 + 10) / 10; // Accounting for 7.1 div.21922193src_end_div_clk_ratio = (src_div_clock * 1000) / dst_div_clock;21942195if (src_end_div_clk_ratio > 1010 || src_end_div_clk_ratio < 990)2196return true;2197else2198return false;2199}22002201static void _save_train_results(emc_table_t *mtc_table_entry, u32 needs_training, u32 dram_dev_num, bool channel1_enabled)2202{2203bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA);2204bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF);2205bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE);2206bool needs_quse_vref_training = !!(needs_training & NEEDS_TRAINING_QUSE_VREF);2207bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR);2208bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF);2209bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD);2210bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF);2211bool needs_training_in_self_refresh = !!(needs_training & NEEDS_TRAINING_IN_SELF_REFRESH);22122213if (needs_ca_training)2214{2215mtc_table_entry->trim_perch_regs.emc_cmd_brlshft_0 = EMC_CH0(EMC_CMD_BRLSHFT_0);2216mtc_table_entry->trim_perch_regs.emc_cmd_brlshft_1 = channel1_enabled ? EMC_CH1(EMC_CMD_BRLSHFT_1) : 0;2217mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_4 = EMC_CH0(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4);2218mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_5 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5) : 0;22192220if (needs_training_in_self_refresh)2221{2222mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd0_0 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0);2223mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd0_1 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1);2224mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd0_2 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2);2225mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd1_0 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0);2226mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd1_1 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1);2227mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd1_2 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2);2228mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd2_0 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0);2229mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd2_1 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1);2230mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd2_2 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2);2231mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd3_0 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0);2232mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd3_1 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1);2233mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_cmd3_2 = EMC(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2);2234}2235}22362237if (needs_ca_vref_training)2238{2239mtc_table_entry->burst_reg_per_ch.emc0_mrw10 = (EMC_CH0(EMC_TRAINING_OPT_CA_VREF) & 0xFFFF) | 0x880C0000;2240mtc_table_entry->burst_reg_per_ch.emc1_mrw10 = (channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_CA_VREF) & 0xFFFF : 0) | 0x880C0000;22412242u32 mrw11_dev_selectn;2243if (dram_dev_num == TWO_RANK)2244mrw11_dev_selectn = 0x480C0000;2245else2246mrw11_dev_selectn = 0xC80C0000;22472248mtc_table_entry->burst_reg_per_ch.emc0_mrw11 =2249((EMC_CH0(EMC_TRAINING_OPT_CA_VREF) >> 16) & 0xFF)2250| (EMC_CH0(EMC_TRAINING_OPT_CA_VREF) >> 24 << 8)2251| (mrw11_dev_selectn & 0xFFFFFF00);22522253mtc_table_entry->burst_reg_per_ch.emc1_mrw11 =2254(((channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_CA_VREF) : 0) >> 16) & 0xFF)2255| ((channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_CA_VREF) : 0) >> 24 << 8)2256| (mrw11_dev_selectn & 0xFFFFFF00);2257}22582259if (needs_quse_training || needs_rd_training)2260{2261mtc_table_entry->trim_perch_regs.emc_quse_brlshft_0 = EMC_CH0(EMC_QUSE_BRLSHFT_0);2262mtc_table_entry->trim_perch_regs.emc_quse_brlshft_1 = channel1_enabled ? EMC_CH1(EMC_QUSE_BRLSHFT_1) : 0;22632264mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank0_0 = EMC_CH0(EMC_PMACRO_QUSE_DDLL_RANK0_0);2265mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank0_1 = EMC_CH0(EMC_PMACRO_QUSE_DDLL_RANK0_1);2266mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank0_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_QUSE_DDLL_RANK0_2) : 0;2267mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank0_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_QUSE_DDLL_RANK0_3) : 0;22682269if (dram_dev_num == TWO_RANK)2270{2271mtc_table_entry->trim_perch_regs.emc_quse_brlshft_2 = EMC_CH0(EMC_QUSE_BRLSHFT_2);2272mtc_table_entry->trim_perch_regs.emc_quse_brlshft_3 = channel1_enabled ? EMC_CH1(EMC_QUSE_BRLSHFT_3) : 0;22732274mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank1_0 = EMC_CH0(EMC_PMACRO_QUSE_DDLL_RANK1_0);2275mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank1_1 = EMC_CH0(EMC_PMACRO_QUSE_DDLL_RANK1_1);2276mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank1_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_QUSE_DDLL_RANK1_2) : 0;2277mtc_table_entry->trim_regs.emc_pmacro_quse_ddll_rank1_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_QUSE_DDLL_RANK1_3) : 0;2278}2279}22802281if (needs_quse_vref_training)2282{2283if (dram_dev_num == TWO_RANK)2284{2285u32 emc0_opt_dqs_array[4] = {0};2286u32 emc1_opt_dqs_array[4] = {0};2287u32 emc1_training_opt_dqs_ib_vref_rank0_val = channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) : 0;2288u32 emc1_training_opt_dqs_ib_vref_rank1_val = channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) : 0;22892290for (u32 i = 0; i < 4; i++)2291{2292emc0_opt_dqs_array[i] = (EMC_CH0(EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) >> (8 * i)) & 0xFF;2293emc1_opt_dqs_array[i] = (emc1_training_opt_dqs_ib_vref_rank0_val >> (8 * i)) & 0xFF;2294}22952296u32 ib_vref_dqs_0 = 0;2297u32 ib_vref_dqs_1 = 0;2298for (u32 i = 0; i < 4; i++)2299{2300ib_vref_dqs_0 |= (emc0_opt_dqs_array[i] + ((EMC_CH0(EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) >> (8 * i)) & 0xFF)) >> 1 << (8 * i);2301ib_vref_dqs_1 |= (emc1_opt_dqs_array[i] + ((emc1_training_opt_dqs_ib_vref_rank1_val >> (8 * i)) & 0xFF)) >> 1 << (8 * i);2302}23032304mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dqs_0 = ib_vref_dqs_0;2305mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dqs_1 = ib_vref_dqs_1;2306}2307else2308{2309mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dqs_0 = EMC(EMC_PMACRO_IB_VREF_DQS_0);2310mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dqs_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_VREF_DQS_1) : 0;2311}2312}23132314if (needs_rd_training)2315{2316mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank0_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0);2317mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank0_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1);2318mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank0_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2) : 0;2319mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank0_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3) : 0;23202321if (dram_dev_num == TWO_RANK)2322{2323mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank1_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0);2324mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank1_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1);2325mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank1_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2) : 0;2326mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_long_dqs_rank1_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3) : 0;2327}23282329if (needs_training_in_self_refresh)2330{2331mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte0_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0);2332mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte0_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1);2333mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte0_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2);2334mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte1_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0);2335mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte1_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1);2336mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte1_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2);2337mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte2_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0);2338mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte2_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1);2339mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte2_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2);2340mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte3_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0);2341mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte3_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1);2342mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte3_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2);2343mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte4_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;2344mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte4_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;2345mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte4_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;2346mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte5_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;2347mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte5_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;2348mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte5_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;2349mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte6_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;2350mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte6_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;2351mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte6_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;2352mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte7_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;2353mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte7_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;2354mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank0_byte7_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;23552356if (dram_dev_num == TWO_RANK)2357{2358mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte0_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0);2359mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte0_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1);2360mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte0_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2);2361mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte1_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0);2362mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte1_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1);2363mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte1_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2);2364mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte2_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0);2365mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte2_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1);2366mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte2_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2);2367mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte3_0 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0);2368mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte3_1 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1);2369mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte3_2 = EMC_CH0(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2);2370mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte4_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;2371mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte4_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;2372mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte4_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;2373mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte5_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;2374mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte5_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;2375mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte5_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;2376mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte6_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;2377mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte6_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;2378mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte6_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;2379mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte7_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;2380mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte7_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;2381mtc_table_entry->trim_regs.emc_pmacro_ib_ddll_short_dq_rank1_byte7_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;2382}2383}23842385if (needs_rd_vref_training)2386{2387char ib_vref_dq_byte0_icr = (EMC(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[0] & 0x7F);2388if (mtc_table_entry->save_restore_mod_regs[0] & 0x80000000) // < 0 check.2389ib_vref_dq_byte0_icr = (EMC(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[0] & 0x7F);23902391char ib_vref_dq_byte1_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[1] & 0x7F);2392if (mtc_table_entry->save_restore_mod_regs[1] & 0x80000000)2393ib_vref_dq_byte1_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[1] & 0x7F);23942395char ib_vref_dq_byte2_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[2] & 0x7F);2396if (mtc_table_entry->save_restore_mod_regs[2] & 0x80000000)2397ib_vref_dq_byte2_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[2] & 0x7F);23982399char ib_vref_dq_byte3_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[3] & 0x7F);2400if (mtc_table_entry->save_restore_mod_regs[3] & 0x80000000)2401ib_vref_dq_byte3_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[3] & 0x7F);24022403mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dq_0 =2404((ib_vref_dq_byte0_icr & 0x7F)2405| (ib_vref_dq_byte1_icr & 0x7F) << 8)2406| ((ib_vref_dq_byte2_icr & 0x7F) << 16)2407| ((ib_vref_dq_byte3_icr & 0x7F) << 24);24082409char ib_vref_dq_byte4_icr = (EMC(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[4] & 0x7F);2410if (mtc_table_entry->save_restore_mod_regs[4] & 0x80000000)2411ib_vref_dq_byte4_icr = (EMC(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[4] & 0x7F);24122413char ib_vref_dq_byte5_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[5] & 0x7F);2414if (mtc_table_entry->save_restore_mod_regs[5] & 0x80000000)2415ib_vref_dq_byte5_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[5] & 0x7F);24162417char ib_vref_dq_byte6_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[6] & 0x7F);2418if (mtc_table_entry->save_restore_mod_regs[6] & 0x80000000)2419ib_vref_dq_byte6_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[6] & 0x7F);24202421char ib_vref_dq_byte7_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) + (mtc_table_entry->save_restore_mod_regs[7] & 0x7F);2422if (mtc_table_entry->save_restore_mod_regs[7] & 0x80000000)2423ib_vref_dq_byte7_icr = ((EMC(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) - (mtc_table_entry->save_restore_mod_regs[7] & 0x7F);24242425mtc_table_entry->trim_regs.emc_pmacro_ib_vref_dq_1 =2426((ib_vref_dq_byte4_icr & 0x7F)2427| (ib_vref_dq_byte5_icr & 0x7F) << 8)2428| ((ib_vref_dq_byte6_icr & 0x7F) << 16)2429| ((ib_vref_dq_byte7_icr & 0x7F) << 24);2430}2431}24322433if (needs_wr_training)2434{2435mtc_table_entry->trim_perch_regs.emc0_data_brlshft_0 = EMC_CH0(EMC_DATA_BRLSHFT_0);2436mtc_table_entry->trim_perch_regs.emc1_data_brlshft_0 = channel1_enabled ? EMC_CH1(EMC_DATA_BRLSHFT_0) : 0;24372438if (dram_dev_num == TWO_RANK)2439{2440mtc_table_entry->trim_perch_regs.emc0_data_brlshft_1 = EMC_CH0(EMC_DATA_BRLSHFT_1);2441mtc_table_entry->trim_perch_regs.emc1_data_brlshft_1 = channel1_enabled ? EMC_CH1(EMC_DATA_BRLSHFT_1) : 0;2442}24432444mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0);2445mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1);2446mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2) : 0;2447mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank0_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3) : 0;24482449if (dram_dev_num == TWO_RANK)2450{2451mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank1_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0);2452mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank1_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1);2453mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank1_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2) : 0;2454mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_long_dq_rank1_3 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3) : 0;2455}24562457if (needs_training_in_self_refresh)2458{2459mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte0_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0);2460mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte0_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1);2461mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte0_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2);2462mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte1_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0);2463mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte1_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1);2464mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte1_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2);2465mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte2_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0);2466mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte2_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1);2467mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte2_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2);2468mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte3_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0);2469mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte3_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1);2470mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte3_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2);2471mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte4_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;2472mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte4_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;2473mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte4_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;2474mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte5_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;2475mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte5_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;2476mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte5_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;2477mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte6_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;2478mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte6_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;2479mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte6_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;2480mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte7_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;2481mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte7_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;2482mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank0_byte7_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;24832484if (dram_dev_num == TWO_RANK)2485{2486mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte0_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0);2487mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte0_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1);2488mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte0_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2);2489mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte1_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0);2490mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte1_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1);2491mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte1_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2);2492mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte2_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0);2493mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte2_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1);2494mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte2_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2);2495mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte3_0 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0);2496mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte3_1 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1);2497mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte3_2 = EMC_CH0(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2);2498mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte4_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;2499mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte4_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;2500mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte4_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;2501mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte5_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;2502mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte5_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;2503mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte5_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;2504mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte6_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;2505mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte6_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;2506mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte6_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;2507mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte7_0 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;2508mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte7_1 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;2509mtc_table_entry->trim_regs.emc_pmacro_ob_ddll_short_dq_rank1_byte7_2 = channel1_enabled ? EMC_CH1(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;2510}2511}25122513if (needs_wr_vref_training) // mode 12/13 (MRW).2514{2515u32 emc1_ranks_sub_partitions = 0;2516emc1_ranks_sub_partitions = channel1_enabled ? EMC_CH1(EMC_TRAINING_OPT_DQ_OB_VREF) : 0;25172518u8 emc0_ib_vref_dq_byte8_modded_plus = mtc_table_entry->save_restore_mod_regs[8] + EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF);2519if (mtc_table_entry->save_restore_mod_regs[8] & 0x80000000) // < 0 check.2520emc0_ib_vref_dq_byte8_modded_plus = EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) - mtc_table_entry->save_restore_mod_regs[8];25212522u8 emc0_mrw12_op_sp1 = ((EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) + mtc_table_entry->save_restore_mod_regs[9];2523if (mtc_table_entry->save_restore_mod_regs[9] & 0x80000000)2524emc0_mrw12_op_sp1 = ((EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) - mtc_table_entry->save_restore_mod_regs[9];25252526u8 emc0_mrw13_op_sp0 = ((EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) + mtc_table_entry->save_restore_mod_regs[8];2527if (mtc_table_entry->save_restore_mod_regs[8] & 0x80000000)2528emc0_mrw13_op_sp0 = ((EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) - mtc_table_entry->save_restore_mod_regs[8];25292530u8 emc0_ib_vref_dq_byte9_modded_a_plus = mtc_table_entry->save_restore_mod_regs[9] + (EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) >> 24);2531if (mtc_table_entry->save_restore_mod_regs[9] & 0x80000000)2532emc0_ib_vref_dq_byte9_modded_a_plus = (EMC_CH0(EMC_TRAINING_OPT_DQ_OB_VREF) >> 24) - (u8)mtc_table_entry->save_restore_mod_regs[9];25332534u8 emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions + mtc_table_entry->save_restore_mod_regs[10];2535if (mtc_table_entry->save_restore_mod_regs[10] & 0x80000000)2536emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions - mtc_table_entry->save_restore_mod_regs[10];25372538u8 emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) + mtc_table_entry->save_restore_mod_regs[11];2539if (mtc_table_entry->save_restore_mod_regs[11] & 0x80000000)2540emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) - mtc_table_entry->save_restore_mod_regs[11];25412542u8 emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) + mtc_table_entry->save_restore_mod_regs[10];2543if (mtc_table_entry->save_restore_mod_regs[10] & 0x80000000)2544emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) - mtc_table_entry->save_restore_mod_regs[10];25452546u8 emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) + mtc_table_entry->save_restore_mod_regs[11];2547if (mtc_table_entry->save_restore_mod_regs[11] & 0x80000000)2548emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) - mtc_table_entry->save_restore_mod_regs[11];25492550u32 mr13_dev_ext_cnt_sp_addr = 0xC80E0000;2551if (dram_dev_num == TWO_RANK)2552mr13_dev_ext_cnt_sp_addr = 0x480E0000;25532554mtc_table_entry->burst_reg_per_ch.emc1_mrw12 = (u8)emc0_ib_vref_dq_byte10_modded_plus | 0x880E0000 | (emc0_ib_vref_dq_byte11_modded_plus << 8);2555mtc_table_entry->burst_reg_per_ch.emc0_mrw12 = emc0_ib_vref_dq_byte8_modded_plus | 0x880E0000 | (emc0_mrw12_op_sp1 << 8);2556mtc_table_entry->burst_reg_per_ch.emc0_mrw13 = emc0_ib_vref_dq_byte9_modded_a_plus << 8 | emc0_mrw13_op_sp0 | mr13_dev_ext_cnt_sp_addr;2557mtc_table_entry->burst_reg_per_ch.emc1_mrw13 = (emc1_mrw13_op_sp1 << 8) | emc1_mrw13_op_sp0 | mr13_dev_ext_cnt_sp_addr;2558}2559}2560}25612562static u32 _minerva_change_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_entry, u32 needs_training, u32 selected_clk_src_emc)2563{2564u32 emc_dbg_o;2565u32 emc_pin_o;2566u32 emc_cfg_pipe_clk_o;2567u32 emc_sel_dpd_ctrl;2568u32 emc_cfg;2569u32 emc_dbg_val;2570u32 emc_zq_cal = 0;2571u32 ramp_up_wait;2572u32 ramp_down_wait;2573u32 bg_regulator_mode_change;2574u32 mr13_flip_fspop;2575u32 mr13_flip_fspwr;2576u32 mr13_catr_enable;25772578/* needs_training flags */2579/*2580| bit | Description |2581|-----|----------------------------------|2582| 0 | Needs CA training |2583| 1 | Needs CA_VREF training |2584| 2 | Needs QUSE training |2585| 3 | Needs QUSE_VREF training |2586| 4 | Needs WR training |2587| 5 | Needs WR_VREF training |2588| 6 | Needs RD training |2589| 7 | Needs RD_VREF training |2590| 8 | Needs SWAP_RANK training |2591| 9 | Needs IN_SELF_REFRESH training |2592*/25932594bool compensate_trimmer_applicable = false;2595bool in_training = !!(needs_training & NEEDS_TRAINING);25962597bool needs_ca_combo_training = !!(needs_training & NEEDS_TRAINING_CA_COMBO);2598bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA);2599bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF);2600bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE);2601bool needs_quse_vref_training = !!(needs_training & NEEDS_TRAINING_QUSE_VREF);2602bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR);2603bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF);2604bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD);2605bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF);2606bool needs_swap_rank_training = !!(needs_training & NEEDS_TRAINING_SWAP_RANK);26072608bool zcal_resistor_shared = (src_emc_entry->burst_regs.emc_zcal_wait_cnt >> 31) & 1;2609bool enable_bg_regulator = (dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 1) ^ 1;2610bool channel1_enabled = (src_emc_entry->burst_regs.emc_fbio_cfg7 >> 2) & 1;2611u32 dram_type = EMC(EMC_FBIO_CFG5) & 3;2612u32 dram_dev_num = (MC(MC_EMEM_ADR_CFG) & 1) + 1;26132614u32 src_clock_period = 1000000000 / src_emc_entry->rate_khz; // In picoseconds.2615u32 dst_clock_period = 1000000000 / dst_emc_entry->rate_khz; // In picoseconds.26162617// Get current FSP op/write value.2618bool enable_fsp_opwr = !(EMC(EMC_MRW3) & 0xC0);26192620if (dram_type != DRAM_TYPE_LPDDR4)2621{2622EPRINTF("MTC Error: DRAM is not LPDDR4");2623return 5;2624}26252626u32 tFC_lpddr4 = dst_emc_entry->dram_timings.t_fc_lpddr4 * 1000;2627u32 tZQCAL_lpddr4 = 1000000;2628if (dst_clock_period <= 2000)2629tZQCAL_lpddr4 -= tFC_lpddr4;2630s32 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 / dst_clock_period;26312632(void)EMC(EMC_CFG);2633(void)EMC(EMC_AUTO_CAL_CONFIG);26342635// Step 1 - Pre DVFS SW sequence.2636EPRINTF("Step 1");2637emc_dbg_o = EMC(EMC_DBG);2638emc_pin_o = EMC(EMC_PIN);2639emc_cfg = dst_emc_entry->burst_regs.emc_cfg & 0xFFFFFFF;2640emc_sel_dpd_ctrl = dst_emc_entry->emc_sel_dpd_ctrl & 0xFFFFFEC3;2641emc_cfg_pipe_clk_o = EMC(EMC_CFG_PIPE_CLK);2642_digital_dll_disable();26432644// Step 1.2 - Disable AUTOCAL temporarily.2645EPRINTF("Step 1.2");2646EMC(EMC_AUTO_CAL_CONFIG) = (dst_emc_entry->emc_auto_cal_config & 0x7FFFF9FF) | 0x600;2647(void)EMC(EMC_AUTO_CAL_CONFIG);26482649// Step 1.3 - Disable other power features.2650EPRINTF("Step 1.3");2651EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;2652EMC(EMC_CFG) = emc_cfg;2653EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl;2654EMC(EMC_DBG) = emc_dbg_o;26552656if (!in_training && dst_emc_entry->periodic_training)2657{2658if (dram_dev_num == TWO_RANK)2659{2660_wait_emc_status(EMC_EMC_STATUS, IN_POWERDOWN_BOTH_MASK, false, EMC_CHANNEL0);2661if (channel1_enabled)2662_wait_emc_status(EMC_EMC_STATUS, IN_POWERDOWN_BOTH_MASK, false, EMC_CHANNEL1);2663}2664else2665{2666_wait_emc_status(EMC_EMC_STATUS, IN_POWERDOWN_1DEV_MASK, false, EMC_CHANNEL0);2667if (channel1_enabled)2668_wait_emc_status(EMC_EMC_STATUS, IN_POWERDOWN_1DEV_MASK, false, EMC_CHANNEL1);2669}26702671_wait_emc_status(EMC_EMC_STATUS, IN_SELF_REFRESH_MASK, false, EMC_CHANNEL0);2672if (channel1_enabled)2673_wait_emc_status(EMC_EMC_STATUS, IN_SELF_REFRESH_MASK, false, EMC_CHANNEL1);26742675// Reset clock tree delays.2676dst_emc_entry->current_dram_clktree_c0d0u0 = dst_emc_entry->trained_dram_clktree_c0d0u0;2677dst_emc_entry->current_dram_clktree_c0d0u1 = dst_emc_entry->trained_dram_clktree_c0d0u1;2678dst_emc_entry->current_dram_clktree_c0d1u0 = dst_emc_entry->trained_dram_clktree_c0d1u0;2679dst_emc_entry->current_dram_clktree_c0d1u1 = dst_emc_entry->trained_dram_clktree_c0d1u1;2680dst_emc_entry->current_dram_clktree_c1d0u0 = dst_emc_entry->trained_dram_clktree_c1d0u0;2681dst_emc_entry->current_dram_clktree_c1d0u1 = dst_emc_entry->trained_dram_clktree_c1d0u1;2682dst_emc_entry->current_dram_clktree_c1d1u0 = dst_emc_entry->trained_dram_clktree_c1d1u0;2683dst_emc_entry->current_dram_clktree_c1d1u1 = dst_emc_entry->trained_dram_clktree_c1d1u1;26842685u32 adelta = _minerva_periodic_compensation_handler(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, DVFS_SEQUENCE);26862687if (((dst_emc_entry->rate_khz / 1000) * 128) * adelta / 1000000 > dst_emc_entry->tree_margin)2688compensate_trimmer_applicable = true;2689}26902691EMC(EMC_INTSTATUS) = CLKCHANGE_COMPLETE_INT;2692EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;2693EMC(EMC_CFG) = emc_cfg;2694EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl;2695EMC(EMC_CFG_PIPE_CLK) = emc_cfg_pipe_clk_o | 1; // CLK_ALWAYS_ON.2696EMC(EMC_FDPD_CTRL_CMD_NO_RAMP) = dst_emc_entry->emc_fdpd_ctrl_cmd_no_ramp & 0xFFFFFFFE;26972698bg_regulator_mode_change = src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 ^ dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0;2699bg_regulator_mode_change = (bg_regulator_mode_change | (bg_regulator_mode_change >> 2)) & 1;27002701if (bg_regulator_mode_change)2702{2703EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;2704if (enable_bg_regulator)2705EMC(EMC_PMACRO_BG_BIAS_CTRL_0) = src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFE;2706else2707EMC(EMC_PMACRO_BG_BIAS_CTRL_0) = src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFB;2708}27092710// Check if we need to turn on VREF generator.2711if ((!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100) && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100)) ||2712(!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1) && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1)))2713{2714EMC(EMC_PMACRO_DATA_PAD_TX_CTRL) =2715(((dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1) | (src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0xFFFFFFFE)) & 0xFFFFFEFF)2716| (((dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl >> 8) & 0x1) << 8);2717}27182719_usleep(1);27202721EMC(EMC_DBG) = emc_dbg_o;27222723// Step 2 - Prelock the DLL.2724EPRINTF("Step 2");2725if (dst_emc_entry->burst_regs.emc_cfg_dig_dll & 1)2726_digital_dll_prelock(dst_emc_entry, in_training, selected_clk_src_emc); // Prelock enabled for target frequency.2727else2728{2729_change_dll_src(dst_emc_entry, selected_clk_src_emc);2730_digital_dll_disable(); // Disabling DLL for target frequency.2731}27322733// Step 3 - Prepare autocal for the clock change.2734EPRINTF("Step 3");2735EMC(EMC_AUTO_CAL_CONFIG) = (dst_emc_entry->emc_auto_cal_config & 0x7FFFF9FF) | 0x600;2736EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;2737EMC(EMC_AUTO_CAL_CONFIG2) = dst_emc_entry->emc_auto_cal_config2;2738EMC(EMC_AUTO_CAL_CONFIG3) = dst_emc_entry->emc_auto_cal_config3;2739EMC(EMC_AUTO_CAL_CONFIG4) = dst_emc_entry->emc_auto_cal_config4;2740EMC(EMC_AUTO_CAL_CONFIG5) = dst_emc_entry->emc_auto_cal_config5;2741EMC(EMC_AUTO_CAL_CONFIG6) = dst_emc_entry->emc_auto_cal_config6;2742EMC(EMC_AUTO_CAL_CONFIG7) = dst_emc_entry->emc_auto_cal_config7;2743EMC(EMC_AUTO_CAL_CONFIG8) = dst_emc_entry->emc_auto_cal_config8;2744EMC(EMC_DBG) = emc_dbg_o;2745EMC(EMC_AUTO_CAL_CONFIG) = (dst_emc_entry->emc_auto_cal_config & 0x7FFFF9FE) | 0x601;27462747// Step 4 - Update EMC_CFG.2748EPRINTF("Step 4");2749if (src_clock_period <= 50000)2750EMC(EMC_CFG_2) = dst_emc_entry->emc_cfg_2;2751else2752_ccfifo_write(EMC_SELF_REF, 1, 0);27532754// Step 5 - Prepare reference variables for ZQCAL regs.2755EPRINTF("Step 5");2756// u32 zq_wait_long = 0;2757// u32 zq_wait_short = 0;27582759// zq_wait_long = _fceil(1000.0f / dst_clock_period);27602761// Step 7 - Bug 200024907 - Patch RP R2P.2762EPRINTF("Step 7");2763if (needs_ca_combo_training && dram_dev_num == TWO_RANK)2764EMC(EMC_PIN) = 0x107;27652766u32 R2P_war = 0;2767u32 TRPab_war = 0;2768u32 RP_war = 0;2769u32 W2P_war = 0;27702771u32 nRTP = 8; // <= 1066MHz.2772if ( src_clock_period < 1000000 / 266 // 266MHz - 3759.39 ps.2773&& src_clock_period < 1000000 / 533 // 533MHz - 1876.17 ps.2774&& src_clock_period < 1000000 / 800 // 800MHz - 1250.00 ps.2775&& src_clock_period < 1000000 / 1066) // 1066MHz - 938.09 ps.2776nRTP = 10; // 1067MHz < x <= 1333MHz.2777if (src_clock_period < 1000000 / 1333) // 1333MHz - 750.19 ps.2778nRTP = 12; // 1333MHz < x <= 1600MHz.2779if (src_clock_period < 1000000 / 1600) // 1600MHz - 625.00 ps.2780nRTP = 14; // 1600MHz < x <= 1866MHz.2781if (src_clock_period < 1000000 / 1866) // 1866MHz - 535.91 ps.2782nRTP = 16; // > 1866MHz27832784u32 tRPST = (src_emc_entry->emc_mrw >> 7) & 1;27852786u32 deltaTWATM = _div_o3(7500, src_clock_period);2787if (deltaTWATM < 8)2788deltaTWATM = 8;27892790u32 tRTM = src_emc_entry->dram_timings.rl + _div_o3(3600, src_clock_period) + deltaTWATM + tRPST + nRTP + 1;27912792if (tRTM <= src_emc_entry->burst_regs.emc_rp + src_emc_entry->burst_regs.emc_r2p)2793{2794TRPab_war = src_emc_entry->burst_regs.emc_trpab;2795R2P_war = src_emc_entry->burst_regs.emc_r2p;2796RP_war = src_emc_entry->burst_regs.emc_rp;2797}2798else2799{2800R2P_war = tRTM - src_emc_entry->burst_regs.emc_rp;2801TRPab_war = src_emc_entry->burst_regs.emc_trpab;2802RP_war = src_emc_entry->burst_regs.emc_rp;2803if (R2P_war > 63)2804{2805RP_war = tRTM - 63;2806R2P_war = 63;2807if (src_emc_entry->burst_regs.emc_trpab < tRTM - 63)2808TRPab_war = tRTM - 63;2809else2810TRPab_war = src_emc_entry->burst_regs.emc_trpab;2811}2812}28132814if (RP_war >= deltaTWATM)2815W2P_war = src_emc_entry->burst_regs.emc_w2p;2816else2817{2818u32 W2P_war_temp = deltaTWATM + src_emc_entry->burst_regs.emc_w2p;2819W2P_war = W2P_war_temp - RP_war;2820if (W2P_war > 63)2821{2822RP_war = W2P_war_temp - 63;2823W2P_war = 63;2824if (TRPab_war < RP_war)2825TRPab_war = RP_war;2826}2827}28282829if (src_emc_entry->burst_regs.emc_w2p != W2P_war ||2830src_emc_entry->burst_regs.emc_rp != RP_war ||2831src_emc_entry->burst_regs.emc_r2p != R2P_war ||2832src_emc_entry->burst_regs.emc_trpab != TRPab_war)2833{2834EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;2835EMC(EMC_RP) = RP_war;2836EMC(EMC_R2P) = R2P_war;2837EMC(EMC_W2P) = W2P_war;2838EMC(EMC_TRPAB) = TRPab_war;2839EMC(EMC_DBG) = emc_dbg_o;2840(void)EMC(EMC_TRPAB);2841_usleep(1);2842}28432844// Step 7.2 - Program FSP reference registers and send MRWs to new FSPWR.2845EPRINTF("Step 7.2");2846if (enable_fsp_opwr)2847{2848mr13_flip_fspop = dst_emc_entry->emc_mrw3 | 0xC0;2849mr13_flip_fspwr = (dst_emc_entry->emc_mrw3 & 0xFFFFFF3F) | 0x40;2850}2851else2852{2853mr13_flip_fspop = dst_emc_entry->emc_mrw3 & 0xFFFFFF3F;2854mr13_flip_fspwr = mr13_flip_fspop | 0x80;2855}28562857if (dram_dev_num == TWO_RANK)2858{2859if (needs_swap_rank_training)2860mr13_catr_enable = (mr13_flip_fspwr & 0x3FFFFFFF) | 0x40000001;2861else2862mr13_catr_enable = (mr13_flip_fspwr & 0x3FFFFFFF) | 0x80000001;28632864if (needs_ca_combo_training)2865{2866if (needs_swap_rank_training)2867mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x80000000;2868else2869mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x40000000;2870}2871}2872else2873mr13_catr_enable = mr13_flip_fspwr | 1;28742875EMC(EMC_MRW3) = mr13_flip_fspwr;2876EMC(EMC_MRW) = dst_emc_entry->emc_mrw;2877EMC(EMC_MRW2) = dst_emc_entry->emc_mrw2;28782879// Step 8 - Program the shadow registers.2880EPRINTF("Step 8");2881// Writing burst_regs.2882u32 reg_addr = 0;2883u32 reg_val = 0;2884burst_regs_table_t *dst_burst_regs = (burst_regs_table_t *)&dst_emc_entry->burst_regs;2885u32 *burst_val_table = dst_burst_regs->burst_regs;28862887if (in_training)2888{2889if (needs_ca_combo_training)2890burst_val_table = dst_burst_regs->shadow_regs_ca_train;2891else if (needs_training & NEEDS_TRAINING_QUSE_COMBO)2892burst_val_table = dst_burst_regs->shadow_regs_quse_train;2893else if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO))2894burst_val_table = dst_burst_regs->shadow_regs_rdwr_train;2895else2896burst_val_table = NULL;2897}28982899for (u32 i = 0; burst_val_table && i < dst_emc_entry->num_burst; i++)2900{2901reg_addr = burst_regs_emc_addr_table[i];2902reg_val = burst_val_table[i];29032904switch (reg_addr)2905{2906case EMC_CFG:2907reg_val &= 0xFFFFFFF;2908break;2909case EMC_ZCAL_INTERVAL:2910reg_val = 0;2911break;2912case EMC_PMACRO_BRICK_CTRL_RFU1:2913reg_val &= 0xF800F800;2914break;2915case EMC_PMACRO_CMD_PAD_TX_CTRL:2916reg_val = (reg_val & 0xFAFEFDFD) | BIT(26); // CMD_CMD_TX_DRVFORCEON.2917break;2918case EMC_PMACRO_DATA_PAD_TX_CTRL:2919reg_val &= 0xFEFEFDFD;2920break;2921case EMC_PMACRO_COMMON_PAD_TX_CTRL:2922reg_val &= 0xFFFFFFF0;2923break;2924case EMC_PMACRO_AUTOCAL_CFG_COMMON:2925reg_val |= BIT(16); // E_CAL_BYPASS_DVFS.2926break;2927case EMC_TRAINING_CTRL:2928reg_val |= needs_swap_rank_training << 14;2929break;2930}29312932EMC(reg_addr) = reg_val;2933}29342935if (in_training)2936EMC(EMC_MRW) = (src_emc_entry->run_clocks & 0xFF) | 0x170000;2937else2938EMC(EMC_MRW) = (dst_emc_entry->run_clocks & 0xFF) | 0x170000;29392940// Writing burst_regs_per_ch.2941for (u32 i = 0; dst_emc_entry->num_burst_per_ch > i; i++)2942{2943reg_addr = burst_reg_per_ch_emc01_addr_table[i];2944if (reg_addr && (channel1_enabled || ((reg_addr - 0x4000) > 0xFFF)))2945{2946EMC(reg_addr) = dst_burst_regs->burst_reg_per_ch[i];2947}2948}29492950// Writing vref_regs.2951trim_regs_table_t *trim_regs_table = (trim_regs_table_t *)&dst_emc_entry->trim_regs;2952for (u32 i = 0; dst_emc_entry->vref_num > i; i++)2953{2954reg_addr = vref_perch_regs_emc01_addr_table[i];2955if (reg_addr && (channel1_enabled || (reg_addr - 0x4000) > 0xFFF))2956EMC(reg_addr) = trim_regs_table->vref_perch_regs[i];2957}29582959// Writing training mod regs.2960if (in_training)2961{2962for (u32 i = 0; dst_emc_entry->training_mod_num > i; i++)2963{2964reg_addr = training_mod_regs_emc01_addr_table[i];2965if (reg_addr && (channel1_enabled || (reg_addr - 0x4000) > 0xFFF))2966EMC(reg_addr) = dst_emc_entry->training_mod_regs[i];2967}2968}29692970// Writing trim_regs2971for (u32 i = 0; dst_emc_entry->num_trim > i; i++)2972{2973reg_addr = trim_regs_emc_addr_table[i];2974if (reg_addr)2975{2976if (((reg_addr & 0xFFFFFFF0) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||2977(reg_addr & 0xFFFFFFF0) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||2978(reg_addr & 0xFFFFFFF8) == EMC_DATA_BRLSHFT_0)2979&& compensate_trimmer_applicable)2980{2981EMC(reg_addr) = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr);2982}2983else2984EMC(reg_addr) = trim_regs_table->trim_regs[i];2985}2986}29872988// Writing trim_regs_per_ch2989for (u32 i = 0; dst_emc_entry->num_trim_per_ch > i; i++)2990{2991reg_addr = trim_perch_regs_emc01_addr_table[i];2992if (reg_addr && (channel1_enabled || (reg_addr - 0x4000) > 0xFFF))2993{2994if (((reg_addr & 0xFFFFFFF8) == EMC0_DATA_BRLSHFT_0 ||2995(reg_addr & 0xFFFFFFF8) == EMC1_DATA_BRLSHFT_0)2996&& compensate_trimmer_applicable)2997{2998EMC(reg_addr) = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr & 0xFFF);2999}3000else3001EMC(reg_addr) = trim_regs_table->trim_perch_regs[i];3002}3003}30043005if (in_training)3006{3007// Check delta wrt previous values (save value if margin exceeds what is set in table).3008if (needs_wr_training && dst_emc_entry->periodic_training)3009_minerva_periodic_compensation_handler(src_emc_entry, dst_emc_entry, dram_dev_num, channel1_enabled, WRITE_TRAINING_SEQUENCE);3010}3011else3012{3013// Writing burst_mc_regs.3014for (u32 i = 0; dst_emc_entry->num_mc_regs > i; i++)3015MC(burst_mc_regs_addr_table[i]) = dst_emc_entry->burst_mc_regs[i];30163017// Writing la_scale_regs.3018if (dst_emc_entry->rate_khz < src_emc_entry->rate_khz)3019{3020for (u32 i = 0; dst_emc_entry->num_up_down > i; i++)3021MC(la_scale_regs_mc_addr_table[i]) = dst_emc_entry->la_scale_regs[i];3022}3023}30243025// Step 9 - LPDDR4.3026EPRINTF("Step 9");3027EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval & 0xFF000000;3028EMC(EMC_ZCAL_WAIT_CNT) = dst_emc_entry->burst_regs.emc_zcal_wait_cnt & 0xFFFFF800;3029EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE;3030EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval & 0xFF000000;3031EMC(EMC_DBG) = emc_dbg_o;30323033if (in_training)3034{3035EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;3036EMC(EMC_PMACRO_AUTOCAL_CFG_COMMON) = dst_emc_entry->burst_regs.emc_pmacro_autocal_cfg_common | 0x10000;30373038if (needs_ca_combo_training)3039EMC(EMC_FBIO_CFG5) = src_emc_entry->burst_regs.emc_fbio_cfg5 | 0x8000000;30403041EMC(EMC_DBG) = emc_dbg_o;30423043if (channel1_enabled)3044_ccfifo_write(EMC_CFG_SYNC, 0, 0);30453046_ccfifo_write(EMC_DBG, (emc_dbg_o & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_CFG_SWAP_SWAP, 0);3047}30483049// Step 10 - Self refresh3050EPRINTF("Step 10");3051_ccfifo_write(EMC_SELF_REF, 0x101, 0);30523053if (!needs_ca_combo_training && (dst_clock_period <= 2000))3054{3055_ccfifo_write(EMC_MRW3, mr13_flip_fspwr ^ 0x40, 0);3056_ccfifo_write(EMC_MRW6, (src_emc_entry->burst_regs.emc_mrw6 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw6 & 0xFFFF3F3F), 0);3057_ccfifo_write(EMC_MRW14, (src_emc_entry->burst_regs.emc_mrw14 & 0x3838) | (dst_emc_entry->burst_regs.emc_mrw14 & 0xFFFF0707), 0);3058if (dram_dev_num == TWO_RANK)3059{3060_ccfifo_write(EMC_MRW7, (src_emc_entry->burst_regs.emc_mrw7 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw7 & 0xFFFF3F3F), 0);3061_ccfifo_write(EMC_MRW15, (src_emc_entry->burst_regs.emc_mrw15 & 0x3838) | (dst_emc_entry->burst_regs.emc_mrw15 & 0xFFFF0707), 0);3062}30633064if (dram_dev_num == ONE_RANK || zcal_resistor_shared)3065emc_zq_cal = 0x80000001;3066else3067emc_zq_cal = 1;30683069_ccfifo_write(EMC_ZQ_CAL, emc_zq_cal, 0);3070}30713072emc_dbg_val = emc_dbg_o;3073u32 tRP_src_timing = (src_emc_entry->dram_timings.t_rp * 1000) / src_clock_period;3074u32 ref_delay = 0;30753076if (in_training)3077{3078emc_dbg_val = (emc_dbg_o & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_CFG_SWAP_SWAP;3079_ccfifo_write(EMC_DBG, emc_dbg_val, 0);3080}30813082if (needs_ca_combo_training)3083{3084_ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, src_emc_entry->burst_regs.emc_pmacro_data_rx_term_mode & 0xFFFFFCCC, 0);30853086if (dram_dev_num == TWO_RANK && needs_swap_rank_training)3087{3088_ccfifo_write(EMC_MRW3, mr13_flip_fspop | 8, tRP_src_timing);3089_ccfifo_write(EMC_MRW3, mr13_catr_enable | 8, 0);3090}3091else3092_ccfifo_write(EMC_MRW3, mr13_catr_enable | 8, tRP_src_timing);30933094_ccfifo_write(EMC_TR_CTRL_0, 0x15A, 0);3095ref_delay = 1000000 / src_clock_period;3096}3097else3098{3099_ccfifo_write(EMC_MRW3, mr13_flip_fspop | 8, tRP_src_timing);3100ref_delay = tFC_lpddr4 / src_clock_period;3101}31023103_ccfifo_write(EMC_INTSTATUS, 0, ref_delay);3104_ccfifo_write(EMC_PIN, emc_pin_o & 0xFFFFFFF8, 30);31053106// Step 11 - Ramp down.3107EPRINTF("Step 11");3108_ccfifo_write(EMC_CFG_SYNC, 0, 0);3109_ccfifo_write(EMC_DBG, emc_dbg_val | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE, 0); // WRITE_MUX_ACTIVE | WRITE_ACTIVE_ONLY31103111ramp_down_wait = _dvfs_power_ramp_down(false, src_emc_entry, dst_emc_entry, src_clock_period);31123113// Step 12 - Trigger clock change.3114EPRINTF("Step 12");3115_ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);3116if (!in_training)3117_ccfifo_write(EMC_DBG, (emc_dbg_val & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);31183119// Step 13 - Ramp up.3120EPRINTF("Step 13");3121ramp_up_wait = _dvfs_power_ramp_up(false, src_emc_entry, dst_emc_entry, needs_training, dst_clock_period);31223123_ccfifo_write(EMC_DBG, emc_dbg_val, 0);31243125// Step 14 - Bringup CKE pins.3126EPRINTF("Step 14");3127u32 emc_pin_val_final = 0;3128if (needs_ca_combo_training)3129{3130emc_pin_val_final = emc_pin_o & 0xFFFFFFF8;3131if (dram_dev_num == TWO_RANK)3132{3133if (needs_swap_rank_training)3134emc_pin_val_final |= 5;3135else3136emc_pin_val_final |= 6;3137}3138}3139else if (dram_dev_num == TWO_RANK)3140emc_pin_val_final = emc_pin_o | 7;3141else3142emc_pin_val_final = (emc_pin_o & 0xFFFFFFF8) | 1;31433144_ccfifo_write(EMC_PIN, emc_pin_val_final, 0);31453146// Step 15 - Zqlatch.3147EPRINTF("Step 15");3148if (!needs_ca_combo_training)3149{3150s32 zq_latch_dvfs_wait_time;3151u32 T_PDEX_timing = _div_o3(dst_emc_entry->dram_timings.t_pdex * 1000, dst_clock_period);31523153if (dst_clock_period > 2000)3154zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - (s32)T_PDEX_timing;3155else3156zq_latch_dvfs_wait_time =3157(s32)tZQCAL_lpddr4_fc_adj - (ramp_up_wait + ramp_down_wait) / dst_clock_period;31583159if (dram_dev_num == ONE_RANK)3160{3161if (dst_clock_period > 2000)3162_ccfifo_write(EMC_ZQ_CAL, 0x80000001, T_PDEX_timing);31633164if (!in_training)3165_ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, T_PDEX_timing);31663167emc_zq_cal = 0x80000002;3168}3169else if (zcal_resistor_shared)3170{3171if (dst_clock_period > 2000)3172_ccfifo_write(EMC_ZQ_CAL, 0x80000001, T_PDEX_timing);31733174s32 T_PDEX_timing_final = zq_latch_dvfs_wait_time + (s32)T_PDEX_timing;31753176if (T_PDEX_timing_final < 0)3177T_PDEX_timing_final = 0;31783179_ccfifo_write(EMC_ZQ_CAL, 0x80000002, T_PDEX_timing_final);3180_ccfifo_write(EMC_ZQ_CAL, 0x40000001, 0);31813182if (!in_training)3183_ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, 0);31843185emc_zq_cal = 0x40000002;3186zq_latch_dvfs_wait_time = 1000000 / dst_clock_period;3187}3188else3189{3190if (dst_clock_period > 2000)3191_ccfifo_write(EMC_ZQ_CAL, 1, T_PDEX_timing);31923193if (!in_training)3194_ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, T_PDEX_timing);31953196emc_zq_cal = 2;3197}31983199// Disable self-refresh.3200if (!in_training)3201{3202_ccfifo_write(EMC_SELF_REF, 0, 0);3203_ccfifo_write(EMC_REF, 0, 0);3204}32053206if (zq_latch_dvfs_wait_time < 0)3207zq_latch_dvfs_wait_time = 0;32083209_ccfifo_write(EMC_ZQ_CAL, emc_zq_cal, (u32)zq_latch_dvfs_wait_time);3210}32113212_ccfifo_write(EMC_INTSTATUS, 0, 10); // WAR: delay for zqlatch.32133214// Step 16 - LPDDR4 Conditional training kickoff.3215EPRINTF("Step 16");3216if (in_training)3217{3218_ccfifo_write(EMC_INTSTATUS, 0, 1020000 / dst_clock_period);32193220u32 training_command = 0;32213222if (needs_ca_training)3223training_command |= BIT(1); // CA: Initiates CA Training.3224if (needs_ca_vref_training)3225training_command |= BIT(5); // CA_VREF: Initiates CA_VREF Training.3226if (needs_quse_training)3227training_command |= BIT(4); // QUSE: Initiates QUSE Training.3228if (needs_quse_vref_training)3229training_command |= BIT(8); // QUSE_VREF: Initiates DQS_VREF Training.3230if (needs_wr_training)3231training_command |= BIT(3); // WR: Initiates WR Training.3232if (needs_wr_vref_training)3233training_command |= BIT(6); // WR_VREF: Initiates OB (write) DRAM_VREF Training.3234if (needs_rd_training)3235training_command |= BIT(2); // RD: Initiates RD Training.3236if (needs_rd_vref_training)3237training_command |= BIT(7); // RD_VREF: Initiates IB_DQ_VREF Training.3238training_command |= BIT(31); // GO: Start the Training.32393240_ccfifo_write(EMC_TRAINING_CMD, training_command, 0);32413242if (bg_regulator_mode_change)3243{3244if (enable_bg_regulator)3245_ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0,3246src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFE, 0);3247else3248_ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0,3249src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFB, 0);3250}32513252_ccfifo_write(EMC_SWITCH_BACK_CTRL, 1, 0);32533254if (!needs_ca_combo_training || needs_swap_rank_training)3255{3256_ccfifo_write(EMC_MRW3, mr13_flip_fspop ^ 0xC0, 0);3257_ccfifo_write(EMC_INTSTATUS, 0, 1000000 / dst_clock_period);3258}32593260_ccfifo_write(EMC_PIN, emc_pin_o & 0xFFFFFFF8, 0);3261_ccfifo_write(EMC_CFG_SYNC, 0, 0);3262_ccfifo_write(EMC_DBG, emc_dbg_val | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE, 0);32633264_dvfs_power_ramp_down(true, src_emc_entry, dst_emc_entry, dst_clock_period);32653266_ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);3267_ccfifo_write(EMC_DBG, (emc_dbg_val & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);32683269_dvfs_power_ramp_up(true, src_emc_entry, dst_emc_entry, needs_training, src_clock_period);32703271_ccfifo_write(EMC_DBG, emc_dbg_val, 0);32723273if (dram_dev_num == TWO_RANK)3274_ccfifo_write(EMC_PIN, emc_pin_o | 7, 0);3275else3276_ccfifo_write(EMC_PIN, (emc_pin_o & 0xFFFFFFF8) | 1, 0);32773278if (needs_ca_combo_training)3279{3280_ccfifo_write(EMC_TR_CTRL_0, 0x4A, 200000 / src_clock_period);3281_ccfifo_write(EMC_TR_CTRL_0, 0x40, 1000000 / src_clock_period);3282_ccfifo_write(EMC_MRW3, mr13_catr_enable & 0xFFFFFFFE, 0);3283_ccfifo_write(EMC_INTSTATUS, 0, 1000000 / src_clock_period);3284_ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, src_emc_entry->burst_regs.emc_pmacro_data_rx_term_mode, 0);3285}32863287_ccfifo_write(EMC_DBG, emc_dbg_o, 0);32883289_ccfifo_write(EMC_ZQ_CAL, 0x80000001, 0);3290_ccfifo_write(EMC_ZQ_CAL, 0x80000002, 1000000 / src_clock_period);32913292if ((!needs_ca_combo_training || needs_swap_rank_training) && dram_dev_num == TWO_RANK)3293{32943295_ccfifo_write(EMC_ZQ_CAL, 0x40000001, 0);3296_ccfifo_write(EMC_ZQ_CAL, 0x40000002, 1000000 / src_clock_period);3297}32983299if (!needs_ca_combo_training)3300_ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) ^ 0xC0000C0, 0);33013302_ccfifo_write(EMC_SELF_REF, 0, 0);3303}33043305// Step 19.2.3306EPRINTF("Step 19.2");3307if (bg_regulator_mode_change)3308{3309_ccfifo_write(EMC_DBG, emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE, 0);33103311u32 bg_regulator_switch_complete_wait_clks = 0;3312if (in_training)3313{3314bg_regulator_switch_complete_wait_clks = 1250000 / src_clock_period;3315_ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0,3316src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0, bg_regulator_switch_complete_wait_clks);3317}3318else3319{3320if (ramp_up_wait <= 1250000)3321bg_regulator_switch_complete_wait_clks = (1250000 - ramp_up_wait) / dst_clock_period;3322_ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0,3323dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0, bg_regulator_switch_complete_wait_clks);3324}33253326_ccfifo_write(EMC_DBG, emc_dbg_o, 0);3327}33283329// Step 20 - Issue ref and optional QRST.3330EPRINTF("Step 20");3331if (in_training)3332_ccfifo_write(EMC_REF, 0, 0);33333334// Step 21 - Restore ZCAL and ZCAL interval.3335EPRINTF("Step 21");3336_ccfifo_write(EMC_DBG, emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE, 0);33373338if (in_training)3339_ccfifo_write(EMC_ZCAL_INTERVAL, src_emc_entry->burst_regs.emc_zcal_interval, 0);33403341_ccfifo_write(EMC_CFG, dst_emc_entry->burst_regs.emc_cfg & 0xEFFFFFFF, 0);33423343// Step 22 - Restore EMC_CFG_PIPE_CLK.3344EPRINTF("Step 22");3345//if (in_training && dram_type == DRAM_TYPE_LPDDR4)////////////////3346if (in_training)3347_ccfifo_write(EMC_SEL_DPD_CTRL, src_emc_entry->emc_sel_dpd_ctrl, 0);33483349_ccfifo_write(EMC_DBG, emc_dbg_o, 0);3350_ccfifo_write(EMC_CFG_PIPE_CLK, emc_cfg_pipe_clk_o, 0);33513352if (bg_regulator_mode_change)3353{3354if (enable_bg_regulator)3355EMC(EMC_PMACRO_BG_BIAS_CTRL_0) = dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFB;3356else3357EMC(EMC_PMACRO_BG_BIAS_CTRL_0) = dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFE;3358}33593360// Step 23 - Clock Change.3361EPRINTF("Step 23 - Clock Change");3362// During training save current clock.3363if (in_training)3364{3365u32 emc_clk_src = CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC);3366CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC_SAFE) = emc_clk_src;3367_change_dll_src(src_emc_entry, emc_clk_src);3368}33693370// Set CFG_DLL_MODE to RUN_PERIODIC.3371EMC(EMC_CFG_DIG_DLL) = (EMC(EMC_CFG_DIG_DLL) & 0xFFFFFF24) | 0x88;3372(void)EMC(EMC_CFG_DIG_DLL);33733374(void)EMC(EMC_FBIO_CFG7);3375(void)MC(MC_EMEM_ADR_CFG);3376(void)EMC(EMC_INTSTATUS);33773378// Do clock change.3379CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC) = selected_clk_src_emc;3380(void)CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC);33813382if (_wait_emc_status(EMC_INTSTATUS, CLKCHANGE_COMPLETE_INT, true, 0))3383return 4; // Clkchange handshake timeout error.33843385// Step 24 - Save training results.3386EPRINTF("Step 24");3387if (in_training)3388{3389(void)MC(MC_EMEM_ADR_CFG);3390emc_dbg_val = EMC(EMC_DBG);3391EMC(EMC_DBG) |= EMC_DBG_READ_MUX_ASSEMBLY;33923393_save_train_results(dst_emc_entry, needs_training, dram_dev_num, channel1_enabled);33943395EMC(EMC_DBG) = emc_dbg_val;3396}33973398// Step 25 - Program MC updown regs.3399EPRINTF("Step 25");3400if (!in_training && (dst_emc_entry->rate_khz > src_emc_entry->rate_khz))3401{3402for (u32 i = 0; dst_emc_entry->num_up_down > i; i++)3403MC(la_scale_regs_mc_addr_table[i]) = dst_emc_entry->la_scale_regs[i];34043405bool dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1);3406if (_timing_update(dual_channel))3407return 4;3408}34093410// Step 26 - Restore ZCAL regs.3411EPRINTF("Step 26");3412EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;3413EMC(EMC_ZCAL_WAIT_CNT) = dst_emc_entry->burst_regs.emc_zcal_wait_cnt;3414EMC(EMC_ZCAL_INTERVAL) = dst_emc_entry->burst_regs.emc_zcal_interval;3415EMC(EMC_DBG) = emc_dbg_o;34163417// Step 27 - Restore EMC_CFG, FDPD regs.3418EPRINTF("Step 27");3419EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;3420EMC(EMC_CFG) = dst_emc_entry->burst_regs.emc_cfg;3421EMC(EMC_DBG) = emc_dbg_o;3422EMC(EMC_FDPD_CTRL_CMD_NO_RAMP) = dst_emc_entry->emc_fdpd_ctrl_cmd_no_ramp;3423EMC(EMC_SEL_DPD_CTRL) = dst_emc_entry->emc_sel_dpd_ctrl;34243425// Step 28 - Training recover.3426EPRINTF("Step 28");3427if (in_training)3428{3429EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;3430EMC(EMC_CFG) = dst_emc_entry->burst_regs.emc_cfg;3431EMC(EMC_SEL_DPD_CTRL) = dst_emc_entry->emc_sel_dpd_ctrl;3432EMC(EMC_ZCAL_WAIT_CNT) = src_emc_entry->burst_regs.emc_zcal_wait_cnt;3433EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval;3434EMC(EMC_AUTO_CAL_CONFIG2) = src_emc_entry->emc_auto_cal_config2;3435EMC(EMC_AUTO_CAL_CONFIG3) = src_emc_entry->emc_auto_cal_config3;3436EMC(EMC_AUTO_CAL_CONFIG4) = src_emc_entry->emc_auto_cal_config4;3437EMC(EMC_AUTO_CAL_CONFIG5) = src_emc_entry->emc_auto_cal_config5;3438EMC(EMC_AUTO_CAL_CONFIG6) = src_emc_entry->emc_auto_cal_config6;3439EMC(EMC_AUTO_CAL_CONFIG7) = src_emc_entry->emc_auto_cal_config7;3440EMC(EMC_AUTO_CAL_CONFIG8) = src_emc_entry->emc_auto_cal_config8;3441EMC(EMC_DBG) = emc_dbg_o;3442EMC(EMC_TR_DVFS) = dst_emc_entry->burst_regs.emc_tr_dvfs & 0xFFFFFFFE;3443}34443445EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE;3446EMC(EMC_PMACRO_AUTOCAL_CFG_COMMON) = dst_emc_entry->burst_regs.emc_pmacro_autocal_cfg_common;3447EMC(EMC_DBG) = emc_dbg_o;34483449// Step 29 - Power fix WAR.3450EPRINTF("Step 29");3451EMC(EMC_PMACRO_CFG_PM_GLOBAL_0) = 0xFF0000;3452EMC(EMC_PMACRO_TRAINING_CTRL_0) = CH0_TRAINING_E_WRPTR;3453EMC(EMC_PMACRO_TRAINING_CTRL_1) = CH0_TRAINING_E_WRPTR;3454EMC(EMC_PMACRO_CFG_PM_GLOBAL_0) = 0;34553456// Step 30 - Re-enable autocal.3457EPRINTF("Step 30");3458if (in_training)3459EMC(EMC_AUTO_CAL_CONFIG) = src_emc_entry->emc_auto_cal_config;3460else3461{3462if (dst_emc_entry->burst_regs.emc_cfg_dig_dll & 1)3463_digital_dll_enable_rs(channel1_enabled);3464EMC(EMC_AUTO_CAL_CONFIG) = dst_emc_entry->emc_auto_cal_config;3465}34663467return 0;3468}34693470static void _minerva_train(emc_table_t *src_emc_entry, emc_table_t *dst_emc_entry, bool switch_rate, u32 selected_clk_src_emc)3471{3472u32 needs_training_num = 0;3473u32 emc_cfg_dig_dll_val = 0;3474u32 needs_training_emc_table[8] = {0};34753476u32 needs_training = dst_emc_entry->needs_training;34773478// Must start as true.3479if (train_ram_patterns)3480{3481u32 train_table_off = dst_emc_entry->training_pattern * 256;3482for (u32 i = 0; i < 256; i++)3483{3484EMC(EMC_TRAINING_PATRAM_DQ) = ram_pattern_dq_table[train_table_off + i];3485EMC(EMC_TRAINING_PATRAM_DMI) = ram_pattern_dmi_table[train_table_off + i] & 0xF;3486EMC(EMC_TRAINING_PATRAM_CTRL) = 0x80000000 + i;3487}3488train_ram_patterns = false;3489}34903491if (!dst_emc_entry->trained)3492{3493if (needs_training & NEEDS_TRAINING_CA_COMBO)3494{3495needs_training_emc_table[needs_training_num++] =3496needs_training & (NEEDS_TRAINING_CA_COMBO | NEEDS_TRAINING_IN_SELF_REFRESH);3497if (MC(MC_EMEM_ADR_CFG) & 1) // if mapping W8 (1KB page).3498needs_training_emc_table[needs_training_num++] =3499needs_training & (NEEDS_TRAINING_CA_COMBO | NEEDS_TRAINING_SWAP_RANK | NEEDS_TRAINING_IN_SELF_REFRESH);3500}35013502if (needs_training & NEEDS_TRAINING_QUSE_COMBO)3503{3504needs_training_emc_table[needs_training_num++] =3505needs_training & (NEEDS_TRAINING_QUSE_COMBO | NEEDS_TRAINING_IN_SELF_REFRESH);3506if (MC(MC_EMEM_ADR_CFG) & 1)3507needs_training_emc_table[needs_training_num++] =3508needs_training & (NEEDS_TRAINING_QUSE | NEEDS_TRAINING_IN_SELF_REFRESH);3509}35103511if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO))3512needs_training_emc_table[needs_training_num++] =3513needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO | NEEDS_TRAINING_IN_SELF_REFRESH);35143515for (u32 i = 0; needs_training_num > i; i++) // Runs more than once for needs_training CA/QUSE/WR/RD.3516{3517_minerva_change_clock(src_emc_entry, dst_emc_entry, needs_training_emc_table[i], selected_clk_src_emc);35183519bool dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1);35203521EMC(EMC_DBG) = (EMC(EMC_DBG) & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_CFG_SWAP_ASSEMBLY_ONLY;3522EMC(EMC_CFG_UPDATE) = (EMC(EMC_CFG_UPDATE) & 0xFFFFFFF9) | 4;3523_timing_update(dual_channel);35243525EMC(EMC_CFG_UPDATE) &= 0xFFFFFFF9;3526EMC(EMC_DBG) &= ~EMC_DBG_CFG_SWAP_MASK; // EMC_DBG_CFG_SWAP_ACTIVE_ONLY.3527EMC(EMC_CFG_DIG_DLL) = (EMC(EMC_CFG_DIG_DLL) & 0xFFFFFF3E) | 0x80;3528_timing_update(dual_channel);35293530emc_cfg_dig_dll_val = EMC(EMC_CFG_DIG_DLL) & 0xFFFFFFFE;3531if (dst_emc_entry->burst_regs.emc_cfg_dig_dll == 1)3532emc_cfg_dig_dll_val = EMC(EMC_CFG_DIG_DLL) | 1;3533EMC(EMC_CFG_DIG_DLL) = (emc_cfg_dig_dll_val & 0xFFFFFF3F) | 0x80;3534_timing_update(dual_channel);35353536while (!(EMC(EMC_DIG_DLL_STATUS) & 0x8000))3537;35383539// Bug 200024907.3540EMC(EMC_RP) = src_emc_entry->burst_regs.emc_rp;3541EMC(EMC_R2P) = src_emc_entry->burst_regs.emc_r2p;3542EMC(EMC_W2P) = src_emc_entry->burst_regs.emc_w2p;3543EMC(EMC_TRPAB) = src_emc_entry->burst_regs.emc_trpab;35443545_timing_update(dual_channel);3546}35473548EPRINTF("Trained");3549dst_emc_entry->trained = 1;3550}35513552if (switch_rate)3553_minerva_change_clock(src_emc_entry, dst_emc_entry, 0, selected_clk_src_emc);3554}35553556void _minerva_do_over_temp_compensation(mtc_config_t *mtc_cfg)3557{3558if (!mtc_cfg->current_emc_table)3559return;35603561u32 dram_type = EMC(EMC_FBIO_CFG5) & 3;35623563// Only LPDDR chips are supported.3564if (dram_type != DRAM_TYPE_LPDDR4)3565return;35663567s32 dram_temp = _get_dram_temperature();35683569if (dram_temp < 0 || mtc_cfg->prev_temp == (u32)dram_temp)3570return;35713572u32 refr = mtc_cfg->current_emc_table->burst_regs.emc_refresh;3573u32 pre_refr = mtc_cfg->current_emc_table->burst_regs.emc_pre_refresh_req_cnt;3574u32 dyn_self_ref = mtc_cfg->current_emc_table->burst_regs.emc_dyn_self_ref_control;35753576switch (dram_temp)3577{3578// Normal temp (<= 85 oC).3579case 0:3580case 1:3581case 2:3582case 3:3583if (mtc_cfg->prev_temp < 4)3584{3585mtc_cfg->prev_temp = (u32)dram_temp;3586return;3587}3588break;3589// Over temp (> 85 oC).3590case 4: // 2x refresh.3591refr = (refr & 0xFFFF0000) | ((refr & 0xFFFF) >> REFRESH_X2);3592pre_refr = (pre_refr & 0xFFFF0000) | ((pre_refr & 0xFFFF) >> REFRESH_X2);3593dyn_self_ref = (dyn_self_ref & 0xFFFF0000) | ((dyn_self_ref & 0xFFFF) >> REFRESH_X2);3594break;3595case 5: // 4x refresh.3596case 6: // Temp 6 normally needs a derating emc table.3597refr = (refr & 0xFFFF0000) | ((refr & 0xFFFF) >> REFRESH_X4);3598pre_refr = (pre_refr & 0xFFFF0000) | ((pre_refr & 0xFFFF) >> REFRESH_X4);3599dyn_self_ref = (dyn_self_ref & 0xFFFF0000) | ((dyn_self_ref & 0xFFFF) >> REFRESH_X4);3600break;3601default:3602break;3603}36043605mtc_cfg->prev_temp = dram_temp;36063607EMC(EMC_REFRESH) = refr;3608EMC(EMC_PRE_REFRESH_REQ_CNT) = pre_refr;3609EMC(EMC_DYN_SELF_REF_CONTROL) = dyn_self_ref;3610}36113612u32 _minerva_do_periodic_compensation(emc_table_t *mtc_table_entry)3613{3614if (mtc_table_entry && mtc_table_entry->periodic_training)3615{3616u32 dram_dev_num = (MC(MC_EMEM_ADR_CFG) & 1) + 1;3617u32 pd_mask = (dram_dev_num == TWO_RANK) ? IN_POWERDOWN_BOTH_MASK : IN_POWERDOWN_1DEV_MASK;3618bool channel1_enabled = (mtc_table_entry->burst_regs.emc_fbio_cfg7 >> 2) & 1;36193620(void)EMC(EMC_DBG);36213622// Safekeep current config.3623u32 emc_cfg_o = EMC(EMC_CFG);3624u32 emc_cfg_dig_dll_o = EMC(EMC_CFG_DIG_DLL);3625u32 emc_cfg_update_o = EMC(EMC_CFG_UPDATE);36263627// Step 1 - Disable digital DLL.3628EMC(EMC_CFG_DIG_DLL) = emc_cfg_dig_dll_o & 0xFFFFFFFE;36293630// Step 1.2 - Always update auto cal in clock change.3631EMC(EMC_CFG_UPDATE) = (emc_cfg_update_o & 0xFFFFF9FF) | 0x400;36323633// Step 1.3 - Disable other power features.3634EMC(EMC_CFG) = emc_cfg_o & 0xFFFFFFF;36353636// Timing update and wait for everything to power down.3637_timing_update(channel1_enabled);36383639_wait_emc_status(EMC_EMC_STATUS, pd_mask, 0, EMC_CHANNEL0);3640if (channel1_enabled)3641_wait_emc_status(EMC_EMC_STATUS, pd_mask, 0, EMC_CHANNEL1);36423643_wait_emc_status(EMC_EMC_STATUS, IN_SELF_REFRESH_MASK, 0, EMC_CHANNEL0);3644if (channel1_enabled)3645_wait_emc_status(EMC_EMC_STATUS, IN_SELF_REFRESH_MASK, 0, EMC_CHANNEL1);36463647_wait_emc_status(EMC_CFG_DIG_DLL, 1, 0, EMC_CHANNEL0);3648if (channel1_enabled)3649_wait_emc_status(EMC_CFG_DIG_DLL, 1, 0, EMC_CHANNEL1);36503651// Step 2 - Osc kick off - this assumes training and dvfs have set correct MR23.3652_start_periodic_compensation();36533654// Step 3 - Let dram capture its clock tree delays.3655_usleep(1000 * _actual_osc_clocks(mtc_table_entry->run_clocks) / mtc_table_entry->rate_khz + 2);36563657// Step 4 - Check delta wrt previous values (save value if margin exceeds what is set in table).3658u32 adelta = _minerva_update_clock_tree_delay(mtc_table_entry, mtc_table_entry, dram_dev_num, channel1_enabled, PERIODIC_TRAINING_UPDATE);36593660// Step 5 - Apply compensation w.r.t. trained values (if clock tree has drifted more than the set margin).3661if (adelta && ((mtc_table_entry->rate_khz / 1000) * 128) * adelta / 1000000 > mtc_table_entry->tree_margin)3662{3663for (u32 i = 0; i < 10; i++)3664{3665EMC(periodic_training_addr[i]) =3666_minerva_apply_periodic_compensation_trimmer(mtc_table_entry, periodic_training_addr[i]);3667}3668}36693670// Step 6 - Restore other power features.3671EMC(EMC_CFG) = emc_cfg_o;36723673// Step 6.1 - Restore the DLL.3674EMC(EMC_CFG_DIG_DLL) = emc_cfg_dig_dll_o;36753676// Step 6.2 - Timing update for applying the new trimmers.3677_timing_update(channel1_enabled);36783679// Step 6.3 - Restore the UPDATE_DLL_IN_UPDATE field.3680EMC(EMC_CFG_UPDATE) = emc_cfg_update_o;3681}36823683return 0;3684}36853686static int _minerva_set_ir_boost(mtc_config_t *mtc_cfg)3687{3688switch (mtc_cfg->train_mode)3689{3690case OP_SWITCH:3691break;3692case OP_TRAIN:3693return 4;3694case OP_TRAIN_SWITCH:3695break;3696default:3697return 4;3698}36993700EPRINTFARGS("Requested op %d from %d to %d.", OP_SWITCH, src_rate_khz, dst_rate_khz);37013702bool boost = mtc_cfg->rate_to > MTC_INIT_FREQUENCY;37033704u32 clk_new = (PLLP_OUT0 << 29u) | 0x188000 | (boost ? 0 : 2);3705u32 clk_cur = CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC);3706u32 clk_cur_src = clk_cur >> 29u;37073708// Already at requested clock.3709if (clk_new == clk_cur)3710{3711mtc_cfg->rate_from = mtc_cfg->rate_to;3712return 0;3713}37143715if (clk_cur_src != PLLP_OUT0)3716return 3;37173718// Clear clock change interrupt.3719EMC(EMC_INTSTATUS) = CLKCHANGE_COMPLETE_INT;37203721u32 emc_dbg_o = EMC(EMC_DBG);3722u32 emc_dbg_c = emc_dbg_o & ~(EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_READ_MUX_ASSEMBLY);3723EMC(EMC_DBG) = emc_dbg_c;3724(void)EMC(EMC_DBG);37253726// Save registers.3727u32 emc_cfg_o = EMC(EMC_CFG);3728u32 emc_auto_cal_cfg_o = EMC(EMC_AUTO_CAL_CONFIG);3729u32 emc_sel_dpd_ctrl_o = EMC(EMC_SEL_DPD_CTRL);3730u32 emc_cfg_pipe_clk_o = EMC(EMC_CFG_PIPE_CLK);3731u32 emc_fdpd_ctl_cmd_o = EMC(EMC_FDPD_CTRL_CMD_NO_RAMP);37323733EMC(EMC_DBG) = emc_dbg_c | EMC_DBG_WRITE_MUX_ACTIVE;3734(void)EMC(EMC_DBG);37353736// Disable autocal.3737EMC(EMC_AUTO_CAL_CONFIG) = (emc_auto_cal_cfg_o & 0x7FFFF9FE) | 0x601;3738(void)EMC(EMC_AUTO_CAL_CONFIG);37393740// Disable other power features.3741EMC(EMC_CFG) = emc_cfg_o & 0xFFFFFFF;3742EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl_o & 0xFFFFFEC3;3743EMC(EMC_CFG_PIPE_CLK) = emc_cfg_pipe_clk_o | BIT(0);3744EMC(EMC_FDPD_CTRL_CMD_NO_RAMP) = emc_fdpd_ctl_cmd_o & ~BIT(0);37453746EMC(EMC_DBG) = emc_dbg_c;3747(void)EMC(EMC_DBG);3748_usleep(5);37493750if (boost)3751{3752// Program arbiter regs.3753MC(MC_EMEM_ARB_CFG) = 0x1000003;3754MC(MC_EMEM_ARB_TIMING_RP) = 1;3755MC(MC_EMEM_ARB_TIMING_RC) = 6;3756MC(MC_EMEM_ARB_TIMING_RAS) = 3;3757MC(MC_EMEM_ARB_TIMING_FAW) = 4;3758MC(MC_EMEM_ARB_TIMING_RFCPB) = 0xE;3759MC(MC_EMEM_ARB_DA_COVERS) = 0x30203;3760MC(MC_EMEM_ARB_MISC0) = 0x71E30507;37613762// Program base timings.3763EMC(EMC_RP) = 8;3764EMC(EMC_TRPAB) = 9;3765EMC(EMC_RC) = 25;3766EMC(EMC_RAS) = 18;3767EMC(EMC_R2W) = 15;3768EMC(EMC_RD_RCD) = 8;3769EMC(EMC_WR_RCD) = 8;3770EMC(EMC_TFAW) = 17; // 13 on 2133 MHz modules.37713772// ZQ calibration.3773EMC(EMC_ZCAL_WAIT_CNT) = 0xE0198;37743775// Set refresh timings.3776EMC(EMC_RFC) = 115;3777EMC(EMC_RFCPB) = 58;3778EMC(EMC_TREFBW) = 0x18DD; // 0x638.3779EMC(EMC_REFRESH) = 0x181E; // 0x607.3780EMC(EMC_PRE_REFRESH_REQ_CNT) = 0x607; // 0x181.3781EMC(EMC_DYN_SELF_REF_CONTROL) = (EMC(EMC_DYN_SELF_REF_CONTROL) & 0xFFFF0000) | 0xD22;3782}3783else3784{3785// Program arbiter regs.3786MC(MC_EMEM_ARB_CFG) = 0x8000001;3787MC(MC_EMEM_ARB_TIMING_RP) = 0;3788MC(MC_EMEM_ARB_TIMING_RC) = 3;3789MC(MC_EMEM_ARB_TIMING_RAS) = 1;3790MC(MC_EMEM_ARB_TIMING_FAW) = 2;3791MC(MC_EMEM_ARB_TIMING_RFCPB) = 7;3792MC(MC_EMEM_ARB_DA_COVERS) = 0x30201;3793MC(MC_EMEM_ARB_MISC0) = 0x72A30504;37943795// Program base timings.3796EMC(EMC_RP) = 4;3797EMC(EMC_TRPAB) = 5;3798EMC(EMC_RC) = 13;3799EMC(EMC_RAS) = 9;3800EMC(EMC_R2W) = 11;3801EMC(EMC_RD_RCD) = 6;3802EMC(EMC_WR_RCD) = 6;3803EMC(EMC_TFAW) = 9; // 8 on 2133 MHz modules.38043805// Program ZQ calibration.3806EMC(EMC_ZCAL_WAIT_CNT) = 0x900CC;38073808// Program refresh timings.3809EMC(EMC_RFC) = 58;3810EMC(EMC_RFCPB) = 29;3811EMC(EMC_TREFBW) = 0x31C;3812EMC(EMC_REFRESH) = 0x304;3813EMC(EMC_PRE_REFRESH_REQ_CNT) = 0xC1;3814EMC(EMC_DYN_SELF_REF_CONTROL) = (EMC(EMC_DYN_SELF_REF_CONTROL) & 0xFFFF0000) | 0x713;3815}38163817// Restore power features and pipe clock.3818EMC(EMC_CFG) = emc_cfg_o & 0xEFFFFFFF;3819EMC(EMC_CFG_PIPE_CLK) = emc_cfg_pipe_clk_o;3820EMC(EMC_DBG) = emc_dbg_o;3821(void)EMC(EMC_DBG);38223823// Do clock change.3824CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC) = clk_new;3825(void)CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC);38263827// Clock change handshake.3828_wait_emc_status(EMC_INTSTATUS, CLKCHANGE_COMPLETE_INT, true, 0);38293830// Restore regs.3831EMC(EMC_CFG) = emc_cfg_o;3832EMC(EMC_FDPD_CTRL_CMD_NO_RAMP) = emc_fdpd_ctl_cmd_o;3833EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl_o;3834EMC(EMC_AUTO_CAL_CONFIG) = emc_auto_cal_cfg_o;38353836EMC(EMC_TIMING_CONTROL) = 1;38373838// Min 20us before next clock change.3839_usleep(20);38403841mtc_cfg->rate_from = mtc_cfg->rate_to;38423843return 0;3844}38453846static u32 _minerva_do_training_switch(mtc_config_t *mtc_cfg)3847{3848u32 src_emc_entry_idx = 100;3849u32 dst_emc_entry_idx = 100;3850u32 selected_clk_src_emc;3851u32 emc_clk_src;3852bool freq_changed = false;3853bool src_is_pllmb;3854emc_table_t *src_emc_entry;3855emc_table_t *dst_emc_entry;38563857if (mtc_cfg->table_entries > 32)3858return 4;38593860if (mtc_cfg->mtc_table[0].rev == MTC_IRB_MAGIC)3861return _minerva_set_ir_boost(mtc_cfg);38623863// Find tables.3864for (u32 i = 0; i < mtc_cfg->table_entries; i++)3865{3866u32 table_entry_rate = mtc_cfg->mtc_table[i].rate_khz;3867if (mtc_cfg->rate_from == table_entry_rate)3868src_emc_entry_idx = i;3869if (mtc_cfg->rate_to == table_entry_rate)3870dst_emc_entry_idx = i;3871}38723873// Check if tables were found.3874if (src_emc_entry_idx >= mtc_cfg->table_entries ||3875dst_emc_entry_idx >= mtc_cfg->table_entries)3876return 4;38773878src_emc_entry = (emc_table_t *)&mtc_cfg->mtc_table[src_emc_entry_idx];3879dst_emc_entry = (emc_table_t *)&mtc_cfg->mtc_table[dst_emc_entry_idx];38803881u32 src_rate_khz = src_emc_entry->rate_khz;3882u32 dst_rate_khz = dst_emc_entry->rate_khz;3883u32 src_clk_src_emc = src_emc_entry->clk_src_emc;3884u32 dst_clk_src_emc = dst_emc_entry->clk_src_emc;38853886freq_changed = _check_freq_changed(dst_rate_khz, dst_clk_src_emc, src_rate_khz, src_clk_src_emc);3887EPRINTFARGS("Requested op %d from %d to %d.", mtc_cfg->train_mode, src_rate_khz, dst_rate_khz);38883889// Get current clock source.3890emc_clk_src = CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC) >> EMC_2X_CLK_SRC_SHIFT;3891src_is_pllmb = emc_clk_src == PLLMB_UD || emc_clk_src == PLLMB_OUT0;38923893if (freq_changed)3894{3895if (emc_clk_src == PLLM_UD ||3896emc_clk_src == PLLM_OUT0) // Clock source is PLLM. Switch based on src_is_pllmb.3897{3898src_is_pllmb = !src_is_pllmb;3899}3900else if (emc_clk_src == PLLMB_UD ||3901emc_clk_src == PLLMB_OUT0) // Clock source is PLLMB. Switch to PLLM.3902{3903src_is_pllmb = false;3904}3905selected_clk_src_emc = _pllm_clk_base_cfg(dst_rate_khz, dst_clk_src_emc, src_is_pllmb);3906}3907else3908{3909selected_clk_src_emc = dst_clk_src_emc;3910emc_clk_src = selected_clk_src_emc >> EMC_2X_CLK_SRC_SHIFT;3911if (src_is_pllmb)3912{3913if (emc_clk_src == PLLM_UD || emc_clk_src == PLLMB_UD)3914selected_clk_src_emc = (selected_clk_src_emc & 0x1FFFFFFF) | (PLLMB_UD << EMC_2X_CLK_SRC_SHIFT);3915else if (emc_clk_src == PLLM_OUT0 || emc_clk_src == PLLMB_OUT0)3916selected_clk_src_emc = (selected_clk_src_emc & 0x1FFFFFFF) | (PLLMB_OUT0 << EMC_2X_CLK_SRC_SHIFT);3917}3918}39193920switch (mtc_cfg->train_mode)3921{3922case OP_SWITCH:3923_minerva_change_clock(src_emc_entry, dst_emc_entry, 0, selected_clk_src_emc);3924mtc_cfg->current_emc_table = dst_emc_entry;3925mtc_cfg->rate_from = dst_emc_entry->rate_khz;3926if (dst_emc_entry->periodic_training)3927_minerva_do_periodic_compensation(dst_emc_entry);3928return 0;3929case OP_TRAIN:3930_minerva_train(src_emc_entry, dst_emc_entry, false, selected_clk_src_emc);3931return 0;3932case OP_TRAIN_SWITCH:3933_minerva_train(src_emc_entry, dst_emc_entry, true, selected_clk_src_emc);3934mtc_cfg->current_emc_table = dst_emc_entry;3935mtc_cfg->rate_from = dst_emc_entry->rate_khz;3936if (dst_emc_entry->periodic_training)3937_minerva_do_periodic_compensation(dst_emc_entry);3938return 0;3939default:3940return 4;3941}3942}39433944static void _minerva_get_table(mtc_config_t *mtc_cfg)3945{3946u32 hidrev = APB_MISC(APB_MISC_GP_HIDREV);3947u32 chip_id = (hidrev >> 8) & 0xFF;3948u32 major = (hidrev >> 4) & 0xF;39493950// Check that SoC is T210/T210B01.3951if (chip_id != HIDREV_CHIPID_T210)3952goto error;39533954// Check if SoC SKU is ODIN.3955if (FUSE(FUSE_SKU_INFO) != SKU_ODIN)3956goto error;39573958// Clear table storage.3959memcpy(mtc_cfg->mtc_table, DVFS_T21X_CC_VERSION, sizeof(DVFS_T21X_CC_VERSION));3960memcpy(mtc_cfg->mtc_table, DVFS_T210_CC_VERSION, sizeof(DVFS_T210_CC_VERSION));3961memset(mtc_cfg->mtc_table, 0, EMC_TABLE_ENTRY_SIZE_R7 * 10);39623963// Check if init rate boost mode.3964if (mtc_cfg->init_done == MTC_IRB_MAGIC)3965{3966mtc_cfg->mtc_table[0].rev = MTC_IRB_MAGIC;3967mtc_cfg->table_entries = 1; // Avoid OOB.39683969goto init_cfg;3970}39713972// Check if ODINX02.3973if (major != HIDREV_MAJOR_T210)3974goto error;39753976// ODINX02 table: 204, 408, 666, 800, 1066, 1333, 1600 MHz.3977mtc_cfg->table_entries = EMC_TABLE_SIZE_ODINX02_R7 / EMC_TABLE_ENTRY_SIZE_R7;39783979switch (mtc_cfg->sdram_id)3980{3981case DRAM_4GB_HYNIX_H9HCNNNBPUMLHR_NLN:3982memcpy(mtc_cfg->mtc_table, odinx02_a2_2_10NoCfgVersion_V9_8_7_V1_6, EMC_TABLE_SIZE_ODINX02_R7);3983break;3984case DRAM_8GB_SAMSUNG_K4FBE3D4HM_MGXX:3985memcpy(mtc_cfg->mtc_table, odinx02_a2_7_10NoCfgVersion_V9_8_7_V1_6, EMC_TABLE_SIZE_ODINX02_R7);3986break;3987case DRAM_4GB_SAMSUNG_K4F6E304HB_MGCH:3988case DRAM_4GB_MICRON_MT53B512M32D2NP_062_WT:3989case DRAM_4GB_COPPER_SAMSUNG:3990case DRAM_6GB_SAMSUNG_K4FHE3D4HM_MFCH:3991default:3992memcpy(mtc_cfg->mtc_table, odinx02_a2_0_3_10NoCfgVersion_V9_8_7_V1_6, EMC_TABLE_SIZE_ODINX02_R7);3993break;3994}39953996init_cfg:3997mtc_cfg->rate_to = 0;3998mtc_cfg->rate_from = 0;3999mtc_cfg->train_mode = -1;4000mtc_cfg->prev_temp = 0;4001mtc_cfg->current_emc_table = NULL;40024003// Important!4004mtc_cfg->train_ram_patterns = true;4005mtc_cfg->init_done = MTC_INIT_MAGIC;40064007return;40084009error:4010mtc_cfg->init_done = 0;4011return;4012}40134014void minerva_entry(mtc_config_t *mtc_cfg, bdk_params_t *bp)4015{4016EPRINTF("-- Minerva Training Cell --");40174018train_ram_patterns = mtc_cfg->train_ram_patterns;40194020if (mtc_cfg->init_done != MTC_INIT_MAGIC)4021{4022if (mtc_cfg->init_done == MTC_NEW_MAGIC ||4023mtc_cfg->init_done == MTC_IRB_MAGIC)4024_minerva_get_table(mtc_cfg);40254026return;4027}40284029switch (mtc_cfg->train_mode)4030{4031case OP_SWITCH:4032EPRINTF("Switching..");4033_minerva_do_training_switch(mtc_cfg);4034break;4035case OP_TRAIN:4036EPRINTF("Training..");4037_minerva_do_training_switch(mtc_cfg);4038break;4039case OP_TRAIN_SWITCH:4040EPRINTF("Training and switching..");4041_minerva_do_training_switch(mtc_cfg);4042break;4043case OP_PERIODIC_TRAIN:4044EPRINTF("Periodic training..");4045_minerva_do_periodic_compensation(mtc_cfg->current_emc_table);4046break;4047case OP_TEMP_COMP:4048EPRINTF("Over temperature compensation..");4049_minerva_do_over_temp_compensation(mtc_cfg);4050break;4051}40524053mtc_cfg->train_ram_patterns = train_ram_patterns;4054}405540564057