blob: 57557d3948d005a77c34be5ab1f38b644713c75e [file] [log] [blame]
Dinh Nguyen3da42852015-06-02 22:52:49 -05001/*
2 * Copyright Altera Corporation (C) 2012-2015
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <common.h>
8#include <asm/io.h>
9#include <asm/arch/sdram.h>
Marek Vasut04372fb2015-07-18 02:46:56 +020010#include <errno.h>
Dinh Nguyen3da42852015-06-02 22:52:49 -050011#include "sequencer.h"
12#include "sequencer_auto.h"
13#include "sequencer_auto_ac_init.h"
14#include "sequencer_auto_inst_init.h"
15#include "sequencer_defines.h"
16
Dinh Nguyen3da42852015-06-02 22:52:49 -050017static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
Marek Vasut6afb4fe2015-07-12 18:46:52 +020018 (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
Dinh Nguyen3da42852015-06-02 22:52:49 -050019
20static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
Marek Vasut6afb4fe2015-07-12 18:46:52 +020021 (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
Dinh Nguyen3da42852015-06-02 22:52:49 -050022
23static struct socfpga_sdr_reg_file *sdr_reg_file =
Marek Vasuta1c654a2015-07-12 18:31:05 +020024 (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
Dinh Nguyen3da42852015-06-02 22:52:49 -050025
26static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
Marek Vasute79025a2015-07-12 18:42:34 +020027 (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
Dinh Nguyen3da42852015-06-02 22:52:49 -050028
29static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
Marek Vasut1bc6f142015-07-12 18:54:37 +020030 (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
Dinh Nguyen3da42852015-06-02 22:52:49 -050031
32static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
Marek Vasut1bc6f142015-07-12 18:54:37 +020033 (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
Dinh Nguyen3da42852015-06-02 22:52:49 -050034
35static struct socfpga_data_mgr *data_mgr =
Marek Vasutc4815f72015-07-12 19:03:33 +020036 (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
Dinh Nguyen3da42852015-06-02 22:52:49 -050037
Marek Vasut6cb9f162015-07-12 20:49:39 +020038static struct socfpga_sdr_ctrl *sdr_ctrl =
39 (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
40
Dinh Nguyen3da42852015-06-02 22:52:49 -050041#define DELTA_D 1
Dinh Nguyen3da42852015-06-02 22:52:49 -050042
43/*
44 * In order to reduce ROM size, most of the selectable calibration steps are
45 * decided at compile time based on the user's calibration mode selection,
46 * as captured by the STATIC_CALIB_STEPS selection below.
47 *
48 * However, to support simulation-time selection of fast simulation mode, where
49 * we skip everything except the bare minimum, we need a few of the steps to
50 * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
51 * check, which is based on the rtl-supplied value, or we dynamically compute
52 * the value to use based on the dynamically-chosen calibration mode
53 */
54
55#define DLEVEL 0
56#define STATIC_IN_RTL_SIM 0
57#define STATIC_SKIP_DELAY_LOOPS 0
58
59#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
60 STATIC_SKIP_DELAY_LOOPS)
61
62/* calibration steps requested by the rtl */
63uint16_t dyn_calib_steps;
64
65/*
66 * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
67 * instead of static, we use boolean logic to select between
68 * non-skip and skip values
69 *
70 * The mask is set to include all bits when not-skipping, but is
71 * zero when skipping
72 */
73
74uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */
75
76#define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
77 ((non_skip_value) & skip_delay_mask)
78
79struct gbl_type *gbl;
80struct param_type *param;
81uint32_t curr_shadow_reg;
82
83static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
84 uint32_t write_group, uint32_t use_dm,
85 uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
86
Dinh Nguyen3da42852015-06-02 22:52:49 -050087static void set_failing_group_stage(uint32_t group, uint32_t stage,
88 uint32_t substage)
89{
90 /*
91 * Only set the global stage if there was not been any other
92 * failing group
93 */
94 if (gbl->error_stage == CAL_STAGE_NIL) {
95 gbl->error_substage = substage;
96 gbl->error_stage = stage;
97 gbl->error_group = group;
98 }
99}
100
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200101static void reg_file_set_group(u16 set_group)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500102{
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200103 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500104}
105
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200106static void reg_file_set_stage(u8 set_stage)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500107{
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200108 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500109}
110
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200111static void reg_file_set_sub_stage(u8 set_sub_stage)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500112{
Marek Vasut2c0d2d92015-07-12 21:10:24 +0200113 set_sub_stage &= 0xff;
114 clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500115}
116
Marek Vasut7c89c2d2015-07-17 01:36:32 +0200117/**
118 * phy_mgr_initialize() - Initialize PHY Manager
119 *
120 * Initialize PHY Manager.
121 */
Marek Vasut9fa9c902015-07-17 01:12:07 +0200122static void phy_mgr_initialize(void)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500123{
Marek Vasut7c89c2d2015-07-17 01:36:32 +0200124 u32 ratio;
125
Dinh Nguyen3da42852015-06-02 22:52:49 -0500126 debug("%s:%d\n", __func__, __LINE__);
Marek Vasut7c89c2d2015-07-17 01:36:32 +0200127 /* Calibration has control over path to memory */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500128 /*
129 * In Hard PHY this is a 2-bit control:
130 * 0: AFI Mux Select
131 * 1: DDIO Mux Select
132 */
Marek Vasut1273dd92015-07-12 21:05:08 +0200133 writel(0x3, &phy_mgr_cfg->mux_sel);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500134
135 /* USER memory clock is not stable we begin initialization */
Marek Vasut1273dd92015-07-12 21:05:08 +0200136 writel(0, &phy_mgr_cfg->reset_mem_stbl);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500137
138 /* USER calibration status all set to zero */
Marek Vasut1273dd92015-07-12 21:05:08 +0200139 writel(0, &phy_mgr_cfg->cal_status);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500140
Marek Vasut1273dd92015-07-12 21:05:08 +0200141 writel(0, &phy_mgr_cfg->cal_debug_info);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500142
Marek Vasut7c89c2d2015-07-17 01:36:32 +0200143 /* Init params only if we do NOT skip calibration. */
144 if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
145 return;
146
147 ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
148 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
149 param->read_correct_mask_vg = (1 << ratio) - 1;
150 param->write_correct_mask_vg = (1 << ratio) - 1;
151 param->read_correct_mask = (1 << RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
152 param->write_correct_mask = (1 << RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
153 ratio = RW_MGR_MEM_DATA_WIDTH /
154 RW_MGR_MEM_DATA_MASK_WIDTH;
155 param->dm_correct_mask = (1 << ratio) - 1;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500156}
157
Marek Vasut080bf642015-07-20 08:15:57 +0200158/**
159 * set_rank_and_odt_mask() - Set Rank and ODT mask
160 * @rank: Rank mask
161 * @odt_mode: ODT mode, OFF or READ_WRITE
162 *
163 * Set Rank and ODT mask (On-Die Termination).
164 */
Marek Vasutb2dfd102015-07-20 08:03:11 +0200165static void set_rank_and_odt_mask(const u32 rank, const u32 odt_mode)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500166{
Marek Vasutb2dfd102015-07-20 08:03:11 +0200167 u32 odt_mask_0 = 0;
168 u32 odt_mask_1 = 0;
169 u32 cs_and_odt_mask;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500170
Marek Vasutb2dfd102015-07-20 08:03:11 +0200171 if (odt_mode == RW_MGR_ODT_MODE_OFF) {
172 odt_mask_0 = 0x0;
173 odt_mask_1 = 0x0;
174 } else { /* RW_MGR_ODT_MODE_READ_WRITE */
Marek Vasut287cdf62015-07-20 08:09:05 +0200175 switch (RW_MGR_MEM_NUMBER_OF_RANKS) {
176 case 1: /* 1 Rank */
177 /* Read: ODT = 0 ; Write: ODT = 1 */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500178 odt_mask_0 = 0x0;
179 odt_mask_1 = 0x1;
Marek Vasut287cdf62015-07-20 08:09:05 +0200180 break;
181 case 2: /* 2 Ranks */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500182 if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
Marek Vasut080bf642015-07-20 08:15:57 +0200183 /*
184 * - Dual-Slot , Single-Rank (1 CS per DIMM)
185 * OR
186 * - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
187 *
188 * Since MEM_NUMBER_OF_RANKS is 2, they
189 * are both single rank with 2 CS each
190 * (special for RDIMM).
191 *
Dinh Nguyen3da42852015-06-02 22:52:49 -0500192 * Read: Turn on ODT on the opposite rank
193 * Write: Turn on ODT on all ranks
194 */
195 odt_mask_0 = 0x3 & ~(1 << rank);
196 odt_mask_1 = 0x3;
197 } else {
198 /*
Marek Vasut080bf642015-07-20 08:15:57 +0200199 * - Single-Slot , Dual-Rank (2 CS per DIMM)
200 *
201 * Read: Turn on ODT off on all ranks
202 * Write: Turn on ODT on active rank
Dinh Nguyen3da42852015-06-02 22:52:49 -0500203 */
204 odt_mask_0 = 0x0;
205 odt_mask_1 = 0x3 & (1 << rank);
206 }
Marek Vasut287cdf62015-07-20 08:09:05 +0200207 break;
208 case 4: /* 4 Ranks */
209 /* Read:
Dinh Nguyen3da42852015-06-02 22:52:49 -0500210 * ----------+-----------------------+
Dinh Nguyen3da42852015-06-02 22:52:49 -0500211 * | ODT |
212 * Read From +-----------------------+
213 * Rank | 3 | 2 | 1 | 0 |
214 * ----------+-----+-----+-----+-----+
215 * 0 | 0 | 1 | 0 | 0 |
216 * 1 | 1 | 0 | 0 | 0 |
217 * 2 | 0 | 0 | 0 | 1 |
218 * 3 | 0 | 0 | 1 | 0 |
219 * ----------+-----+-----+-----+-----+
220 *
221 * Write:
222 * ----------+-----------------------+
Dinh Nguyen3da42852015-06-02 22:52:49 -0500223 * | ODT |
224 * Write To +-----------------------+
225 * Rank | 3 | 2 | 1 | 0 |
226 * ----------+-----+-----+-----+-----+
227 * 0 | 0 | 1 | 0 | 1 |
228 * 1 | 1 | 0 | 1 | 0 |
229 * 2 | 0 | 1 | 0 | 1 |
230 * 3 | 1 | 0 | 1 | 0 |
231 * ----------+-----+-----+-----+-----+
232 */
233 switch (rank) {
234 case 0:
235 odt_mask_0 = 0x4;
236 odt_mask_1 = 0x5;
237 break;
238 case 1:
239 odt_mask_0 = 0x8;
240 odt_mask_1 = 0xA;
241 break;
242 case 2:
243 odt_mask_0 = 0x1;
244 odt_mask_1 = 0x5;
245 break;
246 case 3:
247 odt_mask_0 = 0x2;
248 odt_mask_1 = 0xA;
249 break;
250 }
Marek Vasut287cdf62015-07-20 08:09:05 +0200251 break;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500252 }
Dinh Nguyen3da42852015-06-02 22:52:49 -0500253 }
254
Marek Vasutb2dfd102015-07-20 08:03:11 +0200255 cs_and_odt_mask = (0xFF & ~(1 << rank)) |
256 ((0xFF & odt_mask_0) << 8) |
257 ((0xFF & odt_mask_1) << 16);
Marek Vasut1273dd92015-07-12 21:05:08 +0200258 writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
259 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500260}
261
Marek Vasutc76976d2015-07-12 22:28:33 +0200262/**
263 * scc_mgr_set() - Set SCC Manager register
264 * @off: Base offset in SCC Manager space
265 * @grp: Read/Write group
266 * @val: Value to be set
267 *
268 * This function sets the SCC Manager (Scan Chain Control Manager) register.
269 */
270static void scc_mgr_set(u32 off, u32 grp, u32 val)
271{
272 writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
273}
274
Marek Vasute893f4d2015-07-20 07:16:42 +0200275/**
276 * scc_mgr_initialize() - Initialize SCC Manager registers
277 *
278 * Initialize SCC Manager registers.
279 */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500280static void scc_mgr_initialize(void)
281{
Dinh Nguyen3da42852015-06-02 22:52:49 -0500282 /*
Marek Vasute893f4d2015-07-20 07:16:42 +0200283 * Clear register file for HPS. 16 (2^4) is the size of the
284 * full register file in the scc mgr:
285 * RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
286 * MEM_IF_READ_DQS_WIDTH - 1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500287 */
Marek Vasutc76976d2015-07-12 22:28:33 +0200288 int i;
Marek Vasute893f4d2015-07-20 07:16:42 +0200289
Dinh Nguyen3da42852015-06-02 22:52:49 -0500290 for (i = 0; i < 16; i++) {
Marek Vasut7ac40d22015-06-26 18:56:54 +0200291 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
Dinh Nguyen3da42852015-06-02 22:52:49 -0500292 __func__, __LINE__, i);
Marek Vasutc76976d2015-07-12 22:28:33 +0200293 scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500294 }
295}
296
Marek Vasut5ff825b2015-07-12 22:11:55 +0200297static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
298{
Marek Vasutc76976d2015-07-12 22:28:33 +0200299 scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200300}
301
302static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500303{
Marek Vasutc76976d2015-07-12 22:28:33 +0200304 scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500305}
306
Dinh Nguyen3da42852015-06-02 22:52:49 -0500307static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
308{
Marek Vasutc76976d2015-07-12 22:28:33 +0200309 scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500310}
311
Marek Vasut5ff825b2015-07-12 22:11:55 +0200312static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
313{
Marek Vasutc76976d2015-07-12 22:28:33 +0200314 scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200315}
316
Marek Vasut32675242015-07-17 06:07:13 +0200317static void scc_mgr_set_dqs_io_in_delay(uint32_t delay)
Marek Vasut5ff825b2015-07-12 22:11:55 +0200318{
Marek Vasutc76976d2015-07-12 22:28:33 +0200319 scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
320 delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200321}
322
323static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
324{
Marek Vasutc76976d2015-07-12 22:28:33 +0200325 scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200326}
327
328static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
329{
Marek Vasutc76976d2015-07-12 22:28:33 +0200330 scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200331}
332
Marek Vasut32675242015-07-17 06:07:13 +0200333static void scc_mgr_set_dqs_out1_delay(uint32_t delay)
Marek Vasut5ff825b2015-07-12 22:11:55 +0200334{
Marek Vasutc76976d2015-07-12 22:28:33 +0200335 scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
336 delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200337}
338
339static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
340{
Marek Vasutc76976d2015-07-12 22:28:33 +0200341 scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
342 RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
343 delay);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200344}
345
346/* load up dqs config settings */
347static void scc_mgr_load_dqs(uint32_t dqs)
348{
349 writel(dqs, &sdr_scc_mgr->dqs_ena);
350}
351
352/* load up dqs io config settings */
353static void scc_mgr_load_dqs_io(void)
354{
355 writel(0, &sdr_scc_mgr->dqs_io_ena);
356}
357
358/* load up dq config settings */
359static void scc_mgr_load_dq(uint32_t dq_in_group)
360{
361 writel(dq_in_group, &sdr_scc_mgr->dq_ena);
362}
363
364/* load up dm config settings */
365static void scc_mgr_load_dm(uint32_t dm)
366{
367 writel(dm, &sdr_scc_mgr->dm_ena);
368}
369
Marek Vasut0b69b802015-07-12 23:25:21 +0200370/**
371 * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
372 * @off: Base offset in SCC Manager space
373 * @grp: Read/Write group
374 * @val: Value to be set
375 * @update: If non-zero, trigger SCC Manager update for all ranks
376 *
377 * This function sets the SCC Manager (Scan Chain Control Manager) register
378 * and optionally triggers the SCC update for all ranks.
379 */
380static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
381 const int update)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500382{
Marek Vasut0b69b802015-07-12 23:25:21 +0200383 u32 r;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500384
385 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
386 r += NUM_RANKS_PER_SHADOW_REG) {
Marek Vasut0b69b802015-07-12 23:25:21 +0200387 scc_mgr_set(off, grp, val);
Marek Vasut162d60e2015-07-12 23:14:33 +0200388
Marek Vasut0b69b802015-07-12 23:25:21 +0200389 if (update || (r == 0)) {
390 writel(grp, &sdr_scc_mgr->dqs_ena);
Marek Vasut1273dd92015-07-12 21:05:08 +0200391 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500392 }
393 }
394}
395
Marek Vasut0b69b802015-07-12 23:25:21 +0200396static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
397{
398 /*
399 * USER although the h/w doesn't support different phases per
400 * shadow register, for simplicity our scc manager modeling
401 * keeps different phase settings per shadow reg, and it's
402 * important for us to keep them in sync to match h/w.
403 * for efficiency, the scan chain update should occur only
404 * once to sr0.
405 */
406 scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
407 read_group, phase, 0);
408}
409
Dinh Nguyen3da42852015-06-02 22:52:49 -0500410static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
411 uint32_t phase)
412{
Marek Vasut0b69b802015-07-12 23:25:21 +0200413 /*
414 * USER although the h/w doesn't support different phases per
415 * shadow register, for simplicity our scc manager modeling
416 * keeps different phase settings per shadow reg, and it's
417 * important for us to keep them in sync to match h/w.
418 * for efficiency, the scan chain update should occur only
419 * once to sr0.
420 */
421 scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
422 write_group, phase, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500423}
424
Dinh Nguyen3da42852015-06-02 22:52:49 -0500425static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
426 uint32_t delay)
427{
Dinh Nguyen3da42852015-06-02 22:52:49 -0500428 /*
429 * In shadow register mode, the T11 settings are stored in
430 * registers in the core, which are updated by the DQS_ENA
431 * signals. Not issuing the SCC_MGR_UPD command allows us to
432 * save lots of rank switching overhead, by calling
433 * select_shadow_regs_for_update with update_scan_chains
434 * set to 0.
435 */
Marek Vasut0b69b802015-07-12 23:25:21 +0200436 scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
437 read_group, delay, 1);
Marek Vasut1273dd92015-07-12 21:05:08 +0200438 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500439}
440
Marek Vasut5be355c2015-07-12 23:39:06 +0200441/**
442 * scc_mgr_set_oct_out1_delay() - Set OCT output delay
443 * @write_group: Write group
444 * @delay: Delay value
445 *
446 * This function sets the OCT output delay in SCC manager.
447 */
448static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500449{
Marek Vasut5be355c2015-07-12 23:39:06 +0200450 const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
451 RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
452 const int base = write_group * ratio;
453 int i;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500454 /*
455 * Load the setting in the SCC manager
456 * Although OCT affects only write data, the OCT delay is controlled
457 * by the DQS logic block which is instantiated once per read group.
458 * For protocols where a write group consists of multiple read groups,
459 * the setting must be set multiple times.
460 */
Marek Vasut5be355c2015-07-12 23:39:06 +0200461 for (i = 0; i < ratio; i++)
462 scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500463}
464
Marek Vasut37a37ca2015-07-19 01:32:55 +0200465/**
466 * scc_mgr_set_hhp_extras() - Set HHP extras.
467 *
468 * Load the fixed setting in the SCC manager HHP extras.
469 */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500470static void scc_mgr_set_hhp_extras(void)
471{
472 /*
473 * Load the fixed setting in the SCC manager
Marek Vasut37a37ca2015-07-19 01:32:55 +0200474 * bits: 0:0 = 1'b1 - DQS bypass
475 * bits: 1:1 = 1'b1 - DQ bypass
476 * bits: 4:2 = 3'b001 - rfifo_mode
477 * bits: 6:5 = 2'b01 - rfifo clock_select
478 * bits: 7:7 = 1'b0 - separate gating from ungating setting
479 * bits: 8:8 = 1'b0 - separate OE from Output delay setting
Dinh Nguyen3da42852015-06-02 22:52:49 -0500480 */
Marek Vasut37a37ca2015-07-19 01:32:55 +0200481 const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
482 (1 << 2) | (1 << 1) | (1 << 0);
483 const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
484 SCC_MGR_HHP_GLOBALS_OFFSET |
485 SCC_MGR_HHP_EXTRAS_OFFSET;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500486
Marek Vasut37a37ca2015-07-19 01:32:55 +0200487 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n",
488 __func__, __LINE__);
489 writel(value, addr);
490 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
491 __func__, __LINE__);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500492}
493
Marek Vasutf42af352015-07-20 04:41:53 +0200494/**
495 * scc_mgr_zero_all() - Zero all DQS config
496 *
497 * Zero all DQS config.
Dinh Nguyen3da42852015-06-02 22:52:49 -0500498 */
499static void scc_mgr_zero_all(void)
500{
Marek Vasutf42af352015-07-20 04:41:53 +0200501 int i, r;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500502
503 /*
504 * USER Zero all DQS config settings, across all groups and all
505 * shadow registers
506 */
Marek Vasutf42af352015-07-20 04:41:53 +0200507 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
508 r += NUM_RANKS_PER_SHADOW_REG) {
Dinh Nguyen3da42852015-06-02 22:52:49 -0500509 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
510 /*
511 * The phases actually don't exist on a per-rank basis,
512 * but there's no harm updating them several times, so
513 * let's keep the code simple.
514 */
515 scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
516 scc_mgr_set_dqs_en_phase(i, 0);
517 scc_mgr_set_dqs_en_delay(i, 0);
518 }
519
520 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
521 scc_mgr_set_dqdqs_output_phase(i, 0);
Marek Vasutf42af352015-07-20 04:41:53 +0200522 /* Arria V/Cyclone V don't have out2. */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500523 scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
524 }
525 }
526
Marek Vasutf42af352015-07-20 04:41:53 +0200527 /* Multicast to all DQS group enables. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200528 writel(0xff, &sdr_scc_mgr->dqs_ena);
529 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500530}
531
Marek Vasutc5c5f532015-07-17 02:06:20 +0200532/**
533 * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
534 * @write_group: Write group
535 *
536 * Set bypass mode and trigger SCC update.
537 */
538static void scc_set_bypass_mode(const u32 write_group)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500539{
Marek Vasutc5c5f532015-07-17 02:06:20 +0200540 /* Multicast to all DQ enables. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200541 writel(0xff, &sdr_scc_mgr->dq_ena);
542 writel(0xff, &sdr_scc_mgr->dm_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500543
Marek Vasutc5c5f532015-07-17 02:06:20 +0200544 /* Update current DQS IO enable. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200545 writel(0, &sdr_scc_mgr->dqs_io_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500546
Marek Vasutc5c5f532015-07-17 02:06:20 +0200547 /* Update the DQS logic. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200548 writel(write_group, &sdr_scc_mgr->dqs_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500549
Marek Vasutc5c5f532015-07-17 02:06:20 +0200550 /* Hit update. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200551 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500552}
553
Marek Vasut5e837892015-07-13 00:30:09 +0200554/**
555 * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
556 * @write_group: Write group
557 *
558 * Load DQS settings for Write Group, do not trigger SCC update.
559 */
560static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
Marek Vasut5ff825b2015-07-12 22:11:55 +0200561{
Marek Vasut5e837892015-07-13 00:30:09 +0200562 const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
563 RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
564 const int base = write_group * ratio;
565 int i;
Marek Vasut5ff825b2015-07-12 22:11:55 +0200566 /*
Marek Vasut5e837892015-07-13 00:30:09 +0200567 * Load the setting in the SCC manager
Marek Vasut5ff825b2015-07-12 22:11:55 +0200568 * Although OCT affects only write data, the OCT delay is controlled
569 * by the DQS logic block which is instantiated once per read group.
570 * For protocols where a write group consists of multiple read groups,
Marek Vasut5e837892015-07-13 00:30:09 +0200571 * the setting must be set multiple times.
Marek Vasut5ff825b2015-07-12 22:11:55 +0200572 */
Marek Vasut5e837892015-07-13 00:30:09 +0200573 for (i = 0; i < ratio; i++)
574 writel(base + i, &sdr_scc_mgr->dqs_ena);
Marek Vasut5ff825b2015-07-12 22:11:55 +0200575}
576
Marek Vasutd41ea932015-07-20 08:41:04 +0200577/**
578 * scc_mgr_zero_group() - Zero all configs for a group
579 *
580 * Zero DQ, DM, DQS and OCT configs for a group.
581 */
582static void scc_mgr_zero_group(const u32 write_group, const int out_only)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500583{
Marek Vasutd41ea932015-07-20 08:41:04 +0200584 int i, r;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500585
Marek Vasutd41ea932015-07-20 08:41:04 +0200586 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
587 r += NUM_RANKS_PER_SHADOW_REG) {
588 /* Zero all DQ config settings. */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500589 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
Marek Vasut07aee5b2015-07-12 22:07:33 +0200590 scc_mgr_set_dq_out1_delay(i, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500591 if (!out_only)
Marek Vasut07aee5b2015-07-12 22:07:33 +0200592 scc_mgr_set_dq_in_delay(i, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500593 }
594
Marek Vasutd41ea932015-07-20 08:41:04 +0200595 /* Multicast to all DQ enables. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200596 writel(0xff, &sdr_scc_mgr->dq_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500597
Marek Vasutd41ea932015-07-20 08:41:04 +0200598 /* Zero all DM config settings. */
599 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
Marek Vasut07aee5b2015-07-12 22:07:33 +0200600 scc_mgr_set_dm_out1_delay(i, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500601
Marek Vasutd41ea932015-07-20 08:41:04 +0200602 /* Multicast to all DM enables. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200603 writel(0xff, &sdr_scc_mgr->dm_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500604
Marek Vasutd41ea932015-07-20 08:41:04 +0200605 /* Zero all DQS IO settings. */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500606 if (!out_only)
Marek Vasut32675242015-07-17 06:07:13 +0200607 scc_mgr_set_dqs_io_in_delay(0);
Marek Vasutd41ea932015-07-20 08:41:04 +0200608
609 /* Arria V/Cyclone V don't have out2. */
Marek Vasut32675242015-07-17 06:07:13 +0200610 scc_mgr_set_dqs_out1_delay(IO_DQS_OUT_RESERVE);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500611 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
612 scc_mgr_load_dqs_for_write_group(write_group);
613
Marek Vasutd41ea932015-07-20 08:41:04 +0200614 /* Multicast to all DQS IO enables (only 1 in total). */
Marek Vasut1273dd92015-07-12 21:05:08 +0200615 writel(0, &sdr_scc_mgr->dqs_io_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500616
Marek Vasutd41ea932015-07-20 08:41:04 +0200617 /* Hit update to zero everything. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200618 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500619 }
620}
621
Dinh Nguyen3da42852015-06-02 22:52:49 -0500622/*
623 * apply and load a particular input delay for the DQ pins in a group
624 * group_bgn is the index of the first dq pin (in the write group)
625 */
Marek Vasut32675242015-07-17 06:07:13 +0200626static void scc_mgr_apply_group_dq_in_delay(uint32_t group_bgn, uint32_t delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500627{
628 uint32_t i, p;
629
630 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
Marek Vasut07aee5b2015-07-12 22:07:33 +0200631 scc_mgr_set_dq_in_delay(p, delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500632 scc_mgr_load_dq(p);
633 }
634}
635
Marek Vasut300c2e62015-07-17 05:42:49 +0200636/**
637 * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
638 * @delay: Delay value
639 *
640 * Apply and load a particular output delay for the DQ pins in a group.
641 */
642static void scc_mgr_apply_group_dq_out1_delay(const u32 delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500643{
Marek Vasut300c2e62015-07-17 05:42:49 +0200644 int i;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500645
Marek Vasut300c2e62015-07-17 05:42:49 +0200646 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
647 scc_mgr_set_dq_out1_delay(i, delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500648 scc_mgr_load_dq(i);
649 }
650}
651
652/* apply and load a particular output delay for the DM pins in a group */
Marek Vasut32675242015-07-17 06:07:13 +0200653static void scc_mgr_apply_group_dm_out1_delay(uint32_t delay1)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500654{
655 uint32_t i;
656
657 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
Marek Vasut07aee5b2015-07-12 22:07:33 +0200658 scc_mgr_set_dm_out1_delay(i, delay1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500659 scc_mgr_load_dm(i);
660 }
661}
662
663
664/* apply and load delay on both DQS and OCT out1 */
665static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
666 uint32_t delay)
667{
Marek Vasut32675242015-07-17 06:07:13 +0200668 scc_mgr_set_dqs_out1_delay(delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500669 scc_mgr_load_dqs_io();
670
671 scc_mgr_set_oct_out1_delay(write_group, delay);
672 scc_mgr_load_dqs_for_write_group(write_group);
673}
674
Marek Vasut5cb1b502015-07-17 05:33:28 +0200675/**
676 * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, OCT
677 * @write_group: Write group
678 * @delay: Delay value
679 *
680 * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
681 */
Marek Vasut8eccde32015-07-17 05:30:14 +0200682static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group,
Marek Vasut8eccde32015-07-17 05:30:14 +0200683 const u32 delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500684{
Marek Vasut8eccde32015-07-17 05:30:14 +0200685 u32 i, new_delay;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500686
Marek Vasut8eccde32015-07-17 05:30:14 +0200687 /* DQ shift */
688 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500689 scc_mgr_load_dq(i);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500690
Marek Vasut8eccde32015-07-17 05:30:14 +0200691 /* DM shift */
692 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500693 scc_mgr_load_dm(i);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500694
Marek Vasut5cb1b502015-07-17 05:33:28 +0200695 /* DQS shift */
696 new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500697 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
Marek Vasut5cb1b502015-07-17 05:33:28 +0200698 debug_cond(DLEVEL == 1,
699 "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
700 __func__, __LINE__, write_group, delay, new_delay,
701 IO_IO_OUT2_DELAY_MAX,
Dinh Nguyen3da42852015-06-02 22:52:49 -0500702 new_delay - IO_IO_OUT2_DELAY_MAX);
Marek Vasut5cb1b502015-07-17 05:33:28 +0200703 new_delay -= IO_IO_OUT2_DELAY_MAX;
704 scc_mgr_set_dqs_out1_delay(new_delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500705 }
706
707 scc_mgr_load_dqs_io();
708
Marek Vasut5cb1b502015-07-17 05:33:28 +0200709 /* OCT shift */
710 new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500711 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
Marek Vasut5cb1b502015-07-17 05:33:28 +0200712 debug_cond(DLEVEL == 1,
713 "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
714 __func__, __LINE__, write_group, delay,
715 new_delay, IO_IO_OUT2_DELAY_MAX,
Dinh Nguyen3da42852015-06-02 22:52:49 -0500716 new_delay - IO_IO_OUT2_DELAY_MAX);
Marek Vasut5cb1b502015-07-17 05:33:28 +0200717 new_delay -= IO_IO_OUT2_DELAY_MAX;
718 scc_mgr_set_oct_out1_delay(write_group, new_delay);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500719 }
720
721 scc_mgr_load_dqs_for_write_group(write_group);
722}
723
Marek Vasutf51a7d32015-07-19 02:18:21 +0200724/**
725 * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
726 * @write_group: Write group
727 * @delay: Delay value
728 *
729 * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
Dinh Nguyen3da42852015-06-02 22:52:49 -0500730 */
Marek Vasutf51a7d32015-07-19 02:18:21 +0200731static void
732scc_mgr_apply_group_all_out_delay_add_all_ranks(const u32 write_group,
733 const u32 delay)
Dinh Nguyen3da42852015-06-02 22:52:49 -0500734{
Marek Vasutf51a7d32015-07-19 02:18:21 +0200735 int r;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500736
737 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
Marek Vasutf51a7d32015-07-19 02:18:21 +0200738 r += NUM_RANKS_PER_SHADOW_REG) {
Marek Vasut5cb1b502015-07-17 05:33:28 +0200739 scc_mgr_apply_group_all_out_delay_add(write_group, delay);
Marek Vasut1273dd92015-07-12 21:05:08 +0200740 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500741 }
742}
743
Marek Vasutf936f942015-07-26 11:07:19 +0200744/**
745 * set_jump_as_return() - Return instruction optimization
746 *
747 * Optimization used to recover some slots in ddr3 inst_rom could be
748 * applied to other protocols if we wanted to
749 */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500750static void set_jump_as_return(void)
751{
Dinh Nguyen3da42852015-06-02 22:52:49 -0500752 /*
Marek Vasutf936f942015-07-26 11:07:19 +0200753 * To save space, we replace return with jump to special shared
Dinh Nguyen3da42852015-06-02 22:52:49 -0500754 * RETURN instruction so we set the counter to large value so that
Marek Vasutf936f942015-07-26 11:07:19 +0200755 * we always jump.
Dinh Nguyen3da42852015-06-02 22:52:49 -0500756 */
Marek Vasut1273dd92015-07-12 21:05:08 +0200757 writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
758 writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500759}
760
761/*
762 * should always use constants as argument to ensure all computations are
763 * performed at compile time
764 */
765static void delay_for_n_mem_clocks(const uint32_t clocks)
766{
767 uint32_t afi_clocks;
768 uint8_t inner = 0;
769 uint8_t outer = 0;
770 uint16_t c_loop = 0;
Dinh Nguyen3da42852015-06-02 22:52:49 -0500771
772 debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
773
774
775 afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
776 /* scale (rounding up) to get afi clocks */
777
778 /*
779 * Note, we don't bother accounting for being off a little bit
780 * because of a few extra instructions in outer loops
781 * Note, the loops have a test at the end, and do the test before
782 * the decrement, and so always perform the loop
783 * 1 time more than the counter value
784 */
785 if (afi_clocks == 0) {
786 ;
787 } else if (afi_clocks <= 0x100) {
788 inner = afi_clocks-1;
789 outer = 0;
790 c_loop = 0;
791 } else if (afi_clocks <= 0x10000) {
792 inner = 0xff;
793 outer = (afi_clocks-1) >> 8;
794 c_loop = 0;
795 } else {
796 inner = 0xff;
797 outer = 0xff;
798 c_loop = (afi_clocks-1) >> 16;
799 }
800
801 /*
802 * rom instructions are structured as follows:
803 *
804 * IDLE_LOOP2: jnz cntr0, TARGET_A
805 * IDLE_LOOP1: jnz cntr1, TARGET_B
806 * return
807 *
808 * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
809 * TARGET_B is set to IDLE_LOOP2 as well
810 *
811 * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
812 * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
813 *
814 * a little confusing, but it helps save precious space in the inst_rom
815 * and sequencer rom and keeps the delays more accurate and reduces
816 * overhead
817 */
818 if (afi_clocks <= 0x100) {
Marek Vasut1273dd92015-07-12 21:05:08 +0200819 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
820 &sdr_rw_load_mgr_regs->load_cntr1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500821
Marek Vasut1273dd92015-07-12 21:05:08 +0200822 writel(RW_MGR_IDLE_LOOP1,
823 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500824
Marek Vasut1273dd92015-07-12 21:05:08 +0200825 writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
826 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500827 } else {
Marek Vasut1273dd92015-07-12 21:05:08 +0200828 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
829 &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500830
Marek Vasut1273dd92015-07-12 21:05:08 +0200831 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
832 &sdr_rw_load_mgr_regs->load_cntr1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500833
Marek Vasut1273dd92015-07-12 21:05:08 +0200834 writel(RW_MGR_IDLE_LOOP2,
835 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500836
Marek Vasut1273dd92015-07-12 21:05:08 +0200837 writel(RW_MGR_IDLE_LOOP2,
838 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500839
840 /* hack to get around compiler not being smart enough */
841 if (afi_clocks <= 0x10000) {
842 /* only need to run once */
Marek Vasut1273dd92015-07-12 21:05:08 +0200843 writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
844 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500845 } else {
846 do {
Marek Vasut1273dd92015-07-12 21:05:08 +0200847 writel(RW_MGR_IDLE_LOOP2,
848 SDR_PHYGRP_RWMGRGRP_ADDRESS |
849 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500850 } while (c_loop-- != 0);
851 }
852 }
853 debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
854}
855
Marek Vasut944fe712015-07-13 00:44:30 +0200856/**
857 * rw_mgr_mem_init_load_regs() - Load instruction registers
858 * @cntr0: Counter 0 value
859 * @cntr1: Counter 1 value
860 * @cntr2: Counter 2 value
861 * @jump: Jump instruction value
862 *
863 * Load instruction registers.
864 */
865static void rw_mgr_mem_init_load_regs(u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
866{
867 uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
868 RW_MGR_RUN_SINGLE_GROUP_OFFSET;
869
870 /* Load counters */
871 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
872 &sdr_rw_load_mgr_regs->load_cntr0);
873 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
874 &sdr_rw_load_mgr_regs->load_cntr1);
875 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
876 &sdr_rw_load_mgr_regs->load_cntr2);
877
878 /* Load jump address */
879 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
880 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
881 writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
882
883 /* Execute count instruction */
884 writel(jump, grpaddr);
885}
886
Marek Vasutecd23342015-07-13 00:51:05 +0200887/**
888 * rw_mgr_mem_load_user() - Load user calibration values
889 * @fin1: Final instruction 1
890 * @fin2: Final instruction 2
891 * @precharge: If 1, precharge the banks at the end
892 *
893 * Load user calibration values and optionally precharge the banks.
894 */
895static void rw_mgr_mem_load_user(const u32 fin1, const u32 fin2,
896 const int precharge)
897{
898 u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
899 RW_MGR_RUN_SINGLE_GROUP_OFFSET;
900 u32 r;
901
902 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
903 if (param->skip_ranks[r]) {
904 /* request to skip the rank */
905 continue;
906 }
907
908 /* set rank */
909 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
910
911 /* precharge all banks ... */
912 if (precharge)
913 writel(RW_MGR_PRECHARGE_ALL, grpaddr);
914
915 /*
916 * USER Use Mirror-ed commands for odd ranks if address
917 * mirrorring is on
918 */
919 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
920 set_jump_as_return();
921 writel(RW_MGR_MRS2_MIRR, grpaddr);
922 delay_for_n_mem_clocks(4);
923 set_jump_as_return();
924 writel(RW_MGR_MRS3_MIRR, grpaddr);
925 delay_for_n_mem_clocks(4);
926 set_jump_as_return();
927 writel(RW_MGR_MRS1_MIRR, grpaddr);
928 delay_for_n_mem_clocks(4);
929 set_jump_as_return();
930 writel(fin1, grpaddr);
931 } else {
932 set_jump_as_return();
933 writel(RW_MGR_MRS2, grpaddr);
934 delay_for_n_mem_clocks(4);
935 set_jump_as_return();
936 writel(RW_MGR_MRS3, grpaddr);
937 delay_for_n_mem_clocks(4);
938 set_jump_as_return();
939 writel(RW_MGR_MRS1, grpaddr);
940 set_jump_as_return();
941 writel(fin2, grpaddr);
942 }
943
944 if (precharge)
945 continue;
946
947 set_jump_as_return();
948 writel(RW_MGR_ZQCL, grpaddr);
949
950 /* tZQinit = tDLLK = 512 ck cycles */
951 delay_for_n_mem_clocks(512);
952 }
953}
954
Marek Vasut8e9d7d02015-07-26 10:57:06 +0200955/**
956 * rw_mgr_mem_initialize() - Initialize RW Manager
957 *
958 * Initialize RW Manager.
959 */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500960static void rw_mgr_mem_initialize(void)
961{
Dinh Nguyen3da42852015-06-02 22:52:49 -0500962 debug("%s:%d\n", __func__, __LINE__);
963
964 /* The reset / cke part of initialization is broadcasted to all ranks */
Marek Vasut1273dd92015-07-12 21:05:08 +0200965 writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
966 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500967
968 /*
969 * Here's how you load register for a loop
970 * Counters are located @ 0x800
971 * Jump address are located @ 0xC00
972 * For both, registers 0 to 3 are selected using bits 3 and 2, like
973 * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
974 * I know this ain't pretty, but Avalon bus throws away the 2 least
975 * significant bits
976 */
977
Marek Vasut8e9d7d02015-07-26 10:57:06 +0200978 /* Start with memory RESET activated */
Dinh Nguyen3da42852015-06-02 22:52:49 -0500979
980 /* tINIT = 200us */
981
982 /*
983 * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
984 * If a and b are the number of iteration in 2 nested loops
985 * it takes the following number of cycles to complete the operation:
986 * number_of_cycles = ((2 + n) * a + 2) * b
987 * where n is the number of instruction in the inner loop
988 * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
989 * b = 6A
990 */
Marek Vasut944fe712015-07-13 00:44:30 +0200991 rw_mgr_mem_init_load_regs(SEQ_TINIT_CNTR0_VAL, SEQ_TINIT_CNTR1_VAL,
992 SEQ_TINIT_CNTR2_VAL,
993 RW_MGR_INIT_RESET_0_CKE_0);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500994
Marek Vasut8e9d7d02015-07-26 10:57:06 +0200995 /* Indicate that memory is stable. */
Marek Vasut1273dd92015-07-12 21:05:08 +0200996 writel(1, &phy_mgr_cfg->reset_mem_stbl);
Dinh Nguyen3da42852015-06-02 22:52:49 -0500997
998 /*
999 * transition the RESET to high
1000 * Wait for 500us
1001 */
1002
1003 /*
1004 * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
1005 * If a and b are the number of iteration in 2 nested loops
1006 * it takes the following number of cycles to complete the operation
1007 * number_of_cycles = ((2 + n) * a + 2) * b
1008 * where n is the number of instruction in the inner loop
1009 * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
1010 * b = FF
1011 */
Marek Vasut944fe712015-07-13 00:44:30 +02001012 rw_mgr_mem_init_load_regs(SEQ_TRESET_CNTR0_VAL, SEQ_TRESET_CNTR1_VAL,
1013 SEQ_TRESET_CNTR2_VAL,
1014 RW_MGR_INIT_RESET_1_CKE_0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001015
Marek Vasut8e9d7d02015-07-26 10:57:06 +02001016 /* Bring up clock enable. */
Dinh Nguyen3da42852015-06-02 22:52:49 -05001017
1018 /* tXRP < 250 ck cycles */
1019 delay_for_n_mem_clocks(250);
1020
Marek Vasutecd23342015-07-13 00:51:05 +02001021 rw_mgr_mem_load_user(RW_MGR_MRS0_DLL_RESET_MIRR, RW_MGR_MRS0_DLL_RESET,
1022 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001023}
1024
1025/*
1026 * At the end of calibration we have to program the user settings in, and
1027 * USER hand off the memory to the user.
1028 */
1029static void rw_mgr_mem_handoff(void)
1030{
Marek Vasutecd23342015-07-13 00:51:05 +02001031 rw_mgr_mem_load_user(RW_MGR_MRS0_USER_MIRR, RW_MGR_MRS0_USER, 1);
1032 /*
1033 * USER need to wait tMOD (12CK or 15ns) time before issuing
1034 * other commands, but we will have plenty of NIOS cycles before
1035 * actual handoff so its okay.
1036 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05001037}
1038
1039/*
1040 * performs a guaranteed read on the patterns we are going to use during a
1041 * read test to ensure memory works
1042 */
1043static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1044 uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1045 uint32_t all_ranks)
1046{
1047 uint32_t r, vg;
1048 uint32_t correct_mask_vg;
1049 uint32_t tmp_bit_chk;
1050 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1051 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1052 uint32_t addr;
1053 uint32_t base_rw_mgr;
1054
1055 *bit_chk = param->read_correct_mask;
1056 correct_mask_vg = param->read_correct_mask_vg;
1057
1058 for (r = rank_bgn; r < rank_end; r++) {
1059 if (param->skip_ranks[r])
1060 /* request to skip the rank */
1061 continue;
1062
1063 /* set rank */
1064 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1065
1066 /* Load up a constant bursts of read commands */
Marek Vasut1273dd92015-07-12 21:05:08 +02001067 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
1068 writel(RW_MGR_GUARANTEED_READ,
1069 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001070
Marek Vasut1273dd92015-07-12 21:05:08 +02001071 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
1072 writel(RW_MGR_GUARANTEED_READ_CONT,
1073 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001074
1075 tmp_bit_chk = 0;
1076 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1077 /* reset the fifos to get pointers to known state */
1078
Marek Vasut1273dd92015-07-12 21:05:08 +02001079 writel(0, &phy_mgr_cmd->fifo_reset);
1080 writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1081 RW_MGR_RESET_READ_DATAPATH_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001082
1083 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1084 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1085
Marek Vasutc4815f72015-07-12 19:03:33 +02001086 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02001087 writel(RW_MGR_GUARANTEED_READ, addr +
Dinh Nguyen3da42852015-06-02 22:52:49 -05001088 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1089 vg) << 2));
1090
Marek Vasut1273dd92015-07-12 21:05:08 +02001091 base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001092 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1093
1094 if (vg == 0)
1095 break;
1096 }
1097 *bit_chk &= tmp_bit_chk;
1098 }
1099
Marek Vasutc4815f72015-07-12 19:03:33 +02001100 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02001101 writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05001102
1103 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1104 debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1105 %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1106 (long unsigned int)(*bit_chk == param->read_correct_mask));
1107 return *bit_chk == param->read_correct_mask;
1108}
1109
1110static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1111 (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1112{
1113 return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1114 num_tries, bit_chk, 1);
1115}
1116
1117/* load up the patterns we are going to use during a read test */
1118static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1119 uint32_t all_ranks)
1120{
1121 uint32_t r;
Dinh Nguyen3da42852015-06-02 22:52:49 -05001122 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1123 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1124
1125 debug("%s:%d\n", __func__, __LINE__);
1126 for (r = rank_bgn; r < rank_end; r++) {
1127 if (param->skip_ranks[r])
1128 /* request to skip the rank */
1129 continue;
1130
1131 /* set rank */
1132 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1133
1134 /* Load up a constant bursts */
Marek Vasut1273dd92015-07-12 21:05:08 +02001135 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001136
Marek Vasut1273dd92015-07-12 21:05:08 +02001137 writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
1138 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001139
Marek Vasut1273dd92015-07-12 21:05:08 +02001140 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001141
Marek Vasut1273dd92015-07-12 21:05:08 +02001142 writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
1143 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001144
Marek Vasut1273dd92015-07-12 21:05:08 +02001145 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001146
Marek Vasut1273dd92015-07-12 21:05:08 +02001147 writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
1148 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001149
Marek Vasut1273dd92015-07-12 21:05:08 +02001150 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001151
Marek Vasut1273dd92015-07-12 21:05:08 +02001152 writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
1153 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001154
Marek Vasut1273dd92015-07-12 21:05:08 +02001155 writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1156 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001157 }
1158
1159 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1160}
1161
1162/*
1163 * try a read and see if it returns correct data back. has dummy reads
1164 * inserted into the mix used to align dqs enable. has more thorough checks
1165 * than the regular read test.
1166 */
1167static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1168 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1169 uint32_t all_groups, uint32_t all_ranks)
1170{
1171 uint32_t r, vg;
1172 uint32_t correct_mask_vg;
1173 uint32_t tmp_bit_chk;
1174 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1175 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1176 uint32_t addr;
1177 uint32_t base_rw_mgr;
1178
1179 *bit_chk = param->read_correct_mask;
1180 correct_mask_vg = param->read_correct_mask_vg;
1181
1182 uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1183 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1184
1185 for (r = rank_bgn; r < rank_end; r++) {
1186 if (param->skip_ranks[r])
1187 /* request to skip the rank */
1188 continue;
1189
1190 /* set rank */
1191 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1192
Marek Vasut1273dd92015-07-12 21:05:08 +02001193 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001194
Marek Vasut1273dd92015-07-12 21:05:08 +02001195 writel(RW_MGR_READ_B2B_WAIT1,
1196 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001197
Marek Vasut1273dd92015-07-12 21:05:08 +02001198 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
1199 writel(RW_MGR_READ_B2B_WAIT2,
1200 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001201
Dinh Nguyen3da42852015-06-02 22:52:49 -05001202 if (quick_read_mode)
Marek Vasut1273dd92015-07-12 21:05:08 +02001203 writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001204 /* need at least two (1+1) reads to capture failures */
1205 else if (all_groups)
Marek Vasut1273dd92015-07-12 21:05:08 +02001206 writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001207 else
Marek Vasut1273dd92015-07-12 21:05:08 +02001208 writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001209
Marek Vasut1273dd92015-07-12 21:05:08 +02001210 writel(RW_MGR_READ_B2B,
1211 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001212 if (all_groups)
1213 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1214 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
Marek Vasut1273dd92015-07-12 21:05:08 +02001215 &sdr_rw_load_mgr_regs->load_cntr3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001216 else
Marek Vasut1273dd92015-07-12 21:05:08 +02001217 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001218
Marek Vasut1273dd92015-07-12 21:05:08 +02001219 writel(RW_MGR_READ_B2B,
1220 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001221
1222 tmp_bit_chk = 0;
1223 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1224 /* reset the fifos to get pointers to known state */
Marek Vasut1273dd92015-07-12 21:05:08 +02001225 writel(0, &phy_mgr_cmd->fifo_reset);
1226 writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1227 RW_MGR_RESET_READ_DATAPATH_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001228
1229 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1230 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1231
Marek Vasutc4815f72015-07-12 19:03:33 +02001232 if (all_groups)
1233 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
1234 else
1235 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1236
Marek Vasut17fdc912015-07-12 20:05:54 +02001237 writel(RW_MGR_READ_B2B, addr +
Dinh Nguyen3da42852015-06-02 22:52:49 -05001238 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1239 vg) << 2));
1240
Marek Vasut1273dd92015-07-12 21:05:08 +02001241 base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001242 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1243
1244 if (vg == 0)
1245 break;
1246 }
1247 *bit_chk &= tmp_bit_chk;
1248 }
1249
Marek Vasutc4815f72015-07-12 19:03:33 +02001250 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02001251 writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05001252
1253 if (all_correct) {
1254 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1255 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1256 (%u == %u) => %lu", __func__, __LINE__, group,
1257 all_groups, *bit_chk, param->read_correct_mask,
1258 (long unsigned int)(*bit_chk ==
1259 param->read_correct_mask));
1260 return *bit_chk == param->read_correct_mask;
1261 } else {
1262 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1263 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1264 (%u != %lu) => %lu\n", __func__, __LINE__,
1265 group, all_groups, *bit_chk, (long unsigned int)0,
1266 (long unsigned int)(*bit_chk != 0x00));
1267 return *bit_chk != 0x00;
1268 }
1269}
1270
1271static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1272 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1273 uint32_t all_groups)
1274{
1275 return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1276 bit_chk, all_groups, 1);
1277}
1278
1279static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1280{
Marek Vasut1273dd92015-07-12 21:05:08 +02001281 writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001282 (*v)++;
1283}
1284
1285static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1286{
1287 uint32_t i;
1288
1289 for (i = 0; i < VFIFO_SIZE-1; i++)
1290 rw_mgr_incr_vfifo(grp, v);
1291}
1292
1293static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1294{
1295 uint32_t v;
1296 uint32_t fail_cnt = 0;
1297 uint32_t test_status;
1298
1299 for (v = 0; v < VFIFO_SIZE; ) {
1300 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1301 __func__, __LINE__, v);
1302 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1303 (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1304 if (!test_status) {
1305 fail_cnt++;
1306
1307 if (fail_cnt == 2)
1308 break;
1309 }
1310
1311 /* fiddle with FIFO */
1312 rw_mgr_incr_vfifo(grp, &v);
1313 }
1314
1315 if (v >= VFIFO_SIZE) {
1316 /* no failing read found!! Something must have gone wrong */
1317 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1318 __func__, __LINE__);
1319 return 0;
1320 } else {
1321 return v;
1322 }
1323}
1324
1325static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1326 uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1327 uint32_t *v, uint32_t *d, uint32_t *p,
1328 uint32_t *i, uint32_t *max_working_cnt)
1329{
1330 uint32_t found_begin = 0;
1331 uint32_t tmp_delay = 0;
1332 uint32_t test_status;
1333
1334 for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1335 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1336 *work_bgn = tmp_delay;
1337 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1338
1339 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1340 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1341 IO_DELAY_PER_OPA_TAP) {
1342 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1343
1344 test_status =
1345 rw_mgr_mem_calibrate_read_test_all_ranks
1346 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1347
1348 if (test_status) {
1349 *max_working_cnt = 1;
1350 found_begin = 1;
1351 break;
1352 }
1353 }
1354
1355 if (found_begin)
1356 break;
1357
1358 if (*p > IO_DQS_EN_PHASE_MAX)
1359 /* fiddle with FIFO */
1360 rw_mgr_incr_vfifo(*grp, v);
1361 }
1362
1363 if (found_begin)
1364 break;
1365 }
1366
1367 if (*i >= VFIFO_SIZE) {
1368 /* cannot find working solution */
1369 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1370 ptap/dtap\n", __func__, __LINE__);
1371 return 0;
1372 } else {
1373 return 1;
1374 }
1375}
1376
1377static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1378 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1379 uint32_t *p, uint32_t *max_working_cnt)
1380{
1381 uint32_t found_begin = 0;
1382 uint32_t tmp_delay;
1383
1384 /* Special case code for backing up a phase */
1385 if (*p == 0) {
1386 *p = IO_DQS_EN_PHASE_MAX;
1387 rw_mgr_decr_vfifo(*grp, v);
1388 } else {
1389 (*p)--;
1390 }
1391 tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1392 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1393
1394 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1395 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1396 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1397
1398 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1399 PASS_ONE_BIT,
1400 bit_chk, 0)) {
1401 found_begin = 1;
1402 *work_bgn = tmp_delay;
1403 break;
1404 }
1405 }
1406
1407 /* We have found a working dtap before the ptap found above */
1408 if (found_begin == 1)
1409 (*max_working_cnt)++;
1410
1411 /*
1412 * Restore VFIFO to old state before we decremented it
1413 * (if needed).
1414 */
1415 (*p)++;
1416 if (*p > IO_DQS_EN_PHASE_MAX) {
1417 *p = 0;
1418 rw_mgr_incr_vfifo(*grp, v);
1419 }
1420
1421 scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1422}
1423
1424static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1425 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1426 uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1427 uint32_t *work_end)
1428{
1429 uint32_t found_end = 0;
1430
1431 (*p)++;
1432 *work_end += IO_DELAY_PER_OPA_TAP;
1433 if (*p > IO_DQS_EN_PHASE_MAX) {
1434 /* fiddle with FIFO */
1435 *p = 0;
1436 rw_mgr_incr_vfifo(*grp, v);
1437 }
1438
1439 for (; *i < VFIFO_SIZE + 1; (*i)++) {
1440 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1441 += IO_DELAY_PER_OPA_TAP) {
1442 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1443
1444 if (!rw_mgr_mem_calibrate_read_test_all_ranks
1445 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1446 found_end = 1;
1447 break;
1448 } else {
1449 (*max_working_cnt)++;
1450 }
1451 }
1452
1453 if (found_end)
1454 break;
1455
1456 if (*p > IO_DQS_EN_PHASE_MAX) {
1457 /* fiddle with FIFO */
1458 rw_mgr_incr_vfifo(*grp, v);
1459 *p = 0;
1460 }
1461 }
1462
1463 if (*i >= VFIFO_SIZE + 1) {
1464 /* cannot see edge of failing read */
1465 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1466 failed\n", __func__, __LINE__);
1467 return 0;
1468 } else {
1469 return 1;
1470 }
1471}
1472
1473static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1474 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1475 uint32_t *p, uint32_t *work_mid,
1476 uint32_t *work_end)
1477{
1478 int i;
1479 int tmp_delay = 0;
1480
1481 *work_mid = (*work_bgn + *work_end) / 2;
1482
1483 debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1484 *work_bgn, *work_end, *work_mid);
1485 /* Get the middle delay to be less than a VFIFO delay */
1486 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1487 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1488 ;
1489 debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1490 while (*work_mid > tmp_delay)
1491 *work_mid -= tmp_delay;
1492 debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1493
1494 tmp_delay = 0;
1495 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1496 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1497 ;
1498 tmp_delay -= IO_DELAY_PER_OPA_TAP;
1499 debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1500 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1501 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1502 ;
1503 debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1504
1505 scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1506 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1507
1508 /*
1509 * push vfifo until we can successfully calibrate. We can do this
1510 * because the largest possible margin in 1 VFIFO cycle.
1511 */
1512 for (i = 0; i < VFIFO_SIZE; i++) {
1513 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1514 *v);
1515 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1516 PASS_ONE_BIT,
1517 bit_chk, 0)) {
1518 break;
1519 }
1520
1521 /* fiddle with FIFO */
1522 rw_mgr_incr_vfifo(*grp, v);
1523 }
1524
1525 if (i >= VFIFO_SIZE) {
1526 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1527 failed\n", __func__, __LINE__);
1528 return 0;
1529 } else {
1530 return 1;
1531 }
1532}
1533
1534/* find a good dqs enable to use */
1535static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1536{
1537 uint32_t v, d, p, i;
1538 uint32_t max_working_cnt;
1539 uint32_t bit_chk;
1540 uint32_t dtaps_per_ptap;
1541 uint32_t work_bgn, work_mid, work_end;
1542 uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
Dinh Nguyen3da42852015-06-02 22:52:49 -05001543
1544 debug("%s:%d %u\n", __func__, __LINE__, grp);
1545
1546 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1547
1548 scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1549 scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1550
1551 /* ************************************************************** */
1552 /* * Step 0 : Determine number of delay taps for each phase tap * */
1553 dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1554
1555 /* ********************************************************* */
1556 /* * Step 1 : First push vfifo until we get a failing read * */
1557 v = find_vfifo_read(grp, &bit_chk);
1558
1559 max_working_cnt = 0;
1560
1561 /* ******************************************************** */
1562 /* * step 2: find first working phase, increment in ptaps * */
1563 work_bgn = 0;
1564 if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1565 &p, &i, &max_working_cnt) == 0)
1566 return 0;
1567
1568 work_end = work_bgn;
1569
1570 /*
1571 * If d is 0 then the working window covers a phase tap and
1572 * we can follow the old procedure otherwise, we've found the beginning,
1573 * and we need to increment the dtaps until we find the end.
1574 */
1575 if (d == 0) {
1576 /* ********************************************************* */
1577 /* * step 3a: if we have room, back off by one and
1578 increment in dtaps * */
1579
1580 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1581 &max_working_cnt);
1582
1583 /* ********************************************************* */
1584 /* * step 4a: go forward from working phase to non working
1585 phase, increment in ptaps * */
1586 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1587 &i, &max_working_cnt, &work_end) == 0)
1588 return 0;
1589
1590 /* ********************************************************* */
1591 /* * step 5a: back off one from last, increment in dtaps * */
1592
1593 /* Special case code for backing up a phase */
1594 if (p == 0) {
1595 p = IO_DQS_EN_PHASE_MAX;
1596 rw_mgr_decr_vfifo(grp, &v);
1597 } else {
1598 p = p - 1;
1599 }
1600
1601 work_end -= IO_DELAY_PER_OPA_TAP;
1602 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1603
1604 /* * The actual increment of dtaps is done outside of
1605 the if/else loop to share code */
1606 d = 0;
1607
1608 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1609 vfifo=%u ptap=%u\n", __func__, __LINE__,
1610 v, p);
1611 } else {
1612 /* ******************************************************* */
1613 /* * step 3-5b: Find the right edge of the window using
1614 delay taps * */
1615 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1616 ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1617 v, p, d, work_bgn);
1618
1619 work_end = work_bgn;
1620
1621 /* * The actual increment of dtaps is done outside of the
1622 if/else loop to share code */
1623
1624 /* Only here to counterbalance a subtract later on which is
1625 not needed if this branch of the algorithm is taken */
1626 max_working_cnt++;
1627 }
1628
1629 /* The dtap increment to find the failing edge is done here */
1630 for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1631 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1632 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1633 end-2: dtap=%u\n", __func__, __LINE__, d);
1634 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1635
1636 if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1637 PASS_ONE_BIT,
1638 &bit_chk, 0)) {
1639 break;
1640 }
1641 }
1642
1643 /* Go back to working dtap */
1644 if (d != 0)
1645 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1646
1647 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1648 ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1649 v, p, d-1, work_end);
1650
1651 if (work_end < work_bgn) {
1652 /* nil range */
1653 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1654 failed\n", __func__, __LINE__);
1655 return 0;
1656 }
1657
1658 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1659 __func__, __LINE__, work_bgn, work_end);
1660
1661 /* *************************************************************** */
1662 /*
1663 * * We need to calculate the number of dtaps that equal a ptap
1664 * * To do that we'll back up a ptap and re-find the edge of the
1665 * * window using dtaps
1666 */
1667
1668 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1669 for tracking\n", __func__, __LINE__);
1670
1671 /* Special case code for backing up a phase */
1672 if (p == 0) {
1673 p = IO_DQS_EN_PHASE_MAX;
1674 rw_mgr_decr_vfifo(grp, &v);
1675 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1676 cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1677 v, p);
1678 } else {
1679 p = p - 1;
1680 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1681 phase only: v=%u p=%u", __func__, __LINE__,
1682 v, p);
1683 }
1684
1685 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1686
1687 /*
1688 * Increase dtap until we first see a passing read (in case the
1689 * window is smaller than a ptap),
1690 * and then a failing read to mark the edge of the window again
1691 */
1692
1693 /* Find a passing read */
1694 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1695 __func__, __LINE__);
1696 found_passing_read = 0;
1697 found_failing_read = 0;
1698 initial_failing_dtap = d;
1699 for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1700 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1701 read d=%u\n", __func__, __LINE__, d);
1702 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1703
1704 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1705 PASS_ONE_BIT,
1706 &bit_chk, 0)) {
1707 found_passing_read = 1;
1708 break;
1709 }
1710 }
1711
1712 if (found_passing_read) {
1713 /* Find a failing read */
1714 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1715 read\n", __func__, __LINE__);
1716 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1717 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1718 testing read d=%u\n", __func__, __LINE__, d);
1719 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1720
1721 if (!rw_mgr_mem_calibrate_read_test_all_ranks
1722 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1723 found_failing_read = 1;
1724 break;
1725 }
1726 }
1727 } else {
1728 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1729 calculate dtaps", __func__, __LINE__);
1730 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1731 }
1732
1733 /*
1734 * The dynamically calculated dtaps_per_ptap is only valid if we
1735 * found a passing/failing read. If we didn't, it means d hit the max
1736 * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1737 * statically calculated value.
1738 */
1739 if (found_passing_read && found_failing_read)
1740 dtaps_per_ptap = d - initial_failing_dtap;
1741
Marek Vasut1273dd92015-07-12 21:05:08 +02001742 writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001743 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1744 - %u = %u", __func__, __LINE__, d,
1745 initial_failing_dtap, dtaps_per_ptap);
1746
1747 /* ******************************************** */
1748 /* * step 6: Find the centre of the window * */
1749 if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1750 &work_mid, &work_end) == 0)
1751 return 0;
1752
1753 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1754 vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1755 v, p-1, d);
1756 return 1;
1757}
1758
1759/*
1760 * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1761 * dq_in_delay values
1762 */
1763static uint32_t
1764rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1765(uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1766{
1767 uint32_t found;
1768 uint32_t i;
1769 uint32_t p;
1770 uint32_t d;
1771 uint32_t r;
Dinh Nguyen3da42852015-06-02 22:52:49 -05001772
1773 const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1774 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1775 /* we start at zero, so have one less dq to devide among */
1776
1777 debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1778 test_bgn);
1779
1780 /* try different dq_in_delays since the dq path is shorter than dqs */
1781
1782 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1783 r += NUM_RANKS_PER_SHADOW_REG) {
Marek Vasut32675242015-07-17 06:07:13 +02001784 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++, d += delay_step) {
Dinh Nguyen3da42852015-06-02 22:52:49 -05001785 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1786 vfifo_find_dqs_", __func__, __LINE__);
1787 debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1788 write_group, read_group);
1789 debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
Marek Vasut07aee5b2015-07-12 22:07:33 +02001790 scc_mgr_set_dq_in_delay(p, d);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001791 scc_mgr_load_dq(p);
1792 }
Marek Vasut1273dd92015-07-12 21:05:08 +02001793 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001794 }
1795
1796 found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1797
1798 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1799 en_phase_sweep_dq", __func__, __LINE__);
1800 debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1801 chain to zero\n", write_group, read_group, found);
1802
1803 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1804 r += NUM_RANKS_PER_SHADOW_REG) {
1805 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1806 i++, p++) {
Marek Vasut07aee5b2015-07-12 22:07:33 +02001807 scc_mgr_set_dq_in_delay(p, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001808 scc_mgr_load_dq(p);
1809 }
Marek Vasut1273dd92015-07-12 21:05:08 +02001810 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001811 }
1812
1813 return found;
1814}
1815
1816/* per-bit deskew DQ and center */
1817static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1818 uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1819 uint32_t use_read_test, uint32_t update_fom)
1820{
1821 uint32_t i, p, d, min_index;
1822 /*
1823 * Store these as signed since there are comparisons with
1824 * signed numbers.
1825 */
1826 uint32_t bit_chk;
1827 uint32_t sticky_bit_chk;
1828 int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1829 int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1830 int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
1831 int32_t mid;
1832 int32_t orig_mid_min, mid_min;
1833 int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
1834 final_dqs_en;
1835 int32_t dq_margin, dqs_margin;
1836 uint32_t stop;
1837 uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
1838 uint32_t addr;
1839
1840 debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
1841
Marek Vasutc4815f72015-07-12 19:03:33 +02001842 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02001843 start_dqs = readl(addr + (read_group << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05001844 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
Marek Vasut17fdc912015-07-12 20:05:54 +02001845 start_dqs_en = readl(addr + ((read_group << 2)
Dinh Nguyen3da42852015-06-02 22:52:49 -05001846 - IO_DQS_EN_DELAY_OFFSET));
1847
1848 /* set the left and right edge of each bit to an illegal value */
1849 /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
1850 sticky_bit_chk = 0;
1851 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1852 left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1853 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1854 }
1855
Dinh Nguyen3da42852015-06-02 22:52:49 -05001856 /* Search for the left edge of the window for each bit */
1857 for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
1858 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
1859
Marek Vasut1273dd92015-07-12 21:05:08 +02001860 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001861
1862 /*
1863 * Stop searching when the read test doesn't pass AND when
1864 * we've seen a passing read on every bit.
1865 */
1866 if (use_read_test) {
1867 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1868 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1869 &bit_chk, 0, 0);
1870 } else {
1871 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1872 0, PASS_ONE_BIT,
1873 &bit_chk, 0);
1874 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1875 (read_group - (write_group *
1876 RW_MGR_MEM_IF_READ_DQS_WIDTH /
1877 RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1878 stop = (bit_chk == 0);
1879 }
1880 sticky_bit_chk = sticky_bit_chk | bit_chk;
1881 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1882 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
1883 && %u", __func__, __LINE__, d,
1884 sticky_bit_chk,
1885 param->read_correct_mask, stop);
1886
1887 if (stop == 1) {
1888 break;
1889 } else {
1890 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1891 if (bit_chk & 1) {
1892 /* Remember a passing test as the
1893 left_edge */
1894 left_edge[i] = d;
1895 } else {
1896 /* If a left edge has not been seen yet,
1897 then a future passing test will mark
1898 this edge as the right edge */
1899 if (left_edge[i] ==
1900 IO_IO_IN_DELAY_MAX + 1) {
1901 right_edge[i] = -(d + 1);
1902 }
1903 }
1904 bit_chk = bit_chk >> 1;
1905 }
1906 }
1907 }
1908
1909 /* Reset DQ delay chains to 0 */
Marek Vasut32675242015-07-17 06:07:13 +02001910 scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001911 sticky_bit_chk = 0;
1912 for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
1913 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
1914 %d right_edge[%u]: %d\n", __func__, __LINE__,
1915 i, left_edge[i], i, right_edge[i]);
1916
1917 /*
1918 * Check for cases where we haven't found the left edge,
1919 * which makes our assignment of the the right edge invalid.
1920 * Reset it to the illegal value.
1921 */
1922 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
1923 right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1924 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1925 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
1926 right_edge[%u]: %d\n", __func__, __LINE__,
1927 i, right_edge[i]);
1928 }
1929
1930 /*
1931 * Reset sticky bit (except for bits where we have seen
1932 * both the left and right edge).
1933 */
1934 sticky_bit_chk = sticky_bit_chk << 1;
1935 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
1936 (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1937 sticky_bit_chk = sticky_bit_chk | 1;
1938 }
1939
1940 if (i == 0)
1941 break;
1942 }
1943
Dinh Nguyen3da42852015-06-02 22:52:49 -05001944 /* Search for the right edge of the window for each bit */
1945 for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
1946 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
1947 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
1948 uint32_t delay = d + start_dqs_en;
1949 if (delay > IO_DQS_EN_DELAY_MAX)
1950 delay = IO_DQS_EN_DELAY_MAX;
1951 scc_mgr_set_dqs_en_delay(read_group, delay);
1952 }
1953 scc_mgr_load_dqs(read_group);
1954
Marek Vasut1273dd92015-07-12 21:05:08 +02001955 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05001956
1957 /*
1958 * Stop searching when the read test doesn't pass AND when
1959 * we've seen a passing read on every bit.
1960 */
1961 if (use_read_test) {
1962 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1963 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1964 &bit_chk, 0, 0);
1965 } else {
1966 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1967 0, PASS_ONE_BIT,
1968 &bit_chk, 0);
1969 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1970 (read_group - (write_group *
1971 RW_MGR_MEM_IF_READ_DQS_WIDTH /
1972 RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1973 stop = (bit_chk == 0);
1974 }
1975 sticky_bit_chk = sticky_bit_chk | bit_chk;
1976 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1977
1978 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
1979 %u && %u", __func__, __LINE__, d,
1980 sticky_bit_chk, param->read_correct_mask, stop);
1981
1982 if (stop == 1) {
1983 break;
1984 } else {
1985 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1986 if (bit_chk & 1) {
1987 /* Remember a passing test as
1988 the right_edge */
1989 right_edge[i] = d;
1990 } else {
1991 if (d != 0) {
1992 /* If a right edge has not been
1993 seen yet, then a future passing
1994 test will mark this edge as the
1995 left edge */
1996 if (right_edge[i] ==
1997 IO_IO_IN_DELAY_MAX + 1) {
1998 left_edge[i] = -(d + 1);
1999 }
2000 } else {
2001 /* d = 0 failed, but it passed
2002 when testing the left edge,
2003 so it must be marginal,
2004 set it to -1 */
2005 if (right_edge[i] ==
2006 IO_IO_IN_DELAY_MAX + 1 &&
2007 left_edge[i] !=
2008 IO_IO_IN_DELAY_MAX
2009 + 1) {
2010 right_edge[i] = -1;
2011 }
2012 /* If a right edge has not been
2013 seen yet, then a future passing
2014 test will mark this edge as the
2015 left edge */
2016 else if (right_edge[i] ==
2017 IO_IO_IN_DELAY_MAX +
2018 1) {
2019 left_edge[i] = -(d + 1);
2020 }
2021 }
2022 }
2023
2024 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2025 d=%u]: ", __func__, __LINE__, d);
2026 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2027 (int)(bit_chk & 1), i, left_edge[i]);
2028 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2029 right_edge[i]);
2030 bit_chk = bit_chk >> 1;
2031 }
2032 }
2033 }
2034
2035 /* Check that all bits have a window */
Dinh Nguyen3da42852015-06-02 22:52:49 -05002036 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2037 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2038 %d right_edge[%u]: %d", __func__, __LINE__,
2039 i, left_edge[i], i, right_edge[i]);
2040 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2041 == IO_IO_IN_DELAY_MAX + 1)) {
2042 /*
2043 * Restore delay chain settings before letting the loop
2044 * in rw_mgr_mem_calibrate_vfifo to retry different
2045 * dqs/ck relationships.
2046 */
2047 scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2048 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2049 scc_mgr_set_dqs_en_delay(read_group,
2050 start_dqs_en);
2051 }
2052 scc_mgr_load_dqs(read_group);
Marek Vasut1273dd92015-07-12 21:05:08 +02002053 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002054
2055 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2056 find edge [%u]: %d %d", __func__, __LINE__,
2057 i, left_edge[i], right_edge[i]);
2058 if (use_read_test) {
2059 set_failing_group_stage(read_group *
2060 RW_MGR_MEM_DQ_PER_READ_DQS + i,
2061 CAL_STAGE_VFIFO,
2062 CAL_SUBSTAGE_VFIFO_CENTER);
2063 } else {
2064 set_failing_group_stage(read_group *
2065 RW_MGR_MEM_DQ_PER_READ_DQS + i,
2066 CAL_STAGE_VFIFO_AFTER_WRITES,
2067 CAL_SUBSTAGE_VFIFO_CENTER);
2068 }
2069 return 0;
2070 }
2071 }
2072
2073 /* Find middle of window for each DQ bit */
2074 mid_min = left_edge[0] - right_edge[0];
2075 min_index = 0;
2076 for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2077 mid = left_edge[i] - right_edge[i];
2078 if (mid < mid_min) {
2079 mid_min = mid;
2080 min_index = i;
2081 }
2082 }
2083
2084 /*
2085 * -mid_min/2 represents the amount that we need to move DQS.
2086 * If mid_min is odd and positive we'll need to add one to
2087 * make sure the rounding in further calculations is correct
2088 * (always bias to the right), so just add 1 for all positive values.
2089 */
2090 if (mid_min > 0)
2091 mid_min++;
2092
2093 mid_min = mid_min / 2;
2094
2095 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2096 __func__, __LINE__, mid_min, min_index);
2097
2098 /* Determine the amount we can change DQS (which is -mid_min) */
2099 orig_mid_min = mid_min;
2100 new_dqs = start_dqs - mid_min;
2101 if (new_dqs > IO_DQS_IN_DELAY_MAX)
2102 new_dqs = IO_DQS_IN_DELAY_MAX;
2103 else if (new_dqs < 0)
2104 new_dqs = 0;
2105
2106 mid_min = start_dqs - new_dqs;
2107 debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2108 mid_min, new_dqs);
2109
2110 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2111 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2112 mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2113 else if (start_dqs_en - mid_min < 0)
2114 mid_min += start_dqs_en - mid_min;
2115 }
2116 new_dqs = start_dqs - mid_min;
2117
2118 debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2119 new_dqs=%d mid_min=%d\n", start_dqs,
2120 IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2121 new_dqs, mid_min);
2122
2123 /* Initialize data for export structures */
2124 dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2125 dq_margin = IO_IO_IN_DELAY_MAX + 1;
2126
Dinh Nguyen3da42852015-06-02 22:52:49 -05002127 /* add delay to bring centre of all DQ windows to the same "level" */
2128 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2129 /* Use values before divide by 2 to reduce round off error */
2130 shift_dq = (left_edge[i] - right_edge[i] -
2131 (left_edge[min_index] - right_edge[min_index]))/2 +
2132 (orig_mid_min - mid_min);
2133
2134 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2135 shift_dq[%u]=%d\n", i, shift_dq);
2136
Marek Vasut1273dd92015-07-12 21:05:08 +02002137 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02002138 temp_dq_in_delay1 = readl(addr + (p << 2));
2139 temp_dq_in_delay2 = readl(addr + (i << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05002140
2141 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2142 (int32_t)IO_IO_IN_DELAY_MAX) {
2143 shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2144 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2145 shift_dq = -(int32_t)temp_dq_in_delay1;
2146 }
2147 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2148 shift_dq[%u]=%d\n", i, shift_dq);
2149 final_dq[i] = temp_dq_in_delay1 + shift_dq;
Marek Vasut07aee5b2015-07-12 22:07:33 +02002150 scc_mgr_set_dq_in_delay(p, final_dq[i]);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002151 scc_mgr_load_dq(p);
2152
2153 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2154 left_edge[i] - shift_dq + (-mid_min),
2155 right_edge[i] + shift_dq - (-mid_min));
2156 /* To determine values for export structures */
2157 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2158 dq_margin = left_edge[i] - shift_dq + (-mid_min);
2159
2160 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2161 dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2162 }
2163
2164 final_dqs = new_dqs;
2165 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2166 final_dqs_en = start_dqs_en - mid_min;
2167
2168 /* Move DQS-en */
2169 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2170 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2171 scc_mgr_load_dqs(read_group);
2172 }
2173
2174 /* Move DQS */
2175 scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2176 scc_mgr_load_dqs(read_group);
2177 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2178 dqs_margin=%d", __func__, __LINE__,
2179 dq_margin, dqs_margin);
2180
2181 /*
2182 * Do not remove this line as it makes sure all of our decisions
2183 * have been applied. Apply the update bit.
2184 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002185 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002186
2187 return (dq_margin >= 0) && (dqs_margin >= 0);
2188}
2189
Marek Vasutbce24ef2015-07-17 03:16:45 +02002190/**
Marek Vasut04372fb2015-07-18 02:46:56 +02002191 * rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the device
2192 * @rw_group: Read/Write Group
2193 * @phase: DQ/DQS phase
2194 *
2195 * Because initially no communication ca be reliably performed with the memory
2196 * device, the sequencer uses a guaranteed write mechanism to write data into
2197 * the memory device.
2198 */
2199static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group,
2200 const u32 phase)
2201{
2202 u32 bit_chk;
2203 int ret;
2204
2205 /* Set a particular DQ/DQS phase. */
2206 scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase);
2207
2208 debug_cond(DLEVEL == 1, "%s:%d guaranteed write: g=%u p=%u\n",
2209 __func__, __LINE__, rw_group, phase);
2210
2211 /*
2212 * Altera EMI_RM 2015.05.04 :: Figure 1-25
2213 * Load up the patterns used by read calibration using the
2214 * current DQDQS phase.
2215 */
2216 rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2217
2218 if (gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
2219 return 0;
2220
2221 /*
2222 * Altera EMI_RM 2015.05.04 :: Figure 1-26
2223 * Back-to-Back reads of the patterns used for calibration.
2224 */
2225 ret = rw_mgr_mem_calibrate_read_test_patterns_all_ranks(rw_group, 1,
2226 &bit_chk);
2227 if (!ret) { /* FIXME: 0 means failure in this old code :-( */
2228 debug_cond(DLEVEL == 1,
2229 "%s:%d Guaranteed read test failed: g=%u p=%u\n",
2230 __func__, __LINE__, rw_group, phase);
2231 return -EIO;
2232 }
2233
2234 return 0;
2235}
2236
2237/**
Marek Vasutbce24ef2015-07-17 03:16:45 +02002238 * rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
2239 * @rw_group: Read/Write Group
2240 * @test_bgn: Rank at which the test begins
Dinh Nguyen3da42852015-06-02 22:52:49 -05002241 *
Marek Vasutbce24ef2015-07-17 03:16:45 +02002242 * Stage 1: Calibrate the read valid prediction FIFO.
2243 *
2244 * This function implements UniPHY calibration Stage 1, as explained in
2245 * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
2246 *
2247 * - read valid prediction will consist of finding:
2248 * - DQS enable phase and DQS enable delay (DQS Enable Calibration)
2249 * - DQS input phase and DQS input delay (DQ/DQS Centering)
Dinh Nguyen3da42852015-06-02 22:52:49 -05002250 * - we also do a per-bit deskew on the DQ lines.
2251 */
Marek Vasutc336ca32015-07-17 04:24:18 +02002252static int rw_mgr_mem_calibrate_vfifo(const u32 rw_group, const u32 test_bgn)
Dinh Nguyen3da42852015-06-02 22:52:49 -05002253{
2254 uint32_t p, d, rank_bgn, sr;
2255 uint32_t dtaps_per_ptap;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002256 uint32_t grp_calibrated;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002257 uint32_t failed_substage;
2258
Marek Vasut04372fb2015-07-18 02:46:56 +02002259 int ret;
2260
Marek Vasutc336ca32015-07-17 04:24:18 +02002261 debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002262
Marek Vasut7c0a9df2015-07-18 03:15:34 +02002263 /* Update info for sims */
2264 reg_file_set_group(rw_group);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002265 reg_file_set_stage(CAL_STAGE_VFIFO);
Marek Vasut7c0a9df2015-07-18 03:15:34 +02002266 reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002267
Marek Vasut7c0a9df2015-07-18 03:15:34 +02002268 failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2269
2270 /* USER Determine number of delay taps for each phase tap. */
Marek Vasutd32badb2015-07-17 03:11:06 +02002271 dtaps_per_ptap = DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP,
2272 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) - 1;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002273
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002274 for (d = 0; d <= dtaps_per_ptap; d += 2) {
Dinh Nguyen3da42852015-06-02 22:52:49 -05002275 /*
2276 * In RLDRAMX we may be messing the delay of pins in
Marek Vasutc336ca32015-07-17 04:24:18 +02002277 * the same write rw_group but outside of the current read
2278 * the rw_group, but that's ok because we haven't calibrated
Marek Vasutac70d2f2015-07-17 03:44:26 +02002279 * output side yet.
Dinh Nguyen3da42852015-06-02 22:52:49 -05002280 */
2281 if (d > 0) {
Marek Vasutf51a7d32015-07-19 02:18:21 +02002282 scc_mgr_apply_group_all_out_delay_add_all_ranks(
Marek Vasutc336ca32015-07-17 04:24:18 +02002283 rw_group, d);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002284 }
2285
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002286 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++) {
Marek Vasut04372fb2015-07-18 02:46:56 +02002287 /* 1) Guaranteed Write */
2288 ret = rw_mgr_mem_calibrate_guaranteed_write(rw_group, p);
2289 if (ret)
2290 break;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002291
Marek Vasutd2ea4952015-07-17 03:22:31 +02002292 /* case:56390 */
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002293 if (!rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
Marek Vasutc336ca32015-07-17 04:24:18 +02002294 (rw_group, rw_group, test_bgn)) {
Dinh Nguyen3da42852015-06-02 22:52:49 -05002295 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002296 continue;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002297 }
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002298
2299 /*
2300 * USER Read per-bit deskew can be done on a
2301 * per shadow register basis.
2302 */
2303 grp_calibrated = 1;
2304 for (rank_bgn = 0, sr = 0;
2305 rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2306 rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
2307 /*
2308 * Determine if this set of ranks
2309 * should be skipped entirely.
2310 */
2311 if (param->skip_shadow_regs[sr])
2312 continue;
2313 /*
2314 * If doing read after write
2315 * calibration, do not update
2316 * FOM, now - do it then.
2317 */
2318 if (rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
Marek Vasutc336ca32015-07-17 04:24:18 +02002319 rw_group, rw_group,
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002320 test_bgn, 1, 0))
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002321 continue;
2322
2323 grp_calibrated = 0;
2324 failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
2325 }
2326
2327 if (grp_calibrated)
2328 goto cal_done_ok;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002329 }
2330 }
2331
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002332 /* Calibration Stage 1 failed. */
Marek Vasutc336ca32015-07-17 04:24:18 +02002333 set_failing_group_stage(rw_group, CAL_STAGE_VFIFO, failed_substage);
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002334 return 0;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002335
Marek Vasutfe2d0a22015-07-17 03:50:17 +02002336 /* Calibration Stage 1 completed OK. */
2337cal_done_ok:
Dinh Nguyen3da42852015-06-02 22:52:49 -05002338 /*
2339 * Reset the delay chains back to zero if they have moved > 1
2340 * (check for > 1 because loop will increase d even when pass in
2341 * first case).
2342 */
2343 if (d > 2)
Marek Vasutc336ca32015-07-17 04:24:18 +02002344 scc_mgr_zero_group(rw_group, 1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002345
2346 return 1;
2347}
2348
2349/* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2350static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2351 uint32_t test_bgn)
2352{
2353 uint32_t rank_bgn, sr;
2354 uint32_t grp_calibrated;
2355 uint32_t write_group;
2356
2357 debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2358
2359 /* update info for sims */
2360
2361 reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2362 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2363
2364 write_group = read_group;
2365
2366 /* update info for sims */
2367 reg_file_set_group(read_group);
2368
2369 grp_calibrated = 1;
2370 /* Read per-bit deskew can be done on a per shadow register basis */
2371 for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2372 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2373 /* Determine if this set of ranks should be skipped entirely */
2374 if (!param->skip_shadow_regs[sr]) {
2375 /* This is the last calibration round, update FOM here */
2376 if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2377 write_group,
2378 read_group,
2379 test_bgn, 0,
2380 1)) {
2381 grp_calibrated = 0;
2382 }
2383 }
2384 }
2385
2386
2387 if (grp_calibrated == 0) {
2388 set_failing_group_stage(write_group,
2389 CAL_STAGE_VFIFO_AFTER_WRITES,
2390 CAL_SUBSTAGE_VFIFO_CENTER);
2391 return 0;
2392 }
2393
2394 return 1;
2395}
2396
2397/* Calibrate LFIFO to find smallest read latency */
2398static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2399{
2400 uint32_t found_one;
2401 uint32_t bit_chk;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002402
2403 debug("%s:%d\n", __func__, __LINE__);
2404
2405 /* update info for sims */
2406 reg_file_set_stage(CAL_STAGE_LFIFO);
2407 reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2408
2409 /* Load up the patterns used by read calibration for all ranks */
2410 rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2411 found_one = 0;
2412
Dinh Nguyen3da42852015-06-02 22:52:49 -05002413 do {
Marek Vasut1273dd92015-07-12 21:05:08 +02002414 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002415 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2416 __func__, __LINE__, gbl->curr_read_lat);
2417
2418 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2419 NUM_READ_TESTS,
2420 PASS_ALL_BITS,
2421 &bit_chk, 1)) {
2422 break;
2423 }
2424
2425 found_one = 1;
2426 /* reduce read latency and see if things are working */
2427 /* correctly */
2428 gbl->curr_read_lat--;
2429 } while (gbl->curr_read_lat > 0);
2430
2431 /* reset the fifos to get pointers to known state */
2432
Marek Vasut1273dd92015-07-12 21:05:08 +02002433 writel(0, &phy_mgr_cmd->fifo_reset);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002434
2435 if (found_one) {
2436 /* add a fudge factor to the read latency that was determined */
2437 gbl->curr_read_lat += 2;
Marek Vasut1273dd92015-07-12 21:05:08 +02002438 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002439 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2440 read_lat=%u\n", __func__, __LINE__,
2441 gbl->curr_read_lat);
2442 return 1;
2443 } else {
2444 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2445 CAL_SUBSTAGE_READ_LATENCY);
2446
2447 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2448 read_lat=%u\n", __func__, __LINE__,
2449 gbl->curr_read_lat);
2450 return 0;
2451 }
2452}
2453
2454/*
2455 * issue write test command.
2456 * two variants are provided. one that just tests a write pattern and
2457 * another that tests datamask functionality.
2458 */
2459static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2460 uint32_t test_dm)
2461{
2462 uint32_t mcc_instruction;
2463 uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2464 ENABLE_SUPER_QUICK_CALIBRATION);
2465 uint32_t rw_wl_nop_cycles;
2466 uint32_t addr;
2467
2468 /*
2469 * Set counter and jump addresses for the right
2470 * number of NOP cycles.
2471 * The number of supported NOP cycles can range from -1 to infinity
2472 * Three different cases are handled:
2473 *
2474 * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2475 * mechanism will be used to insert the right number of NOPs
2476 *
2477 * 2. For a number of NOP cycles equals to 0, the micro-instruction
2478 * issuing the write command will jump straight to the
2479 * micro-instruction that turns on DQS (for DDRx), or outputs write
2480 * data (for RLD), skipping
2481 * the NOP micro-instruction all together
2482 *
2483 * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2484 * turned on in the same micro-instruction that issues the write
2485 * command. Then we need
2486 * to directly jump to the micro-instruction that sends out the data
2487 *
2488 * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2489 * (2 and 3). One jump-counter (0) is used to perform multiple
2490 * write-read operations.
2491 * one counter left to issue this command in "multiple-group" mode
2492 */
2493
2494 rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2495
2496 if (rw_wl_nop_cycles == -1) {
2497 /*
2498 * CNTR 2 - We want to execute the special write operation that
2499 * turns on DQS right away and then skip directly to the
2500 * instruction that sends out the data. We set the counter to a
2501 * large number so that the jump is always taken.
2502 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002503 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002504
2505 /* CNTR 3 - Not used */
2506 if (test_dm) {
2507 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002508 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
Marek Vasut1273dd92015-07-12 21:05:08 +02002509 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002510 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
Marek Vasut1273dd92015-07-12 21:05:08 +02002511 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002512 } else {
2513 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
Marek Vasut1273dd92015-07-12 21:05:08 +02002514 writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
2515 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2516 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2517 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002518 }
2519 } else if (rw_wl_nop_cycles == 0) {
2520 /*
2521 * CNTR 2 - We want to skip the NOP operation and go straight
2522 * to the DQS enable instruction. We set the counter to a large
2523 * number so that the jump is always taken.
2524 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002525 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002526
2527 /* CNTR 3 - Not used */
2528 if (test_dm) {
2529 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002530 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
Marek Vasut1273dd92015-07-12 21:05:08 +02002531 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002532 } else {
2533 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
Marek Vasut1273dd92015-07-12 21:05:08 +02002534 writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
2535 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002536 }
2537 } else {
2538 /*
2539 * CNTR 2 - In this case we want to execute the next instruction
2540 * and NOT take the jump. So we set the counter to 0. The jump
2541 * address doesn't count.
2542 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002543 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
2544 writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002545
2546 /*
2547 * CNTR 3 - Set the nop counter to the number of cycles we
2548 * need to loop for, minus 1.
2549 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002550 writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002551 if (test_dm) {
2552 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
Marek Vasut1273dd92015-07-12 21:05:08 +02002553 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2554 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002555 } else {
2556 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
Marek Vasut1273dd92015-07-12 21:05:08 +02002557 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2558 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002559 }
2560 }
2561
Marek Vasut1273dd92015-07-12 21:05:08 +02002562 writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
2563 RW_MGR_RESET_READ_DATAPATH_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002564
Dinh Nguyen3da42852015-06-02 22:52:49 -05002565 if (quick_write_mode)
Marek Vasut1273dd92015-07-12 21:05:08 +02002566 writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002567 else
Marek Vasut1273dd92015-07-12 21:05:08 +02002568 writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002569
Marek Vasut1273dd92015-07-12 21:05:08 +02002570 writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002571
2572 /*
2573 * CNTR 1 - This is used to ensure enough time elapses
2574 * for read data to come back.
2575 */
Marek Vasut1273dd92015-07-12 21:05:08 +02002576 writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002577
Dinh Nguyen3da42852015-06-02 22:52:49 -05002578 if (test_dm) {
Marek Vasut1273dd92015-07-12 21:05:08 +02002579 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
2580 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002581 } else {
Marek Vasut1273dd92015-07-12 21:05:08 +02002582 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
2583 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002584 }
2585
Marek Vasutc4815f72015-07-12 19:03:33 +02002586 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02002587 writel(mcc_instruction, addr + (group << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05002588}
2589
2590/* Test writes, can check for a single bit pass or multiple bit pass */
2591static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2592 uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2593 uint32_t *bit_chk, uint32_t all_ranks)
2594{
Dinh Nguyen3da42852015-06-02 22:52:49 -05002595 uint32_t r;
2596 uint32_t correct_mask_vg;
2597 uint32_t tmp_bit_chk;
2598 uint32_t vg;
2599 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2600 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2601 uint32_t addr_rw_mgr;
2602 uint32_t base_rw_mgr;
2603
2604 *bit_chk = param->write_correct_mask;
2605 correct_mask_vg = param->write_correct_mask_vg;
2606
2607 for (r = rank_bgn; r < rank_end; r++) {
2608 if (param->skip_ranks[r]) {
2609 /* request to skip the rank */
2610 continue;
2611 }
2612
2613 /* set rank */
2614 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2615
2616 tmp_bit_chk = 0;
Marek Vasuta4bfa462015-07-12 17:52:36 +02002617 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
Dinh Nguyen3da42852015-06-02 22:52:49 -05002618 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2619 /* reset the fifos to get pointers to known state */
Marek Vasut1273dd92015-07-12 21:05:08 +02002620 writel(0, &phy_mgr_cmd->fifo_reset);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002621
2622 tmp_bit_chk = tmp_bit_chk <<
2623 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2624 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2625 rw_mgr_mem_calibrate_write_test_issue(write_group *
2626 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2627 use_dm);
2628
Marek Vasut17fdc912015-07-12 20:05:54 +02002629 base_rw_mgr = readl(addr_rw_mgr);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002630 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2631 if (vg == 0)
2632 break;
2633 }
2634 *bit_chk &= tmp_bit_chk;
2635 }
2636
2637 if (all_correct) {
2638 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2639 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2640 %u => %lu", write_group, use_dm,
2641 *bit_chk, param->write_correct_mask,
2642 (long unsigned int)(*bit_chk ==
2643 param->write_correct_mask));
2644 return *bit_chk == param->write_correct_mask;
2645 } else {
2646 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2647 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2648 write_group, use_dm, *bit_chk);
2649 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2650 (long unsigned int)(*bit_chk != 0));
2651 return *bit_chk != 0x00;
2652 }
2653}
2654
2655/*
2656 * center all windows. do per-bit-deskew to possibly increase size of
2657 * certain windows.
2658 */
2659static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2660 uint32_t write_group, uint32_t test_bgn)
2661{
2662 uint32_t i, p, min_index;
2663 int32_t d;
2664 /*
2665 * Store these as signed since there are comparisons with
2666 * signed numbers.
2667 */
2668 uint32_t bit_chk;
2669 uint32_t sticky_bit_chk;
2670 int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2671 int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2672 int32_t mid;
2673 int32_t mid_min, orig_mid_min;
2674 int32_t new_dqs, start_dqs, shift_dq;
2675 int32_t dq_margin, dqs_margin, dm_margin;
2676 uint32_t stop;
2677 uint32_t temp_dq_out1_delay;
2678 uint32_t addr;
2679
2680 debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2681
2682 dm_margin = 0;
2683
Marek Vasutc4815f72015-07-12 19:03:33 +02002684 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02002685 start_dqs = readl(addr +
Dinh Nguyen3da42852015-06-02 22:52:49 -05002686 (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2687
2688 /* per-bit deskew */
2689
2690 /*
2691 * set the left and right edge of each bit to an illegal value
2692 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2693 */
2694 sticky_bit_chk = 0;
2695 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2696 left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2697 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2698 }
2699
2700 /* Search for the left edge of the window for each bit */
Dinh Nguyen3da42852015-06-02 22:52:49 -05002701 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
Marek Vasut300c2e62015-07-17 05:42:49 +02002702 scc_mgr_apply_group_dq_out1_delay(write_group, d);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002703
Marek Vasut1273dd92015-07-12 21:05:08 +02002704 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002705
2706 /*
2707 * Stop searching when the read test doesn't pass AND when
2708 * we've seen a passing read on every bit.
2709 */
2710 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2711 0, PASS_ONE_BIT, &bit_chk, 0);
2712 sticky_bit_chk = sticky_bit_chk | bit_chk;
2713 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2714 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2715 == %u && %u [bit_chk= %u ]\n",
2716 d, sticky_bit_chk, param->write_correct_mask,
2717 stop, bit_chk);
2718
2719 if (stop == 1) {
2720 break;
2721 } else {
2722 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2723 if (bit_chk & 1) {
2724 /*
2725 * Remember a passing test as the
2726 * left_edge.
2727 */
2728 left_edge[i] = d;
2729 } else {
2730 /*
2731 * If a left edge has not been seen
2732 * yet, then a future passing test will
2733 * mark this edge as the right edge.
2734 */
2735 if (left_edge[i] ==
2736 IO_IO_OUT1_DELAY_MAX + 1) {
2737 right_edge[i] = -(d + 1);
2738 }
2739 }
2740 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2741 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2742 (int)(bit_chk & 1), i, left_edge[i]);
2743 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2744 right_edge[i]);
2745 bit_chk = bit_chk >> 1;
2746 }
2747 }
2748 }
2749
2750 /* Reset DQ delay chains to 0 */
Marek Vasut32675242015-07-17 06:07:13 +02002751 scc_mgr_apply_group_dq_out1_delay(0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002752 sticky_bit_chk = 0;
2753 for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2754 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2755 %d right_edge[%u]: %d\n", __func__, __LINE__,
2756 i, left_edge[i], i, right_edge[i]);
2757
2758 /*
2759 * Check for cases where we haven't found the left edge,
2760 * which makes our assignment of the the right edge invalid.
2761 * Reset it to the illegal value.
2762 */
2763 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2764 (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2765 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2766 debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2767 right_edge[%u]: %d\n", __func__, __LINE__,
2768 i, right_edge[i]);
2769 }
2770
2771 /*
2772 * Reset sticky bit (except for bits where we have
2773 * seen the left edge).
2774 */
2775 sticky_bit_chk = sticky_bit_chk << 1;
2776 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2777 sticky_bit_chk = sticky_bit_chk | 1;
2778
2779 if (i == 0)
2780 break;
2781 }
2782
2783 /* Search for the right edge of the window for each bit */
Dinh Nguyen3da42852015-06-02 22:52:49 -05002784 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2785 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2786 d + start_dqs);
2787
Marek Vasut1273dd92015-07-12 21:05:08 +02002788 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002789
2790 /*
2791 * Stop searching when the read test doesn't pass AND when
2792 * we've seen a passing read on every bit.
2793 */
2794 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2795 0, PASS_ONE_BIT, &bit_chk, 0);
2796
2797 sticky_bit_chk = sticky_bit_chk | bit_chk;
2798 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2799
2800 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2801 %u && %u\n", d, sticky_bit_chk,
2802 param->write_correct_mask, stop);
2803
2804 if (stop == 1) {
2805 if (d == 0) {
2806 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2807 i++) {
2808 /* d = 0 failed, but it passed when
2809 testing the left edge, so it must be
2810 marginal, set it to -1 */
2811 if (right_edge[i] ==
2812 IO_IO_OUT1_DELAY_MAX + 1 &&
2813 left_edge[i] !=
2814 IO_IO_OUT1_DELAY_MAX + 1) {
2815 right_edge[i] = -1;
2816 }
2817 }
2818 }
2819 break;
2820 } else {
2821 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2822 if (bit_chk & 1) {
2823 /*
2824 * Remember a passing test as
2825 * the right_edge.
2826 */
2827 right_edge[i] = d;
2828 } else {
2829 if (d != 0) {
2830 /*
2831 * If a right edge has not
2832 * been seen yet, then a future
2833 * passing test will mark this
2834 * edge as the left edge.
2835 */
2836 if (right_edge[i] ==
2837 IO_IO_OUT1_DELAY_MAX + 1)
2838 left_edge[i] = -(d + 1);
2839 } else {
2840 /*
2841 * d = 0 failed, but it passed
2842 * when testing the left edge,
2843 * so it must be marginal, set
2844 * it to -1.
2845 */
2846 if (right_edge[i] ==
2847 IO_IO_OUT1_DELAY_MAX + 1 &&
2848 left_edge[i] !=
2849 IO_IO_OUT1_DELAY_MAX + 1)
2850 right_edge[i] = -1;
2851 /*
2852 * If a right edge has not been
2853 * seen yet, then a future
2854 * passing test will mark this
2855 * edge as the left edge.
2856 */
2857 else if (right_edge[i] ==
2858 IO_IO_OUT1_DELAY_MAX +
2859 1)
2860 left_edge[i] = -(d + 1);
2861 }
2862 }
2863 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
2864 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2865 (int)(bit_chk & 1), i, left_edge[i]);
2866 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2867 right_edge[i]);
2868 bit_chk = bit_chk >> 1;
2869 }
2870 }
2871 }
2872
2873 /* Check that all bits have a window */
2874 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2875 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2876 %d right_edge[%u]: %d", __func__, __LINE__,
2877 i, left_edge[i], i, right_edge[i]);
2878 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
2879 (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
2880 set_failing_group_stage(test_bgn + i,
2881 CAL_STAGE_WRITES,
2882 CAL_SUBSTAGE_WRITES_CENTER);
2883 return 0;
2884 }
2885 }
2886
2887 /* Find middle of window for each DQ bit */
2888 mid_min = left_edge[0] - right_edge[0];
2889 min_index = 0;
2890 for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2891 mid = left_edge[i] - right_edge[i];
2892 if (mid < mid_min) {
2893 mid_min = mid;
2894 min_index = i;
2895 }
2896 }
2897
2898 /*
2899 * -mid_min/2 represents the amount that we need to move DQS.
2900 * If mid_min is odd and positive we'll need to add one to
2901 * make sure the rounding in further calculations is correct
2902 * (always bias to the right), so just add 1 for all positive values.
2903 */
2904 if (mid_min > 0)
2905 mid_min++;
2906 mid_min = mid_min / 2;
2907 debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
2908 __LINE__, mid_min);
2909
2910 /* Determine the amount we can change DQS (which is -mid_min) */
2911 orig_mid_min = mid_min;
2912 new_dqs = start_dqs;
2913 mid_min = 0;
2914 debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
2915 mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
2916 /* Initialize data for export structures */
2917 dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
2918 dq_margin = IO_IO_OUT1_DELAY_MAX + 1;
2919
2920 /* add delay to bring centre of all DQ windows to the same "level" */
Dinh Nguyen3da42852015-06-02 22:52:49 -05002921 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
2922 /* Use values before divide by 2 to reduce round off error */
2923 shift_dq = (left_edge[i] - right_edge[i] -
2924 (left_edge[min_index] - right_edge[min_index]))/2 +
2925 (orig_mid_min - mid_min);
2926
2927 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
2928 [%u]=%d\n", __func__, __LINE__, i, shift_dq);
2929
Marek Vasut1273dd92015-07-12 21:05:08 +02002930 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
Marek Vasut17fdc912015-07-12 20:05:54 +02002931 temp_dq_out1_delay = readl(addr + (i << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05002932 if (shift_dq + (int32_t)temp_dq_out1_delay >
2933 (int32_t)IO_IO_OUT1_DELAY_MAX) {
2934 shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
2935 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
2936 shift_dq = -(int32_t)temp_dq_out1_delay;
2937 }
2938 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
2939 i, shift_dq);
Marek Vasut07aee5b2015-07-12 22:07:33 +02002940 scc_mgr_set_dq_out1_delay(i, temp_dq_out1_delay + shift_dq);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002941 scc_mgr_load_dq(i);
2942
2943 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
2944 left_edge[i] - shift_dq + (-mid_min),
2945 right_edge[i] + shift_dq - (-mid_min));
2946 /* To determine values for export structures */
2947 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2948 dq_margin = left_edge[i] - shift_dq + (-mid_min);
2949
2950 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2951 dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2952 }
2953
2954 /* Move DQS */
2955 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
Marek Vasut1273dd92015-07-12 21:05:08 +02002956 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002957
2958 /* Centre DM */
2959 debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
2960
2961 /*
2962 * set the left and right edge of each bit to an illegal value,
2963 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
2964 */
2965 left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
2966 right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
2967 int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
2968 int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
2969 int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
2970 int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
2971 int32_t win_best = 0;
2972
2973 /* Search for the/part of the window with DM shift */
Dinh Nguyen3da42852015-06-02 22:52:49 -05002974 for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
Marek Vasut32675242015-07-17 06:07:13 +02002975 scc_mgr_apply_group_dm_out1_delay(d);
Marek Vasut1273dd92015-07-12 21:05:08 +02002976 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05002977
2978 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
2979 PASS_ALL_BITS, &bit_chk,
2980 0)) {
2981 /* USE Set current end of the window */
2982 end_curr = -d;
2983 /*
2984 * If a starting edge of our window has not been seen
2985 * this is our current start of the DM window.
2986 */
2987 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
2988 bgn_curr = -d;
2989
2990 /*
2991 * If current window is bigger than best seen.
2992 * Set best seen to be current window.
2993 */
2994 if ((end_curr-bgn_curr+1) > win_best) {
2995 win_best = end_curr-bgn_curr+1;
2996 bgn_best = bgn_curr;
2997 end_best = end_curr;
2998 }
2999 } else {
3000 /* We just saw a failing test. Reset temp edge */
3001 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3002 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3003 }
3004 }
3005
3006
3007 /* Reset DM delay chains to 0 */
Marek Vasut32675242015-07-17 06:07:13 +02003008 scc_mgr_apply_group_dm_out1_delay(0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003009
3010 /*
3011 * Check to see if the current window nudges up aganist 0 delay.
3012 * If so we need to continue the search by shifting DQS otherwise DQS
3013 * search begins as a new search. */
3014 if (end_curr != 0) {
3015 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3016 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3017 }
3018
3019 /* Search for the/part of the window with DQS shifts */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003020 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
3021 /*
3022 * Note: This only shifts DQS, so are we limiting ourselve to
3023 * width of DQ unnecessarily.
3024 */
3025 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
3026 d + new_dqs);
3027
Marek Vasut1273dd92015-07-12 21:05:08 +02003028 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003029 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3030 PASS_ALL_BITS, &bit_chk,
3031 0)) {
3032 /* USE Set current end of the window */
3033 end_curr = d;
3034 /*
3035 * If a beginning edge of our window has not been seen
3036 * this is our current begin of the DM window.
3037 */
3038 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3039 bgn_curr = d;
3040
3041 /*
3042 * If current window is bigger than best seen. Set best
3043 * seen to be current window.
3044 */
3045 if ((end_curr-bgn_curr+1) > win_best) {
3046 win_best = end_curr-bgn_curr+1;
3047 bgn_best = bgn_curr;
3048 end_best = end_curr;
3049 }
3050 } else {
3051 /* We just saw a failing test. Reset temp edge */
3052 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3053 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3054
3055 /* Early exit optimization: if ther remaining delay
3056 chain space is less than already seen largest window
3057 we can exit */
3058 if ((win_best-1) >
3059 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3060 break;
3061 }
3062 }
3063 }
3064
3065 /* assign left and right edge for cal and reporting; */
3066 left_edge[0] = -1*bgn_best;
3067 right_edge[0] = end_best;
3068
3069 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3070 __LINE__, left_edge[0], right_edge[0]);
3071
3072 /* Move DQS (back to orig) */
3073 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3074
3075 /* Move DM */
3076
3077 /* Find middle of window for the DM bit */
3078 mid = (left_edge[0] - right_edge[0]) / 2;
3079
3080 /* only move right, since we are not moving DQS/DQ */
3081 if (mid < 0)
3082 mid = 0;
3083
3084 /* dm_marign should fail if we never find a window */
3085 if (win_best == 0)
3086 dm_margin = -1;
3087 else
3088 dm_margin = left_edge[0] - mid;
3089
Marek Vasut32675242015-07-17 06:07:13 +02003090 scc_mgr_apply_group_dm_out1_delay(mid);
Marek Vasut1273dd92015-07-12 21:05:08 +02003091 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003092
3093 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3094 dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3095 right_edge[0], mid, dm_margin);
3096 /* Export values */
3097 gbl->fom_out += dq_margin + dqs_margin;
3098
3099 debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3100 dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3101 dq_margin, dqs_margin, dm_margin);
3102
3103 /*
3104 * Do not remove this line as it makes sure all of our
3105 * decisions have been applied.
3106 */
Marek Vasut1273dd92015-07-12 21:05:08 +02003107 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003108 return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3109}
3110
3111/* calibrate the write operations */
3112static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3113 uint32_t test_bgn)
3114{
3115 /* update info for sims */
3116 debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3117
3118 reg_file_set_stage(CAL_STAGE_WRITES);
3119 reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3120
3121 reg_file_set_group(g);
3122
3123 if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3124 set_failing_group_stage(g, CAL_STAGE_WRITES,
3125 CAL_SUBSTAGE_WRITES_CENTER);
3126 return 0;
3127 }
3128
3129 return 1;
3130}
3131
Marek Vasut4b0ac262015-07-20 07:33:33 +02003132/**
3133 * mem_precharge_and_activate() - Precharge all banks and activate
3134 *
3135 * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
3136 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003137static void mem_precharge_and_activate(void)
3138{
Marek Vasut4b0ac262015-07-20 07:33:33 +02003139 int r;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003140
3141 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
Marek Vasut4b0ac262015-07-20 07:33:33 +02003142 /* Test if the rank should be skipped. */
3143 if (param->skip_ranks[r])
Dinh Nguyen3da42852015-06-02 22:52:49 -05003144 continue;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003145
Marek Vasut4b0ac262015-07-20 07:33:33 +02003146 /* Set rank. */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003147 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3148
Marek Vasut4b0ac262015-07-20 07:33:33 +02003149 /* Precharge all banks. */
Marek Vasut1273dd92015-07-12 21:05:08 +02003150 writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3151 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003152
Marek Vasut1273dd92015-07-12 21:05:08 +02003153 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
3154 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
3155 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003156
Marek Vasut1273dd92015-07-12 21:05:08 +02003157 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
3158 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
3159 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003160
Marek Vasut4b0ac262015-07-20 07:33:33 +02003161 /* Activate rows. */
Marek Vasut1273dd92015-07-12 21:05:08 +02003162 writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3163 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003164 }
3165}
3166
Marek Vasut16502a02015-07-17 01:57:41 +02003167/**
3168 * mem_init_latency() - Configure memory RLAT and WLAT settings
3169 *
3170 * Configure memory RLAT and WLAT parameters.
3171 */
3172static void mem_init_latency(void)
Dinh Nguyen3da42852015-06-02 22:52:49 -05003173{
Marek Vasut16502a02015-07-17 01:57:41 +02003174 /*
3175 * For AV/CV, LFIFO is hardened and always runs at full rate
3176 * so max latency in AFI clocks, used here, is correspondingly
3177 * smaller.
3178 */
3179 const u32 max_latency = (1 << MAX_LATENCY_COUNT_WIDTH) - 1;
3180 u32 rlat, wlat;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003181
3182 debug("%s:%d\n", __func__, __LINE__);
Marek Vasut16502a02015-07-17 01:57:41 +02003183
3184 /*
3185 * Read in write latency.
3186 * WL for Hard PHY does not include additive latency.
3187 */
Marek Vasut1273dd92015-07-12 21:05:08 +02003188 wlat = readl(&data_mgr->t_wl_add);
3189 wlat += readl(&data_mgr->mem_t_add);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003190
Marek Vasut16502a02015-07-17 01:57:41 +02003191 gbl->rw_wl_nop_cycles = wlat - 1;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003192
Marek Vasut16502a02015-07-17 01:57:41 +02003193 /* Read in readl latency. */
Marek Vasut1273dd92015-07-12 21:05:08 +02003194 rlat = readl(&data_mgr->t_rl_add);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003195
Marek Vasut16502a02015-07-17 01:57:41 +02003196 /* Set a pretty high read latency initially. */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003197 gbl->curr_read_lat = rlat + 16;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003198 if (gbl->curr_read_lat > max_latency)
3199 gbl->curr_read_lat = max_latency;
3200
Marek Vasut1273dd92015-07-12 21:05:08 +02003201 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003202
Marek Vasut16502a02015-07-17 01:57:41 +02003203 /* Advertise write latency. */
3204 writel(wlat, &phy_mgr_cfg->afi_wlat);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003205}
3206
Marek Vasut51cea0b2015-07-26 10:54:15 +02003207/**
3208 * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
3209 *
3210 * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
3211 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003212static void mem_skip_calibrate(void)
3213{
3214 uint32_t vfifo_offset;
3215 uint32_t i, j, r;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003216
3217 debug("%s:%d\n", __func__, __LINE__);
3218 /* Need to update every shadow register set used by the interface */
3219 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
Marek Vasut51cea0b2015-07-26 10:54:15 +02003220 r += NUM_RANKS_PER_SHADOW_REG) {
Dinh Nguyen3da42852015-06-02 22:52:49 -05003221 /*
3222 * Set output phase alignment settings appropriate for
3223 * skip calibration.
3224 */
3225 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3226 scc_mgr_set_dqs_en_phase(i, 0);
3227#if IO_DLL_CHAIN_LENGTH == 6
3228 scc_mgr_set_dqdqs_output_phase(i, 6);
3229#else
3230 scc_mgr_set_dqdqs_output_phase(i, 7);
3231#endif
3232 /*
3233 * Case:33398
3234 *
3235 * Write data arrives to the I/O two cycles before write
3236 * latency is reached (720 deg).
3237 * -> due to bit-slip in a/c bus
3238 * -> to allow board skew where dqs is longer than ck
3239 * -> how often can this happen!?
3240 * -> can claim back some ptaps for high freq
3241 * support if we can relax this, but i digress...
3242 *
3243 * The write_clk leads mem_ck by 90 deg
3244 * The minimum ptap of the OPA is 180 deg
3245 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3246 * The write_clk is always delayed by 2 ptaps
3247 *
3248 * Hence, to make DQS aligned to CK, we need to delay
3249 * DQS by:
3250 * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3251 *
3252 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3253 * gives us the number of ptaps, which simplies to:
3254 *
3255 * (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3256 */
Marek Vasut51cea0b2015-07-26 10:54:15 +02003257 scc_mgr_set_dqdqs_output_phase(i,
3258 1.25 * IO_DLL_CHAIN_LENGTH - 2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003259 }
Marek Vasut1273dd92015-07-12 21:05:08 +02003260 writel(0xff, &sdr_scc_mgr->dqs_ena);
3261 writel(0xff, &sdr_scc_mgr->dqs_io_ena);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003262
Dinh Nguyen3da42852015-06-02 22:52:49 -05003263 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
Marek Vasut1273dd92015-07-12 21:05:08 +02003264 writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3265 SCC_MGR_GROUP_COUNTER_OFFSET);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003266 }
Marek Vasut1273dd92015-07-12 21:05:08 +02003267 writel(0xff, &sdr_scc_mgr->dq_ena);
3268 writel(0xff, &sdr_scc_mgr->dm_ena);
3269 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003270 }
3271
3272 /* Compensate for simulation model behaviour */
3273 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3274 scc_mgr_set_dqs_bus_in_delay(i, 10);
3275 scc_mgr_load_dqs(i);
3276 }
Marek Vasut1273dd92015-07-12 21:05:08 +02003277 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003278
3279 /*
3280 * ArriaV has hard FIFOs that can only be initialized by incrementing
3281 * in sequencer.
3282 */
3283 vfifo_offset = CALIB_VFIFO_OFFSET;
Marek Vasut51cea0b2015-07-26 10:54:15 +02003284 for (j = 0; j < vfifo_offset; j++)
Marek Vasut1273dd92015-07-12 21:05:08 +02003285 writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
Marek Vasut1273dd92015-07-12 21:05:08 +02003286 writel(0, &phy_mgr_cmd->fifo_reset);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003287
3288 /*
Marek Vasut51cea0b2015-07-26 10:54:15 +02003289 * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
3290 * setting from generation-time constant.
Dinh Nguyen3da42852015-06-02 22:52:49 -05003291 */
3292 gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
Marek Vasut1273dd92015-07-12 21:05:08 +02003293 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003294}
3295
Marek Vasut3589fbf2015-07-20 04:34:51 +02003296/**
3297 * mem_calibrate() - Memory calibration entry point.
3298 *
3299 * Perform memory calibration.
3300 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003301static uint32_t mem_calibrate(void)
3302{
3303 uint32_t i;
3304 uint32_t rank_bgn, sr;
3305 uint32_t write_group, write_test_bgn;
3306 uint32_t read_group, read_test_bgn;
3307 uint32_t run_groups, current_run;
3308 uint32_t failing_groups = 0;
3309 uint32_t group_failed = 0;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003310
Marek Vasut33c42bb2015-07-17 02:21:47 +02003311 const u32 rwdqs_ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
3312 RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
3313
Dinh Nguyen3da42852015-06-02 22:52:49 -05003314 debug("%s:%d\n", __func__, __LINE__);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003315
Marek Vasut16502a02015-07-17 01:57:41 +02003316 /* Initialize the data settings */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003317 gbl->error_substage = CAL_SUBSTAGE_NIL;
3318 gbl->error_stage = CAL_STAGE_NIL;
3319 gbl->error_group = 0xff;
3320 gbl->fom_in = 0;
3321 gbl->fom_out = 0;
3322
Marek Vasut16502a02015-07-17 01:57:41 +02003323 /* Initialize WLAT and RLAT. */
3324 mem_init_latency();
3325
3326 /* Initialize bit slips. */
3327 mem_precharge_and_activate();
Dinh Nguyen3da42852015-06-02 22:52:49 -05003328
Dinh Nguyen3da42852015-06-02 22:52:49 -05003329 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
Marek Vasut1273dd92015-07-12 21:05:08 +02003330 writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3331 SCC_MGR_GROUP_COUNTER_OFFSET);
Marek Vasutfa5d8212015-07-19 01:34:43 +02003332 /* Only needed once to set all groups, pins, DQ, DQS, DM. */
3333 if (i == 0)
3334 scc_mgr_set_hhp_extras();
3335
Marek Vasutc5c5f532015-07-17 02:06:20 +02003336 scc_set_bypass_mode(i);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003337 }
3338
Marek Vasut722c9682015-07-17 02:07:12 +02003339 /* Calibration is skipped. */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003340 if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3341 /*
3342 * Set VFIFO and LFIFO to instant-on settings in skip
3343 * calibration mode.
3344 */
3345 mem_skip_calibrate();
Dinh Nguyen3da42852015-06-02 22:52:49 -05003346
Marek Vasut722c9682015-07-17 02:07:12 +02003347 /*
3348 * Do not remove this line as it makes sure all of our
3349 * decisions have been applied.
3350 */
3351 writel(0, &sdr_scc_mgr->update);
3352 return 1;
3353 }
Dinh Nguyen3da42852015-06-02 22:52:49 -05003354
Marek Vasut722c9682015-07-17 02:07:12 +02003355 /* Calibration is not skipped. */
3356 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3357 /*
3358 * Zero all delay chain/phase settings for all
3359 * groups and all shadow register sets.
3360 */
3361 scc_mgr_zero_all();
Dinh Nguyen3da42852015-06-02 22:52:49 -05003362
Marek Vasut722c9682015-07-17 02:07:12 +02003363 run_groups = ~param->skip_groups;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003364
Marek Vasut722c9682015-07-17 02:07:12 +02003365 for (write_group = 0, write_test_bgn = 0; write_group
3366 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3367 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
Marek Vasutc452dcd2015-07-17 02:50:56 +02003368
3369 /* Initialize the group failure */
Marek Vasut722c9682015-07-17 02:07:12 +02003370 group_failed = 0;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003371
Marek Vasut722c9682015-07-17 02:07:12 +02003372 current_run = run_groups & ((1 <<
3373 RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3374 run_groups = run_groups >>
3375 RW_MGR_NUM_DQS_PER_WRITE_GROUP;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003376
Marek Vasut722c9682015-07-17 02:07:12 +02003377 if (current_run == 0)
3378 continue;
3379
3380 writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
3381 SCC_MGR_GROUP_COUNTER_OFFSET);
3382 scc_mgr_zero_group(write_group, 0);
3383
Marek Vasut33c42bb2015-07-17 02:21:47 +02003384 for (read_group = write_group * rwdqs_ratio,
3385 read_test_bgn = 0;
Marek Vasutc452dcd2015-07-17 02:50:56 +02003386 read_group < (write_group + 1) * rwdqs_ratio;
Marek Vasut33c42bb2015-07-17 02:21:47 +02003387 read_group++,
3388 read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
3389 if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
3390 continue;
Marek Vasut722c9682015-07-17 02:07:12 +02003391
Marek Vasut33c42bb2015-07-17 02:21:47 +02003392 /* Calibrate the VFIFO */
3393 if (rw_mgr_mem_calibrate_vfifo(read_group,
3394 read_test_bgn))
3395 continue;
3396
Marek Vasutc452dcd2015-07-17 02:50:56 +02003397 if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
3398 return 0;
3399
3400 /* The group failed, we're done. */
3401 goto grp_failed;
3402 }
3403
3404 /* Calibrate the output side */
3405 for (rank_bgn = 0, sr = 0;
3406 rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
3407 rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
3408 if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
3409 continue;
3410
3411 /* Not needed in quick mode! */
3412 if (STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS)
3413 continue;
3414
3415 /*
3416 * Determine if this set of ranks
3417 * should be skipped entirely.
3418 */
3419 if (param->skip_shadow_regs[sr])
3420 continue;
3421
3422 /* Calibrate WRITEs */
3423 if (rw_mgr_mem_calibrate_writes(rank_bgn,
3424 write_group, write_test_bgn))
3425 continue;
3426
Marek Vasut33c42bb2015-07-17 02:21:47 +02003427 group_failed = 1;
3428 if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
3429 return 0;
Marek Vasut722c9682015-07-17 02:07:12 +02003430 }
3431
Marek Vasutc452dcd2015-07-17 02:50:56 +02003432 /* Some group failed, we're done. */
3433 if (group_failed)
3434 goto grp_failed;
Marek Vasut4ac21612015-07-17 02:31:04 +02003435
Marek Vasutc452dcd2015-07-17 02:50:56 +02003436 for (read_group = write_group * rwdqs_ratio,
3437 read_test_bgn = 0;
3438 read_group < (write_group + 1) * rwdqs_ratio;
3439 read_group++,
3440 read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
3441 if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
3442 continue;
Marek Vasut4ac21612015-07-17 02:31:04 +02003443
Marek Vasutc452dcd2015-07-17 02:50:56 +02003444 if (rw_mgr_mem_calibrate_vfifo_end(read_group,
3445 read_test_bgn))
3446 continue;
Marek Vasut4ac21612015-07-17 02:31:04 +02003447
Marek Vasutc452dcd2015-07-17 02:50:56 +02003448 if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
3449 return 0;
Marek Vasut4ac21612015-07-17 02:31:04 +02003450
Marek Vasutc452dcd2015-07-17 02:50:56 +02003451 /* The group failed, we're done. */
3452 goto grp_failed;
Marek Vasut722c9682015-07-17 02:07:12 +02003453 }
3454
Marek Vasutc452dcd2015-07-17 02:50:56 +02003455 /* No group failed, continue as usual. */
3456 continue;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003457
Marek Vasutc452dcd2015-07-17 02:50:56 +02003458grp_failed: /* A group failed, increment the counter. */
3459 failing_groups++;
Marek Vasut722c9682015-07-17 02:07:12 +02003460 }
Dinh Nguyen3da42852015-06-02 22:52:49 -05003461
Marek Vasut722c9682015-07-17 02:07:12 +02003462 /*
3463 * USER If there are any failing groups then report
3464 * the failure.
3465 */
3466 if (failing_groups != 0)
3467 return 0;
3468
Marek Vasutc50ae302015-07-17 02:40:21 +02003469 if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
3470 continue;
3471
3472 /*
3473 * If we're skipping groups as part of debug,
3474 * don't calibrate LFIFO.
3475 */
3476 if (param->skip_groups != 0)
3477 continue;
3478
Marek Vasut722c9682015-07-17 02:07:12 +02003479 /* Calibrate the LFIFO */
Marek Vasutc50ae302015-07-17 02:40:21 +02003480 if (!rw_mgr_mem_calibrate_lfifo())
3481 return 0;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003482 }
3483
3484 /*
3485 * Do not remove this line as it makes sure all of our decisions
3486 * have been applied.
3487 */
Marek Vasut1273dd92015-07-12 21:05:08 +02003488 writel(0, &sdr_scc_mgr->update);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003489 return 1;
3490}
3491
Marek Vasut23a040c2015-07-17 01:20:21 +02003492/**
3493 * run_mem_calibrate() - Perform memory calibration
3494 *
3495 * This function triggers the entire memory calibration procedure.
3496 */
3497static int run_mem_calibrate(void)
Dinh Nguyen3da42852015-06-02 22:52:49 -05003498{
Marek Vasut23a040c2015-07-17 01:20:21 +02003499 int pass;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003500
3501 debug("%s:%d\n", __func__, __LINE__);
3502
3503 /* Reset pass/fail status shown on afi_cal_success/fail */
Marek Vasut1273dd92015-07-12 21:05:08 +02003504 writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003505
Marek Vasut23a040c2015-07-17 01:20:21 +02003506 /* Stop tracking manager. */
3507 clrbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003508
Marek Vasut9fa9c902015-07-17 01:12:07 +02003509 phy_mgr_initialize();
Dinh Nguyen3da42852015-06-02 22:52:49 -05003510 rw_mgr_mem_initialize();
3511
Marek Vasut23a040c2015-07-17 01:20:21 +02003512 /* Perform the actual memory calibration. */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003513 pass = mem_calibrate();
3514
3515 mem_precharge_and_activate();
Marek Vasut1273dd92015-07-12 21:05:08 +02003516 writel(0, &phy_mgr_cmd->fifo_reset);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003517
Marek Vasut23a040c2015-07-17 01:20:21 +02003518 /* Handoff. */
3519 rw_mgr_mem_handoff();
Dinh Nguyen3da42852015-06-02 22:52:49 -05003520 /*
Marek Vasut23a040c2015-07-17 01:20:21 +02003521 * In Hard PHY this is a 2-bit control:
3522 * 0: AFI Mux Select
3523 * 1: DDIO Mux Select
Dinh Nguyen3da42852015-06-02 22:52:49 -05003524 */
Marek Vasut23a040c2015-07-17 01:20:21 +02003525 writel(0x2, &phy_mgr_cfg->mux_sel);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003526
Marek Vasut23a040c2015-07-17 01:20:21 +02003527 /* Start tracking manager. */
3528 setbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
3529
3530 return pass;
3531}
3532
3533/**
3534 * debug_mem_calibrate() - Report result of memory calibration
3535 * @pass: Value indicating whether calibration passed or failed
3536 *
3537 * This function reports the results of the memory calibration
3538 * and writes debug information into the register file.
3539 */
3540static void debug_mem_calibrate(int pass)
3541{
3542 uint32_t debug_info;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003543
3544 if (pass) {
3545 printf("%s: CALIBRATION PASSED\n", __FILE__);
3546
3547 gbl->fom_in /= 2;
3548 gbl->fom_out /= 2;
3549
3550 if (gbl->fom_in > 0xff)
3551 gbl->fom_in = 0xff;
3552
3553 if (gbl->fom_out > 0xff)
3554 gbl->fom_out = 0xff;
3555
3556 /* Update the FOM in the register file */
3557 debug_info = gbl->fom_in;
3558 debug_info |= gbl->fom_out << 8;
Marek Vasut1273dd92015-07-12 21:05:08 +02003559 writel(debug_info, &sdr_reg_file->fom);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003560
Marek Vasut1273dd92015-07-12 21:05:08 +02003561 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3562 writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003563 } else {
3564 printf("%s: CALIBRATION FAILED\n", __FILE__);
3565
3566 debug_info = gbl->error_stage;
3567 debug_info |= gbl->error_substage << 8;
3568 debug_info |= gbl->error_group << 16;
3569
Marek Vasut1273dd92015-07-12 21:05:08 +02003570 writel(debug_info, &sdr_reg_file->failing_stage);
3571 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3572 writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003573
3574 /* Update the failing group/stage in the register file */
3575 debug_info = gbl->error_stage;
3576 debug_info |= gbl->error_substage << 8;
3577 debug_info |= gbl->error_group << 16;
Marek Vasut1273dd92015-07-12 21:05:08 +02003578 writel(debug_info, &sdr_reg_file->failing_stage);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003579 }
3580
Marek Vasut23a040c2015-07-17 01:20:21 +02003581 printf("%s: Calibration complete\n", __FILE__);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003582}
3583
Marek Vasutbb064342015-07-19 06:12:42 +02003584/**
3585 * hc_initialize_rom_data() - Initialize ROM data
3586 *
3587 * Initialize ROM data.
3588 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003589static void hc_initialize_rom_data(void)
3590{
Marek Vasutbb064342015-07-19 06:12:42 +02003591 u32 i, addr;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003592
Marek Vasutc4815f72015-07-12 19:03:33 +02003593 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
Marek Vasutbb064342015-07-19 06:12:42 +02003594 for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
3595 writel(inst_rom_init[i], addr + (i << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05003596
Marek Vasutc4815f72015-07-12 19:03:33 +02003597 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
Marek Vasutbb064342015-07-19 06:12:42 +02003598 for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
3599 writel(ac_rom_init[i], addr + (i << 2));
Dinh Nguyen3da42852015-06-02 22:52:49 -05003600}
3601
Marek Vasut9c1ab2c2015-07-19 06:13:37 +02003602/**
3603 * initialize_reg_file() - Initialize SDR register file
3604 *
3605 * Initialize SDR register file.
3606 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003607static void initialize_reg_file(void)
3608{
Dinh Nguyen3da42852015-06-02 22:52:49 -05003609 /* Initialize the register file with the correct data */
Marek Vasut1273dd92015-07-12 21:05:08 +02003610 writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
3611 writel(0, &sdr_reg_file->debug_data_addr);
3612 writel(0, &sdr_reg_file->cur_stage);
3613 writel(0, &sdr_reg_file->fom);
3614 writel(0, &sdr_reg_file->failing_stage);
3615 writel(0, &sdr_reg_file->debug1);
3616 writel(0, &sdr_reg_file->debug2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003617}
3618
Marek Vasut2ca151f2015-07-19 06:14:04 +02003619/**
3620 * initialize_hps_phy() - Initialize HPS PHY
3621 *
3622 * Initialize HPS PHY.
3623 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003624static void initialize_hps_phy(void)
3625{
3626 uint32_t reg;
Dinh Nguyen3da42852015-06-02 22:52:49 -05003627 /*
3628 * Tracking also gets configured here because it's in the
3629 * same register.
3630 */
3631 uint32_t trk_sample_count = 7500;
3632 uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3633 /*
3634 * Format is number of outer loops in the 16 MSB, sample
3635 * count in 16 LSB.
3636 */
3637
3638 reg = 0;
3639 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3640 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3641 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3642 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3643 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3644 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3645 /*
3646 * This field selects the intrinsic latency to RDATA_EN/FULL path.
3647 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3648 */
3649 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3650 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3651 trk_sample_count);
Marek Vasut6cb9f162015-07-12 20:49:39 +02003652 writel(reg, &sdr_ctrl->phy_ctrl0);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003653
3654 reg = 0;
3655 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3656 trk_sample_count >>
3657 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3658 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3659 trk_long_idle_sample_count);
Marek Vasut6cb9f162015-07-12 20:49:39 +02003660 writel(reg, &sdr_ctrl->phy_ctrl1);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003661
3662 reg = 0;
3663 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3664 trk_long_idle_sample_count >>
3665 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
Marek Vasut6cb9f162015-07-12 20:49:39 +02003666 writel(reg, &sdr_ctrl->phy_ctrl2);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003667}
3668
Marek Vasut880e46f2015-07-17 00:45:11 +02003669/**
3670 * initialize_tracking() - Initialize tracking
3671 *
3672 * Initialize the register file with usable initial data.
3673 */
Dinh Nguyen3da42852015-06-02 22:52:49 -05003674static void initialize_tracking(void)
3675{
Marek Vasut880e46f2015-07-17 00:45:11 +02003676 /*
3677 * Initialize the register file with the correct data.
3678 * Compute usable version of value in case we skip full
3679 * computation later.
3680 */
3681 writel(DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP) - 1,
3682 &sdr_reg_file->dtaps_per_ptap);
3683
3684 /* trk_sample_count */
3685 writel(7500, &sdr_reg_file->trk_sample_count);
3686
3687 /* longidle outer loop [15:0] */
3688 writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003689
3690 /*
Marek Vasut880e46f2015-07-17 00:45:11 +02003691 * longidle sample count [31:24]
3692 * trfc, worst case of 933Mhz 4Gb [23:16]
3693 * trcd, worst case [15:8]
3694 * vfifo wait [7:0]
Dinh Nguyen3da42852015-06-02 22:52:49 -05003695 */
Marek Vasut880e46f2015-07-17 00:45:11 +02003696 writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
3697 &sdr_reg_file->delays);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003698
Marek Vasut880e46f2015-07-17 00:45:11 +02003699 /* mux delay */
3700 writel((RW_MGR_IDLE << 24) | (RW_MGR_ACTIVATE_1 << 16) |
3701 (RW_MGR_SGLE_READ << 8) | (RW_MGR_PRECHARGE_ALL << 0),
3702 &sdr_reg_file->trk_rw_mgr_addr);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003703
Marek Vasut880e46f2015-07-17 00:45:11 +02003704 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH,
3705 &sdr_reg_file->trk_read_dqs_width);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003706
Marek Vasut880e46f2015-07-17 00:45:11 +02003707 /* trefi [7:0] */
3708 writel((RW_MGR_REFRESH_ALL << 24) | (1000 << 0),
3709 &sdr_reg_file->trk_rfsh);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003710}
3711
3712int sdram_calibration_full(void)
3713{
3714 struct param_type my_param;
3715 struct gbl_type my_gbl;
3716 uint32_t pass;
Marek Vasut84e0b0c2015-07-17 01:05:36 +02003717
3718 memset(&my_param, 0, sizeof(my_param));
3719 memset(&my_gbl, 0, sizeof(my_gbl));
Dinh Nguyen3da42852015-06-02 22:52:49 -05003720
3721 param = &my_param;
3722 gbl = &my_gbl;
3723
Dinh Nguyen3da42852015-06-02 22:52:49 -05003724 /* Set the calibration enabled by default */
3725 gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3726 /*
3727 * Only sweep all groups (regardless of fail state) by default
3728 * Set enabled read test by default.
3729 */
3730#if DISABLE_GUARANTEED_READ
3731 gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3732#endif
3733 /* Initialize the register file */
3734 initialize_reg_file();
3735
3736 /* Initialize any PHY CSR */
3737 initialize_hps_phy();
3738
3739 scc_mgr_initialize();
3740
3741 initialize_tracking();
3742
Dinh Nguyen3da42852015-06-02 22:52:49 -05003743 printf("%s: Preparing to start memory calibration\n", __FILE__);
3744
3745 debug("%s:%d\n", __func__, __LINE__);
Marek Vasut23f62b32015-07-13 01:05:27 +02003746 debug_cond(DLEVEL == 1,
3747 "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
3748 RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
3749 RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
3750 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
3751 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
3752 debug_cond(DLEVEL == 1,
3753 "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
3754 RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3755 RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
3756 IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
3757 debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
3758 IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
3759 debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
3760 IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
3761 IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
3762 debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
3763 IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
3764 IO_IO_OUT2_DELAY_MAX);
3765 debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
3766 IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003767
3768 hc_initialize_rom_data();
3769
3770 /* update info for sims */
3771 reg_file_set_stage(CAL_STAGE_NIL);
3772 reg_file_set_group(0);
3773
3774 /*
3775 * Load global needed for those actions that require
3776 * some dynamic calibration support.
3777 */
3778 dyn_calib_steps = STATIC_CALIB_STEPS;
3779 /*
3780 * Load global to allow dynamic selection of delay loop settings
3781 * based on calibration mode.
3782 */
3783 if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
3784 skip_delay_mask = 0xff;
3785 else
3786 skip_delay_mask = 0x0;
3787
3788 pass = run_mem_calibrate();
Marek Vasut23a040c2015-07-17 01:20:21 +02003789 debug_mem_calibrate(pass);
Dinh Nguyen3da42852015-06-02 22:52:49 -05003790 return pass;
3791}