arch/mips/mach-octeon/cvmx-helper-ilk.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2018-2022 Marvell International Ltd.
 *
 * Functions for ILK initialization, configuration,
 * and monitoring.
 */

#include <time.h>
#include <log.h>
#include <linux/delay.h>

#include <mach/cvmx-regs.h>
#include <mach/cvmx-csr.h>
#include <mach/cvmx-bootmem.h>
#include <mach/octeon-model.h>
#include <mach/cvmx-fuse.h>
#include <mach/octeon-feature.h>
#include <mach/cvmx-qlm.h>
#include <mach/octeon_qlm.h>
#include <mach/cvmx-pcie.h>
#include <mach/cvmx-coremask.h>

#include <mach/cvmx-agl-defs.h>
#include <mach/cvmx-bgxx-defs.h>
#include <mach/cvmx-ciu-defs.h>
#include <mach/cvmx-gmxx-defs.h>
#include <mach/cvmx-ilk-defs.h>
#include <mach/cvmx-ipd-defs.h>
#include <mach/cvmx-pcsx-defs.h>
#include <mach/cvmx-pcsxx-defs.h>
#include <mach/cvmx-pki-defs.h>
#include <mach/cvmx-pko-defs.h>
#include <mach/cvmx-xcv-defs.h>

#include <mach/cvmx-hwpko.h>
#include <mach/cvmx-ilk.h>
#include <mach/cvmx-pki.h>

#include <mach/cvmx-helper.h>
#include <mach/cvmx-helper-board.h>
#include <mach/cvmx-helper-cfg.h>

int __cvmx_helper_ilk_enumerate(int xiface)
{
	struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);

	xi.interface -= CVMX_ILK_GBL_BASE();
	return cvmx_ilk_chans[xi.node][xi.interface];
}

/**
 * @INTERNAL
 * Initialize all tx calendar entries to the xoff state.
 * Initialize all rx calendar entries to the xon state. The rx calendar entries
 * must be in the xon state to allow new pko pipe assignments. If a calendar
 * entry is assigned a different pko pipe while in the xoff state, the old pko
 * pipe will stay in the xoff state even when no longer used by ilk.
 *
 * @param intf Interface whose calendar are to be initialized.
 */
static void __cvmx_ilk_clear_cal_cn78xx(int intf)
{
	cvmx_ilk_txx_cal_entryx_t tx_entry;
	cvmx_ilk_rxx_cal_entryx_t rx_entry;
	int i;
	int node = (intf >> 4) & 0xf;
	int interface = (intf & 0xf);

	/* Initialize all tx calendar entries to off */
	tx_entry.u64 = 0;
	tx_entry.s.ctl = XOFF;
	for (i = 0; i < CVMX_ILK_MAX_CAL; i++) {
		csr_wr_node(node, CVMX_ILK_TXX_CAL_ENTRYX(i, interface),
			    tx_entry.u64);
	}

	/* Initialize all rx calendar entries to on */
	rx_entry.u64 = 0;
	rx_entry.s.ctl = XOFF;
	for (i = 0; i < CVMX_ILK_MAX_CAL; i++) {
		csr_wr_node(node, CVMX_ILK_RXX_CAL_ENTRYX(i, interface),
			    rx_entry.u64);
	}
}

/**
 * @INTERNAL
 * Initialize all tx calendar entries to the xoff state.
 * Initialize all rx calendar entries to the xon state. The rx calendar entries
 * must be in the xon state to allow new pko pipe assignments. If a calendar
 * entry is assigned a different pko pipe while in the xoff state, the old pko
 * pipe will stay in the xoff state even when no longer used by ilk.
 *
 * @param interface whose calendar are to be initialized.
 */
static void __cvmx_ilk_clear_cal_cn68xx(int interface)
{
	cvmx_ilk_txx_idx_cal_t tx_idx;
	cvmx_ilk_txx_mem_cal0_t tx_cal0;
	cvmx_ilk_txx_mem_cal1_t tx_cal1;
	cvmx_ilk_rxx_idx_cal_t rx_idx;
	cvmx_ilk_rxx_mem_cal0_t rx_cal0;
	cvmx_ilk_rxx_mem_cal1_t rx_cal1;
	int i;

	/*
	 * First we initialize the tx calendar starting from entry 0,
	 * incrementing the entry with every write.
	 */
	tx_idx.u64 = 0;
	tx_idx.s.inc = 1;
	csr_wr(CVMX_ILK_TXX_IDX_CAL(interface), tx_idx.u64);

	/* Set state to xoff for all entries */
	tx_cal0.u64 = 0;
	tx_cal0.s.entry_ctl0 = XOFF;
	tx_cal0.s.entry_ctl1 = XOFF;
	tx_cal0.s.entry_ctl2 = XOFF;
	tx_cal0.s.entry_ctl3 = XOFF;

	tx_cal1.u64 = 0;
	tx_cal1.s.entry_ctl4 = XOFF;
	tx_cal1.s.entry_ctl5 = XOFF;
	tx_cal1.s.entry_ctl6 = XOFF;
	tx_cal1.s.entry_ctl7 = XOFF;

	/* Write all 288 entries */
	for (i = 0; i < CVMX_ILK_MAX_CAL_IDX; i++) {
		csr_wr(CVMX_ILK_TXX_MEM_CAL0(interface), tx_cal0.u64);
		csr_wr(CVMX_ILK_TXX_MEM_CAL1(interface), tx_cal1.u64);
	}

	/*
	 * Next we initialize the rx calendar starting from entry 0,
	 * incrementing the entry with every write.
	 */
	rx_idx.u64 = 0;
	rx_idx.s.inc = 1;
	csr_wr(CVMX_ILK_RXX_IDX_CAL(interface), rx_idx.u64);

	/* Set state to xon for all entries */
	rx_cal0.u64 = 0;
	rx_cal0.s.entry_ctl0 = XON;
	rx_cal0.s.entry_ctl1 = XON;
	rx_cal0.s.entry_ctl2 = XON;
	rx_cal0.s.entry_ctl3 = XON;

	rx_cal1.u64 = 0;
	rx_cal1.s.entry_ctl4 = XON;
	rx_cal1.s.entry_ctl5 = XON;
	rx_cal1.s.entry_ctl6 = XON;
	rx_cal1.s.entry_ctl7 = XON;

	/* Write all 288 entries */
	for (i = 0; i < CVMX_ILK_MAX_CAL_IDX; i++) {
		csr_wr(CVMX_ILK_RXX_MEM_CAL0(interface), rx_cal0.u64);
		csr_wr(CVMX_ILK_RXX_MEM_CAL1(interface), rx_cal1.u64);
	}
}

/**
 * @INTERNAL
 * Initialize all calendar entries.
 *
 * @param interface whose calendar is to be initialized.
 */
void __cvmx_ilk_clear_cal(int interface)
{
	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
		__cvmx_ilk_clear_cal_cn68xx(interface);
	else if (OCTEON_IS_MODEL(OCTEON_CN78XX))
		__cvmx_ilk_clear_cal_cn78xx(interface);
}

void __cvmx_ilk_write_tx_cal_entry_cn68xx(int interface, int channel,
					  unsigned char bpid)
{
	cvmx_ilk_txx_idx_cal_t tx_idx;
	cvmx_ilk_txx_mem_cal0_t tx_cal0;
	cvmx_ilk_txx_mem_cal1_t tx_cal1;
	int entry;
	int window;
	int window_entry;

	/*
	 * The calendar has 288 entries. Each calendar entry represents a
	 * channel's flow control state or the link flow control state.
	 * Starting with the first entry, every sixteenth entry is used for the
	 * link flow control state. The other 15 entries are used for the
	 * channels flow control state:
	 * entry 0   ----> link flow control state
	 * entry 1   ----> channel 0 flow control state
	 * entry 2   ----> channel 1 flow control state
	 * ...
	 * entry 15  ----> channel 14 flow control state
	 * entry 16  ----> link flow control state
	 * entry 17  ----> channel 15 flow control state
	 *
	 * Also, the calendar is accessed via windows into it. Each window maps
	 * to 8 entries.
	 */
	entry = 1 + channel + (channel / 15);
	window = entry / 8;
	window_entry = entry % 8;

	/* Indicate the window we need to access */
	tx_idx.u64 = 0;
	tx_idx.s.index = window;
	csr_wr(CVMX_ILK_TXX_IDX_CAL(interface), tx_idx.u64);

	/* Get the window's current value */
	tx_cal0.u64 = csr_rd(CVMX_ILK_TXX_MEM_CAL0(interface));
	tx_cal1.u64 = csr_rd(CVMX_ILK_TXX_MEM_CAL1(interface));

	/* Force every sixteenth entry as link flow control state */
	if ((window & 1) == 0)
		tx_cal0.s.entry_ctl0 = LINK;

	/* Update the entry */
	switch (window_entry) {
	case 0:
		tx_cal0.s.entry_ctl0 = 0;
		tx_cal0.s.bpid0 = bpid;
		break;
	case 1:
		tx_cal0.s.entry_ctl1 = 0;
		tx_cal0.s.bpid1 = bpid;
		break;
	case 2:
		tx_cal0.s.entry_ctl2 = 0;
		tx_cal0.s.bpid2 = bpid;
		break;
	case 3:
		tx_cal0.s.entry_ctl3 = 0;
		tx_cal0.s.bpid3 = bpid;
		break;
	case 4:
		tx_cal1.s.entry_ctl4 = 0;
		tx_cal1.s.bpid4 = bpid;
		break;
	case 5:
		tx_cal1.s.entry_ctl5 = 0;
		tx_cal1.s.bpid5 = bpid;
		break;
	case 6:
		tx_cal1.s.entry_ctl6 = 0;
		tx_cal1.s.bpid6 = bpid;
		break;
	case 7:
		tx_cal1.s.entry_ctl7 = 0;
		tx_cal1.s.bpid7 = bpid;
		break;
	}

	/* Write the window new value */
	csr_wr(CVMX_ILK_TXX_MEM_CAL0(interface), tx_cal0.u64);
	csr_wr(CVMX_ILK_TXX_MEM_CAL1(interface), tx_cal1.u64);
}

void __cvmx_ilk_write_tx_cal_entry_cn78xx(int intf, int channel,
					  unsigned char bpid)
{
	cvmx_ilk_txx_cal_entryx_t tx_cal;
	int calender_16_block = channel / 15;
	int calender_16_index = channel % 15 + 1;
	int index = calender_16_block * 16 + calender_16_index;
	int node = (intf >> 4) & 0xf;
	int interface = intf & 0xf;

	/* Program the link status on first channel */
	if (calender_16_index == 1) {
		tx_cal.u64 = 0;
		tx_cal.s.ctl = 1;
		csr_wr_node(node, CVMX_ILK_TXX_CAL_ENTRYX(index - 1, interface),
			    tx_cal.u64);
	}
	tx_cal.u64 = 0;
	tx_cal.s.ctl = 0;
	tx_cal.s.channel = channel;
	csr_wr_node(node, CVMX_ILK_TXX_CAL_ENTRYX(index, interface),
		    tx_cal.u64);
}

/**
 * @INTERNAL
 * Setup the channel's tx calendar entry.
 *
 * @param interface channel belongs to
 * @param channel whose calendar entry is to be updated
 * @param bpid assigned to the channel
 */
void __cvmx_ilk_write_tx_cal_entry(int interface, int channel,
				   unsigned char bpid)
{
	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
		__cvmx_ilk_write_tx_cal_entry_cn68xx(interface, channel, bpid);
	else
		__cvmx_ilk_write_tx_cal_entry_cn78xx(interface, channel, bpid);
}

void __cvmx_ilk_write_rx_cal_entry_cn78xx(int intf, int channel,
					  unsigned char bpid)
{
	cvmx_ilk_rxx_cal_entryx_t rx_cal;
	int calender_16_block = channel / 15;
	int calender_16_index = channel % 15 + 1;
	int index = calender_16_block * 16 + calender_16_index;
	int node = (intf >> 4) & 0xf;
	int interface = intf & 0xf;

	/* Program the link status on first channel */
	if (calender_16_index == 1) {
		rx_cal.u64 = 0;
		rx_cal.s.ctl = 1;
		csr_wr_node(node, CVMX_ILK_RXX_CAL_ENTRYX(index - 1, interface),
			    rx_cal.u64);
	}
	rx_cal.u64 = 0;
	rx_cal.s.ctl = 0;
	rx_cal.s.channel = channel;
	csr_wr_node(node, CVMX_ILK_RXX_CAL_ENTRYX(index, interface),
		    rx_cal.u64);
}

void __cvmx_ilk_write_rx_cal_entry_cn68xx(int interface, int channel,
					  unsigned char pipe)
{
	cvmx_ilk_rxx_idx_cal_t rx_idx;
	cvmx_ilk_rxx_mem_cal0_t rx_cal0;
	cvmx_ilk_rxx_mem_cal1_t rx_cal1;
	int entry;
	int window;
	int window_entry;

	/*
	 * The calendar has 288 entries. Each calendar entry represents a
	 * channel's flow control state or the link flow control state.
	 * Starting with the first entry, every sixteenth entry is used for the
	 * link flow control state. The other 15 entries are used for the
	 * channels flow control state:
	 * entry 0   ----> link flow control state
	 * entry 1   ----> channel 0 flow control state
	 * entry 2   ----> channel 1 flow control state
	 * ...
	 * entry 15  ----> channel 14 flow control state
	 * entry 16  ----> link flow control state
	 * entry 17  ----> channel 15 flow control state
	 *
	 * Also, the calendar is accessed via windows into it. Each window maps
	 * to 8 entries.
	 */
	entry = 1 + channel + (channel / 15);
	window = entry / 8;
	window_entry = entry % 8;

	/* Indicate the window we need to access */
	rx_idx.u64 = 0;
	rx_idx.s.index = window;
	csr_wr(CVMX_ILK_RXX_IDX_CAL(interface), rx_idx.u64);

	/* Get the window's current value */
	rx_cal0.u64 = csr_rd(CVMX_ILK_RXX_MEM_CAL0(interface));
	rx_cal1.u64 = csr_rd(CVMX_ILK_RXX_MEM_CAL1(interface));

	/* Force every sixteenth entry as link flow control state */
	if ((window & 1) == 0)
		rx_cal0.s.entry_ctl0 = LINK;

	/* Update the entry */
	switch (window_entry) {
	case 0:
		rx_cal0.s.entry_ctl0 = 0;
		rx_cal0.s.port_pipe0 = pipe;
		break;
	case 1:
		rx_cal0.s.entry_ctl1 = 0;
		rx_cal0.s.port_pipe1 = pipe;
		break;
	case 2:
		rx_cal0.s.entry_ctl2 = 0;
		rx_cal0.s.port_pipe2 = pipe;
		break;
	case 3:
		rx_cal0.s.entry_ctl3 = 0;
		rx_cal0.s.port_pipe3 = pipe;
		break;
	case 4:
		rx_cal1.s.entry_ctl4 = 0;
		rx_cal1.s.port_pipe4 = pipe;
		break;
	case 5:
		rx_cal1.s.entry_ctl5 = 0;
		rx_cal1.s.port_pipe5 = pipe;
		break;
	case 6:
		rx_cal1.s.entry_ctl6 = 0;
		rx_cal1.s.port_pipe6 = pipe;
		break;
	case 7:
		rx_cal1.s.entry_ctl7 = 0;
		rx_cal1.s.port_pipe7 = pipe;
		break;
	}

	/* Write the window new value */
	csr_wr(CVMX_ILK_RXX_MEM_CAL0(interface), rx_cal0.u64);
	csr_wr(CVMX_ILK_RXX_MEM_CAL1(interface), rx_cal1.u64);
}

/**
 * @INTERNAL
 * Setup the channel's rx calendar entry.
 *
 * @param interface channel belongs to
 * @param channel whose calendar entry is to be updated
 * @param pipe PKO assigned to the channel
 */
void __cvmx_ilk_write_rx_cal_entry(int interface, int channel,
				   unsigned char pipe)
{
	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
		__cvmx_ilk_write_rx_cal_entry_cn68xx(interface, channel, pipe);
	else
		__cvmx_ilk_write_rx_cal_entry_cn78xx(interface, channel, pipe);
}

/**
 * @INTERNAL
 * Probe a ILK interface and determine the number of ports
 * connected to it. The ILK interface should still be down
 * after this call.
 *
 * @param xiface Interface to probe
 *
 * @return Number of ports on the interface. Zero to disable.
 */
int __cvmx_helper_ilk_probe(int xiface)
{
	int res = 0;
	int interface;
	struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);

	if (!octeon_has_feature(OCTEON_FEATURE_ILK))
		return res;

	interface = xi.interface - CVMX_ILK_GBL_BASE();
	if (interface >= CVMX_NUM_ILK_INTF)
		return 0;

	/* the configuration should be done only once */
	if (cvmx_ilk_get_intf_ena(xiface))
		return cvmx_ilk_chans[xi.node][interface];

	/* configure lanes and enable the link */
	res = cvmx_ilk_start_interface(((xi.node << 4) | interface),
				       cvmx_ilk_lane_mask[xi.node][interface]);
	if (res < 0)
		return 0;

	res = __cvmx_helper_ilk_enumerate(xiface);

	return res;
}

static int __cvmx_helper_ilk_init_port_cn68xx(int xiface)
{
	int i, j, res = -1;
	static int pipe_base = 0, pknd_base;
	static cvmx_ilk_pipe_chan_t *pch = NULL, *tmp;
	static cvmx_ilk_chan_pknd_t *chpknd = NULL, *tmp1;
	static cvmx_ilk_cal_entry_t *calent = NULL, *tmp2;
	int enable_rx_cal = 1;
	int interface;
	struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);
	int intf;
	int num_chans;

	interface = xi.interface - CVMX_ILK_GBL_BASE();
	intf = (xi.node << 4) | interface;
	if (interface >= CVMX_NUM_ILK_INTF)
		return 0;

	num_chans = cvmx_ilk_chans[0][interface];

	/* set up channel to pkind mapping */
	if (pknd_base == 0)
		pknd_base = cvmx_helper_get_pknd(xiface, 0);

	/* set up the group of pipes available to ilk */
	if (pipe_base == 0)
		pipe_base =
			__cvmx_pko_get_pipe(interface + CVMX_ILK_GBL_BASE(), 0);

	if (pipe_base == -1) {
		pipe_base = 0;
		return 0;
	}

	res = cvmx_ilk_set_pipe(xiface, pipe_base,
				cvmx_ilk_chans[0][interface]);
	if (res < 0)
		return 0;

	/* set up pipe to channel mapping */
	i = pipe_base;
	if (!pch) {
		pch = (cvmx_ilk_pipe_chan_t *)cvmx_bootmem_alloc(
			num_chans * sizeof(cvmx_ilk_pipe_chan_t),
			sizeof(cvmx_ilk_pipe_chan_t));
		if (!pch)
			return 0;
	}

	memset(pch, 0, num_chans * sizeof(cvmx_ilk_pipe_chan_t));
	tmp = pch;
	for (j = 0; j < num_chans; j++) {
		tmp->pipe = i++;
		tmp->chan = j;
		tmp++;
	}
	res = cvmx_ilk_tx_set_channel(interface, pch,
				      cvmx_ilk_chans[0][interface]);
	if (res < 0) {
		res = 0;
		goto err_free_pch;
	}
	pipe_base += cvmx_ilk_chans[0][interface];
	i = pknd_base;
	if (!chpknd) {
		chpknd = (cvmx_ilk_chan_pknd_t *)cvmx_bootmem_alloc(
			CVMX_ILK_MAX_PKNDS * sizeof(cvmx_ilk_chan_pknd_t),
			sizeof(cvmx_ilk_chan_pknd_t));
		if (!chpknd) {
			pipe_base -= cvmx_ilk_chans[xi.node][interface];
			res = 0;
			goto err_free_pch;
		}
	}

	memset(chpknd, 0, CVMX_ILK_MAX_PKNDS * sizeof(cvmx_ilk_chan_pknd_t));
	tmp1 = chpknd;
	for (j = 0; j < cvmx_ilk_chans[xi.node][interface]; j++) {
		tmp1->chan = j;
		tmp1->pknd = i++;
		tmp1++;
	}

	res = cvmx_ilk_rx_set_pknd(xiface, chpknd,
				   cvmx_ilk_chans[xi.node][interface]);
	if (res < 0) {
		pipe_base -= cvmx_ilk_chans[xi.node][interface];
		res = 0;
		goto err_free_chpknd;
	}
	pknd_base += cvmx_ilk_chans[xi.node][interface];

	/* Set up tx calendar */
	if (!calent) {
		calent = (cvmx_ilk_cal_entry_t *)cvmx_bootmem_alloc(
			CVMX_ILK_MAX_PIPES * sizeof(cvmx_ilk_cal_entry_t),
			sizeof(cvmx_ilk_cal_entry_t));
		if (!calent) {
			pipe_base -= cvmx_ilk_chans[xi.node][interface];
			pknd_base -= cvmx_ilk_chans[xi.node][interface];
			res = 0;
			goto err_free_chpknd;
		}
	}

	memset(calent, 0, CVMX_ILK_MAX_PIPES * sizeof(cvmx_ilk_cal_entry_t));
	tmp1 = chpknd;
	tmp2 = calent;
	for (j = 0; j < cvmx_ilk_chans[xi.node][interface]; j++) {
		tmp2->pipe_bpid = tmp1->pknd;
		tmp2->ent_ctrl = PIPE_BPID;
		tmp1++;
		tmp2++;
	}
	res = cvmx_ilk_cal_setup_tx(intf, cvmx_ilk_chans[xi.node][interface],
				    calent, 1);
	if (res < 0) {
		pipe_base -= cvmx_ilk_chans[xi.node][interface];
		pknd_base -= cvmx_ilk_chans[xi.node][interface];
		res = 0;
		goto err_free_calent;
	}

	/* set up rx calendar. allocated memory can be reused.
	 * this is because max pkind is always less than max pipe
	 */
	memset(calent, 0, CVMX_ILK_MAX_PIPES * sizeof(cvmx_ilk_cal_entry_t));
	tmp = pch;
	tmp2 = calent;
	for (j = 0; j < cvmx_ilk_chans[0][interface]; j++) {
		tmp2->pipe_bpid = tmp->pipe;
		tmp2->ent_ctrl = PIPE_BPID;
		tmp++;
		tmp2++;
	}
	if (cvmx_ilk_use_la_mode(interface, 0))
		enable_rx_cal = cvmx_ilk_la_mode_enable_rx_calendar(interface);
	else
		enable_rx_cal = 1;

	res = cvmx_ilk_cal_setup_rx(intf, cvmx_ilk_chans[xi.node][interface],
				    calent, CVMX_ILK_RX_FIFO_WM, enable_rx_cal);
	if (res < 0) {
		pipe_base -= cvmx_ilk_chans[xi.node][interface];
		pknd_base -= cvmx_ilk_chans[xi.node][interface];
		res = 0;
		goto err_free_calent;
	}
	goto out;

err_free_calent:
	/* no free() for cvmx_bootmem_alloc() */

err_free_chpknd:
	/* no free() for cvmx_bootmem_alloc() */

err_free_pch:
	/* no free() for cvmx_bootmem_alloc() */
out:
	return res;
}

static int __cvmx_helper_ilk_init_port_cn78xx(int xiface)
{
	struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);
	int interface;
	int intf;

	interface = xi.interface - CVMX_ILK_GBL_BASE();
	intf = (xi.node << 4) | interface;
	if (interface >= CVMX_NUM_ILK_INTF)
		return 0;

	if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
		struct cvmx_pki_style_config style_cfg;
		int num_channels = cvmx_ilk_chans[xi.node][interface];
		int index, i;

		for (i = 0; i < num_channels; i++) {
			int pknd;

			index = (i % 8);

			/* Set jabber to allow max sized packets */
			if (i == 0)
				csr_wr_node(xi.node,
					    CVMX_ILK_RXX_JABBER(interface),
					    0xfff8);

			/* Setup PKND */
			pknd = cvmx_helper_get_pknd(xiface, index);
			csr_wr_node(xi.node, CVMX_ILK_RXX_CHAX(i, interface),
				    pknd);
			cvmx_pki_read_style_config(
				0, pknd, CVMX_PKI_CLUSTER_ALL, &style_cfg);
			style_cfg.parm_cfg.qpg_port_sh = 0;
			/* 256 channels */
			style_cfg.parm_cfg.qpg_port_msb = 8;
			cvmx_pki_write_style_config(
				0, pknd, CVMX_PKI_CLUSTER_ALL, &style_cfg);
		}

		cvmx_ilk_cal_setup_tx(intf, num_channels, NULL, 1);
		cvmx_ilk_cal_setup_rx(intf, num_channels, NULL,
				      CVMX_ILK_RX_FIFO_WM, 1);
	}
	return 0;
}

static int __cvmx_helper_ilk_init_port(int xiface)
{
	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
		return __cvmx_helper_ilk_init_port_cn68xx(xiface);
	else
		return __cvmx_helper_ilk_init_port_cn78xx(xiface);
}

/**
 * @INTERNAL
 * Bringup and enable ILK interface. After this call packet
 * I/O should be fully functional. This is called with IPD
 * enabled but PKO disabled.
 *
 * @param xiface Interface to bring up
 *
 * @return Zero on success, negative on failure
 */
int __cvmx_helper_ilk_enable(int xiface)
{
	if (__cvmx_helper_ilk_init_port(xiface) < 0)
		return -1;

	return cvmx_ilk_enable(xiface);
}

/**
 * @INTERNAL
 * Return the link state of an IPD/PKO port as returned by ILK link status.
 *
 * @param ipd_port IPD/PKO port to query
 *
 * @return Link state
 */
cvmx_helper_link_info_t __cvmx_helper_ilk_link_get(int ipd_port)
{
	cvmx_helper_link_info_t result;
	int xiface = cvmx_helper_get_interface_num(ipd_port);
	struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);
	int interface;
	int retry_count = 0;
	cvmx_ilk_rxx_cfg1_t ilk_rxx_cfg1;
	cvmx_ilk_rxx_int_t ilk_rxx_int;
	int lane_mask = 0;
	int i;
	int node = xi.node;

	result.u64 = 0;
	interface = xi.interface - CVMX_ILK_GBL_BASE();

retry:
	retry_count++;
	if (retry_count > 200)
		goto fail;

	/* Read RX config and status bits */
	ilk_rxx_cfg1.u64 = csr_rd_node(node, CVMX_ILK_RXX_CFG1(interface));
	ilk_rxx_int.u64 = csr_rd_node(node, CVMX_ILK_RXX_INT(interface));

	if (ilk_rxx_cfg1.s.rx_bdry_lock_ena == 0) {
		/* (GSER-21957) GSER RX Equalization may make >= 5gbaud non-KR
		 * channel better
		 */
		if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
			int qlm, lane_mask;

			for (qlm = 4; qlm < 8; qlm++) {
				lane_mask = 1 << (qlm - 4) * 4;
				if (lane_mask &
				    cvmx_ilk_lane_mask[node][interface]) {
					if (__cvmx_qlm_rx_equalization(
						    node, qlm, -1))
						goto retry;
				}
			}
		}

		/* Clear the boundary lock status bit */
		ilk_rxx_int.u64 = 0;
		ilk_rxx_int.s.word_sync_done = 1;
		csr_wr_node(node, CVMX_ILK_RXX_INT(interface), ilk_rxx_int.u64);

		/* We need to start looking for word boundary lock */
		ilk_rxx_cfg1.s.rx_bdry_lock_ena =
			cvmx_ilk_lane_mask[node][interface];
		ilk_rxx_cfg1.s.rx_align_ena = 0;
		csr_wr_node(node, CVMX_ILK_RXX_CFG1(interface),
			    ilk_rxx_cfg1.u64);
		//debug("ILK%d: Looking for word boundary lock\n", interface);
		udelay(50);
		goto retry;
	}

	if (ilk_rxx_cfg1.s.rx_align_ena == 0) {
		if (ilk_rxx_int.s.word_sync_done) {
			/* Clear the lane align status bits */
			ilk_rxx_int.u64 = 0;
			ilk_rxx_int.s.lane_align_fail = 1;
			ilk_rxx_int.s.lane_align_done = 1;
			csr_wr_node(node, CVMX_ILK_RXX_INT(interface),
				    ilk_rxx_int.u64);

			ilk_rxx_cfg1.s.rx_align_ena = 1;
			csr_wr_node(node, CVMX_ILK_RXX_CFG1(interface),
				    ilk_rxx_cfg1.u64);
			//printf("ILK%d: Looking for lane alignment\n", interface);
		}
		udelay(50);
		goto retry;
	}

	if (ilk_rxx_int.s.lane_align_fail) {
		ilk_rxx_cfg1.s.rx_bdry_lock_ena = 0;
		ilk_rxx_cfg1.s.rx_align_ena = 0;
		csr_wr_node(node, CVMX_ILK_RXX_CFG1(interface),
			    ilk_rxx_cfg1.u64);
		//debug("ILK%d: Lane alignment failed\n", interface);
		goto fail;
	}

	lane_mask = ilk_rxx_cfg1.s.rx_bdry_lock_ena;

	if (ilk_rxx_cfg1.s.pkt_ena == 0 && ilk_rxx_int.s.lane_align_done) {
		cvmx_ilk_txx_cfg1_t ilk_txx_cfg1;

		ilk_txx_cfg1.u64 =
			csr_rd_node(node, CVMX_ILK_TXX_CFG1(interface));
		ilk_rxx_cfg1.u64 =
			csr_rd_node(node, CVMX_ILK_RXX_CFG1(interface));
		ilk_rxx_cfg1.s.pkt_ena = ilk_txx_cfg1.s.pkt_ena;
		csr_wr_node(node, CVMX_ILK_RXX_CFG1(interface),
			    ilk_rxx_cfg1.u64);

		if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
			/*
			 * Enable rxf_ctl_perr, rxf_lnk0_perr, rxf_lnk1_perr,
			 * pop_empty, push_full.
			 */
			csr_wr(CVMX_ILK_GBL_INT_EN, 0x1f);
			/* Enable bad_pipe, bad_seq, txf_err */
			csr_wr(CVMX_ILK_TXX_INT_EN(interface), 0x7);

			/*
			 * Enable crc24_err, lane_bad_word,
			 * pkt_drop_{rid,rxf,sop}
			 */
			csr_wr(CVMX_ILK_RXX_INT_EN(interface), 0x1e2);
		}
		/* Need to enable ILK interrupts for 78xx */

		for (i = 0; i < CVMX_ILK_MAX_LANES(); i++) {
			if ((1 << i) & lane_mask) {
				/* clear pending interrupts, before enabling. */
				csr_wr_node(node, CVMX_ILK_RX_LNEX_INT(i),
					    0x1ff);
				/* Enable bad_64b67b, bdry_sync_loss, crc32_err,
				 * dskew_fifo_ovfl, scrm_sync_loss,
				 * serdes_lock_loss, stat_msg, ukwn_cntl_word
				 */
				if (OCTEON_IS_MODEL(OCTEON_CN68XX))
					csr_wr(CVMX_ILK_RX_LNEX_INT_EN(i),
					       0x1ff);
			}
		}

		//debug("ILK%d: Lane alignment complete\n", interface);
	}

	/* Enable error interrupts, now link is up */
	cvmx_error_enable_group(CVMX_ERROR_GROUP_ILK,
				node | (interface << 2) | (lane_mask << 4));

	result.s.link_up = 1;
	result.s.full_duplex = 1;
	if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
		int qlm = cvmx_qlm_lmac(xiface, 0);

		result.s.speed = cvmx_qlm_get_gbaud_mhz(qlm) * 64 / 67;
	} else {
		result.s.speed =
			cvmx_qlm_get_gbaud_mhz(1 + interface) * 64 / 67;
	}
	result.s.speed *= cvmx_pop(lane_mask);

	return result;

fail:
	if (ilk_rxx_cfg1.s.pkt_ena) {
		/* Disable the interface */
		ilk_rxx_cfg1.s.pkt_ena = 0;
		csr_wr_node(node, CVMX_ILK_RXX_CFG1(interface),
			    ilk_rxx_cfg1.u64);

		/* Disable error interrupts */
		for (i = 0; i < CVMX_ILK_MAX_LANES(); i++) {
			/* Disable bad_64b67b, bdry_sync_loss, crc32_err,
			 * dskew_fifo_ovfl, scrm_sync_loss, serdes_lock_loss,
			 * stat_msg, ukwn_cntl_word
			 */
			if ((1 << i) & lane_mask) {
				csr_wr_node(node, CVMX_ILK_RX_LNEX_INT(i),
					    0x1ff);
				if (OCTEON_IS_MODEL(OCTEON_CN68XX))
					csr_wr(CVMX_ILK_RX_LNEX_INT_EN(i),
					       ~0x1ff);
			}
		}
		/* Disable error interrupts */
		cvmx_error_enable_group(CVMX_ERROR_GROUP_ILK, 0);
	}

	return result;
}

/**
 * @INTERNAL
 * Set the link state of an IPD/PKO port.
 *
 * @param ipd_port  IPD/PKO port to configure
 * @param link_info The new link state
 *
 * @return Zero on success, negative on failure
 */
int __cvmx_helper_ilk_link_set(int ipd_port, cvmx_helper_link_info_t link_info)
{
	/* Do nothing */
	return 0;
}