in hw/hfi1/pcie.c [942:1403]
int do_pcie_gen3_transition(struct hfi1_devdata *dd)
{
struct pci_dev *parent = dd->pcidev->bus->self;
u64 fw_ctrl;
u64 reg, therm;
u32 reg32, fs, lf;
u32 status, err;
int ret;
int do_retry, retry_count = 0;
int intnum = 0;
uint default_pset;
uint pset = pcie_pset;
u16 target_vector, target_speed;
u16 lnkctl2, vendor;
u8 div;
const u8 (*eq)[3];
const u8 (*ctle_tunings)[4];
uint static_ctle_mode;
int return_error = 0;
u32 target_width;
/* PCIe Gen3 is for the ASIC only */
if (dd->icode != ICODE_RTL_SILICON)
return 0;
if (pcie_target == 1) { /* target Gen1 */
target_vector = PCI_EXP_LNKCTL2_TLS_2_5GT;
target_speed = 2500;
} else if (pcie_target == 2) { /* target Gen2 */
target_vector = PCI_EXP_LNKCTL2_TLS_5_0GT;
target_speed = 5000;
} else if (pcie_target == 3) { /* target Gen3 */
target_vector = PCI_EXP_LNKCTL2_TLS_8_0GT;
target_speed = 8000;
} else {
/* off or invalid target - skip */
dd_dev_info(dd, "%s: Skipping PCIe transition\n", __func__);
return 0;
}
/* if already at target speed, done (unless forced) */
if (dd->lbus_speed == target_speed) {
dd_dev_info(dd, "%s: PCIe already at gen%d, %s\n", __func__,
pcie_target,
pcie_force ? "re-doing anyway" : "skipping");
if (!pcie_force)
return 0;
}
/*
* The driver cannot do the transition if it has no access to the
* upstream component
*/
if (!parent) {
dd_dev_info(dd, "%s: No upstream, Can't do gen3 transition\n",
__func__);
return 0;
}
/* Previous Gen1/Gen2 bus width */
target_width = dd->lbus_width;
/*
* Do the Gen3 transition. Steps are those of the PCIe Gen3
* recipe.
*/
/* step 1: pcie link working in gen1/gen2 */
/* step 2: if either side is not capable of Gen3, done */
if (pcie_target == 3 && !dd->link_gen3_capable) {
dd_dev_err(dd, "The PCIe link is not Gen3 capable\n");
ret = -ENOSYS;
goto done_no_mutex;
}
/* hold the SBus resource across the firmware download and SBR */
ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
if (ret) {
dd_dev_err(dd, "%s: unable to acquire SBus resource\n",
__func__);
return ret;
}
/* make sure thermal polling is not causing interrupts */
therm = read_csr(dd, ASIC_CFG_THERM_POLL_EN);
if (therm) {
write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
msleep(100);
dd_dev_info(dd, "%s: Disabled therm polling\n",
__func__);
}
retry:
/* the SBus download will reset the spico for thermal */
/* step 3: download SBus Master firmware */
/* step 4: download PCIe Gen3 SerDes firmware */
dd_dev_info(dd, "%s: downloading firmware\n", __func__);
ret = load_pcie_firmware(dd);
if (ret) {
/* do not proceed if the firmware cannot be downloaded */
return_error = 1;
goto done;
}
/* step 5: set up device parameter settings */
dd_dev_info(dd, "%s: setting PCIe registers\n", __func__);
/*
* PcieCfgSpcie1 - Link Control 3
* Leave at reset value. No need to set PerfEq - link equalization
* will be performed automatically after the SBR when the target
* speed is 8GT/s.
*/
/* clear all 16 per-lane error bits (PCIe: Lane Error Status) */
pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, 0xffff);
/* step 5a: Set Synopsys Port Logic registers */
/*
* PcieCfgRegPl2 - Port Force Link
*
* Set the low power field to 0x10 to avoid unnecessary power
* management messages. All other fields are zero.
*/
reg32 = 0x10ul << PCIE_CFG_REG_PL2_LOW_PWR_ENT_CNT_SHIFT;
pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL2, reg32);
/*
* PcieCfgRegPl100 - Gen3 Control
*
* turn off PcieCfgRegPl100.Gen3ZRxDcNonCompl
* turn on PcieCfgRegPl100.EqEieosCnt
* Everything else zero.
*/
reg32 = PCIE_CFG_REG_PL100_EQ_EIEOS_CNT_SMASK;
pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL100, reg32);
/*
* PcieCfgRegPl101 - Gen3 EQ FS and LF
* PcieCfgRegPl102 - Gen3 EQ Presets to Coefficients Mapping
* PcieCfgRegPl103 - Gen3 EQ Preset Index
* PcieCfgRegPl105 - Gen3 EQ Status
*
* Give initial EQ settings.
*/
if (dd->pcidev->device == PCI_DEVICE_ID_INTEL0) { /* discrete */
/* 1000mV, FS=24, LF = 8 */
fs = 24;
lf = 8;
div = 3;
eq = discrete_preliminary_eq;
default_pset = DEFAULT_DISCRETE_PSET;
ctle_tunings = discrete_ctle_tunings;
/* bit 0 - discrete on/off */
static_ctle_mode = pcie_ctle & 0x1;
} else {
/* 400mV, FS=29, LF = 9 */
fs = 29;
lf = 9;
div = 1;
eq = integrated_preliminary_eq;
default_pset = DEFAULT_MCP_PSET;
ctle_tunings = integrated_ctle_tunings;
/* bit 1 - integrated on/off */
static_ctle_mode = (pcie_ctle >> 1) & 0x1;
}
pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,
(fs <<
PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_FS_SHIFT) |
(lf <<
PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_LF_SHIFT));
ret = load_eq_table(dd, eq, fs, div);
if (ret)
goto done;
/*
* PcieCfgRegPl106 - Gen3 EQ Control
*
* Set Gen3EqPsetReqVec, leave other fields 0.
*/
if (pset == UNSET_PSET)
pset = default_pset;
if (pset > 10) { /* valid range is 0-10, inclusive */
dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n",
__func__, pset, default_pset);
pset = default_pset;
}
dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset);
pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106,
((1 << pset) <<
PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) |
PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK |
PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK);
/*
* step 5b: Do post firmware download steps via SBus
*/
dd_dev_info(dd, "%s: doing pcie post steps\n", __func__);
pcie_post_steps(dd);
/*
* step 5c: Program gasket interrupts
*/
/* set the Rx Bit Rate to REFCLK ratio */
write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050);
/* disable pCal for PCIe Gen3 RX equalization */
/* select adaptive or static CTLE */
write_gasket_interrupt(dd, intnum++, 0x0026,
0x5b01 | (static_ctle_mode << 3));
/*
* Enable iCal for PCIe Gen3 RX equalization, and set which
* evaluation of RX_EQ_EVAL will launch the iCal procedure.
*/
write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202);
if (static_ctle_mode) {
/* apply static CTLE tunings */
u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;
pcie_dc = ctle_tunings[pset][0];
pcie_lf = ctle_tunings[pset][1];
pcie_hf = ctle_tunings[pset][2];
pcie_bw = ctle_tunings[pset][3];
write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc);
write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf);
write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf);
write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 | pcie_bw);
}
/* terminate list */
write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000);
/*
* step 5d: program XMT margin
*/
write_xmt_margin(dd, __func__);
/*
* step 5e: disable active state power management (ASPM). It
* will be enabled if required later
*/
dd_dev_info(dd, "%s: clearing ASPM\n", __func__);
aspm_hw_disable_l1(dd);
/*
* step 5f: clear DirectSpeedChange
* PcieCfgRegPl67.DirectSpeedChange must be zero to prevent the
* change in the speed target from starting before we are ready.
* This field defaults to 0 and we are not changing it, so nothing
* needs to be done.
*/
/* step 5g: Set target link speed */
/*
* Set target link speed to be target on both device and parent.
* On setting the parent: Some system BIOSs "helpfully" set the
* parent target speed to Gen2 to match the ASIC's initial speed.
* We can set the target Gen3 because we have already checked
* that it is Gen3 capable earlier.
*/
dd_dev_info(dd, "%s: setting parent target link speed\n", __func__);
ret = pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2);
if (ret) {
dd_dev_err(dd, "Unable to read from PCI config\n");
return_error = 1;
goto done;
}
dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
(u32)lnkctl2);
/* only write to parent if target is not as high as ours */
if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) {
lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
lnkctl2 |= target_vector;
dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
(u32)lnkctl2);
ret = pcie_capability_write_word(parent,
PCI_EXP_LNKCTL2, lnkctl2);
if (ret) {
dd_dev_err(dd, "Unable to write to PCI config\n");
return_error = 1;
goto done;
}
} else {
dd_dev_info(dd, "%s: ..target speed is OK\n", __func__);
}
dd_dev_info(dd, "%s: setting target link speed\n", __func__);
ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2);
if (ret) {
dd_dev_err(dd, "Unable to read from PCI config\n");
return_error = 1;
goto done;
}
dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
(u32)lnkctl2);
lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
lnkctl2 |= target_vector;
dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
(u32)lnkctl2);
ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2);
if (ret) {
dd_dev_err(dd, "Unable to write to PCI config\n");
return_error = 1;
goto done;
}
/* step 5h: arm gasket logic */
/* hold DC in reset across the SBR */
write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
(void)read_csr(dd, CCE_DC_CTRL); /* DC reset hold */
/* save firmware control across the SBR */
fw_ctrl = read_csr(dd, MISC_CFG_FW_CTRL);
dd_dev_info(dd, "%s: arming gasket logic\n", __func__);
arm_gasket_logic(dd);
/*
* step 6: quiesce PCIe link
* The chip has already been reset, so there will be no traffic
* from the chip. Linux has no easy way to enforce that it will
* not try to access the device, so we just need to hope it doesn't
* do it while we are doing the reset.
*/
/*
* step 7: initiate the secondary bus reset (SBR)
* step 8: hardware brings the links back up
* step 9: wait for link speed transition to be complete
*/
dd_dev_info(dd, "%s: calling trigger_sbr\n", __func__);
ret = trigger_sbr(dd);
if (ret)
goto done;
/* step 10: decide what to do next */
/* check if we can read PCI space */
ret = pci_read_config_word(dd->pcidev, PCI_VENDOR_ID, &vendor);
if (ret) {
dd_dev_info(dd,
"%s: read of VendorID failed after SBR, err %d\n",
__func__, ret);
return_error = 1;
goto done;
}
if (vendor == 0xffff) {
dd_dev_info(dd, "%s: VendorID is all 1s after SBR\n", __func__);
return_error = 1;
ret = -EIO;
goto done;
}
/* restore PCI space registers we know were reset */
dd_dev_info(dd, "%s: calling restore_pci_variables\n", __func__);
ret = restore_pci_variables(dd);
if (ret) {
dd_dev_err(dd, "%s: Could not restore PCI variables\n",
__func__);
return_error = 1;
goto done;
}
/* restore firmware control */
write_csr(dd, MISC_CFG_FW_CTRL, fw_ctrl);
/*
* Check the gasket block status.
*
* This is the first CSR read after the SBR. If the read returns
* all 1s (fails), the link did not make it back.
*
* Once we're sure we can read and write, clear the DC reset after
* the SBR. Then check for any per-lane errors. Then look over
* the status.
*/
reg = read_csr(dd, ASIC_PCIE_SD_HOST_STATUS);
dd_dev_info(dd, "%s: gasket block status: 0x%llx\n", __func__, reg);
if (reg == ~0ull) { /* PCIe read failed/timeout */
dd_dev_err(dd, "SBR failed - unable to read from device\n");
return_error = 1;
ret = -ENOSYS;
goto done;
}
/* clear the DC reset */
write_csr(dd, CCE_DC_CTRL, 0);
/* Set the LED off */
setextled(dd, 0);
/* check for any per-lane errors */
ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, ®32);
if (ret) {
dd_dev_err(dd, "Unable to read from PCI config\n");
return_error = 1;
goto done;
}
dd_dev_info(dd, "%s: per-lane errors: 0x%x\n", __func__, reg32);
/* extract status, look for our HFI */
status = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_SHIFT)
& ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_MASK;
if ((status & (1 << dd->hfi1_id)) == 0) {
dd_dev_err(dd,
"%s: gasket status 0x%x, expecting 0x%x\n",
__func__, status, 1 << dd->hfi1_id);
ret = -EIO;
goto done;
}
/* extract error */
err = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_SHIFT)
& ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_MASK;
if (err) {
dd_dev_err(dd, "%s: gasket error %d\n", __func__, err);
ret = -EIO;
goto done;
}
/* update our link information cache */
update_lbus_info(dd);
dd_dev_info(dd, "%s: new speed and width: %s\n", __func__,
dd->lbus_info);
if (dd->lbus_speed != target_speed ||
dd->lbus_width < target_width) { /* not target */
/* maybe retry */
do_retry = retry_count < pcie_retry;
dd_dev_err(dd, "PCIe link speed or width did not match target%s\n",
do_retry ? ", retrying" : "");
retry_count++;
if (do_retry) {
msleep(100); /* allow time to settle */
goto retry;
}
ret = -EIO;
}
done:
if (therm) {
write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
msleep(100);
dd_dev_info(dd, "%s: Re-enable therm polling\n",
__func__);
}
release_chip_resource(dd, CR_SBUS);
done_no_mutex:
/* return no error if it is OK to be at current speed */
if (ret && !return_error) {
dd_dev_err(dd, "Proceeding at current speed PCIe speed\n");
ret = 0;
}
dd_dev_info(dd, "%s: done\n", __func__);
return ret;
}