// SPDX-License-Identifier: GPL-2.0 use kernel::{device, devres::Devres, error::code::*, pci, prelude::*, sync::Arc}; use crate::driver::Bar0; use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon}; use crate::fb::FbLayout; use crate::fb::SysmemFlush; use crate::firmware::fwsec::{FwsecCommand, FwsecFirmware}; use crate::firmware::{Firmware, FIRMWARE_VERSION}; use crate::gfw; use crate::regs; use crate::util; use crate::vbios::Vbios; use core::fmt; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => { /// Enum representation of the GPU chipset. #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub(crate) enum Chipset { $($variant = $value),*, } impl Chipset { pub(crate) const ALL: &'static [Chipset] = &[ $( Chipset::$variant, )* ]; pub(crate) const NAMES: [&'static str; Self::ALL.len()] = [ $( util::const_bytes_to_str( util::to_lowercase_bytes::<{ stringify!($variant).len() }>( stringify!($variant) ).as_slice() ), )* ]; } // TODO[FPRI]: replace with something like derive(FromPrimitive) impl TryFrom for Chipset { type Error = kernel::error::Error; fn try_from(value: u32) -> Result { match value { $( $value => Ok(Chipset::$variant), )* _ => Err(ENODEV), } } } } } define_chipset!({ // Turing TU102 = 0x162, TU104 = 0x164, TU106 = 0x166, TU117 = 0x167, TU116 = 0x168, // Ampere GA100 = 0x170, GA102 = 0x172, GA103 = 0x173, GA104 = 0x174, GA106 = 0x176, GA107 = 0x177, // Ada AD102 = 0x192, AD103 = 0x193, AD104 = 0x194, AD106 = 0x196, AD107 = 0x197, }); impl Chipset { pub(crate) fn arch(&self) -> Architecture { match self { Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => { Architecture::Turing } Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { Architecture::Ampere } Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { Architecture::Ada } } } } // TODO // // The resulting strings are used to generate firmware paths, hence the // generated strings have to be stable. // // Hence, replace with something like strum_macros derive(Display). // // For now, redirect to fmt::Debug for convenience. impl fmt::Display for Chipset { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self:?}") } } /// Enum representation of the GPU generation. #[derive(fmt::Debug)] pub(crate) enum Architecture { Turing = 0x16, Ampere = 0x17, Ada = 0x19, } impl TryFrom for Architecture { type Error = Error; fn try_from(value: u8) -> Result { match value { 0x16 => Ok(Self::Turing), 0x17 => Ok(Self::Ampere), 0x19 => Ok(Self::Ada), _ => Err(ENODEV), } } } pub(crate) struct Revision { major: u8, minor: u8, } impl Revision { fn from_boot0(boot0: regs::NV_PMC_BOOT_0) -> Self { Self { major: boot0.major_revision(), minor: boot0.minor_revision(), } } } impl fmt::Display for Revision { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:x}.{:x}", self.major, self.minor) } } /// Structure holding the metadata of the GPU. pub(crate) struct Spec { chipset: Chipset, /// The revision of the chipset. revision: Revision, } impl Spec { fn new(bar: &Bar0) -> Result { let boot0 = regs::NV_PMC_BOOT_0::read(bar); Ok(Self { chipset: boot0.chipset()?, revision: Revision::from_boot0(boot0), }) } } /// Structure holding the resources required to operate the GPU. #[pin_data(PinnedDrop)] pub(crate) struct Gpu { spec: Spec, /// MMIO mapping of PCI BAR 0 bar: Arc>, fw: Firmware, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). sysmem_flush: SysmemFlush, } #[pinned_drop] impl PinnedDrop for Gpu { fn drop(self: Pin<&mut Self>) { // Unregister the sysmem flush page before we release it. self.bar .try_access_with(|b| self.sysmem_flush.unregister(b)); } } impl Gpu { /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly /// created the WPR2 region. /// /// TODO: this needs to be moved into a larger type responsible for booting the whole GSP /// (`GspBooter`?). fn run_fwsec_frts( dev: &device::Device, falcon: &Falcon, bar: &Bar0, bios: &Vbios, fb_layout: &FbLayout, ) -> Result<()> { // Check that the WPR2 region does not already exists - if it does, we cannot run // FWSEC-FRTS until the GPU is reset. if regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound() != 0 { dev_err!( dev, "WPR2 region already exists - GPU needs to be reset to proceed\n" ); return Err(EBUSY); } let fwsec_frts = FwsecFirmware::new( dev, falcon, bar, bios, FwsecCommand::Frts { frts_addr: fb_layout.frts.start, frts_size: fb_layout.frts.end - fb_layout.frts.start, }, )?; // Run FWSEC-FRTS to create the WPR2 region. fwsec_frts.run(dev, falcon, bar)?; // SCRATCH_E contains the error code for FWSEC-FRTS. let frts_status = regs::NV_PBUS_SW_SCRATCH_0E::read(bar).frts_err_code(); if frts_status != 0 { dev_err!( dev, "FWSEC-FRTS returned with error code {:#x}", frts_status ); return Err(EIO); } // Check that the WPR2 region has been created as we requested. let (wpr2_lo, wpr2_hi) = ( regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO::read(bar).lower_bound(), regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound(), ); match (wpr2_lo, wpr2_hi) { (_, 0) => { dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); Err(EIO) } (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { dev_err!( dev, "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", wpr2_lo, fb_layout.frts.start, ); Err(EIO) } (wpr2_lo, wpr2_hi) => { dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); dev_dbg!(dev, "GPU instance built\n"); Ok(()) } } } pub(crate) fn new( pdev: &pci::Device, devres_bar: Arc>, ) -> Result> { let bar = devres_bar.access(pdev.as_ref())?; let spec = Spec::new(bar)?; let fw = Firmware::new(pdev.as_ref(), spec.chipset, FIRMWARE_VERSION)?; dev_info!( pdev.as_ref(), "NVIDIA (Chipset: {}, Architecture: {:?}, Revision: {})\n", spec.chipset, spec.chipset.arch(), spec.revision ); // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. gfw::wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?; let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?; let gsp_falcon = Falcon::::new( pdev.as_ref(), spec.chipset, bar, spec.chipset > Chipset::GA100, )?; gsp_falcon.clear_swgen0_intr(bar); let _sec2_falcon = Falcon::::new(pdev.as_ref(), spec.chipset, bar, true)?; let fb_layout = FbLayout::new(spec.chipset, bar)?; dev_dbg!(pdev.as_ref(), "{:#x?}\n", fb_layout); let bios = Vbios::new(pdev, bar)?; Self::run_fwsec_frts(pdev.as_ref(), &gsp_falcon, bar, &bios, &fb_layout)?; Ok(pin_init!(Self { spec, bar: devres_bar, fw, sysmem_flush, })) } }