PCIE学习系列 五（Linux之PCIe设备驱动开发框架）

概述

本文讲述一个开源的PCIe设备驱动，通过这个例子可以基本上理解所有的PCIe设备驱动。后续也会做关于Linux各类驱动的文章。

通过前面的学习，我们知道PCIe设备访问之前需要先做枚举。一般来说，PCI设备的枚举操作不需要我们来做，BIOS或者系统初始化时已经做好了，当系统枚举完所有设备之后，PCI设备就会添加进系统，在Linux下使用 “lspci” 就能看到系统扫描到的所有PCI设备，我们只需要关注PCI设备driver的实现就好了。

在Linux源码中随便找了一个开源代码，tsi721（一款PCIe转RapidIO芯片）的一些源码，基本上一个普通的PCIE设备驱动模型都是这样的，其中在加上一些设备独有的处理流程。

那么PCIe驱动的入口在哪呢？

当系统枚举到的PCI设备的vendor id和device id与driver中的id匹配上之后，就会调用driver中的probe函数。

staticconststructpci_device_id tsi721_pci_tbl[]={{PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721)},{0,}/* terminate list */};MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);staticstructpci_driver tsi721_driver ={.name       ="tsi721",.id_table   = tsi721_pci_tbl,.probe      = tsi721_probe,.remove     = tsi721_remove,.shutdown   = tsi721_shutdown,};module_pci_driver(tsi721_driver);

PCIe设备驱动不是通过name匹配，而是通过id去匹配的，当驱动与设备匹配上之后，就会执行tsi721_probe函数，一般的PCIe驱动基本上会用到以下步骤：

驱动加载流程

1. pci_enable_device(pdev); 使能pci设备
2. pci_resource_len(pdev, BAR_0);获取pci设备的资源大小，这是枚举时得到的值
3. pci_request_regions(pdev, DRV_NAME); 申请pci设备资源
4. pci_ioremap_bar(pdev, BAR_0); 映射虚拟内存
5. pci_set_master(pdev); 设置pci设备为master，master模式才能主动发起数据传输
6. pci_enable_msi(pdev); 使能MSI中断
7. request_irq(priv->pdev->irq, tsi721_irqhandler,(priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,DRV_NAME, (void *)priv); 注册中断处理函数初始化流程中关于PCIe的就基本上完成了，其余还有关于具体业务的初始化，下面可以结合源码看看。

staticinttsi721_probe(structpci_dev*pdev,conststructpci_device_id*id){structtsi721_device*priv;int err;

    priv =kzalloc(sizeof(structtsi721_device), GFP_KERNEL);if(!priv){
        err =-ENOMEM;goto err_exit;}

    err =pci_enable_device(pdev);// 使能pci设备if(err){tsi_err(&pdev->dev,"Failed to enable PCI device");goto err_clean;}

    priv->pdev = pdev;#ifdefDEBUG{int i;for(i =0; i < PCI_STD_NUM_BARS; i++){tsi_debug(INIT,&pdev->dev,"res%d %pR",
                  i,&pdev->resource[i]);}}#endif/*
     * Verify BAR configuration
     *//* BAR_0 (registers) must be 512KB+ in 32-bit address space */if(!(pci_resource_flags(pdev, BAR_0)& IORESOURCE_MEM)||pci_resource_flags(pdev, BAR_0)& IORESOURCE_MEM_64 ||pci_resource_len(pdev, BAR_0)< TSI721_REG_SPACE_SIZE){// 获取pci设备bar0的资源大小，这是枚举时得到的值tsi_err(&pdev->dev,"Missing or misconfigured CSR BAR0");
        err =-ENODEV;goto err_disable_pdev;}/* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */if(!(pci_resource_flags(pdev, BAR_1)& IORESOURCE_MEM)||pci_resource_flags(pdev, BAR_1)& IORESOURCE_MEM_64 ||pci_resource_len(pdev, BAR_1)< TSI721_DB_WIN_SIZE){tsi_err(&pdev->dev,"Missing or misconfigured Doorbell BAR1");
        err =-ENODEV;goto err_disable_pdev;}/*
     * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
     * space.
     * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
     * It may be a good idea to keep them disabled using HW configuration
     * to save PCI memory space.
     */

    priv->p2r_bar[0].size = priv->p2r_bar[1].size =0;if(pci_resource_flags(pdev, BAR_2)& IORESOURCE_MEM_64){if(pci_resource_flags(pdev, BAR_2)& IORESOURCE_PREFETCH)tsi_debug(INIT,&pdev->dev,"Prefetchable OBW BAR2 will not be used");else{
            priv->p2r_bar[0].base =pci_resource_start(pdev, BAR_2);// 获取bar2的起始地址
            priv->p2r_bar[0].size =pci_resource_len(pdev, BAR_2);// 获取bar2的起始长度}}if(pci_resource_flags(pdev, BAR_4)& IORESOURCE_MEM_64){if(pci_resource_flags(pdev, BAR_4)& IORESOURCE_PREFETCH)tsi_debug(INIT,&pdev->dev,"Prefetchable OBW BAR4 will not be used");else{
            priv->p2r_bar[1].base =pci_resource_start(pdev, BAR_4);// 获取bar4的起始地址
            priv->p2r_bar[1].size =pci_resource_len(pdev, BAR_4);// 获取bar4的起始长度}}

    err =pci_request_regions(pdev, DRV_NAME);// 申请pci设备资源if(err){tsi_err(&pdev->dev,"Unable to obtain PCI resources");goto err_disable_pdev;}pci_set_master(pdev);// 设置pci设备为master模式

    priv->regs =pci_ioremap_bar(pdev, BAR_0);// 映射虚拟内存if(!priv->regs){tsi_err(&pdev->dev,"Unable to map device registers space");
        err =-ENOMEM;goto err_free_res;}

    priv->odb_base =pci_ioremap_bar(pdev, BAR_1);// 映射虚拟内存if(!priv->odb_base){tsi_err(&pdev->dev,"Unable to map outbound doorbells space");
        err =-ENOMEM;goto err_unmap_bars;}/* Configure DMA attributes. */if(pci_set_dma_mask(pdev,DMA_BIT_MASK(64))){
        err =pci_set_dma_mask(pdev,DMA_BIT_MASK(32));// 设置dma掩码if(err){tsi_err(&pdev->dev,"Unable to set DMA mask");goto err_unmap_bars;}if(pci_set_consistent_dma_mask(pdev,DMA_BIT_MASK(32)))tsi_info(&pdev->dev,"Unable to set consistent DMA mask");}else{
        err =pci_set_consistent_dma_mask(pdev,DMA_BIT_MASK(64));if(err)tsi_info(&pdev->dev,"Unable to set consistent DMA mask");}BUG_ON(!pci_is_pcie(pdev));/* Clear "no snoop" and "relaxed ordering" bits. */pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
        PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN,0);/* Override PCIe Maximum Read Request Size setting if requested */if(pcie_mrrs >=0){if(pcie_mrrs <=5)pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
                    PCI_EXP_DEVCTL_READRQ, pcie_mrrs <<12);elsetsi_info(&pdev->dev,"Invalid MRRS override value %d", pcie_mrrs);}/* Set PCIe completion timeout to 1-10ms */pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2,
                       PCI_EXP_DEVCTL2_COMP_TIMEOUT,0x2);/*
     * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
     */pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL,0x01);pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
                        TSI721_MSIXTBL_OFFSET);pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
                        TSI721_MSIXPBA_OFFSET);pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL,0);/* End of FIXUP */tsi721_disable_ints(priv);tsi721_init_pc2sr_mapping(priv);tsi721_init_sr2pc_mapping(priv);if(tsi721_bdma_maint_init(priv)){tsi_err(&pdev->dev,"BDMA initialization failed");
        err =-ENOMEM;goto err_unmap_bars;}

    err =tsi721_doorbell_init(priv);if(err)goto err_free_bdma;tsi721_port_write_init(priv);

    err =tsi721_messages_init(priv);if(err)goto err_free_consistent;

    err =tsi721_setup_mport(priv);if(err)goto err_free_consistent;pci_set_drvdata(pdev, priv);tsi721_interrupts_init(priv);return0;

err_free_consistent:tsi721_port_write_free(priv);tsi721_doorbell_free(priv);
err_free_bdma:tsi721_bdma_maint_free(priv);
err_unmap_bars:if(priv->regs)iounmap(priv->regs);if(priv->odb_base)iounmap(priv->odb_base);
err_free_res:pci_release_regions(pdev);pci_clear_master(pdev);
err_disable_pdev:pci_disable_device(pdev);
err_clean:kfree(priv);
err_exit:return err;}

tsi721_probe调用tsi721_setup_mport(priv)，如下所示，和pcie相关的就是msi 中断的使能，中断函数的注册，其他就是芯片具体的业务相关的初始化。

staticinttsi721_setup_mport(structtsi721_device*priv){structpci_dev*pdev = priv->pdev;int err =0;structrio_mport*mport =&priv->mport;

    err =rio_mport_initialize(mport);if(err)return err;

    mport->ops =&tsi721_rio_ops;
    mport->index =0;
    mport->sys_size =0;/* small system */
    mport->priv =(void*)priv;
    mport->phys_efptr =0x100;
    mport->phys_rmap =1;
    mport->dev.parent =&pdev->dev;
    mport->dev.release = tsi721_mport_release;INIT_LIST_HEAD(&mport->dbells);rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE],0,0xffff);rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE],0,3);rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE],0,3);snprintf(mport->name, RIO_MAX_MPORT_NAME,"%s(%s)",dev_driver_string(&pdev->dev),dev_name(&pdev->dev));/* Hook up interrupt handler */#ifdefCONFIG_PCI_MSIif(!tsi721_enable_msix(priv))// 使能MSI-X中断
        priv->flags |= TSI721_USING_MSIX;elseif(!pci_enable_msi(pdev))// 使能MSI中断
        priv->flags |= TSI721_USING_MSI;elsetsi_debug(MPORT,&pdev->dev,"MSI/MSI-X is not available. Using legacy INTx.");#endif/* CONFIG_PCI_MSI */

    err =tsi721_request_irq(priv);// 中断注册，展开即为 request_irqif(err){tsi_err(&pdev->dev,"Unable to get PCI IRQ %02X (err=0x%x)",
            pdev->irq, err);return err;}#ifdefCONFIG_RAPIDIO_DMA_ENGINE
    err =tsi721_register_dma(priv);if(err)goto err_exit;#endif/* Enable SRIO link */iowrite32(ioread32(priv->regs + TSI721_DEVCTL)|
          TSI721_DEVCTL_SRBOOT_CMPL,
          priv->regs + TSI721_DEVCTL);if(mport->host_deviceid >=0)iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
              RIO_PORT_GEN_DISCOVERED,
              priv->regs +(0x100+ RIO_PORT_GEN_CTL_CSR));elseiowrite32(0, priv->regs +(0x100+ RIO_PORT_GEN_CTL_CSR));

    err =rio_register_mport(mport);if(err){tsi721_unregister_dma(priv);goto err_exit;}return0;

err_exit:tsi721_free_irq(priv);return err;}

驱动加载初始化的代码就结束了。

驱动卸载流程

当驱动卸载时会调用到tsi721_remove函数，其流程基本上就是probe的逆向操作。

1. free_irq(priv->pdev->irq, (void *)priv); 释放中断
2. iounmap(priv->regs); 地址解映射
3. pci_disable_msi(priv->pdev);失能msi中断
4. pci_release_regions(pdev);释放申请的资源
5. pci_clear_master(pdev); 清除设备master属性
6. pci_disable_device(pdev);失能pci设备

下面一起看看源码

staticvoidtsi721_remove(structpci_dev*pdev){structtsi721_device*priv =pci_get_drvdata(pdev);tsi_debug(EXIT,&pdev->dev,"enter");tsi721_disable_ints(priv);tsi721_free_irq(priv);// 释放中断flush_scheduled_work();rio_unregister_mport(&priv->mport);tsi721_unregister_dma(priv);tsi721_bdma_maint_free(priv);tsi721_doorbell_free(priv);tsi721_port_write_free(priv);tsi721_close_sr2pc_mapping(priv);if(priv->regs)iounmap(priv->regs);// 地址解映射if(priv->odb_base)iounmap(priv->odb_base);#ifdefCONFIG_PCI_MSIif(priv->flags & TSI721_USING_MSIX)pci_disable_msix(priv->pdev);// 失能msi-x中断elseif(priv->flags & TSI721_USING_MSI)pci_disable_msi(priv->pdev);// 失能msi中断#endifpci_release_regions(pdev);// 释放申请的资源pci_clear_master(pdev);// 清除设备master属性pci_disable_device(pdev);// 失能pci设备pci_set_drvdata(pdev,NULL);kfree(priv);tsi_debug(EXIT,&pdev->dev,"exit");}

至此关于PCIe设备驱动的卸载就完成了。

以上就是一个PCIe设备的基本驱动框架。