Tegra TX2一瞥2：IO子系統

05-03

TX2的啟動系統使用的是UEFI+dtb=>grub=>kernel的方案，文件系統使用標準的Ubuntu ARM64版本（可能經過定製，但至少apt source是標準的）。

（但我懷疑這裡做錯了，如果用UEFI方案，正確的方法應該是把單獨UEFI放在一個fat目錄下，這樣最初的載入程序只需要解釋fat目錄就可以了。現在把UEFI和它東西放在一起，就發揮不出UEFI和grub載入多種文件系統的效果了）

直接查dtb的compatible，有如下外設（我感興趣的都標記了對應的驅動，方括弧是未入主線的代碼）：

compatible = "a-edp,1080p-14-0"; [drivers/video/tegra/dc/panel/panel-a-edp-1080p-14-0.c]
compatible = "a-edp,1080p-14-0-bl";

compatible = "ak,ak89xx";(iio/imu/inv_mpu/inv_compass/inv_ak89xx_core.c)
compatible = "android,CustomIPI";(kernel/smp.c)
compatible = "android,firmware";(drivers/platform/tegra/firmwares-all.c)
compatible = "android,trusty-fiq-v1";
compatible = "android,trusty-irq-v1";
compatible = "android,trusty-log-v1";
compatible = "android,trusty-ote-v1";
compatible = "android,trusty-smc-v1";
compatible = "android,trusty-virtio-v1";
compatible = "arm,armv8-pmuv3";

compatible = "arm,armv8-timer";
compatible = "arm,coresight-etm3x", "arm,primecell";
compatible = "arm,coresight-etm4x", "arm,primecell";
compatible = "arm,coresight-funnel", "arm,primecell";
compatible = "arm,coresight-replicator";
compatible = "arm,coresight-stm", "arm,primecell";
compatible = "arm,coresight-tmc", "arm,primecell";
compatible = "arm,coresight-tpiu", "arm,primecell";
compatible = "arm,cortex-a15-gic";
compatible = "arm,cortex-a57-64bit", "arm,armv8";

compatible = "arm,mmu-500";
compatible = "arm,psci-1.0";
compatible = "bmp,bmpX80";（drivers/iio/pressure/nvi_bmpX80.c）
compatible = "bosch,mttcan";[drivers/staging/mttcan/m_ttcan_linux.c]
compatible = "bosch,mttcan-ivc";
compatible = "cache";
compatible = "capella,cm32180";（drivers/iio/light/nvs_cm3218.c）
compatible = "dp, display";
compatible = "dummy-cooling-dev";
compatible = "extcon-gpio-states";

compatible = "fixed-clock";
compatible = "gpio-keys";
compatible = "gps-wake";
compatible = "hdmi,display";
compatible = "invensense,mpu6xxx";（iio/imu/nvi_mpu/nvi.c）
compatible = "linux,spdif-dit";（sound/soc/codecs/spdif_transmitter.c）
compatible = "maxim,max16984-tegra186-cdp-phy";（drivers/phy/phy-max16984-cdp.c）
compatible = "maxim,max20024";（drivers/mfd/max77620.c）
compatible = "null,dsi-hotplug";
compatible = "nvidia, tegra-camera-platform";（drivers/video/tegra/camera/tegra_camera_platform.c）

compatible = "nvidia, tegra186-mipical";（drivers/media/platform/tegra/mipical/mipi_cal.c）
compatible = "nvidia,bwmgr";（drivers/platform/tegra/mc/emc_bwmgr.c）
compatible = "nvidia,carveouts";[drivers/video/tegra/nvmap/nvmap_init.c]
compatible = "nvidia,denver", "arm,armv8";[drivers/platform/tegra/tegra18_perf_uncore.c]
compatible = "nvidia,denver-hardwood";
compatible = "nvidia,denver15-pmu";
compatible = "nvidia,eqos";[net/ethernet/nvidia/eqos/init.c]
compatible = "nvidia,fiq-debugger";(drivers/staging/android/fiq_debugger/fiq_debugger.c)
compatible = "nvidia,generic_carveout";
compatible = "nvidia,imx185"; --i2c

compatible = "nvidia,imx219";
compatible = "nvidia,imx274";
compatible = "nvidia,mods-clocks";
compatible = "nvidia,ov23850";(drivers/media/i2c/ov23850.c)
compatible = "nvidia,ov5693";
compatible = "nvidia,pca9570";
compatible = "nvidia,ramoops";
compatible = "nvidia,smmu_test";
compatible = "nvidia,storm", "nvidia,tegra186";
compatible = "nvidia,t18x-cluster-clk-priv";

compatible = "nvidia,tegra-audio-t186ref-mobile-rt565x";
compatible = "nvidia,tegra-gic";
compatible = "nvidia,tegra-t18x-mc";
compatible = "nvidia,tegra-wdt-t18x";
compatible = "nvidia,tegra18-rtc";
compatible = "nvidia,tegra186";
compatible = "nvidia,tegra186-AXI2APB-bridge";[drivers/platform/tegra/bridge_mca.c]
compatible = "nvidia,tegra186-AXIP2P-bridge";
compatible = "nvidia,tegra186-adma";
compatible = "nvidia,tegra186-adsp-pd";(drivers/platform/tegra/pm_domains.c)

compatible = "nvidia,tegra186-ahc";[drivers/misc/tegra186-ahc/tegra186_ahc.c]
compatible = "nvidia,tegra186-ahci-sata";(drivers/ata/tegra/ahci_tegra.c)
compatible = "nvidia,tegra186-aon";[drivers/platform/tegra/tegra-aon.c]
compatible = "nvidia,tegra186-aon-ivc-echo";
compatible = "nvidia,tegra186-aon-spi";
compatible = "nvidia,tegra186-aondbg";
compatible = "nvidia,tegra186-aowake";
compatible = "nvidia,tegra186-ape-ivc";[drivers/platform/tegra/tegra-camera-rtcpu.c]
compatible = "nvidia,tegra186-ape-pd";
compatible = "nvidia,tegra186-arad";

compatible = "nvidia,tegra186-asrc";
compatible = "nvidia,tegra186-bpmp";[drivers/thermal/tegra_bpmp_thermal.c]
compatible = "nvidia,tegra186-bpmp-i2c";
compatible = "nvidia,tegra186-bpmp-thermal";
compatible = "nvidia,tegra186-cactmon";(drivers/platform/tegra/central_actmon/actmon_common.c)
compatible = "nvidia,tegra186-camera-ivc-protocol-capture";
compatible = "nvidia,tegra186-camera-ivc-protocol-capture-control";
compatible = "nvidia,tegra186-camera-ivc-protocol-dbg";
compatible = "nvidia,tegra186-camera-ivc-protocol-debug";
compatible = "nvidia,tegra186-camera-ivc-protocol-echo";
compatible = "nvidia,tegra186-camera-ivc-protocol-mods";
compatible = "nvidia,tegra186-camera-ivc-protocol-vinotify";
compatible = "nvidia,tegra186-cec";(drivers/misc/tegra-cec/tegra_cec.c)
compatible = "nvidia,tegra186-chipid";(drivers/platform/tegra/tegra_chipid.c)
compatible = "nvidia,tegra186-combined-uart";
compatible = "nvidia,tegra186-cpuidle-a57";
compatible = "nvidia,tegra186-cpuidle-a57-cluster";
compatible = "nvidia,tegra186-cpuidle-a57-thresholds";
compatible = "nvidia,tegra186-cpuidle-denver";
compatible = "nvidia,tegra186-cpuidle-denver-cluster";
compatible = "nvidia,tegra186-cpuidle-denver-thresholds";
compatible = "nvidia,tegra186-dc";[drivers/video/tegra/dc/dc.c]
compatible = "nvidia,tegra186-disa-pd";[drivers/video/tegra/dc/nvdisp/nvdisp.c]
compatible = "nvidia,tegra186-dpaux";
compatible = "nvidia,tegra186-dpaux-pinctrl";
compatible = "nvidia,tegra186-dpaux1";
compatible = "nvidia,tegra186-dpaux1-pinctrl";
compatible = "nvidia,tegra186-dsi";
compatible = "nvidia,tegra186-dspk";
compatible = "nvidia,tegra186-dspk";
compatible = "nvidia,tegra186-efuse", "nvidia,tegra210-efuse";
compatible = "nvidia,tegra186-efuse-burn";
compatible = "nvidia,tegra186-gp10b", "nvidia,gp10b";[drivers/gpu/nvgpu/nvgpu_gpuid_t18x.h]
compatible = "nvidia,tegra186-gpcdma";[drivers/dma/tegra186-gpc-dma.c]
compatible = "nvidia,tegra186-gpio";
compatible = "nvidia,tegra186-gpio-aon";
compatible = "nvidia,tegra186-host1x", "simple-bus";
compatible = "nvidia,tegra186-host1x-pd";
compatible = "nvidia,tegra186-hsp";
compatible = "nvidia,tegra186-hsp-mailbox";
compatible = "nvidia,tegra186-hsuart";
compatible = "nvidia,tegra186-i2c";
compatible = "nvidia,tegra186-iommu-context";
compatible = "nvidia,tegra186-isp";[drivers/video/tegra/host/isp/isp.c]
compatible = "nvidia,tegra186-ispa-pd";
compatible = "nvidia,tegra186-kfuse";
compatible = "nvidia,tegra186-mc-sid";
compatible = "nvidia,tegra186-mce";
compatible = "nvidia,tegra186-miscreg";
compatible = "nvidia,tegra186-msenc-pd";
compatible = "nvidia,tegra186-nvcsi";[drivers/video/tegra/host/nvcsi/nvcsi.c]
compatible = "nvidia,tegra186-nvdec";[drivers/video/tegra/host/nvdec/nvdec.c]
compatible = "nvidia,tegra186-nvdec-pd";
compatible = "nvidia,tegra186-nvdumper";
compatible = "nvidia,tegra186-nvenc"; --drm驅動，主線和外部皆有
compatible = "nvidia,tegra186-nvjpg"; --drm驅動，主線和外部皆有
compatible = "nvidia,tegra186-nvjpg-pd";
compatible = "nvidia,tegra186-pcie";
compatible = "nvidia,tegra186-pcie-pd";
compatible = "nvidia,tegra186-pinmux";
compatible = "nvidia,tegra186-pm-irq";
compatible = "nvidia,tegra186-pmc";
compatible = "nvidia,tegra186-pmc-iopower";
compatible = "nvidia,tegra186-pwm";
compatible = "nvidia,tegra186-qspi";
compatible = "nvidia,tegra186-roc-flush";
compatible = "nvidia,tegra186-safety-cmd-resp";
compatible = "nvidia,tegra186-safety-hb";
compatible = "nvidia,tegra186-safety-ivc";
compatible = "nvidia,tegra186-sata-pd";
compatible = "nvidia,tegra186-sce-ivc";
compatible = "nvidia,tegra186-sdhci";
compatible = "nvidia,tegra186-se-elp";（drivers/crypto/tegra-se-elp.c）
compatible = "nvidia,tegra186-se-pd";
compatible = "nvidia,tegra186-se1-nvhost"; --這個是未主線化的
compatible = "nvidia,tegra186-se2-nvhost";
compatible = "nvidia,tegra186-se3-nvhost";
compatible = "nvidia,tegra186-se4-nvhost";
compatible = "nvidia,tegra186-sor";
compatible = "nvidia,tegra186-sor1";
compatible = "nvidia,tegra186-spi";
compatible = "nvidia,tegra186-spi";
compatible = "nvidia,tegra186-system-config";
compatible = "nvidia,tegra186-tachometer";
compatible = "nvidia,tegra186-timer";
compatible = "nvidia,tegra186-tsec";
compatible = "nvidia,tegra186-tsec-pd";
compatible = "nvidia,tegra186-usb-cd";
compatible = "nvidia,tegra186-ve-pd";
compatible = "nvidia,tegra186-vi"; [drivers/video/tegra/host/vi/vi4.c]
compatible = "nvidia,tegra186-vi-bypass";
compatible = "nvidia,tegra186-vic";
compatible = "nvidia,tegra186-vic03-pd";
compatible = "nvidia,tegra186-xhci";
compatible = "nvidia,tegra186-xotg";
compatible = "nvidia,tegra186-xudc";
compatible = "nvidia,tegra186-xusb-mbox";
compatible = "nvidia,tegra186-xusb-padctl";
compatible = "nvidia,tegra186-xusba-pd";
compatible = "nvidia,tegra186-xusbb-pd";
compatible = "nvidia,tegra186-xusbc-pd";
compatible = "nvidia,tegra18x-adsp";
compatible = "nvidia,tegra18x-adsp-carveout";
compatible = "nvidia,tegra18x-agic";
compatible = "nvidia,tegra18x-balanced-throttle";
compatible = "nvidia,tegra18x-car";
compatible = "nvidia,tegra18x-cpufreq";
compatible = "nvidia,tegra18x-cpuidle";
compatible = "nvidia,tegra18x-eqos-ape";
compatible = "nvidia,tegra20-uart", "nvidia,tegra186-hsuart";
compatible = "nvidia,tegra210-admaif";
compatible = "nvidia,tegra210-adsp-audio";
compatible = "nvidia,tegra210-adx";
compatible = "nvidia,tegra210-afc";
ompatible = "nvidia,tegra210-amixer";
compatible = "nvidia,tegra210-amx";(sound/soc/tegra-alt/tegra210_amx_alt.c)
compatible = "nvidia,tegra210-axbar";
compatible = "nvidia,tegra210-dmic";
compatible = "nvidia,tegra210-i2s";
compatible = "nvidia,tegra210-iqc";
compatible = "nvidia,tegra210-mvc";
compatible = "nvidia,tegra210-mvc";
compatible = "nvidia,tegra210-ope";
compatible = "nvidia,tegra210-pmc-blink-pwm";
compatible = "nvidia,tegra210-sfc";
compatible = "nvidia,tegra210-spdif";
compatible = "nvidia,tegra210-spkprot";
compatible = "nvidia,tegra30-hda";
compatible = "nvidia,vpr-carveout";
compatible = "nxp,pca9546";
compatible = "pwm-fan";
compatible = "realtek,rt5658";(sound/soc/codecs/rt5659.c)
compatible = "regulator-fixed";
compatible = "regulator-fixed-sync";
compatible = "s,wqxga-10-1";
compatible = "s,wqxga-10-1-bl";
compatible = "s,wuxga-8-0";
compatible = "s,wuxga-8-0-bl";
compatible = "s-edp,uhdtv-15-6";
compatible = "s-edp,uhdtv-15-6-bl";
compatible = "sharp,lr388k7_ts";(drivers/input/touchscreen/lr388k7_ts.c)
compatible = "simple-bus";
compatible = "softdog-platform";
compatible = "synopsys,dwc_eqos_virt_test";
compatible = "tegra,ufs_variant";
compatible = "tegra-power-domains";
compatible = "thermal-fan-est";
compatible = "ti,ina3221x";--大部分是iio驅動
compatible = "ti,lp8556";
compatible = "ti,tas2552";
compatible = "ti,tca6408";
compatible = "ti,tca6416";
compatible = "ti,tca9539";
compatible = "ti,tmp451";
compatible = "ti,tps65132";

顯示部分傳到主線的大部分都是drm和fb的，沒有上傳的統一放在一個叫flcn的框架中，裡面的東西包羅萬有。

用來支持cuda運算的驅動都沒有主線化，也不在flcn中，而是放在driver/gpu目錄下，提供的主要是用戶態的介面（通過一組字元設備），和內核的關係不大。

從大量的電源域和調頻的設計看，這是從手機改過來的片子。當然，Tegra系列本來就是給手機做的，這也沒有什麼可說的。

gk20a的驅動是2014年開始上傳主線的，當時的內核版本是3.16，但一直僅包含drm的驅動。從現在這個代碼的樣子，應該是一個內部的ML的產品線集成了計算核心到手機SoC中，然後把對應的代碼拼上來了。

查iommu_group，可以看到大部分外設都用iommu_group分割：

sdhci x2
i2c x7
spi x3
serial x4
ether_qos
rtcpu
ahci-sata
aon
smmu_test
xhci
xudc
host1x x10
host1x:ctx0
nvcsi
vi
isp
nvdisplay
vic
nvenc
nvdec
i2c
nvjpg
tsec
tsecb
se x4
gp10b
bpmp
dma
pcie-controller
sound
hda
adsp_audio
adsp

片內帶一個gpu設備（/sys/device/gpu.0），支持虛擬化，但僅包含一個iommu_group（所以才支持了vfio-mdev？），驅動是/sys/bus/platform/drivers/gk20a。

gk20a的構架是一個統一的PCI化的設備框架nvgpu，不同的硬體實現則通過一個稱為platform的數據結構進行封裝（放在device的priv中），從gk20a的實現看，主要是是些初始化，時鐘一類的東西，也就是說，標準的數據流是統一的設計。我猜這樣的構架撐不了三代（當然，Tegra也不需要），這可能是沒有主線化的主要原因。

用戶態介面主要暴露為如下字元設備：

/dev/nvhost-gpu：共享內存和通道管理

/dev/nvhost-as-gpu：Address Space

/dev/nvhost-ctrl-gpu：設備狀態控制

/dev/nvhost-dbg-gpu：設備調試信息控制

/dev/nvhost-prof-gpu: GPU profiling事件導出和控制

/dev/nvhost-tsg-gpu: 不知道是啥，看起來是做任務控制的

/dev/nvhost-sched-gpu：看起來是tsg的輔助功能，可能前者是任務組管理，後者是調度

其他媒體codec也有類似的結構：

/dev/nvhost-as-gpu /dev/nvhost-ctrl-isp /dev/nvhost-ctrl-vi /dev/nvhost-gpu /dev/nvhost-nvcsi /dev/nvhost-prof-gpu /dev/nvhost-tsecb /dev/nvhost-vic
/dev/nvhost-ctrl /dev/nvhost-ctrl-nvcsi /dev/nvhost-ctxsw-gpu /dev/nvhost-isp /dev/nvhost-nvdec /dev/nvhost-sched-gpu /dev/nvhost-tsg-gpu
/dev/nvhost-ctrl-gpu /dev/nvhost-ctrl-nvdec /dev/nvhost-dbg-gpu /dev/nvhost-msenc /dev/nvhost-nvjpg /dev/nvhost-tsec /dev/nvhost-vi

如果直接看一個cuda的調用過程，主要包括如下動作：

openat(AT_FDCWD, "/dev/nvhost-ctrl-gpu", O_RDWR|O_CLOEXEC) = 4ioctl(4, NVGPU_GPU_IOCTL_GET_CHARACTERISTICSioctl(4, NVGPU_GPU_IOCTL_GET_TPC_MASKSioctl(4, NVGPU_GPU_IOCTL_GET_FBP_L2_MASKSioctl(4, NVGPU_GPU_IOCTL_ZCULL_GET_CTX_SIZEioctl(4, NVGPU_GPU_IOCTL_ZCULL_GET_INFOioctl(4, NVGPU_GPU_IOCTL_GET_ENGINE_INFOioctl(4, NVGPU_GPU_IOCTL_ALLOC_AS //5是這裡分配的句柄ioctl(5, NVGPU_AS_IOCTL_GET_VA_REGIONSioctl(5, NVGPU_AS_IOCTL_GET_VA_REGIONSioctl(5, NVGPU_AS_IOCTL_ALLOC_SPACEioctl(5, NVGPU_AS_IOCTL_ALLOC_SPACEopenat(AT_FDCWD, "/dev/nvmap", O_RDWR|O_SYNC|O_CLOEXEC) = 9ioctl(9, NVMAP_IOC_CREATE //和nvmap有關的我們先放下一篇分析里ioctl(9, NVMAP_IOC_ALLOCioctl(5, NVGPU_AS_IOCTL_UNMAP_BUFFERioctl(9, NVMAP_IOC_CREATEioctl(9, NVMAP_IOC_ALLOC...上兩個調用的多次重複ioctl(4, NVGPU_GPU_IOCTL_OPEN_TSG ioctl(4, NVGPU_GPU_IOCTL_OPEN_CHANNEL //這裡也是創建句柄，從設備分配一個通道ioctl(5, NVGPU_AS_IOCTL_BIND_CHANNEL //把channel和AS關聯起來ioctl(4, NVGPU_GPU_IOCTL_OPEN_CHANNEL ioctl(5, NVGPU_AS_IOCTL_BIND_CHANNEL ioctl(4, NVGPU_GPU_IOCTL_OPEN_CHANNEL...

簡單從這個語義上下文理解，一次通訊是這樣的：先從進程和GPU建立一個關聯，驅動GPU的能力，然後分配一個或者多個獨立的地址空間（AS），可能是把MMU的一部分複製給SMMU，然後在設備和進程間建立一個通道，然後把AS和通道綁定，估計這個時候設定streamid和substreamid給設備，這樣剩下的缺頁行為就可以交給設備，設備發起缺頁中斷的時候，靠進程這邊接管對應的mm.handle_fault()，完成後面的過程。

這個設計優先保性能，上傳是下一步的事情（也不一定會上傳）。

其他沒有什麼感興趣了的了，就這樣吧。