H88K系统iStoreOS的编译

发表于 | 分类于

配置编译环境

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# Debian 11下为H88K编译istoreos
sudo apt install build-essential asciidoc binutils bzip2 gawk \
gettext git libncurses5-dev libz-dev patch unzip zlib1g-dev curl \
lib32gcc-s1 libc6-dev-i386 subversion flex uglifyjs libssl-dev wget \
gcc-multilib p7zip p7zip-full msmtp texinfo libglib2.0-dev xmlto rsync \
qemu-utils libelf-dev autoconf automake libtool autopoint device-tree-compiler python-is-python3 

# WSL1编译的话需要如下剔除Windows的PATH内容
sudo vi /etc/wsl.conf
[interop]
appendWindowsPath = false

# WSL1设置代理来加速源码下载，没有代理可跳过或根据需求设置。如下是WSL1设置网络代理的例子
export all_proxy="http://172.23.240.1:7890"


# WSL2上由于网络隔离了，所以需要先宿主机防火墙放行再设置虚拟机的代理
# PowerShell中运行来放开对wsl2的局域网访问权限, 设置 -> 防火墙 -> 高级设置 -> 入站规则中可看到
New-NetFirewallRule -DisplayName "WSL" -Direction Inbound -InterfaceAlias "vEthernet (WSL)" -Action Allow

# 设置虚拟机的代理
proxy_port=7890
proxy_host=$(ip route | awk '/default/ {print $3}')
export https_proxy="http://${proxy_host}:${proxy_port}"
export http_proxy="http://${proxy_host}:${proxy_port}"
export all_proxy="socks5://${proxy_host}:${proxy_port}"

获取官方源码

1
2
3
4
5
6
7
8
9
# 获取iStore官方源码
git clone --depth=1 -b istoreos-22.03 https://github.com/istoreos/istoreos.git
cd istoreos
wget https://fw0.koolcenter.com/iStoreOS/h88k/feeds.conf

# 以后每次编译前建议执行以下三行命令更新源码
git pull
./scripts/feeds update -a
./scripts/feeds install -a

开始编译源码

1
2
3
4
5
6
7
8
9
10
11
12
# 下载iStore官方的H88K编译配置文件
wget -O .config https://fw0.koolcenter.com/iStoreOS/h88k/config.buildinfo

# 开始编译
make menuconfig		# 进入固件配置界面
make download V=s -j8	# 下载所需源码，请尽量使用梯子
make V=s -j$(nproc)	# 首次编译推荐用单线程

# 清理相关
make clean		# 在源码有大规模更新或者内核更新后执行，以保证编译质量，此操作会删除/bin和/build_dir目录中的文件
make dirclean		# 更换架构编译前必须执行，此操作相当于 make clean 以及删除/staging_dir、/toolchain、/tmp和/logs中的文件
make distclean		# 除非打算 push 到远程仓库 否则几乎用不到，此操作相当于 make dirclean 外加删除/dl、/feeds目录和.config文件

解决编译问题

  1. 由于官方太臃肿，你可以精简不需要的luci-app，以下是部分可选配置
1
2
3
4
5
6
7
8
9
10
> LuCI > Collections  > luci				# 必选，不添加这个固件将只有 <在线升级> 一个选项
> LuCI > Modules      > luci-base和luci-compat		# 解决插件兼容性问题
> LuCI > Protocols    > luci-proto-modemmanager		# 5G模块上网
> LuCI > Applications > luci-app-argon-config		# Argon 主题设置插件
> LuCI > Applications > luci-app-ddnsto			# 内网穿透神器
> LuCI > Applications > luci-app-linkease		# 易有云
> LuCI > Applications > luci-app-quickstart		# 网络向导
> LuCI > Applications > luci-app-store			# iStoreOS的应用商店

> Base system > Miscellaneous Utilities > microcom	# 添加串口工具
  1. 在Debian 12中因没有python2报错可以试试：sudo ln -s /usr/bin/python3 /usr/bin/python2
  1. 在 make menuconfig 中存在luci-app-store时需要添加额外的源到源码根目录：wget https://fw0.koolcenter.com/iStoreOS/h88k/feeds.conf

  1. cannot find dependency attr for base-files：首次编译需要进入 make menuconfig 后无需修改保存退出即可，否则可能会报该错

  2. 在官方配置文件中需要将 kmod-backlight-pwm 编进内核，否则会报找不到kmod-backlight的相关错误

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# 如下编译通过，不选backlight仅去掉下面vvop的话会报找不到kmod-rknpu的相关错误
> Kernel modules > Video Support > <*> kmod-backlight-pwm
> Kernel modules > Video Support > <*> kmod-drm-rockchip-rk35xx-vvop

# iStore已经为rk35xx添加了开源的panfrost驱动，但panfrost依赖的kmod-backlight不存在会报错
> Kernel modules > Video Support > kmod-backlight-pwm	# 添加这个可以解决依赖问题
> Kernel modules > Video Support > kmod-drm-panfrost

# iStore已经为rk35xx添加了闭源的mali驱动，跑Android In Docker的可以用到
# 闭源部分 /lib/firmware/mali_csffw.bin 在 https://github.com/JeffyCN/mirrors/commits/libmali/firmware/g610 可获取
> Kernel modules > Other modules > kmod-rkgpu-bifrost

# 开源驱动 panfrost 支持openGL     不支持openCL 只能使用wayland显示服务器
# 闭源驱动 libmali  支持openGL ES  支持openCL   只能使用 xorg  显示服务器
  1. golang编译时出现proxy问题
1
2
3
4
5
go: github.com/fsnotify/fsnotify@v1.4.9: Get "https://proxy.golang.org/github.com/fsnotify/fsnotify/@v/v1.4.9.mod": 
dial tcp 172.217.160.113:443: connect: connection refused

export GO111MODULE=on
export GOPROXY=https://goproxy.cn
  1. 如果需要编译所有内核模块，请如下全部清理一次
1
2
3
4
5
6
cd openwrt
rm -rf tmp/ .config*
make dirclean

# 再重新选择
【Global build settings】--->【Select all target specific packages by default】--->【Select all kernel module packages by default】
  1. glib编译时无法下载pcre问题
1
2
3
4
5
6
7
8
9
10
# 查看istoreos/build_dir/hostpkg/glib-2.74.0/subprojects/pcre2.wrap中pcre的相关信息如下：
source_url = https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.40/pcre2-10.40.tar.bz2
source_filename = pcre2-10.40.tar.bz2
source_hash = 14e4b83c4783933dc17e964318e6324f7cae1bc75d8f3c79bc6969f00c159d68
patch_filename = pcre2_10.40-3_patch.zip
patch_url = https://wrapdb.mesonbuild.com/v2/pcre2_10.40-3/get_patch

# 手动下载source_url和patch_url后拷贝到subprojects/packagecache/下即可
cp pcre2-10.40.tar.bz2 istoreos/build_dir/hostpkg/glib-2.74.0/subprojects/packagecache/
cp pcre2_10.40-3_patch.zip istoreos/build_dir/hostpkg/glib-2.74.0/subprojects/packagecache/

额外配置内核

1
2
3
4
5
6
7
8
9
10
11
12
13
14
make kernel_menuconfig	# 的修改是保存在 target/linux/rockchip/rk35xx/config-5.10 中

内核的配置文件分为两部分，一部分是通用的内核配置项，一部分是板子相关的配置项，
通用linux选项是放在target/linux/generic目录下的，板子相关的配置项是由顶层目录中的.config中的CONFIG_TARGET_BOARD来决定的。
最终内核使用的配置文件是这两个文件合并出来的文件
具体合并代码是在$(TOPDIR)/scripts/kconfig.pl中完成的

$(TOPDIR)/target/linux/generic/config-5.10
$(TOPDIR)/target/linux/rockchip/rk35xx/config-5.10
$(TOPDIR)/build_dir/target-aarch64_cortex-a53_musl/linux-rockchip_rk35xx/linux-5.10.176/.config 也是最终编译内核使用的配置文件。

make target/linux/clean V=s
make target/linux/compile V=s
make target/linux/install V=s

添加KVM支持

  1. 将如下添加到 package/kernel/linux/modules/virt.mk 的后面，再在菜单中选择它即可 > Kernel modules > Virtualization > <*> kmod-kvm-rockchip...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# 添加RK3588的KVM支持
define KernelPackage/kvm-rockchip
  SUBMENU:=Virtualization
  TITLE:=Kernel-based Virtual Machine (KVM) support
  DEPENDS:=@(TARGET_rockchip||TARGET_sunxi)
  KCONFIG:=\
	  CONFIG_TASKSTATS=y \
	  CONFIG_ARM64_VHE=y \
	  CONFIG_TASK_DELAY_ACCT=y \
	  CONFIG_ARM64_ERRATUM_834220=y \
	  CONFIG_IRQ_BYPASS_MANAGER=y \
	  CONFIG_VIRTUALIZATION=y \
	  CONFIG_KVM=y \
	  CONFIG_HAVE_KVM_IRQCHIP=y \
	  CONFIG_HAVE_KVM_IRQFD=y \
	  CONFIG_HAVE_KVM_IRQ_ROUTING=y \
	  CONFIG_HAVE_KVM_EVENTFD=y \
	  CONFIG_KVM_MMIO=y \
	  CONFIG_HAVE_KVM_MSI=y \
	  CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT=y \
	  CONFIG_KVM_VFIO=y \
	  CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL=y \
	  CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT=y \
	  CONFIG_HAVE_KVM_IRQ_BYPASS=y \
	  CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE=y \
	  CONFIG_KVM_ARM_PMU=y \
	  CONFIG_PREEMPT_NOTIFIERS=y \
	  CONFIG_MMU_NOTIFIER=y \
	  CONFIG_VIRTIO=y \
	  CONFIG_VIRTIO_PCI=y \
	  CONFIG_VIRTIO_PCI_LEGACY=y \
	  CONFIG_BLK_MQ_VIRTIO=y \
	  CONFIG_VIRTIO_NET=y \
	  CONFIG_FAILOVER=y \
	  CONFIG_NET_FAILOVER=y \
	  CONFIG_DRM_VIRTIO_GPU=y \
	  CONFIG_DRM_GEM_SHMEM_HELPER=y \
	  CONFIG_VIRTIO_DMA_SHARED_BUFFER=y \
	  CONFIG_VIRTIO_BALLOON=y \
	  CONFIG_MEMORY_BALLOON=y \
	  CONFIG_BALLOON_COMPACTION=y \
	  CONFIG_PAGE_REPORTING=y \
	  CONFIG_VIRTIO_CONSOLE=y \
	  CONFIG_HVC_DRIVER=y \
	  CONFIG_VIRTIO_IOMMU=y \
	  CONFIG_VIRTIO_BLK=y \
	  CONFIG_SCSI_VIRTIO=y \
	  CONFIG_VIRTIO_INPUT=y \
	  CONFIG_HW_RANDOM_VIRTIO=y \
	  CONFIG_CRYPTO_DEV_VIRTIO=y
	  CONFIG_VIRTIO_MMIO=y \
	  CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
endef

define KernelPackage/kvm-rockchip/description
  Support hosting fully virtualized guest machines using hardware
  virtualization extensions.  You will need a fairly recent
  processor equipped with virtualization extensions. You will also
  need to select one or more of the processor modules.

  This module provides access to the hardware capabilities through
  a character device node named /dev/kvm.
endef

$(eval $(call KernelPackage,kvm-rockchip))
  1. 查看KVM是否添加成功
1
2
3
4
5
6
7
8
9
10
11
# 有如下输出即代表添加成功了
root@iStoreOS:~# dmesg | grep kvm
[    0.151444] kvm [1]: IPA Size Limit: 40 bits
[    0.151456] kvm [1]: GICv3: no GICV resource entry
[    0.151459] kvm [1]: disabling GICv2 emulation
[    0.151473] kvm [1]: GIC system register CPU interface enabled
[    0.151578] kvm [1]: vgic interrupt IRQ9
[    0.151693] kvm [1]: VHE mode initialized successfully

# 也可执行 ls /sys/module/ 查看是否有如下模块显示
kvm virtio_blk virtio_gpu virtio_net virtio_pci

添加QEMU包

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
# 默认OPENWRT是不为x86_64之外架构提供QEMU的，所以需要添加TARGET_rockchip支持
# 修改 feeds/packages/utils/qemu/Makefile 中第31和32行如下内容为这样：
QEMU_DEPS_IN_GUEST := @(TARGET_x86_64||TARGET_armvirt||TARGET_malta||TARGET_rockchip)
QEMU_DEPS_IN_HOST := @(TARGET_x86_64||TARGET_sunxi||TARGET_rockchip)

# 在222行如下内容中添加：aarch64-softmmu  即可添加新包 qemu-aarch64-softmmu
# > Utilities > Virtualization > < > qemu-aarch64-softmmu...... QEMU target aarch64-softmmu
qemu-target-list :=	\
	x86_64-softmmu	\
	arm-softmmu	\
	aarch64-softmmu	\

# 然后就可以编译QEMU了
make package/feeds/packages/qemu/compile V=s


# 安装的先后顺序，后面某些依赖前面
opkg install 
qemu-keymaps_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-firmware-efi_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-arm-softmmu_6.2.0-2_aarch64_cortex-a53.ipk \

qemu-firmware-pxe_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-firmware-seabios_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-firmware-seavgabios_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-x86_64-softmmu_6.2.0-2_aarch64_cortex-a53.ipk \

qemu-bridge-helper_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-img_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-nbd_6.2.0-2_aarch64_cortex-a53.ipk \

qemu-aarch64-softmmu_6.2.0-2_aarch64_cortex-a53.ipk \
virtio-console-helper_6.2.0-2_aarch64_cortex-a53.ipk \
qemu-ga_6.2.0-2_aarch64_cortex-a53.ipk

# 现在可运行Debian安装镜像测试下了，VNC连接iStore的IP:5900即可
qemu-system-aarch64 \
-M virt,gic-version=max -smp 4 -m 2G -cpu host -enable-kvm \
-bios /usr/share/qemu/QEMU_EFI.fd -device ramfb -vnc :0 \
-device qemu-xhci,id=xhci -device usb-kbd -device usb-tablet -k en-us \
-device usb-storage,drive=install \
-drive if=none,id=install,media=cdrom,file=/mnt/nvme0n1-1/debian-12.2.0-arm64-netinst.iso

安装WOA教程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# 相关资源下载链接，解包 qemu-efi-aarch64 可找到 /usr/share/qemu-efi-aarch64/QEMU_EFI.fd
# https://packages.debian.org/stable/qemu-efi-aarch64
# https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/latest-virtio/virtio-win.iso
# ed2k://|file|SW_DVD9_Win_Pro_10_22H2.3_64ARM_ChnSimp_Pro_Ent_EDU_N_MLF_X23-36949.ISO|5333610496|401601D6C86121111B85019BAB8CD79D|/

qemu-img create -f qcow2 system.qcow2 30G	# 创建30G的虚拟硬盘
qemu-img resize system.qcow2 +10G		# 调整虚拟磁盘大小，扩容+10G，缩减-10G

# 安装Windows的命令，安装后去掉后面的install，装完驱动后去掉drivers
# 因为WoA不支持GICv3，所以在支持GICv3的设备上 its=off 必需的：-machine virt,gic-version=3,its=off
# 将虚拟机22端口TCP报文转到宿主机的8022端口：hostfwd=tcp::8022-:22		
qemu-system-aarch64 \
-M virt,gic-version=max,its=off -smp 4 -m 8G -cpu host -enable-kvm \
-bios /usr/share/qemu/QEMU_EFI.fd -device ramfb -vnc :0 \
-device qemu-xhci,id=xhci -device usb-kbd -device usb-tablet -k en-us \
-device virtio-net,disable-legacy=on,netdev=net0 \
-netdev user,id=net0,hostfwd=tcp::3389-:3389 \
-device virtio-blk,drive=system \
-drive if=none,id=system,cache=unsafe,file=/mnt/nvme0n1-1/qemu/system.qcow2 \
-device usb-storage,drive=drivers \
-drive if=none,id=drivers,media=cdrom,file=/mnt/nvme0n1-1/qemu/virtio-win-0.1.240.iso \
-device usb-storage,drive=install \
-drive if=none,id=install,format=raw,media=cdrom,file=/mnt/nvme0n1-1/qemu/SW_DVD9_Win_Pro_10_22H2.3_64ARM_ChnSimp_Pro_Ent_EDU_N_MLF_X23-36949.ISO 

# Virtio-win的相关驱动
viostor/	# Virtio 块驱动
vioscsi/	# Virtio SCSI驱动
viorng/		# Virtio RNG驱动
vioser/		# Virtio 串口驱动
NetKVM/		# Virtio 网络驱动
Balloon/	# Virtio 内存气球驱动
pvpanic/	# QEMU pvpanic 设备驱动
qxldod/		# Win8及之后版本的QXL显卡驱动
qxl/		# Win7及之前版本的QXL显卡驱动
guest-agent/	# QEMU Guest Agent 32bit 和 64bit 安装包
qemupciserial/	# QEMU PCI 串口设备驱动