From 1c0d4977de14af877366c65e1706b013c7a69f2a Mon Sep 17 00:00:00 2001 From: andreili Date: Sat, 5 Jul 2025 21:48:53 +0200 Subject: [PATCH] Add basic image creation process. --- build.py | 2 +- config/board/btt_cb1.json | 7 +- config/board/btt_pi2.json | 13 ++- config/board/opi_zero2.json | 17 ++- scripts/board.py | 68 +----------- scripts/os.py | 199 +++++++++++++++++++++++++++++++++++- scripts/target.py | 12 +++ 7 files changed, 238 insertions(+), 80 deletions(-) diff --git a/build.py b/build.py index f1ad92c..58b1b30 100755 --- a/build.py +++ b/build.py @@ -40,4 +40,4 @@ if (args.os_act != ""): os.action(act) if (args.install != ""): - target_board.install(args.install) + os.install(args.install) diff --git a/config/board/btt_cb1.json b/config/board/btt_cb1.json index 461da4a..c3c3e1e 100644 --- a/config/board/btt_cb1.json +++ b/config/board/btt_cb1.json @@ -40,6 +40,7 @@ { "file": "u-boot-sunxi-with-spl.bin", "store_type": "dd", + "block_size": "1k", "img_offset": 8 } ], @@ -91,12 +92,14 @@ "install": { "target": "image", - "block_size": "1k", + "type": "mbr", + "block_size": "512b", "partitions": [ { "name": "boot", - "size": "2g" + "size": "1g", + "first_sector": "2048" }, { "name": "rw", diff --git a/config/board/btt_pi2.json b/config/board/btt_pi2.json index 02b3212..a43b7b4 100644 --- a/config/board/btt_pi2.json +++ b/config/board/btt_pi2.json @@ -8,7 +8,8 @@ "CROSS_C:aarch64-linux-gnu-", "TPL_BIN:rk3566_ddr_1056MHz_v1.23.bin", "BL31_BIN:rk3568_bl31_v1.44.elf", - "ARCH:aarch64" + "ARCH:aarch64", + "DTB_FILE:rockchip/rk3566-bigtreetech-pi2.dtb" ], "targets": [ @@ -40,12 +41,14 @@ { "file": "idbloader.img", "store_type": "dd", + "block_size": "512b", "img_offset": 64 }, { "file": "u-boot.itb", "store_type": "dd", - "img_offset": 16368 + "block_size": "512b", + "img_offset": 16384 } ], "target": [ "" ], @@ -65,7 +68,7 @@ "store_type": "boot" }, { - "file": "arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-*.dtb", + "file": "arch/arm64/boot/dts/%{DTB_FILE}%", "store_type": "boot", "subdir": "dtb/rockchip" } @@ -76,12 +79,14 @@ "install": { "target": "image", + "type": "gpt", "block_size": "512b", "partitions": [ { "name": "boot", - "size": "2g" + "size": "1g", + "first_sector": "32768" }, { "name": "rw", diff --git a/config/board/opi_zero2.json b/config/board/opi_zero2.json index b7a1426..54e64ba 100644 --- a/config/board/opi_zero2.json +++ b/config/board/opi_zero2.json @@ -40,6 +40,7 @@ { "file": "u-boot-sunxi-with-spl.bin", "store_type": "dd", + "block_size": "1k", "img_offset": 8 } ], @@ -77,7 +78,19 @@ "install": { "target": "image", - "block_size": "1k", - "image_size": "2g" + "block_size": "512b", + "type": "mbr", + "partitions": + [ + { + "name": "boot", + "size": "1g", + "first_sector": "2048" + }, + { + "name": "rw", + "size": "2g" + } + ] } } diff --git a/scripts/board.py b/scripts/board.py index 04b502d..f11a938 100644 --- a/scripts/board.py +++ b/scripts/board.py @@ -1,9 +1,7 @@ -import json, os, stat, re +import json, os from pathlib import Path from . import * -units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 } - class Board: def __init__(self, name, js_fn, targets_meta): self.name = name @@ -88,67 +86,3 @@ class Board: break if (not is_finded): Logger.error("Don't find target!") - - def __do_cmd(self, args, cwd=None, env=None, stdin=None): - if (stdin != None): - p = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, text=True) - else: - p = subprocess.Popen(args, cwd=cwd, env=env) - if (stdin != None): - p.communicate(input=stdin) - if (p.wait() != 0): - Logger.error(f"Command '{args[0]}' finished with error code!") - - def __make_blk_struct(self, dev): - Logger.install("\tBlock device. Prepare and mount it...") - - def __create_img_file(self, path, size): - Logger.install("\tCreate image file...") - img_f = Path(path) - if (img_f.is_file()): - shutil.rmtree(path, ignore_errors=True) - blk_size = 1024*1024 - blk_count = int(size / blk_size) - self.__do_cmd(["dd", "if=/dev/zero", f"of={path}", f"bs={blk_size}", f"count={blk_count}"]) - - def __parse_size(elf, size): - size = size.upper() - if not re.match(r' ', size): - size = re.sub(r'([KMGT])', r' \1', size) - number, unit = [string.strip() for string in size.split()] - return int(float(number)*units[unit]) - - def __create_parts(self, img_or_dev): - args = "" - args += "o\n" - for part in self.installs["partitions"]: - part_sz = part["size"] - args += "n\n" - args += "p\n" - args += "\n" - args += "\n" - args += f"+{part_sz}\n" - args += "w\n" - args += "q\n" - self.__do_cmd(["fdisk", img_or_dev], stdin=args) - - def __install_to_img(self): - Logger.install("\tImage. Prepare and mount it...") - img_fn = f"{self.out_dir}/all.img" - # basic image offset - space for partition table and bootloader - img_sz = 1024*1024 + 512 - parts = self.installs["partitions"] - for part in parts: - img_sz += self.__parse_size(part["size"]) - self.__create_img_file(img_fn, img_sz) - self.__create_parts(img_fn) - - def install(self, dir): - Logger.install(f"Install to '{dir}'") - if (self.installs["target"] == "image"): - self.__install_to_img() - else: - Logger.error("Unsupported instalation type!") - #is_blk = False - #if (stat.S_ISBLK(os.stat(dir).st_mode)): - # dir = self.__make_blk_struct(dir) diff --git a/scripts/os.py b/scripts/os.py index 67bc5ea..4224334 100644 --- a/scripts/os.py +++ b/scripts/os.py @@ -1,11 +1,17 @@ -import subprocess, os, sys, datetime, getpass, shutil, requests +import subprocess, os, sys, datetime, getpass, shutil, requests, stat, re from pathlib import Path if __name__ != '__main__': from . import * +units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 } + +class Partition(object): + pass + class OS: def __init__(self): self.root_dir = f"{ROOT_DIR}/root" + self.mount_dir = f"{ROOT_DIR}/build/mnt_tmp" self.actions = [ [ "chroot", self.chroot ], [ "sync", self.sync_repo ], @@ -42,9 +48,9 @@ class OS: self.board = board self.arch = board.parse_variables("%{ARCH}%") - def __sudo(self, args, cwd=None, env=None): + def __sudo(self, args, cwd=None, env=None, stdout=None): args.insert(0, "sudo") - p = subprocess.Popen(args, cwd=cwd, env=env) + p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout) if (p.wait() != 0): Logger.error(f"Command '{args[1]}' finished with error code!") @@ -198,7 +204,8 @@ class OS: self.__extract_tar(arch_path, temp_dir) sqh_fn = f"{ROOT_DIR}/out/root_{date}.sqh" self.__make_sqh(temp_dir, sqh_fn) - #os.symlink(sqh_fn, f"{ROOT_DIR}/out/root.sqh") + os.symlink(sqh_fn, f"{ROOT_DIR}/out/root.sqh.tmp") + os.rename(f"{ROOT_DIR}/out/root.sqh.tmp", f"{ROOT_DIR}/out/root.sqh") self.__tmp_clean(temp_dir) def action(self, action): @@ -207,6 +214,190 @@ class OS: act[1]() break + def __do_cmd(self, args, cwd=None, env=None, stdin=None, stdout=None): + if (stdin != None): + p = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=stdout, text=True) + else: + p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout) + if (stdin != None): + p.communicate(input=stdin) + if (p.wait() != 0): + Logger.error(f"Command '{args[0]}' finished with error code!") + + def __part_prepare(self): + self.block_size = self.__parse_size(self.board.installs["block_size"]) + self.partitions = [] + for part in self.board.installs["partitions"]: + part_obj = Partition() + part_obj.name = part["name"] + if "first_sector" in part: + part_obj.first_sector = int(part["first_sector"]) + else: + part_obj.first_sector = -1 + part_obj.size = self.__parse_size(part["size"]) + part_obj.size_blk = int(part_obj.size / self.block_size) - 1 + self.partitions.append(part_obj) + + def __create_img_file(self, path, size): + Logger.install("\tCreate image file...") + img_f = Path(path) + if (img_f.is_file()): + shutil.rmtree(path, ignore_errors=True) + blk_size = 1024*1024 + blk_count = int(size / blk_size) + self.__do_cmd(["dd", "if=/dev/zero", f"of={path}", f"bs={blk_size}", + f"count={blk_count}"], stdout=subprocess.DEVNULL) + + def __parse_size(self, size): + size = size.upper() + if not re.match(r' ', size): + size = re.sub(r'([BKMGT])', r' \1', size) + number, unit = [string.strip() for string in size.split()] + return int(float(number)*units[unit]) + + def __get_img_size(self): + offset_fix = 1024 * 1024 * 1 + img_sz = offset_fix + for part in self.partitions: + if part.first_sector > -1: + img_sz = offset_fix + (part.first_sector * self.block_size) + img_sz += part.size + return img_sz + + def __create_parts(self, img_or_dev, from_sudo): + Logger.install("\tCreate partitions table...") + args = "" + part_type = "p\n" + if (self.board.installs["type"] == "mbr"): + args += "o\n" + elif (self.board.installs["type"] == "gpt"): + args += "g\n" + part_type = "" + offset = 0 + for part in self.partitions: + args += f"n\n{part_type}\n" + if part.first_sector > -1: + offset = part.first_sector + args += f"{offset}\n" + args += f"+{part.size_blk}\n" + offset += part.size_blk + 1 + args += "w\nq\n" + cmd = [] + if (from_sudo): + cmd.append("sudo") + cmd.append("fdisk") + cmd.append(img_or_dev) + self.__do_cmd(cmd, stdin=args, stdout=subprocess.DEVNULL) + + def __prepare_img(self, out_dir): + Logger.install("\tImage. Prepare and mount it...") + img_fn = f"{out_dir}/all.img" + img_sz = self.__get_img_size() + self.__create_img_file(img_fn, img_sz) + return img_fn + + def __mount_loop(self, img_or_blk, idx): + offset = 0 + i = 0 + for part in self.partitions: + if part.first_sector > -1: + offset = part.first_sector * self.block_size + part_size = part.size + if (part_size > (90 * 1024 * 1024)) and (i == idx): + # required partition + #print(f"\tIdx:{i} Size:{part_size}") + self.__sudo(["losetup", "-o", str(offset), "--sizelimit", + str(part_size), "/dev/loop0", img_or_blk], + cwd=ROOT_DIR)#, stdout=subprocess.DEVNULL) + return True + i += 1 + offset += part_size + return False + + def __umount_loop(self): + self.__sudo(["losetup", "-d", "/dev/loop0"], stdout=subprocess.DEVNULL) + + def __mount_dev(self, dev, dir): + self.__sudo(["mount", dev, dir], stdout=subprocess.DEVNULL) + + def __umount_dev(self, dir): + self.__sudo(["umount", dir], stdout=subprocess.DEVNULL) + + def __create_fs(self, img_or_blk): + Logger.install("\tCreate filesystems...") + for i in range(len(self.partitions)): + if (self.__mount_loop(img_or_blk, i)): + self.__sudo(["mkfs.ext2", "/dev/loop0"], stdout=subprocess.DEVNULL) + self.__umount_loop() + + def __copy_file(self, src, dst): + Logger.install(f"\tCopy {src}") + self.__sudo(["mkdir", "-p", dst], stdout=subprocess.DEVNULL) + self.__sudo(["cp", src, dst], stdout=subprocess.DEVNULL) + + def __dd_bin(self, src, block_size, offset): + blk_sz = self.__parse_size(block_size) + Logger.install(f"\tDD {src} (+{offset}:{blk_sz})") + self.__sudo(["dd", f"if={src}", f"of={self.out_path}", + f"bs={blk_sz}", f"seek={offset}", "conv=notrunc"], stdout=subprocess.DEVNULL) + + def __install_boot(self, out_dir): + extl_dir = f"{out_dir}/extlinux" + extl_fn = f"{extl_dir}/extlinux.conf" + dtb_file = self.board.parse_variables("%{DTB_FILE}%") + cmd = f"mkdir -p {extl_dir} && touch {out_dir}/livecd && " + cmd += f"echo 'menu title Boot Options.\n\ntimeout 20\ndefault Kernel_def\n\n" + cmd += f"label Kernel_def\n\tkernel /Image\n\tfdtdir /dtb/\n\tdevicetree /dtb/{dtb_file}\n\tinitrd /uInitrd\n' >> {extl_fn}" + self.__sudo(["sh", "-c", f"{cmd}"], stdout=subprocess.DEVNULL) + for target in self.board.targets: + target.install_files(out_dir, self.board.out_dir, "boot", self.__copy_file, self.__dd_bin) + self.__copy_file(f"{self.board.out_sh}/uInitrd", f"{out_dir}/") + Logger.install(f"\tCopy root.sqh") + self.__sudo(["cp", "-H", f"{self.board.out_sh}/root.sqh", f"{out_dir}/"]) + + def __install_rw(self, out_dir): + self.__sudo(["touch", f"{out_dir}/rw_part"], stdout=subprocess.DEVNULL) + self.__sudo(["mkdir", "-p", f"{out_dir}/.upper"], stdout=subprocess.DEVNULL) + self.__sudo(["mkdir", "-p", f"{out_dir}/.work"], stdout=subprocess.DEVNULL) + + def __do_boot(self, img_or_blk): + Logger.install("\tCreate boot files...") + i = 0 + os.makedirs(self.mount_dir, exist_ok=True) + for part in self.partitions: + if (part.name == "boot"): + self.__mount_loop(img_or_blk, i) + self.__mount_dev("/dev/loop0", self.mount_dir) + self.__install_boot(self.mount_dir) + self.__umount_dev(self.mount_dir) + self.__umount_loop() + if (part.name == "rw"): + self.__mount_loop(img_or_blk, i) + self.__mount_dev("/dev/loop0", self.mount_dir) + self.__install_rw(self.mount_dir) + self.__umount_dev(self.mount_dir) + self.__umount_loop() + i += 1 + + def install(self, dir_or_dev): + Logger.install(f"Install to '{dir_or_dev}'") + is_blk = False + dir_ch = Path(dir_or_dev) + self.__part_prepare() + if (not dir_ch.is_dir()) and (stat.S_ISBLK(os.stat(dir_or_dev).st_mode)): + is_blk = True + Logger.install(f"\tBlock device, need to use a sudo.") + if (self.board.installs["target"] == "image"): + if (not is_blk): + dir_or_dev = self.__prepare_img(dir_or_dev) + else: + Logger.error("Unsupported instalation type!") + self.out_path = dir_or_dev + self.__create_parts(dir_or_dev, is_blk) + self.__create_fs(dir_or_dev) + self.__do_boot(dir_or_dev) + Logger.install(f"Finished!") + if __name__ == '__main__': f = open("/proc/sys/fs/binfmt_misc/register","wb") if (len(sys.argv) < 2) or (sys.argv[1] == "aarch64"): diff --git a/scripts/target.py b/scripts/target.py index c47fe6d..049f811 100644 --- a/scripts/target.py +++ b/scripts/target.py @@ -90,3 +90,15 @@ class Target: self.sources.compile(opts_tmp, self.config_name) if (sub_target != "config"): self.sources.copy_artifacts(self.artifacts, out_dir) + + def install_files(self, dir, tmp_dir, part_name, on_file, on_dd): + Logger.install(f"'{self.name}': Install artifacts") + for art in self.artifacts: + art_fn = os.path.basename(art["file"]) + if (art["store_type"] == part_name): + subdir = "" + if "subdir" in art: + subdir = art["subdir"] + "/" + on_file(f"{tmp_dir}/{subdir}{art_fn}", f"{dir}/{subdir}") + if (art["store_type"] == "dd"): + on_dd(f"{tmp_dir}/{art_fn}", art["block_size"], int(art["img_offset"]))