From 13eb58357fa075050d41b4a0b18523b064849d58 Mon Sep 17 00:00:00 2001 From: andreili Date: Sun, 20 Jul 2025 19:39:01 +0200 Subject: [PATCH] Add final steps for deployment. --- config/os_aarch64.json | 71 +++++++++--- files/backups/excl_min.lst | 3 +- .../portage/andreil/virtual/klipper/Manifest | 2 +- .../andreil/virtual/klipper/klipper-11.ebuild | 2 + scripts/board.py | 21 +++- scripts/chroot.sh | 9 +- scripts/os.py | 105 +++++++----------- 7 files changed, 124 insertions(+), 89 deletions(-) diff --git a/config/os_aarch64.json b/config/os_aarch64.json index dca31ad..0677ad6 100644 --- a/config/os_aarch64.json +++ b/config/os_aarch64.json @@ -1,4 +1,8 @@ { + "variables": + [ + "USER_LOGIN:klipper" + ], "stage3_info": { "marker": "stage3_extracted", @@ -64,6 +68,15 @@ "location = /usr/portage/andreil" ] }, + { + "file": "/etc/portage/repos.conf/crossdev.conf", + "append": false, + "lines": + [ + "[crossdev]", + "location = /usr/portage/crossdev" + ] + }, { "file": "/etc/portage/savedconfig/sys-kernel/linux-firmware", "append": false, @@ -138,9 +151,7 @@ "sys-fs/e2fsprogs static-libs" ] }, - { - "chroot": "emerge-webrsync" - } + { "chroot": "emerge-webrsync" } ] }, "update": @@ -153,18 +164,52 @@ "append": false, "lines": [ "en_US.UTF-8 UTF-8" ] }, + { "chroot": "locale-gen" }, + { "chroot": "eselect kernel set 1" }, + { "chroot": "eselect news read" }, + { "chroot": "eselect editor set nano" }, + { "action": "update" } + ] + }, + "install": + { + "marker": "stage3_installed", + "steps": + [ { - "chroot": "locale-gen" + "soft_inst": + [ + "app-editors/nano", + "app-misc/mc", + "app-portage/eix", + "app-portage/gentoolkit", + "sys-devel/crossdev", + "sys-process/htop", + "virtual/klipper" + ], + "oneshot": false }, - { - "chroot": "eselect kernel set 1" - }, - { - "chroot": "eselect news read" - }, - { - "action": "update" - } + { "chroot": "crossdev -s4 arm-none-eabi" }, + { "chroot": "eselect news read" }, + { "soft_clean": "default" } + ] + }, + "finalize": + { + "steps": + [ + { "chroot": "systemctl enable NetworkManager ntpdate sshd" }, + { "sudo": "sed -i -E 's/^# (%wheel ALL)/\\1/' ./etc/sudoers" }, + { "sudo": "sed -i -E 's/^#(\\S+MaxUse)=$/\\1=10M/' ./etc/systemd/journald.conf" }, + { "sudo": "sed -i -E 's/^#(\\S+MaxFileSize)=$/\\1=10M/' ./etc/systemd/journald.conf" }, + { "copy": [ "%{ROOT_DIR}%/files/firmware/usr", "."] }, + { "sudo": "chmod u+s ./usr/bin/Xorg" }, + { "sudo": "ln -sf /usr/share/zoneinfo/Europe/Warsaw ./etc/localtime" }, + { "chroot": "useradd -m -G wheel,video,audio,disk,usb %{USER_LOGIN}% --password %{USER_LOGIN}%" }, + { "chroot": "echo '%{USER_LOGIN}%:%{USER_LOGIN}%' | chpasswd" }, + { "chroot": "echo 'root:root' | chpasswd" }, + { "chroot": "sudo -i -u klipper python -m venv /home/%{USER_LOGIN}%/venv" }, + { "soft_clean": "bdeps" } ] } } diff --git a/files/backups/excl_min.lst b/files/backups/excl_min.lst index 202cf4f..0a55625 100644 --- a/files/backups/excl_min.lst +++ b/files/backups/excl_min.lst @@ -6,6 +6,7 @@ usr/lib64/perl5* #usr/src/* var/cache/binpkgs/* var/cache/distfiles/* -var/cache/edb/* +#var/cache/edb/* var/cache/eix/* var/log/*.log +#var/db/repos/* diff --git a/files/portage/andreil/virtual/klipper/Manifest b/files/portage/andreil/virtual/klipper/Manifest index 2fd764d..216b5c2 100644 --- a/files/portage/andreil/virtual/klipper/Manifest +++ b/files/portage/andreil/virtual/klipper/Manifest @@ -1 +1 @@ -EBUILD klipper-11.ebuild 603 BLAKE2B fe8ca3d46a257c6c056f8a2480afdb5270b3e370495d2fad3ebbf7b066b456c2b365c2e83c3c6d42a81018f6bbc2f15cb36be502fff3589343620e844cc21d41 SHA512 14dad1041e1ecaf645bb6c508089d292bc30cf644c656c127e8299c67e1866f5cafe016af8794c2a81f26f7d405c1ff558e9aca1e8c43a9b595ebe63c5b977fd +EBUILD klipper-11.ebuild 636 BLAKE2B fe1685f4bfbba06f544f6bb0a01cbaca30f938dc8963bb2f01f7b57c3643738caa9403049916319eff3f626d1507e7577762188bd03ada31a4e6ea04ce763abb SHA512 381bb6899bfe71456e0148abfeedf8db1dccb5b5d4998068e9eb03a8d0008e9b39ea671e82f1b8afc4aba6527cba801f59d824c2800657d4d96e5f7a12235c49 diff --git a/files/portage/andreil/virtual/klipper/klipper-11.ebuild b/files/portage/andreil/virtual/klipper/klipper-11.ebuild index 2193601..5d7b7f7 100644 --- a/files/portage/andreil/virtual/klipper/klipper-11.ebuild +++ b/files/portage/andreil/virtual/klipper/klipper-11.ebuild @@ -26,4 +26,6 @@ RDEPEND=" dev-python/matplotlib sys-devel/crossdev sys-apps/pciutils + media-libs/libv4l + net-misc/ntp " diff --git a/scripts/board.py b/scripts/board.py index bf3713a..7181c71 100644 --- a/scripts/board.py +++ b/scripts/board.py @@ -1,4 +1,4 @@ -import json, os +import json, os, re from pathlib import Path from . import * @@ -47,18 +47,27 @@ class Board: return 0 def __load_vars(self): - for var_def in self.json["variables"]: - self.variables.append(var_def.split(":")) + self.add_vars(self.json["variables"]) self.variables.append(["board_name", self.name]) self.variables.append(["build_dir", f"{ROOT_DIR}/build/{self.name}"]) self.variables.append(["common_dir", f"{ROOT_DIR}/build/common"]) self.variables.append(["out_dir", self.out_dir]) self.variables.append(["out_sh", self.out_sh]) + self.variables.append(["ROOT_DIR", ROOT_DIR]) + + def add_var(self, name, val): + self.variables.append([name, val]) + + def add_vars(self, lst): + for var_def in lst: + self.variables.append(var_def.split(":")) def parse_variables(self, string): - for var_d in self.variables: - string = string.replace("%{"+var_d[0]+"}%", var_d[1]) - #out_dir + while True: + for var_d in self.variables: + string = string.replace("%{"+var_d[0]+"}%", str(var_d[1])) + if not re.compile('%{\\S+}%').match(string): + break return string def sync(self): diff --git a/scripts/chroot.sh b/scripts/chroot.sh index 165ecfa..078e63b 100644 --- a/scripts/chroot.sh +++ b/scripts/chroot.sh @@ -3,13 +3,18 @@ OS_DIR_DEF="./root/" DDIR=$(realpath "$1") ROOT_DIR="$2" -KV=$(make -C ./build/common/kernel/ --silent kernelversion) if [ -z "${DDIR}" ] then - echo "No directory specified!" + echo "No root directory specified!" exit 1 fi +if [ -z "${ROOT_DIR}" ] +then + echo "No main directory specified!" + exit 1 +fi +KV=$(make -C "${ROOT_DIR}/build/common/kernel/" --silent kernelversion) mkdir -p ${DDIR}/usr/portage mount --bind ${ROOT_DIR}/files/portage ${DDIR}/usr/portage diff --git a/scripts/os.py b/scripts/os.py index f74069a..0e848a7 100644 --- a/scripts/os.py +++ b/scripts/os.py @@ -4,7 +4,7 @@ if __name__ != '__main__': from . import * units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 } -MARKER_ROOTFS_READY = ".rootfs_ready" +MARKER_ROOTFS_READY = "rootfs_ready" class Partition(object): pass @@ -29,6 +29,10 @@ class OS: self.st3_info = js_data["stage3_info"] self.st3_prepare = js_data["prepare"] self.st3_update = js_data["update"] + self.st3_install = js_data["install"] + self.finalize = js_data["finalize"] + self.board.add_vars(js_data["variables"]) + self.board.add_var("ROOT_FS", self.root_dir) def actions_list(self): lst = [] @@ -48,7 +52,7 @@ class OS: return [arch_url, stage3_fn] def __stage3_apply(self, info, text): - self.__tmp_clean(f"{ROOT_DIR}/root") + self.__tmp_clean(self.root_dir) [url,fn] = self.__get_stage3_url() Logger.os(f"Download Stage3 archive '{fn}'...") temp_dir = f"{ROOT_DIR}/build/tmp" @@ -65,7 +69,9 @@ class OS: self.__extract_tar(arch_fn, self.root_dir) self.__tmp_clean(temp_dir) - def __stage3_steps(self, info, text): + def __stage3_steps(self, info, text, dir=""): + if (dir == ""): + dir = self.root_dir Logger.os(text) self.__sudo(["cp", "/etc/resolv.conf", f"{self.root_dir}/etc/resolv.conf"]) for step in info["steps"]: @@ -77,36 +83,54 @@ class OS: cmd = f"mkdir -p {self.root_dir}{directory} && echo '{lines}'" cmd += f" | sudo tee {is_append} {self.root_dir}{path} > /dev/null" Logger.os(f"\tCreate file {path}...") - self.__sudo(cmd, shell=True) + self.__sudo(cmd, shell=True, cwd=dir) if ("chroot" in step): - self.__chroot(step["chroot"], stdout=subprocess.DEVNULL) + cmd = self.board.parse_variables(step["chroot"]) + self.__chroot(cmd, dir=dir) if ("action" in step): action = step["action"] for act in self.actions: if (act[0] == action): act[1]() break + if ("soft_inst" in step): + sw_list = " ".join(step["soft_inst"]) + oneshot = "1" if step["oneshot"] else "" + self.__chroot(f"emerge -avb{oneshot} {sw_list} -j2", dir=dir) + if ("soft_clean" in step): + clean_type = step["soft_clean"] + if (clean_type == "default"): + self.__chroot(f"emerge -ac", dir=dir) + if (clean_type == "bdeps"): + self.__chroot(f"emerge --ask --depclean --with-bdeps=n --exclude sys-devel/gcc && ldconfig", dir=dir) + if ("sudo" in step): + cmd = self.board.parse_variables(step["sudo"]) + Logger.os(f"\tSudo command {cmd}...") + self.__sudo(cmd, cwd=dir, shell=True) def check_rootfs(self): if marker_check(MARKER_ROOTFS_READY): return stages = [ - [self.st3_info, self.__stage3_apply, "" ], - [self.st3_prepare, self.__stage3_steps, "Basic preparation..."], - [self.st3_update, self.__stage3_steps, "System update..." ], + [self.st3_info, self.__stage3_apply, "" ], + [self.st3_prepare, self.__stage3_steps, "Basic preparation..."], + [self.st3_update, self.__stage3_steps, "System update..." ], + [self.st3_install, self.__stage3_steps, "Software installation..."], ] for st in stages: if (not marker_check(st[0]["marker"])): st[1](st[0], st[2]) marker_set(st[0]["marker"]) + marker_set(MARKER_ROOTFS_READY) def set_board(self, board): self.board = board + self.board.add_var("ROOTFS", self.root_dir) self.arch = board.parse_variables("%{ARCH}%") def __sudo(self, args, cwd=None, env=None, stdout=None, shell=None): if isinstance(args, str): - args = "sudo " + args + args = self.board.parse_variables("sudo " + args) err_n = args else: args.insert(0, "sudo") @@ -173,10 +197,6 @@ class OS: def pack(self): return self.__do_archive("excl_min", "FULL", self.root_dir) - def __fix_xorg(self): - Logger.os("Fix Xorg permissions") - self.__sudo(["chmod", "u+s", f"{self.root_dir}/usr/bin/Xorg"]) - def __tmp_clean(self, path): Logger.os(f"Clean directory '{path}'...") t_dir = Path(path) @@ -199,66 +219,19 @@ class OS: user = getpass.getuser() self.__sudo(["chown", user + ":" + user, to_file]) - def __remove_bdeps(self, temp_dir): - # remove unneccessarry packages - # emerge --ask --depclean --with-bdeps=n - #list_to_rm = " virtual/perl-JSON-PP virtual/perl-podlators" - #list_to_rm += " virtual/perl-Getopt-Long virtual/perl-Parse-CPAN-Meta" - #list_to_rm += " virtual/perl-ExtUtils-CBuilder virtual/perl-ExtUtils-ParseXS" - #list_to_rm += " virtual/perl-Unicode-Collate virtual/perl-Text-ParseWords" - #list_to_rm += " virtual/perl-ExtUtils-MakeMaker virtual/perl-Module-Metadata" - #list_to_rm += " virtual/perl-version virtual/perl-CPAN-Meta" - #list_to_rm += " virtual/perl-File-Spec perl-core/Getopt-Long" - #list_to_rm += " dev-perl/Module-Build" - #list_to_rm += " x11-base/xcb-proto x11-libs/xtrans" - #list_to_rm += " app-alternatives/ninja app-eselect/eselect-rust" - #list_to_rm += " dev-libs/vala-common dev-util/glib-utils" - #list_to_rm += " dev-util/gdbus-codegen media-fonts/font-util" - #list_to_rm += " dev-libs/libxslt" - #list_to_rm += " dev-build/gtk-doc-am sys-apps/help2man" - #list_to_rm += " app-text/docbook-xml-dtd:4.1.2" - #list_to_rm += " app-text/docbook-xml-dtd:4.2 app-text/docbook-xml-dtd:4.3" - #list_to_rm += " app-text/docbook-xml-dtd:4.4 app-text/docbook-xml-dtd:4.5" - #list_to_rm += " app-text/docbook-xsl-ns-stylesheets app-text/docbook-xsl-stylesheets" - #list_to_rm += " app-text/build-docbook-catalog app-text/xmlto" - #list_to_rm += " app-text/asciidoc app-text/sgml-common" - #list_to_rm += " dev-lang/rust-common dev-lang/rust" - #list_to_rm += " llvm-core/llvm llvm-core/llvm-toolchain-symlinks" - #list_to_rm += " llvm-core/llvmgold dev-libs/oniguruma" - #list_to_rm += " llvm-core/llvm-common sys-libs/binutils-libs" - #list_to_rm += " dev-build/ninja dev-build/meson dev-build/meson-format-array" - #list_to_rm += " dev-cpp/eigen" - #list_to_rm += " " - #self.__chroot(f"emerge -aC {list_to_rm} && ldconfig", temp_dir) - # remove build deps, exclude gcc/binutils - it's required to build a software after it updates - self.__chroot(f"emerge --ask --depclean --with-bdeps=n --exclude sys-devel/gcc && ldconfig", temp_dir) - self.__do_archive("excl_min", "FULL_min_bdeps", temp_dir) - def __finalize(self, dir): - Logger.os(f"Finalize system installation...") - # enable network services - services = "systemctl enable NetworkManager ntpdate sshd" - self.__chroot(services, dir) - # minimize systemd journal files - journal_min = "sed -i -E 's/^#(\\S+MaxUse)=$/\\1=10M/' /etc/systemd/journald.conf &&" - journal_min += "sed -i -E 's/^#(\\S+MaxFileSize)=$/\\1=10M/' /etc/systemd/journald.conf" - self.__chroot(journal_min, dir) - self.__sudo(["cp", "-r", f"{ROOT_DIR}/files/firmware/usr", f"{dir}/"]) - soft = Software(self) - soft.finalize(dir) + self.__stage3_steps(self.finalize, "Finalize system installation...", dir=dir) def sqh(self): - self.__fix_xorg() date = datetime.datetime.today().strftime('%Y_%m_%d') temp_dir = f"{ROOT_DIR}/build/tmp" # pack full system via tar - arch_full_path = self.pack() - self.__tmp_clean(temp_dir) - self.__extract_tar(arch_full_path, temp_dir) - # remove unnecessary packages - self.__remove_bdeps(temp_dir) - # prepare system + #arch_full_path = self.pack() + #self.__tmp_clean(temp_dir) + #self.__extract_tar(arch_full_path, temp_dir) + # prepare system, remove unnecessary packages self.__finalize(temp_dir) + self.__do_archive("excl_min", "FULL_min_bdeps", temp_dir) # pack a minimal archive arch_path = self.__do_archive("excl", "OS", temp_dir) # remove temp directory