Add basic OS deploy on clean run.

build.py

@@ -22,6 +22,7 @@ if (args.board == ''):
 targets_meta = Target.load_meta(f"config/target_meta.json")
 target_board = Board(args.board, f"config/board/{args.board}.json", targets_meta)
 os.set_board(target_board)
+os.load_info()
 
 os.check_rootfs()
 

config/os_aarch64.json

@@ -0,0 +1,170 @@
+{
+    "stage3_info":
+    {
+        "marker": "stage3_extracted",
+        "server_dir": "https://distfiles.gentoo.org/releases/arm64/autobuilds/current-stage3-arm64-systemd/",
+        "info_file": "latest-stage3-arm64-systemd.txt"
+    },
+    "prepare":
+    {
+        "marker": "stage3_prepared",
+        "steps":
+        [
+            {
+                "file": "/etc/portage/modules",
+                "append": false,
+                "lines": [ "portdbapi.auxdbmodule = portage.cache.sqlite.database" ]
+            },
+            {
+                "file": "/etc/portage/make.conf",
+                "append": false,
+                "lines":
+                [
+                    "COMMON_FLAGS=\"-O3 -pipe\"",
+                    "CFLAGS=\"${COMMON_FLAGS}\"",
+                    "CXXFLAGS=\"${COMMON_FLAGS}\"",
+                    "FCFLAGS=\"${COMMON_FLAGS}\"",
+                    "FFLAGS=\"${COMMON_FLAGS}\"",
+                    "CHOST=\"aarch64-unknown-linux-gnu\"",
+                    "",
+                    "EMERGE_DEFAULT_OPTS=\"--load-average 8.0\"",
+                    "USE=\"${USE} ${ARCH}",
+                    "    -handbook -compiler-rt -sanitize -gtk-doc -gtk -jit",
+                    "    -vulkan -cups -wayland -opengl -egl -eglfs -gles2 -kms",
+                    "    -vala -spell -tk -tcl -fortran",
+                    "    -slang -gpg -doc -test lto -nls",
+                    "    sqlite\"",
+                    "FEATURES=\"${FEATURES} metadata-transfer nodoc noinfo noman -pid-sandbox\"",
+                    "",
+                    "VIDEO_CARDS=\"panfrost fbdev\"",
+                    "INPUT_DEVICES=\"libinput evdev\"",
+                    "LLVM_TARGETS=\"\"",
+                    "PYTHON_TARGETS=\"python3_13\"",
+                    "PYTHON_SINGLE_TARGET=\"${PYTHON_TARGETS}\"",
+                    "",
+                    "LC_MESSAGES=C.utf8"
+                ]
+            },
+            {
+                "file": "/etc/portage/package.accept_keywords/all",
+                "append": false,
+                "lines": [ "*/* ~arm64" ]
+            },
+            {
+                "file": "/etc/portage/package.mask/python",
+                "append": false,
+                "lines": [ ">dev-lang/python-3.13.999" ]
+            },
+            {
+                "file": "/etc/portage/repos.conf/andreil.conf",
+                "append": false,
+                "lines":
+                [
+                    "[andreil]",
+                    "location = /usr/portage/andreil"
+                ]
+            },
+            {
+                "file": "/etc/portage/savedconfig/sys-kernel/linux-firmware",
+                "append": false,
+                "lines": [ "regulatory.db" ]
+            },
+            {
+                "file": "/etc/portage/package.use/test",
+                "append": false,
+                "lines":
+                [
+                    "net-misc/networkmanager -bluetooth -modemmanager",
+                    "net-misc/networkmanager -ppp"
+                ]
+            },
+            {
+                "file": "/etc/portage/package.use/system",
+                "append": false,
+                "lines":
+                [
+                    "sys-kernel/linux-firmware savedconfig",
+                    "sys-kernel/genkernel -firmware",
+                    "net-misc/networkmanager tools",
+                    "sys-apps/util-linux static-libs"
+                ]
+            },
+            {
+                "file": "/etc/portage/package.use/minimal",
+                "append": false,
+                "lines":
+                [
+                    "sys-apps/portage -rsync-verify",
+                    "app-admin/sudo -sendmail",
+                    "sys-apps/systemd -dns-over-tls -gcrypt -kernel-install",
+                    "media-libs/mesa -llvm",
+                    "sys-process/htop lm-sensors"
+                ]
+            },
+            {
+                "file": "/etc/portage/package.use/klipper",
+                "append": false,
+                "lines":
+                [
+                    "virtual/klipper mainsail",
+                    "app-misc/mime-types nginx",
+                    "media-libs/mesa -proprietary-codecs",
+                    "x11-base/xorg-server minimal",
+                    "sys-apps/iproute2 minimal",
+                    "dev-perl/* minimal",
+                    "dev-lang/perl dev-perl",
+                    "sys-apps/systemd policykit",
+                    "x11-libs/cairo X",
+                    "dev-python/pycairo X",
+                    "media-libs/libepoxy -egl",
+                    "net-wireless/wpa_supplicant dbus",
+                    "#x11-base/xorg-server suid",
+                    "cross-arm-none-eabi/newlib nano",
+                    "media-libs/libepoxy egl",
+                    "net-misc/networkmanager -tools",
+                    "media-libs/libglvnd X",
+                    "media-libs/libv4l bpf",
+                    "media-video/ffmpeg x264 x265",
+                    "#media-libs/libv4l -bpf",
+                    "sys-devel/gcc fortran",
+                    "dev-python/pillow webp",
+                    "virtual/imagemagick-tools jpeg tiff",
+                    "media-gfx/imagemagick jpeg tiff",
+                    "sci-libs/atlas lapack threads",
+                    "dev-lang/rust system-llvm",
+                    "sys-apps/busybox static -pam savedconfig",
+                    "virtual/libcrypt static-libs",
+                    "sys-libs/libxcrypt static-libs",
+                    "sys-fs/e2fsprogs static-libs"
+                ]
+            },
+            {
+                "chroot": "emerge-webrsync"
+            }
+        ]
+    },
+    "update":
+    {
+        "marker": "stage3_updated",
+        "steps":
+        [
+            {
+                "file": "/etc/locale.gen",
+                "append": false,
+                "lines": [ "en_US.UTF-8 UTF-8" ]
+            },
+            {
+                "chroot": "locale-gen"
+            },
+            {
+                "chroot": "eselect kernel set 1"
+            },
+            {
+                "chroot": "eselect news read"
+            },
+            {
+                "action": "update"
+            }
+        ]
+    }
+}

scripts/chroot.sh

@@ -3,6 +3,7 @@
 OS_DIR_DEF="./root/"
 DDIR=$(realpath "$1")
 ROOT_DIR="$2"
+KV=$(make -C ./build/common/kernel/ --silent kernelversion)
 
 if [ -z "${DDIR}" ]
 then
@@ -12,29 +13,25 @@ fi
 
 mkdir -p ${DDIR}/usr/portage
 mount --bind ${ROOT_DIR}/files/portage ${DDIR}/usr/portage
-mkdir -p ${DDIR}/usr/src/linux-6.14-rc7
-mount --bind ${ROOT_DIR}/build/common/kernel ${DDIR}/usr/src/linux-6.14-rc7
+mkdir -p ${DDIR}/usr/src/linux-${KV}
+mount --bind ${ROOT_DIR}/build/common/kernel ${DDIR}/usr/src/linux-${KV}
 
 mount --bind /dev ${DDIR}/dev
 mount --bind /dev/shm ${DDIR}/dev/shm
 mount --bind /dev/pts ${DDIR}/dev/pts
 mount --bind /sys ${DDIR}/sys
 mount --bind /proc ${DDIR}/proc
-if [ -d "/var/db/repos" ]; then
-    mount --bind /var/db/repos ${DDIR}/var/db/repos
-fi
 mount -t tmpfs tmpfs ${DDIR}/var/tmp/
 if [ -z "$3" ]
 then
     chroot ${DDIR}/ /bin/bash
+    ret=$?
 else
     chroot ${DDIR}/ /bin/bash -c "${@:3}"
+    ret=$?
 fi
 umount ${DDIR}/var/tmp
-umount ${DDIR}/usr/src/linux-6.14-rc7
-if [ -d "/var/db/repos" ]; then
-    umount ${DDIR}/var/db/repos
-fi
+umount ${DDIR}/usr/src/linux-${KV}
 umount ${DDIR}/proc
 umount ${DDIR}/sys
 umount ${DDIR}/dev/pts
@@ -44,3 +41,4 @@ if [ -n "$1" ]
 then
     umount ${DDIR}/usr/portage
 fi
+exit ${ret}

scripts/lib.py

@@ -2,3 +2,14 @@ import os
 from pathlib import Path
 
 ROOT_DIR=Path(os.path.abspath(__file__)).parent.parent
+
+def marker_check(name):
+    fn = f"{ROOT_DIR}/build/.{name}_marker"
+    marker = Path(fn)
+    if (marker.is_file()):
+        return True
+    return False
+
+def marker_set(name):
+    fn = f"{ROOT_DIR}/build/.{name}_marker"
+    Path(fn).touch()

scripts/os.py

@@ -4,6 +4,7 @@ if __name__ != '__main__':
     from . import *
 
 units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 }
+MARKER_ROOTFS_READY = ".rootfs_ready"
 
 class Partition(object):
     pass
@@ -21,38 +22,99 @@ class OS:
             [ "sqh",       self.sqh         ]
         ]
 
+    def load_info(self):
+        with open(f"{ROOT_DIR}/config/os_{self.arch}.json") as json_data:
+            js_data = json.load(json_data)
+            json_data.close()
+            self.st3_info = js_data["stage3_info"]
+            self.st3_prepare = js_data["prepare"]
+            self.st3_update = js_data["update"]
+
     def actions_list(self):
         lst = []
         for act in self.actions:
             lst.append(act[0])
         return lst
 
-    def check_rootfs(self):
-        root_marker = Path(self.root_dir)
-        if (not root_marker.is_dir()):
-            Logger.os(f"Download base OS rootfs archive...")
+    def __get_stage3_url(self):
+        url_descr = self.st3_info["server_dir"] + self.st3_info["info_file"]
+        r = requests.get(url_descr, stream=True)
+        descr = r.content.decode('utf-8').splitlines()
+        stage3_fn = ""
+        for d in descr:
+            if (d.startswith("stage3")):
+                stage3_fn = d.split()[0]
+        arch_url = self.st3_info["server_dir"] + stage3_fn
+        return [arch_url, stage3_fn]
+
+    def __stage3_apply(self, info, text):
+        self.__tmp_clean(f"{ROOT_DIR}/root")
+        [url,fn] = self.__get_stage3_url()
+        Logger.os(f"Download Stage3 archive '{fn}'...")
         temp_dir = f"{ROOT_DIR}/build/tmp"
+        self.__tmp_clean(temp_dir)
         os.makedirs(temp_dir, exist_ok=True)
-            r = requests.get("https://cloud.andreil.by/public.php/dav/files/sbc-rootfs-archive", stream=True)
-            arch_fn = f"{temp_dir}/sbc_rootfs_archive.tar.xz"
+        r = requests.get(url, stream=True)
+        arch_fn = f"{temp_dir}/{fn}"
         with open(arch_fn, 'wb') as f:
             total_length = int(r.headers.get('content-length'))
             for chunk in r.iter_content(chunk_size=1024):
                 f.write(chunk)
+        Logger.os(f"Extract Stage3 archive...")
         os.makedirs(self.root_dir, exist_ok=True)
         self.__extract_tar(arch_fn, self.root_dir)
         self.__tmp_clean(temp_dir)
-            self.__chroot("eix-sync -v")
+
+    def __stage3_steps(self, info, text):
+        Logger.os(text)
+        self.__sudo(["cp", "/etc/resolv.conf", f"{self.root_dir}/etc/resolv.conf"])
+        for step in info["steps"]:
+            if ("file" in step):
+                is_append = "-a" if step["append"] else ""
+                lines = "\n".join(step["lines"])
+                path = step["file"]
+                directory = Path(path).parent
+                cmd  = f"mkdir -p {self.root_dir}{directory} && echo '{lines}'"
+                cmd += f" | sudo tee {is_append} {self.root_dir}{path} > /dev/null"
+                Logger.os(f"\tCreate file {path}...")
+                self.__sudo(cmd, shell=True)
+            if ("chroot" in step):
+                self.__chroot(step["chroot"], stdout=subprocess.DEVNULL)
+            if ("action" in step):
+                action = step["action"]
+                for act in self.actions:
+                    if (act[0] == action):
+                        act[1]()
+                        break
+
+    def check_rootfs(self):
+        if marker_check(MARKER_ROOTFS_READY):
+            return
+        stages = [
+            [self.st3_info,    self.__stage3_apply, ""                    ],
+            [self.st3_prepare, self.__stage3_steps, "Basic preparation..."],
+            [self.st3_update,  self.__stage3_steps, "System update..."    ],
+        ]
+        for st in stages:
+            if (not marker_check(st[0]["marker"])):
+                st[1](st[0], st[2])
+                marker_set(st[0]["marker"])
 
     def set_board(self, board):
         self.board = board
         self.arch = board.parse_variables("%{ARCH}%")
 
-    def __sudo(self, args, cwd=None, env=None, stdout=None):
+    def __sudo(self, args, cwd=None, env=None, stdout=None, shell=None):
+        if isinstance(args, str):
+            args = "sudo " + args
+            err_n = args
+        else:
             args.insert(0, "sudo")
-        p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout)
-        if (p.wait() != 0):
-            Logger.error(f"Command '{args[1]}' finished with error code!")
+            err_n = args[1]
+        p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout, shell=shell)
+        p.wait()
+        if (p.returncode != 0):
+            Logger.error(f"Command '{err_n}' finished with error code {p.returncode}!")
 
     def __prepare(self):
         qemu_f = Path(f"/proc/sys/fs/binfmt_misc/qemu-{self.arch}")
@@ -60,12 +122,12 @@ class OS:
             self.__sudo(["python", os.path.abspath(__file__), self.arch])
         self.__sudo(["cp", f"{ROOT_DIR}/files/qemu/qemu-{self.arch}", f"{self.root_dir}/bin/"])
 
-    def __chroot(self, command, dir=""):
+    def __chroot(self, command, dir="", stdout=None):
         self.__prepare()
         if (dir == ""):
             dir = self.root_dir
         Logger.os(f"Start chroot'ed command '{command}' into '{dir}'")
-        self.__sudo(["bash", f"{ROOT_DIR}/scripts/chroot.sh", dir, ROOT_DIR, command])
+        self.__sudo(["bash", f"{ROOT_DIR}/scripts/chroot.sh", dir, ROOT_DIR, command], stdout=stdout)
 
     def umount_safe(self):
         self.__sudo(["umount", "--all-targets", "--recursive", self.root_dir])
@@ -116,7 +178,7 @@ class OS:
         self.__sudo(["chmod", "u+s", f"{self.root_dir}/usr/bin/Xorg"])
 
     def  __tmp_clean(self, path):
-        Logger.os("Clean temporary directory...")
+        Logger.os(f"Clean directory '{path}'...")
         t_dir = Path(path)
         if (t_dir.is_dir()):
             self.__sudo(["rm", "-rf", path])