Add final steps for deployment.

config/os_aarch64.json

@@ -1,4 +1,8 @@
 {
+    "variables":
+    [
+        "USER_LOGIN:klipper"
+    ],
     "stage3_info":
     {
         "marker": "stage3_extracted",
@@ -64,6 +68,15 @@
                     "location = /usr/portage/andreil"
                 ]
             },
+            {
+                "file": "/etc/portage/repos.conf/crossdev.conf",
+                "append": false,
+                "lines":
+                [
+                    "[crossdev]",
+                    "location = /usr/portage/crossdev"
+                ]
+            },
             {
                 "file": "/etc/portage/savedconfig/sys-kernel/linux-firmware",
                 "append": false,
@@ -138,9 +151,7 @@
                     "sys-fs/e2fsprogs static-libs"
                 ]
             },
-            {
+            { "chroot": "emerge-webrsync" }
-                "chroot": "emerge-webrsync"
-            }
         ]
     },
     "update":
@@ -153,18 +164,52 @@
                 "append": false,
                 "lines": [ "en_US.UTF-8 UTF-8" ]
             },
+            { "chroot": "locale-gen" },
+            { "chroot": "eselect kernel set 1" },
+            { "chroot": "eselect news read" },
+            { "chroot": "eselect editor set nano" },
+            { "action": "update" }
+        ]
+    },
+    "install":
+    {
+        "marker": "stage3_installed",
+        "steps":
+        [
             {
-                "chroot": "locale-gen"
+                "soft_inst":
+                [
+                    "app-editors/nano",
+                    "app-misc/mc",
+                    "app-portage/eix",
+                    "app-portage/gentoolkit",
+                    "sys-devel/crossdev",
+                    "sys-process/htop",
+                    "virtual/klipper"
+                ],
+                "oneshot": false
             },
-            {
+            { "chroot": "crossdev -s4 arm-none-eabi" },
-                "chroot": "eselect kernel set 1"
+            { "chroot": "eselect news read" },
-            },
+            { "soft_clean": "default" }
-            {
+        ]
-                "chroot": "eselect news read"
+    },
-            },
+    "finalize":
-            {
+    {
-                "action": "update"
+        "steps":
-            }
+        [
+            { "chroot": "systemctl enable NetworkManager ntpdate sshd" },
+            { "sudo": "sed -i -E 's/^# (%wheel ALL)/\\1/' ./etc/sudoers" },
+            { "sudo": "sed -i -E 's/^#(\\S+MaxUse)=$/\\1=10M/' ./etc/systemd/journald.conf" },
+            { "sudo": "sed -i -E 's/^#(\\S+MaxFileSize)=$/\\1=10M/' ./etc/systemd/journald.conf" },
+            { "copy": [ "%{ROOT_DIR}%/files/firmware/usr", "."] },
+            { "sudo": "chmod u+s ./usr/bin/Xorg" },
+            { "sudo": "ln -sf /usr/share/zoneinfo/Europe/Warsaw ./etc/localtime" },
+            { "chroot": "useradd -m -G wheel,video,audio,disk,usb %{USER_LOGIN}% --password %{USER_LOGIN}%" },
+            { "chroot": "echo '%{USER_LOGIN}%:%{USER_LOGIN}%' | chpasswd" },
+            { "chroot": "echo 'root:root' | chpasswd" },
+            { "chroot": "sudo -i -u klipper python -m venv /home/%{USER_LOGIN}%/venv" },
+            { "soft_clean": "bdeps" }
         ]
     }
 }

files/backups/excl_min.lst

@@ -6,6 +6,7 @@ usr/lib64/perl5*
 #usr/src/*
 var/cache/binpkgs/*
 var/cache/distfiles/*
-var/cache/edb/*
+#var/cache/edb/*
 var/cache/eix/*
 var/log/*.log
+#var/db/repos/*

files/portage/andreil/virtual/klipper/Manifest

@@ -1 +1 @@
-EBUILD klipper-11.ebuild 603 BLAKE2B fe8ca3d46a257c6c056f8a2480afdb5270b3e370495d2fad3ebbf7b066b456c2b365c2e83c3c6d42a81018f6bbc2f15cb36be502fff3589343620e844cc21d41 SHA512 14dad1041e1ecaf645bb6c508089d292bc30cf644c656c127e8299c67e1866f5cafe016af8794c2a81f26f7d405c1ff558e9aca1e8c43a9b595ebe63c5b977fd
+EBUILD klipper-11.ebuild 636 BLAKE2B fe1685f4bfbba06f544f6bb0a01cbaca30f938dc8963bb2f01f7b57c3643738caa9403049916319eff3f626d1507e7577762188bd03ada31a4e6ea04ce763abb SHA512 381bb6899bfe71456e0148abfeedf8db1dccb5b5d4998068e9eb03a8d0008e9b39ea671e82f1b8afc4aba6527cba801f59d824c2800657d4d96e5f7a12235c49

files/portage/andreil/virtual/klipper/klipper-11.ebuild

@@ -26,4 +26,6 @@ RDEPEND="
 	dev-python/matplotlib
 	sys-devel/crossdev
 	sys-apps/pciutils
+	media-libs/libv4l
+	net-misc/ntp
 "

scripts/board.py

@@ -1,4 +1,4 @@
-import json, os
+import json, os, re
 from pathlib import Path
 from . import *
 
@@ -47,18 +47,27 @@ class Board:
         return 0
 
     def __load_vars(self):
-        for var_def in self.json["variables"]:
+        self.add_vars(self.json["variables"])
-            self.variables.append(var_def.split(":"))
         self.variables.append(["board_name", self.name])
         self.variables.append(["build_dir", f"{ROOT_DIR}/build/{self.name}"])
         self.variables.append(["common_dir", f"{ROOT_DIR}/build/common"])
         self.variables.append(["out_dir", self.out_dir])
         self.variables.append(["out_sh", self.out_sh])
+        self.variables.append(["ROOT_DIR", ROOT_DIR])
+
+    def add_var(self, name, val):
+        self.variables.append([name, val])
+
+    def add_vars(self, lst):
+        for var_def in lst:
+            self.variables.append(var_def.split(":"))
 
     def parse_variables(self, string):
-        for var_d in self.variables:
+        while True:
-            string = string.replace("%{"+var_d[0]+"}%", var_d[1])
+            for var_d in self.variables:
-        #out_dir
+                string = string.replace("%{"+var_d[0]+"}%", str(var_d[1]))
+            if not re.compile('%{\\S+}%').match(string):
+                break
         return string
 
     def sync(self):

scripts/chroot.sh

@@ -3,13 +3,18 @@
 OS_DIR_DEF="./root/"
 DDIR=$(realpath "$1")
 ROOT_DIR="$2"
-KV=$(make -C ./build/common/kernel/ --silent kernelversion)
 
 if [ -z "${DDIR}" ]
 then
-    echo "No directory specified!"
+    echo "No root directory specified!"
     exit 1
 fi
+if [ -z "${ROOT_DIR}" ]
+then
+    echo "No main directory specified!"
+    exit 1
+fi
+KV=$(make -C "${ROOT_DIR}/build/common/kernel/" --silent kernelversion)
 
 mkdir -p ${DDIR}/usr/portage
 mount --bind ${ROOT_DIR}/files/portage ${DDIR}/usr/portage

scripts/os.py

@@ -4,7 +4,7 @@ if __name__ != '__main__':
     from . import *
 
 units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 }
-MARKER_ROOTFS_READY = ".rootfs_ready"
+MARKER_ROOTFS_READY = "rootfs_ready"
 
 class Partition(object):
     pass
@@ -29,6 +29,10 @@ class OS:
             self.st3_info = js_data["stage3_info"]
             self.st3_prepare = js_data["prepare"]
             self.st3_update = js_data["update"]
+            self.st3_install = js_data["install"]
+            self.finalize = js_data["finalize"]
+            self.board.add_vars(js_data["variables"])
+            self.board.add_var("ROOT_FS", self.root_dir)
 
     def actions_list(self):
         lst = []
@@ -48,7 +52,7 @@ class OS:
         return [arch_url, stage3_fn]
 
     def __stage3_apply(self, info, text):
-        self.__tmp_clean(f"{ROOT_DIR}/root")
+        self.__tmp_clean(self.root_dir)
         [url,fn] = self.__get_stage3_url()
         Logger.os(f"Download Stage3 archive '{fn}'...")
         temp_dir = f"{ROOT_DIR}/build/tmp"
@@ -65,7 +69,9 @@ class OS:
         self.__extract_tar(arch_fn, self.root_dir)
         self.__tmp_clean(temp_dir)
 
-    def __stage3_steps(self, info, text):
+    def __stage3_steps(self, info, text, dir=""):
+        if (dir == ""):
+            dir = self.root_dir
         Logger.os(text)
         self.__sudo(["cp", "/etc/resolv.conf", f"{self.root_dir}/etc/resolv.conf"])
         for step in info["steps"]:
@@ -77,36 +83,54 @@ class OS:
                 cmd  = f"mkdir -p {self.root_dir}{directory} && echo '{lines}'"
                 cmd += f" | sudo tee {is_append} {self.root_dir}{path} > /dev/null"
                 Logger.os(f"\tCreate file {path}...")
-                self.__sudo(cmd, shell=True)
+                self.__sudo(cmd, shell=True, cwd=dir)
             if ("chroot" in step):
-                self.__chroot(step["chroot"], stdout=subprocess.DEVNULL)
+                cmd = self.board.parse_variables(step["chroot"])
+                self.__chroot(cmd, dir=dir)
             if ("action" in step):
                 action = step["action"]
                 for act in self.actions:
                     if (act[0] == action):
                         act[1]()
                         break
+            if ("soft_inst" in step):
+                sw_list = " ".join(step["soft_inst"])
+                oneshot = "1" if step["oneshot"] else ""
+                self.__chroot(f"emerge -avb{oneshot} {sw_list} -j2", dir=dir)
+            if ("soft_clean" in step):
+                clean_type = step["soft_clean"]
+                if (clean_type == "default"):
+                    self.__chroot(f"emerge -ac", dir=dir)
+                if (clean_type == "bdeps"):
+                    self.__chroot(f"emerge --ask --depclean --with-bdeps=n --exclude sys-devel/gcc && ldconfig", dir=dir)
+            if ("sudo" in step):
+                cmd = self.board.parse_variables(step["sudo"])
+                Logger.os(f"\tSudo command {cmd}...")
+                self.__sudo(cmd, cwd=dir, shell=True)
 
     def check_rootfs(self):
         if marker_check(MARKER_ROOTFS_READY):
             return
         stages = [
-            [self.st3_info,    self.__stage3_apply, ""                    ],
+            [self.st3_info,     self.__stage3_apply, ""                    ],
-            [self.st3_prepare, self.__stage3_steps, "Basic preparation..."],
+            [self.st3_prepare,  self.__stage3_steps, "Basic preparation..."],
-            [self.st3_update,  self.__stage3_steps, "System update..."    ],
+            [self.st3_update,   self.__stage3_steps, "System update..."    ],
+            [self.st3_install,  self.__stage3_steps, "Software installation..."],
         ]
         for st in stages:
             if (not marker_check(st[0]["marker"])):
                 st[1](st[0], st[2])
                 marker_set(st[0]["marker"])
+        marker_set(MARKER_ROOTFS_READY)
 
     def set_board(self, board):
         self.board = board
+        self.board.add_var("ROOTFS", self.root_dir)
         self.arch = board.parse_variables("%{ARCH}%")
 
     def __sudo(self, args, cwd=None, env=None, stdout=None, shell=None):
         if isinstance(args, str):
-            args = "sudo " + args
+            args = self.board.parse_variables("sudo " + args)
             err_n = args
         else:
             args.insert(0, "sudo")
@@ -173,10 +197,6 @@ class OS:
     def pack(self):
         return self.__do_archive("excl_min", "FULL", self.root_dir)
 
-    def __fix_xorg(self):
-        Logger.os("Fix Xorg permissions")
-        self.__sudo(["chmod", "u+s", f"{self.root_dir}/usr/bin/Xorg"])
-
     def  __tmp_clean(self, path):
         Logger.os(f"Clean directory '{path}'...")
         t_dir = Path(path)
@@ -199,66 +219,19 @@ class OS:
         user = getpass.getuser()
         self.__sudo(["chown", user + ":" + user, to_file])
 
-    def __remove_bdeps(self, temp_dir):
-        # remove unneccessarry packages
-        # emerge --ask --depclean --with-bdeps=n
-        #list_to_rm  = " virtual/perl-JSON-PP virtual/perl-podlators"
-        #list_to_rm += " virtual/perl-Getopt-Long virtual/perl-Parse-CPAN-Meta"
-        #list_to_rm += " virtual/perl-ExtUtils-CBuilder virtual/perl-ExtUtils-ParseXS"
-        #list_to_rm += " virtual/perl-Unicode-Collate virtual/perl-Text-ParseWords"
-        #list_to_rm += " virtual/perl-ExtUtils-MakeMaker virtual/perl-Module-Metadata"
-        #list_to_rm += " virtual/perl-version virtual/perl-CPAN-Meta"
-        #list_to_rm += " virtual/perl-File-Spec perl-core/Getopt-Long"
-        #list_to_rm += " dev-perl/Module-Build"
-        #list_to_rm += " x11-base/xcb-proto x11-libs/xtrans"
-        #list_to_rm += " app-alternatives/ninja app-eselect/eselect-rust"
-        #list_to_rm += " dev-libs/vala-common dev-util/glib-utils"
-        #list_to_rm += " dev-util/gdbus-codegen media-fonts/font-util"
-        #list_to_rm += " dev-libs/libxslt"
-        #list_to_rm += " dev-build/gtk-doc-am sys-apps/help2man"
-        #list_to_rm += " app-text/docbook-xml-dtd:4.1.2"
-        #list_to_rm += " app-text/docbook-xml-dtd:4.2 app-text/docbook-xml-dtd:4.3"
-        #list_to_rm += " app-text/docbook-xml-dtd:4.4 app-text/docbook-xml-dtd:4.5"
-        #list_to_rm += " app-text/docbook-xsl-ns-stylesheets app-text/docbook-xsl-stylesheets"
-        #list_to_rm += " app-text/build-docbook-catalog app-text/xmlto"
-        #list_to_rm += " app-text/asciidoc app-text/sgml-common"
-        #list_to_rm += " dev-lang/rust-common dev-lang/rust"
-        #list_to_rm += " llvm-core/llvm llvm-core/llvm-toolchain-symlinks"
-        #list_to_rm += " llvm-core/llvmgold dev-libs/oniguruma"
-        #list_to_rm += " llvm-core/llvm-common sys-libs/binutils-libs"
-        #list_to_rm += " dev-build/ninja dev-build/meson dev-build/meson-format-array"
-        #list_to_rm += " dev-cpp/eigen"
-        #list_to_rm += " "
-        #self.__chroot(f"emerge -aC {list_to_rm} && ldconfig", temp_dir)
-        # remove build deps, exclude gcc/binutils - it's required to build a software after it updates
-        self.__chroot(f"emerge --ask --depclean --with-bdeps=n --exclude sys-devel/gcc && ldconfig", temp_dir)
-        self.__do_archive("excl_min", "FULL_min_bdeps", temp_dir)
-
     def __finalize(self, dir):
-        Logger.os(f"Finalize system installation...")
+        self.__stage3_steps(self.finalize, "Finalize system installation...", dir=dir)
-        # enable network services
-        services = "systemctl enable NetworkManager ntpdate sshd"
-        self.__chroot(services, dir)
-        # minimize systemd journal files
-        journal_min  = "sed -i -E 's/^#(\\S+MaxUse)=$/\\1=10M/' /etc/systemd/journald.conf &&"
-        journal_min += "sed -i -E 's/^#(\\S+MaxFileSize)=$/\\1=10M/' /etc/systemd/journald.conf"
-        self.__chroot(journal_min, dir)
-        self.__sudo(["cp", "-r", f"{ROOT_DIR}/files/firmware/usr", f"{dir}/"])
-        soft = Software(self)
-        soft.finalize(dir)
 
     def sqh(self):
-        self.__fix_xorg()
         date = datetime.datetime.today().strftime('%Y_%m_%d')
         temp_dir = f"{ROOT_DIR}/build/tmp"
         # pack full system via tar
-        arch_full_path = self.pack()
+        #arch_full_path = self.pack()
-        self.__tmp_clean(temp_dir)
+        #self.__tmp_clean(temp_dir)
-        self.__extract_tar(arch_full_path, temp_dir)
+        #self.__extract_tar(arch_full_path, temp_dir)
-        # remove unnecessary packages
+        # prepare system, remove unnecessary packages
-        self.__remove_bdeps(temp_dir)
-        # prepare system
         self.__finalize(temp_dir)
+        self.__do_archive("excl_min", "FULL_min_bdeps", temp_dir)
         # pack a minimal archive
         arch_path = self.__do_archive("excl", "OS", temp_dir)
         # remove temp directory