diff --git a/build.py b/build.py index 58b1b30..cba2a34 100755 --- a/build.py +++ b/build.py @@ -22,6 +22,7 @@ if (args.board == ''): targets_meta = Target.load_meta(f"config/target_meta.json") target_board = Board(args.board, f"config/board/{args.board}.json", targets_meta) os.set_board(target_board) +os.load_info() os.check_rootfs() diff --git a/config/os_aarch64.json b/config/os_aarch64.json new file mode 100644 index 0000000..dca31ad --- /dev/null +++ b/config/os_aarch64.json @@ -0,0 +1,170 @@ +{ + "stage3_info": + { + "marker": "stage3_extracted", + "server_dir": "https://distfiles.gentoo.org/releases/arm64/autobuilds/current-stage3-arm64-systemd/", + "info_file": "latest-stage3-arm64-systemd.txt" + }, + "prepare": + { + "marker": "stage3_prepared", + "steps": + [ + { + "file": "/etc/portage/modules", + "append": false, + "lines": [ "portdbapi.auxdbmodule = portage.cache.sqlite.database" ] + }, + { + "file": "/etc/portage/make.conf", + "append": false, + "lines": + [ + "COMMON_FLAGS=\"-O3 -pipe\"", + "CFLAGS=\"${COMMON_FLAGS}\"", + "CXXFLAGS=\"${COMMON_FLAGS}\"", + "FCFLAGS=\"${COMMON_FLAGS}\"", + "FFLAGS=\"${COMMON_FLAGS}\"", + "CHOST=\"aarch64-unknown-linux-gnu\"", + "", + "EMERGE_DEFAULT_OPTS=\"--load-average 8.0\"", + "USE=\"${USE} ${ARCH}", + " -handbook -compiler-rt -sanitize -gtk-doc -gtk -jit", + " -vulkan -cups -wayland -opengl -egl -eglfs -gles2 -kms", + " -vala -spell -tk -tcl -fortran", + " -slang -gpg -doc -test lto -nls", + " sqlite\"", + "FEATURES=\"${FEATURES} metadata-transfer nodoc noinfo noman -pid-sandbox\"", + "", + "VIDEO_CARDS=\"panfrost fbdev\"", + "INPUT_DEVICES=\"libinput evdev\"", + "LLVM_TARGETS=\"\"", + "PYTHON_TARGETS=\"python3_13\"", + "PYTHON_SINGLE_TARGET=\"${PYTHON_TARGETS}\"", + "", + "LC_MESSAGES=C.utf8" + ] + }, + { + "file": "/etc/portage/package.accept_keywords/all", + "append": false, + "lines": [ "*/* ~arm64" ] + }, + { + "file": "/etc/portage/package.mask/python", + "append": false, + "lines": [ ">dev-lang/python-3.13.999" ] + }, + { + "file": "/etc/portage/repos.conf/andreil.conf", + "append": false, + "lines": + [ + "[andreil]", + "location = /usr/portage/andreil" + ] + }, + { + "file": "/etc/portage/savedconfig/sys-kernel/linux-firmware", + "append": false, + "lines": [ "regulatory.db" ] + }, + { + "file": "/etc/portage/package.use/test", + "append": false, + "lines": + [ + "net-misc/networkmanager -bluetooth -modemmanager", + "net-misc/networkmanager -ppp" + ] + }, + { + "file": "/etc/portage/package.use/system", + "append": false, + "lines": + [ + "sys-kernel/linux-firmware savedconfig", + "sys-kernel/genkernel -firmware", + "net-misc/networkmanager tools", + "sys-apps/util-linux static-libs" + ] + }, + { + "file": "/etc/portage/package.use/minimal", + "append": false, + "lines": + [ + "sys-apps/portage -rsync-verify", + "app-admin/sudo -sendmail", + "sys-apps/systemd -dns-over-tls -gcrypt -kernel-install", + "media-libs/mesa -llvm", + "sys-process/htop lm-sensors" + ] + }, + { + "file": "/etc/portage/package.use/klipper", + "append": false, + "lines": + [ + "virtual/klipper mainsail", + "app-misc/mime-types nginx", + "media-libs/mesa -proprietary-codecs", + "x11-base/xorg-server minimal", + "sys-apps/iproute2 minimal", + "dev-perl/* minimal", + "dev-lang/perl dev-perl", + "sys-apps/systemd policykit", + "x11-libs/cairo X", + "dev-python/pycairo X", + "media-libs/libepoxy -egl", + "net-wireless/wpa_supplicant dbus", + "#x11-base/xorg-server suid", + "cross-arm-none-eabi/newlib nano", + "media-libs/libepoxy egl", + "net-misc/networkmanager -tools", + "media-libs/libglvnd X", + "media-libs/libv4l bpf", + "media-video/ffmpeg x264 x265", + "#media-libs/libv4l -bpf", + "sys-devel/gcc fortran", + "dev-python/pillow webp", + "virtual/imagemagick-tools jpeg tiff", + "media-gfx/imagemagick jpeg tiff", + "sci-libs/atlas lapack threads", + "dev-lang/rust system-llvm", + "sys-apps/busybox static -pam savedconfig", + "virtual/libcrypt static-libs", + "sys-libs/libxcrypt static-libs", + "sys-fs/e2fsprogs static-libs" + ] + }, + { + "chroot": "emerge-webrsync" + } + ] + }, + "update": + { + "marker": "stage3_updated", + "steps": + [ + { + "file": "/etc/locale.gen", + "append": false, + "lines": [ "en_US.UTF-8 UTF-8" ] + }, + { + "chroot": "locale-gen" + }, + { + "chroot": "eselect kernel set 1" + }, + { + "chroot": "eselect news read" + }, + { + "action": "update" + } + ] + } +} diff --git a/scripts/chroot.sh b/scripts/chroot.sh index ea9ffb2..165ecfa 100644 --- a/scripts/chroot.sh +++ b/scripts/chroot.sh @@ -3,6 +3,7 @@ OS_DIR_DEF="./root/" DDIR=$(realpath "$1") ROOT_DIR="$2" +KV=$(make -C ./build/common/kernel/ --silent kernelversion) if [ -z "${DDIR}" ] then @@ -12,29 +13,25 @@ fi mkdir -p ${DDIR}/usr/portage mount --bind ${ROOT_DIR}/files/portage ${DDIR}/usr/portage -mkdir -p ${DDIR}/usr/src/linux-6.14-rc7 -mount --bind ${ROOT_DIR}/build/common/kernel ${DDIR}/usr/src/linux-6.14-rc7 +mkdir -p ${DDIR}/usr/src/linux-${KV} +mount --bind ${ROOT_DIR}/build/common/kernel ${DDIR}/usr/src/linux-${KV} mount --bind /dev ${DDIR}/dev mount --bind /dev/shm ${DDIR}/dev/shm mount --bind /dev/pts ${DDIR}/dev/pts mount --bind /sys ${DDIR}/sys mount --bind /proc ${DDIR}/proc -if [ -d "/var/db/repos" ]; then - mount --bind /var/db/repos ${DDIR}/var/db/repos -fi mount -t tmpfs tmpfs ${DDIR}/var/tmp/ if [ -z "$3" ] then chroot ${DDIR}/ /bin/bash + ret=$? else chroot ${DDIR}/ /bin/bash -c "${@:3}" + ret=$? fi umount ${DDIR}/var/tmp -umount ${DDIR}/usr/src/linux-6.14-rc7 -if [ -d "/var/db/repos" ]; then - umount ${DDIR}/var/db/repos -fi +umount ${DDIR}/usr/src/linux-${KV} umount ${DDIR}/proc umount ${DDIR}/sys umount ${DDIR}/dev/pts @@ -44,3 +41,4 @@ if [ -n "$1" ] then umount ${DDIR}/usr/portage fi +exit ${ret} diff --git a/scripts/lib.py b/scripts/lib.py index 29c0847..0056dd6 100644 --- a/scripts/lib.py +++ b/scripts/lib.py @@ -2,3 +2,14 @@ import os from pathlib import Path ROOT_DIR=Path(os.path.abspath(__file__)).parent.parent + +def marker_check(name): + fn = f"{ROOT_DIR}/build/.{name}_marker" + marker = Path(fn) + if (marker.is_file()): + return True + return False + +def marker_set(name): + fn = f"{ROOT_DIR}/build/.{name}_marker" + Path(fn).touch() diff --git a/scripts/os.py b/scripts/os.py index 8a82346..f74069a 100644 --- a/scripts/os.py +++ b/scripts/os.py @@ -4,6 +4,7 @@ if __name__ != '__main__': from . import * units = { "B": 1, "K": 2**10, "M": 2**20, "G": 2**30 } +MARKER_ROOTFS_READY = ".rootfs_ready" class Partition(object): pass @@ -21,38 +22,99 @@ class OS: [ "sqh", self.sqh ] ] + def load_info(self): + with open(f"{ROOT_DIR}/config/os_{self.arch}.json") as json_data: + js_data = json.load(json_data) + json_data.close() + self.st3_info = js_data["stage3_info"] + self.st3_prepare = js_data["prepare"] + self.st3_update = js_data["update"] + def actions_list(self): lst = [] for act in self.actions: lst.append(act[0]) return lst + def __get_stage3_url(self): + url_descr = self.st3_info["server_dir"] + self.st3_info["info_file"] + r = requests.get(url_descr, stream=True) + descr = r.content.decode('utf-8').splitlines() + stage3_fn = "" + for d in descr: + if (d.startswith("stage3")): + stage3_fn = d.split()[0] + arch_url = self.st3_info["server_dir"] + stage3_fn + return [arch_url, stage3_fn] + + def __stage3_apply(self, info, text): + self.__tmp_clean(f"{ROOT_DIR}/root") + [url,fn] = self.__get_stage3_url() + Logger.os(f"Download Stage3 archive '{fn}'...") + temp_dir = f"{ROOT_DIR}/build/tmp" + self.__tmp_clean(temp_dir) + os.makedirs(temp_dir, exist_ok=True) + r = requests.get(url, stream=True) + arch_fn = f"{temp_dir}/{fn}" + with open(arch_fn, 'wb') as f: + total_length = int(r.headers.get('content-length')) + for chunk in r.iter_content(chunk_size=1024): + f.write(chunk) + Logger.os(f"Extract Stage3 archive...") + os.makedirs(self.root_dir, exist_ok=True) + self.__extract_tar(arch_fn, self.root_dir) + self.__tmp_clean(temp_dir) + + def __stage3_steps(self, info, text): + Logger.os(text) + self.__sudo(["cp", "/etc/resolv.conf", f"{self.root_dir}/etc/resolv.conf"]) + for step in info["steps"]: + if ("file" in step): + is_append = "-a" if step["append"] else "" + lines = "\n".join(step["lines"]) + path = step["file"] + directory = Path(path).parent + cmd = f"mkdir -p {self.root_dir}{directory} && echo '{lines}'" + cmd += f" | sudo tee {is_append} {self.root_dir}{path} > /dev/null" + Logger.os(f"\tCreate file {path}...") + self.__sudo(cmd, shell=True) + if ("chroot" in step): + self.__chroot(step["chroot"], stdout=subprocess.DEVNULL) + if ("action" in step): + action = step["action"] + for act in self.actions: + if (act[0] == action): + act[1]() + break + def check_rootfs(self): - root_marker = Path(self.root_dir) - if (not root_marker.is_dir()): - Logger.os(f"Download base OS rootfs archive...") - temp_dir = f"{ROOT_DIR}/build/tmp" - os.makedirs(temp_dir, exist_ok=True) - r = requests.get("https://cloud.andreil.by/public.php/dav/files/sbc-rootfs-archive", stream=True) - arch_fn = f"{temp_dir}/sbc_rootfs_archive.tar.xz" - with open(arch_fn, 'wb') as f: - total_length = int(r.headers.get('content-length')) - for chunk in r.iter_content(chunk_size=1024): - f.write(chunk) - os.makedirs(self.root_dir, exist_ok=True) - self.__extract_tar(arch_fn, self.root_dir) - self.__tmp_clean(temp_dir) - self.__chroot("eix-sync -v") + if marker_check(MARKER_ROOTFS_READY): + return + stages = [ + [self.st3_info, self.__stage3_apply, "" ], + [self.st3_prepare, self.__stage3_steps, "Basic preparation..."], + [self.st3_update, self.__stage3_steps, "System update..." ], + ] + for st in stages: + if (not marker_check(st[0]["marker"])): + st[1](st[0], st[2]) + marker_set(st[0]["marker"]) def set_board(self, board): self.board = board self.arch = board.parse_variables("%{ARCH}%") - def __sudo(self, args, cwd=None, env=None, stdout=None): - args.insert(0, "sudo") - p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout) - if (p.wait() != 0): - Logger.error(f"Command '{args[1]}' finished with error code!") + def __sudo(self, args, cwd=None, env=None, stdout=None, shell=None): + if isinstance(args, str): + args = "sudo " + args + err_n = args + else: + args.insert(0, "sudo") + err_n = args[1] + p = subprocess.Popen(args, cwd=cwd, env=env, stdout=stdout, stderr=stdout, shell=shell) + p.wait() + if (p.returncode != 0): + Logger.error(f"Command '{err_n}' finished with error code {p.returncode}!") def __prepare(self): qemu_f = Path(f"/proc/sys/fs/binfmt_misc/qemu-{self.arch}") @@ -60,12 +122,12 @@ class OS: self.__sudo(["python", os.path.abspath(__file__), self.arch]) self.__sudo(["cp", f"{ROOT_DIR}/files/qemu/qemu-{self.arch}", f"{self.root_dir}/bin/"]) - def __chroot(self, command, dir=""): + def __chroot(self, command, dir="", stdout=None): self.__prepare() if (dir == ""): dir = self.root_dir Logger.os(f"Start chroot'ed command '{command}' into '{dir}'") - self.__sudo(["bash", f"{ROOT_DIR}/scripts/chroot.sh", dir, ROOT_DIR, command]) + self.__sudo(["bash", f"{ROOT_DIR}/scripts/chroot.sh", dir, ROOT_DIR, command], stdout=stdout) def umount_safe(self): self.__sudo(["umount", "--all-targets", "--recursive", self.root_dir]) @@ -116,7 +178,7 @@ class OS: self.__sudo(["chmod", "u+s", f"{self.root_dir}/usr/bin/Xorg"]) def __tmp_clean(self, path): - Logger.os("Clean temporary directory...") + Logger.os(f"Clean directory '{path}'...") t_dir = Path(path) if (t_dir.is_dir()): self.__sudo(["rm", "-rf", path])