[openwrt/openwrt] mikrotik: switch to Yafut for building MikroTik NOR images

Wed Jun 5 08:03:30 PDT 2024

robimarko pushed a commit to openwrt/openwrt.git, branch main:
https://git.openwrt.org/a60721f2ed3b0152821d2a82e5e7ed115a8ae9cc

commit a60721f2ed3b0152821d2a82e5e7ed115a8ae9cc
Author: Michał Kępień <openwrt at kempniu.pl>
AuthorDate: Mon May 13 19:26:15 2024 +0200

    mikrotik: switch to Yafut for building MikroTik NOR images
    
    The Yafut tool now has limited capabilities for working on filesystem
    images stored in regular files.  This enables preparing Yaffs2 images
    for devices with NOR flash using upstream Yaffs2 filesystem code instead
    of the custom kernel2minor tool.
    
    Since minimizing the size of the resulting filesystem image size is
    important and upstream Yaffs2 code requires two allocator reserve blocks
    to be available when writing a file to the filesystem, a trick is
    employed while preparing an OpenWRT image: the blank filesystem image
    that Yafut operates on initially contains two extra erase blocks that
    are chopped off after the kernel file is written.  This is safe to do
    because Yaffs2 has a true log structure and therefore only ever writes
    sequentially (and the size of the kernel file is known beforehand).
    While the two extra erase blocks are necessary for writes, Yaffs2 code
    seems to be perfectly capable of reading back files from a "truncated"
    filesystem that does not contain these extra erase blocks.
    
    In terms of image size, this new approach is only marginally worse than
    the current kernel2minor-based one: specifically, upstream Yaffs2 code
    needs to write three object headers (each of which takes up an entire
    data chunk) when the kernel file is written to the filesystem:
    
      - an object header for the kernel file when it is created,
    
      - an object header for the root directory when the kernel file is
        created,
    
      - an updated object header for the kernel file when the latter is
        fully written (so that its new size can be recorded).
    
    kernel2minor only writes two of these headers, which is the absolute
    minimum required for reading the file back.  This means that the
    Yafut-based approach causes firmware images to be at most one erase
    block (64 kB) larger than those created using kernel2minor, but only in
    the very unfortunate scenario where the size of the kernel file is
    really close to a multiple of the erase block size.
    
    The rest of the calculations performed when the empty filesystem image
    is first prepared stems from the Yaffs2 layout used by MikroTik NOR
    devices: each 65,536-byte erase block contains 63 chunks, each of which
    consists of 1024 bytes of data followed by 16-byte Yaffs tags without
    ECC data; each such group of 63 chunks is then followed by 16 bytes of
    padding, which translates to "-C 1040 -B 64k -E" in the Yafut
    invocation.  Yaffs2 checkpoints and summaries are disabled (using
    Yafut's -P and -S switches, respectively) as they are merely performance
    optimizations that require extra storage space.  The -L and -M switches
    are used to force little-endian or big-endian byte order (respectively)
    in the resulting filesystem image, no matter what byte order the build
    host uses.  The tr invocation is used to ensure that the filesystem
    image is initialized with 0xFF bytes (which are an indicator of unused
    space for Yaffs2 code).
    
    Signed-off-by: Michał Kępień <openwrt at kempniu.pl>
    Link: https://github.com/openwrt/openwrt/pull/13453
    Signed-off-by: Robert Marko <robimarko at gmail.com>
---
 include/image-commands.mk                   | 14 ++++++++++++++
 target/linux/ath79/image/common-mikrotik.mk |  2 +-
 target/linux/ipq40xx/image/mikrotik.mk      |  2 +-
 target/linux/ramips/image/mt7621.mk         |  2 +-
 tools/Makefile                              |  2 ++
 5 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/include/image-commands.mk b/include/image-commands.mk
index 83ecf7c520..f2c3f76f91 100644
--- a/include/image-commands.mk
+++ b/include/image-commands.mk
@@ -438,6 +438,20 @@ define Build/kernel2minor
 	rm -f $(temp_file)
 endef
 
+define Build/yaffs-filesystem
+	let \
+		kernel_size="$$(stat -c%s $@)" \
+		kernel_chunks="(kernel_size / 1024) + 1" \
+		filesystem_chunks="kernel_chunks + 3" \
+		filesystem_blocks="(filesystem_chunks / 63) + 1" \
+		filesystem_size="filesystem_blocks * 64 * 1024" \
+		filesystem_size_with_reserve="(filesystem_blocks + 2) * 64 * 1024"; \
+		head -c $$filesystem_size_with_reserve /dev/zero | tr "\000" "\377" > $@.img \
+		&& yafut -d $@.img -w -i $@ -o kernel -C 1040 -B 64k -E -P -S $(1) \
+		&& truncate -s $$filesystem_size $@.img \
+		&& mv $@.img $@
+endef
+
 define Build/kernel-bin
 	rm -f $@
 	cp $< $@
diff --git a/target/linux/ath79/image/common-mikrotik.mk b/target/linux/ath79/image/common-mikrotik.mk
index b37c8b7197..94c29d8cb6 100644
--- a/target/linux/ath79/image/common-mikrotik.mk
+++ b/target/linux/ath79/image/common-mikrotik.mk
@@ -10,7 +10,7 @@ endef
 define Device/mikrotik_nor
   $(Device/mikrotik)
   DEVICE_PACKAGES := -yafut
-  IMAGE/sysupgrade.bin := append-kernel | kernel2minor -s 1024 -e | \
+  IMAGE/sysupgrade.bin := append-kernel | yaffs-filesystem -M | \
 	pad-to $$$$(BLOCKSIZE) | append-rootfs | pad-rootfs | \
 	check-size | append-metadata
 endef
diff --git a/target/linux/ipq40xx/image/mikrotik.mk b/target/linux/ipq40xx/image/mikrotik.mk
index f0e1f1aad3..8c2c6fa08e 100644
--- a/target/linux/ipq40xx/image/mikrotik.mk
+++ b/target/linux/ipq40xx/image/mikrotik.mk
@@ -5,7 +5,7 @@ define Device/mikrotik_nor
 	KERNEL_NAME := vmlinux
 	KERNEL := kernel-bin | append-dtb-elf
 	IMAGES = sysupgrade.bin
-	IMAGE/sysupgrade.bin := append-kernel | kernel2minor -s 1024 | \
+	IMAGE/sysupgrade.bin := append-kernel | yaffs-filesystem -L | \
 		pad-to $$$$(BLOCKSIZE) | append-rootfs | pad-rootfs | \
 		check-size | append-metadata
 endef
diff --git a/target/linux/ramips/image/mt7621.mk b/target/linux/ramips/image/mt7621.mk
index 8886456bbb..0643cd29f4 100644
--- a/target/linux/ramips/image/mt7621.mk
+++ b/target/linux/ramips/image/mt7621.mk
@@ -1770,7 +1770,7 @@ define Device/MikroTik
   DEVICE_PACKAGES := kmod-usb3 -uboot-envtools
   KERNEL_NAME := vmlinuz
   KERNEL := kernel-bin | append-dtb-elf
-  IMAGE/sysupgrade.bin := append-kernel | kernel2minor -s 1024 | \
+  IMAGE/sysupgrade.bin := append-kernel | yaffs-filesystem -L | \
 	pad-to $$$$(BLOCKSIZE) | append-rootfs | pad-rootfs | check-size | \
 	append-metadata
 endef
diff --git a/tools/Makefile b/tools/Makefile
index f60a458f98..f4e749d710 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -83,6 +83,7 @@ tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_TARGET_mxs),y) += elftosb sdim
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_TARGET_realtek),y) += 7z
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_TARGET_tegra),y) += cbootimage cbootimage-configs
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USES_MINOR),y) += kernel2minor
+tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USES_MINOR),y) += yafut
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USE_SPARSE),y) += sparse
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USE_LLVM_BUILD),y) += llvm-bpf
 tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USE_MOLD),y) += mold
@@ -130,6 +131,7 @@ $(curdir)/sdcc/compile := $(curdir)/bison/compile
 $(curdir)/squashfs3-lzma/compile := $(curdir)/lzma-old/compile
 $(curdir)/squashfs4/compile := $(curdir)/xz/compile $(curdir)/zlib/compile
 $(curdir)/util-linux/compile := $(curdir)/bison/compile
+$(curdir)/yafut/compile := $(curdir)/cmake/compile
 
 ifneq ($(HOST_OS),Linux)
   $(curdir)/coreutils/compile += $(curdir)/automake/compile $(curdir)/bison/compile $(curdir)/gnulib/compile


