[FS#1421] "imagebuilder: reuse rootfs preparation from rootfs.mk" breaks TARGET_PER_DEVICE_ROOTFS

Thu Mar 8 20:34:04 PST 2018

A new Flyspray task has been opened.  Details are below. 

User who did this - Stefan Lippers-Hollmann (slh) 

Attached to Project - OpenWrt/LEDE Project
Summary - "imagebuilder: reuse rootfs preparation from rootfs.mk" breaks TARGET_PER_DEVICE_ROOTFS
Task Type - Bug Report
Category - Toolchain
Status - Unconfirmed
Assigned To - 
Operating System - All
Severity - Medium
Priority - Very Low
Reported Version - Trunk
Due in Version - Undecided
Due Date - Undecided
Details - Building LEDE with TARGET_MULTI_PROFILE/ TARGET_PER_DEVICE_ROOTFS set fails to include the selected TARGET_DEVICE_PACKAGES after 2fbf669730bfa7342b36711d275bfe4c7951ac1b "imagebuilder: reuse rootfs preparation from rootfs.mk" has been merged.

$ ./scripts/diffconfig.sh
CONFIG_TARGET_ipq806x=y
CONFIG_TARGET_MULTI_PROFILE=y
CONFIG_TARGET_PER_DEVICE_ROOTFS=y
CONFIG_TARGET_DEVICE_ipq806x_DEVICE_netgear_r7800=y
CONFIG_TARGET_DEVICE_PACKAGES_ipq806x_DEVICE_netgear_r7800="6in4 blockd ca-certificates collectd-mod-conntrack collectd-mod-cpufreq collectd-mod-entropy collectd-mod-ping collectd-mod-thermal collectd-mod-uptime collectd-mod-wireless htop irqbalance kmod-fs-ext4 kmod-fs-f2fs kmod-nls-utf8 kmod-usb2 kmod-usb-printer kmod-usb-storage luci-app-adblock luci-app-bcp38 luci-app-ddns luci-app-p910nd luci-app-sqm luci-app-statistics luci-app-uhttpd luci-app-vnstat luci-app-wol luci-ssl-openssl openssl-util opkg sqm-scripts-extra strongswan-default strongswan-mod-addrblock strongswan-mod-curve25519 strongswan-mod-dhcp strongswan-mod-eap-identity strongswan-mod-eap-md5 strongswan-mod-eap-mschapv2 strongswan-mod-eap-tls strongswan-mod-openssl strongswan-mod-xauth-eap strongswan-ipsec strongswan-pki wpad"
CONFIG_TARGET_DEVICE_ipq806x_DEVICE_zyxel_nbg6817=y
CONFIG_TARGET_DEVICE_PACKAGES_ipq806x_DEVICE_zyxel_nbg6817="6in4 blockd ca-certificates collectd-mod-conntrack collectd-mod-cpufreq collectd-mod-entropy collectd-mod-ping collectd-mod-thermal collectd-mod-uptime collectd-mod-wireless htop irqbalance kmod-fs-ext4 kmod-fs-f2fs kmod-nls-utf8 kmod-usb2 kmod-usb-printer kmod-usb-storage luci-app-adblock luci-app-advanced-reboot luci-app-bcp38 luci-app-ddns luci-app-p910nd luci-app-sqm luci-app-statistics luci-app-uhttpd luci-app-vnstat luci-app-wol luci-ssl-openssl nbg6817-tools openssl-util opkg sqm-scripts-extra strongswan-default strongswan-mod-addrblock strongswan-mod-curve25519 strongswan-mod-dhcp strongswan-mod-eap-identity strongswan-mod-eap-md5 strongswan-mod-eap-mschapv2 strongswan-mod-eap-tls strongswan-mod-openssl strongswan-mod-xauth-eap strongswan-ipsec strongswan-pki uboot-envtools wpad"
CONFIG_TARGET_PER_DEVICE_ROOTFS=y
CONFIG_IMAGEOPT=y
# CONFIG_KERNEL_DEBUG_INFO is not set
# CONFIG_KERNEL_DEBUG_KERNEL is not set
CONFIG_OPENSSL_OPTIMIZE_SPEED=y
CONFIG_OPENSSL_WITH_EC=y
CONFIG_OPENSSL_WITH_NPN=y
CONFIG_OPENSSL_WITH_PSK=y
CONFIG_OPENSSL_WITH_SRP=y
CONFIG_PACKAGE_6in4=m
CONFIG_PACKAGE_adblock=m
CONFIG_PACKAGE_bcp38=m
CONFIG_PACKAGE_block-mount=m
CONFIG_PACKAGE_blockd=m
CONFIG_PACKAGE_ca-certificates=m
CONFIG_PACKAGE_collectd=m
CONFIG_PACKAGE_collectd-mod-conntrack=m
CONFIG_PACKAGE_collectd-mod-cpu=m
CONFIG_PACKAGE_collectd-mod-cpufreq=m
CONFIG_PACKAGE_collectd-mod-entropy=m
CONFIG_PACKAGE_collectd-mod-interface=m
CONFIG_PACKAGE_collectd-mod-iwinfo=m
CONFIG_PACKAGE_collectd-mod-load=m
CONFIG_PACKAGE_collectd-mod-memory=m
CONFIG_PACKAGE_collectd-mod-network=m
CONFIG_PACKAGE_collectd-mod-ping=m
CONFIG_PACKAGE_collectd-mod-rrdtool=m
CONFIG_PACKAGE_collectd-mod-thermal=m
CONFIG_PACKAGE_collectd-mod-uptime=m
CONFIG_PACKAGE_collectd-mod-wireless=m
CONFIG_PACKAGE_ddns-scripts=m
CONFIG_PACKAGE_etherwake=m
CONFIG_PACKAGE_htop=m
CONFIG_PACKAGE_ipset=m
CONFIG_PACKAGE_iptables-mod-conntrack-extra=m
CONFIG_PACKAGE_iptables-mod-ipopt=m
CONFIG_PACKAGE_iptables-mod-ipsec=m
CONFIG_PACKAGE_irqbalance=m
CONFIG_PACKAGE_kmod-crypto-aead=m
CONFIG_PACKAGE_kmod-crypto-authenc=m
CONFIG_PACKAGE_kmod-crypto-cbc=m
CONFIG_PACKAGE_kmod-crypto-crc32=m
CONFIG_PACKAGE_kmod-crypto-deflate=m
CONFIG_PACKAGE_kmod-crypto-des=m
CONFIG_PACKAGE_kmod-crypto-echainiv=m
CONFIG_PACKAGE_kmod-crypto-hmac=m
CONFIG_PACKAGE_kmod-crypto-iv=m
CONFIG_PACKAGE_kmod-crypto-manager=m
CONFIG_PACKAGE_kmod-crypto-md5=m
CONFIG_PACKAGE_kmod-crypto-null=m
CONFIG_PACKAGE_kmod-crypto-pcompress=m
CONFIG_PACKAGE_kmod-crypto-rng=m
CONFIG_PACKAGE_kmod-crypto-sha1=m
CONFIG_PACKAGE_kmod-crypto-sha256=m
CONFIG_PACKAGE_kmod-crypto-wq=m
CONFIG_PACKAGE_kmod-fs-autofs4=m
CONFIG_PACKAGE_kmod-fs-f2fs=m
CONFIG_PACKAGE_kmod-ifb=m
CONFIG_PACKAGE_kmod-ipsec=m
CONFIG_PACKAGE_kmod-ipsec4=m
CONFIG_PACKAGE_kmod-ipsec6=m
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=m
CONFIG_PACKAGE_kmod-ipt-ipopt=m
CONFIG_PACKAGE_kmod-ipt-ipsec=m
CONFIG_PACKAGE_kmod-ipt-ipset=m
CONFIG_PACKAGE_kmod-iptunnel=m
CONFIG_PACKAGE_kmod-iptunnel4=m
CONFIG_PACKAGE_kmod-iptunnel6=m
CONFIG_PACKAGE_kmod-lib-zlib-deflate=m
CONFIG_PACKAGE_kmod-lib-zlib-inflate=m
CONFIG_PACKAGE_kmod-mtd-rw=m
CONFIG_PACKAGE_kmod-nfnetlink=m
CONFIG_PACKAGE_kmod-nls-utf8=m
CONFIG_PACKAGE_kmod-sched-cake=m
CONFIG_PACKAGE_kmod-sched-core=m
CONFIG_PACKAGE_kmod-sit=m
CONFIG_PACKAGE_kmod-usb-ehci=m
CONFIG_PACKAGE_kmod-usb-printer=m
CONFIG_PACKAGE_kmod-usb-storage=m
CONFIG_PACKAGE_kmod-usb2=m
# CONFIG_PACKAGE_lede-keyring is not set
CONFIG_PACKAGE_libgd=m
CONFIG_PACKAGE_libgmp=m
CONFIG_PACKAGE_libipset=m
CONFIG_PACKAGE_libiwinfo-lua=m
CONFIG_PACKAGE_libjpeg=m
CONFIG_PACKAGE_libltdl=m
CONFIG_PACKAGE_liblua=m
CONFIG_PACKAGE_libmnl=m
CONFIG_PACKAGE_libncurses=m
CONFIG_PACKAGE_libopenssl=m
CONFIG_PACKAGE_liboping=m
CONFIG_PACKAGE_libpng=m
CONFIG_PACKAGE_libpthread=m
CONFIG_PACKAGE_librrd1=m
CONFIG_PACKAGE_libubus-lua=m
CONFIG_PACKAGE_libuci-lua=m
CONFIG_PACKAGE_libustream-openssl=m
CONFIG_PACKAGE_lua=m
CONFIG_PACKAGE_luci=m
CONFIG_PACKAGE_luci-app-adblock=m
CONFIG_PACKAGE_luci-app-advanced-reboot=m
CONFIG_PACKAGE_luci-app-bcp38=m
CONFIG_PACKAGE_luci-app-ddns=m
CONFIG_PACKAGE_luci-app-firewall=m
CONFIG_PACKAGE_luci-app-p910nd=m
CONFIG_PACKAGE_luci-app-sqm=m
CONFIG_PACKAGE_luci-app-statistics=m
CONFIG_PACKAGE_luci-app-uhttpd=y
CONFIG_PACKAGE_luci-app-vnstat=m
CONFIG_PACKAGE_luci-app-wol=m
CONFIG_PACKAGE_luci-base=m
CONFIG_PACKAGE_luci-lib-ip=m
CONFIG_PACKAGE_luci-lib-jsonc=m
CONFIG_PACKAGE_luci-lib-nixio=m
CONFIG_PACKAGE_luci-mod-admin-full=m
CONFIG_PACKAGE_luci-proto-ipv6=m
CONFIG_PACKAGE_luci-proto-ppp=m
CONFIG_PACKAGE_luci-ssl-openssl=m
CONFIG_PACKAGE_luci-theme-bootstrap=m
CONFIG_PACKAGE_nbg6817-tools=m
CONFIG_PACKAGE_openssl-util=m
CONFIG_PACKAGE_opkg=m
CONFIG_PACKAGE_p910nd=m
CONFIG_PACKAGE_rpcd=m
CONFIG_PACKAGE_rpcd-mod-rrdns=m
CONFIG_PACKAGE_rrdtool1=m
CONFIG_PACKAGE_sqm-scripts=m
CONFIG_PACKAGE_sqm-scripts-extra=m
CONFIG_PACKAGE_strongswan=m
CONFIG_PACKAGE_strongswan-charon=m
CONFIG_PACKAGE_strongswan-default=m
CONFIG_PACKAGE_strongswan-ipsec=m
CONFIG_PACKAGE_strongswan-libtls=m
CONFIG_PACKAGE_strongswan-mod-addrblock=m
CONFIG_PACKAGE_strongswan-mod-aes=m
CONFIG_PACKAGE_strongswan-mod-attr=m
CONFIG_PACKAGE_strongswan-mod-connmark=m
CONFIG_PACKAGE_strongswan-mod-constraints=m
CONFIG_PACKAGE_strongswan-mod-curve25519=m
CONFIG_PACKAGE_strongswan-mod-des=m
CONFIG_PACKAGE_strongswan-mod-dhcp=m
CONFIG_PACKAGE_strongswan-mod-dnskey=m
CONFIG_PACKAGE_strongswan-mod-eap-identity=m
CONFIG_PACKAGE_strongswan-mod-eap-md5=m
CONFIG_PACKAGE_strongswan-mod-eap-mschapv2=m
CONFIG_PACKAGE_strongswan-mod-eap-tls=m
CONFIG_PACKAGE_strongswan-mod-fips-prf=m
CONFIG_PACKAGE_strongswan-mod-gmp=m
CONFIG_PACKAGE_strongswan-mod-hmac=m
CONFIG_PACKAGE_strongswan-mod-kernel-netlink=m
CONFIG_PACKAGE_strongswan-mod-md4=m
CONFIG_PACKAGE_strongswan-mod-md5=m
CONFIG_PACKAGE_strongswan-mod-nonce=m
CONFIG_PACKAGE_strongswan-mod-openssl=m
CONFIG_PACKAGE_strongswan-mod-pem=m
CONFIG_PACKAGE_strongswan-mod-pgp=m
CONFIG_PACKAGE_strongswan-mod-pkcs1=m
CONFIG_PACKAGE_strongswan-mod-pubkey=m
CONFIG_PACKAGE_strongswan-mod-random=m
CONFIG_PACKAGE_strongswan-mod-rc2=m
CONFIG_PACKAGE_strongswan-mod-resolve=m
CONFIG_PACKAGE_strongswan-mod-revocation=m
CONFIG_PACKAGE_strongswan-mod-sha1=m
CONFIG_PACKAGE_strongswan-mod-sha2=m
CONFIG_PACKAGE_strongswan-mod-socket-default=m
CONFIG_PACKAGE_strongswan-mod-sshkey=m
CONFIG_PACKAGE_strongswan-mod-stroke=m
CONFIG_PACKAGE_strongswan-mod-updown=m
CONFIG_PACKAGE_strongswan-mod-x509=m
CONFIG_PACKAGE_strongswan-mod-xauth-eap=m
CONFIG_PACKAGE_strongswan-mod-xauth-generic=m
CONFIG_PACKAGE_strongswan-mod-xcbc=m
CONFIG_PACKAGE_strongswan-pki=m
CONFIG_PACKAGE_tc=m
CONFIG_PACKAGE_terminfo=m
CONFIG_PACKAGE_uboot-envtools=m
CONFIG_PACKAGE_uhttpd=y
CONFIG_PACKAGE_uhttpd-mod-ubus=m
# CONFIG_PACKAGE_usign is not set
CONFIG_PACKAGE_vnstat=m
CONFIG_PACKAGE_vnstati=m
CONFIG_PACKAGE_wpad=m
CONFIG_PACKAGE_wpad-mini=m
CONFIG_PACKAGE_zlib=m
CONFIG_PREINITOPT=y
# CONFIG_SIGNED_PACKAGES is not set
CONFIG_STRONGSWAN_ROUTING_TABLE="220"
CONFIG_STRONGSWAN_ROUTING_TABLE_PRIO="220"
CONFIG_TARGET_PREINIT_TIMEOUT=5
CONFIG_WPA_SUPPLICANT_INTERNAL=y
CONFIG_ZLIB_OPTIMIZE_SPEED=y
# CONFIG_OPENSSL_WITH_DEPRECATED is not set

The resulting targets/ipq806x/generic/openwrt-ipq806x-zyxel_nbg6817-squashfs-sysupgrade.bin should be around 11 MB, but now newly generated images are only about 5.8 MB in size, missing the enabled TARGET_DEVICE_PACKAGES.

Bisecting LEDE between 606cea8f1b (last known good version) and current HEAD (332b736a3e, known bad) results in this:

$ git bisect good
2fbf669730bfa7342b36711d275bfe4c7951ac1b is the first bad commit
commit 2fbf669730bfa7342b36711d275bfe4c7951ac1b
Author: Matthias Schiffer 
Date:   Tue Mar 6 09:38:29 2018 +0100

    imagebuilder: reuse rootfs preparation from rootfs.mk

    In addition to removing redundant code, this fixes various issues in
    IB-generated images that have been fixed in prepare_rootfs before,
    including better handling of CONFIG_CLEAN_IPKG and enabling of initscripts
    from FILES.

    We also reuse the opkg macro and remove --force-... flags that have been
    removed from rootfs.mk as well.

    Signed-off-by: Matthias Schiffer 

:040000 040000 f7cacc1b459d94d227eb3bd64e072b9c1433bf4a 6e8757234d8adc7b8d281022c9147b800d1898ea M    include
:040000 040000 dadf23de6ee13ada6a2be33764a4361634fdbb71 6b2dae981ed2913cc68a23d8ec638ad56b0938c4 M    package
:100644 100644 84e0aaadc66b9f19e0e4456d8b6d4f181c2557fe a97b2d2155695c3a15f361ea2fe95b977be05959 M    rules.mk
:040000 040000 5fc6f3aefd4451b59a40b7023ebf5984503d9ec7 bae7b1d4032985ef8187219ddbb21c942efad6fb M    target

$ git bisect log
git bisect start
# good: [606cea8f1b2d1017e238890157a431ccf4988071] mt76: update to the latest version
git bisect good 606cea8f1b2d1017e238890157a431ccf4988071
# bad: [332b736a3e86aaaf5ce4efebb999e999bad7b3b1] ebtables: update to latest git 2018-01-17
git bisect bad 332b736a3e86aaaf5ce4efebb999e999bad7b3b1
# good: [6ab50bb10ded770f023e14d63cac2522076f9daf] include/rootfs.mk: retain list of conffiles with CONFIG_CLEAN_IPKG
git bisect good 6ab50bb10ded770f023e14d63cac2522076f9daf
# bad: [359273d7f6e5733b84a263f8d3023e9d4adc7d40] include/package-defaults.mk: fix default Build/Prepare with empty ./src
git bisect bad 359273d7f6e5733b84a263f8d3023e9d4adc7d40
# bad: [2fbf669730bfa7342b36711d275bfe4c7951ac1b] imagebuilder: reuse rootfs preparation from rootfs.mk
git bisect bad 2fbf669730bfa7342b36711d275bfe4c7951ac1b
# good: [cf1c7c0f17a45bba3e7ce1a3cd3f8f7efa7196c4] include/rootfs.mk: pass additional files dir to prepare_rootfs as an argument
git bisect good cf1c7c0f17a45bba3e7ce1a3cd3f8f7efa7196c4
# first bad commit: [2fbf669730bfa7342b36711d275bfe4c7951ac1b] imagebuilder: reuse rootfs preparation from rootfs.mk

Reverting just 2fbf669730bfa7342b36711d275bfe4c7951ac1b results in the expected ~11 MB image again.

More information can be found at the following URL:
https://bugs.openwrt.org/index.php?do=details&task_id=1421