systemd/test/test-functions

#!/usr/bin/env bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin
export PATH

os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
LOOKS_LIKE_DEBIAN=$(source $os_release && [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && echo yes || :)
LOOKS_LIKE_ARCH=$(source $os_release && [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && echo yes || :)
LOOKS_LIKE_SUSE=$(source $os_release && [[ " $ID_LIKE " = *" suse "* ]] && echo yes || :)
KERNEL_VER=${KERNEL_VER-$(uname -r)}
KERNEL_MODS="/lib/modules/$KERNEL_VER/"
QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
[[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
QEMU_MEM="${QEMU_MEM:-512M}"
# Note that defining a different IMAGE_NAME in a test setup script will only result
# in default.img being copied and renamed. It can then be extended by defining
# a test_append_files() function. The $1 parameter will be the root directory.
# To force creating a new image from scratch (eg: to encrypt it), also define
# TEST_FORCE_NEWIMAGE=1 in the test setup script.
IMAGE_NAME=${IMAGE_NAME:-default}
TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
LOOPDEV=

# Decide if we can (and want to) run QEMU with KVM acceleration.
# Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
# check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
# running under KVM. If these conditions are met, enable KVM (and possibly
# nested KVM), otherwise disable it.
if [[ -n "$TEST_NESTED_KVM" || ( -z "$TEST_NO_KVM" && $(systemd-detect-virt -v) != kvm ) ]]; then
    QEMU_KVM=yes
else
    QEMU_KVM=no
fi

if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
    echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
    ROOTLIBDIR=/usr/lib/systemd
fi

# The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath $(dirname "$BASH_SOURCE"))}
TEST_UNITS_DIR="$TEST_BASE_DIR/units"
SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
TOOLS_DIR="$SOURCE_DIR/tools"

# note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
if ! BUILD_DIR=$($TOOLS_DIR/find-build-dir.sh); then
    if [ "$NO_BUILD" ]; then
        BUILD_DIR=$SOURCE_DIR
    else
        echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
        exit 1
    fi
fi

PATH_TO_INIT=$ROOTLIBDIR/systemd
[ "$SYSTEMD_JOURNALD" ] || SYSTEMD_JOURNALD=$(which -a $BUILD_DIR/systemd-journald $ROOTLIBDIR/systemd-journald 2>/dev/null | grep '^/' -m1)
[ "$SYSTEMD_JOURNAL_REMOTE" ] || SYSTEMD_JOURNAL_REMOTE=$(which -a $BUILD_DIR/systemd-journal-remote $ROOTLIBDIR/systemd-journal-remote 2>/dev/null | grep '^/' -m1)
[ "$SYSTEMD" ] || SYSTEMD=$(which -a $BUILD_DIR/systemd $ROOTLIBDIR/systemd 2>/dev/null | grep '^/' -m1)
[ "$SYSTEMD_NSPAWN" ] || SYSTEMD_NSPAWN=$(which -a $BUILD_DIR/systemd-nspawn systemd-nspawn 2>/dev/null | grep '^/' -m1)
[ "$JOURNALCTL" ] || JOURNALCTL=$(which -a $BUILD_DIR/journalctl journalctl 2>/dev/null | grep '^/' -m1)

TESTFILE=${BASH_SOURCE[1]}
if [ -z "$TESTFILE" ]; then
    echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
    exit 1
fi
TESTNAME=$(basename $(dirname $(realpath $TESTFILE)))
STATEDIR="$BUILD_DIR/test/$TESTNAME"
STATEFILE="$STATEDIR/.testdir"
IMAGESTATEDIR="$STATEDIR/.."
TESTLOG="$STATEDIR/test.log"

BASICTOOLS=(
    awk
    basename
    bash
    busybox
    capsh
    cat
    chmod
    chown
    cmp
    cryptsetup
    cut
    date
    dd
    diff
    dirname
    dmsetup
    echo
    env
    false
    getconf
    getent
    getfacl
    grep
    gunzip
    gzip
    head
    ionice
    ip
    ln
    loadkeys
    login
    lz4cat
    mkfifo
    mktemp
    modprobe
    mount
    mountpoint
    mv
    nc
    nproc
    readlink
    rev
    rm
    rmdir
    sed
    seq
    setfattr
    setfont
    setsid
    sfdisk
    sh
    sleep
    socat
    stat
    su
    sulogin
    sysctl
    tail
    tar
    tee
    test
    timeout
    touch
    tr
    true
    truncate
    umount
    uname
    unshare
    xargs
    xzcat
)

DEBUGTOOLS=(
    cp
    df
    dhclient
    dmesg
    du
    find
    free
    grep
    hostname
    id
    less
    ln
    ls
    mkdir
    ping
    ps
    route
    sort
    strace
    stty
    tty
    vi
)

is_built_with_asan() {
    if ! type -P objdump >/dev/null; then
        ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
        return 1
    fi

    # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
    local _asan_calls=$(objdump -dC $SYSTEMD_JOURNALD | egrep "callq?\s+[0-9a-f]+\s+<__asan" -c)
    if (( $_asan_calls < 1000 )); then
        return 1
    else
        return 0
    fi
}

IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)

if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
    STRIP_BINARIES=no
    SKIP_INITRD="${SKIP_INITRD:-yes}"
    PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
    QEMU_MEM="2048M"
    QEMU_SMP="${QEMU_SMP:-4}"

    # We need to correctly distinguish between gcc's and clang's ASan DSOs.
    if ASAN_RT_NAME="$(ldd "$SYSTEMD" | awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}')"; then
        ASAN_COMPILER=gcc
        ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
    elif ASAN_RT_NAME="$(ldd "$SYSTEMD" | awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}')"; then
        ASAN_COMPILER=clang
        ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"

        # As clang's ASan DSO is usually in a non-standard path, let's check if
        # the environment is set accordingly. If not, warn the user and exit.
        # We're not setting the LD_LIBRARY_PATH automagically here, because
        # user should encounter (and fix) the same issue when running the unit
        # tests (meson test)
        if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
            echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
            echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
            exit 1
        fi
    else
        echo >&2 "systemd is not linked against the ASan DSO"
        echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
        exit 1
    fi

    echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
fi

function find_qemu_bin() {
    # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
    if [[ $QEMU_KVM == "yes" ]]; then
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)
    fi

    [ "$ARCH" ] || ARCH=$(uname -m)
    case $ARCH in
    x86_64)
        # QEMU's own build system calls it qemu-system-x86_64
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
        ;;
    i*86)
        # new i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)

        # i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
        ;;
    ppc64*)
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-ppc64 2>/dev/null | grep '^/' -m1)
        ;;
    esac

    if [ ! -e "$QEMU_BIN" ]; then
        echo "Could not find a suitable QEMU binary" >&2
        return 1
    fi
}

# Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
# returns 0 if newer or equal
# returns 1 if older
# returns 2 if failing
function qemu_min_version() {
    find_qemu_bin || return 2

    # get version from binary
    qemu_ver=$($QEMU_BIN --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')

    # Check version string format
    echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
    echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2

    # compare as last command to return that value
    printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
}

# Return 0 if QEMU did run (then you must check the result state/logs for actual
# success), or 1 if QEMU is not available.
run_qemu() {
    if [ -f /etc/machine-id ]; then
        read MACHINE_ID < /etc/machine-id
        [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
            && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
        [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
            && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
    fi

    CONSOLE=ttyS0

    rm -f "$initdir"/{testok,failed,skipped}
    # make sure the initdir is not mounted to avoid concurrent access
    cleanup_initdir
    umount_loopback

    if [[ ! "$KERNEL_BIN" ]]; then
        if [[ "$LOOKS_LIKE_ARCH" ]]; then
            KERNEL_BIN=/boot/vmlinuz-linux
        else
            [ "$ARCH" ] || ARCH=$(uname -m)
            case $ARCH in
                ppc64*)
                KERNEL_BIN=/boot/vmlinux-$KERNEL_VER
                CONSOLE=hvc0
                ;;
                *)
                KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
                ;;
            esac
        fi
    fi

    default_fedora_initrd=/boot/initramfs-${KERNEL_VER}.img
    default_debian_initrd=/boot/initrd.img-${KERNEL_VER}
    default_arch_initrd=/boot/initramfs-linux-fallback.img
    default_suse_initrd=/boot/initrd-${KERNEL_VER}
    if [[ ! "$INITRD" ]]; then
        if [[ -e "$default_fedora_initrd" ]]; then
            INITRD="$default_fedora_initrd"
        elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
            INITRD="$default_debian_initrd"
        elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
            INITRD="$default_arch_initrd"
        elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
            INITRD="$default_suse_initrd"
        fi
    fi

    # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
    # i.e. use the number of online CPUs on the host machine. If the nproc utility
    # is not installed or there's some other error when calling it, fall back
    # to the original value (QEMU_SMP=1).
    if ! [ "$QEMU_SMP" ]; then
        if ! QEMU_SMP=$(nproc); then
            dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
            QEMU_SMP=1
        fi
    fi

    find_qemu_bin || return 1

    # Umount initdir to avoid concurrent access to the filesystem
    _umount_dir $initdir

    local _cgroup_args
    if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=yes"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=no"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
        dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
        exit 1
    fi

    if [[ "$LOOKS_LIKE_SUSE" ]]; then
        PARAMS+="rd.hostonly=0"
    fi

    local _end
    if [[ ! "$INTERACTIVE_DEBUG" ]]; then
        _end="systemd.wants=end.service"
    else
        _end=""
    fi

    KERNEL_APPEND="$PARAMS \
root=/dev/sda1 \
rw \
raid=noautodetect \
rd.luks=0 \
loglevel=2 \
init=$PATH_TO_INIT \
console=$CONSOLE \
selinux=0 \
$_cgroup_args \
SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units: \
systemd.unit=testsuite.target \
systemd.wants=testsuite-$1.service ${_end} \
$KERNEL_APPEND \
"

    [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
    QEMU_OPTIONS="-smp $QEMU_SMP \
-net none \
-m $QEMU_MEM \
-nographic \
-kernel $KERNEL_BIN \
-drive format=raw,cache=unsafe,file=$image \
$QEMU_OPTIONS \
"

    if [[ "$INITRD" && "$SKIP_INITRD" != "yes" ]]; then
        QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
    fi

    # Let's use KVM if possible
    if [[ -c /dev/kvm && $QEMU_KVM == "yes" ]]; then
        QEMU_OPTIONS="$QEMU_OPTIONS -machine accel=kvm -enable-kvm -cpu host"
    fi

    if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
        QEMU_BIN="timeout --foreground $QEMU_TIMEOUT $QEMU_BIN"
    fi
    (set -x; $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND")
    rc=$?
    if [ "$rc" = 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
        derror "test timed out after $QEMU_TIMEOUT s"
        TIMED_OUT=1
    else
        [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
    fi
    return 0
}

# Return 0 if nspawn did run (then you must check the result state/logs for actual
# success), or 1 if nspawn is not available.
run_nspawn() {
    [[ -d /run/systemd/system ]] || return 1
    rm -f "$initdir"/{testok,failed,skipped}

    local _nspawn_cmd=(
        --register=no
        --kill-signal=SIGKILL
        --directory=$1
        --setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:
        $PATH_TO_INIT
        $KERNEL_APPEND
        systemd.unit=testsuite.target
        systemd.wants=testsuite-$2.service
    )

    if [[ ! "$INTERACTIVE_DEBUG" ]]; then
        _nspawn_cmd+=( systemd.wants=end.service )
    fi

    local _nspawn_pre
    if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
        _nspawn_pre=(timeout --foreground $NSPAWN_TIMEOUT)
    else
        _nspawn_pre=()
    fi

    if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
        dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
        exit
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
        _nspawn_pre=("${_nspawn_pre[@]}" env SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY)
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
        _nspawn_pre=("${_nspawn_pre[@]}" env --unset=UNIFIED_CGROUP_HIERARCHY --unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY)
    else
        dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
        exit 1
    fi

    (set -x; "${_nspawn_pre[@]}" "$SYSTEMD_NSPAWN" $NSPAWN_ARGUMENTS "${_nspawn_cmd[@]}")
    rc=$?
    if [ "$rc" = 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
        derror "test timed out after $NSPAWN_TIMEOUT s"
        TIMED_OUT=1
    else
        [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
    fi
    return 0
}

# Build two very minimal root images, with two units, one is the same and one is different across them
install_verity_minimal() {
    if [ -e $initdir/usr/share/minimal.raw ]; then
        return
    fi
    if ! command -v mksquashfs >/dev/null 2>&1; then
        dfatal "mksquashfs not found"
        exit 1
    fi
    if ! command -v veritysetup >/dev/null 2>&1; then
        dfatal "veritysetup not found"
        exit 1
    fi
    (
        BASICTOOLS=(
            bash
            cat
            grep
            mount
            sleep
        )
        oldinitdir=$initdir
        rm -rfv $TESTDIR/minimal
        export initdir=$TESTDIR/minimal
        mkdir -p $initdir/usr/lib/systemd/system $initdir/usr/lib/extension-release.d $initdir/etc $initdir/var/tmp $initdir/opt
        setup_basic_dirs
        install_basic_tools
        if [[ -v ASAN_RT_PATH ]]; then
            # If we're compiled with ASan, install the ASan RT (and its dependencies)
            # into the verity images to get rid of the annoying errors about
            # missing $LD_PRELOAD libraries.
            inst_libs "$ASAN_RT_PATH"
            inst_library "$ASAN_RT_PATH"
        fi
        cp $os_release $initdir/usr/lib/os-release
        ln -s ../usr/lib/os-release $initdir/etc/os-release
        touch $initdir/etc/machine-id $initdir/etc/resolv.conf
        touch $initdir/opt/some_file
        echo MARKER=1 >> $initdir/usr/lib/os-release
        echo -e "[Service]\nExecStartPre=cat /usr/lib/os-release\nExecStart=sleep 120" > $initdir/usr/lib/systemd/system/app0.service
        cp $initdir/usr/lib/systemd/system/app0.service $initdir/usr/lib/systemd/system/app0-foo.service

        mksquashfs $initdir $oldinitdir/usr/share/minimal_0.raw
        veritysetup format $oldinitdir/usr/share/minimal_0.raw $oldinitdir/usr/share/minimal_0.verity | \
            grep '^Root hash:' | cut -f2 | tr -d '\n' > $oldinitdir/usr/share/minimal_0.roothash

        sed -i "s/MARKER=1/MARKER=2/g" $initdir/usr/lib/os-release
        rm $initdir/usr/lib/systemd/system/app0-foo.service
        cp $initdir/usr/lib/systemd/system/app0.service $initdir/usr/lib/systemd/system/app0-bar.service

        mksquashfs $initdir $oldinitdir/usr/share/minimal_1.raw
        veritysetup format $oldinitdir/usr/share/minimal_1.raw $oldinitdir/usr/share/minimal_1.verity | \
            grep '^Root hash:' | cut -f2 | tr -d '\n' > $oldinitdir/usr/share/minimal_1.roothash

        # Rolling distros like Arch do not set VERSION_ID
        local version_id=""
        if grep -q "^VERSION_ID=" $os_release; then
            version_id="$(grep "^VERSION_ID=" $os_release)"
        fi

        export initdir=$TESTDIR/app0
        mkdir -p $initdir/usr/lib/extension-release.d $initdir/usr/lib/systemd/system $initdir/opt
        grep "^ID=" $os_release > $initdir/usr/lib/extension-release.d/extension-release.app0
        echo "${version_id}" >> $initdir/usr/lib/extension-release.d/extension-release.app0
        cat <<EOF > $initdir/usr/lib/systemd/system/app0.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/opt/script0.sh
EOF
        cat <<EOF > $initdir/opt/script0.sh
#!/bin/bash
set -e
test -e /usr/lib/os-release
cat /usr/lib/extension-release.d/extension-release.app0
EOF
        chmod +x $initdir/opt/script0.sh
        echo MARKER=1 > $initdir/usr/lib/systemd/system/some_file
        mksquashfs $initdir $oldinitdir/usr/share/app0.raw

        export initdir=$TESTDIR/app1
        mkdir -p $initdir/usr/lib/extension-release.d $initdir/usr/lib/systemd/system $initdir/opt
        grep "^ID=" $os_release > $initdir/usr/lib/extension-release.d/extension-release.app1
        echo "${version_id}" >> $initdir/usr/lib/extension-release.d/extension-release.app1
        cat <<EOF > $initdir/usr/lib/systemd/system/app1.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/opt/script1.sh
EOF
        cat <<EOF > $initdir/opt/script1.sh
#!/bin/bash
set -e
test -e /usr/lib/os-release
cat /usr/lib/extension-release.d/extension-release.app1
EOF
        chmod +x $initdir/opt/script1.sh
        echo MARKER=1 > $initdir/usr/lib/systemd/system/other_file
        mksquashfs $initdir $oldinitdir/usr/share/app1.raw
    )
}

setup_basic_environment() {
    # create the basic filesystem layout
    setup_basic_dirs

    install_systemd
    install_missing_libraries
    install_config_files
    install_zoneinfo
    create_rc_local
    install_basic_tools
    install_libnss
    install_pam
    install_dbus
    install_fonts
    install_keymaps
    install_terminfo
    install_execs
    install_fsck
    install_plymouth
    install_debug_tools
    install_ld_so_conf
    install_testuser
    has_user_dbus_socket && install_user_dbus
    setup_selinux
    strip_binaries
    install_depmod_files
    generate_module_dependencies
    if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
        create_asan_wrapper
    fi
    if [ -n "$TEST_INSTALL_VERITY_MINIMAL" ]; then
        install_verity_minimal
    fi
}

setup_selinux() {
    # don't forget KERNEL_APPEND='... selinux=1 ...'
    if [[ "$SETUP_SELINUX" != "yes" ]]; then
        ddebug "Don't setup SELinux"
        return 0
    fi
    ddebug "Setup SELinux"
    local _conf_dir=/etc/selinux
    local _fixfiles_tools="bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles"

    rm -rf $initdir/$_conf_dir
    if ! cp -ar $_conf_dir $initdir/$_conf_dir; then
        dfatal "Failed to copy $_conf_dir"
        exit 1
    fi

    touch $initdir/.autorelabel
    mkdir -p $initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants
    ln -sf ../autorelabel.service $initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/

    dracut_install $_fixfiles_tools
    dracut_install fixfiles
    dracut_install sestatus
}

install_valgrind() {
    if ! type -p valgrind; then
        dfatal "Failed to install valgrind"
        exit 1
    fi

    local _valgrind_bins=$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
    dracut_install $_valgrind_bins

    local _valgrind_libs=$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
    dracut_install $_valgrind_libs

    local _valgrind_dbg_and_supp=$(
        strace -e open valgrind /bin/true 2>&1 >/dev/null |
        perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
    )
    dracut_install $_valgrind_dbg_and_supp
}

create_valgrind_wrapper() {
    local _valgrind_wrapper=$initdir/$ROOTLIBDIR/systemd-under-valgrind
    ddebug "Create $_valgrind_wrapper"
    cat >$_valgrind_wrapper <<EOF
#!/usr/bin/env bash

mount -t proc proc /proc
exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
EOF
    chmod 0755 $_valgrind_wrapper
}

create_asan_wrapper() {
    local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan
    local _asan_rt_pattern
    ddebug "Create $_asan_wrapper"

    [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"

    # clang: install llvm-symbolizer to generate useful reports
    # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
    [[ "$ASAN_COMPILER" == "clang" ]] && dracut_install "llvm-symbolizer"

    cat >$_asan_wrapper <<EOF
#!/usr/bin/env bash

set -x

echo "ASan RT: $ASAN_RT_PATH"
if [[ ! -e "$ASAN_RT_PATH" ]]; then
    echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
    exit 1
fi

DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"

# As right now bash is the PID 1, we can't expect PATH to have a sane value.
# Let's make one to prevent unexpected "<bin> not found" issues in the future
export PATH="/sbin:/bin:/usr/sbin:/usr/bin"

mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -o remount,rw /

# A lot of services (most notably dbus) won't start without preloading libasan
# See https://github.com/systemd/systemd/issues/5004
DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=$ASAN_RT_PATH"

if [[ "$ASAN_COMPILER" == "clang" ]]; then
  # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
  # unnecessary for gcc & libasan, however, for clang this is crucial, as its
  # runtime ASan DSO is in a non-standard (library) path.
  echo "${ASAN_RT_PATH%/*}" > /etc/ld.so.conf.d/asan-path-override.conf
  ldconfig
fi
echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf

# ASAN and syscall filters aren't compatible with each other.
find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'

# The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
# But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
mkdir -p "\$JOURNALD_CONF_DIR"
printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"

# Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
# Let's try to catch them by redirecting stderr (and stdout just in case) to a file
# See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"

# 90s isn't enough for some services to finish when literally everything is run
# under ASan+UBSan in containers, which, in turn, are run in VMs.
# Let's limit which environments such services should be executed in.
mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf

# Let's override another hard-coded timeout that kicks in too early
mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf

# D-Bus has troubles during system shutdown causing it to fail. Although it's
# harmless, it causes unnecessary noise in the logs, so let's disable LSan's
# at_exit check just for the dbus.service
mkdir -p /etc/systemd/system/dbus.service.d
printf "[Service]\nEnvironment=ASAN_OPTIONS=leak_check_at_exit=false\n" >/etc/systemd/system/dbus.service.d/disable-lsan.conf

# Some utilities run via IMPORT/RUN/PROGRAM udev directives fail because
# they're uninstrumented (like dmsetup). Let's add a simple rule which sets
# LD_PRELOAD to the ASan RT library to fix this.
mkdir -p /etc/udev/rules.d
cat > /etc/udev/rules.d/00-set-LD_PRELOAD.rules << INNER_EOF
SUBSYSTEM=="block", ENV{LD_PRELOAD}="$ASAN_RT_PATH"
INNER_EOF

# The 'mount' utility doesn't behave well under libasan, causing unexpected
# fails during boot and subsequent test results check:
# bash-5.0# mount -o remount,rw -v /
# mount: /dev/sda1 mounted on /.
# bash-5.0# echo \$?
# 1
# Let's workaround this by clearing the previously set LD_PRELOAD env variable,
# so the libasan library is not loaded for this particular service
unset_ld_preload() {
    local _dropin_dir="/etc/systemd/system/\$1.service.d"
    mkdir -p "\$_dropin_dir"
    printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$_dropin_dir/unset_ld_preload.conf"
}

unset_ld_preload systemd-remount-fs
unset_ld_preload testsuite-

export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
exec  $ROOTLIBDIR/systemd "\$@"
EOF

    chmod 0755 $_asan_wrapper
}

create_strace_wrapper() {
    local _strace_wrapper=$initdir/$ROOTLIBDIR/systemd-under-strace
    ddebug "Create $_strace_wrapper"
    cat >$_strace_wrapper <<EOF
#!/usr/bin/env bash

exec strace -f -D -o /strace.out $ROOTLIBDIR/systemd "\$@"
EOF
    chmod 0755 $_strace_wrapper
}

install_fsck() {
    dracut_install /sbin/fsck*
    dracut_install -o /bin/fsck*

    # fskc.reiserfs calls reiserfsck. so, install it
    dracut_install -o reiserfsck
}

install_dmevent() {
    instmods dm_crypt =crypto
    inst_binary dmeventd
    if [[ "$LOOKS_LIKE_DEBIAN" ]]; then
        # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
        # and since buster/bionic 95-dm-notify.rules
        # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
        inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
    else
        inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
    fi
    if [[ "$LOOKS_LIKE_SUSE" ]]; then
        inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
    fi
}

install_compiled_systemd() {
    ddebug "Install compiled systemd"

    local _ninja_bin=$(type -P ninja || type -P ninja-build)
    if [[ -z "$_ninja_bin" ]]; then
        dfatal "ninja was not found"
        exit 1
    fi
    (set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install)
}

install_debian_systemd() {
    ddebug "Install debian systemd"

    local _systemd_pkgs=$(grep -E '^Package:' ${SOURCE_DIR}/debian/control | cut -d ':' -f 2)
    local _files=""
    for deb in $_systemd_pkgs; do
        _files=$(dpkg-query -L $deb 2>/dev/null) || continue
        ddebug "Install debian files from package $deb"
        for file in $_files; do
            [ -e "$file" ] || continue
            [ -d "$file" ] && continue
            inst $file
        done
    done
}

install_distro_systemd() {
    ddebug "Install distro systemd"

    if [ "$LOOKS_LIKE_DEBIAN" ]; then
        install_debian_systemd
    else
        dfatal "NO_BUILD not supported for this distro"
        exit 1
    fi
}

install_systemd() {
    if [ "$NO_BUILD" ]; then
        install_distro_systemd
    else
        install_compiled_systemd
    fi

    # remove unneeded documentation
    rm -fr $initdir/usr/share/{man,doc}

    [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse

    # enable debug logging in PID1
    echo LogLevel=debug >> $initdir/etc/systemd/system.conf
    # store coredumps in journal
    echo Storage=journal >> $initdir/etc/systemd/coredump.conf
}

get_ldpath() {
    local _bin="$1"
    local rpath=$(objdump -p "$_bin" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)

    if [ -z "$rpath" ] ; then
        echo $BUILD_DIR
    else
        echo $rpath
    fi
}

install_missing_libraries() {
    # install possible missing libraries
    for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
        LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath $i)" inst_libs $i
    done

    # A number of dependencies is now optional via dlopen, so the install
    # script will not pick them up, since it looks at linkage.
    for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu libfido2; do
        ddebug "Searching for $lib via pkg-config"
        if pkg-config --exists ${lib}; then
                path=$(pkg-config --variable=libdir ${lib})
                if [ -z "${path}" ]; then
                    ddebug "$lib.pc does not contain a libdir variable, skipping"
                    continue
                fi

                if ! [[ ${lib} =~ ^lib ]]; then
                        lib="lib${lib}"
                fi
                # Some pkg-config files are broken and give out the wrong paths
                # (eg: libcryptsetup), so just ignore them
                inst_libs "${path}/${lib}.so" || true
                inst_library "${path}/${lib}.so" || true
        else
            ddebug "$lib.pc not found, skipping"
            continue
        fi
    done
}

cleanup_loopdev() {
    if [ -n "${LOOPDEV}" ]; then
        ddebug "losetup -d $LOOPDEV"
        losetup -d "${LOOPDEV}"
        unset LOOPDEV
    fi
}

trap cleanup_loopdev EXIT INT QUIT PIPE

create_empty_image() {
    if [ -z "$IMAGE_NAME" ]; then
        echo "create_empty_image: \$IMAGE_NAME not set"
        exit 1
    fi

    local _size=500
    if [[ "$STRIP_BINARIES" = "no" ]]; then
        _size=$((4*_size))
    fi

    echo "Setting up $IMAGE_PUBLIC (${_size} MB)"
    rm -f "$IMAGE_PRIVATE" "$IMAGE_PUBLIC"

    # Create the blank file to use as a root filesystem
    truncate -s "${_size}M" "$IMAGE_PUBLIC"

    LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
    [ -b "$LOOPDEV" ] || return 1
    sfdisk "$LOOPDEV" <<EOF
,$((_size-50))M
,
EOF

    udevadm settle

    local _label="-L systemd.${name}"
    # mkfs.reiserfs doesn't know -L. so, use --label instead
    [[ "$FSTYPE" == "reiserfs" ]] && _label="--label systemd.${name}"
    mkfs -t "${FSTYPE}" ${_label} "${LOOPDEV}p1" -q; ret=$?
    if [ $ret -ne 0 ] ; then
        dfatal "Failed to mkfs -t ${FSTYPE}"
        exit 1
    fi
}

mount_initdir() {
    if [ -z "${LOOPDEV}" ]; then
        [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
        LOOPDEV=$(losetup --show -P -f "$image")
        [ -b "$LOOPDEV" ] || return 1

        udevadm settle
    fi

    if ! mountpoint -q $initdir; then
        mkdir -p $initdir
        mount ${LOOPDEV}p1 $initdir
        TEST_SETUP_CLEANUP_ROOTDIR=1
    fi
}

cleanup_initdir() {
    # only umount if create_empty_image_rootdir() was called to mount it
    [[ -z $TEST_SETUP_CLEANUP_ROOTDIR ]] || _umount_dir $initdir
}

umount_loopback() {
    # unmount the loopback device from all places. Otherwise we risk file
    # system corruption.
    for device in $(losetup -l | awk '$6=="'"$IMAGE_PUBLIC"'" {print $1}'); do
        ddebug "Unmounting all uses of $device"
        mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
    done
}

create_empty_image_rootdir() {
    create_empty_image
    mount_initdir
}

check_asan_reports() {
    local ret=0
    local root="$1"

    if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
        ls -l "$root"
        if [[ -e "$root/systemd.asan.log.1" ]]; then
            cat "$root/systemd.asan.log.1"
            ret=$(($ret+1))
        fi

        journald_report=$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)
        if [[ ! -z "$journald_report" ]]; then
            printf "%s\n" "$journald_report"
            cat "$root/systemd-journald.out" || :
            ret=$(($ret+1))
        fi

        pids=$(
            "$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
                 BEGIN {
                     %services_to_ignore = (
                         "dbus-daemon" => undef,
                     );
                 }
                 print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
        )
        if [[ ! -z "$pids" ]]; then
            ret=$(($ret+1))
            for pid in $pids; do
                "$JOURNALCTL" -D "$root/var/log/journal" _PID=$pid --no-pager
            done
        fi
    fi

    return $ret
}

save_journal() {
    if [ -n "${ARTIFACT_DIRECTORY}" ]; then
        dest="${ARTIFACT_DIRECTORY}/${testname}.journal"
    else
        dest="$TESTDIR/system.journal"
    fi

    for j in $1/*; do
        $SYSTEMD_JOURNAL_REMOTE \
            -o $dest \
            --getter="$JOURNALCTL -o export -D $j"

        if [ -n "${TEST_SHOW_JOURNAL}" ]; then
            echo "---- $j ----"
            $JOURNALCTL --no-pager -o short-monotonic --no-hostname --priority=${TEST_SHOW_JOURNAL} -D $j
        fi

        rm -r $j
    done

    # we want to print this sometime later, so save this in a variable
    JOURNAL_LIST="$(ls -l $dest*)"
}

check_result_nspawn() {
    local ret=1
    local journald_report=""
    local pids=""
    [[ -e $1/testok ]] && ret=0
    [[ -f $1/failed ]] && cp -a $1/failed $TESTDIR
    save_journal $1/var/log/journal
    [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
    echo $JOURNAL_LIST
    test -s $TESTDIR/failed && ret=$(($ret+1))
    [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
    check_asan_reports "$1" || ret=$(($ret+1))
    if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f $1/strace.out ]; then
        cp $1/strace.out "${ARTIFACT_DIRECTORY}/"
    fi
    _umount_dir $initdir
    return $ret
}

# can be overridden in specific test
check_result_qemu() {
    local ret=1
    mount_initdir
    [[ -e $initdir/testok ]] && ret=0
    [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
    save_journal $initdir/var/log/journal
    check_asan_reports "$initdir" || ret=$(($ret+1))
    if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f $initdir/strace.out ]; then
        cp $initdir/strace.out "${ARTIFACT_DIRECTORY}/"
    fi
    _umount_dir $initdir
    [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
    echo $JOURNAL_LIST
    test -s $TESTDIR/failed && ret=$(($ret+1))
    [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
    return $ret
}

strip_binaries() {
    if [[ "$STRIP_BINARIES" = "no" ]]; then
        ddebug "Don't strip binaries"
        return 0
    fi
    ddebug "Strip binaries"
    find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | \
        xargs strip --strip-unneeded |& \
        grep -vi 'file format not recognized' | \
        ddebug
}

create_rc_local() {
    mkdir -p $initdir/etc/rc.d
    cat >$initdir/etc/rc.d/rc.local <<EOF
#!/usr/bin/env bash
exit 0
EOF
    chmod 0755 $initdir/etc/rc.d/rc.local
}

install_execs() {
    ddebug "install any Execs from the service files"
    (
    export PKG_CONFIG_PATH=$BUILD_DIR/src/core/
    systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
    systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
    sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
         | sort -u | while read i; do
         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
         # also, plymouth is pulled in by rescue.service, but even there the exit code
         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
         dinfo "Attempting to install $i (based on unit file reference)"
         inst $i || [ "${i%.local}" != "$i" ] || [ "${i%systemd-update-done}" != "$i" ] || [ "${i##*/}" == "plymouth" ]
     done
    )
}

generate_module_dependencies() {
    if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
        ! depmod -a -b "$initdir" $KERNEL_VER; then
            dfatal "\"depmod -a $KERNEL_VER\" failed."
            exit 1
    fi
}

install_depmod_files() {
    inst /lib/modules/$KERNEL_VER/modules.order
    inst /lib/modules/$KERNEL_VER/modules.builtin
}

install_plymouth() {
    # install plymouth, if found... else remove plymouth service files
    # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
    #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
    #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
    #         dracut_install plymouth plymouthd
    # else
        rm -f $initdir/{usr/lib,lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,lib,etc}/systemd/system/*/plymouth*
    # fi
}

install_ld_so_conf() {
    cp -a /etc/ld.so.conf* $initdir/etc
    ldconfig -r "$initdir"
}

install_testuser() {
    # create unprivileged user for user manager tests
    mkdir -p $initdir/etc/sysusers.d
    cat >$initdir/etc/sysusers.d/testuser.conf <<EOF
u testuser    4711     "Test User" /home/testuser
EOF

    mkdir -p $initdir/home/testuser -m 0700
    chown 4711:4711 $initdir/home/testuser
}

install_config_files() {
    inst /etc/sysconfig/init || :
    inst /etc/passwd
    inst /etc/shadow
    inst_any /etc/login.defs /usr/etc/login.defs
    inst /etc/group
    inst /etc/shells
    inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
    inst /etc/pam.conf || :
    inst_any /etc/os-release /usr/lib/os-release
    inst /etc/localtime
    # we want an empty environment
    > $initdir/etc/environment
    > $initdir/etc/machine-id
    > $initdir/etc/resolv.conf

    # set the hostname
    echo systemd-testsuite > $initdir/etc/hostname

    # let's set up just one image with the traditional verbose output
    if [ ${IMAGE_NAME} != "basic" ]; then
        mkdir -p $initdir/etc/systemd/system.conf.d
        echo -e '[Manager]\nStatusUnitFormat=name' >$initdir/etc/systemd/system.conf.d/status.conf
    fi
}

install_basic_tools() {
    dracut_install "${BASICTOOLS[@]}"
    dracut_install -o sushell
    # in Debian ldconfig is just a shell script wrapper around ldconfig.real
    dracut_install -o ldconfig.real
}

install_debug_tools() {
    dracut_install "${DEBUGTOOLS[@]}"

    if [[ $INTERACTIVE_DEBUG ]]; then
        # Set default TERM from vt220 to linux, so at least basic key shortcuts work
        local _getty_override="$initdir/etc/systemd/system/serial-getty@.service.d"
        mkdir -p "$_getty_override"
        echo -e "[Service]\nEnvironment=TERM=linux" > "$_getty_override/default-TERM.conf"

        cat > "$initdir/etc/motd" << EOF
To adjust the terminal size use:
    export COLUMNS=xx
    export LINES=yy
or
    stty cols xx rows yy
EOF
    fi
}

install_libnss() {
    # install libnss_files for login
    NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
    dracut_install $NSS_LIBS
}

install_dbus() {
    inst $ROOTLIBDIR/system/dbus.socket

    # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
    if [ -f $ROOTLIBDIR/system/dbus-broker.service ]; then
        inst $ROOTLIBDIR/system/dbus-broker.service
        inst_symlink /etc/systemd/system/dbus.service
        inst /usr/bin/dbus-broker
        inst /usr/bin/dbus-broker-launch
    elif [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
        # Fedora rawhide replaced dbus.service with dbus-daemon.service
        inst $ROOTLIBDIR/system/dbus-daemon.service
        # Alias symlink
        inst_symlink /etc/systemd/system/dbus.service
    else
        inst $ROOTLIBDIR/system/dbus.service
    fi

    find \
        /etc/dbus-1 /usr/share/dbus-1 -xtype f \
        | while read file; do
        inst $file
    done

    # setup policy for Type=dbus test
    mkdir -p $initdir/etc/dbus-1/system.d
    cat > $initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf <<EOF
<?xml version="1.0"?>
<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
        "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
    <policy user="root">
        <allow own="systemd.test.ExecStopPost"/>
    </policy>
</busconfig>
EOF
}

install_user_dbus() {
    local userunitdir
    if ! userunitdir=$(pkg-config --variable=systemduserunitdir systemd); then
        echo "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user" >&2
        local userunitdir=/usr/lib/systemd/user
    fi

    inst $userunitdir/dbus.socket
    inst_symlink $userunitdir/sockets.target.wants/dbus.socket || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket

    # Append the After= dependency on dbus in case it isn't already set up
    mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
    cat <<EOF >"$initdir/etc/systemd/system/user@.service.d/dbus.conf"
[Unit]
After=dbus.service
EOF

    # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
    if [ -f $userunitdir/dbus-broker.service ]; then
        inst $userunitdir/dbus-broker.service
        inst_symlink /etc/systemd/user/dbus.service
    elif [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
        # Fedora rawhide replaced dbus.service with dbus-daemon.service
        inst $userunitdir/dbus-daemon.service
        # Alias symlink
        inst_symlink /etc/systemd/user/dbus.service
    else
        inst $userunitdir/dbus.service
    fi
}

install_pam() {
    (
    if [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null; then
        find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
    else
        find /lib*/security -xtype f
    fi
    for d in /etc/pam.d /etc/security /usr/lib/pam.d; do
        [ -d "$d" ] && find $d -xtype f
    done
    ) | while read file; do
        inst $file
    done

    # pam_unix depends on unix_chkpwd.
    # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
    dracut_install -o unix_chkpwd

    # set empty root password for easy debugging
    sed -i 's/^root:x:/root::/' $initdir/etc/passwd
}

install_keymaps() {
    # The first three paths may be deprecated.
    # It seems now the last two paths are used by many distributions.
    for i in \
        /usr/lib/kbd/keymaps/include/* \
        /usr/lib/kbd/keymaps/i386/include/* \
        /usr/lib/kbd/keymaps/i386/qwerty/us.* \
        /usr/lib/kbd/keymaps/legacy/include/* \
        /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
            [[ -f $i ]] || continue
            inst $i
    done

    # When it takes any argument, then install more keymaps.
    if [[ -n $1 ]]; then
        for i in \
        /usr/lib/kbd/keymaps/i386/*/* \
        /usr/lib/kbd/keymaps/legacy/i386/*/*; do
            [[ -f $i ]] || continue
            inst $i
        done
    fi
}

install_zoneinfo() {
    inst_any /usr/share/zoneinfo/Asia/Seoul
    inst_any /usr/share/zoneinfo/Asia/Vladivostok
    inst_any /usr/share/zoneinfo/Australia/Sydney
    inst_any /usr/share/zoneinfo/Europe/Berlin
    inst_any /usr/share/zoneinfo/Europe/Kiev
    inst_any /usr/share/zoneinfo/Pacific/Auckland
    inst_any /usr/share/zoneinfo/Pacific/Honolulu
    inst_any /usr/share/zoneinfo/CET
    inst_any /usr/share/zoneinfo/EET
    inst_any /usr/share/zoneinfo/UTC
}

install_fonts() {
    for i in \
        /usr/lib/kbd/consolefonts/eurlatgr* \
        /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
            [[ -f $i ]] || continue
            inst $i
    done
}

install_terminfo() {
    for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
        [ -f ${_terminfodir}/l/linux ] && break
    done
    dracut_install -o ${_terminfodir}/l/linux
}

has_user_dbus_socket() {
    if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
        return 0
    else
        echo "Per-user instances are not supported. Skipping..."
        return 1
    fi
}

setup_nspawn_root() {
    if [ -z "${initdir}" ]; then
        dfatal "\$initdir not defined"
        exit 1
    fi

    rm -rf "$TESTDIR/unprivileged-nspawn-root"

    if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
        ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
        cp -ar $initdir $TESTDIR/unprivileged-nspawn-root
    fi
}

setup_basic_dirs() {
    mkdir -p $initdir/run
    mkdir -p $initdir/etc/systemd/system
    mkdir -p $initdir/var/log/journal

    for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
        if [ -L "/$d" ]; then
            inst_symlink "/$d"
        else
            inst_dir "/$d"
        fi
    done

    ln -sfn /run "$initdir/var/run"
    ln -sfn /run/lock "$initdir/var/lock"
}

mask_supporting_services() {
    # mask some services that we do not want to run in these tests
    ln -fs /dev/null $initdir/etc/systemd/system/systemd-hwdb-update.service
    ln -fs /dev/null $initdir/etc/systemd/system/systemd-journal-catalog-update.service
    ln -fs /dev/null $initdir/etc/systemd/system/systemd-networkd.service
    ln -fs /dev/null $initdir/etc/systemd/system/systemd-networkd.socket
    ln -fs /dev/null $initdir/etc/systemd/system/systemd-resolved.service
}

inst_libs() {
    local _bin=$1
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    local _file _line

    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
}

import_testdir() {
    # make sure we don't get a stale LOOPDEV value from old times
    __LOOPDEV=$LOOPDEV
    [[ -e $STATEFILE ]] && . $STATEFILE
    LOOPDEV=$__LOOPDEV
    if [[ ! -d "$TESTDIR" ]]; then
        if [[ -z "$TESTDIR" ]]; then
            TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
        else
            mkdir -p "$TESTDIR"
        fi

        cat >$STATEFILE<<EOF
TESTDIR="$TESTDIR"
EOF
        export TESTDIR
    fi

    IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME}.img"
    IMAGE_PUBLIC="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
}

import_initdir() {
    initdir=$TESTDIR/root
    mkdir -p $initdir
    export initdir
}

## @brief Converts numeric logging level to the first letter of level name.
#
# @param lvl Numeric logging level in range from 1 to 6.
# @retval 1 if @a lvl is out of range.
# @retval 0 if @a lvl is correct.
# @result Echoes first letter of level name.
_lvl2char() {
    case "$1" in
        1) echo F;;
        2) echo E;;
        3) echo W;;
        4) echo I;;
        5) echo D;;
        6) echo T;;
        *) return 1;;
    esac
}

## @brief Internal helper function for _do_dlog()
#
# @param lvl Numeric logging level.
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
#
# @note This function is not supposed to be called manually. Please use
# dtrace(), ddebug(), or others instead which wrap this one.
#
# This function calls _do_dlog() either with parameter msg, or if
# none is given, it will read standard input and will use every line as
# a message.
#
# This enables:
# dwarn "This is a warning"
# echo "This is a warning" | dwarn
LOG_LEVEL=${LOG_LEVEL:-4}

dlog() {
    [ -z "$LOG_LEVEL" ] && return 0
    [ $1 -le $LOG_LEVEL ] || return 0
    local lvl="$1"; shift
    local lvlc=$(_lvl2char "$lvl") || return 0

    if [ $# -ge 1 ]; then
        echo "$lvlc: $*"
    else
        while read line; do
            echo "$lvlc: " "$line"
        done
    fi
}

## @brief Logs message at TRACE level (6)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dtrace() {
    set +x
    dlog 6 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at DEBUG level (5)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
ddebug() {
#    set +x
    dlog 5 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at INFO level (4)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dinfo() {
    set +x
    dlog 4 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at WARN level (3)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dwarn() {
    set +x
    dlog 3 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at ERROR level (2)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
derror() {
#    set +x
    dlog 2 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at FATAL level (1)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dfatal() {
    set +x
    dlog 1 "$@"
    [ -n "$debug" ] && set -x || :
}


# Generic substring function.  If $2 is in $1, return 0.
strstr() { [ "${1#*$2*}" != "$1" ]; }

# normalize_path <path>
# Prints the normalized path, where it removes any duplicated
# and trailing slashes.
# Example:
# $ normalize_path ///test/test//
# /test/test
normalize_path() {
    shopt -q -s extglob
    set -- "${1//+(\/)//}"
    shopt -q -u extglob
    echo "${1%/}"
}

# convert_abs_rel <from> <to>
# Prints the relative path, when creating a symlink to <to> from <from>.
# Example:
# $ convert_abs_rel /usr/bin/test /bin/test-2
# ../../bin/test-2
# $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
convert_abs_rel() {
    local __current __absolute __abssize __cursize __newpath
    local -i __i __level

    set -- "$(normalize_path "$1")" "$(normalize_path "$2")"

    # corner case #1 - self looping link
    [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }

    # corner case #2 - own dir link
    [[ "${1%/*}" == "$2" ]] && { echo "."; return; }

    IFS="/" __current=($1)
    IFS="/" __absolute=($2)

    __abssize=${#__absolute[@]}
    __cursize=${#__current[@]}

    while [[ ${__absolute[__level]} == ${__current[__level]} ]]
    do
        (( __level++ ))
        if (( __level > __abssize || __level > __cursize ))
        then
            break
        fi
    done

    for ((__i = __level; __i < __cursize-1; __i++))
    do
        if ((__i > __level))
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath".."
    done

    for ((__i = __level; __i < __abssize; __i++))
    do
        if [[ -n $__newpath ]]
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath${__absolute[__i]}
    done

    echo "$__newpath"
}


# Install a directory, keeping symlinks as on the original system.
# Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
# will create ${initdir}/lib64, ${initdir}/lib64/file,
# and a symlink ${initdir}/lib -> lib64.
inst_dir() {
    [[ -e ${initdir}/"$1" ]] && return 0  # already there

    local _dir="$1" _part="${1%/*}" _file
    while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
        _dir="$_part $_dir"
        _part=${_part%/*}
    done

    # iterate over parent directories
    for _file in $_dir; do
        [[ -e "${initdir}/$_file" ]] && continue
        if [[ -L $_file ]]; then
            inst_symlink "$_file"
        else
            # create directory
            mkdir -m 0755 -p "${initdir}/$_file" || return 1
            [[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
            chmod u+w "${initdir}/$_file"
        fi
    done
}

# $1 = file to copy to ramdisk
# $2 (optional) Name for the file on the ramdisk
# Location of the image dir is assumed to be $initdir
# We never overwrite the target if it exists.
inst_simple() {
    [[ -f "$1" ]] || return 1
    strstr "$1" "/" || return 1

    local _src=$1 target="${2:-$1}"
    if ! [[ -d ${initdir}/$target ]]; then
        [[ -e ${initdir}/$target ]] && return 0
        [[ -L ${initdir}/$target ]] && return 0
        [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
    fi
    # install checksum files also
    if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
        inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
    fi
    ddebug "Installing $_src"
    cp --sparse=always -pfL "$_src" "${initdir}/$target"
}

# find symlinks linked to given library file
# $1 = library file
# Function searches for symlinks by stripping version numbers appended to
# library filename, checks if it points to the same target and finally
# prints the list of symlinks to stdout.
#
# Example:
# rev_lib_symlinks libfoo.so.8.1
# output: libfoo.so.8 libfoo.so
# (Only if libfoo.so.8 and libfoo.so exists on host system.)
rev_lib_symlinks() {
    [[ ! $1 ]] && return 0

    local fn="$1" orig="$(readlink -f "$1")" links=''

    [[ ${fn} =~ .*\.so\..* ]] || return 1

    until [[ ${fn##*.} == so ]]; do
        fn="${fn%.*}"
        [[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
    done

    echo "${links}"
}

# Same as above, but specialized to handle dynamic libraries.
# It handles making symlinks according to how the original library
# is referenced.
inst_library() {
    local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
    strstr "$1" "/" || return 1
    [[ -e $initdir/$_dest ]] && return 0
    if [[ -L $_src ]]; then
        # install checksum files also
        if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
            inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
        fi
        _reallib=$(readlink -f "$_src")
        inst_simple "$_reallib" "$_reallib"
        inst_dir "${_dest%/*}"
        [[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
        ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
    else
        inst_simple "$_src" "$_dest"
    fi

    # Create additional symlinks.  See rev_symlinks description.
    for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
        [[ -e $initdir/$_symlink ]] || {
            ddebug "Creating extra symlink: $_symlink"
            inst_symlink $_symlink
        }
    done
}

# find a binary.  If we were not passed the full path directly,
# search in the usual places to find the binary.
find_binary() {
    if [[ -z ${1##/*} ]]; then
        if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; };  then
            echo $1
            return 0
        fi
    fi

    type -P $1
}

# Same as above, but specialized to install binary executables.
# Install binary executable, and all shared library dependencies, if any.
inst_binary() {
    local _bin _target

    # In certain cases we might attempt to install a binary which is already
    # present in the test image, yet it's missing from the host system.
    # In such cases, let's check if the binary indeed exists in the image
    # before doing any other chcecks. If it does, immediately return with
    # success.
    [[ $# -eq 1 && -e $initdir/$1 || -e $initdir/bin/$1 || -e $initdir/sbin/$1 || -e $initdir/usr/bin/$1 || -e $initdir/usr/sbin/$1 ]] && return 0

    _bin=$(find_binary "$1") || return 1
    _target=${2:-$_bin}
    [[ -e $initdir/$_target ]] && return 0
    [[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
    local _file _line
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    # I love bash!
    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
    inst_simple "$_bin" "$_target"
}

# same as above, except for shell scripts.
# If your shell script does not start with shebang, it is not a shell script.
inst_script() {
    local _bin
    _bin=$(find_binary "$1") || return 1
    shift
    local _line _shebang_regex
    read -r -n 80 _line <"$_bin"
    # If debug is set, clean unprintable chars to prevent messing up the term
    [[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
    _shebang_regex='(#! *)(/[^ ]+).*'
    [[ $_line =~ $_shebang_regex ]] || return 1
    inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
}

# same as above, but specialized for symlinks
inst_symlink() {
    local _src=$1 _target=${2:-$1} _realsrc
    strstr "$1" "/" || return 1
    [[ -L $1 ]] || return 1
    [[ -L $initdir/$_target ]] && return 0
    _realsrc=$(readlink -f "$_src")
    if ! [[ -e $initdir/$_realsrc ]]; then
        if [[ -d $_realsrc ]]; then
            inst_dir "$_realsrc"
        else
            inst "$_realsrc"
        fi
    fi
    [[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
    [[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
    ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
}

# attempt to install any programs specified in a udev rule
inst_rule_programs() {
    local _prog _bin

    if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
        for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
            if [ -x /lib/udev/$_prog ]; then
                _bin=/lib/udev/$_prog
            else
                _bin=$(find_binary "$_prog") || {
                    dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
                    continue;
                }
            fi

            #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
            dracut_install "$_bin"
        done
    fi
}

# udev rules always get installed in the same place, so
# create a function to install them to make life simpler.
inst_rules() {
    local _target=/etc/udev/rules.d _rule _found

    inst_dir "/lib/udev/rules.d"
    inst_dir "$_target"
    for _rule in "$@"; do
        if [ "${rule#/}" = "$rule" ]; then
            for r in /lib/udev/rules.d /etc/udev/rules.d; do
                if [[ -f $r/$_rule ]]; then
                    _found="$r/$_rule"
                    inst_simple "$_found"
                    inst_rule_programs "$_found"
                fi
            done
        fi
        for r in '' ./ $dracutbasedir/rules.d/; do
            if [[ -f ${r}$_rule ]]; then
                _found="${r}$_rule"
                inst_simple "$_found" "$_target/${_found##*/}"
                inst_rule_programs "$_found"
            fi
        done
        [[ $_found ]] || dinfo "Skipping udev rule: $_rule"
        _found=
    done
}

# general purpose installation function
# Same args as above.
inst() {
    local _x

    case $# in
        1) ;;
        2) [[ ! $initdir && -d $2 ]] && export initdir=$2
            [[ $initdir = $2 ]] && set $1;;
        3) [[ -z $initdir ]] && export initdir=$2
            set $1 $3;;
        *) dfatal "inst only takes 1 or 2 or 3 arguments"
            exit 1;;
    esac
    for _x in inst_symlink inst_script inst_binary inst_simple; do
        $_x "$@" && return 0
    done
    return 1
}

# install any of listed files
#
# If first argument is '-d' and second some destination path, first accessible
# source is installed into this path, otherwise it will installed in the same
# path as source.  If none of listed files was installed, function return 1.
# On first successful installation it returns with 0 status.
#
# Example:
#
# inst_any -d /bin/foo /bin/bar /bin/baz
#
# Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
# initramfs.
inst_any() {
    local to f

    [[ $1 = '-d' ]] && to="$2" && shift 2

    for f in "$@"; do
        if [[ -e $f ]]; then
            [[ $to ]] && inst "$f" "$to" && return 0
            inst "$f" && return 0
        fi
    done

    return 1
}

# dracut_install [-o ] <file> [<file> ... ]
# Install <file> to the initramfs image
# -o optionally install the <file> and don't fail, if it is not there
dracut_install() {
    local _optional=no
    if [[ $1 = '-o' ]]; then
        _optional=yes
        shift
    fi
    while (($# > 0)); do
        if ! inst "$1" ; then
            if [[ $_optional = yes ]]; then
                dinfo "Skipping program $1 as it cannot be found and is" \
                    "flagged to be optional"
            else
                dfatal "Failed to install $1"
                exit 1
            fi
        fi
        shift
    done
}

# Install a single kernel module along with any firmware it may require.
# $1 = full path to kernel module to install
install_kmod_with_fw() {
    # no need to go further if the module is already installed

    [[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
        && return 0

    [[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0

    if [[ $omit_drivers ]]; then
        local _kmod=${1##*/}
        _kmod=${_kmod%.ko}
        _kmod=${_kmod/-/_}
        if [[ "$_kmod" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
        if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
    fi

    [ -d "$initdir/.kernelmodseen" ] && \
        > "$initdir/.kernelmodseen/${1##*/}"

    inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
        || return $?

    local _modname=${1##*/} _fwdir _found _fw
    _modname=${_modname%.ko*}
    for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
        _found=''
        for _fwdir in $fw_dir; do
            if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
                inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
                _found=yes
            fi
        done
        if [[ $_found != yes ]]; then
            if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
                dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            else
                dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            fi
        fi
    done
    return 0
}

# Do something with all the dependencies of a kernel module.
# Note that kernel modules depend on themselves using the technique we use
# $1 = function to call for each dependency we find
#      It will be passed the full path to the found kernel module
# $2 = module to get dependencies for
# rest of args = arguments to modprobe
# _fderr specifies FD passed from surrounding scope
for_each_kmod_dep() {
    local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
    shift 2
    modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
        while read _cmd _modpath _options; do
            [[ $_cmd = insmod ]] || continue
            $_func ${_modpath} || exit $?
            _found=1
        done
        [[ $_found -eq 0 ]] && exit 1
        exit 0
    )
}

# filter kernel modules to install certain modules that meet specific
# requirements.
# $1 = search only in subdirectory of /kernel/$1
# $2 = function to call with module name to filter.
#      This function will be passed the full path to the module to test.
# The behavior of this function can vary depending on whether $hostonly is set.
# If it is, we will only look at modules that are already in memory.
# If it is not, we will look at all kernel modules
# This function returns the full filenames of modules that match $1
filter_kernel_modules_by_path () (
    local _modname _filtercmd
    if ! [[ $hostonly ]]; then
        _filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
        _filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
        _filtercmd+=' -o -name "*.ko.xz"'
        _filtercmd+=' 2>/dev/null'
    else
        _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
        _filtercmd+='-k $KERNEL_VER 2>/dev/null'
    fi
    for _modname in $(eval $_filtercmd); do
        case $_modname in
            *.ko) "$2" "$_modname" && echo "$_modname";;
            *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
            *.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
        esac
    done
)
find_kernel_modules_by_path () (
    if ! [[ $hostonly ]]; then
        find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
          -name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
    else
        cut -d " " -f 1 </proc/modules \
        | xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
    fi
)

filter_kernel_modules () {
    filter_kernel_modules_by_path  drivers  "$1"
}

find_kernel_modules () {
    find_kernel_modules_by_path  drivers
}

# instmods [-c] <kernel module> [<kernel module> ... ]
# instmods [-c] <kernel subsystem>
# install kernel modules along with all their dependencies.
# <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
instmods() {
    [[ $no_kernel = yes ]] && return
    # called [sub]functions inherit _fderr
    local _fderr=9
    local _check=no
    if [[ $1 = '-c' ]]; then
        _check=yes
        shift
    fi

    function inst1mod() {
        local _ret=0 _mod="$1"
        case $_mod in
            =*)
                if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
                    | instmods
                else
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      find "$KERNEL_MODS" -path "*/${_mod#=}/*" -type f -printf '%f\n' ) \
                    | instmods
                fi
                ;;
            --*) _mpargs+=" $_mod" ;;
            i2o_scsi) return ;; # Do not load this diagnostic-only module
            *)
                _mod=${_mod##*/}
                # if we are already installed, skip this module and go on
                # to the next one.
                [[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return

                if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
                    dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
                    return
                fi
                # If we are building a host-specific initramfs and this
                # module is not already loaded, move on to the next one.
                [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
                    && ! echo $add_drivers | grep -qe "\<${_mod}\>" \
                    && return

                # We use '-d' option in modprobe only if modules prefix path
                # differs from default '/'.  This allows us to use Dracut with
                # old version of modprobe which doesn't have '-d' option.
                local _moddirname=${KERNEL_MODS%%/lib/modules/*}
                [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"

                # ok, load the module, all its dependencies, and any firmware
                # it may require
                for_each_kmod_dep install_kmod_with_fw $_mod \
                    --set-version $KERNEL_VER ${_moddirname} $_mpargs
                ((_ret+=$?))
                ;;
        esac
        return $_ret
    }

    function instmods_1() {
        local _mod _mpargs
        if (($# == 0)); then  # filenames from stdin
            while read _mod; do
                inst1mod "${_mod%.ko*}" || {
                    if [ "$_check" = "yes" ]; then
                        dfatal "Failed to install $_mod"
                        return 1
                    fi
                }
            done
        fi
        while (($# > 0)); do  # filenames as arguments
            inst1mod ${1%.ko*} || {
                if [ "$_check" = "yes" ]; then
                    dfatal "Failed to install $1"
                    return 1
                fi
            }
            shift
        done
        return 0
    }

    local _ret _filter_not_found='FATAL: Module .* not found.'
    set -o pipefail
    # Capture all stderr from modprobe to _fderr. We could use {var}>...
    # redirections, but that would make dracut require bash4 at least.
    eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
    | while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
    _ret=$?
    set +o pipefail
    return $_ret
}

setup_suse() {
    ln -fs ../usr/bin/systemctl $initdir/bin/
    ln -fs ../usr/lib/systemd $initdir/lib/
    inst_simple "/usr/lib/systemd/system/haveged.service"
    instmods ext4
}

_umount_dir() {
    if mountpoint -q $1; then
        ddebug "umount $1"
        umount $1
    fi
}

# can be overridden in specific test
test_setup_cleanup() {
    cleanup_initdir
}

_test_cleanup() {
    # (post-test) cleanup should always ignore failure and cleanup as much as possible
    (
        set +e
        _umount_dir $initdir
        rm -vf "$IMAGE_PUBLIC"
        # If multiple setups/cleans are ran in parallel, this can cause a race
        if [ ${TEST_PARALLELIZE} -ne 1 ]; then
            rm -vf "${IMAGESTATEDIR}/default.img"
        fi
        rm -vfr "$TESTDIR"
        rm -vf "$STATEFILE"
    ) || :
}

# can be overridden in specific test
test_cleanup() {
    _test_cleanup
}

test_cleanup_again() {
    [ -n "$TESTDIR" ] || return
    rm -rf "$TESTDIR/unprivileged-nspawn-root"
    _umount_dir $initdir
}

test_create_image() {
    create_empty_image_rootdir

    # Create what will eventually be our root filesystem onto an overlay
    (
        LOG_LEVEL=5
        setup_basic_environment
        mask_supporting_services
    )
}

test_setup() {
    if [ ${TEST_REQUIRE_INSTALL_TESTS} -ne 0 ] && \
            type -P meson >/dev/null && \
            [[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
        dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
        exit 1
    fi

    if [ -e "$IMAGE_PRIVATE" ]; then
        echo "Reusing existing image $IMAGE_PRIVATE → $(realpath $IMAGE_PRIVATE)"
        mount_initdir
    else
        if [ ! -e "$IMAGE_PUBLIC" ]; then
            # default.img is the base that every test uses and optionally appends to
            if [ ! -e "${IMAGESTATEDIR}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE}" ]; then
                # Create the backing public image, but then completely unmount
                # it and drop the loopback device responsible for it, since we're
                # going to symlink/copy the image and mount it again from
                # elsewhere.
                local image_old=${IMAGE_PUBLIC}
                if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
                    IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
                fi
                test_create_image
                test_setup_cleanup
                umount_loopback
                cleanup_loopdev
                IMAGE_PUBLIC="${image_old}"
            fi
            if [ "${IMAGE_NAME}" != "default" ] && [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
                cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
            fi
        fi

        local hook_defined=1
        if declare -f -F test_append_files > /dev/null; then
            hook_defined=$?
        fi

        echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath $IMAGE_PUBLIC)"
        if [ ${TEST_PARALLELIZE} -ne 0 ] || [ ${hook_defined} -eq 0 ]; then
            cp -v "$(realpath $IMAGE_PUBLIC)" "$IMAGE_PRIVATE"
        else
            ln -sv "$(realpath $IMAGE_PUBLIC)" "$IMAGE_PRIVATE"
        fi

        mount_initdir
        if [ ${hook_defined} -eq 0 ]; then
            test_append_files "$initdir"
        fi
    fi

    setup_nspawn_root
}

test_run() {
    mount_initdir

    if [ -z "$TEST_NO_QEMU" ]; then
        if run_qemu "$1"; then
            check_result_qemu || { echo "QEMU test failed"; return 1; }
        else
            dwarn "can't run QEMU, skipping"
        fi
    fi
    if [ -z "$TEST_NO_NSPAWN" ]; then
        mount_initdir
        if run_nspawn "$initdir" "$1"; then
            check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
        else
            dwarn "can't run systemd-nspawn, skipping"
        fi

        if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
            dir="$TESTDIR/unprivileged-nspawn-root"
            if NSPAWN_ARGUMENTS="-U --private-network $NSPAWN_ARGUMENTS" run_nspawn "$dir" "$1"; then
                check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
            else
                dwarn "can't run systemd-nspawn, skipping"
            fi
        fi
    fi
    return 0
}

do_test() {
    if [[ $UID != "0" ]]; then
        echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
        exit 0
    fi

    if [ -n "$TEST_NO_QEMU" ] && [ -n "$TEST_NO_NSPAWN" ]; then
        echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both QEMU and nspawn disabled" >&2
        exit 0
    fi

    if [ -n "$TEST_QEMU_ONLY" ] && [ -z "$TEST_NO_NSPAWN" ]; then
        echo "TEST: $TEST_DESCRIPTION [SKIPPED]: QEMU-only tests requested" >&2
        exit 0
    fi

    if [ -n "$TEST_PREFER_NSPAWN" ] && [ -z "$TEST_NO_NSPAWN" ]; then
        TEST_NO_QEMU=1
    fi

    # Detect lib paths
    [[ $libdir ]] || for libdir in /lib64 /lib; do
        [[ -d $libdir ]] && libdirs+=" $libdir" && break
    done

    [[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
        [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
    done

    mkdir -p "$STATEDIR"

    import_testdir
    import_initdir

    testname="$(basename $PWD)"

    while (($# > 0)); do
        case $1 in
            --run)
                echo "${testname} RUN: $TEST_DESCRIPTION"
                test_run "$2"
                ret=$?
                if (( $ret == 0 )); then
                    echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
                else
                    echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
                fi
                exit $ret;;
            --setup)
                echo "${testname} SETUP: $TEST_DESCRIPTION"
                test_setup
                test_setup_cleanup
                ;;
            --clean)
                echo "${testname} CLEANUP: $TEST_DESCRIPTION"
                test_cleanup
                ;;
            --clean-again)
                echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
                test_cleanup_again
                ;;
            --all)
                ret=0
                echo -n "${testname}: $TEST_DESCRIPTION "
                # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
                # and loop devices will leak
                test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
                if [ $ret -eq 0 ]; then
                    test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
                fi
                if [ $ret -eq 0 ]; then
                    test_run "$2" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
                fi
                test_cleanup
                if [ $ret -eq 0 ]; then
                    rm "$TESTLOG"
                    echo "[OK]"
                else
                    echo "[FAILED]"
                    echo "see $TESTLOG"
                fi
                exit $ret;;
            *) break ;;
        esac
        shift
    done
}