new appimg builder framework

appimg-builder/README

@@ -0,0 +1,87 @@
+Application Image Builder
+=========================
+
+Application Images (or appimgs for short) are created with this builder
+framework. The build is controlled by a configuration file and this
+configuration file is actually a shell script that follows the conventions
+described in the Configuration File section of this document.
+
+Stage One
+---------
+
+The Stage One builder uses debootstrap to build a very minimal debian
+installation.  Then a chroot is set up and stage-two.sh is executed inside the
+chroot to perform most of the installation.
+
+Stage Two
+---------
+
+The stage-two.sh script mostly just orchestrates the execution of small
+fragments of shell script code that are called 'modules'.  The base framework
+modules can be found in the directory /usr/share/appimg-builder/appimg-modules.
+
+It imports the configuration file with the 'source' command after all the key
+variables and functions have been defined.  It's possible to override any of
+these variables and functions simply by defining another version with the same
+name in the configuration file, but you should almost never need to do this.
+
+Configuration File
+------------------
+
+- Variables
+
+PACKAGES can be set to a list of additional packages to add to the base set of
+packages.
+
+  PACKAGES="extremetuxracer biff anarchism" 
+
+PRE_INSTALL_MODULES can be set to a list of modules to run before the main
+package installation stage happens.  The contents will be appended to a
+pre-defined list of 'base' modules that run.
+
+  PRE_INSTALL_MODULES="my-cool-module another-module"
+
+If complete control over the modules to run is required, you can override the
+variable BASE_PRE_INSTALL_MODULES entirely rather than providing
+PRE_INSTALL_MODULES.  Other modules depend on 'utility-library' so it is usually
+required and should be the first module listed.
+
+  BASE_PRE_INSTALL_MODULES="utility-library configure-locale custom-create-user"
+
+POST_INSTALL_MODULES is a list of modules to execute after packages have been
+installed.  It works exactly the same way as PRE_INSTALL_MODULES and also has a
+corresponding 'base' variable that could be overidden if necessary.
+
+- Modules
+
+Modules can be functions that you define or they can be loaded from files on
+disk. To use files rather than functions a directory named 'appimg-modules'
+must exist as a subdirectory of the directory containing the configuration file.
+Any files you place in this directory will be found by name during the module
+execution stages.
+
+- Installing Files
+
+If you would like to have external files such as configuration files copied into
+the image, create 'appimg-files' as a subdirectory of the directory containing
+the configuration file.  You can then use the install_file command inside of a
+module to copy the files from this directory. You can either store the files to
+install in a flat directory or organize them into subdirectories mirroring the
+location in which they will be installed.  Depending on which option you use,
+the install_file command how two different modes.  In the examples below BASE
+refers to the directory in which your configuration file is located.
+
+   (1): install_file [mode] [file] [target directory]
+  
+   Example: Install BASE/appimg-files/my_config.conf 
+              to /etc/mydaemon/my_config.conf 
+  
+        install_file 0644 my_config.conf /etc/mydaemon
+     
+   (2): install_file [mode] [full path] 
+  
+   Example: Install BASE/appimg-files/etc/mydaemon/my_config.conf 
+              to /etc/mydaemon/my_config.conf
+  
+        install_file 0644 /etc/mydaemon/my_config.conf
+  

appimg-builder/appimg-files/configure-host0.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run script to configure host0 interface
+Before=network.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/libexec/configure-host0.sh
+
+[Install]
+WantedBy=sysinit.target

appimg-builder/appimg-files/configure-host0.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# inspired by last section of
+# 
+#     https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+#
+SYSTEMD_ENV=$(xargs -a /proc/1/environ --null echo)
+
+process_var() {
+    case ${1} in
+        "IFCONFIG_IP")
+            echo "IP: ${2}"
+            ip addr add ${2} dev host0
+            ip link set host0 up
+            ;;
+        "IFCONFIG_GW")
+            echo "GW: ${2}"
+            ip route add default via ${2}
+            ;;
+    esac
+}
+
+for var in ${SYSTEMD_ENV}; do
+    IFS="=" read -a PAIR <<< ${var}
+    if [[ ${#PAIR[@]} -eq 2 ]]; then
+        process_var ${PAIR[0]} ${PAIR[1]}
+    fi
+done

appimg-builder/appimg-files/launch

@@ -0,0 +1,7 @@
+#!/bin/bash
+export DISPLAY=:0
+export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus
+export XDG_RUNTIME_DIR=/run/user/1000
+export PULSE_SERVER=unix:/run/user/host/pulse/native
+export GNOME_DESKTOP_SESSION_ID=this-is-deprecated
+$@

appimg-builder/appimg-modules/configure-locale

@@ -0,0 +1,6 @@
+info "Setting up locale for en_US.UTF8"
+
+echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
+locale-gen
+update-locale LANG=en_US.UTF-8 LC_COLLATE=C
+export LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8

appimg-builder/appimg-modules/configure-systemd

@@ -0,0 +1,16 @@
+info "Adding configuration files to /etc/systemd"
+
+local userconfdir=/etc/systemd/user.conf.d
+local loginconfdir=/etc/systemd/logind.conf.d
+
+mkdir -p ${userconfdir} ${loginconfdir}
+
+cat > ${userconfdir}/50-display-env.conf << 'EOF'
+[Manager]
+DefaultEnvironment="DISPLAY=:0"
+EOF
+
+cat > ${loginconfdir}/50-no-kill-user-processes.conf << EOF
+[Login]
+KillUserProcesses=no
+EOF

appimg-builder/appimg-modules/create-user

@@ -0,0 +1,6 @@
+info "Creating user account"
+
+useradd -s /bin/bash -m user
+echo "user:user" | chpasswd 
+usermod -aG sudo user
+echo "export DISPLAY=:0" >> /home/user/.bashrc

appimg-builder/appimg-modules/install-configure-host0

@@ -0,0 +1,5 @@
+info "Installing scripts to configure host0 network interface"
+
+install_file 0755 configure-host0.sh /usr/libexec
+install_file 0644 configure-host0.service /usr/lib/systemd/system
+systemctl -q enable configure-host0.service

appimg-builder/appimg-modules/install-launch-script

@@ -0,0 +1,2 @@
+info "Installing launch script to /usr/libexec/launch"
+install_file 0755 launch /usr/libexec

appimg-builder/appimg-modules/install-packages

@@ -0,0 +1,9 @@
+info "Beginning package installation"
+
+apt-get update
+apt-get --assume-yes upgrade
+apt-get --assume-yes install ${PACKAGES}
+
+printf "\n\nInstalled Packages\n\n"
+dpkg -l
+printf "\n\n"

appimg-builder/appimg-modules/loginctl-enable-linger

@@ -0,0 +1,6 @@
+info "Setting enable-linger for user account"
+# Otherwise gnome-terminal won't work if nothing else has been launched
+# see 'enable-linger' in loginctl(1)
+local lingerdir="/var/lib/systemd/linger"
+mkdir -p ${lingerdir}
+touch ${lingerdir}/user

appimg-builder/appimg-modules/no-install-recommends

@@ -0,0 +1,4 @@
+info "Disabling Install-Recommends in apt.conf"
+
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";'  >> /etc/apt/apt.conf.d/99no-install-recommends

appimg-builder/appimg-modules/set-hostname

@@ -0,0 +1,4 @@
+local hostname=${APPIMG_HOSTNAME:-"subgraph"}
+info "Setting hostname to '$hostname' in /etc/hosts and /etc/hostname"
+printf "127.0.0.1\t${hostname} localhost\n" > /etc/hosts
+printf "${hostname}\n" > /etc/hostname

appimg-builder/appimg-modules/utility-library

@@ -0,0 +1,67 @@
+
+info "utility library loaded"
+SEARCH_PATH="/tmp/appimg-build;${APPIMG_BUILDER_BASE}"
+
+# Searches for modules and files in SEARCH_PATH directories
+#
+#   search_item module <module name>
+#   search_item file   <file name>
+#
+search_item() {
+    local IFS=";" subtype=${1} item=${2}
+    local subdir="appimg-${subtype}s"
+
+    for dir in ${SEARCH_PATH}; do
+        local fullpath=${dir}/${subdir}/${item}
+        if [[ -f ${fullpath} ]]; then
+            printf ${fullpath}
+            return
+        fi
+    done
+
+    fatal "Could not find ${subtype} named '${item}'. Bailing.."
+}
+
+#
+# Execute a module by name.  If a function exists with the
+# requested module name it will be executed, otherwise scripts 
+# with this name are searched for in SEARCH_PATH /appimg-modules
+# subdirectories.
+#
+
+module() {
+    if [[ $(type -t ${1}) == "function" ]]; then
+        local fname=${1}
+        ${fname} 
+    else
+        local modpath=$(search_item module ${1})
+        source ${modpath}
+    fi
+}
+
+#
+# See README for more information about this function
+#
+install_file() {
+
+    local mode=${1}
+
+    if [[ $# -eq 3 ]]; then
+        # Type (1)
+        # strip trailing / from ${3} if present
+        local target=${3%/}/${2}
+    elif [[ $# -eq 2 ]]; then
+        # Type (2)
+        # accept either /usr/bin/foo or usr/bin/foo
+        # add leading slash if necessary to construct absolute path
+        local target="/${2#/}"
+    else
+       fatal "Bad number of arguments $# to install_file()"
+    fi
+
+    # strip leading slash from ${2} if present
+    local srcpath=$(search_item file ${2#/})
+    local dstdir=$(dirname ${target})
+    /usr/bin/install -d ${dstdir}
+    /usr/bin/install -m ${mode} ${srcpath} ${target}
+}

appimg-builder/appimg-modules/write-apt-sources

@@ -0,0 +1,8 @@
+info "Writing /etc/apt/sources.list"
+
+{
+    echo "deb ${DEBIAN_MIRROR} ${DEBIAN_RELEASE} main contrib non-free"
+    echo "deb ${DEBIAN_MIRROR}-security ${DEBIAN_RELEASE}/updates main contrib non-free"
+    echo "deb ${DEBIAN_MIRROR} unstable main"
+
+} > /etc/apt/sources.list

appimg-builder/basic-image.conf

@@ -0,0 +1,2 @@
+PACKAGES="man manpages vim-nox iputils-ping tmux vifm gnome-terminal firefox nautilus eog evince unzip x264"
+

appimg-builder/build-template.conf

@@ -0,0 +1,79 @@
+#
+#  Application Image Builder configuration file template
+#
+#  Synopsis:
+#
+#    mkdir work && cd work
+#    appimg-builder --new
+#    vim build.conf
+#    sudo appimg-builder build.conf
+#
+#  Full documentation in /usr/share/appimg-builder/README
+#
+
+# 
+# Add space separated list of extra packages to install to PACKAGES.  
+#
+# If you need full control over the packages installed, you can set 
+# the variable BASE_PACKAGES and it will override the value of this
+# variable in the main builder.
+#
+
+PACKAGES="man manpages vim-nox iputils-ping tmux gnome-terminal firefox nautilus evince unzip x264"
+
+# 
+# These variables are set to the default values, if you don't want to change them 
+# they can just be deleted.  Or you can keep them, the result will be the same.
+#
+
+DEBIAN_RELEASE="buster"
+DEBIAN_MIRROR="https://deb.debian.org/debian"
+
+#
+# All the following is about how you can add code to run either 
+# before or after packages are installed.  
+#
+# If you just want to add some extra packages to install in your image, you don't 
+# need any of this and it can all be deleted.
+# 
+
+#
+# PRE_INSTALL_MODULES run before packages are installed
+#
+
+PRE_INSTALL_MODULES=""
+
+#
+# POST_INSTALL_MODULES run after packages are installed
+#
+
+POST_INSTALL_MODULES="example-module another-module"
+
+#
+# Writing shell code functions inside your config file is the easiest way to 
+# implement modules, but you can also put module code into a directory of 
+# external files.  See README for information about how to do that.
+#
+
+example-module() {
+    info "Hello, this is example-module, configuring password-less sudo for user account"
+    # add a sudoers config line
+    echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/10-no-password-for-user
+}
+
+#
+# If you add a subdirectory called 'appimg-files' to the 
+# directory containing this build config file and then 
+# add a file 'example.service':
+#
+#   THISDIR/appimg-files/example.service
+#
+# You can then install it by running install_file in a module 
+# like this:
+#
+
+another-module() {
+    info "This is another example module"
+    # install_file 0644 example.service /usr/lib/systemd/system
+    # systemctl enable example.service
+}

appimg-builder/common.inc

@@ -0,0 +1,21 @@
+
+: ${DEBIAN_MIRROR:="https://deb.debian.org/debian"}
+: ${DEBIAN_RELEASE:="buster"}
+
+BASE_PACKAGES="debootstrap iproute2 less xz-utils sudo dbus libpam-systemd openssh-client packagekit-gtk3-module libcanberra-gtk3-module libpulse0 fonts-roboto-hinted bash-completion"
+
+BASE_PRE_INSTALL_MODULES="utility-library configure-locale create-user set-hostname no-install-recommends write-apt-sources"
+
+BASE_POST_INSTALL_MODULES="loginctl-enable-linger install-configure-host0 install-launch-script"
+
+trap "exit 1" TERM
+export TOP_PID=$$
+
+fatal() {
+    >&2 echo "Fatal: ${1}"
+    kill -s TERM $TOP_PID
+}
+
+info() {
+    printf "[+] ${1}\n"
+}

appimg-builder/stage-one.sh

@@ -0,0 +1,237 @@
+#!/bin/bash
+
+: ${APPIMG_BUILDER_BASE:="/usr/share/appimg-builder"}
+source ${APPIMG_BUILDER_BASE}/common.inc
+
+umount_if_tmpfs() {
+    if findmnt -t tmpfs -M ${1} > /dev/null; then
+        info "unmounting existing rootfs tmpfs"
+        umount ${1}
+    fi
+}
+
+setup_rootfs() {
+    umount_if_tmpfs ${ROOTFS}
+    if [[ -d ${ROOTFS} ]]; then
+        info "Removing existing rootfs at ${ROOTFS}"
+        rm -rf ${ROOTFS}
+    fi
+
+    mkdir --parents ${ROOTFS}
+
+    if [[ ${USE_TMPFS} -eq 1 ]]; then
+        mount -t tmpfs rootfs-tmp ${ROOTFS}
+    fi
+}
+
+run_debootstrap() {
+
+    [[ -f ${CACHE_DIR}/lock ]] && rm -f ${CACHE_DIR}/lock
+    mkdir --parents ${CACHE_DIR} ${ROOTFS}/var/cache/apt/archives
+
+    info "Bind mounting ${CACHE_DIR} to ${ROOTFS}/var/cache/apt/archives"
+    mount --bind ${CACHE_DIR} ${ROOTFS}/var/cache/apt/archives
+
+    info "Launching debootstrap"
+
+    debootstrap --verbose --merged-usr --variant=minbase \
+        --include=systemd-sysv,locales \
+        --exclude=sysv-rc,initscripts,startpar,lsb-base,insserv \
+        ${DEBIAN_RELEASE} ${ROOTFS} ${DEBIAN_MIRROR}
+}
+
+setup_chroot() {
+
+    mount chproc ${ROOTFS}/proc -t proc
+    mount chsys ${ROOTFS}/sys -t sysfs
+    mount chtmp ${ROOTFS}/tmp -t tmpfs
+
+    # Install a copy of appimg-builder inside new image
+    mkdir -p ${ROOTFS}/usr/share
+    cp -a ${APPIMG_BUILDER_BASE} ${ROOTFS}/usr/share
+    ln -s /usr/share/appimg-builder/stage-one.sh ${ROOTFS}/usr/bin/appimg-builder
+
+    # $BUILDFILE and any extra files go in /tmp/appimg-build of rootfs
+    mkdir -p ${ROOTFS}/tmp/appimg-build
+    cp ${BUILDFILE} ${ROOTFS}/tmp/appimg-build/build.conf
+}
+
+cleanup_chroot() {
+    umount ${ROOTFS}/proc
+    umount ${ROOTFS}/sys
+    umount ${ROOTFS}/tmp
+    umount ${ROOTFS}/var/cache/apt/archives
+
+    # Remove cache files in case we are creating a tarball for distribution
+    rm -f ${ROOTFS}/var/cache/apt/pkgcache.bin
+    rm -f ${ROOTFS}/var/cache/apt/srcpkgcache.bin
+}
+
+run_chroot_stage() {
+
+    setup_chroot
+
+    #
+    # Run second-stage.sh inside chroot(), pass in various environment variables
+    #
+
+    DEBIAN_FRONTEND=noninteractive \
+        DEBCONF_NONINTERACTIVE_SEEN=true \
+        LC_ALL=C LANGUAGE=C LANG=C \
+        DEBIAN_RELEASE=${DEBIAN_RELEASE} DEBIAN_MIRROR=${DEBIAN_MIRROR} \
+        chroot ${ROOTFS} /usr/share/appimg-builder/stage-two.sh /tmp/appimg-build/build.conf
+
+    info "chroot installation stage finished, cleaning chroot setup"
+    cleanup_chroot
+}
+
+generate_tarball() {
+    local tarball=${WORKDIR}/appimg-rootfs.tar
+
+    info "----- Generating rootfs tarball -----"
+    tar -C ${ROOTFS} --numeric-owner -c --xattrs --xattrs-include=* -f $tarball .
+    if [[ ${DO_XZ} -eq 1 ]]; then
+        info "Compressing $tarball"
+        xz --force --threads=0 $tarball
+        tarball=${tarball}.xz
+    fi
+    echo
+    ls -hl $tarball
+    echo
+}
+
+usage() {
+cat <<-EOF
+USAGE: appimg-builder [options] [config-file]
+
+OPTIONS
+
+    --new               Create a configuration file template called build.conf in the current directory
+    -d <directory>      Choose a non-default directory for build output (currently: $(pwd)/appimg)
+    -t                  Create a tarball but don't compress it
+    -z                  Create a tarball compressed with xz
+    --no-tmpfs          Do not use tmpfs as rootfs build directory
+    --no-confirm        Do not ask for confirmation before beginning
+
+For more documentation see /usr/share/appimg-builder/README
+
+EOF
+exit 0
+}
+
+ask_confirm() {
+    local use_tmpfs="No"
+    [[ ${USE_TMPFS} -eq 1 ]] && use_tmpfs="Yes"
+
+    printf "About to build application image with the following parameters:\n\n"
+    printf "\tBuild Configuration File : ${BUILDFILE}\n"
+    printf "\tOutput rootfs directory  : ${ROOTFS}\n"
+    printf "\tBuild rootfs on tmpfs    : ${use_tmpfs}\n"
+    [[ -e ${ROOTFS} ]] && printf "\nWarning: rootfs directory from a prior build exists and will be deleted before building new image\n\n"
+
+    read -p "Ok to proceed? (y/N): " confirm
+
+    [[ $confirm =~ ^[Yy] ]] || exit 0
+}
+
+try_config() {
+    local rp
+    rp=$(realpath ${1} 2> /dev/null) || return 1
+    [[ -f ${rp} ]] || return 1
+    printf "${rp}"
+}
+    
+WORKDIR="$(pwd)/appimg"
+
+DO_TAR=0
+DO_XZ=0
+USE_TMPFS=1
+NO_CONFIRM=0
+
+while [[ $# -gt 0 ]]; do
+    key=${1}
+    case $key in
+        -d)
+            WORKDIR="$(realpath ${2})"
+            shift
+            shift
+            ;;
+
+        -t)
+            DO_TAR=1
+            shift
+            ;;
+
+        -z)
+            DO_TAR=1 DO_XZ=1
+            shift
+            ;;
+
+        --no-tmpfs)
+            USE_TMPFS=0
+            shift
+            ;;
+
+        --no-confirm)
+            NO_CONFIRM=1
+            shift
+            ;;
+
+        --new)
+            cp --verbose ${APPIMG_BUILDER_BASE}/build-template.conf build.conf
+            exit 0
+            ;;
+
+        --help|-h|help)
+            usage
+            ;;
+        -*)
+            printf "Unknown option ${key}\n"
+            usage
+            ;;
+
+        *)
+            BUILDFILE=$(try_config ${1}) || fatal "Cannot find config file '${1}'"
+            shift
+            ;;
+    esac
+done
+
+if [ "$EUID" -ne 0 ]; then
+    echo "The appimg-builder must be run with root privileges."
+    exit 1
+fi
+
+if [[ -z ${BUILDFILE} ]]; then
+    BUILDFILE=$(try_config "${PWD}/build.conf" || try_config "${APPIMG_BUILDER_BASE}/basic-image.conf") || fatal "Could not find a configuration file to use"
+fi
+
+ROOTFS=${WORKDIR}/rootfs
+CACHE_DIR=${WORKDIR}/var-cache-apt-archives
+
+[[ ${NO_CONFIRM} -ne 1 ]] && ask_confirm
+
+# black magick from stack overflow
+exec > >(tee -a $WORKDIR/appimg-build.log) 2>&1
+
+info "Starting build of application image from configuration file ${BUILDFILE}"
+
+source ${BUILDFILE}
+
+setup_rootfs
+
+run_debootstrap
+
+run_chroot_stage
+
+info "rootfs build is completed:"
+info "    $(du -sh ${ROOTFS})"
+
+if [[ ${DO_TAR} -eq 1 ]]; then
+    generate_tarball
+fi
+
+if [[ ${USE_TMPFS} -eq 1 ]]; then
+    info "Root directory is a tmpfs. To remove mount and directory contents:"
+    info "    sudo umount ${ROOTFS}"
+fi

appimg-builder/stage-two.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+APPIMG_BUILDER_BASE="/usr/share/appimg-builder"
+source ${APPIMG_BUILDER_BASE}/common.inc
+
+if [ "$EUID" -ne 0 ]; then
+    fatal "The stage-two.sh script is not running as root."
+fi
+
+if [ $# -ne 1 ]; then
+    fatal "The stage-two.sh script expects a single argument (configuration file path)"
+fi
+
+if [ ! -f ${1} ]; then
+    fatal "Configuration file '${1}' does not exist."
+fi
+
+# running module 'utility-library' replaces this function with a more powerful version
+module() {
+    local modpath=${APPIMG_BUILDER_BASE}/appimg-modules/${1}
+    [[ -f ${modpath} ]] || fatal "Could not find module '${1}'"
+    source ${modpath}
+}
+
+pre_install_packages() {
+    info "Running pre package install modules"
+    for mod in ${PRE_INSTALL_MODULES}; do
+        module ${mod}
+    done
+}
+
+post_install_packages() {
+    info "Running post package install modules"
+    for mod in ${POST_INSTALL_MODULES}; do
+        module ${mod}
+    done
+}
+
+run_build() {
+    pre_install_packages
+    module install-packages
+    post_install_packages
+}
+
+
+# config file imported here
+source ${1}
+
+#
+# Define any of PACKAGES, PRE_INSTALL_MODULES, POST_INSTALL_MODULES in config file
+# to append to the corresponding BASE list.  Also you can override the BASE lists
+# entirely by setting the variable name in config file.
+#
+PACKAGES="${BASE_PACKAGES} ${PACKAGES:-}"
+PRE_INSTALL_MODULES="${BASE_PRE_INSTALL_MODULES} ${PRE_INSTALL_MODULES:-}" 
+POST_INSTALL_MODULES="${BASE_POST_INSTALL_MODULES} ${POST_INSTALL_MODULES:-}"
+
+run_build