ffho-firmware-build/build.sh

#!/bin/bash
# (c) 2014-2018 Freifunk Hochstift <kontakt@hochstift.freifunk.net>
#
# This script builds the firmware by the environment variables given, the
# first two being mandatory:
#
# BASE        = Gluon Version (branch, tag or commit, i.e. v2014.4)
# BRANCH      = Firmware Branch (stable/testing/experimental)
# VERSION     = the version tag (can only be empty if BRANCH=experimental)
#
# optional:
# AUTOUPDATER = force Autoupdater Branch (stable/testing/experimental/off)
# BROKEN      = 0 (default) or 1, build the untested hardware model firmwares, too
# BUILD_TS    = build timestamp (format: %Y-%m-%d %H:%M:%S)
# CLEAN       = 'dirclean' to perform "make dirclean" before build or 'clean' to perform "make clean" (default: 'none')
# DEVICES     = build some specific devices and not the entire target
# KEY_DIR     = specify directory for gluon-opkg-key
# MAKEJOBS    = number of compiler processes running in parallel (default: number of CPUs/Cores)
# PRIORITY    = determines the number of day a rollout phase should last at most
# UPLOAD      = 0 (default) or 1, upload firmware at the end
# UPDATEGIT   = 0 (default) or 1, update firmware website git at the end
# SITE        = site repository version (branch, tag or commit, default: '$BRANCH')
# TARGETS     = a space separated list of target platforms (if unset, all platforms will be build)
# VERBOSE     = 0 (default) or 1, call the make commands with 'V=s' to see actual errors better
# VERSIONS_DIR = specify directory for version files
#


### includes
. functions.sh

### static variables
DEFAULT_BASE="v2021.1.2"
DEFAULT_BRANCH="experimental"
MY_DIR=$(dirname $0)
MY_DIR=$(readlink -f "${MY_DIR}")
DEFAULT_KEY_DIR="${MY_DIR}/opkg-keys"
DEFAULT_VERSIONS_DIR="${MY_DIR}/versions"
CODE_DIR="${MY_DIR}/src"
GLUON_BUILD_DIR="${CODE_DIR}/lede"
SITE_DIR="${CODE_DIR}/site"
PATCH_DIR="${SITE_DIR}/patches"
OUTPUT_DIR="${MY_DIR}/output"
IMAGE_DIR="${CODE_DIR}/output/images"
PACKAGES_DIR="${CODE_DIR}/output/packages"
MANIFEST_DIR="${CODE_DIR}/output/images/sysupgrade"
SITE_GEN_SCRIPT="${SITE_DIR}/scripts/gensites.py"
SRV_URL="firmware.in.ffho.net"
SRV_USER="firmware"
SRV_PATH="/srv/firmware"

BUILD_INFO_FILENAME="build-info.txt"
SITE_REPO_URL="https://git.ffho.net/freifunkhochstift/ffho-site.git"
GLUON_REPO_URL="https://github.com/freifunk-gluon/gluon.git"
LANG=C

pushd ${MY_DIR} > /dev/null

### set reasonable defaults for unset environment variables
[ -n "${BRANCH}" ] || BRANCH=${DEFAULT_BRANCH}
[ -n "${BASE}" ] || BASE=${DEFAULT_BASE}
[ -n "${AUTOUPDATER}" ] || AUTOUPDATER=${BRANCH}
[ "${BROKEN}" == "1" ] && export BROKEN || unset BROKEN
[ -n "${BUILD_TS}" ] || BUILD_TS=$(date +"%Y-%m-%d %H:%M:%S")
[ -n "${CLEAN}" ] || CLEAN="none"

if [ -n "${KEY_DIR}" ]; then
	KEY_DIR=$(readlink -f "${KEY_DIR}")
	[ -e "${KEY_DIR}" ] || abort "Can not find specified key directory: ${KEY_DIR}"
else
	KEY_DIR="${DEFAULT_KEY_DIR}"
fi

[ -n "${MAKEJOBS}" ] || MAKEJOBS=$(grep -c "^processor" /proc/cpuinfo)
[ -n "${PRIORITY}" ] || PRIORITY=0
[ -n "${UPLOAD}" ] || UPLOAD=0
[ -n "${UPDATEGIT}" ] || UPDATEGIT=0
[ -n "${SITE}" ] || SITE=${BRANCH}
[ -n "${VERBOSE}" ] || VERBOSE=0

if [ -n "${VERSIONS_DIR}" ]; then
	VERSIONS_DIR=$(readlink -f "${VERSIONS_DIR}")
else
	VERSIONS_DIR="${DEFAULT_VERSIONS_DIR}"
fi

MAKE_PARAM=""
[ "${VERBOSE}" -eq "1" ] && MAKE_PARAM="${MAKE_PARAM} V=s"
[ -n "${DEVICE}" ] && MAKE_PARAM="${MAKE_PARAM} DEVICE=${DEVICE}"

### ERROR handling
[ -n "${BASE}" ] || abort "Please specify BASE environment variable (Gluon, i.e. 'v2014.3' or commit-id)."
[ "${BASE}" == "HEAD" ] && abort "HEAD is not an allowed BASE-identifier. Either use a branch, a tagged commit or the commit-SHA itself."
[ -n "${BRANCH}" ] || abort "Please specify BRANCH environment variable."
[ -n "${VERSION}" -o "${BRANCH}" == "experimental" ] || abort "Please specify VERSION environment variable (not necessary for experimental branch)."
[ "${BRANCH}" == "experimental" -o ! -r "${VERSIONS_DIR}/${VERSION}" ] || abort "There exists a version file for '${VERSION}' ... you are trying to do something really stupid, aren't you?"

### init VERSIONS_DIR if necessary
if [ ! -d "${VERSIONS_DIR}" ]; then
	info "Versions directory does not exist yet - creating it ..."
	mkdir -p ${VERSIONS_DIR}
	[ "$?" -eq "0" ] || abort "Unable to create versions directory: ${VERSIONS_DIR}"
fi

### init CODE_DIR if necessary
if [ ! -d "${CODE_DIR}" ]; then
	info "Code directory does not exist yet - fetching Gluon ..."
	git clone "${GLUON_REPO_URL}" "${CODE_DIR}"
	[ "$?" -eq "0" ] || abort "Failed to fetch Gluon repository."
fi

### init SITE_DIR if necessary
if [ ! -d "${SITE_DIR}" ]; then
	info "Site repository does not exist, fetching it ..."
	git clone "${SITE_REPO_URL}" "${SITE_DIR}"
	[ "$?" -eq "0" ] || abort "Failed to fetch SITE repository."
fi


### CHECKOUT GLUON
progress "Checking out GLUON '${BASE}' ..."
pushd ${CODE_DIR} > /dev/null
# check if gluon got modified and bail out if necessary
[ "$(git status --porcelain)" ] && abort "Local changes to gluon directory. Cowardly refusing to update repository." >&2
git fetch
git show-ref --verify --quiet refs/remotes/origin/${BASE}
if [ "$?" -eq "0" ]; then
	git checkout -B build origin/${BASE}
	[ "$?" -eq "0" ] || abort "Failed to checkout gluon origin/${BASE}."
else
	git checkout -q ${BASE}
	[ "$?" -eq "0" ] || abort "Failed to checkout gluon ${BASE}." >&2
fi
GLUON_COMMIT=$(git rev-list --max-count=1 HEAD)
popd > /dev/null #${CODE_DIR}


### CHECKOUT SITE REPO
progress "Checking out SITE REPO ..."
pushd ${SITE_DIR} > /dev/null
# check if site repo got modified and bail out if necessary
[ "$(git status --porcelain)" ] && abort "Local changes to site directory. Cowardly refusing to update repository." >&2
git fetch
git show-ref --verify --quiet refs/remotes/origin/${SITE}
if [ "$?" -eq "0" ]; then
	git checkout -B build origin/${SITE}
	[ "$?" -eq "0" ] || abort "Failed to checkout site repo 'origin/${SITE}'."
else
	git checkout -q ${SITE}
	[ "$?" -eq "0" ] || abort "Failed to checkout site repo '${SITE}'." >&2
fi
SITE_COMMIT=$(git rev-list --max-count=1 HEAD)
popd > /dev/null #${SITE_DIR}

pushd ${CODE_DIR} > /dev/null

### APPLY PATCHES TO GLUON
progress "Applying Patches ..."
git checkout -B patching "${BASE}"
if [ -d "${PATCH_DIR}" -a "$(echo ${PATCH_DIR}/*.patch)" ]; then
	git am --whitespace=nowarn ${PATCH_DIR}/*.patch || (
		git am --abort
		git checkout patched
		git branch -D patching
		false
	)
	[ "$?" -eq "0" ] || abort "Failed to apply patches, mimimi."
fi
git branch -M patched

### DIRCLEAN
if [ -d "${GLUON_BUILD_DIR}/" -a "${CLEAN}" == "dirclean" ]; then
	progress "Cleaning your build environment (make dirclean) ..."
	make dirclean
fi

### PREPARE
progress "Preparing the build environment (make update) ..."
make update
[ "$?" -eq "0" ] || abort "Failed to update the build environment, mimimi."

### set reasonable defaults for TARGETS and VERSION if unset
if [ -z "${TARGETS}" ]; then
	TARGETS=$(make list-targets | sed ':a;N;$!ba;s/\n/ /g')
	info "building all targets: '${TARGETS}'"
fi
if [ -z "${VERSION}" ] ; then
	VERSION=$(make show-release)
	info "${BRANCH} firmware: using version tag '${VERSION}'"
fi
popd > /dev/null #${CODE_DIR}

# we are now ready to produce the firmware images, so let's "save" our state
BUILD_INFO="${VERSIONS_DIR}/${VERSION}"
progress "Saving build information to: ${BUILD_INFO}"
mkdir -p ${VERSIONS_DIR}
[ "$?" -eq "0" ] || abort "Unable to create output directory: ${VERSIONS_DIR}"
touch ${BUILD_INFO}
[ "$?" -eq "0" ] || abort "Cannot create build information file: ${BUILD_INFO}"
echo "VERSION='${VERSION}'"              >  ${BUILD_INFO}
echo "GLUON='${GLUON_COMMIT}' # ${BASE}" >> ${BUILD_INFO}
echo "BRANCH='${BRANCH}'"                >> ${BUILD_INFO}
echo "SITE='${SITE_COMMIT}'  # ${SITE}"  >> ${BUILD_INFO}
echo "TARGETS='${TARGETS}'"              >> ${BUILD_INFO}
echo "TS='${BUILD_TS}'"                  >> ${BUILD_INFO}

### restore opkg-keys
if [ -e "${KEY_DIR}" ]; then
	info "build key already exists, restoring it."
	mkdir -p ${GLUON_BUILD_DIR}/
	[ "$?" -eq "0" ] || abort "Unable to create directory: ${GLUON_BUILD_DIR}/"
	cp -f ${KEY_DIR}/* ${GLUON_BUILD_DIR}/
	[ "$?" -eq "0" ] || abort "Unable to copy build key."
fi

### create site.conf
if [ -x "${SITE_GEN_SCRIPT}" ]; then
	progress "Generating defaut site.conf"
	eval GLUON_SITEDIR=${SITE_DIR} ${SITE_GEN_SCRIPT}
fi

### BUILD FIRMWARE
progress "Building the firmware - please stand by!"
pushd ${CODE_DIR} > /dev/null

export GLUON_RELEASE="${VERSION}"
if [ "${AUTOUPDATER}" != "off" ] ; then
	export GLUON_AUTOUPDATER_BRANCH="${AUTOUPDATER}"
	export GLUON_AUTOUPDATER_ENABLED=1
fi

for target in ${TARGETS} ; do
	[[ "$EXCLUDE_TARGETS" =~ "$target" ]] && continue
	# configure build environment for our current target
	export GLUON_TARGET="${target}"

	# prepare build environment for our current target
	if [ "${CLEAN}" == "clean" ]; then
		progress "${target}: Preparing build environment. (make clean)"
		make clean
		[ "$?" -eq "0" ] || abort "${target}: Unable to clean environment."
	fi

	# now we can start building the images for the target platform
	progress "${target}: Building FFHO-flavoured Gluon firmware. You'd better go and fetch some c0ffee!"
	make -j ${MAKEJOBS} ${MAKE_PARAM}
	[ "$?" -eq "0" ] || abort "${target}: Unable to build firmware."
done

# generate manifest
progress "Generating manifest ..."
GLUON_PRIORITY=${PRIORITY} GLUON_BRANCH=${BRANCH} make manifest
[ "$?" -eq "0" ] || abort "Failed to generate the manifest."

popd > /dev/null #${CODE_DIR}

if [ "${UPLOAD}" -eq "1" ]; then
	# copying firmware to the server
	if [ -d "${IMAGE_DIR}" ]; then
		progress "Copying firmware images ..."
		rsync -rlutzc  --filter="+ */" --filter="+ *-${VERSION}-*" --filter="- *" -e ssh ${IMAGE_DIR}/ ${SRV_USER}@${SRV_URL}:${SRV_PATH}/${VERSION}
		[ "$?" -eq "0" ] || abort "Failed to copy firmware images."
	fi

	# copying packages to the server
	PKG_SUB_DIR=$(find ${PACKAGES_DIR}/ -maxdepth 1 -name *${VERSION})
	if [ -d "${PKG_SUB_DIR}" ]; then
		progress "Copying packages ..."
		rsync -rlutzc -e ssh ${PKG_SUB_DIR}/ ${SRV_USER}@${SRV_URL}:${SRV_PATH}/${VERSION}/packages
		[ "$?" -eq "0" ] || abort "Failed to copy packages."
	fi
fi
if [ "${UPDATEGIT}" -eq "1" ]; then
	# copy manifest and the build info file
	progress "Copying manifest and build info file ..."
	pushd ${OUTPUT_DIR} > /dev/null
	git fetch
	git checkout -q signing
	git merge --ff-only origin/master
	mkdir -p "${OUTPUT_DIR}/${VERSION}/sysupgrade"
	cp -f "${MANIFEST_DIR}/${BRANCH}.manifest" "${OUTPUT_DIR}/${VERSION}/sysupgrade/"
	cp -f "${BUILD_INFO}" "${OUTPUT_DIR}/${VERSION}/${BUILD_INFO_FILENAME}"
	popd > /dev/null #${OUTPUT_DIR}

	# Update symlinks
	progress "Update symlink for ${BRANCH}"
	pushd ${OUTPUT_DIR} > /dev/null
	ln -fsn ${VERSION} ${BRANCH}
	popd > /dev/null #${OUTPUT_DIR}
fi

# The end. Finally.
success "We're done, go and enjoy your new firmware (${VERSION})!"
popd > /dev/null #${MY_DIR}