modify build.sh for new gluon-firmware

build.sh

@@ -1,158 +1,141 @@
 #!/bin/bash
-# (c) 2014-2015 Freifunk Paderborn <maschinenraum@paderborn.freifunk.net>
+# (c) 2014-2016 Freifunk Hochstift <kontakt@hochstift.freifunk.net>
 #
 # This script builds the firmware by the environment variables given, the
 # first two being mandatory:
 #
-# BASE      = Gluon Version (tag or commit, i.e. v2014.4)
-# BRANCH    = Firmware Branch (stable/testing/experimental)
-# SITE      = site repository to use
-# SITE_ID   = specific site repository commit-id (leave blank to use HEAD)
-# VERSION   = the version tag (can only be empty if BRANCH=experimental)
-# BUILD_TS  = build timestamp (format: %Y-%m-%d %H:%M:%S)
-# BROKEN    = 0 (default) or 1, build the untested hardware model firmwares, too
-# MAKEJOBS  = number of compiler processes running in parallel (default: number of CPUs/Cores)
-# TARGETS   = a space separated list of target platforms (if unset, all platforms will be build)
-# PRIORITY  = determines the number of day a rollout phase should last at most
-# VERBOSE   = 0 (default) or 1, call the make commands with 'V=s' to see actual errors better
-# SITE_REPO_FETCH_METHOD = http, everything except "git" will use the HTTP method for fetchting site repo
+# BASE        = Gluon Version (tag or commit, i.e. v2014.4)
+# BRANCH      = Firmware Branch (stable/testing/experimental)
+# VERSION     = the version tag (can only be empty if BRANCH=experimental)
 #
+# optional:
+# AUTOUPDATER = force Autoupdater Branch (stable/testing/experimental/off)
+# BROKEN      = 0 (default) or 1, build the untested hardware model firmwares, too 
+# BUILD_TS    = build timestamp (format: %Y-%m-%d %H:%M:%S)
+# DIRCLEAN    = 1 perform "make dirclean" before build (BRANCH==stable/testing) or 0 perform "make clean" (BRANCH==experimental)
+# MAKEJOBS    = number of compiler processes running in parallel (default: number of CPUs/Cores)
+# PRIORITY    = determines the number of day a rollout phase should last at most
+# SITE_ID     = specific site repository commit-id (leave blank to use HEAD)
+# SITE_REPO_FETCH_METHOD = http, everything except "git" will use the HTTP method for fetchting site repo
+# TARGETS     = a space separated list of target platforms (if unset, all platforms will be build)
+# VERBOSE     = 0 (default) or 1, call the make commands with 'V=s' to see actual errors better
 
-function get_all_supported_platforms()
-{
-	local buffer;
-	for val in $(ls ${1}) ; do
-		[ -d "./${1}/${val}" ] || continue
-	buffer="${buffer} ${val}"
-	done
-	echo ${buffer}
-}
-
-
-if [ "_$BRANCH" == "_" ]; then
-	echo "Please specify BRANCH environment variable." >&2
-	exit 1
-fi
-
-if [ "_$BASE" == "_" ]; then
-	echo "Please specify BASE environment variable (Gluon, i.e. 'v2014.3' or commit-id)." >&2
-	exit 1
-fi
 
-if [ "${BRANCH}" != "experimental" -a "${BASE}" == "HEAD" ] ; then
-	echo "HEAD is not an allowed BASE-identifier for non-experimental builds." >&2
-	echo "Either use a tagged commit or the commit-SHA itself." >&2
-	exit 1
-fi
-
-if [ "_$VERSION" == "_" -a "$BRANCH" != "experimental" ]; then
-	echo "Please specify VERSION environment variable (not necessary for experimental branch)." >&2
-	exit 1
-fi
+### includes
+. functions.sh
 
+### static variables
 MY_DIR=$(dirname $0)
-MY_DIR=$(readlink -f "$MY_DIR")
-CODE_DIR="src"
-PATCH_DIR="site/patches"
+MY_DIR=$(readlink -f "${MY_DIR}")
+CODE_DIR="${MY_DIR}/src"
+SITE_DIR="${CODE_DIR}/site"
+PATCH_DIR="${SIZE_DIR}/patches"
 OUTPUT_DIR="${MY_DIR}/output"
+IMAGE_DIR="${CODE_DIR}/output/images"
+MODULE_DIR="${CODE_DIR}/output/modules"
 BUILD_INFO_FILENAME="build-info.txt"
-VERSIONS_INFO_DIR="versions"
+VERSIONS_INFO_DIR="${MY_DIR}/versions"
+SITE_REPO_URL="git.c3pb.de/freifunk-pb/site-ffho.git"
 LANG=C
 
-pushd $MY_DIR > /dev/null
-[ "_$BUILD_TS" == "_" ] && export BUILD_TS=$(date +"%Y-%m-%d %H:%M:%S")
+pushd ${MY_DIR} > /dev/null
 
-. functions.sh
+### ERROR handling
+[ -n "${BASE}" ] || abort "Please specify BASE environment variable (Gluon, i.e. 'v2014.3' or commit-id)."
+[ -n "${BRANCH}" ] || abort "Please specify BRANCH environment variable."
+[ "${BRANCH}" == "experimental" -o "${BASE}" != "HEAD" ] || abort "HEAD is not an allowed BASE-identifier for non-experimental builds. Either use a tagged commit or the commit-SHA itself."
+[ -n "${VERSION}" -o "${BRANCH}" == "experimental" ] || abort "Please specify VERSION environment variable (not necessary for experimental branch)."
+[ "${BRANCH}" == "experimental" -o ! -r "${VERSIONS_INFO_DIR}/${VERSION}" ] || abort "There exists a version file for '${VERSION}' ... you are trying to do something really stupid, aren't you?"
 
-### CHECK THAT VERSION DOES NOT YET EXISTS
-[ -n "$VERSION" -a -x "${VERSIONS_INFO_DIR}/${VERSION}" ] && abort "There exists a version file for '$VERSION' ... you are trying to do something really stupid, aren't you?"
+### set reasonable defaults for unset environment variables
+[ -n "${AUTOUPDATER}" ] || AUTOUPDATER=${BRANCH}
+# ToDo: [ "${BASE}" == "master" ] && BASE="origin/master"
+[ -n "${BROKEN}" ] || BROKEN=0
+[ -n "${BUILD_TS}" ] || BUILD_TS=$(date +"%Y-%m-%d %H:%M:%S")
 
-### INIT /src IF NECESSARY
-if [ ! -d "$CODE_DIR" ]; then
-	info "Code directory does not exist yet - fetching Gluon ..."
-	git clone https://github.com/freifunk-gluon/gluon.git "$CODE_DIR"
+if [ -z "${DIRCLEAN}" ]; then
+	if [ "${BRANCH}" == "experimental" ]; then
+		DIRCLEAN=0
+	else
+		DIRCLEAN=1
+	fi
 fi
 
-if [ "_${SITE_REPO_FETCH_METHOD}" != "_git" ]; then
-	SITE_REPO_URL="https://git.c3pb.de/freifunk-pb/site-${SITE}.git"
+[ -n "${MAKEJOBS}" ] || MAKEJOBS=$(grep -c "^processor" /proc/cpuinfo)
+[ -n "${PRIORITY}" ] || PRIORITY=0
+[ -n "${SITE_REPO_FETCH_METHOD}" ] || SITE_REPO_FETCH_METHOD="http"
+[ -n "${TARGETS}" ] || TARGETS=$(BROKEN=${BROKEN} make list-targets | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+
+if [ "${BRANCH}" == "experimental" -a -z "${VERSION}" ] ; then
+	default_release_pattern=$( awk -F" := " '/^DEFAULT_GLUON_RELEASE/ { gsub("shell ", "", $2); print $2; }' ${CODE_DIR}/site/site.mk )
+	VERSION=$(eval echo ${default_release_pattern})
+
+	info "EXPERIMENTAL FIRMWARE: using version tag '$VERSION'"
+fi
+
+if [ "${SITE_REPO_FETCH_METHOD}" != "git" ]; then
+	SITE_REPO_URL="https://${SITE_REPO_URL}"
 else
-	SITE_REPO_URL="git@git.c3pb.de:freifunk-pb/site-${SITE}.git"
+	SITE_REPO_URL="git@${SITE_REPO_URL}"
 fi
 
-### INIT /src/site IF NECESSARY
-if [ -d "$CODE_DIR/site" ]; then
-	# verify the site-repo is the correct one ($SITE), otherwise delete the repo
-	pushd "$CODE_DIR/site"
-	url=$(git remote show origin | awk '/Fetch URL/ { print $3; }')
-	if [ "$SITE_REPO_URL" != "$url" ]; then
-		info "The site repository is not the correct one."
-		if ! git diff-index --quiet HEAD --; then
-			popd > /dev/null
-			abort "The site repo is the wrong one but has local modifications, please fix this manually."
-		fi
-		# check on the actual branch, not the target one given as parameter
-		local_branch=$(git branch --list --no-color | awk '/^*/ { print $2; }')
-		commits=$(git log origin/${local_branch}..HEAD)
-		if [ -n "$commits" ]; then
-			popd > /dev/null
-			abort "The site repo is the wrong one but has unpushed commits, please fix this manually."
-		fi
-
-		# remove the directory without asking further questions
-		popd > /dev/null
-		rm -Rf "$CODE_DIR/site" || abort "Failed to remove wrong site-repository."
-		success "Removed old site directory in order to be able to clone the correct one."
-	else
-		popd > /dev/null
-	fi
+MAKE_PARAM=""
+[ "$VERBOSE" -eq "1" ] && MAKE_PARAM="${MAKE_PARAM} V=s"
+
+### INIT
+### INIT /src IF NECESSARY
+if [ ! -d "${CODE_DIR}" ]; then
+	info "Code directory does not exist yet - fetching Gluon ..."
+	git clone https://github.com/freifunk-gluon/gluon.git "${CODE_DIR}"
 fi
 
-if [ ! -d "$CODE_DIR/site" ]; then
+
+### INIT /src/site IF NECESSARY
+if [ ! -d "${SITE_DIR}" ]; then
 	info "Site repository does not exist, fetching it ..."
-	git clone "$SITE_REPO_URL" "$CODE_DIR/site" || abort "Failed to fetch SITE repository."
+	git clone "${SITE_REPO_URL}" "${SITE_DIR}"
+	[ "$?" -eq "0" ] || abort "Failed to fetch SITE repository."
 fi
 
 ### CHECKOUT GLUON
 progress "Checking out GLUON '$BASE' ..."
-cd $CODE_DIR
+pushd ${CODE_DIR}
 # TODO: check if gluon got modified and bail out if necessary
 git fetch
-if [ "$BASE" = "master" ]; then
-	git checkout -q origin/master
-else
-	git checkout -q $BASE
-fi
+git checkout -q ${BASE}
 [ "$?" -eq "0" ] || abort "Failed to checkout '$BASE' gluon base version, mimimi." >&2
 GLUON_COMMIT=$(git rev-list --max-count=1 HEAD)
 
+
 ### CHECKOUT SITE REPO
 progress "Checking out SITE REPO ..."
-cd site
+pushd ${SITE_DIR}
 # TODO: check if site got modified locally and bail out if necessary
-if [ "_${SITE_ID}" == "_" ]; then
+if [ -z "${SITE_ID}" ]; then
 	# no specific site given - get the most current one
-	git checkout -q $BRANCH ; git pull
-	[ "$?" -eq "0" ] || abort "Failed to get newest '$BRANCH' in site repository, mimimi."
+	git checkout -q ${BRANCH} ; git pull
+	[ "$?" -eq "0" ] || abort "Failed to get newest '${BRANCH}' in site repository, mimimi."
 else
 	# fetch site repo updates
 	git fetch || true
 	# commit given - use this one
-	git checkout -q ${SITE_ID} || abort "Failed to checkout requested site commit, mimimi."
+	git checkout -q ${SITE_ID} || abort "Failed to checkout requested site commit '${SITE_ID}', mimimi."
 fi
 SITE_COMMIT=$(git rev-list --max-count=1 HEAD)
-
-cd ..
+popd
 
 ### APPLY PATCHES TO GLUON
+# ToDo:
+
 if [ -d "$PATCH_DIR" ]; then
 	progress "Applying Patches ..."
-	git am $PATCH_DIR/*.patch
+	git am ${PATCH_DIR}/*.patch
 	[ "$?" -eq "0" ] || abort "Failed to apply patches, mimimi."
 fi
 
 
-### CLEAN
-if [ -d "./build/" -a "$BRANCH" != "experimental" ]; then
+### DIRCLEAN
+if [ -d "./build/" -a "${DIRCLEAN}" -eq "1" ]; then
 	progress "Cleaning your build environment ..."
 	make dirclean
 fi
@@ -162,23 +145,8 @@ progress "Preparing the build environment (make update) ..."
 make update
 [ "$?" -eq "0" ] || abort "Failed to update the build environment, mimimi."
 
-# determine VERSION for BRANCH=experimental if it is not set
-if [ "${BRANCH}" == "experimental" -a -z "${VERSION}" ] ; then
-	default_release_pattern=$( awk -F" := " '/^DEFAULT_GLUON_RELEASE/ { gsub("shell ", "", $2); print $2; }' ./site/site.mk )
-	VERSION=$(eval echo ${default_release_pattern})
 
-	info "EXPERIMENTAL FIRMWARE: using version tag '$VERSION'"
-fi
-
-# set reasonable defaults for unset environment variables
-[ -n "${BROKEN}" ] || BROKEN=0
-[ -n "${MAKEJOBS}" ] || MAKEJOBS=$(grep -c "^processor" /proc/cpuinfo)
-[ -n "${TARGETS}" ] || TARGETS=$(get_all_supported_platforms "./targets")
-[ -n "${PRIORITY}" ] || PRIORITY=0
-MAKE_PARAM=""
-[ "_$VERBOSE" = "_1" ] && MAKE_PARAM="${MAKE_PARAM} V=s"
-
-# we are now ready to produke the firmware images, so let's "save" our state
+# we are now ready to produce the firmware images, so let's "save" our state
 build_info_path="${OUTPUT_DIR}/${BRANCH}/${BUILD_INFO_FILENAME}"
 progress "Saving build information to: ${build_info_path}"
 [ -n "${build_info_path}" -a -f "${build_info_path}" ] && rm -f ${build_info_path}
@@ -189,7 +157,6 @@ touch $(dirname ${build_info_path})
 echo "VERSION=${VERSION}" >> ${build_info_path}
 echo "GLUON=${GLUON_COMMIT} # ${BASE}" >> ${build_info_path}
 echo "BRANCH=${BRANCH}" >> ${build_info_path}
-echo "SITE_REPO=${SITE}" >> ${build_info_path}
 echo "SITE=${SITE_COMMIT} # ${VERSION}" >> ${build_info_path}
 echo "TARGETS=${TARGETS}" >> ${build_info_path}
 echo "TS=${BUILD_TS}" >> ${build_info_path}
@@ -200,47 +167,58 @@ progress "Building the firmware - please stand by!"
 for target in ${TARGETS} ; do
 	# configure build environment for our current target
 	export GLUON_TARGET="${target}"
-	gluon_build_env_vars="GLUON_TARGET=\"${target}\" GLUON_BRANCH=\"${BRANCH}\" GLUON_RELEASE=\"${VERSION}\" BROKEN=\"${BROKEN}\""
+	gluon_build_env_vars="GLUON_TARGET=\"${target}\" GLUON_RELEASE=\"${VERSION}\" BROKEN=\"${BROKEN}\""
+	if [ "${AUTOUPDATER}" != "off" ]; then
+		gluon_build_env_vars="${gluon_build_env_vars} GLUON_BRANCH=\"${AUTOUPDATER}\""
+	fi
 
 	# prepare build environment for our current target
 	progress "Preparing build environment for target ${target}."
-	[ "${BRANCH}" == "experimental" ] || make clean
+	[ "${DIRCLEAN}" -eq "0" ] || make clean
 	make -j ${MAKEJOBS} prepare-target${MAKE_PARAM}
 
 	# need to have a toolchain for the particular target 
 	progress "Building toolchain for target ${target}."
 	make -j ${MAKEJOBS} toolchain/install${MAKE_PARAM}
-	[ "$?" -eq "0" ] || abort "Unable to build toolchain for target. Aborting."
+	[ "$?" -eq "0" ] || abort "Unable to build toolchain for target-platform ${target}."
 
 	# now we can start building the images for the target platform
 	progress "Building FFPB-flavoured Gluon firmware for target ${target}. You'd better go and fetch some c0ffee!"
 	make_targets="prepare"
-	eval "${gluon_build_env_vars} faketime \"$BUILD_TS\" make -j ${MAKEJOBS} ${make_targets}${MAKE_PARAM}"
+	eval "${gluon_build_env_vars} faketime \"${BUILD_TS}\" make -j ${MAKEJOBS} ${make_targets}${MAKE_PARAM}"
 	[ "$?" -eq "0" ] || abort "Failed to build firmware for target-platform ${target}."
 
+	eval "${gluon_build_env_vars} faketime \"${BUILD_TS}\" make -j ${MAKEJOBS} modules${MAKE_PARAM}"
+	[ "$?" -eq "0" ] || abort "Failed to build modules for target-platform ${target}."
+
 	# finally compile the firmware binaries
 	progress "Compiling binary firmware images."
-	faketime "$BUILD_TS" make images${MAKE_PARAM}
+	faketime "${BUILD_TS}" make images${MAKE_PARAM}
 	[ "$?" -eq "0" ] || abort "Failed to assemble images for target-platform ${target}."
 done
-
-cd ..
+popd
 
 # compress all binaries into 7z archive
 progress "Assembling images.7z ..."
 [ -e "${OUTPUT_DIR}/${BRANCH}/images.7z" ] && rm "${OUTPUT_DIR}/${BRANCH}/images.7z"
-if [ ${BRANCH} == "experimental" ]; then
-        7z a -xr!*.manifest "${OUTPUT_DIR}/${BRANCH}/images.7z" ${CODE_DIR}/output/images/sysupgrade/* ${CODE_DIR}/output/images/factory/* || abort "Failed to assemble images (did you install p7zip-full?)."
-else
-        7z a -xr!*.manifest "${OUTPUT_DIR}/${BRANCH}/images.7z" ${CODE_DIR}/images/sysupgrade/* ${CODE_DIR}/images/factory/* || abort "Failed to assemble images (did you install p7zip-full?)."
-fi
+7z a -xr!*.manifest "${OUTPUT_DIR}/${BRANCH}/images.7z" ${IMAGE_DIR}/sysupgrade/* ${IMAGE_DIR}/factory/*
+[ "$?" -eq "0" ] || abort "Failed to assemble images (did you install p7zip-full?)."
+
+# compress modules into 7z archive
+progress "Assembling modules.7z ..."
+pushd ${MODULE_DIR}
+[ -e "${OUTPUT_DIR}/${BRANCH}/modules.7z" ] && rm "${OUTPUT_DIR}/${BRANCH}/modules.7z"
+7z a "${OUTPUT_DIR}/${BRANCH}/modules.7z" ./gluon-ffho-${VERSION}
+[ "$?" -eq "0" ] || abort "Failed to assemble modules."
+popd
 
-# generate, franken-merge, and copy manifests
+# generate and copy manifests
 progress "Generating and copying manifest ..."
-pushd $CODE_DIR
-GLUON_TARGET="ar71xx-generic" GLUON_BRANCH=$BRANCH make manifest || abort "Failed to generate the manifest, try running 'make manifest' in '$CODE_DIR' directory manually."
+pushd ${CODE_DIR}
+faketime "${BUILD_TS}" GLUON_PRIORITY=${PRIORITY} GLUON_BRANCH=${BRANCH} make manifest
+[ "$?" -eq "0" ] || abort "Failed to generate the manifest, try running 'make manifest' in '$CODE_DIR' directory manually."
+cp "${CODE_DIR}/output/images/sysupgrade/${BRANCH}.manifest" "${OUTPUT_DIR}/${BRANCH}/"
 popd
-cp "${CODE_DIR}/images/sysupgrade/${BRANCH}.manifest" "${OUTPUT_DIR}/${BRANCH}/"
 
 # Saving a copy of the build info file as reference
 progress "Building a greater and brighter firmware finished successfully. Saving build information at: ${VERSIONS_INFO_DIR}/${VERSION}"
@@ -248,4 +226,5 @@ cp -p "${build_info_path}" "${VERSIONS_INFO_DIR}/${VERSION}"
 
 # The end. Finally.
 success "We're done, go and enjoy your new firmware in ${OUTPUT_DIR}/${BRANCH}!"
-popd > /dev/null
+popd > /dev/null
+