slackware-current/source/make_world.sh

#!/bin/bash
# Copyright 2018  Patrick J. Volkerding, Sebeka, Minnesota, USA
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
# permitted provided that the following conditions are met:
#
# 1. Redistributions of this script must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
#  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
#  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
#  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
#  EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
#  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
#  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
#  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# make_world for Slackware: rebuilds all the SlackBuild scripts that are
# listed in the given build list.
# Each line needs to look like (for example, and without beginning with "# "):
# a/grep/grep.SlackBuild
# For x11/KDE packages, you may specify a specific package to be built using
# the arguments understood by the SlackBuild. For example:
# kde/kde.SlackBuild kdelibs:kdelibs
# Any line beginning with "#" will be skipped.

# WARNING: This script has the potential to mess up your system.
# It is not recommended to run this on a production machine.
# This script is meant to be used on a fully installed and updated system.
# Using it on a partially installed system may result in broken packages,
# packages with missing features, or build failures. If there are circular
# dependencies, more than one build may be needed to get a correct package.
#
# Slackware is not Gentoo.
# For Amusement Purposes Only.

# Not all SlackBuilds are compatible with this script. In order for a build
# script to work, it needs these features:
# 1) cd into the script directory when run i.e.: cd $(dirname $0) ; CWD=$(pwd)
# 2) Handle $TMP properly (less of an issue if you don't change $TMP)
# 3) Support output package name reporting with PRINT_PACKAGE_NAME=yes

cd $(dirname $0) ; CWD=$(pwd)
# Some SlackBuilds do not work (yet) with a different $TMP setting.
TMP=${TMP:-/tmp}
export TMP
# Where the SlackBuild script puts the built package (i.e., $TMP).
OUTPUT_LOCATION=${OUTPUT_LOCATION:-$TMP}
# Used for logs and lock files.
LOGDIR=$TMP/make_world
# Lockfiles. You might need to purge these before restarting a build.
mkdir -p $LOGDIR/lock

# Set a custom Slackware source directory. By default we assume we are already
# in the source directory.
SLACKWARE_SOURCE_DIRECTORY=${SLACKWARE_SOURCE_DIRECTORY:-}
if [ ! -z "$SLACKWARE_SOURCE_DIRECTORY" ]; then
  # Make sure this ends in '/':
  if [ ! "$(echo $SLACKWARE_SOURCE_DIRECTORY | rev | cut -b 1)" = "/" ]; then
    SLACKWARE_SOURCE_DIRECTORY="${SLACKWARE_SOURCE_DIRECTORY}/"
  fi
fi 

# To wipe build directories and package creation directories after each
# package is built, set this to anything other than "no". You might need
# to use this if you're short on build space. NOTE: if you use this
# feature, you can NOT run more than one copy of this script at the same
# time! It will wipe build trees for other packages before they can finish.
# Otherwise, file locking is used and you may run as many parallel copies
# of this script as you think will help to speed things along.
WIPE_AFTER_BUILD=${WIPE_AFTER_BUILD:-no}

# Be kind, don't hit control-c! If you do, you might leave broken packages,
# logfiles, and locks in $TMP that will cause problems for you later. If you're
# not in a huge hurry to quit, create this file (replace with $TMP if needed):
# /tmp/make_world/lock/abort
# This will cause all instances of make_world.sh to exit when they complete the
# task they are working on.
rm -f $LOGDIR/lock/abort

BUILDLIST=${BUILDLIST:-$LOGDIR/buildlist}
if [ ! -r $BUILDLIST -a ! -r ${BUILDLIST}.lock ]; then
  # The buildlist does not exist, so attempt to create one that builds
  # everything except for the kernels (the kernel scripts are not compatible
  # with make_world.sh, mostly because of the need to reboot the new kernel).
  touch ${BUILDLIST} ${BUILDLIST}.lock
  echo "Generating list of packages to build in ${BUILDLIST}..."
  for script in ${SLACKWARE_SOURCE_DIRECTORY}*/*/*.SlackBuild ; do
    # Only add the script if the SlackBuild name matches the directory name:
    if [ "$(basename $(echo $script | cut -f 1 -d ' ') .SlackBuild)" = "$(echo $(dirname $(echo $script | cut -f 1 -d ' ')) | rev | cut -f 1 -d / | rev)" ]; then
      if [ "$uname -m)" = "x86_64" -a "$(basename $(echo $script | cut -f 1 -d ' '))" = "isapnptools.SlackBuild" ]; then
        # Don't try to build isapnptools on x86_64.
        continue
      fi
      echo $script >> $BUILDLIST
    fi
  done
  if [ -r ${SLACKWARE_SOURCE_DIRECTORY}kde/kde.SlackBuild ]; then
    echo "${SLACKWARE_SOURCE_DIRECTORY}kde/kde.SlackBuild" >> $BUILDLIST
  fi
  rm -f ${BUILDLIST}.lock
  # Set GEN_LIST_ONLY=yes if you'd like to exit after generating a build list.
  # You might want to do this to comment some build scripts out first, or if
  # you'd like to sort it into a "magic build order". ;-) This script is pretty
  # good at just brute-forcing things, though (with a few runs).
  if [ "$GEN_LIST_ONLY" = "yes" ]; then
    echo "Generated ${BUILDLIST}. Exiting."
    exit 0
  fi
fi
if [ -r ${BUILDLIST}.lock ]; then
  echo -n "Waiting for ${BUILDLIST}.lock to be removed..."
  while [ 0 ]; do
    if [ ! -r ${BUILDLIST}.lock ]; then
      break
    fi
    sleep 5
  done
  echo " done."
fi
echo "Using buildlist $BUILDLIST."

# To use shuffle mode (build packages in a random order each time through),
# pass SHUFFLE=yes (or anything other than "no") to this script.
SHUFFLE=${SHUFFLE:-no}
if [ "$SHUFFLE" = "no" ]; then
  SHUF=cat
else
  SHUF=shuf
fi

# To keep repeating the build list, set $REPEAT to anything other than "no":
REPEAT=${REPEAT:-no}

# To always rebuild a SlackBuild even if already built packages are found, set
# FORCE_BUILD=yes:
FORCE_BUILD=${FORCE_BUILD:-no}

# Function to do the build:
do_build() {
  if [ "$HAVE_GLOBAL_LOCK" = "true" ]; then
    # Wait for other builds to complete
    echo -n "have global lock, waiting for other builds to complete... "
    while [ 0 ]; do
      sleep 5
      if ! /bin/ls $LOGDIR/lock/*.lock 1> /dev/null 2> /dev/null ; then
        echo -n "done, continuing... "
        break
      fi
    done
  fi
  # If we're trying again, we don't want to leave a failure log in the logs
  # directory. But save it just in case...
  if [ -r $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed ]; then
    mkdir -p $LOGDIR/faillog-backups
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed $LOGDIR/faillog-backups
  fi
  $buildscript &> $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log
  if [ ! $? = 0 ]; then
    # Exit code from SlackBuild indicated an error:
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) failed to build."
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed
  elif [ ! -r $OUTPUT_LOCATION/$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) ]; then
    # No error code returned from SlackBuild, but the package(s) were not found.
    # Possibly the SlackBuild doesn't honor $TMP, and a non-/tmp $TMP variable was set?
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) failed to build."
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed
  else
    # Figure out a progress report to include with the successful build message:
    cat $BUILDLIST | grep -v "^$" | grep -v "^#" | sort | uniq | cut -f 1 -d ' ' | rev | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-to-build.$$
    # OK, we don't know if every *.log is actually finished if we're running
    # more than one make_world.sh, but whatever. It's an estimate.
    /bin/ls $LOGDIR/*.log | rev | cut -f 2,3 -d . | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-built-or-building.$$
    NUMTOTAL="$(cat $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    NUMBUILT="$(grep -x -f $LOGDIR/tmp-pkgs-built-or-building.$$ $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    rm -f $LOGDIR/tmp-pkgs-to-build.$$ $LOGDIR/tmp-pkgs-built-or-building.$$
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) built successfully ($NUMBUILT/$NUMTOTAL)."
    for package in $(PRINT_PACKAGE_NAME=foo $buildscript) ; do
      upgradepkg --install-new --reinstall $OUTPUT_LOCATION/$package > /dev/null 2>&1
    done
    # The post-build cleanup is pretty sloppy. It will not clean up all of
    # the build-related residue, and it may possibly delete some things that
    # this script did not put there. It's also not compatible with running
    # more than one copy of make_world.sh simultaneously.
    # Think more than twice before using this.
    if [ ! "$WIPE_AFTER_BUILD" = "no" ]; then
      ( cd $TMP
        rm -f configure CMakeLists.txt
        for findconfigure in */configure ; do
          rm -rf "$(dirname $findconfigure)"
        done
        for findautogen in */autogen.sh ; do
          rm -rf "$(dirname $findautogen)"
        done
        for findcmake in */CMakeLists.txt ; do
          rm -rf "$(dirname $findcmake)"
        done
        for findmake in */Makefile ; do
          rm -rf "$(dirname $findmake)"
        done
        for findmeson in */meson.build ; do
          rm -rf "$(dirname $findmeson)"
        done
        for findpython in */setup.py ; do
          rm -rf "$(dirname $findpython)"
        done
        rm -rf package-*
      )
    fi
  fi
}

# Main loop:
while [ 0 ]; do
  # Skip any blank lines or lines in the buildlist that begin with #:
  cat $BUILDLIST | grep -v "^$" | grep -v "^#" | $SHUF | while read buildscript ; do
    if [ -r $LOGDIR/lock/abort ]; then
      echo "Exiting (abort requested)."
      exit 0
    fi
    # If there's a global lock, we have to wait for it to be released:
    if [ -r $LOGDIR/lock/lock.global ]; then
      HELD_BY="$(cat $LOGDIR/lock/lock.global)"
      echo -n "Waiting for global lock release (held by ${HELD_BY})... "
      while [ 0 ]; do
        sleep 10
        if [ ! -r $LOGDIR/lock/lock.global ]; then
          echo "released."
          break
        fi
        if [ ! "${HELD_BY}" = "$(cat $LOGDIR/lock/lock.global)" ]; then
          HELD_BY="$(cat $LOGDIR/lock/lock.global)"
          echo
          echo -n "Waiting for global lock release (held by ${HELD_BY})... "
        fi
      done
    fi
    echo -n "Working on $buildscript... "
    if [ -r $OUTPUT_LOCATION/$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) -a $FORCE_BUILD = no ]; then
      echo "skipping because built package(s) were found."
      continue
    fi
    # Use flock to only allow one instance of this script to work on a given
    # SlackBuild script at a time. If the SlackBuild is already locked, we'll
    # just move on to the next one. Yes, you can run more than one copy of this
    # script at the same time to speed things up!
    #
    # See if we need a global lock. Some SlackBuilds are disruptive and other
    # builds should not take place until they have completed. For example, perl
    # removes itself from the system during the build. Assume that we need a
    # global lock for any package that uses removepkg, upgradepkg, slacktrack,
    # or trackbuild. Also, you may add the identifier REQUIRE_GLOBAL_LOCK
    # anywhere in a SlackBuild script to make it require the global lock.
    HAVE_GLOBAL_LOCK=false
    if grep -q -e removepkg -e upgradepkg -e slacktrack -e trackbuild -e REQUIRE_GLOBAL_LOCK $(dirname $(echo $buildscript | cut -f 1 -d ' '))/$(basename $(echo $buildscript | cut -f 1 -d ' ')) ; then
      # pkgtools, x11, and KDE all trigger the detection above, but none of them
      # really need the global lock. So only request the lock if the build
      # script is not one of those.
      if ! echo "$(basename $(echo $buildscript | cut -f 1 -d ' '))" | grep -q -x -e pkgtools.SlackBuild -e x11.SlackBuild -e kde.SlackBuild ; then
        HAVE_GLOBAL_LOCK=true
      fi
    fi
    if [ "$HAVE_GLOBAL_LOCK" = "true" ]; then
      ( flock 9 || exit 11
        echo $(basename $(echo $buildscript | cut -f 1 -d ' ')) >> $LOGDIR/lock/lock.global
        do_build
      ) 9> $LOGDIR/lock/lock.global
      # Remove lock file:
      rm -f $LOGDIR/lock/lock.global
    else
      ( flock -n 9 || exit 11
        do_build
      ) 9> $LOGDIR/lock/$(basename $(echo $buildscript | cut -f 1 -d ' ')).lock
      if [ $? = 11 ]; then
        echo "skipping (locked by another make_world.sh instance)."
        continue
      fi
      # Remove lock file:
      rm -f $LOGDIR/lock/$(basename $(echo $buildscript | cut -f 1 -d ' ')).lock
    fi
  done
  if [ "$REPEAT" = "no" ]; then
    break
  else
    # Figure out if we skipped everything and exit REPEAT mode if we did:
    cat $BUILDLIST | grep -v "^$" | grep -v "^#" | sort | uniq | cut -f 1 -d ' ' | rev | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-to-build.$$
    /bin/ls $LOGDIR/*.log | rev | cut -f 2,3 -d . | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-built-or-building.$$
    NUMTOTAL="$(cat $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    NUMBUILT="$(grep -x -f $LOGDIR/tmp-pkgs-built-or-building.$$ $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    rm -f $LOGDIR/tmp-pkgs-to-build.$$ $LOGDIR/tmp-pkgs-built-or-building.$$
    if [ "$NUMTOTAL" = "$NUMBUILT" ]; then
      echo "All packages have been built ($NUMBUILT/$NUMTOTAL). Exiting."
      break
    else
      echo "Repeating BUILDLIST since some packages are not built yet ($NUMBUILT/$NUMTOTAL complete)."
    fi
  fi
done