build_arm64v8_ubuntu_cross_compilation_toolchain

#!/bin/bash
#
# Based on
#     - https://github.com/apple/swift-package-manager/blob/master/Utilities/build_ubuntu_cross_compilation_toolchain

# by Johannes Weiß
# Raspi adjustments by Helge Heß <me@helgehess.eu>

set -eu

export PATH="/bin:/usr/bin"

# Note: aarch64-linux-gnu is rejected by swift-pm. We need
#       aarch64-unknown-linux (arch-vendor-os)
TARGET_ARCH=aarch64
export TC_TARGET="${TARGET_ARCH}-linux-gnu"
export SPM_TARGET="${TARGET_ARCH}-unknown-linux"

function usage() {
    echo >&2 "Usage: $0 SWIFT-FOR-MACOS.pkg SWIFT-FOR-LINUX.tar.gz"
    echo >&2
    echo >&2 "Example: $0 /tmp/ ~/Downloads/swift-4.2.1-RELEASE-osx.pkg ~/Downloads/swift-4.2.1-futurejones-ubuntu-bionic.tar.gz"
    echo >&2
    echo >&2 "Complete example:"
    echo >&2 "  # Download the Swift binaries for Ubuntu and macOS"
    echo >&2 "  curl -L -o ~/Downloads/swift-4.2.1-futurejones-ubuntu-bionic.tar.gz https://www.dropbox.com/s/fzaxi1hz18w01rz/swift-4.2.1-futurejones-ubuntu-bionic.tar.gz?dl=1"
    echo >&2 "  curl -o ~/Downloads/swift-4.2.1-osx.pkg https://swift.org/builds/swift-4.2.1-release/xcode/swift-4.2.1-RELEASE/swift-4.2.1-RELEASE-osx.pkg"
    echo >&2 "  # Compile the SDK and toolchain from that"
    echo >&2 "  $0 /tmp/ ~/Downloads/swift-4.2.1-RELEASE-osx.pkg ~/Downloads/swift-4.2.1-futurejones-ubuntu-bionic.tar.gz"
    echo >&2 "  # Create a test application"
    echo >&2 "  mkdir my-test-app"
    echo >&2 "  cd my-test-app"
    echo >&2 "  swift package init --type=executable"
    echo >&2 "  # Build it for Ubuntu"
    echo >&2 "  swift build --destination /tmp/cross-toolchain/arm64v8-ubuntu-bionic-destination.json"

}

if [[ $# -ne 3 ]]; then
    usage
    exit 1
fi

function realpath() {
    if [[ "${1:0:1}" = / ]]; then
        echo "$1"
    else
        (
        cd "$(dirname "$1")"
        echo "$(pwd)/$(basename "$1")"
        )
    fi
}

# This is a funny function. The Glibc.modulemap contains absolute include
# pathes like so:
#      header "/usr/include/aarch64-linux-gnu/sys/ioctl.h"
# This thing creates a new directory:
#   swift.xctoolchain/usr/lib/swift/linux/aarch64/private_includes
# and for each header in the modmap it creates a shim header which includes
# a relative path, like:
#   swift.xctoolchain/usr/lib/swift/linux/aarch64/private_includes/aarch64-linux-gnu_sys_ioctl.h
# which includes:
#   #include <aarch64-linux-gnu/sys/ioctl.h>
function fix_glibc_modulemap() {
    local glc_mm
    local tmp
    local inc_dir

    glc_mm="$1"
    echo "glibc.modulemap at '$glc_mm' (`pwd`)"
    if ! test -f "$glc_mm"; then
      echo "Missing: $glc_mm"
      exit 42
    fi
    
    tmp=$(mktemp "$glc_mm"_orig_XXXXXX)
    inc_dir="$(dirname "$glc_mm")/private_includes"
    cat "$glc_mm" >> "$tmp"
    echo "Paths:"
    echo " - original glibc.modulemap: $tmp"
    echo " - new      glibc.modulemap: $glc_mm"
    echo " - private includes dir    : $inc_dir"
    echo -n > "$glc_mm"
    rm -rf "$inc_dir"
    mkdir "$inc_dir"
    cat "$tmp" | while IFS='' read line; do
        # hh: apparently the modmap started w/ two slashes? ///usr/local/
        # if [[ "$line" =~ ^(\ *header\ )\"\/\/\/usr\/include\/(${TC_TARGET}\/)?([^\"]+)\" ]]; then
        if [[ "$line" =~ ^(\ *header\ )\"\/usr\/include\/(${TC_TARGET}\/)?([^\"]+)\" ]]; then
            local orig_inc
            local rel_repl_inc
            local repl_inc

            orig_inc="${BASH_REMATCH[3]}"
            rel_repl_inc="$(echo "$orig_inc" | tr / _)"
            repl_inc="$inc_dir/$rel_repl_inc"
            echo "${BASH_REMATCH[1]} \"$(basename "$inc_dir")/$rel_repl_inc\"" >> "$glc_mm"
            if [[ "$orig_inc" == "uuid/uuid.h" ]]; then
                # no idea why ;)
                echo "#include <linux/uuid.h>" >> "$repl_inc"
            else
                echo "#include <$orig_inc>" >> "$repl_inc"
            fi
            true
        else
            echo "$line" >> "$glc_mm"
        fi
    done
}

# set -xv
# where to get stuff from
dest=$(realpath "$1")
macos_swift_pkg=$(realpath "$2")
linux_swift_pkg=$(realpath "$3")
if ! test -f "$macos_swift_pkg"; then
  echo "Missing macOS toolchain: $macos_swift_pkg"
  exit 42
fi
if ! test -f "$linux_swift_pkg"; then
  echo "Missing Raspi toolchain: $linux_swift_pkg"
  exit 42
fi

# config
blocks_h_url="https://raw.githubusercontent.com/apple/swift-corelibs-libdispatch/master/src/BlocksRuntime/Block.h"
xc_tc_name="swift.xctoolchain"
linux_sdk_name="arm64v8-ubuntu-bionic.sdk"
cross_tc_basename="cross-toolchain"
binutils_pkg_url="https://ftp.gnu.org/gnu/binutils/binutils-2.31.tar.gz"
ubuntu_mirror="http://ports.ubuntu.com/ubuntu-ports"
packages_file="$ubuntu_mirror/dists/bionic/main/binary-arm64/Packages.gz"
# was libgcc-5-dev, on Bionic we need libgcc-7-dev, libstdc++-7-dev
pkg_names=( libc6-dev linux-libc-dev libicu60 libgcc-7-dev libicu-dev libc6 libgcc1 libstdc++-7-dev libstdc++6 zlib1g zlib1g-dev)
pkgs=()

# url
function download_stdout() {
    curl --fail -s "$1"
}

# url, key
function download_with_cache() {
    mkdir -p "$dest/cache"
    local out
    out="$dest/cache/$2"
    if [[ ! -f "$out" ]]; then
        curl --fail -s -o "$out" "$1"
    fi
    echo "$out"
}

# dst, file
function unpack_deb() {
    local tmp
    tmp=$(mktemp -d /tmp/.unpack_deb_XXXXXX)
    (
    cd "$tmp"
    ar -x "$2"
    tar -C "$1" -xf data.tar.*
    )
    rm -rf "$tmp"
}

# dst, file
function unpack_pkg() {
    local tmp
    tmp=$(mktemp -d /tmp/.unpack_pkg_XXXXXX)
    (
    cd "$tmp"
    xar -xf "$2"
    )
    (
    cd "$1"
    cat "$tmp"/*.pkg/Payload | gunzip -dc | cpio -i
    )
    rm -rf "$tmp"
}

# dst, file
function unpack() {
    ext=${2##*.}
    "unpack_$ext" "$@"
}

cd "$dest"

rm -rf $cross_tc_basename
mkdir -p "$cross_tc_basename/$linux_sdk_name"

# oopsie, this is slow but seemingly fast enough :)
while read -r line; do
    for pkg_name in "${pkg_names[@]}"; do
        if [[ "$line" =~ ^Filename:\ (.*\/([^/_]+)_.*$) ]]; then
            # echo "${BASH_REMATCH[2]}"
            if [[ "${BASH_REMATCH[2]}" == "$pkg_name" ]]; then
                new_pkg="$ubuntu_mirror/${BASH_REMATCH[1]}"
                pkgs+=( "$new_pkg" )
                echo "- will download $new_pkg"
            fi
        fi
    done
done < <(download_stdout "$packages_file" | gunzip -d -c | grep ^Filename:)


# --- Unpack Linux Platform SDK ---------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
(
cd "$tmp"
for f in "${pkgs[@]}"; do
    name="$(basename "$f")"
    archive="$(download_with_cache "$f" "$name")"
    unpack "$dest/$cross_tc_basename/$linux_sdk_name" "$archive"
done
)
rm -rf "$tmp"


# --- Build Binutils and Gold ------------------------------------------------
(
cd $cross_tc_basename
mkdir -p "$xc_tc_name/usr/bin"

binutils_pkg="$(download_with_cache "$binutils_pkg_url" binutils.tar.gz)"
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
(
cd "$tmp"
echo "Unpacking binutils: ${binutils_pkg} to ${tmp} .."
tar -xf "$binutils_pkg"
cd binutils-*
# 4secs on ZeaPro
echo "  Configuring binutils in ${tmp} (~4s) .."
./configure --enable-gold > xt-binutils-configure.log 2>&1
echo "  .. building binutils (~1m) .."
time make -j > xt-binutils-make.log 2>&1
echo "  done."
cd gold
echo "  Configuring gold (~11s).."
./configure --enable-gold > xt-gold-configure.log 2>&1
echo "  .. building gold (~1m).."
time make -j > xt-gold-make.log 2>&1
echo "  done."
)
echo "done with gold, copy to: ${xc_tc_name}/usr/bin/ld.gold"
cp "$tmp"/binutils-*/gold/ld-new "$xc_tc_name/usr/bin/ld.gold"
rm -rf "$tmp"


# --- Patch Absolute Links ---------------------------------------------------
echo "Fixing absolute links in ${linux_sdk_name} ..."
# fix absolute symlinks
find "$linux_sdk_name" -type l | while read -r line; do
    dst=$(readlink "$line")
    if [[ "${dst:0:1}" = / ]]; then
        echo "  Patch Link: ${line} dest ${dst}"
        rm "$line"
        #echo ln -s "$dest/$cross_tc_basename/$linux_sdk_name$dst" "$line"
        ln -s "$dest/$cross_tc_basename/$linux_sdk_name$dst" "$line"
    #else
    #    echo "  Link: ${line} OK."
    fi
done
# HH: can't find that
#ln -s 5 "$linux_sdk_name/usr/lib/gcc/${TC_TARGET}/5.4.0"
echo "Fixed absolute links."


# --- Unpack macOS Swift Toolchain -------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
echo "Unpacking macOS toolchain: ${macos_swift_pkg} to ${tmp} .."
unpack "$tmp" "$macos_swift_pkg"
echo ".. copy to ${xc_tc_name} .."
rsync -a "$tmp/" "$xc_tc_name"
echo ".. delete tmpdir ..."
rm -rf "$tmp"
echo "Did unpack into ${xc_tc_name}."


# --- Unpack Raspi Swift toolchain -------------------------------------------
tmp=$(mktemp -d "$dest/tmp_pkgs_XXXXXX")
# official release tarballs have a dirname, the Raspi ones usually don't but
# just directly contain 'usr'
#   --strip-components 1
echo "Unpacking Raspi toolchain: ${linux_swift_pkg} to ${tmp} .."
tar -C "$tmp" -xf "$linux_swift_pkg"

echo ".. copy to ${xc_tc_name} .."
rsync -a "$tmp/usr/lib/swift/linux"        "$xc_tc_name/usr/lib/swift/"
rsync -a "$tmp/usr/lib/swift_static/linux" "$xc_tc_name/usr/lib/swift_static/"
# HH: not necessary, name is aarch64
#(cd "$xc_tc_name/usr/lib/swift/linux"; ln -s armv7 arm)

rsync -a "$tmp/usr/lib/swift/dispatch"     "$linux_sdk_name/usr/include/"
rsync -a "$tmp/usr/lib/swift/os"           "$linux_sdk_name/usr/include/"
rsync -a "$tmp/usr/lib/swift/CoreFoundation" "$linux_sdk_name/usr/include/"
echo ".. delete tmpdir ..."
rm -rf "$tmp"
echo "Did unpack into ${xc_tc_name}."


# ----------------------------------------------------------------------------
# HH: we do have a Block.h
#curl --fail -s -o "$linux_sdk_name/usr/include/Block.h" "$blocks_h_url"
# this exists
#ln -s swift "$xc_tc_name/usr/bin/swift-autolink-extract"
ln -s "$dest/$cross_tc_basename/$xc_tc_name/usr/lib/swift/linux/libdispatch.so" "$linux_sdk_name/usr/lib/"
#ln -s "$dest/$cross_tc_basename/$xc_tc_name/usr/lib/swift/linux/libdispatch.a" "$linux_sdk_name/usr/lib/"

# fix up
)

# HH DEBUG
ls -la "$cross_tc_basename/$xc_tc_name/usr/lib/swift/linux/"
fix_glibc_modulemap "$cross_tc_basename/$xc_tc_name/usr/lib/swift/linux/${TARGET_ARCH}/glibc.modulemap"


# ----------------------------------------------------------------------------
echo "**** emit json"

cat > "$cross_tc_basename/arm64v8-ubuntu-bionic-destination.json" <<EOF
{
    "version": 1,
    "sdk": "$(pwd)/${cross_tc_basename}/${linux_sdk_name}",
    "toolchain-bin-dir": "$(pwd)/${cross_tc_basename}/${xc_tc_name}/usr/bin",
    "target": "${SPM_TARGET}",
    "dynamic-library-extension": "so",
    "extra-cc-flags": [
        "-fPIC"
    ],
    "extra-swiftc-flags": [
        "-use-ld=gold", "-tools-directory", "$(pwd)/$cross_tc_basename/$xc_tc_name/usr/bin"
    ],
    "extra-cpp-flags": [
        "-lstdc++"
    ]
}
EOF

echo
echo "OK, your cross compilation toolchain for Raspi 64-bit Ubuntu Bionic is now ready to be used"
echo " - SDK: $(pwd)/$cross_tc_basename/$linux_sdk_name"
echo " - toolchain: $(pwd)/$cross_tc_basename/$xc_tc_name"
echo " - SwiftPM destination.json: $(pwd)/$cross_tc_basename/arm64v8-ubuntu-bionic-destination.json"