Resolve MacOS dylib loading issues for gdal-warp-bindings (#99)

* Switch to conda gdal for MacOS

* Add rpath=/usr/local/lib on macos

* Update container build image

* Add rpath ld flag to circleci build

* Add CHANGELOG and docs

* Update to GT org image

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 executors:
   gdal-build-executor:
     docker:
-      - image: daunnc/gdal-build-environment:6
+      - image: quay.io/geotrellis/gdal-warp-bindings-environment:1
     working_directory: /workdir
 
 jobs:
@@ -37,14 +37,14 @@ jobs:
       CROSS_TRIPLE: x86_64-apple-darwin14
       OS: darwin
       SO: dylib
-      CC: cc 
+      CC: cc
       CXX: c++
       CFLAGS: "-Wall -Werror -O0 -ggdb3"
       JAVA_HOME: "/macintosh/jdk8u202-b08/Contents/Home"
       GDALCFLAGS: "-I/macintosh/gdal/3.1.2/include"
       CXXFLAGS: "-I/usr/osxcross/SDK/MacOSX10.10.sdk/usr/include/c++/v1"
       BOOST_ROOT: "/usr/local/include/boost_1_69_0"
-      LDFLAGS: "-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread"
+      LDFLAGS: "-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib"
       LD_LIBRARY_PATH_ORIGIN: ""
       CROSS_ROOT: /usr/x86_64-apple-darwin14
       PATH: /usr/x86_64-apple-darwin14/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Use Conda GDAL in MacOS to fix dylib load errors [#99](https://github.com/geotrellis/gdal-warp-bindings/pull/99). See the MacOS installation section of the README for more details.
+
 ## [v1.1.0] - 2020-08-10
 ### Changed
 - GDAL 3.x Support [#96](https://github.com/geotrellis/gdal-warp-bindings/pull/96)

Docker/Dockerfile.crossbuild

@@ -99,4 +99,4 @@ WORKDIR /workdir
 RUN wget -k 'https://raw.githubusercontent.com/multiarch/crossbuild/master/assets/crossbuild' -O /usr/bin/crossbuild && \
     chmod ugo+x /usr/bin/crossbuild
 
-# docker build -f Dockerfile.crossbuild -t jamesmcclain/crossbuild:0 .
+# docker build -f Dockerfile.crossbuild -t quay.io/geotrellis/gdal-warp-bindings-crossbuild:latest .

Docker/Dockerfile.environment

@@ -1,5 +1,5 @@
-FROM jamesmcclain/crossbuild:0
-LABEL maintainer="James McClain <james.mcclain@gmail.com>"
+FROM quay.io/geotrellis/gdal-warp-bindings-crossbuild:1
+LABEL maintainer="Azavea <info@azavea.com>"
 
 RUN apt-get update -y && \
     apt-get install build-essential pkg-config openjdk-8-jdk -y -q && \
@@ -39,10 +39,10 @@ RUN mkdir -p /macintosh && \
     wget "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u202-b08/OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz" && \
     tar axvf OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz && \
     rm -f OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz && \
-    wget "http://homebrew.bintray.com/bottles/gdal-3.1.2.high_sierra.bottle.tar.gz" && \
-    tar axvf gdal-3.1.2.high_sierra.bottle.tar.gz && \
-    rm -f gdal-3.1.2.high_sierra.bottle.tar.gz
-
+    wget "https://anaconda.org/conda-forge/libgdal/3.1.2/download/osx-64/libgdal-3.1.2-hd7bf8dc_4.tar.bz2" && \
+    mkdir -p gdal/3.1.2 && \
+    tar axvf libgdal-3.1.2-hd7bf8dc_4.tar.bz2 -C gdal/3.1.2 && \
+    rm -f libgdal-3.1.2-hd7bf8dc_4.tar.bz2
 
 # Windows
 RUN mkdir -p /windows && \
@@ -59,4 +59,4 @@ RUN mkdir -p /windows && \
 # Linkage
 RUN echo '/usr/local/lib' >> /etc/ld.so.conf && ldconfig
 
-# docker build -f Dockerfile.environment -t jamesmcclain/gdal-build-environment:6 .
+# docker build -f Dockerfile.environment -t quay.io/geotrellis/gdal-warp-bindings-environment:latest .

Docker/README.md

@@ -1,6 +1,6 @@
 # Dockerfile.crossbuild #
 
-The file [`Dockerfile.crossbuild`](Dockerfile.crossbuild) is used to generate the Docker image `jamesmcclain/crossbuild` which contains the cross-compilers needed to generate Macintosh and Windows shared libraries.
+The file [`Dockerfile.crossbuild`](Dockerfile.crossbuild) is used to generate the Docker image `quay.io/geotrellis/gdal-warp-bindings-crossbuild` which contains the cross-compilers needed to generate Macintosh and Windows shared libraries.
 
 This image is based heavily on Manfred Touron's [`multiarch/crossbuild`](https://github.com/multiarch/crossbuild) image (in fact, it is just that image with non-x86 compilers removed, some unneeded packages removed, and based on Ubuntu 18.04 instead of Debian Jessie).
 
@@ -14,7 +14,7 @@ OSX/Darwin/Apple builds:
 
 # Dockerfile.environment #
 
-The file [`Dockerfile.environment`](Dockerfile.environment) is used to generate the Docker image `jamesmcclain/gdal-build-environment` which is derived from `jamesmcclain/crossbuild`.  The former adds Macintosh and Windows versions of OpenJDK (which provide necessary header files) as well as binary version of GDAL for Macintosh and Windows (which need to be linked-against).
+The file [`Dockerfile.environment`](Dockerfile.environment) is used to generate the Docker image `quay.io/geotrellis/gdal-warp-bindings-environment` which is derived from `quay.io/geotrellis/gdal-warp-bindings-crossbuild`.  The former adds Macintosh and Windows versions of OpenJDK (which provide necessary header files) as well as binary version of GDAL for Macintosh and Windows (which need to be linked-against).
 
 # License #
 

README.md

@@ -7,6 +7,29 @@ If one wishes to use [GDAL's VRT functionality](https://www.gdal.org/gdal_vrttut
 The code in this repository attempts to address that issue by wrapping GDAL datasets in objects which abstract one or more identical datasets; the wrapped datasets can be safely used from multiple threads with less contention than would be the case with a simple mutex around one GDAL dataset.
 APIs are provided for C and Java.
 
+# Installation #
+
+These bindings require a GDAL installation on your machine with the appropriate matching version:
+
+| GDAL Warp Bindings | OS | GDAL  | Shared Library {so,dylib,dll} |
+|--------------------|----|-------|-------------------------------|
+| 1.1.x | Linux, MacOS | 3.1.2 | libgdal.27 |
+| 1.1.x | Windows | 3.0.4 | -- |
+
+## MacOS ##
+
+For MacOS users, you must also ensure that the appropriate shared libary file (dylib) above is symlinked to `/usr/local/lib` from your GDAL installation with matching version.
+
+For best results, we recommend explicitly installing the appropriate GDAL version via Conda and symlinking the appropriate lib. Here's an example for GDAL Warp Bindings 1.1.0 using GDAL 3.1.2:
+
+```shell
+conda create -n gdal-3.1.2
+conda activate gdal-3.1.2
+conda install -c conda-forge gdal==3.1.2
+sudo ln -s $(conda info --base)/envs/gdal-3.1.2/lib/libgdal.27.dylib /usr/local/lib/libgdal.27.dylib
+conda deactivate
+```
+
 # APIs #
 
 The C and Java APIs are very similar to each other.

scripts/publish-local.sh

@@ -8,13 +8,13 @@ docker run -it --rm \
       -e CFLAGS="-Wall -Wno-sign-compare -Werror -O0 -ggdb3 -DSO_FINI -D_GNU_SOURCE" \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
       -e JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \
-      daunnc/gdal-build-environment:6 make -j4 -C src tests || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src tests || exit -1
 
 docker run -it --rm \
       -v $(pwd):/workdir \
       -e CC=gcc -e CXX=g++ \
       -e JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \
-      daunnc/gdal-build-environment:6 make -j4 -C src/experiments/thread pattern oversubscribe || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src/experiments/thread pattern oversubscribe || exit -1
 
 rm -f $(find ./src | grep '\.o$')
 docker run -it --rm \
@@ -28,8 +28,8 @@ docker run -it --rm \
       -e GDALCFLAGS="-I/macintosh/gdal/3.1.2/include" \
       -e CXXFLAGS="-I/usr/osxcross/SDK/MacOSX10.10.sdk/usr/include/c++/v1"  \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
-      -e LDFLAGS="-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread" \
-      daunnc/gdal-build-environment:6 make -j4 -C src libgdalwarp_bindings.dylib || exit -1
+      -e LDFLAGS="-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib" \
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src libgdalwarp_bindings.dylib || exit -1
 
 docker run -it --rm \
       -v $(pwd):/workdir \
@@ -40,7 +40,7 @@ docker run -it --rm \
       -e GDALCFLAGS="-I/usr/local/include" \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
       -e LDFLAGS="-L/windows/gdal/lib -lgdal_i -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic -lws2_32" \
-      daunnc/gdal-build-environment:6 make -j4 -C src gdalwarp_bindings.dll || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src gdalwarp_bindings.dll || exit -1
 
 rm -f src/main/resources/*.so
 mv src/libgdalwarp_bindings.so src/main/resources/resources/

scripts/tests.sh

@@ -8,13 +8,13 @@ docker run -it --rm \
       -e CFLAGS="-Wall -Wno-sign-compare -Werror -O0 -ggdb3 -DSO_FINI -D_GNU_SOURCE" \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
       -e JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \
-      daunnc/gdal-build-environment:6 make -j4 -C src tests || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src tests || exit -1
 
 docker run -it --rm \
       -v $(pwd):/workdir \
       -e CC=gcc -e CXX=g++ \
       -e JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" \
-      daunnc/gdal-build-environment:6 make -j4 -C src/experiments/thread pattern oversubscribe || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src/experiments/thread pattern oversubscribe || exit -1
 
 rm -f $(find | grep '\.o$')
 docker run -it --rm \
@@ -28,8 +28,8 @@ docker run -it --rm \
       -e GDALCFLAGS="-I/macintosh/gdal/3.1.2/include" \
       -e CXXFLAGS="-I/usr/osxcross/SDK/MacOSX10.10.sdk/usr/include/c++/v1"  \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
-      -e LDFLAGS="-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread" \
-      daunnc/gdal-build-environment:6 make -j4 -C src libgdalwarp_bindings.dylib || exit -1
+      -e LDFLAGS="-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib" \
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src libgdalwarp_bindings.dylib || exit -1
 
 docker run -it --rm \
       -v $(pwd):/workdir \
@@ -40,7 +40,7 @@ docker run -it --rm \
       -e GDALCFLAGS="-I/usr/local/include" \
       -e BOOST_ROOT="/usr/local/include/boost_1_69_0" \
       -e LDFLAGS="-L/windows/gdal/lib -lgdal_i -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic -lws2_32" \
-      daunnc/gdal-build-environment:6 make -j4 -C src gdalwarp_bindings.dll || exit -1
+      quay.io/geotrellis/gdal-warp-bindings-environment:1 make -j4 -C src gdalwarp_bindings.dll || exit -1
 
 rm -f src/main/resources/*.so
 mv src/libgdalwarp_bindings.so src/main/resources/resources/