From 5d6617c4a26be2983336e1baba1a16effe412e9c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 6 May 2020 05:48:16 +0900 Subject: [PATCH 1/4] [CI][C++] Add a test job for s390x --- .travis.yml | 78 ++++++++++++++++++++++++ ci/docker/ubuntu-20.04-cpp.dockerfile | 6 +- cpp/build-support/run-test.sh | 2 +- cpp/cmake_modules/SetupCxxFlags.cmake | 2 + dev/archery/archery/cli.py | 11 +++- dev/archery/archery/docker.py | 7 ++- dev/archery/archery/tests/test_docker.py | 11 ++++ docker-compose.yml | 6 +- 8 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000..d82e934d21f9c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +dist: bionic + +language: minimal + +addons: + apt: + packages: + - python3-pip + +services: + - docker + +jobs: + include: + - name: "C++ on s390x" + os: linux + arch: s390x + env: + ARCH: s390x + ARROW_CI_MODULES: "CPP" + DOCKER_IMAGE_ID: ubuntu-cpp + UBUNTU: "20.04" + allow_failures: + - arch: s390x + +env: + DOCKER_BUILDKIT: 0 + COMPOSE_DOCKER_CLI_BUILD: 1 + ARROW_ENABLE_TIMING_TESTS: "OFF" + +before_install: + - eval "$(python ci/detect-changes.py)" + - | + arrow_ci_affected=no + for arrow_ci_module in ${ARROW_CI_MODULES}; do + arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED + if [ "$(eval 'echo ${arrow_ci_affected_variable}')" != "1" ]; then + arrow_ci_affected=yes + fi + done + if [ "${arrow_ci_affected}" = "no" ]; then + travis_terminate 0 + fi + +install: + - pip3 install -e dev/archery[docker] + +script: + - sudo sysctl -w kernel.core_pattern="core.%e.%p" + # This isn't allowed on Travis CI: + # /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted + - | + ulimit -c unlimited || : + - | + archery docker run \ + --volume ${PWD}/build:/build \ + ${DOCKER_IMAGE_ID} + +after_success: + - | + archery docker push ${DOCKER_IMAGE_ID} || : diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 897f794483370..726a2fdc18225 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -66,6 +66,7 @@ RUN apt-get update -y -q && \ libssl-dev \ libthrift-dev \ libzstd-dev \ + make \ ninja-build \ pkg-config \ protobuf-compiler \ @@ -104,6 +105,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_ZSTD=ON \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 20170bf5fba7d..7e2a22f069a0a 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -97,7 +97,7 @@ function run_test() { rm -f $XMLFILE $TEST_EXECUTABLE "$@" 2>&1 \ - | $ROOT/build-support/asan_symbolize.py \ + | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \ | ${CXXFILT:-c++filt} \ | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ | $pipe_cmd 2>&1 | tee $LOGFILE diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index ae8c8d0b21a32..726de0adec250 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -29,6 +29,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv7") set(ARROW_CPU_FLAG "armv7") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc") set(ARROW_CPU_FLAG "ppc") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") + set(ARROW_CPU_FLAG "s390x") endif() # Check architecture specific compiler flags if(ARROW_CPU_FLAG STREQUAL "x86") diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 8915b65a9dc7d..d276eaaa479f9 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -696,9 +695,11 @@ def docker_compose(obj, src): @click.option('--dry-run/--execute', default=False, help="Display the docker-compose commands instead of executing " "them.") +@click.option('--volume', '-v', multiple=True, + help="Set volume within the container") @click.pass_obj def docker_compose_run(obj, image, command, env, force_pull, force_build, - use_cache, use_leaf_cache, dry_run): + use_cache, use_leaf_cache, dry_run, volume): """Execute docker-compose builds. To see the available builds run `archery docker list`. @@ -723,6 +724,9 @@ def docker_compose_run(obj, image, command, env, force_pull, force_build, # pass runtime parameters via docker environment variables archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp + # set a volume + archery docker run -v $PWD/build:/build ubuntu-cpp + # starting an interactive bash session for debugging archery docker run ubuntu-cpp bash """ @@ -749,7 +753,8 @@ def _print_command(self, *args, **kwargs): force_pull=force_pull, force_build=force_build, use_cache=use_cache, - use_leaf_cache=use_leaf_cache + use_leaf_cache=use_leaf_cache, + volumes=volume ) except UndefinedImage as e: raise click.ClickException( diff --git a/dev/archery/archery/docker.py b/dev/archery/archery/docker.py index 0c0a49aae9802..6e63abfad5518 100644 --- a/dev/archery/archery/docker.py +++ b/dev/archery/archery/docker.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -162,7 +161,8 @@ def build(self, image, use_cache=True, use_leaf_cache=True): self._execute('build', '--no-cache', image) def run(self, image, command=None, env=None, force_pull=False, - force_build=False, use_cache=True, use_leaf_cache=True): + force_build=False, use_cache=True, use_leaf_cache=True, + volumes=None): self._validate_image(image) if force_pull: @@ -175,6 +175,9 @@ def run(self, image, command=None, env=None, force_pull=False, if env is not None: for k, v in env.items(): args.extend(['-e', '{}={}'.format(k, v)]) + if volumes is not None: + for volume in volumes: + args.extend(['--volume', volume]) args.append(image) if command is not None: diff --git a/dev/archery/archery/tests/test_docker.py b/dev/archery/archery/tests/test_docker.py index f9c9cd92bc9ef..e521bfe87c45e 100644 --- a/dev/archery/archery/tests/test_docker.py +++ b/dev/archery/archery/tests/test_docker.py @@ -384,6 +384,17 @@ def test_compose_run(arrow_compose_path): ]) compose.run('conda-python', env=env) + expected_calls = [ + ( + "run --rm --volume /host/build:/build " + "--volume /host/ccache:/ccache:delegated conda-python" + ) + ] + compose = DockerCompose(arrow_compose_path) + with assert_compose_calls(compose, expected_calls): + volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated") + compose.run('conda-python', volume=volumes) + def test_compose_run_force_pull_and_build(arrow_compose_path): compose = DockerCompose(arrow_compose_path) diff --git a/docker-compose.yml b/docker-compose.yml index a877dc69c9dc9..37ca4c45455df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,8 @@ # defaults are set in .env file. # # Example: -# $ ARCH=arm64v8 docker-compose build ubuntu-bionic-cpp -# $ ARCH=arm64v8 docker-compose run ubuntu-bionic-cpp +# $ ARCH=arm64v8 docker-compose build ubuntu-cpp +# $ ARCH=arm64v8 docker-compose run ubuntu-cpp # # # Coredumps @@ -257,7 +257,7 @@ services: # docker-compose build ubuntu-cpp # docker-compose run --rm ubuntu-cpp # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64v8, s390x, ... # UBUNTU: 16.04, 18.04, 20.04 image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp build: From 974689e171240dd7e6b1ab0119de2ee00da8f166 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 9 May 2020 06:19:51 +0900 Subject: [PATCH 2/4] Fix Archery tests --- dev/archery/archery/tests/test_cli.py | 36 ++++++++++++++++++++---- dev/archery/archery/tests/test_docker.py | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dev/archery/archery/tests/test_cli.py b/dev/archery/archery/tests/test_cli.py index db5b88d866cb9..d18684f40f2cc 100644 --- a/dev/archery/archery/tests/test_cli.py +++ b/dev/archery/archery/tests/test_cli.py @@ -34,7 +34,8 @@ force_pull=True, force_build=True, use_cache=True, - use_leaf_cache=True + use_leaf_cache=True, + volumes=() ) ), ( @@ -46,7 +47,8 @@ force_pull=True, force_build=True, use_cache=True, - use_leaf_cache=True + use_leaf_cache=True, + volumes=() ) ), ( @@ -58,7 +60,8 @@ force_pull=False, force_build=False, use_cache=True, - use_leaf_cache=True + use_leaf_cache=True, + volumes=() ) ), ( @@ -73,11 +76,12 @@ force_pull=False, force_build=True, use_cache=False, - use_leaf_cache=False + use_leaf_cache=False, + volumes=() ) ), ( - ['-e', 'ARROW_GANDIVA=OFF', '-e', 'ARROW_FLIGHT=ON', 'ubuntu-cpp'], + ['-e', 'ARROW_GANDIVA=OFF', '-e', 'ARROW_FLIGHT=ON', 'ubuntu-cpp'], ['ubuntu-cpp'], dict( command=None, @@ -88,7 +92,27 @@ force_pull=True, force_build=True, use_cache=True, - use_leaf_cache=True + use_leaf_cache=True, + volumes=() + ) + ), + ( + [ + '--volume', './build:/build', '-v', './ccache:/ccache:delegated', + 'ubuntu-cpp' + ], + ['ubuntu-cpp'], + dict( + command=None, + env={}, + force_pull=True, + force_build=True, + use_cache=True, + use_leaf_cache=True, + volumes=( + './build:/build', + './ccache:/ccache:delegated', + ) ) ) ]) diff --git a/dev/archery/archery/tests/test_docker.py b/dev/archery/archery/tests/test_docker.py index e521bfe87c45e..06a156de35253 100644 --- a/dev/archery/archery/tests/test_docker.py +++ b/dev/archery/archery/tests/test_docker.py @@ -393,7 +393,7 @@ def test_compose_run(arrow_compose_path): compose = DockerCompose(arrow_compose_path) with assert_compose_calls(compose, expected_calls): volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated") - compose.run('conda-python', volume=volumes) + compose.run('conda-python', volumes=volumes) def test_compose_run_force_pull_and_build(arrow_compose_path): From 7dabdcdbdf954bcb5c26d7f34d5a64f4d8034f10 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 9 May 2020 11:15:24 +0900 Subject: [PATCH 3/4] Disable modules --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.travis.yml b/.travis.yml index d82e934d21f9c..ce0d9419c45fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,14 @@ jobs: env: ARCH: s390x ARROW_CI_MODULES: "CPP" + ARROW_DATASET: "OFF" + ARROW_GANDIVA: "OFF" + ARROW_HDFS: "OFF" + ARROW_ORC: "OFF" + ARROW_PARQUET: "OFF" + ARROW_PLASMA: "OFF" + PARQUET_BUILD_EXAMPLES: "OFF" + PARQUET_BUILD_EXECUTABLES: "OFF" DOCKER_IMAGE_ID: ubuntu-cpp UBUNTU: "20.04" allow_failures: @@ -70,6 +78,14 @@ script: ulimit -c unlimited || : - | archery docker run \ + -e ARROW_DATASET=${ARROW_DATASET:-ON} \ + -e ARROW_GANDIVA=${ARROW_GANDIVA:-ON} \ + -e ARROW_HDFS=${ARROW_HDFS:-ON} \ + -e ARROW_ORC=${ARROW_ORC:-ON} \ + -e ARROW_PARQUET=${ARROW_PARQUET:-ON} \ + -e ARROW_PLASMA=${ARROW_PLASMA:-ON} \ + -e PARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-ON} \ + -e PARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-ON} \ --volume ${PWD}/build:/build \ ${DOCKER_IMAGE_ID} From 6e044ccb03d409d625164ab3ff77628abf892195 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 9 May 2020 11:38:19 +0900 Subject: [PATCH 4/4] Disable more components --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index ce0d9419c45fe..71925acb6d352 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,9 +35,14 @@ jobs: env: ARCH: s390x ARROW_CI_MODULES: "CPP" + ARROW_COMPUTE: "OFF" + ARROW_CSV: "OFF" ARROW_DATASET: "OFF" + ARROW_FILESYSTEM: "OFF" + ARROW_FLIGHT: "OFF" ARROW_GANDIVA: "OFF" ARROW_HDFS: "OFF" + ARROW_IPC: "OFF" ARROW_ORC: "OFF" ARROW_PARQUET: "OFF" ARROW_PLASMA: "OFF" @@ -78,9 +83,14 @@ script: ulimit -c unlimited || : - | archery docker run \ + -e ARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ + -e ARROW_CSV=${ARROW_CSV:-ON} \ -e ARROW_DATASET=${ARROW_DATASET:-ON} \ + -e ARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ + -e ARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ -e ARROW_GANDIVA=${ARROW_GANDIVA:-ON} \ -e ARROW_HDFS=${ARROW_HDFS:-ON} \ + -e ARROW_IPC=${ARROW_IPC:-ON} \ -e ARROW_ORC=${ARROW_ORC:-ON} \ -e ARROW_PARQUET=${ARROW_PARQUET:-ON} \ -e ARROW_PLASMA=${ARROW_PLASMA:-ON} \