Skip to content

Commit

Permalink
🚚 Move variant-lite.Singularity → Singularity.latest-lite (#1464)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shnizzedy
shnizzedy authored Mar 15, 2021
2 parents fc88ab7 + 874c970 commit 0926b45
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 61 deletions.
163 changes: 102 additions & 61 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,42 @@
version: 2.1

commands:
build-images:
parameters:
variant:
type: string
default: ""
steps:
- set-up-variant:
variant: "<< parameters.variant >>"
- run:
name: "Building Docker image"
command: |
docker build -t fcpindi/c-pac:${DOCKER_TAG} . ${DOCKERFILE}
docker save fcpindi/c-pac:${DOCKER_TAG} | gzip > ${DOCKER_TARBALL}.tar.gz
no_output_timeout: 5h
- run:
name: "Starting local registry"
command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
- install-singularity-requirements
- when:
condition:
or:
- equal: [ "", "<< parameters.variant >>" ]
- equal: [ "lite", "<< parameters.variant >>" ]
steps:
- run:
name: "Cloning Singularity 2.5.2"
command: git clone -b 2.5.2 https://github.com/sylabs/singularity
- set-up-singularity
- run:
name: "Building Singularity image from Docker image"
command: |
docker load < ${DOCKER_TARBALL}.tar.gz
docker tag fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
docker push localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
no_output_timeout: 5h
check-for-built-images:
steps:
- run:
Expand All @@ -10,6 +46,25 @@ commands:
then
circleci step halt
fi
configure-git-user:
steps:
- add_ssh_keys:
fingerprints:
- "12:bc:f2:e4:31:cc:72:54:54:bc:f5:5b:89:e6:d8:ee"
- run:
name: "Configuring git user"
command: |
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
sudo apt-get update
sudo apt-get install git -y
git config --global user.email "${CIRCLE_USERNAME}@users.noreply.github.com"
git config --global user.name "${CIRCLE_USERNAME} @ CircleCI"
get-sample-bids-data:
steps:
- run:
name: Getting Sample BIDS Data
command: git clone https://github.com/bids-standard/bids-examples.git
set-python-version:
steps:
- run:
Expand All @@ -24,6 +79,14 @@ commands:
name: "Installing Singularity requirements"
command: |
sudo apt-get update && sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 libarchive-dev libgpgme11-dev libseccomp-dev -y
run-pytest-docker:
steps:
- run:
name: Running pytest on Docker image
command: |
docker load < cpac-docker-image.tar.gz
docker run -dit -P -v /home/circleci/project/test-results:/code/test-results -v /home/circleci/project/htmlcov:/code/htmlcov --entrypoint=/bin/bash --name docker_test fcpindi/c-pac:${DOCKER_TAG}
docker exec docker_test /bin/bash ./code/dev/circleci_data/test_in_image.sh
set-up-singularity:
steps:
- run:
Expand All @@ -35,11 +98,10 @@ commands:
make
sudo make install
cd ..
build-images:
set-up-variant:
parameters:
variant:
type: string
default: ""
steps:
- run:
name: "Setting up variant"
Expand All @@ -48,81 +110,50 @@ commands:
if [[ -n "<< parameters.variant >>" ]]
then
DOCKERFILE="variant-<< parameters.variant >>.Dockerfile"
sed -i "s|:latest|:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
if [[ -f ${DOCKERFILE} ]]
then
sed -i "s|:latest|:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
fi
DOCKER_TARBALL="${DOCKER_TARBALL}-<< parameters.variant >>"
VARIANT="-<< parameters.variant >>"
echo "export DOCKERFILE=\"-f ${DOCKERFILE}\"" >> $BASH_ENV
fi
echo "export DOCKER_TARBALL=${DOCKER_TARBALL}" >> $BASH_ENV
echo "export VARIANT=${VARIANT}" >> $BASH_ENV
- run:
name: "Building Docker image"
command: |
docker build -t fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} . ${DOCKERFILE}
docker save fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} | gzip > ${DOCKER_TARBALL}.tar.gz
no_output_timeout: 5h
# Persist the specified paths (workspace/echo-output) into the workspace for use in downstream job.
- run:
name: "Starting local registry"
command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
- install-singularity-requirements
- when:
condition:
equal: [ "", "<< parameters.variant >>" ]
steps:
- run:
name: "Cloning Singularity 2.5.2"
command: git clone -b 2.5.2 https://github.com/sylabs/singularity
- set-up-singularity
- run:
name: "Building Singularity image from Docker image"
command: |
docker load < ${DOCKER_TARBALL}.tar.gz
docker tag fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
docker push localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
no_output_timeout: 5h
configure-git-user:
DOCKER_TAG=${CIRCLE_BRANCH//\//_}${VARIANT}
echo "export DOCKER_TAG=${DOCKER_TAG}" >> $BASH_ENV
test-singularity-installation:
steps:
- add_ssh_keys:
fingerprints:
- "12:bc:f2:e4:31:cc:72:54:54:bc:f5:5b:89:e6:d8:ee"
- run:
name: "Configuring git user"
name: Testing Singularity installation
command: |
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
sudo apt-get update
sudo apt-get install git -y
git config --global user.email "${CIRCLE_USERNAME}@users.noreply.github.com"
git config --global user.name "${CIRCLE_USERNAME} @ CircleCI"
pip install -r dev/circleci_data/requirements.txt
coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors dev/circleci_data/test_install.py
jobs:
pytest-docker:
parameters:
variant:
type: string
default: ""
machine:
image: ubuntu-2004:202010-01
steps:
- attach_workspace:
# Must be absolute path or relative path from working_directory
at: /home/circleci/
- set-up-variant:
variant: "<< parameters.variant >>"
- check-for-built-images
- set-python-version
- run:
name: Getting Sample BIDS Data
command: git clone https://github.com/bids-standard/bids-examples.git
- run:
name: Running pytest on Docker image
command: |
docker load < cpac-docker-image.tar.gz
docker run -dit -P -v /home/circleci/project/test-results:/code/test-results -v /home/circleci/project/htmlcov:/code/htmlcov --entrypoint=/bin/bash --name docker_test fcpindi/c-pac:${CIRCLE_BRANCH//\//_}
docker exec docker_test /bin/bash ./code/dev/circleci_data/test_in_image.sh
- get-sample-bids-data
- run-pytest-docker
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
- store_artifacts:
path: htmlcov
no_output_timeout: 5h

pytest-singularity:
machine:
image: ubuntu-2004:202010-01
Expand All @@ -134,11 +165,8 @@ jobs:
- set-python-version
- install-singularity-requirements
- set-up-singularity
- run:
name: Testing Singularity installation
command: |
pip install -r dev/circleci_data/requirements.txt
coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors dev/circleci_data/test_install.py
- test-singularity-installation

build:
machine:
image: ubuntu-2004:202010-01
Expand Down Expand Up @@ -196,11 +224,24 @@ workflows:
version: 2
build-and-test:
jobs:
- build
- build-lite
- build:
name: "Build images"
- build-lite:
name: "Build lite variant images"
- pytest-docker:
name: "Test in Docker"
requires:
- "Build images"
- pytest-docker:
name: "Test lite variant in Docker"
requires:
- "Build lite variant images"
variant: lite
- pytest-singularity:
name: "Test in Singularity"
requires:
- build
- "Build images"
- pytest-singularity:
name: "Test lite variant in Singularity"
requires:
- build
- "Build lite variant images"
0 variant-lite.Singularity → Singularity.latest-lite
View file
File renamed without changes.

0 comments on commit 0926b45

Please sign in to comment.