From 62dd4854ec047139873ba550f275c7b12d41d7ad Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 20 Dec 2018 13:57:58 +0100 Subject: [PATCH 1/4] improve config file priorities --- conf/base.config | 1 - conf/munin.config | 2 ++ nextflow.config | 16 ++++++++-------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf/base.config b/conf/base.config index b890e8b076..29095984e2 100644 --- a/conf/base.config +++ b/conf/base.config @@ -10,7 +10,6 @@ wf_repository = 'maxulysse' params { // set up default params - containerPath = '' // Path to Singularity images docker = false // Don't use Docker to build buildContainers.nf download = false // Don't download reference files in buildReferences.nf explicitBqsrNeeded = true // Enable recalibration in main.nf diff --git a/conf/munin.config b/conf/munin.config index f3a55a7b39..5fa78adf71 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -11,6 +11,8 @@ env { } params { + containerPath = '/btb/containers/' + genome == 'GRCh38' genome_base = params.genome == 'GRCh37' ? '/btb/references/Homo_sapiens/GATK/GRCh37/' : params.genome == 'GRCh38' ? '/btb/references/Homo_sapiens/GATK/GRCh38/' : 'References/smallGRCh37' singleCPUMem = 15.GB totalMemory = 754.GB diff --git a/nextflow.config b/nextflow.config index f8eca2dc6b..ec881b74f6 100644 --- a/nextflow.config +++ b/nextflow.config @@ -27,9 +27,9 @@ profiles { // Docker images will be pulled automatically awsbatch { includeConfig 'conf/base.config' - includeConfig 'conf/igenomes.config' includeConfig 'conf/aws-batch.config' includeConfig 'conf/docker.config' + includeConfig 'conf/igenomes.config' includeConfig 'conf/resources.config' includeConfig 'conf/containers.config' } @@ -49,8 +49,8 @@ profiles { // Singularity images need to be set up btb { includeConfig 'conf/base.config' - includeConfig 'conf/igenomes.config' includeConfig 'conf/munin.config' + includeConfig 'conf/igenomes.config' includeConfig 'conf/singularity-path.config' } // Default config for CFC cluster in Tuebingen/Germany @@ -66,8 +66,8 @@ profiles { // Docker images will be pulled automatically docker { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/travis.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/docker.config' includeConfig 'conf/containers.config' } @@ -75,8 +75,8 @@ profiles { // Singularity images will be pulled automatically singularity { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/travis.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/singularity.config' includeConfig 'conf/containers.config' } @@ -84,8 +84,8 @@ profiles { // Singularity images need to be set up singularityPath { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/travis.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/singularity-path.config' } // slurm profile for UPPMAX secure clusters @@ -93,8 +93,8 @@ profiles { // Singularity images need to be set up slurm { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/uppmax-slurm.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/singularity-path.config' } // slurm profile for UPPMAX clusters @@ -102,8 +102,8 @@ profiles { // Singularity images will be pulled automatically slurmDownload { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/uppmax-slurm.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/singularity.config' includeConfig 'conf/containers.config' } @@ -112,8 +112,8 @@ profiles { // Singularity images need to be set up standard { includeConfig 'conf/base.config' - includeConfig 'conf/genomes.config' includeConfig 'conf/uppmax-localhost.config' + includeConfig 'conf/genomes.config' includeConfig 'conf/singularity-path.config' } } From a63174dba9419386202db5ec84172b0bff98aca7 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 20 Dec 2018 14:27:20 +0100 Subject: [PATCH 2/4] update CHANGELOG [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7952903460..aaf352769f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### `Changed` - [#710](https://github.com/SciLifeLab/Sarek/pull/710) - Improve release checklist and script +- [#711](https://github.com/SciLifeLab/Sarek/pull/711) - Improve configuration priorities ## [2.2.2] - 2018-12-19 From e6ce37cdaf10967a607a223d36dc57dbbebd5512 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 20 Dec 2018 14:30:09 +0100 Subject: [PATCH 3/4] remove unnecessary params --- conf/munin.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 5fa78adf71..0b78972776 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -12,7 +12,6 @@ env { params { containerPath = '/btb/containers/' - genome == 'GRCh38' genome_base = params.genome == 'GRCh37' ? '/btb/references/Homo_sapiens/GATK/GRCh37/' : params.genome == 'GRCh38' ? '/btb/references/Homo_sapiens/GATK/GRCh38/' : 'References/smallGRCh37' singleCPUMem = 15.GB totalMemory = 754.GB From bc331a8b8f57f150142e079d3eb4b6ace5c1d37e Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Fri, 21 Dec 2018 11:22:46 +0100 Subject: [PATCH 4/4] update docs --- docs/CONFIG.md | 20 ++------------------ docs/INSTALL_BIANCA.md | 4 +--- docs/INSTALL_RACKHAM.md | 17 +++++++++-------- nextflow.config | 15 ++------------- 4 files changed, 14 insertions(+), 42 deletions(-) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index df32a2b9c2..4b8786778f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -90,16 +90,6 @@ The default profile is `standard`, but Sarek has multiple predefined profiles wh ```bash nextflow run SciLifeLab/Sarek --sample mysample.tsv -profile myprofile ``` -awsbatch { -binac { -btb { -cfc { -docker { -singularity { -singularityPath { -slurm { -slurmDownload { -standard { ### `awsbatch` @@ -136,18 +126,12 @@ Singularity images needs to be set up. This is another profile for use on a UPPMAX cluster using the job scheduler slurm with Singularity. Will run the workflow on `/scratch`. -Singularity images need to be set up. - -### `slurmDownload` - -This is another profile for use on a UPPMAX cluster using the job scheduler slurm with Singularity. -Will run the workflow on `/scratch`. -Singularity images will be pulled automatically. +Singularity images are already set up. ### `standard` This is the default profile for use on a localhost on a UPPMAX cluster with Singularity. -Singularity images need to be set up. +Singularity images are already set up. ## Customisation The recommended way to use custom settings is to supply Sarek with an additional configuration file. You can use the files in the [`conf/`](https://github.com/SciLifeLab/Sarek/tree/master/conf) directory as an inspiration to make this new `.config` file and specify it using the `-c` flag: diff --git a/docs/INSTALL_BIANCA.md b/docs/INSTALL_BIANCA.md index d7ddb71f87..943b2f3ad4 100644 --- a/docs/INSTALL_BIANCA.md +++ b/docs/INSTALL_BIANCA.md @@ -58,14 +58,12 @@ For more information about using Singularity with UPPMAX, follow the [Singularit ## Install Sarek Sarek use Singularity containers to package all the different tools. -All containers are already stored on UPPMAX. +All containers, and all Reference files are already stored on UPPMAX. As `bianca` is secure, no direct download is available, so Sarek will have to be installed and updated manually. You can either download Sarek on your computer or on `rackham`, make an archive, and send it to `bianca` using `FileZilla` or `sftp` given your preferences. -All Reference files are already stored in `bianca`. - ```bash # Connect to rackham > ssh -AX [USER]@rackham.uppmax.uu.se diff --git a/docs/INSTALL_RACKHAM.md b/docs/INSTALL_RACKHAM.md index 8d6101b164..67ead8dc46 100644 --- a/docs/INSTALL_RACKHAM.md +++ b/docs/INSTALL_RACKHAM.md @@ -8,16 +8,17 @@ This workflow itself needs little installation. You need to install [Nextflow][nextflow-link] and put it somewhere in your `$PATH` -The Reference files are already stored in `rackham`. +Sarek use Singularity containers to package all the different tools. + +All containers, and all Reference files are already stored on UPPMAX. Nextflow will automatically fetch Sarek from GitHub when launched if `SciLifeLab/Sarek` is specified as the workflow name. -Sarek use Singularity containers to package all the different tools. -All containers are already stored on UPPMAX. ## Test Sarek with small dataset and small reference -For more information, follow the [reference files documentation](REFERENCES.md). The following tutorial explain how to run Sarek on a small dataset using a small reference. +For more information, follow the [reference files documentation](REFERENCES.md). +The following tutorial explain how to run Sarek on a small dataset using a small reference. ```bash # Connect to rackham @@ -45,10 +46,10 @@ For more information, follow the [reference files documentation](REFERENCES.md). > cd test_Sarek # Build the smallGRCh37 reference -> nextflow run SciLifeLab/Sarek/buildReferences.nf --download --genome smallGRCh37 --project [PROJECT] -profile download +> nextflow run SciLifeLab/Sarek/buildReferences.nf --download --genome smallGRCh37 --project [PROJECT] # Test the workflow on a test tiny set -> nextflow run SciLifeLab/Sarek --test --genome smallGRCh37 --noReports --project [PROJECT] -profile download +> nextflow run SciLifeLab/Sarek --test --genome smallGRCh37 --noReports --project [PROJECT] ``` ## Update Sarek @@ -63,12 +64,12 @@ For more information, follow the [reference files documentation](REFERENCES.md). ## Use Sarek with slurm -To use Sarek on rackham you will need to use the `slurmDownload` profile. +To use Sarek on rackham you will need to use the `slurm` profile. ```bash # Connect to rackham > ssh -AX [USER]@rackham.uppmax.uu.se # Run the workflow directly on the login node -> nextflow run SciLifeLab/Sarek/main.nf --project [PROJECT] -profile slurmDownload +> nextflow run SciLifeLab/Sarek/main.nf --project [PROJECT] -profile slurm ``` diff --git a/nextflow.config b/nextflow.config index ec881b74f6..325a82a37c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -90,26 +90,16 @@ profiles { } // slurm profile for UPPMAX secure clusters // Runs the pipeline using the job scheduler - // Singularity images need to be set up + // Singularity images are already set up slurm { includeConfig 'conf/base.config' includeConfig 'conf/uppmax-slurm.config' includeConfig 'conf/genomes.config' includeConfig 'conf/singularity-path.config' } - // slurm profile for UPPMAX clusters - // Runs the pipeline using the job scheduler - // Singularity images will be pulled automatically - slurmDownload { - includeConfig 'conf/base.config' - includeConfig 'conf/uppmax-slurm.config' - includeConfig 'conf/genomes.config' - includeConfig 'conf/singularity.config' - includeConfig 'conf/containers.config' - } // Default profile for UPPMAX secure clusters // Runs the pipeline locally on a single 16-core node - // Singularity images need to be set up + // Singularity images are already set up standard { includeConfig 'conf/base.config' includeConfig 'conf/uppmax-localhost.config' @@ -118,7 +108,6 @@ profiles { } } - // Function to ensure that resource requirements don't go beyond // a maximum limit def check_max(obj, type) {