v2.0 Release Candidate

.editorconfig

@@ -8,12 +8,9 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = space
 
-[*.{yml,yaml}]
+[*.{md,yml,yaml,html,css,scss,js}]
 indent_size = 2
 
-[*.json]
-insert_final_newline = unset
-
 # These files are edited and tested upstream in nf-core/modules
 [/modules/nf-core/**]
 charset = unset

.github/CONTRIBUTING.md

@@ -15,8 +15,7 @@ Contributions to the code are even more welcome ;)
 
 If you'd like to write some code for nf-core/cutandrun, the standard workflow is as follows:
 
-1. Check that there isn't already an issue about your idea in the [nf-core/cutandrun issues](https://github.com/nf-core/cutandrun/issues) to avoid duplicating work
-    * If there isn't one already, please create one so that others know you're working on this
+1. Check that there isn't already an issue about your idea in the [nf-core/cutandrun issues](https://github.com/nf-core/cutandrun/issues) to avoid duplicating work. If there isn't one already, please create one so that others know you're working on this
 2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [nf-core/cutandrun repository](https://github.com/nf-core/cutandrun) to your GitHub account
 3. Make the necessary changes / additions within your forked repository following [Pipeline conventions](#pipeline-contribution-conventions)
 4. Use `nf-core schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10).
@@ -49,9 +48,9 @@ These tests are run both with the latest available version of `Nextflow` and als
 
 :warning: Only in the unlikely and regretful event of a release happening with a bug.
 
-* On your own fork, make a new branch `patch` based on `upstream/master`.
-* Fix the bug, and bump version (X.Y.Z+1).
-* A PR should be made on `master` from patch to directly this particular bug.
+- On your own fork, make a new branch `patch` based on `upstream/master`.
+- Fix the bug, and bump version (X.Y.Z+1).
+- A PR should be made on `master` from patch to directly this particular bug.
 
 ## Getting help
 
@@ -73,7 +72,7 @@ If you wish to contribute a new step, please use the following coding standards:
 6. Add sanity checks and validation for all relevant parameters.
 7. Perform local tests to validate that the new code works as expected.
 8. If applicable, add a new test command in `.github/workflow/ci.yml`.
-9. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, file name clean up and module plots are in the appropriate order. If applicable, add a [MultiQC](https://https://multiqc.info/) module.
+9. Update MultiQC config `assets/multiqc_config.yml` so relevant suffixes, file name clean up and module plots are in the appropriate order. If applicable, add a [MultiQC](https://https://multiqc.info/) module.
 10. Add a description of the output files and if relevant any appropriate images from the MultiQC report to `docs/output.md`.
 
 ### Default values
@@ -92,8 +91,8 @@ The process resources can be passed on to the tool dynamically within the proces
 
 Please use the following naming schemes, to make it easy to understand what is going where.
 
-* initial process channel: `ch_output_from_<process>`
-* intermediate and terminal channels: `ch_<previousprocess>_for_<nextprocess>`
+- initial process channel: `ch_output_from_<process>`
+- intermediate and terminal channels: `ch_<previousprocess>_for_<nextprocess>`
 
 ### Nextflow version bumping
 

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,9 +1,7 @@
-
 name: Bug report
 description: Report something that is broken or incorrect
 labels: bug
 body:
-
   - type: markdown
     attributes:
       value: |

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,16 +10,15 @@ Remember that PRs should be made against the dev branch, unless you're preparing
 
 Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/cutandrun/tree/master/.github/CONTRIBUTING.md)
 -->
-<!-- markdownlint-disable ul-indent -->
 
 ## PR checklist
 
 - [ ] This comment contains a description of changes (with reason).
 - [ ] If you've fixed a bug or added code that should be tested, add tests!
-    - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/cutandrun/tree/master/.github/CONTRIBUTING.md)
-    - [ ] If necessary, also make a PR on the nf-core/cutandrun _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
+  - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/cutandrun/tree/master/.github/CONTRIBUTING.md)
+  - [ ] If necessary, also make a PR on the nf-core/cutandrun _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
-- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`).
+- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.
 - [ ] `CHANGELOG.md` is updated.

.github/workflows/awsfulltest.yml

@@ -14,21 +14,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
+        uses: nf-core/tower-action@v3
         # TODO nf-core: You can customise AWS full pipeline tests as required
         # Add full size test data (but still relatively small datasets for few samples)
         # on the `test_full.config` test runs with only one set of parameters
-
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
-          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/cutandrun/work-${{ github.sha }}
           parameters: |
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/cutandrun/results-${{ github.sha }}"
             }
           profiles: test_full,aws_tower
-          pre_run_script: 'export NXF_VER=21.10.3'
+          nextflow_config: |
+            process.errorStrategy = 'retry'
+            process.maxRetries = 3

.github/workflows/awstest.yml

@@ -10,19 +10,19 @@ jobs:
     if: github.repository == 'nf-core/cutandrun'
     runs-on: ubuntu-latest
     steps:
+      # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
-
+        uses: nf-core/tower-action@v3
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
-          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/cutandrun/work-${{ github.sha }}
           parameters: |
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/cutandrun/results-test-${{ github.sha }}"
             }
           profiles: test,aws_tower
-          pre_run_script: 'export NXF_VER=21.10.3'
+          nextflow_config: |
+            process.errorStrategy = 'retry'
+            process.maxRetries = 3

.github/workflows/branch.yml

@@ -13,8 +13,7 @@ jobs:
       - name: Check PRs
         if: github.repository == 'nf-core/cutandrun'
         run: |
-          { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/cutandrun ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
-
+          "{ [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/cutandrun ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]"
 
       # If the above check failed, post a comment on the PR explaining the failure
       # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
@@ -43,4 +42,4 @@ jobs:
             Thanks again for your contribution!
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allow-repeats: false
-
+#

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
   test:
     name: Run standard pipeline test
     # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/cutandrun') }}
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/cutandrun') }}"
     runs-on: ubuntu-latest
     env:
       NXF_VER: ${{ matrix.nxf_ver }}
@@ -26,9 +26,9 @@ jobs:
         # Nextflow versions
         include:
           # Test pipeline minimum Nextflow version
-          - NXF_VER: '21.10.3'
+          - NXF_VER: "21.10.3"
           # Test latest nextflow version
-          - NXF_VER: ''
+          - NXF_VER: ""
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -39,7 +39,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
 
       # Work around for the unexpected end of file error that github actions seems to get when downloading compressed
-      # files during pipelie execution
+      # files during pipeline execution
       - name: Download data
         run: |
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/hg38-chr20.fa.gz
@@ -49,9 +49,11 @@ jobs:
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz
 
-      - name: Run pipeline
-        run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip
+      - name: Run Pipeline
+        uses: Wandalen/[email protected]
+        with:
+          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip
+          attempt_limit: 2
 
   ##############################
   ### LARGE INTEGRATION TEST ###
@@ -68,7 +70,7 @@ jobs:
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
-        nxf_ver: ['21.10.3', '']
+        nxf_ver: ["21.10.3", ""]
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -79,7 +81,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
 
       # Work around for the unexpected end of file error that github actions seems to get when downloading compressed
-      # files during pipelie execution
+      # files during pipeline execution
       - name: Download data
         run: |
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/hg38-chr20.fa.gz
@@ -89,9 +91,11 @@ jobs:
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz
 
-      - name: Run pipeline with test data
-        run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_full_small_local_zip
+      - name: Run Pipeline
+        uses: Wandalen/[email protected]
+        with:
+          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_full_small_local_zip
+          attempt_limit: 2
 
   #####################################
   ### EDGE VERSION INTEGRATION TEST ###
@@ -102,7 +106,7 @@ jobs:
     if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/cutandrun') }}
     runs-on: ubuntu-latest
     env:
-      NXF_VER: ''
+      NXF_VER: ""
       NXF_EDGE: 1
       NXF_ANSI_LOG: false
       CAPSULE_LOG: none
@@ -117,7 +121,7 @@ jobs:
           nextflow self-update
 
       # Work around for the unexpected end of file error that github actions seems to get when downloading compressed
-      # files during pipelie execution
+      # files during pipeline execution
       - name: Download data
         run: |
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/hg38-chr20.fa.gz
@@ -127,9 +131,11 @@ jobs:
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz
           wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz
 
-      - name: Run pipeline with test data
-        run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip
+      - name: Run Pipeline
+        uses: Wandalen/[email protected]
+        with:
+          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip
+          attempt_limit: 2
 
   ##############################
   ###      UNIT TESTS        ###
@@ -144,45 +150,87 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        nxf_version: ['21.10.3', '']
-        tags: ["samplesheet", "verify_output_input", "verify_output_align_short", "verify_output_align_long", "verify_output_peak_calling", "verify_output_reporting", "verify_output_save", "verify_output_skip"]
+        nxf_version: ["21.10.3", ""]
+        tags:
+          - test_params
+          - test_samplesheet
+          - verify_output_save_ref
+          - verify_output_only_input
+          - verify_output_save_merged
+          - verify_output_skip_fastqc
+          - verify_output_save_trimmed
+          - verify_output_skip_trimming
+          - verify_output_align_intermed
+          - verify_output_align_only_align
+          - verify_output_align_save_spikein_align
+          - verify_output_align_save_unaligned
+          - verify_output_only_filtering
+          - verify_output_align_duplicates_mark
+          - verify_output_align_duplicates_remove
+          - verify_output_align_duplicates_remove_target
+          - verify_output_peak_calling_only_peak_calling
+          - test_bam_scale_none
+          - test_bam_scale_spikein
+          - test_bam_scale_cpm
+          - test_bam_scale_rpkm
+          - test_bam_scale_bpm
+          - test_bam_scale_cpm_iggscale
+          - test_peak_callers_seacr
+          - test_peak_callers_macs2
+          - test_peak_callers_invalid
+          - test_peak_callers_seacr_macs2
+          - test_peak_callers_macs2_seacr
+          - test_peak_callers_seacr_macs2_noigg
+          - test_peak_callers_ctrl_tests
+          - test_conseneus_peaks_group
+          - test_conseneus_peaks_all
+          - test_conseneus_peaks_invalid
+          - verify_output_reporting_skip_reporting
+          - verify_output_reporting_skip_igv
+          - verify_output_reporting_skip_heatmaps
+          - verify_output_reporting_skip_multiqc
+          - verify_output_skip_frip
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout Code
+        uses: actions/checkout@v2
 
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: "3.x"
 
-      - uses: actions/cache@v2
+      - name: Setup Pip Cache
+        uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
 
       - name: Install Python dependencies
         run: python -m pip install --upgrade pip pytest-workflow
 
-      - uses: actions/cache@v2
+      - name: Restore Keys
+        uses: actions/cache@v2
         with:
           path: /usr/local/bin/nextflow
           key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }}
           restore-keys: |
             ${{ runner.os }}-nextflow-
+
       - name: Install Nextflow
         env:
           NXF_VER: ${{ matrix.nxf_version }}
           CAPSULE_LOG: none
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
+
       - name: Run pytest-workflow
-        run: pytest --tag ${{ matrix.tags }} --kwdof
+        uses: Wandalen/[email protected]
+        with:
+          command: pytest --tag ${{ matrix.tags }} --kwdof
+          attempt_limit: 3
 
       - name: Upload logs on failure
         if: failure()