Merge branch 'release/1.42.0'

futureverse · Jan 30, 2025 · feebe7b · feebe7b
2 parents bf44f96 + 995e99f
commit feebe7b
Show file tree

Hide file tree

Showing 24 changed files with 207 additions and 89 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,5 +1,5 @@
 Package: parallelly
-Version: 1.41.0
+Version: 1.42.0
 Title: Enhancing the 'parallel' Package
 Imports:
     parallel,

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,28 @@
+# Version 1.42.0 [2025-01-30]
+
+## New Features
+
+ * Now `availableCores()` and `availableWorkers()` support also when
+   both CGroups v1 and CGroups v2 are enabled on the
+   machine. Previously, such configurations were completely ignored.
+
+## Bug Fixes
+
+ * Call `isNodeAlive()` and `killNode()` on cluster nodes running on
+   external machines would produce `Error in match.arg(type, choices =
+   known_types, several.ok = FALSE) : 'arg' must be of length 1`. This
+   bug was introduced in version 1.38.0 (2024-07-27), when adding
+   richer support for the `rscript_sh` argument.
+
+ * Call `isNodeAlive()` and `killNode()` on cluster nodes running on
+   external machines would produce `Error: ‘length(rsh_call) == 1L’ is
+   not TRUE` if option `rshopts` were specified during creation.
+
+ * The value of `availableCores()` was numeric rather than integer as
+   documented. This harmless bug was introduced in version 1.31.0
+   (2022-04-07).
+
+
 # Version 1.41.0 [2024-12-17]
 
 ## New Features

diff --git a/R/availableCores.R b/R/availableCores.R
@@ -122,7 +122,8 @@
 #'
 #'  \item `"SGE"` -
 #'    Query Sun Grid Engine/Oracle Grid Engine/Son of Grid Engine (SGE)
-#'    and Univa Grid Engine (UGE) environment variable \env{NSLOTS}.
+#'    and Univa Grid Engine (UGE)/Altair Grid Engine (AGE) environment
+#'    variable \env{NSLOTS}.
 #'    An example of a job submission that results in this is
 #'    `qsub -pe smp 2` (or `qsub -pe by_node 2`), which
 #'    requests two cores on a single machine.
@@ -305,14 +306,18 @@ availableCores <- function(constraints = NULL, methods = getOption2("parallelly.
     } else if (method == "cgroups.cpuquota") {
       ## Number of cores according to Unix cgroups v1 CPU quota
       n <- getCGroups1CpuQuota()
-      if (!is.na(n)) {
+      if (is.na(n)) {
+        n <- NA_integer_
+      } else {
         n <- as.integer(floor(n + 0.5))
 	if (n == 0L) n <- 1L  ## If CPU quota < 0.5, round up to one CPU
       }
     } else if (method == "cgroups2.cpu.max") {
       ## Number of cores according to Unix cgroups v2 CPU max quota
       n <- getCGroups2CpuMax()
-      if (!is.na(n)) {
+      if (is.na(n)) {
+        n <- NA_integer_
+      } else {
         n <- as.integer(floor(n + 0.5))
 	if (n == 0L) n <- 1L  ## If CPU max quota < 0.5, round up to one CPU
       }
@@ -332,10 +337,10 @@ availableCores <- function(constraints = NULL, methods = getOption2("parallelly.
         on.exit(options(oopts))
         fcn()
       })
-      n <- as.integer(n)
       if (length(n) != 1L) {
         stop("Function specified by option 'parallelly.availableCores.custom' does not a single value")
       }
+      n <- as.integer(n)
     } else {
       ## covr: skip=3
       ## Fall back to querying option and system environment variable

diff --git a/R/availableWorkers.R b/R/availableWorkers.R
@@ -57,7 +57,8 @@
 #'
 #'  \item `"SGE"` -
 #'    Query Sun Grid Engine/Oracle Grid Engine/Son of Grid Engine (SGE)
-#'    and Univa Grid Engine (UGE) environment variable \env{PE_HOSTFILE}.
+#'    and Univa Grid Engine (UGE)/Altair Grid Engine (AGE) environment
+#'    variable \env{PE_HOSTFILE}.
 #'    An example of a job submission that results in this is
 #'    `qsub -pe mpi 8` (or `qsub -pe ompi 8`), which
 #'    requests eight cores on a any number of machines.

diff --git a/R/cgroups.R b/R/cgroups.R
@@ -120,16 +120,13 @@ cloneCGroups <- function(tarfile = "cgroups.tar.gz") {
 
   ## Mixed CGroups versions are not supported
   utypes <- unique(mounts$type)
-  if (length(utypes) > 1) {
-    stop("Mixed CGroups versions are not supported: ", paste(sQuote(utypes), collapse = ", "))	
+
+  controllers <- c()
+   if ("cgroup" %in% utypes) {
+    controllers <- c(controllers, "cpu", "cpuset")
   }
-
-  if (utypes == "cgroup") {
-    controllers <- c("cpu", "cpuset")
-  } else if (utypes == "cgroup2") {
-    controllers <- ""
-  } else {
-    stop("Unknown CGroups version: ", sQuote(utypes))
+  if ("cgroup2" %in% utypes) {
+    controllers <- c(controllers, "")
   }
 
   ## Write CGroups mountpoints
@@ -328,15 +325,6 @@ getCGroupsRoot <- local({
     ## Look up the CGroups mountpoint
     mounts <- getCGroupsMounts()
 
-    ## Mixed CGroups versions are not supported
-    utypes <- unique(mounts$type)
-    if (length(utypes) > 1) {
-      warning("Mixed CGroups versions are not supported: ", paste(sQuote(utypes), collapse = ", "))
-      path <- NA_character_
-      .cache[[controller]] <<- path
-      return(path)
-    }
-
     ## Filter by CGroups v1 or v2?
     if (nzchar(controller)) {
       mounts <- subset(mounts, type == "cgroup")

diff --git a/R/isNodeAlive.R b/R/isNodeAlive.R
@@ -99,18 +99,19 @@ isNodeAlive.RichSOCKnode <- function(x, timeout = 0.0, ...) {
 
   ## Command to call Rscript -e 
   code <- sprintf("cat(%s:::pid_exists(%d))", .packageName, pid)
-  rscript_args <- paste(c("-e", shQuote(code, type = rscript_sh)), collapse = " ")
+  rscript_args <- paste(c("-e", shQuote(code, type = rscript_sh[1])), collapse = " ")
   cmd <- paste(rscript, rscript_args)
   debug && mdebugf("- Rscript command to be called on the other host: %s", cmd)
   stop_if_not(length(cmd) == 1L)
 
   rshopts <- args_org$rshopts
   if (length(args_org$user) == 1L) rshopts <- c("-l", args_org$user, rshopts)
+  rshopts <- paste(rshopts, collapse = " ")
   rsh_call <- paste(paste(shQuote(rshcmd), collapse = " "), rshopts, worker)
   debug && mdebugf("- Command to connect to the other host: %s", rsh_call)
   stop_if_not(length(rsh_call) == 1L)
 
-  local_cmd <- paste(rsh_call, shQuote(cmd, type = rscript_sh))
+  local_cmd <- paste(rsh_call, shQuote(cmd, type = rscript_sh[2]))
   debug && mdebugf("- System call: %s", local_cmd)
   stop_if_not(length(local_cmd) == 1L)
 

diff --git a/R/killNode.R b/R/killNode.R
@@ -131,18 +131,19 @@ killNode.RichSOCKnode <- function(x, signal = tools::SIGTERM, timeout = 0.0, ...
   signal_str <- paste(sprintf("%s", signal), collapse = ", ")
   if (length(signal) > 1) signal_str <- sprintf("c(%s)", signal_str)
   code <- sprintf("cat(tools::pskill(%d, signal = %s))", pid, signal_str)
-  rscript_args <- paste(c("-e", shQuote(code, type = rscript_sh)), collapse = " ")
+  rscript_args <- paste(c("-e", shQuote(code, type = rscript_sh[1])), collapse = " ")
   cmd <- paste(rscript, rscript_args)
   debug && mdebugf("- Rscript command to be called on the other host: %s", cmd)
   stop_if_not(length(cmd) == 1L)
 
   rshopts <- args_org$rshopts
   if (length(args_org$user) == 1L) rshopts <- c("-l", args_org$user, rshopts)
+  rshopts <- paste(rshopts, collapse = " ")
   rsh_call <- paste(paste(shQuote(rshcmd), collapse = " "), rshopts, worker)
   debug && mdebugf("- Command to connect to the other host: %s", rsh_call)
   stop_if_not(length(rsh_call) == 1L)
 
-  local_cmd <- paste(rsh_call, shQuote(cmd, type = rscript_sh))
+  local_cmd <- paste(rsh_call, shQuote(cmd, type = rscript_sh[2]))
   debug && mdebugf("- System call: %s", local_cmd)
   stop_if_not(length(local_cmd) == 1L)
 

diff --git a/R/makeNodePSOCK.R b/R/makeNodePSOCK.R
@@ -53,8 +53,8 @@
 #' @param rscript_sh The type of shell used where `rscript` is launched,
 #' which should be `"sh"` is launched via a POSIX shell and `"cmd"` if 
 #' launched via an MS Windows shell.  This controls how shell command-line
-#' options are quoted, via
-#' \code{\link[base:shQuote]{shQuote(..., type = rscript_sh)}}.
+#' options are quoted, but also how R string expression are quoted when
+#' passed to `Rscript`.
 #' If `"none"`, then no quoting is done.
 #' If `"auto"` (default), and the cluster node is launched locally, then it
 #' is set to `"sh"` or `"cmd"` according to the current platform.

diff --git a/R/utils,cluster.R b/R/utils,cluster.R
@@ -385,7 +385,7 @@ shQuote <- local({
       if (.Platform$OS.type == "windows") {
         type <- "cmd"
       } else {
-        type <- known_types
+        type <- known_types[1]
       }
     }
     type <- match.arg(type, choices = known_types, several.ok = FALSE)

diff --git a/README.md b/README.md
@@ -128,3 +128,8 @@ This will install the package from source.  Because of this and because this pac
 
 <!-- pkgdown-drop-below -->
 
+
+## Contributing
+
+To contribute to this package, please see [CONTRIBUTING.md](CONTRIBUTING.md).
+
diff --git a/cran-comments.md b/cran-comments.md
@@ -1,3 +1,3 @@
-I've verified this submission has no negative impact on any of the 60 reverse package dependencies available on CRAN (n = 54) and Bioconductor (n = 6).
+I've verified this submission has no negative impact on any of the 63 reverse package dependencies available on CRAN (n = 57) and Bioconductor (n = 6).
 
 Thank you
diff --git a/inst/test-data/mixed-cgroups/piavpn-cgroups.tar.gz b/inst/test-data/mixed-cgroups/piavpn-cgroups.tar.gz
diff --git a/inst/test-data/mixed-cgroups/piavpn.R b/inst/test-data/mixed-cgroups/piavpn.R
@@ -0,0 +1,6 @@
+ncores <- parallelly::availableCores(which = "all")
+print(ncores)
+
+stopifnot(
+  !any(grepl("^cgroups", names(ncores)))
+)
diff --git a/inst/test-data/mixed-cgroups/slurm-cgroups.tar.gz b/inst/test-data/mixed-cgroups/slurm-cgroups.tar.gz
diff --git a/inst/test-data/mixed-cgroups/slurm.R b/inst/test-data/mixed-cgroups/slurm.R
@@ -0,0 +1,7 @@
+ncores <- parallelly::availableCores(which = "all")
+print(ncores)
+
+stopifnot(
+  "cgroups.cpuset" %in% names(ncores),
+  ncores[["cgroups.cpuset"]] == 1L
+)
diff --git a/man/availableCores.Rd b/man/availableCores.Rd
diff --git a/man/availableWorkers.Rd b/man/availableWorkers.Rd
diff --git a/man/makeClusterPSOCK.Rd b/man/makeClusterPSOCK.Rd
diff --git a/revdep/README.md b/revdep/README.md
@@ -10,33 +10,33 @@
 |collate  |en_US.UTF-8                                           |
 |ctype    |en_US.UTF-8                                           |
 |tz       |America/Los_Angeles                                   |
-|date     |2024-12-15                                            |
+|date     |2025-01-29                                            |
 |pandoc   |3.6 @ /software/c4/cbi/software/pandoc-3.6/bin/pandoc |
 
 # Dependencies
 
 |package    |old    |new         |Δ  |
 |:----------|:------|:-----------|:--|
-|parallelly |1.40.1 |1.40.1-9007 |*  |
+|parallelly |1.41.0 |1.41.0-9011 |*  |
 
 # Revdeps
 
-## All (60)
+## All (63)
 
 |package             |version |error |warning |note |
 |:-------------------|:-------|:-----|:-------|:----|
 |[adea](problems.md#adea)|1.5.2   |1     |        |     |
 |alookr              |0.3.9   |      |        |     |
-|amadeus             |1.1.6   |      |        |     |
+|bbw                 |0.3.0   |      |        |     |
 |bigparallelr        |0.3.2   |      |        |     |
 |bigreadr            |0.2.5   |      |        |     |
 |[bootUR](problems.md#bootur)|1.0.4   |      |        |2    |
-|[breathteststan](problems.md#breathteststan)|0.8.8   |      |        |2    |
+|[breathteststan](problems.md#breathteststan)|0.8.9   |      |        |2    |
 |cccrm               |3.0.3   |      |        |     |
-|[COTAN](problems.md#cotan)|2.6.1   |      |        |3    |
+|[COTAN](problems.md#cotan)|2.6.2   |      |        |3    |
 |CptNonPar           |0.2.1   |      |        |     |
 |dataquieR           |2.1.0   |      |        |     |
-|[decoupleR](problems.md#decoupler)|2.12.0  |      |1       |1    |
+|[decoupleR](problems.md#decoupler)|2.12.0  |3     |1       |1    |
 |DEoptim             |2.2-8   |      |        |     |
 |[desla](problems.md#desla)|0.3.0   |      |        |1    |
 |fiery               |1.2.1   |      |        |     |
@@ -45,12 +45,12 @@
 |future              |1.34.0  |      |        |     |
 |future.batchtools   |0.12.1  |      |        |     |
 |future.mirai        |0.2.2   |      |        |     |
-|[gastempt](problems.md#gastempt)|0.6.2   |      |        |2    |
+|[gastempt](problems.md#gastempt)|0.7.0   |      |        |2    |
 |GREENeR             |1.0.0   |      |        |     |
 |greta               |0.5.0   |      |        |     |
-|[gtfstools](problems.md#gtfstools)|1.3.0   |      |        |1    |
+|[gtfstools](problems.md#gtfstools)|1.4.0   |      |        |1    |
 |IFAA                |1.8.0   |      |        |     |
-|[InPAS](problems.md#inpas)|2.14.0  |      |1       |2    |
+|[InPAS](problems.md#inpas)|2.14.1  |      |1       |2    |
 |[JMbayes2](problems.md#jmbayes2)|0.5-0   |      |        |1    |
 |kernelboot          |0.1.10  |      |        |     |
 |LWFBrook90R         |0.6.1   |      |        |     |
@@ -59,25 +59,28 @@
 |mlr3                |0.22.1  |      |        |     |
 |[mmrm](problems.md#mmrm)|0.3.14  |      |        |1    |
 |modeltime           |1.3.1   |      |        |     |
-|multilevelmediation |0.3.1   |      |        |     |
+|multilevelmediation |0.4.1   |      |        |     |
 |[NCC](problems.md#ncc)|1.0     |      |        |1    |
 |[nebula](problems.md#nebula)|1.5.3   |      |        |1    |
 |[outliers.ts.oga](problems.md#outlierstsoga)|0.0.1   |      |1       |     |
-|parseRPDR           |1.1.1   |      |        |     |
+|parseRPDR           |1.1.2   |      |        |     |
 |[pmartR](problems.md#pmartr)|2.4.6   |      |        |2    |
 |proffer             |0.2.2   |      |        |     |
 |[QDNAseq](problems.md#qdnaseq)|1.42.0  |      |1       |     |
 |[qtl2pleio](problems.md#qtl2pleio)|1.4.3   |      |        |2    |
+|rbiom               |2.0.13  |      |        |     |
 |[ResIN](problems.md#resin)|2.0.0   |      |        |1    |
 |rivnet              |0.5.0   |      |        |     |
 |Rtapas              |1.2     |      |        |     |
 |SCDB                |0.4.1   |      |        |     |
 |[scruff](problems.md#scruff)|1.24.0  |      |        |5    |
+|seqimpute           |2.2.0   |      |        |     |
 |[SimDesign](problems.md#simdesign)|2.18    |      |        |1    |
 |simIDM              |0.1.0   |      |        |     |
+|spanishoddata       |0.1.0   |      |        |     |
 |specr               |1.0.0   |      |        |     |
-|[streetscape](problems.md#streetscape)|1.0.3   |      |1       |1    |
-|[targets](problems.md#targets)|1.9.1   |      |        |1    |
+|[streetscape](problems.md#streetscape)|1.0.5   |      |1       |1    |
+|[targets](problems.md#targets)|1.10.0  |      |        |1    |
 |TaxaNorm            |2.4     |      |        |     |
 |[TDApplied](problems.md#tdapplied)|3.0.4   |      |        |1    |
 |tern.mmrm           |0.3.2   |      |        |     |

diff --git a/revdep/cran.md b/revdep/cran.md
@@ -1,6 +1,6 @@
 ## revdepcheck results
 
-We checked 60 reverse dependencies (54 from CRAN + 6 from Bioconductor), comparing R CMD check results across CRAN and dev versions of this package.
+We checked 63 reverse dependencies (57 from CRAN + 6 from Bioconductor), comparing R CMD check results across CRAN and dev versions of this package.
 
  * We saw 0 new problems
  * We failed to check 0 packages
Original file line number	Diff line number	Diff line change
Expand Up		@@ -128,3 +128,8 @@ This will install the package from source. Because of this and because this pac

		<!-- pkgdown-drop-below -->


		## Contributing

		To contribute to this package, please see [CONTRIBUTING.md](CONTRIBUTING.md).