Skip to content

Commit

Permalink
Add support for modules
Browse files Browse the repository at this point in the history
Support for modules is something that has come up many times in the
past. Most recently in FCOS:

coreos/fedora-coreos-tracker#767

This commit adds support for both composing with modules on the server
side and package layering modules (or just enabling them) on the client
side.

This doesn't support modules in `rpm-ostree compose extensions` yet,
which is relevant for RHCOS. We can look at adding that in a follow-up.
  • Loading branch information
jlebon committed Jul 26, 2021
1 parent 711b528 commit 8de71b8
Show file tree
Hide file tree
Showing 25 changed files with 838 additions and 30 deletions.
9 changes: 9 additions & 0 deletions docs/treefile.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,15 @@ It supports the following parameters:
* `packages`: Array of strings, required: List of packages to install.
* `repo`: String, required: Name of the repo from which to fetch packages.

* `modules`: Object, optional: Describes RPM modules to enable or install. Two
keys are supported:
* `enable`: Array of strings, required: Set of RPM module specs to enable
(the same formats as dnf are supported, e.g. `NAME[:STREAM]`).
One can then cherry-pick specific packages from the enabled modules via
`packages`.
* `install`: Array of strings, required: Set of RPM module specs to install
(the same formats as dnf are supported, e.g. `NAME[:STREAM][/PROFILE]`).

* `ostree-layers`: Array of strings, optional: After all packages are unpacked,
check out these OSTree refs, which must already be in the destination repository.
Any conflicts with packages will be an error.
Expand Down
10 changes: 10 additions & 0 deletions rust/src/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ fn deployment_populate_variant_origin(

// Package mappings. Note these are inserted unconditionally, even if empty.
vdict_insert_optvec(&dict, "requested-packages", tf.packages.as_ref());
vdict_insert_optvec(
&dict,
"requested-modules",
tf.modules.as_ref().map(|m| m.install.as_ref()).flatten(),
);
vdict_insert_optvec(
&dict,
"modules-enabled",
tf.modules.as_ref().map(|m| m.enable.as_ref()).flatten(),
);
vdict_insert_optmap(
&dict,
"requested-local-packages",
Expand Down
3 changes: 3 additions & 0 deletions rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,8 @@ pub mod ffi {
fn get_packages_local(&self) -> Vec<String>;
fn get_packages_override_replace_local(&self) -> Vec<String>;
fn get_packages_override_remove(&self) -> Vec<String>;
fn get_modules_enable(&self) -> Vec<String>;
fn get_modules_install(&self) -> Vec<String>;
fn get_exclude_packages(&self) -> Vec<String>;
fn get_install_langs(&self) -> Vec<String>;
fn format_install_langs_macro(&self) -> String;
Expand Down Expand Up @@ -618,6 +620,7 @@ mod lockfile;
pub(crate) use self::lockfile::*;
mod live;
pub(crate) use self::live::*;
pub mod modularity;
mod nameservice;
mod origin;
pub(crate) use self::origin::*;
Expand Down
1 change: 1 addition & 0 deletions rust/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ fn inner_main() -> Result<i32> {
Some("countme") => rpmostree_rust::countme::entrypoint(&args).map(|_| 0),
Some("cliwrap") => rpmostree_rust::cliwrap::entrypoint(&args).map(|_| 0),
Some("ex-container") => rpmostree_rust::container::entrypoint(&args).map(|_| 0),
Some("module") => rpmostree_rust::modularity::entrypoint(&args).map(|_| 0),
_ => {
// Otherwise fall through to C++ main().
Ok(rpmostree_rust::ffi::rpmostree_main(&args)?)
Expand Down
130 changes: 130 additions & 0 deletions rust/src/modularity.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//! Implementation of the client-side of "rpm-ostree module".
// SPDX-License-Identifier: Apache-2.0 OR MIT

use anyhow::{anyhow, bail, Result};
use gio::DBusProxyExt;
use ostree_ext::variant_utils;
use structopt::StructOpt;

use crate::utils::print_treepkg_diff;

#[derive(Debug, StructOpt)]
#[structopt(name = "rpm-ostree module", no_version)]
#[structopt(rename_all = "kebab-case")]
enum Opt {
/// Enable a module
Enable(InstallOpts),
/// Disable a module
Disable(InstallOpts),
/// Install a module
Install(InstallOpts),
/// Uninstall a module
Uninstall(InstallOpts),
}

#[derive(Debug, StructOpt)]
struct InstallOpts {
#[structopt(parse(from_str))]
modules: Vec<String>,
#[structopt(long)]
reboot: bool,
#[structopt(long)]
lock_finalization: bool,
#[structopt(long)]
dry_run: bool,
#[structopt(long)]
experimental: bool,
}

const OPT_KEY_ENABLE_MODULES: &str = "enable-modules";
const OPT_KEY_DISABLE_MODULES: &str = "disable-modules";
const OPT_KEY_INSTALL_MODULES: &str = "install-modules";
const OPT_KEY_UNINSTALL_MODULES: &str = "uninstall-modules";

pub fn entrypoint(args: &[&str]) -> Result<()> {
match Opt::from_iter(args.iter().skip(1)) {
Opt::Enable(ref opts) => enable(opts),
Opt::Disable(ref opts) => disable(opts),
Opt::Install(ref opts) => install(opts),
Opt::Uninstall(ref opts) => uninstall(opts),
}
}

// XXX: Should split out a lot of the below into a more generic Rust wrapper around
// UpdateDeployment() like we have on the C side.

fn get_modifiers_variant(key: &str, modules: &[String]) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert_value(key, &crate::variant_utils::new_variant_strv(modules));
Ok(r.end())
}

fn get_options_variant(opts: &InstallOpts) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert("no-pull-base", &true);
r.insert("reboot", &opts.reboot);
r.insert("lock-finalization", &opts.lock_finalization);
r.insert("dry-run", &opts.dry_run);
Ok(r.end())
}

fn enable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_ENABLE_MODULES, opts)
}

fn disable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_DISABLE_MODULES, opts)
}

fn install(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_INSTALL_MODULES, opts)
}

fn uninstall(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_UNINSTALL_MODULES, opts)
}

fn modules_impl(key: &str, opts: &InstallOpts) -> Result<()> {
if !opts.experimental {
bail!("Modularity support is experimental and subject to change. Use --experimental.");
}

if opts.modules.is_empty() {
bail!("At least one module must be specified");
}

let client = &mut crate::client::ClientConnection::new()?;
let previous_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
let modifiers = get_modifiers_variant(key, &opts.modules)?;
let options = get_options_variant(opts)?;
let params = variant_utils::new_variant_tuple(&[modifiers, options]);
let reply = &client.get_os_proxy().call_sync(
"UpdateDeployment",
Some(&params),
gio::DBusCallFlags::NONE,
-1,
gio::NONE_CANCELLABLE,
)?;
let reply_child = crate::variant_utils::variant_tuple_get(reply, 0)
.ok_or_else(|| anyhow!("Invalid reply"))?;
let txn_address = reply_child
.get_str()
.ok_or_else(|| anyhow!("Expected string transaction address"))?;
client.transaction_connect_progress_sync(txn_address)?;
if opts.dry_run {
println!("Exiting because of '--dry-run' option");
} else if !opts.reboot {
let new_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
if previous_deployment != new_deployment {
print_treepkg_diff("/");
}
}
Ok(())
}
30 changes: 30 additions & 0 deletions rust/src/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::result::Result as StdResult;
const ORIGIN: &str = "origin";
const RPMOSTREE: &str = "rpmostree";
const PACKAGES: &str = "packages";
const MODULES: &str = "modules";
const OVERRIDES: &str = "overrides";

/// The set of keys that we parse as BTreeMap and need to ignore ordering changes.
Expand All @@ -43,6 +44,14 @@ pub(crate) fn origin_to_treefile_inner(kf: &KeyFile) -> Result<Box<Treefile>> {
cfg.derive.base_refspec = Some(refspec_str);
cfg.packages = parse_stringlist(&kf, PACKAGES, "requested")?;
cfg.derive.packages_local = parse_localpkglist(&kf, PACKAGES, "requested-local")?;
let modules_enable = parse_stringlist(&kf, MODULES, "enable")?;
let modules_install = parse_stringlist(&kf, MODULES, "install")?;
if modules_enable.is_some() || modules_install.is_some() {
cfg.modules = Some(crate::treefile::ModulesConfig {
enable: modules_enable,
install: modules_install,
});
}
cfg.derive.override_remove = parse_stringlist(&kf, OVERRIDES, "remove")?;
cfg.derive.override_replace_local = parse_localpkglist(&kf, OVERRIDES, "replace-local")?;

Expand Down Expand Up @@ -143,6 +152,16 @@ fn treefile_to_origin_inner(tf: &Treefile) -> Result<glib::KeyFile> {
if let Some(pkgs) = tf.derive.override_replace_local.as_ref() {
set_sha256_nevra_pkgs(&kf, OVERRIDES, "replace-local", pkgs)
}
if let Some(ref modcfg) = tf.modules {
if let Some(modules) = modcfg.enable.as_deref() {
let modules = modules.iter().map(|s| s.as_str());
kf_set_string_list(&kf, MODULES, "enable", modules)
}
if let Some(modules) = modcfg.install.as_deref() {
let modules = modules.iter().map(|s| s.as_str());
kf_set_string_list(&kf, MODULES, "install", modules)
}
}

// Initramfs bits
if let Some(initramfs) = tf.derive.initramfs.as_ref() {
Expand Down Expand Up @@ -332,6 +351,10 @@ pub(crate) mod test {
requested=libvirt;fish;
requested-local=4ed748ba060fce4571e7ef19f3f5ed6209f67dbac8327af0d38ea70b96d2f723:foo-1.2-3.x86_64;
[modules]
enable=foo:2.0;bar:rolling;
install=baz:next/development;
[overrides]
remove=docker;
replace-local=0c7072500af2758e7dc7d7700fed82c3c5f4da7453b4d416e79f75384eee96b0:rpm-ostree-devel-2021.1-2.fc33.x86_64;648ab3ff4d4b708ea180269297de5fa3e972f4481d47b7879c6329272e474d68:rpm-ostree-2021.1-2.fc33.x86_64;8b29b78d0ade6ec3aedb8e3846f036f6f28afe64635d83cb6a034f1004607678:rpm-ostree-libs-2021.1-2.fc33.x86_64;
Expand Down Expand Up @@ -393,6 +416,13 @@ pub(crate) mod test {
tf.parsed.derive.override_commit.unwrap(),
"41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3"
);
assert_eq!(
tf.parsed.modules,
Some(crate::treefile::ModulesConfig {
enable: Some(vec!["foo:2.0".into(), "bar:rolling".into()]),
install: Some(vec!["baz:next/development".into()]),
})
);
Ok(())
}

Expand Down
81 changes: 81 additions & 0 deletions rust/src/treefile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,17 @@ fn treefile_parse_stream<R: io::Read>(
}
}

// to be consistent, we also support whitespace-separated modules
if let Some(mut modules) = treefile.modules.take() {
if let Some(enable) = modules.enable.take() {
modules.enable = Some(whitespace_split_packages(&enable)?);
}
if let Some(install) = modules.install.take() {
modules.install = Some(whitespace_split_packages(&install)?);
}
treefile.modules = Some(modules);
}

if let Some(repo_packages) = treefile.repo_packages.take() {
treefile.repo_packages = Some(
repo_packages
Expand Down Expand Up @@ -313,6 +324,18 @@ fn merge_hashset_field<T: Eq + std::hash::Hash>(
}
}

/// Merge modules fields.
fn merge_modules(dest: &mut Option<ModulesConfig>, src: &mut Option<ModulesConfig>) {
if let Some(mut srcv) = src.take() {
if let Some(mut destv) = dest.take() {
merge_vec_field(&mut destv.enable, &mut srcv.enable);
merge_vec_field(&mut destv.install, &mut srcv.install);
srcv = destv;
}
*dest = Some(srcv);
}
}

/// Given two configs, merge them.
fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
macro_rules! merge_basics {
Expand Down Expand Up @@ -384,6 +407,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
);

merge_basic_field(&mut dest.derive.base_refspec, &mut src.derive.base_refspec);
merge_modules(&mut dest.modules, &mut src.modules);
}

/// Merge the treefile externals. There are currently only two keys that
Expand Down Expand Up @@ -595,6 +619,30 @@ impl Treefile {
.collect()
}

pub(crate) fn get_modules_enable(&self) -> Vec<String> {
self.parsed
.modules
.as_ref()
.map(|m| m.enable.as_ref())
.flatten()
.cloned()
.into_iter()
.flatten()
.collect()
}

pub(crate) fn get_modules_install(&self) -> Vec<String> {
self.parsed
.modules
.as_ref()
.map(|m| m.install.as_ref())
.flatten()
.cloned()
.into_iter()
.flatten()
.collect()
}

pub(crate) fn get_packages_override_remove(&self) -> Vec<String> {
self.parsed
.derive
Expand Down Expand Up @@ -1097,6 +1145,8 @@ pub(crate) struct TreeComposeConfig {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(rename = "repo-packages")]
pub(crate) repo_packages: Option<Vec<RepoPackage>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) modules: Option<ModulesConfig>,
// Deprecated option
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) bootstrap_packages: Option<Vec<String>>,
Expand Down Expand Up @@ -1224,6 +1274,14 @@ pub(crate) struct RepoPackage {
pub(crate) packages: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
pub(crate) struct ModulesConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) enable: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) install: Option<Vec<String>>,
}

#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
pub(crate) struct LegacyTreeComposeConfigFields {
#[serde(skip_serializing)]
Expand Down Expand Up @@ -1426,6 +1484,12 @@ pub(crate) mod tests {
- repo: baserepo
packages:
- blah bloo
modules:
enable:
- foobar:2.0
install:
- nodejs:15
- swig:3.0/complete sway:rolling
"#};

// This one has "comments" (hence unknown keys)
Expand Down Expand Up @@ -1720,6 +1784,11 @@ pub(crate) mod tests {
- repo: foo2
packages:
- qwert
modules:
enable:
- dodo
install:
- bazboo
"},
)?;
let mut buf = VALID_PRELUDE.to_string();
Expand All @@ -1741,6 +1810,18 @@ pub(crate) mod tests {
}
])
);
assert_eq!(
tf.parsed.modules,
Some(ModulesConfig {
enable: Some(vec!["dodo".into(), "foobar:2.0".into()]),
install: Some(vec![
"bazboo".into(),
"nodejs:15".into(),
"swig:3.0/complete".into(),
"sway:rolling".into(),
])
},)
);
Ok(())
}

Expand Down
Loading

0 comments on commit 8de71b8

Please sign in to comment.