diff --git a/Cargo.lock b/Cargo.lock index d4d1ad43..7da61001 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2594,13 +2594,17 @@ dependencies = [ "css-syntax-types", "csv", "flate2", + "indexmap", "rari-types", "rari-utils", "reqwest", + "semver", "serde", "serde_json", "tar", "thiserror 1.0.69", + "tracing", + "url", ] [[package]] @@ -2739,11 +2743,13 @@ dependencies = [ "indexmap", "normalize-path", "schemars", + "semver", "serde", "serde_json", "serde_variant", "strum", "thiserror 1.0.69", + "tracing", ] [[package]] @@ -3167,6 +3173,9 @@ name = "semver" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +dependencies = [ + "serde", +] [[package]] name = "serde" diff --git a/Cargo.toml b/Cargo.toml index ff28091b..db310877 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,7 @@ pretty_yaml = "0.5" yaml_parser = "0.2" const_format = "0.2" dialoguer = "0.11" +semver = { version = "1", features = ["serde"] } [dependencies] rari-doc.workspace = true diff --git a/crates/rari-cli/main.rs b/crates/rari-cli/main.rs index e6f838e7..b4c7e386 100644 --- a/crates/rari-cli/main.rs +++ b/crates/rari-cli/main.rs @@ -229,15 +229,6 @@ fn main() -> Result<(), Error> { info!("Using env_file: {}", env_file.display()) } let cli = Cli::parse(); - if !cli.skip_updates { - rari_deps::webref_css::update_webref_css(rari_types::globals::data_dir())?; - rari_deps::web_features::update_web_features(rari_types::globals::data_dir())?; - rari_deps::bcd::update_bcd(rari_types::globals::data_dir())?; - rari_deps::mdn_data::update_mdn_data(rari_types::globals::data_dir())?; - rari_deps::web_ext_examples::update_web_ext_examples(rari_types::globals::data_dir())?; - rari_deps::popularities::update_popularities(rari_types::globals::data_dir())?; - } - let fmt_filter = filter::Targets::new().with_target("rari_doc", cli.verbose.tracing_level_filter()); @@ -249,6 +240,7 @@ fn main() -> Result<(), Error> { let cli_filter = filter::Targets::new() .with_target("rari", cli_level) .with_target("rari_tools", cli_level) + .with_target("rari_deps", cli_level) .with_target("rari_doc", LevelFilter::OFF); let memory_filter = filter::Targets::new() @@ -273,6 +265,15 @@ fn main() -> Result<(), Error> { .with(memory_layer.clone().with_filter(memory_filter)) .init(); + if !cli.skip_updates { + rari_deps::webref_css::update_webref_css(rari_types::globals::data_dir())?; + rari_deps::web_features::update_web_features(rari_types::globals::data_dir())?; + rari_deps::bcd::update_bcd(rari_types::globals::data_dir())?; + rari_deps::mdn_data::update_mdn_data(rari_types::globals::data_dir())?; + rari_deps::web_ext_examples::update_web_ext_examples(rari_types::globals::data_dir())?; + rari_deps::popularities::update_popularities(rari_types::globals::data_dir())?; + } + match cli.command { Commands::Build(args) => { let mut settings = Settings::new()?; diff --git a/crates/rari-deps/Cargo.toml b/crates/rari-deps/Cargo.toml index 30f45823..2f774c3f 100644 --- a/crates/rari-deps/Cargo.toml +++ b/crates/rari-deps/Cargo.toml @@ -14,6 +14,10 @@ serde_json.workspace = true chrono.workspace = true thiserror.workspace = true reqwest.workspace = true +url.workspace = true +indexmap.workspace = true +tracing.workspace = true +semver.workspace = true css-syntax-types = { path = "../css-syntax-types" } tar = "0.4" diff --git a/crates/rari-deps/src/bcd.rs b/crates/rari-deps/src/bcd.rs index 6451af2b..c679f099 100644 --- a/crates/rari-deps/src/bcd.rs +++ b/crates/rari-deps/src/bcd.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; +use rari_types::globals::deps; use rari_utils::io::read_to_string; use serde_json::Value; @@ -9,10 +10,10 @@ use crate::error::DepsError; use crate::npm::get_package; pub fn update_bcd(base_path: &Path) -> Result<(), DepsError> { - if let Some(path) = get_package("@mdn/browser-compat-data", None, base_path)? { + if let Some(path) = get_package("@mdn/browser-compat-data", &deps().bcd, base_path)? { extract_spec_urls(&path)?; } - get_package("web-specs", None, base_path)?; + get_package("web-specs", &deps().web_specs, base_path)?; Ok(()) } diff --git a/crates/rari-deps/src/client.rs b/crates/rari-deps/src/client.rs new file mode 100644 index 00000000..25c65d5f --- /dev/null +++ b/crates/rari-deps/src/client.rs @@ -0,0 +1,9 @@ +use reqwest::blocking::Response; + +pub fn get(url: impl AsRef) -> reqwest::Result { + reqwest::blocking::ClientBuilder::new() + .user_agent("mdn/rari") + .build()? + .get(url.as_ref()) + .send() +} diff --git a/crates/rari-deps/src/current.rs b/crates/rari-deps/src/current.rs index 32a42a71..02c917cd 100644 --- a/crates/rari-deps/src/current.rs +++ b/crates/rari-deps/src/current.rs @@ -1,8 +1,9 @@ use chrono::{DateTime, Utc}; +use semver::Version; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Default, Debug)] pub struct Current { pub latest_last_check: Option>, - pub version: String, + pub current_version: Option, } diff --git a/crates/rari-deps/src/error.rs b/crates/rari-deps/src/error.rs index 07a47d46..f6f4857a 100644 --- a/crates/rari-deps/src/error.rs +++ b/crates/rari-deps/src/error.rs @@ -18,4 +18,6 @@ pub enum DepsError { WebRefMissingTarballError, #[error("Invalid github version")] InvalidGitHubVersion, + #[error("Invalid github version")] + VersionNotFound, } diff --git a/crates/rari-deps/src/external_json.rs b/crates/rari-deps/src/external_json.rs index 18b8e0e5..77132b2d 100644 --- a/crates/rari-deps/src/external_json.rs +++ b/crates/rari-deps/src/external_json.rs @@ -6,6 +6,7 @@ use chrono::{DateTime, Duration, Utc}; use rari_utils::io::read_to_string; use serde::{Deserialize, Serialize}; +use crate::client::get; use crate::error::DepsError; #[derive(Deserialize, Serialize, Default, Debug)] @@ -26,7 +27,7 @@ pub fn get_json(name: &str, url: &str, out_path: &Path) -> Result, +} + +type Releases = Vec; + +fn get_version(repo: &str, version_req: &VersionReq) -> Result<(Version, String), DepsError> { + let releases = get(format!( + "https://api.github.com/repos/{repo}/releases?per_page=10" + ))?; + let releases: Releases = serde_json::from_value(releases.json()?)?; + if let Some(version) = releases.iter().find_map(|k| { + let version = k + .tag_name + .as_ref() + .and_then(|v| Version::parse(v.trim_start_matches('v')).ok()); + if version + .as_ref() + .map(|v| version_req.matches(v)) + .unwrap_or_default() + { + version.map(|version| (version, k.tag_name.clone().unwrap())) + } else { + None + } + }) { + Ok(version) + } else { + Err(DepsError::VersionNotFound) + } +} /// Download a github release artifact for a given version (defaults to latest). pub fn get_artifact( - base_url: &str, + repo: &str, artifact: &str, name: &str, - version: Option<&str>, + version_req: &Option, out_path: &Path, ) -> Result, DepsError> { - let version = version.unwrap_or("latest"); + let star = VersionReq::default(); + + let version_req = version_req.as_ref().unwrap_or(&star); let package_path = out_path.join(name); let last_check_path = package_path.join("last_check.json"); let now = Utc::now(); @@ -25,44 +61,27 @@ pub fn get_artifact( .ok() .and_then(|current| serde_json::from_str::(¤t).ok()) .unwrap_or_default(); - if version != current.version - || version == "latest" - && current.latest_last_check.unwrap_or_default() < now - Duration::days(1) + if !current + .current_version + .as_ref() + .map(|v| version_req.matches(v)) + .unwrap_or_default() + || current.latest_last_check.unwrap_or_default() < now - Duration::days(1) { - let prev_url = format!( - "{base_url}/releases/download/{}/{artifact}", - current.version - ); - let url = if version == "latest" { - let client = reqwest::blocking::ClientBuilder::default() - .redirect(Policy::none()) - .build()?; - let res = client - .get(format!("{base_url}/releases/latest/download/{artifact}")) - .send()?; - res.headers() - .get("location") - .ok_or(DepsError::InvalidGitHubVersion)? - .to_str()? - .to_string() - } else { - format!("{base_url}/releases/download/{version}/{artifact}") - }; + let (version, tag_name) = get_version(repo, version_req)?; + let url = format!("https://github.com/{repo}/releases/download/{tag_name}/{artifact}"); let artifact_path = package_path.join(artifact); - let download_update = if artifact_path.exists() { - prev_url != url - } else { - true - }; + let download_update = current.current_version.as_ref() != Some(&version); if download_update { + tracing::info!("Updating {repo} ({artifact}) to {version}"); if package_path.exists() { fs::remove_dir_all(&package_path)?; } fs::create_dir_all(&package_path)?; let mut buf = vec![]; - let _ = reqwest::blocking::get(url)?.read_to_end(&mut buf)?; + let _ = get(url)?.read_to_end(&mut buf)?; let file = File::create(artifact_path).unwrap(); let mut buffed = BufWriter::new(file); @@ -70,15 +89,13 @@ pub fn get_artifact( buffed.write_all(&buf)?; } - if version == "latest" { - fs::write( - package_path.join("last_check.json"), - serde_json::to_string_pretty(&Current { - version: version.to_string(), - latest_last_check: Some(now), - })?, - )?; - } + fs::write( + package_path.join("last_check.json"), + serde_json::to_string_pretty(&Current { + current_version: Some(version), + latest_last_check: Some(now), + })?, + )?; if download_update { return Ok(Some(package_path)); } diff --git a/crates/rari-deps/src/lib.rs b/crates/rari-deps/src/lib.rs index e6fa0202..7e246177 100644 --- a/crates/rari-deps/src/lib.rs +++ b/crates/rari-deps/src/lib.rs @@ -1,4 +1,5 @@ pub mod bcd; +pub mod client; pub mod current; pub mod error; pub mod external_json; diff --git a/crates/rari-deps/src/mdn_data.rs b/crates/rari-deps/src/mdn_data.rs index ace060bf..97dff199 100644 --- a/crates/rari-deps/src/mdn_data.rs +++ b/crates/rari-deps/src/mdn_data.rs @@ -1,9 +1,11 @@ use std::path::Path; +use rari_types::globals::deps; + use crate::error::DepsError; use crate::npm::get_package; pub fn update_mdn_data(base_path: &Path) -> Result<(), DepsError> { - get_package("mdn-data", None, base_path)?; + get_package("mdn-data", &deps().mdn_data, base_path)?; Ok(()) } diff --git a/crates/rari-deps/src/npm.rs b/crates/rari-deps/src/npm.rs index d35d5ee9..786faa7a 100644 --- a/crates/rari-deps/src/npm.rs +++ b/crates/rari-deps/src/npm.rs @@ -4,20 +4,63 @@ use std::path::{Path, PathBuf}; use chrono::{Duration, Utc}; use flate2::read::GzDecoder; +use indexmap::IndexMap; use rari_utils::io::read_to_string; -use serde_json::Value; +use semver::{Version, VersionReq}; +use serde::Deserialize; use tar::Archive; +use url::Url; +use crate::client::get; use crate::current::Current; use crate::error::DepsError; +#[derive(Deserialize, Debug, Clone)] +struct Dist { + tarball: Url, +} + +#[derive(Deserialize, Debug, Clone)] +struct VersionEntry { + version: Version, + dist: Dist, +} + +#[derive(Deserialize, Debug, Clone)] +struct Package { + versions: IndexMap, +} + +fn get_version(package_name: &str, version_req: &VersionReq) -> Result { + let package: Package = get(format!("https://registry.npmjs.org/{package_name}"))?.json()?; + if let Some((_, entry)) = package + .versions + .iter() + .rfind(|(k, _)| version_req.matches(k)) + { + if let Some((latest, _)) = package.versions.last() { + if latest > &entry.version { + tracing::warn!( + "Update for {package_name} available {} -> {}", + entry.version, + latest + ); + } + } + Ok(entry.clone()) + } else { + Err(DepsError::VersionNotFound) + } +} + /// Download and unpack an npm package for a given version (defaults to latest). pub fn get_package( package: &str, - version: Option<&str>, + version_req: &Option, out_path: &Path, ) -> Result, DepsError> { - let version = version.unwrap_or("latest"); + let star = VersionReq::default(); + let version_req = version_req.as_ref().unwrap_or(&star); let package_path = out_path.join(package); let last_check_path = package_path.join("last_check.json"); let now = Utc::now(); @@ -25,56 +68,56 @@ pub fn get_package( .ok() .and_then(|current| serde_json::from_str::(¤t).ok()) .unwrap_or_default(); - if version != current.version - || version == "latest" - && current.latest_last_check.unwrap_or_default() < now - Duration::days(1) - { - let body: Value = - reqwest::blocking::get(format!("https://registry.npmjs.org/{package}/{version}"))? - .json()?; - let latest_version = body["version"] - .as_str() - .ok_or(DepsError::WebRefMissingVersionError)?; - let tarball_url = body["dist"]["tarball"] - .as_str() - .ok_or(DepsError::WebRefMissingTarballError)?; - let package_json_path = package_path.join("package").join("package.json"); - let download_update = if package_json_path.exists() { - let json_str = read_to_string(package_json_path)?; - let package_json: Value = serde_json::from_str(&json_str)?; - let current_version = package_json["version"] - .as_str() - .ok_or(DepsError::WebRefMissingVersionError)?; - current_version != latest_version - } else { - true - }; + if !current + .current_version + .as_ref() + .map(|v| version_req.matches(v)) + .unwrap_or_default() + || current.latest_last_check.unwrap_or_default() < now - Duration::days(1) + { + let version_entry = get_version(package, version_req)?; + let tarball_url = version_entry.dist.tarball; + let download_update = current.current_version.as_ref() != Some(&version_entry.version); if download_update { + tracing::info!("Updating {package} to {}", version_entry.version); if package_path.exists() { fs::remove_dir_all(&package_path)?; } fs::create_dir_all(&package_path)?; let mut buf = vec![]; - let _ = reqwest::blocking::get(tarball_url)?.read_to_end(&mut buf)?; + let _ = get(tarball_url)?.read_to_end(&mut buf)?; let gz = GzDecoder::new(&buf[..]); let mut ar = Archive::new(gz); ar.unpack(&package_path)?; } - if version == "latest" { - fs::write( - package_path.join("last_check.json"), - serde_json::to_string_pretty(&Current { - version: version.to_string(), - latest_last_check: Some(now), - })?, - )?; - } + fs::write( + package_path.join("last_check.json"), + serde_json::to_string_pretty(&Current { + current_version: Some(version_entry.version), + latest_last_check: Some(now), + })?, + )?; if download_update { return Ok(Some(package_path)); } } Ok(None) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_get_version() { + let e = get_version( + "/@mdn/browser-compat-data", + &VersionReq::parse("5.6.33").unwrap(), + ) + .unwrap(); + println!("{} -> {}", e.version, e.dist.tarball) + } +} diff --git a/crates/rari-deps/src/popularities.rs b/crates/rari-deps/src/popularities.rs index e8084217..cda7e138 100644 --- a/crates/rari-deps/src/popularities.rs +++ b/crates/rari-deps/src/popularities.rs @@ -34,7 +34,6 @@ fn should_update(now: &DateTime, current: &Option>) -> bool { } pub fn update_popularities(base_path: &Path) -> Result, DepsError> { - let version = "latest"; let package_path = base_path.join("popularities"); let last_check_path = package_path.join("last_check.json"); let now = Utc::now(); @@ -77,7 +76,7 @@ pub fn update_popularities(base_path: &Path) -> Result, DepsErro fs::write( package_path.join("last_check.json"), serde_json::to_string_pretty(&Current { - version: version.to_string(), + current_version: None, latest_last_check: Some(now), })?, )?; diff --git a/crates/rari-deps/src/web_features.rs b/crates/rari-deps/src/web_features.rs index 79eb4ba4..ef63bd91 100644 --- a/crates/rari-deps/src/web_features.rs +++ b/crates/rari-deps/src/web_features.rs @@ -1,15 +1,18 @@ use std::path::Path; +use rari_types::globals::deps; + use crate::error::DepsError; use crate::github_release::get_artifact; pub fn update_web_features(base_path: &Path) -> Result<(), DepsError> { //get_package("web-features", None, base_path)?; + get_artifact( - "https://github.com/web-platform-dx/web-features", + "web-platform-dx/web-features", "data.extended.json", "baseline", - None, + &deps().web_features, base_path, )?; Ok(()) diff --git a/crates/rari-deps/src/webref_css.rs b/crates/rari-deps/src/webref_css.rs index 7d99801b..6c80824b 100644 --- a/crates/rari-deps/src/webref_css.rs +++ b/crates/rari-deps/src/webref_css.rs @@ -3,6 +3,7 @@ use std::fs; use std::path::Path; use css_syntax_types::Css; +use rari_types::globals::deps; use rari_utils::io::read_to_string; use serde_json::Value; @@ -84,7 +85,7 @@ fn list_all(folder: &Path) -> Result, DepsError> { } pub fn update_webref_css(base_path: &Path) -> Result<(), DepsError> { - if let Some(package_path) = get_package("@webref/css", None, base_path)? { + if let Some(package_path) = get_package("@webref/css", &deps().webref_css, base_path)? { let webref_css_dest_path = package_path.join("webref_css.json"); let webref_css = list_all(&package_path)?; fs::write(webref_css_dest_path, serde_json::to_string(&webref_css)?)?; diff --git a/crates/rari-types/Cargo.toml b/crates/rari-types/Cargo.toml index 3fec71c3..035a797a 100644 --- a/crates/rari-types/Cargo.toml +++ b/crates/rari-types/Cargo.toml @@ -17,6 +17,8 @@ serde_json.workspace = true indexmap.workspace = true chrono.workspace = true schemars.workspace = true +semver.workspace = true +tracing.workspace = true serde_variant = "0.1" normalize-path = "0.2" diff --git a/crates/rari-types/src/globals.rs b/crates/rari-types/src/globals.rs index f5c3acb3..c1398b49 100644 --- a/crates/rari-types/src/globals.rs +++ b/crates/rari-types/src/globals.rs @@ -7,7 +7,7 @@ use serde::Deserialize; use crate::error::EnvError; use crate::locale::Locale; -use crate::settings::Settings; +use crate::settings::{Deps, Settings}; use crate::{HistoryEntry, Popularities}; #[inline(always)] @@ -74,6 +74,11 @@ pub fn settings() -> &'static Settings { SETTINGS.get_or_init(|| Settings::new().expect("error generating settings")) } +pub static DEPS: OnceLock = OnceLock::new(); +pub fn deps() -> &'static Deps { + DEPS.get_or_init(|| Deps::new().expect("error generating deps")) +} + #[derive(Debug, Deserialize)] pub struct JsonSpecData { pub url: String, diff --git a/crates/rari-types/src/settings.rs b/crates/rari-types/src/settings.rs index fcbfd816..891a4c36 100644 --- a/crates/rari-types/src/settings.rs +++ b/crates/rari-types/src/settings.rs @@ -1,10 +1,55 @@ -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use config::{Config, ConfigError, Environment, File}; +use semver::VersionReq; use serde::{Deserialize, Serialize}; use crate::locale::Locale; +#[derive(Serialize, Deserialize, Default, Debug)] +#[serde(default)] +pub struct Deps { + #[serde(alias = "@mdn/browser-compat-data")] + pub bcd: Option, + #[serde(alias = "mdn-data")] + pub mdn_data: Option, + #[serde(alias = "web-features")] + pub web_features: Option, + #[serde(alias = "web-specs")] + pub web_specs: Option, + #[serde(alias = "@webref/css")] + pub webref_css: Option, +} + +#[derive(Serialize, Deserialize, Default, Debug)] +#[serde(default)] +pub struct DepsPackageJson { + dependencies: Deps, +} + +impl Deps { + pub fn new() -> Result { + if let Some(package_json) = std::env::var_os("deps_package_json") { + let path = Path::new(&package_json); + if let Some(deps) = fs::read_to_string(path) + .ok() + .and_then(|json_str| serde_json::from_str(&json_str).ok()) + { + return Ok(deps); + } else { + tracing::error!("unable to parse {}", path.display()); + } + } + let s = Config::builder() + .add_source(Environment::default().prefix("deps").try_parsing(true)) + .build()?; + + let deps: Self = s.try_deserialize::()?; + Ok(deps) + } +} + #[derive(Serialize, Deserialize, Default, Debug)] #[serde(default)] pub struct Settings { @@ -26,6 +71,7 @@ pub struct Settings { pub data_issues: bool, pub json_issues: bool, pub blog_unpublished: bool, + pub deps: Deps, } impl Settings {