From edbaf358ec0a58ef0a9e8cd223934f5613737fe4 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Fri, 18 Oct 2024 15:44:39 -0500 Subject: [PATCH 1/2] WIP: Autogenerate Telegraf docs from influxdata/telegraf repo. This is a rough first pass at autogenerating Telegraf docs from READMEs in the influxdata/telegraf repo. I expect we'll need to tweak separation of concerns between the two repos and maybe there's a better Hugo-ish approach for templating and leveraging data. --- .gitignore | 4 +- build-telegraf/README.md | 1 + build-telegraf/plugins.sh | 112 +++++++++++++++++++++++ layouts/shortcodes/telegraf/plugins.html | 1 + 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 build-telegraf/README.md create mode 100644 build-telegraf/plugins.sh diff --git a/.gitignore b/.gitignore index 254ec132e7..35f2d43a93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store *~ +external public .*.swp node_modules @@ -14,4 +15,5 @@ node_modules .idea **/config.toml package-lock.json -tmp \ No newline at end of file +tmp +content/telegraf/v1/input-plugins diff --git a/build-telegraf/README.md b/build-telegraf/README.md new file mode 100644 index 0000000000..c962e4dd0a --- /dev/null +++ b/build-telegraf/README.md @@ -0,0 +1 @@ +Build Telegraf documentation from https://github.com/influxdata/telegraf diff --git a/build-telegraf/plugins.sh b/build-telegraf/plugins.sh new file mode 100644 index 0000000000..c04ef11b2b --- /dev/null +++ b/build-telegraf/plugins.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Determine the root project directory +ROOT_DIR=$(git rev-parse --show-toplevel) + +# Clone an external repository +git clone https://github.com/influxdata/telegraf.git "$ROOT_DIR/external/telegraf" + +# Update the external repository +cd "$ROOT_DIR/external/telegraf" +git pull origin master + +# Function to create a Hugo page +create_page() { + local plugin_dir=$1 + local dest_dir=$2 + local dest_file="$dest_dir/_index.md" + local parent=$3 + + # Check if the plugin directory exists + if [ -d "$plugin_dir" ]; then + mkdir -p "$dest_dir" + # Use a heredoc to write the frontmatter to the destination file + cat < "$dest_file" +--- +description: "Telegraf plugin for collecting metrics from $plugin_dir" +menu: + telegraf_v1_ref: + parent: $parent + name: $(basename "$plugin_dir") +tags: [$(basename "$plugin_dir")] +--- + +EOF + else + echo "Plugin directory not found: $plugin_dir" + fi +} + +# Function to copy README.md content to _index.md +copy_readme() { + local plugin_dir=$1 + local src_readme="$plugin_dir/README.md" + local dest_dir=$2 + local dest_file="$dest_dir/_index.md" + + # If src_readme doesn't exist, then return an error + if [ ! -f "$src_readme" ]; then + echo "README.md not found in $plugin_dir" + return 1 + fi + # If $dest_file doesn't exist, then return an error + if [ ! -f "$dest_file" ]; then + echo "$dest_file not found" + return 1 + fi + + # Copy the README.md content to _index.md + cat $src_readme >> $dest_file + echo "Copied $src_readme to $dest_file" +} + +# Semaphore function to limit concurrent processes +semaphore() { + local max_concurrent=$1 + while [ "$(jobs -r | wc -l)" -ge "$max_concurrent" ]; do + sleep 0.1 + done +} + +# Limit the number of concurrent processes +MAX_CONCURRENT=10 + +build() { + # For each plugin in external/telegraf/plugins/inputs, + # copy the README.md file to content. + local input_plugins_dir="$ROOT_DIR/content/telegraf/v1/input-plugins" + cat < "$input_plugins_dir/_index.md" +--- +title: "Telegraf Input Plugins" +description: "Telegraf input plugins collect metrics from the system, services, and third-party APIs." +menu: + telegraf_v1_ref: + name: Input Plugins + identifier: input-plugins + weight: 10 +tags: [input-plugins] +--- + +EOF + + for plugin_dir in "$ROOT_DIR/external/telegraf/plugins/inputs"/*; do + if [ -d "$plugin_dir" ]; then + # If the directory name is "all", then skip it + if [ "$(basename "$plugin_dir")" == "all" ]; then + continue + fi + local plugin_name=$(basename "$plugin_dir") + local dest_dir=$input_plugins_dir/$plugin_name + semaphore "$MAX_CONCURRENT" + ( + create_page "$plugin_dir" "$dest_dir" "Input Plugins" + copy_readme "$plugin_dir" "$dest_dir" + ) & + fi + done + + # Wait for all background jobs to finish + wait +} + +build diff --git a/layouts/shortcodes/telegraf/plugins.html b/layouts/shortcodes/telegraf/plugins.html index 753f69f3e5..4bfa5180d4 100644 --- a/layouts/shortcodes/telegraf/plugins.html +++ b/layouts/shortcodes/telegraf/plugins.html @@ -23,5 +23,6 @@

{{ .name }}

View +
{{ .Site.GetPage "/telegraf/v1/input-plugins/apache/_index.md" }}
{{ end }} From d5ce256b20b637daa86217558f7e18f981f7527d Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Mon, 21 Oct 2024 12:11:53 -0500 Subject: [PATCH 2/2] WIP(telegraf): autogenerate output plugin docs, refactor. --- .gitignore | 2 + build-telegraf/plugins.sh | 162 +++++++++++++++++++++++++++----------- 2 files changed, 117 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 35f2d43a93..fb50125413 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ node_modules package-lock.json tmp content/telegraf/v1/input-plugins +content/telegraf/v1/output-plugins + diff --git a/build-telegraf/plugins.sh b/build-telegraf/plugins.sh index c04ef11b2b..f988fc3b12 100644 --- a/build-telegraf/plugins.sh +++ b/build-telegraf/plugins.sh @@ -3,38 +3,23 @@ # Determine the root project directory ROOT_DIR=$(git rev-parse --show-toplevel) -# Clone an external repository -git clone https://github.com/influxdata/telegraf.git "$ROOT_DIR/external/telegraf" +# If the Telegraf repo isn't already cloned in ./external, clone it +if [ ! -d "$ROOT_DIR/external/telegraf" ]; then + git clone https://github.com/influxdata/telegraf.git "$ROOT_DIR/external/telegraf" +fi # Update the external repository cd "$ROOT_DIR/external/telegraf" git pull origin master # Function to create a Hugo page create_page() { - local plugin_dir=$1 - local dest_dir=$2 + local dest_dir=$1 local dest_file="$dest_dir/_index.md" - local parent=$3 - - # Check if the plugin directory exists - if [ -d "$plugin_dir" ]; then - mkdir -p "$dest_dir" - # Use a heredoc to write the frontmatter to the destination file - cat < "$dest_file" ---- -description: "Telegraf plugin for collecting metrics from $plugin_dir" -menu: - telegraf_v1_ref: - parent: $parent - name: $(basename "$plugin_dir") -tags: [$(basename "$plugin_dir")] ---- + local content=$2 -EOF - else - echo "Plugin directory not found: $plugin_dir" - fi + mkdir -p "$dest_dir" + echo "$content" > "$dest_file" } # Function to copy README.md content to _index.md @@ -56,7 +41,7 @@ copy_readme() { fi # Copy the README.md content to _index.md - cat $src_readme >> $dest_file + cat "$src_readme" >> "$dest_file" echo "Copied $src_readme to $dest_file" } @@ -69,39 +54,72 @@ semaphore() { } # Limit the number of concurrent processes -MAX_CONCURRENT=10 +MAX_CONCURRENT=5 -build() { - # For each plugin in external/telegraf/plugins/inputs, - # copy the README.md file to content. - local input_plugins_dir="$ROOT_DIR/content/telegraf/v1/input-plugins" - cat < "$input_plugins_dir/_index.md" +# Function to generate frontmatter for input plugins +input_plugin_frontmatter() { + local plugin_name=$1 + + cat < "$dest_dir/_index.md" + + for plugin_dir in "$src_dir"/*; do + # If the directory name is "all", then skip it + if [ "$(basename "$plugin_dir")" == "all" ]; then + continue + fi + local plugin_name=$(basename "$plugin_dir") + local plugin_dest_dir=$dest_dir/$plugin_name if [ -d "$plugin_dir" ]; then - # If the directory name is "all", then skip it - if [ "$(basename "$plugin_dir")" == "all" ]; then - continue - fi - local plugin_name=$(basename "$plugin_dir") - local dest_dir=$input_plugins_dir/$plugin_name semaphore "$MAX_CONCURRENT" ( - create_page "$plugin_dir" "$dest_dir" "Input Plugins" - copy_readme "$plugin_dir" "$dest_dir" - ) & + local frontmatter=$($frontmatter_func "$plugin_name") + create_page "$plugin_dest_dir" "$frontmatter" + copy_readme "$plugin_dir" "$plugin_dest_dir" + ) & fi done @@ -109,4 +127,54 @@ EOF wait } -build +# Configuration for input plugins +SRC_DIR_INPUT="$ROOT_DIR/external/telegraf/plugins/inputs" +DEST_DIR_INPUT="$ROOT_DIR/content/telegraf/v1/input-plugins" +FRONTMATTER_FUNC_INPUT="input_plugin_frontmatter" +MAIN_FRONTMATTER_INPUT=$(cat <}} + +EOF +) + +# Run the build function for input plugins +build "$SRC_DIR_INPUT" "$DEST_DIR_INPUT" "$FRONTMATTER_FUNC_INPUT" "$MAIN_FRONTMATTER_INPUT" + +# Configuration for output plugins +SRC_DIR_OUTPUT="$ROOT_DIR/external/telegraf/plugins/outputs" +DEST_DIR_OUTPUT="$ROOT_DIR/content/telegraf/v1/output-plugins" +FRONTMATTER_FUNC_OUTPUT="output_plugin_frontmatter" +MAIN_FRONTMATTER_OUTPUT=$(cat <}} + +EOF +) + +# Run the build function for output plugins +build "$SRC_DIR_OUTPUT" "$DEST_DIR_OUTPUT" "$FRONTMATTER_FUNC_OUTPUT" "$MAIN_FRONTMATTER_OUTPUT" \ No newline at end of file