Merge pull request intelsdi-x#1 from librato/plugin-directory

Add support for configuring plugins from directory

control/config.go

@@ -23,9 +23,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
@@ -34,6 +37,7 @@ import (
 	"github.com/intelsdi-x/snap/core"
 	"github.com/intelsdi-x/snap/core/cdata"
 	"github.com/intelsdi-x/snap/core/ctypes"
+	"github.com/intelsdi-x/snap/pkg/cfgfile"
 )
 
 // default configuration values
@@ -160,6 +164,29 @@ const (
 				"additionalProperties": false
 			}
 	`
+	PLUGIN_CONFIG_DIR_CONSTRAINTS = `{
+		"$schema": "http://json-schema.org/draft-04/schema#",
+		"title": "snapteld plugin config directory schema",
+		"type": ["object", "null"],
+		"properties" : {
+			"collector": {
+				"type": ["object", "null"],
+				"properties" : {},
+				"additionalProperties": true
+			},
+			"processor": {
+				"type": ["object", "null"],
+				"properties" : {},
+				"additionalProperties": true
+			},
+			"publisher": {
+				"type": ["object", "null"],
+				"properties" : {},
+				"additionalProperties": true
+			}
+        },
+		"additionalProperties": false
+}`
 )
 
 // get the default snapteld configuration
@@ -283,7 +310,13 @@ func (p *pluginConfig) UnmarshalJSON(data []byte) error {
 		p.All = cdn
 	}
 
-	//process the hierarchy of plugins
+	// load config plugin configuration directory into t, if specified
+	if err := unmarshalPluginDir(t); err != nil {
+		controlLogger.WithFields(log.Fields{"err": err}).Warning("Error loading plugin directory")
+		// continue loading configuration
+	}
+
+	//process the hierarchy of plugins and include dir
 	for _, typ := range []string{"collector", "processor", "publisher"} {
 		if err := unmarshalPluginConfig(typ, p, t); err != nil {
 			return err
@@ -428,6 +461,101 @@ func (p *pluginConfig) getPluginConfigDataNode(pluginType core.PluginType, name
 	return p.pluginCache[key]
 }
 
+// unmarshalPluginDir loads from a specified "include" path, if configured, and loads the
+// JSON or YAML files in those directories into t.
+func unmarshalPluginDir(t map[string]interface{}) error {
+	var paths []string
+	// split include path string into multiple paths
+	v, ok := t["include"]
+	if !ok {
+		return nil
+	}
+	val, ok := v.(string)
+	if !ok || val == "" {
+		return nil
+	}
+	paths = filepath.SplitList(val)
+	// load paths, based on pluginControl.Start autodiscovery
+	for _, pa := range paths {
+		fullPath, err := filepath.Abs(pa)
+		if err != nil {
+			log.WithFields(log.Fields{"path": pa, "error": err}).Error("Abs")
+			continue
+		}
+		log.WithFields(log.Fields{"path": fullPath}).Info("Autoloading plugins")
+		files, err := ioutil.ReadDir(fullPath)
+		if err != nil {
+			log.WithFields(log.Fields{"path": fullPath, "error": err}).Error("Error reading directory")
+			continue
+		}
+		for _, file := range files {
+			fname := file.Name()
+			absfile := filepath.Join(fullPath, fname)
+
+			statCheck := file
+			if file.Mode()&os.ModeSymlink != 0 {
+				realPath, err := filepath.EvalSymlinks(filepath.Join(fullPath, fname))
+				if err != nil {
+					log.WithFields(log.Fields{"file": absfile, "error": err}).
+						Error("Cannot follow symlink")
+					continue
+				}
+				statCheck, err = os.Stat(realPath)
+				if err != nil {
+					log.WithFields(log.Fields{"file": absfile, "error": err, "target-path": realPath}).
+						Error("Target of symlink inacessible")
+					continue
+				}
+			}
+
+			if statCheck.IsDir() {
+				log.WithFields(log.Fields{"file": absfile}).Warning("Ignoring subdirectory")
+				continue
+			}
+			if strings.HasSuffix(fname, ".json") || strings.HasSuffix(fname, ".yaml") || strings.HasSuffix(fname, ".yml") {
+				cfg := make(map[string]interface{})
+				// read config and and validate schema
+				log.WithFields(log.Fields{"file": absfile}).Debug("Loading plugin")
+				if errs := cfgfile.Read(absfile, &cfg, PLUGIN_CONFIG_DIR_CONSTRAINTS); errs != nil {
+					for _, err := range errs {
+						log.WithFields(err.Fields()).WithFields(log.Fields{"file": absfile}).Error("Config read error")
+					}
+					log.WithFields(log.Fields{"file": absfile}).Warning("Error parsing plugin config, skipping")
+					continue // skip this file
+				}
+
+				// include plugins from configuration file into provided map t
+				for pluginType, v := range cfg {
+					switch pluginType {
+					case "collector", "processor", "publisher":
+						plugins, ok := v.(map[string]interface{})
+						if !ok {
+							continue
+						}
+						for plugin, pluginCfg := range plugins {
+							log.WithFields(log.Fields{"file": absfile, "type": pluginType, "plugin": plugin}).
+								Info("Loading plugin from file")
+							if _, exists := t[pluginType]; !exists {
+								t[pluginType] = make(map[string]interface{})
+							}
+							pluginMap, ok := t[pluginType].(map[string]interface{})
+							if !ok {
+								continue
+							}
+							if _, exists := pluginMap[plugin]; exists {
+								log.WithFields(log.Fields{"file": absfile, "type": pluginType, "plugin": plugin}).
+									Warning("Overwriting existing plugin configuration")
+							}
+							pluginMap[plugin] = pluginCfg
+						}
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func unmarshalPluginConfig(typ string, p *pluginConfig, t map[string]interface{}) error {
 	if v, ok := t[typ]; ok {
 		switch plugins := v.(type) {

control/config_test.go

@@ -287,6 +287,10 @@ func TestControlConfigYaml(t *testing.T) {
 		Convey("Plugins.Publisher should not be nil", func() {
 			So(cfg.Plugins.Publisher, ShouldNotBeNil)
 		})
+		Convey("Config for nginx plugin from plugin-conf.d should be set", func() {
+			So(cfg.Plugins.Collector.Plugins["nginx"], ShouldNotBeNil)
+			So(cfg.Plugins.Collector.Plugins["nginx"].ConfigDataNode.Table()["nginx_server_url"], ShouldResemble, ctypes.ConfigValueStr{Value: "http://localhost/status"})
+		})
 	})
 
 }

examples/configs/plugin-conf.d/nginx.yaml

@@ -0,0 +1,4 @@
+collector:
+  nginx:
+    all:
+      nginx_server_url: "http://localhost/status"

examples/configs/snap-config-sample.yaml

@@ -93,6 +93,7 @@ control:
   # plugins section contains plugin config settings that will be applied for
   # plugins across tasks.
   plugins:
+    include: /tmp/small-setup-plugin-conf.d:/tmp/medium-setup-plugin-conf.d:/tmp/plugin-conf.d
     all:
       password: p@ssw0rd
     collector:

scripts/test.sh

@@ -49,6 +49,12 @@ if [[ "${SNAP_TEST_TYPE}" == "build" ]]; then
   exit 0
 fi
 
+# Put example plugin include file in place for TestControlConfigYaml
+if [[ "${SNAP_TEST_TYPE}" == "legacy" ]]; then
+    mkdir -p /tmp/plugin-conf.d
+    cp "${__proj_dir}/examples/configs/plugin-conf.d/nginx.yaml" /tmp/plugin-conf.d/nginx.yaml
+fi
+
 _go_path
 # If the following tools don't exist, get them
 _go_get github.com/smartystreets/goconvey