Skip to content

Commit

Permalink
json. add_error_key: false not honored
Browse files Browse the repository at this point in the history 
fixes elastic#11298
  • Loading branch information
@olimpias
olimpias committed Jun 30, 2019
1 parent 66caea4 commit 5bcf085
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 16 deletions.
7 changes: 7 additions & 0 deletions libbeat/beat/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,10 @@ func metadataKey(key string) (string, bool) {
}
return "", false
}

// SetErrorWithOption sets jsonErr value in the event fields according to addErrKey value.
func (e *Event) SetErrorWithOption(jsonErr common.MapStr, addErrKey bool) {
if addErrKey {
e.Fields["error"] = jsonErr
}
}
14 changes: 7 additions & 7 deletions libbeat/common/jsontransform/jsonhelper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import (
"github.com/elastic/beats/libbeat/logp"
)

// WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option
func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys bool) {
// WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option and the addErrKey
func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys bool, addErrKey bool) {
if !overwriteKeys {
for k, v := range keys {
if _, exists := event.Fields[k]; !exists && k != "@timestamp" && k != "@metadata" {
Expand All @@ -43,15 +43,15 @@ func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys
vstr, ok := v.(string)
if !ok {
logp.Err("JSON: Won't overwrite @timestamp because value is not string")
event.Fields["error"] = createJSONError("@timestamp not overwritten (not string)")
event.SetErrorWithOption(createJSONError("@timestamp not overwritten (not string)"), addErrKey)
continue
}

// @timestamp must be of format RFC3339
ts, err := time.Parse(time.RFC3339, vstr)
if err != nil {
logp.Err("JSON: Won't overwrite @timestamp because of parsing error: %v", err)
event.Fields["error"] = createJSONError(fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr))
event.SetErrorWithOption(createJSONError(fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr)), addErrKey)
continue
}
event.Timestamp = ts
Expand All @@ -67,19 +67,19 @@ func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, overwriteKeys
event.Meta.Update(common.MapStr(m))

default:
event.Fields["error"] = createJSONError("failed to update @metadata")
event.SetErrorWithOption(createJSONError("failed to update @metadata"), addErrKey)
}

case "type":
vstr, ok := v.(string)
if !ok {
logp.Err("JSON: Won't overwrite type because value is not string")
event.Fields["error"] = createJSONError("type not overwritten (not string)")
event.SetErrorWithOption(createJSONError("type not overwritten (not string)"), addErrKey)
continue
}
if len(vstr) == 0 || vstr[0] == '_' {
logp.Err("JSON: Won't overwrite type because value is empty or starts with an underscore")
event.Fields["error"] = createJSONError(fmt.Sprintf("type not overwritten (invalid value [%s])", vstr))
event.SetErrorWithOption(createJSONError(fmt.Sprintf("type not overwritten (invalid value [%s])", vstr)), addErrKey)
continue
}
event.Fields[k] = vstr
Expand Down
8 changes: 5 additions & 3 deletions libbeat/processors/actions/decode_json_fields.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type decodeJSONFields struct {
fields []string
maxDepth int
overwriteKeys bool
addErrorKey bool
processArray bool
target *string
}
Expand All @@ -45,6 +46,7 @@ type config struct {
Fields []string `config:"fields"`
MaxDepth int `config:"max_depth" validate:"min=1"`
OverwriteKeys bool `config:"overwrite_keys"`
AddErrorKey bool `config:"add_error_key"`
ProcessArray bool `config:"process_array"`
Target *string `config:"target"`
}
Expand All @@ -63,7 +65,7 @@ func init() {
processors.RegisterPlugin("decode_json_fields",
checks.ConfigChecked(NewDecodeJSONFields,
checks.RequireFields("fields"),
checks.AllowedFields("fields", "max_depth", "overwrite_keys", "process_array", "target", "when")))
checks.AllowedFields("fields", "max_depth", "overwrite_keys", "add_error_key", "process_array", "target", "when")))
}

// NewDecodeJSONFields construct a new decode_json_fields processor.
Expand All @@ -76,7 +78,7 @@ func NewDecodeJSONFields(c *common.Config) (processors.Processor, error) {
return nil, fmt.Errorf("fail to unpack the decode_json_fields configuration: %s", err)
}

f := &decodeJSONFields{fields: config.Fields, maxDepth: config.MaxDepth, overwriteKeys: config.OverwriteKeys, processArray: config.ProcessArray, target: config.Target}
f := &decodeJSONFields{fields: config.Fields, maxDepth: config.MaxDepth, overwriteKeys: config.OverwriteKeys, addErrorKey: config.AddErrorKey, processArray: config.ProcessArray, target: config.Target}
return f, nil
}

Expand Down Expand Up @@ -115,7 +117,7 @@ func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) {
} else {
switch t := output.(type) {
case map[string]interface{}:
jsontransform.WriteJSONKeys(event, t, f.overwriteKeys)
jsontransform.WriteJSONKeys(event, t, f.overwriteKeys, f.addErrorKey)
default:
errs = append(errs, "failed to add target to root")
}
Expand Down
2 changes: 1 addition & 1 deletion libbeat/reader/readjson/json.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func MergeJSONFields(data common.MapStr, jsonFields common.MapStr, text *string,
Timestamp: ts,
Fields: data,
}
jsontransform.WriteJSONKeys(event, jsonFields, config.OverwriteKeys)
jsontransform.WriteJSONKeys(event, jsonFields, config.OverwriteKeys, config.AddErrorKey)

return event.Timestamp
}
Expand Down
29 changes: 24 additions & 5 deletions libbeat/reader/readjson/json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,13 @@ func TestDecodeJSON(t *testing.T) {
ExpectedText: ``,
ExpectedMap: common.MapStr{"message": "test", "value": int64(1)},
},
{
// If AddErrorKey set to false, error event should not be set.
Text: `{"message": "test", "value": "`,
Config: Config{MessageKey: "value", AddErrorKey: false},
ExpectedText: `{"message": "test", "value": "`,
ExpectedMap: nil,
},
}

for _, test := range tests {
Expand Down Expand Up @@ -265,7 +272,7 @@ func TestAddJSONFields(t *testing.T) {
Name: "fail to parse @timestamp",
Data: common.MapStr{"@timestamp": common.Time(now), "type": "test_type", "json": common.MapStr{"type": "test", "@timestamp": "2016-04-05T18:47:18.44XX4Z"}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true},
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: true},
ExpectedItems: common.MapStr{
"type": "test",
"error": common.MapStr{"type": "json", "message": "@timestamp not overwritten (parse error on 2016-04-05T18:47:18.44XX4Z)"},
Expand All @@ -278,7 +285,7 @@ func TestAddJSONFields(t *testing.T) {
Name: "wrong @timestamp format",
Data: common.MapStr{"@timestamp": common.Time(now), "type": "test_type", "json": common.MapStr{"type": "test", "@timestamp": 42}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true},
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: true},
ExpectedItems: common.MapStr{
"type": "test",
"error": common.MapStr{"type": "json", "message": "@timestamp not overwritten (not string)"},
Expand All @@ -290,7 +297,7 @@ func TestAddJSONFields(t *testing.T) {
Name: "ignore non-string type field",
Data: common.MapStr{"type": "test_type", "json": common.MapStr{"type": 42}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true},
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: true},
ExpectedItems: common.MapStr{
"type": "test_type",
"error": common.MapStr{"type": "json", "message": "type not overwritten (not string)"},
Expand All @@ -302,7 +309,7 @@ func TestAddJSONFields(t *testing.T) {
Name: "ignore empty type field",
Data: common.MapStr{"type": "test_type", "json": common.MapStr{"type": ""}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true},
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: true},
ExpectedItems: common.MapStr{
"type": "test_type",
"error": common.MapStr{"type": "json", "message": "type not overwritten (invalid value [])"},
Expand All @@ -314,13 +321,25 @@ func TestAddJSONFields(t *testing.T) {
Name: "ignore type names starting with underscore",
Data: common.MapStr{"@timestamp": common.Time(now), "type": "test_type", "json": common.MapStr{"type": "_type"}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true},
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: true},
ExpectedItems: common.MapStr{
"type": "test_type",
"error": common.MapStr{"type": "json", "message": "type not overwritten (invalid value [_type])"},
},
ExpectedTimestamp: time.Time{},
},
{
// if AddErrorKey is false, err should not be set.
Name: "ignore type names starting with underscore",
Data: common.MapStr{"@timestamp": common.Time(now), "type": "test_type", "json": common.MapStr{"type": "_type"}},
Text: &text,
JSONConfig: Config{KeysUnderRoot: true, OverwriteKeys: true, AddErrorKey: false},
ExpectedItems: common.MapStr{
"type": "test_type",
"error": nil,
},
ExpectedTimestamp: time.Time{},
},
}

for _, test := range tests {
Expand Down

0 comments on commit 5bcf085

Please sign in to comment.