Skip to content
This repository has been archived by the owner on Sep 4, 2024. It is now read-only.

Commit

Permalink
Return the jobs logs with the API responses (#65)
Browse files Browse the repository at this point in the history 
* move the Log to the BuildInfo
* remove redundant Job.Output and Job.Log, add Job.BuildInfo
* populate the BuildInfo with the Log on new job finish, and on job listing
* move common code to read a job's BuildInfo to job functions
* update HTML template to use the updated structs
* update CLI to output the job logs
  • Loading branch information
@dtheodor
dtheodor authored May 3, 2018
1 parent ba66451 commit c56d263
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 94 deletions.
11 changes: 9 additions & 2 deletions cmd/mistry/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,9 @@ EXAMPLES:
}

if verbose {
fmt.Printf("Result after unmarshalling: %#v\n", bi)
fmt.Printf(
"\nResult:\nStarted at: %s ExitCode: %v Params: %s Cached: %v Coalesced: %v\n\nLogs:\n%s\n",
bi.StartedAt, bi.ExitCode, bi.Params, bi.Cached, bi.Coalesced, bi.Log)
}

if jsonResult {
Expand All @@ -254,11 +256,17 @@ EXAMPLES:
return fmt.Errorf("Build failed with exit code %d", bi.ExitCode)
}

if verbose {
fmt.Printf("Copying artifacts to %s", target)
}
out, err := ts.Copy(transportUser, host, project, bi.Path+"/*", target, clearTarget)
fmt.Println(out)
if err != nil {
return err
}
if verbose {
fmt.Printf("Artifacts copied to %s successfully\n", target)
}

return nil
},
Expand Down Expand Up @@ -289,7 +297,6 @@ func sendRequest(url string, reqBody []byte, verbose bool) ([]byte, error) {

if verbose {
fmt.Printf("Server response: %#v\n", resp)
fmt.Printf("Body: %s\n", respBody)
}

if resp.StatusCode != http.StatusCreated {
Expand Down
33 changes: 28 additions & 5 deletions cmd/mistryd/end_to_end_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package main

import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
Expand Down Expand Up @@ -171,25 +172,48 @@ func TestImageBuildFailure(t *testing.T) {
}
}

func TestImageBuildLogsNotJson(t *testing.T) {
func TestLogs(t *testing.T) {
// trigger a job
cmdout, cmderr, err := cliBuildJob("--json-result", "--project", "simple", "--", "--testing=logs")
if err != nil {
t.Fatalf("mistry-cli stdout: %s, stderr: %s, err: %#v", cmdout, cmderr, err)
}

br, err := parseClientJSON(cmdout)
if err != nil {
t.Fatal(err)
}

// find the log file
j, err := NewJob("simple", types.Params{"testing": "logs"}, "", testcfg)
if err != nil {
t.Fatalf("failed to create job: err: %#v", err)
}
f, err := os.Open(filepath.Join(j.ReadyBuildPath, BuildLogFname))
log, err := ReadJobLogs(j.ReadyBuildPath)
if err != nil {
t.Fatalf("failed to read job log: err: %#v", err)
}

assertEq(br.Log, string(log), t)
}

func TestLogsNotJson(t *testing.T) {
// trigger a job and grab the logs
cmdout, cmderr, err := cliBuildJob("--json-result", "--project", "simple", "--", "--testing=logsnotjson")
if err != nil {
t.Fatalf("mistry-cli stdout: %s, stderr: %s, err: %#v", cmdout, cmderr, err)
}
j, err := NewJob("simple", types.Params{"testing": "logsnotjson"}, "", testcfg)
if err != nil {
t.Fatalf("failed to create job: err: %#v", err)
}
logs, err := ReadJobLogs(j.ReadyBuildPath)
if err != nil {
t.Fatalf("failed to read job log: err: %#v", err)
}
defer f.Close()

// if any line in the log file can be parsed into a JSON, fail
scanner := bufio.NewScanner(f)
scanner := bufio.NewScanner(bytes.NewReader(logs))
for scanner.Scan() {
line := scanner.Bytes()
var v interface{}
Expand All @@ -199,7 +223,6 @@ func TestImageBuildLogsNotJson(t *testing.T) {
t.Fatalf("found JSON line in the logs: %s", line)
}
}

}

func TestExitCode(t *testing.T) {
Expand Down
83 changes: 58 additions & 25 deletions cmd/mistryd/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
Expand Down Expand Up @@ -56,10 +56,8 @@ type Job struct {

StartedAt time.Time

// webview-related
Output string
Log template.HTML
State string
BuildInfo *types.BuildInfo
State string
}

// NewJobFromRequest returns a new Job from the JobRequest
Expand Down Expand Up @@ -119,13 +117,15 @@ func NewJob(project string, params types.Params, group string, cfg *Config) (*Jo
j.PendingBuildPath = filepath.Join(j.RootBuildPath, "pending", j.ID)
j.ReadyBuildPath = filepath.Join(j.RootBuildPath, "ready", j.ID)
j.ReadyDataPath = filepath.Join(j.ReadyBuildPath, DataDir)
j.BuildLogPath = filepath.Join(j.PendingBuildPath, BuildLogFname)
j.BuildLogPath = BuildLogPath(j.PendingBuildPath)
j.BuildInfoFilePath = filepath.Join(j.PendingBuildPath, BuildInfoFname)

j.Image = ImgCntPrefix + j.ID
j.Container = ImgCntPrefix + j.ID

j.StartedAt = time.Now()
j.BuildInfo = new(types.BuildInfo)
j.State = "pending"

return j, nil
}
Expand Down Expand Up @@ -229,18 +229,16 @@ func (j *Job) String() string {
// MarshalJSON serializes the Job to JSON
func (j *Job) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
ID string `json:"id"`
Project string `json:"project"`
StartedAt string `json:"startedAt"`
Output string `json:"output"`
Log template.HTML `json:"log"`
State string `json:"state"`
ID string `json:"id"`
Project string `json:"project"`
StartedAt string `json:"startedAt"`
BuildInfo types.BuildInfo `json:"buildInfo"`
State string `json:"state"`
}{
ID: j.ID,
Project: j.Project,
StartedAt: j.StartedAt.Format(DateFmt),
Output: j.Output,
Log: template.HTML(j.Log),
BuildInfo: *j.BuildInfo,
State: j.State,
})
}
Expand All @@ -249,12 +247,11 @@ func (j *Job) MarshalJSON() ([]byte, error) {
// with them
func (j *Job) UnmarshalJSON(data []byte) error {
jData := &struct {
ID string `json:"id"`
Project string `json:"project"`
StartedAt string `json:"startedAt"`
Output string `json:"output"`
Log string `json:"log"`
State string `json:"state"`
ID string `json:"id"`
Project string `json:"project"`
StartedAt string `json:"startedAt"`
BuildInfo types.BuildInfo `json:"buildInfo"`
State string `json:"state"`
}{}
err := json.Unmarshal(data, &jData)
if err != nil {
Expand All @@ -266,8 +263,7 @@ func (j *Job) UnmarshalJSON(data []byte) error {
if err != nil {
return err
}
j.Output = jData.Output
j.Log = template.HTML(jData.Log)
j.BuildInfo = &jData.BuildInfo
j.State = jData.State

return nil
Expand Down Expand Up @@ -355,7 +351,44 @@ func (j *Job) BootstrapBuildDir(fs filesystem.FileSystem, log *log.Logger) (bool
return shouldCleanup, nil
}

// RemoveBuildDir removes the existing build directory
func (j *Job) RemoveBuildDir(fs filesystem.FileSystem, log *log.Logger) error {
return fs.Remove(j.ReadyBuildPath)
// BuildLogPath returns the path of the job logs found at jobPath
func BuildLogPath(jobPath string) string {
return filepath.Join(jobPath, BuildLogFname)
}

// ReadJobLogs returns the job logs found at jobPath
func ReadJobLogs(jobPath string) ([]byte, error) {
buildLogPath := BuildLogPath(jobPath)

log, err := ioutil.ReadFile(buildLogPath)
if err != nil {
return nil, err
}
return log, nil
}

// ReadJobBuildInfo returns the BuildInfo found at jobPath
func ReadJobBuildInfo(path string, logs bool) (*types.BuildInfo, error) {
buildInfoPath := filepath.Join(path, BuildInfoFname)
buildInfo := &types.BuildInfo{}

buildInfoBytes, err := ioutil.ReadFile(buildInfoPath)
if err != nil {
return nil, err
}

err = json.Unmarshal(buildInfoBytes, &buildInfo)
if err != nil {
return nil, err
}

if logs {
log, err := ReadJobLogs(path)
if err != nil {
return nil, err
}
buildInfo.Log = string(log)
}

return buildInfo, nil
}
24 changes: 12 additions & 12 deletions cmd/mistryd/public/templates/show.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,29 @@ <h4> Details </h4>
<div class="card-divider">
<h4> Logs </h4>
</div>
<p id="js-job-log">{{.Log}}</p>
<p id="js-job-log"></p>
</div>
</div>
</div>
</div>

<script type="text/javascript">
const jobLog = document.getElementById('js-job-log')
const logs = {{.BuildInfo.Log}};
const jobInfo = document.getElementById('js-job-info');
const state = {{.State}}
const output = {{.Output}}
const jobJSON = JSON.parse(output);

jobInfo.innerHTML += "Project: ".big() + {{.Project}} + "<br>";
jobInfo.innerHTML += "State: ".big() + {{.State}} + "<br>";
jobInfo.innerHTML += "Started At: ".big() + new Date(jobJSON["StartedAt"]) + "<br>";
jobInfo.innerHTML += "Path: ".big() + jobJSON["Path"] + "<br>";
jobInfo.innerHTML += "Params: ".big() + JSON.stringify(jobJSON.Params) + "<br>";
jobInfo.innerHTML += "Cached: ".big() + jobJSON["Cached"] + "<br>";
jobInfo.innerHTML += "Coalesced: ".big() + jobJSON["Coalesced"] + "<br>";
jobInfo.innerHTML += "Error: ".big() + jobJSON["Err"] + "<br>";
jobInfo.innerHTML += "ExitCode: ".big() + jobJSON["ExitCode"] + "<br>";
jobInfo.innerHTML += "Transport method: ".big() + jobJSON["TransportMethod"] + "<br>";
jobInfo.innerHTML += "Started At: ".big() + new Date({{.BuildInfo.StartedAt}}) + "<br>";
jobInfo.innerHTML += "Path: ".big() + {{.BuildInfo.Path}} + "<br>";
jobInfo.innerHTML += "Params: ".big() + JSON.stringify({{.BuildInfo.Params}}) + "<br>";
jobInfo.innerHTML += "Cached: ".big() + {{.BuildInfo.Cached}} + "<br>";
jobInfo.innerHTML += "Coalesced: ".big() + {{.BuildInfo.Coalesced}} + "<br>";
jobInfo.innerHTML += "ExitCode: ".big() + {{.BuildInfo.ExitCode}} + "<br>";
jobInfo.innerHTML += "Transport method: ".big() + {{.BuildInfo.TransportMethod}} + "<br>";

jobLog.innerHTML += logs.split('\n').join('<br>')

if (state == "pending") {
setInterval(checkState, 3000);
Expand All @@ -76,7 +77,6 @@ <h4> Logs </h4>
}

const source = new EventSource('/log/{{.Project}}/{{.ID}}');
const jobLog = document.getElementById('js-job-log')
source.onmessage = function(e) {
jobLog.innerHTML += e.data + "</br>"
};
Expand Down
37 changes: 10 additions & 27 deletions cmd/mistryd/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,6 @@ func (s *Server) HandleIndex(w http.ResponseWriter, r *http.Request) {
// HandleShowJob receives requests for a job and produces the appropriate output
// based on the content type of the request.
func (s *Server) HandleShowJob(w http.ResponseWriter, r *http.Request) {
var log []byte
var buildInfo []byte

if r.Method != "GET" {
http.Error(w, "Expected GET, got "+r.Method, http.StatusMethodNotAllowed)
return
Expand All @@ -212,29 +209,19 @@ func (s *Server) HandleShowJob(w http.ResponseWriter, r *http.Request) {
return
}
jPath := filepath.Join(s.cfg.BuildPath, project, state, id)
buildLogPath := filepath.Join(jPath, BuildLogFname)
buildInfoPath := filepath.Join(jPath, BuildInfoFname)

buildInfo, err = ioutil.ReadFile(buildInfoPath)
if err != nil {
s.Log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
return
}

log, err = ioutil.ReadFile(buildLogPath)
buildInfo, err := ReadJobBuildInfo(jPath, true)
if err != nil {
s.Log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
return
}

j := Job{
Output: string(buildInfo),
Log: template.HTML(strings.Replace(string(log), "\n", "<br />", -1)),
ID: id,
Project: project,
State: state,
BuildInfo: buildInfo,
ID: id,
Project: project,
State: state,
}

if r.Header.Get("Content-type") == "application/json" {
Expand Down Expand Up @@ -262,7 +249,7 @@ func (s *Server) HandleShowJob(w http.ResponseWriter, r *http.Request) {
s.br.CloseClientC[id] = make(chan struct{})
}

tl, err := tailer.New(buildLogPath)
tl, err := tailer.New(BuildLogPath(jPath))
if err != nil {
s.Log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
Expand Down Expand Up @@ -454,21 +441,17 @@ func (s *Server) getJobs() ([]Job, error) {
}

getJob := func(path, jobID, project, state string) (Job, error) {
bi := types.BuildInfo{}
biBlob, err := ioutil.ReadFile(filepath.Join(path, jobID, BuildInfoFname))
if err != nil {
return Job{}, fmt.Errorf("cannot read build_info file; %s", err)
}
err = json.Unmarshal(biBlob, &bi)
bi, err := ReadJobBuildInfo(filepath.Join(path, jobID), false)
if err != nil {
return Job{}, fmt.Errorf("cannot read build_info file of job %s; %s", jobID, err)
return Job{}, err
}

return Job{
ID: jobID,
Project: project,
StartedAt: bi.StartedAt,
State: state}, nil
State: state,
BuildInfo: bi}, nil
}

for _, j := range pendingJobs {
Expand Down
4 changes: 2 additions & 2 deletions cmd/mistryd/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,8 @@ func TestHandleShowJob(t *testing.T) {
if err != nil {
t.Fatal(err)
}
jobID := (job[0].ID)
project := (job[0].Project)
jobID := job[0].ID
project := job[0].Project

// Request the show page of the job selected from the index page.
showPath := path.Join("/job", project, jobID)
Expand Down
Loading

0 comments on commit c56d263

Please sign in to comment.