Gateway renders pretty 404 pages if available

In the same way that an `index.html` file is rendered, if one is present, when a
path requested of the gateway is a directory, a `404.html` file is rendered if
the requested file is not present within the specified IPFS object.

`404.html` files are looked for in the directory of the requested path and each
parent until one is found, falling back on the previous error message.

License: MIT
Signed-off-by: JP Hastings-Spital <[email protected]>

core/corehttp/gateway_handler.go

@@ -168,6 +168,8 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 		return
 	}
 
+	preventCache := false
+
 	// Resolve path to the final DAG node for the ETag
 	resolvedPath, err := i.api.ResolvePath(ctx, parsedPath)
 	switch err {
@@ -179,8 +181,17 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 		}
 		fallthrough
 	default:
-		webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
-		return
+		resolved404Path, err404 := searchUpTreeFor404(r.Header["Accept"], ctx, parsedPath, i)
+
+		if err404 != nil {
+			webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
+			return
+		}
+
+		w.WriteHeader(http.StatusNotFound)
+		preventCache = true
+		resolvedPath = resolved404Path
+		log.Debugf("using pretty 404 file for %s", escapedURLPath)
 	}
 
 	dr, err := i.api.Unixfs().Cat(ctx, resolvedPath)
@@ -191,6 +202,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 		defer dr.Close()
 	case coreiface.ErrIsDir:
 		dir = true
+		preventCache = true
 	default:
 		webError(w, "ipfs cat "+escapedURLPath, err, http.StatusNotFound)
 		return
@@ -259,7 +271,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 	// TODO: break this out when we split /ipfs /ipns routes.
 	modtime := time.Now()
 
-	if strings.HasPrefix(urlPath, ipfsPathPrefix) && !dir {
+	if strings.HasPrefix(urlPath, ipfsPathPrefix) && !preventCache {
 		w.Header().Set("Cache-Control", "public, max-age=29030400, immutable")
 
 		// set modtime to a really long time ago, since files are immutable and should stay cached
@@ -598,3 +610,42 @@ func webErrorWithCode(w http.ResponseWriter, message string, err error, code int
 func internalWebError(w http.ResponseWriter, err error) {
 	webErrorWithCode(w, "internalWebError", err, http.StatusInternalServerError)
 }
+
+func searchUpTreeFor404(acceptHeaders []string, ctx context.Context, parsedPath coreiface.Path, i *gatewayHandler) (coreiface.Path, error) {
+	filename404, err := preferred404Filename(acceptHeaders)
+	if err != nil {
+		return nil, err
+	}
+
+	pathComponents := strings.SplitAfter(gopath.Dir(parsedPath.String()), "/")
+
+	for idx := len(pathComponents); idx >= 3; idx-- {
+		pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
+		parsedPath, err := coreapi.ParsePath(pretty404)
+		if err == nil {
+			resolvedPath, err := i.api.ResolvePath(ctx, parsedPath)
+			if err == nil {
+				return resolvedPath, nil
+			}
+		}
+	}
+
+	return nil, errors.New("No pretty 404 in any parent folder")
+}
+
+func preferred404Filename(acceptHeaders []string) (string, error) {
+	// If we ever want to offer a 404 file for a different content type
+	// then this function will need to parse q weightings, but for now
+	// the presence of anything matching HTML is enough.
+	for _, acceptHeader := range acceptHeaders {
+		accepted := strings.Split(acceptHeader, ",")
+		for _, spec := range accepted {
+			contentType := strings.Split(spec, ";")[0]
+			if contentType == "*/*" || contentType == "text/*" || contentType == "text/html" {
+				return "404.html", nil
+			}
+		}
+	}
+
+	return "", errors.New("There is no 404 file for the requested content types")
+}

core/corehttp/gateway_test.go

@@ -173,6 +173,76 @@ func TestGatewayGet(t *testing.T) {
 	}
 }
 
+func TestPretty404(t *testing.T) {
+	ns := mockNamesys{}
+	ts, n := newTestServerAndNode(t, ns)
+	defer ts.Close()
+
+	// Create files for test hash
+	_, dagRoot, err := coreunix.AddWrapped(n, strings.NewReader("Custom 404"), "404.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, dagDeep404, err := coreunix.AddWrapped(n, strings.NewReader("Deep custom 404"), "404.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = dagRoot.(*dag.ProtoNode).AddNodeLink("deeper", dagDeep404)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = n.DAG.Add(dagRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k := dagRoot.Cid()
+	basePath := "/ipfs/" + k.String()
+
+	for _, test := range []struct {
+		path   string
+		accept string
+		status int
+		text   string
+	}{
+		{"/404.html", "text/html", http.StatusOK, "Custom 404"},
+		{"/nope", "text/html", http.StatusNotFound, "Custom 404"},
+		{"/nope", "text/*", http.StatusNotFound, "Custom 404"},
+		{"/nope", "*/*", http.StatusNotFound, "Custom 404"},
+		{"/nope", "application/json", http.StatusNotFound, "ipfs resolve -r /ipfs/QmY51Xk3B2JDTLyGDQuq9Z45Xkg6qFyQENEy3j5NywymmG/nope: no link named \"nope\" under QmY51Xk3B2JDTLyGDQuq9Z45Xkg6qFyQENEy3j5NywymmG\n"},
+		{"/deeper/nope", "text/html", http.StatusNotFound, "Deep custom 404"},
+		{"/nope/nope", "text/html", http.StatusNotFound, "Custom 404"},
+	} {
+		var c http.Client
+		req, err := http.NewRequest("GET", ts.URL+basePath+test.path, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Add("Accept", test.accept)
+		resp, err := c.Do(req)
+
+		if err != nil {
+			t.Fatalf("error requesting %s: %s", test.path, err)
+		}
+
+		defer resp.Body.Close()
+		if resp.StatusCode != test.status {
+			t.Fatalf("got %d, expected %d, from %s", resp.StatusCode, test.status, test.path)
+		}
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("error reading response from %s: %s", test.path, err)
+		}
+
+		if string(body) != test.text {
+			t.Fatalf("unexpected response body from %s: got %q, expected %q", test.path, body, test.text)
+		}
+	}
+}
+
 func TestIPNSHostnameRedirect(t *testing.T) {
 	ns := mockNamesys{}
 	ts, n := newTestServerAndNode(t, ns)
@@ -190,7 +260,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
+	err = dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
 	if err != nil {
 		t.Fatal(err)
 	}