Skip to content

Commit

Permalink
cmd/pkgsite: add command
Browse files Browse the repository at this point in the history
Functionality for running the pkgsite frontend locally is moved from
cmd/frontend to cmd/pkgsite, since cmd/frontend is currently overloaded
with flag options and running locally does not need all the dependencies
for running cmd/frontend.

Additional functionality will be added to cmd/pkgsite in future CLs.

For golang/go#40371

Change-Id: I4230aa9539c94e01a68eda33cc6492ae377debff
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/290134
Reviewed-by: Jamal Carvalho <[email protected]>
Trust: Julie Qiu <[email protected]>
  • Loading branch information
julieqiu committed Feb 5, 2021
1 parent e10f26f commit b317742
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 73 deletions.
92 changes: 27 additions & 65 deletions cmd/frontend/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"flag"
"net/http"
"os"
"path/filepath"
"time"

"cloud.google.com/go/profiler"
Expand All @@ -21,7 +20,6 @@ import (
"golang.org/x/pkgsite/internal/config"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/frontend"
"golang.org/x/pkgsite/internal/localdatasource"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/middleware"
"golang.org/x/pkgsite/internal/postgres"
Expand All @@ -42,8 +40,6 @@ var (
"for direct proxy mode and frontend fetches")
directProxy = flag.Bool("direct_proxy", false, "if set to true, uses the module proxy referred to by this URL "+
"as a direct backend, bypassing the database")
localPaths = flag.String("local", "", "run locally, accepts a GOPATH-like collection of local paths for modules to load to memory")
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src, used only with -local")
bypassLicenseCheck = flag.Bool("bypass_license_check", false, "display all information, even for non-redistributable paths")
)

Expand Down Expand Up @@ -75,41 +71,36 @@ func main() {
expg := cmdconfig.ExperimentGetter(ctx, cfg)
log.Infof(ctx, "cmd/frontend: initialized cmdconfig.ExperimentGetter")

if *localPaths != "" {
lds := localdatasource.New()
dsg = func(context.Context) internal.DataSource { return lds }
proxyClient, err := proxy.New(*proxyURL)
if err != nil {
log.Fatal(ctx, err)
}

if *directProxy {
var pds *proxydatasource.DataSource
if *bypassLicenseCheck {
pds = proxydatasource.NewBypassingLicenseCheck(proxyClient)
} else {
pds = proxydatasource.New(proxyClient)
}
dsg = func(context.Context) internal.DataSource { return pds }
} else {
proxyClient, err := proxy.New(*proxyURL)
db, err := cmdconfig.OpenDB(ctx, cfg, *bypassLicenseCheck)
if err != nil {
log.Fatal(ctx, err)
log.Fatalf(ctx, "%v", err)
}

if *directProxy {
var pds *proxydatasource.DataSource
if *bypassLicenseCheck {
pds = proxydatasource.NewBypassingLicenseCheck(proxyClient)
} else {
pds = proxydatasource.New(proxyClient)
}
dsg = func(context.Context) internal.DataSource { return pds }
} else {
db, err := cmdconfig.OpenDB(ctx, cfg, *bypassLicenseCheck)
if err != nil {
log.Fatalf(ctx, "%v", err)
}
defer db.Close()
dsg = func(context.Context) internal.DataSource { return db }
sourceClient := source.NewClient(config.SourceTimeout)
// The closure passed to queue.New is only used for testing and local
// execution, not in production. So it's okay that it doesn't use a
// per-request connection.
fetchQueue, err = queue.New(ctx, cfg, queueName, *workers, expg,
func(ctx context.Context, modulePath, version string) (int, error) {
return frontend.FetchAndUpdateState(ctx, modulePath, version, proxyClient, sourceClient, db)
})
if err != nil {
log.Fatalf(ctx, "queue.New: %v", err)
}
defer db.Close()
dsg = func(context.Context) internal.DataSource { return db }
sourceClient := source.NewClient(config.SourceTimeout)
// The closure passed to queue.New is only used for testing and local
// execution, not in production. So it's okay that it doesn't use a
// per-request connection.
fetchQueue, err = queue.New(ctx, cfg, queueName, *workers, expg,
func(ctx context.Context, modulePath, version string) (int, error) {
return frontend.FetchAndUpdateState(ctx, modulePath, version, proxyClient, sourceClient, db)
})
if err != nil {
log.Fatalf(ctx, "queue.New: %v", err)
}
}

Expand All @@ -135,13 +126,6 @@ func main() {
log.Fatalf(ctx, "frontend.NewServer: %v", err)
}

if *localPaths != "" {
lds, ok := dsg(ctx).(*localdatasource.DataSource)
if ok {
load(ctx, lds, *localPaths)
}
}

router := dcensus.NewRouter(frontend.TagRoute)
var cacheClient *redis.Client
if cfg.RedisCacheHost != "" {
Expand Down Expand Up @@ -204,25 +188,3 @@ func main() {
log.Infof(ctx, "Listening on addr %s", addr)
log.Fatal(ctx, http.ListenAndServe(addr, mw(router)))
}

// load loads local modules from pathList.
func load(ctx context.Context, ds *localdatasource.DataSource, pathList string) {
paths := filepath.SplitList(pathList)
loaded := len(paths)
for _, path := range paths {
var err error
if *gopathMode {
err = ds.LoadFromGOPATH(ctx, path)
} else {
err = ds.Load(ctx, path)
}
if err != nil {
log.Error(ctx, err)
loaded--
}
}

if loaded == 0 {
log.Fatalf(ctx, "failed to load module(s) at %s", pathList)
}
}
99 changes: 99 additions & 0 deletions cmd/pkgsite/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This is a work in progress.
//
// Pkgsite extracts and generates documentation for Go programs.
// It runs as a web server and presents the documentation as a
// web page.
// Usage:
//
// pkgsite [flag]
//
// The flags are:
//
// -local=path1,path2
// Accepts a GOPATH-like collection of local paths for modules to load to memory
// -gopath_mode=false
// Assume that local modules' paths are relative to GOPATH/src
package main

import (
"context"
"flag"
"net/http"
"path/filepath"
"time"

"github.com/google/safehtml/template"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/frontend"
"golang.org/x/pkgsite/internal/localdatasource"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/middleware"
)

var (
_ = flag.String("static", "content/static", "path to folder containing static files served")
localPaths = flag.String("local", "", "run locally, accepts a GOPATH-like collection of local paths for modules to load to memory")
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src, used only with -local")
)

func main() {
flag.Parse()
ctx := context.Background()
var dsg func(context.Context) internal.DataSource
if *localPaths == "" {
log.Fatalf(ctx, "-local is not set")
}

lds := localdatasource.New()
dsg = func(context.Context) internal.DataSource { return lds }
server, err := frontend.NewServer(frontend.ServerConfig{
DataSourceGetter: dsg,
StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
})
if err != nil {
log.Fatalf(ctx, "frontend.NewServer: %v", err)
}
lds, ok := dsg(ctx).(*localdatasource.DataSource)
if ok {
load(ctx, lds, *localPaths)
}

router := dcensus.NewRouter(frontend.TagRoute)
server.Install(router.Handle, nil, nil)

mw := middleware.Chain(
middleware.RedirectedFrom(),
middleware.LatestVersions(server.GetLatestInfo), // must come before caching for version badge to work
middleware.Timeout(54*time.Second),
)
addr := "localhost:6060"
log.Infof(ctx, "Listening on addr %s", addr)
log.Fatal(ctx, http.ListenAndServe(addr, mw(router)))
}

// load loads local modules from pathList.
func load(ctx context.Context, ds *localdatasource.DataSource, pathList string) {
paths := filepath.SplitList(pathList)
loaded := len(paths)
for _, path := range paths {
var err error
if *gopathMode {
err = ds.LoadFromGOPATH(ctx, path)
} else {
err = ds.Load(ctx, path)
}
if err != nil {
log.Error(ctx, err)
loaded--
}
}

if loaded == 0 {
log.Fatalf(ctx, "failed to load module(s) at %s", pathList)
}
}
20 changes: 12 additions & 8 deletions doc/frontend.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@ See [experiment.md](experiment.md) for instructions how to enable experiments.

You can run the frontend locally like so:

go run ./cmd/frontend [-dev] [-direct_proxy] [-local .]
go run ./cmd/frontend [-dev] [-direct_proxy]

- The `-dev` flag reloads templates on each page load.

The frontend can use one of three datasources:

- Postgres database
- Proxy service
- Local filesystem

The `Datasource` interface implementation is available at internal/datasource.go.

Expand All @@ -39,16 +38,21 @@ your local database with packages of your choice.

You can then run the frontend with: `go run ./cmd/frontend`

You can also use `-local` flag to run the frontend with an in-memory datasource
populated with modules loaded from your local filesystem. This allows you to run
the frontend without setting up a database and to view documentation of local
modules without requiring a proxy. `-local` accepts a GOPATH-like string containing
paths of modules to load into memory.

If you add, change or remove any inline scripts in templates, run
`devtools/cmd/csphash` to update the hashes. Running `all.bash`
will do that as well.

### Local mode

You can also use run the frontend locally with an in-memory datasource
populated with modules loaded from your local filesystem.

go run ./cmd/pkgsite [-local .]

This allows you to run the frontend without setting up a database and to view
documentation of local modules without requiring a proxy. `-local` accepts a
GOPATH-like string containing paths of modules to load into memory.

### Testing

In addition to tests inside internal/frontend and internal/testing/integration,
Expand Down

0 comments on commit b317742

Please sign in to comment.