Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Datadog profiler #314

Merged
merged 12 commits into from
Apr 5, 2022
Merged

Add Datadog profiler #314

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9b80f30
Add Datadog as a supported profiler
daxmc99 Mar 25, 2022
bb666dd
Print an error if GOMAXPROCs fails
daxmc99 Mar 25, 2022
7da02b0
sp
daxmc99 Mar 25, 2022
180619b
go mod tidy
daxmc99 Mar 25, 2022
775808d
Revert "Print an error if GOMAXPROCs fails"
daxmc99 Mar 25, 2022
775f45f
review comments
daxmc99 Mar 25, 2022
877e8aa
review comments pt2
daxmc99 Mar 25, 2022
32ea12a
Review comments v3
daxmc99 Mar 25, 2022
f8d27c3
review comments v4
daxmc99 Mar 25, 2022
8ff1af6
Merge branch 'master' into dax/add_dd_profiler
daxmc99 Mar 29, 2022
c92a228
Update internal/profiler/profiler.go
daxmc99 Mar 29, 2022
5fbe4e0
Move profiler back to where it was
daxmc99 Apr 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 3 additions & 21 deletions cmd/zoekt-sourcegraph-indexserver/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"syscall"
"time"

"cloud.google.com/go/profiler"
"github.com/hashicorp/go-retryablehttp"
"github.com/keegancsmith/tmpfriend"
"github.com/peterbourgon/ff/v3/ffcli"
Expand All @@ -39,6 +38,7 @@ import (
"github.com/google/zoekt"
"github.com/google/zoekt/build"
"github.com/google/zoekt/debugserver"
"github.com/google/zoekt/internal/profiler"
)

var (
Expand Down Expand Up @@ -667,24 +667,6 @@ func printShardStats(fn string) error {
return zoekt.PrintNgramStats(iFile)
}

func initializeGoogleCloudProfiler() {
// Google cloud profiler is opt-in since we only want to run it on
// Sourcegraph.com.
if os.Getenv("GOOGLE_CLOUD_PROFILER_ENABLED") == "" {
return
}

err := profiler.Start(profiler.Config{
Service: "zoekt-sourcegraph-indexserver",
ServiceVersion: zoekt.Version,
MutexProfiling: true,
AllocForceGC: true,
})
if err != nil {
log.Printf("could not initialize google cloud profiler: %s", err.Error())
}
}

func srcLogLevelIsDebug() bool {
lvl := os.Getenv("SRC_LOG_LEVEL")
return strings.EqualFold(lvl, "dbug") || strings.EqualFold(lvl, "debug")
Expand Down Expand Up @@ -784,7 +766,6 @@ func startServer(conf rootConfig) error {
return err
}

initializeGoogleCloudProfiler()
setCompoundShardCounter(s.IndexDir)

if conf.listen != "" {
Expand Down Expand Up @@ -820,9 +801,10 @@ func newServer(conf rootConfig) (*Server, error) {
_, _ = maxprocs.Set()

// Set the sampling rate of Go's block profiler: https://github.com/DataDog/go-profiler-notes/blob/main/guide/README.md#block-profiler.
// The block profiler is disabled by default.
// The block profiler is disabled by default and should be enabled with care in production
runtime.SetBlockProfileRate(conf.blockProfileRate)

profiler.Init("zoekt-sourcegraph-indexserver", zoekt.Version, conf.blockProfileRate)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you just make it so that we call runtime.SetBlockProfileRate inside of profiler.Init? That way webserver also gets to benefit from it.

cc @ggilmore do we actually use this anymore? Should we just get rid of it?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you talking about the block profiler itself? We don't run it continuously, but we have the machinery to enable it by setting an environment variable (this would we be useful while we're debugging something).

// Automatically prepend our own path at the front, to minimize
// required configuration.
if l, err := os.Readlink("/proc/self/exe"); err == nil {
Expand Down
23 changes: 3 additions & 20 deletions cmd/zoekt-webserver/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ import (
"syscall"
"time"

"cloud.google.com/go/profiler"
"github.com/google/zoekt"
"github.com/google/zoekt/build"
"github.com/google/zoekt/debugserver"
"github.com/google/zoekt/internal/profiler"
"github.com/google/zoekt/query"
"github.com/google/zoekt/shards"
"github.com/google/zoekt/stream"
Expand Down Expand Up @@ -135,6 +135,7 @@ func main() {
templateDir := flag.String("template_dir", "", "set directory from which to load custom .html.tpl template files")
dumpTemplates := flag.Bool("dump_templates", false, "dump templates into --template_dir and exit.")
version := flag.Bool("version", false, "Print version number")

flag.Parse()

if *version {
Expand All @@ -150,7 +151,7 @@ func main() {
}

initializeJaeger()
initializeGoogleCloudProfiler()
profiler.Init("zoekt-webserver", zoekt.Version, -1)

if *logDir != "" {
if fi, err := os.Lstat(*logDir); err != nil || !fi.IsDir() {
Expand Down Expand Up @@ -528,24 +529,6 @@ func (l *jaegerLogger) Infof(msg string, args ...interface{}) {
log.Printf(msg, args...)
}

func initializeGoogleCloudProfiler() {
// Google cloud profiler is opt-in since we only want to run it on
// Sourcegraph.com.
if os.Getenv("GOOGLE_CLOUD_PROFILER_ENABLED") == "" {
return
}

err := profiler.Start(profiler.Config{
Service: "zoekt-webserver",
ServiceVersion: zoekt.Version,
MutexProfiling: true,
AllocForceGC: true,
})
if err != nil {
log.Printf("could not initialize google cloud profiler: %s", err.Error())
}
}

var (
metricWatchdogErrors = promauto.NewGauge(prometheus.GaugeOpts{
Name: "zoekt_webserver_watchdog_errors",
Expand Down
12 changes: 4 additions & 8 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ module github.com/google/zoekt

require (
cloud.google.com/go v0.82.0
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210707164159-52430bf6b52c // indirect
github.com/RoaringBitmap/roaring v0.9.4
github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57
Expand All @@ -14,12 +13,11 @@ require (
github.com/go-enry/go-enry/v2 v2.8.0
github.com/go-git/go-git/v5 v5.4.2
github.com/gobwas/glob v0.2.3
github.com/google/go-cmp v0.5.5
github.com/google/go-cmp v0.5.7
github.com/google/go-github/v27 v27.0.6
github.com/google/slothfs v0.0.0-20190417171004-6b42407d9230
github.com/grafana/regexp v0.0.0-20220202152701-6a046c4caf32
github.com/hashicorp/go-hclog v0.12.1 // indirect
github.com/hashicorp/go-retryablehttp v0.6.4
github.com/hashicorp/go-retryablehttp v0.6.6
github.com/keegancsmith/rpc v1.1.0
github.com/keegancsmith/tmpfriend v0.0.0-20180423180255-86e88902a513
github.com/kevinburke/ssh_config v1.1.0 // indirect
Expand All @@ -35,13 +33,11 @@ require (
github.com/uber/jaeger-client-go v2.25.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible
github.com/xanzy/go-gitlab v0.28.0
go.uber.org/atomic v1.9.0
go.uber.org/automaxprocs v1.3.0
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985
golang.org/x/net v0.0.0-20211020060615-d418f374d309
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
gopkg.in/DataDog/dd-trace-go.v1 v1.37.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
humungus.tedunangst.com/r/gerc v0.1.2
)
Expand Down
Loading