From 2007eba90d7f40c73da83044dd112c75879dedf3 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 26 Nov 2019 13:46:57 -0500 Subject: [PATCH 1/4] go.mod: update go to 1.12 --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index f1c0b8f..c31db9e 100644 --- a/go.mod +++ b/go.mod @@ -4,3 +4,5 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-metrics-interface v0.0.1 ) + +go 1.12 From fb847172ff982c28a79fc7d67e81b1b2559b3360 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 26 Nov 2019 13:47:50 -0500 Subject: [PATCH 2/4] testing: add datastore testsuite --- .travis.yml | 2 +- measure_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 measure_test.go diff --git a/.travis.yml b/.travis.yml index 4cfe98c..ce8b00b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ os: language: go go: - - 1.11.x + - 1.12.x env: global: diff --git a/measure_test.go b/measure_test.go new file mode 100644 index 0000000..c927455 --- /dev/null +++ b/measure_test.go @@ -0,0 +1,12 @@ +package measure + +import ( + "testing" + + dstest "github.com/ipfs/go-datastore/test" +) + +func TestSuite(t *testing.T) { + d := New("measure", dstest.NewTestDatastore(false)) + dstest.SubtestAll(t, d) +} From 6b5bc5615a4e02f80ffe4990b2f9a9df97aa82d9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 26 Nov 2019 17:53:54 -0500 Subject: [PATCH 3/4] fix: batched put, delete and commit operations are now measured separately from regular operations --- measure.go | 99 +++++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/measure.go b/measure.go index 8955a55..e53cd7e 100644 --- a/measure.go +++ b/measure.go @@ -77,6 +77,23 @@ func New(prefix string, ds datastore.Datastore) *measure { duErr: metrics.New(prefix+".du.errors_total", "Number of errored Datastore.DiskUsage calls").Counter(), duLatency: metrics.New(prefix+".du.latency_seconds", "Latency distribution of Datastore.DiskUsage calls").Histogram(datastoreLatencyBuckets), + + batchPutNum: metrics.New(prefix+".batchput_total", "Total number of Batch.Put calls").Counter(), + batchPutErr: metrics.New(prefix+".batchput.errors_total", "Number of errored Batch.Put calls").Counter(), + batchPutLatency: metrics.New(prefix+".batchput.latency_seconds", + "Latency distribution of Batch.Put calls").Histogram(datastoreLatencyBuckets), + batchPutSize: metrics.New(prefix+".batchput.size_bytes", + "Size distribution of byte slices put into batches").Histogram(datastoreSizeBuckets), + + batchDeleteNum: metrics.New(prefix+".batchdelete_total", "Total number of Batch.Delete calls").Counter(), + batchDeleteErr: metrics.New(prefix+".batchdelete.errors_total", "Number of errored Batch.Delete calls").Counter(), + batchDeleteLatency: metrics.New(prefix+".batchdelete.latency_seconds", + "Latency distribution of Batch.Delete calls").Histogram(datastoreLatencyBuckets), + + batchCommitNum: metrics.New(prefix+".batchcommit_total", "Total number of Batch.Commit calls").Counter(), + batchCommitErr: metrics.New(prefix+".batchcommit.errors_total", "Number of errored Batch.Commit calls").Counter(), + batchCommitLatency: metrics.New(prefix+".batchcommit.latency_seconds", + "Latency distribution of Batch.Commit calls").Histogram(datastoreLatencyBuckets), } return m } @@ -125,6 +142,19 @@ type measure struct { duNum metrics.Counter duErr metrics.Counter duLatency metrics.Histogram + + batchPutNum metrics.Counter + batchPutErr metrics.Counter + batchPutLatency metrics.Histogram + batchPutSize metrics.Histogram + + batchDeleteNum metrics.Counter + batchDeleteErr metrics.Counter + batchDeleteLatency metrics.Histogram + + batchCommitNum metrics.Counter + batchCommitErr metrics.Counter + batchCommitLatency metrics.Histogram } func recordLatency(h metrics.Histogram, start time.Time) { @@ -251,12 +281,7 @@ func (m *measure) DiskUsage() (uint64, error) { } type measuredBatch struct { - puts int - deletes int - - putts datastore.Batch - delts datastore.Batch - + b datastore.Batch m *measure } @@ -265,64 +290,46 @@ func (m *measure) Batch() (datastore.Batch, error) { if !ok { return nil, datastore.ErrBatchUnsupported } - pb, err := bds.Batch() - if err != nil { - return nil, err - } - - db, err := bds.Batch() + batch, err := bds.Batch() if err != nil { return nil, err } return &measuredBatch{ - putts: pb, - delts: db, - + b: batch, m: m, }, nil } func (mt *measuredBatch) Put(key datastore.Key, val []byte) error { - mt.puts++ - mt.m.putSize.Observe(float64(len(val))) - return mt.putts.Put(key, val) -} - -func (mt *measuredBatch) Delete(key datastore.Key) error { - mt.deletes++ - return mt.delts.Delete(key) -} - -func (mt *measuredBatch) Commit() error { - err := logBatchCommit(mt.delts, mt.deletes, mt.m.deleteNum, mt.m.deleteErr, mt.m.deleteLatency) + defer recordLatency(mt.m.batchPutLatency, time.Now()) + mt.m.batchPutNum.Inc() + mt.m.batchPutSize.Observe(float64(len(val))) + err := mt.b.Put(key, val) if err != nil { - return err + mt.m.batchPutErr.Inc() } + return err +} - err = logBatchCommit(mt.putts, mt.puts, mt.m.putNum, mt.m.putErr, mt.m.putLatency) +func (mt *measuredBatch) Delete(key datastore.Key) error { + defer recordLatency(mt.m.batchDeleteLatency, time.Now()) + mt.m.batchDeleteNum.Inc() + err := mt.b.Delete(key) if err != nil { - return err + mt.m.batchDeleteErr.Inc() } - - return nil + return err } -func logBatchCommit(b datastore.Batch, n int, num, errs metrics.Counter, lat metrics.Histogram) error { - if n > 0 { - before := time.Now() - err := b.Commit() - took := time.Since(before) / time.Duration(n) - num.Add(float64(n)) - for i := 0; i < n; i++ { - lat.Observe(took.Seconds()) - } - if err != nil { - errs.Inc() - return err - } +func (mt *measuredBatch) Commit() error { + defer recordLatency(mt.m.batchCommitLatency, time.Now()) + mt.m.batchCommitNum.Inc() + err := mt.b.Commit() + if err != nil { + mt.m.batchCommitErr.Inc() } - return nil + return err } func (m *measure) Close() error { From 788dc2a1648e4aa5b25a1c5db75cc44c8c5ad1ef Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 26 Nov 2019 21:04:04 -0500 Subject: [PATCH 4/4] build: remove gx build files and CI --- .travis.yml | 2 -- Makefile | 9 --------- package.json | 30 ------------------------------ 3 files changed, 41 deletions(-) delete mode 100644 Makefile delete mode 100644 package.json diff --git a/.travis.yml b/.travis.yml index ce8b00b..ba87d9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ env: global: - GOTFLAGS="-race" matrix: - - BUILD_DEPTYPE=gx - BUILD_DEPTYPE=gomod @@ -24,7 +23,6 @@ script: cache: directories: - - $GOPATH/src/gx - $GOPATH/pkg/mod - $HOME/.cache/go-build diff --git a/Makefile b/Makefile deleted file mode 100644 index 5415256..0000000 --- a/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -export IPFS_API ?= v04x.ipfs.io - -gx: - go get -u github.com/whyrusleeping/gx - go get -u github.com/whyrusleeping/gx-go - -deps: gx - gx --verbose install --global - gx-go rewrite diff --git a/package.json b/package.json deleted file mode 100644 index 9d31f48..0000000 --- a/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "author": "whyrusleeping", - "bugs": { - "url": "https://github.com/ipfs/go-ds-measure" - }, - "gx": { - "dvcsimport": "github.com/ipfs/go-ds-measure" - }, - "gxDependencies": [ - { - "author": "ipfs", - "hash": "QmekzFM3hPZjTjUFGTABdQkEnQ3PTiMstY198PwSFr5w1Q", - "name": "go-metrics-interface", - "version": "0.2.0" - }, - { - "author": "jbenet", - "hash": "QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9", - "name": "go-datastore", - "version": "3.6.1" - } - ], - "gxVersion": "0.8.0", - "language": "go", - "license": "", - "name": "go-ds-measure", - "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "1.4.5" -} -