From 26665ad5a8ed592f3c7c50ea626619a9170c8b9b Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Fri, 25 Aug 2023 12:15:28 +0100 Subject: [PATCH 1/2] Add config to specify a list of SPs used for deal making Add CLI flags to specify a list of storage providers to make deals with. In a case where no SPs are specified, Motion will fall back to local storage only, i.e. the existing behaviour. Update comments and docs to use consistent casing for "Filecoin". Fixes #59 --- README.md | 13 ++++++++----- blob/ribs_store.go | 2 +- blob/singularity_store.go | 2 +- cmd/motion/main.go | 26 ++++++++++++++++++++++++-- motion.go | 2 +- openapi.yaml | 4 ++-- options.go | 17 +++++++++++++---- 7 files changed, 50 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1c02891..d63f3d6 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,14 @@ COMMANDS: help, h Shows a list of commands or help for one command GLOBAL OPTIONS: - --storeDir value The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR] - --experimentalRibsStore Whether to use experimental RIBS as the storage and deal making (default: Local storage is used) - --localWalletDir value The path to the local wallet directory. (default: Defaults to '/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.) [$MOTION_LOCAL_WALLET_DIR] - --localWalletGenerateIfNotExist Whether to generate the local wallet key if none is found (default: true) - --help, -h + --storeDir value The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR] + --experimentalRibsStore Whether to use experimental RIBS as the storage and deal making (default: Local storage is used) + --localWalletDir value The path to the local wallet directory. (default: Defaults to '/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.) [$MOTION_LOCAL_WALLET_DIR] + --localWalletGenerateIfNotExist Whether to generate the local wallet key if none is found (default: true) + --experimentalSingularityStore whether to use experimental Singularity store as the storage and deal making engine (default: Local storage is used) + --experimentalRemoteSingularityAPIUrl value when using a singularity as the storage engine, if set, uses a remote HTTP API to interface with Singularity (default: use singularity as a code library) + --storageProvider value, --sp value [ --storageProvider value, --sp value ] Storage providers to which to make deals with. Multiple providers may be specified. (default: No deals are made to replicate data onto storage providers.) + --help, -h show help ``` ## Run Server Locally diff --git a/blob/ribs_store.go b/blob/ribs_store.go index 74d558b..7ce95f0 100644 --- a/blob/ribs_store.go +++ b/blob/ribs_store.go @@ -179,7 +179,7 @@ func (r *RibsStore) describeRibsStoredBlob(_ context.Context, id ID) (*ribsStore case err == nil: var storedBlob ribsStoredBlob err := json.NewDecoder(index).Decode(&storedBlob) - // TODO: populate descriptor status with FileCoin chain data about the stored blob. + // TODO: populate descriptor status with Filecoin chain data about the stored blob. return &storedBlob, err case errors.Is(err, os.ErrNotExist): return nil, ErrBlobNotFound diff --git a/blob/singularity_store.go b/blob/singularity_store.go index a10c4bf..20385ff 100644 --- a/blob/singularity_store.go +++ b/blob/singularity_store.go @@ -91,7 +91,7 @@ func (s *SingularityStore) Put(ctx context.Context, reader io.ReadCloser) (*Desc defer idFile.Close() _, err = idFile.Write([]byte(strconv.FormatUint(model.ID, 10))) if err != nil { - os.Remove(idFile.Name()) + _ = os.Remove(idFile.Name()) return nil, err } if err = os.Rename(idFile.Name(), path.Join(s.local.dir, desc.ID.String()+".id")); err != nil { diff --git a/cmd/motion/main.go b/cmd/motion/main.go index 3db9000..1f98b89 100644 --- a/cmd/motion/main.go +++ b/cmd/motion/main.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net/http" "os" "os/signal" @@ -9,6 +10,7 @@ import ( httpclient "github.com/data-preservation-programs/singularity/client/http" libclient "github.com/data-preservation-programs/singularity/client/lib" "github.com/data-preservation-programs/singularity/database" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/motion" "github.com/filecoin-project/motion/blob" @@ -25,7 +27,7 @@ func main() { } app := cli.App{ Name: "motion", - Usage: "Propelling data onto FileCoin", + Usage: "Propelling data onto Filecoin", Flags: []cli.Flag{ &cli.StringFlag{ Name: "storeDir", @@ -60,6 +62,12 @@ func main() { Usage: "when using a singularity as the storage engine, if set, uses a remote HTTP API to interface with Singularity", DefaultText: "use singularity as a code library", }, + &cli.StringSliceFlag{ + Name: "storageProvider", + Aliases: []string{"sp"}, + Usage: "Storage providers to which to make deals with. Multiple providers may be specified.", + DefaultText: "No deals are made to replicate data onto storage providers.", + }, }, Action: func(cctx *cli.Context) error { @@ -120,7 +128,21 @@ func main() { logger.Infow("Using local blob store", "storeDir", storeDir) } - m, err := motion.New(motion.WithBlobStore(store), motion.WithWallet(wallet)) + // Parse any configured storage povider addresses. + sps := cctx.StringSlice("storageProvider") + spAddrs := make([]address.Address, 0, len(sps)) + for _, sp := range sps { + spAddr, err := address.NewFromString(sp) + if err != nil { + return fmt.Errorf("storage provider '%s' is not a valid address: %w", sp, err) + } + spAddrs = append(spAddrs, spAddr) + } + m, err := motion.New( + motion.WithBlobStore(store), + motion.WithWallet(wallet), + motion.WithStorageProviders(spAddrs...), + ) if err != nil { logger.Fatalw("Failed to instantiate Motion", "err", err) } diff --git a/motion.go b/motion.go index 6331638..a416212 100644 --- a/motion.go +++ b/motion.go @@ -11,7 +11,7 @@ var ( logger = log.Logger("motion") ) -// Motion is a service that exposes a simple HTTP API to create / retrieve blobs of data to and from FileCoin Network. +// Motion is a service that exposes a simple HTTP API to create / retrieve blobs of data to and from Filecoin Network. type Motion struct { *options httpServer *server.HttpServer diff --git a/openapi.yaml b/openapi.yaml index 9af33c7..1008832 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Motion Data Storage and Retrieval API - description: This is the API for exchanging data with FileCoin network via Motion. + description: This is the API for exchanging data with Filecoin network via Motion. version: 0.0.0 servers: - url: 'http://localhost:40080' @@ -106,7 +106,7 @@ paths: properties: provider: type: string - description: 'ID of the FileCoin storage provider.' + description: 'ID of the Filecoin storage provider.' status: type: string description: 'Status of this replica. Can be "active", "slashed" or "expired".' diff --git a/options.go b/options.go index 5a6e02e..d9b5b6a 100644 --- a/options.go +++ b/options.go @@ -3,6 +3,7 @@ package motion import ( "os" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/motion/api/server" "github.com/filecoin-project/motion/blob" "github.com/filecoin-project/motion/wallet" @@ -12,9 +13,10 @@ type ( // Option represents a configurable parameter in Motion service. Option func(*options) error options struct { - serverOptions []server.Option - blobStore blob.Store - wallet *wallet.Wallet + serverOptions []server.Option + blobStore blob.Store + wallet *wallet.Wallet + storageProviders []address.Address } ) @@ -52,7 +54,7 @@ func WithBlobStore(s blob.Store) Option { } } -// WithWallet sets the wallet used by Motion to interact with FileCoin network. +// WithWallet sets the wallet used by Motion to interact with Filecoin network. // Defaults to wallet.New. func WithWallet(w *wallet.Wallet) Option { return func(o *options) error { @@ -60,3 +62,10 @@ func WithWallet(w *wallet.Wallet) Option { return nil } } + +func WithStorageProviders(sp ...address.Address) Option { + return func(o *options) error { + o.storageProviders = sp + return nil + } +} From fef8a61f106141b9b7c459aaf064d104f28e1373 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Fri, 25 Aug 2023 12:18:30 +0100 Subject: [PATCH 2/2] Update godoc for storage provider option Update godoc for storage procvider option. --- options.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/options.go b/options.go index d9b5b6a..3348b7e 100644 --- a/options.go +++ b/options.go @@ -63,6 +63,8 @@ func WithWallet(w *wallet.Wallet) Option { } } +// WithStorageProviders sets the list of Filecoin storage providers to make deals with. +// Defaults to no deals, i.e. local storage only if unspecified. func WithStorageProviders(sp ...address.Address) Option { return func(o *options) error { o.storageProviders = sp