Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rewrite add command to use dagwriter #236

Merged
merged 2 commits into from
Oct 30, 2014
Merged

rewrite add command to use dagwriter #236

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18ada93
rewrite add command to use dagwriter, moved a pinner into the dagwrit…
whyrusleeping Oct 30, 2014
71ac117
Address concerns in PR
whyrusleeping Oct 30, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/commands/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func addDir(n *core.IpfsNode, fpath string, depth int, out io.Writer) (*dag.Node
}

func addFile(n *core.IpfsNode, fpath string, depth int, out io.Writer) (*dag.Node, error) {
root, err := importer.NewDagFromFile(fpath)
root, err := importer.NewDagFromFileWServer(fpath, n.DAG, n.Pinning)
if err != nil {
return nil, err
}
Expand All @@ -98,7 +98,7 @@ func addFile(n *core.IpfsNode, fpath string, depth int, out io.Writer) (*dag.Nod
log.Info("adding subblock: %s %s", l.Name, l.Hash.B58String())
}

return root, addNode(n, root, fpath, out)
return root, nil
}

// addNode adds the node to the graph + local storage
Expand Down
41 changes: 41 additions & 0 deletions importer/importer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package importer

import (
"errors"
"fmt"
"io"
"os"

"github.com/jbenet/go-ipfs/importer/chunk"
dag "github.com/jbenet/go-ipfs/merkledag"
"github.com/jbenet/go-ipfs/pin"
ft "github.com/jbenet/go-ipfs/unixfs"
uio "github.com/jbenet/go-ipfs/unixfs/io"
"github.com/jbenet/go-ipfs/util"
)

Expand Down Expand Up @@ -72,3 +75,41 @@ func NewDagFromFile(fpath string) (*dag.Node, error) {

return NewDagFromReader(f)
}

func NewDagFromFileWServer(fpath string, dserv dag.DAGService, p pin.Pinner) (*dag.Node, error) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

...WServer ? ...WithService ?

Since these are more complicated construction funcs, no need to adhere to the New... form. Could easily be named: ImportFileDag

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cool. Ill give them nicer names now, lol

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, seems like you could just pass the DagWriter in directly. As in, maybe this function should just be taking a dagwriter.

dw, err := uio.NewDagWriter(dserv, chunk.DefaultSplitter)
if err != nil { ... }
nd, err := import.ImportFileDag(filename, dw) // arg lint error.

stat, err := os.Stat(fpath)
if err != nil {
return nil, err
}

if stat.IsDir() {
return nil, fmt.Errorf("`%s` is a directory", fpath)
}

f, err := os.Open(fpath)
if err != nil {
return nil, err
}
defer f.Close()

return NewDagFromReaderWServer(f, dserv, p)
}

func NewDagFromReaderWServer(r io.Reader, dserv dag.DAGService, p pin.Pinner) (*dag.Node, error) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or ImportReaderDag

dw := uio.NewDagWriter(dserv, chunk.DefaultSplitter)

mp, ok := p.(pin.ManualPinner)
if !ok {
return nil, errors.New("Needed to be passed a manual pinner!")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if really only works with a ManualPinner, could make this a param of the function. If this is liable to change later, then keep the pin.Pinner type.

}
dw.Pinner = mp
_, err := io.Copy(dw, r)
if err != nil {
return nil, err
}
err = dw.Close()
if err != nil {
return nil, err
}
return dw.GetNode(), nil
}
26 changes: 26 additions & 0 deletions pin/pin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,28 @@ var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys")
var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys")
var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys")

type PinMode int

const (
Recursive PinMode = iota
Direct
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Strong, weak? (common vocabulary for this operation)

Indirect
)

type Pinner interface {
IsPinned(util.Key) bool
Pin(*mdag.Node, bool) error
Unpin(util.Key, bool) error
Flush() error
}

// ManualPinner is for manually editing the pin structure
// Use with care
type ManualPinner interface {
PinWithMode(util.Key, PinMode)
Pinner
}

type pinner struct {
lock sync.RWMutex
recursePin set.BlockSet
Expand Down Expand Up @@ -228,3 +243,14 @@ func loadSet(d ds.Datastore, k ds.Key, val interface{}) error {
}
return json.Unmarshal(bf, val)
}

func (p *pinner) PinWithMode(k util.Key, mode PinMode) {
switch mode {
case Recursive:
p.recursePin.AddBlock(k)
case Direct:
p.directPin.AddBlock(k)
case Indirect:
p.indirPin.Increment(k)
}
}
12 changes: 10 additions & 2 deletions unixfs/io/dagwriter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package io
import (
"github.com/jbenet/go-ipfs/importer/chunk"
dag "github.com/jbenet/go-ipfs/merkledag"
"github.com/jbenet/go-ipfs/pin"
ft "github.com/jbenet/go-ipfs/unixfs"
"github.com/jbenet/go-ipfs/util"
)
Expand All @@ -17,6 +18,7 @@ type DagWriter struct {
done chan struct{}
splitter chunk.BlockSplitter
seterr error
Pinner pin.ManualPinner
}

func NewDagWriter(ds dag.DAGService, splitter chunk.BlockSplitter) *DagWriter {
Expand Down Expand Up @@ -48,7 +50,10 @@ func (dw *DagWriter) startSplitter() {
// Store the block size in the root node
mbf.AddBlockSize(uint64(len(blkData)))
node := &dag.Node{Data: ft.WrapData(blkData)}
_, err := dw.dagserv.Add(node)
nk, err := dw.dagserv.Add(node)
if dw.Pinner != nil {
dw.Pinner.PinWithMode(nk, pin.Indirect)
}
if err != nil {
dw.seterr = err
log.Critical("Got error adding created node to dagservice: %s", err)
Expand All @@ -75,12 +80,15 @@ func (dw *DagWriter) startSplitter() {
root.Data = data

// Add root node to the dagservice
_, err = dw.dagserv.Add(root)
rootk, err := dw.dagserv.Add(root)
if err != nil {
dw.seterr = err
log.Critical("Got error adding created node to dagservice: %s", err)
return
}
if dw.Pinner != nil {
dw.Pinner.PinWithMode(rootk, pin.Recursive)
}
dw.node = root
dw.done <- struct{}{}
}
Expand Down