diff --git a/README.md b/README.md index 22826ff9..1d454854 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ ipfs repo version | go-ipfs versions 4 | 0.4.3 - 0.4.5 5 | 0.4.6 - 0.4.10 6 | 0.4.11 - 0.4.15 - 7 | 0.4.16 - current + 7 | 0.4.16 - 0.4.19 + 8 | 0.4.20 - current ### How to Run Migrations diff --git a/ipfs-7-to-8/base32/base32.go b/ipfs-7-to-8/base32/base32.go new file mode 100644 index 00000000..768a2350 --- /dev/null +++ b/ipfs-7-to-8/base32/base32.go @@ -0,0 +1,505 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base32 implements base32 encoding as specified by RFC 4648. +package base32 + +import ( + "io" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 32 encoding/decoding scheme, defined by a +// 32-character alphabet. The most common is the "base32" encoding +// introduced for SASL GSSAPI and standardized in RFC 4648. +// The alternate "base32hex" encoding is used in DNSSEC. +type Encoding struct { + encode string + decodeMap [256]byte + padChar rune +} + +// Alphabet returns the Base32 alphabet used +func (enc *Encoding) Alphabet() string { + return enc.encode +} + +const ( + StdPadding rune = '=' + NoPadding rune = -1 +) + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 32-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.padChar = StdPadding + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// NewEncoding returns a new case insensitive Encoding defined by the +// given alphabet, which must be a 32-byte string. +func NewEncodingCI(encoder string) *Encoding { + e := new(Encoding) + e.padChar = StdPadding + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[asciiToLower(encoder[i])] = byte(i) + e.decodeMap[asciiToUpper(encoder[i])] = byte(i) + } + return e +} + +func asciiToLower(c byte) byte { + if c >= 'A' && c <= 'Z' { + return c + 32 + } + return c +} + +func asciiToUpper(c byte) byte { + if c >= 'a' && c <= 'z' { + return c - 32 + } + return c +} + +// WithPadding creates a new encoding identical to enc except +// with a specified padding character, or NoPadding to disable padding. +func (enc Encoding) WithPadding(padding rune) *Encoding { + enc.padChar = padding + return &enc +} + +// StdEncoding is the standard base32 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncodingCI(encodeStd) + +// HexEncoding is the ``Extended Hex Alphabet'' defined in RFC 4648. +// It is typically used in DNS. +var HexEncoding = NewEncodingCI(encodeHex) + +var RawStdEncoding = NewEncodingCI(encodeStd).WithPadding(NoPadding) +var RawHexEncoding = NewEncodingCI(encodeHex).WithPadding(NoPadding) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 8 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + var carry byte + + // Unpack 8x 5-bit source blocks into a 5 byte + // destination quantum + switch len(src) { + default: + dst[7] = enc.encode[src[4]&0x1F] + carry = src[4] >> 5 + fallthrough + case 4: + dst[6] = enc.encode[carry|(src[3]<<3)&0x1F] + dst[5] = enc.encode[(src[3]>>2)&0x1F] + carry = src[3] >> 7 + fallthrough + case 3: + dst[4] = enc.encode[carry|(src[2]<<1)&0x1F] + carry = (src[2] >> 4) & 0x1F + fallthrough + case 2: + dst[3] = enc.encode[carry|(src[1]<<4)&0x1F] + dst[2] = enc.encode[(src[1]>>1)&0x1F] + carry = (src[1] >> 6) & 0x1F + fallthrough + case 1: + dst[1] = enc.encode[carry|(src[0]<<2)&0x1F] + dst[0] = enc.encode[src[0]>>3] + } + + // Pad the final quantum + if len(src) < 5 { + if enc.padChar != NoPadding { + dst[7] = byte(enc.padChar) + if len(src) < 4 { + dst[6] = byte(enc.padChar) + dst[5] = byte(enc.padChar) + if len(src) < 3 { + dst[4] = byte(enc.padChar) + if len(src) < 2 { + dst[3] = byte(enc.padChar) + dst[2] = byte(enc.padChar) + } + } + } + } + break + } + src = src[5:] + dst = dst[8:] + } +} + +// EncodeToString returns the base32 encoding of src. +func (enc *Encoding) EncodeToString(src []byte) string { + buf := make([]byte, enc.EncodedLen(len(src))) + enc.Encode(buf, src) + return string(buf) +} + +type encoder struct { + err error + enc *Encoding + w io.Writer + buf [5]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 5; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 5 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:8]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 5 { + nn := len(e.out) / 8 * 5 + if nn > len(p) { + nn = len(p) + nn -= nn % 5 + } + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { + return n, e.err + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:8]) + } + return e.err +} + +// NewEncoder returns a new base32 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base32 encodings operate in 5-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base32 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { + if enc.padChar == NoPadding { + return (n*8 + 4) / 5 // minimum # chars at 5 bits per char + } + return (n + 4) / 5 * 8 +} + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) Error() string { + return "illegal base32 data at input byte " + strconv.FormatInt(int64(e), 10) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. This method assumes that src has been +// stripped of all supported whitespace ('\r' and '\n'). +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { + olen := len(src) + for len(src) > 0 && !end { + // Decode quantum using the base32 alphabet + var dbuf [8]byte + dlen := 8 + + for j := 0; j < 8; { + if len(src) == 0 { + if enc.padChar != NoPadding { + return n, false, CorruptInputError(olen - len(src) - j) + } + dlen = j + break + } + in := src[0] + src = src[1:] + if in == byte(enc.padChar) && j >= 2 && len(src) < 8 { + if enc.padChar == NoPadding { + return n, false, CorruptInputError(olen) + } + + // We've reached the end and there's padding + if len(src)+j < 8-1 { + // not enough padding + return n, false, CorruptInputError(olen) + } + for k := 0; k < 8-1-j; k++ { + if len(src) > k && src[k] != byte(enc.padChar) { + // incorrect padding + return n, false, CorruptInputError(olen - len(src) + k - 1) + } + } + dlen, end = j, true + // 7, 5 and 2 are not valid padding lengths, and so 1, 3 and 6 are not + // valid dlen values. See RFC 4648 Section 6 "Base 32 Encoding" listing + // the five valid padding lengths, and Section 9 "Illustrations and + // Examples" for an illustration for how the 1st, 3rd and 6th base32 + // src bytes do not yield enough information to decode a dst byte. + if dlen == 1 || dlen == 3 || dlen == 6 { + return n, false, CorruptInputError(olen - len(src) - 1) + } + break + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(olen - len(src) - 1) + } + j++ + } + + // Pack 8x 5-bit source blocks into 5 byte destination + // quantum + switch dlen { + case 8: + dst[4] = dbuf[6]<<5 | dbuf[7] + fallthrough + case 7: + dst[3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 + fallthrough + case 5: + dst[2] = dbuf[3]<<4 | dbuf[4]>>1 + fallthrough + case 4: + dst[1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 + fallthrough + case 2: + dst[0] = dbuf[0]<<3 | dbuf[1]>>2 + } + + if len(dst) > 5 { + dst = dst[5:] + } + + switch dlen { + case 2: + n += 1 + case 4: + n += 2 + case 5: + n += 3 + case 7: + n += 4 + case 8: + n += 5 + } + } + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base32 data, it will return the +// number of bytes successfully written and CorruptInputError. +// New line characters (\r and \n) are ignored. +func (enc *Encoding) Decode(dst, s []byte) (n int, err error) { + // FIXME: if dst is the same as s use decodeInPlace + stripped := make([]byte, 0, len(s)) + for _, c := range s { + if c != '\r' && c != '\n' { + stripped = append(stripped, c) + } + } + n, _, err = enc.decode(dst, stripped) + return +} + +func (enc *Encoding) decodeInPlace(strb []byte) (n int, err error) { + off := 0 + for _, b := range strb { + if b == '\n' || b == '\r' { + continue + } + strb[off] = b + off++ + } + n, _, err = enc.decode(strb, strb[:off]) + return +} + +// DecodeString returns the bytes represented by the base32 string s. +func (enc *Encoding) DecodeString(s string) ([]byte, error) { + strb := []byte(s) + n, err := enc.decodeInPlace(strb) + if err != nil { + return nil, err + } + return strb[:n], nil +} + +type decoder struct { + err error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 8 * 5]byte +} + +func (d *decoder) Read(p []byte) (n int, err error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 5 * 8 + if nn < 8 { + nn = 8 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + d.nbuf += nn + if d.nbuf < 8 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 8 * 8 + nw := d.nbuf / 8 * 5 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +type newlineFilteringReader struct { + wrapped io.Reader +} + +func (r *newlineFilteringReader) Read(p []byte) (int, error) { + n, err := r.wrapped.Read(p) + for n > 0 { + offset := 0 + for i, b := range p[0:n] { + if b != '\r' && b != '\n' { + if i != offset { + p[offset] = b + } + offset++ + } + } + if offset > 0 { + return offset, err + } + // Previous buffer entirely whitespace, read again + n, err = r.wrapped.Read(p) + } + return n, err +} + +// NewDecoder constructs a new base32 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: &newlineFilteringReader{r}} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base32-encoded data. +func (enc *Encoding) DecodedLen(n int) int { + if enc.padChar == NoPadding { + return (n*5 + 7) / 8 + } + + return n / 8 * 5 +} diff --git a/ipfs-7-to-8/base32/base32_test.go b/ipfs-7-to-8/base32/base32_test.go new file mode 100644 index 00000000..c33a8ee6 --- /dev/null +++ b/ipfs-7-to-8/base32/base32_test.go @@ -0,0 +1,365 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base32 + +import ( + "bytes" + "io" + "io/ioutil" + "math/rand" + "strings" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairsUpper = []testpair{ + // RFC 4648 examples + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + + // Wikipedia examples, converted to base32 + {"sure.", "ON2XEZJO"}, + {"sure", "ON2XEZI="}, + {"sur", "ON2XE==="}, + {"su", "ON2Q===="}, + {"leasure.", "NRSWC43VOJSS4==="}, + {"easure.", "MVQXG5LSMUXA===="}, + {"asure.", "MFZXK4TFFY======"}, + {"sure.", "ON2XEZJO"}, +} + +var pairs = pairsUpper + +func init() { + // also test lower case versions + for _, p := range pairsUpper { + pairs = append(pairs, testpair{p.decoded, strings.ToLower(p.encoded)}) + } +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "KR3WC4ZAMJZGS3DMNFTSYIDBNZSCA5DIMUQHG3DJORUHSIDUN53GK4Y=", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairsUpper { + got := StdEncoding.EncodeToString([]byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, got, p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairsUpper { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, + string(dbuf[0:count]), + p.decoded) + + dbuf, err = StdEncoding.DecodeString(p.encoded) + testEqual(t, "DecodeString(%q) = error %v, want %v", p.encoded, err, error(nil)) + testEqual(t, "DecodeString(%q) = %q, want %q", p.encoded, string(dbuf), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, strings.NewReader(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != io.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != io.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, io.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + testCases := []struct { + input string + offset int // -1 means no corruption. + }{ + {"", -1}, + {"!!!!", 0}, + {"!===", 0}, + {"AA=A====", 2}, + {"AAA=AAAA", 3}, + {"MMMMMMMMM", 8}, + {"MMMMMM", 0}, + {"A=", 1}, + {"AA=", 3}, + {"AA==", 4}, + {"AA===", 5}, + {"AAAA=", 5}, + {"AAAA==", 6}, + {"AAAAA=", 6}, + {"AAAAA==", 7}, + {"A=======", 1}, + {"AA======", -1}, + {"AAA=====", 3}, + {"AAAA====", -1}, + {"AAAAA===", -1}, + {"AAAAAA==", 6}, + {"AAAAAAA=", -1}, + {"AAAAAAAA", -1}, + } + for _, tc := range testCases { + dbuf := make([]byte, StdEncoding.DecodedLen(len(tc.input))) + _, err := StdEncoding.Decode(dbuf, []byte(tc.input)) + if tc.offset == -1 { + if err != nil { + t.Error("Decoder wrongly detected corruption in", tc.input) + } + continue + } + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", tc.input, int(err), tc.offset) + default: + t.Error("Decoder failed to detect corruption in", tc) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} + +func testStringEncoding(t *testing.T, expected string, examples []string) { + for _, e := range examples { + buf, err := StdEncoding.DecodeString(e) + if err != nil { + t.Errorf("Decode(%q) failed: %v", e, err) + continue + } + if s := string(buf); s != expected { + t.Errorf("Decode(%q) = %q, want %q", e, s, expected) + } + } +} + +func TestNewLineCharacters(t *testing.T) { + // Each of these should decode to the string "sure", without errors. + examples := []string{ + "ON2XEZI=", + "ON2XEZI=\r", + "ON2XEZI=\n", + "ON2XEZI=\r\n", + "ON2XEZ\r\nI=", + "ON2X\rEZ\nI=", + "ON2X\nEZ\rI=", + "ON2XEZ\nI=", + "ON2XEZI\n=", + } + testStringEncoding(t, "sure", examples) + + // Each of these should decode to the string "foobar", without errors. + examples = []string{ + "MZXW6YTBOI======", + "MZXW6YTBOI=\r\n=====", + } + testStringEncoding(t, "foobar", examples) +} + +func TestDecoderIssue4779(t *testing.T) { + encoded := `JRXXEZLNEBUXA43VNUQGI33MN5ZCA43JOQQGC3LFOQWCAY3PNZZWKY3UMV2HK4 +RAMFSGS4DJONUWG2LOM4QGK3DJOQWCA43FMQQGI3YKMVUXK43NN5SCA5DFNVYG64RANFXGG2LENFSH +K3TUEB2XIIDMMFRG64TFEBSXIIDEN5WG64TFEBWWCZ3OMEQGC3DJOF2WCLRAKV2CAZLONFWQUYLEEB +WWS3TJNUQHMZLONFQW2LBAOF2WS4ZANZXXG5DSOVSCAZLYMVZGG2LUMF2GS33OEB2WY3DBNVRW6IDM +MFRG64TJOMQG42LTNEQHK5AKMFWGS4LVNFYCAZLYEBSWCIDDN5WW233EN4QGG33OONSXC5LBOQXCAR +DVNFZSAYLVORSSA2LSOVZGKIDEN5WG64RANFXAU4TFOBZGK2DFNZSGK4TJOQQGS3RAOZXWY5LQORQX +IZJAOZSWY2LUEBSXG43FEBRWS3DMOVWSAZDPNRXXEZJAMV2SAZTVM5UWC5BANZ2WY3DBBJYGC4TJMF +2HK4ROEBCXQY3FOB2GK5LSEBZWS3TUEBXWGY3BMVRWC5BAMN2XA2LEMF2GC5BANZXW4IDQOJXWSZDF +NZ2CYIDTOVXHIIDJNYFGG5LMOBQSA4LVNEQG6ZTGNFRWSYJAMRSXGZLSOVXHIIDNN5WGY2LUEBQW42 +LNEBUWIIDFON2CA3DBMJXXE5LNFY== +====` + encodedShort := strings.Replace(encoded, "\n", "", -1) + + dec := NewDecoder(StdEncoding, strings.NewReader(encoded)) + res1, err := ioutil.ReadAll(dec) + if err != nil { + t.Errorf("ReadAll failed: %v", err) + } + + dec = NewDecoder(StdEncoding, strings.NewReader(encodedShort)) + var res2 []byte + res2, err = ioutil.ReadAll(dec) + if err != nil { + t.Errorf("ReadAll failed: %v", err) + } + + if !bytes.Equal(res1, res2) { + t.Error("Decoded results not equal") + } +} + +func BenchmarkEncodeToString(b *testing.B) { + data := make([]byte, 8192) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.EncodeToString(data) + } +} + +func BenchmarkDecodeString(b *testing.B) { + data := StdEncoding.EncodeToString(make([]byte, 8192)) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.DecodeString(data) + } +} + +func TestNoPadding(t *testing.T) { + examples := []string{ + "a", + "ab", + "abc", + "abcd", + "abcde", + "", + string([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}), + } + + for _, e := range examples { + t.Log("testing: ", e) + enc := RawStdEncoding.EncodeToString([]byte(e)) + if strings.Contains(enc, "=") { + t.Fatal("shouldnt have any padding") + } + + out, err := RawStdEncoding.DecodeString(enc) + if err != nil { + t.Fatal(err) + } + + if string(out) != e { + t.Fatalf("round trip of %q failed", e) + } + } +} + +func TestNoPaddingRand(t *testing.T) { + buf := make([]byte, 1024) + for i := 0; i < 1000; i++ { + l := rand.Intn(len(buf)) + seg := buf[:l] + rand.Read(seg) + + enc := RawStdEncoding.EncodeToString(seg) + if strings.Contains(enc, "=") { + t.Fatal("shouldnt have any padding") + } + + out, err := RawStdEncoding.DecodeString(enc) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out, seg) { + t.Fatalf("round trip of %q failed", seg) + } + } +} + diff --git a/ipfs-7-to-8/base32/example_test.go b/ipfs-7-to-8/base32/example_test.go new file mode 100644 index 00000000..2a302d88 --- /dev/null +++ b/ipfs-7-to-8/base32/example_test.go @@ -0,0 +1,45 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Keep in sync with ../base64/example_test.go. + +package base32_test + +import ( + "encoding/base32" + "fmt" + "os" +) + +func ExampleEncoding_EncodeToString() { + data := []byte("any + old & data") + str := base32.StdEncoding.EncodeToString(data) + fmt.Println(str) + // Output: + // MFXHSIBLEBXWYZBAEYQGIYLUME====== +} + +func ExampleEncoding_DecodeString() { + str := "ONXW2ZJAMRQXIYJAO5UXI2BAAAQGC3TEEDX3XPY=" + data, err := base32.StdEncoding.DecodeString(str) + if err != nil { + fmt.Println("error:", err) + return + } + fmt.Printf("%q\n", data) + // Output: + // "some data with \x00 and \ufeff" +} + +func ExampleNewEncoder() { + input := []byte("foo\x00bar") + encoder := base32.NewEncoder(base32.StdEncoding, os.Stdout) + encoder.Write(input) + // Must close the encoder when finished to flush any partial blocks. + // If you comment out the following line, the last partial block "r" + // won't be encoded. + encoder.Close() + // Output: + // MZXW6ADCMFZA==== +} diff --git a/ipfs-7-to-8/base32/package.json b/ipfs-7-to-8/base32/package.json new file mode 100644 index 00000000..f78c51b6 --- /dev/null +++ b/ipfs-7-to-8/base32/package.json @@ -0,0 +1,15 @@ +{ + "author": "whyrusleeping", + "bugs": { + "url": "https://github.com/whyrusleeping/base32" + }, + "gx": { + "dvcsimport": "github.com/whyrusleeping/base32" + }, + "gxVersion": "0.7.0", + "language": "go", + "license": "", + "name": "base32", + "version": "0.0.2" +} + diff --git a/ipfs-7-to-8/main.go b/ipfs-7-to-8/main.go new file mode 100644 index 00000000..3e9a7b51 --- /dev/null +++ b/ipfs-7-to-8/main.go @@ -0,0 +1,11 @@ +package main + +import ( + migrate "github.com/ipfs/fs-repo-migrations/go-migrate" + mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration" +) + +func main() { + m := mg7.Migration{} + migrate.Main(&m) +} diff --git a/ipfs-7-to-8/migration/migration.go b/ipfs-7-to-8/migration/migration.go new file mode 100644 index 00000000..001fb249 --- /dev/null +++ b/ipfs-7-to-8/migration/migration.go @@ -0,0 +1,153 @@ +package mg6 + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + migrate "github.com/ipfs/fs-repo-migrations/go-migrate" + mfsr "github.com/ipfs/fs-repo-migrations/mfsr" + log "github.com/ipfs/fs-repo-migrations/stump" + + base32 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/base32" +) + +type Migration struct{} + +func (m Migration) Versions() string { + return "7-to-8" +} + +func (m Migration) Reversible() bool { + return true +} + +const keyFilenamePrefix = "key_" + +func isEncoded(name string) bool { + if !strings.HasPrefix(name, keyFilenamePrefix) { + return false + } + + _, err := decode(name) + + return err == nil +} + +func encode(name string) (string, error) { + if name == "" { + return "", fmt.Errorf("key name must be at least one character") + } + + encodedName := base32.RawStdEncoding.EncodeToString([]byte(name)) + return keyFilenamePrefix + strings.ToLower(encodedName), nil +} + +func decode(name string) (string, error) { + if !strings.HasPrefix(name, keyFilenamePrefix) { + return "", fmt.Errorf("key's filename has unexpected format") + } + + nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):]) + data, err := base32.RawStdEncoding.DecodeString(nameWithoutPrefix) + + if err != nil { + return "", err + } + + decodedName := string(data[:]) + + return decodedName, nil +} + +func (m Migration) Apply(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("applying %s repo migration", m.Versions()) + + keystoreRoot := filepath.Join(opts.Path, "keystore") + fileInfos, err := ioutil.ReadDir(keystoreRoot) + + if err != nil { + return err + } + + for _, info := range fileInfos { + if info.IsDir() { + log.Log("skipping ", info.Name(), " as it is directory!") + continue + } + + if isEncoded(info.Name()) { + log.Log("skipping ", info.Name(), " as it is already encoded!") + continue + } + + log.VLog("migrating key's filename: ", info.Name()) + encodedName, err := encode(info.Name()) + if err != nil { + return err + } + + os.Rename( + filepath.Join(keystoreRoot, info.Name()), + filepath.Join(keystoreRoot, encodedName), + ) + } + + err = mfsr.RepoPath(opts.Path).WriteVersion("8") + if err != nil { + log.Error("failed to update version file to 8") + return err + } + + log.Log("updated version file") + + return nil +} + +func (m Migration) Revert(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("reverting migration") + + keystoreRoot := filepath.Join(opts.Path, "keystore") + fileInfos, err := ioutil.ReadDir(keystoreRoot) + + if err != nil { + return err + } + + for _, info := range fileInfos { + if info.IsDir() { + log.Log("skipping", info.Name(), "as it is directory!") + continue + } + + if !isEncoded(info.Name()) { + log.Log("skipping", info.Name(), "as it is not encoded!") + continue + } + + log.VLog("reverting key's filename:", info.Name()) + decodedName, err := decode(info.Name()) + if err != nil { + return err + } + + os.Rename( + filepath.Join(keystoreRoot, info.Name()), + filepath.Join(keystoreRoot, decodedName), + ) + } + + err = mfsr.RepoPath(opts.Path).WriteVersion("7") + if err != nil { + log.Error("failed to update version file to 7") + return err + } + + log.Log("updated version file") + + return nil +} diff --git a/ipfs-7-to-8/package.json b/ipfs-7-to-8/package.json new file mode 100644 index 00000000..40d00d16 --- /dev/null +++ b/ipfs-7-to-8/package.json @@ -0,0 +1,22 @@ +{ + "author": "AuHau", + "bugs": {}, + "gx": { + "dvcsimport": "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8" + }, + "gxDependencies": [ + { + "author": "whyrusleeping", + "hash": "QmfVj3x4D6Jkq9SEoi5n2NmoUomLwoeiwnYz2KQa15wRw6", + "name": "base32", + "version": "0.0.2" + } + ], + "gxVersion": "0.14.0", + "language": "go", + "license": "", + "name": "ipfs-7-to-8", + "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", + "version": "0.0.0" +} + diff --git a/main.go b/main.go index 4edf2a1a..f57882a5 100644 --- a/main.go +++ b/main.go @@ -16,10 +16,11 @@ import ( mg4 "github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/migration" mg5 "github.com/ipfs/fs-repo-migrations/ipfs-5-to-6/migration" mg6 "github.com/ipfs/fs-repo-migrations/ipfs-6-to-7/migration" + mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration" mfsr "github.com/ipfs/fs-repo-migrations/mfsr" ) -var CurrentVersion = 7 +var CurrentVersion = 8 var migrations = []gomigrate.Migration{ &mg0.Migration{}, @@ -29,6 +30,7 @@ var migrations = []gomigrate.Migration{ &mg4.Migration{}, &mg5.Migration{}, &mg6.Migration{}, + &mg7.Migration{}, } func GetIpfsDir() (string, error) {