#18 WIP on efficient binary RDF format

xrb/serialize.go

@@ -1,10 +1,12 @@
 package xrb
 
 import (
+	"bufio"
 	"encoding/binary"
 	"fmt"
 	"github.com/regen-network/regen-ledger/x/schema"
 	"io"
+	"strconv"
 )
 
 type Serializer struct {
@@ -17,10 +19,10 @@ func (sz *Serializer) SerializeGraph(g Graph, w io.Writer) (hash []byte, err err
 
 type szContext struct {
 	*Serializer
-	w io.Writer
+	w bufio.Writer
 	//resGraph graph
 	//curResNode node
-	hashText string
+	hashText hashAcc
 }
 
 type propInfo struct {
@@ -52,7 +54,20 @@ func (s *szContext) serializeGraph(g Graph) {
 	}
 	nodes := g.Nodes()
 	writeVarInt(s.w, len(nodes))
+	var last string
 	for _, n := range nodes {
+		if n == "" {
+			panic("node ID cannot be nil")
+		}
+		if last != "" {
+			if n < last {
+				panic("nodes not in sorted order") // Node implementation error so panic
+			}
+			if last == n {
+				panic("duplicate node ID")
+			}
+			last = n
+		}
 		s.serializeNode(false, g.GetNode(n))
 	}
 }
@@ -61,47 +76,54 @@ func (s *szContext) serializeNode(root bool, n Node) {
 	if !root {
 		writeString(s.w, n.ID())
 	}
+	id := n.ID()
 	for _, url := range n.Properties() {
+		if root {
+			s.hashText.write("_:_\n")
+		} else {
+			s.hashText.writeIRI(id)
+			s.hashText.write("\n")
+		}
 		s.writeProperty(s.w, url, n.GetProperty(url))
 	}
 }
 
-func writeVarUint64(w io.Writer, x uint64) {
+func writeVarUint64(w bufio.Writer, x uint64) {
 	buf := make([]byte, binary.MaxVarintLen64)
 	n := binary.PutUvarint(buf, x)
 	mustWrite(w, buf[:n])
 }
 
-func writeVarInt(w io.Writer, x int) {
+func writeVarInt(w bufio.Writer, x int) {
 	buf := make([]byte, binary.MaxVarintLen64)
 	n := binary.PutVarint(buf, int64(x))
 	mustWrite(w, buf[:n])
 }
 
-func mustWrite(w io.Writer, buf []byte) {
+func mustWrite(w bufio.Writer, buf []byte) {
 	_, err := w.Write(buf)
 	if err != nil {
 		panic(err)
 	}
 }
 
-func writeByte(w io.Writer, x byte) {
+func writeByte(w bufio.Writer, x byte) {
 	mustWrite(w, []byte{x})
 }
 
-func writeString(w io.Writer, str string) {
+func writeString(w bufio.Writer, str string) {
 	panic("TODO")
 }
 
-func writeBool(w io.Writer, x bool) {
+func writeBool(w bufio.Writer, x bool) {
 	panic("TODO")
 }
 
-func writeFloat64(w io.Writer, x float64) {
+func writeFloat64(w bufio.Writer, x float64) {
 	panic("TODO")
 }
 
-func (s *szContext) writeProperty(w io.Writer, name string, value interface{}) {
+func (s *szContext) writeProperty(w bufio.Writer, name string, value interface{}) {
 	info, found := s.properties[name]
 	if !found {
 		panic("TODO")
@@ -110,14 +132,20 @@ func (s *szContext) writeProperty(w io.Writer, name string, value interface{}) {
 	writeByte(w, 0)
 	writeVarUint64(w, uint64(info.id))
 
-	s.writePropertyValue(w, info.def.Many, info.def.PropertyType, value)
+	s.writePropertyValue(w, info.def.Arity, info.def.PropertyType, value)
 }
 
-func (s *szContext) writePropertyValue(w io.Writer, many bool, ty schema.PropertyType, value interface{}) {
-	if !many {
+func (s *szContext) writePropertyValue(w bufio.Writer, arity schema.Arity, ty schema.PropertyType, value interface{}) {
+	switch arity {
+	case schema.One:
 		s.writePropertyOne(ty, value)
-	} else {
-		s.writePropertyMany(ty, value)
+	case schema.UnorderedSet:
+		s.writePropertyMany(ty, value, false)
+	case schema.OrderedSet:
+		s.writePropertyMany(ty, value, true)
+	default:
+		panic("unknown arity")
+
 	}
 }
 
@@ -149,7 +177,7 @@ func (s *szContext) writePropertyOne(ty schema.PropertyType, value interface{})
 	}
 }
 
-func (s *szContext) writePropertyMany(ty schema.PropertyType, value interface{}) {
+func (s *szContext) writePropertyMany(ty schema.PropertyType, value interface{}, ordered bool) {
 	switch ty {
 	case schema.TyString:
 		arr, ok := value.([]string)
@@ -185,3 +213,40 @@ func (s *szContext) writePropertyMany(ty schema.PropertyType, value interface{})
 	default:
 	}
 }
+
+type hashAcc struct {
+	bufio.Writer
+}
+
+func (ha *hashAcc) write(x string) {
+	_, err := ha.WriteString(x)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (ha *hashAcc) writeStringLiteral(x string, typeIri string, lang string) {
+	ha.write("\"")
+	ha.write(strconv.Quote(x))
+	ha.write("\"")
+	if typeIri != "" {
+		if lang != "" {
+			panic("cannot specify both a data type IRI and a language tag")
+		}
+		ha.write("^^")
+		ha.writeIRI(typeIri)
+	} else if lang != "" {
+		ha.write("@")
+		ha.write(lang)
+	}
+}
+
+func (ha *hashAcc) writeIRI(x string) {
+	ha.write("<")
+	ha.write(x)
+	ha.write("> ")
+}
+
+func (ha *hashAcc) finishLine() {
+	ha.write(".\n")
+}

xrb/xrb_test.go

@@ -1 +1,5 @@
 package xrb
+
+// TODO mock schema
+// TODO graph generator
+// verify graph can be serialized and deserialized and is equivalent and has same hash