~whereswaldon/arbor-dev

forest-go: Add metadata handling v3 APPLIED

Andrew Thorp: 2
 Add metadata handling
 Use twig key parser

 6 files changed, 120 insertions(+), 31 deletions(-)
Thanks, this is applied! I appreciate your patience on the
multiple revisions.
That’s what code review is for ;) 

Sent from my iPhone
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~whereswaldon/arbor-dev/patches/11552/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH forest-go v3 1/2] Add metadata handling Export this patch

Add twig read/write functionality to node creation

Address feedback

Fix struct accessibility

Address some feedback

Use valid key string

Add Contains and Get functions to sprig data

Use <key>/<version> to add metadata
Update docs

Do propper string splitting and update sanity check
---
 cmd/forest/main.go         | 105 +++++++++++++++++++++++++++++--------
 cmd/forest/sanity-check.sh |   2 +
 hashable.go                |   2 +-
 twig/twig.go               |  22 ++++++++
 twig/twig_test.go          |  10 ++++
 5 files changed, 117 insertions(+), 24 deletions(-)

diff --git a/cmd/forest/main.go b/cmd/forest/main.go
index 21f7171..bb2e7df 100644
--- a/cmd/forest/main.go
+++ b/cmd/forest/main.go
@@ -8,9 +8,12 @@ import (
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
	"git.sr.ht/~whereswaldon/forest-go/twig"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/packet"
)
@@ -106,37 +109,82 @@ func create(args []string) error {
	return nil
}

type Metadata struct {
	Version uint   `json:"version" `
	Data    string `json:"data" `
}

func decodeMetadata(input string) ([]byte, error) {

	var metadata map[string]string
	err := json.Unmarshal([]byte(input), &metadata)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshalling metadata: %v", err)
	}

	var twigData = twig.New()

	for k, d := range metadata {
		key := strings.Split(k, "/")
		n := key[0]
		v, err := strconv.Atoi(key[1])
		if err != nil {
			return nil, fmt.Errorf("Error converting version to int: %w", err)
		}

		_, err = twigData.Set(n, uint(v), []byte(d))
		if err != nil {
			return nil, fmt.Errorf("Error adding twig data: %v", err)
		}
	}

	mdBlob, err := twigData.MarshalBinary()
	if err != nil {
		return nil, fmt.Errorf("Error marshalling data to binary: %v", err)
	}

	return mdBlob, nil
}

func createIdentity(args []string) error {
	var (
		name, keyfile, gpguser string
		name, keyfile, gpguser, metadata string
	)
	flags := flag.NewFlagSet(commandCreate+" "+commandIdentity, flag.ExitOnError)
	flags.StringVar(&name, "name", "forest", "username for the identity node")
	flags.StringVar(&keyfile, "key", "arbor.privkey", "the openpgp private key for the identity node")
	flags.StringVar(&gpguser, "gpguser", "", "gpg2 user whose private key should be used to create this node. Supercedes -key.")
	flags.StringVar(&metadata, "metadata", "{}", "Twig metadata fields for the node: {\"<key>/<version>\": \"data\",...}")

	usage := func() {
		flags.PrintDefaults()
	}
	if err := flags.Parse(args); err != nil {
		usage()
		return err
		return fmt.Errorf("Error parsing arguments: %v", err)
	}
	signer, err := getSigner(gpguser, keyfile)
	if err != nil {
		return err
		return fmt.Errorf("Error getting signer: %v", err)
	}

	metadataRaw, err := decodeMetadata(metadata)
	if err != nil {
		return fmt.Errorf("Error decoding metadata: %v", err)
	}
	identity, err := forest.NewIdentity(signer, name, []byte{})

	identity, err := forest.NewIdentity(signer, name, metadataRaw)
	if err != nil {
		return err
		return fmt.Errorf("Error creating identity: %v", err)
	}

	fname, err := identity.ID().MarshalString()
	if err != nil {
		return err
		return fmt.Errorf("Error marshalling identity: %v", err)
	}

	if err := saveAs(fname, identity); err != nil {
		return err
		return fmt.Errorf("Error saving identity: %v", err)
	}

	fmt.Println(fname)
@@ -146,41 +194,46 @@ func createIdentity(args []string) error {

func createCommunity(args []string) error {
	var (
		name, keyfile, identity, gpguser string
		name, keyfile, identity, gpguser, metadata string
	)
	flags := flag.NewFlagSet(commandCreate+" "+commandCommunity, flag.ExitOnError)
	flags.StringVar(&name, "name", "forest", "username for the community node")
	flags.StringVar(&keyfile, "key", "arbor.privkey", "the openpgp private key for the signing identity node")
	flags.StringVar(&identity, "as", "", "[required] the id of the signing identity node")
	flags.StringVar(&gpguser, "gpguser", "", "gpg2 user whose private key should be used to create this node. Supercedes -key.")
	flags.StringVar(&metadata, "metadata", "{}", "Twig metadata fields for the node: {\"<key>/<version>\": \"data\",...}")
	usage := func() {
		flags.PrintDefaults()
	}
	if err := flags.Parse(args); err != nil {
		usage()
		return err
		return fmt.Errorf("Error parsing arguments: %v", err)
	}
	signer, err := getSigner(gpguser, keyfile)
	if err != nil {
		return err
		return fmt.Errorf("Error getting signer: %v", err)
	}
	idNode, err := getIdentity(identity)
	if err != nil {
		return err
		return fmt.Errorf("Error gettig identity: %v", err)
	}
	metadataRaw, err := decodeMetadata(metadata)
	if err != nil {
		return fmt.Errorf("Error decoding metadata: %v", err)
	}

	community, err := forest.As(idNode, signer).NewCommunity(name, []byte{})
	community, err := forest.As(idNode, signer).NewCommunity(name, metadataRaw)
	if err != nil {
		return err
		return fmt.Errorf("Error creating community: %v", err)
	}

	fname, err := community.ID().MarshalString()
	if err != nil {
		return err
		return fmt.Errorf("Error marshalling community: %v", err)
	}

	if err := saveAs(fname, community); err != nil {
		return err
		return fmt.Errorf("Error saving community: %v", err)
	}

	fmt.Println(fname)
@@ -190,7 +243,7 @@ func createCommunity(args []string) error {

func createReply(args []string) error {
	var (
		content, parent, keyfile, identity, gpguser string
		content, parent, keyfile, identity, gpguser, metadata string
	)
	flags := flag.NewFlagSet(commandCreate+" "+commandReply, flag.ExitOnError)
	flags.StringVar(&keyfile, "key", "arbor.privkey", "the openpgp private key for the signing identity node")
@@ -198,6 +251,7 @@ func createReply(args []string) error {
	flags.StringVar(&identity, "as", "", "[required] the id of the signing identity node")
	flags.StringVar(&parent, "to", "", "[required] the id of the parent reply or community node")
	flags.StringVar(&content, "content", "", "[required] content of the reply node")
	flags.StringVar(&metadata, "metadata", "{}", "Twig metadata fields for the node: {\"<key>/<version>\": \"data\",...}")

	usage := func() {
		flags.PrintDefaults()
@@ -209,30 +263,35 @@ func createReply(args []string) error {

	signer, err := getSigner(gpguser, keyfile)
	if err != nil {
		return err
		return fmt.Errorf("Error getting signer: %v", err)
	}
	idNode, err := getIdentity(identity)
	if err != nil {
		return err
		return fmt.Errorf("Error getting Identity: %v", err)
	}

	parentNode, err := getReplyOrCommunity(parent)
	if err != nil {
		return err
		return fmt.Errorf("Error getting Reply/Community: %v", err)
	}

	reply, err := forest.As(idNode, signer).NewReply(parentNode, content, []byte{})
	metadataRaw, err := decodeMetadata(metadata)
	if err != nil {
		return err
		return fmt.Errorf("Error decoding metadata: %v", err)
	}

	reply, err := forest.As(idNode, signer).NewReply(parentNode, content, metadataRaw)
	if err != nil {
		return fmt.Errorf("Error during creating new reply: %v", err)
	}

	fname, err := reply.ID().MarshalString()
	if err != nil {
		return err
		return fmt.Errorf("Error marshalling reply.ID: %v", err)
	}

	if err := saveAs(fname, reply); err != nil {
		return err
		return fmt.Errorf("Error saving reply: %v", err)
	}

	fmt.Println(fname)
diff --git a/cmd/forest/sanity-check.sh b/cmd/forest/sanity-check.sh
index f9cb13c..cb4c44e 100755
--- a/cmd/forest/sanity-check.sh
+++ b/cmd/forest/sanity-check.sh
@@ -13,8 +13,10 @@ identity=$("$forest_cmd" create identity)
community=$("$forest_cmd" create community --as "$identity")
reply1=$("$forest_cmd" create reply --as "$identity" --to "$community" --content test1)
reply2=$("$forest_cmd" create reply --as "$identity" --to "$reply1" --content test2)
reply3=$("$forest_cmd" create reply --as "$identity" --to "$reply1" --content "this node has metadata" --metadata '{"meta/1": "some-value" }')

"$forest_cmd" show "$identity"
"$forest_cmd" show "$community"
"$forest_cmd" show "$reply1"
"$forest_cmd" show "$reply2"
"$forest_cmd" show "$reply3"
diff --git a/hashable.go b/hashable.go
index 307e120..2cfc7cf 100644
--- a/hashable.go
+++ b/hashable.go
@@ -55,7 +55,7 @@ func ValidateID(h Hashable, expected fields.QualifiedHash) (bool, error) {
	}
	computedID := fields.QualifiedHash{
		Descriptor: *h.HashDescriptor(),
		Blob:      fields.Blob(id),
		Blob:       fields.Blob(id),
	}
	return expected.Equals(&computedID), nil
}
diff --git a/twig/twig.go b/twig/twig.go
index d96ff6f..d600d76 100644
--- a/twig/twig.go
+++ b/twig/twig.go
@@ -80,6 +80,25 @@ func New() *Data {
	return &Data{Values: make(map[Key][]byte)}
}

// Set sets a twig key-version data entry. If the entry does not exist, it is created
func (d *Data) Set(name string, version uint, value []byte) (*Data, error) {
	d.Values[Key{Name: name, Version: version}] = value
	return d, nil
}

// Get fetches a value from the value store by key name and version, and whether or
// not the key was in the values
func (d *Data) Get(name string, version uint) ([]byte, bool) {
	data, inValues := d.Values[Key{Name: name, Version: version}]
	return data, inValues
}

// Contains checks whether or not a key exists in the data values by name and version
func (d *Data) Contains(name string, version uint) bool {
	_, inValues := d.Get(name, version)
	return inValues
}

// UnmarshalBinary populates a Data from raw binary in Twig format
func (d *Data) UnmarshalBinary(b []byte) error {
	if len(b) == 0 {
@@ -101,6 +120,9 @@ func (d *Data) UnmarshalBinary(b []byte) error {

// MarshalBinary converts this Data into twig binary form.
func (d *Data) MarshalBinary() ([]byte, error) {
	if len(d.Values) == 0 {
		return []byte{}, nil
	}
	buf := new(bytes.Buffer)
	for key, value := range d.Values {
		// gotta check here because the Values map is exported and could be
diff --git a/twig/twig_test.go b/twig/twig_test.go
index a01513f..e0c3841 100644
--- a/twig/twig_test.go
+++ b/twig/twig_test.go
@@ -80,3 +80,13 @@ func TestDataMarshalBadKey(t *testing.T) {
		t.Fatalf("Should have returned nil slice when failing to marshal")
	}
}

func TestDataMarshalNoBytes(t *testing.T) {
	data := twig.New()
	asBin, err := data.MarshalBinary()
	if err != nil {
		t.Fatalf("Should not error on marshallign empty twig store: %v", nil)
	} else if len(asBin) != 0 {
		t.Fatalf("Empty data store should return an empty.")
	}
}
-- 
2.17.1

[PATCH forest-go v3 2/2] Use twig key parser Export this patch

---
 cmd/forest/main.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/cmd/forest/main.go b/cmd/forest/main.go
index bb2e7df..1b57c50 100644
--- a/cmd/forest/main.go
+++ b/cmd/forest/main.go
@@ -8,8 +8,6 @@ import (
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
@@ -125,14 +123,12 @@ func decodeMetadata(input string) ([]byte, error) {
	var twigData = twig.New()

	for k, d := range metadata {
		key := strings.Split(k, "/")
		n := key[0]
		v, err := strconv.Atoi(key[1])
		key, err := twig.FromString(k)
		if err != nil {
			return nil, fmt.Errorf("Error converting version to int: %w", err)
			return nil, fmt.Errorf("Error parsing twig key: %w", err)
		}

		_, err = twigData.Set(n, uint(v), []byte(d))
		twigData.Values[key] = []byte(d)
		if err != nil {
			return nil, fmt.Errorf("Error adding twig data: %v", err)
		}
-- 
2.17.1
Thanks, this is applied! I appreciate your patience on the
multiple revisions.