~samwhited/mellium-devel

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch
1

[PATCH xmpp v3] styling: use an iterator pattern to make looping easier

Dane David
Details
Message ID
<20210609141803.64634-1-dndavid102@gmail.com>
DKIM signature
pass
Download raw message
Patch: +135 -84
Previously the Token method returned a token value & and error value, so looping over a message styling document required a lot of extra handling. To gain better usability, have switched to an iterator pattern by adding a Next method & altering the Token method. Fixes #128.

Signed-off-by: Dane David <dndavid102@gmail.com>
---
Changes after patch review.

 styling/example_test.go |  20 +++----
 styling/styling.go      | 118 +++++++++++++++++++++++++---------------
 styling/styling_test.go |  81 +++++++++++++++++----------
 3 files changed, 135 insertions(+), 84 deletions(-)

diff --git a/styling/example_test.go b/styling/example_test.go
index 1e2fbfc..46c1855 100644
--- a/styling/example_test.go
+++ b/styling/example_test.go
@@ -21,17 +21,8 @@ but *most* people shorten it.`)

	var out strings.Builder
	out.WriteString("<!doctype HTML>\n")
	for {
		tok, err := d.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			out.WriteString("<mark>")
			out.WriteString(html.EscapeString(fmt.Sprintf("Error encountered while parsing tokens: %v", err)))
			out.WriteString("</mark>")
			break
		}
	for d.Next() {
		tok := d.Token()

		switch {
		case tok.Mask&styling.SpanEmphStart == styling.SpanEmphStart:
@@ -55,6 +46,13 @@ but *most* people shorten it.`)
			out.WriteString(html.EscapeString(string(tok.Data)))
		}
	}

	err := d.Err()
	if err != nil && err != io.EOF {
		out.WriteString("<mark>")
		out.WriteString(html.EscapeString(fmt.Sprintf("Error encountered while parsing tokens: %v", err)))
		out.WriteString("</mark>")
	}
	fmt.Println(out.String())

	// Output:
diff --git a/styling/styling.go b/styling/styling.go
index 0813462..87ef16b 100644
--- a/styling/styling.go
+++ b/styling/styling.go
@@ -159,15 +159,17 @@ func Scan() bufio.SplitFunc {
// A Decoder represents a styling lexer reading a particular input stream.
// The parser assumes that input is encoded in UTF-8.
type Decoder struct {
	s             *bufio.Scanner
	clearMask     Style
	mask          Style
	quoteSplit    *Decoder
	quoteStarted  bool
	lastNewline   bool
	hasRun        bool
	spanStack     []byte
	bufferedToken *Token
	s                *bufio.Scanner
	clearMask        Style
	mask             Style
	quoteSplit       *Decoder
	quoteStarted     bool
	lastNewline      bool
	hasRun           bool
	spanStack        []byte
	insertBlockClose bool
	bufferedToken    *Token
	lastError        error
}

// NewDecoder creates a new styling parser reading from r.
@@ -181,7 +183,6 @@ func NewDecoder(r io.Reader) *Decoder {
}

// Token returns the next styling token in the input stream.
// At the end of the input stream, it returns an empty token and io.EOF.
// Returned tokens do not always correspond directly to the input stream.
// For example, at the end of a block quote an empty token is returned with the
// mask BlockQuote|BlockQuoteEnd, but there is no explicit block quote
@@ -190,22 +191,39 @@ func NewDecoder(r io.Reader) *Decoder {
// Slices of bytes in the returned token data refer to the parser's internal
// buffer and remain valid only until the next call to Token.
// To acquire a copy of the bytes call the token's Copy method.
func (d *Decoder) Token() (Token, error) {
	if d.bufferedToken != nil {
		t := *d.bufferedToken
		d.bufferedToken = nil
		return t, nil
func (d *Decoder) Token() Token {
	ret := d.bufferedToken
	d.bufferedToken = nil
	return *ret
}

// Next prepares the next styling token in the input stream for reading with
// the Token method.
// It returns true on success, or false if end of input stream is reached
// or an error happened while preparing it.
// Err should be consulted to distinguish between the two cases.
// Every call to Token, even the first one, must be preceded by a call to Next.
func (d *Decoder) Next() bool {
	if d.insertBlockClose {
		d.insertBlockClose = false
		d.bufferedToken = &Token{
			Mask: d.Style(),
			Data: d.s.Bytes(),
		}
		return true
	}

	prevLevel := d.Quote()
	if s := d.s.Scan(); !s {
		if err := d.s.Err(); err != nil {
			return Token{}, err
			d.lastError = err
			return false
		}
		return Token{}, io.EOF
		d.lastError = io.EOF
		return false
	}

	t := Token{
	t := &Token{
		Mask: d.Style(),
		Data: d.s.Bytes(),
	}
@@ -222,66 +240,76 @@ func (d *Decoder) Token() (Token, error) {
	// at the end of the quote since block quotes have no explicit terminator.
	currLevel := d.Quote()
	if currLevel < prevLevel {
		d.bufferedToken = &t
		return Token{Mask: d.Style()}, nil
		d.insertBlockClose = true
		d.bufferedToken = &Token{Mask: d.Style()}
		return true
	}

	return t, nil
	d.bufferedToken = t
	return true
}

// Err returns the error, if any, that was encountered while reading the input stream.
// In case of EOF, Err returns io.EOF
func (d *Decoder) Err() error {
	return d.lastError
}

// SkipSpan pops tokens from the decoder until it reaches the end of the current
// span, positioning the token stream at the beginning of the next span or
// block.
// If SkipSpan is called while no span is entered it behaves like SkipBlock.
// It returns any errors it encounters along the way.
func (d *Decoder) SkipSpan() error {
	for {
		prevLevel := d.Quote()
		tok, err := d.Token()
		if err != nil {
			return err
		}
// It returns true if it succeeds without errors, false otherwise
// Err should be consulted to check for errors.
func (d *Decoder) SkipSpan() bool {
	for prevLevel := d.Quote(); d.Next(); prevLevel = d.Quote() {
		tok := d.Token()

		switch {
		case tok.Mask&StartDirective&^BlockQuoteStart > 0 || (tok.Mask&BlockQuoteStart == BlockQuoteStart && d.Quote() > prevLevel):
			if err := d.SkipSpan(); err != nil {
				return err
			if !d.SkipSpan() {
				return false
			}
		case tok.Mask&EndDirective > 0 || (tok.Mask == 0 && d.lastNewline):
			return nil
			return true
		}
	}
	if err := d.Err(); err != nil {
		return false
	}
	return true
}

// SkipBlock pops tokens from the decoder until it reaches the end of the
// current block, positioning the token stream at the beginning of the next
// block.
// It returns any errors it encounters along the way.
// It returns true if it succeeds without errors, false otherwise
// Err should be consulted to check for errors.
// If SkipBlock is called at the beginning of the input stream before any tokens
// have been popped or any blocks have been entered, it will skip the entire
// input stream as if everything were contained in an imaginary "root" block.
func (d *Decoder) SkipBlock() error {
	for {
		prevLevel := d.Quote()
		tok, err := d.Token()
		if err != nil {
			return err
		}
func (d *Decoder) SkipBlock() bool {
	for prevLevel := d.Quote(); d.Next(); prevLevel = d.Quote() {
		tok := d.Token()
		switch {
		case tok.Mask&BlockStartDirective&^BlockQuoteStart > 0 || (tok.Mask&BlockQuoteStart == BlockQuoteStart && d.Quote() > prevLevel):
			// If we're a start directive (other than starting a block quote), or
			// we're a block quote start directive that's deeper than the previous
			// level of block quote (ie. starting a new blockquote, not continuing an
			// existing one), recurse down into the inner block, skipping tokens.
			if err := d.SkipBlock(); err != nil {
				return err
			if !d.SkipBlock() {
				return false
			}
		case tok.Mask&BlockEndDirective > 0 || (tok.Mask == 0 && d.lastNewline):
			// If this is an end directive (or the end of a plain block), we're done
			// with skipping the current level, so end the current level of recursion.
			return nil
			return true
		}
	}
	if err := d.Err(); err != nil {
		return false
	}
	return true
}

func (d *Decoder) scan(data []byte, atEOF bool) (advance int, token []byte, err error) {
@@ -377,7 +405,7 @@ func (d *Decoder) Quote() uint {
	}
	// If the next token is a virtual block quote end token, pretend we haven't
	// dropped down a level yet.
	if d.bufferedToken != nil {
	if d.insertBlockClose {
		level++
	}
	return level
@@ -386,7 +414,7 @@ func (d *Decoder) Quote() uint {
// Style returns a bitmask representing the currently applied styles at the
// current position in the document.
func (d *Decoder) Style() Style {
	if d.bufferedToken != nil {
	if d.insertBlockClose {
		return BlockQuoteEnd | BlockQuote
	}
	if d.hasRun {
diff --git a/styling/styling_test.go b/styling/styling_test.go
index b76709c..126059c 100644
--- a/styling/styling_test.go
+++ b/styling/styling_test.go
@@ -711,30 +711,11 @@ func TestToken(t *testing.T) {
			r := strings.NewReader(tc.input)
			d := styling.NewDecoder(r)
			var n int
			for {
				tok, err := d.Token()
				switch err {
				case nil:
				case io.EOF:
					if tc.Err != nil {
						t.Errorf("Expected error but got io.EOF: %v", err)
					}
					if len(toks) != 0 {
						var tokStrs []string
						for _, tok := range toks {
							tokStrs = append(tokStrs, string(tok.Data))
						}
						t.Fatalf("Reached EOF at token %d, but expected remaining tokens: %+v (%v)", n, tc.toks, tokStrs)
					}
					return
				default:
					if tc.Err != err {
						t.Errorf("Unexpected error: want=%v, got=%v", tc.Err, err)
					}
				}
			for d.Next() {
				tok := d.Token()

				if len(toks) == 0 {
					t.Fatalf("Did not expect more tokens but got %+v (%q), %v", tok, tok.Data, err)
					t.Fatalf("Did not expect more tokens but got %+v (%q)", tok, tok.Data)
				}

				var expectedTok tokenAndStyle
@@ -756,6 +737,28 @@ func TestToken(t *testing.T) {
				}
				n++
			}

			err := d.Err()

			switch err {
			case nil:
			case io.EOF:
				if tc.Err != nil && tc.Err != io.EOF {
					t.Errorf("Expected error but got io.EOF: %v", tc.Err)
				}
				if len(toks) != 0 {
					var tokStrs []string
					for _, tok := range toks {
						tokStrs = append(tokStrs, string(tok.Data))
					}
					t.Fatalf("Reached EOF at token %d, but expected remaining tokens: %+v (%v)", n, tc.toks, tokStrs)
				}
				return
			default:
				if tc.Err != err {
					t.Errorf("Unexpected error: want=%v, got=%v", tc.Err, err)
				}
			}
		})
	}
}
@@ -802,7 +805,11 @@ func (EOFRead) Read(b []byte) (int, error) {

func TestEOFPre(t *testing.T) {
	d := styling.NewDecoder(EOFRead{})
	tok, err := d.Token()
	var tok styling.Token
	if d.Next() {
		tok = d.Token()
	}
	err := d.Err()
	if err != nil {
		t.Fatal(err)
	}
@@ -821,7 +828,10 @@ func (ErrRead) Read([]byte) (int, error) {

func TestDecodeErr(t *testing.T) {
	d := styling.NewDecoder(ErrRead{})
	_, err := d.Token()
	if d.Next() {
		t.Fatal("Next() should return false, got true")
	}
	err := d.Err()
	if err != errTesting {
		t.Errorf("Want error passed through, got %v", err)
	}
@@ -862,24 +872,39 @@ func TestSkip(t *testing.T) {
			r := strings.NewReader(tc.input)
			d := styling.NewDecoder(r)
			for i := 0; i < tc.pop; i++ {
				_, err := d.Token()
				if d.Next() {
					d.Token()
				}
				err := d.Err()
				if err != nil {
					t.Fatalf("error at token %d: %v", i, err)
				}
			}
			var err error
			var skipSuccess bool
			if tc.span {
				err = d.SkipSpan()
				skipSuccess = d.SkipSpan()
			} else {
				err = d.SkipBlock()
				skipSuccess = d.SkipBlock()
			}
			err = d.Err()
			if err != tc.err {
				t.Errorf("final error does not match: want=%v, got=%v", tc.err, err)
			}
			if err != nil {
				if skipSuccess {
					t.Errorf("expected return value of skip function to be false")
				}
				return
			}
			tok, err := d.Token()
			if !skipSuccess {
				t.Errorf("expected return value of skip function to be true")
			}
			var tok styling.Token
			if d.Next() {
				tok = d.Token()
			}
			err = d.Err()
			if err != nil {
				t.Fatalf("error on expected token: %v", err)
			}
-- 
2.30.1 (Apple Git-130)

[xmpp/patches] build failed

builds.sr.ht
Details
Message ID
<CBZ5YCYCNAPE.3GDY5KB3SANXG@cirno>
In-Reply-To
<20210609141803.64634-1-dndavid102@gmail.com> (view parent)
DKIM signature
missing
Download raw message
xmpp/patches: FAILED in 21m29s

[styling: use an iterator pattern to make looping easier][0] v3 from [Dane David][1]

[0]: https://lists.sr.ht/~samwhited/mellium-devel/patches/23223
[1]: mailto:dndavid102@gmail.com

✗ #522545 FAILED  xmpp/patches/validate.yml    https://builds.sr.ht/~samwhited/job/522545
✗ #522543 FAILED  xmpp/patches/sync.yml        https://builds.sr.ht/~samwhited/job/522543
✓ #522544 SUCCESS xmpp/patches/test.yml        https://builds.sr.ht/~samwhited/job/522544
✗ #522542 FAILED  xmpp/patches/integration.yml https://builds.sr.ht/~samwhited/job/522542
Reply to thread Export thread (mbox)