~rockorager/offmap

offmap: Updated sync and cancellation v1 APPLIED

This series does a few things, but first off it is based on the account patch
set, if you are testing this out please apply that one first:

https://lists.sr.ht/~rockorager/offmap/patches/37388

The build will also fail because of this.

First, this series changes to fetching mail directly into the maildir. This is
done by sending fetched mail across a channel to maildir, via a function in the
Diff routine. Maildir still syncs to IMAP the same way as before, this should be
cleaned up in a subsequent patchset.

Second, the sync order is changed to facilitate better cancellation and
partial-sync state saves. The new order is:
1. Create and/or rename mailboxes
2. Sync email ops
3. (not implemented) Delete mailboxes

Third, the improved cancellation is implemented. If you are downloading files in
a large mailbox, it will take some time before offmap will close if you were to
try to cancel it. This will be improved in subsequent patches.

The combination of all of these is that: a partial sync, either due to loss of
network or user cancellation saves the current synced state, allowing for
picking up where you left off if you run the command again.

Tim Culverhouse (3):
  sync: fetch mail directly to maildir
  sync: sync by mailbox
  cancellation: close after current mailbox finishes sync

 cmd/offmap.go          |   3 +-
 cmd/sync.go            |  38 +++++--
 diff.go                |   6 ++
 email.go               |  26 ++++-
 imap/diff.go           |   2 +-
 imap/imap.go           | 133 ++++++++++-------------
 imap/update.go         | 232 +++++++++++++++++++++++------------------
 maildir/update.go      |  79 +++++++-------
 maildir/update_test.go |  22 ++--
 state.go               |   4 +
 10 files changed, 300 insertions(+), 245 deletions(-)

-- 
2.38.1
#900382 .build.yml success
offmap/patches/.build.yml: SUCCESS in 13m42s

[Updated sync and cancellation][0] from [Tim Culverhouse][1]

[0]: https://lists.sr.ht/~rockorager/offmap/patches/37404
[1]: mailto:tim@timculverhouse.com

✓ #900382 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/900382



          
          
          
        
      

      
      
      
      

      

      
      
      
      
      
      

      
      
        
          






Thanks! Applied.
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~rockorager/offmap/patches/37404/mbox | git am -3
Learn more about email & git

[PATCH offmap 1/3] sync: fetch mail directly to maildir Export this patch

Send mail body in a wrapper struct over a channel to the maildir. This
allows for cleaner partial syncs and cancellations.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
 diff.go                |   6 ++
 email.go               |  26 ++++++--
 imap/diff.go           |   2 +-
 imap/imap.go           | 133 ++++++++++++++++++-----------------------
 maildir/update.go      |  23 ++++---
 maildir/update_test.go |  22 ++++---
 6 files changed, 106 insertions(+), 106 deletions(-)

diff --git a/diff.go b/diff.go
index 39ffdb35a900..21c962d5275c 100644
--- a/diff.go
+++ b/diff.go
@@ -12,6 +12,8 @@ type Diff struct {
	// Emails is a map of EmailDiffs to mailbox name. The key will always be
	// in reference to the local name, ("/" as a delimiter)
	Emails map[string]*EmailDiff

	FetchFunc func(string, []*Email) chan *FullEmail
}

// NewDiff creates a new Diff object
@@ -22,6 +24,10 @@ func NewDiff() *Diff {
	}
}

func (d *Diff) Fetch(name string, emls []*Email) chan *FullEmail {
	return d.FetchFunc(name, emls)
}

type MailboxDiff struct {
	// Mailboxes created since last sync
	Created []*Mailbox
diff --git a/email.go b/email.go
index 7d9fdf50627d..419bf392d1fc 100644
--- a/email.go
+++ b/email.go
@@ -32,6 +32,24 @@ type Email struct {
	size int
}

type FullEmail struct {
	*Email
	Body io.Reader
	Size int
}

func (fe *FullEmail) Read(p []byte) (int, error) {
	return fe.Body.Read(p)
}

func (fe *FullEmail) SetSize(i int) {
	fe.size = i
}

func (fe *FullEmail) Len() int {
	return fe.size
}

// NewEmail creates a new Email object
func NewEmail(mbox *Mailbox, uid uint32, key string, flags []maildir.Flag) *Email {
	return &Email{
@@ -48,10 +66,10 @@ func (e *Email) SetFilename(path string) {
}

func (e *Email) Read(p []byte) (int, error) {
	if e.filename == "" {
		return 0, fmt.Errorf("email: no body")
	}
	if e.body == nil {
	// if e.filename == "" {
	// 	return 0, fmt.Errorf("email: no body")
	// }
	if e.body == nil && e.filename != "" {
		b, err := os.ReadFile(e.filename)
		if err != nil {
			return 0, fmt.Errorf("email: %v", err)
diff --git a/imap/diff.go b/imap/diff.go
index c7de4bd93889..79d8d2abb155 100644
--- a/imap/diff.go
+++ b/imap/diff.go
@@ -43,11 +43,11 @@ func (s *Store) Diff(cached []*offmap.Mailbox) (*offmap.Diff, error) {
	)
	var c, u, d int
	for _, emls := range diff.Emails {
		s.DownloadEmail(emls.Created)
		c = c + len(emls.Created)
		u = u + len(emls.FlagRemoved) + len(emls.FlagAdded)
		d = d + len(emls.Deleted)
	}
	diff.FetchFunc = s.DownloadEmail
	log.Debugf("imap: emails   : {%d created, %d updated, %d deleted}", c, u, d)
	log.Debugf("imap: calculated diff in %s", time.Since(start).Round(1*time.Millisecond))
	return diff, nil
diff --git a/imap/imap.go b/imap/imap.go
index dde579884347..228669db260e 100644
--- a/imap/imap.go
+++ b/imap/imap.go
@@ -2,7 +2,6 @@ package imap

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
@@ -152,90 +151,72 @@ func (s *Store) getCurrentState() ([]*offmap.Mailbox, error) {
	return mailboxes, nil
}

func (s *Store) DownloadEmail(emls []*offmap.Email) []*offmap.Email {
	// TODO redo this batching part, it's not needed like this anymore
	// Batch is a map of mailbox name to eml UID
	batch := map[string][]*offmap.Email{}
	emlMap := make(map[string]*offmap.Email, len(emls))
func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.FullEmail {
	mbox = strings.ReplaceAll(mbox, "/", s.Delimiter())
	emlMap := make(map[uint32]*offmap.Email, len(emls))
	for _, eml := range emls {
		_, ok := batch[eml.Mailbox.RemoteName]
		if !ok {
			// We haven't seen this mailbox yet
			batch[eml.Mailbox.RemoteName] = []*offmap.Email{}
		}
		id := fmt.Sprintf("%d.%d", eml.Mailbox.UIDValidity, eml.UID)
		emlMap[id] = eml
		batch[eml.Mailbox.RemoteName] = append(batch[eml.Mailbox.RemoteName], eml)
		emlMap[eml.UID] = eml
	}
	ch := make(chan *offmap.FullEmail)

	var dled int32 = 0
	total := len(emls)
	// Fetch the mail by mailbox
	wg := sync.WaitGroup{}
	emlMu := sync.Mutex{}
	for mbox, emls := range batch {
		seq := &imap.SeqSet{}
		for _, eml := range emls {
			seq.AddNum(eml.UID)
	seq := &imap.SeqSet{}
	for _, eml := range emls {
		seq.AddNum(eml.UID)
	}
	client := s.getConn()
	go func() {
		defer client.done()
		defer close(ch)
		if seq.Empty() {
			return
		}
		client := s.getConn()
		wg.Add(1)
		go func(client *conn, mbox string, seq *imap.SeqSet) {
			defer client.done()
			defer wg.Done()
			status, err := client.Select(mbox, true)
			if err != nil {
				log.Errorf("imap: %v", err)
				return
			}
			section := &imap.BodySectionName{Peek: true}
			items := []imap.FetchItem{section.FetchItem()}
			msgCh := make(chan *imap.Message)
			msgWg := sync.WaitGroup{}
			msgWg.Add(1)
			go func() {
				for msg := range msgCh {
					filename := fmt.Sprintf("%d.%d", status.UidValidity, msg.Uid)
					// TODO should we create this in the
					// maildir's tmp folder instead??
					filepath := path.Join(s.tmp, filename)
					f, err := os.Create(filepath)
					if err != nil {
						log.Errorf("imap: %v", err)
						continue
					}
					r := msg.GetBody(section)
					if r == nil {
						log.Errorf("imap: no body")
						continue
					}
					io.Copy(f, r) //nolint:errcheck // We don't care if it actually can't write at this point
					atomic.AddInt32(&dled, 1)
					cur := atomic.LoadInt32(&dled)
					f.Close()
					emlMu.Lock()
					eml, ok := emlMap[filename]
					emlMu.Unlock()
					if !ok {
						log.Errorf("imap: %s: eml not found: %s", status.Name, filename)
						continue
					}
					eml.SetFilename(filepath)
					log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
		status, err := client.Select(mbox, true)
		if err != nil {
			log.Errorf("imap: %v", err)
			return
		}
		section := &imap.BodySectionName{Peek: true}
		items := []imap.FetchItem{section.FetchItem()}
		msgCh := make(chan *imap.Message)
		msgWg := sync.WaitGroup{}
		msgWg.Add(1)
		go func() {
			for msg := range msgCh {
				atomic.AddInt32(&dled, 1)
				cur := atomic.LoadInt32(&dled)
				emlMu.Lock()
				eml, ok := emlMap[msg.Uid]
				emlMu.Unlock()
				if !ok {
					log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
					continue
				}
				msgWg.Done()
			}()
			log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
			err = client.UidFetch(seq, items, msgCh)
			if err != nil {
				log.Errorf("imap (client %d): %v", client.id, err)
				return
				r := msg.GetBody(section)
				if r == nil {
					log.Errorf("imap: no body")
					continue
				}
				fe := &offmap.FullEmail{
					Email: eml,
					Body:  r,
				}
				ch <- fe
				log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
			}
			msgWg.Wait()
		}(client, mbox, seq)
	}
	wg.Wait()
	return emls
			msgWg.Done()
		}()
		log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
		err = client.UidFetch(seq, items, msgCh)
		if err != nil {
			log.Errorf("imap (client %d): %v", client.id, err)
			return
		}
		msgWg.Wait()
	}()
	return ch
}

// listMailboxes fetches mailbox statuses from the IMAP server. If the server
diff --git a/maildir/update.go b/maildir/update.go
index 5fa55af67390..61c2783dccbb 100644
--- a/maildir/update.go
+++ b/maildir/update.go
@@ -2,6 +2,7 @@ package maildir

import (
	"fmt"
	"io"
	"os"
	"path"

@@ -43,7 +44,8 @@ func (s *Store) ApplyDiff(state *offmap.State, c *offmap.Diff) error {
	// Handle EmailDiff
	for mbox, emlDiff := range c.Emails {
		// Create emails
		err := s.createEmail(mbox, emlDiff.Created)
		ch := c.Fetch(mbox, emlDiff.Created)
		err := s.createEmail(mbox, ch)
		if err != nil {
			return fmt.Errorf("maildir: could not apply diff: %v", err)
		}
@@ -105,25 +107,20 @@ func (s *Store) renameMaildir(orig string, dest string) error {
}

// createEmail creates an email in the maildir
func (s *Store) createEmail(mbox string, emls []*offmap.Email) error {
func (s *Store) createEmail(mbox string, emls chan *offmap.FullEmail) error {
	dir := maildir.Dir(path.Join(s.root, mbox))
	for _, eml := range emls {
	for eml := range emls {
		key, wc, err := dir.Create(eml.Flags)
		if err != nil {
			return fmt.Errorf("maildir: could not create email: %v", err)
		}
		wc.Close()
		dest, err := dir.Filename(key)
		if err != nil {
			return fmt.Errorf("maildir: could not write email: %v", err)
			return fmt.Errorf("maildir: could not create email: %w", err)
		}
		err = os.Rename(eml.Filename(), dest)
		_, err = io.Copy(wc, eml)
		if err != nil {
			return fmt.Errorf("maildir: could not write email: %v", err)
			return fmt.Errorf("maildir: could not create email: %w", err)
		}
		eml.Key = key
		eml.Email.Key = key
		if s.state != nil {
			s.state.CreateEmail(mbox, eml)
			s.state.CreateEmail(mbox, eml.Email)
		}
	}
	return nil
diff --git a/maildir/update_test.go b/maildir/update_test.go
index 4d6c1e5a3a06..f7f8e2caa016 100644
--- a/maildir/update_test.go
+++ b/maildir/update_test.go
@@ -1,8 +1,7 @@
package maildir

import (
	"os"
	"path"
	"bytes"
	"testing"

	"git.sr.ht/~rockorager/offmap"
@@ -71,20 +70,19 @@ func TestCreateEmail(t *testing.T) {
	mailbox := mboxes["Inbox"]
	eml := offmap.NewEmail(mailbox, 0, "key", []maildir.Flag{'S'})

	tmpEml := path.Join(t.TempDir(), "tmpEmail")
	f, err := os.Create(tmpEml)
	if err != nil {
		t.Fatal(err)
	}
	_, _ = f.WriteString("email body")
	f.Close()
	eml.SetFilename(tmpEml)

	mboxes, err = store.readCurrentState()
	assert.NoError(t, err)

	assert.Equal(t, 100, len(mboxes["Inbox"].Emails))
	err = store.createEmail("Inbox", []*offmap.Email{eml})
	ch := make(chan *offmap.FullEmail)
	go func() {
		ch <- &offmap.FullEmail{
			Email: eml,
			Body:  bytes.NewBuffer([]byte("email body")),
		}
		close(ch)
	}()
	err = store.createEmail("Inbox", ch)
	assert.NoError(t, err)
	mboxes, err = store.readCurrentState()
	assert.NoError(t, err)
-- 
2.38.1

[PATCH offmap 2/3] sync: sync by mailbox Export this patch

Perform sync operations in a slightly different order:
1. Create and/or rename mailboxes on both sides
2. Loop through all mailboxes and apply local then remote changes

This enables better cancellation (future patches coming), by fully
syncing a mailbox pair before moving on to the next one. In the case of
CONDSTORE, this has significant advantages.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
 cmd/sync.go       |  27 +++++-
 imap/update.go    | 232 ++++++++++++++++++++++++++--------------------
 maildir/update.go |  60 ++++++------
 state.go          |   4 +
 4 files changed, 189 insertions(+), 134 deletions(-)

diff --git a/cmd/sync.go b/cmd/sync.go
index ddd444486001..f537572de0d1 100644
--- a/cmd/sync.go
+++ b/cmd/sync.go
@@ -74,8 +74,8 @@ func sync(cmd *cobra.Command, args []string) {
			log.Infof("offmap: sync aborted, state saved")
		}()

		// Apply local changes to remote
		err = remote.ApplyDiff(state, localDiff)
		// Do mailboxes first
		err = remote.ApplyMailboxDiff(state, localDiff)
		if err != nil {
			// Save any changes we've made. log.Fatalf calls os.Exit(1).
			// Defers are bypassed with this call
@@ -86,7 +86,7 @@ func sync(cmd *cobra.Command, args []string) {
		}

		// Apply remote changes to local
		err = local.ApplyDiff(state, remoteDiff)
		err = local.ApplyMailboxDiff(state, remoteDiff)
		if err != nil {
			// Save any changes we've made. log.Fatalf calls os.Exit(1).
			// Defers are bypassed with this call
@@ -96,6 +96,27 @@ func sync(cmd *cobra.Command, args []string) {
			log.Fatalf("%v", err)
		}

		// Now the state has an up to date picture of mailboxes. Sync
		// each mailbox, if it has changes
		for _, mbox := range state.Mailboxes {
			ok1, err := remote.ApplyEmailDiff(state, mbox.RemoteName, localDiff)
			if err != nil {
				log.Errorf("%v", err)
				continue
			}
			ok2, err := local.ApplyEmailDiff(state, mbox.LocalName, remoteDiff)
			if err != nil {
				log.Errorf("%v", err)
				continue
			}
			if ok1 || ok2 {
				err = remote.UpdateMailboxStatus(state, mbox.LocalName)
				if err != nil {
					log.Errorf("%v", err)
				}
			}
		}

		// Save and clean up
		if err := state.Save(); err != nil {
			log.Errorf("offmap: could not save state: %v", err)
diff --git a/imap/update.go b/imap/update.go
index a43d5427b6e5..e3a7dd03e7b6 100644
--- a/imap/update.go
+++ b/imap/update.go
@@ -6,14 +6,13 @@ import (
	"time"

	"git.sr.ht/~rockorager/offmap"
	"git.sr.ht/~rockorager/offmap/imap/condstore"
	"github.com/emersion/go-imap"
	uidplus "github.com/emersion/go-imap-uidplus"
	"github.com/emersion/go-maildir"
)

// ApplyDiff applies the changes from offmap, updates the internal state, and saves
// the new state to disk.
func (s *Store) ApplyDiff(state *offmap.State, c *offmap.Diff) error {
func (s *Store) ApplyMailboxDiff(state *offmap.State, c *offmap.Diff) error {
	// Apply changes
	for _, mbox := range c.Mailboxes.Created {
		conn := s.getConn()
@@ -49,118 +48,147 @@ func (s *Store) ApplyDiff(state *offmap.State, c *offmap.Diff) error {
		// Update state
		state.RenameMailbox(orig, dest)
	}
	return nil
}

	// Emails
func (s *Store) ApplyEmailDiff(state *offmap.State, remote string, c *offmap.Diff) (bool, error) {
	emlDiff, ok := c.Emails[remote]
	if !ok {
		// no changes
		return ok, nil
	}
	// IMAP diffs always have all mailboxes, reset ok to false and only set
	// it to true if we end up in one of the loops
	ok = false
	local := strings.ReplaceAll(remote, s.Delimiter(), "/")
	conn := s.getConn()
	defer conn.done()
	for local, emlDiff := range c.Emails {
		// Create emails
		remote := strings.ReplaceAll(local, "/", s.Delimiter())
		// TODO Multiappend?
		for _, eml := range emlDiff.Created {
			_, err := conn.Select(remote, false)
			// TODO handle the error
			if err != nil {
				return fmt.Errorf("imap/create email: %v", err)
			}
			client := uidplus.NewClient(conn.Client)
			flags := flagMaildirToIMAP(eml.Flags)
			if eml.Date.IsZero() {
				eml.Date = time.Now()
			}
			_, uid, err := client.Append(remote, flags, eml.Date, eml)
			if err != nil {
				return fmt.Errorf("imap: couldn't create email: %v", err)
			}
			eml.UID = uid
			// Update state
			state.CreateEmail(local, eml)
	// Create emails
	// TODO Multiappend?
	for _, eml := range emlDiff.Created {
		ok = true
		_, err := conn.Select(remote, false)
		// TODO handle the error
		if err != nil {
			return ok, fmt.Errorf("imap/create email: %v", err)
		}

		// add flags
		for flag, emls := range emlDiff.FlagAdded {
			_, err := conn.Select(remote, false)
			// TODO handle the error
			if err != nil {
				return fmt.Errorf("imap/add flags: %v", err)
			}
			flags := flagMaildirToIMAP([]maildir.Flag{flag})
			// Take the second part of the ID
			seq := &imap.SeqSet{}
			for _, e := range emls {
				seq.AddNum(e.UID)
			}
			val := []interface{}{}
			for _, flag := range flags {
				val = append(val, flag)
			}
			err = conn.UidStore(seq, imap.AddFlags, val, nil)
			if err != nil {
				return fmt.Errorf("imap/add flags: %v", err)
			}
			// Update state
			for _, eml := range emls {
				state.AddFlag(local, flag, eml)
			}
		client := uidplus.NewClient(conn.Client)
		flags := flagMaildirToIMAP(eml.Flags)
		if eml.Date.IsZero() {
			eml.Date = time.Now()
		}
		_, uid, err := client.Append(remote, flags, eml.Date, eml)
		if err != nil {
			return ok, fmt.Errorf("imap: couldn't create email: %v", err)
		}
		eml.UID = uid
		// Update state
		state.CreateEmail(local, eml)
	}

		// Remove flags
		for flag, emls := range emlDiff.FlagRemoved {
			_, err := conn.Select(remote, false)
			if err != nil {
				return fmt.Errorf("imap/remove flags: %v", err)
			}
			flags := flagMaildirToIMAP([]maildir.Flag{flag})
			seq := &imap.SeqSet{}
			for _, e := range emls {
				seq.AddNum(e.UID)
			}
			val := []interface{}{}
			for _, flag := range flags {
				val = append(val, flag)
			}
			err = conn.UidStore(seq, imap.RemoveFlags, val, nil)
			if err != nil {
				return fmt.Errorf("imap/removing flags: %v", err)
			}
			// Update state
			for _, eml := range emls {
				state.RemoveFlag(local, flag, eml)
			}
	// add flags
	for flag, emls := range emlDiff.FlagAdded {
		ok = true
		_, err := conn.Select(remote, false)
		// TODO handle the error
		if err != nil {
			return ok, fmt.Errorf("imap/add flags: %v", err)
		}
		flags := flagMaildirToIMAP([]maildir.Flag{flag})
		// Take the second part of the ID
		seq := &imap.SeqSet{}
		for _, e := range emls {
			seq.AddNum(e.UID)
		}
		val := []interface{}{}
		for _, flag := range flags {
			val = append(val, flag)
		}
		err = conn.UidStore(seq, imap.AddFlags, val, nil)
		if err != nil {
			return ok, fmt.Errorf("imap/add flags: %v", err)
		}
		// Update state
		for _, eml := range emls {
			state.AddFlag(local, flag, eml)
		}
	}

		// Delete emails
		if len(emlDiff.Deleted) > 0 {
			_, err := conn.Select(remote, false)
			if err != nil {
				return fmt.Errorf("imap/expunge: %v", err)
			}
			seq := &imap.SeqSet{}
			for _, eml := range emlDiff.Deleted {
				seq.AddNum(eml.UID)
			}
			val := []interface{}{imap.DeletedFlag}
			err = conn.UidStore(seq, imap.AddFlags, val, nil)
			if err != nil {
				return fmt.Errorf("imap/expunge: %v", err)
			}
			err = conn.Expunge(nil)
			if err != nil {
				return fmt.Errorf("imap/expunge: %v", err)
			}
			// Update state
			for _, eml := range emlDiff.Deleted {
				state.DeleteEmail(local, eml)
			}
	// Remove flags
	for flag, emls := range emlDiff.FlagRemoved {
		ok = true
		_, err := conn.Select(remote, false)
		if err != nil {
			return ok, fmt.Errorf("imap/remove flags: %v", err)
		}
		flags := flagMaildirToIMAP([]maildir.Flag{flag})
		seq := &imap.SeqSet{}
		for _, e := range emls {
			seq.AddNum(e.UID)
		}
		val := []interface{}{}
		for _, flag := range flags {
			val = append(val, flag)
		}
		err = conn.UidStore(seq, imap.RemoveFlags, val, nil)
		if err != nil {
			return ok, fmt.Errorf("imap/removing flags: %v", err)
		}
		// Update state
		for _, eml := range emls {
			state.RemoveFlag(local, flag, eml)
		}
	}

	// TODO delete mailboxes
	// Delete emails
	if len(emlDiff.Deleted) > 0 {
		ok = true
		_, err := conn.Select(remote, false)
		if err != nil {
			return ok, fmt.Errorf("imap/expunge: %v", err)
		}
		seq := &imap.SeqSet{}
		for _, eml := range emlDiff.Deleted {
			seq.AddNum(eml.UID)
		}
		val := []interface{}{imap.DeletedFlag}
		err = conn.UidStore(seq, imap.AddFlags, val, nil)
		if err != nil {
			return ok, fmt.Errorf("imap/expunge: %v", err)
		}
		err = conn.Expunge(nil)
		if err != nil {
			return ok, fmt.Errorf("imap/expunge: %v", err)
		}
		// Update state
		for _, eml := range emlDiff.Deleted {
			state.DeleteEmail(local, eml)
		}
	}
	return ok, nil
}

func (s *Store) UpdateMailboxStatus(state *offmap.State, local string) error {
	client := s.getConn()
	defer client.done()
	items := []imap.StatusItem{
		imap.StatusUidNext,
		imap.StatusUidValidity,
	}
	if s.SupportCondstore() {
		for _, mbox := range s.mailboxes {
			state.UpdateMailboxState(mbox.LocalName, mbox.HighestModSeq, mbox.UIDNext)
		}
		items = append(items, condstore.StatusHighestModSeq)
	}
	remote := strings.ReplaceAll(local, "/", s.Delimiter())
	status, err := client.Status(remote, items)
	if err != nil {
		return err
	}
	var modseq uint64
	if s.SupportCondstore() {
		modseq = parseHighestModSeq(status)
	}
	state.UpdateMailboxState(local, modseq, status.UidNext)
	return nil
}

// TODO delete mailboxes
diff --git a/maildir/update.go b/maildir/update.go
index 61c2783dccbb..0092a4eccea3 100644
--- a/maildir/update.go
+++ b/maildir/update.go
@@ -11,8 +11,7 @@ import (
	"github.com/emersion/go-maildir"
)

// ApplyDiff applies the changes from offmap, updates the internal state
func (s *Store) ApplyDiff(state *offmap.State, c *offmap.Diff) error {
func (s *Store) ApplyMailboxDiff(state *offmap.State, c *offmap.Diff) error {
	s.state = state
	// Apply changes, updating state as we go
	for _, mbox := range c.Mailboxes.Created {
@@ -40,41 +39,44 @@ func (s *Store) ApplyDiff(state *offmap.State, c *offmap.Diff) error {
		s.mailboxes[dest.LocalName] = dest
		state.RenameMailbox(orig, dest)
	}
	return nil
}

	// Handle EmailDiff
	for mbox, emlDiff := range c.Emails {
		// Create emails
		ch := c.Fetch(mbox, emlDiff.Created)
		err := s.createEmail(mbox, ch)
		if err != nil {
			return fmt.Errorf("maildir: could not apply diff: %v", err)
		}

		// Add flags
		for flag, emls := range emlDiff.FlagAdded {
			err := s.addFlags(mbox, emls, flag)
			if err != nil {
				return fmt.Errorf("maildir: could not apply diff: %v", err)
			}
		}
func (s *Store) ApplyEmailDiff(state *offmap.State, mbox string, c *offmap.Diff) (bool, error) {
	// Create emails
	emlDiff, ok := c.Emails[mbox]
	if !ok {
		// no changes
		return ok, nil
	}
	ch := c.Fetch(mbox, emlDiff.Created)
	err := s.createEmail(mbox, ch)
	if err != nil {
		return ok, fmt.Errorf("maildir: could not apply diff: %v", err)
	}

		// Remove flags
		for flag, emls := range emlDiff.FlagRemoved {
			err := s.removeFlags(mbox, emls, flag)
			if err != nil {
				return fmt.Errorf("maildir: could not apply diff: %v", err)
			}
	// Add flags
	for flag, emls := range emlDiff.FlagAdded {
		err := s.addFlags(mbox, emls, flag)
		if err != nil {
			return ok, fmt.Errorf("maildir: could not apply diff: %v", err)
		}
	}

		// Delete emails
		err = s.deleteEmail(mbox, emlDiff.Deleted)
	// Remove flags
	for flag, emls := range emlDiff.FlagRemoved {
		err := s.removeFlags(mbox, emls, flag)
		if err != nil {
			return fmt.Errorf("maildir: could not apply diff: %v", err)
			return ok, fmt.Errorf("maildir: could not apply diff: %v", err)
		}
	}
	return nil

	// TODO Delete mailboxes
	// Delete emails
	err = s.deleteEmail(mbox, emlDiff.Deleted)
	if err != nil {
		return ok, fmt.Errorf("maildir: could not apply diff: %v", err)
	}
	return ok, nil
}

// createMaildir creates a new maildir with the name of the given mbox
diff --git a/state.go b/state.go
index 970c5e425340..f57b5fc56fa8 100644
--- a/state.go
+++ b/state.go
@@ -102,6 +102,10 @@ func (s *State) CreateMailbox(mbox *Mailbox) {
	// Zero out the emails or we'll get double the count after
	// appending newly created messages below
	mbox.Emails = []*Email{}
	// Zero out UIDNext and HighestModSeq, these will be updated after
	// adding emails
	mbox.UIDNext = 0
	mbox.HighestModSeq = 0
	s.Mailboxes = append(s.Mailboxes, mbox)
}

-- 
2.38.1

[PATCH offmap 3/3] cancellation: close after current mailbox finishes sync Export this patch

End offmap after current mailbox finishes sync.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
 cmd/offmap.go |  3 +--
 cmd/sync.go   | 11 ++++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/cmd/offmap.go b/cmd/offmap.go
index 94d05b866c4a..8a2176f28164 100644
--- a/cmd/offmap.go
+++ b/cmd/offmap.go
@@ -15,9 +15,8 @@ func main() {
	sigCh := make(chan os.Signal, 4)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-sigCh
		<-sigCh
		cancel()
		log.Fatalf("Aborting sync: %s", sig)
	}()
	cmd := &cobra.Command{
		Use:   "offmap",
diff --git a/cmd/sync.go b/cmd/sync.go
index f537572de0d1..328e84175a0e 100644
--- a/cmd/sync.go
+++ b/cmd/sync.go
@@ -1,6 +1,7 @@
package main

import (
	"sync/atomic"
	"time"

	"git.sr.ht/~rockorager/offmap"
@@ -64,14 +65,11 @@ func sync(cmd *cobra.Command, args []string) {
			log.Fatalf("%v", err)
		}

		var quit atomic.Bool
		go func() {
			// Save our state if we cancel the command
			<-cmd.Context().Done()
			if err := state.Save(); err != nil {
				log.Errorf("offmap: could not save state: %v", err)
			}
			remote.Cleanup()
			log.Infof("offmap: sync aborted, state saved")
			quit.Store(true)
		}()

		// Do mailboxes first
@@ -99,6 +97,9 @@ func sync(cmd *cobra.Command, args []string) {
		// Now the state has an up to date picture of mailboxes. Sync
		// each mailbox, if it has changes
		for _, mbox := range state.Mailboxes {
			if quit.Load() {
				break
			}
			ok1, err := remote.ApplyEmailDiff(state, mbox.RemoteName, localDiff)
			if err != nil {
				log.Errorf("%v", err)
-- 
2.38.1
offmap/patches/.build.yml: SUCCESS in 13m42s

[Updated sync and cancellation][0] from [Tim Culverhouse][1]

[0]: https://lists.sr.ht/~rockorager/offmap/patches/37404
[1]: mailto:tim@timculverhouse.com

✓ #900382 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/900382