~sircmpwn/aerc

Fix crash on archiving large numbers of messages v1 PROPOSED

Daniel Patterson: 1
 Fix crash on archiving large numbers of messages

 1 files changed, 118 insertions(+), 44 deletions(-)
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~sircmpwn/aerc/patches/22833/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH] Fix crash on archiving large numbers of messages Export this patch

---
Currently, if you attempt to archive messages that do not have their
headers downloaded, aerc panics. This can be easily replicated by going
to a folder with many emails downloaded, entering ":mark -a" followed by
":archive year". This patch attempts to fix this.

 commands/msg/archive.go | 162 +++++++++++++++++++++++++++++-----------
 1 file changed, 118 insertions(+), 44 deletions(-)

diff --git a/commands/msg/archive.go b/commands/msg/archive.go
index 59ca985..09793cf 100644
--- a/commands/msg/archive.go
+++ b/commands/msg/archive.go
@@ -52,62 +52,136 @@ func (Archive) Execute(aerc *widgets.Aerc, args []string) error {
		return err
	}
	archiveDir := acct.AccountConfig().Archive
	store.Next()
	_, isMsgView := h.msgProvider.(*widgets.MessageViewer)
	if isMsgView {
		aerc.RemoveTab(h.msgProvider)
	}
	acct.Messages().Invalidate()

	var uidMap map[string][]uint32
	switch args[1] {
	case ARCHIVE_MONTH:
		uidMap = groupBy(msgs, func(msg *models.MessageInfo) string {
			dir := path.Join(archiveDir,
				fmt.Sprintf("%d", msg.Envelope.Date.Year()),
				fmt.Sprintf("%02d", msg.Envelope.Date.Month()))
			return dir
		})
	case ARCHIVE_YEAR:
		uidMap = groupBy(msgs, func(msg *models.MessageInfo) string {
			dir := path.Join(archiveDir, fmt.Sprintf("%v",
				msg.Envelope.Date.Year()))
			return dir
		})
	case ARCHIVE_FLAT:
		uidMap = make(map[string][]uint32)
		uidMap[archiveDir] = commands.UidsFromMessageInfos(msgs)
	}
	// Asynchronously handle the archival
	go func() {
		uidMap := make(map[string][]uint32)
		var foundMap map[uint32]struct{}
		notFlat := true

		switch args[1] {
		case ARCHIVE_MONTH:
			uidMap, foundMap = groupBy(msgs, archiveDir, monthGrouper)
		case ARCHIVE_YEAR:
			uidMap, foundMap = groupBy(msgs, archiveDir, yearGrouper)
		case ARCHIVE_FLAT:
			notFlat = false
			uidMap = make(map[string][]uint32)
			uids, _ := commands.MarkedOrSelected(h.msgProvider)
			uidMap[archiveDir] = uids
		}

		// Select the next message after the current has been stored.
		store.Next()

	var wg sync.WaitGroup
	wg.Add(len(uidMap))
	success := true

	for dir, uids := range uidMap {
		store.Move(uids, dir, true, func(
			msg types.WorkerMessage) {
			switch msg := msg.(type) {
			case *types.Done:
				wg.Done()
			case *types.Error:
				aerc.PushError(msg.Error.Error())
				success = false
				wg.Done()
		missing := len(msgs) - len(foundMap) // no. of msgs with missing header

		if missing > 0 && notFlat {

			// We need to only fetch the headers which we don't already have.
			missingHeaderIDs := make([]uint32, 0, missing)
			for _, v := range store.Uids() {
				store.Deleted[v] = nil
				if _, ok := foundMap[v]; !ok {
					missingHeaderIDs = append(missingHeaderIDs, v)
				}
			}
		})
	}
	// we need to do that in the background, else we block the main thread
	go func() {
		wg.Wait()

			// Prepare for all the missing headers to be send on callbackChan
			callbackChan := make(chan *models.MessageInfo)
			var fetchHeaderWG sync.WaitGroup
			fetchHeaderWG.Add(missing)

			// Each message header gets its own callback to get sent on callbackChan
			store.FetchHeaders(missingHeaderIDs, func(info *types.MessageInfo) {
				callbackChan <- info.Info
				fetchHeaderWG.Done()
			})
			go func() {
				// Close callbackChan when there are no more headers coming
				fetchHeaderWG.Wait()
				close(callbackChan)
			}()

			for info := range callbackChan {
				// Each message gets added to the correct group
				switch args[1] {
				case ARCHIVE_MONTH:
					directory := monthGrouper(info, archiveDir)
					uidMap[directory] = append(uidMap[directory], info.Uid)
				case ARCHIVE_YEAR:
					directory := yearGrouper(info, archiveDir)
					uidMap[directory] = append(uidMap[directory], info.Uid)
				}
			}
		}

		var moveWG sync.WaitGroup
		moveWG.Add(len(uidMap))
		success := true
		for dir, uids := range uidMap {
			store.Move(uids, dir, true, func(
				msg types.WorkerMessage) {
				switch msg := msg.(type) {
				case *types.Done:
					moveWG.Done()
				case *types.Error:
					aerc.PushError(msg.Error.Error())
					success = false
					moveWG.Done()
				}
			})
		}
		if success {
			aerc.PushStatus("Messages archived.", 10*time.Second)
		}
	}()
	// returns after setting up all goroutines so that the main thread doesn't
	// block during the potentially lengthy header fetching session.
	return nil
}

func groupBy(msgs []*models.MessageInfo,
	grouper func(*models.MessageInfo) string) map[string][]uint32 {
	m := make(map[string][]uint32)
// groupBy returns a map of groupings, as produced by grouper, to uids, as well
// as a map containing the uids which were successfully found.
// If len(msgs) < len(foundMap), then not all headers were available and need to
// be fetched.
func groupBy(
	msgs []*models.MessageInfo, archiveDir string,
	grouper func(*models.MessageInfo, string) string,
) (
	m map[string][]uint32, foundMap map[uint32]struct{},
) {
	m = make(map[string][]uint32)
	foundMap = make(map[uint32]struct{})
	for _, msg := range msgs {
		group := grouper(msg)
		if msg == nil {
			continue
		}
		foundMap[msg.Uid] = struct{}{}
		group := grouper(msg, archiveDir)
		m[group] = append(m[group], msg.Uid)
	}
	return m
	return m, foundMap
}

// monthGrouper produces a string of the form "archiveDir/YYYY/MM" from
// msg.Envelope.Date
func monthGrouper(msg *models.MessageInfo, archiveDir string) string {
	dir := path.Join(archiveDir,
		fmt.Sprintf("%d", msg.Envelope.Date.Year()),
		fmt.Sprintf("%02d", msg.Envelope.Date.Month()))
	return dir
}

// yearGrouper produces a string of the form "archiveDir/YYYY" from
// msg.Envelope.Date
func yearGrouper(msg *models.MessageInfo, archiveDir string) string {
	dir := path.Join(archiveDir, fmt.Sprintf("%v",
		msg.Envelope.Date.Year()))
	return dir
}
-- 
2.27.0