offmap: imap: download emails in batches v2 NEEDS REVISION

Tim Culverhouse: 1
 imap: download emails in batches

 1 files changed, 40 insertions(+), 38 deletions(-)

While it seems to work in principle, I would prefer that an Outlook user
who previously encountered this problem gives this patch a spin.

If not:
Tested-by: Moritz Poldrack <moritz@poldrack.dev

Aerc really needs the "fetch thread" command, because I have no idea now what
this is referring to :D

Moritz Poldrack <moritz@poldrack.dev>

4 months ago

:toggle-threads -g

While useful, I will *not* touch that ^^

Bence Ferdinandy <bence@ferdinandy.com>

4 months ago

But seriously, which patch is this that needs testing from an outlook user?

Moritz Poldrack <moritz@poldrack.dev>

4 months ago

Batched fetching

Bence Ferdinandy <bence@ferdinandy.com>

4 months ago

We tested that on IRC, I think the optimal batch size for me was 1700, but the
conclusion was that is should be probably configurable per account, because it
might be different for people or MS might randomly change the accepted batch
sizes.

Best,
Bence


--
+36305425054
bence.ferdinandy.com

--
+36305425054
bence.ferdinandy.com

Best,
Bence
--
+36305425054
bence.ferdinandy.com

[PATCH offmap v2] imap: download emails in batches Export this patch

Download email in batches (of size 2000) to prevent issues with Office
365. Requests can be too large for O365 to initiate the download.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
v2: rearrange goroutine structure for better readability

 imap/imap.go | 78 +++++++++++++++++++++++++++-------------------------
 1 file changed, 40 insertions(+), 38 deletions(-)

diff --git a/imap/imap.go b/imap/imap.go
index 228669db260e..791248b04615 100644
--- a/imap/imap.go
+++ b/imap/imap.go
@@ -162,15 +162,11 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
 	var dled int32 = 0
 	total := len(emls)
 	emlMu := sync.Mutex{}
-	seq := &imap.SeqSet{}
-	for _, eml := range emls {
-		seq.AddNum(eml.UID)
-	}
 	client := s.getConn()
 	go func() {
 		defer client.done()
 		defer close(ch)
-		if seq.Empty() {
+		if len(emls) == 0 {
 			return
 		}
 		status, err := client.Select(mbox, true)
@@ -180,41 +176,47 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
 		}
 		section := &imap.BodySectionName{Peek: true}
 		items := []imap.FetchItem{section.FetchItem()}
-		msgCh := make(chan *imap.Message)
-		msgWg := sync.WaitGroup{}
-		msgWg.Add(1)
-		go func() {
-			for msg := range msgCh {
-				atomic.AddInt32(&dled, 1)
-				cur := atomic.LoadInt32(&dled)
-				emlMu.Lock()
-				eml, ok := emlMap[msg.Uid]
-				emlMu.Unlock()
-				if !ok {
-					log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
-					continue
-				}
-				r := msg.GetBody(section)
-				if r == nil {
-					log.Errorf("imap: no body")
-					continue
-				}
-				fe := &offmap.FullEmail{
-					Email: eml,
-					Body:  r,
-				}
-				ch <- fe
-				log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
-			}
-			msgWg.Done()
-		}()
 		log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
-		err = client.UidFetch(seq, items, msgCh)
-		if err != nil {
-			log.Errorf("imap (client %d): %v", client.id, err)
-			return
+		seq := &imap.SeqSet{}
+		batch := 0
+		for i, eml := range emls {
+			seq.AddNum(eml.UID)
+			batch = batch + 1
+			if batch == 2000 || i == len(emls)-1 {
+				msgCh := make(chan *imap.Message)
+				go func() {
+					err = client.UidFetch(seq, items, msgCh)
+					if err != nil {
+						log.Errorf("imap (client %d): %v", client.id, err)
+						return
+					}
+				}()
+				for msg := range msgCh {
+					atomic.AddInt32(&dled, 1)
+					cur := atomic.LoadInt32(&dled)
+					emlMu.Lock()
+					eml, ok := emlMap[msg.Uid]
+					emlMu.Unlock()
+					if !ok {
+						log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
+						continue
+					}
+					r := msg.GetBody(section)
+					if r == nil {
+						log.Errorf("imap: no body")
+						continue
+					}
+					fe := &offmap.FullEmail{
+						Email: eml,
+						Body:  r,
+					}
+					ch <- fe
+					log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
+				}
+				seq.Clear()
+				batch = 0
+			}
 		}
-		msgWg.Wait()
 	}()
 	return ch
 }
-- 
2.39.0

builds.sr.ht <builds@sr.ht>

5 months ago

offmap/patches/.build.yml: SUCCESS in 1m56s

[imap: download emails in batches][0] v2 from [Tim Culverhouse][1]

[0]: https://lists.sr.ht/~rockorager/offmap/patches/37785
[1]: mailto:tim@timculverhouse.com

✓ #912154 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/912154