~rockorager/offmap

offmap: imap: download emails in batches v2 NEEDS REVISION

Tim Culverhouse: 1
 imap: download emails in batches

 1 files changed, 40 insertions(+), 38 deletions(-)
#912154 .build.yml success
While it seems to work in principle, I would prefer that an Outlook user
who previously encountered this problem gives this patch a spin.

If not:
Tested-by: Moritz Poldrack <moritz@poldrack.dev



          
          
          
        
      
      
        
          






Aerc really needs the "fetch thread" command, because I have no idea now what
this is referring to :D

Best,
Bence
--
+36305425054
bence.ferdinandy.com



          
          
          
          
Next



          
          
          
          
Next
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~rockorager/offmap/patches/37785/mbox | git am -3
Learn more about email & git

[PATCH offmap v2] imap: download emails in batches Export this patch

Download email in batches (of size 2000) to prevent issues with Office
365. Requests can be too large for O365 to initiate the download.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
v2: rearrange goroutine structure for better readability

 imap/imap.go | 78 +++++++++++++++++++++++++++-------------------------
 1 file changed, 40 insertions(+), 38 deletions(-)

diff --git a/imap/imap.go b/imap/imap.go
index 228669db260e..791248b04615 100644
--- a/imap/imap.go
+++ b/imap/imap.go
@@ -162,15 +162,11 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
	var dled int32 = 0
	total := len(emls)
	emlMu := sync.Mutex{}
	seq := &imap.SeqSet{}
	for _, eml := range emls {
		seq.AddNum(eml.UID)
	}
	client := s.getConn()
	go func() {
		defer client.done()
		defer close(ch)
		if seq.Empty() {
		if len(emls) == 0 {
			return
		}
		status, err := client.Select(mbox, true)
@@ -180,41 +176,47 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
		}
		section := &imap.BodySectionName{Peek: true}
		items := []imap.FetchItem{section.FetchItem()}
		msgCh := make(chan *imap.Message)
		msgWg := sync.WaitGroup{}
		msgWg.Add(1)
		go func() {
			for msg := range msgCh {
				atomic.AddInt32(&dled, 1)
				cur := atomic.LoadInt32(&dled)
				emlMu.Lock()
				eml, ok := emlMap[msg.Uid]
				emlMu.Unlock()
				if !ok {
					log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
					continue
				}
				r := msg.GetBody(section)
				if r == nil {
					log.Errorf("imap: no body")
					continue
				}
				fe := &offmap.FullEmail{
					Email: eml,
					Body:  r,
				}
				ch <- fe
				log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
			}
			msgWg.Done()
		}()
		log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
		err = client.UidFetch(seq, items, msgCh)
		if err != nil {
			log.Errorf("imap (client %d): %v", client.id, err)
			return
		seq := &imap.SeqSet{}
		batch := 0
		for i, eml := range emls {
			seq.AddNum(eml.UID)
			batch = batch + 1
			if batch == 2000 || i == len(emls)-1 {
				msgCh := make(chan *imap.Message)
				go func() {
					err = client.UidFetch(seq, items, msgCh)
					if err != nil {
						log.Errorf("imap (client %d): %v", client.id, err)
						return
					}
				}()
				for msg := range msgCh {
					atomic.AddInt32(&dled, 1)
					cur := atomic.LoadInt32(&dled)
					emlMu.Lock()
					eml, ok := emlMap[msg.Uid]
					emlMu.Unlock()
					if !ok {
						log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
						continue
					}
					r := msg.GetBody(section)
					if r == nil {
						log.Errorf("imap: no body")
						continue
					}
					fe := &offmap.FullEmail{
						Email: eml,
						Body:  r,
					}
					ch <- fe
					log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
				}
				seq.Clear()
				batch = 0
			}
		}
		msgWg.Wait()
	}()
	return ch
}
-- 
2.39.0
offmap/patches/.build.yml: SUCCESS in 1m56s

[imap: download emails in batches][0] v2 from [Tim Culverhouse][1]

[0]: https://lists.sr.ht/~rockorager/offmap/patches/37785
[1]: mailto:tim@timculverhouse.com

✓ #912154 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/912154