[PATCH offmap] imap: download emails in batches
Export this patch
Download email in batches (of size 2000) to prevent issues with Office
365. Requests can be too large for O365 to initiate the download.
Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
imap/imap.go | 82 ++++++++++++++++++++++++++++------------------------
1 file changed, 44 insertions(+), 38 deletions(-)
diff --git a/imap/imap.go b/imap/imap.go
index 228669db260e..0624c73aacc2 100644
--- a/imap/imap.go
+++ b/imap/imap.go
@@ -162,15 +162,11 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
var dled int32 = 0
total := len(emls)
emlMu := sync.Mutex{}
- seq := &imap.SeqSet{}
- for _, eml := range emls {
- seq.AddNum(eml.UID)
- }
client := s.getConn()
go func() {
defer client.done()
defer close(ch)
- if seq.Empty() {
+ if len(emls) == 0 {
return
}
status, err := client.Select(mbox, true)
@@ -180,41 +176,51 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
}
section := &imap.BodySectionName{Peek: true}
items := []imap.FetchItem{section.FetchItem()}
- msgCh := make(chan *imap.Message)
- msgWg := sync.WaitGroup{}
- msgWg.Add(1)
- go func() {
- for msg := range msgCh {
- atomic.AddInt32(&dled, 1)
- cur := atomic.LoadInt32(&dled)
- emlMu.Lock()
- eml, ok := emlMap[msg.Uid]
- emlMu.Unlock()
- if !ok {
- log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
- continue
- }
- r := msg.GetBody(section)
- if r == nil {
- log.Errorf("imap: no body")
- continue
- }
- fe := &offmap.FullEmail{
- Email: eml,
- Body: r,
- }
- ch <- fe
- log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
- }
- msgWg.Done()
- }()
log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
- err = client.UidFetch(seq, items, msgCh)
- if err != nil {
- log.Errorf("imap (client %d): %v", client.id, err)
- return
+ seq := &imap.SeqSet{}
+ msgWg := sync.WaitGroup{}
+ batch := 0
+ for i, eml := range emls {
+ seq.AddNum(eml.UID)
+ batch = batch + 1
+ if batch == 2000 || i == len(emls)-1 {
+ msgCh := make(chan *imap.Message)
+ msgWg.Add(1)
+ go func() {
I think keeping the larger logic-block outside the goroutine would make
this more legible.
+ for msg := range msgCh {
+ atomic.AddInt32(&dled, 1)
+ cur := atomic.LoadInt32(&dled)
+ emlMu.Lock()
+ eml, ok := emlMap[msg.Uid]
+ emlMu.Unlock()
+ if !ok {
+ log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
+ continue
+ }
+ r := msg.GetBody(section)
+ if r == nil {
+ log.Errorf("imap: no body")
+ continue
+ }
+ fe := &offmap.FullEmail{
+ Email: eml,
+ Body: r,
+ }
+ ch <- fe
+ log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
+ }
+ msgWg.Done()
+ }()
+ err = client.UidFetch(seq, items, msgCh)
+ if err != nil {
+ log.Errorf("imap (client %d): %v", client.id, err)
+ return
+ }
+ msgWg.Wait()
+ seq.Clear()
+ batch = 0
+ }
}
- msgWg.Wait()
}()
return ch
}
--
2.39.0
offmap/patches/.build.yml: SUCCESS in 2m5s
[imap: download emails in batches][0] from [Tim Culverhouse][1]
[0]: https://lists.sr.ht/~rockorager/offmap/patches/37658
[1]: mailto:tim@timculverhouse.com
✓ #908629 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/908629
I think this warrants a comment.