[PATCH offmap v2] imap: download emails in batches
Export this patch
Download email in batches (of size 2000) to prevent issues with Office
365. Requests can be too large for O365 to initiate the download.
Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
---
v2: rearrange goroutine structure for better readability
imap/imap.go | 78 +++++++++++++++++++++++++++ -------------------------
1 file changed, 40 insertions(+), 38 deletions(-)
diff --git a/imap/imap.go b/imap/imap.go
index 228669db260e..791248b04615 100644
--- a/imap/imap.go
+++ b/imap/imap.go
@@ -162,15 +162,11 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
var dled int32 = 0
total := len(emls)
emlMu := sync.Mutex{}
- seq := &imap.SeqSet{}
- for _, eml := range emls {
- seq.AddNum(eml.UID)
- }
client := s.getConn()
go func() {
defer client.done()
defer close(ch)
- if seq.Empty() {
+ if len(emls) == 0 {
return
}
status, err := client.Select(mbox, true)
@@ -180,41 +176,47 @@ func (s *Store) DownloadEmail(mbox string, emls []*offmap.Email) chan *offmap.Fu
}
section := &imap.BodySectionName{Peek: true}
items := []imap.FetchItem{section.FetchItem()}
- msgCh := make(chan *imap.Message)
- msgWg := sync.WaitGroup{}
- msgWg.Add(1)
- go func() {
- for msg := range msgCh {
- atomic.AddInt32(&dled, 1)
- cur := atomic.LoadInt32(&dled)
- emlMu.Lock()
- eml, ok := emlMap[msg.Uid]
- emlMu.Unlock()
- if !ok {
- log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
- continue
- }
- r := msg.GetBody(section)
- if r == nil {
- log.Errorf("imap: no body")
- continue
- }
- fe := &offmap.FullEmail{
- Email: eml,
- Body: r,
- }
- ch <- fe
- log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
- }
- msgWg.Done()
- }()
log.Debugf("imap (client %d): downloading uids in mailbox: %s", client.id, mbox)
- err = client.UidFetch(seq, items, msgCh)
- if err != nil {
- log.Errorf("imap (client %d): %v", client.id, err)
- return
+ seq := &imap.SeqSet{}
+ batch := 0
+ for i, eml := range emls {
+ seq.AddNum(eml.UID)
+ batch = batch + 1
+ if batch == 2000 || i == len(emls)-1 {
+ msgCh := make(chan *imap.Message)
+ go func() {
+ err = client.UidFetch(seq, items, msgCh)
+ if err != nil {
+ log.Errorf("imap (client %d): %v", client.id, err)
+ return
+ }
+ }()
+ for msg := range msgCh {
+ atomic.AddInt32(&dled, 1)
+ cur := atomic.LoadInt32(&dled)
+ emlMu.Lock()
+ eml, ok := emlMap[msg.Uid]
+ emlMu.Unlock()
+ if !ok {
+ log.Errorf("imap: %s: eml not found: %d", status.Name, msg.Uid)
+ continue
+ }
+ r := msg.GetBody(section)
+ if r == nil {
+ log.Errorf("imap: no body")
+ continue
+ }
+ fe := &offmap.FullEmail{
+ Email: eml,
+ Body: r,
+ }
+ ch <- fe
+ log.Tracef("imap: (client %d) downloaded %d of %d", client.id, cur, total)
+ }
+ seq.Clear()
+ batch = 0
+ }
}
- msgWg.Wait()
}()
return ch
}
--
2.39.0
offmap/patches/.build.yml: SUCCESS in 1m56s
[imap: download emails in batches][0] v2 from [Tim Culverhouse][1]
[0]: https://lists.sr.ht/~rockorager/offmap/patches/37785
[1]: mailto:tim@timculverhouse.com
✓ #912154 SUCCESS offmap/patches/.build.yml https://builds.sr.ht/~rockorager/job/912154