From b888c37346cad5671fcf74a9dd0c5b33b6e1bd5f Mon Sep 17 00:00:00 2001 From: Glenn Wilkinson Date: Tue, 24 Mar 2020 12:48:02 +0000 Subject: [PATCH] Updated imap to use better supported library --- imap/imap.go | 421 ++++++++++++++++++------------------------------ imap/monitor.go | 8 +- 2 files changed, 159 insertions(+), 270 deletions(-) diff --git a/imap/imap.go b/imap/imap.go index f895aac8..dad3c4d9 100644 --- a/imap/imap.go +++ b/imap/imap.go @@ -1,34 +1,37 @@ package imap -// Functionality taken from https://github.com/jprobinson/eazye +//package main import ( "bytes" "crypto/tls" "fmt" + "io" "strconv" + "strings" "time" + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/client" + "github.com/emersion/go-message/mail" log "github.com/gophish/gophish/logger" "github.com/gophish/gophish/models" + "github.com/jordan-wright/email" - "github.com/mxk/go-imap/imap" ) // Client interface for IMAP interactions type Client interface { - Close(expunge bool) (cmd *imap.Command, err error) Login(username, password string) (cmd *imap.Command, err error) Logout(timeout time.Duration) (cmd *imap.Command, err error) - Select(mbox string, readonly bool) (cmd *imap.Command, err error) - UIDFetch(seq *imap.SeqSet, items ...string) (cmd *imap.Command, err error) - UIDSearch(spec ...imap.Field) (cmd *imap.Command, err error) - UIDStore(seq *imap.SeqSet, item string, value imap.Field) (cmd *imap.Command, err error) + Select(name string, readOnly bool) (mbox *imap.MailboxStatus, err error) + Store(seq *imap.SeqSet, item imap.StoreItem, value interface{}, ch chan *imap.Message) (err error) + Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) (err error) } -// Email represents an email.Email with an included IMAP UID +// Email represents an email.Email with an included IMAP Sequence Number type Email struct { - UID uint32 `json:"uid"` + SeqNum uint32 `json:"seqnum"` *email.Email } @@ -44,95 +47,6 @@ type Mailbox struct { ReadOnly bool } -// GetAll will pull all emails from the email folder and return them as a list. -func (mbox *Mailbox) GetAll(markAsRead, delete bool) ([]Email, error) { - // call chan, put 'em in a list, return - var emails []Email - responses, err := mbox.GenerateAll(markAsRead, delete) - if err != nil { - return emails, err - } - - for resp := range responses { - if resp.Err != nil { - return emails, resp.Err - } - emails = append(emails, resp.Email) - } - - return emails, nil -} - -// GenerateAll will find all emails in the email folder and pass them along to the responses channel. -func (mbox *Mailbox) GenerateAll(markAsRead, delete bool) (chan Response, error) { - return mbox.generateMail("ALL", nil, markAsRead, delete) -} - -// GetUnread will find all unread emails in the folder and return them as a list. -func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) { - // call chan, put 'em in a list, return - var emails []Email - - responses, err := mbox.GenerateUnread(markAsRead, delete) - if err != nil { - return emails, err - } - - for resp := range responses { - if resp.Err != nil { - return emails, resp.Err - } - emails = append(emails, resp.Email) - } - - return emails, nil -} - -// GenerateUnread will find all unread emails in the folder and pass them along to the responses channel. -func (mbox *Mailbox) GenerateUnread(markAsRead, delete bool) (chan Response, error) { - return mbox.generateMail("UNSEEN", nil, markAsRead, delete) -} - -// MarkAsUnread will set the UNSEEN flag on a supplied slice of UIDs -func (mbox *Mailbox) MarkAsUnread(uids []uint32) error { - client, err := mbox.newClient() - if err != nil { - return err - } - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - }() - for _, u := range uids { - err := alterEmail(client, u, "\\SEEN", false) - if err != nil { - return err //return on first failure - } - } - return nil - -} - -// DeleteEmails will delete emails from the supplied slice of UIDs -func (mbox *Mailbox) DeleteEmails(uids []uint32) error { - client, err := mbox.newClient() - if err != nil { - return err - } - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - }() - for _, u := range uids { - err := deleteEmail(client, u) - if err != nil { - return err //return on first failure - } - } - return nil - -} - // Validate validates supplied IMAP model by connecting to the server func Validate(s *models.IMAP) error { @@ -150,180 +64,53 @@ func Validate(s *models.IMAP) error { Pwd: s.Password, Folder: s.Folder} - client, err := mailServer.newClient() + imapclient, err := mailServer.newClient() if err != nil { log.Error(err.Error()) } else { - client.Close(true) - client.Logout(30 * time.Second) + imapclient.Logout() } return err } -// Response is a helper struct to wrap the email responses and possible errors. -type Response struct { - Email Email - Err error -} - -// newClient will initiate a new IMAP connection with the given creds. -func (mbox *Mailbox) newClient() (*imap.Client, error) { - var client *imap.Client - var err error - if mbox.TLS { - client, err = imap.DialTLS(mbox.Host, new(tls.Config)) - if err != nil { - return client, err - } - } else { - client, err = imap.Dial(mbox.Host) - if err != nil { - return client, err - } - } - - _, err = client.Login(mbox.User, mbox.Pwd) +// MarkAsUnread will set the UNSEEN flag on a supplied slice of SeqNums +func (mbox *Mailbox) MarkAsUnread(seqs []uint32) error { + imapclient, err := mbox.newClient() if err != nil { - return client, err + return err } - _, err = imap.Wait(client.Select(mbox.Folder, mbox.ReadOnly)) + defer imapclient.Logout() + + seqSet := new(imap.SeqSet) + seqSet.AddNum(seqs...) + + item := imap.FormatFlagsOp(imap.RemoveFlags, true) + flags := []interface{}{imap.SeenFlag} + err = imapclient.Store(seqSet, item, flags, nil) if err != nil { - return client, err + return err } - return client, nil + return nil + } -const dateFormat = "02-Jan-2006" - -// findEmails will run a find the UIDs of any emails that match the search.: -func findEmails(client Client, search string, since *time.Time) (*imap.Command, error) { - var specs []imap.Field - if len(search) > 0 { - specs = append(specs, search) - } - - if since != nil { - sinceStr := since.Format(dateFormat) - specs = append(specs, "SINCE", sinceStr) - } - - // get headers and UID for UnSeen message in src inbox... - cmd, err := imap.Wait(client.UIDSearch(specs...)) +// DeleteEmails will delete emails from the supplied slice of SeqNums +func (mbox *Mailbox) DeleteEmails(seqs []uint32) error { + imapclient, err := mbox.newClient() if err != nil { - return &imap.Command{}, fmt.Errorf("uid search failed: %s", err) - } - return cmd, nil -} - -const GenerateBufferSize = 100 - -func (mbox *Mailbox) generateMail(search string, since *time.Time, markAsRead, delete bool) (chan Response, error) { - responses := make(chan Response, GenerateBufferSize) - client, err := mbox.newClient() - if err != nil { - close(responses) - return responses, fmt.Errorf("failed to create IMAP connection: %s", err) + return err } - go func() { - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - close(responses) - }() + defer imapclient.Logout() - var cmd *imap.Command - // find all the UIDs - cmd, err = findEmails(client, search, since) - if err != nil { - responses <- Response{Err: err} - return - } - // gotta fetch 'em all - getEmails(client, cmd, markAsRead, delete, responses) - }() + seqSet := new(imap.SeqSet) + seqSet.AddNum(seqs...) - return responses, nil -} - -func getEmails(client Client, cmd *imap.Command, markAsRead, delete bool, responses chan Response) { - seq := &imap.SeqSet{} - msgCount := 0 - for _, rsp := range cmd.Data { - for _, uid := range rsp.SearchResults() { - msgCount++ - seq.AddNum(uid) - } - } - - if seq.Empty() { - return - } - - fCmd, err := imap.Wait(client.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER")) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to perform uid fetch: %s", err)} - return - } - - var email Email - for _, msgData := range fCmd.Data { - msgFields := msgData.MessageInfo().Attrs - - // make sure is a legit response before we attempt to parse it - // deal with unsolicited FETCH responses containing only flags - // I'm lookin' at YOU, Gmail! - // http://mailman13.u.washington.edu/pipermail/imap-protocol/2014-October/002355.html - // http://stackoverflow.com/questions/26262472/gmail-imap-is-sometimes-returning-bad-results-for-fetch - if _, ok := msgFields["RFC822.HEADER"]; !ok { - continue - } - - email, err = NewEmail(msgFields) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to parse email: %s", err)} - return - } - - responses <- Response{Email: email} - - if !markAsRead { - err = removeSeen(client, imap.AsNumber(msgFields["UID"])) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to remove seen flag: %s", err)} - return - } - } - - if delete { - err = deleteEmail(client, imap.AsNumber(msgFields["UID"])) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to delete email: %s", err)} - return - } - } - } - return -} - -func deleteEmail(client Client, UID uint32) error { - return alterEmail(client, UID, "\\DELETED", true) -} - -func removeSeen(client Client, UID uint32) error { - return alterEmail(client, UID, "\\SEEN", false) -} - -func alterEmail(client Client, UID uint32, flag string, plus bool) error { - flg := "-FLAGS" - if plus { - flg = "+FLAGS" - } - fSeq := &imap.SeqSet{} - fSeq.AddNum(UID) - _, err := imap.Wait(client.UIDStore(fSeq, flg, flag)) + item := imap.FormatFlagsOp(imap.AddFlags, true) + flags := []interface{}{imap.DeletedFlag} + err = imapclient.Store(seqSet, item, flags, nil) if err != nil { return err } @@ -331,22 +118,124 @@ func alterEmail(client Client, UID uint32, flag string, plus bool) error { return nil } -// NewEmail will parse an imap.FieldMap into an Email. This -// will expect the message to container the internaldate and the body with -// all headers included. -func NewEmail(msgFields imap.FieldMap) (Email, error) { +// GetUnread will find all unread emails in the folder and return them as a list. +func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) { + var emails []Email - rawBody := imap.AsBytes(msgFields["BODY[]"]) - - rawBodyStream := bytes.NewReader(rawBody) - em, err := email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library + imapclient, err := mbox.newClient() if err != nil { - return Email{}, err - } - iem := Email{ - Email: em, - UID: imap.AsNumber(msgFields["UID"]), + return emails, fmt.Errorf("failed to create IMAP connection: %s", err) } - return iem, err + defer imapclient.Logout() + + // Search for unread emails + criteria := imap.NewSearchCriteria() + criteria.WithoutFlags = []string{"\\Seen"} + seqs, err := imapclient.Search(criteria) + if err != nil { + return emails, err + } + + if len(seqs) > 0 { + seqset := new(imap.SeqSet) + seqset.AddNum(seqs...) + section := &imap.BodySectionName{} + items := []imap.FetchItem{imap.FetchEnvelope, imap.FetchFlags, imap.FetchInternalDate, section.FetchItem()} // Check this + messages := make(chan *imap.Message) + + go func() { + if err := imapclient.Fetch(seqset, items, messages); err != nil { + log.Error("Error fetching emails: ", err.Error()) // TODO: How to handle this, need to propogate error out + } + }() + + // Step through each email + for msg := range messages { + // Extract raw message body. I can't find a better way to do this with the emersion library + var em *email.Email + var buf []byte + for _, value := range msg.Body { + buf = make([]byte, value.Len()) + value.Read(buf) + break // There should only ever be one item in this map, but I'm not 100% sure + } + + //Remove CR characters, see https://github.com/jordan-wright/email/issues/106 + tmp := string(buf) + tmp = strings.ReplaceAll(tmp, "\r", "") + buf = []byte(tmp) + + rawBodyStream := bytes.NewReader(buf) + em, err = email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library + if err != nil { + return emails, err + } + + // Reload the reader 🔫 + rawBodyStream = bytes.NewReader(buf) + mr, err := mail.CreateReader(rawBodyStream) + if err != nil { + return emails, err + } + + // Step over each part of the email, parsing attachments and attaching them to Jordan's email + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } else if err != nil { + return emails, err + } + h := p.Header + + s, ok := h.(*mail.AttachmentHeader) + if ok { + filename, _ := s.Filename() + typ, _, _ := s.ContentType() + _, err := em.Attach(p.Body, filename, typ) + if err != nil { + return emails, err //Unable to attach file + } + } + } + + emtmp := Email{Email: em, SeqNum: msg.SeqNum} // Not sure why msg.Uid is always 0, so swapped to sequence numbers + emails = append(emails, emtmp) + + } // On to the next email + } else { + //log.Println("No new messages") + } + + return emails, nil +} + +// newClient will initiate a new IMAP connection with the given creds. +func (mbox *Mailbox) newClient() (*client.Client, error) { + var imapclient *client.Client + var err error + if mbox.TLS { + imapclient, err = client.DialTLS(mbox.Host, new(tls.Config)) + if err != nil { + return imapclient, err + } + } else { + imapclient, err = client.Dial(mbox.Host) + if err != nil { + return imapclient, err + } + } + + err = imapclient.Login(mbox.User, mbox.Pwd) + if err != nil { + return imapclient, err + } + + _, err = imapclient.Select(mbox.Folder, mbox.ReadOnly) + if err != nil { + return imapclient, err + } + + return imapclient, nil } diff --git a/imap/monitor.go b/imap/monitor.go index 4c0a61a9..ecdc691e 100644 --- a/imap/monitor.go +++ b/imap/monitor.go @@ -134,8 +134,8 @@ func checkForNewEmails(im models.IMAP) { err = models.SuccessfulLogin(&im) if len(msgs) > 0 { - var reportingFailed []uint32 // UIDs of emails that were unable to be reported to phishing server, mark as unread - var campaignEmails []uint32 // UIDs of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these + var reportingFailed []uint32 // SeqNums of emails that were unable to be reported to phishing server, mark as unread + var campaignEmails []uint32 // SeqNums of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these for _, m := range msgs { // Check if sender is from company's domain, if enabled. TODO: Make this an IMAP filter if im.RestrictDomain != "" { // e.g domainResitct = widgets.com @@ -156,14 +156,14 @@ func checkForNewEmails(im models.IMAP) { result, err := models.GetResult(rid) if err != nil { log.Error("Error reporting GoPhish email with rid ", rid, ": ", err.Error()) - reportingFailed = append(reportingFailed, m.UID) + reportingFailed = append(reportingFailed, m.SeqNum) } else { err = result.HandleEmailReport(models.EventDetails{}) if err != nil { log.Error("Error updating GoPhish email with rid ", rid, ": ", err.Error()) } else { if im.DeleteReportedCampaignEmail == true { - campaignEmails = append(campaignEmails, m.UID) + campaignEmails = append(campaignEmails, m.SeqNum) } } }