From 6f95da00ba005caca29db6fcf47eae081f829948 Mon Sep 17 00:00:00 2001 From: Glenn Wilkinson Date: Wed, 17 Jun 2020 02:02:09 +0100 Subject: [PATCH] IMAP update; new library and attachment support (#1791) Updates the IMAP processing to use a more mature library. This allows for more robust IMAP support. Additionally, this adds support for reporting emails as attachments. --- imap/imap.go | 389 +++++++++++++++--------------------------------- imap/monitor.go | 116 ++++++++++----- 2 files changed, 200 insertions(+), 305 deletions(-) diff --git a/imap/imap.go b/imap/imap.go index ac2798a4..0707185e 100644 --- a/imap/imap.go +++ b/imap/imap.go @@ -1,34 +1,34 @@ package imap -// Functionality taken from https://github.com/jprobinson/eazye - import ( "bytes" "crypto/tls" "fmt" + "regexp" "strconv" "time" + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/client" + "github.com/emersion/go-message/charset" log "github.com/gophish/gophish/logger" "github.com/gophish/gophish/models" + "github.com/jordan-wright/email" - "github.com/mxk/go-imap/imap" ) // Client interface for IMAP interactions type Client interface { - Close(expunge bool) (cmd *imap.Command, err error) Login(username, password string) (cmd *imap.Command, err error) Logout(timeout time.Duration) (cmd *imap.Command, err error) - Select(mbox string, readonly bool) (cmd *imap.Command, err error) - UIDFetch(seq *imap.SeqSet, items ...string) (cmd *imap.Command, err error) - UIDSearch(spec ...imap.Field) (cmd *imap.Command, err error) - UIDStore(seq *imap.SeqSet, item string, value imap.Field) (cmd *imap.Command, err error) + Select(name string, readOnly bool) (mbox *imap.MailboxStatus, err error) + Store(seq *imap.SeqSet, item imap.StoreItem, value interface{}, ch chan *imap.Message) (err error) + Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) (err error) } -// Email represents an email.Email with an included IMAP UID +// Email represents an email.Email with an included IMAP Sequence Number type Email struct { - UID uint32 `json:"uid"` + SeqNum uint32 `json:"seqnum"` *email.Email } @@ -44,98 +44,8 @@ type Mailbox struct { ReadOnly bool } -// GetAll will pull all emails from the email folder and return them as a list. -func (mbox *Mailbox) GetAll(markAsRead, delete bool) ([]Email, error) { - // call chan, put 'em in a list, return - var emails []Email - responses, err := mbox.GenerateAll(markAsRead, delete) - if err != nil { - return emails, err - } - - for resp := range responses { - if resp.Err != nil { - return emails, resp.Err - } - emails = append(emails, resp.Email) - } - - return emails, nil -} - -// GenerateAll will find all emails in the email folder and pass them along to the responses channel. -func (mbox *Mailbox) GenerateAll(markAsRead, delete bool) (chan Response, error) { - return mbox.generateMail("ALL", nil, markAsRead, delete) -} - -// GetUnread will find all unread emails in the folder and return them as a list. -func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) { - // call chan, put 'em in a list, return - var emails []Email - - responses, err := mbox.GenerateUnread(markAsRead, delete) - if err != nil { - return emails, err - } - - for resp := range responses { - if resp.Err != nil { - return emails, resp.Err - } - emails = append(emails, resp.Email) - } - - return emails, nil -} - -// GenerateUnread will find all unread emails in the folder and pass them along to the responses channel. -func (mbox *Mailbox) GenerateUnread(markAsRead, delete bool) (chan Response, error) { - return mbox.generateMail("UNSEEN", nil, markAsRead, delete) -} - -// MarkAsUnread will set the UNSEEN flag on a supplied slice of UIDs -func (mbox *Mailbox) MarkAsUnread(uids []uint32) error { - client, err := mbox.newClient() - if err != nil { - return err - } - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - }() - for _, u := range uids { - err := alterEmail(client, u, "\\SEEN", false) - if err != nil { - return err //return on first failure - } - } - return nil - -} - -// DeleteEmails will delete emails from the supplied slice of UIDs -func (mbox *Mailbox) DeleteEmails(uids []uint32) error { - client, err := mbox.newClient() - if err != nil { - return err - } - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - }() - for _, u := range uids { - err := deleteEmail(client, u) - if err != nil { - return err //return on first failure - } - } - return nil - -} - // Validate validates supplied IMAP model by connecting to the server func Validate(s *models.IMAP) error { - err := s.Validate() if err != nil { log.Error(err) @@ -150,179 +60,51 @@ func Validate(s *models.IMAP) error { Pwd: s.Password, Folder: s.Folder} - client, err := mailServer.newClient() + imapClient, err := mailServer.newClient() if err != nil { log.Error(err.Error()) } else { - client.Close(true) - client.Logout(30 * time.Second) + imapClient.Logout() } return err } -// Response is a helper struct to wrap the email responses and possible errors. -type Response struct { - Email Email - Err error -} - -// newClient will initiate a new IMAP connection with the given creds. -func (mbox *Mailbox) newClient() (*imap.Client, error) { - var client *imap.Client - var err error - if mbox.TLS { - client, err = imap.DialTLS(mbox.Host, new(tls.Config)) - if err != nil { - return client, err - } - } else { - client, err = imap.Dial(mbox.Host) - if err != nil { - return client, err - } - } - - _, err = client.Login(mbox.User, mbox.Pwd) +// MarkAsUnread will set the UNSEEN flag on a supplied slice of SeqNums +func (mbox *Mailbox) MarkAsUnread(seqs []uint32) error { + imapClient, err := mbox.newClient() if err != nil { - return client, err + return err } - _, err = imap.Wait(client.Select(mbox.Folder, mbox.ReadOnly)) + defer imapClient.Logout() + + seqSet := new(imap.SeqSet) + seqSet.AddNum(seqs...) + + item := imap.FormatFlagsOp(imap.RemoveFlags, true) + err = imapClient.Store(seqSet, item, imap.SeenFlag, nil) if err != nil { - return client, err + return err } - return client, nil + return nil + } -const dateFormat = "02-Jan-2006" - -// findEmails will run a find the UIDs of any emails that match the search.: -func findEmails(client Client, search string, since *time.Time) (*imap.Command, error) { - var specs []imap.Field - if len(search) > 0 { - specs = append(specs, search) - } - - if since != nil { - sinceStr := since.Format(dateFormat) - specs = append(specs, "SINCE", sinceStr) - } - - // get headers and UID for UnSeen message in src inbox... - cmd, err := imap.Wait(client.UIDSearch(specs...)) +// DeleteEmails will delete emails from the supplied slice of SeqNums +func (mbox *Mailbox) DeleteEmails(seqs []uint32) error { + imapClient, err := mbox.newClient() if err != nil { - return &imap.Command{}, fmt.Errorf("uid search failed: %s", err) - } - return cmd, nil -} - -const GenerateBufferSize = 100 - -func (mbox *Mailbox) generateMail(search string, since *time.Time, markAsRead, delete bool) (chan Response, error) { - responses := make(chan Response, GenerateBufferSize) - client, err := mbox.newClient() - if err != nil { - close(responses) - return responses, fmt.Errorf("failed to create IMAP connection: %s", err) + return err } - go func() { - defer func() { - client.Close(true) - client.Logout(30 * time.Second) - close(responses) - }() + defer imapClient.Logout() - var cmd *imap.Command - // find all the UIDs - cmd, err = findEmails(client, search, since) - if err != nil { - responses <- Response{Err: err} - return - } - // gotta fetch 'em all - getEmails(client, cmd, markAsRead, delete, responses) - }() + seqSet := new(imap.SeqSet) + seqSet.AddNum(seqs...) - return responses, nil -} - -func getEmails(client Client, cmd *imap.Command, markAsRead, delete bool, responses chan Response) { - seq := &imap.SeqSet{} - msgCount := 0 - for _, rsp := range cmd.Data { - for _, uid := range rsp.SearchResults() { - msgCount++ - seq.AddNum(uid) - } - } - - if seq.Empty() { - return - } - - fCmd, err := imap.Wait(client.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER")) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to perform uid fetch: %s", err)} - return - } - - var email Email - for _, msgData := range fCmd.Data { - msgFields := msgData.MessageInfo().Attrs - - // make sure is a legit response before we attempt to parse it - // deal with unsolicited FETCH responses containing only flags - // I'm lookin' at YOU, Gmail! - // http://mailman13.u.washington.edu/pipermail/imap-protocol/2014-October/002355.html - // http://stackoverflow.com/questions/26262472/gmail-imap-is-sometimes-returning-bad-results-for-fetch - if _, ok := msgFields["RFC822.HEADER"]; !ok { - continue - } - - email, err = NewEmail(msgFields) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to parse email: %s", err)} - return - } - - responses <- Response{Email: email} - - if !markAsRead { - err = removeSeen(client, imap.AsNumber(msgFields["UID"])) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to remove seen flag: %s", err)} - return - } - } - - if delete { - err = deleteEmail(client, imap.AsNumber(msgFields["UID"])) - if err != nil { - responses <- Response{Err: fmt.Errorf("unable to delete email: %s", err)} - return - } - } - } -} - -func deleteEmail(client Client, UID uint32) error { - return alterEmail(client, UID, "\\DELETED", true) -} - -func removeSeen(client Client, UID uint32) error { - return alterEmail(client, UID, "\\SEEN", false) -} - -func alterEmail(client Client, UID uint32, flag string, plus bool) error { - flg := "-FLAGS" - if plus { - flg = "+FLAGS" - } - fSeq := &imap.SeqSet{} - fSeq.AddNum(UID) - _, err := imap.Wait(client.UIDStore(fSeq, flg, flag)) + item := imap.FormatFlagsOp(imap.AddFlags, true) + err = imapClient.Store(seqSet, item, imap.DeletedFlag, nil) if err != nil { return err } @@ -330,22 +112,95 @@ func alterEmail(client Client, UID uint32, flag string, plus bool) error { return nil } -// NewEmail will parse an imap.FieldMap into an Email. This -// will expect the message to container the internaldate and the body with -// all headers included. -func NewEmail(msgFields imap.FieldMap) (Email, error) { +// GetUnread will find all unread emails in the folder and return them as a list. +func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) { + imap.CharsetReader = charset.Reader + var emails []Email - rawBody := imap.AsBytes(msgFields["BODY[]"]) - - rawBodyStream := bytes.NewReader(rawBody) - em, err := email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library + imapClient, err := mbox.newClient() if err != nil { - return Email{}, err - } - iem := Email{ - Email: em, - UID: imap.AsNumber(msgFields["UID"]), + return emails, fmt.Errorf("failed to create IMAP connection: %s", err) } - return iem, err + defer imapClient.Logout() + + // Search for unread emails + criteria := imap.NewSearchCriteria() + criteria.WithoutFlags = []string{imap.SeenFlag} + seqs, err := imapClient.Search(criteria) + if err != nil { + return emails, err + } + + if len(seqs) == 0 { + return emails, nil + } + + seqset := new(imap.SeqSet) + seqset.AddNum(seqs...) + section := &imap.BodySectionName{} + items := []imap.FetchItem{imap.FetchEnvelope, imap.FetchFlags, imap.FetchInternalDate, section.FetchItem()} + messages := make(chan *imap.Message) + + go func() { + if err := imapClient.Fetch(seqset, items, messages); err != nil { + log.Error("Error fetching emails: ", err.Error()) // TODO: How to handle this, need to propogate error out + } + }() + + // Step through each email + for msg := range messages { + // Extract raw message body. I can't find a better way to do this with the emersion library + var em *email.Email + var buf []byte + for _, value := range msg.Body { + buf = make([]byte, value.Len()) + value.Read(buf) + break // There should only ever be one item in this map, but I'm not 100% sure + } + + //Remove CR characters, see https://github.com/jordan-wright/email/issues/106 + tmp := string(buf) + re := regexp.MustCompile(`\r`) + tmp = re.ReplaceAllString(tmp, "") + buf = []byte(tmp) + + rawBodyStream := bytes.NewReader(buf) + em, err = email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library + if err != nil { + return emails, err + } + + emtmp := Email{Email: em, SeqNum: msg.SeqNum} // Not sure why msg.Uid is always 0, so swapped to sequence numbers + emails = append(emails, emtmp) + + } + return emails, nil +} + +// newClient will initiate a new IMAP connection with the given creds. +func (mbox *Mailbox) newClient() (*client.Client, error) { + var imapClient *client.Client + var err error + if mbox.TLS { + imapClient, err = client.DialTLS(mbox.Host, new(tls.Config)) + } else { + imapClient, err = client.Dial(mbox.Host) + } + if err != nil { + return imapClient, err + } + + + err = imapClient.Login(mbox.User, mbox.Pwd) + if err != nil { + return imapClient, err + } + + _, err = imapClient.Select(mbox.Folder, mbox.ReadOnly) + if err != nil { + return imapClient, err + } + + return imapClient, nil } diff --git a/imap/monitor.go b/imap/monitor.go index 34b62e79..ff4d6f51 100644 --- a/imap/monitor.go +++ b/imap/monitor.go @@ -7,19 +7,22 @@ package imap * - Add field to User for numner of non-campaign emails reported */ import ( + "bytes" "context" "regexp" "strconv" "strings" "time" + "path/filepath" log "github.com/gophish/gophish/logger" + "github.com/jordan-wright/email" "github.com/gophish/gophish/models" ) // Pattern for GoPhish emails e.g ?rid=AbC123 -var goPhishRegex = regexp.MustCompile("(\\?rid=[A-Za-z0-9]{7})") +var goPhishRegex = regexp.MustCompile("(\\?rid=(3D)?([A-Za-z0-9]{7}))") // We include the optional quoted-printable 3D at the front, just in case decoding fails // Monitor is a worker that monitors IMAP servers for reported campaign emails type Monitor struct { @@ -30,7 +33,6 @@ type Monitor struct { // As each account can have its own polling frequency set we need to run one Go routine for // each, as well as keeping an eye on newly created user accounts. func (im *Monitor) start(ctx context.Context) { - usermap := make(map[int64]int) // Keep track of running go routines, one per user. We assume incrementing non-repeating UIDs (for the case where users are deleted and re-added). for { @@ -58,7 +60,6 @@ func (im *Monitor) start(ctx context.Context) { // monitor will continuously login to the IMAP settings associated to the supplied user id (if the user account has IMAP settings, and they're enabled.) // It also verifies the user account exists, and returns if not (for the case of a user being deleted). func monitor(uid int64, ctx context.Context) { - for { select { case <-ctx.Done(): @@ -80,7 +81,7 @@ func monitor(uid int64, ctx context.Context) { im := imapSettings[0] // 3. Check if IMAP is enabled if im.Enabled { - log.Debug("Checking IMAP for user ", uid, ": ", im.Username, "@", im.Host) + log.Debug("Checking IMAP for user ", uid, ": ", im.Username, " -> ", im.Host) checkForNewEmails(im) time.Sleep((time.Duration(im.IMAPFreq) - 10) * time.Second) // Subtract 10 to compensate for the default sleep of 10 at the bottom } @@ -92,7 +93,6 @@ func monitor(uid int64, ctx context.Context) { // NewMonitor returns a new instance of imap.Monitor func NewMonitor() *Monitor { - im := &Monitor{} return im } @@ -116,7 +116,6 @@ func (im *Monitor) Shutdown() error { // checkForNewEmails logs into an IMAP account and checks unread emails // for the rid campaign identifier. func checkForNewEmails(im models.IMAP) { - im.Host = im.Host + ":" + strconv.Itoa(int(im.Port)) // Append port mailServer := Mailbox{ Host: im.Host, @@ -134,8 +133,9 @@ func checkForNewEmails(im models.IMAP) { err = models.SuccessfulLogin(&im) if len(msgs) > 0 { - var reportingFailed []uint32 // UIDs of emails that were unable to be reported to phishing server, mark as unread - var campaignEmails []uint32 // UIDs of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these + log.Debugf("%d new emails for %s", len(msgs), im.Username) + var reportingFailed []uint32 // SeqNums of emails that were unable to be reported to phishing server, mark as unread + var deleteEmails []uint32 // SeqNums of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these for _, m := range msgs { // Check if sender is from company's domain, if enabled. TODO: Make this an IMAP filter if im.RestrictDomain != "" { // e.g domainResitct = widgets.com @@ -147,48 +147,88 @@ func checkForNewEmails(im models.IMAP) { } } - body := string(append(m.Email.Text, m.Email.HTML...)) // Not sure if we need to check the Text as well as the HTML. Perhaps sometimes Text only emails won't have an HTML component? - rid := goPhishRegex.FindString(body) + rids, err := matchEmail(m.Email) // Search email Text, HTML, and each attachment for rid parameters - if rid != "" { - rid = rid[5:] + if err != nil { + log.Errorf("Error searching email for rids from user '%s': %s", m.Email.From, err.Error()) + continue + } + if len(rids) < 1 { + // In the future this should be an alert in Gophish + log.Infof("User '%s' reported email with subject '%s'. This is not a GoPhish campaign; you should investigate it.", m.Email.From, m.Email.Subject) + } + for rid := range rids { log.Infof("User '%s' reported email with rid %s", m.Email.From, rid) result, err := models.GetResult(rid) if err != nil { log.Error("Error reporting GoPhish email with rid ", rid, ": ", err.Error()) - reportingFailed = append(reportingFailed, m.UID) - } else { - err = result.HandleEmailReport(models.EventDetails{}) - if err != nil { - log.Error("Error updating GoPhish email with rid ", rid, ": ", err.Error()) - } else { - if im.DeleteReportedCampaignEmail { - campaignEmails = append(campaignEmails, m.UID) - } - } + reportingFailed = append(reportingFailed, m.SeqNum) + continue } - } else { - // In the future this should be an alert in Gophish - log.Debugf("User '%s' reported email with subject '%s'. This is not a GoPhish campaign; you should investigate it.\n", m.Email.From, m.Email.Subject) - } - // Check if any emails were unable to be reported, so we can mark them as unread - if len(reportingFailed) > 0 { - log.Debugf("Marking %d emails as unread as failed to report\n", len(reportingFailed)) - err := mailServer.MarkAsUnread(reportingFailed) // Set emails as unread that we failed to report to GoPhish + err = result.HandleEmailReport(models.EventDetails{}) if err != nil { - log.Error("Unable to mark emails as unread: ", err.Error()) + log.Error("Error updating GoPhish email with rid ", rid, ": ", err.Error()) + continue + } + if im.DeleteReportedCampaignEmail == true { + deleteEmails = append(deleteEmails, m.SeqNum) } } - // If the DeleteReportedCampaignEmail flag is set, delete reported Gophish campaign emails - if im.DeleteReportedCampaignEmail && len(campaignEmails) > 0 { - log.Debugf("Deleting %d campaign emails\n", len(campaignEmails)) - err := mailServer.DeleteEmails(campaignEmails) // Delete GoPhish campaign emails. - if err != nil { - log.Error("Failed to delete emails: ", err.Error()) - } + + } + // Check if any emails were unable to be reported, so we can mark them as unread + if len(reportingFailed) > 0 { + log.Debugf("Marking %d emails as unread as failed to report", len(reportingFailed)) + err := mailServer.MarkAsUnread(reportingFailed) // Set emails as unread that we failed to report to GoPhish + if err != nil { + log.Error("Unable to mark emails as unread: ", err.Error()) } } + // If the DeleteReportedCampaignEmail flag is set, delete reported Gophish campaign emails + if len(deleteEmails) > 0 { + log.Debugf("Deleting %d campaign emails", len(deleteEmails)) + err := mailServer.DeleteEmails(deleteEmails) // Delete GoPhish campaign emails. + if err != nil { + log.Error("Failed to delete emails: ", err.Error()) + } + } + } else { log.Debug("No new emails for ", im.Username) } } + +func checkRIDs(em *email.Email, rids map[string]bool){ + // Check Text and HTML + emailContent := string(em.Text) + string(em.HTML) + for _, r := range goPhishRegex.FindAllStringSubmatch(emailContent, -1) { + newrid := r[len(r)-1] + if !rids[newrid] { + rids[newrid] = true + } + } +} + +// returns a slice of gophish rid paramters found in the email HTML, Text, and attachments +func matchEmail(em *email.Email) (map[string]bool, error) { + rids := make(map[string]bool) + checkRIDs(em, rids) + + // Next check each attachment + for _, a := range em.Attachments { + ext := filepath.Ext(a.Filename) + if a.Header.Get("Content-Type") == "message/rfc822" || ext == ".eml" { + + // Let's decode the email + rawBodyStream := bytes.NewReader(a.Content) + attachmentEmail, err := email.NewEmailFromReader(rawBodyStream) + if err != nil { + return rids, err + } + + checkRIDs(attachmentEmail, rids) + } + } + + return rids, nil +}