cristiancmoises
/
gophish
mirror of https://github.com/gophish/gophish
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

IMAP update; new library and attachment support (#1791) 

Updates the IMAP processing to use a more mature library. This allows for more robust IMAP support.

Additionally, this adds support for reporting emails as attachments.
pull/1867/head
Glenn Wilkinson 2020-06-17 02:02:09 +01:00 committed by GitHub
parent 8ebdb43469
commit 6f95da00ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 200 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

395
imap/imap.go
View File

@ -1,34 +1,34 @@
package imap package imap
// Functionality taken from https://github.com/jprobinson/eazye
import ( import (
"bytes" "bytes"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
"regexp"
"strconv" "strconv"
"time" "time"
"github.com/emersion/go-imap"
"github.com/emersion/go-imap/client"
"github.com/emersion/go-message/charset"
log "github.com/gophish/gophish/logger" log "github.com/gophish/gophish/logger"
"github.com/gophish/gophish/models" "github.com/gophish/gophish/models"
"github.com/jordan-wright/email" "github.com/jordan-wright/email"
"github.com/mxk/go-imap/imap"
) )
// Client interface for IMAP interactions // Client interface for IMAP interactions
type Client interface { type Client interface {
Close(expunge bool) (cmd *imap.Command, err error)
Login(username, password string) (cmd *imap.Command, err error) Login(username, password string) (cmd *imap.Command, err error)
Logout(timeout time.Duration) (cmd *imap.Command, err error) Logout(timeout time.Duration) (cmd *imap.Command, err error)
Select(mbox string, readonly bool) (cmd *imap.Command, err error) Select(name string, readOnly bool) (mbox *imap.MailboxStatus, err error)
UIDFetch(seq *imap.SeqSet, items ...string) (cmd *imap.Command, err error) Store(seq *imap.SeqSet, item imap.StoreItem, value interface{}, ch chan *imap.Message) (err error)
UIDSearch(spec ...imap.Field) (cmd *imap.Command, err error) Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) (err error)
UIDStore(seq *imap.SeqSet, item string, value imap.Field) (cmd *imap.Command, err error)
} }
// Email represents an email.Email with an included IMAP UID // Email represents an email.Email with an included IMAP Sequence Number
type Email struct { type Email struct {
UID uint32 `json:"uid"` SeqNum uint32 `json:"seqnum"`
*email.Email *email.Email
} }
@ -44,98 +44,8 @@ type Mailbox struct {
ReadOnly bool ReadOnly bool
} }
// GetAll will pull all emails from the email folder and return them as a list.
func (mbox *Mailbox) GetAll(markAsRead, delete bool) ([]Email, error) {
// call chan, put 'em in a list, return
var emails []Email
responses, err := mbox.GenerateAll(markAsRead, delete)
if err != nil {
return emails, err
}
for resp := range responses {
if resp.Err != nil {
return emails, resp.Err
}
emails = append(emails, resp.Email)
}
return emails, nil
}
// GenerateAll will find all emails in the email folder and pass them along to the responses channel.
func (mbox *Mailbox) GenerateAll(markAsRead, delete bool) (chan Response, error) {
return mbox.generateMail("ALL", nil, markAsRead, delete)
}
// GetUnread will find all unread emails in the folder and return them as a list.
func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) {
// call chan, put 'em in a list, return
var emails []Email
responses, err := mbox.GenerateUnread(markAsRead, delete)
if err != nil {
return emails, err
}
for resp := range responses {
if resp.Err != nil {
return emails, resp.Err
}
emails = append(emails, resp.Email)
}
return emails, nil
}
// GenerateUnread will find all unread emails in the folder and pass them along to the responses channel.
func (mbox *Mailbox) GenerateUnread(markAsRead, delete bool) (chan Response, error) {
return mbox.generateMail("UNSEEN", nil, markAsRead, delete)
}
// MarkAsUnread will set the UNSEEN flag on a supplied slice of UIDs
func (mbox *Mailbox) MarkAsUnread(uids []uint32) error {
client, err := mbox.newClient()
if err != nil {
return err
}
defer func() {
client.Close(true)
client.Logout(30 * time.Second)
}()
for _, u := range uids {
err := alterEmail(client, u, "\\SEEN", false)
if err != nil {
return err //return on first failure
}
}
return nil
}
// DeleteEmails will delete emails from the supplied slice of UIDs
func (mbox *Mailbox) DeleteEmails(uids []uint32) error {
client, err := mbox.newClient()
if err != nil {
return err
}
defer func() {
client.Close(true)
client.Logout(30 * time.Second)
}()
for _, u := range uids {
err := deleteEmail(client, u)
if err != nil {
return err //return on first failure
}
}
return nil
}
// Validate validates supplied IMAP model by connecting to the server // Validate validates supplied IMAP model by connecting to the server
func Validate(s *models.IMAP) error { func Validate(s *models.IMAP) error {
err := s.Validate() err := s.Validate()
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -150,179 +60,51 @@ func Validate(s *models.IMAP) error {
Pwd: s.Password, Pwd: s.Password,
Folder: s.Folder} Folder: s.Folder}
client, err := mailServer.newClient() imapClient, err := mailServer.newClient()
if err != nil { if err != nil {
log.Error(err.Error()) log.Error(err.Error())
} else { } else {
client.Close(true) imapClient.Logout()
client.Logout(30 * time.Second)
} }
return err return err
} }
// Response is a helper struct to wrap the email responses and possible errors. // MarkAsUnread will set the UNSEEN flag on a supplied slice of SeqNums
type Response struct { func (mbox *Mailbox) MarkAsUnread(seqs []uint32) error {
Email Email imapClient, err := mbox.newClient()
Err error
}
// newClient will initiate a new IMAP connection with the given creds.
func (mbox *Mailbox) newClient() (*imap.Client, error) {
var client *imap.Client
var err error
if mbox.TLS {
client, err = imap.DialTLS(mbox.Host, new(tls.Config))
if err != nil { if err != nil {
return client, err return err
} }
} else {
client, err = imap.Dial(mbox.Host) defer imapClient.Logout()
seqSet := new(imap.SeqSet)
seqSet.AddNum(seqs...)
item := imap.FormatFlagsOp(imap.RemoveFlags, true)
err = imapClient.Store(seqSet, item, imap.SeenFlag, nil)
if err != nil { if err != nil {
return client, err return err
}
} }
_, err = client.Login(mbox.User, mbox.Pwd) return nil
}
// DeleteEmails will delete emails from the supplied slice of SeqNums
func (mbox *Mailbox) DeleteEmails(seqs []uint32) error {
imapClient, err := mbox.newClient()
if err != nil { if err != nil {
return client, err return err
} }
_, err = imap.Wait(client.Select(mbox.Folder, mbox.ReadOnly)) defer imapClient.Logout()
if err != nil {
return client, err
}
return client, nil seqSet := new(imap.SeqSet)
} seqSet.AddNum(seqs...)
const dateFormat = "02-Jan-2006" item := imap.FormatFlagsOp(imap.AddFlags, true)
err = imapClient.Store(seqSet, item, imap.DeletedFlag, nil)
// findEmails will run a find the UIDs of any emails that match the search.:
func findEmails(client Client, search string, since *time.Time) (*imap.Command, error) {
var specs []imap.Field
if len(search) > 0 {
specs = append(specs, search)
}
if since != nil {
sinceStr := since.Format(dateFormat)
specs = append(specs, "SINCE", sinceStr)
}
// get headers and UID for UnSeen message in src inbox...
cmd, err := imap.Wait(client.UIDSearch(specs...))
if err != nil {
return &imap.Command{}, fmt.Errorf("uid search failed: %s", err)
}
return cmd, nil
}
const GenerateBufferSize = 100
func (mbox *Mailbox) generateMail(search string, since *time.Time, markAsRead, delete bool) (chan Response, error) {
responses := make(chan Response, GenerateBufferSize)
client, err := mbox.newClient()
if err != nil {
close(responses)
return responses, fmt.Errorf("failed to create IMAP connection: %s", err)
}
go func() {
defer func() {
client.Close(true)
client.Logout(30 * time.Second)
close(responses)
}()
var cmd *imap.Command
// find all the UIDs
cmd, err = findEmails(client, search, since)
if err != nil {
responses <- Response{Err: err}
return
}
// gotta fetch 'em all
getEmails(client, cmd, markAsRead, delete, responses)
}()
return responses, nil
}
func getEmails(client Client, cmd *imap.Command, markAsRead, delete bool, responses chan Response) {
seq := &imap.SeqSet{}
msgCount := 0
for _, rsp := range cmd.Data {
for _, uid := range rsp.SearchResults() {
msgCount++
seq.AddNum(uid)
}
}
if seq.Empty() {
return
}
fCmd, err := imap.Wait(client.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER"))
if err != nil {
responses <- Response{Err: fmt.Errorf("unable to perform uid fetch: %s", err)}
return
}
var email Email
for _, msgData := range fCmd.Data {
msgFields := msgData.MessageInfo().Attrs
// make sure is a legit response before we attempt to parse it
// deal with unsolicited FETCH responses containing only flags
// I'm lookin' at YOU, Gmail!
// http://mailman13.u.washington.edu/pipermail/imap-protocol/2014-October/002355.html
// http://stackoverflow.com/questions/26262472/gmail-imap-is-sometimes-returning-bad-results-for-fetch
if _, ok := msgFields["RFC822.HEADER"]; !ok {
continue
}
email, err = NewEmail(msgFields)
if err != nil {
responses <- Response{Err: fmt.Errorf("unable to parse email: %s", err)}
return
}
responses <- Response{Email: email}
if !markAsRead {
err = removeSeen(client, imap.AsNumber(msgFields["UID"]))
if err != nil {
responses <- Response{Err: fmt.Errorf("unable to remove seen flag: %s", err)}
return
}
}
if delete {
err = deleteEmail(client, imap.AsNumber(msgFields["UID"]))
if err != nil {
responses <- Response{Err: fmt.Errorf("unable to delete email: %s", err)}
return
}
}
}
}
func deleteEmail(client Client, UID uint32) error {
return alterEmail(client, UID, "\\DELETED", true)
}
func removeSeen(client Client, UID uint32) error {
return alterEmail(client, UID, "\\SEEN", false)
}
func alterEmail(client Client, UID uint32, flag string, plus bool) error {
flg := "-FLAGS"
if plus {
flg = "+FLAGS"
}
fSeq := &imap.SeqSet{}
fSeq.AddNum(UID)
_, err := imap.Wait(client.UIDStore(fSeq, flg, flag))
if err != nil { if err != nil {
return err return err
} }
@ -330,22 +112,95 @@ func alterEmail(client Client, UID uint32, flag string, plus bool) error {
return nil return nil
} }
// NewEmail will parse an imap.FieldMap into an Email. This // GetUnread will find all unread emails in the folder and return them as a list.
// will expect the message to container the internaldate and the body with func (mbox *Mailbox) GetUnread(markAsRead, delete bool) ([]Email, error) {
// all headers included. imap.CharsetReader = charset.Reader
func NewEmail(msgFields imap.FieldMap) (Email, error) { var emails []Email
rawBody := imap.AsBytes(msgFields["BODY[]"]) imapClient, err := mbox.newClient()
rawBodyStream := bytes.NewReader(rawBody)
em, err := email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library
if err != nil { if err != nil {
return Email{}, err return emails, fmt.Errorf("failed to create IMAP connection: %s", err)
}
iem := Email{
Email: em,
UID: imap.AsNumber(msgFields["UID"]),
} }
return iem, err defer imapClient.Logout()
// Search for unread emails
criteria := imap.NewSearchCriteria()
criteria.WithoutFlags = []string{imap.SeenFlag}
seqs, err := imapClient.Search(criteria)
if err != nil {
return emails, err
}
if len(seqs) == 0 {
return emails, nil
}
seqset := new(imap.SeqSet)
seqset.AddNum(seqs...)
section := &imap.BodySectionName{}
items := []imap.FetchItem{imap.FetchEnvelope, imap.FetchFlags, imap.FetchInternalDate, section.FetchItem()}
messages := make(chan *imap.Message)
go func() {
if err := imapClient.Fetch(seqset, items, messages); err != nil {
log.Error("Error fetching emails: ", err.Error()) // TODO: How to handle this, need to propogate error out
}
}()
// Step through each email
for msg := range messages {
// Extract raw message body. I can't find a better way to do this with the emersion library
var em *email.Email
var buf []byte
for _, value := range msg.Body {
buf = make([]byte, value.Len())
value.Read(buf)
break // There should only ever be one item in this map, but I'm not 100% sure
}
//Remove CR characters, see https://github.com/jordan-wright/email/issues/106
tmp := string(buf)
re := regexp.MustCompile(`\r`)
tmp = re.ReplaceAllString(tmp, "")
buf = []byte(tmp)
rawBodyStream := bytes.NewReader(buf)
em, err = email.NewEmailFromReader(rawBodyStream) // Parse with @jordanwright's library
if err != nil {
return emails, err
}
emtmp := Email{Email: em, SeqNum: msg.SeqNum} // Not sure why msg.Uid is always 0, so swapped to sequence numbers
emails = append(emails, emtmp)
}
return emails, nil
}
// newClient will initiate a new IMAP connection with the given creds.
func (mbox *Mailbox) newClient() (*client.Client, error) {
var imapClient *client.Client
var err error
if mbox.TLS {
imapClient, err = client.DialTLS(mbox.Host, new(tls.Config))
} else {
imapClient, err = client.Dial(mbox.Host)
}
if err != nil {
return imapClient, err
}
err = imapClient.Login(mbox.User, mbox.Pwd)
if err != nil {
return imapClient, err
}
_, err = imapClient.Select(mbox.Folder, mbox.ReadOnly)
if err != nil {
return imapClient, err
}
return imapClient, nil
} }

92
imap/monitor.go
View File

@ -7,19 +7,22 @@ package imap
* - Add field to User for numner of non-campaign emails reported * - Add field to User for numner of non-campaign emails reported
*/ */
import ( import (
"bytes"
"context" "context"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"path/filepath"
log "github.com/gophish/gophish/logger" log "github.com/gophish/gophish/logger"
"github.com/jordan-wright/email"
"github.com/gophish/gophish/models" "github.com/gophish/gophish/models"
) )
// Pattern for GoPhish emails e.g ?rid=AbC123 // Pattern for GoPhish emails e.g ?rid=AbC123
var goPhishRegex = regexp.MustCompile("(\\?rid=[A-Za-z0-9]{7})") var goPhishRegex = regexp.MustCompile("(\\?rid=(3D)?([A-Za-z0-9]{7}))") // We include the optional quoted-printable 3D at the front, just in case decoding fails
// Monitor is a worker that monitors IMAP servers for reported campaign emails // Monitor is a worker that monitors IMAP servers for reported campaign emails
type Monitor struct { type Monitor struct {
@ -30,7 +33,6 @@ type Monitor struct {
// As each account can have its own polling frequency set we need to run one Go routine for // As each account can have its own polling frequency set we need to run one Go routine for
// each, as well as keeping an eye on newly created user accounts. // each, as well as keeping an eye on newly created user accounts.
func (im *Monitor) start(ctx context.Context) { func (im *Monitor) start(ctx context.Context) {
usermap := make(map[int64]int) // Keep track of running go routines, one per user. We assume incrementing non-repeating UIDs (for the case where users are deleted and re-added). usermap := make(map[int64]int) // Keep track of running go routines, one per user. We assume incrementing non-repeating UIDs (for the case where users are deleted and re-added).
for { for {
@ -58,7 +60,6 @@ func (im *Monitor) start(ctx context.Context) {
// monitor will continuously login to the IMAP settings associated to the supplied user id (if the user account has IMAP settings, and they're enabled.) // monitor will continuously login to the IMAP settings associated to the supplied user id (if the user account has IMAP settings, and they're enabled.)
// It also verifies the user account exists, and returns if not (for the case of a user being deleted). // It also verifies the user account exists, and returns if not (for the case of a user being deleted).
func monitor(uid int64, ctx context.Context) { func monitor(uid int64, ctx context.Context) {
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -80,7 +81,7 @@ func monitor(uid int64, ctx context.Context) {
im := imapSettings[0] im := imapSettings[0]
// 3. Check if IMAP is enabled // 3. Check if IMAP is enabled
if im.Enabled { if im.Enabled {
log.Debug("Checking IMAP for user ", uid, ": ", im.Username, "@", im.Host) log.Debug("Checking IMAP for user ", uid, ": ", im.Username, " -> ", im.Host)
checkForNewEmails(im) checkForNewEmails(im)
time.Sleep((time.Duration(im.IMAPFreq) - 10) * time.Second) // Subtract 10 to compensate for the default sleep of 10 at the bottom time.Sleep((time.Duration(im.IMAPFreq) - 10) * time.Second) // Subtract 10 to compensate for the default sleep of 10 at the bottom
} }
@ -92,7 +93,6 @@ func monitor(uid int64, ctx context.Context) {
// NewMonitor returns a new instance of imap.Monitor // NewMonitor returns a new instance of imap.Monitor
func NewMonitor() *Monitor { func NewMonitor() *Monitor {
im := &Monitor{} im := &Monitor{}
return im return im
} }
@ -116,7 +116,6 @@ func (im *Monitor) Shutdown() error {
// checkForNewEmails logs into an IMAP account and checks unread emails // checkForNewEmails logs into an IMAP account and checks unread emails
// for the rid campaign identifier. // for the rid campaign identifier.
func checkForNewEmails(im models.IMAP) { func checkForNewEmails(im models.IMAP) {
im.Host = im.Host + ":" + strconv.Itoa(int(im.Port)) // Append port im.Host = im.Host + ":" + strconv.Itoa(int(im.Port)) // Append port
mailServer := Mailbox{ mailServer := Mailbox{
Host: im.Host, Host: im.Host,
@ -134,8 +133,9 @@ func checkForNewEmails(im models.IMAP) {
err = models.SuccessfulLogin(&im) err = models.SuccessfulLogin(&im)
if len(msgs) > 0 { if len(msgs) > 0 {
var reportingFailed []uint32 // UIDs of emails that were unable to be reported to phishing server, mark as unread log.Debugf("%d new emails for %s", len(msgs), im.Username)
var campaignEmails []uint32 // UIDs of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these var reportingFailed []uint32 // SeqNums of emails that were unable to be reported to phishing server, mark as unread
var deleteEmails []uint32 // SeqNums of campaign emails. If DeleteReportedCampaignEmail is true, we will delete these
for _, m := range msgs { for _, m := range msgs {
// Check if sender is from company's domain, if enabled. TODO: Make this an IMAP filter // Check if sender is from company's domain, if enabled. TODO: Make this an IMAP filter
if im.RestrictDomain != "" { // e.g domainResitct = widgets.com if im.RestrictDomain != "" { // e.g domainResitct = widgets.com
@ -147,48 +147,88 @@ func checkForNewEmails(im models.IMAP) {
} }
} }
body := string(append(m.Email.Text, m.Email.HTML...)) // Not sure if we need to check the Text as well as the HTML. Perhaps sometimes Text only emails won't have an HTML component? rids, err := matchEmail(m.Email) // Search email Text, HTML, and each attachment for rid parameters
rid := goPhishRegex.FindString(body)
if rid != "" { if err != nil {
rid = rid[5:] log.Errorf("Error searching email for rids from user '%s': %s", m.Email.From, err.Error())
continue
}
if len(rids) < 1 {
// In the future this should be an alert in Gophish
log.Infof("User '%s' reported email with subject '%s'. This is not a GoPhish campaign; you should investigate it.", m.Email.From, m.Email.Subject)
}
for rid := range rids {
log.Infof("User '%s' reported email with rid %s", m.Email.From, rid) log.Infof("User '%s' reported email with rid %s", m.Email.From, rid)
result, err := models.GetResult(rid) result, err := models.GetResult(rid)
if err != nil { if err != nil {
log.Error("Error reporting GoPhish email with rid ", rid, ": ", err.Error()) log.Error("Error reporting GoPhish email with rid ", rid, ": ", err.Error())
reportingFailed = append(reportingFailed, m.UID) reportingFailed = append(reportingFailed, m.SeqNum)
} else { continue
}
err = result.HandleEmailReport(models.EventDetails{}) err = result.HandleEmailReport(models.EventDetails{})
if err != nil { if err != nil {
log.Error("Error updating GoPhish email with rid ", rid, ": ", err.Error()) log.Error("Error updating GoPhish email with rid ", rid, ": ", err.Error())
} else { continue
if im.DeleteReportedCampaignEmail { }
campaignEmails = append(campaignEmails, m.UID) if im.DeleteReportedCampaignEmail == true {
deleteEmails = append(deleteEmails, m.SeqNum)
} }
} }
}
} else {
// In the future this should be an alert in Gophish
log.Debugf("User '%s' reported email with subject '%s'. This is not a GoPhish campaign; you should investigate it.\n", m.Email.From, m.Email.Subject)
} }
// Check if any emails were unable to be reported, so we can mark them as unread // Check if any emails were unable to be reported, so we can mark them as unread
if len(reportingFailed) > 0 { if len(reportingFailed) > 0 {
log.Debugf("Marking %d emails as unread as failed to report\n", len(reportingFailed)) log.Debugf("Marking %d emails as unread as failed to report", len(reportingFailed))
err := mailServer.MarkAsUnread(reportingFailed) // Set emails as unread that we failed to report to GoPhish err := mailServer.MarkAsUnread(reportingFailed) // Set emails as unread that we failed to report to GoPhish
if err != nil { if err != nil {
log.Error("Unable to mark emails as unread: ", err.Error()) log.Error("Unable to mark emails as unread: ", err.Error())
} }
} }
// If the DeleteReportedCampaignEmail flag is set, delete reported Gophish campaign emails // If the DeleteReportedCampaignEmail flag is set, delete reported Gophish campaign emails
if im.DeleteReportedCampaignEmail && len(campaignEmails) > 0 { if len(deleteEmails) > 0 {
log.Debugf("Deleting %d campaign emails\n", len(campaignEmails)) log.Debugf("Deleting %d campaign emails", len(deleteEmails))
err := mailServer.DeleteEmails(campaignEmails) // Delete GoPhish campaign emails. err := mailServer.DeleteEmails(deleteEmails) // Delete GoPhish campaign emails.
if err != nil { if err != nil {
log.Error("Failed to delete emails: ", err.Error()) log.Error("Failed to delete emails: ", err.Error())
} }
} }
}
} else { } else {
log.Debug("No new emails for ", im.Username) log.Debug("No new emails for ", im.Username)
} }
} }
func checkRIDs(em *email.Email, rids map[string]bool){
// Check Text and HTML
emailContent := string(em.Text) + string(em.HTML)
for _, r := range goPhishRegex.FindAllStringSubmatch(emailContent, -1) {
newrid := r[len(r)-1]
if !rids[newrid] {
rids[newrid] = true
}
}
}
// returns a slice of gophish rid paramters found in the email HTML, Text, and attachments
func matchEmail(em *email.Email) (map[string]bool, error) {
rids := make(map[string]bool)
checkRIDs(em, rids)
// Next check each attachment
for _, a := range em.Attachments {
ext := filepath.Ext(a.Filename)
if a.Header.Get("Content-Type") == "message/rfc822" || ext == ".eml" {
// Let's decode the email
rawBodyStream := bytes.NewReader(a.Content)
attachmentEmail, err := email.NewEmailFromReader(rawBodyStream)
if err != nil {
return rids, err
}
checkRIDs(attachmentEmail, rids)
}
}
return rids, nil
}