From 2149bb7fb6b6f732ac8364e45d072a22921957db Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Mon, 22 Jun 2020 00:45:14 +0000 Subject: Rework the data model to improve the performance Signed-off-by: Max Magorsch --- pkg/app/home/home.go | 20 +-- pkg/app/home/utils.go | 4 +- pkg/app/list/messages.go | 20 +-- pkg/app/list/show.go | 18 +-- pkg/app/list/threads.go | 23 ++-- pkg/app/message/show.go | 8 +- pkg/app/message/utils.go | 53 +++++--- pkg/app/popular/utils.go | 17 ++- pkg/app/search/search.go | 8 +- pkg/database/connection.go | 15 ++- pkg/importer/importer.go | 23 +--- pkg/importer/utils.go | 225 +++++++++++++++++++++++++------- pkg/models/message.go | 116 ++++------------ web/templates/home/home.tmpl | 4 +- web/templates/list/messages.tmpl | 2 +- web/templates/list/threads.tmpl | 2 +- web/templates/message/show.tmpl | 16 +-- web/templates/popular/threads.tmpl | 4 +- web/templates/search/searchresults.tmpl | 2 +- 19 files changed, 326 insertions(+), 254 deletions(-) diff --git a/pkg/app/home/home.go b/pkg/app/home/home.go index e2d3955..fe8a56e 100644 --- a/pkg/app/home/home.go +++ b/pkg/app/home/home.go @@ -21,17 +21,17 @@ func Show(w http.ResponseWriter, r *http.Request) { var messages []*models.Message database.DBCon.Model(&messages). WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + mailingList[0] + `]%'`). - WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + mailingList[0] + `]%'`) - return q, nil - }). - WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`) + q = q.WhereOr(`subject LIKE '[` + mailingList[0] + `]%'`). + WhereOr(`subject LIKE 'Re: [` + mailingList[0] + `]%'`) return q, nil }). + //WhereGroup(func(q *orm.Query) (*orm.Query, error) { + // q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`) + // return q, nil + //}). Order("date DESC"). Limit(5). Select() @@ -57,7 +57,7 @@ func Show(w http.ResponseWriter, r *http.Request) { templateData := struct { MailingLists []models.MailingList - PopularThreads models.Threads + PopularThreads []*models.Message MessageCount string CurrentMonth string }{ diff --git a/pkg/app/home/utils.go b/pkg/app/home/utils.go index f854549..767b60d 100644 --- a/pkg/app/home/utils.go +++ b/pkg/app/home/utils.go @@ -19,7 +19,7 @@ func renderIndexTemplate(w http.ResponseWriter, templateData interface{}) { Funcs(template.FuncMap{ "makeMessage": func(headers map[string][]string) models.Message { return models.Message{ - Headers: headers, + //Headers: headers, } }, }). @@ -35,7 +35,7 @@ func getAllMessagesCount() int { var messsageCount int database.DBCon.Model((*models.Message)(nil)).QueryOne(pg.Scan(&messsageCount), ` SELECT - count(DISTINCT messages.headers->>'Message-Id') + count(DISTINCT messages.message_id) FROM messages; `) diff --git a/pkg/app/list/messages.go b/pkg/app/list/messages.go index 383e891..7a78811 100644 --- a/pkg/app/list/messages.go +++ b/pkg/app/list/messages.go @@ -33,20 +33,20 @@ func Messages(w http.ResponseWriter, r *http.Request) { var messages []*models.Message query := database.DBCon.Model(&messages). - Column("id", "headers", "date"). + Column("id", "subject", "from", "date"). Where("to_char(date, 'YYYY-MM') = ?", combinedDate). WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`). - WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`) - return q, nil - }). - WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`) + q = q.WhereOr(`subject LIKE '[` + listName + `]%'`). + WhereOr(`subject LIKE 'Re: [` + listName + `]%'`) return q, nil }). + //WhereGroup(func(q *orm.Query) (*orm.Query, error) { + // q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`) + // return q, nil + //}). Order("date DESC") messagesCount, _ := query.Count() diff --git a/pkg/app/list/show.go b/pkg/app/list/show.go index 8db8778..d90f236 100644 --- a/pkg/app/list/show.go +++ b/pkg/app/list/show.go @@ -18,17 +18,17 @@ func Show(w http.ResponseWriter, r *http.Request) { } err := database.DBCon.Model((*models.Message)(nil)). WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`). - WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`) - return q, nil - }). - WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`) + q = q.WhereOr(`subject LIKE '[` + listName + `]%'`). + WhereOr(`subject LIKE 'Re: [` + listName + `]%'`) return q, nil }). + //WhereGroup(func(q *orm.Query) (*orm.Query, error) { + // q = q.WhereOr(`to LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`cc LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`to LIKE '%` + listName + `@gentoo.org%'`). + // WhereOr(`cc LIKE '%` + listName + `@gentoo.org%'`) + // return q, nil + //}). ColumnExpr("to_char(date, 'YYYY-MM') AS combined_date"). ColumnExpr("count(*) AS message_count"). Group("combined_date"). diff --git a/pkg/app/list/threads.go b/pkg/app/list/threads.go index 33ade3c..c069673 100644 --- a/pkg/app/list/threads.go +++ b/pkg/app/list/threads.go @@ -32,22 +32,21 @@ func Threads(w http.ResponseWriter, r *http.Request) { var messages []*models.Message query := database.DBCon.Model(&messages). - Column("id", "headers", "date"). + Column("id", "subject", "from", "date"). Where("to_char(date, 'YYYY-MM') = ?", combinedDate). - Where(`NOT headers::jsonb ? 'References'`). - Where(`NOT headers::jsonb ? 'In-Reply-To'`). + Where(`starts_thread = TRUE`). WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`). - WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`) - return q, nil - }). - WhereGroup(func(q *orm.Query) (*orm.Query, error) { - q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`). - WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`). - WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`) + q = q.WhereOr(`subject LIKE '[` + listName + `]%'`). + WhereOr(`subject LIKE 'Re: [` + listName + `]%'`) return q, nil }). + //WhereGroup(func(q *orm.Query) (*orm.Query, error) { + // q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`). + // WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`). + // WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`) + // return q, nil + //}). Order("date DESC") messagesCount, _ := query.Count() diff --git a/pkg/app/message/show.go b/pkg/app/message/show.go index a027596..3d56544 100644 --- a/pkg/app/message/show.go +++ b/pkg/app/message/show.go @@ -26,9 +26,9 @@ func Show(w http.ResponseWriter, r *http.Request) { var inReplyTos []*models.Message var inReplyTo *models.Message - if message.HasHeaderField("In-Reply-To") { + if message.InReplyTo != nil { err = database.DBCon.Model(&inReplyTos). - Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.GetHeaderField("In-Reply-To") + `'`). + Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.InReplyTo.Id + `'`). Select() if err != nil || len(inReplyTos) < 1 { inReplyTo = nil @@ -41,8 +41,8 @@ func Show(w http.ResponseWriter, r *http.Request) { var replies []*models.Message database.DBCon.Model(&replies). - Where(`(headers::jsonb->>'References')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`). - WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`). + Where(`(headers::jsonb->>'References')::jsonb ? '` + message.Id + `'`). + WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.Id + `'`). Order("date ASC").Select() renderMessageTemplate(w, listName, message, inReplyTo, replies) diff --git a/pkg/app/message/utils.go b/pkg/app/message/utils.go index 0cb40f5..f79e6f3 100644 --- a/pkg/app/message/utils.go +++ b/pkg/app/message/utils.go @@ -35,29 +35,40 @@ func renderMessageTemplate(w http.ResponseWriter, listName string, message *mode func getFuncMap() template.FuncMap { return template.FuncMap{ - "formatAddr": func(addr string) string { - if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") { - addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o") - addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o") - } else { - start := false - for i := len(addr) - 1; i > 0; i-- { - if addr[i] == '@' { - break - } - if start { - out := []rune(addr) - out[i] = '×' - addr = string(out) - } - if addr[i] == '.' { - start = true - } - } + "formatAddr": formatAddr, + "formatAddrList": formatAddrList, + } +} + +func formatAddr(addr string) string { + if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") { + addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o") + addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o") + } else { + start := false + for i := len(addr) - 1; i > 0; i-- { + if addr[i] == '@' { + break + } + if start { + out := []rune(addr) + out[i] = '×' + addr = string(out) } - return addr - }, + if addr[i] == '.' { + start = true + } + } + } + return addr +} + +func formatAddrList(addrList []string) string { + var formatedAddrList []string + for _, addr := range addrList { + formatedAddrList = append(formatedAddrList, formatAddr(addr)) } + return strings.Join(formatedAddrList, ", ") } func replaceAtIndex(in string, r rune, i int) string { diff --git a/pkg/app/popular/utils.go b/pkg/app/popular/utils.go index c5772d7..bc2a12f 100644 --- a/pkg/app/popular/utils.go +++ b/pkg/app/popular/utils.go @@ -16,7 +16,7 @@ func renderPopularThreads(w http.ResponseWriter, templateData interface{}) { Funcs(template.FuncMap{ "makeMessage": func(headers map[string][]string) models.Message { return models.Message{ - Headers: headers, + //Headers: headers, } }, }). @@ -28,17 +28,16 @@ func renderPopularThreads(w http.ResponseWriter, templateData interface{}) { // utility methods -func GetPopularThreads(n int, date string) (models.Threads, error) { - var popularThreads models.Threads - err := database.DBCon.Model(&popularThreads). - TableExpr(`(SELECT id, headers, regexp_replace(regexp_replace(regexp_replace(regexp_replace(headers::jsonb->>'Subject','^\["',''),'"\]$',''),'^Re:\s',''), '^\[.*\]', '') AS c FROM messages WHERE date >= '2020-06-12'::date) t`). - ColumnExpr(`c as Subject, jsonb_agg(id)->>0 as Id, jsonb_agg(headers)->>0 as Headers, Count(*) as Count`). - GroupExpr(`c`). - OrderExpr(`count DESC`). +func GetPopularThreads(n int, date string) ([]*models.Message, error) { + + var recentMessages []*models.Message + + err := database.DBCon.Model(&recentMessages). + OrderExpr("date DESC"). Limit(n). Select() - return popularThreads, err + return recentMessages, err } func GetMessagesFromPopularThreads(threads models.Threads) []*models.Message { diff --git a/pkg/app/search/search.go b/pkg/app/search/search.go index f6498e9..072bf2f 100644 --- a/pkg/app/search/search.go +++ b/pkg/app/search/search.go @@ -44,10 +44,10 @@ func Search(w http.ResponseWriter, r *http.Request) { // var searchResults []*models.Message query := database.DBCon.Model(&searchResults). - WhereOr(`headers::jsonb->>'From' LIKE ?`, "%"+searchTerm+"%"). + WhereOr(`message.from LIKE ?`, "%"+searchTerm+"%"). Order("date DESC") if showThreads { - query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`) + query = query.Where(`starts_thread = TRUE`) } messagesCount, _ := query.Count() @@ -65,7 +65,7 @@ func Search(w http.ResponseWriter, r *http.Request) { query = database.DBCon.Model(&searchResults). Where(`tsv_subject @@ to_tsquery(''?'')`, searchTerm) if showThreads { - query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`) + query = query.Where(`starts_thread = TRUE`) } messagesCount, _ = query.Count() @@ -83,7 +83,7 @@ func Search(w http.ResponseWriter, r *http.Request) { query = database.DBCon.Model(&searchResults). Where(`tsv_body @@ to_tsquery(''?'')`, searchTerm) if showThreads { - query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`) + query = query.Where(`starts_thread = TRUE`) } messagesCount, _ = query.Count() diff --git a/pkg/database/connection.go b/pkg/database/connection.go index 18fa2e6..06514df 100644 --- a/pkg/database/connection.go +++ b/pkg/database/connection.go @@ -21,9 +21,16 @@ var ( func CreateSchema() error { if !tableExists("messages") { - err := DBCon.CreateTable((*models.Message)(nil), &orm.CreateTableOptions{ - IfNotExists: true, - }) + for _, model := range []interface{}{(*models.Message)(nil), + (*models.MessageToReferences)(nil)} { + + err := DBCon.CreateTable(model, &orm.CreateTableOptions{ + IfNotExists: true, + }) + if err != nil { + return err + } + } // Add tsvector column for subjects DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_subject tsvector;") @@ -33,7 +40,7 @@ func CreateSchema() error { DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_body tsvector;") DBCon.Exec("CREATE INDEX body_idx ON messages USING gin(tsv_body);") - return err + return nil } return nil } diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go index 379332c..cdb278d 100644 --- a/pkg/importer/importer.go +++ b/pkg/importer/importer.go @@ -2,25 +2,14 @@ package importer import ( "archives/pkg/config" - "log" - "os" + "fmt" "path/filepath" ) func FullImport() { - err := filepath.Walk(config.MailDirPath(), - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 { - if isPublicList(path) { - importMail(info.Name(), path, config.MailDirPath()) - } - } - return nil - }) - if err != nil { - log.Println(err) - } + fmt.Println("Init import...") + filepath.Walk(config.MailDirPath(), initImport) + fmt.Println("Start import...") + filepath.Walk(config.MailDirPath(), importMail) + fmt.Println("Finished import.") } diff --git a/pkg/importer/utils.go b/pkg/importer/utils.go index b2a5b63..8383ad0 100644 --- a/pkg/importer/utils.go +++ b/pkg/importer/utils.go @@ -15,74 +15,217 @@ import ( "time" ) -func importMail(name, path, maildirPath string) { - file, _ := os.Open(path) - m, _ := mail.ReadMessage(file) - - msg := models.Message{ - Id: m.Header.Get("X-Archives-Hash"), - Filename: name, - Headers: m.Header, - Attachments: nil, - Body: getBody(m.Header, m.Body), - Date: getDate(m.Header), - Lists: getLists(m.Header), - List: getListName(path), - Comment: "", - Hidden: false, +type MailIdentifier struct { + ArchivesHash string + MessageId string + To string +} + +// TODO +var mails []*models.Message + +// TODO +func initImport(path string, info os.FileInfo, err error) error { + if err != nil { + return err } + if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) { - err := insertMessage(msg) + file, _ := os.Open(path) + m, _ := mail.ReadMessage(file) + mails = append(mails, &models.Message{ + Id: m.Header.Get("X-Archives-Hash"), + Filename: info.Name(), + From: m.Header.Get("From"), + To: strings.Split(m.Header.Get("To"), ","), + Subject: m.Header.Get("Subject"), + MessageId: m.Header.Get("Message-Id"), + }) + } + return nil +} + +// TODO +func importMail(path string, info os.FileInfo, err error) error { if err != nil { - fmt.Println("Error during importing Mail") - fmt.Println(err) + return err + } + if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) { + file, _ := os.Open(path) + m, _ := mail.ReadMessage(file) + + msg := models.Message{ + Id: m.Header.Get("X-Archives-Hash"), + MessageId: m.Header.Get("Message-Id"), + Filename: info.Name(), + From: m.Header.Get("From"), + To: strings.Split(m.Header.Get("To"), ","), + Cc: strings.Split(m.Header.Get("Cc"), ","), + Subject: m.Header.Get("Subject"), + + List: getListName(path), + + // TODO + Date: getDate(m.Header), + InReplyToId: getInReplyToMail(m.Header.Get("In-Reply-To"), m.Header.Get("From")), + //References: getReferencesToMail(strings.Split(m.Header.Get("References"), ","), m.Header.Get("From")), + Body: getBody(m.Header, m.Body), + Attachments: getAttachments(m.Header, m.Body), + + StartsThread: m.Header.Get("In-Reply-To") == "" && m.Header.Get("References") == "", + + Comment: "", + Hidden: false, + } + + err := insertMessage(msg) + + if err != nil { + fmt.Println("Error during importing Mail") + fmt.Println(err) + } + + insertReferencesToMail(strings.Split(m.Header.Get("References"), ","), m.Header.Get("X-Archives-Hash"), m.Header.Get("From")) + + } + return nil +} + +func getInReplyToMail(messageId, from string) string { + // step 1 TODO add description + for _, mail := range mails { + if mail.MessageId == messageId && strings.Contains(strings.Join(mail.To, ", "), from) { + return mail.Id + } + } + // step 2 TODO add description + for _, mail := range mails { + if mail.MessageId == messageId { + return mail.Id + } + } + return "" +} + + +func insertReferencesToMail(references []string, messageId, from string) []*models.Message { + var referencesToMail []*models.Message + for _, reference := range references { + // step 1 TODO add description + for _, mail := range mails { + if mail.MessageId == reference && strings.Contains(strings.Join(mail.To, ", "), from) { + referencesToMail = append(referencesToMail, mail) + } + } + // step 2 TODO add description + for _, mail := range mails { + if mail.MessageId == reference { + referencesToMail = append(referencesToMail, mail) + } + } + } + + for _, reference := range referencesToMail { + _, err := database.DBCon.Model(&models.MessageToReferences{ + MessageId: messageId, + ReferenceId: reference.Id, + }).Insert() + + if err != nil { + fmt.Println("Err inserting Message to references") + fmt.Println(err) + } } + + return referencesToMail } func getDepth(path, maildirPath string) int { return strings.Count(strings.ReplaceAll(path, maildirPath, ""), "/") } -func getBody(header mail.Header, body io.Reader) map[string]string { +func getBody(header mail.Header, body io.Reader) string { if isMultipartMail(header) { boundary := regexp.MustCompile(`boundary="(.*?)"`). FindStringSubmatch( header.Get("Content-Type")) if len(boundary) != 2 { //err - return map[string]string{ - "text/plain": "", + return "" + } + parsedBody := "" + mr := multipart.NewReader(body, boundary[1]) + for { + p, err := mr.NextPart() + if err != nil { + return parsedBody + } + bodyContent, err := ioutil.ReadAll(p) + if err != nil { + fmt.Println("Error while reading the body:") + fmt.Println(err) + continue + } + if strings.Contains(p.Header.Get("Content-Type"), "text/plain") { + return string(bodyContent) + } else if strings.Contains(p.Header.Get("Content-Type"), "text/html") { + parsedBody = string(bodyContent) } } - return getBodyParts(body, boundary[1]) + return parsedBody } else { content, _ := ioutil.ReadAll(body) - return map[string]string{ - getContentType(header): string(content), - } + return string(content) } } -func getBodyParts(body io.Reader, boundary string) map[string]string { - bodyParts := make(map[string]string) - mr := multipart.NewReader(body, boundary) + +func getAttachments(header mail.Header, body io.Reader) []models.Attachment { + + if !isMultipartMail(header) { + return nil + } + + boundary := regexp.MustCompile(`boundary="(.*?)"`). + FindStringSubmatch( + header.Get("Content-Type")) + if len(boundary) != 2 { + return nil + } + var attachments []models.Attachment + mr := multipart.NewReader(body, boundary[1]) for { p, err := mr.NextPart() if err != nil { - return bodyParts + return attachments } - slurp, err := ioutil.ReadAll(p) + content, err := ioutil.ReadAll(p) if err != nil { fmt.Println("Error while reading the body:") fmt.Println(err) continue } - bodyParts[p.Header.Get("Content-Type")] = string(slurp) + + attachments = append(attachments, models.Attachment{ + Filename: getAttachmentFileName(p.Header.Get("Content-Type")), + Mime: p.Header.Get("Content-Type"), + Content: string(content), + }) + } - return bodyParts + return attachments } +func getAttachmentFileName(contentTypeHeader string) string { + parts := strings.Split(contentTypeHeader, "name=") + if len(parts) < 2 { + return "unknown" + } + return strings.ReplaceAll(parts[1], "\"", "") +} + + func getContentType(header mail.Header) string { contentTypes := regexp.MustCompile(`(.*?);`). FindStringSubmatch( @@ -103,20 +246,6 @@ func isMultipartMail(header mail.Header) bool { return strings.Contains(getContentType(header), "multipart") } -func getLists(header mail.Header) []string { - var lists []string - // To - adr, _ := mail.ParseAddressList(header.Get("To")) - for _, v := range adr { - lists = append(lists, v.Address) - } - // Cc - adr, _ = mail.ParseAddressList(header.Get("Cc")) - for _, v := range adr { - lists = append(lists, v.Address) - } - return lists -} func getListName(path string) string { listName := strings.ReplaceAll(path, config.MailDirPath() + ".", "") @@ -126,8 +255,8 @@ func getListName(path string) string { func insertMessage(message models.Message) error { _, err := database.DBCon.Model(&message). - Value("tsv_subject", "to_tsvector(?)", message.GetSubject()). - Value("tsv_body", "to_tsvector(?)", message.GetBody()). + Value("tsv_subject", "to_tsvector(?)", message.Subject). + Value("tsv_body", "to_tsvector(?)", message.Body). OnConflict("(id) DO NOTHING"). Insert() return err diff --git a/pkg/models/message.go b/pkg/models/message.go index 6d8b299..89d7d73 100644 --- a/pkg/models/message.go +++ b/pkg/models/message.go @@ -1,7 +1,6 @@ package models import ( - "mime" "net/mail" "strings" "time" @@ -9,23 +8,35 @@ import ( type Message struct { Id string `pg:",pk"` + MessageId string Filename string - Headers map[string][]string - Body map[string]string - Attachments []Attachment - - Lists []string List string - Date time.Time - //Search types.ValueAppender // tsvector + From string + To []string + Cc []string + + Subject string + Body string + + Date time.Time + + // fk + InReplyTo *Message `pg:"fk:in_reply_to_id"` // fk specifies foreign key + InReplyToId string + + // many to many + //References []string + References []Message `pg:"many2many:message_to_references,joinFK:reference_id"` + + Attachments []Attachment + + StartsThread bool Comment string Hidden bool - //ParentId string - //Parent Message -> pg fk? } type Header struct { @@ -44,12 +55,13 @@ type Attachment struct { Content string } -func (m Message) GetSubject() string { - return m.GetHeaderField("Subject") +type MessageToReferences struct { + MessageId string + ReferenceId string } func (m Message) GetListNameFromSubject() string { - subject := m.GetSubject() + subject := m.Subject listName := strings.Split(subject, "]")[0] listName = strings.ReplaceAll(listName, "[", "") listName = strings.ReplaceAll(listName, "Re:", "") @@ -58,7 +70,7 @@ func (m Message) GetListNameFromSubject() string { } func (m Message) GetAuthorName() string { - addr, err := mail.ParseAddress(m.GetHeaderField("From")) + addr, err := mail.ParseAddress(m.From) if err != nil { return "" } @@ -66,7 +78,7 @@ func (m Message) GetAuthorName() string { } func (m Message) GetMessageId() string { - messageId := m.GetHeaderField("Message-Id") + messageId := m.MessageId messageId = strings.ReplaceAll(messageId, "<", "") messageId = strings.ReplaceAll(messageId, ">", "") messageId = strings.ReplaceAll(messageId, "\"", "") @@ -74,83 +86,11 @@ func (m Message) GetMessageId() string { } func (m Message) GetInReplyTo() string { - inReplyTo := m.GetHeaderField("In-Reply-To") + inReplyTo := m.InReplyTo.MessageId inReplyTo = strings.ReplaceAll(inReplyTo, "<", "") inReplyTo = strings.ReplaceAll(inReplyTo, ">", "") inReplyTo = strings.ReplaceAll(inReplyTo, " ", "") return inReplyTo } -func (m Message) GetHeaderField(key string) string { - subject, found := m.Headers[key] - if !found { - return "" - } - header := strings.Join(subject, " ") - if strings.Contains(header, "=?") { - dec := new(mime.WordDecoder) - decodedHeader, err := dec.DecodeHeader(header) - if err != nil { - return "" - } - return decodedHeader - } - return header -} - -func (m Message) HasHeaderField(key string) bool { - _, found := m.Headers[key] - return found -} - -func (m Message) GetBody() string { - // Get text/plain body - for contentType, content := range m.Body { - if strings.Contains(contentType, "text/plain") { - return content - } - } - - // If text/plain is not present, fall back to html - for contentType, content := range m.Body { - if strings.Contains(contentType, "text/html") { - return content - } - } - - // If neither text/plain nor text/html is available return nothing - return "" -} -func (m Message) HasAttachments() bool { - for key, _ := range m.Body { - if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) { - return true - } - } - return false -} - -func (m Message) GetAttachments() []Attachment { - var attachments []Attachment - for key, content := range m.Body { - if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) { - attachments = append(attachments, Attachment{ - Filename: getAttachmentFileName(key), - Mime: strings.Split(key, ";")[0], - Content: content, - }) - } - } - return attachments -} - -// utility methods - -func getAttachmentFileName(contentTypeHeader string) string { - parts := strings.Split(contentTypeHeader, "name=") - if len(parts) < 2 { - return "unknown" - } - return strings.ReplaceAll(parts[1], "\"", "") -} diff --git a/web/templates/home/home.tmpl b/web/templates/home/home.tmpl index 20fa1ba..39cf1fb 100644 --- a/web/templates/home/home.tmpl +++ b/web/templates/home/home.tmpl @@ -45,7 +45,7 @@

Recent Popular Threads

{{range .PopularThreads}} -

{{(makeMessage .Headers).GetHeaderField "Subject"}}

+

{{.Subject}}

{{end}}
@@ -87,7 +87,7 @@ {{$listName:=.Name}} {{range .Messages}} - {{.GetHeaderField "Subject"}} + {{.Subject}} {{.GetAuthorName}} {{end}} diff --git a/web/templates/list/messages.tmpl b/web/templates/list/messages.tmpl index 31084bd..fff6a53 100644 --- a/web/templates/list/messages.tmpl +++ b/web/templates/list/messages.tmpl @@ -28,7 +28,7 @@ {{range .Messages}} - {{.GetHeaderField "Subject"}} + {{.Subject}} {{.GetAuthorName}} {{.Date.Format "Mon, 2 Jan 2006 15:04:05"}} diff --git a/web/templates/list/threads.tmpl b/web/templates/list/threads.tmpl index ec07e49..933417a 100644 --- a/web/templates/list/threads.tmpl +++ b/web/templates/list/threads.tmpl @@ -28,7 +28,7 @@ {{range .Messages}} - {{.GetHeaderField "Subject"}} + {{.Subject}} {{.GetAuthorName}} {{.Date.Format "Mon, 2 Jan 2006 15:04:05"}} diff --git a/web/templates/message/show.tmpl b/web/templates/message/show.tmpl index 7b88c35..76ffdf4 100644 --- a/web/templates/message/show.tmpl +++ b/web/templates/message/show.tmpl @@ -13,22 +13,22 @@ - + - + - {{if .Message.HasHeaderField "Cc"}} + {{if .Message.Cc}} - + {{end}} - + @@ -49,9 +49,9 @@
From:{{formatAddr (.Message.GetHeaderField "From")}}{{formatAddr (.Message.From)}}
To:{{formatAddr (.Message.GetHeaderField "To")}}{{formatAddrList (.Message.To)}}
Cc:{{formatAddr (.Message.GetHeaderField "Cc")}}{{formatAddrList (.Message.Cc)}}
Subject:{{.Message.GetHeaderField "Subject"}}{{.Message.Subject}}
Date:
- 
{{.Message.GetBody}}
+ 
{{.Message.Body}}
- {{if .Message.HasAttachments }} + {{if .Message.Attachments }}

Attachments

@@ -61,7 +61,7 @@ MIME type - {{range .Message.GetAttachments}} + {{range .Message.Attachments}} {{.Filename}} {{.Mime}} diff --git a/web/templates/popular/threads.tmpl b/web/templates/popular/threads.tmpl index 5b849eb..4dfcbbc 100644 --- a/web/templates/popular/threads.tmpl +++ b/web/templates/popular/threads.tmpl @@ -12,13 +12,11 @@ - {{range .}} - - + {{end}} diff --git a/web/templates/search/searchresults.tmpl b/web/templates/search/searchresults.tmpl index ecd64f0..19ed5e4 100644 --- a/web/templates/search/searchresults.tmpl +++ b/web/templates/search/searchresults.tmpl @@ -28,7 +28,7 @@ {{range .Messages}} - + -- cgit v1.2.3-65-gdbad
ThreadNumber of messages
{{(makeMessage .Headers).GetSubject}}{{.Count}}{{.Subject}}
{{.GetHeaderField "Subject"}}{{.Subject}} {{.GetAuthorName}} {{.Date.Format "Mon, 2 Jan 2006 15:04:05"}}