Browse Source
Rewrite reference processing code in preparation for opening/closing from comment references (#8261)
Rewrite reference processing code in preparation for opening/closing from comment references (#8261)
* Add a markdown stripper for mentions and xrefs * Improve comments * Small code simplification * Move reference code to modules/references * Fix typo * Make MarkdownStripper return [][]byte * Implement preliminary keywords parsing * Add FIXME comment * Fix comment * make fmt * Fix permissions check * Fix text assumptions * Fix imports * Fix lint, fmt * Fix unused import * Add missing export comment * Bypass revive on implemented interface * Move mdstripper into its own package * Support alphanumeric patterns * Refactor FindAllMentions * Move mentions test to references * Parse mentions from reference package * Refactor code to implement renderizable references * Fix typo * Move patterns and tests to the references package * Fix nil reference * Preliminary rendering attempt of closing keywords * Normalize names, comments, general tidy-up * Add CSS style for action keywords * Fix permission for admin and owner * Fix golangci-lint * Fix golangci-lintrelease/v1.10
guillep2k
3 years ago
committed by
zeripath
zeripath
16 changed files with 1116 additions and 431 deletions
-
13integrations/issue_test.go
-
177models/action.go
-
53models/action_test.go
-
11models/issue_comment.go
-
111models/issue_xref.go
-
124modules/markup/html.go
-
92modules/markup/html_internal_test.go
-
260modules/markup/mdstripper/mdstripper.go
-
71modules/markup/mdstripper/mdstripper_test.go
-
3modules/markup/sanitizer.go
-
322modules/references/references.go
-
296modules/references/references_test.go
-
1public/css/index.css
-
5public/less/_repository.less
-
4services/mailer/mail_comment.go
-
4services/mailer/mail_issue.go
@ -0,0 +1,260 @@ |
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package mdstripper |
|||
|
|||
import ( |
|||
"bytes" |
|||
|
|||
"github.com/russross/blackfriday" |
|||
) |
|||
|
|||
// MarkdownStripper extends blackfriday.Renderer
|
|||
type MarkdownStripper struct { |
|||
blackfriday.Renderer |
|||
links []string |
|||
coallesce bool |
|||
} |
|||
|
|||
const ( |
|||
blackfridayExtensions = 0 | |
|||
blackfriday.EXTENSION_NO_INTRA_EMPHASIS | |
|||
blackfriday.EXTENSION_TABLES | |
|||
blackfriday.EXTENSION_FENCED_CODE | |
|||
blackfriday.EXTENSION_STRIKETHROUGH | |
|||
blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK | |
|||
blackfriday.EXTENSION_DEFINITION_LISTS | |
|||
blackfriday.EXTENSION_FOOTNOTES | |
|||
blackfriday.EXTENSION_HEADER_IDS | |
|||
blackfriday.EXTENSION_AUTO_HEADER_IDS | |
|||
// Not included in modules/markup/markdown/markdown.go;
|
|||
// required here to process inline links
|
|||
blackfriday.EXTENSION_AUTOLINK |
|||
) |
|||
|
|||
//revive:disable:var-naming Implementing the Rendering interface requires breaking some linting rules
|
|||
|
|||
// StripMarkdown parses markdown content by removing all markup and code blocks
|
|||
// in order to extract links and other references
|
|||
func StripMarkdown(rawBytes []byte) (string, []string) { |
|||
stripper := &MarkdownStripper{ |
|||
links: make([]string, 0, 10), |
|||
} |
|||
body := blackfriday.Markdown(rawBytes, stripper, blackfridayExtensions) |
|||
return string(body), stripper.GetLinks() |
|||
} |
|||
|
|||
// StripMarkdownBytes parses markdown content by removing all markup and code blocks
|
|||
// in order to extract links and other references
|
|||
func StripMarkdownBytes(rawBytes []byte) ([]byte, []string) { |
|||
stripper := &MarkdownStripper{ |
|||
links: make([]string, 0, 10), |
|||
} |
|||
body := blackfriday.Markdown(rawBytes, stripper, blackfridayExtensions) |
|||
return body, stripper.GetLinks() |
|||
} |
|||
|
|||
// block-level callbacks
|
|||
|
|||
// BlockCode dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) BlockCode(out *bytes.Buffer, text []byte, infoString string) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// BlockQuote dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) BlockQuote(out *bytes.Buffer, text []byte) { |
|||
// FIXME: perhaps it's better to leave out block quote for this?
|
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// BlockHtml dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) BlockHtml(out *bytes.Buffer, text []byte) { //nolint
|
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// Header dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Header(out *bytes.Buffer, text func() bool, level int, id string) { |
|||
text() |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// HRule dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) HRule(out *bytes.Buffer) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// List dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) List(out *bytes.Buffer, text func() bool, flags int) { |
|||
text() |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// ListItem dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) ListItem(out *bytes.Buffer, text []byte, flags int) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// Paragraph dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Paragraph(out *bytes.Buffer, text func() bool) { |
|||
text() |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// Table dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) { |
|||
r.processString(out, header, false) |
|||
r.processString(out, body, false) |
|||
} |
|||
|
|||
// TableRow dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) TableRow(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// TableHeaderCell dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// TableCell dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) TableCell(out *bytes.Buffer, text []byte, flags int) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// Footnotes dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Footnotes(out *bytes.Buffer, text func() bool) { |
|||
text() |
|||
} |
|||
|
|||
// FootnoteItem dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// TitleBlock dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) TitleBlock(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// Span-level callbacks
|
|||
|
|||
// AutoLink dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) AutoLink(out *bytes.Buffer, link []byte, kind int) { |
|||
r.processLink(out, link, []byte{}) |
|||
} |
|||
|
|||
// CodeSpan dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) CodeSpan(out *bytes.Buffer, text []byte) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// DoubleEmphasis dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) DoubleEmphasis(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// Emphasis dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Emphasis(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// Image dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// LineBreak dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) LineBreak(out *bytes.Buffer) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// Link dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { |
|||
r.processLink(out, link, content) |
|||
} |
|||
|
|||
// RawHtmlTag dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) RawHtmlTag(out *bytes.Buffer, tag []byte) { //nolint
|
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// TripleEmphasis dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) TripleEmphasis(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// StrikeThrough dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) StrikeThrough(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, false) |
|||
} |
|||
|
|||
// FootnoteRef dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) FootnoteRef(out *bytes.Buffer, ref []byte, id int) { |
|||
// Not rendered
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// Low-level callbacks
|
|||
|
|||
// Entity dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) Entity(out *bytes.Buffer, entity []byte) { |
|||
// FIXME: literal entities are not parsed; perhaps they should
|
|||
r.coallesce = false |
|||
} |
|||
|
|||
// NormalText dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) NormalText(out *bytes.Buffer, text []byte) { |
|||
r.processString(out, text, true) |
|||
} |
|||
|
|||
// Header and footer
|
|||
|
|||
// DocumentHeader dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) DocumentHeader(out *bytes.Buffer) { |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// DocumentFooter dummy function to proceed with rendering
|
|||
func (r *MarkdownStripper) DocumentFooter(out *bytes.Buffer) { |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// GetFlags returns rendering flags
|
|||
func (r *MarkdownStripper) GetFlags() int { |
|||
return 0 |
|||
} |
|||
|
|||
//revive:enable:var-naming
|
|||
|
|||
func doubleSpace(out *bytes.Buffer) { |
|||
if out.Len() > 0 { |
|||
out.WriteByte('\n') |
|||
} |
|||
} |
|||
|
|||
func (r *MarkdownStripper) processString(out *bytes.Buffer, text []byte, coallesce bool) { |
|||
// Always break-up words
|
|||
if !coallesce || !r.coallesce { |
|||
doubleSpace(out) |
|||
} |
|||
out.Write(text) |
|||
r.coallesce = coallesce |
|||
} |
|||
func (r *MarkdownStripper) processLink(out *bytes.Buffer, link []byte, content []byte) { |
|||
// Links are processed out of band
|
|||
r.links = append(r.links, string(link)) |
|||
r.coallesce = false |
|||
} |
|||
|
|||
// GetLinks returns the list of link data collected while parsing
|
|||
func (r *MarkdownStripper) GetLinks() []string { |
|||
return r.links |
|||
} |
|||
@ -0,0 +1,71 @@ |
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package mdstripper |
|||
|
|||
import ( |
|||
"strings" |
|||
"testing" |
|||
|
|||
"github.com/stretchr/testify/assert" |
|||
) |
|||
|
|||
func TestMarkdownStripper(t *testing.T) { |
|||
type testItem struct { |
|||
markdown string |
|||
expectedText []string |
|||
expectedLinks []string |
|||
} |
|||
|
|||
list := []testItem{ |
|||
{ |
|||
` |
|||
## This is a title |
|||
|
|||
This is [one](link) to paradise. |
|||
This **is emphasized**. |
|||
This: should coallesce. |
|||
|
|||
` + "```" + ` |
|||
This is a code block. |
|||
This should not appear in the output at all. |
|||
` + "```" + ` |
|||
|
|||
* Bullet 1 |
|||
* Bullet 2 |
|||
|
|||
A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. |
|||
`, |
|||
[]string{ |
|||
"This is a title", |
|||
"This is", |
|||
"to paradise.", |
|||
"This", |
|||
"is emphasized", |
|||
".", |
|||
"This: should coallesce.", |
|||
"Bullet 1", |
|||
"Bullet 2", |
|||
"A HIDDEN", |
|||
"IN THIS LINE.", |
|||
}, |
|||
[]string{ |
|||
"link", |
|||
}}, |
|||
} |
|||
|
|||
for _, test := range list { |
|||
text, links := StripMarkdown([]byte(test.markdown)) |
|||
rawlines := strings.Split(text, "\n") |
|||
lines := make([]string, 0, len(rawlines)) |
|||
for _, line := range rawlines { |
|||
line := strings.TrimSpace(line) |
|||
if line != "" { |
|||
lines = append(lines, line) |
|||
} |
|||
} |
|||
assert.EqualValues(t, test.expectedText, lines) |
|||
assert.EqualValues(t, test.expectedLinks, links) |
|||
} |
|||
} |
|||
@ -0,0 +1,322 @@ |
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package references |
|||
|
|||
import ( |
|||
"net/url" |
|||
"regexp" |
|||
"strconv" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"code.gitea.io/gitea/modules/markup/mdstripper" |
|||
"code.gitea.io/gitea/modules/setting" |
|||
) |
|||
|
|||
var ( |
|||
// validNamePattern performs only the most basic validation for user or repository names
|
|||
// Repository name should contain only alphanumeric, dash ('-'), underscore ('_') and dot ('.') characters.
|
|||
validNamePattern = regexp.MustCompile(`^[a-z0-9_.-]+$`) |
|||
|
|||
// NOTE: All below regex matching do not perform any extra validation.
|
|||
// Thus a link is produced even if the linked entity does not exist.
|
|||
// While fast, this is also incorrect and lead to false positives.
|
|||
// TODO: fix invalid linking issue
|
|||
|
|||
// mentionPattern matches all mentions in the form of "@user"
|
|||
mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[0-9a-zA-Z-_\.]+)(?:\s|$|\)|\])`) |
|||
// issueNumericPattern matches string that references to a numeric issue, e.g. #1287
|
|||
issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(#[0-9]+)(?:\s|$|\)|\]|:|\.(\s|$))`) |
|||
// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
|
|||
issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`) |
|||
// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
|
|||
// e.g. gogits/gogs#12345
|
|||
crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+)(?:\s|$|\)|\]|\.(\s|$))`) |
|||
|
|||
// Same as GitHub. See
|
|||
// https://help.github.com/articles/closing-issues-via-commit-messages
|
|||
issueCloseKeywords = []string{"close", "closes", "closed", "fix", "fixes", "fixed", "resolve", "resolves", "resolved"} |
|||
issueReopenKeywords = []string{"reopen", "reopens", "reopened"} |
|||
|
|||
issueCloseKeywordsPat, issueReopenKeywordsPat *regexp.Regexp |
|||
|
|||
giteaHostInit sync.Once |
|||
giteaHost string |
|||
) |
|||
|
|||
// XRefAction represents the kind of effect a cross reference has once is resolved
|
|||
type XRefAction int64 |
|||
|
|||
const ( |
|||
// XRefActionNone means the cross-reference is simply a comment
|
|||
XRefActionNone XRefAction = iota // 0
|
|||
// XRefActionCloses means the cross-reference should close an issue if it is resolved
|
|||
XRefActionCloses // 1
|
|||
// XRefActionReopens means the cross-reference should reopen an issue if it is resolved
|
|||
XRefActionReopens // 2
|
|||
// XRefActionNeutered means the cross-reference will no longer affect the source
|
|||
XRefActionNeutered // 3
|
|||
) |
|||
|
|||
// IssueReference contains an unverified cross-reference to a local issue or pull request
|
|||
type IssueReference struct { |
|||
Index int64 |
|||
Owner string |
|||
Name string |
|||
Action XRefAction |
|||
} |
|||
|
|||
// RenderizableReference contains an unverified cross-reference to with rendering information
|
|||
type RenderizableReference struct { |
|||
Issue string |
|||
Owner string |
|||
Name string |
|||
RefLocation *RefSpan |
|||
Action XRefAction |
|||
ActionLocation *RefSpan |
|||
} |
|||
|
|||
type rawReference struct { |
|||
index int64 |
|||
owner string |
|||
name string |
|||
action XRefAction |
|||
issue string |
|||
refLocation *RefSpan |
|||
actionLocation *RefSpan |
|||
} |
|||
|
|||
func rawToIssueReferenceList(reflist []*rawReference) []IssueReference { |
|||
refarr := make([]IssueReference, len(reflist)) |
|||
for i, r := range reflist { |
|||
refarr[i] = IssueReference{ |
|||
Index: r.index, |
|||
Owner: r.owner, |
|||
Name: r.name, |
|||
Action: r.action, |
|||
} |
|||
} |
|||
return refarr |
|||
} |
|||
|
|||
// RefSpan is the position where the reference was found within the parsed text
|
|||
type RefSpan struct { |
|||
Start int |
|||
End int |
|||
} |
|||
|
|||
func makeKeywordsPat(keywords []string) *regexp.Regexp { |
|||
return regexp.MustCompile(`(?i)(?:\s|^|\(|\[)(` + strings.Join(keywords, `|`) + `):? $`) |
|||
} |
|||
|
|||
func init() { |
|||
issueCloseKeywordsPat = makeKeywordsPat(issueCloseKeywords) |
|||
issueReopenKeywordsPat = makeKeywordsPat(issueReopenKeywords) |
|||
} |
|||
|
|||
// getGiteaHostName returns a normalized string with the local host name, with no scheme or port information
|
|||
func getGiteaHostName() string { |
|||
giteaHostInit.Do(func() { |
|||
if uapp, err := url.Parse(setting.AppURL); err == nil { |
|||
giteaHost = strings.ToLower(uapp.Host) |
|||
} else { |
|||
giteaHost = "" |
|||
} |
|||
}) |
|||
return giteaHost |
|||
} |
|||
|
|||
// FindAllMentionsMarkdown matches mention patterns in given content and
|
|||
// returns a list of found unvalidated user names **not including** the @ prefix.
|
|||
func FindAllMentionsMarkdown(content string) []string { |
|||
bcontent, _ := mdstripper.StripMarkdownBytes([]byte(content)) |
|||
locations := FindAllMentionsBytes(bcontent) |
|||
mentions := make([]string, len(locations)) |
|||
for i, val := range locations { |
|||
mentions[i] = string(bcontent[val.Start+1 : val.End]) |
|||
} |
|||
return mentions |
|||
} |
|||
|
|||
// FindAllMentionsBytes matches mention patterns in given content
|
|||
// and returns a list of locations for the unvalidated user names, including the @ prefix.
|
|||
func FindAllMentionsBytes(content []byte) []RefSpan { |
|||
mentions := mentionPattern.FindAllSubmatchIndex(content, -1) |
|||
ret := make([]RefSpan, len(mentions)) |
|||
for i, val := range mentions { |
|||
ret[i] = RefSpan{Start: val[2], End: val[3]} |
|||
} |
|||
return ret |
|||
} |
|||
|
|||
// FindFirstMentionBytes matches the first mention in then given content
|
|||
// and returns the location of the unvalidated user name, including the @ prefix.
|
|||
func FindFirstMentionBytes(content []byte) (bool, RefSpan) { |
|||
mention := mentionPattern. | |||