From 62d0a4d8829f214af4f9c00ecf8a81907d86ef06 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 7 Nov 2017 14:33:06 +0800 Subject: [PATCH] Add external markup render support (#2570) * add external markup render support * bug fixed * refacotr codes and fix wrong error log * fix comments and add check to prevent leaks * add check for config file and improve the example * check file close error * use ioutil.TempFile instead uuid * correct Render -> Parser * improve warning when incorrect markup setting * fix typos --- cmd/web.go | 3 + conf/app.ini | 9 +++ modules/markup/external/external.go | 88 +++++++++++++++++++++++++++++ modules/setting/setting.go | 50 +++++++++++++++- 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 modules/markup/external/external.go diff --git a/cmd/web.go b/cmd/web.go index 21f12ee16..811771aa5 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -14,6 +14,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers/routes" @@ -59,6 +60,8 @@ func runWeb(ctx *cli.Context) error { routers.GlobalInit() + external.RegisterParsers() + m := routes.NewMacaron() routes.RegisterRoutes(m) diff --git a/conf/app.ini b/conf/app.ini index 07c60ea0a..603f8c14f 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -573,3 +573,12 @@ SHOW_FOOTER_BRANDING = false SHOW_FOOTER_VERSION = true ; Show time of template execution in the footer SHOW_FOOTER_TEMPLATE_LOAD_TIME = true + +[markup.asciidoc] +ENABLED = false +; List of file extensions that should be rendered by an external command +FILE_EXTENSIONS = .adoc,.asciidoc +; External command to render all matching extensions +RENDER_COMMAND = "asciidoc --out-file=- -" +; Input is not a standard input but a file +IS_INPUT_FILE = false \ No newline at end of file diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go new file mode 100644 index 000000000..676ec4b4c --- /dev/null +++ b/modules/markup/external/external.go @@ -0,0 +1,88 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package external + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "os/exec" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" +) + +// RegisterParsers registers all supported third part parsers according settings +func RegisterParsers() { + for _, parser := range setting.ExternalMarkupParsers { + if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 { + markup.RegisterParser(&Parser{parser}) + } + } +} + +// Parser implements markup.Parser for external tools +type Parser struct { + setting.MarkupParser +} + +// Name returns the external tool name +func (p *Parser) Name() string { + return p.MarkupName +} + +// Extensions returns the supported extensions of the tool +func (p *Parser) Extensions() []string { + return p.FileExtensions +} + +// Render renders the data of the document to HTML via the external tool. +func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { + var ( + bs []byte + buf = bytes.NewBuffer(bs) + rd = bytes.NewReader(rawBytes) + commands = strings.Fields(p.Command) + args = commands[1:] + ) + + if p.IsInputFile { + // write to temp file + f, err := ioutil.TempFile("", "gitea_input") + if err != nil { + log.Error(4, "%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err) + return []byte("") + } + defer os.Remove(f.Name()) + + _, err = io.Copy(f, rd) + if err != nil { + f.Close() + log.Error(4, "%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err) + return []byte("") + } + + err = f.Close() + if err != nil { + log.Error(4, "%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err) + return []byte("") + } + args = append(args, f.Name()) + } + + cmd := exec.Command(commands[0], args...) + if !p.IsInputFile { + cmd.Stdin = rd + } + cmd.Stdout = buf + if err := cmd.Run(); err != nil { + log.Error(4, "%s render run command %s %v failed: %v", p.Name(), commands[0], args, err) + return []byte("") + } + return buf.Bytes() +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 62d4d5eeb..df80ebdb5 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -60,6 +60,15 @@ const ( LandingPageExplore LandingPage = "/explore" ) +// MarkupParser defines the external parser configured in ini +type MarkupParser struct { + Enabled bool + MarkupName string + Command string + FileExtensions []string + IsInputFile bool +} + // settings var ( // AppVer settings @@ -515,6 +524,8 @@ var ( HasRobotsTxt bool InternalToken string // internal access token IterateBufferSize int + + ExternalMarkupParsers []MarkupParser ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1073,6 +1084,44 @@ func NewContext() { UI.ShowUserEmail = Cfg.Section("ui").Key("SHOW_USER_EMAIL").MustBool(true) HasRobotsTxt = com.IsFile(path.Join(CustomPath, "robots.txt")) + + extensionReg := regexp.MustCompile(`\.\w`) + for _, sec := range Cfg.Section("markup").ChildSections() { + name := strings.TrimLeft(sec.Name(), "markup.") + if name == "" { + log.Warn("name is empty, markup " + sec.Name() + "ignored") + continue + } + + extensions := sec.Key("FILE_EXTENSIONS").Strings(",") + var exts = make([]string, 0, len(extensions)) + for _, extension := range extensions { + if !extensionReg.MatchString(extension) { + log.Warn(sec.Name() + " file extension " + extension + " is invalid. Extension ignored") + } else { + exts = append(exts, extension) + } + } + + if len(exts) == 0 { + log.Warn(sec.Name() + " file extension is empty, markup " + name + " ignored") + continue + } + + command := sec.Key("RENDER_COMMAND").MustString("") + if command == "" { + log.Warn(" RENDER_COMMAND is empty, markup " + name + " ignored") + continue + } + + ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{ + Enabled: sec.Key("ENABLED").MustBool(false), + MarkupName: name, + FileExtensions: exts, + Command: command, + IsInputFile: sec.Key("IS_INPUT_FILE").MustBool(false), + }) + } } // Service settings @@ -1133,7 +1182,6 @@ func newService() { Service.OpenIDBlacklist[i] = regexp.MustCompilePOSIX(p) } } - } var logLevels = map[string]string{