From 551f3cbe420dcb72bcb784205451d5b01b811041 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Wed, 29 Nov 2017 02:50:39 +0100 Subject: [PATCH] Memory usage improvements (#3013) * govendor update code.gitea.io/git Signed-off-by: Duncan Ogilvie * Greatly improve memory usage Signed-off-by: Duncan Ogilvie --- modules/context/repo.go | 3 ++ routers/repo/download.go | 3 +- routers/repo/editor.go | 9 ++++-- routers/repo/issue.go | 3 ++ routers/repo/view.go | 29 +++++++++++------ vendor/code.gitea.io/git/blob.go | 50 +++++++++++++++++++++++++++--- vendor/code.gitea.io/git/commit.go | 3 +- vendor/code.gitea.io/git/git.go | 2 +- vendor/vendor.json | 6 ++-- 9 files changed, 86 insertions(+), 22 deletions(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 38baa4230..e8d084c34 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { if err != nil { return nil, err } + if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize { + return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"} + } reader, err := treeEntry.Blob().Data() if err != nil { return nil, err diff --git a/routers/repo/download.go b/routers/repo/download.go index 78c608860..78c4b519b 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error { // ServeBlob download a git.Blob func ServeBlob(ctx *context.Context, blob *git.Blob) error { - dataRc, err := blob.Data() + dataRc, err := blob.DataAsync() if err != nil { return err } + defer dataRc.Close() return ServeData(ctx, ctx.Repo.TreePath, dataRc) } diff --git a/routers/repo/editor.go b/routers/repo/editor.go index a6cc92236..82b04a84d 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) { // No way to edit a directory online. if entry.IsDir() { - ctx.Handle(404, "", nil) + ctx.Handle(404, "entry.IsDir", nil) return } blob := entry.Blob() + if blob.Size() >= setting.UI.MaxDisplayFileSize { + ctx.Handle(404, "blob.Size", err) + return + } + dataRc, err := blob.Data() if err != nil { ctx.Handle(404, "blob.Data", err) @@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) { // Only text file are editable online. if !base.IsTextFile(buf) { - ctx.Handle(404, "", nil) + ctx.Handle(404, "base.IsTextFile", nil) return } diff --git a/routers/repo/issue.go b/routers/repo/issue.go index c24a4e436..b45d521e5 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str if err != nil { return "", false } + if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize { + return "", false + } r, err = entry.Blob().Data() if err != nil { return "", false diff --git a/routers/repo/view.go b/routers/repo/view.go index d43b4d7f7..a02acb0d6 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) { ctx.Data["ReadmeInList"] = true ctx.Data["ReadmeExist"] = true - dataRc, err := readmeFile.Data() + dataRc, err := readmeFile.DataAsync() if err != nil { ctx.Handle(500, "Data", err) return } + defer dataRc.Close() buf := make([]byte, 1024) n, _ := dataRc.Read(buf) @@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) { ctx.Data["FileName"] = readmeFile.Name() // FIXME: what happens when README file is an image? if isTextFile { - d, _ := ioutil.ReadAll(dataRc) - buf = append(buf, d...) - if markup.Type(readmeFile.Name()) != "" { - ctx.Data["IsMarkup"] = true - ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas())) + if readmeFile.Size() >= setting.UI.MaxDisplayFileSize { + // Pretend that this is a normal text file to display 'This file is too large to be shown' + ctx.Data["IsFileTooLarge"] = true + ctx.Data["IsTextFile"] = true + ctx.Data["FileSize"] = readmeFile.Size() } else { - ctx.Data["IsRenderedHTML"] = true - ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`
`), -1)) + d, _ := ioutil.ReadAll(dataRc) + buf = append(buf, d...) + if markup.Type(readmeFile.Name()) != "" { + ctx.Data["IsMarkup"] = true + ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas())) + } else { + ctx.Data["IsRenderedHTML"] = true + ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`
`), -1)) + } } } } @@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["IsViewFile"] = true blob := entry.Blob() - dataRc, err := blob.Data() + dataRc, err := blob.DataAsync() if err != nil { - ctx.Handle(500, "Data", err) + ctx.Handle(500, "DataAsync", err) return } + defer dataRc.Close() ctx.Data["FileSize"] = blob.Size() ctx.Data["FileName"] = blob.Name() diff --git a/vendor/code.gitea.io/git/blob.go b/vendor/code.gitea.io/git/blob.go index 10b8ea4c9..a6e392eeb 100644 --- a/vendor/code.gitea.io/git/blob.go +++ b/vendor/code.gitea.io/git/blob.go @@ -6,7 +6,11 @@ package git import ( "bytes" + "fmt" "io" + "io/ioutil" + "os" + "os/exec" ) // Blob represents a Git object. @@ -18,14 +22,52 @@ type Blob struct { // Data gets content of blob all at once and wrap it as io.Reader. // This can be very slow and memory consuming for huge content. func (b *Blob) Data() (io.Reader, error) { - stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path) - if err != nil { - return nil, err + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + // Preallocate memory to save ~50% memory usage on big files. + stdout.Grow(int(b.Size() + 2048)) + + if err := b.DataPipeline(stdout, stderr); err != nil { + return nil, concatenateError(err, stderr.String()) } - return bytes.NewBuffer(stdout), nil + return stdout, nil } // DataPipeline gets content of blob and write the result or error to stdout or stderr func (b *Blob) DataPipeline(stdout, stderr io.Writer) error { return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr) } + +type cmdReadCloser struct { + cmd *exec.Cmd + stdout io.Reader +} + +func (c cmdReadCloser) Read(p []byte) (int, error) { + return c.stdout.Read(p) +} + +func (c cmdReadCloser) Close() error { + io.Copy(ioutil.Discard, c.stdout) + return c.cmd.Wait() +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + cmd := exec.Command("git", "show", b.ID.String()) + cmd.Dir = b.repo.Path + cmd.Stderr = os.Stderr + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("StdoutPipe: %v", err) + } + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("Start: %v", err) + } + + return cmdReadCloser{stdout: stdout, cmd: cmd}, nil +} diff --git a/vendor/code.gitea.io/git/commit.go b/vendor/code.gitea.io/git/commit.go index c29541238..299a2381b 100644 --- a/vendor/code.gitea.io/git/commit.go +++ b/vendor/code.gitea.io/git/commit.go @@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool { return false } - dataRc, err := blob.Data() + dataRc, err := blob.DataAsync() if err != nil { return false } + defer dataRc.Close() buf := make([]byte, 1024) n, _ := dataRc.Read(buf) buf = buf[:n] diff --git a/vendor/code.gitea.io/git/git.go b/vendor/code.gitea.io/git/git.go index 9ec20c97e..150b80fb0 100644 --- a/vendor/code.gitea.io/git/git.go +++ b/vendor/code.gitea.io/git/git.go @@ -25,7 +25,7 @@ var ( // Prefix the log prefix Prefix = "[git-module] " // GitVersionRequired is the minimum Git version required - GitVersionRequired = "1.8.1.6" + GitVersionRequired = "1.7.2" ) func log(format string, args ...interface{}) { diff --git a/vendor/vendor.json b/vendor/vendor.json index 54cbbf330..e3084fd53 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -3,10 +3,10 @@ "ignore": "test appengine", "package": [ { - "checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=", + "checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=", "path": "code.gitea.io/git", - "revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a", - "revisionTime": "2017-10-23T00:52:09Z" + "revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54", + "revisionTime": "2017-11-28T15:25:05Z" }, { "checksumSHA1": "QQ7g7B9+EIzGjO14KCGEs9TNEzM=",