Language statistics bar for repositories (#8037)
* Implementation for calculating language statistics Impement saving code language statistics to database Implement rendering langauge stats Add primary laguage to show in repository list Implement repository stats indexer queue Add indexer test Refactor to use queue module * Do not timeout for queuesmj
Lauris BH
4 years ago
committed by
GitHub
parent
37892be635
commit
ad2642a8aa
@ -0,0 +1,45 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
func addLanguageStats(x *xorm.Engine) error {
|
||||
// LanguageStat see models/repo_language_stats.go
|
||||
type LanguageStat struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
||||
CommitID string
|
||||
IsPrimary bool
|
||||
Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
|
||||
Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
|
||||
Color string `xorm:"-"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
|
||||
}
|
||||
|
||||
type RepoIndexerType int
|
||||
|
||||
// RepoIndexerStatus see models/repo_stats_indexer.go
|
||||
type RepoIndexerStatus struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
RepoID int64 `xorm:"INDEX(s)"`
|
||||
CommitSha string `xorm:"VARCHAR(40)"`
|
||||
IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
|
||||
}
|
||||
|
||||
if err := x.Sync2(new(LanguageStat)); err != nil {
|
||||
return fmt.Errorf("Sync2: %v", err)
|
||||
}
|
||||
if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
|
||||
return fmt.Errorf("Sync2: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,137 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/src-d/enry/v2"
|
||||
)
|
||||
|
||||
// LanguageStat describes language statistics of a repository
|
||||
type LanguageStat struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
||||
CommitID string
|
||||
IsPrimary bool
|
||||
Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
|
||||
Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
|
||||
Color string `xorm:"-"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
|
||||
}
|
||||
|
||||
// LanguageStatList defines a list of language statistics
|
||||
type LanguageStatList []*LanguageStat
|
||||
|
||||
func (stats LanguageStatList) loadAttributes() {
|
||||
for i := range stats {
|
||||
stats[i].Color = enry.GetColor(stats[i].Language)
|
||||
}
|
||||
}
|
||||
|
||||
func (repo *Repository) getLanguageStats(e Engine) (LanguageStatList, error) {
|
||||
stats := make(LanguageStatList, 0, 6)
|
||||
if err := e.Where("`repo_id` = ?", repo.ID).Desc("`percentage`").Find(&stats); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats.loadAttributes()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// GetLanguageStats returns the language statistics for a repository
|
||||
func (repo *Repository) GetLanguageStats() (LanguageStatList, error) {
|
||||
return repo.getLanguageStats(x)
|
||||
}
|
||||
|
||||
// GetTopLanguageStats returns the top language statistics for a repository
|
||||
func (repo *Repository) GetTopLanguageStats(limit int) (LanguageStatList, error) {
|
||||
stats, err := repo.getLanguageStats(x)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
topstats := make(LanguageStatList, 0, limit)
|
||||
var other float32
|
||||
for i := range stats {
|
||||
if stats[i].Language == "other" || len(topstats) >= limit {
|
||||
other += stats[i].Percentage
|
||||
continue
|
||||
}
|
||||
topstats = append(topstats, stats[i])
|
||||
}
|
||||
if other > 0 {
|
||||
topstats = append(topstats, &LanguageStat{
|
||||
RepoID: repo.ID,
|
||||
Language: "other",
|
||||
Color: "#cccccc",
|
||||
Percentage: float32(math.Round(float64(other)*10) / 10),
|
||||
})
|
||||
}
|
||||
return topstats, nil
|
||||
}
|
||||
|
||||
// UpdateLanguageStats updates the language statistics for repository
|
||||
func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]float32) error {
|
||||
sess := x.NewSession()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer sess.Close()
|
||||
|
||||
oldstats, err := repo.getLanguageStats(sess)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var topLang string
|
||||
var p float32
|
||||
for lang, perc := range stats {
|
||||
if perc > p {
|
||||
p = perc
|
||||
topLang = strings.ToLower(lang)
|
||||
}
|
||||
}
|
||||
|
||||
for lang, perc := range stats {
|
||||
upd := false
|
||||
llang := strings.ToLower(lang)
|
||||
for _, s := range oldstats {
|
||||
// Update already existing language
|
||||
if strings.ToLower(s.Language) == llang {
|
||||
s.CommitID = commitID
|
||||
s.IsPrimary = llang == topLang
|
||||
s.Percentage = perc
|
||||
if _, err := sess.ID(s.ID).Cols("`commit_id`", "`percentage`", "`is_primary`").Update(s); err != nil {
|
||||
return err
|
||||
}
|
||||
upd = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// Insert new language
|
||||
if !upd {
|
||||
if _, err := sess.Insert(&LanguageStat{
|
||||
RepoID: repo.ID,
|
||||
CommitID: commitID,
|
||||
IsPrimary: llang == topLang,
|
||||
Language: lang,
|
||||
Percentage: perc,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// Delete old languages
|
||||
if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sess.Commit()
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/src-d/enry/v2"
|
||||
"gopkg.in/src-d/go-git.v4"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
||||
)
|
||||
|
||||
const fileSizeLimit int64 = 16 * 1024 * 1024
|
||||
|
||||
// GetLanguageStats calculates language stats for git repository at specified commit
|
||||
func (repo *Repository) GetLanguageStats(commitID string) (map[string]float32, error) {
|
||||
r, err := git.PlainOpen(repo.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commit, err := r.CommitObject(*rev)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tree, err := commit.Tree()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sizes := make(map[string]int64)
|
||||
var total int64
|
||||
err = tree.Files().ForEach(func(f *object.File) error {
|
||||
if enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
|
||||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: Use .gitattributes file for linguist overrides
|
||||
|
||||
language, ok := enry.GetLanguageByExtension(f.Name)
|
||||
if !ok {
|
||||
if language, ok = enry.GetLanguageByFilename(f.Name); !ok {
|
||||
content, err := readFile(f, fileSizeLimit)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
language = enry.GetLanguage(filepath.Base(f.Name), content)
|
||||
if language == enry.OtherLanguage {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if language != "" {
|
||||
sizes[language] += f.Size
|
||||
total += f.Size
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stats := make(map[string]float32)
|
||||
var otherPerc float32 = 100
|
||||
for language, size := range sizes {
|
||||
perc := float32(math.Round(float64(size)/float64(total)*1000) / 10)
|
||||
if perc <= 0.1 {
|
||||
continue
|
||||
}
|
||||
otherPerc -= perc
|
||||
stats[language] = perc
|
||||
}
|
||||
otherPerc = float32(math.Round(float64(otherPerc)*10) / 10)
|
||||
if otherPerc > 0 {
|
||||
stats["other"] = otherPerc
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func readFile(f *object.File, limit int64) ([]byte, error) {
|
||||
r, err := f.Reader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
if limit <= 0 {
|
||||
return ioutil.ReadAll(r)
|
||||
}
|
||||
|
||||
size := f.Size
|
||||
if limit > 0 && size > limit {
|
||||
size = limit
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
buf.Grow(int(size))
|
||||
_, err = io.Copy(buf, io.LimitReader(r, limit))
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
)
|
||||
|
||||
// DBIndexer implements Indexer interface to use database's like search
|
||||
type DBIndexer struct {
|
||||
}
|
||||
|
||||
// Index repository status function
|
||||
func (db *DBIndexer) Index(id int64) error {
|
||||
repo, err := models.GetRepositoryByID(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := repo.GetIndexerStatus(models.RepoIndexerTypeStats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gitRepo, err := git.OpenRepository(repo.RepoPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gitRepo.Close()
|
||||
|
||||
// Get latest commit for default branch
|
||||
commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do not recalculate stats if already calculated for this commit
|
||||
if status.CommitSha == commitID {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate and save language statistics to database
|
||||
stats, err := gitRepo.GetLanguageStats(commitID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return repo.UpdateLanguageStats(commitID, stats)
|
||||
}
|
||||
|
||||
// Close dummy function
|
||||
func (db *DBIndexer) Close() {
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
)
|
||||
|
||||
// Indexer defines an interface to index repository stats
|
||||
type Indexer interface {
|
||||
Index(id int64) error
|
||||
Close()
|
||||
}
|
||||
|
||||
// indexer represents a indexer instance
|
||||
var indexer Indexer
|
||||
|
||||
// Init initialize the repo indexer
|
||||
func Init() error {
|
||||
indexer = &DBIndexer{}
|
||||
|
||||
if err := initStatsQueue(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go populateRepoIndexer()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
||||
// should only be run when the indexer is created for the first time.
|
||||
func populateRepoIndexer() {
|
||||
log.Info("Populating the repo stats indexer with existing repositories")
|
||||
|
||||
isShutdown := graceful.GetManager().IsShutdown()
|
||||
|
||||
exist, err := models.IsTableNotEmpty("repository")
|
||||
if err != nil {
|
||||
log.Fatal("System error: %v", err)
|
||||
} else if !exist {
|
||||
return
|
||||
}
|
||||
|
||||
var maxRepoID int64
|
||||
if maxRepoID, err = models.GetMaxID("repository"); err != nil {
|
||||
log.Fatal("System error: %v", err)
|
||||
}
|
||||
|
||||
// start with the maximum existing repo ID and work backwards, so that we
|
||||
// don't include repos that are created after gitea starts; such repos will
|
||||
// already be added to the indexer, and we don't need to add them again.
|
||||
for maxRepoID > 0 {
|
||||
select {
|
||||
case <-isShutdown:
|
||||
log.Info("Repository Stats Indexer population shutdown before completion")
|
||||
return
|
||||
default:
|
||||
}
|
||||
ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeStats, maxRepoID, 0, 50)
|
||||
if err != nil {
|
||||
log.Error("populateRepoIndexer: %v", err)
|
||||
return
|
||||
} else if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
for _, id := range ids {
|
||||
select {
|
||||
case <-isShutdown:
|
||||
log.Info("Repository Stats Indexer population shutdown before completion")
|
||||
return
|
||||
default:
|
||||
}
|
||||
if err := statsQueue.Push(id); err != nil {
|
||||
log.Error("statsQueue.Push: %v", err)
|
||||
}
|
||||
maxRepoID = id - 1
|
||||
}
|
||||
}
|
||||
log.Info("Done (re)populating the repo stats indexer with existing repositories")
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"gopkg.in/ini.v1"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
models.MainTest(m, filepath.Join("..", "..", ".."))
|
||||
}
|
||||
|
||||
func TestRepoStatsIndex(t *testing.T) {
|
||||
assert.NoError(t, models.PrepareTestDatabase())
|
||||
setting.Cfg = ini.Empty()
|
||||
|
||||
setting.NewQueueService()
|
||||
|
||||
err := Init()
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
repo, err := models.GetRepositoryByID(1)
|
||||
assert.NoError(t, err)
|
||||
langs, err := repo.GetTopLanguageStats(5)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, langs, 1)
|
||||
assert.Equal(t, "other", langs[0].Language)
|
||||
assert.Equal(t, float32(100), langs[0].Percentage)
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
)
|
||||
|
||||
// statsQueue represents a queue to handle repository stats updates
|
||||
var statsQueue queue.Queue
|
||||
|
||||
// handle passed PR IDs and test the PRs
|
||||
func handle(data ...queue.Data) {
|
||||
for _, datum := range data {
|
||||
opts := datum.(int64)
|
||||
if err := indexer.Index(opts); err != nil {
|
||||
log.Error("stats queue idexer.Index(%d) failed: %v", opts, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initStatsQueue() error {
|
||||
statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue)
|
||||
if statsQueue == nil {
|
||||
return fmt.Errorf("Unable to create repo_stats_update Queue")
|
||||
}
|
||||
|
||||
go graceful.GetManager().RunWithShutdownFns(statsQueue.Run)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateRepoIndexer update a repository's entries in the indexer
|
||||
func UpdateRepoIndexer(repo *models.Repository) error {
|
||||
return statsQueue.Push(repo.ID)
|
||||
}
|
||||
@ -1,17 +1,42 @@
|
||||
<div class="ui segment sub-menu">
|
||||
<div class="ui two horizontal center link list">
|
||||
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}}
|
||||
<div class="item{{if .PageIsCommits}} active{{end}}">
|
||||
<a class="ui" href="{{.RepoLink}}/commits{{if .IsViewBranch}}/branch{{else if .IsViewTag}}/tag{{else if .IsViewCommit}}/commit{{end}}/{{EscapePound .BranchName}}"><i class="octicon octicon-history"></i> <b>{{.CommitsCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }}</a>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }}
|
||||
<div class="item{{if .PageIsBranches}} active{{end}}">
|
||||
<a class="ui" href="{{.RepoLink}}/branches/"><i class="octicon octicon-git-branch"></i> <b>{{.BranchesCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }}</a>
|
||||
</div>
|
||||
<div class="ui segments">
|
||||
<div class="ui segment sub-menu repository-menu">
|
||||
<div class="ui two horizontal center link list">
|
||||
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}}
|
||||
<div class="item{{if .PageIsCommits}} active{{end}}">
|
||||
<a class="ui" href="{{.RepoLink}}/commits{{if .IsViewBranch}}/branch{{else if .IsViewTag}}/tag{{else if .IsViewCommit}}/commit{{end}}/{{EscapePound .BranchName}}"><i class="octicon octicon-history"></i> <b>{{.CommitsCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }}</a>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }}
|
||||
<div class="item{{if .PageIsBranches}} active{{end}}">
|
||||
<a class="ui" href="{{.RepoLink}}/branches/"><i class="octicon octicon-git-branch"></i> <b>{{.BranchesCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }}</a>
|
||||
</div>
|
||||
<div class="item">
|
||||
<a class="ui" href="#"><i class="octicon octicon-database"></i> <b>{{SizeFmt .Repository.Size}}</b></a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) .LanguageStats }}
|
||||
<div class="ui segment sub-menu language-stats-details" style="display: none">
|
||||
<div class="ui horizontal center link list">
|
||||
{{range .LanguageStats}}
|
||||
<div class="item">
|
||||
<a class="ui" href="#"><i class="octicon octicon-database"></i> <b>{{SizeFmt .Repository.Size}}</b></a>
|
||||
<i class="color-icon" style="background-color: {{ .Color }}"></i>
|
||||
<span class="ui"><b>
|
||||
{{if eq .Language "other" }}
|
||||
{{ $.i18n.Tr "repo.language_other" }}
|
||||
{{else}}
|
||||
{{ .Language }}
|
||||
{{end}}
|
||||
</b> {{ .Percentage }}%</span>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
<a class="ui segment language-stats">
|
||||
{{range .LanguageStats}}
|
||||
<div class="bar" style="width: {{ .Percentage }}%; background-color: {{ .Color }}"> </div>
|
||||
{{end}}
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
.linguist
|
||||
benchmarks/output
|
||||
.ci
|
||||
Makefile.main
|
||||
.shared
|
||||
.idea
|
||||
.docsrv-resources
|
||||
build/
|
||||
vendor/
|
||||
java/lib/
|
||||
.vscode/
|
||||
@ -0,0 +1,132 @@
|
||||
dist: trusty
|
||||
language: go
|
||||
go:
|
||||
- '1.12.x'
|
||||
- '1.11.x'
|
||||
env:
|
||||
global:
|
||||
- GO_VERSION_FOR_JVM='1.11.x'
|
||||
- CGO_ENABLED=0
|
||||
- GO111MODULE=on
|
||||
- ONIGURUMA_VERSION='6.9.1'
|
||||
matrix:
|
||||
- ONIGURUMA=0
|
||||
- ONIGURUMA=1
|
||||
matrix:
|
||||
fast_finish: true
|
||||
|
||||
stages:
|
||||
- name: test
|
||||
- name: release
|
||||
if: tag IS present
|
||||
- name: publish
|
||||
if: tag IS present
|
||||
|
||||
stage: test
|
||||
install:
|
||||
- >
|
||||
if [[ "${ONIGURUMA}" -gt 0 ]]; then
|
||||
export CGO_ENABLED=1
|
||||
export GO_TAGS='oniguruma'
|
||||
# install oniguruma manually as trusty has only ancient 5.x
|
||||
sudo apt-get install -y dpkg # dpkg >= 1.17.5ubuntu5.8 fixes https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627
|
||||
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
|
||||
sudo dpkg -i "libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
|
||||
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
|
||||
sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
|
||||
fi;
|
||||
script:
|
||||
- make test-coverage
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
jobs:
|
||||
include:
|
||||
- name: 'java unit-tests'
|
||||
stage: test
|
||||
language: scala
|
||||
jdk: oraclejdk8
|
||||
install:
|
||||
- export CGO_ENABLED=1
|
||||
- eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)"
|
||||
- go version
|
||||
before_script:
|
||||
- cd java
|
||||
- make
|
||||
script:
|
||||
- make test
|
||||
|
||||
- name: 'linux packages'
|
||||
stage: release
|
||||
install:
|
||||
- go version
|
||||
script: make packages
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key:
|
||||
secure: $GITHUB_TOKEN
|
||||
file_glob: true
|
||||
file: build/*.tar.gz
|
||||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
|
||||
- name: 'linux shared lib'
|
||||
stage: release
|
||||
install:
|
||||
- go version
|
||||
script: make linux-shared
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key:
|
||||
secure: $GITHUB_TOKEN
|
||||
file:
|
||||
- ./.shared/linux-x86-64/libenry.so
|
||||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
|
||||
- name: 'macOS shared lib'
|
||||
stage: release
|
||||
env:
|
||||
- OSXCROSS_PACKAGE="osxcross_3034f7149716d815bc473d0a7b35d17e4cf175aa.tar.gz"
|
||||
- OSXCROSS_URL="https://github.com/bblfsh/client-scala/releases/download/v1.5.2/${OSXCROSS_PACKAGE}"
|
||||
- PATH="/$HOME/osxcross/bin:$PATH"
|
||||
install:
|
||||
- go version
|
||||
- sudo apt-get update
|
||||
- sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils
|
||||
- cd ${HOME}
|
||||
- curl -sfSL ${OSXCROSS_URL} | tar -C ${HOME} -xzf -
|
||||
- cd $GOPATH/src/github.com/src-d/enry
|
||||
script: make darwin-shared
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key:
|
||||
secure: $GITHUB_TOKEN
|
||||
file: ./.shared/darwin/libenry.dylib
|
||||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
|
||||
- name: 'java: publish to maven'
|
||||
stage: publish
|
||||
language: scala
|
||||
jdk: oraclejdk8
|
||||
install:
|
||||
- export CGO_ENABLED=1
|
||||
- eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)"
|
||||
- go version
|
||||
before_script:
|
||||
- cd java
|
||||
- make
|
||||
- curl -o ./shared/linux-x86-64/libenry.so -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.so" || travis_terminate 1
|
||||
- mkdir -p ./shared/darwin
|
||||
- curl -o ./shared/darwin/libenry.dylib -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.dylib" || travis_terminate 1
|
||||
- openssl aes-256-cbc -K $encrypted_a0e1c69dbbc7_key -iv $encrypted_a0e1c69dbbc7_iv -in key.asc.enc -out key.asc -d
|
||||
- gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
|
||||
script:
|
||||
- make test # ensure the shared objects are functional
|
||||
- ./sbt publishLocal
|
||||
- ./sbt publishSigned
|
||||
- ./sbt sonatypeRelease
|
||||
@ -0,0 +1,61 @@
|
||||
# source{d} Contributing Guidelines
|
||||
|
||||
source{d} projects accept contributions via GitHub pull requests.
|
||||
This document outlines some of the
|
||||
conventions on development workflow, commit message formatting, contact points,
|
||||
and other resources to make it easier to get your contribution accepted.
|
||||
|
||||
## Certificate of Origin
|
||||
|
||||
By contributing to this project, you agree to the [Developer Certificate of
|
||||
Origin (DCO)](DCO). This document was created by the Linux Kernel community and is a
|
||||
simple statement that you, as a contributor, have the legal right to make the
|
||||
contribution.
|
||||
|
||||
In order to show your agreement with the DCO you should include at the end of the commit message,
|
||||
the following line: `Signed-off-by: John Doe <john.doe@example.com>`, using your real name.
|
||||
|
||||
This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`.
|
||||
|
||||
If you find yourself pushed a few commits without `Signed-off-by`, you can still add it afterwards. We wrote a manual which can help: [fix-DCO.md](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md).
|
||||
|
||||
## Support Channels
|
||||
|
||||
The official support channels, for both users and contributors, are:
|
||||
|
||||
- GitHub issues: each repository has its own list of issues.
|
||||
- Slack: join the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) community.
|
||||
|
||||
*Before opening a new issue or submitting a new pull request, it's helpful to
|
||||
search the project - it's likely that another user has already reported the
|
||||
issue you're facing, or it's a known issue that we're already aware of.
|
||||
|
||||
|
||||
## How to Contribute
|
||||
|
||||
Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects.
|
||||
In order for a PR to be accepted it needs to pass this list of requirements:
|
||||
|
||||
- The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it.
|
||||
- All PRs must be written idiomaticly:
|
||||
- for Go: formatted according to [gofmt](https://golang.org/cmd/gofmt/), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/)
|
||||
- for other languages, similar constraints apply.
|
||||
- They should in general include tests, and those shall pass.
|
||||
- If the PR is a bug fix, it has to include a new unit test that fails before the patch is merged.
|
||||
- If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality.
|
||||
- In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project.
|
||||
|
||||
|
||||
### Format of the commit message
|
||||
|
||||
Every commit message should describe what was changed, under which context and, if applicable, the GitHub issue it relates to:
|
||||
|
||||
```
|
||||
plumbing: packp, Skip argument validations for unknown capabilities. Fixes #623
|
||||
```
|
||||
|
||||
The format can be described more formally as follows:
|
||||
|
||||
```
|
||||
<package>: <subpackage>, <what changed>. [Fixes #<issue-number>]
|
||||
```
|
||||
@ -0,0 +1,25 @@
|
||||
Developer's Certificate of Origin 1.1
|
||||
|
||||
By making a contribution to this project, I certify that:
|
||||
|
||||
(a) The contribution was created in whole or in part by me and I
|
||||
have the right to submit it under the open source license
|
||||
indicated in the file; or
|
||||
|
||||
(b) The contribution is based upon previous work that, to the best
|
||||
of my knowledge, is covered under an appropriate open source
|
||||
license and I have the right under that license to submit that
|
||||
work with modifications, whether created in whole or in part
|
||||
by me, under the same open source license (unless I am
|
||||
permitted to submit under a different license), as indicated
|
||||
in the file; or
|
||||
|
||||
(c) The contribution was provided directly to me by some other
|
||||
person who certified (a), (b) or (c) and I have not modified
|
||||
it.
|
||||
|
||||
(d) I understand and agree that this project and the contribution
|
||||
are public and that a record of the contribution (including all
|
||||
personal information I submit with it, including my sign-off) is
|
||||
maintained indefinitely and may be redistributed consistent with
|
||||
this project or the open source license(s) involved.
|
||||
@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||