Browse Source

Language statistics bar for repositories (#8037)

* Implementation for calculating language statistics

Impement saving code language statistics to database

Implement rendering langauge stats

Add primary laguage to show in repository list

Implement repository stats indexer queue

Add indexer test

Refactor to use queue module

* Do not timeout for queues
mj
Lauris BH 3 years ago
committed by GitHub
parent
commit
ad2642a8aa
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      go.mod
  2. 9
      go.sum
  3. 3
      integrations/testlogger.go
  4. 2
      models/migrations/migrations.go
  5. 45
      models/migrations/v127.go
  6. 1
      models/models.go
  7. 9
      models/repo.go
  8. 81
      models/repo_indexer.go
  9. 137
      models/repo_language_stats.go
  10. 31
      models/repo_list.go
  11. 116
      modules/git/repo_language_stats.go
  12. 2
      modules/indexer/code/bleve.go
  13. 7
      modules/indexer/code/git.go
  14. 2
      modules/indexer/code/queue.go
  15. 54
      modules/indexer/stats/db.go
  16. 85
      modules/indexer/stats/indexer.go
  17. 42
      modules/indexer/stats/indexer_test.go
  18. 43
      modules/indexer/stats/queue.go
  19. 7
      modules/notification/indexer/indexer.go
  20. 1
      options/locale/locale_en-US.ini
  21. 4
      routers/init.go
  22. 1
      routers/org/home.go
  23. 15
      routers/repo/view.go
  24. 1
      routers/user/profile.go
  25. 3
      templates/explore/repo_list.tmpl
  26. 51
      templates/repo/sub_menu.tmpl
  27. 11
      vendor/github.com/src-d/enry/v2/.gitignore
  28. 132
      vendor/github.com/src-d/enry/v2/.travis.yml
  29. 61
      vendor/github.com/src-d/enry/v2/CONTRIBUTING.md
  30. 25
      vendor/github.com/src-d/enry/v2/DCO
  31. 201
      vendor/github.com/src-d/enry/v2/LICENSE
  32. 1
      vendor/github.com/src-d/enry/v2/MAINTAINERS
  33. 82
      vendor/github.com/src-d/enry/v2/Makefile
  34. 328
      vendor/github.com/src-d/enry/v2/README.md
  35. 107
      vendor/github.com/src-d/enry/v2/classifier.go
  36. 472
      vendor/github.com/src-d/enry/v2/common.go
  37. 783
      vendor/github.com/src-d/enry/v2/data/alias.go
  38. 254
      vendor/github.com/src-d/enry/v2/data/colors.go
  39. 7
      vendor/github.com/src-d/enry/v2/data/commit.go
  40. 1358
      vendor/github.com/src-d/enry/v2/data/content.go
  41. 3
      vendor/github.com/src-d/enry/v2/data/doc.go
  42. 26
      vendor/github.com/src-d/enry/v2/data/documentation.go
  43. 1629
      vendor/github.com/src-d/enry/v2/data/extension.go
  44. 241
      vendor/github.com/src-d/enry/v2/data/filename.go
  45. 170527
      vendor/github.com/src-d/enry/v2/data/frequencies.go
  46. 35
      vendor/github.com/src-d/enry/v2/data/heuristics.go
  47. 124
      vendor/github.com/src-d/enry/v2/data/interpreter.go
  48. 226
      vendor/github.com/src-d/enry/v2/data/mimeType.go
  49. 109
      vendor/github.com/src-d/enry/v2/data/rule/rule.go
  50. 526
      vendor/github.com/src-d/enry/v2/data/type.go
  51. 166
      vendor/github.com/src-d/enry/v2/data/vendor.go
  52. 16
      vendor/github.com/src-d/enry/v2/enry.go
  53. 11
      vendor/github.com/src-d/enry/v2/go.mod
  54. 17
      vendor/github.com/src-d/enry/v2/go.sum
  55. 7
      vendor/github.com/src-d/enry/v2/internal/tokenizer/common.go
  56. 2226
      vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/lex.linguist_yy.c
  57. 336
      vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/lex.linguist_yy.h
  58. 15
      vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/linguist.h
  59. 71
      vendor/github.com/src-d/enry/v2/internal/tokenizer/flex/tokenize_c.go
  60. 214
      vendor/github.com/src-d/enry/v2/internal/tokenizer/tokenize.go
  61. 15
      vendor/github.com/src-d/enry/v2/internal/tokenizer/tokenize_c.go
  62. 17
      vendor/github.com/src-d/enry/v2/regex/oniguruma.go
  63. 17
      vendor/github.com/src-d/enry/v2/regex/standard.go
  64. 84
      vendor/github.com/src-d/enry/v2/utils.go
  65. 20
      vendor/github.com/src-d/go-oniguruma/.travis.yml
  66. 19
      vendor/github.com/src-d/go-oniguruma/LICENSE
  67. 20
      vendor/github.com/src-d/go-oniguruma/README.md
  68. 184
      vendor/github.com/src-d/go-oniguruma/chelper.c
  69. 14
      vendor/github.com/src-d/go-oniguruma/chelper.h
  70. 27
      vendor/github.com/src-d/go-oniguruma/constants.go
  71. 1
      vendor/github.com/src-d/go-oniguruma/go.mod
  72. 36
      vendor/github.com/src-d/go-oniguruma/quotemeta.go
  73. 668
      vendor/github.com/src-d/go-oniguruma/regex.go
  74. 22
      vendor/github.com/toqueteos/trie/LICENSE.txt
  75. 7
      vendor/github.com/toqueteos/trie/README.md
  76. 1
      vendor/github.com/toqueteos/trie/go.mod
  77. 102
      vendor/github.com/toqueteos/trie/trie.go
  78. 24
      vendor/gopkg.in/toqueteos/substring.v1/.gitignore
  79. 11
      vendor/gopkg.in/toqueteos/substring.v1/.travis.yml
  80. 22
      vendor/gopkg.in/toqueteos/substring.v1/LICENSE
  81. 80
      vendor/gopkg.in/toqueteos/substring.v1/README.md
  82. 229
      vendor/gopkg.in/toqueteos/substring.v1/bytes.go
  83. 10
      vendor/gopkg.in/toqueteos/substring.v1/lib.go
  84. 216
      vendor/gopkg.in/toqueteos/substring.v1/string.go
  85. 13
      vendor/modules.txt
  86. 8
      web_src/js/index.js
  87. 10
      web_src/less/_base.less
  88. 20
      web_src/less/_repository.less
  89. 6
      web_src/less/themes/theme-arc-green.less

1
go.mod

@ -85,6 +85,7 @@ require (
github.com/sergi/go-diff v1.0.0
github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
github.com/src-d/enry/v2 v2.1.0
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect
github.com/stretchr/testify v1.4.0
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect

9
go.sum

@ -508,8 +508,13 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/src-d/enry v1.7.3 h1:jG2fmEaQaURh0qqU/sn82BRzVa6d4EVHJIw6gc98bak=
github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI=
github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -530,6 +535,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk=
github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM=
github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ=
@ -747,6 +754,8 @@ gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw=
gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=
gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

3
integrations/testlogger.go

@ -13,7 +13,6 @@ import (
"strings"
"sync"
"testing"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
@ -101,7 +100,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
}
writerCloser.setT(&t)
return func() {
if err := queue.GetManager().FlushAll(context.Background(), 20*time.Second); err != nil {
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
t.Errorf("Flushing queues failed with error %v", err)
}
_ = writerCloser.Close()

2
models/migrations/migrations.go

@ -186,6 +186,8 @@ var migrations = []Migration{
NewMigration("Add some columns on review for migration", addReviewMigrateInfo),
// v126 -> v127
NewMigration("Fix topic repository count", fixTopicRepositoryCount),
// v127 -> v128
NewMigration("add repository code language statistics", addLanguageStats),
}
// Migrate database to current version

45
models/migrations/v127.go

@ -0,0 +1,45 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/xorm"
)
func addLanguageStats(x *xorm.Engine) error {
// LanguageStat see models/repo_language_stats.go
type LanguageStat struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
CommitID string
IsPrimary bool
Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
Color string `xorm:"-"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
}
type RepoIndexerType int
// RepoIndexerStatus see models/repo_stats_indexer.go
type RepoIndexerStatus struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX(s)"`
CommitSha string `xorm:"VARCHAR(40)"`
IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
}
if err := x.Sync2(new(LanguageStat)); err != nil {
return fmt.Errorf("Sync2: %v", err)
}
if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
return fmt.Errorf("Sync2: %v", err)
}
return nil
}

1
models/models.go

@ -116,6 +116,7 @@ func init() {
new(OAuth2AuthorizationCode),
new(OAuth2Grant),
new(Task),
new(LanguageStat),
)
gonicNames := []string{"SSL", "UID"}

9
models/repo.go

@ -175,8 +175,9 @@ type Repository struct {
*Mirror `xorm:"-"`
Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
RenderingMetas map[string]string `xorm:"-"`
Units []*RepoUnit `xorm:"-"`
RenderingMetas map[string]string `xorm:"-"`
Units []*RepoUnit `xorm:"-"`
PrimaryLanguage *LanguageStat `xorm:"-"`
IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"`
ForkID int64 `xorm:"INDEX"`
@ -185,7 +186,8 @@ type Repository struct {
TemplateID int64 `xorm:"INDEX"`
TemplateRepo *Repository `xorm:"-"`
Size int64 `xorm:"NOT NULL DEFAULT 0"`
IndexerStatus *RepoIndexerStatus `xorm:"-"`
CodeIndexerStatus *RepoIndexerStatus `xorm:"-"`
StatsIndexerStatus *RepoIndexerStatus `xorm:"-"`
IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"`
CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"`
Topics []string `xorm:"TEXT JSON"`
@ -1504,6 +1506,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
&Notification{RepoID: repoID},
&CommitStatus{RepoID: repoID},
&RepoIndexerStatus{RepoID: repoID},
&LanguageStat{RepoID: repoID},
&Comment{RefRepoID: repoID},
&Task{RepoID: repoID},
); err != nil {

81
models/repo_indexer.go

@ -10,21 +10,32 @@ import (
"xorm.io/builder"
)
// RepoIndexerType specifies the repository indexer type
type RepoIndexerType int
const (
// RepoIndexerTypeCode code indexer
RepoIndexerTypeCode RepoIndexerType = iota // 0
// RepoIndexerTypeStats repository stats indexer
RepoIndexerTypeStats // 1
)
// RepoIndexerStatus status of a repo's entry in the repo indexer
// For now, implicitly refers to default branch
type RepoIndexerStatus struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
CommitSha string `xorm:"VARCHAR(40)"`
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX(s)"`
CommitSha string `xorm:"VARCHAR(40)"`
IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
}
// GetUnindexedRepos returns repos which do not have an indexer status
func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageSize int) ([]int64, error) {
ids := make([]int64, 0, 50)
cond := builder.Cond(builder.IsNull{
"repo_indexer_status.id",
})
sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id")
sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
if maxRepoID > 0 {
cond = builder.And(cond, builder.Lte{
"repository.id": maxRepoID,
@ -43,40 +54,64 @@ func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
return ids, err
}
// GetIndexerStatus loads repo codes indxer status
func (repo *Repository) GetIndexerStatus() error {
if repo.IndexerStatus != nil {
return nil
// getIndexerStatus loads repo codes indxer status
func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
switch indexerType {
case RepoIndexerTypeCode:
if repo.CodeIndexerStatus != nil {
return repo.CodeIndexerStatus, nil
}
case RepoIndexerTypeStats:
if repo.StatsIndexerStatus != nil {
return repo.StatsIndexerStatus, nil
}
}
status := &RepoIndexerStatus{RepoID: repo.ID}
has, err := x.Get(status)
status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
has, err := e.Get(status)
if err != nil {
return err
return nil, err
} else if !has {
status.CommitSha = ""
}
repo.IndexerStatus = status
return nil
switch indexerType {
case RepoIndexerTypeCode:
repo.CodeIndexerStatus = status
case RepoIndexerTypeStats:
repo.StatsIndexerStatus = status
}
return status, nil
}
// UpdateIndexerStatus updates indexer status
func (repo *Repository) UpdateIndexerStatus(sha string) error {
if err := repo.GetIndexerStatus(); err != nil {
// GetIndexerStatus loads repo codes indxer status
func (repo *Repository) GetIndexerStatus(indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
return repo.getIndexerStatus(x, indexerType)
}
// updateIndexerStatus updates indexer status
func (repo *Repository) updateIndexerStatus(e Engine, indexerType RepoIndexerType, sha string) error {
status, err := repo.getIndexerStatus(e, indexerType)
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %v", repo.FullName(), err)
}
if len(repo.IndexerStatus.CommitSha) == 0 {
repo.IndexerStatus.CommitSha = sha
_, err := x.Insert(repo.IndexerStatus)
if len(status.CommitSha) == 0 {
status.CommitSha = sha
_, err := e.Insert(status)
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
}
return nil
}
repo.IndexerStatus.CommitSha = sha
_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
Update(repo.IndexerStatus)
status.CommitSha = sha
_, err = e.ID(status.ID).Cols("commit_sha").
Update(status)
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
}
return nil
}
// UpdateIndexerStatus updates indexer status
func (repo *Repository) UpdateIndexerStatus(indexerType RepoIndexerType, sha string) error {
return repo.updateIndexerStatus(x, indexerType, sha)
}

137
models/repo_language_stats.go

@ -0,0 +1,137 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"math"
"strings"
"code.gitea.io/gitea/modules/timeutil"
"github.com/src-d/enry/v2"
)
// LanguageStat describes language statistics of a repository
type LanguageStat struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
CommitID string
IsPrimary bool
Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
Color string `xorm:"-"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
}
// LanguageStatList defines a list of language statistics
type LanguageStatList []*LanguageStat
func (stats LanguageStatList) loadAttributes() {
for i := range stats {
stats[i].Color = enry.GetColor(stats[i].Language)
}
}
func (repo *Repository) getLanguageStats(e Engine) (LanguageStatList, error) {
stats := make(LanguageStatList, 0, 6)
if err := e.Where("`repo_id` = ?", repo.ID).Desc("`percentage`").Find(&stats); err != nil {
return nil, err
}
stats.loadAttributes()
return stats, nil
}
// GetLanguageStats returns the language statistics for a repository
func (repo *Repository) GetLanguageStats() (LanguageStatList, error) {
return repo.getLanguageStats(x)
}
// GetTopLanguageStats returns the top language statistics for a repository
func (repo *Repository) GetTopLanguageStats(limit int) (LanguageStatList, error) {
stats, err := repo.getLanguageStats(x)
if err != nil {
return nil, err
}
topstats := make(LanguageStatList, 0, limit)
var other float32
for i := range stats {
if stats[i].Language == "other" || len(topstats) >= limit {
other += stats[i].Percentage
continue
}
topstats = append(topstats, stats[i])
}
if other > 0 {
topstats = append(topstats, &LanguageStat{
RepoID: repo.ID,
Language: "other",
Color: "#cccccc",
Percentage: float32(math.Round(float64(other)*10) / 10),
})
}
return topstats, nil
}
// UpdateLanguageStats updates the language statistics for repository
func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]float32) error {
sess := x.NewSession()
if err := sess.Begin(); err != nil {
return err
}
defer sess.Close()
oldstats, err := repo.getLanguageStats(sess)
if err != nil {
return err
}
var topLang string
var p float32
for lang, perc := range stats {
if perc > p {
p = perc
topLang = strings.ToLower(lang)
}
}
for lang, perc := range stats {
upd := false
llang := strings.ToLower(lang)
for _, s := range oldstats {
// Update already existing language
if strings.ToLower(s.Language) == llang {
s.CommitID = commitID
s.IsPrimary = llang == topLang
s.Percentage = perc
if _, err := sess.ID(s.ID).Cols("`commit_id`", "`percentage`", "`is_primary`").Update(s); err != nil {
return err
}
upd = true
break
}
}
// Insert new language
if !upd {
if _, err := sess.Insert(&LanguageStat{
RepoID: repo.ID,
CommitID: commitID,
IsPrimary: llang == topLang,
Language: lang,
Percentage: perc,
}); err != nil {
return err
}
}
}
// Delete old languages
if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil {
return err
}
if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil {
return err
}
return sess.Commit()
}

31
models/repo_list.go

@ -46,11 +46,14 @@ func (repos RepositoryList) loadAttributes(e Engine) error {
return nil
}
// Load owners.
set := make(map[int64]struct{})
repoIDs := make([]int64, len(repos))
for i := range repos {
set[repos[i].OwnerID] = struct{}{}
repoIDs[i] = repos[i].ID
}
// Load owners.
users := make(map[int64]*User, len(set))
if err := e.
Where("id > 0").
@ -61,6 +64,25 @@ func (repos RepositoryList) loadAttributes(e Engine) error {
for i := range repos {
repos[i].Owner = users[repos[i].OwnerID]
}
// Load primary language.
stats := make(LanguageStatList, 0, len(repos))
if err := e.
Where("`is_primary` = ? AND `language` != ?", true, "other").
In("`repo_id`", repoIDs).
Find(&stats); err != nil {
return fmt.Errorf("find primary languages: %v", err)
}
stats.loadAttributes()
for i := range repos {
for _, st := range stats {
if st.RepoID == repos[i].ID {
repos[i].PrimaryLanguage = st
break
}
}
}
return nil
}
@ -119,7 +141,6 @@ type SearchRepoOptions struct {
OrderBy SearchOrderBy
Private bool // Include private repositories in results
StarredByID int64
IsProfile bool
AllPublic bool // Include also all public repositories of users and public organisations
AllLimited bool // Include also all public repositories of limited organisations
// None -> include collaborative AND non-collaborative
@ -306,10 +327,8 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
return nil, 0, fmt.Errorf("Repo: %v", err)
}
if !opts.IsProfile {
if err = repos.loadAttributes(sess); err != nil {
return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}
if err = repos.loadAttributes(sess); err != nil {
return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}
return repos, count, nil

116
modules/git/repo_language_stats.go

@ -0,0 +1,116 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"bytes"
"io"
"io/ioutil"
"math"
"path/filepath"
"github.com/src-d/enry/v2"
"gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
)
const fileSizeLimit int64 = 16 * 1024 * 1024
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]float32, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
if err != nil {
return nil, err
}
commit, err := r.CommitObject(*rev)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
var total int64
err = tree.Files().ForEach(func(f *object.File) error {
if enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
// TODO: Use .gitattributes file for linguist overrides
language, ok := enry.GetLanguageByExtension(f.Name)
if !ok {
if language, ok = enry.GetLanguageByFilename(f.Name); !ok {
content, err := readFile(f, fileSizeLimit)
if err != nil {
return nil
}
language = enry.GetLanguage(filepath.Base(f.Name), content)
if language == enry.OtherLanguage {
return nil
}
}
}
if language != "" {
sizes[language] += f.Size
total += f.Size
}
return nil
})
if err != nil {
return nil, err
}
stats := make(map[string]float32)
var otherPerc float32 = 100
for language, size := range sizes {
perc := float32(math.Round(float64(size)/float64(total)*1000) / 10)
if perc <= 0.1 {
continue
}
otherPerc -= perc
stats[language] = perc
}
otherPerc = float32(math.Round(float64(otherPerc)*10) / 10)
if otherPerc > 0 {
stats["other"] = otherPerc
}
return stats, nil
}
func readFile(f *object.File, limit int64) ([]byte, error) {
r, err := f.Reader()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := f.Size
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

2
modules/indexer/code/bleve.go

@ -267,7 +267,7 @@ func (b *BleveIndexer) Index(repoID int64) error {
if err = batch.Flush(); err != nil {
return err
}
return repo.UpdateIndexerStatus(sha)
return repo.UpdateIndexerStatus(models.RepoIndexerTypeCode, sha)
}
// Delete deletes indexes by ids

7
modules/indexer/code/git.go

@ -35,11 +35,12 @@ func getDefaultBranchSha(repo *models.Repository) (string, error) {
// getRepoChanges returns changes to repo since last indexer update
func getRepoChanges(repo *models.Repository, revision string) (*repoChanges, error) {
if err := repo.GetIndexerStatus(); err != nil {
status, err := repo.GetIndexerStatus(models.RepoIndexerTypeCode)
if err != nil {
return nil, err
}
if len(repo.IndexerStatus.CommitSha) == 0 {
if len(status.CommitSha) == 0 {
return genesisChanges(repo, revision)
}
return nonGenesisChanges(repo, revision)
@ -98,7 +99,7 @@ func genesisChanges(repo *models.Repository, revision string) (*repoChanges, err
// nonGenesisChanges get changes since the previous indexer update
func nonGenesisChanges(repo *models.Repository, revision string) (*repoChanges, error) {
diffCmd := git.NewCommand("diff", "--name-status",
repo.IndexerStatus.CommitSha, revision)
repo.CodeIndexerStatus.CommitSha, revision)
stdout, err := diffCmd.RunInDir(repo.RepoPath())
if err != nil {
// previous commit sha may have been removed by a force push, so

2
modules/indexer/code/queue.go

@ -109,7 +109,7 @@ func populateRepoIndexer() {
return
default:
}
ids, err := models.GetUnindexedRepos(maxRepoID, 0, 50)
ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeCode, maxRepoID, 0, 50)
if err != nil {
log.Error("populateRepoIndexer: %v", err)
return

54
modules/indexer/stats/db.go

@ -0,0 +1,54 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package stats
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
)
// DBIndexer implements Indexer interface to use database's like search
type DBIndexer struct {
}
// Index repository status function
func (db *DBIndexer) Index(id int64) error {
repo, err := models.GetRepositoryByID(id)
if err != nil {
return err
}
status, err := repo.GetIndexerStatus(models.RepoIndexerTypeStats)
if err != nil {
return err
}
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return err
}
defer gitRepo.Close()
// Get latest commit for default branch
commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
if err != nil {
return err
}
// Do not recalculate stats if already calculated for this commit
if status.CommitSha == commitID {
return nil
}
// Calculate and save language statistics to database
stats, err := gitRepo.GetLanguageStats(commitID)
if err != nil {
return err
}
return repo.UpdateLanguageStats(commitID, stats)
}
// Close dummy function
func (db *DBIndexer) Close() {
}

85
modules/indexer/stats/indexer.go

@ -0,0 +1,85 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package stats
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
)
// Indexer defines an interface to index repository stats
type Indexer interface {
Index(id int64) error
Close()
}
// indexer represents a indexer instance
var indexer Indexer
// Init initialize the repo indexer
func Init() error {
indexer = &DBIndexer{}
if err := initStatsQueue(); err != nil {
return err
}
go populateRepoIndexer()
return nil
}
// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
func populateRepoIndexer() {
log.Info("Populating the repo stats indexer with existing repositories")
isShutdown := graceful.GetManager().IsShutdown()
exist, err := models.IsTableNotEmpty("repository")
if err != nil {
log.Fatal("System error: %v", err)
} else if !exist {
return
}
var maxRepoID int64
if maxRepoID, err = models.GetMaxID("repository"); err != nil {
log.Fatal("System error: %v", err)
}
// start with the maximum existing repo ID and work backwards, so that we
// don't include repos that are created after gitea starts; such repos will
// already be added to the indexer, and we don't need to add them again.
for maxRepoID > 0 {
select {
case <-isShutdown:
log.Info("Repository Stats Indexer population shutdown before completion")
return
default:
}
ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeStats, maxRepoID, 0, 50)
if err != nil {
log.Error("populateRepoIndexer: %v", err)
return
} else if len(ids) == 0 {
break
}
for _, id := range ids {
select {
case <-isShutdown:
log.Info("Repository Stats Indexer population shutdown before completion")
return
default:
}
if err := statsQueue.Push(id); err != nil {
log.Error("statsQueue.Push: %v", err)
}
maxRepoID = id - 1
}
}
log.Info("Done (re)populating the repo stats indexer with existing repositories")
}

42
modules/indexer/stats/indexer_test.go

@ -0,0 +1,42 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package stats
import (
"path/filepath"
"testing"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/setting"
"gopkg.in/ini.v1"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
models.MainTest(m, filepath.Join("..", "..", ".."))
}
func TestRepoStatsIndex(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())
setting.Cfg = ini.Empty()
setting.NewQueueService()
err := Init()
assert.NoError(t, err)
time.Sleep(5 * time.Second)
repo, err := models.GetRepositoryByID(1)
assert.NoError(t, err)
langs, err := repo.GetTopLanguageStats(5)
assert.NoError(t, err)
assert.Len(t, langs, 1)
assert.Equal(t, "other", langs[0].Language)
assert.Equal(t, float32(100), langs[0].Percentage)
}

43
modules/indexer/stats/queue.go

@ -0,0 +1,43 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package stats
import (
"fmt"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
)
// statsQueue represents a queue to handle repository stats updates
var statsQueue queue.Queue
// handle passed PR IDs and test the PRs
func handle(data ...queue.Data) {
for _, datum := range data {
opts := datum.(int64)
if err := indexer.Index(opts); err != nil {
log.Error("stats queue idexer.Index(%d) failed: %v", opts, err)
}
}
}
func initStatsQueue() error {
statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue)
if statsQueue == nil {
return fmt.Errorf("Unable to create repo_stats_update Queue")
}
go graceful.GetManager().RunWithShutdownFns(statsQueue.Run)
return nil
}
// UpdateRepoIndexer update a repository's entries in the indexer
func UpdateRepoIndexer(repo *models.Repository) error {
return statsQueue.Push(repo.ID)
}

7
modules/notification/indexer/indexer.go

@ -9,6 +9,7 @@ import (
"code.gitea.io/gitea/modules/git"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
stats_indexer "code.gitea.io/gitea/modules/indexer/stats"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification/base"
"code.gitea.io/gitea/modules/repository"
@ -117,12 +118,18 @@ func (r *indexerNotifier) NotifyMigrateRepository(doer *models.User, u *models.U
if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty {
code_indexer.UpdateRepoIndexer(repo)
}
if err := stats_indexer.UpdateRepoIndexer(repo); err != nil {
log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err)
}
}
func (r *indexerNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) {
if setting.Indexer.RepoIndexerEnabled && refName == git.BranchPrefix+repo.DefaultBranch {
code_indexer.UpdateRepoIndexer(repo)
}
if err := stats_indexer.UpdateRepoIndexer(repo); err != nil {
log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err)
}
}
func (r *indexerNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string) {

1
options/locale/locale_en-US.ini

@ -641,6 +641,7 @@ forks = Forks
pick_reaction = Pick your reaction
reactions_more = and %d more
unit_disabled = The site administrator has disabled this repository section.
language_other = Other
template.items = Template Items
template.git_content = Git Content (Default Branch)

4
routers/init.go

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/highlight"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
stats_indexer "code.gitea.io/gitea/modules/indexer/stats"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external"
@ -111,6 +112,9 @@ func GlobalInit(ctx context.Context) {
cron.NewContext()
issue_indexer.InitIssueIndexer(false)
code_indexer.Init()
if err := stats_indexer.Init(); err != nil {
log.Fatal("Failed to initialize repository stats indexer queue: %v", err)
}
mirror_service.InitSyncMirrors()
webhook.InitDeliverHooks()
if err := pull_service.Init(); err != nil {

1
routers/org/home.go

@ -85,7 +85,6 @@ func Home(ctx *context.Context) {
OrderBy: orderBy,
Private: ctx.IsSigned,
Actor: ctx.User,
IsProfile: true,
IncludeDescription: setting.UI.SearchRepoDescription,
})
if err != nil {

15
routers/repo/view.go

@ -457,6 +457,16 @@ func Home(ctx *context.Context) {
ctx.NotFound("Home", fmt.Errorf(ctx.Tr("units.error.no_unit_allowed_repo")))
}
func renderLanguageStats(ctx *context.Context) {
langs, err := ctx.Repo.Repository.GetTopLanguageStats(5)
if err != nil {
ctx.ServerError("Repo.GetTopLanguageStats", err)
return
}
ctx.Data["LanguageStats"] = langs
}
func renderCode(ctx *context.Context) {
ctx.Data["PageIsViewCode"] = true
@ -497,6 +507,11 @@ func renderCode(ctx *context.Context) {
return
}
renderLanguageStats(ctx)
if ctx.Written() {
return
}
if entry.IsDir() {
renderDirectory(ctx, treeLink)
} else {

1
routers/user/profile.go

@ -220,7 +220,6 @@ func Profile(ctx *context.Context) {
OwnerID: ctxUser.ID,
OrderBy: orderBy,
Private: ctx.IsSigned,
IsProfile: true,
Collaborate: util.OptionalBoolFalse,
TopicOnly: topicOnly,
IncludeDescription: setting.UI.SearchRepoDescription,

3
templates/explore/repo_list.tmpl

@ -21,6 +21,9 @@
{{end}}
{{end}}
<div class="ui right metas">
{{if .PrimaryLanguage }}
<span class="text grey"><i class="color-icon" style="background-color: {{.PrimaryLanguage.Color}}"></i>{{ .PrimaryLanguage.Language }}</span>
{{end}}
<span class="text grey"><i class="octicon octicon-star"></i> {{.NumStars}}</span>
<span class="text grey"><i class="octicon octicon-git-branch"></i> {{.NumForks}}</span>
</div>

51
templates/repo/sub_menu.tmpl

@ -1,17 +1,42 @@
<div class="ui segment sub-menu">
<div class="ui two horizontal center link list">
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}}
<div class="item{{if .PageIsCommits}} active{{end}}">
<a class="ui" href="{{.RepoLink}}/commits{{if .IsViewBranch}}/branch{{else if .IsViewTag}}/tag{{else if .IsViewCommit}}/commit{{end}}/{{EscapePound .BranchName}}"><i class="octicon octicon-history"></i> <b>{{.CommitsCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }}</a>
</div>
{{end}}
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }}
<div class="item{{if .PageIsBranches}} active{{end}}">
<a class="ui" href="{{.RepoLink}}/branches/"><i class="octicon octicon-git-branch"></i> <b>{{.BranchesCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }}</a>
</div>
<div class="ui segments">
<div class="ui segment sub-menu repository-menu">
<div class="ui two horizontal center link list">
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}}
<div class="item{{if .PageIsCommits}} active{{end}}">
<a class="ui" href="{{.RepoLink}}/commits{{if .IsViewBranch}}/branch{{else if .IsViewTag}}/tag{{else if .IsViewCommit}}/commit{{end}}/{{EscapePound .BranchName}}"><i class="octicon octicon-history"></i> <b>{{.CommitsCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }}</a>
</div>
{{end}}
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) }}
<div class="item{{if .PageIsBranches}} active{{end}}">
<a class="ui" href="{{.RepoLink}}/branches/"><i class="octicon octicon-git-branch"></i> <b>{{.BranchesCount}}</b> {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }}</a>
</div>
<div class="item">
<a class="ui" href="#"><i class="octicon octicon-database"></i> <b>{{SizeFmt .Repository.Size}}</b></a>
</div>
{{end}}
</div>
</div>
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo) .LanguageStats }}
<div class="ui segment sub-menu language-stats-details" style="display: none">
<div class="ui horizontal center link list">
{{range .LanguageStats}}
<div class="item">
<a class="ui" href="#"><i class="octicon octicon-database"></i> <b>{{SizeFmt .Repository.Size}}</b></a>
<i class="color-icon" style="background-color: {{ .Color }}"></i>
<span class="ui"><b>
{{if eq .Language "other" }}
{{ $.i18n.Tr "repo.language_other" }}
{{else}}
{{ .Language }}
{{end}}
</b> {{ .Percentage }}%</span>
</div>
{{end}}
{{end}}
</div>
</div>
<a class="ui segment language-stats">
{{range .LanguageStats}}
<div class="bar" style="width: {{ .Percentage }}%; background-color: {{ .Color }}">&nbsp;</div>
{{end}}
</a>
{{end}}
</div>

11
vendor/github.com/src-d/enry/v2/.gitignore

@ -0,0 +1,11 @@
.linguist
benchmarks/output
.ci
Makefile.main
.shared
.idea
.docsrv-resources
build/
vendor/
java/lib/
.vscode/

132
vendor/github.com/src-d/enry/v2/.travis.yml

@ -0,0 +1,132 @@
dist: trusty
language: go
go:
- '1.12.x'
- '1.11.x'
env:
global:
- GO_VERSION_FOR_JVM='1.11.x'
- CGO_ENABLED=0
- GO111MODULE=on
- ONIGURUMA_VERSION='6.9.1'
matrix:
- ONIGURUMA=0
- ONIGURUMA=1
matrix:
fast_finish: true
stages:
- name: test
- name: release
if: tag IS present
- name: publish
if: tag IS present
stage: test
install:
- >
if [[ "${ONIGURUMA}" -gt 0 ]]; then
export CGO_ENABLED=1
export GO_TAGS='oniguruma'
# install oniguruma manually as trusty has only ancient 5.x
sudo apt-get install -y dpkg # dpkg >= 1.17.5ubuntu5.8 fixes https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
sudo dpkg -i "libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
fi;
script:
- make test-coverage
after_success:
- bash <(curl -s https://codecov.io/bash)
jobs:
include:
- name: 'java unit-tests'
stage: test
language: scala
jdk: oraclejdk8
install:
- export CGO_ENABLED=1
- eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)"
- go version
before_script:
- cd java
- make
script:
- make test
- name: 'linux packages'
stage: release
install:
- go version
script: make packages
deploy:
provider: releases
api_key:
secure: $GITHUB_TOKEN
file_glob: true
file: build/*.tar.gz
skip_cleanup: true
on:
tags: true
- name: 'linux shared lib'
stage: release
install:
- go version
script: make linux-shared
deploy:
provider: releases
api_key:
secure: $GITHUB_TOKEN
file:
- ./.shared/linux-x86-64/libenry.so
skip_cleanup: true
on:
tags: true
- name: 'macOS shared lib'
stage: release
env:
- OSXCROSS_PACKAGE="osxcross_3034f7149716d815bc473d0a7b35d17e4cf175aa.tar.gz"
- OSXCROSS_URL="https://github.com/bblfsh/client-scala/releases/download/v1.5.2/${OSXCROSS_PACKAGE}"
- PATH="/$HOME/osxcross/bin:$PATH"
install:
- go version
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils
- cd ${HOME}
- curl -sfSL ${OSXCROSS_URL} | tar -C ${HOME} -xzf -
- cd $GOPATH/src/github.com/src-d/enry
script: make darwin-shared
deploy:
provider: releases
api_key:
secure: $GITHUB_TOKEN
file: ./.shared/darwin/libenry.dylib
skip_cleanup: true
on:
tags: true
- name: 'java: publish to maven'
stage: publish
language: scala
jdk: oraclejdk8
install:
- export CGO_ENABLED=1
- eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=$GO_VERSION_FOR_JVM bash)"
- go version
before_script:
- cd java
- make
- curl -o ./shared/linux-x86-64/libenry.so -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.so" || travis_terminate 1
- mkdir -p ./shared/darwin
- curl -o ./shared/darwin/libenry.dylib -sfL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.dylib" || travis_terminate 1
- openssl aes-256-cbc -K $encrypted_a0e1c69dbbc7_key -iv $encrypted_a0e1c69dbbc7_iv -in key.asc.enc -out key.asc -d
- gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
script:
- make test # ensure the shared objects are functional
- ./sbt publishLocal
- ./sbt publishSigned
- ./sbt sonatypeRelease

61
vendor/github.com/src-d/enry/v2/CONTRIBUTING.md

@ -0,0 +1,61 @@
# source{d} Contributing Guidelines
source{d} projects accept contributions via GitHub pull requests.
This document outlines some of the
conventions on development workflow, commit message formatting, contact points,
and other resources to make it easier to get your contribution accepted.
## Certificate of Origin
By contributing to this project, you agree to the [Developer Certificate of
Origin (DCO)](DCO). This document was created by the Linux Kernel community and is a
simple statement that you, as a contributor, have the legal right to make the
contribution.
In order to show your agreement with the DCO you should include at the end of the commit message,
the following line: `Signed-off-by: John Doe <john.doe@example.com>`, using your real name.
This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`.
If you find yourself pushed a few commits without `Signed-off-by`, you can still add it afterwards. We wrote a manual which can help: [fix-DCO.md](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md).
## Support Channels
The official support channels, for both users and contributors, are:
- GitHub issues: each repository has its own list of issues.
- Slack: join the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) community.
*Before opening a new issue or submitting a new pull request, it's helpful to
search the project - it's likely that another user has already reported the
issue you're facing, or it's a known issue that we're already aware of.
## How to Contribute
Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects.
In order for a PR to be accepted it needs to pass this list of requirements:
- The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it.
- All PRs must be written idiomaticly:
- for Go: formatted according to [gofmt](https://golang.org/cmd/gofmt/), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/)
- for other languages, similar constraints apply.
- They should in general include tests, and those shall pass.
- If the PR is a bug fix, it has to include a new unit test that fails before the patch is merged.
- If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality.
- In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project.
### Format of the commit message
Every commit message should describe what was changed, under which context and, if applicable, the GitHub issue it relates to:
```
plumbing: packp, Skip argument validations for unknown capabilities. Fixes #623
```
The format can be described more formally as follows:
```
<package>: <subpackage>, <what changed>. [Fixes #<issue-number>]
```

25
vendor/github.com/src-d/enry/v2/DCO

@ -0,0 +1,25 @@
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.

201
vendor/github.com/src-d/enry/v2/LICENSE

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2017 Sourced Technologies, S.L.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY