Browse Source

Graceful: Xorm, RepoIndexer, Cron and Others (#9282)

* Change graceful to use a singleton obtained through GetManager instead of a global.
* Graceful: Make TestPullRequests shutdownable
* Graceful: Make the cron tasks graceful
* Graceful: AddTestPullRequest run in graceful ctx
* Graceful: SyncMirrors shutdown
* Graceful: SetDefaultContext for Xorm to be HammerContext
* Avoid starting graceful for migrate commands and checkout
* Graceful: DeliverHooks now can be shutdown
* Fix multiple syncing errors in modules/sync/UniqueQueue &  Make UniqueQueue closable
* Begin the process of making the repo indexer shutdown gracefully
lunny/display_deleted_branch2
zeripath 3 years ago
committed by GitHub
parent
commit
e3c3b33ea7
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      cmd/migrate.go
  2. 10
      cmd/web.go
  3. 4
      cmd/web_graceful.go
  4. 3
      contrib/pr/checkout.go
  5. 6
      integrations/auth_ldap_test.go
  6. 8
      integrations/integration_test.go
  7. 3
      integrations/migration-test/migration_test.go
  8. 4
      models/branches.go
  9. 5
      models/models.go
  10. 9
      models/pull_list.go
  11. 64
      models/repo.go
  12. 43
      models/repo_indexer.go
  13. 35
      models/user.go
  14. 9
      modules/cron/cron.go
  15. 3
      modules/git/git.go
  16. 3
      modules/git/git_test.go
  17. 6
      modules/graceful/context.go
  18. 140
      modules/graceful/manager.go
  19. 45
      modules/graceful/manager_unix.go
  20. 31
      modules/graceful/manager_windows.go
  21. 2
      modules/graceful/net_unix.go
  22. 2
      modules/graceful/restart_unix.go
  23. 6
      modules/graceful/server.go
  24. 6
      modules/graceful/server_hooks.go
  25. 82
      modules/indexer/code/bleve.go
  26. 35
      modules/indexer/code/repo.go
  27. 2
      modules/indexer/issues/indexer.go
  28. 26
      modules/migrations/update.go
  29. 2
      modules/ssh/ssh_graceful.go
  30. 56
      modules/sync/unique_queue.go
  31. 63
      modules/webhook/deliver.go
  32. 5
      routers/admin/admin.go
  33. 11
      routers/init.go
  34. 3
      routers/install.go
  35. 32
      services/mirror/mirror.go
  36. 90
      services/pull/check.go
  37. 63
      services/pull/pull.go

4
cmd/migrate.go

@ -5,6 +5,8 @@
package cmd
import (
"context"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
@ -32,7 +34,7 @@ func runMigrate(ctx *cli.Context) error {
log.Trace("Log path: %s", setting.LogRootPath)
setting.InitDBConfig()
if err := models.NewEngine(migrations.Migrate); err != nil {
if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil {
log.Fatal("Failed to initialize ORM engine: %v", err)
return err
}

10
cmd/web.go

@ -5,6 +5,7 @@
package cmd
import (
"context"
"fmt"
"net/http"
_ "net/http/pprof" // Used for debugging if enabled and a web server is running
@ -96,6 +97,10 @@ func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
}
func runWeb(ctx *cli.Context) error {
managerCtx, cancel := context.WithCancel(context.Background())
graceful.InitManager(managerCtx)
defer cancel()
if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
} else {
@ -108,7 +113,7 @@ func runWeb(ctx *cli.Context) error {
}
// Perform global initialization
routers.GlobalInit()
routers.GlobalInit(graceful.GetManager().HammerContext())
// Set up Macaron
m := routes.NewMacaron()
@ -199,8 +204,7 @@ func runWeb(ctx *cli.Context) error {
log.Critical("Failed to start server: %v", err)
}
log.Info("HTTP Listener: %s Closed", listenAddr)
graceful.Manager.WaitForServers()
graceful.Manager.WaitForTerminate()
<-graceful.GetManager().Done()
log.Info("PID: %d Gitea Web Finished", os.Getpid())
log.Close()
return nil

4
cmd/web_graceful.go

@ -28,13 +28,13 @@ func runHTTPSWithTLSConfig(network, listenAddr string, tlsConfig *tls.Config, m
// NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
func NoHTTPRedirector() {
graceful.Manager.InformCleanup()
graceful.GetManager().InformCleanup()
}
// NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
// for our main HTTP/HTTPS service
func NoMainListener() {
graceful.Manager.InformCleanup()
graceful.GetManager().InformCleanup()
}
func runFCGI(network, listenAddr string, m http.Handler) error {

3
contrib/pr/checkout.go

@ -5,6 +5,7 @@ Checkout a PR and load the tests data into sqlite database
*/
import (
"context"
"flag"
"fmt"
"io/ioutil"
@ -92,7 +93,7 @@ func runPR() {
//x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
var helper testfixtures.Helper = &testfixtures.SQLite{}
models.NewEngine(func(_ *xorm.Engine) error {
models.NewEngine(context.Background(), func(_ *xorm.Engine) error {
return nil
})
models.HasEngine = true

6
integrations/auth_ldap_test.go

@ -5,6 +5,7 @@
package integrations
import (
"context"
"net/http"
"os"
"strings"
@ -147,7 +148,7 @@ func TestLDAPUserSync(t *testing.T) {
}
defer prepareTestEnv(t)()
addAuthSourceLDAP(t, "")
models.SyncExternalUsers()
models.SyncExternalUsers(context.Background())
session := loginUser(t, "user1")
// Check if users exists
@ -206,7 +207,8 @@ func TestLDAPUserSSHKeySync(t *testing.T) {
}
defer prepareTestEnv(t)()
addAuthSourceLDAP(t, "sshPublicKey")
models.SyncExternalUsers()
models.SyncExternalUsers(context.Background())
// Check if users has SSH keys synced
for _, u := range gitLDAPUsers {

8
integrations/integration_test.go

@ -6,6 +6,7 @@ package integrations
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
@ -24,6 +25,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/routers"
"code.gitea.io/gitea/routers/routes"
@ -55,6 +57,10 @@ func NewNilResponseRecorder() *NilResponseRecorder {
}
func TestMain(m *testing.M) {
managerCtx, cancel := context.WithCancel(context.Background())
graceful.InitManager(managerCtx)
defer cancel()
initIntegrationTest()
mac = routes.NewMacaron()
routes.RegisterRoutes(mac)
@ -171,7 +177,7 @@ func initIntegrationTest() {
}
defer db.Close()
}
routers.GlobalInit()
routers.GlobalInit(graceful.GetManager().HammerContext())
}
func prepareTestEnv(t testing.TB, skip ...int) func() {

3
integrations/migration-test/migration_test.go

@ -6,6 +6,7 @@ package migrations
import (
"compress/gzip"
"context"
"database/sql"
"fmt"
"io/ioutil"
@ -220,7 +221,7 @@ func doMigrationTest(t *testing.T, version string) {
err := models.SetEngine()
assert.NoError(t, err)
err = models.NewEngine(wrappedMigrate)
err = models.NewEngine(context.Background(), wrappedMigrate)
assert.NoError(t, err)
currentEngine.Close()
}

4
models/branches.go

@ -5,6 +5,7 @@
package models
import (
"context"
"fmt"
"time"
@ -525,7 +526,8 @@ func (deletedBranch *DeletedBranch) LoadUser() {
}
// RemoveOldDeletedBranches removes old deleted branches
func RemoveOldDeletedBranches() {
func RemoveOldDeletedBranches(ctx context.Context) {
// Nothing to do for shutdown or terminate
log.Trace("Doing: DeletedBranchesCleanup")
deleteBefore := time.Now().Add(-setting.Cron.DeletedBranchesCleanup.OlderThan)

5
models/models.go

@ -6,6 +6,7 @@
package models
import (
"context"
"database/sql"
"errors"
"fmt"
@ -164,11 +165,13 @@ func SetEngine() (err error) {
}
// NewEngine initializes a new xorm.Engine
func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) {
func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
if err = SetEngine(); err != nil {
return err
}
x.SetDefaultContext(ctx)
if err = x.Ping(); err != nil {
return err
}

9
models/pull_list.go

@ -68,11 +68,12 @@ func GetUnmergedPullRequestsByBaseInfo(repoID int64, branch string) ([]*PullRequ
Find(&prs)
}
// GetPullRequestsByCheckStatus returns all pull requests according the special checking status.
func GetPullRequestsByCheckStatus(status PullRequestStatus) ([]*PullRequest, error) {
prs := make([]*PullRequest, 0, 10)
return prs, x.
// GetPullRequestIDsByCheckStatus returns all pull requests according the special checking status.
func GetPullRequestIDsByCheckStatus(status PullRequestStatus) ([]int64, error) {
prs := make([]int64, 0, 10)
return prs, x.Table("pull_request").
Where("status=?", status).
Cols("pull_request.id").
Find(&prs)
}

64
models/repo.go

@ -7,6 +7,7 @@ package models
import (
"bytes"
"context"
"crypto/md5"
"errors"
"fmt"
@ -2098,19 +2099,27 @@ func DeleteRepositoryArchives() error {
}
// DeleteOldRepositoryArchives deletes old repository archives.
func DeleteOldRepositoryArchives() {
func DeleteOldRepositoryArchives(ctx context.Context) {
log.Trace("Doing: ArchiveCleanup")
if err := x.Where("id > 0").Iterate(new(Repository), deleteOldRepositoryArchives); err != nil {
if err := x.Where("id > 0").Iterate(new(Repository), func(idx int, bean interface{}) error {
return deleteOldRepositoryArchives(ctx, idx, bean)
}); err != nil {
log.Error("ArchiveClean: %v", err)
}
}
func deleteOldRepositoryArchives(idx int, bean interface{}) error {
func deleteOldRepositoryArchives(ctx context.Context, idx int, bean interface{}) error {
repo := bean.(*Repository)
basePath := filepath.Join(repo.RepoPath(), "archives")
for _, ty := range []string{"zip", "targz"} {
select {
case <-ctx.Done():
return fmt.Errorf("Aborted due to shutdown:\nin delete of old repository archives %v\nat delete file %s", repo, ty)
default:
}
path := filepath.Join(basePath, ty)
file, err := os.Open(path)
if err != nil {
@ -2133,6 +2142,11 @@ func deleteOldRepositoryArchives(idx int, bean interface{}) error {
minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan)
for _, info := range files {
if info.ModTime().Before(minimumOldestTime) && !info.IsDir() {
select {
case <-ctx.Done():
return fmt.Errorf("Aborted due to shutdown:\nin delete of old repository archives %v\nat delete file %s - %s", repo, ty, info.Name())
default:
}
toDelete := filepath.Join(path, info.Name())
// This is a best-effort purge, so we do not check error codes to confirm removal.
if err = os.Remove(toDelete); err != nil {
@ -2226,13 +2240,17 @@ func SyncRepositoryHooks() error {
}
// GitFsck calls 'git fsck' to check repository health.
func GitFsck() {
func GitFsck(ctx context.Context) {
log.Trace("Doing: GitFsck")
if err := x.
Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository),
func(idx int, bean interface{}) error {
select {
case <-ctx.Done():
return fmt.Errorf("Aborted due to shutdown")
default:
}
repo := bean.(*Repository)
repoPath := repo.RepoPath()
log.Trace("Running health check on repository %s", repoPath)
@ -2278,13 +2296,19 @@ type repoChecker struct {
desc string
}
func repoStatsCheck(checker *repoChecker) {
func repoStatsCheck(ctx context.Context, checker *repoChecker) {
results, err := x.Query(checker.querySQL)
if err != nil {
log.Error("Select %s: %v", checker.desc, err)
return
}
for _, result := range results {
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Aborting due to shutdown")
return
default:
}
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating %s: %d", checker.desc, id)
_, err = x.Exec(checker.correctSQL, id, id)
@ -2295,7 +2319,7 @@ func repoStatsCheck(checker *repoChecker) {
}
// CheckRepoStats checks the repository stats
func CheckRepoStats() {
func CheckRepoStats(ctx context.Context) {
log.Trace("Doing: CheckRepoStats")
checkers := []*repoChecker{
@ -2331,7 +2355,13 @@ func CheckRepoStats() {
},
}
for i := range checkers {
repoStatsCheck(checkers[i])
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Aborting due to shutdown")
return
default:
repoStatsCheck(ctx, checkers[i])
}
}
// ***** START: Repository.NumClosedIssues *****
@ -2341,6 +2371,12 @@ func CheckRepoStats() {
log.Error("Select %s: %v", desc, err)
} else {
for _, result := range results {
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Aborting due to shutdown")
return
default:
}
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating %s: %d", desc, id)
_, err = x.Exec("UPDATE `repository` SET num_closed_issues=(SELECT COUNT(*) FROM `issue` WHERE repo_id=? AND is_closed=? AND is_pull=?) WHERE id=?", id, true, false, id)
@ -2358,6 +2394,12 @@ func CheckRepoStats() {
log.Error("Select %s: %v", desc, err)
} else {
for _, result := range results {
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Aborting due to shutdown")
return
default:
}
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating %s: %d", desc, id)
_, err = x.Exec("UPDATE `repository` SET num_closed_pulls=(SELECT COUNT(*) FROM `issue` WHERE repo_id=? AND is_closed=? AND is_pull=?) WHERE id=?", id, true, true, id)
@ -2375,6 +2417,12 @@ func CheckRepoStats() {
log.Error("Select repository count 'num_forks': %v", err)
} else {
for _, result := range results {
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Aborting due to shutdown")
return
default:
}
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating repository count 'num_forks': %d", id)

43
models/repo_indexer.go

@ -4,6 +4,12 @@
package models
import (
"fmt"
"xorm.io/builder"
)
// RepoIndexerStatus status of a repo's entry in the repo indexer
// For now, implicitly refers to default branch
type RepoIndexerStatus struct {
@ -12,6 +18,31 @@ type RepoIndexerStatus struct {
CommitSha string `xorm:"VARCHAR(40)"`
}
// GetUnindexedRepos returns repos which do not have an indexer status
func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
ids := make([]int64, 0, 50)
cond := builder.Cond(builder.IsNull{
"repo_indexer_status.id",
})
sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id")
if maxRepoID > 0 {
cond = builder.And(cond, builder.Lte{
"repository.id": maxRepoID,
})
}
if page >= 0 && pageSize > 0 {
start := 0
if page > 0 {
start = (page - 1) * pageSize
}
sess.Limit(pageSize, start)
}
sess.Where(cond).Cols("repository.id").Desc("repository.id")
err := sess.Find(&ids)
return ids, err
}
// GetIndexerStatus loads repo codes indxer status
func (repo *Repository) GetIndexerStatus() error {
if repo.IndexerStatus != nil {
@ -31,15 +62,21 @@ func (repo *Repository) GetIndexerStatus() error {
// UpdateIndexerStatus updates indexer status
func (repo *Repository) UpdateIndexerStatus(sha string) error {
if err := repo.GetIndexerStatus(); err != nil {
return err
return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s/%s Error: %v", repo.MustOwnerName(), repo.Name, err)
}
if len(repo.IndexerStatus.CommitSha) == 0 {
repo.IndexerStatus.CommitSha = sha
_, err := x.Insert(repo.IndexerStatus)
return err
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s/%s Sha: %s Error: %v", repo.MustOwnerName(), repo.Name, sha, err)
}
return nil
}
repo.IndexerStatus.CommitSha = sha
_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
Update(repo.IndexerStatus)
return err
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s/%s Sha: %s Error: %v", repo.MustOwnerName(), repo.Name, sha, err)
}
return nil
}

35
models/user.go

@ -7,6 +7,7 @@ package models
import (
"container/list"
"context"
"crypto/md5"
"crypto/sha256"
"crypto/subtle"
@ -1695,7 +1696,7 @@ func synchronizeLdapSSHPublicKeys(usr *User, s *LoginSource, sshPublicKeys []str
}
// SyncExternalUsers is used to synchronize users with external authorization source
func SyncExternalUsers() {
func SyncExternalUsers(ctx context.Context) {
log.Trace("Doing: SyncExternalUsers")
ls, err := LoginSources()
@ -1710,6 +1711,12 @@ func SyncExternalUsers() {
if !s.IsActived || !s.IsSyncEnabled {
continue
}
select {
case <-ctx.Done():
log.Warn("SyncExternalUsers: Aborted due to shutdown before update of %s", s.Name)
return
default:
}
if s.IsLDAP() {
log.Trace("Doing: SyncExternalUsers[%s]", s.Name)
@ -1727,6 +1734,12 @@ func SyncExternalUsers() {
log.Error("SyncExternalUsers: %v", err)
return
}
select {
case <-ctx.Done():
log.Warn("SyncExternalUsers: Aborted due to shutdown before update of %s", s.Name)
return
default:
}
sr, err := s.LDAP().SearchEntries()
if err != nil {
@ -1735,6 +1748,19 @@ func SyncExternalUsers() {
}
for _, su := range sr {
select {
case <-ctx.Done():
log.Warn("SyncExternalUsers: Aborted due to shutdown at update of %s before completed update of users", s.Name)
// Rewrite authorized_keys file if LDAP Public SSH Key attribute is set and any key was added or removed
if sshKeysNeedUpdate {
err = RewriteAllPublicKeys()
if err != nil {
log.Error("RewriteAllPublicKeys: %v", err)
}
}
return
default:
}
if len(su.Username) == 0 {
continue
}
@ -1819,6 +1845,13 @@ func SyncExternalUsers() {
}
}
select {
case <-ctx.Done():
log.Warn("SyncExternalUsers: Aborted due to shutdown at update of %s before delete users", s.Name)
return
default:
}
// Deactivate users not present in LDAP
if updateExisting {
for _, usr := range users {

9
modules/cron/cron.go

@ -6,9 +6,11 @@
package cron
import (
"context"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/setting"
@ -37,17 +39,19 @@ var taskStatusTable = sync.NewStatusTable()
type Func func()
// WithUnique wrap a cron func with an unique running check
func WithUnique(name string, body Func) Func {
func WithUnique(name string, body func(context.Context)) Func {
return func() {
if !taskStatusTable.StartIfNotRunning(name) {
return
}
defer taskStatusTable.Stop(name)
body()
graceful.GetManager().RunWithShutdownContext(body)
}
}
// NewContext begins cron tasks
// Each cron task is run within the shutdown context as a running server
// AtShutdown the cron server is stopped
func NewContext() {
var (
entry *cron.Entry
@ -129,6 +133,7 @@ func NewContext() {
go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
c.Start()
graceful.GetManager().RunAtShutdown(context.Background(), c.Stop)
}
// ListTasks returns all running cron tasks.

3
modules/git/git.go

@ -106,7 +106,8 @@ func SetExecutablePath(path string) error {
}
// Init initializes git module
func Init() error {
func Init(ctx context.Context) error {
DefaultContext = ctx
// Git requires setting user.name and user.email in order to commit changes.
for configKey, defaultValue := range map[string]string{"user.name": "Gitea", "user.email": "gitea@fake.local"} {
if stdout, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", configKey); err != nil || strings.TrimSpace(stdout) == "" {

3
modules/git/git_test.go

@ -5,6 +5,7 @@
package git
import (
"context"
"fmt"
"os"
"testing"
@ -16,7 +17,7 @@ func fatalTestError(fmtStr string, args ...interface{}) {
}
func TestMain(m *testing.M) {
if err := Init(); err != nil {
if err := Init(context.Background()); err != nil {
fatalTestError("Init failed: %v", err)
}

6
modules/graceful/context.go

@ -62,7 +62,7 @@ func (ctx *ChannelContext) Value(key interface{}) interface{} {
// ShutdownContext returns a context.Context that is Done at shutdown
// Callers using this context should ensure that they are registered as a running server
// in order that they are waited for.
func (g *gracefulManager) ShutdownContext() context.Context {
func (g *Manager) ShutdownContext() context.Context {
return &ChannelContext{
done: g.IsShutdown(),
err: ErrShutdown,
@ -72,7 +72,7 @@ func (g *gracefulManager) ShutdownContext() context.Context {
// HammerContext returns a context.Context that is Done at hammer
// Callers using this context should ensure that they are registered as a running server
// in order that they are waited for.
func (g *gracefulManager) HammerContext() context.Context {
func (g *Manager) HammerContext() context.Context {
return &ChannelContext{
done: g.IsHammer(),
err: ErrHammer,
@ -82,7 +82,7 @@ func (g *gracefulManager) HammerContext() context.Context {
// TerminateContext returns a context.Context that is Done at terminate
// Callers using this context should ensure that they are registered as a terminating server
// in order that they are waited for.
func (g *gracefulManager) TerminateContext() context.Context {
func (g *Manager) TerminateContext() context.Context {
return &ChannelContext{
done: g.IsTerminate(),
err: ErrTerminate,

140
modules/graceful/manager.go

@ -6,9 +6,9 @@ package graceful
import (
"context"
"sync"
"time"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
@ -34,14 +34,24 @@ const (
const numberOfServersToCreate = 3
// Manager represents the graceful server manager interface
var Manager *gracefulManager
func init() {
Manager = newGracefulManager(context.Background())
// Set the git default context to the HammerContext
git.DefaultContext = Manager.HammerContext()
// Set the process default context to the HammerContext
process.DefaultContext = Manager.HammerContext()
var manager *Manager
var initOnce = sync.Once{}
// GetManager returns the Manager
func GetManager() *Manager {
InitManager(context.Background())
return manager
}
// InitManager creates the graceful manager in the provided context
func InitManager(ctx context.Context) {
initOnce.Do(func() {
manager = newGracefulManager(ctx)
// Set the process default context to the HammerContext
process.DefaultContext = manager.HammerContext()
})
}
// CallbackWithContext is combined runnable and context to watch to see if the caller has finished
@ -61,7 +71,7 @@ type RunnableWithShutdownFns func(atShutdown, atTerminate func(context.Context,
// Please note that use of the atShutdown and atTerminate callbacks will create go-routines that will wait till their respective signals
// - users must therefore be careful to only call these as necessary.
// If run is not expected to run indefinitely RunWithShutdownChan is likely to be more appropriate.
func (g *gracefulManager) RunWithShutdownFns(run RunnableWithShutdownFns) {
func (g *Manager) RunWithShutdownFns(run RunnableWithShutdownFns) {
g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done()
run(func(ctx context.Context, atShutdown func()) {
@ -90,7 +100,7 @@ type RunnableWithShutdownChan func(atShutdown <-chan struct{}, atTerminate Callb
// (Optionally IsHammer may be waited for instead however, this should be avoided if possible.)
// The callback function provided to atTerminate must return once termination is complete.
// Please note that use of the atTerminate function will create a go-routine that will wait till terminate - users must therefore be careful to only call this as necessary.
func (g *gracefulManager) RunWithShutdownChan(run RunnableWithShutdownChan) {
func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) {
g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done()
run(g.IsShutdown(), func(ctx context.Context, atTerminate func()) {
@ -101,14 +111,14 @@ func (g *gracefulManager) RunWithShutdownChan(run RunnableWithShutdownChan) {
// RunWithShutdownContext takes a function that has a context to watch for shutdown.
// After the provided context is Done(), the main function must return once shutdown is complete.
// (Optionally the HammerContext may be obtained and waited for however, this should be avoided if possible.)
func (g *gracefulManager) RunWithShutdownContext(run func(context.Context)) {
func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done()
run(g.ShutdownContext())
}
// RunAtTerminate adds to the terminate wait group and creates a go-routine to run the provided function at termination
func (g *gracefulManager) RunAtTerminate(ctx context.Context, terminate func()) {
func (g *Manager) RunAtTerminate(ctx context.Context, terminate func()) {
g.terminateWaitGroup.Add(1)
go func() {
select {
@ -121,7 +131,7 @@ func (g *gracefulManager) RunAtTerminate(ctx context.Context, terminate func())
}
// RunAtShutdown creates a go-routine to run the provided function at shutdown
func (g *gracefulManager) RunAtShutdown(ctx context.Context, shutdown func()) {
func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
go func() {
select {
case <-g.IsShutdown():
@ -132,7 +142,7 @@ func (g *gracefulManager) RunAtShutdown(ctx context.Context, shutdown func()) {
}
// RunAtHammer creates a go-routine to run the provided function at shutdown
func (g *gracefulManager) RunAtHammer(ctx context.Context, hammer func()) {
func (g *Manager) RunAtHammer(ctx context.Context, hammer func()) {
go func() {
select {
case <-g.IsHammer():
@ -141,7 +151,7 @@ func (g *gracefulManager) RunAtHammer(ctx context.Context, hammer func()) {
}
}()
}
func (g *gracefulManager) doShutdown() {
func (g *Manager) doShutdown() {
if !g.setStateTransition(stateRunning, stateShuttingDown) {
return
}
@ -158,48 +168,47 @@ func (g *gracefulManager) doShutdown() {
g.doHammerTime(0)
<-time.After(1 * time.Second)
g.doTerminate()
g.WaitForTerminate()
g.lock.Lock()
close(g.done)
g.lock.Unlock()
}()
}
func (g *gracefulManager) doHammerTime(d time.Duration) {
func (g *Manager) doHammerTime(d time.Duration) {
time.Sleep(d)
g.lock.Lock()
select {
case <-g.hammer:
default:
log.Warn("Setting Hammer condition")
close(g.hammer)
}
g.lock.Unlock()
}
func (g *gracefulManager) doTerminate() {
func (g *Manager) doTerminate() {
if !g.setStateTransition(stateShuttingDown, stateTerminate) {
return
}
g.lock.Lock()
close(g.terminate)
select {
case <-g.terminate:
default:
log.Warn("Terminating")
close(g.terminate)
}
g.lock.Unlock()
}
// IsChild returns if the current process is a child of previous Gitea process
func (g *gracefulManager) IsChild() bool {
func (g *Manager) IsChild() bool {
return g.isChild
}
// IsShutdown returns a channel which will be closed at shutdown.
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
func (g *gracefulManager) IsShutdown() <-chan struct{} {
g.lock.RLock()
if g.shutdown == nil {
g.lock.RUnlock()
g.lock.Lock()
if g.shutdown == nil {
g.shutdown = make(chan struct{})
}
defer g.lock.Unlock()
return g.shutdown
}
defer g.lock.RUnlock()
func (g *Manager) IsShutdown() <-chan struct{} {
return g.shutdown
}
@ -207,65 +216,43 @@ func (g *gracefulManager) IsShutdown() <-chan struct{} {
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
// Servers running within the running server wait group should respond to IsHammer
// if not shutdown already
func (g *gracefulManager) IsHammer() <-chan struct{} {
g.lock.RLock()
if g.hammer == nil {
g.lock.RUnlock()
g.lock.Lock()
if g.hammer == nil {
g.hammer = make(chan struct{})
}
defer g.lock.Unlock()
return g.hammer
}
defer g.lock.RUnlock()
func (g *Manager) IsHammer() <-chan struct{} {
return g.hammer
}
// IsTerminate returns a channel which will be closed at terminate
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
// IsTerminate will only close once all running servers have stopped
func (g *gracefulManager) IsTerminate() <-chan struct{} {
g.lock.RLock()
if g.terminate == nil {
g.lock.RUnlock()
g.lock.Lock()
if g.terminate == nil {
g.terminate = make(chan struct{})
}
defer g.lock.Unlock()
return g.terminate
}
defer g.lock.RUnlock()
func (g *Manager) IsTerminate() <-chan struct{} {
return g.terminate
}
// ServerDone declares a running server done and subtracts one from the
// running server wait group. Users probably do not want to call this
// and should use one of the RunWithShutdown* functions
func (g *gracefulManager) ServerDone() {
func (g *Manager) ServerDone() {
g.runningServerWaitGroup.Done()
}
// WaitForServers waits for all running servers to finish. Users should probably
// instead use AtTerminate or IsTerminate
func (g *gracefulManager) WaitForServers() {
func (g *Manager) WaitForServers() {
g.runningServerWaitGroup.Wait()
}
// WaitForTerminate waits for all terminating actions to finish.
// Only the main go-routine should use this
func (g *gracefulManager) WaitForTerminate() {
func (g *Manager) WaitForTerminate() {
g.terminateWaitGroup.Wait()
}
func (g *gracefulManager) getState() state {
func (g *Manager) getState() state {
g.lock.RLock()
defer g.lock.RUnlock()
return g.state
}
func (g *gracefulManager) setStateTransition(old, new state) bool {
func (g *Manager) setStateTransition(old, new state) bool {
if old != g.getState() {
return false
}
@ -279,7 +266,7 @@ func (g *gracefulManager) setStateTransition(old, new state) bool {
return true
}
func (g *gracefulManager) setState(st state) {
func (g *Manager) setState(st state) {
g.lock.Lock()
defer g.lock.Unlock()
@ -288,6 +275,31 @@ func (g *gracefulManager) setState(st state) {
// InformCleanup tells the cleanup wait group that we have either taken a listener
// or will not be taking a listener
func (g *gracefulManager) InformCleanup() {
func (g *Manager) InformCleanup() {
g.createServerWaitGroup.Done()
}
// Done allows the manager to be viewed as a context.Context, it returns a channel that is closed when the server is finished terminating
func (g *Manager) Done() <-chan struct{} {
return g.done
}
// Err allows the manager to be viewed as a context.Context done at Terminate, it returns ErrTerminate
func (g *Manager) Err() error {
select {
case <-g.Done():
return ErrTerminate
default:
return nil
}
}
// Value allows the manager to be viewed as a context.Context done at Terminate, it has no values
func (g *Manager) Value(key interface{}) interface{} {
return nil
}
// Deadline returns nil as there is no fixed Deadline for the manager, it allows the manager to be viewed as a context.Context
func (g *Manager) Deadline() (deadline time.Time, ok bool) {
return
}

45
modules/graceful/manager_unix.go

@ -19,7 +19,8 @@ import (
"code.gitea.io/gitea/modules/setting"
)
type gracefulManager struct {
// Manager manages the graceful shutdown process
type Manager struct {
isChild bool
forked bool
lock *sync.RWMutex
@ -27,27 +28,37 @@ type gracefulManager struct {
shutdown chan struct{}
hammer chan struct{}
terminate chan struct{}
done chan struct{}
runningServerWaitGroup sync.WaitGroup
createServerWaitGroup sync.WaitGroup
terminateWaitGroup sync.WaitGroup
}
func newGracefulManager(ctx context.Context) *gracefulManager {
manager := &gracefulManager{
func newGracefulManager(ctx context.Context) *Manager {
manager := &Manager{
isChild: len(os.Getenv(listenFDs)) > 0 && os.Getppid() > 1,
lock: &sync.RWMutex{},
}
manager.createServerWaitGroup.Add(numberOfServersToCreate)
manager.Run(ctx)
manager.start(ctx)
return manager
}
func (g *gracefulManager) Run(ctx context.Context) {
func (g *Manager) start(ctx context.Context) {
// Make channels
g.terminate = make(chan struct{})
g.shutdown = make(chan struct{})
g.hammer = make(chan struct{})
g.done = make(chan struct{})
// Set the running state & handle signals
g.setState(stateRunning)
go g.handleSignals(ctx)
c := make(chan struct{})
// Handle clean up of unused provided listeners and delayed start-up
startupDone := make(chan struct{})
go func() {
defer close(c)
defer close(startupDone)
// Wait till we're done getting all of the listeners and then close
// the unused ones
g.createServerWaitGroup.Wait()
@ -58,9 +69,19 @@ func (g *gracefulManager) Run(ctx context.Context) {
if setting.StartupTimeout > 0 {
go func() {
select {
case <-c:
case <-startupDone:
return
case <-g.IsShutdown():
func() {
// When waitgroup counter goes negative it will panic - we don't care about this so we can just ignore it.
defer func() {
_ = recover()
}()
// Ensure that the createServerWaitGroup stops waiting
for {
g.createServerWaitGroup.Done()
}
}()
return
case <-time.After(setting.StartupTimeout):
log.Error("Startup took too long! Shutting down")
@ -70,7 +91,7 @@ func (g *gracefulManager) Run(ctx context.Context) {
}
}
func (g *gracefulManager) handleSignals(ctx context.Context) {
func (g *Manager) handleSignals(ctx context.Context) {
signalChannel := make(chan os.Signal, 1)
signal.Notify(
@ -123,7 +144,7 @@ func (g *gracefulManager) handleSignals(ctx context.Context) {
}
}
func (g *gracefulManager) doFork() error {
func (g *Manager) doFork() error {
g.lock.Lock()
if g.forked {
g.lock.Unlock()
@ -139,7 +160,9 @@ func (g *gracefulManager) doFork() error {
return err
}
func (g *gracefulManager) RegisterServer() {
// RegisterServer registers the running of a listening server, in the case of unix this means that the parent process can now die.
// Any call to RegisterServer must be matched by a call to ServerDone
func (g *Manager) RegisterServer() {
KillParent()
g.runningServerWaitGroup.Add(1)
}

31
modules/graceful/manager_windows.go

@ -30,7 +30,8 @@ const (
acceptHammerCode = svc.Accepted(hammerCode)
)
type gracefulManager struct {
// Manager manages the graceful shutdown process
type Manager struct {
ctx context.Context
isChild bool
lock *sync.RWMutex
@ -38,27 +39,37 @@ type gracefulManager struct {
shutdown chan struct{}
hammer chan struct{}
terminate chan struct{}
done chan struct{}
runningServerWaitGroup sync.WaitGroup
createServerWaitGroup sync.WaitGroup
terminateWaitGroup sync.WaitGroup
}
func newGracefulManager(ctx context.Context) *gracefulManager {
manager := &gracefulManager{
func newGracefulManager(ctx context.Context) *Manager {
manager := &Manager{
isChild: false,
lock: &sync.RWMutex{},
ctx: ctx,
}
manager.createServerWaitGroup.Add(numberOfServersToCreate)
manager.Run()
manager.start()
return manager
}
func (g *gracefulManager) Run() {
func (g *Manager) start() {
// Make channels
g.terminate = make(chan struct{})
g.shutdown = make(chan struct{})
g.hammer = make(chan struct{})
g.done = make(chan struct{})
// Set the running state
g.setState(stateRunning)
if skip, _ := strconv.ParseBool(os.Getenv("SKIP_MINWINSVC")); skip {
return
}
// Make SVC process
run := svc.Run
isInteractive, err := svc.IsAnInteractiveSession()
if err != nil {
@ -71,8 +82,8 @@ func (g *gracefulManager) Run() {
go run(WindowsServiceName, g)
}
// Execute makes gracefulManager implement svc.Handler
func (g *gracefulManager) Execute(args []string, changes <-chan svc.ChangeRequest, status chan<- svc.Status) (svcSpecificEC bool, exitCode uint32) {
// Execute makes Manager implement svc.Handler
func (g *Manager) Execute(args []string, changes <-chan svc.ChangeRequest, status chan<- svc.Status) (svcSpecificEC bool, exitCode uint32) {
if setting.StartupTimeout > 0 {
status <- svc.Status{State: svc.StartPending}
} else {
@ -141,11 +152,13 @@ hammerLoop:
return false, 0
}
func (g *gracefulManager) RegisterServer() {
// RegisterServer registers the running of a listening server.
// Any call to RegisterServer must be matched by a call to ServerDone
func (g *Manager) RegisterServer() {
g.runningServerWaitGroup.Add(1)
}
func (g *gracefulManager) awaitServer(limit time.Duration) bool {
func (g *Manager) awaitServer(limit time.Duration) bool {
c := make(chan struct{})
go func() {
defer close(c)

2
modules/graceful/net_unix.go

@ -101,7 +101,7 @@ func CloseProvidedListeners() error {
// creates a new one using net.Listen.
func GetListener(network, address string) (net.Listener, error) {
// Add a deferral to say that we've tried to grab a listener
defer Manager.InformCleanup()
defer GetManager().InformCleanup()
switch network {
case "tcp", "tcp4", "tcp6":
tcpAddr, err := net.ResolveTCPAddr(network, address)

2
modules/graceful/restart_unix.go

@ -22,7 +22,7 @@ var killParent sync.Once
// KillParent sends the kill signal to the parent process if we are a child
func KillParent() {
killParent.Do(func() {
if Manager.IsChild() {
if GetManager().IsChild() {
ppid := syscall.Getppid()
if ppid > 1 {
_ = syscall.Kill(ppid, syscall.SIGTERM)

6
modules/graceful/server.go

@ -47,7 +47,7 @@ type Server struct {
// NewServer creates a server on network at provided address
func NewServer(network, address string) *Server {
if Manager.IsChild() {
if GetManager().IsChild() {
log.Info("Restarting new server: %s:%s on PID: %d", network, address, os.Getpid())
} else {
log.Info("Starting new server: %s:%s on PID: %d", network, address, os.Getpid())
@ -138,12 +138,12 @@ func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFun
func (srv *Server) Serve(serve ServeFunction) error {
defer log.Debug("Serve() returning... (PID: %d)", syscall.Getpid())
srv.setState(stateRunning)
Manager.RegisterServer()
GetManager().RegisterServer()
err := serve(srv.listener)
log.Debug("Waiting for connections to finish... (PID: %d)", syscall.Getpid())
srv.wg.Wait()
srv.setState(stateTerminate)
Manager.ServerDone()
GetManager().ServerDone()
// use of closed means that the listeners are closed - i.e. we should be shutting down - return nil
if err != nil && strings.Contains(err.Error(), "use of closed") {
return nil

6
modules/graceful/server_hooks.go

@ -14,15 +14,15 @@ import (
// awaitShutdown waits for the shutdown signal from the Manager
func (srv *Server) awaitShutdown() {
select {
case <-Manager.IsShutdown():
case <-GetManager().IsShutdown():
// Shutdown
srv.doShutdown()
case <-Manager.IsHammer():
case <-GetManager().IsHammer():
// Hammer
srv.doShutdown()
srv.doHammer()
}
<-Manager.IsHammer()
<-GetManager().IsHammer()
srv.doHammer()
}

82
modules/indexer/code/bleve.go

@ -6,6 +6,7 @@ package code
import (
"fmt"
"os"
"strconv"
"strings"
"time"
@ -34,10 +35,11 @@ func InitRepoIndexer() {
return
}
waitChannel := make(chan time.Duration)
// FIXME: graceful: This should use a persistable queue
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
go func() {
start := time.Now()
log.Info("Initializing Repository Indexer")
log.Info("PID: %d: Initializing Repository Indexer", os.Getpid())
initRepoIndexer(populateRepoIndexerAsynchronously)
go processRepoIndexerOperationQueue()
waitChannel <- time.Since(start)
@ -45,7 +47,7 @@ func InitRepoIndexer() {
if setting.Indexer.StartupTimeout > 0 {
go func() {
timeout := setting.Indexer.StartupTimeout
if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 {
timeout += setting.GracefulHammerTime
}
select {
@ -70,13 +72,6 @@ func populateRepoIndexerAsynchronously() error {
return nil
}
// if there is any existing repo indexer metadata in the DB, delete it
// since we are starting afresh. Also, xorm requires deletes to have a
// condition, and we want to delete everything, thus 1=1.
if err := models.DeleteAllRecords("repo_indexer_status"); err != nil {
return err
}
var maxRepoID int64
if maxRepoID, err = models.GetMaxID("repository"); err != nil {
return err
@ -87,44 +82,59 @@ func populateRepoIndexerAsynchronously() error {
// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
// FIXME: graceful: This should use a persistable queue
func populateRepoIndexer(maxRepoID int64) {
log.Info("Populating the repo indexer with existing repositories")
isShutdown := graceful.GetManager().IsShutdown()
// start with the maximum existing repo ID and work backwards, so that we
// don't include repos that are created after gitea starts; such repos will
// already be added to the indexer, and we don't need to add them again.
for maxRepoID > 0 {
repos := make([]*models.Repository, 0, models.RepositoryListDefaultPageSize)
err := models.FindByMaxID(maxRepoID, models.RepositoryListDefaultPageSize, &repos)
select {
case <-isShutdown:
log.Info("Repository Indexer population shutdown before completion")
return
default:
}
ids, err := models.GetUnindexedRepos(maxRepoID, 0, 50)
if err != nil {
log.Error("populateRepoIndexer: %v", err)
return
} else if len(repos) == 0 {
} else if len(ids) == 0 {
break
}
for _, repo := range repos {
for _, id := range ids {
select {
case <-isShutdown:
log.Info("Repository Indexer population shutdown before completion")
return
default:
}
repoIndexerOperationQueue <- repoIndexerOperation{
repoID: repo.ID,
repoID: id,
deleted: false,
}
maxRepoID = repo.ID - 1
maxRepoID = id - 1
}
}
log.Info("Done populating the repo indexer with existing repositories")
log.Info("Done (re)populating the repo indexer with existing repositories")
}
func updateRepoIndexer(repoID int64) error {
repo, err := models.GetRepositoryByID(repoID)
if err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to GetRepositoryByID: %d, Error: %v", repoID, err)
}
sha, err := getDefaultBranchSha(repo)
if err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to GetDefaultBranchSha for: %s/%s, Error: %v", repo.MustOwnerName(), repo.Name, err)
}
changes, err := getRepoChanges(repo, sha)
if err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to GetRepoChanges for: %s/%s Sha: %s Error: %v", repo.MustOwnerName(), repo.Name, sha, err)
} else if changes == nil {
return nil
}
@ -132,16 +142,16 @@ func updateRepoIndexer(repoID int64) error {
batch := RepoIndexerBatch()
for _, update := range changes.Updates {
if err := addUpdate(update, repo, batch); err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to addUpdate to: %s/%s Sha: %s, update: %s(%s) Error: %v", repo.MustOwnerName(), repo.Name, sha, update.Filename, update.BlobSha, err)
}
}
for _, filename := range changes.RemovedFilenames {
if err := addDelete(filename, repo, batch); err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to addDelete to: %s/%s Sha: %s, filename: %s Error: %v", repo.MustOwnerName(), repo.Name, sha, filename, err)
}
}
if err = batch.Flush(); err != nil {
return err
return fmt.Errorf("UpdateRepoIndexer: Unable to flush batch to indexer for repo: %s/%s Error: %v", repo.MustOwnerName(), repo.Name, err)
}
return repo.UpdateIndexerStatus(sha)
}