Queue: Make WorkerPools and Queues flushable (#10001)
* Make WorkerPools and Queues flushable Adds Flush methods to Queues and the WorkerPool Further abstracts the WorkerPool Adds a final step to Flush the queues in the defer from PrintCurrentTest Fixes an issue with Settings inheritance in queues Signed-off-by: Andrew Thornton <art27@cantab.net> * Change to for loop * Add IsEmpty and begin just making the queues composed WorkerPools * subsume workerpool into the queues and create a flushable interface * Add manager command * Move flushall to queue.Manager and add to testlogger * As per @guillep2k * as per @guillep2k * Just make queues all implement flushable and clean up the wrapped queue flushes * cope with no timeout Co-authored-by: Lauris BH <lauris@nix.lv>mj
zeripath
4 years ago
committed by
GitHub
parent
7c84dbca4f
commit
c01221e70f
@ -0,0 +1,92 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/private"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var (
|
||||
// CmdManager represents the manager command
|
||||
CmdManager = cli.Command{
|
||||
Name: "manager",
|
||||
Usage: "Manage the running gitea process",
|
||||
Description: "This is a command for managing the running gitea process",
|
||||
Subcommands: []cli.Command{
|
||||
subcmdShutdown,
|
||||
subcmdRestart,
|
||||
subcmdFlushQueues,
|
||||
},
|
||||
}
|
||||
subcmdShutdown = cli.Command{
|
||||
Name: "shutdown",
|
||||
Usage: "Gracefully shutdown the running process",
|
||||
Action: runShutdown,
|
||||
}
|
||||
subcmdRestart = cli.Command{
|
||||
Name: "restart",
|
||||
Usage: "Gracefully restart the running process - (not implemented for windows servers)",
|
||||
Action: runRestart,
|
||||
}
|
||||
subcmdFlushQueues = cli.Command{
|
||||
Name: "flush-queues",
|
||||
Usage: "Flush queues in the running process",
|
||||
Action: runFlushQueues,
|
||||
Flags: []cli.Flag{
|
||||
cli.DurationFlag{
|
||||
Name: "timeout",
|
||||
Value: 60 * time.Second,
|
||||
Usage: "Timeout for the flushing process",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "non-blocking",
|
||||
Usage: "Set to true to not wait for flush to complete before returning",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func runShutdown(c *cli.Context) error {
|
||||
setup("manager", false)
|
||||
statusCode, msg := private.Shutdown()
|
||||
switch statusCode {
|
||||
case http.StatusInternalServerError:
|
||||
fail("InternalServerError", msg)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stdout, msg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runRestart(c *cli.Context) error {
|
||||
setup("manager", false)
|
||||
statusCode, msg := private.Restart()
|
||||
switch statusCode {
|
||||
case http.StatusInternalServerError:
|
||||
fail("InternalServerError", msg)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stdout, msg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runFlushQueues(c *cli.Context) error {
|
||||
setup("manager", false)
|
||||
statusCode, msg := private.FlushQueues(c.Duration("timeout"), c.Bool("non-blocking"))
|
||||
switch statusCode {
|
||||
case http.StatusInternalServerError:
|
||||
fail("InternalServerError", msg)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stdout, msg)
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package private
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
// Shutdown calls the internal shutdown function
|
||||
func Shutdown() (int, string) {
|
||||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/shutdown")
|
||||
|
||||
req := newInternalRequest(reqURL, "POST")
|
||||
resp, err := req.Response()
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return resp.StatusCode, decodeJSONError(resp).Err
|
||||
}
|
||||
|
||||
return http.StatusOK, "Shutting down"
|
||||
}
|
||||
|
||||
// Restart calls the internal restart function
|
||||
func Restart() (int, string) {
|
||||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/restart")
|
||||
|
||||
req := newInternalRequest(reqURL, "POST")
|
||||
resp, err := req.Response()
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return resp.StatusCode, decodeJSONError(resp).Err
|
||||
}
|
||||
|
||||
return http.StatusOK, "Restarting"
|
||||
}
|
||||
|
||||
// FlushOptions represents the options for the flush call
|
||||
type FlushOptions struct {
|
||||
Timeout time.Duration
|
||||
NonBlocking bool
|
||||
}
|
||||
|
||||
// FlushQueues calls the internal flush-queues function
|
||||
func FlushQueues(timeout time.Duration, nonBlocking bool) (int, string) {
|
||||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/flush-queues")
|
||||
|
||||
req := newInternalRequest(reqURL, "POST")
|
||||
if timeout > 0 {
|
||||
req.SetTimeout(timeout+10*time.Second, timeout+10*time.Second)
|
||||
}
|
||||
req = req.Header("Content-Type", "application/json")
|
||||
jsonBytes, _ := json.Marshal(FlushOptions{
|
||||
Timeout: timeout,
|
||||
NonBlocking: nonBlocking,
|
||||
})
|
||||
req.Body(jsonBytes)
|
||||
resp, err := req.Response()
|
||||
if err != nil {
|
||||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return resp.StatusCode, decodeJSONError(resp).Err
|
||||
}
|
||||
|
||||
return http.StatusOK, "Flushed"
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
|
||||
//
|
||||
// It will tolerate the cfg being passed as a []byte or string of a json representation of the
|
||||
// exemplar or the correct type of the exemplar itself
|
||||
func toConfig(exemplar, cfg interface{}) (interface{}, error) {
|
||||
if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
configBytes, ok := cfg.([]byte)
|
||||
if !ok {
|
||||
configStr, ok := cfg.(string)
|
||||
if !ok {
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg}
|
||||
}
|
||||
configBytes = []byte(configStr)
|
||||
}
|
||||
newVal := reflect.New(reflect.TypeOf(exemplar))
|
||||
if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
|
||||
}
|
||||
return newVal.Elem().Interface(), nil
|
||||
}
|
||||
|
||||
// unmarshalAs will attempt to unmarshal provided bytes as the provided exemplar
|
||||
func unmarshalAs(bs []byte, exemplar interface{}) (data Data, err error) {
|
||||
if exemplar != nil {
|
||||
t := reflect.TypeOf(exemplar)
|
||||
n := reflect.New(t)
|
||||
ne := n.Elem()
|
||||
err = json.Unmarshal(bs, ne.Addr().Interface())
|
||||
data = ne.Interface().(Data)
|
||||
} else {
|
||||
err = json.Unmarshal(bs, &data)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// assignableTo will check if provided data is assignable to the same type as the exemplar
|
||||
// if the provided exemplar is nil then it will always return true
|
||||
func assignableTo(data Data, exemplar interface{}) bool {
|
||||
if exemplar == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// Assert data is of same type as exemplar
|
||||
t := reflect.TypeOf(data)
|
||||
exemplarType := reflect.TypeOf(exemplar)
|
||||
|
||||
return t.AssignableTo(exemplarType) && data != nil
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package private
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/private"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
|
||||
"gitea.com/macaron/macaron"
|
||||
)
|
||||
|
||||
// FlushQueues flushes all the Queues
|
||||
func FlushQueues(ctx *macaron.Context, opts private.FlushOptions) {
|
||||
if opts.NonBlocking {
|
||||
// Save the hammer ctx here - as a new one is created each time you call this.
|
||||
baseCtx := graceful.GetManager().HammerContext()
|
||||
go func() {
|
||||
err := queue.GetManager().FlushAll(baseCtx, opts.Timeout)
|
||||
if err != nil {
|
||||
log.Error("Flushing request timed-out with error: %v", err)
|
||||
}
|
||||
}()
|
||||
ctx.JSON(http.StatusAccepted, map[string]interface{}{
|
||||
"err": "Flushing",
|
||||
})
|
||||
return
|
||||
}
|
||||
err := queue.GetManager().FlushAll(ctx.Req.Request.Context(), opts.Timeout)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusRequestTimeout, map[string]interface{}{
|
||||
"err": err,
|
||||
})
|
||||
}
|
||||
ctx.PlainText(http.StatusOK, []byte("success"))
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
// +build !windows
|
||||
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package private
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
|
||||
"gitea.com/macaron/macaron"
|
||||
)
|
||||
|
||||
// Restart causes the server to perform a graceful restart
|
||||
func Restart(ctx *macaron.Context) {
|
||||
graceful.GetManager().DoGracefulRestart()
|
||||
ctx.PlainText(http.StatusOK, []byte("success"))
|
||||
|
||||
}
|
||||
|
||||
// Shutdown causes the server to perform a graceful shutdown
|
||||
func Shutdown(ctx *macaron.Context) {
|
||||
graceful.GetManager().DoGracefulShutdown()
|
||||
ctx.PlainText(http.StatusOK, []byte("success"))
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
// +build windows
|
||||
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package private
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
|
||||
"gitea.com/macaron/macaron"
|
||||
)
|
||||
|
||||
// Restart is not implemented for Windows based servers as they can't fork
|
||||
func Restart(ctx *macaron.Context) {
|
||||
ctx.JSON(http.StatusNotImplemented, map[string]interface{}{
|
||||
"err": "windows servers cannot be gracefully restarted - shutdown and restart manually",
|
||||
})
|
||||
}
|
||||
|
||||
// Shutdown causes the server to perform a graceful shutdown
|
||||
func Shutdown(ctx *macaron.Context) {
|
||||
graceful.GetManager().DoGracefulShutdown()
|
||||
ctx.PlainText(http.StatusOK, []byte("success"))
|
||||
}
|
||||
Loading…
Reference in new issue