Browse Source
Queue: Make WorkerPools and Queues flushable (#10001)
Queue: Make WorkerPools and Queues flushable (#10001)
* Make WorkerPools and Queues flushable Adds Flush methods to Queues and the WorkerPool Further abstracts the WorkerPool Adds a final step to Flush the queues in the defer from PrintCurrentTest Fixes an issue with Settings inheritance in queues Signed-off-by: Andrew Thornton <art27@cantab.net> * Change to for loop * Add IsEmpty and begin just making the queues composed WorkerPools * subsume workerpool into the queues and create a flushable interface * Add manager command * Move flushall to queue.Manager and add to testlogger * As per @guillep2k * as per @guillep2k * Just make queues all implement flushable and clean up the wrapped queue flushes * cope with no timeout Co-authored-by: Lauris BH <lauris@nix.lv>mj
zeripath
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1030 additions and 327 deletions
-
92cmd/manager.go
-
6integrations/testlogger.go
-
1main.go
-
46modules/graceful/manager_unix.go
-
33modules/graceful/manager_windows.go
-
83modules/private/manager.go
-
63modules/queue/helper.go
-
159modules/queue/manager.go
-
43modules/queue/queue.go
-
59modules/queue/queue_channel.go
-
30modules/queue/queue_channel_test.go
-
87modules/queue/queue_disk.go
-
126modules/queue/queue_disk_channel.go
-
36modules/queue/queue_disk_test.go
-
102modules/queue/queue_redis.go
-
97modules/queue/queue_wrapped.go
-
24modules/queue/setting.go
-
94modules/queue/workerpool.go
-
20modules/setting/queue.go
-
4options/locale/locale_en-US.ini
-
30routers/admin/admin.go
-
4routers/private/internal.go
-
41routers/private/manager.go
-
28routers/private/manager_unix.go
-
28routers/private/manager_windows.go
-
1routers/routes/routes.go
-
20templates/admin/queue.tmpl
@ -0,0 +1,92 @@ |
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package cmd |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net/http" |
|||
"os" |
|||
"time" |
|||
|
|||
"code.gitea.io/gitea/modules/private" |
|||
|
|||
"github.com/urfave/cli" |
|||
) |
|||
|
|||
var ( |
|||
// CmdManager represents the manager command
|
|||
CmdManager = cli.Command{ |
|||
Name: "manager", |
|||
Usage: "Manage the running gitea process", |
|||
Description: "This is a command for managing the running gitea process", |
|||
Subcommands: []cli.Command{ |
|||
subcmdShutdown, |
|||
subcmdRestart, |
|||
subcmdFlushQueues, |
|||
}, |
|||
} |
|||
subcmdShutdown = cli.Command{ |
|||
Name: "shutdown", |
|||
Usage: "Gracefully shutdown the running process", |
|||
Action: runShutdown, |
|||
} |
|||
subcmdRestart = cli.Command{ |
|||
Name: "restart", |
|||
Usage: "Gracefully restart the running process - (not implemented for windows servers)", |
|||
Action: runRestart, |
|||
} |
|||
subcmdFlushQueues = cli.Command{ |
|||
Name: "flush-queues", |
|||
Usage: "Flush queues in the running process", |
|||
Action: runFlushQueues, |
|||
Flags: []cli.Flag{ |
|||
cli.DurationFlag{ |
|||
Name: "timeout", |
|||
Value: 60 * time.Second, |
|||
Usage: "Timeout for the flushing process", |
|||
}, |
|||
cli.BoolFlag{ |
|||
Name: "non-blocking", |
|||
Usage: "Set to true to not wait for flush to complete before returning", |
|||
}, |
|||
}, |
|||
} |
|||
) |
|||
|
|||
func runShutdown(c *cli.Context) error { |
|||
setup("manager", false) |
|||
statusCode, msg := private.Shutdown() |
|||
switch statusCode { |
|||
case http.StatusInternalServerError: |
|||
fail("InternalServerError", msg) |
|||
} |
|||
|
|||
fmt.Fprintln(os.Stdout, msg) |
|||
return nil |
|||
} |
|||
|
|||
func runRestart(c *cli.Context) error { |
|||
setup("manager", false) |
|||
statusCode, msg := private.Restart() |
|||
switch statusCode { |
|||
case http.StatusInternalServerError: |
|||
fail("InternalServerError", msg) |
|||
} |
|||
|
|||
fmt.Fprintln(os.Stdout, msg) |
|||
return nil |
|||
} |
|||
|
|||
func runFlushQueues(c *cli.Context) error { |
|||
setup("manager", false) |
|||
statusCode, msg := private.FlushQueues(c.Duration("timeout"), c.Bool("non-blocking")) |
|||
switch statusCode { |
|||
case http.StatusInternalServerError: |
|||
fail("InternalServerError", msg) |
|||
} |
|||
|
|||
fmt.Fprintln(os.Stdout, msg) |
|||
return nil |
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package private |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"fmt" |
|||
"net/http" |
|||
"time" |
|||
|
|||
"code.gitea.io/gitea/modules/setting" |
|||
) |
|||
|
|||
// Shutdown calls the internal shutdown function
|
|||
func Shutdown() (int, string) { |
|||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/shutdown") |
|||
|
|||
req := newInternalRequest(reqURL, "POST") |
|||
resp, err := req.Response() |
|||
if err != nil { |
|||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
if resp.StatusCode != http.StatusOK { |
|||
return resp.StatusCode, decodeJSONError(resp).Err |
|||
} |
|||
|
|||
return http.StatusOK, "Shutting down" |
|||
} |
|||
|
|||
// Restart calls the internal restart function
|
|||
func Restart() (int, string) { |
|||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/restart") |
|||
|
|||
req := newInternalRequest(reqURL, "POST") |
|||
resp, err := req.Response() |
|||
if err != nil { |
|||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
if resp.StatusCode != http.StatusOK { |
|||
return resp.StatusCode, decodeJSONError(resp).Err |
|||
} |
|||
|
|||
return http.StatusOK, "Restarting" |
|||
} |
|||
|
|||
// FlushOptions represents the options for the flush call
|
|||
type FlushOptions struct { |
|||
Timeout time.Duration |
|||
NonBlocking bool |
|||
} |
|||
|
|||
// FlushQueues calls the internal flush-queues function
|
|||
func FlushQueues(timeout time.Duration, nonBlocking bool) (int, string) { |
|||
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/flush-queues") |
|||
|
|||
req := newInternalRequest(reqURL, "POST") |
|||
if timeout > 0 { |
|||
req.SetTimeout(timeout+10*time.Second, timeout+10*time.Second) |
|||
} |
|||
req = req.Header("Content-Type", "application/json") |
|||
jsonBytes, _ := json.Marshal(FlushOptions{ |
|||
Timeout: timeout, |
|||
NonBlocking: nonBlocking, |
|||
}) |
|||
req.Body(jsonBytes) |
|||
resp, err := req.Response() |
|||
if err != nil { |
|||
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
if resp.StatusCode != http.StatusOK { |
|||
return resp.StatusCode, decodeJSONError(resp).Err |
|||
} |
|||
|
|||
return http.StatusOK, "Flushed" |
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package queue |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"reflect" |
|||
) |
|||
|
|||
// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
|
|||
//
|
|||
// It will tolerate the cfg being passed as a []byte or string of a json representation of the
|
|||
// exemplar or the correct type of the exemplar itself
|
|||
func toConfig(exemplar, cfg interface{}) (interface{}, error) { |
|||
if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { |
|||
return cfg, nil |
|||
} |
|||
|
|||
configBytes, ok := cfg.([]byte) |
|||
if !ok { |
|||
configStr, ok := cfg.(string) |
|||
if !ok { |
|||
return nil, ErrInvalidConfiguration{cfg: cfg} |
|||
} |
|||
configBytes = []byte(configStr) |
|||
} |
|||
newVal := reflect.New(reflect.TypeOf(exemplar)) |
|||
if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { |
|||
return nil, ErrInvalidConfiguration{cfg: cfg, err: err} |
|||
} |
|||
return newVal.Elem().Interface(), nil |
|||
} |
|||
|
|||
// unmarshalAs will attempt to unmarshal provided bytes as the provided exemplar
|
|||
func unmarshalAs(bs []byte, exemplar interface{}) (data Data, err error) { |
|||
if exemplar != nil { |
|||
t := reflect.TypeOf(exemplar) |
|||
n := reflect.New(t) |
|||
ne := n.Elem() |
|||
err = json.Unmarshal(bs, ne.Addr().Interface()) |
|||
data = ne.Interface().(Data) |
|||
} else { |
|||
err = json.Unmarshal(bs, &data) |
|||
} |
|||
|
|||
return |
|||
} |
|||
|
|||
// assignableTo will check if provided data is assignable to the same type as the exemplar
|
|||
// if the provided exemplar is nil then it will always return true
|
|||
func assignableTo(data Data, exemplar interface{}) bool { |
|||
if exemplar == nil { |
|||
return true |
|||
} |
|||
|
|||
// Assert data is of same type as exemplar
|
|||
t := reflect.TypeOf(data) |
|||
exemplarType := reflect.TypeOf(exemplar) |
|||
|
|||
return t.AssignableTo(exemplarType) && data != nil |
|||
} |
|||